cartethyia 2.7.9 → 2.8.0

package/src/network/pool-health.ts

@@ -1,12 +1,11 @@
-import { randomUUID } from "node:crypto";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
-import { healthEvents } from "../persistence/schema";
 import type { NetworkPoolSelector } from "./pool/selector";
-import type { GatewayError } from "../protocol/gateway-error";
+import { GatewayError } from "../protocol/gateway-error";
 import { metrics } from "../observability/metrics";
 import { log } from "../observability/logger";
-import { upsertProviderCooldownMarker } from "./pool-cooldown-store";
+import { appendPoolProviderEgressAuditRow, upsertProviderCooldownMarker } from "./pool-cooldown-store";
 import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
+import type { EgressVerdict } from "./rate-limit-verdict";
 
 /**
  * Durable pool rate-limit history.
@@ -29,9 +28,6 @@ import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
  * needs one row per cooldown episode — not one per throttled request.
  */
 const POOL_COOLDOWN_EVENT_WINDOW_MS = 15 * 60 * 1000;
-const MAX_TRACKED_PAIRS = 5_000;
-
-const lastRecordedByPoolProvider = new Map<string, number>();
 
 /** Durable half of `flagPoolCooldown`; not a second public entry point. */
 async function recordPoolCooldownEvent(
@@ -40,35 +36,13 @@ async function recordPoolCooldownEvent(
   providerId: string,
   reason: string,
 ): Promise<void> {
-  const key = `${poolId}::${providerId.toLowerCase()}`;
-  const now = Date.now();
-  const last = lastRecordedByPoolProvider.get(key);
-  if (last !== undefined && now - last < POOL_COOLDOWN_EVENT_WINDOW_MS) return;
-  try {
-    await db.insert(healthEvents).values({
-      id: randomUUID(),
-      entityKind: "pool",
-      networkPoolId: poolId,
-      fromStatus: null,
-      toStatus: "cooldown",
-      reason: `provider ${providerId}: ${reason}`,
-      errorCategory: "rate_limit_transient",
-      createdAt: new Date(),
-    });
-  } catch {
-    // Non-fatal: observability writes must never fail dispatch. The
-    // coalescing stamp is only written below, so a failed insert does not
-    // suppress the next attempt for the whole window.
-    return;
-  }
-  lastRecordedByPoolProvider.set(key, now);
-  if (lastRecordedByPoolProvider.size > MAX_TRACKED_PAIRS) {
-    for (const [trackedKey, recordedAt] of lastRecordedByPoolProvider) {
-      if (now - recordedAt >= POOL_COOLDOWN_EVENT_WINDOW_MS) {
-        lastRecordedByPoolProvider.delete(trackedKey);
-      }
-    }
-  }
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: false,
+    coalesceWindowMs: POOL_COOLDOWN_EVENT_WINDOW_MS,
+  });
 }
 
 /**
@@ -106,14 +80,7 @@ export function flagPoolBlocked(
         error: String(cause),
       });
     });
-  void recordPoolBlockEvent(db, poolId, providerId, reason, error).catch((cause: unknown) => {
-    metrics.pool_cooldown_record_failed.inc(1);
-    log.warn("pool block event failed", {
-      networkPoolId: poolId,
-      providerId,
-      error: String(cause),
-    });
-  });
+  void recordPoolBlockEvent(db, poolId, providerId, reason, error);
 }
 
 /**
@@ -131,15 +98,12 @@ async function recordPoolBlockEvent(
   reason: string,
   error: GatewayError,
 ): Promise<void> {
-  await db.insert(healthEvents).values({
-    id: randomUUID(),
-    entityKind: "pool",
-    networkPoolId: poolId,
-    fromStatus: null,
-    toStatus: "cooldown",
-    reason: `provider ${providerId}: ${reason}${providerCodeSuffix(error)}`,
-    errorCategory: "egress_blocked",
-    createdAt: new Date(),
+  await appendPoolProviderEgressAuditRow(db, {
+    poolId,
+    providerId,
+    reason,
+    blocked: true,
+    codeSuffix: providerCodeSuffix(error),
   });
 }
 
@@ -199,12 +163,36 @@ export function flagPoolCooldown(
         error: String(cause),
       });
     });
-  void recordPoolCooldownEvent(db, poolId, providerId, reason).catch((cause: unknown) => {
-    metrics.pool_cooldown_record_failed.inc(1);
-    log.warn("pool cooldown event failed", {
-      networkPoolId: poolId,
-      providerId,
-      error: String(cause),
-    });
-  });
+  void recordPoolCooldownEvent(db, poolId, providerId, reason);
+}
+
+/**
+ * One egress-failure verdict → pool-flag dispatch, shared by the attempt loop
+ * and the streaming finalizer.
+ *
+ * Both paths classify the same failure with `classifyEgressFailure`, then must
+ * retire the same pool the same way; a policy change — which verdict maps to a
+ * cooldown vs a block, or a new verdict — belongs here so the two response
+ * modes cannot drift in telemetry.
+ *
+ * The call sites compute the verdict (the attempt loop also needs it for its
+ * retry decision) and pass it in; the gating is identical to what both
+ * previously applied by hand: only a real pool id + selector pair, only a
+ * `GatewayError`, and never a `"none"` verdict. The flags themselves are
+ * fire-and-forget advisory writes that never fail the request.
+ */
+export function flagPoolOnEgressFailure(
+  poolSelector: NetworkPoolSelector | undefined,
+  db: CartethyiaDatabase,
+  poolId: string | undefined,
+  providerId: string,
+  error: unknown,
+  verdict: EgressVerdict,
+): void {
+  if (poolId === undefined || poolSelector === undefined || !(error instanceof GatewayError)) return;
+  if (verdict.kind === "cooldown") {
+    flagPoolCooldown(poolSelector, db, poolId, providerId, error);
+  } else if (verdict.kind === "blocked") {
+    flagPoolBlocked(poolSelector, db, poolId, providerId, error);
+  }
 }

package/src/network/rate-limit-verdict.ts

@@ -21,7 +21,7 @@ import { GatewayError } from "../protocol/gateway-error";
 import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
 
 /** What one upstream failure means for the egress that carried it. */
-type EgressVerdict =
+export type EgressVerdict =
   | { kind: "none" }
   | { kind: "cooldown"; retryAfterMs?: number }
   | { kind: "blocked" };

package/src/network/ssrf.ts

@@ -3,6 +3,7 @@ import { lookup } from "node:dns/promises";
 import { isIP } from "node:net";
 import { GatewayError } from "../protocol/gateway-error";
 import type { SsrfPolicy } from "../config/network";
+import { BoundedLruMap } from "../transport/routing/bounded-lru";
 import {
   cidrContainsAddress,
   ipv4ToNumberStrict,
@@ -12,17 +13,16 @@ import {
 
 const DNS_CACHE_TTL_MS = 60_000;
 const DNS_CACHE_MAX = 64;
-const dnsCache = new Map<string, { addrs: string[]; at: number }>();
+// Capacity-bounded: `set` re-inserts the host (moving it to the tail) and
+// evicts the least-recently-used entry when over the cap — the same
+// insertion-order eviction the hand-rolled loop performed, since every write
+// here re-inserts.
+const dnsCache = new BoundedLruMap<string, { addrs: string[]; at: number }>(DNS_CACHE_MAX);
 
 /** Bounds cache memory: evicts oldest entries once size exceeds the cap, not just expired ones. */
 function cacheDnsResult(hostname: string, addrs: string[], now: number): void {
   dnsCache.delete(hostname);
   dnsCache.set(hostname, { addrs, at: now });
-  while (dnsCache.size > DNS_CACHE_MAX) {
-    const oldest = dnsCache.keys().next().value;
-    if (oldest === undefined) break;
-    dnsCache.delete(oldest);
-  }
 }
 
 

package/src/observability/event-log.ts

@@ -1,15 +1,15 @@
-import { open, readdir } from "node:fs/promises";
 import { join } from "node:path";
-import { resolveTelemetryRetentionDays } from "../config/telemetry";
 import { resolveDataDir } from "../persistence/db-mode";
-import { hourStamp } from "../shared/time";
 
 /**
- * Per-request telemetry lives in hourly JSONL files, not in Postgres. The
- * database keeps only the durable aggregate; the files are the single source
- * for recent requests and readers must go through this module, not a
- * `telemetry_events` table.
+ * Per-request telemetry is mirrored to hourly JSONL files and the durable
+ * `telemetry_events` table. This module is the shared home for the raw-file
+ * directory constant and the raw record type; the canonical reader is
+ * `event-query.ts` (SQL), and the reconciliation job (`telemetry-reconciliation.ts`)
+ * replays the raw mirror. The file-based reader that once lived here is gone —
+ * it was used only by tests and by this module itself.
  */
+
 /**
  * Where raw hourly event files live.
  *
@@ -64,357 +64,3 @@ export interface TelemetryEventRecord {
    */
   readonly degenerateStop?: boolean;
 }
-
-function asString(value: unknown): string | null {
-  return typeof value === "string" ? value : null;
-}
-
-function asNumber(value: unknown): number | null {
-  if (typeof value === "number" && Number.isFinite(value)) return value;
-  if (typeof value === "string" && value.length > 0 && Number.isFinite(Number(value))) return Number(value);
-  return null;
-}
-
-function normalize(raw: Record<string, unknown>): TelemetryEventRecord | undefined {
-  const tenantId = asString(raw["tenantId"]);
-  const requestId = asString(raw["requestId"]);
-  const createdAt = asString(raw["createdAt"]);
-  // No time means the row cannot be ordered or aged out. Inventing one made
-  // every undated row reappear as "just now" on each read.
-  if (!tenantId || !requestId || !createdAt) return undefined;
-  return {
-    id: asString(raw["id"]) ?? requestId,
-    createdAt,
-    tenantId,
-    requestId,
-    sourceSurface: asString(raw["sourceSurface"]),
-    requestedModel: asString(raw["requestedModel"]),
-    reportedModel: asString(raw["reportedModel"]),
-    requestedEffort: asString(raw["requestedEffort"]),
-    providerId: asString(raw["providerId"]),
-    accountId: asString(raw["accountId"]),
-    networkPoolId: asString(raw["networkPoolId"]),
-    endpoint: asString(raw["endpoint"]),
-    apiKeyId: asString(raw["apiKeyId"]),
-    userAgent: asString(raw["userAgent"]),
-    clientIp: asString(raw["clientIp"]),
-    latencyMs: asNumber(raw["latencyMs"]),
-    ttfbMs: asNumber(raw["ttfbMs"]),
-    stream: typeof raw["stream"] === "boolean" ? raw["stream"] : null,
-    status: asString(raw["status"]),
-    httpStatus: asNumber(raw["httpStatus"]),
-    errorOrigin: asString(raw["errorOrigin"]),
-    errorCategory: asString(raw["errorCategory"]),
-    inputTokens: asNumber(raw["inputTokens"]),
-    cachedInputTokens: asNumber(raw["cachedInputTokens"]),
-    outputTokens: asNumber(raw["outputTokens"]),
-    reasoningTokens: asNumber(raw["reasoningTokens"]),
-    estimatedCostUsd: asNumber(raw["estimatedCostUsd"]),
-    creditUsed: asNumber(raw["creditUsed"]),
-    resolveMs: asNumber(raw["resolveMs"]),
-    tokensPerSec: asNumber(raw["tokensPerSec"]),
-    firstContentDeltaAtMs: asNumber(raw["firstContentDeltaAtMs"]),
-    lastEventAtMs: asNumber(raw["lastEventAtMs"]),
-    ...(raw["degenerateStop"] === true ? { degenerateStop: true } : {}),
-  };
-}
-
-/**
- * The comparable hour of an event file's name.
- *
- * A name is either `2026-10-22T05-15.jsonl` (an older window file) or
- * `2026-10-22.events.jsonl` (a day file). Both sort against `hourStamp`: only
- * the leading date and hour are taken, so the day file's `.events` suffix cannot
- * make it compare as later or earlier than the window files beside it.
- */
-function eventFileHour(file: string): string {
-  const stamp = file.endsWith(".events.jsonl") ? `${file.slice(0, 10)}T00` : file.slice(0, 13);
-  return stamp;
-}
-
-/** Chunk size for the streaming scan: one line can be long, a file cannot. */
-const READ_CHUNK_BYTES = 256 * 1024;
-
-/**
- * The newest matching events in one file, read backwards from the end.
- *
- * Reads fixed blocks from the tail until the page is full, so a day file of any
- * size costs one block per page rather than a full parse. A partial line at the
- * very end is dropped: the writer may still be appending to it.
- */
-async function* readEventFileTail(
-  file: string,
-  tenantId: string | undefined,
-  want: number,
-  since: number | undefined,
-  until: number | undefined,
-): AsyncGenerator<TelemetryEventRecord> {
-  if (want <= 0) return;
-  const handle = await open(join(TELEMETRY_EVENT_DIR, file), "r").catch(() => undefined);
-  if (handle === undefined) return;
-  try {
-    const size = (await handle.stat().catch(() => undefined))?.size ?? 0;
-    if (size === 0) return;
-    const BLOCK = 64 * 1024;
-    let end = size;
-    let carry = "";
-    const found: TelemetryEventRecord[] = [];
-    while (end > 0 && found.length < want) {
-      const start = Math.max(0, end - BLOCK);
-      const length = end - start;
-      const buffer = Buffer.alloc(length);
-      await handle.read(buffer, 0, length, start);
-      const text = buffer.toString("utf8") + carry;
-      const lines = text.split("\n");
-      carry = lines.shift() ?? "";
-      // Newest first: the block was read from the end, so its lines run
-      // backwards through the file.
-      for (let index = lines.length - 1; index >= 0; index -= 1) {
-        const line = lines[index] ?? "";
-        if (line.length === 0) continue;
-        if (tenantId !== undefined && !line.includes(tenantId)) continue;
-        let parsed: TelemetryEventRecord | undefined;
-        try {
-          const value: unknown = JSON.parse(line);
-          if (typeof value === "object" && value !== null) {
-            parsed = value as TelemetryEventRecord;
-          }
-        } catch {
-          continue;
-        }
-        if (parsed === undefined) continue;
-        if (parsed.tenantId !== tenantId) continue;
-        const at = Date.parse(parsed.createdAt);
-        if (Number.isFinite(at)) {
-          if (since !== undefined && at < since) continue;
-          if (until !== undefined && at > until) continue;
-        }
-        found.push(parsed);
-        if (found.length >= want) break;
-      }
-      end = start;
-    }
-    for (const event of found) yield event;
-  } finally {
-    await handle.close().catch(() => undefined);
-  }
-}
-
-async function* readEventFiles(
-  files: readonly string[],
-  allow?: (line: string) => boolean,
-): AsyncGenerator<TelemetryEventRecord> {
-  for (const file of files) {
-    const handle = await open(join(TELEMETRY_EVENT_DIR, file), "r").catch(() => undefined);
-    if (handle === undefined) continue;
-    // Chunked, because `readFile` + `split` held the whole file as a string plus
-    // an array of one string per line — several times the file's size, per
-    // concurrent read, on the same process that serves every request. Peak
-    // memory is now one chunk and one line.
-    const decoder = new TextDecoder();
-    const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES);
-    let carry = "";
-    try {
-      for (;;) {
-        const { bytesRead } = await handle.read(buffer, 0, READ_CHUNK_BYTES);
-        if (bytesRead === 0) break;
-        const text = carry + decoder.decode(buffer.subarray(0, bytesRead), { stream: true });
-        const lines = text.split(String.fromCharCode(10));
-        // The tail is a partial line unless the chunk ended exactly on a newline.
-        carry = lines.pop() ?? "";
-        for (const line of lines) {
-          if (!line) continue;
-          // A line that cannot match is skipped before `JSON.parse`. The log is
-          // shared by every tenant, so most of its bytes belong to someone else —
-          // parsing them was the entire cost of a console read.
-          if (allow !== undefined && !allow(line)) continue;
-          try {
-            const event = normalize(JSON.parse(line) as Record<string, unknown>);
-            if (event) yield event;
-          } catch {
-            // A torn tail from a crash is skipped, not fatal. The next line stands alone.
-          }
-        }
-      }
-      if (carry.length > 0) {
-        const line = carry + decoder.decode();
-        if (allow === undefined || allow(line)) {
-          try {
-            const event = normalize(JSON.parse(line) as Record<string, unknown>);
-            if (event) yield event;
-          } catch {
-            // A torn tail from a crash is skipped, not fatal.
-          }
-        }
-      }
-    } finally {
-      await handle.close();
-    }
-  }
-}
-
-/** Reads every retained event, newest first. A missing directory is an empty log. */
-export async function readTelemetryEvents(): Promise<TelemetryEventRecord[]> {
-  const files = await readdir(TELEMETRY_EVENT_DIR).catch(() => [] as string[]);
-  const events: TelemetryEventRecord[] = [];
-  for await (const event of readEventFiles(files.filter((file) => file.endsWith(".jsonl")))) events.push(event);
-  events.sort((a, b) => (a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0));
-  return events;
-}
-
-interface EventQuery {
-  readonly tenantId?: string;
-  readonly since?: Date;
-  /**
-   * Exclusive upper bound on `createdAt`, used to push a pagination cursor into
-   * the query instead of filtering after the whole window was parsed. Without
-   * it a cursor page drops `limit` — the reader cannot know where the page
-   * ends — so only the first page of a list is bounded.
-   */
-  readonly until?: Date;
-  readonly apiKeyIds?: readonly string[];
-  readonly accountIds?: readonly string[];
-  readonly requestId?: string;
-  /**
-   * Distinct from {@link requestId}: `id` defaults to the request id when a row
-   * does not carry one of its own (`event-log.ts:68`), so a row can match by
-   * either. A lookup by `id` short-circuits like `requestId` does, which is the
-   * whole point — without it a single-event detail read parses the retention
-   * window and then scans it in JS.
-   */
-  readonly id?: string;
-  readonly httpStatus?: number;
-  readonly limit?: number;
-}
-
-function matches(event: TelemetryEventRecord, query: EventQuery, since: number | undefined, until: number | undefined, keys: ReadonlySet<string> | undefined, accounts: ReadonlySet<string> | undefined): boolean {
-  if (query.tenantId !== undefined && event.tenantId !== query.tenantId) return false;
-  const at = new Date(event.createdAt).getTime();
-  if (since !== undefined && at < since) return false;
-  if (until !== undefined && at > until) return false;
-  if (keys && (event.apiKeyId === null || !keys.has(event.apiKeyId))) return false;
-  if (accounts && (event.accountId === null || !accounts.has(event.accountId))) return false;
-  if (query.requestId !== undefined && event.requestId !== query.requestId) return false;
-  if (query.id !== undefined && event.id !== query.id) return false;
-  if (query.httpStatus !== undefined && event.httpStatus !== query.httpStatus) return false;
-  return true;
-}
-
-/** The events matching a query, already newest first. */
-export async function queryTelemetryEvents(query: EventQuery): Promise<TelemetryEventRecord[]> {
-  // Hourly request files only. A `.compacted.jsonl` also ends in `.jsonl`, and
-  // letting one through meant every summary row was parsed as if it were a
-  // request — a row with no `tenantId`/`createdAt` that then failed `matches`,
-  // but only after paying for the parse.
-  //
-  // The lower bound is the retention window, not just `since`: a caller that
-  // asks for all time still must not scan files that have already been aged
-  // out, and one that passes no `since` at all must not read every file ever
-  // written. Retention removes whole raw files; no summary is substituted.
-  const retentionFloor = hourStamp(
-    new Date(Date.now() - resolveTelemetryRetentionDays() * 24 * 60 * 60_000),
-  );
-  const sinceFloor =
-    query.since === undefined ? retentionFloor : hourStamp(query.since);
-  const floor = sinceFloor > retentionFloor ? sinceFloor : retentionFloor;
-  const files = (await readdir(TELEMETRY_EVENT_DIR).catch(() => [] as string[]))
-    .filter(
-      (file) =>
-        file.endsWith(".jsonl") &&
-        !file.endsWith(".compacted.jsonl") &&
-        eventFileHour(file) >= floor,
-    )
-    .sort((a, b) => (a < b ? 1 : a > b ? -1 : 0));
-  const since = query.since?.getTime();
-  const until = query.until?.getTime();
-  const keys = query.apiKeyIds ? new Set(query.apiKeyIds) : undefined;
-  const accounts = query.accountIds ? new Set(query.accountIds) : undefined;
-  const events: TelemetryEventRecord[] = [];
-  // The tenant and request ids are plain substrings of the stored line, so they
-  // are checked on the raw text: a UUID appearing anywhere in the row is that
-  // field, and a false negative is not possible while a false positive costs
-  // only the parse it would have paid anyway.
-  const allow =
-    query.tenantId === undefined && query.requestId === undefined && query.id === undefined
-      ? undefined
-      : (line: string): boolean =>
-          (query.tenantId === undefined || line.includes(query.tenantId)) &&
-          (query.requestId === undefined || line.includes(query.requestId)) &&
-          (query.id === undefined || line.includes(query.id));
-  // A single-row lookup: stop at the first (newest) match instead of parsing
-  // the whole retention window.
-  const oneRow = query.requestId !== undefined || query.id !== undefined;
-  // With one file per hour, a plain page of the newest rows does not need to
-  // parse the whole retention window to return fifty lines. When nothing filters on
-  // content — no status, no key, no account — the newest rows are the last
-  // lines, so they are read backwards from the end and the scan stops once the
-  // page is full.
-  const tailOnly =
-    query.limit !== undefined &&
-    !oneRow &&
-    query.httpStatus === undefined &&
-    query.apiKeyIds === undefined &&
-    query.accountIds === undefined;
-  if (tailOnly) {
-    const newest: TelemetryEventRecord[] = [];
-    for (const file of files) {
-      for await (const event of readEventFileTail(file, query.tenantId, query.limit - newest.length, since, until)) {
-        newest.push(event);
-      }
-      if (newest.length >= query.limit) break;
-    }
-    return newest.slice(0, query.limit);
-  }
-  for (const file of files) {
-    const matched: TelemetryEventRecord[] = [];
-    for await (const event of readEventFiles([file], allow)) {
-      if (!matches(event, query, since, until, keys, accounts)) continue;
-      matched.push(event);
-      if (oneRow) return matched;
-    }
-    matched.sort((a, b) => (a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0));
-    events.push(...matched);
-    if (query.limit !== undefined && events.length >= query.limit) return events.slice(0, query.limit);
-  }
-  return events;
-}
-
-interface EventTotals {
-  requests: number;
-  succeeded: number;
-  failed: number;
-  cancelled: number;
-  truncated: number;
-  inputTokens: number;
-  cachedTokens: number;
-  outputTokens: number;
-  estimatedCost: number;
-  unpriced: number;
-  latencySum: number;
-  latencyCount: number;
-  tokensPerSecSum: number;
-  tokensPerSecCount: number;
-}
-
-export function emptyTotals(): EventTotals {
-  return { requests: 0, succeeded: 0, failed: 0, cancelled: 0, truncated: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, estimatedCost: 0, unpriced: 0, latencySum: 0, latencyCount: 0, tokensPerSecSum: 0, tokensPerSecCount: 0 };
-}
-
-/**
-
-/** Folds one event into a totals bucket. `failed` counts any non-completed terminal status. */
-export function accumulate(totals: EventTotals, event: TelemetryEventRecord): void {
-  totals.requests += 1;
-  if (event.status === "completed") totals.succeeded += 1;
-  else if (event.status === "cancelled") totals.cancelled += 1;
-  else if (event.status === "truncated") totals.truncated += 1;
-  else if (event.status !== null) totals.failed += 1;
-  totals.inputTokens += event.inputTokens ?? 0;
-  totals.cachedTokens += event.cachedInputTokens ?? 0;
-  totals.outputTokens += event.outputTokens ?? 0;
-  if (event.estimatedCostUsd !== null) totals.estimatedCost += event.estimatedCostUsd;
-  else if (event.status === "completed") totals.unpriced += 1;
-  if (event.latencyMs !== null) { totals.latencySum += event.latencyMs; totals.latencyCount += 1; }
-  if (event.tokensPerSec !== null) { totals.tokensPerSecSum += event.tokensPerSec; totals.tokensPerSecCount += 1; }
-}
-

package/src/observability/performance-metrics.ts

@@ -62,6 +62,15 @@ export function trackMemoryUsage(component: string, bytes: number): void {
   memoryUsage.record(component, bytes);
 }
 
+/**
+ * The recorded memory figures keyed by component name (`rss`, `heap_used`,
+ * `heap_total`, `external`). This series is the canonical memory view the
+ * Prometheus gauges are derived from, so the two registries cannot disagree.
+ */
+export function memoryUsageBytes(): Record<string, number> {
+  return memoryUsage.snapshot();
+}
+
 /** Returns the current snapshot for the console performance endpoint. */
 export function performanceMetricsSnapshot(): PerformanceMetricsSnapshot {
   return {

package/src/observability/runtime-metrics.ts

@@ -10,7 +10,7 @@
 import { readFileSync } from "node:fs";
 import os from "node:os";
 import { metrics } from "./metrics";
-import { trackMemoryUsage } from "./performance-metrics";
+import { memoryUsageBytes, trackMemoryUsage } from "./performance-metrics";
 import { log } from "./logger";
 import { resolveMemoryLimitOverrideBytes } from "../config/timeouts";
 import { CONFIG_SPEC, readFlag } from "../config/spec";
@@ -190,19 +190,24 @@ export class RuntimeMetricsSampler {
 
   sample(): void {
     const memory = process.memoryUsage();
-    metrics.cartethyia_memory_rss_bytes.set(memory.rss);
-    metrics.cartethyia_memory_heap_used_bytes.set(memory.heapUsed);
-    metrics.cartethyia_memory_heap_total_bytes.set(memory.heapTotal);
+    // The performance-metrics memory series is the single memory view: record
+    // the four figures once, then derive the Prometheus gauges from that same
+    // series. `external` has no gauge equivalent, so it lives only in the
+    // series (and in the `/system/performance` snapshot).
+    trackMemoryUsage("rss", memory.rss);
+    trackMemoryUsage("heap_used", memory.heapUsed);
+    trackMemoryUsage("heap_total", memory.heapTotal);
+    trackMemoryUsage("external", memory.external);
+    const recordedMemory = memoryUsageBytes();
+    metrics.cartethyia_memory_rss_bytes.set(recordedMemory["rss"] ?? 0);
+    metrics.cartethyia_memory_heap_used_bytes.set(recordedMemory["heap_used"] ?? 0);
+    metrics.cartethyia_memory_heap_total_bytes.set(recordedMemory["heap_total"] ?? 0);
     metrics.cartethyia_memory_limit_bytes.set(this.limitBytes ?? 0);
     this.snapshot = {
-      rssBytes: memory.rss,
+      rssBytes: recordedMemory["rss"] ?? 0,
       limitBytes: this.limitBytes,
       sampledAtMs: Date.now(),
     };
-    trackMemoryUsage("rss", memory.rss);
-    trackMemoryUsage("heap_used", memory.heapUsed);
-    trackMemoryUsage("heap_total", memory.heapTotal);
-    trackMemoryUsage("external", memory.external);
 
     const roundRobin = this.sources.routingRoundRobinEntries();
     metrics.cartethyia_routing_roundrobin_entries.set(roundRobin.combo, { scope: "combo" });