cartethyia 2.5.9 → 2.6.0

package/src/persistence/stores/telemetry-store.ts

@@ -1,4 +1,4 @@
-import { appendFile, mkdir, readdir, rm } from "node:fs/promises";
+import { readdir, rm } from "node:fs/promises";
 import { join } from "node:path";
 import { sql } from "drizzle-orm";
 import { affectedRows, currentTimestamp, laterOf } from "../dialect";
@@ -11,7 +11,8 @@ import {
 } from "../schema";
 import { isGatewayError } from "../../observability/telemetry-status";
 import { TELEMETRY_EVENT_DIR } from "../../observability/event-log";
-import { dayEventFileName, utcDay } from "../../observability/event-day-index";
+import { RawTelemetryEventWriter } from "../../observability/raw-event-writer";
+import { hourStamp } from "../../shared/time";
 
 interface UsageTotalDelta {
   readonly tenantId: string;
@@ -71,94 +72,153 @@ function aggregateUsage(rows: readonly (typeof telemetryEvents.$inferInsert)[])
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 
 /**
- * Deletes whole days past the retention cutoff: the event file and its index
- * together.
+ * Deletes hourly event files whose whole hour is older than the retention
+ * cutoff.
  *
- * The day is the retention unit now, so both go at once. Removing one without
- * the other is the failure this avoids — an index with no events reports totals
- * for requests that no longer exist, and events with no index are invisible to
- * every usage read.
- *
- * The cutoff is a whole day, so today's file — still being appended to — is
- * never a candidate. Comparing filenames hour by hour would match a day file's
- * `.events` suffix against an hour stamp and could drop the open file.
+ * The CURRENT and PREVIOUS hour are never touched. The current one is still
+ * being appended; the previous one is why retention cannot be an hour
+ * boundary: a batch flushed just after a boundary still carries events from
+ * the hour that just ended, and the writer files them under that hour. Under
+ * multi-process/multi-writer operation a file for the previous hour can also
+ * still be open, so deleting it would either fail (Windows) or unlink a file
+ * another writer has open (POSIX). Files are named `<hour>-<uuid>.jsonl` now,
+ * so an hour is complete only once it is two hours old.
  */
 async function pruneEventFiles(before: Date): Promise<void> {
   const files = await readdir(TELEMETRY_EVENT_DIR).catch(() => [] as string[]);
+  // Two hours back: the current hour is open, the previous one can be too.
+  const floor = new Date(Math.min(before.getTime(), Date.now() - 2 * 3_600_000));
+  const cutoff = hourStamp(floor);
+  for (const file of files) {
+    if (!file.endsWith(".jsonl") || file.endsWith(".events.jsonl")) continue;
+    const hour = file.slice(0, 13);
+    if (hour < cutoff) await rm(join(TELEMETRY_EVENT_DIR, file), { force: true });
+  }
+  // Read-only legacy day logs are pruned only after their whole UTC day has
+  // aged out; index files are never consulted for usage anymore.
   const cutoffDay = before.toISOString().slice(0, 10);
   for (const file of files) {
-    const day = file.slice(0, 10);
-    if (day.length !== 10 || day >= cutoffDay) continue;
-    if (!file.endsWith(".events.jsonl") && !file.endsWith(".index.json")) continue;
+    if (!file.endsWith(".events.jsonl") || file.slice(0, 10) >= cutoffDay) continue;
     await rm(join(TELEMETRY_EVENT_DIR, file), { force: true });
   }
 }
 
 /**
- * Appends one JSON line per event to the day's event file.
- *
- * One file per UTC day, appended to and never rewritten while the day is open.
- * A file per `windowMinutes` window — 96 a day at the default — would have to be
- * found, opened and parsed one by one on every usage query, and which of those
- * files was still being written could only be inferred from the name, which is
- * what lets the open one be deleted mid-write.
- *
- * A single growing file has no such ambiguity, and the day index tracks how much
- * of it has been counted by byte offset.
+ * Appends one JSON line per event. One file per hour, so retention is a file
+ * delete and no request is replaced by a compacted summary.
  */
-async function appendEvents(rows: readonly (typeof telemetryEvents.$inferInsert)[]): Promise<void> {
-  await mkdir(TELEMETRY_EVENT_DIR, { recursive: true });
-  const file = join(TELEMETRY_EVENT_DIR, dayEventFileName(utcDay(new Date())));
-  const newline = String.fromCharCode(10);
-  const body = rows.map((row) => JSON.stringify(row)).join(newline) + newline;
-  await appendFile(file, body, "utf8");
-}
-
 export class DrizzleTelemetryStore {
-  constructor(private readonly db: CartethyiaDatabase) {}
+  private readonly eventWriter: RawTelemetryEventWriter;
 
-  /** Inserts events and their durable identity aggregates atomically. */
+  constructor(
+    private readonly db: CartethyiaDatabase,
+    eventWriter = new RawTelemetryEventWriter(TELEMETRY_EVENT_DIR),
+  ) {
+    this.eventWriter = eventWriter;
+  }
+
+  /**
+   * Applies usage deltas inside an enclosing transaction.
+   *
+   * Takes the transaction handle, not this.db: `insertEvents` runs the event
+   * insert and the aggregate upsert in ONE transaction, so usage is charged
+   * from the rows actually inserted — never from rows that `ON CONFLICT DO
+   * NOTHING` skipped. The deltas are computed by the caller from the
+   * inserted-rows result for exactly that reason.
+   */
+  private async applyUsageDeltas(
+    tx: Parameters<Parameters<CartethyiaDatabase["transaction"]>[0]>[0],
+    deltas: readonly UsageTotalDelta[],
+  ): Promise<void> {
+    if (deltas.length === 0) return;
+    // Deterministic order so two processes racing the same identities lock
+    // rows in the same order and cannot deadlock.
+    const ordered = [...deltas].sort((left, right) =>
+      left.tenantId.localeCompare(right.tenantId)
+      || left.identityType.localeCompare(right.identityType)
+      || left.entityId.localeCompare(right.entityId));
+    const now = new Date();
+    await tx
+      .insert(telemetryUsageTotals)
+      .values(ordered.map((delta) => ({
+        ...delta,
+        requests: delta.requests,
+        errors: delta.errors,
+        inputTokens: delta.inputTokens,
+        outputTokens: delta.outputTokens,
+        lastUsedAt: now,
+      })))
+      .onConflictDoUpdate({
+        target: [
+          telemetryUsageTotals.tenantId,
+          telemetryUsageTotals.identityType,
+          telemetryUsageTotals.entityId,
+        ],
+        set: {
+          requests: sql`${telemetryUsageTotals.requests} + excluded.requests`,
+          errors: sql`${telemetryUsageTotals.errors} + excluded.errors`,
+          inputTokens: sql`${telemetryUsageTotals.inputTokens} + excluded.input_tokens`,
+          outputTokens: sql`${telemetryUsageTotals.outputTokens} + excluded.output_tokens`,
+          lastUsedAt: laterOf(sql`${telemetryUsageTotals.lastUsedAt}`, sql`excluded.last_used_at`, this.db),
+          updatedAt: currentTimestamp(),
+        },
+      });
+  }
+
+  /** Writes raw events and maintains their durable identity aggregates. */
   async insertEvents(rows: Array<typeof telemetryEvents.$inferInsert>): Promise<void> {
     if (rows.length === 0) return;
     // The file accepts anything, so a malformed row would be written and then
     // retried forever without ever failing. Rejecting it here keeps the buffer's
     // drop-and-count contract: one bad row fails the batch, once.
+    const unique: Array<typeof telemetryEvents.$inferInsert> = [];
+    const ids = new Set<string>();
     for (const row of rows) {
       if (!UUID_RE.test(row.tenantId)) {
         throw new Error("telemetry event rejected: tenantId is not a uuid");
       }
+      // In-batch dedupe on the stable id: keep the first occurrence. A batch
+      // containing the same request twice (a buffer bug or a replayed
+      // enqueue) writes one row and charges once, instead of relying on the
+      // DB to skip the second insert silently.
+      if (row.id !== undefined) {
+        if (ids.has(row.id)) continue;
+        ids.add(row.id);
+      }
+      unique.push(row);
     }
-    const deltas = aggregateUsage(rows);
-    // The per-request row goes to an append-only file, not to Postgres. The
-    // database keeps only the durable aggregate below, so a request never
-    // adds a row to the largest table. Readers of recent requests read the
-    // file; readers of long-term totals read the aggregate.
+    // The raw append happens FIRST: the file is the at-least-once mirror. A
+    // crash after the append but before the SQL commit is recovered by
+    // `carte telemetry reconcile` (the backfill skips rows the unique index
+    // already holds). A raw append failure rejects the whole store call so
+    // the buffer retries the batch; nothing is committed, nothing charged.
+    await this.eventWriter.append(unique);
+    // One transaction: insert (skip conflicts), then charge usage from the
+    // rows actually inserted. A retried drain or a cross-process race
+    // converges to one row and one charge; no compensation transactions.
     await this.db.transaction(async (tx) => {
-      if (deltas.length === 0) return;
-      const now = new Date();
-      await tx
-        .insert(telemetryUsageTotals)
-        .values(deltas.map((delta) => ({ ...delta, lastUsedAt: now })))
-        .onConflictDoUpdate({
-          target: [
-            telemetryUsageTotals.tenantId,
-            telemetryUsageTotals.identityType,
-            telemetryUsageTotals.entityId,
-          ],
-          set: {
-            requests: sql`${telemetryUsageTotals.requests} + excluded.requests`,
-            errors: sql`${telemetryUsageTotals.errors} + excluded.errors`,
-            inputTokens: sql`${telemetryUsageTotals.inputTokens} + excluded.input_tokens`,
-            outputTokens: sql`${telemetryUsageTotals.outputTokens} + excluded.output_tokens`,
-            lastUsedAt: laterOf(sql`${telemetryUsageTotals.lastUsedAt}`, sql`excluded.last_used_at`),
-            updatedAt: currentTimestamp(),
-          },
+      const inserted = await tx
+        .insert(telemetryEvents)
+        .values(unique)
+        .onConflictDoNothing({
+          target: [telemetryEvents.tenantId, telemetryEvents.requestId],
+        })
+        .returning({
+          tenantId: telemetryEvents.tenantId,
+          requestId: telemetryEvents.requestId,
+          accountId: telemetryEvents.accountId,
+          apiKeyId: telemetryEvents.apiKeyId,
+          status: telemetryEvents.status,
+          httpStatus: telemetryEvents.httpStatus,
+          inputTokens: telemetryEvents.inputTokens,
+          outputTokens: telemetryEvents.outputTokens,
         });
+      // Usage is charged only from what actually landed. `returning` on
+      // `ON CONFLICT DO NOTHING` yields exactly the inserted rows on both
+      // backends; skipped conflicts contribute nothing.
+      const deltas = aggregateUsage(inserted);
+      await this.applyUsageDeltas(tx, deltas);
     });
-    // After the aggregate commits, so a rejected batch is retried once, not
-    // appended once per attempt. The file is the record of requests whose
-    // totals were actually counted.
-    await appendEvents(rows);
   }
 
   /** Inserts one redacted payload and returns its generated identity. */
@@ -225,6 +285,7 @@ export class DrizzleTelemetryStore {
    * the payload sweeper's `cleanupExpired`.
    */
   async pruneTelemetry(before: Date): Promise<void> {
+    await this.eventWriter.idle();
     await pruneEventFiles(before);
     const MAX_BATCHES_PER_SWEEP = 40;
     for (let batch = 0; batch < MAX_BATCHES_PER_SWEEP; batch += 1) {

package/src/protocol/reasoning-text.ts

@@ -0,0 +1,30 @@
+/** The reasoning-shaped part this reads. Structural, so any reasoning part —
+ *  including a `ContentPart` narrowed to `kind: "reasoning"` — is accepted. */
+export interface ReasoningTextSource {
+  readonly payload?: unknown;
+  readonly summary?: string;
+  readonly opaque?: boolean;
+}
+
+/**
+ * The one reader for visible reasoning text.
+ *
+ * A reasoning part carries two distinct fields: `payload` (the readable
+ * reasoning stream — Responses `reasoning_text`, Chat `reasoning_content`) and
+ * `summary` (Responses `reasoning_summary_text`, a digest of the same thought).
+ * Two call sites used to read them in opposite orders, so a part that populates
+ * both rendered different text per surface — and a part that populated both
+ * with the *same* string was emitted twice by surfaces that read each field.
+ *
+ * Precedence is `payload` → `summary`: `payload` is the stream, `summary` is a
+ * digest of it, and when only one is present it is the one to show. Opaque
+ * parts (redacted thinking, encrypted reasoning) have no readable text at all.
+ */
+export function reasoningTextOf(part: ReasoningTextSource): string | undefined {
+  if (part.opaque === true) return undefined;
+  const payload = part.payload;
+  if (typeof payload === "string" && payload.length > 0) return payload;
+  const summary = part.summary;
+  if (typeof summary === "string" && summary.length > 0) return summary;
+  return undefined;
+}

package/src/protocol/registry.ts

@@ -110,6 +110,8 @@ async function loadCodec(wireFamily: WireFamily): Promise<CodecModule> {
   const cached = codecCache.get(wireFamily);
   if (cached !== undefined) return cached;
   const promise = (async (): Promise<CodecModule> => {
+    // dynamic-import: codec picked by wire family at request time; static
+    // imports here would couple every request path to every codec module.
     switch (wireFamily) {
       case "chat":
         return (await import("./request/chat")).chatCodecModule;

package/src/protocol/response/chat.ts

@@ -258,6 +258,10 @@ export async function* decodeChatSseStream(
   // Single emission ledger: one entry per logical call.
   const ledger = createToolEmitLedger();
   const think = new MimoThinkSplitter();
+  // Separate splitter for the reasoning channel: `reasoning_content` is its own
+  // stream, and interleaving it into the answer splitter would let a tag in one
+  // channel swallow the other's text.
+  const reasoningThink = new MimoThinkSplitter();
   const declaredToolNames = declaredToolNamesForRequest(request);
   const callableToolNames = callableToolNamesForRequest(request);
   const contentMarkup =
@@ -344,13 +348,22 @@ export async function* decodeChatSseStream(
         const reasoningMarkupResult =
           typeof reasoningRaw === "string" ? reasoningMarkup?.push(reasoningRaw) : undefined;
         // Strip stray `<think>` tags the upstream double-wrapped inside its own
-        // reasoning stream (buddy's hy4-preview-f leaks them per-delta). Done
-        // per delta rather than once at the end because an open-ended tag that
-        // spans multiple deltas is a splitter job, not this one.
-        const reasoningText =
+        // reasoning stream (buddy's hy4-preview-f leaks them per-delta). This
+        // is the splitter's job, not `stripThinkingTags`'s: a stateless strip
+        // cannot see a tag split across two deltas, so `"<thi"` was emitted
+        // verbatim and only the following `</think>` removed.
+        const reasoningSplit =
           typeof reasoningRaw === "string"
-            ? stripThinkingTags(reasoningMarkupResult?.text ?? reasoningRaw)
-            : reasoningRaw;
+            ? reasoningThink.push(reasoningMarkupResult?.text ?? reasoningRaw)
+            : undefined;
+        // Both sides are reasoning. The splitter routes text outside a
+        // `<think>` span to its `.text` side, but this field is the reasoning
+        // stream in its entirety: untagged prose here is still thought, not
+        // answer. Taking only `.reasoning` dropped every un-tagged delta.
+        const reasoningText =
+          reasoningSplit === undefined
+            ? undefined
+            : `${reasoningSplit.reasoning}${reasoningSplit.text}`;
         if (typeof reasoningText === "string" && reasoningText.length > 0) {
           yield {
             type: "content_delta",
@@ -510,7 +523,9 @@ export async function* decodeChatSseStream(
   }
   const reasoningMarkupTail = reasoningMarkup?.flush();
   if (reasoningMarkupTail?.text) {
-    const reasoningText = stripThinkingTags(reasoningMarkupTail.text);
+    const split = reasoningThink.push(reasoningMarkupTail.text);
+    // Both sides are reasoning: see the per-delta branch above.
+    const reasoningText = `${split.reasoning}${split.text}`;
     if (reasoningText.length > 0) {
       yield {
         type: "content_delta",
@@ -519,6 +534,17 @@ export async function* decodeChatSseStream(
       } as CanonicalEvent;
     }
   }
+  // Release anything the reasoning splitter held for a tag that never
+  // completed, before `think.flush()` and the terminal.
+  const reasoningRest = reasoningThink.flush();
+  const reasoningTail = `${reasoningRest.reasoning}${reasoningRest.text}`;
+  if (reasoningTail.length > 0) {
+    yield {
+      type: "content_delta",
+      sequence_number: seq++,
+      content: { kind: "reasoning", payload: null, summary: reasoningTail },
+    } as CanonicalEvent;
+  }
   const rest = think.flush();
   if (rest.reasoning.length > 0) {
     yield {

package/src/protocol/response/codex.ts

@@ -629,11 +629,23 @@ export class CodexStreamFrameProcessor {
       if (streamError !== undefined) throw streamError;
       this.terminalStatus = "failed";
     } else if (typeof json["delta"] === "string" && json["delta"].length > 0) {
+      // An unrecognized frame carrying a bare `delta` used to be forwarded as
+      // answer text. Reasoning frames that fall through the chain above — the
+      // last `else if` is the catch-all for every type this decoder does not
+      // name — then leaked into the answer verbatim. Keeping the payload
+      // visible as an extension part is the unknown-event convention the
+      // Responses stream already uses (`responses:<type>`), so the frame stays
+      // in telemetry without ever reaching the answer. Dropping it instead
+      // would silently discard real content.
       events.push({
         type: "content_delta",
         sequence_number: this.#seq++,
-        content: { kind: "text", text: json["delta"] as string },
-      } as unknown as CanonicalEvent);
+        content: {
+          kind: "extension",
+          name: "codex:untyped_delta",
+          payload: json["delta"] as string,
+        },
+      } as CanonicalEvent);
     } else {
       const choices = json["choices"] as
         | Array<Record<string, unknown>>

package/src/protocol/response/messages.ts

@@ -92,8 +92,12 @@ export function parseClaudeContent(value: unknown, isOAuth = false): ContentPart
     const thinking = stringValue(value.thinking) ? value.thinking : "";
     return {
       kind: "reasoning",
+      // `thinking` is the readable reasoning stream, not a digest of itself.
+      // Putting it in both `payload` and `summary` made the Responses encoder
+      // emit the same text twice — once as `reasoning_text` and again as
+      // `reasoning_summary_text` — the same bug already fixed on the Codex
+      // surface (see `reasoningEvent` in `response/codex.ts`).
       payload: thinking,
-      summary: thinking,
       ...(stringValue(value.signature) ? { signature: value.signature } : {}),
     };
   }
@@ -375,7 +379,6 @@ function streamEventToCanonical(
           content: {
             kind: "reasoning",
             payload: delta.thinking,
-            summary: delta.thinking,
           },
         });
       }

package/src/providers/authentication/oauth-flow-store.ts

@@ -2,6 +2,7 @@
 
 import { createHash, randomBytes } from "node:crypto";
 import { base64url, decodeJwt } from "jose";
+import { asRecord as record, asTrimmedString as nonEmptyTrimmedString } from "../../shared/coerce";
 import type { RedisClient } from "../../persistence/redis";
 import { TtlCache } from "../../persistence/ttl-cache";
 
@@ -268,18 +269,10 @@ export function devicePollBackoff(
 }
 
 /** Trimmed non-empty string, or `undefined` for anything else. */
-export function nonEmptyTrimmedString(value: unknown): string | undefined {
-  if (typeof value !== "string") return undefined;
-  const trimmed = value.trim();
-  return trimmed.length > 0 ? trimmed : undefined;
-}
+export { nonEmptyTrimmedString };
 
 /** Object (non-array) narrowing for untyped upstream JSON. */
-export function record(value: unknown): Record<string, unknown> | undefined {
-  return value !== null && typeof value === "object" && !Array.isArray(value)
-    ? (value as Record<string, unknown>)
-    : undefined;
-}
+export { record };
 
 
 interface PkcePair {