cartethyia 2.4.5 → 2.5.0

package/src/persistence/usage-flush.ts

@@ -80,11 +80,30 @@ async function writeLifetimeDeltas(
 const FLUSH_STATEMENT_MAX_KEYS = 500;
 /** Max distinct keys retained across windows; oldest dropped first. */
 const FLUSH_PENDING_MAX_KEYS = 5000;
+
+/**
+ * Consecutive fully-failed flushes before the writer backs off.
+ *
+ * A database that is down stays down: retrying every `intervalMs` means a
+ * write attempt per second against a backend that is already overloaded, which
+ * is load the operator cannot distinguish from real traffic and which delays
+ * recovery. After this many failures the buffer stops probing and waits out a
+ * cooldown instead.
+ */
+const FLUSH_FAILURE_STREAK_LIMIT = 3;
+/** How long a failing writer waits before probing again. */
+const FLUSH_BACKOFF_MS = 15_000;
 export class UsageFlushBuffer {
   private readonly pending = new Map<string, number>();
   private timer: ReturnType<typeof setTimeout> | undefined;
   private flushing = false;
   private stopped = false;
+  /** Consecutive flushes that failed outright; drives the backoff below. */
+  private failureStreak = 0;
+  /** Wall-clock time before which no flush is attempted. */
+  private backoffUntil = 0;
+  /** Deltas abandoned because the buffer could not persist them. */
+  private droppedDelta = 0;
 
   constructor(
     private readonly db: CartethyiaDatabase,
@@ -104,6 +123,19 @@ export class UsageFlushBuffer {
     }
   }
 
+  /**
+   * Deltas recorded but not yet persisted, for one key.
+   *
+   * Exposed so a reader of `lifetime_tokens_consumed` can add what is in
+   * flight to what Postgres last stored. Without it such a reader saw a value
+   * that was behind by up to one flush interval, and — because the flush is
+   * coalescing by design — behind by the largest amount exactly when the key
+   * was busiest.
+   */
+  pendingFor(apiKeyId: string): number {
+    return this.pending.get(apiKeyId) ?? 0;
+  }
+
   /**
    * Writes everything accumulated so far, in bounded statements of at most
    * 500 keys each. One giant statement grows the SQL string with the
@@ -116,9 +148,14 @@ export class UsageFlushBuffer {
    */
   async flush(): Promise<void> {
     if (this.flushing || this.pending.size === 0) return;
+    // Backing off during a sustained outage: without this the writer probed a
+    // failing database on every tick, adding write load to the very overload
+    // that caused the failure.
+    if (Date.now() < this.backoffUntil) return;
     this.flushing = true;
     const batch = new Map(this.pending);
     this.pending.clear();
+    let failed = 0;
     try {
       const entries = [...batch];
       for (let offset = 0; offset < entries.length; offset += FLUSH_STATEMENT_MAX_KEYS) {
@@ -134,15 +171,38 @@ export class UsageFlushBuffer {
           try {
             await this.write(this.db, apiKeyId, delta);
           } catch {
-            this.pending.set(apiKeyId, (this.pending.get(apiKeyId) ?? 0) + delta);
+            // A transient failure retries, but only while the buffer is not
+            // itself saturated: an outage that outlasts the cap must drop
+            // accounting rather than grow the pending map without bound.
+            if (this.pending.size < FLUSH_PENDING_MAX_KEYS) {
+              this.pending.set(apiKeyId, (this.pending.get(apiKeyId) ?? 0) + delta);
+            } else {
+              this.droppedDelta += delta;
+            }
+            failed += 1;
           }
         }
       }
     } finally {
+      if (failed > 0) {
+        this.failureStreak += 1;
+        if (this.failureStreak >= FLUSH_FAILURE_STREAK_LIMIT) {
+          this.backoffUntil = Date.now() + FLUSH_BACKOFF_MS;
+          this.failureStreak = 0;
+        }
+      } else {
+        this.failureStreak = 0;
+        this.backoffUntil = 0;
+      }
       this.flushing = false;
     }
   }
 
+  /** Total usage delta abandoned after sustained write failure. */
+  get droppedTotal(): number {
+    return this.droppedDelta;
+  }
+
   /** Stops the timer. Pass `{ flush: true }` to persist what is still pending. */
   async stop(options?: { flush?: boolean }): Promise<void> {
     this.stopped = true;

package/src/protocol/canonical-model.ts

@@ -33,7 +33,16 @@ export type WireFamily = (typeof WIRE_FAMILIES)[number];
  * the console's Elysia body schema and its validator both project this list, so
  * a new kind cannot reach one and miss the other.
  */
-export const SERVICE_KINDS = ["llm", "systemone", "websearch", "imageGeneration", "videoGeneration"] as const;
+export const SERVICE_KINDS = [
+  "llm",
+  "systemone",
+  "websearch",
+  "imageGeneration",
+  "videoGeneration",
+  "embedding",
+  "audioTranscription",
+  "audioSpeech",
+] as const;
 
 /** Provider service kind selected by a model's catalog row. */
 export type ServiceKind = (typeof SERVICE_KINDS)[number];
@@ -354,6 +363,19 @@ export interface CanonicalTerminalEvent {
   readonly response_id?: string;
   /** Event timestamp in milliseconds since epoch (optional for profiling). */
   readonly timestamp?: number;
+  /**
+   * Set by an adapter whose turn is not finished: the stream ended because the
+   * model asked for a tool the client must answer, and the provider's
+   * authoritative usage arrives with the *later* request that continues the
+   * same upstream turn.
+   *
+   * The settle path reads this to skip charging estimated tokens for the
+   * interim response — otherwise one logical turn bills an input/output
+   * estimate on the tool-call response and the real usage on the continuation,
+   * which double-counts the turn. Only a provider that genuinely spans client
+   * requests with one upstream turn sets it.
+   */
+  readonly usage_deferred?: true;
 }
 
 /** An ordered canonical stream event; sequence numbers must increase monotonically. */

package/src/protocol/response/chat.ts

@@ -11,6 +11,11 @@ import { usageFromProvider, readReasoningText } from "../../providers/provider-u
 import { createToolEmitLedger } from "../tool-identity";
 import { MimoThinkSplitter, stripThinkingTags } from "../../providers/integrations/xiaomi-mimo/think-stream";
 import { parseDsmlToolCalls } from "./dsml";
+import {
+  OmpToolCallMarkupParser,
+  parseOmpToolCallMarkup,
+  type OmpTextToolCall,
+} from "./omp-tool-call-markup";
 
 
 export function mapChatStopReason(
@@ -43,14 +48,23 @@ export function parseChatResponseToEvents(
   const choices = (json["choices"] as Array<Record<string, unknown>>) ?? [];
   const first = choices[0] as Record<string, unknown> | undefined;
   const message = first?.["message"] as Record<string, unknown> | undefined;
+  const declaredToolNames = declaredToolNamesForRequest(request);
+  const callableToolNames = callableToolNamesForRequest(request);
+  let recoveredToolCalls = 0;
   if (message) {
     const reasoningRaw = readReasoningText(message);
+    const reasoningMarkup =
+      typeof reasoningRaw === "string" && (request.tools?.length ?? 0) > 0
+        ? parseOmpToolCallMarkup(reasoningRaw, declaredToolNames)
+        : undefined;
     // hy4-preview-f (and other buddy reasoning models) sometimes wraps its own
     // reasoning in `<think>…</think>` inside `reasoning_content`, which then
     // reaches the client as literal tag bytes. The field itself is the stream
     // for reasoning here, so the tags cannot carry structure — strip them.
     const reasoning =
-      typeof reasoningRaw === "string" ? stripThinkingTags(reasoningRaw) : reasoningRaw;
+      typeof reasoningRaw === "string"
+        ? stripThinkingTags(reasoningMarkup?.text ?? reasoningRaw)
+        : reasoningRaw;
     if (typeof reasoning === "string" && reasoning.length > 0) {
       events.push({
         type: "content_delta",
@@ -62,8 +76,16 @@ export function parseChatResponseToEvents(
       });
     }
     const content = message["content"] as string | null;
-    const recovered = typeof content === "string" ? parseDsmlToolCalls(content) : undefined;
-    const visible = recovered === undefined ? content : recovered.text;
+    const recoveredMarkup =
+      typeof content === "string" && (request.tools?.length ?? 0) > 0
+        ? parseOmpToolCallMarkup(content, declaredToolNames)
+        : undefined;
+    const contentWithoutMarkup = recoveredMarkup?.text ?? content;
+    const recovered =
+      typeof contentWithoutMarkup === "string"
+        ? parseDsmlToolCalls(contentWithoutMarkup)
+        : undefined;
+    const visible = recovered === undefined ? contentWithoutMarkup : recovered.text;
     if (typeof visible === "string" && visible.includes("<think")) {
       const think = new MimoThinkSplitter();
       const split = think.push(visible);
@@ -104,9 +126,39 @@ export function parseChatResponseToEvents(
       });
     }
     const structuredCalls = message["tool_calls"];
-    const toolCalls = Array.isArray(structuredCalls) && structuredCalls.length > 0
+    const hasStructuredCalls =
+      Array.isArray(structuredCalls) &&
+      structuredCalls.some((candidate) => {
+        if (typeof candidate !== "object" || candidate === null) return false;
+        const record = candidate as Record<string, unknown>;
+        const fn =
+          typeof record["function"] === "object" && record["function"] !== null
+            ? (record["function"] as Record<string, unknown>)
+            : undefined;
+        const callId = typeof record["id"] === "string" && record["id"].length > 0;
+        const name =
+          typeof fn?.["name"] === "string" && fn["name"].length > 0
+            ? fn["name"]
+            : typeof record["name"] === "string"
+              ? record["name"]
+              : undefined;
+        return callId && name !== undefined && name.length > 0;
+      });
+    const toolCalls = hasStructuredCalls
       ? structuredCalls
-      : recovered?.calls ?? [];
+      : recoveredMarkup?.calls.some((call) => callableToolNames.has(call.name))
+        ? recoveredMarkup.calls
+            .filter((call) => callableToolNames.has(call.name))
+            .map((call) => ({
+              id: call.id,
+              function: { name: call.name, arguments: call.arguments },
+            }))
+        : recovered?.calls ?? [];
+    if (!hasStructuredCalls && recoveredMarkup !== undefined) {
+      recoveredToolCalls = recoveredMarkup.calls.filter((call) =>
+        callableToolNames.has(call.name),
+      ).length;
+    }
     for (const tc of toolCalls) {
       if (typeof tc !== "object" || tc === null) continue;
       const record = tc as Record<string, unknown>;
@@ -166,7 +218,7 @@ export function parseChatResponseToEvents(
       sequenceNumber: events.length + 1,
       responseId: id,
       state: "complete",
-      stopReason: chatJsonStopReason,
+      stopReason: recoveredToolCalls > 0 ? "tool_use" : chatJsonStopReason,
       providerStopReason: chatJsonProviderStopReason,
       usage: usageRecord,
     }),
@@ -194,9 +246,17 @@ export async function* decodeChatSseStream(
   const lastToolNameByIndex = new Map<number, string>();
   let lastSeenToolId: string | undefined;
   let lastSeenToolName: string | undefined;
+  let structuredToolCallEmitted = false;
+  const recoveredMarkupCalls: OmpTextToolCall[] = [];
   // Single emission ledger: one entry per logical call.
   const ledger = createToolEmitLedger();
   const think = new MimoThinkSplitter();
+  const declaredToolNames = declaredToolNamesForRequest(request);
+  const callableToolNames = callableToolNamesForRequest(request);
+  const contentMarkup =
+    (request.tools?.length ?? 0) > 0 ? new OmpToolCallMarkupParser(declaredToolNames) : undefined;
+  const reasoningMarkup =
+    (request.tools?.length ?? 0) > 0 ? new OmpToolCallMarkupParser(declaredToolNames) : undefined;
   yield {
     type: "response_start",
     sequence_number: seq++,
@@ -240,8 +300,10 @@ export async function* decodeChatSseStream(
         // text/thinking blocks.
         if (typeof delta?.["content"] === "string" && delta["content"].length > 0) {
           const raw = delta["content"] as string;
-          const recovered = parseDsmlToolCalls(raw);
-          const visible = recovered?.text ?? raw;
+          const markup = contentMarkup?.push(raw);
+          if (markup !== undefined) recoveredMarkupCalls.push(...markup.calls);
+          const recovered = parseDsmlToolCalls(markup?.text ?? raw);
+          const visible = recovered?.text ?? markup?.text ?? raw;
           const split = think.push(visible);
           if (split.reasoning.length > 0) {
             yield {
@@ -270,12 +332,16 @@ export async function* decodeChatSseStream(
           }
         }
         const reasoningRaw = delta ? readReasoningText(delta) : undefined;
+        const reasoningMarkupResult =
+          typeof reasoningRaw === "string" ? reasoningMarkup?.push(reasoningRaw) : undefined;
         // Strip stray `<think>` tags the upstream double-wrapped inside its own
         // reasoning stream (buddy's hy4-preview-f leaks them per-delta). Done
         // per delta rather than once at the end because an open-ended tag that
         // spans multiple deltas is a splitter job, not this one.
         const reasoningText =
-          typeof reasoningRaw === "string" ? stripThinkingTags(reasoningRaw) : reasoningRaw;
+          typeof reasoningRaw === "string"
+            ? stripThinkingTags(reasoningMarkupResult?.text ?? reasoningRaw)
+            : reasoningRaw;
         if (typeof reasoningText === "string" && reasoningText.length > 0) {
           yield {
             type: "content_delta",
@@ -353,6 +419,7 @@ export async function* decodeChatSseStream(
             ledger.isDuplicateDefinition(id, fragName, args);
           if (!duplicate) {
             if (establishedNew) ledger.claimFirstEmission(id, fragName, args);
+            structuredToolCallEmitted = true;
             yield {
               type: "tool_call_delta",
               sequence_number: seq++,
@@ -390,6 +457,38 @@ export async function* decodeChatSseStream(
       );
     }
   }
+  const contentMarkupTail = contentMarkup?.flush();
+  if (contentMarkupTail !== undefined) {
+    recoveredMarkupCalls.push(...contentMarkupTail.calls);
+    if (contentMarkupTail.text.length > 0) {
+      const split = think.push(contentMarkupTail.text);
+      if (split.reasoning.length > 0) {
+        yield {
+          type: "content_delta",
+          sequence_number: seq++,
+          content: { kind: "reasoning", payload: null, summary: split.reasoning },
+        } as CanonicalEvent;
+      }
+      if (split.text.length > 0) {
+        yield {
+          type: "content_delta",
+          sequence_number: seq++,
+          content: { kind: "text", text: split.text },
+        } as CanonicalEvent;
+      }
+    }
+  }
+  const reasoningMarkupTail = reasoningMarkup?.flush();
+  if (reasoningMarkupTail?.text) {
+    const reasoningText = stripThinkingTags(reasoningMarkupTail.text);
+    if (reasoningText.length > 0) {
+      yield {
+        type: "content_delta",
+        sequence_number: seq++,
+        content: { kind: "reasoning", payload: null, summary: reasoningText },
+      } as CanonicalEvent;
+    }
+  }
   const rest = think.flush();
   if (rest.reasoning.length > 0) {
     yield {
@@ -405,6 +504,20 @@ export async function* decodeChatSseStream(
       content: { kind: "text", text: rest.text },
     } as CanonicalEvent;
   }
+  let recoveredMarkupToolCallsEmitted = 0;
+  if (!structuredToolCallEmitted) {
+    for (const call of recoveredMarkupCalls) {
+      if (!callableToolNames.has(call.name)) continue;
+      yield {
+        type: "tool_call_delta",
+        sequence_number: seq++,
+        call_id: call.id,
+        name: call.name,
+        arguments_delta: call.arguments,
+      } as CanonicalEvent;
+      recoveredMarkupToolCallsEmitted += 1;
+    }
+  }
   const chatStopReason = mapChatStopReason(finishReason);
   const chatProviderStopReason =
     typeof finishReason === "string" ? finishReason : undefined;
@@ -423,8 +536,28 @@ export async function* decodeChatSseStream(
     // `stop`. `chatFinishReason(undefined)` falls back to `stop`, so the client
     // saw a completed turn while telemetry recorded a 502 upstream transport
     // failure — the failure this terminal already carries was invisible to it.
-    stopReason: truncated && signal?.aborted !== true ? "error" : chatStopReason,
+    stopReason:
+      truncated && signal?.aborted !== true
+        ? "error"
+        : !structuredToolCallEmitted && recoveredMarkupToolCallsEmitted > 0
+          ? "tool_use"
+          : chatStopReason,
     providerStopReason: chatProviderStopReason,
     usage: usageFromProvider(rawUsage),
   });
 }
+
+function declaredToolNamesForRequest(request: CanonicalRequest): ReadonlySet<string> {
+  return new Set((request.tools ?? []).map((tool) => tool.name));
+}
+
+function callableToolNamesForRequest(request: CanonicalRequest): ReadonlySet<string> {
+  const declared = declaredToolNamesForRequest(request);
+  const choice = request.tool_choice;
+  if (choice === "none") return new Set();
+  if (choice === undefined || typeof choice === "string") return declared;
+  if (choice.type === "tool" || choice.type === "custom") {
+    return declared.has(choice.name) ? new Set([choice.name]) : new Set();
+  }
+  return new Set(choice.names.filter((name) => declared.has(name)));
+}

package/src/protocol/response/codex.ts

@@ -281,9 +281,7 @@ export class CodexStreamFrameProcessor {
   // output_index routing + `output_item.added` registration.
   readonly #openItems = new Map<string, OpenItemEntry>();
   readonly #openItemsByOutputIndex = new Map<number, OpenItemEntry>();
-  // Track tool call argument accumulation, retained for diagnostics and
-  // future reconciliation of streamed arguments against authoritative ones.
-  readonly #toolCallArgs = new Map<string, string[]>();
+
   // Whitespace-loop guard: a model stuck emitting only whitespace tool-call
   // argument deltas would otherwise hang the dispatch loop forever.
   readonly #whitespaceDeltaTracking = new Map<
@@ -427,9 +425,6 @@ export class CodexStreamFrameProcessor {
           : undefined;
       const key = itemId ?? callId;
       this.#checkWhitespaceLoopGuard(key, json["delta"] as string);
-      const chunks = this.#toolCallArgs.get(key) ?? [];
-      chunks.push(json["delta"] as string);
-      this.#toolCallArgs.set(key, chunks);
       events.push({
         type: "tool_call_delta",
         sequence_number: this.#seq++,
@@ -443,14 +438,7 @@ export class CodexStreamFrameProcessor {
       typeof json["delta"] === "string"
     ) {
       const { callId, name } = this.#resolveToolIdentity(json);
-      const itemId =
-        typeof json["item_id"] === "string"
-          ? (json["item_id"] as string)
-          : undefined;
-      const key = itemId ?? callId;
-      const chunks = this.#toolCallArgs.get(key) ?? [];
-      chunks.push(json["delta"] as string);
-      this.#toolCallArgs.set(key, chunks);
+
       events.push({
         type: "tool_call_delta",
         sequence_number: this.#seq++,
@@ -505,37 +493,13 @@ export class CodexStreamFrameProcessor {
         },
       });
     } else if (t === "response.function_call_arguments.done") {
-      const rawCallId =
-        (json["call_id"] as string | undefined) ??
-        (json["item_id"] as string | undefined);
-      const itemId =
-        typeof json["item_id"] === "string"
-          ? (json["item_id"] as string)
-          : undefined;
-      let key: string | undefined;
-      if (itemId !== undefined) key = itemId;
-      else if (rawCallId !== undefined) key = decodeCodexToolCallId(rawCallId);
-      else {
-        const outputIndex = readOutputIndex(json);
-        if (outputIndex !== undefined) key = `idx_${outputIndex}`;
-      }
-      const finalArgs = (json as Record<string, unknown>)["arguments"];
-      if (typeof finalArgs === "string" && key !== undefined) {
-        this.#toolCallArgs.set(key, [finalArgs]);
-      }
+      // The arguments themselves are emitted by the delta events above; the
+      // `.done` event only marks the call complete.
+
       this.hasToolCall = true;
     } else if (t === "response.custom_tool_call_input.done") {
-      const itemId =
-        typeof json["item_id"] === "string"
-          ? (json["item_id"] as string)
-          : undefined;
-      const rawCallId = (json["call_id"] as string | undefined) ?? itemId;
-      let key: string | undefined;
-      if (itemId !== undefined) key = itemId;
-      else if (rawCallId !== undefined) key = decodeCodexToolCallId(rawCallId);
-      const finalInput = (json as Record<string, unknown>)["input"];
-      if (typeof finalInput === "string" && key !== undefined)
-        this.#toolCallArgs.set(key, [finalInput]);
+      // As above: the input arrives as deltas, so `.done` only completes it.
+
       this.hasToolCall = true;
     } else if (t === "response.output_item.done") {
       const item = json["item"];
@@ -598,43 +562,13 @@ export class CodexStreamFrameProcessor {
         }
         if (outputItem["type"] === "function_call") {
           this.hasToolCall = true;
-          const callIdRaw =
-            typeof outputItem["call_id"] === "string"
-              ? (outputItem["call_id"] as string)
-              : undefined;
-          const decodedCallId =
-            callIdRaw !== undefined
-              ? decodeCodexToolCallId(callIdRaw)
-              : undefined;
-          const authoritative =
-            typeof outputItem["arguments"] === "string"
-              ? (outputItem["arguments"] as string)
-              : undefined;
-          if (authoritative !== undefined && decodedCallId !== undefined) {
-            const key = outputItemId ?? decodedCallId;
-            this.#toolCallArgs.set(key, [authoritative]);
-          }
+
           if (outputItemId !== undefined) this.#openItems.delete(outputItemId);
           if (outputIndex !== undefined)
             this.#openItemsByOutputIndex.delete(outputIndex);
         } else if (outputItem["type"] === "custom_tool_call") {
           this.hasToolCall = true;
-          const callIdRaw =
-            typeof outputItem["call_id"] === "string"
-              ? (outputItem["call_id"] as string)
-              : undefined;
-          const decodedCallId =
-            callIdRaw !== undefined
-              ? decodeCodexToolCallId(callIdRaw)
-              : undefined;
-          const authoritative =
-            typeof outputItem["input"] === "string"
-              ? (outputItem["input"] as string)
-              : undefined;
-          if (authoritative !== undefined && decodedCallId !== undefined) {
-            const key = outputItemId ?? decodedCallId;
-            this.#toolCallArgs.set(key, [authoritative]);
-          }
+
           if (outputItemId !== undefined) this.#openItems.delete(outputItemId);
           if (outputIndex !== undefined)
             this.#openItemsByOutputIndex.delete(outputIndex);