cartethyia 2.7.8 → 2.8.0

package/src/observability/telemetry-row.ts

@@ -0,0 +1,125 @@
+// Shared telemetry insert-row mapping: the single column spelling for both the
+// write path (telemetry-buffer.ts) and the reconciliation backfill
+// (telemetry-reconciliation.ts). Before this module existed the two paths kept
+// hand-maintained mirrors that drifted — the backfill dropped creditUsed,
+// resolveMs, tokensPerSec, firstContentDeltaAtMs, lastEventAtMs,
+// payloadCaptureState, and degenerateStop, so a recovered row read as if those
+// events never happened. One mapper means a column change lands in both paths.
+import { telemetryEvents } from "../persistence/schema";
+import { clientNameFromUserAgent } from "./client-name";
+
+/**
+ * A normalized telemetry event, shaped like the raw `TelemetryEventRecord` the
+ * reconciliation job parses off disk. The write path normalizes its
+ * `TelemetryEventInput` (nested `usage`, token-count coercion) into this shape
+ * before calling {@link telemetryInsertRow}; the reconciliation path passes its
+ * parsed record directly.
+ */
+export interface TelemetryInsertRowSource {
+  readonly id: string;
+  readonly tenantId: string;
+  readonly requestId: string;
+  readonly createdAt: Date | string;
+  readonly sourceSurface: string | null;
+  readonly requestedModel: string | null;
+  readonly reportedModel: string | null;
+  readonly requestedEffort: string | null;
+  readonly providerId: string | null;
+  readonly accountId: string | null;
+  readonly networkPoolId: string | null;
+  readonly endpoint: string | null;
+  readonly apiKeyId: string | null;
+  readonly userAgent: string | null;
+  readonly clientIp: string | null;
+  readonly latencyMs: number | null;
+  readonly ttfbMs: number | null;
+  readonly stream: boolean | null;
+  readonly status: string | null;
+  readonly httpStatus: number | null;
+  readonly errorOrigin: string | null;
+  readonly errorCategory: string | null;
+  readonly inputTokens: number | null;
+  readonly cachedInputTokens: number | null;
+  readonly outputTokens: number | null;
+  readonly reasoningTokens: number | null;
+  readonly estimatedCostUsd: number | null;
+  readonly creditUsed: number | null;
+  readonly resolveMs: number | null;
+  readonly tokensPerSec: number | null;
+  readonly firstContentDeltaAtMs: number | null;
+  readonly lastEventAtMs: number | null;
+  /** Why the drawer payloads are empty, or `captured`; absent when unrecorded. */
+  readonly payloadCaptureState?: string | null;
+  /** `true` for a degenerate-stop request; absent/null for everything else. */
+  readonly degenerateStop?: boolean;
+}
+
+/** Coerces a token count to the integer the schema columns store; `undefined` for an unavailable count. */
+export function tokenCountToInt(value: number | "unavailable" | undefined): number | undefined {
+  return typeof value === "number" ? Math.round(value) : undefined;
+}
+
+/**
+ * Maps a normalized event to its `telemetry_events` insert row.
+ *
+ * Every column below has a live producer on the write path, and the
+ * reconciliation backfill relies on this exact spelling to recover the same
+ * columns from a raw line. Contract details that are load-bearing:
+ *
+ * - `id = requestId` is what makes the unique index a working idempotency
+ *   boundary: a retry of the same request carries the same id.
+ * - `estimatedCostUsd` persists NULL when the route is unpriced, never the
+ *   text `"null"` — the analytics `partial` flag counts completed rows with no
+ *   persisted cost.
+ * - `creditUsed`/`tokensPerSec` use `!= null` so a measured zero is preserved
+ *   (numeric columns are string-typed in Drizzle).
+ * - `clientName` is derived here, once, via `clientNameFromUserAgent`, so the
+ *   write path and the backfill group the same client the same way.
+ */
+export function telemetryInsertRow(event: TelemetryInsertRowSource): typeof telemetryEvents.$inferInsert {
+  return {
+    id: event.requestId,
+    tenantId: event.tenantId,
+    requestId: event.requestId,
+    // Stamped here, not by the reader. A row without it has no real time, and a
+    // reader that invented one made every such row look like it happened just now.
+    createdAt: new Date(event.createdAt),
+    sourceSurface: event.sourceSurface as typeof telemetryEvents.$inferInsert.sourceSurface,
+    requestedModel: event.requestedModel,
+    reportedModel: event.reportedModel,
+    requestedEffort: event.requestedEffort,
+    endpoint: event.endpoint,
+    apiKeyId: event.apiKeyId,
+    userAgent: event.userAgent,
+    // Canonical client grouping, stamped once so SQL breakdowns group by it.
+    clientName: clientNameFromUserAgent(event.userAgent),
+    clientIp: event.clientIp,
+    providerId: event.providerId,
+    accountId: event.accountId,
+    networkPoolId: event.networkPoolId,
+    latencyMs: event.latencyMs,
+    ttfbMs: event.ttfbMs,
+    stream: event.stream,
+    httpStatus: event.httpStatus,
+    status: event.status as typeof telemetryEvents.$inferInsert.status,
+    errorCategory: event.errorCategory,
+    errorOrigin: event.errorOrigin,
+    inputTokens: event.inputTokens,
+    cachedInputTokens: event.cachedInputTokens,
+    outputTokens: event.outputTokens,
+    reasoningTokens: event.reasoningTokens,
+    // `estimated_cost` is `null` when the route is unpriced; persist NULL so
+    // the analytics `partial` flag can count it. `String(null)` would store the
+    // text "null", which the numeric column rejects.
+    estimatedCostUsd: event.estimatedCostUsd === null ? null : String(event.estimatedCostUsd),
+    // Provider-billed credit (Tencent buddy-meter `credit`, Qoder `credits`);
+    // NULL when the upstream reported no credit field.
+    creditUsed: event.creditUsed != null ? String(event.creditUsed) : null,
+    resolveMs: event.resolveMs,
+    tokensPerSec: event.tokensPerSec != null ? String(event.tokensPerSec) : null,
+    firstContentDeltaAtMs: event.firstContentDeltaAtMs,
+    lastEventAtMs: event.lastEventAtMs,
+    payloadCaptureState: event.payloadCaptureState ?? null,
+    degenerateStop: event.degenerateStop ?? null,
+  };
+}

package/src/persistence/db-mode.ts

@@ -48,7 +48,10 @@ export function resolveDbMode(env: NodeJS.ProcessEnv = process.env): DbMode {
  */
 export function resolveDataDir(env: NodeJS.ProcessEnv = process.env): string {
   const override = env.CARTETHYIA_DATA_DIR?.trim();
-  if (override) return override;
+  // Guard against a literal `"undefined"` value: a launcher that stringifies
+  // a missing env var (`String(undefined)`) would otherwise make the gateway
+  // write its state into a folder named `undefined/` in the CWD.
+  if (override && override !== "undefined") return override;
   if (process.platform === "win32") {
     const roaming = env.APPDATA?.trim();
     if (roaming) return join(roaming, "Cartethyia");

package/src/persistence/dialect.ts

@@ -16,6 +16,7 @@
 import { sql, type SQL } from "drizzle-orm";
 import { SQLiteBunSession } from "drizzle-orm/bun-sqlite";
 import { resolveDbMode } from "./db-mode";
+import { providerAccounts } from "./schema";
 
 /** True when the running backend is the embedded SQLite database. */
 export function sqliteDialect(): boolean {
@@ -55,6 +56,35 @@ function hasSqliteDialect(database: unknown): boolean {
   return (database as { session?: unknown } | undefined)?.session instanceof SQLiteBunSession;
 }
 
+/**
+ * Dialect of a concrete Drizzle handle, rather than of the process's env.
+ *
+ * A test can hold an in-memory SQLite database while `CARTETHYIA_DB_MODE`
+ * names `full` (the backend suite sets that globally). A raw SQL predicate
+ * chosen from the env would emit `jsonb_each_text` into SQLite and fail; the
+ * handle's session is the authority, exactly as `mergeJson` already treats it.
+ */
+export function isSqliteDatabase(database: unknown): boolean {
+  return hasSqliteDialect(database);
+}
+
+/**
+ * The `EXISTS` subquery for "the column's cooldown object has an entry whose
+ * ISO deadline is still in the future". The JSON check is dialect-specific:
+ * PostgreSQL uses `jsonb_each_text` (with a shape guard), SQLite `json_each`
+ * on its `text` column (D07). Timestamps are stored as ISO strings on both
+ * backends, so the lexical comparison against `nowIso` is stable.
+ *
+ * Callers keep their own scoping around this fragment — a tenant path must
+ * never touch `tenant_id IS NULL` rows, and the global path must only touch
+ * them — so only the shared JSON subquery lives here.
+ */
+export function liveModelCooldownExistsSql(database: unknown, nowIso: string): SQL {
+  return isSqliteDatabase(database)
+    ? sql`EXISTS (SELECT 1 FROM json_each(${providerAccounts.modelCooldowns}) AS entry WHERE json_valid(${providerAccounts.modelCooldowns}) AND json_type(${providerAccounts.modelCooldowns}) = 'object' AND entry.value > ${nowIso})`
+    : sql`EXISTS (SELECT 1 FROM jsonb_each_text(${providerAccounts.modelCooldowns}) AS entry WHERE jsonb_typeof(${providerAccounts.modelCooldowns}) = 'object' AND entry.value > ${nowIso})`;
+}
+
 /**
  * The per-model cooldown object with every entry whose timestamp has passed
  * removed, or `'{}'` when nothing remains.

package/src/persistence/migrate.ts

@@ -25,11 +25,14 @@ export function resolveMigrationsFolder(): string {
   return folder;
 }
 
-export function migrationFiles(folder: string): readonly string[] {
+export function migrationFileIds(folder: string): readonly string[] {
   return readdirSync(folder)
     .filter((file) => /^\d{4}_.+\.sql$/.test(file))
-    .sort()
-    .map((file) => resolve(folder, file));
+    .sort();
+}
+
+export function migrationFiles(folder: string): readonly string[] {
+  return migrationFileIds(folder).map((file) => resolve(folder, file));
 }
 
 export function migrationIdFor(file: string): string {

package/src/persistence/schema.ts

@@ -219,8 +219,8 @@ export interface NativeIdentityRecord {
   readonly machineCode: string;
   /** The seed-file content this identity was minted from, if captured. */
   readonly seed?: string;
-  /** How this identity was produced: the genuine SG bridge or the fallback. */
-  readonly source?: "runtime-info" | "generated";
+  /** How this identity was produced: the genuine SG bridge, the OS fingerprint, or the fallback. */
+  readonly source?: "runtime-info" | "generated" | "os";
 }
 
 export const providerAccounts = pgTable("provider_accounts", {
@@ -309,6 +309,14 @@ export const providerAccounts = pgTable("provider_accounts", {
    * to an account that may well be refused.
    */
   quotaPlan: text("quota_plan"),
+  /**
+   * When `quota_plan` was last written by a successful quota read.
+   *
+   * The plan column is a bare string with no timestamp, so routing cannot tell
+   * a fresh plan from one recorded weeks ago. With the freshness horizon, a
+   * stale plan is downgraded to `unknown` instead of being trusted blindly.
+   */
+  quotaPlanFetchedAt: timestamp("quota_plan_fetched_at", { withTimezone: true }),
   /**
    * The last successful quota read for this account, as a JSON snapshot.
    *
@@ -432,6 +440,11 @@ export const models = pgTable(
     reasoning: boolean("reasoning").notNull().default(false),
     toolCall: boolean("tool_call").notNull().default(false),
     cost: jsonb("cost"),
+    // Display-only warning metadata for this row: Postgres `jsonb` (SQLite
+    // `text` carrying the JSON string), nullable, `NULL` = no warnings. Shape:
+    // `{ warnings: [{ kind, message, severity?, updatedAt? }] }`. Part of the
+    // provider-account model-update proposal §3.4 area 06; not read by routing.
+    warningMetadata: jsonb("warning_metadata"),
     source: text("source"),
     sourceUpdatedAt: timestamp("source_updated_at", { withTimezone: true }),
     enabled: boolean("enabled").notNull().default(true),
@@ -1288,6 +1301,19 @@ export interface ConsoleSettingsPreferences {
    * placeholder behavior.
    */
   visionHandoffModels?: readonly string[];
+  /**
+   * Auto-disable provider accounts that keep failing with upstream rate-limit
+   * or quota verdicts. Off by default: without it, those failures land on a
+   * per-model/account cooldown and the account stays rotatable.
+   */
+  autoDisableRateLimited?: boolean;
+  /**
+   * Modelprobe mode: `detail` (full identity probe) or `fast` (minimal
+   * 32-token check). `detail` when unset.
+   */
+  modelProbeMode?: "fast" | "detail";
+  /** Fast-probe timeout in seconds; `null`/absent means the built-in default (10). */
+  modelProbeFastTimeoutSeconds?: number | null;
 }
 
 export const consoleSettings = pgTable("console_settings", {

package/src/persistence/ttl-cache.ts

@@ -11,6 +11,12 @@ interface TtlCacheOptions {
   readonly ttlMs?: number;
   readonly maxEntries?: number;
   readonly now?: () => number;
+  /**
+   * LRU eviction order. When set, re-setting an existing key moves it to the
+   * newest position (delete-then-set), so oldest-first eviction becomes
+   * least-recently-used-first. Off by default: plain FIFO insertion order.
+   */
+  readonly lru?: boolean;
 }
 
 interface TtlCacheEntry<T> {
@@ -27,11 +33,13 @@ export class TtlCache<T> {
   private readonly ttlMs: number;
   private readonly maxEntries: number;
   private readonly now: () => number;
+  private readonly lru: boolean;
 
   constructor(opts: TtlCacheOptions = {}) {
     this.ttlMs = opts.ttlMs ?? 5_000;
     this.maxEntries = opts.maxEntries ?? 128;
     this.now = opts.now ?? Date.now;
+    this.lru = opts.lru ?? false;
   }
 
   /**
@@ -54,10 +62,21 @@ export class TtlCache<T> {
    */
   set(key: string, value: T): void {
     const now = this.now();
+    if (this.lru) this.cache.delete(key); // move to newest position
     this.cache.set(key, { value, at: now });
     this.evictIfNeeded(now);
   }
 
+  /**
+   * Peek at a stored value without checking the TTL or deleting on expiry.
+   * Returns only undefined when the key is absent. Callers that need the
+   * possibly-expired value (e.g. a stale fallback after a failed refetch)
+   * must read it before `get`, which removes expired entries.
+   */
+  peek(key: string): T | undefined {
+    return this.cache.get(key)?.value;
+  }
+
   /**
    * Delete a specific key.
    */
@@ -144,8 +163,7 @@ interface TtlCacheFamilyOptions {
 /**
  * A set of {@link TtlCache} instances keyed by TTL, created on first use.
  *
- * Two provider-side caches (`operations/provider-version-cache.ts` and
- * `operations/model-discovery-cache.ts`) had each hand-rolled the same
+ * Two provider-side caches in `operations/provider-caches.ts` had each hand-rolled the same
  * `Map<number, TtlCache<T>>` + `cacheFor(ttlMs)` memo plus the same
  * `for (const cache of caches.values()) cache.clear()` reset. Tests pass
  * `ttlMs: 0` to force a reload, which is why the per-TTL split exists at all —

package/src/protocol/codecs.ts

@@ -14,7 +14,7 @@ import type { CanonicalStopReason, CanonicalTerminalEvent, UsageRecord } from ".
 import { type WireFamily } from "./canonical-model";
 import type { GenerationControls } from "./canonical-model";
 import { BASE_PROTECTED_HEADERS, HEADER_CONTROL, HEADER_TOKEN } from "../security/outbound-headers";
-import { unwrapProviderToken } from "../providers/provider-credential-envelope";
+import { unwrapProviderToken } from "../providers/provider-credentials";
 
 const TEXT_ENCODER = new TextEncoder();
 const TEXT_DECODER = new TextDecoder();

package/src/protocol/dsml.ts

@@ -5,9 +5,10 @@
  * markup inside the plain-text content stream instead of the structured
  * `tool_calls` field. The gateway sees it from one upstream family:
  *
- * - CodeBuddy/WorkBuddy emit strict blocks (`<｜DSML｜ calls>…</｜DSML｜ calls>`)
+ * - CodeBuddy emits strict blocks (`<｜DSML｜ calls>…</｜DSML｜ calls>`)
  *   that must be parsed exactly, with declared-tool allowlisting.
  */
+import { MarkupStreamParser } from "./markup-stream-parser";
 export interface DsmlToolCall {
   readonly id: string;
   readonly name: string;
@@ -74,102 +75,49 @@ interface DsmlParseResult {
  * With no allowlist the generic name rule applies; with one, undeclared names
  * reject the whole block. The caller feeds chunks with `push` and drains the
  * tail with `flush`.
+ *
+ * The stateful prefix-buffer loop is the shared {@link MarkupStreamParser}
+ * scaffold; this class supplies the DSML grammar as a thin configuration.
  */
 export class DsmlStreamParser {
-  readonly #separators: readonly string[];
-  readonly #blockNames: readonly string[];
-  readonly #openers: readonly string[];
-  readonly #allowedNames: ReadonlySet<string>;
-  readonly #params: "json" | "string";
-  readonly #sortKeys: boolean;
-  readonly #maxBlockBytes: number;
-  readonly #id: (call: { readonly name: string }, index: number, seq: number) => string;
-  #pending = "";
-  #callSequence = 0;
+  readonly #parser: MarkupStreamParser<DsmlToolCall>;
 
   constructor(options: DsmlParseOptions = {}) {
-    this.#separators = options.separators ?? DEFAULT_SEPARATORS;
-    this.#blockNames = options.blockNames ?? DEFAULT_BLOCK_NAMES;
-    this.#openers = openers(this.#separators, this.#blockNames);
-    this.#allowedNames = options.allowedNames ?? new Set();
-    this.#params = options.params ?? "json";
-    this.#sortKeys = options.sortKeys ?? true;
-    this.#maxBlockBytes = options.maxBlockBytes ?? DEFAULT_MAX_BLOCK_BYTES;
-    this.#id = options.id ?? sequentialId;
+    const separators = options.separators ?? DEFAULT_SEPARATORS;
+    const blockNames = options.blockNames ?? DEFAULT_BLOCK_NAMES;
+    const allowedNames = options.allowedNames ?? new Set();
+    const params = options.params ?? "json";
+    const sortKeys = options.sortKeys ?? true;
+    const maxBlockBytes = options.maxBlockBytes ?? DEFAULT_MAX_BLOCK_BYTES;
+    const id = options.id ?? sequentialId;
+    this.#parser = new MarkupStreamParser<DsmlToolCall>({
+      openers: openers(separators, blockNames),
+      classifyOpener: (_pending, _index, opener) => {
+        const separator = separators.find((value) => opener.startsWith(`<${value}DSML`));
+        const blockName = blockNames.find((value) => opener.endsWith(` ${value}>`));
+        if (separator === undefined || blockName === undefined) return { kind: "literal" };
+        const close = `</${separator}DSML${separator} ${blockName}>`;
+        return {
+          kind: "found",
+          closer: close,
+          parse: (block) =>
+            parseStrictBlock(block, opener, close, separator, allowedNames, params, sortKeys),
+          callId: (call, index, seq) => ({ ...call, id: id(call, index, seq) }),
+        };
+      },
+      onUnclosed: (pending, openIndex) =>
+        pending.length - openIndex > maxBlockBytes
+          ? { kind: "passThrough" }
+          : "hold",
+    });
   }
 
   push(chunk: string): DsmlParseResult {
-    this.#pending += chunk;
-    let cursor = 0;
-    let visible = "";
-    const calls: DsmlToolCall[] = [];
-    for (;;) {
-      let openIndex = -1;
-      let opener = "";
-      for (const candidate of this.#openers) {
-        const index = this.#pending.indexOf(candidate, cursor);
-        if (index >= 0 && (openIndex < 0 || index < openIndex)) {
-          openIndex = index;
-          opener = candidate;
-        }
-      }
-      if (openIndex < 0) {
-        const safeEnd = this.#settledEnd(cursor);
-        visible += this.#pending.slice(cursor, safeEnd);
-        cursor = safeEnd;
-        break;
-      }
-      visible += this.#pending.slice(cursor, openIndex);
-      const separator = this.#separators.find((value) => opener.startsWith(`<${value}DSML`));
-      const blockName = this.#blockNames.find((value) => opener.endsWith(` ${value}>`));
-      if (separator === undefined || blockName === undefined) {
-        visible += "<";
-        cursor = openIndex + 1;
-        continue;
-      }
-      const close = `</${separator}DSML${separator} ${blockName}>`;
-      const closeIndex = this.#pending.indexOf(close, openIndex + opener.length);
-      if (closeIndex < 0) {
-        // An unclosed block past the cap is not markup — pass it through as
-        // text instead of buffering it indefinitely. Same fail-open as
-        // `feed()` in the reference bridge's `MAX_BLOCK_BYTES` guard.
-        if (this.#pending.length - openIndex > this.#maxBlockBytes) {
-          visible += this.#pending.slice(openIndex);
-          cursor = this.#pending.length;
-          break;
-        }
-        cursor = openIndex;
-        break;
-      }
-      const end = closeIndex + close.length;
-      const block = this.#pending.slice(openIndex, end);
-      const parsed = parseStrictBlock(block, opener, close, separator, this.#allowedNames, this.#params, this.#sortKeys);
-      if (parsed === undefined) visible += block;
-      else {
-        parsed.forEach((call, index) => {
-          calls.push({ ...call, id: this.#id(call, index, this.#callSequence++) });
-        });
-      }
-      cursor = end;
-    }
-    this.#pending = this.#pending.slice(cursor);
-    return { text: visible, calls };
+    return this.#parser.push(chunk);
   }
 
   flush(): DsmlParseResult {
-    const text = this.#pending;
-    this.#pending = "";
-    return { text, calls: [] };
-  }
-
-  #settledEnd(start: number): number {
-    const remaining = this.#pending.slice(start);
-    const maxOpener = this.#openers.reduce((max, opener) => Math.max(max, opener.length - 1), 0);
-    for (let length = Math.min(maxOpener, remaining.length); length > 0; length -= 1) {
-      const suffix = remaining.slice(-length);
-      if (this.#openers.some((opener) => opener.startsWith(suffix))) return this.#pending.length - length;
-    }
-    return this.#pending.length;
+    return this.#parser.flush();
   }
 }
 

package/src/protocol/gateway-error.ts

@@ -174,6 +174,7 @@ const PUBLIC_ERROR_DETAIL_KEYS = new Set([
   "status",
   "upstreamRequestId",
   "upstreamStatus",
+  "upstreamMessageAsIs",
   "maxInflight",
   "weight",
 ]);

package/src/protocol/markup-stream-parser.ts

@@ -0,0 +1,195 @@
+/**
+ * Generic streaming prefix-buffer parser for in-band tool-call markup.
+ *
+ * Three grammars (DSML blocks, OMP `<tool_calls:8hex>` envelopes, Qwen
+ * `<tool_call>` JSON blocks) share one stateful loop: append chunks to a
+ * pending buffer, scan for the earliest opener, settle visible text up to a
+ * safe end that holds back a possible trailing opener fragment, parse a
+ * complete block when its closer appears, fail open past a block-size cap, and
+ * flush the tail. The grammar-specific parts are supplied as configuration:
+ * opener strings, an opener classifier (some grammars must validate beyond the
+ * bare prefix, e.g. OMP's hex marker), closer/parse functions, an unclosed
+ * block policy (hold vs fail-open pass-through), a flush policy (pass through
+ * vs discard an unterminated envelope), and a call-id factory. Each parser
+ * instance keeps its own buffer; nothing is shared.
+ */
+
+/** Result of one push/flush pass: text that settled plus recovered calls. */
+export interface MarkupStreamParseResult<Call extends { readonly id: string }> {
+  readonly text: string;
+  readonly calls: readonly Call[];
+}
+
+/** Raw call shape produced by a grammar's block parser, without its id. */
+export type MarkupRawCall<Call extends { readonly id: string }> = Omit<Call, "id">;
+
+/** Builds a complete call (id included) from a parsed raw call. */
+export type MarkupCallFactory<Call extends { readonly id: string }> = (
+  call: MarkupRawCall<Call>,
+  index: number,
+  seq: number,
+) => Call;
+
+/** Classification of one opener candidate occurrence. */
+export type MarkupOpenerClassification<Call extends { readonly id: string }> =
+  | {
+      /**
+       * The candidate may be the start of a real opener (e.g. a partial hex
+       * marker): hold everything from the candidate until a later chunk
+       * completes or disproves it.
+       */
+      readonly kind: "hold";
+    }
+  | {
+      /**
+       * The candidate is definitely not an opener: emit it as literal text
+       * (one character) and rescan after it.
+       */
+      readonly kind: "literal";
+    }
+  | {
+      readonly kind: "found";
+      /** Tag that closes the block opened at this position. */
+      readonly closer: string;
+      /**
+       * Parses one complete block (opener through closer). `undefined` means
+       * the block is not valid markup and is passed through as text verbatim.
+       */
+      readonly parse: (block: string) => readonly MarkupRawCall<Call>[] | undefined;
+      /**
+       * Call factory for the calls recovered from this block; defaults to the
+       * parser-level `callId`. Some grammars derive ids from block-local state
+       * (e.g. OMP embeds the envelope marker).
+       */
+      readonly callId?: MarkupCallFactory<Call>;
+    };
+
+export interface MarkupStreamParserOptions<Call extends { readonly id: string }> {
+  /**
+   * Opener frontier. The earliest occurrence wins; trailing fragments that
+   * could start one are held back from settled text.
+   */
+  readonly openers: readonly string[];
+  /** Classifies an opener candidate found at a scan position. */
+  readonly classifyOpener: (
+    pending: string,
+    index: number,
+    opener: string,
+  ) => MarkupOpenerClassification<Call>;
+  /**
+   * Policy when a block's closer has not arrived yet. Default `"hold"`: keep
+   * the whole pending buffer from the opener until a later chunk completes or
+   * disproves the block. Returning `{ kind: "passThrough" }` fails open:
+   * everything from the opener is emitted as text immediately (the DSML
+   * block-size cap).
+   */
+  readonly onUnclosed?: (
+    pending: string,
+    openIndex: number,
+  ) => "hold" | { readonly kind: "passThrough" };
+  /**
+   * Tail policy at flush. Returns the text to emit for whatever remains in the
+   * buffer (default: everything, i.e. pass through). A grammar that discards
+   * unterminated envelopes returns `""` when the tail looks like an opener.
+   */
+  readonly flush?: (pending: string) => string;
+  /** Call factory for recovered calls, unless a block overrides it. */
+  readonly callId?: MarkupCallFactory<Call>;
+}
+
+export class MarkupStreamParser<Call extends { readonly id: string }> {
+  readonly #options: MarkupStreamParserOptions<Call>;
+  #pending = "";
+  #callSequence = 0;
+
+  constructor(options: MarkupStreamParserOptions<Call>) {
+    this.#options = options;
+  }
+
+  /** Feeds one chunk and returns only settled text and complete calls. */
+  push(chunk: string): MarkupStreamParseResult<Call> {
+    this.#pending += chunk;
+    let cursor = 0;
+    let visible = "";
+    const calls: Call[] = [];
+    for (;;) {
+      let openIndex = -1;
+      let opener = "";
+      for (const candidate of this.#options.openers) {
+        const index = this.#pending.indexOf(candidate, cursor);
+        if (index >= 0 && (openIndex < 0 || index < openIndex)) {
+          openIndex = index;
+          opener = candidate;
+        }
+      }
+      if (openIndex < 0) {
+        const safeEnd = this.#settledEnd(cursor);
+        visible += this.#pending.slice(cursor, safeEnd);
+        cursor = safeEnd;
+        break;
+      }
+      visible += this.#pending.slice(cursor, openIndex);
+      const classified = this.#options.classifyOpener(this.#pending, openIndex, opener);
+      if (classified.kind === "literal") {
+        visible += "<";
+        cursor = openIndex + 1;
+        continue;
+      }
+      if (classified.kind === "hold") {
+        cursor = openIndex;
+        break;
+      }
+      const closeIndex = this.#pending.indexOf(classified.closer, openIndex + opener.length);
+      if (closeIndex < 0) {
+        const decision = this.#options.onUnclosed?.(this.#pending, openIndex) ?? "hold";
+        if (decision === "hold") {
+          cursor = openIndex;
+          break;
+        }
+        visible += this.#pending.slice(openIndex);
+        cursor = this.#pending.length;
+        break;
+      }
+      const end = closeIndex + classified.closer.length;
+      const block = this.#pending.slice(openIndex, end);
+      const parsed = classified.parse(block);
+      if (parsed === undefined) {
+        visible += block;
+      } else {
+        const callFor = classified.callId ?? this.#options.callId;
+        if (callFor === undefined) {
+          throw new Error("MarkupStreamParser: no call factory configured");
+        }
+        parsed.forEach((call, index) => {
+          calls.push(callFor(call, index, this.#callSequence++));
+        });
+      }
+      cursor = end;
+    }
+    this.#pending = this.#pending.slice(cursor);
+    return { text: visible, calls };
+  }
+
+  /** Settles trailing text per the configured flush policy. */
+  flush(): MarkupStreamParseResult<Call> {
+    const text = this.#options.flush?.(this.#pending) ?? this.#pending;
+    this.#pending = "";
+    return { text, calls: [] };
+  }
+
+  /** Holds back a trailing fragment that could be the start of an opener. */
+  #settledEnd(start: number): number {
+    const remaining = this.#pending.slice(start);
+    const maxOpener = this.#options.openers.reduce(
+      (max, open) => Math.max(max, open.length - 1),
+      0,
+    );
+    for (let length = Math.min(maxOpener, remaining.length); length > 0; length -= 1) {
+      const suffix = remaining.slice(-length);
+      if (this.#options.openers.some((open) => open.startsWith(suffix))) {
+        return this.#pending.length - length;
+      }
+    }
+    return this.#pending.length;
+  }
+}