cartethyia 2.5.3 → 2.5.6

package/src/persistence/db-mode.ts

@@ -8,17 +8,17 @@ export type DbMode = "full" | "lite";
 /**
  * Reads `CARTETHYIA_DB_MODE`, which is required and has no default.
  *
- * It used to default to `full`, which left the same key carrying three
- * different answers: `.env.example` shipped `lite`, the `bun setup` prompt
- * offered `lite`, and an environment that set nothing at all booted `full`.
- * The third one is the one that bit — a clone without a `.env` came up in Full
- * mode and then failed on a missing `DATABASE_URL`, when the documented intent
- * was the self-contained Lite path.
+ * It must not default to `full`: that leaves the same key carrying three
+ * different answers, since `.env.example` ships `lite`, the `bun setup` prompt
+ * offers `lite`, and an environment that sets nothing at all would boot
+ * `full`. The third one is the one that bites — a clone without a `.env` comes
+ * up in Full mode and then fails on a missing `DATABASE_URL`, when the
+ * documented intent is the self-contained Lite path.
  *
  * Failing closed is the honest fix: the two modes are different databases with
  * different migration sets (`migrations/lite-sqlite3/` vs
  * `migrations/full-postgres/`), so guessing one silently picks which migrations
- * an operator's data is read against. `bun setup`, `scripts/install/lite.sh`,
+ * an operator's data is read against. `bun setup`, `scripts/install.sh`,
  * `.env.example`, and `docker-compose.yml` all set it explicitly,
  * so requiring it costs an existing deployment nothing.
  */

package/src/persistence/health-event-flush.ts

@@ -67,6 +67,12 @@ export class HealthEventFlushBuffer {
   private timer: ReturnType<typeof setInterval> | undefined;
   private flushing = false;
   private dropped = 0;
+  /**
+   * Settles once while an INSERT is in flight, so a purge that drops the
+   * queue can wait that write out instead of letting it land behind its own
+   * `DELETE` and resurrect the rows it just removed.
+   */
+  private inflightDone: Promise<void> | undefined;
 
   constructor(
     private readonly db: CartethyiaDatabase,
@@ -97,10 +103,38 @@ export class HealthEventFlushBuffer {
     return this.dropped;
   }
 
+  /** Drops queued rows — everything, or one account's — without writing them. */
+  clear(accountId?: string): void {
+    if (accountId === undefined) {
+      this.pending.length = 0;
+      return;
+    }
+    for (let i = this.pending.length - 1; i >= 0; i -= 1) {
+      if (this.pending[i]?.accountId === accountId) this.pending.splice(i, 1);
+    }
+  }
+
+  /**
+   * Drops queued rows and then blocks until a write already in flight lands.
+   *
+   * The barrier is what makes a purge sound: deleting only the rows visible at
+   * one instant is final, while an insert in flight for the same instant would
+   * re-add rows after the `DELETE` finished. Rows enqueued *after* the drop
+   * are new events, not resurrected ones, so they are left alone.
+   */
+  async drain(accountId?: string): Promise<void> {
+    this.clear(accountId);
+    await this.inflightDone;
+  }
+
   /** Writes everything accumulated so far. Safe to call concurrently. */
   async flush(): Promise<void> {
     if (this.flushing || this.pending.length === 0) return;
     this.flushing = true;
+    let settle!: () => void;
+    this.inflightDone = new Promise<void>((resolve) => {
+      settle = resolve;
+    });
     // Drained before the await: rows enqueued while the insert runs stay for
     // the next pass rather than being lost if this one throws.
     const batch = this.pending.splice(0, this.pending.length);
@@ -123,6 +157,8 @@ export class HealthEventFlushBuffer {
       // are dropped; the durable status change was written synchronously.
     } finally {
       this.flushing = false;
+      this.inflightDone = undefined;
+      settle();
     }
   }
 

package/src/persistence/readiness.ts

@@ -9,7 +9,7 @@ import { withTimeout } from "../boot/timeout";
 /**
  * Memoized readiness: the first caller runs the real `SELECT 1`/`PING`/
  * migration checks, and every subsequent caller within the TTL window reuses
- * that result — so a burst of concurrent requests no longer each spend a
+ * that result — so a burst of concurrent requests does not each spend a
  * Postgres pool checkout plus a Redis PING on readiness validation.
  */
 

package/src/persistence/schema.ts

@@ -182,7 +182,7 @@ export const providers = pgTable(
 // `tenant_id` null means the account is shared pool-wide; populated means
 // tenant-owned/BYOK. The concurrency ceiling lives in provider routing
 // (`provider_routing_settings.max_inflight`) and network pools
-// (`network_pools.max_inflight`); the per-account override no longer exists.
+// (`network_pools.max_inflight`); there is no per-account override.
 export const providerAccounts = pgTable("provider_accounts", {
   id: uuid("id").primaryKey().defaultRandom(),
   providerId: text("provider_id")
@@ -469,11 +469,11 @@ export const networkPools = pgTable("network_pools", {
    * (`egress_ip`, `last_latency_ms`, the speed-test columns) rather than in the
    * browser.
    *
-   * It lived in `localStorage` first, and that made it decorative: the verdict
-   * was visible to the operator who ran it and to nobody else — not to another
-   * machine, not to the gateway, and not to admission. A pool could read
-   * "OpenCode blocked" on screen while dispatch kept sending traffic through
-   * it, because nothing outside that one browser had ever heard the verdict.
+   * Browser storage is not enough: a verdict held only there is visible to the
+   * operator who ran it and to nobody else — not to another machine, not to the
+   * gateway, and not to admission. A pool would read "OpenCode blocked" on
+   * screen while dispatch kept sending traffic through it, because nothing
+   * outside that one browser had heard the verdict.
    *
    * `null` means never probed. These describe the *address* recorded in
    * `egress_ip` at the time, so a changed endpoint clears them.
@@ -527,7 +527,7 @@ export const healthEvents = pgTable(
     index("health_events_account_id_created_at_idx").on(table.accountId, table.createdAt),
     // Descending `created_at` is deliberate: health history is always read
     // newest-first. The SQLite baseline drops the direction (see
-    // `scripts/generate/sqlite-ddl.ts` — SQLite scans an index either way),
+    // `scripts/tools/generate-sqlite.ts` — SQLite scans an index either way),
     // so this is a Postgres-only access-path choice, not a cross-backend
     // disagreement.
     index("health_events_account_model_created_idx").on(
@@ -544,8 +544,7 @@ export const healthEvents = pgTable(
 );
 
 
-// Model aliasing & combos: replaces the dormant, disconnected
-// `routing_policies` scaffolding. Aliases resolve a client-facing name to a
+// Model aliasing & combos. Aliases resolve a client-facing name to a
 // real model; combos back a client-facing name with >=1 real models,
 // auto-selected via `fallback` or `round_robin`.
 export const modelComboStrategy = pgEnum("model_combo_strategy", [
@@ -556,7 +555,7 @@ export const modelComboStrategy = pgEnum("model_combo_strategy", [
 /** Canonical `ComboStrategy` union, derived from the enum above. Console
  * domain and routing layers import this instead of hand-typing their own mirror.
  * `ProviderRoutingStrategy` covers the same two values but persists on
- * `provider_routing_settings` beside `rotateCount`; sticky routing is gone. */
+ * `provider_routing_settings` beside `rotateCount`. */
 export type ComboStrategy = (typeof modelComboStrategy.enumValues)[number];
 
 export const modelAliases = pgTable(
@@ -635,9 +634,8 @@ export const providerRoutingSettings = pgTable(
      * provider; this one bounds one provider's share of one pool. All three
      * are enforced together — none replaces another.
      *
-     * `null` = unlimited, and that is the default: a provider nobody has
-     * configured behaves exactly as it did before this column existed, with no
-     * extra Redis key and no extra round trip.
+     * `null` = unlimited, and that is the default: an unconfigured provider
+     * costs no extra Redis key and no extra round trip.
      */
     maxInflightPerProxy: integer("max_inflight_per_proxy"),
     /**
@@ -752,7 +750,7 @@ export const apiKeys = pgTable(
     /**
      * `whitelist` = only the names in `modelList` may be used (empty = all);
      * `blacklist` = the names in `modelList` are refused (empty = none). One
-     * list, one mode — the old separate allow/deny columns are retired.
+     * list, one mode.
      */
     modelAccessMode: text("model_access_mode")
       .$type<ApiKeyModelAccessMode>()
@@ -787,11 +785,9 @@ export const apiKeys = pgTable(
     /**
      * One *live* default key per tenant, enforced by the database.
      *
-     * The baseline SQL has always created this; the schema never declared it,
-     * so the two descriptions disagreed and nothing noticed. It is the only
-     * database-level guarantee behind `isDefault`, and
+     * This is the only database-level guarantee behind `isDefault`, and
      * `test/persistence/sqlite-baseline.test.ts` pins it by name — so the
-     * schema was the incomplete side, not the migration.
+     * schema must keep declaring it to match the baseline.
      *
      * Partial on `revoked_at IS NULL` so a tenant can rotate its default key
      * without the old, revoked row blocking the new one.
@@ -879,8 +875,7 @@ export const shareLinks = pgTable(
       .references(() => apiKeys.id, { onDelete: "cascade" }),
     tokenHash: text("token_hash").notNull(),
     // Encrypted bearer token. Present so the console can re-display a stable
-    // link; historical rows written before this column carry NULL and simply
-    // cannot be shown again.
+    // link; nullable, and a NULL row simply cannot be shown again.
     tokenEncrypted: bytea("token_encrypted"),
     /**
      * The public slug this page is reachable at, in the clear.
@@ -891,7 +886,7 @@ export const shareLinks = pgTable(
      * unencrypted keeps every restored share page reachable under the address
      * it was created with.
      *
-     * Nullable: rows written before this column have no slug, and inventing one
+     * Nullable: a row without a slug has no address, and inventing one
      * would publish an address the page never had.
      */
     slug: text("slug"),
@@ -903,8 +898,8 @@ export const shareLinks = pgTable(
   },
   (table) => [
     uniqueIndex("share_links_token_hash_idx").on(table.tokenHash),
-    // Partial: rows without a slug (written before the column) are not
-    // constrained, so backfilling is never blocked by a uniqueness collision.
+    // Partial: rows without a slug are not constrained, so backfilling is never
+    // blocked by a uniqueness collision.
     uniqueIndex("share_links_slug_uidx").on(table.slug).where(sql`${table.slug} IS NOT NULL`),
     index("share_links_api_key_idx").on(table.apiKeyId),
     index("share_links_active_kind_expires_idx").on(table.active, table.kind, table.expiresAt),
@@ -1045,7 +1040,59 @@ export interface ConsoleSettingsPreferences {
    * - `maximum` — 16 MiB combined cap, expect CPU/memory spikes while active
    */
   telemetryPayloadDepth?: "minimum" | "moderate" | "maximum";
+  /**
+   * How long a captured payload frame is kept, in whole minutes.
+   *
+   * `null` or absent means the deployment default
+   * (`CARTETHYIA_TELEMETRY_PAYLOAD_RETENTION_MS`, 15 minutes). Raising it is a
+   * real trade: a frame is a gzipped copy of up to four request bodies, so
+   * keeping them for a week is keeping a week of bodies on disk. The upper
+   * bound is seven days, enforced by the capture path rather than trusted here.
+   */
+  telemetryPayloadRetentionMinutes?: number | null;
   privacyMode?: "masked" | "full";
+  /**
+   * Ignores an account-wide cooldown when building route candidates.
+   *
+   * Off by default, and deliberately dangerous: an account in cooldown was
+   * parked because the upstream refused it, and dialing it anyway reproduces
+   * that refusal on every attempt. Turning this on can exhaust the candidate
+   * list and fail a request that would otherwise have been routed.
+   *
+   * The override is read-only — the cooldown is still recorded, still shown in
+   * Health, and still recovered by the sweep. Nothing is written back, so the
+   * toggle can be turned off again without cleaning anything up.
+   *
+   * Per-(account, model) cooldowns are NOT overridden. Those are usually a
+   * named quota ("daily free limit reached, try again in 11h") where the
+   * upstream stated a real reset; ignoring it only buys repeated refusals.
+   */
+  forceDisableCooldown?: boolean;
+  /**
+   * Per-tenant upstream request timeout in seconds, overriding
+   * `CARTETHYIA_UPSTREAM_TIMEOUT_MS`. `null` or absent means use the
+   * deployment default.
+   */
+  upstreamTimeoutSeconds?: number | null;
+  /**
+   * Per-tenant telemetry compaction age in days, overriding
+   * `CARTETHYIA_TELEMETRY_COMPACT_AFTER_DAYS`. `null` or absent means use the
+   * deployment default. Lower keeps less history but bounds the usage-list
+   * scan; see the compaction notes in `event-compaction.ts`.
+   */
+  telemetryWindowMinutes?: number | null;
+  /**
+   * Per-tenant ceiling for one inline attachment, in bytes, overriding the
+   * deployment default of 10 MB.
+   *
+   * Raising this asserts that this tenant's upstreams accept larger
+   * attachments. The clamp is a hard send gate precisely because an oversized
+   * block poisons every retry and every replayed turn — the upstream rejects
+   * the whole request rather than the one block — so an override larger than
+   * the provider actually accepts trades that protection away. Lowering it is
+   * the safer direction and the more common need.
+   */
+  imageClampMaxBytes?: number | null;
 }
 
 export const consoleSettings = pgTable("console_settings", {
@@ -1247,9 +1294,8 @@ export const telemetryPayloads = pgTable(
     expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
     /**
      * Typed index into the on-disk frame store. Prompt/response bodies live
-     * only in the frame file these columns name — never in Postgres. The
-     * previous `request_body jsonb` column is gone so a body cannot be
-     * written here by accident.
+     * only in the frame file these columns name — never in Postgres, which is
+     * why this table carries no body column to write one into by accident.
      */
     storage: text("storage").notNull(),
     file: text("file").notNull(),
@@ -1264,8 +1310,8 @@ export const telemetryPayloads = pgTable(
      * reads it off the indexed row.
      *
      * Bits: 1=request, 2=provider_request, 4=provider_response,
-     * 8=client_response. Nullable: NULL means the row predates this column,
-     * which renders as all icons greyed rather than inventing presence.
+     * 8=client_response. Nullable: NULL renders as all icons greyed rather
+     * than inventing presence.
      */
     stages: integer("stages"),
   },
@@ -1273,7 +1319,7 @@ export const telemetryPayloads = pgTable(
     index("telemetry_payloads_request_id_idx").on(table.requestId),
     index("telemetry_payloads_tenant_request_idx").on(table.tenantId, table.requestId),
     index("telemetry_payloads_expires_idx").on(table.expiresAt),
-    check("telemetry_payloads_storage_check", sql`${table.storage} IN ('jsonb-file', 'jsonl-file')`),
+    check("telemetry_payloads_storage_check", sql`${table.storage} = 'jsonl-file'`),
     check("telemetry_payloads_version_check", sql`${table.version} = 1`),
     check("telemetry_payloads_offset_check", sql`${table.offset} >= 0`),
     check("telemetry_payloads_length_check", sql`${table.length} > 0`),

package/src/persistence/shared-cache.ts

@@ -1,7 +1,6 @@
 // Cross-process cache coordination: a shared L2 in front of Postgres.
 //
-// Every hot read in the request path is cached in-process (see
-// `documentation/research/iomemory/01-request-path-census.md`). That is correct
+// Every hot read in the request path is cached in-process. That is correct
 // for one process and wrong for several: `main.ts` sets `reusePort: true`, so N
 // gateway processes each hold their own copy. With no shared layer, a key
 // revoked through the console stays valid for up to a full TTL in every other

package/src/persistence/stores/api-key-store.ts

@@ -187,6 +187,7 @@ export interface ApiKeyStore {
   reorder(tenantId: string, keyIds: readonly string[]): Promise<void>;
   update(tenantId: string, keyId: string, patch: ApiKeyPatch): Promise<ApiKeyRecord | undefined>;
   revoke(tenantId: string, keyId: string, revokedAt: Date): Promise<boolean>;
+  remove(tenantId: string, keyId: string): Promise<boolean>;
   findActiveByHash?(hash: string): Promise<typeof apiKeys.$inferSelect | undefined>;
   findActiveById?(keyId: string): Promise<typeof apiKeys.$inferSelect | undefined>;
   sumChildrenConsumed?(parentKeyId: string): Promise<number>;
@@ -380,10 +381,6 @@ export class DrizzleApiKeyStore implements ApiKeyStore {
         .set({
           ...(patch.enabled !== undefined ? { enabled: patch.enabled } : {}),
           ...(patch.label !== undefined ? { label: patch.label } : {}),
-          // Promotes a legacy default key: `isDefaultGatewayApiKey` recognizes
-          // it by its canonical label, but recognition was never written back,
-          // so the row stayed unflagged and a later rename — which changes the
-          // only field that identifies it — dropped the default entirely.
           ...(patch.isDefault !== undefined ? { isDefault: patch.isDefault } : {}),
           ...(patch.keyHash !== undefined ? { keyHash: patch.keyHash } : {}),
           ...(patch.keyMode !== undefined ? { keyMode: patch.keyMode } : {}),
@@ -413,8 +410,8 @@ export class DrizzleApiKeyStore implements ApiKeyStore {
             : {}),
           // Scopes are validated once, in `validateApiKeyRequest`, against the
           // tenant scope set; the store writes what it is handed. Omitting the
-          // field here meant an update was type-checked, accepted, and then
-          // silently dropped — a grant like `routing:cli_mapping` could be
+          // field here means an update is type-checked, accepted, and then
+          // silently dropped — a grant like `routing:cli_mapping` can be
           // turned on and never stick.
           ...(patch.scopes === undefined ? {} : { scopes: patch.scopes }),
         })
@@ -453,6 +450,21 @@ export class DrizzleApiKeyStore implements ApiKeyStore {
       return true;
     });
   }
+  async remove(tenantId: string, keyId: string): Promise<boolean> {
+    return this.db.transaction(async (tx) => {
+      const rows = await tx
+        .delete(apiKeys)
+        .where(
+          and(
+            eq(apiKeys.tenantId, tenantId),
+            eq(apiKeys.id, keyId),
+            eq(apiKeys.isDefault, false),
+          ),
+        )
+        .returning({ id: apiKeys.id });
+      return rows.length > 0;
+    });
+  }
   async findActiveByHash(hash: string): Promise<typeof apiKeys.$inferSelect | undefined> {
     // The SQLite backend holds this lookup prepared. The builder path rebuilds
     // the query per call, which measured 60x the lookup itself, so the hot

package/src/persistence/stores/share-store.ts

@@ -308,9 +308,10 @@ export class DrizzleShareLinkStore implements ShareLinkStore {
           .set({
             tokenHash: input.tokenHash,
             tokenEncrypted: input.tokenEncrypted,
-            // The slug is the address and survives a rotate, so an operator who
-            // rotates does not invalidate every copy of the link they shared.
-            ...(current.slug === null ? { slug: input.slug } : {}),
+            // Rotating the slug changes the public URL, so the old link stops resolving.
+            // Callers always generate a fresh slug via generateShareSlug() before calling
+            // create() with rotate: true.
+            slug: input.slug,
             expiresAt: input.expiresAt,
             lastViewedAt: null,
           })

package/src/persistence/stores/telemetry-store.ts

@@ -10,6 +10,8 @@ import {
   type TelemetryUsageIdentityType,
 } from "../schema";
 import { isGatewayError } from "../../observability/telemetry-status";
+import { TELEMETRY_EVENT_DIR } from "../../observability/event-log";
+import { dayEventFileName, utcDay } from "../../observability/event-day-index";
 
 interface UsageTotalDelta {
   readonly tenantId: string;
@@ -66,27 +68,47 @@ function aggregateUsage(rows: readonly (typeof telemetryEvents.$inferInsert)[])
  * operations, lifetime aggregates, and retention.
  */
 
-const EVENT_DIR = "./data/telemetry-events";
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 
 /**
- * Deletes hourly event files whose whole hour is older than the retention
- * cutoff. The current hour's file is never touched: it is still being appended.
+ * Deletes whole days past the retention cutoff: the event file and its index
+ * together.
+ *
+ * The day is the retention unit now, so both go at once. Removing one without
+ * the other is the failure this avoids — an index with no events reports totals
+ * for requests that no longer exist, and events with no index are invisible to
+ * every usage read.
+ *
+ * The cutoff is a whole day, so today's file — still being appended to — is
+ * never a candidate. Comparing filenames hour by hour would match a day file's
+ * `.events` suffix against an hour stamp and could drop the open file.
  */
 async function pruneEventFiles(before: Date): Promise<void> {
-  const files = await readdir(EVENT_DIR).catch(() => [] as string[]);
-  const cutoff = before.toISOString().slice(0, 13).replaceAll(":", "-");
+  const files = await readdir(TELEMETRY_EVENT_DIR).catch(() => [] as string[]);
+  const cutoffDay = before.toISOString().slice(0, 10);
   for (const file of files) {
-    if (!file.endsWith(".jsonl")) continue;
-    const hour = file.slice(0, 13);
-    if (hour < cutoff) await rm(join(EVENT_DIR, file), { force: true });
+    const day = file.slice(0, 10);
+    if (day.length !== 10 || day >= cutoffDay) continue;
+    if (!file.endsWith(".events.jsonl") && !file.endsWith(".index.json")) continue;
+    await rm(join(TELEMETRY_EVENT_DIR, file), { force: true });
   }
 }
 
-/** Appends one JSON line per event. One file per hour, so retention is a file delete. */
+/**
+ * Appends one JSON line per event to the day's event file.
+ *
+ * One file per UTC day, appended to and never rewritten while the day is open.
+ * A file per `windowMinutes` window — 96 a day at the default — would have to be
+ * found, opened and parsed one by one on every usage query, and which of those
+ * files was still being written could only be inferred from the name, which is
+ * what lets the open one be deleted mid-write.
+ *
+ * A single growing file has no such ambiguity, and the day index tracks how much
+ * of it has been counted by byte offset.
+ */
 async function appendEvents(rows: readonly (typeof telemetryEvents.$inferInsert)[]): Promise<void> {
-  await mkdir(EVENT_DIR, { recursive: true });
-  const file = join(EVENT_DIR, new Date().toISOString().slice(0, 13).replaceAll(":", "-") + ".jsonl");
+  await mkdir(TELEMETRY_EVENT_DIR, { recursive: true });
+  const file = join(TELEMETRY_EVENT_DIR, dayEventFileName(utcDay(new Date())));
   const newline = String.fromCharCode(10);
   const body = rows.map((row) => JSON.stringify(row)).join(newline) + newline;
   await appendFile(file, body, "utf8");

package/src/protocol/canonical-model.ts

@@ -308,6 +308,13 @@ export type ContentPart =
       /** Responses reasoning summary block index, preserved across surfaces. */
       summary_index?: number;
       signature?: string;
+      /**
+       * Responses item id the `encrypted_content` was issued for. The upstream
+       * binds encrypted reasoning to its item and rejects a replay whose item
+       * id does not match, so the id travels with the payload it belongs to
+       * rather than being re-derived at request time.
+       */
+      item_id?: string;
       encrypted_content?: string;
     }
   | { kind: "extension"; name: string; payload: unknown };
@@ -596,9 +603,10 @@ export function hasEncryptedReasoning(parts: readonly ContentPart[]): boolean {
  * Single-pass summary of a request's content parts.
  *
  * `deriveRequiredCapabilities` + `assessTranslation` run on every candidate
- * preflight and previously each flattened the whole request (`allParts`) and
- * then scanned the copy up to ~6 times. This walks the messages once, builds
- * no intermediate array, and answers every predicate the pipeline needs.
+ * preflight, and flattening the whole request (`allParts`) in each and then
+ * scanning the copy up to ~6 times is the cost this avoids. It walks the
+ * messages once, builds no intermediate array, and answers every predicate the
+ * pipeline needs.
  * Results are memoized per request object in a `WeakMap`, so the preflight +
  * translate passes over one request share a single summary and entries are
  * collected with the request itself — no manual invalidation, no leak.