cartethyia 2.5.3 → 2.5.6

package/src/console/auth/credentials.ts

@@ -28,9 +28,9 @@ export class ConsoleCredentialService {
    * undecodable hashes: a corrupted `password_hash` row must surface as an
    * ordinary failed login (counted by the lockout service), never as a 500
    * that lets an attacker probe hash integrity or bypass the failure counter.
-   * Legacy/migrated hashes are rejected here rather than verified — a hash
-   * that does not start with `$argon2id$` can only originate from a foreign
-   * scheme this deployment never issued.
+   * A hash that does not start with `$argon2id$` is rejected here rather than
+   * verified — it can only originate from a foreign scheme this deployment
+   * never issued.
    */
   async verifyPassword(password: string, hash: string): Promise<boolean> {
     if (

package/src/console/auth/setup-service.ts

@@ -160,8 +160,8 @@ export class FirstBootSetupService {
     // First boot creates the default tenant, the platform-admin user, and the
     // gateway key. The advisory lock is what stops two concurrent boots from
     // both passing the "no admin yet" check and racing to create them, so a
-    // handle that cannot take the lock must fail closed — the previous
-    // `typeof` probe skipped the lock and ran the setup unserialized.
+    // handle that cannot take the lock must fail closed — running the setup
+    // unserialized lets two boots both pass the check.
     await this.db.transaction(async (transaction) => {
       await transaction.execute(firstBootSetupLock());
       await create(transaction as unknown as CartethyiaDatabase);

package/src/console/backup/contracts.ts

@@ -20,6 +20,7 @@
  * as sensitive as the database.
  */
 import { getTableColumns, getTableName, type Column, type Table } from "drizzle-orm";
+import { resolveMaxBodyBytes } from "../../config/runtime";
 import {
   apiKeys,
   cliToolMappings,
@@ -45,8 +46,20 @@ import {
 export const BACKUP_APP = "cartethyia";
 export const BACKUP_VERSION = 1;
 
-/** Upper bound for restore payloads; the listener enforces the same ceiling. */
-export const MAX_BACKUP_BYTES = 8 * 1024 * 1024 * 1024;
+/**
+ * Upper bound for restore payloads.
+ *
+ * Deliberately not larger than the listener's own body cap. It used to claim
+ * 8 GiB while the server admitted 4 MiB, so the number described a limit that
+ * could never be reached: a file above the listener's cap is cut off before
+ * this check runs, and a large upload fails as a reset connection rather than
+ * the 413 this constant promises. The bound now states what the install
+ * actually admits, so the message an operator sees matches what happened.
+ *
+ * Raise `CARTETHYIA_SERVER_MAX_BODY_BYTES` to accept larger backups; this
+ * constant follows it rather than the two drifting apart again.
+ */
+export const MAX_BACKUP_BYTES = resolveMaxBodyBytes();
 
 /** Defense-in-depth row cap per table, after the byte bound. */
 export const MAX_ROWS_PER_TABLE = 500_000;

package/src/console/backup/export-endpoint.ts

@@ -47,10 +47,14 @@ export function createBackupExportHandler(
   deps: BackupExportDeps,
 ): (context: { request: Request; body: unknown }) => Promise<Response> {
   return async ({ request, body }) => {
+    // `policy_rejected` / `authentication_failed`, not `invalid_request` for
+    // both: neither refusal is about the body. The first is where the request
+    // came from, the second is the credential it carried — and `invalid_request`
+    // told an operator to fix a body that was never the problem.
     if (!isLoopbackPeer(deps.resolvePeerAddress(request)))
-      throw new GatewayError("invalid_request", 403, "backup export is only available from loopback");
+      throw new GatewayError("policy_rejected", 403, "backup export is only available from loopback");
     if (!tokenMatches(request.headers.get("x-drain-token"), deps.token))
-      throw new GatewayError("invalid_request", 401, "invalid export token");
+      throw new GatewayError("authentication_failed", 401, "invalid export token");
 
     const directory =
       body !== null && typeof body === "object" && "directory" in body

package/src/console/backup/nine-router.ts

@@ -21,6 +21,7 @@
  */
 import type { BackupPayload, BackupRow } from "./contracts";
 import { BACKUP_APP, BACKUP_VERSION } from "./contracts";
+import { isRecord } from "../../protocol/codecs";
 import { hashSecret } from "../../security/crypto";
 import {
   AUTH_STATE_KEYS,
@@ -97,7 +98,7 @@ const OAUTH_PROVIDERS = new Set(["claude", "codex", "antigravity", "kimi", "clin
 
 /** The non-secret per-account config, as our `auth_state` bag. */
 function authStateOf(entry: Row): Record<string, unknown> {
-  const specific = isRecord(entry.providerSpecificData) ? entry.providerSpecificData : {};
+  const specific = isRecord(entry.providerSpecificData) ? entry.providerSpecificData as Row : {};
   const state: Record<string, unknown> = {};
   for (const key of AUTH_STATE_KEYS) {
     const value = specific[key];
@@ -111,10 +112,6 @@ function emailOf(entry: Row): Record<string, unknown> {
   return email === null ? {} : { account_email: email };
 }
 
-function isRecord(value: unknown): value is Row {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
 /**
  * Splits `user:pass@host` userinfo out of a proxy URL.
  *

package/src/console/backup/service.ts

@@ -374,6 +374,10 @@ export class BackupService {
     const validation = validateRestorePayload(candidate, "");
     const warnings = validation.ok ? validation.value.warnings : [];
     const supported = new Set(validation.ok ? [...validation.value.tables.keys()].map(tableName) : []);
+    // `tenants` is diverted to `validation.value.tenantRows` during validate —
+    // not included in `tables` — so it would show as "unsupported" in the
+    // preview for any backup that carries it. Mark it supported explicitly.
+    if (validation.ok) supported.add("tenants");
 
     const rows: BackupInspectionRow[] = [];
     let total = 0;

package/src/console/backup/store.ts

@@ -283,13 +283,24 @@ export async function exportBackup(
   const config: Record<string, readonly BackupRow[]> = {};
   const telemetry: Record<string, readonly BackupRow[]> = {};
   const counts: Record<string, number> = {};
-  const configSet = new Set<Table>([...CONFIG_TABLES, TENANT_TABLE]);
+  const configSet = new Set<Table>([...CONFIG_TABLES]);
   const telemetrySet = new Set<Table>(TELEMETRY_TABLES);
 
-  for (const table of tables) {
-    const rows = table === telemetryEvents
-      ? await exportEventFile(tenantId)
-      : await exportTable(db, table, tenantId);
+  // Table reads are independent tenant-scoped `SELECT`s plus one disk read for
+  // the event file; firing them concurrently cuts a many-table export from
+  // one round-trip-at-a-time to the single slowest query. The DB driver pools
+  // these; the table list is bounded by `CONFIG_TABLES + TELEMETRY_TABLES`
+  // (small and known), so fan-out is bounded without an extra limiter.
+  const reads = tables.map(async (table) =>
+    [
+      table,
+      table === telemetryEvents ? await exportEventFile(tenantId) : await exportTable(db, table, tenantId),
+    ] as const,
+  );
+  // Preserve the caller's `tables` order when writing into the output maps so
+  // the payload's key order is deterministic across runs.
+  const settled = await Promise.all(reads);
+  for (const [table, rows] of settled) {
     const name = tableName(table);
     counts[name] = rows.length;
     if (configSet.has(table)) config[name] = rows;
@@ -517,12 +528,22 @@ async function clearOwnedRows(
 }
 
 /**
- * Inserts rows one at a time so a single bad row names its own failure.
+ * Inserts rows in bulk with a per-row fallback for diagnostics.
  *
  * `conflictTarget` is supplied for tables a restore may not delete from: the
  * payload's rows are upserted by the table's unique key, so re-importing a
  * backup updates what is already there instead of failing on the first
  * duplicate.
+ *
+ * Historically rows went in one statement at a time so a single bad row could
+ * name its own failure. On a real backup — thousands of telemetry events, a
+ * tenant's models and accounts — that cost one DB round-trip per row and the
+ * restore stalled on network latency rather than the server's write path. The
+ * fast path collapses those statements into one `INSERT ... VALUES (...), ...`
+ * per chunk; if a chunk fails for any reason (bad row, constraint, oversized
+ * parameter list) the chunk is re-run one row at a time so the diagnostic
+ * surface is unchanged. The slow path is only reached on real errors, so a
+ * clean restore pays zero penalty for the diagnostic.
  */
 async function insertRows(
   tx: CartethyiaDatabase,
@@ -535,46 +556,171 @@ async function insertRows(
     readonly ignoreConflicts?: boolean;
   } = {},
 ): Promise<number> {
-  let inserted = 0;
+  // Normalise rows up front: filter skips, drop empty-value rows (no
+  // insertable columns), and project to Drizzle values. Doing it once keeps
+  // the two paths from diverging in what counts as insertable.
+  const prepared: Record<string, unknown>[] = [];
   for (const row of rows) {
     if (options.skipKeys !== undefined && options.keyOf !== undefined && options.skipKeys.has(options.keyOf(row))) {
       continue;
     }
     const values = toValues(table, row);
     if (Object.keys(values).length === 0) continue;
+    prepared.push(values);
+  }
+  if (prepared.length === 0) return 0;
+
+  // The `set` clause for upserts: every column the payload carries is written,
+  // so a re-import restores the file's state rather than leaving stale values
+  // behind. `excluded` is the row that failed to insert, and column names come
+  // from the schema rather than the JS key — guessing the database spelling
+  // from a camelCase key breaks the moment the two differ.
+  const columnNameMap = new Map(
+    Object.entries(getTableColumns(table)).map(([key, column]) => [key, column.name]),
+  );
+  const buildUpsertSet = (values: Record<string, unknown>): Record<string, unknown> =>
+    Object.fromEntries(
+      Object.keys(values).map((key) => {
+        const name = columnNameMap.get(key);
+        if (name === undefined) throw new Error(`${tableName(table)} has no column ${key}`);
+        return [key, sql.raw(`excluded."${name}"`)];
+      }),
+    );
+
+  // One row's path, kept as its own function so the bulk fallback can reuse it
+  // without duplicating the control flow.
+  const insertOne = async (values: Record<string, unknown>): Promise<number> => {
     const statement = insertRow(tx, table, values);
     if (options.ignoreConflicts) {
-      if (options.conflictTarget === undefined)
+      if (options.conflictTarget === undefined) {
         throw new Error(`ignoreConflicts requires a target for ${tableName(table)}`);
+      }
       const insertedRows = await statement
         .onConflictDoNothing({ target: options.conflictTarget as never })
         .returning();
-      inserted += insertedRows.length;
-      continue;
+      return insertedRows.length;
     }
     if (options.conflictTarget === undefined) {
       await statement;
-    } else {
-      // Every column the payload carries is written, so a re-import restores
-      // the file's state rather than leaving stale values behind. `excluded`
-      // is the row that failed to insert, and the column names come from the
-      // schema rather than from the JS key — guessing the database spelling
-      // from a camelCase key breaks the moment the two differ.
-      const names = new Map(
-        Object.entries(getTableColumns(table)).map(([key, column]) => [key, column.name]),
-      );
-      await statement.onConflictDoUpdate({
-        target: options.conflictTarget as never,
-        set: Object.fromEntries(
-          Object.keys(values).map((key) => {
-            const name = names.get(key);
-            if (name === undefined) throw new Error(`${tableName(table)} has no column ${key}`);
-            return [key, sql.raw(`excluded."${name}"`)];
-          }),
-        ) as never,
+      return 1;
+    }
+    await statement.onConflictDoUpdate({
+      target: options.conflictTarget as never,
+      set: buildUpsertSet(values) as never,
+    });
+    return 1;
+  };
+
+  // Bulk chunk size: keeps each statement's parameter count comfortably under
+  // Postgres's 65535 limit even for the widest table. 500 rows × ~60 columns
+  // leaves plenty of headroom; a narrower table just means a faster chunk.
+  const BULK_CHUNK = 500;
+
+  // Attempts one chunk in a single statement. Returns the inserted count on
+  // success, or `null` to signal "fall back to per-row for this chunk". The
+  // chunk is wrapped in a savepoint so a bulk failure can be rolled back
+  // *inside* the outer restore transaction — without this, Postgres marks the
+  // outer transaction as aborted on the first error and the per-row diagnostic
+  // fallback would itself fail with "current transaction is aborted".
+  const insertChunk = async (
+    chunk: readonly Record<string, unknown>[],
+  ): Promise<{ count: number } | { error: unknown }> => {
+    try {
+      // `tx.transaction(...)` nests as a `SAVEPOINT ... ROLLBACK TO SAVEPOINT`
+      // on both Postgres and SQLite drivers Drizzle supports here, which is
+      // exactly what the fallback needs: on failure the outer transaction
+      // stays writable and the per-row path can isolate the offending row.
+      const count = await tx.transaction(async (innerTx) => {
+        // Drizzle types the nested transaction callback with the dialect's
+        // own transaction handle; cast once to a named local with a reason,
+        // never inline on member access.
+        const inner = innerTx as unknown as CartethyiaDatabase; // nested-tx handle; mirrors applyRestore's outer `scoped` cast on line 607.
+        const statement = inner.insert(table).values(chunk as never);
+        if (options.ignoreConflicts) {
+          if (options.conflictTarget === undefined) {
+            throw new Error(`ignoreConflicts requires a target for ${tableName(table)}`);
+          }
+          const insertedRows = await statement
+            .onConflictDoNothing({ target: options.conflictTarget as never })
+            .returning();
+          return insertedRows.length;
+        }
+        if (options.conflictTarget === undefined) {
+          await statement;
+          return chunk.length;
+        }
+        // For a bulk upsert `excluded.*` references the row the driver tried
+        // to insert, so each column mapping is the same regardless of which
+        // row in the chunk conflicts. Build the set clause from the first
+        // row's keys — every prepared row for one table carries the same key
+        // set because `toValues` is deterministic for a given schema.
+        const firstRow = chunk[0];
+        if (firstRow === undefined) return 0;
+        await statement.onConflictDoUpdate({
+          target: options.conflictTarget as never,
+          set: buildUpsertSet(firstRow) as never,
+        });
+        return chunk.length;
       });
+      return { count };
+    } catch (error) {
+      // Signal "re-run row by row": a bulk failure must not swallow the
+      // specific row's error, because the per-row path is what names the
+      // offending row to the operator. Keep the bulk error too: when the
+      // per-row fallback also fails (connection drop, aborted transaction,
+      // oversized statement), the derived row error would otherwise hide the
+      // real cause from the operator.
+      return { error };
     }
-    inserted += 1;
+  };
+
+  let inserted = 0;
+  // Group adjacent rows that share the same column key set. Different rows in
+  // one table can carry different keys (an optional column omitted when the
+  // payload cell is absent; `parent_key_id` present on child api_keys but not
+  // parents), and Drizzle's bulk `.values([...])` compiles one SQL statement
+  // with a column list taken from the first row — so a bulk of mixed shapes
+  // produces malformed SQL. Grouping by signature keeps each bulk statement
+  // uniform while preserving row order within the group.
+  //
+  // Signature is a stable `|`-joined list of sorted keys; empty columns were
+  // already dropped by the `toValues` → `Object.keys(values).length === 0`
+  // check above, so empty signatures never occur here.
+  const signatureOf = (values: Record<string, unknown>): string =>
+    Object.keys(values).sort().join("|");
+  let groupStart = 0;
+  while (groupStart < prepared.length) {
+    const groupSignature = signatureOf(prepared[groupStart]!);
+    let groupEnd = groupStart + 1;
+    while (
+      groupEnd < prepared.length &&
+      signatureOf(prepared[groupEnd]!) === groupSignature &&
+      groupEnd - groupStart < BULK_CHUNK
+    ) {
+      groupEnd += 1;
+    }
+    const chunk = prepared.slice(groupStart, groupEnd);
+    const bulk = await insertChunk(chunk);
+    if ("count" in bulk) {
+      inserted += bulk.count;
+    } else {
+      // Diagnostic fallback: let the single statement that actually errors
+      // reach the caller with the row that caused it. If the fallback itself
+      // fails, the row-level error is derived — chain the original bulk error
+      // so a connection-level fault is not reported as a bad row.
+      try {
+        for (const values of chunk) {
+          inserted += await insertOne(values);
+        }
+      } catch (perRowError) {
+        const bulkMessage = bulk.error instanceof Error ? bulk.error.message : String(bulk.error);
+        throw new Error(
+          `bulk insert into ${tableName(table)} failed (${bulkMessage}); per-row diagnostic also failed`,
+          { cause: perRowError },
+        );
+      }
+    }
+    groupStart = groupEnd;
   }
   return inserted;
 }
@@ -610,6 +756,14 @@ export async function applyRestore(
     // its PK is `id`, so a conflict means "already there" rather than an error.
     // The payload's tenant is forced to the restoring tenant — a payload naming
     // someone else's tenant id must not create or claim that tenant.
+    //
+    // Current exports deliberately omit the `tenants` section, so
+    // `tenantRows` is normally empty. Child rows still carry
+    // `tenant_id = tenantId`, which means a restore into a tenant that does
+    // not exist yet would fail the `api_keys` (and other) foreign keys unless
+    // we ensure the receiving tenant here. The minimal placeholder row is
+    // written only for a tenant that did not already exist; on conflict it is
+    // a no-op and the existing tenant's name/status are left untouched.
     if (validation.tenantRows.length > 0) {
       let inserted = 0;
       for (const row of validation.tenantRows) {
@@ -621,6 +775,15 @@ export async function applyRestore(
         inserted += result.length;
       }
       restored["tenants"] = inserted;
+    } else {
+      const result = await insertRow(scoped, TENANT_TABLE, {
+        id: tenantId,
+        name: "restored",
+        status: "active",
+      })
+        .onConflictDoNothing()
+        .returning({ id: sql<number>`1` });
+      restored["tenants"] = result.length;
     }
 
     const replaceTables = order.filter(

package/src/console/backup/validate.ts

@@ -36,18 +36,13 @@ import {
   type ValidatedRestore,
   type ValidatedTable,
 } from "./contracts";
-import { DEFAULT_API_KEY_LABEL } from "../api-keys/contracts";
 import {
-  apiKeys,
   modelCombos,
   networkPools,
   providerRoutingSettings,
   shareLinks,
 } from "../../persistence/schema";
 
-const LEGACY_PROVIDER_USER_AGENT = "codex_cli_rs/0.156.1";
-const CURRENT_PROVIDER_USER_AGENT = "claude-cli/2.1.293 (external, cli)";
-
 /**
  * Provider ids that were renamed, old spelling to current.
  *
@@ -68,51 +63,14 @@ function currentProviderId(value: unknown): unknown {
   return RENAMED_PROVIDER_IDS[value] ?? value;
 }
 
-function legacyModelList(value: unknown): readonly string[] | undefined {
-  if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) return undefined;
-  return value;
-}
-
-function normalizeLegacyApiKeyPolicy(row: Record<string, unknown>): Record<string, unknown> {
-  const allowlist = legacyModelList(row["model_allowlist"]);
-  const denylist = legacyModelList(row["model_denylist"]);
-  if (row["model_access_mode"] !== undefined || row["model_list"] !== undefined) return row;
-  if (denylist !== undefined && denylist.length > 0) {
-    return { ...row, model_access_mode: "blacklist", model_list: denylist };
-  }
-  if (allowlist !== undefined) {
-    return { ...row, model_access_mode: "whitelist", model_list: allowlist };
-  }
-  return row;
-}
-
 function normalizeLegacyRow(table: Table, row: Record<string, unknown>): Record<string, unknown> {
   if (row["provider_id"] !== undefined) {
     const renamed = currentProviderId(row["provider_id"]);
     if (renamed !== row["provider_id"]) row = { ...row, provider_id: renamed };
   }
-  if (
-    table === apiKeys &&
-    row["is_default"] === undefined &&
-    (row["key_mode"] === undefined || row["key_mode"] === "personal") &&
-    row["parent_key_id"] == null &&
-    row["label"] === DEFAULT_API_KEY_LABEL
-  ) {
-    return { ...row, is_default: true };
-  }
   if (table === providerRoutingSettings) {
-    const { bypass_proxy: legacySkipProxy, ...current } = row;
-    let normalized = current;
-    if (normalized["skip_proxy"] === undefined && legacySkipProxy !== undefined) {
-      normalized = { ...normalized, skip_proxy: legacySkipProxy };
-    }
-    if (normalized["provider_id"] === "cline") {
-      normalized = { ...normalized, skip_proxy: true };
-    }
-    if (normalized["user_agent"] === LEGACY_PROVIDER_USER_AGENT) {
-      normalized = { ...normalized, user_agent: CURRENT_PROVIDER_USER_AGENT };
-    }
-    return normalized;
+    if (row["provider_id"] === "cline") return { ...row, skip_proxy: true };
+    return row;
   }
   if (table === networkPools && row["kind"] === "bridge") {
     return { ...row, kind: "https" };
@@ -342,7 +300,7 @@ export function validateRestorePayload(payload: unknown, tenantId: string): Rest
         // the current single policy before dropping retired column names.
         const sourceRow = normalizeLegacyRow(
           table,
-          table === apiKeys ? normalizeLegacyApiKeyPolicy(row) : row,
+          row,
         );
         // A stale backup may still carry columns the schema has since dropped.
         // Those are accepted but removed here, so they never reach the store