@humwick/server 0.130.1 → 0.130.3

package/src/db/checkout-refresh-schema.ts

@@ -0,0 +1,47 @@
+import type { getDb } from "./connection";
+
+type Database = ReturnType<typeof getDb>;
+
+// Keep one referenced command per checkout; generic command retention can expire its history (#3797).
+export const CHECKOUT_REFRESH_SCHEMA = `
+  CREATE TABLE IF NOT EXISTS checkout_refresh_heads (
+    project_id TEXT NOT NULL,
+    orchestrator_id TEXT NOT NULL,
+    root_path TEXT NOT NULL,
+    command_id TEXT NOT NULL REFERENCES commands(id) ON DELETE CASCADE,
+    created_at INTEGER NOT NULL,
+    PRIMARY KEY (project_id, orchestrator_id, root_path)
+  );
+  CREATE INDEX IF NOT EXISTS idx_checkout_refresh_heads_command ON checkout_refresh_heads(command_id);
+  CREATE TRIGGER IF NOT EXISTS checkout_refresh_head_insert AFTER INSERT ON commands
+  WHEN NEW.type = 'project.checkout.refresh' AND json_valid(NEW.params)
+    AND json_type(NEW.params, '$.projectId') = 'text'
+    AND json_type(NEW.params, '$.orchestratorId') = 'text'
+    AND json_type(NEW.params, '$.rootPath') = 'text'
+  BEGIN
+    INSERT INTO checkout_refresh_heads VALUES (
+      json_extract(NEW.params, '$.projectId'), json_extract(NEW.params, '$.orchestratorId'),
+      json_extract(NEW.params, '$.rootPath'), NEW.id, NEW.created_at)
+    ON CONFLICT(project_id, orchestrator_id, root_path) DO UPDATE SET
+      command_id = excluded.command_id, created_at = excluded.created_at
+    WHERE excluded.created_at >= checkout_refresh_heads.created_at;
+  END;
+`;
+
+export function migrateCheckoutRefreshHeads(db: Database): void {
+  db.transaction(() => {
+    const installed = db.query("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'checkout_refresh_heads'").get();
+    db.run(CHECKOUT_REFRESH_SCHEMA);
+    if (installed) return;
+    db.run(`INSERT INTO checkout_refresh_heads
+      SELECT project_id, orchestrator_id, root_path, id, created_at FROM (
+        SELECT id, created_at, json_extract(params, '$.projectId') AS project_id,
+          json_extract(params, '$.orchestratorId') AS orchestrator_id, json_extract(params, '$.rootPath') AS root_path,
+          row_number() OVER (PARTITION BY json_extract(params, '$.projectId'), json_extract(params, '$.orchestratorId'),
+            json_extract(params, '$.rootPath') ORDER BY created_at DESC, rowid DESC) AS ordinal
+        FROM commands WHERE type = 'project.checkout.refresh' AND json_valid(params)
+          AND json_type(params, '$.projectId') = 'text' AND json_type(params, '$.orchestratorId') = 'text'
+          AND json_type(params, '$.rootPath') = 'text'
+      ) WHERE ordinal = 1`);
+  })();
+}

package/src/db/checkout-refresh.ts

@@ -4,12 +4,12 @@ import { getOrchestrator } from "./orchestrators";
 import { stringValue } from "@humwick/sdk";
 import { getDb } from "./connection";
 
+export const CHECKOUT_REFRESH_COMMANDS_SQL = `SELECT c.id, c.status, c.params, c.result, c.error, c.updated_at
+  FROM checkout_refresh_heads h CROSS JOIN commands c ON c.id = h.command_id
+  WHERE h.project_id = ?`;
+
 export function checkoutRefreshCommands(projectId: string): CheckoutRefreshStatus[] {
-  const rows = getDb().query(`SELECT * FROM (
-    SELECT id, status, params, result, error, updated_at,
-      row_number() OVER (PARTITION BY json_extract(params, '$.orchestratorId'), json_extract(params, '$.rootPath') ORDER BY created_at DESC, rowid DESC) AS ordinal
-    FROM commands WHERE type = 'project.checkout.refresh' AND json_extract(params, '$.projectId') = ?
-  ) WHERE ordinal = 1`).all(projectId) as Array<{ id: string; status: Command["status"]; params: string; result: string | null; error: string | null; updated_at: number }>;
+  const rows = getDb().query(CHECKOUT_REFRESH_COMMANDS_SQL).all(projectId) as Array<{ id: string; status: Command["status"]; params: string; result: string | null; error: string | null; updated_at: number }>;
   return rows.map((row) => {
     const params = JSON.parse(row.params);
     return { orchestratorId: params.orchestratorId, rootPath: params.rootPath, commandId: row.id, status: row.status, updatedAt: row.updated_at,

package/src/db/ci-incidents-async.ts

@@ -35,15 +35,15 @@ const write = <T>(op: string, args: unknown, idempotencyKey?: string): Promise<T
 export function upsertWatchStateAsync(input: Parameters<typeof upsertWatchState>[0]): Promise<ReturnType<typeof upsertWatchState>> {
   return write("ciIncidents.watch.upsert", input);
 }
-/** One gateway writer transaction for the complete watch + incident lifecycle decision. */
-export function ingestCiObservationTransitionAsync(input: CiObservationTransitionInput): Promise<CiObservationTransitionResult> {
-  return write("ciIncidents.observation.ingest", input);
+/** One gateway writer transaction for the complete watch + incident lifecycle decision. `now` pins a test clock. */
+export function ingestCiObservationTransitionAsync(input: CiObservationTransitionInput, now?: number): Promise<CiObservationTransitionResult> {
+  return write("ciIncidents.observation.ingest", now === undefined ? input : { ...input, now });
 }
 export function markCiWatchNeverMatchedNotifiedAsync(watchKey: string): Promise<ReturnType<typeof markCiWatchNeverMatchedNotified>> {
   return write("ciIncidents.watch.markNeverMatched", { watchKey });
 }
-export function markCiRootMissingNotifiedAsync(repository: string): Promise<ReturnType<typeof markCiRootMissingNotified>> {
-  return write("ciIncidents.root.markMissing", { repository });
+export function markCiRootMissingNotifiedAsync(repository: string, now?: number): Promise<ReturnType<typeof markCiRootMissingNotified>> {
+  return write("ciIncidents.root.markMissing", now === undefined ? { repository } : { repository, now });
 }
 export function openOrDedupeIncidentAsync(input: Parameters<typeof openOrDedupeIncident>[0]): Promise<ReturnType<typeof openOrDedupeIncident>> {
   return write("ciIncidents.openOrDedupe", input);

package/src/db/ci-incidents-write-ops.ts

@@ -27,7 +27,7 @@ export const CI_INCIDENTS_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Ob
   "ciIncidents.observation.ingest": {
     offLoop: true,
     idempotent: true,
-    fn: (a: Parameters<typeof applyCiObservationTransitionInternal>[0], ctx) => applyCiObservationTransitionInternal(a, ctx.now),
+    fn: (a: Parameters<typeof applyCiObservationTransitionInternal>[0] & { now?: number }, ctx) => applyCiObservationTransitionInternal(a, a.now ?? ctx.now),
   },
   "ciIncidents.gateRun.record": { offLoop: true, idempotent: true, fn: (a: Parameters<typeof recordCiGateRuns>[0]) => recordCiGateRuns(a) },
   "ciIncidents.gateBackfill.start": { offLoop: true, idempotent: true, fn: (a: Parameters<typeof startCiGateBackfill>[0]) => startCiGateBackfill(a) },
@@ -36,7 +36,7 @@ export const CI_INCIDENTS_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Ob
   "ciIncidents.sha.prune": { offLoop: true, idempotent: true, fn: (_a: { now: number }, ctx) => pruneCiShaObservations(ctx.now) },
   "ciIncidents.watch.upsert": { offLoop: true, idempotent: true, fn: (a: Parameters<typeof upsertWatchState>[0], ctx) => upsertWatchState(a, ctx.now) },
   "ciIncidents.watch.markNeverMatched": { offLoop: true, idempotent: true, fn: (a: { watchKey: string }, ctx) => markCiWatchNeverMatchedNotified(a.watchKey, ctx.now) },
-  "ciIncidents.root.markMissing": { offLoop: true, idempotent: true, fn: (a: { repository: string }, ctx) => markCiRootMissingNotified(a.repository, ctx.now) },
+  "ciIncidents.root.markMissing": { offLoop: true, idempotent: true, fn: (a: { repository: string; now?: number }, ctx) => markCiRootMissingNotified(a.repository, a.now ?? ctx.now) },
   "ciIncidents.openOrDedupe": { offLoop: true, idempotent: true, fn: (a: Parameters<typeof openOrDedupeIncidentInternal>[0], ctx) => openOrDedupeIncidentInternal(a, ctx.now, false) },
   // Guarded transitions throw after they have advanced; use the durable retry ledger rather than
   // misclassifying that as safe replay. Distinct steward attempts carry distinct request ids.

package/src/db/command-agent-profile-backfill.ts

@@ -0,0 +1,39 @@
+// #3606/#3610 — spawn commands written before the content-hash store (command-agent-profiles.ts)
+// still carry `params.agentProfile` inline (~0.4 MB each). Retention keeps some of them past the
+// terminal window (managed agents, unconfirmed spawns), so move those profiles into the store in
+// bounded chunks from the retention pass. `updated_at` is untouched: retention windows key off it.
+
+import { storeCommandAgentProfile } from "./command-agent-profiles";
+import { TERMINAL_COMMAND_STATUS_SQL_LIST } from "./command-terminal-status";
+import type { getDb } from "./connection";
+
+type Database = ReturnType<typeof getDb>;
+const WRITE_TRANSACTION_BUDGET_MS = 25;
+
+/** One bounded chunk: returns the same shape as a retention prune chunk (`pruned` = rows slimmed). */
+export function slimSpawnProfileChunk(db: Database, limit: number): { candidates: number; attempted: number; pruned: number } {
+  // Only terminal rows: a live spawn is still being claimed. A profile carrying vault ciphertext stays
+  // inline (the #3488 settlement trigger prunes it there), so those rows are not candidates.
+  const ids = db.query(`
+    SELECT id FROM commands
+    WHERE type = 'agent.spawn' AND status IN (${TERMINAL_COMMAND_STATUS_SQL_LIST}) AND agent_profile_hash IS NULL
+      AND instr(params, '"agentProfile"') > 0 AND json_valid(params) AND json_type(params, '$.agentProfile') = 'object'
+      AND instr(params, 'sealed:v2:') = 0
+    LIMIT ?
+  `).all(limit) as Array<{ id: string }>;
+  if (ids.length === 0) return { candidates: 0, attempted: 0, pruned: 0 };
+  const read = db.query("SELECT params FROM commands WHERE id = ?");
+  const write = db.query("UPDATE commands SET params = ?, agent_profile_hash = ? WHERE id = ? AND agent_profile_hash IS NULL");
+  return db.transaction(() => {
+    const startedAt = performance.now(); let attempted = 0; let pruned = 0;
+    for (const { id } of ids) {
+      if (attempted > 0 && performance.now() - startedAt >= WRITE_TRANSACTION_BUDGET_MS) break;
+      attempted += 1;
+      const row = read.get(id) as { params: string } | null;
+      if (!row) continue;
+      const stored = storeCommandAgentProfile(JSON.parse(row.params) as Record<string, unknown>, Date.now(), db);
+      if (stored.agentProfileHash) pruned += write.run(JSON.stringify(stored.params), stored.agentProfileHash, id).changes;
+    }
+    return { candidates: ids.length, attempted, pruned };
+  })();
+}

package/src/db/command-agent-profiles.ts

@@ -7,6 +7,7 @@
 import { createHash } from "node:crypto";
 import { isRecord } from "@humwick/sdk";
 import { getDb } from "./connection";
+import { COMMAND_PROFILE_WARN_BYTES, warnIfOversized } from "./agent-meta";
 
 type Database = ReturnType<typeof getDb>;
 
@@ -41,7 +42,10 @@ export function storeCommandAgentProfile(sealed: Record<string, unknown>, now: n
   const body = JSON.stringify(profile);
   if (body.includes("sealed:v2:")) return { params: sealed, agentProfileHash: null };
   const hash = createHash("sha256").update(body).digest("hex");
-  db.query("INSERT OR IGNORE INTO command_agent_profiles (hash, profile, created_at) VALUES (?, ?, ?)").run(hash, body, now);
+  const inserted = db.query("INSERT OR IGNORE INTO command_agent_profiles (hash, profile, created_at) VALUES (?, ?, ?)").run(hash, body, now);
+  if (inserted.changes && Buffer.byteLength(body) > COMMAND_PROFILE_WARN_BYTES) {
+    warnIfOversized("command-profile", body, COMMAND_PROFILE_WARN_BYTES, "[commands] stored agentProfile");
+  }
   const { agentProfile: _stored, ...params } = sealed;
   return { params, agentProfileHash: hash };
 }

package/src/db/command-retention.ts

@@ -0,0 +1,126 @@
+import { quoteIdentifier } from "@humwick/kernel/persistence-schema";
+import { getDb } from "./connection";
+import { TERMINAL_COMMAND_STATUS_SQL_LIST } from "./command-terminal-status";
+
+type Database = ReturnType<typeof getDb>;
+
+const ELIGIBLE = `type != 'agent.spawn' AND status IN (${TERMINAL_COMMAND_STATUS_SQL_LIST})`;
+export const COMMAND_RETENTION_SCHEMA = `
+  CREATE INDEX IF NOT EXISTS idx_commands_terminal_retention ON commands(updated_at, id) WHERE ${ELIGIBLE};
+  CREATE INDEX IF NOT EXISTS idx_commands_refresh_retention ON commands(updated_at, id)
+    WHERE type = 'project.checkout.refresh' AND status IN (${TERMINAL_COMMAND_STATUS_SQL_LIST});
+  CREATE TABLE IF NOT EXISTS command_retention_cursor (
+    id INTEGER PRIMARY KEY CHECK (id = 1), updated_at INTEGER NOT NULL, command_id TEXT NOT NULL);
+`;
+
+// Sampled status stats favor a broad status scan plus sort; retain the bounded age seek (#3619).
+export const TERMINAL_COMMAND_CANDIDATES_SQL = `SELECT id, updated_at FROM commands INDEXED BY idx_commands_terminal_retention
+  WHERE ${ELIGIBLE} AND updated_at < ? AND (updated_at, id) > (?, ?)
+  ORDER BY updated_at, id LIMIT ?`;
+
+export const REFRESH_COMMAND_CANDIDATES_SQL = `SELECT id, updated_at FROM commands INDEXED BY idx_commands_refresh_retention
+  WHERE type = 'project.checkout.refresh' AND status IN (${TERMINAL_COMMAND_STATUS_SQL_LIST})
+    AND updated_at >= ? AND updated_at < ? AND (updated_at, id) > (?, ?)
+  ORDER BY updated_at, id LIMIT ?`;
+
+const JSON_REFERENCES = [
+  ["activity_events", "metadata", ["$.commandId", "$.mergeCommandId"]],
+  ["task_events", "metadata", ["$.commandId", "$.mergeCommandId"]],
+  ["workspaces", "metadata", ["$.mergeCommandId", "$.rekeyCommandId", "$.lastStewardRekey.commandId"]],
+  ["scheduled_timers", "command", ["$.commandId"]],
+  ["workflow_instances", "wait_for", ["$.commandId"]],
+  ["workflow_instances", "initial_state", ["$.commandId"]],
+  ["workflow_step_journal", "wait_for", ["$.commandId"]],
+  ["workflow_step_journal", "state_snapshot", ["$.commandId"]],
+] as const;
+
+function jsonReference(column: string, path: string): string {
+  return `json_extract(CASE WHEN json_valid(${column}) THEN ${column} ELSE '{}' END, '${path}')`;
+}
+
+function commandReferences(db: Database): { tbl: string; col: string }[] {
+  return db.query(`SELECT s.name AS tbl, c.name AS col FROM sqlite_schema s, pragma_table_info(s.name) c
+    WHERE s.type = 'table' AND s.name NOT LIKE 'sqlite_%' AND s.name != 'command_retention_cursor'
+      AND (lower(replace(c.name, '_', '')) LIKE '%commandid' OR lower(replace(c.name, '_', '')) LIKE '%commandids')
+    UNION SELECT s.name, f."from" FROM sqlite_schema s, pragma_foreign_key_list(s.name) f
+      WHERE s.type = 'table' AND f."table" = 'commands'`).all() as { tbl: string; col: string }[];
+}
+
+/** Equality checks in a DELETE must seek references, never repeatedly scan their history (#3619). */
+export function ensureCommandRetentionReferenceIndexes(db: Database): void {
+  for (const { tbl, col } of commandReferences(db)) {
+    if (col.replaceAll("_", "").toLowerCase().endsWith("commandids")) continue;
+    const indexes = db.query(`PRAGMA index_list(${quoteIdentifier(tbl)})`).all() as { name: string; partial: number }[];
+    const covered = indexes.some(index => !index.partial &&
+      (db.query(`PRAGMA index_info(${quoteIdentifier(index.name)})`).get() as { name: string } | null)?.name === col);
+    if (!covered) db.run(`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`idx_retention_${tbl}_${col}`)} ON ${quoteIdentifier(tbl)}(${quoteIdentifier(col)})`);
+  }
+  for (const [table, column, paths] of JSON_REFERENCES) {
+    if (table === "scheduled_timers") continue; // idx_scheduled_timers_command_created already owns this expression.
+    for (const [i, path] of paths.entries()) db.run(`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`idx_retention_${table}_${column}_${i}`)}
+      ON ${table}(${jsonReference(column, path)})`);
+  }
+}
+
+const predicates = new WeakMap<Database, { version: number; sql: string }>();
+
+/** Resolve each reference index for the whole candidate set, including CASCADE/SET NULL FKs. */
+function referencedCommandIds(db: Database): string {
+  const version = (db.query("PRAGMA schema_version").get() as { schema_version: number }).schema_version;
+  const cached = predicates.get(db);
+  if (cached?.version === version) return cached.sql;
+  const refs = commandReferences(db);
+  const selections = refs.map(({ tbl, col }) => {
+    const table = quoteIdentifier(tbl), column = quoteIdentifier(col);
+    return col.replaceAll("_", "").toLowerCase().endsWith("commandids")
+      ? `SELECT candidate.id FROM ${table} ref, json_each(CASE WHEN json_valid(ref.${column}) THEN ref.${column} ELSE '[]' END) item
+          JOIN candidates candidate ON CAST(item.value AS TEXT) = candidate.id`
+      : `SELECT candidate.id FROM candidates candidate CROSS JOIN ${table} ref ON ref.${column} = candidate.id`;
+  });
+  // Strip candidate TEXT affinity so JSON expression indexes can seek (#3619).
+  for (const [table, column, paths] of JSON_REFERENCES) {
+    for (const path of paths) {
+      const expression = table === "scheduled_timers" ? `json_extract(ref.${column}, '${path}')` : jsonReference(`ref.${column}`, path);
+      selections.push(`SELECT candidate.id FROM candidates candidate CROSS JOIN ${table} ref ON ${expression} = +candidate.id`);
+    }
+  }
+  const sql = selections.join(" UNION ");
+  predicates.set(db, { version, sql });
+  return sql;
+}
+
+/** #3619 — advance past referenced rows so they cannot starve unrelated expired commands. */
+export function pruneTerminalCommands(cutoff: number, limit: number, refreshCutoff = cutoff): { pruned: number; scanned: number; moreWork: boolean } {
+  const db = getDb();
+  const start = performance.now();
+  const cursor = db.query("SELECT updated_at, command_id FROM command_retention_cursor WHERE id = 1").get() as
+    { updated_at: number; command_id: string } | null;
+  const candidates = db.query(TERMINAL_COMMAND_CANDIDATES_SQL)
+    .all(cutoff, cursor?.updated_at ?? -1, cursor?.command_id ?? "", limit) as { id: string; updated_at: number }[];
+  // Merge two bounded age seeks; one shared cursor also advances past referenced refresh receipts (#3797).
+  if (refreshCutoff > cutoff) candidates.push(...db.query(REFRESH_COMMAND_CANDIDATES_SQL)
+    .all(cutoff, refreshCutoff, cursor?.updated_at ?? -1, cursor?.command_id ?? "", limit) as typeof candidates);
+  const rows = candidates.sort((a, b) => a.updated_at - b.updated_at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)).slice(0, limit);
+  if (!rows.length) {
+    db.run("DELETE FROM command_retention_cursor WHERE id = 1");
+    return { pruned: 0, scanned: 0, moreWork: false };
+  }
+  const references = referencedCommandIds(db);
+  let pruned = 0, scanned = 0;
+  // Visit each reference index once per chunk, rather than switch indexes per command (#3807).
+  for (let offset = 0; offset < rows.length; offset += 25) {
+    if (scanned > 0 && performance.now() - start >= 25) break;
+    const chunk = rows.slice(offset, offset + 25);
+    pruned += db.query(`WITH candidates AS MATERIALIZED (
+        SELECT id FROM commands WHERE id IN (${chunk.map(() => "?").join(",")}) AND ${ELIGIBLE}
+          AND (updated_at < ? OR (type = 'project.checkout.refresh' AND updated_at < ?))),
+      referenced(id) AS MATERIALIZED (${references})
+      DELETE FROM commands WHERE id IN (SELECT id FROM candidates EXCEPT SELECT id FROM referenced)`)
+      .run(...chunk.map(row => row.id), cutoff, refreshCutoff).changes;
+    scanned += chunk.length;
+  }
+  const last = rows[scanned - 1]!;
+  db.query(`INSERT INTO command_retention_cursor VALUES (1, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET updated_at = excluded.updated_at, command_id = excluded.command_id`).run(last.updated_at, last.id);
+  return { pruned, scanned, moreWork: true };
+}

package/src/db/command-row.ts

@@ -41,13 +41,15 @@ export interface CommandRow {
 }
 
 
-export function rowToCommand(row: CommandRow): Command {
+/** `slim` (#3391): listings omit `params.agentProfile` (~0.5 MB per spawn row) instead of hydrating it;
+ *  `GET /api/commands/:id` returns the full row. */
+export function rowToCommand(row: CommandRow, options: { slim?: boolean } = {}): Command {
   return {
     id: row.id,
     type: row.type,
     source: row.source,
     target: row.target,
-    params: hydrateCommandAgentProfile(parseObject(row.params), row.agent_profile_hash),
+    params: options.slim ? withoutAgentProfile(parseObject(row.params)) : hydrateCommandAgentProfile(parseObject(row.params), row.agent_profile_hash),
     status: row.status,
     result: row.result ? parseObject(row.result) : undefined,
     error: row.error ?? undefined,
@@ -78,6 +80,12 @@ export function rowToCommand(row: CommandRow): Command {
   };
 }
 
+function withoutAgentProfile(params: Record<string, unknown>): Record<string, unknown> {
+  if (!("agentProfile" in params)) return params;
+  const { agentProfile: _omitted, ...rest } = params;
+  return rest;
+}
+
 function parseObject(raw: string): Record<string, unknown> {
   try {
     const parsed = JSON.parse(raw);

package/src/db/commands-list-query.ts

@@ -9,6 +9,8 @@ export interface CommandFilters {
   dispatchableOnly?: boolean;
   /** #2663 — oldest-first for the executor DISPATCH feed, whose window is capped. See `getCommands`. */
   oldestFirst?: boolean;
+  /** #3391 — omit `params.agentProfile` from rows (listing views; fetch the row by id for the profile). */
+  slim?: boolean;
 }
 
 export function buildListCommandsQuery(filters: CommandFilters = {}): { sql: string; params: unknown[] } {

package/src/db/commands-planner-stats.test-fixture.ts

@@ -39,6 +39,10 @@ export const PROD_SAMPLED_COMMANDS_STATS: CommandsStatsProfile = {
     idx_commands_target_updated: "499038 6 1 1",
     idx_commands_correlation: "499038 5",
     idx_commands_type_status_created: "499038 18063 6168 1 1",
+    // Undeployed (#3619): conservative full-history upper bound for terminal non-spawn rows.
+    idx_commands_terminal_retention: "499038 2 1",
+    // Undeployed (#3797): conservative refresh-history census, before the one-hour pruning drains it.
+    idx_commands_refresh_retention: "65773 2 1",
     idx_commands_created: "499038 1",
     idx_commands_status: "499038 134",
     idx_commands_target: "499038 6",
@@ -78,6 +82,10 @@ export const PROD_EXACT_COMMANDS_STATS: CommandsStatsProfile = {
     idx_commands_target_updated: "505748 157 1 1",
     idx_commands_correlation: "505748 12",
     idx_commands_type_status_created: "505748 18063 6168 1 1",
+    // Undeployed (#3619): conservative full-history upper bound for terminal non-spawn rows.
+    idx_commands_terminal_retention: "505748 2 1",
+    // Undeployed (#3797): conservative refresh-history census, before the one-hour pruning drains it.
+    idx_commands_refresh_retention: "65773 2 1",
     idx_commands_created: "505748 2",
     idx_commands_status: "505748 72250",
     idx_commands_target: "505748 157",

package/src/db/commands-read-ops.ts

@@ -27,6 +27,21 @@ function hostLegacyLandingRecoveryCommands(args: { target: string; afterId?: str
   return rows.map((row) => getCommand(row.id)).filter((command): command is Command => command !== null);
 }
 
+/** The newest attempt of a retried command: `baseId` is attempt 1, `baseId:<n>` attempt n (#3827). One PK range
+ *  scan instead of a point read per past attempt. */
+function latestCommandAttempt(args: { baseId: string }): { attempt: number; command: Command } | null {
+  const rows = getDb().query("SELECT id FROM commands WHERE id = ?1 OR (id > ?2 AND id < ?3)")
+    .all(args.baseId, `${args.baseId}:`, `${args.baseId};`) as Array<{ id: string }>;
+  let latest: { attempt: number; id: string } | undefined;
+  for (const { id } of rows) {
+    const suffix = id.slice(args.baseId.length + 1);
+    const attempt = id === args.baseId ? 1 : /^[1-9]\d*$/.test(suffix) ? Number(suffix) : 0;
+    if (attempt > (latest?.attempt ?? 0)) latest = { attempt, id };
+  }
+  const command = latest ? getCommand(latest.id) : null;
+  return latest && command ? { attempt: latest.attempt, command } : null;
+}
+
 export const COMMANDS_READ_OPS = Object.freeze({
   "commands.latestByCorrelation": (args: { type: string; correlationId: string }) => findLatestCommandByCorrelation(args.type, args.correlationId),
   "commands.list": (args: Parameters<typeof listCommands>[0]) => listCommands(args),
@@ -39,6 +54,7 @@ export const COMMANDS_READ_OPS = Object.freeze({
     return command ? { status: command.status, error: command.error, registered: command.params.registrationConfirmed === true, admission: command.admission, orchestratorId: typeof command.params.orchestratorId === "string" ? command.params.orchestratorId : undefined } : null;
   },
   "commands.get": (args: { id: string }): Command | null => getCommand(args.id),
+  "commands.latestAttempt": latestCommandAttempt,
   "commands.effortChanges": (args: { agentId: string; from: number; to: number; after?: EffortChangeCursor }) => listEffortChanges(args.agentId, args.from, args.to, args.after),
   "commands.listLegacyLandingBackfillCandidates": listLegacyLandingBackfillCandidatesBody,
   // #2102 — the merge⟺deploy interlock drain signal, read off the loop so polling it can never add

package/src/db/commands-spawn-read.ts

@@ -29,5 +29,5 @@ export const IN_FLIGHT_SPAWNS_BY_PARENT_SQL = `SELECT * FROM commands INDEXED BY
 /** One parent's agent.spawn commands still in flight, newest first. */
 export function listInFlightSpawnCommands(parentId: string, limit = 500): Command[] {
   const rows = getDb().query(IN_FLIGHT_SPAWNS_BY_PARENT_SQL).all(...SPAWN_IN_FLIGHT_STATUSES, parentId, limit) as CommandRow[];
-  return rows.map(rowToCommand);
+  return rows.map((row) => rowToCommand(row));
 }

package/src/db/config-store-write-ops.ts

@@ -13,14 +13,15 @@
 // `_writer_idempotency`. That result is a `ConfigEntry` whose `value` is the SAME operator data already
 // persisted verbatim in the `config` table of the SAME database file — so this introduces NO new
 // at-rest exposure class (unlike a signed token, which is never otherwise stored). We therefore keep
-// the real result in the ledger (redacting it would corrupt a keyed phantom-retry's return value) and
-// deliberately omit `serializeIdempotencyResult`.
+// the real result in the ledger, except provider credentials: the ledger is a copy outside the live row,
+// so it masks them like config_history does (#3540, #1870 S7).
 
 import {
   deleteConfigInternal,
   setConfigInternal,
 } from "../config-store.ts";
 import type { ConfigEntry } from "../types.ts";
+import { credentialFreeConfigValue } from "./provider-config-history.ts";
 import type { WriterOpContext, WriterOpSpec } from "./write-ops.ts";
 
 /** Structured-clone-safe args for `config.set` (H6: plain data only). `value` is arbitrary JSON. */
@@ -46,6 +47,10 @@ export const CONFIG_STORE_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Ob
     offLoop: true,
     idempotent: false,
     idempotencySnapshotKey: (args: SetConfigOpArgs) => JSON.stringify([args.namespace, args.key]),
+    serializeIdempotencyResult: (result: unknown) => {
+      const entry = result as ConfigEntry;
+      return { ...entry, value: credentialFreeConfigValue(entry) };
+    },
     fn: (args: SetConfigOpArgs, ctx: WriterOpContext): ConfigEntry =>
       setConfigInternal(args.namespace, args.key, args.value, args.updatedBy, ctx.now),
   },

package/src/db/connection.ts

@@ -66,7 +66,6 @@ export function closeDb(): void {
   }
   db = undefined as unknown as Database;
   dbPath = undefined;
-  closeFastFailConnection();
 }
 
 /**
@@ -175,156 +174,6 @@ export const openExistingDb = (path: string): Database => {
 /** A read-only connection to `path` for a reader that must not share the main RW handle (#3012). */
 export const openReadOnlyDb = (path: string): Database => new Database(path, { readonly: true });
 
-// #1835/#1840 — a SEPARATE connection to the same database file, opened with a much shorter
-// busy_timeout than the shared app connection, lazily created and cached by (path, timeout).
-let fastFailConn: Database | undefined;
-let fastFailConnPath: string | undefined;
-let fastFailConnBusyTimeoutMs: number | undefined;
-
-function closeFastFailConnection(): void {
-  if (fastFailConn) {
-    try { fastFailConn.close(); } catch { /* a half-open handle must not wedge the caller */ }
-  }
-  fastFailConn = undefined;
-  fastFailConnPath = undefined;
-  fastFailConnBusyTimeoutMs = undefined;
-}
-
-/** Forbids `fn` from returning a `PromiseLike` — see `withFastFailConnection`'s own doc for why an
- *  async callback would silently defeat the whole point of this helper. Resolves to `never` for any
- *  `T` that is (or could be) a promise, which TypeScript then rejects at the CALL SITE (the argument
- *  no longer matches `() => never`) rather than merely in a comment. */
-type NotPromiseLike<T> = T extends PromiseLike<unknown> ? never : T;
-
-function isThenable(value: unknown): value is PromiseLike<unknown> {
-  return typeof value === "object" && value !== null && typeof (value as { then?: unknown }).then === "function";
-}
-
-/**
- * Run `fn` with `getDb()` temporarily pointing at a SEPARATE connection to the same database file,
- * using `busyTimeoutMs` instead of the shared app connection's own (much longer) busy_timeout — so a
- * write-lock conflict during `fn` fails FAST (a `SQLITE_BUSY`/`SQLITE_LOCKED` throw after
- * `busyTimeoutMs`) instead of blocking the event loop for up to the shared connection's own budget.
- * Built for `memory-store-health-probe.ts`'s per-scope DB cycle (#1840): a health probe under real
- * write-lock contention should report `cannot-tell` fast, not become a multi-second freeze.
- *
- * LOAD-BEARING INVARIANT — `fn` MUST be fully synchronous: no `await`, no callback deferred to a
- * later microtask/macrotask. The swap works by reassigning the SAME module-level `db` binding
- * `getDb()` reads, for the exact duration of `fn`'s synchronous call; nothing else can observe the
- * swapped connection only because nothing else CAN run while `fn` occupies the (single-threaded)
- * call stack, and the shared connection is restored in `finally` before control ever returns to the
- * event loop. This is exactly why the shared app connection's OWN busy_timeout is never touched by
- * this helper. A caller that needs fast-fail behavior spanning a real await needs a genuinely
- * different design (e.g. #1809/#1822's offload primitive), not this one.
- *
- * ENFORCED AT COMPILE TIME, backstopped (not fixed) at runtime (#1840 re-review MED-1/MED-2) — an
- * `async fn` would still return its Promise SYNCHRONOUSLY (the function body only runs to its first
- * `await` before returning), so a naive version of this helper would restore the shared connection in
- * `finally` and only THEN let the callback's post-`await` continuation run — silently against the
- * ordinary shared connection, with no error, defeating the fast-fail guarantee for exactly the call it
- * was supposed to protect. `NotPromiseLike<T>` is the REAL protection: it rejects an `async fn` at the
- * CALL SITE, so no caller reachable by the type checker can ever produce this shape.
- *
- * The runtime `isThenable` check below is NOT a second way to prevent the same outcome — it CANNOT be:
- * by the time `fn()` has returned a thenable, that thenable's continuation is already scheduled
- * independent of anything this function does next, so throwing here does not (and cannot) stop it
- * from eventually running against whatever connection is active when it resumes. What the runtime
- * check buys is FAIL-LOUD, not fail-safe: a caller that routes an async callback around the type
- * system (a cast through `any`, for instance) gets an immediate, unmissable `TypeError` at the call
- * site instead of a silent, delayed connection mix-up nobody notices until a later bug report. Treat a
- * caller ever seeing this error as a real bug in that call site, not a recoverable runtime condition.
- *
- * Falls back to running `fn` against the ordinary shared connection for `:memory:` databases (this
- * repo's whole test suite) or before any database is open: `bun:sqlite` gives every `:memory:` handle
- * a DISTINCT, empty database (no shared-cache URI in use here), so a second `:memory:` connection
- * would silently see no data at all rather than fail fast on contention — and there is no meaningful
- * lock contention to guard against in a single-process test run in the first place. The thenable
- * guard still applies on this path, so misuse is caught the same way regardless of environment.
- */
-export function withFastFailConnection<T>(busyTimeoutMs: number, fn: () => NotPromiseLike<T>): T {
-  if (!Number.isSafeInteger(busyTimeoutMs) || busyTimeoutMs < 0) {
-    throw new RangeError("busyTimeoutMs must be a non-negative integer");
-  }
-  const path = getDbPath();
-  const useFastFail = Boolean(path) && path !== ":memory:";
-  let original: Database | undefined;
-  if (useFastFail) {
-    if (!fastFailConn || fastFailConnPath !== path || fastFailConnBusyTimeoutMs !== busyTimeoutMs) {
-      closeFastFailConnection();
-      const conn = new Database(path!, { readwrite: true, create: false });
-      // The live connection already established WAL/foreign_keys/etc.; only busy_timeout needs to
-      // differ, applied AFTER the shared pragmas so it overrides the ordinary default.
-      applyConnectionPragmas(conn, { ensureWal: false });
-      conn.run(`PRAGMA busy_timeout = ${busyTimeoutMs}`);
-      fastFailConn = conn;
-      fastFailConnPath = path;
-      fastFailConnBusyTimeoutMs = busyTimeoutMs;
-    }
-    original = db;
-    db = fastFailConn;
-  }
-  try {
-    const result = fn();
-    if (isThenable(result)) {
-      // Fail LOUD, not fail-safe — see this function's doc for why this can only make the misuse
-      // visible, not undo it: the thenable's continuation is already scheduled and will still run
-      // against whatever connection is active when it resumes, regardless of this throw.
-      throw new TypeError(
-        "withFastFailConnection: fn returned a thenable — this is a caller bug, not a recoverable " +
-          "condition. fn must be fully synchronous; an async callback's post-await continuation runs " +
-          "AFTER this call restores the shared connection, so it silently escapes the fast-fail " +
-          "guarantee even though this check caught it here. Fix the caller — do not catch this and " +
-          "continue.",
-      );
-    }
-    return result as T;
-  } finally {
-    if (useFastFail) db = original!;
-  }
-}
-
-/** #2712 — is the write lock free right now? One empty `BEGIN IMMEDIATE` on the fast-fail connection,
- *  so a paced sweep can step aside for another writer instead of waiting out `busy_timeout` per write.
- *  Always true for `:memory:` (no second writer can exist), like {@link withFastFailConnection}. */
-export function writerLockAvailable(busyTimeoutMs: number): boolean {
-  try {
-    return withFastFailConnection(busyTimeoutMs, () => {
-      const conn = getDb();
-      if (conn.inTransaction) return true;
-      conn.run("BEGIN IMMEDIATE");
-      conn.run("ROLLBACK");
-      return true;
-    });
-  } catch (err) {
-    if (isSqliteBusyError(err)) return false;
-    throw err;
-  }
-}
-
-// SQLITE_BUSY resilience for WRITE transactions (#1428).
-//
-// bun:sqlite's `db.transaction(fn)()` begins DEFERRED: the write lock is only taken when the first
-// write statement runs. A read-then-write transaction (the hot message-send path reads getAgent /
-// idempotency / duplicate checks, THEN inserts) therefore starts as a reader and UPGRADES to a
-// writer. If another connection holds the write lock at that instant — e.g. the maintenance-worker
-// subprocess mid-checkpoint/prune — SQLite returns SQLITE_BUSY *immediately* and IGNORES
-// busy_timeout, because waiting for the upgrade could deadlock two readers. That is exactly the
-// production `database is locked` on `POST /api/messages`.
-//
-// `BEGIN IMMEDIATE` takes the write lock up front, so busy_timeout governs contention: a concurrent
-// writer WAITS for the lock instead of erroring on the upgrade. Route every read-then-write
-// transaction through `writeTransaction`; keep pure read-only transactions on the deferred default
-// (they never upgrade, and IMMEDIATE would needlessly serialize reads).
-//
-// SHORT sync-block bias (#1428): `writeTransaction` makes a SINGLE immediate attempt — it never
-// synchronously sleeps-and-retries, because that would only lengthen the main-thread stall. With
-// busy_timeout kept low, that one attempt blocks the loop only briefly. Resilience for the rare BUSY
-// that survives busy_timeout lives in `runWriteWithBusyRetry`, which awaits between attempts so the
-// event loop breathes (other requests/timers run) instead of the thread spinning.
-const SQLITE_BUSY_RETRY_ATTEMPTS = 4;
-const SQLITE_BUSY_RETRY_BASE_MS = 15;
-const SQLITE_BUSY_RETRY_MAX_MS = 120;
-
 /** True for the transient lock-contention codes SQLite raises under concurrency. */
 export function isSqliteBusyError(err: unknown): boolean {
   if (!err || typeof err !== "object") return false;
@@ -337,13 +186,7 @@ export function isSqliteBusyError(err: unknown): boolean {
   return typeof message === "string" && /database (is|table is) locked|SQLITE_BUSY|SQLITE_LOCKED/i.test(message);
 }
 
-/**
- * Run `fn` inside a single `BEGIN IMMEDIATE` transaction. The write lock is taken at BEGIN, so
- * busy_timeout (short, by design) governs contention and the loop stalls only briefly. On a BUSY
- * that survives busy_timeout this THROWS — the caller on the hot path wraps it in
- * `runWriteWithBusyRetry` for an async, loop-yielding retry; other callers surface the error as
- * before. Keep the transaction body small: it runs synchronously with the write lock held.
- */
+// Writer-thread transaction context; boot and in-memory fixtures use the same bodies.
 declare const WriteTransactionContextBrand: unique symbol;
 
 /** Unforgeable caller capability proving code is executing synchronously inside `writeTransaction`.
@@ -413,30 +256,6 @@ export function inWriteTransaction<T>(fn: (tx: WriteTransactionContext, db: Data
   return active ? fn(active, getDb()) : writeTransaction((tx) => fn(tx, getDb()));
 }
 
-/**
- * Hot-path write with a bounded, ASYNC retry around SQLITE_BUSY (#1428). Each attempt is one short
- * synchronous `writeTransaction`; between attempts we `await` a jittered backoff via setTimeout,
- * which YIELDS the event loop so other work runs and the contending writer (another process, a
- * checkpoint) can release the lock — the opposite of a long synchronous busy_timeout spin. A rare
- * contention spike thus degrades to a slightly slower request instead of a 500 or a multi-second
- * main-thread stall.
- */
-export async function runWriteWithBusyRetry<T>(fn: () => T, attempts: number = SQLITE_BUSY_RETRY_ATTEMPTS): Promise<T> {
-  for (let attempt = 1; ; attempt++) {
-    try {
-      return writeTransaction(fn);
-    } catch (err) {
-      if (attempt < attempts && isSqliteBusyError(err)) {
-        const cap = Math.min(SQLITE_BUSY_RETRY_MAX_MS, SQLITE_BUSY_RETRY_BASE_MS * 2 ** (attempt - 1));
-        const delay = cap / 2 + Math.random() * (cap / 2);
-        await new Promise((resolve) => setTimeout(resolve, delay));
-        continue;
-      }
-      throw err;
-    }
-  }
-}
-
 // Slow-query log. Off unless AGENT_RELAY_DB_SLOW_MS is set (>0). Wraps the
 // execution of the heavy dynamic queries so the next slow query surfaces before
 // it wedges something (see issue #197 — the 8.6s reply-obligation query).