create-walle 0.10.2 → 0.10.3

package/template/coding-task-manager/db.js

@@ -34,6 +34,10 @@ const { ensureApprovalDecisionColumns } = require('./lib/approval-decision-colum
 // workers/read-pool-worker.js's 'wallePermissionReconcileRead' case.
 const { queryNonTerminalPtyRuntimeApprovals, queryForwardingWallePermissionResolutions } = require('./lib/walle-permission-reconcile-query');
 const fs = require('fs');
+// D-5 (docs/temp-file-leaks-disk-full-2026-10-25.html §5): one SQLite destination is a file SET —
+// the base plus its -journal/-wal/-shm sidecars. Every backup and live-copy writer reclaims the
+// whole set when it throws, so no orphan sidecar (or base) survives a failed write.
+const { removeSqliteFileSet } = require('./lib/sqlite-file-set');
 const crypto = require('crypto');
 const os = require('os');
 const zlib = require('zlib');
@@ -43,6 +47,10 @@ const { reapGhostRows, agentRowGhostVerdict } = require('./lib/session-identity-
 const conversationSourceRegistry = require('./lib/conversation-source-registry');
 const conversationCommitKernel = require('./lib/conversation-commit-kernel');
 const conversationStoreV2 = require('./lib/conversation-store-v2');
+// join-sync-followups-2 WP-4: the clock's identity seam (it cannot import instance-identity-access; see its header).
+require('./lib/conversation-item-clock').setConversationInstanceIdResolver(() => {
+  try { return getCurrentHostedInstanceIdStrict(); } catch { /* expected: identity not readable yet (pre-initDb / bare fixture) — the clock keeps origin '' rather than mislabelling */ return ''; }
+});
 const walleHistoryCutoverV2 = require('./lib/walle-history-cutover-v2');
 const { createReviewReposStore } = require('./lib/storage/review-repos-store');
 const { insertResumeLineage } = require('./lib/storage/resume-lineage-store');
@@ -172,6 +180,8 @@ const {
 } = require('../shared/db-migration-runner');
 // WP-3a (§6.15): pure release schema authority — data + pure functions, no app/native dependency.
 const { assertMigrationRegistry } = require('../shared/release-contract');
+// Backup retention policy shared with Wall-E (design: docs/backup-retention-days-setting-2026-10-31.html).
+const backupRetention = require('../shared/backup-retention');
 
 let dbErrorSink = createErrorSink({ name: 'db' });
 function setErrorSink(sink) {
@@ -300,13 +310,17 @@ function _pathIsWithin(candidate, parent) {
 }
 
 function _assertDevDbPathIsNotPrimary(dbPath) {
-  if (!process.env.DEV_CTM_PORT || require('./lib/env-flags').boolFlag('CTM_ALLOW_PRIMARY_DB', false)) return;
+  if (require('./lib/env-flags').boolFlag('CTM_ALLOW_PRIMARY_DB', false)) return;
+  // 2026-10-31: unmerged branch code (a linked git worktree) must never open -- and so never migrate --
+  // the primary DB; see shared/linked-worktree.js. A primary or npm install is never a linked worktree.
+  const linkedWorktree = require('../shared/linked-worktree').isLinkedWorktreeCheckout(__dirname);
+  if (!process.env.DEV_CTM_PORT && !linkedWorktree) return;
   const defaultPrimaryDir = require('./lib/data-dir').getDefaultDataDir();
   const configuredPrimaryDir = process.env.CTM_PRIMARY_DATA_DIR || process.env.CTM_PRODUCTION_DATA_DIR;
   const primaryDirs = [defaultPrimaryDir, configuredPrimaryDir].filter(Boolean);
   if (!primaryDirs.some((primaryDir) => _pathIsWithin(dbPath, primaryDir))) return;
 
-  const error = new Error(`CTM_PRIMARY_DB_REFUSED_IN_DEV: refusing ${path.resolve(dbPath)} while DEV_CTM_PORT is set`);
+  const error = new Error(`CTM_PRIMARY_DB_REFUSED_IN_DEV: refusing ${path.resolve(dbPath)} while ${linkedWorktree ? 'running from a linked git worktree (unmerged branch code); set CTM_DATA_DIR to a scratch dir' : 'DEV_CTM_PORT is set'}`);
   error.code = 'CTM_PRIMARY_DB_REFUSED_IN_DEV';
   throw error;
 }
@@ -316,6 +330,8 @@ function _assertDevDbPathIsNotPrimary(dbPath) {
 // keep all backup call sites reading this module-level value instead of capturing it.
 const DEFAULT_BACKUP_DIR = _normalizeUserPath(process.env.CTM_BACKUP_DIR || path.join(DATA_DIR, 'backups'));
 let BACKUP_DIR = DEFAULT_BACKUP_DIR;
+// Retention is a module value read at prune time (design §5 R-8); the default is 3.
+let BACKUP_RETENTION_DAYS = backupRetention.DEFAULT_BACKUP_RETENTION_DAYS;
 const IMAGE_HASH_CHUNK_BYTES = Math.max(
   16 * 1024,
   Number(process.env.CTM_IMAGE_HASH_CHUNK_BYTES || 512 * 1024) || 512 * 1024
@@ -551,6 +567,22 @@ function _applyPersistedBackupDirSetting() {
   }
 }
 
+// Backup retention (design §5 R-5/R-8): a plain module value; the setter validates but never
+// persists, so writing the setting is the caller's job (api/backup-retention.js PUT).
+function getBackupRetentionDays() { return BACKUP_RETENTION_DAYS; }
+function setBackupRetentionDays(days) {
+  BACKUP_RETENTION_DAYS = backupRetention.normalizeRetentionDays(days);
+  return BACKUP_RETENTION_DAYS;
+}
+function _applyPersistedBackupRetentionSetting() {
+  try {
+    if (!db || !_tableExists('settings')) return;
+    setBackupRetentionDays(getSetting(backupRetention.BACKUP_RETENTION_SETTING_KEY, null));
+  } catch (e) {
+    console.error('[db] Backup retention setting ignored:', e.message);
+  }
+}
+
 // Cache for resolveGitRoot — avoids repeated git calls for the same path
 const _gitRootCache = new Map();
 
@@ -743,6 +775,7 @@ function _baseResidentDb() {
   return _baseFileHandle;
 }
 function _isDomainFlippedInSettings(domain) {
+  if (domain !== 'core' && domain !== 'telemetry') return false; // only EXTRACTABLE_DOMAINS (lib/domain-split-migration.js) flip; never read settings for firehose in a request zone
   if (_domainFlipStickyCache.get(domain)) return true;
   try {
     const row = _baseResidentDb().prepare('SELECT value FROM settings WHERE key = ?').get(`domain_routing_flip:${domain}`);
@@ -4951,6 +4984,7 @@ function initDb(dbPath) {
     _resetHostedPrimarySchemaMemo();
     _ensureHostedPrimarySchema(db);
     _applyPersistedBackupDirSetting();
+    _applyPersistedBackupRetentionSetting();
     cleanupScrollbackChunkSeqIntegrity();
     return db;
   } catch (err) {
@@ -7569,6 +7603,31 @@ function _defaultMigrationSteps() {
     53: () => _migrateToV53Impl({ getDb: () => dbFor('session_model_intent'), setSchemaVersion }),
     // v54 (WP-0) the per-session lead model pin — an explicit model choice replayed on resume; v55 (WP-2, multi-machine) stamps blank agent_sessions.origin_instance_id rows with this database's identity (one idempotent UPDATE, sharing this line because db.js is at its size ceiling); v56 (WP-1, machine pool) is host_samples + machine_self + machine_pairing_offers + machines.urls_json, all domain 'core'.
     54: () => _migrateToV54Impl({ getDb: () => dbFor('session_lead_model'), setSchemaVersion }), 55: () => require('./lib/migrations/v55-agent-origin-backfill').migrateToV55({ getDb: () => dbFor('agent_sessions'), getIdentityDb: () => dbFor('instance_identity'), setSchemaVersion }), 56: () => require('./lib/migrations/v56-host-samples-and-machine-pool').migrateToV56({ getDb: () => dbFor('machines'), setSchemaVersion }),
+    // v57 (opencode-lead-and-delegate-model-selection-2026-10-28.html §6.4, §8 WP-4, S-17/S-18):
+    // both delegate-preference tables admit agent='opencode'. SQLite cannot ALTER a CHECK, so the
+    // migration REBUILDS them from their own stored DDL (one transaction, every row and column);
+    // v48 is untouched. Idempotent by inspection of sqlite_master, so a re-run is a no-op.
+    57: () => require('./lib/migrations/v57-delegate-pref-mirror-opencode').migrateToV57({ getDb: () => dbFor('delegate_pref_mirror'), setSchemaVersion }),
+    // v58 (WP-1, codex-native-subagent-pick-enforcement-2026-10-28.html §6): the launch row gains
+    // the native-sub-agents deny fact (nullable TEXT, no default), so the UI states what a launch
+    // DID instead of re-deriving the rule. One ALTER on the v48 launch table; core domain, like v48/v50.
+    58: () => require('./lib/migrations/v58-subagent-launch-native-subagents').migrateToV58({ getDb: () => dbFor('session_subagent_pick_launch'), setSchemaVersion }),
+    // v59 (WP-8, ctm-deepseek-harness-acp-agent-design §9b): coding_session_model_state admits the
+    // DeepSeek Harness agent 'dsh'. SQLite cannot ALTER a CHECK, so v59 REBUILDS the table from its
+    // own stored DDL in one transaction; v42 is untouched and a re-run is a no-op.
+    59: () => require('./lib/migrations/v59-coding-session-model-state-dsh').migrateToV59({ getDb: () => dbFor('coding_session_model_state'), setSchemaVersion }),
+    // v60 (WP-10 step C, ctm-deepseek-harness-acp-agent-design §9b): the delegate pref mirror admits
+    // agent 'dsh', the CTM half of the brain's migration 103 widening of `default:dsh'. SQLite cannot
+    // ALTER a CHECK, so v60 REBUILDS the table from its own stored DDL in one transaction; v48 is
+    // untouched and a re-run is a no-op. session_subagent_pick_launch is deliberately NOT widened:
+    // no writer can reach it with 'dsh' (see the module header).
+    60: () => require('./lib/migrations/v60-delegate-pref-mirror-dsh').migrateToV60({ getDb: () => dbFor('delegate_pref_mirror'), setSchemaVersion }),
+    // v61 (WP-5, shared-memory-every-machine-2026-10-31.html §5 D-3, §6.8): the conversation-item
+    // change clock — three additive tables + one index, declared by the migration instead of the
+    // store's own DDL (the store is a ratcheted lazy store; lib/migrations/** is sanctioned). The
+    // clock is stamped in the same transaction as conversation_text_items_v2, a 'firehose' table, so
+    // this migration hands the same handle/file as the text table.
+    61: () => require('./lib/migrations/v61-conversation-item-clock').migrateToV61({ getDb: () => dbFor('conversation_text_items_v2'), setSchemaVersion }),
   }, { reservedVersions: [18] });
 }
 let _migrationSteps = _defaultMigrationSteps();
@@ -12969,6 +13028,17 @@ function _discardPartialBackupSet(names) {
   }
 }
 
+// D-5: a writer that throws reclaims the destination's whole file set (base + sidecars, via
+// lib/sqlite-file-set.js) — including the .db.gz companion when the function writes one.
+// Best-effort by construction: the caller rethrows the ORIGINAL error, so a reclaim failure
+// must never replace it.
+function _reclaimBackupDestinationSet(...paths) {
+  for (const target of paths) {
+    if (!target) continue;
+    try { removeSqliteFileSet(target); } catch { /* expected: the reclaim is best-effort and must never mask the original writer error, which the caller rethrows */ }
+  }
+}
+
 function _finalizeBackup(tmpPath, gzPath, now, tag, basePath) {
   const ts = now.toISOString().replace(/[:.]/g, '-').slice(0, 19);
   const check = _quickCheckFile(tmpPath);
@@ -13010,8 +13080,13 @@ async function createBackup(label) {
   const { now, tmpPath, gzPath, tag } = _backupPaths(label);
   fs.rmSync(tmpPath, { force: true });
   fs.rmSync(gzPath, { force: true });
-  await db.backup(tmpPath);
-  return _finalizeBackupAsync(tmpPath, gzPath, now, tag, getBaseDbPath() || currentDbPath);
+  try {
+    await db.backup(tmpPath);
+    return await _finalizeBackupAsync(tmpPath, gzPath, now, tag, getBaseDbPath() || currentDbPath);
+  } catch (err) {
+    _reclaimBackupDestinationSet(tmpPath, gzPath);
+    throw err;
+  }
 }
 
 // Live-snapshot entry point for the dedicated backup worker. It intentionally does not call
@@ -13023,6 +13098,7 @@ async function createBackupFromPath(sourcePath, label, options = {}) {
   if (!requestedSource) throw new Error('Backup source database path required');
   const resolvedSource = path.resolve(requestedSource);
   if (options.backupDir) setBackupDir(options.backupDir, { persist: false });
+  if (options.retentionDays != null) setBackupRetentionDays(options.retentionDays);
 
   const { now, tmpPath, gzPath, tag } = _backupPaths(label);
   fs.rmSync(tmpPath, { force: true });
@@ -13036,12 +13112,15 @@ async function createBackupFromPath(sourcePath, label, options = {}) {
   });
   try {
     source.prepare('VACUUM INTO ?').run(tmpPath);
+    // The worker is handed the BASE path, so its siblings resolve from the same directory —
+    // this is the path the daily backup actually takes (setBackupRunner -> online-backup-worker).
+    return await _finalizeBackupAsync(tmpPath, gzPath, now, tag, resolvedSource);
+  } catch (err) {
+    _reclaimBackupDestinationSet(tmpPath, gzPath);
+    throw err;
   } finally {
     try { source.close(); } catch {}
   }
-  // The worker is handed the BASE path, so its siblings resolve from the same directory —
-  // this is the path the daily backup actually takes (setBackupRunner -> online-backup-worker).
-  return _finalizeBackupAsync(tmpPath, gzPath, now, tag, resolvedSource);
 }
 
 async function copyLiveDatabaseTo(destPath) {
@@ -13059,22 +13138,16 @@ async function copyLiveDatabaseTo(destPath) {
   }
   const tmpPath = path.join(destDir, `.${path.basename(resolved)}.${process.pid}.${Date.now()}.tmp`);
   fs.mkdirSync(destDir, { recursive: true });
-  for (const p of [tmpPath, `${tmpPath}-wal`, `${tmpPath}-shm`]) {
-    try { fs.rmSync(p, { force: true }); } catch {}
-  }
+  _reclaimBackupDestinationSet(tmpPath);
   try {
     await db.backup(tmpPath);
     const check = _quickCheckFile(tmpPath);
     if (!check.ok) throw new Error(`Live database copy quick_check failed: ${check.error}`);
-    for (const p of [resolved, `${resolved}-wal`, `${resolved}-shm`]) {
-      try { fs.rmSync(p, { force: true }); } catch {}
-    }
+    _reclaimBackupDestinationSet(resolved);
     fs.renameSync(tmpPath, resolved);
     return { path: resolved, size: fs.statSync(resolved).size };
   } catch (err) {
-    for (const p of [tmpPath, `${tmpPath}-wal`, `${tmpPath}-shm`]) {
-      try { fs.rmSync(p, { force: true }); } catch {}
-    }
+    _reclaimBackupDestinationSet(tmpPath);
     throw err;
   }
 }
@@ -13084,8 +13157,13 @@ function createBackupSync(label) {
   const { now, tmpPath, gzPath, tag } = _backupPaths(label);
   fs.rmSync(tmpPath, { force: true });
   fs.rmSync(gzPath, { force: true });
-  db.exec(`VACUUM INTO ${_quoteSqlString(tmpPath)}`);
-  return _finalizeBackup(tmpPath, gzPath, now, tag, getBaseDbPath() || currentDbPath);
+  try {
+    db.exec(`VACUUM INTO ${_quoteSqlString(tmpPath)}`);
+    return _finalizeBackup(tmpPath, gzPath, now, tag, getBaseDbPath() || currentDbPath);
+  } catch (err) {
+    _reclaimBackupDestinationSet(tmpPath, gzPath);
+    throw err;
+  }
 }
 
 function listBackups() {
@@ -13412,16 +13490,9 @@ async function restoreAndVerifyPromptPurgeBackupAbsent(baseline = {}, closure =
   }
 }
 
-function cleanOldBackups(maxAge = 7) {
-  // Keep backups for maxAge days, but always keep at least 5
-  const backups = listBackups();
-  if (backups.length <= 5) return;
-
-  const cutoff = Date.now() - (maxAge * 24 * 60 * 60 * 1000);
-  const toDelete = backups.slice(5).filter(b => new Date(b.createdAt).getTime() < cutoff);
-  for (const b of toDelete) {
-    deleteBackup(b.name);
-  }
+// Design §5 R-1: keep every backup from the N most recent UTC days that have one (shared policy).
+function cleanOldBackups(retentionDays = BACKUP_RETENTION_DAYS) {
+  for (const b of backupRetention.selectExpiredBackups(listBackups(), retentionDays)) deleteBackup(b.name);
 }
 
 // Optional hook (registered by server.js via setBackupRunner) that dispatches a backup to a
@@ -16052,11 +16123,36 @@ function listStartupTasks(opts = {}) {
 // listStartupTasks() above drops exited rows — so "does this hub's ledger know the id, and who owns
 // it" is answered here. Deliberately throw-through: the catalog's per-clause guard counts the fault
 // and fails OPEN, which a silent empty answer here would hide.
+//
+// D-5 (docs/remote-session-rename-forward-and-empty-rows-2026-10-30.html §5 D-5, §6, §9 S-7): a NAME
+// is not ownership. The pre-WP-1 hub rename/star of a box-owned id wrote a TITLE-ONLY stub into this
+// table — provider/project_path/cwd/origin_instance_id/current_owner_instance_id all '' and no
+// agent_sessions row — and this read then reported it as an owned ledger row, so clause (d) fenced the
+// box's REAL row out and the session vanished from the hub. Such a stub is not ownership: it answers
+// `{ exists: false, currentOwner: '', titleOnlyStub: true }` and the box row is kept. Every other row
+// answers exactly what it always did. Nothing is deleted (R-143).
 function getCtmSessionOwnerRow(ctmSessionId) {
   const id = String(ctmSessionId || '').trim();
   if (!id) return { exists: false, currentOwner: '' };
-  const row = getDomainDb('core').prepare('SELECT current_owner_instance_id FROM ctm_sessions WHERE id = ?').get(id);
+  // current_owner_instance_id is selected unconditionally: a schema without it already threw here
+  // (throw-through, per the comment above). The four D-5 columns and agent_sessions may be absent on
+  // an older schema — a missing one is selected as NULL and `evaluable` then refuses to call the row
+  // a stub, so a shape this read cannot fully evaluate never newly fences a box row out.
+  const d5 = ['provider', 'project_path', 'cwd', 'origin_instance_id'];
+  const missing = d5.filter((c) => !_tableHasColumn('ctm_sessions', c));
+  const hasAgent = _tableExists('agent_sessions') && _tableHasColumn('agent_sessions', 'ctm_session_id');
+  const row = getDomainDb('core').prepare(`
+    SELECT current_owner_instance_id, ${d5.map((c) => (missing.includes(c) ? `NULL AS ${c}` : c)).join(', ')},
+           ${hasAgent ? 'EXISTS(SELECT 1 FROM agent_sessions a WHERE a.ctm_session_id = ctm_sessions.id)' : '0'} AS has_agent
+    FROM ctm_sessions WHERE id = ?
+  `).get(id);
   if (!row) return { exists: false, currentOwner: '' };
+  const blank = (v) => String(v == null ? '' : v).trim() === '';
+  const evaluable = !missing.length;
+  if (evaluable && !Number(row.has_agent) && blank(row.provider) && blank(row.project_path)
+    && blank(row.cwd) && blank(row.origin_instance_id) && blank(row.current_owner_instance_id)) {
+    return { exists: false, currentOwner: '', titleOnlyStub: true };
+  }
   return { exists: true, currentOwner: String(row.current_owner_instance_id || '').trim() };
 }
 
@@ -19983,7 +20079,29 @@ const _WORKER_SESSION_PATCHABLE_COLUMNS = new Set([
   'failure_json', 'consumed_at', 'acknowledged_at', 'hidden_at', 'hide_at', 'launch_json', 'runtime_json', 'generation',
 ]);
 
-function updateWorkerSession(ctmSessionId, patch = {}) {
+// C7-B1 (worker-terminal-retry-generation-fence-2026-10-27.html §3.1): the optional third
+// argument is an INTERNAL storage fence, not a patch field and not a general predicate engine.
+// `options === undefined` (or omitted) is the unchanged legacy two-argument call; a PROVIDED
+// options object must carry EXACTLY the own key expectedGeneration, a positive safe integer, and
+// is validated BEFORE any DB access -- a present-but-malformed guard is a programming error, never
+// a silent unfenced write. A valid guard appends one bound predicate (never an interpolated
+// value) so a stale generation matches zero rows and the existing boolean result already reports
+// the refusal. Generation stays a SETtable column: the guard never writes it.
+function updateWorkerSession(ctmSessionId, patch = {}, options) {
+  let expectedGeneration;
+  if (options !== undefined) {
+    if (!options || typeof options !== 'object' || Array.isArray(options)) {
+      throw new TypeError('updateWorkerSession: options must be an object carrying exactly { expectedGeneration }');
+    }
+    const guardKeys = Reflect.ownKeys(options);
+    if (guardKeys.length !== 1 || guardKeys[0] !== 'expectedGeneration') {
+      throw new TypeError('updateWorkerSession: options must carry exactly one own key, expectedGeneration');
+    }
+    expectedGeneration = options.expectedGeneration;
+    if (!Number.isSafeInteger(expectedGeneration) || expectedGeneration <= 0) {
+      throw new TypeError('updateWorkerSession: expectedGeneration must be a positive safe integer');
+    }
+  }
   const id = String(ctmSessionId || '').trim();
   if (!id) return false;
   const columns = Object.keys(patch || {}).filter((k) => _WORKER_SESSION_PATCHABLE_COLUMNS.has(k));
@@ -19991,8 +20109,10 @@ function updateWorkerSession(ctmSessionId, patch = {}) {
   const setClause = columns.map((c) => `${c} = @${c}`).join(', ');
   const params = { id };
   for (const c of columns) params[c] = patch[c];
+  const generationPredicate = options === undefined ? '' : ' AND generation = @expectedGeneration';
+  if (options !== undefined) params.expectedGeneration = expectedGeneration;
   const result = dbFor('worker_sessions').prepare(
-    `UPDATE worker_sessions SET ${setClause} WHERE ctm_session_id = @id`
+    `UPDATE worker_sessions SET ${setClause} WHERE ctm_session_id = @id${generationPredicate}`
   ).run(params);
   return result.changes > 0;
 }
@@ -20515,6 +20635,10 @@ const DB_OWNER_MUTATION_ACTIONS = Object.freeze({
   ackConversationChanges: (payload) => conversationChangeFeed.ackConversationChanges(
     dbFor('conversation_projection_outbox_v2'), payload,
   ),
+  // join-sync-followups-2 WP-3: /api/sync/conversation/v2/* run here, never in the request zone.
+  conversationSyncVector: (opts) => require('./lib/conversation-sync').ownerVector(dbFor('conversation_text_items_v2'), opts),
+  conversationSyncChanges: (opts) => require('./lib/conversation-sync').ownerChanges(dbFor('conversation_text_items_v2'), opts),
+  conversationSyncApply: (opts) => require('./lib/conversation-sync').ownerApply(dbFor('conversation_text_items_v2'), opts),
 });
 
 function runDbOwnerMutationAction(action, args = []) {
@@ -20547,7 +20671,7 @@ module.exports = {
   _getMigrationLadderInfo,
   // Canonical filename for bin/migrate.js; avoids a second DEFAULT_DB_PATH basename.
   CTM_DB_FILENAME: path.basename(DEFAULT_DB_PATH),
-  initDb, connectDb, getDb, closeDb, getDbPath: () => currentDbPath, getBackupDir: () => BACKUP_DIR, getBackupDirInfo, getHostedBackupStatus, setBackupDir, moveBackupsToDir, flushWal, resolveGitRoot,
+  initDb, connectDb, getDb, closeDb, getDbPath: () => currentDbPath, getBackupDir: () => BACKUP_DIR, getBackupDirInfo, getHostedBackupStatus, setBackupDir, moveBackupsToDir, flushWal, resolveGitRoot, getBackupRetentionDays, setBackupRetentionDays, cleanOldBackups,
   getDomainDb,
   getBaseDbPath,
   setBaseDbPath,