create-walle 0.9.25 → 0.9.26

package/template/claude-task-manager/db.js

@@ -8,7 +8,7 @@ const { threadId, isMainThread } = require('node:worker_threads');
 const { execFileSync } = require('child_process');
 const { normalizeAgentType } = require('./lib/agent-capabilities');
 const { codexRolloutIdFromPath, readCodexRolloutMetadata } = require('./lib/session-history');
-const { ensureTranscriptTables } = require('./lib/transcript-store');
+const { ensureTranscriptTables, healSelfLinkedTranscriptKeys } = require('./lib/transcript-store');
 const { isProviderGeneratedUserContextText } = require('./lib/provider-user-context');
 const { buildSessionImageRefs } = require('./lib/session-image-refs');
 const {
@@ -693,6 +693,26 @@ function _openDbConnection(dbPath) {
   d.pragma('journal_mode = WAL');
   d.pragma(`busy_timeout = ${busyTimeoutMs}`);
   d.pragma('foreign_keys = ON');
+  // synchronous=NORMAL (safe under WAL): the default FULL fsync's on EVERY commit, which is the
+  // bulk of how long a write txn holds the cross-process write lock — and that hold is what parks
+  // the main thread OFF-CPU (the "(unknown)" lock-wait freezes that stall Codex keystrokes). Under
+  // WAL, NORMAL only fsync's at checkpoint, not per commit; the documented trade-off is that a
+  // power-loss/OS-crash can lose the last few committed txns (NOT corrupt the db). For a local
+  // session cache that's continuously re-derivable from the JSONL transcripts, that's an easy win.
+  // Override with CTM_SQLITE_SYNCHRONOUS=FULL to restore the old behavior.
+  const _sync = String(process.env.CTM_SQLITE_SYNCHRONOUS || 'NORMAL').toUpperCase();
+  if (['OFF', 'NORMAL', 'FULL', 'EXTRA'].includes(_sync)) d.pragma(`synchronous = ${_sync}`);
+  // Disable SQLite's INLINE auto-checkpoint (default 1000 pages). Otherwise any commit that
+  // pushes the WAL past the threshold runs a PASSIVE checkpoint SYNCHRONOUSLY on that connection,
+  // holding CTM's cross-process write lock for the checkpoint I/O while every other writer stalls.
+  // Once the WAL bloats (reader-pool snapshots pin frames so PASSIVE can't truncate, observed 49MB)
+  // that inline checkpoint scans the whole WAL — measured 457ms on a trivial cursor upsert and up to
+  // 3s on the periodic pass — landing the cost on a random user-facing write (the off-CPU lock-wait
+  // freezes). We OWN checkpointing explicitly off the event loop: the db-owner worker runs a periodic
+  // TRUNCATE (bounds the file) plus flushWal PASSIVE nudges after critical writes. Set
+  // CTM_WAL_AUTOCHECKPOINT to a positive page count to restore SQLite's inline backstop.
+  const _autoCkpt = Number(process.env.CTM_WAL_AUTOCHECKPOINT);
+  d.pragma(`wal_autocheckpoint = ${Number.isFinite(_autoCkpt) && _autoCkpt >= 0 ? Math.floor(_autoCkpt) : 0}`);
   return d;
 }
 
@@ -1127,6 +1147,13 @@ function runMigrations() {
     getDb().exec('ALTER TABLE perm_rules ADD COLUMN always_ask INTEGER DEFAULT 0');
   }
 
+  // Add exceptions column to perm_rules (migration) — allow-exceptions attached
+  // to deny rules ("deny rm except under /tmp"); JSON array, NULL = none.
+  const hasExceptions = getDb().prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='perm_rules'").get();
+  if (hasExceptions && !hasExceptions.sql.includes('exceptions')) {
+    getDb().exec('ALTER TABLE perm_rules ADD COLUMN exceptions TEXT DEFAULT NULL');
+  }
+
   // Collapse CTM permission rules into a single GLOBAL config (decoupled from
   // per-project Claude/Codex settings). CTM owns permissions in its own DB and
   // applies them to all sessions; the project column is legacy. Idempotent: runs
@@ -1305,6 +1332,17 @@ function runMigrations() {
     getDb().exec("ALTER TABLE approval_decisions ADD COLUMN command_signature TEXT DEFAULT ''");
   }
 
+  // When a Wall-E coding turn parks on approval it registers the park here so it
+  // surfaces in the shared "Pending" tab and survives reload. walle_request_id is
+  // the Wall-E PermissionService request id (permId); resolving the row calls back
+  // that id to unpark the live turn. walle_session_id scopes the cascade re-check.
+  try {
+    getDb().prepare("SELECT walle_request_id FROM approval_decisions LIMIT 1").get();
+  } catch {
+    getDb().exec("ALTER TABLE approval_decisions ADD COLUMN walle_request_id TEXT DEFAULT ''");
+    getDb().exec("ALTER TABLE approval_decisions ADD COLUMN walle_session_id TEXT DEFAULT ''");
+  }
+
   // Approval observations are the pre-decision control-plane audit trail.
   // approval_decisions records what the agent did; this table records every
   // approval-shaped terminal screen, including prompts rejected by structural
@@ -1866,9 +1904,16 @@ function runMigrations() {
   // table-create block above only fires for fresh DBs; run this every init
   // (idempotent) so existing DBs get the index too. Turns a filtered partial
   // scan + sort into a pre-ordered index range scan.
-  getDb().exec(
-    "CREATE INDEX IF NOT EXISTS idx_session_messages_user_prompts ON session_messages(ctm_session_id, role, message_index);"
-  );
+  // Guard on the column: a v0 DB still has session_messages.session_id at this
+  // point — the rename to ctm_session_id happens later in this same pass — so an
+  // unguarded CREATE INDEX here throws "no such column: ctm_session_id" and aborts
+  // the whole v0→current migration. migrateToV1() builds it itself right after
+  // the rename so a migrating DB still gets the index in this same pass.
+  if (_tableHasColumn('session_messages', 'ctm_session_id')) {
+    getDb().exec(
+      "CREATE INDEX IF NOT EXISTS idx_session_messages_user_prompts ON session_messages(ctm_session_id, role, message_index);"
+    );
+  }
 
   // --- FTS5 on session_messages ---
   const hasMsgFts = getDb().prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='session_messages_fts'").get();
@@ -1937,6 +1982,13 @@ function runMigrations() {
   // migration time instead of the first user search.
   try {
     ensureTranscriptTables(getDb());
+    // Heal any transcript rows orphaned by the historical self-link (rollouts keyed by their
+    // own agent_session_id instead of the owning ctm_session_id — splits a resumed session's
+    // Conv tab). Idempotent + bounded; a no-op once clean. See healSelfLinkedTranscriptKeys.
+    const healed = healSelfLinkedTranscriptKeys(getDb());
+    if (healed && (healed.files || healed.events)) {
+      console.log(`[db] healed self-linked transcript keys: sessions=${healed.sessions}, files=${healed.files}, events=${healed.events}`);
+    }
   } catch (e) {
     console.error('[db] transcript row-store migration failed:', e.message);
   }
@@ -2911,6 +2963,10 @@ function migrateToV1() {
     const smCols = d.prepare("PRAGMA table_info(session_messages)").all();
     if (smCols.find(c => c.name === 'session_id')) {
       d.exec('ALTER TABLE session_messages RENAME COLUMN session_id TO ctm_session_id');
+      // The "every init" index build earlier in runMigrations skips a v0 table
+      // (column not renamed yet); build it here so a migrating DB gets the
+      // covering index in this same pass instead of one boot later.
+      d.exec('CREATE INDEX IF NOT EXISTS idx_session_messages_user_prompts ON session_messages(ctm_session_id, role, message_index)');
     }
 
     // session_analyses: session_id -> ctm_session_id (it is the PK)
@@ -3073,10 +3129,63 @@ function addSessionDiagnostic(sessionId, event, details = {}) {
           LIMIT 500
        )
   `).run(cleanSessionId, cleanSessionId);
-  flushWal();
+  // No flushWal() here — the WAL is durable on commit, and the throttled
+  // periodic checkpoint (flushWal trailing timer) owns flushing. Calling it on
+  // every diagnostic write made the checkpoint contend with the write storm for
+  // the single writer lock (see flushSessionDiagnostics for the batched path).
   return { id: result.lastInsertRowid, session_id: cleanSessionId, event: cleanEvent, ...payload };
 }
 
+// UTC timestamp string matching SQLite's datetime('now') format, so batched
+// inserts carry the same shape as the legacy per-row datetime('now') default.
+function _utcSqliteTimestamp(ts) {
+  const d = ts != null ? new Date(ts) : new Date();
+  const iso = Number.isNaN(d.getTime()) ? new Date().toISOString() : d.toISOString();
+  return iso.replace('T', ' ').replace(/\.\d+Z$/, '');
+}
+
+// Batched diagnostic writer: persist a coalesced array of diagnostics in ONE
+// transaction (one writer-lock acquire / one commit for the whole batch) and run
+// the per-session retention sweep at most once per affected session. Replaces the
+// per-event INSERT+DELETE+flushWal storm that saturated the single-writer DB under
+// heavy multi-session load. Sanitization + JSON happen OUTSIDE the txn to minimize
+// lock-hold time. `entries`: [{ sessionId, event, details, createdAt }].
+function flushSessionDiagnostics(entries) {
+  const list = Array.isArray(entries) ? entries : [];
+  const prepared = [];
+  const sessions = new Set();
+  for (const e of list) {
+    if (!e) continue;
+    const sid = String(e.sessionId || '').trim();
+    if (!sid) continue;
+    const event = String(e.event || '').trim() || 'event';
+    const payload = _sanitizeDiagnosticDetails(e.details || {});
+    prepared.push([sid, event, JSON.stringify(payload), e.createdAt || _utcSqliteTimestamp()]);
+    sessions.add(sid);
+  }
+  if (!prepared.length) return { inserted: 0, sessionsSwept: 0 };
+  const d = getDb();
+  const insert = d.prepare(
+    'INSERT INTO session_diagnostics (session_id, event, details_json, created_at) VALUES (?, ?, ?, ?)'
+  );
+  const sweep = d.prepare(`
+    DELETE FROM session_diagnostics
+     WHERE session_id = ?
+       AND id NOT IN (
+         SELECT id FROM session_diagnostics
+          WHERE session_id = ?
+          ORDER BY id DESC
+          LIMIT 500
+       )
+  `);
+  const txn = d.transaction(() => {
+    for (const row of prepared) insert.run(row[0], row[1], row[2], row[3]);
+    for (const sid of sessions) sweep.run(sid, sid);
+  });
+  txn();
+  return { inserted: prepared.length, sessionsSwept: sessions.size };
+}
+
 function listSessionDiagnostics(sessionId, limit = 120) {
   const cleanSessionId = String(sessionId || '').trim();
   if (!cleanSessionId) return [];
@@ -4097,11 +4206,12 @@ function listPermRules({ project, listType } = {}) {
   return getDb().prepare(sql).all(...params);
 }
 
-function addPermRule({ rule, listType, scope, project }) {
+function addPermRule({ rule, listType, scope, project, exceptions }) {
   getDb().prepare(
-    `INSERT OR IGNORE INTO perm_rules (rule, list_type, scope, project)
-     VALUES (?, ?, ?, ?)`
-  ).run(rule, listType || 'allow', scope || 'global', project || '__global__');
+    `INSERT OR IGNORE INTO perm_rules (rule, list_type, scope, project, exceptions)
+     VALUES (?, ?, ?, ?, ?)`
+  ).run(rule, listType || 'allow', scope || 'global', project || '__global__',
+    Array.isArray(exceptions) && exceptions.length ? JSON.stringify(exceptions) : null);
 }
 
 function removePermRule({ rule, listType, project }) {
@@ -4110,15 +4220,28 @@ function removePermRule({ rule, listType, project }) {
   ).run(rule, listType || 'allow', project || '__global__');
 }
 
+// Replace the exceptions array on one rule row. `exceptions` is a validated
+// array (the API layer validates shape/limits) or [] to clear. Returns the
+// number of rows updated (0 = rule not found).
+function setPermRuleExceptions({ rule, listType, project, exceptions }) {
+  const json = Array.isArray(exceptions) && exceptions.length ? JSON.stringify(exceptions) : null;
+  const res = getDb().prepare(
+    'UPDATE perm_rules SET exceptions = ? WHERE rule = ? AND list_type = ? AND project = ?'
+  ).run(json, rule, listType || 'deny', project || '__global__');
+  return res.changes;
+}
+
 function bulkSetPermRules(rules) {
-  // rules: [{ rule, listType, scope, project }]
+  // rules: [{ rule, listType, scope, project, exceptions? }]
   const txn = getDb().transaction(() => {
     getDb().exec('DELETE FROM perm_rules');
     const ins = getDb().prepare(
-      'INSERT OR IGNORE INTO perm_rules (rule, list_type, scope, project) VALUES (?, ?, ?, ?)'
+      'INSERT OR IGNORE INTO perm_rules (rule, list_type, scope, project, exceptions) VALUES (?, ?, ?, ?, ?)'
     );
     for (const r of rules) {
-      ins.run(r.rule, r.listType || 'allow', r.scope || 'global', r.project || '__global__');
+      const exc = Array.isArray(r.exceptions) && r.exceptions.length ? JSON.stringify(r.exceptions)
+        : (typeof r.exceptions === 'string' && r.exceptions ? r.exceptions : null);
+      ins.run(r.rule, r.listType || 'allow', r.scope || 'global', r.project || '__global__', exc);
     }
   });
   txn();
@@ -4244,6 +4367,12 @@ function _lastMessageAtFromMessages(messages) {
 }
 
 const _overCapImportLogged = new Set(); // session_id → already warned about an over-parse-cap import (log once)
+// session_id → message count at the last full token estimate. The estimate is an O(N) walk of the
+// WHOLE message array; recomputing it on every re-import of an actively-appending session (just to
+// nudge an APPROXIMATE (~) count by one message) is wasted work under the write lock. Only re-estimate
+// when the count grew past a delta (or the model changed); otherwise the upsert's COALESCE preserves
+// the prior stored value — exact recompute still happens on the live path for Claude/Codex.
+const _importTokenEstimateLen = new Map();
 function importSessionConversation({
   session_id, project_path, messages, user_msg_count, assistant_msg_count,
   search_messages,
@@ -4253,6 +4382,10 @@ function importSessionConversation({
   // v3 title-fallback fields; optional for backward compatibility with any
   // caller that doesn't yet populate them.
   last_user_content, first_assistant_text, rename_name,
+  // When true, DON'T write session_message_rows here — the caller owns the row write because the
+  // render shape differs from `messages` (Wall-E persists review-shaped rows, not the raw transcript
+  // blob shape). The metadata upsert + blob still happen so the freshness gate + legacy reads work.
+  skipMessageRows,
 }) {
   // Attribution: the whole import is one synchronous span — a JSON.stringify of
   // the full message array (multi-MB for 2000+ prompt sessions) + an upsert + a
@@ -4293,13 +4426,23 @@ function importSessionConversation({
   // null when there are no messages, and the COALESCE guards below then preserve
   // any existing value rather than wiping it.
   let _tokTotal = null, _tokCtx = null, _tokWindow = null, _tokExact = 0, _tokBreakdown = null;
-  if (!_overParseCap) {
+  // Only pay the O(N) full-array estimate when it can actually move the (~) number: first time for
+  // this session, a model change, or the count grew past a delta. Otherwise leave the fields null so
+  // the upsert COALESCEs the prior stored value (no loss, no recompute). Delta default 32 messages;
+  // CTM_IMPORT_TOKEN_REESTIMATE_DELTA tunes it, 0 = always re-estimate (old behavior).
+  const _msgLen = Array.isArray(messages) ? messages.length : 0;
+  const _reestimateDelta = Math.max(0, Number(process.env.CTM_IMPORT_TOKEN_REESTIMATE_DELTA ?? 32));
+  const _prevEst = _importTokenEstimateLen.get(session_id);
+  const _modelChanged = !_prevEst || _prevEst.model !== (model_id || '');
+  const _grewEnough = !_prevEst || _reestimateDelta === 0 || (_msgLen - _prevEst.len) >= _reestimateDelta;
+  if (!_overParseCap && (_modelChanged || _grewEnough)) {
     try {
       const summary = require('./lib/session-token-usage').estimateFromMessages(messages || [], model_id || '');
       if (summary && summary.total > 0) {
         _tokTotal = summary.total; _tokCtx = summary.ctx; _tokWindow = summary.ctxWindow;
         _tokExact = summary.exact ? 1 : 0;
         _tokBreakdown = summary.breakdown ? JSON.stringify(summary.breakdown) : null;
+        _importTokenEstimateLen.set(session_id, { len: _msgLen, model: model_id || '' });
       }
     } catch { /* token estimate is best-effort */ }
   }
@@ -4370,19 +4513,22 @@ function importSessionConversation({
       // Faithful per-message render store (the blob replacement). Uses `messages` (the exact
       // blob source array, full text) so the rows are byte-faithful to the blob. O(Δ) append.
       // Stamps extracted_source_len (the render gate's HWM) so it's self-sufficient.
-      try {
-        replaceSessionMessageRows(session_id, messages || []);
-      } catch (e) {
-        console.error('[db] replaceSessionMessageRows error:', e.message);
-        // If we retired the blob ('[]') but the faithful rows failed to write, this session
-        // would read empty. Restore the real blob (the rare failure path pays the stringify)
-        // so a read is never blank — correctness wins over the perf optimization here.
-        if (_retireBlob) {
-          try {
-            getDb().prepare('UPDATE session_conversations SET messages = ? WHERE ctm_session_id = ?')
-              .run(JSON.stringify(messages || []), session_id);
-          } catch (e2) {
-            console.error('[db] blob-restore after row-write failure also failed:', e2.message);
+      // skipMessageRows: the caller writes rows itself in a different (render) shape — see Wall-E.
+      if (!skipMessageRows) {
+        try {
+          replaceSessionMessageRows(session_id, messages || []);
+        } catch (e) {
+          console.error('[db] replaceSessionMessageRows error:', e.message);
+          // If we retired the blob ('[]') but the faithful rows failed to write, this session
+          // would read empty. Restore the real blob (the rare failure path pays the stringify)
+          // so a read is never blank — correctness wins over the perf optimization here.
+          if (_retireBlob) {
+            try {
+              getDb().prepare('UPDATE session_conversations SET messages = ? WHERE ctm_session_id = ?')
+                .run(JSON.stringify(messages || []), session_id);
+            } catch (e2) {
+              console.error('[db] blob-restore after row-write failure also failed:', e2.message);
+            }
           }
         }
       }
@@ -5085,11 +5231,11 @@ function incrementApprovalRuleMatch(id) {
 }
 
 // --- Approval Decisions (audit log) ---
-function addApprovalDecision({ sessionId, toolName, commandSummary, fullContext, warning, decision, reasoning, decidedBy, ruleId, riskLevel, commandSignature }) {
+function addApprovalDecision({ sessionId, toolName, commandSummary, fullContext, warning, decision, reasoning, decidedBy, ruleId, riskLevel, commandSignature, walleRequestId, walleSessionId }) {
   const result = getDb().prepare(
-    `INSERT INTO approval_decisions (session_id, tool_name, command_summary, full_context, warning_text, decision, reasoning, decided_by, rule_id, risk_level, command_signature)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-  ).run(sessionId || '', toolName || '', commandSummary || '', fullContext || '', warning || '', decision, reasoning || '', decidedBy || 'ai', ruleId || null, riskLevel || 'low', commandSignature || '');
+    `INSERT INTO approval_decisions (session_id, tool_name, command_summary, full_context, warning_text, decision, reasoning, decided_by, rule_id, risk_level, command_signature, walle_request_id, walle_session_id)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+  ).run(sessionId || '', toolName || '', commandSummary || '', fullContext || '', warning || '', decision, reasoning || '', decidedBy || 'ai', ruleId || null, riskLevel || 'low', commandSignature || '', walleRequestId || '', walleSessionId || '');
   return result.lastInsertRowid;
 }
 
@@ -7237,9 +7383,45 @@ function replaceSessionMessageRows(sessionId, messages) {
     }
   }
 
-  // Full rewrite (first import / compaction / prefix change). Chunked so the write lock is
-  // never held for the whole conversation; AFTER DELETE/INSERT triggers re-sync the FTS.
   const CHUNK = Math.max(50, Math.min(5000, Number(process.env.CTM_MESSAGE_REWRITE_CHUNK) || 500));
+
+  // Incremental tail diff (the Codex freeze fix). The pure-extension fast path above misses whenever
+  // the LAST stored message's text changed between imports — and Codex mutates its trailing assistant
+  // message as tokens stream, so EVERY re-import of a long Codex session fell through to the full
+  // rewrite below: DELETE + re-INSERT all N rows, re-tokenizing the whole conversation through the
+  // FTS triggers while holding the cross-process write lock (~1s/16k rows measured). That lock-hold is
+  // the off-CPU main-loop freeze that lagged Codex typing. Claude appends immutably so it always hit
+  // the fast path — hence "especially in codex". Here, for a DENSE store, find the first index where
+  // the stored rows and the new array diverge (forward scan; a lock-free read) and rewrite ONLY from
+  // there. A last-message edit then rewrites ~1 row instead of N. If divergence is at the very start
+  // (true compaction / prefix change) this naturally degrades to a full rewrite (same cost as before).
+  // Gate: CTM_MESSAGE_ROWS_INCREMENTAL_DIFF=0 forces the old full-rewrite behavior.
+  const _denseStore = !!(minRow && maxRow) && minRow.idx === 0 && count === maxRow.idx + 1 && count > 0;
+  if (_denseStore && process.env.CTM_MESSAGE_ROWS_INCREMENTAL_DIFF !== '0') {
+    const existing = d.prepare(
+      'SELECT message_index AS idx, role, text FROM session_message_rows WHERE ctm_session_id = ? ORDER BY message_index ASC'
+    ).all(sessionId);
+    const sameAt = (i) => {
+      const s = existing[i]; const m = normalized[i] || {};
+      return String(m.role || '') === s.role && String(m.text != null ? m.text : (m.content != null ? m.content : '')) === s.text;
+    };
+    let div = 0;
+    const lim = Math.min(existing.length, normalized.length);
+    while (div < lim && sameAt(div)) div++;
+    // Delete the stale tail [div, count) and (re)insert the new tail [div, len). Both chunked so the
+    // write lock is never held for the whole conversation; the FTS triggers re-sync per affected row.
+    const tailIds = d.prepare('SELECT id FROM session_message_rows WHERE ctm_session_id = ? AND message_index >= ?').all(sessionId, div);
+    const delById = d.prepare('DELETE FROM session_message_rows WHERE id = ?');
+    const delChunk = d.transaction((slice) => { for (const r of slice) delById.run(r.id); });
+    for (let i = 0; i < tailIds.length; i += CHUNK) delChunk(tailIds.slice(i, i + CHUNK));
+    const insChunk = d.transaction((start, end) => { for (let i = start; i < end; i++) writeRow(i); });
+    for (let i = div; i < normalized.length; i += CHUNK) insChunk(i, Math.min(i + CHUNK, normalized.length));
+    _setExtractedSourceLen(d, sessionId, normalized.length, _lastMessageAtFromMessages(normalized));
+    return normalized.length;
+  }
+
+  // Full rewrite (first import / compaction / prefix change / non-dense store). Chunked so the write
+  // lock is never held for the whole conversation; AFTER DELETE/INSERT triggers re-sync the FTS.
   const oldIds = d.prepare('SELECT id FROM session_message_rows WHERE ctm_session_id = ?').all(sessionId);
   const delById = d.prepare('DELETE FROM session_message_rows WHERE id = ?');
   const delChunk = d.transaction((slice) => { for (const r of slice) delById.run(r.id); });
@@ -7250,6 +7432,128 @@ function replaceSessionMessageRows(sessionId, messages) {
   return normalized.length;
 }
 
+// O(Δ) append of NEW messages after a verified-dense, unchanged-prefix base. Unlike
+// replaceSessionMessageRows this NEVER loads or rewrites the base array, so it stays O(Δ) for an
+// active multi-thousand-message session (that full-array load + re-process was the giant-transcript
+// freeze). Returns the new total length on success, or null when the base is not safe to append
+// onto — no rows, not a dense 0..n-1 block, or the row count disagrees with `expectedBaseCount`
+// (the conversation's known message HWM). On null the caller MUST fall back to the full path.
+function appendSessionMessageRows(sessionId, newMessages, expectedBaseCount) {
+  if (!sessionId || !_tableExists('session_message_rows')) return null;
+  const add = Array.isArray(newMessages) ? newMessages : [];
+  if (add.length === 0) return null;
+  const d = getDb();
+  const minRow = d.prepare('SELECT MIN(message_index) AS idx FROM session_message_rows WHERE ctm_session_id = ?').get(sessionId);
+  const maxRow = d.prepare('SELECT MAX(message_index) AS idx FROM session_message_rows WHERE ctm_session_id = ?').get(sessionId);
+  const count = Number(d.prepare('SELECT COUNT(*) AS n FROM session_message_rows WHERE ctm_session_id = ?').get(sessionId).n);
+  // Dense 0..count-1 block — no gaps, starts at 0 — else appending would misorder.
+  if (count === 0 || minRow == null || minRow.idx !== 0 || count !== Number(maxRow.idx) + 1) return null;
+  // The row count must equal the base length the delta was parsed against. When the caller can't
+  // prove it (HWM unset / -1), refuse and let the full path re-establish the invariant.
+  if (!Number.isFinite(expectedBaseCount) || expectedBaseCount < 0 || count !== expectedBaseCount) return null;
+  const insertRow = d.prepare(
+    'INSERT OR IGNORE INTO session_message_rows (ctm_session_id, message_index, role, text, timestamp, meta) VALUES (?, ?, ?, ?, ?, ?)'
+  );
+  const writeRow = (i, m) => insertRow.run(
+    sessionId, i, String(m.role || ''),
+    String(m.text != null ? m.text : (m.content != null ? m.content : '')),
+    m.timestamp != null ? String(m.timestamp) : '', _messageRowMeta(m)
+  );
+  const newLen = count + add.length;
+  d.transaction(() => { for (let k = 0; k < add.length; k++) writeRow(count + k, add[k] || {}); })();
+  _setExtractedSourceLen(d, sessionId, newLen, _lastMessageAtFromMessages(add));
+  return newLen;
+}
+
+// Incremental counterpart of importSessionConversation for an APPEND (file grew, prefix
+// unchanged). Appends only the Δ messages to the faithful row store and bumps the conversation
+// metadata (counts, last_message_at/last_user_content, file_size, token ESTIMATE) without loading
+// or re-serialising the full conversation. Returns { ok:true, length } on success, or
+// { ok:false, reason } when the base isn't safe to append onto — the caller then does a full
+// importSessionConversation. Only the (~) estimate token tier is touched; the live exact path wins.
+function appendSessionConversation({
+  session_id, new_messages, model_id, file_size,
+  last_user_content, model_provider, import_parser_version,
+  // Cheap metadata that can be corrected on re-import (provider cwd, branch, title, host, rename).
+  // first_message / first_assistant_text / session_created_at are cold-import-only and never change
+  // on an append, so they are deliberately NOT touched here (that would need the full array).
+  project_path, git_branch, title, hostname, rename_name,
+} = {}) {
+  if (!session_id) return { ok: false, reason: 'no-session' };
+  const add = Array.isArray(new_messages) ? new_messages : [];
+  if (add.length === 0) return { ok: false, reason: 'empty-delta' };
+  // The append path only maintains the row store. If the legacy session_messages index is being
+  // dual-written (or rows are off), defer to the full path so that index stays in lockstep.
+  if (process.env.CTM_SESSION_ROWS === '0' || process.env.CTM_DUAL_WRITE_SESSION_MESSAGES === '1') {
+    return { ok: false, reason: 'legacy-index-dual-write' };
+  }
+  const d = getDb();
+  const existing = d.prepare(
+    'SELECT extracted_source_len, tokens_total, tokens_exact FROM session_conversations WHERE ctm_session_id = ?'
+  ).get(session_id);
+  if (!existing) return { ok: false, reason: 'no-existing' };
+  const baseCount = Number(existing.extracted_source_len);
+  // Append the rows first — the strong dense+count guard lives there.
+  const newLen = appendSessionMessageRows(session_id, add, baseCount);
+  if (newLen == null) return { ok: false, reason: 'rows-not-appendable' };
+
+  // Δ-only counts (O(Δ), never re-walks the base).
+  let dUsers = 0, dAssist = 0;
+  for (const m of add) {
+    const role = String(m && m.role || '');
+    if (role === 'user') dUsers++;
+    else if (role === 'assistant') dAssist++;
+  }
+  const lastAt = _lastMessageAtFromMessages(add) || '';
+
+  // Incremental token estimate: add ONLY the delta's tokens to the running total. Skip entirely
+  // when an exact (live) total already owns the row — never downgrade exact → estimate.
+  let dTokens = 0;
+  if (!Number(existing.tokens_exact)) {
+    try {
+      const summ = require('./lib/session-token-usage').estimateFromMessages(add, model_id || '');
+      dTokens = (summ && summ.total) || 0;
+    } catch { /* estimate is best-effort */ }
+  }
+  const newTokens = dTokens > 0 ? (Number(existing.tokens_total) || 0) + dTokens : null;
+
+  d.prepare(
+    `UPDATE session_conversations SET
+       user_msg_count = user_msg_count + ?,
+       assistant_msg_count = assistant_msg_count + ?,
+       file_size = ?,
+       last_message_at = CASE
+         WHEN ? = '' THEN last_message_at
+         WHEN last_message_at IS NULL OR last_message_at = '' THEN ?
+         WHEN ? > last_message_at THEN ?
+         ELSE last_message_at END,
+       last_user_content = COALESCE(NULLIF(?, ''), last_user_content),
+       project_path = COALESCE(NULLIF(?, ''), project_path),
+       git_branch = COALESCE(NULLIF(?, ''), git_branch),
+       title = COALESCE(NULLIF(?, ''), title),
+       hostname = COALESCE(NULLIF(?, ''), hostname),
+       rename_name = COALESCE(NULLIF(?, ''), rename_name),
+       model_provider = COALESCE(NULLIF(?, ''), model_provider),
+       model_id = COALESCE(NULLIF(?, ''), model_id),
+       import_parser_version = ?,
+       tokens_total = COALESCE(?, tokens_total),
+       tokens_ctx = COALESCE(?, tokens_ctx),
+       tokens_updated_at = CASE WHEN ? IS NULL THEN tokens_updated_at ELSE datetime('now') END,
+       imported_at = datetime('now')
+     WHERE ctm_session_id = ?`
+  ).run(
+    dUsers, dAssist, Number(file_size) || 0,
+    lastAt, lastAt, lastAt, lastAt,
+    last_user_content || '',
+    project_path || '', git_branch || '', title || '', hostname || '', rename_name || '',
+    model_provider || '', model_id || '',
+    Number(import_parser_version || 0),
+    newTokens, newTokens, newTokens,
+    session_id
+  );
+  return { ok: true, length: newLen };
+}
+
 // Whole conversation as the faithful message array (blob-identical), from rows; blob fallback.
 function getSessionMessagesArray(sessionId, { fallbackToBlob = true } = {}) {
   const d = getDb();
@@ -7283,17 +7587,19 @@ function sessionContentRowsAvailable(sessionId) {
 }
 
 // Newest-first offset page (matches lib/message-pagination.js semantics), O(limit) rows.
+// Delegates to the shared lib/session-messages-page module so the SAME logic runs on the main
+// thread (here) and on the read-pool worker (off-thread, read-only handle) — parity by construction.
 function getSessionMessagesPage(sessionId, { offset = 0, limit = 200 } = {}) {
-  const d = getDb();
-  const total = Number(d.prepare('SELECT COUNT(*) AS n FROM session_message_rows WHERE ctm_session_id = ?').get(sessionId).n);
-  const lim = Math.max(1, Math.min(1000, Number(limit) || 200));
-  const off = Math.max(0, Number(offset) || 0);
-  const start = Math.max(0, total - off - lim);
-  const take = Math.max(0, Math.min(lim, total - off - start));
-  const messages = take === 0 ? [] : d.prepare(
-    'SELECT role, text, timestamp, meta FROM session_message_rows WHERE ctm_session_id = ? ORDER BY message_index ASC LIMIT ? OFFSET ?'
-  ).all(sessionId, take, start).map(_messageRowToObject);
-  return { messages, total, has_more: start > 0, next_offset: off + take };
+  return require('./lib/session-messages-page').getMessagesPage(getDb(), { sessionId, offset, limit });
+}
+
+// Cheap row count (no fetch) — a content-version signal so callers can skip an
+// expensive page fetch + re-render when the message set is unchanged.
+function countSessionMessageRows(sessionId) {
+  try {
+    if (!_tableExists('session_message_rows')) return 0;
+    return Number(getDb().prepare('SELECT COUNT(*) AS n FROM session_message_rows WHERE ctm_session_id = ?').get(sessionId).n) || 0;
+  } catch { return 0; }
 }
 
 function _defaultSessionTurnStart(message) {
@@ -7392,9 +7698,14 @@ function getSessionMessagesTurnPage(sessionId, { offset = 0, limit = 50, isTurnS
 // backfill sweep using THIS module's writer + handle, so the worker/scheduler never thread
 // the handle through. Idempotent + resumable; returns the sweep summary {swept,migrated,rows,remaining}.
 function runContentRowsBackfillSweep(options = {}) {
-  const limit = Math.max(1, Math.min(500, Number(options && options.limit) || 50));
+  const o = options && typeof options === 'object' ? options : {};
+  // Smaller per-sweep batch by default (was 50) — this background migration shares the write lock
+  // with interactive work; keep each sweep cheap and let it converge across ticks.
+  const limit = Math.max(1, Math.min(500, Number(o.limit) || 25));
+  const budgetMs = Math.max(0, Number(o.budgetMs ?? process.env.CTM_ROWS_BACKFILL_BUDGET_MS ?? 250));
+  const maxRows = Math.max(0, Number(o.maxRows ?? process.env.CTM_ROWS_BACKFILL_MAX_ROWS ?? 3000));
   const bf = require('./lib/session-content-backfill');
-  return bf.runContentRowsBackfillSweep(getDb(), replaceSessionMessageRows, { limit });
+  return bf.runContentRowsBackfillSweep(getDb(), replaceSessionMessageRows, { limit, budgetMs, maxRows });
 }
 
 // Migration completeness for GET /api/ctm/session-rows-status (+ the cutover gate).
@@ -8766,6 +9077,45 @@ function getSessionTitleNew(id) {
   return row && row.title ? { title: row.title, userRenamed: !!row.user_renamed } : null;
 }
 
+// Batch form of getSessionTitleNew: resolve titles for many ids in TWO queries instead of N point
+// reads, so a per-session gather loop (e.g. model-sync) doesn't block the main loop. Returns a
+// Map<id,{title,userRenamed}> keyed by the ORIGINAL input id; ids with no title are omitted (a miss
+// is `map.get(id) === undefined`, parity with getSessionTitleNew returning null). Precedence matches
+// getSessionTitleNew EXACTLY: a direct ctm_sessions row wins — and a titleless direct row resolves to
+// a miss WITHOUT falling through to the agent-session-id alias (getSessionTitleNew only tries the
+// alias when no ctm row exists for the id).
+function getSessionTitlesByIds(ids) {
+  const out = new Map();
+  const list = Array.from(new Set((Array.isArray(ids) ? ids : []).map((x) => String(x || '')).filter(Boolean)));
+  if (list.length === 0) return out;
+  const db = getDb();
+  const ph = list.map(() => '?').join(',');
+  const directRows = db.prepare(`SELECT id, title, user_renamed FROM ctm_sessions WHERE id IN (${ph})`).all(...list);
+  const directById = new Map(directRows.map((r) => [String(r.id), r]));
+  // Only ids with NO ctm row at all fall through to the agent_session_id alias.
+  const unresolved = list.filter((id) => !directById.has(id));
+  let aliasByAgentId = new Map();
+  if (unresolved.length) {
+    const aph = unresolved.map(() => '?').join(',');
+    const aliasRows = db.prepare(
+      `SELECT a.agent_session_id AS aid, c.title AS title, c.user_renamed AS user_renamed
+         FROM agent_sessions a JOIN ctm_sessions c ON c.id = a.ctm_session_id
+        WHERE a.agent_session_id IN (${aph})`
+    ).all(...unresolved);
+    aliasByAgentId = new Map(aliasRows.map((r) => [String(r.aid), r]));
+  }
+  for (const id of list) {
+    if (directById.has(id)) {
+      const direct = directById.get(id);
+      if (direct && direct.title) out.set(id, { title: direct.title, userRenamed: !!direct.user_renamed });
+      continue; // titleless direct row → miss, no alias (matches getSessionTitleNew)
+    }
+    const alias = aliasByAgentId.get(id);
+    if (alias && alias.title) out.set(id, { title: alias.title, userRenamed: !!alias.user_renamed });
+  }
+  return out;
+}
+
 function getSessionDisplayTitleInfo(id) {
   const cleanId = String(id || '').trim();
   if (!cleanId) return null;
@@ -9226,7 +9576,7 @@ module.exports = {
   getStorageRisk,
   getSqliteDriverStatus,
   getSetting, getSettingsByPrefix, setSetting,
-  addSessionDiagnostic, listSessionDiagnostics,
+  addSessionDiagnostic, flushSessionDiagnostics, listSessionDiagnostics,
   upsertRuntimeTask, updateRuntimeTask, getRuntimeTask, listRuntimeTasks, deleteRuntimeTasksForSession,
   createPrompt, updatePrompt, getPrompt, listPrompts, listChildPrompts, setPromptParent, createGroupFromPrompts, deletePrompt, duplicatePrompt, reorderPrompts,
   getPromptVersions, restorePromptVersion,
@@ -9236,8 +9586,8 @@ module.exports = {
   listPermissionRules, upsertPermissionRule, deletePermissionRule,
   listPermissionLog, addPermissionLog,
   getAlwaysAskTools, setAlwaysAsk,
-  importSessionConversation, listSessionConversations, getSessionConversation, getSessionConversationMeta, getSessionConversationMessages, updateSessionModel,
-  replaceSessionMessageRows, getSessionMessagesArray, sessionContentRowsAvailable, getSessionMessagesPage, getSessionMessagesTurnPage,
+  importSessionConversation, appendSessionConversation, appendSessionMessageRows, listSessionConversations, getSessionConversation, getSessionConversationMeta, getSessionConversationMessages, updateSessionModel,
+  replaceSessionMessageRows, getSessionMessagesArray, sessionContentRowsAvailable, getSessionMessagesPage, getSessionMessagesTurnPage, countSessionMessageRows,
   runContentRowsBackfillSweep, getSessionRowsStatus, retireLegacyStores, runVacuum,
   updateSessionTokens, getSessionTokens,
   pickDisplayTitle,
@@ -9247,6 +9597,7 @@ module.exports = {
   checkpointWal, checkpointWalOrThrow, setWalCheckpointRunner, createBackup, copyLiveDatabaseTo, createBackupSync, listBackups, restoreBackup, deleteBackup, startDailyBackup,
   saveQueue, loadQueue, loadAllQueues, deleteQueueDb,
   listPermRules, addPermRule, removePermRule, bulkSetPermRules, getPermRulesByProject,
+  setPermRuleExceptions,
   listAutoApprovals, upsertAutoApproval, toggleAutoApproval, deleteAutoApproval, getEnabledAutoApprovals,
   listApprovalRules, upsertApprovalRule, findApprovalRuleBySignature, toggleApprovalRule, deleteApprovalRule, incrementApprovalRuleMatch,
   addApprovalDecision, listApprovalDecisions, resolveApprovalDecision, getPendingEscalations,
@@ -9285,7 +9636,7 @@ module.exports = {
   upsertSessionIndex, resolveSessionId,
   // CTM Sessions + Agent Sessions (v1 schema)
   getSessionIdentity, getLatestAgentSessionForCtm, getSessionConversationSourceIds, getSessionMessageIndexFreshness, getSessionUserPromptFreshness, resolveSession, batchRestoreIdentities, upsertSession, upsertAgentSessionIdentity, setSessionStar, getStarredSessionIds,
-  setSessionTitleNew, setSessionTitleStatusNew, getSessionTitleNew, getSessionDisplayTitleInfo, isUserSessionTitleCandidate, promoteStartupTaskTitleIfUserNamed, repairSessionTitlesFromStartupTasks, getAllSessionsData,
+  setSessionTitleNew, setSessionTitleStatusNew, getSessionTitleNew, getSessionTitlesByIds, getSessionDisplayTitleInfo, isUserSessionTitleCandidate, promoteStartupTaskTitleIfUserNamed, repairSessionTitlesFromStartupTasks, getAllSessionsData,
   repairCodexSessionMetadataFromConversations, repairCodexRolloutAgentIdentities, detachPromotedProviderChildSessions,
   listProviderChildAgentOwnerMappings,
   getAgentSessions, getAgentSession, deleteCtmSession,