create-walle 0.9.30 → 0.9.32

package/template/claude-task-manager/db.js

@@ -1814,6 +1814,13 @@ function runMigrations() {
   if (!scCols2.find(c => c.name === 'extracted_source_len')) {
     getDb().prepare("ALTER TABLE session_conversations ADD COLUMN extracted_source_len INTEGER DEFAULT -1").run();
   }
+  // Byte size of the .jsonl.bak at last full compact-pair stitch. The .bak is byte-stable between
+  // compactions, so a later .jsonl-only growth can append just the new tail (priorJsonlSize =
+  // file_size - compact_bak_size) instead of re-reading+re-stitching the whole (100MB+) pair — the
+  // giant compacted-session import freeze. 0 = not a compact-pair / unknown → take the full path.
+  if (!scCols2.find(c => c.name === 'compact_bak_size')) {
+    getDb().prepare("ALTER TABLE session_conversations ADD COLUMN compact_bak_size INTEGER DEFAULT 0").run();
+  }
   if (!scCols2.find(c => c.name === 'last_message_at')) {
     getDb().prepare("ALTER TABLE session_conversations ADD COLUMN last_message_at TEXT").run();
     try {
@@ -4164,7 +4171,7 @@ function getPrompt(id) {
 }
 
 function listPrompts({ folder_id, context_type, search, starred, limit, offset, include_children, lifecycle_status } = {}) {
-  let sql = `SELECT p.*, (SELECT COUNT(*) FROM prompts c WHERE c.parent_id = p.id) as child_count FROM prompts p WHERE 1=1`;
+  let sql = `SELECT p.*, (SELECT COUNT(*) FROM prompts c WHERE c.parent_id = p.id) as child_count, (SELECT COUNT(*) FROM prompt_usage u WHERE u.prompt_id = p.id) as use_count, (SELECT MAX(v.version) FROM prompt_versions v WHERE v.prompt_id = p.id) as version FROM prompts p WHERE 1=1`;
   const params = [];
 
   // By default, only show top-level prompts (parent_id IS NULL)
@@ -5030,6 +5037,19 @@ const _importTokenEstimateLen = new Map();
 function _blobRetirementActive() {
   return process.env.CTM_DUAL_WRITE_BLOB !== '1' && process.env.CTM_SESSION_ROWS !== '0';
 }
+// Belt-and-suspenders: the over-cap guard above should already skip the whole-array stringify for a
+// multi-GB conversation, but if any caller ever under-reports the size again, a >512MB JSON.stringify
+// would throw a RangeError and abort the import (and the loop). Degrade to an empty blob instead — the
+// session then renders from rows / the streaming transcript-store, exactly like the over-cap path.
+function _safeBlobStringify(messages) {
+  try {
+    return JSON.stringify(messages || []);
+  } catch (e) {
+    console.error(`[db] blob stringify failed (${e.message}) — storing empty blob, reads fall back to rows.`);
+    return '[]';
+  }
+}
+
 function importSessionConversation({
   session_id, project_path, messages, user_msg_count, assistant_msg_count,
   search_messages,
@@ -5047,6 +5067,16 @@ function importSessionConversation({
   // while content-rows-backfill fills rows across slices (chunked/cold-import route). Over-cap still
   // wins — a multi-GB stringify is never safe regardless of this flag.
   forceBlobWrite,
+  // Byte size of the .jsonl.bak at this compact-pair stitch (0 / omitted for non-compact imports).
+  // Persisted so a later .jsonl-only growth can append just the new tail instead of re-stitching the
+  // whole pair. Always written so a session that stops being a compact-pair resets to 0.
+  compact_bak_size,
+  // Truthful on-disk transcript size, used ONLY for the over-cap decision below. Some callers pass a
+  // `file_size` that under-reports the real conversation size (codex `importedFileSize` is a
+  // consumed-bytes high-water-mark), which would let a multi-GB rollout slip under the parse cap and
+  // then throw at JSON.stringify (>512MB). Decision-only: the stored `file_size` column is unchanged.
+  // Defaults to `file_size` when omitted (back-compat for every existing caller).
+  parse_size_bytes,
 }) {
   // Attribution: the whole import is one synchronous span — a JSON.stringify of
   // the full message array (multi-MB for 2000+ prompt sessions) + an upsert + a
@@ -5064,7 +5094,12 @@ function importSessionConversation({
   // heavy work; the streaming transcript-store rows remain the source of truth.
   let _isOverParseCap = null;
   try { _isOverParseCap = require('./lib/size-cap').isOverParseCap; } catch { /* optional */ }
-  const _overParseCap = typeof _isOverParseCap === 'function' ? _isOverParseCap(file_size) : false;
+  // Prefer the truthful on-disk size when the caller provides it; fall back to file_size. isOverParseCap
+  // already fails open on a non-finite/zero value, so the fallback is robust.
+  const _parseSizeBytes = Number.isFinite(Number(parse_size_bytes)) && Number(parse_size_bytes) > 0
+    ? Number(parse_size_bytes)
+    : file_size;
+  const _overParseCap = typeof _isOverParseCap === 'function' ? _isOverParseCap(_parseSizeBytes) : false;
   // Phase 7 (blob retirement): reads are served from session_message_rows (the
   // faithful per-message store), so the monolithic `messages` blob is dead weight
   // and its full-array JSON.stringify on every append is the O(N²) write cost we
@@ -5080,7 +5115,7 @@ function importSessionConversation({
   const _skipBlobWrite = _overParseCap || (_retireBlob && !forceBlobWrite);
   if (_overParseCap && !_overCapImportLogged.has(session_id)) {
     _overCapImportLogged.add(session_id);
-    console.warn(`[db] session_conversations import over parse cap (file_size=${file_size}) for ${String(session_id || '').slice(0, 8)} — storing empty blob, skipping full stringify/row-rewrite.`);
+    console.warn(`[db] session_conversations import over parse cap (parse_size=${_parseSizeBytes}, file_size=${file_size}) for ${String(session_id || '').slice(0, 8)} — storing empty blob, skipping full stringify/row-rewrite.`);
   }
   // Estimate-tier token fallback, FOLDED INTO the upsert below (no extra write /
   // WAL flush). Stored so providers without an authoritative transcript
@@ -5115,8 +5150,8 @@ function importSessionConversation({
 
   const _doImport = () => {
     getDb().prepare(
-      `INSERT INTO session_conversations (ctm_session_id, project_path, messages, user_msg_count, assistant_msg_count, title, first_message, git_branch, file_size, session_created_at, last_message_at, hostname, model_provider, model_id, import_parser_version, last_user_content, first_assistant_text, rename_name, tokens_total, tokens_ctx, tokens_window, tokens_exact, tokens_breakdown, tokens_updated_at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
+      `INSERT INTO session_conversations (ctm_session_id, project_path, messages, user_msg_count, assistant_msg_count, title, first_message, git_branch, file_size, session_created_at, last_message_at, hostname, model_provider, model_id, import_parser_version, last_user_content, first_assistant_text, rename_name, tokens_total, tokens_ctx, tokens_window, tokens_exact, tokens_breakdown, compact_bak_size, tokens_updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
        ON CONFLICT(ctm_session_id) DO UPDATE SET
          project_path=excluded.project_path,
          messages=excluded.messages, user_msg_count=excluded.user_msg_count,
@@ -5143,16 +5178,18 @@ function importSessionConversation({
          tokens_exact=CASE WHEN excluded.tokens_total IS NOT NULL THEN excluded.tokens_exact ELSE session_conversations.tokens_exact END,
          tokens_breakdown=COALESCE(excluded.tokens_breakdown, session_conversations.tokens_breakdown),
          tokens_updated_at=CASE WHEN excluded.tokens_total IS NOT NULL THEN excluded.tokens_updated_at ELSE session_conversations.tokens_updated_at END,
+         compact_bak_size=excluded.compact_bak_size,
          imported_at=datetime('now')`
     ).run(
-      session_id, project_path || '', _skipBlobWrite ? '[]' : JSON.stringify(messages || []),
+      session_id, project_path || '', _skipBlobWrite ? '[]' : _safeBlobStringify(messages),
       user_msg_count || 0, assistant_msg_count || 0, title || '',
       first_message || '', git_branch || '', file_size || 0, session_created_at || '',
       _lastMessageAt,
       hostname || require('os').hostname(), model_provider || '', model_id || '',
       Number(import_parser_version || 0),
       last_user_content || '', first_assistant_text || '', rename_name || '',
-      _tokTotal, _tokCtx, _tokWindow, _tokExact, _tokBreakdown
+      _tokTotal, _tokCtx, _tokWindow, _tokExact, _tokBreakdown,
+      Number(compact_bak_size) || 0
     );
 
     // Chunked-import route: rows are written later by content-rows-backfill (in windows). Stamp the
@@ -8190,9 +8227,16 @@ function appendSessionMessageRows(sessionId, newMessages, expectedBaseCount) {
   const count = Number(d.prepare('SELECT COUNT(*) AS n FROM session_message_rows WHERE ctm_session_id = ?').get(sessionId).n);
   // Dense 0..count-1 block — no gaps, starts at 0 — else appending would misorder.
   if (count === 0 || minRow == null || minRow.idx !== 0 || count !== Number(maxRow.idx) + 1) return null;
-  // The row count must equal the base length the delta was parsed against. When the caller can't
-  // prove it (HWM unset / -1), refuse and let the full path re-establish the invariant.
-  if (!Number.isFinite(expectedBaseCount) || expectedBaseCount < 0 || count !== expectedBaseCount) return null;
+  // The dense row base must COVER the base length the delta was parsed against (count >= HWM).
+  // A base SHORTER than the HWM (count < expectedBaseCount) is still backfilling / incomplete →
+  // refuse so the full path re-establishes the invariant. A base AHEAD of the HWM (count > HWM) is
+  // the giant compact-pair drift: a chunked full re-stitch stamps extracted_source_len = parseCount
+  // but never rewrites the rows, so a piecewise-append row count sits a few ahead of the HWM. The
+  // delta's byte boundary (file_size - bakSize) is independent of this count drift, so the delta is
+  // still strictly-new bytes — appending onto the (dense, gap-free, 0-based) base at `count` is
+  // correct and re-aligns the HWM (newLen = count + add.length). Refusing here instead forced a
+  // perpetual full re-stitch of the 100MB+ pair every growth = the multi-second main-loop freeze.
+  if (!Number.isFinite(expectedBaseCount) || expectedBaseCount < 0 || count < expectedBaseCount) return null;
   const insertRow = d.prepare(
     'INSERT OR IGNORE INTO session_message_rows (ctm_session_id, message_index, role, text, timestamp, meta) VALUES (?, ?, ?, ?, ?, ?)'
   );
@@ -8354,6 +8398,22 @@ function sessionContentRowsAvailable(sessionId) {
   } catch { return false; }
 }
 
+// True when a chunked-route import deferred this session's per-message rows (extracted_source_len
+// stamped as the HWM) but the windowed content-rows-backfill hasn't filled them yet. This is the
+// exact window where the O(Δ) appendSessionConversation fast-path can't prove its row base and
+// refuses ('rows-not-appendable') — without this signal the caller falls through to a multi-second
+// full-rebuild blob re-parse (the giant-session restart freeze). A session with no deferred-rows HWM
+// (expected <= 0) is NOT pending: it's either a normal inline import or simply has no rows to wait on.
+function sessionRowsBackfillPending(sessionId) {
+  try {
+    const d = getDb();
+    const conv = d.prepare('SELECT extracted_source_len FROM session_conversations WHERE ctm_session_id = ?').get(sessionId);
+    const expected = conv ? Number(conv.extracted_source_len) : 0;
+    if (!(expected > 0)) return false;              // no deferred-rows HWM → nothing to wait on
+    return !sessionContentRowsAvailable(sessionId); // HWM set but rows < HWM → backfill still pending
+  } catch { return false; }
+}
+
 // Newest-first offset page (matches lib/message-pagination.js semantics), O(limit) rows.
 // Delegates to the shared lib/session-messages-page module so the SAME logic runs on the main
 // thread (here) and on the read-pool worker (off-thread, read-only handle) — parity by construction.
@@ -8473,8 +8533,12 @@ function runContentRowsBackfillSweep(options = {}) {
   const budgetMs = Math.max(0, Number(o.budgetMs ?? process.env.CTM_ROWS_BACKFILL_BUDGET_MS ?? 250));
   const maxRows = Math.max(0, Number(o.maxRows ?? process.env.CTM_ROWS_BACKFILL_MAX_ROWS ?? 3000));
   // Per-session row cap: migrate a giant session in bounded slices across ticks (resumable) so one
-  // session can't hold the write lock for its whole length. 0 = whole-session (legacy/tests).
-  const maxRowsPerSession = Math.max(0, Number(o.maxRowsPerSession ?? process.env.CTM_ROWS_BACKFILL_MAX_ROWS_PER_SESSION ?? 2000));
+  // session can't hold the write lock for its whole length. 0 = whole-session (legacy/tests). Default
+  // 500: appendChunk holds the cross-process write lock for the whole row batch, and at the measured
+  // ~6-7ms/row under Dropbox-WAL contention a 2000-row batch ran ~13.6s — over the coop 5s ceiling,
+  // starving interactive writers during a giant session's first backfill. 500 keeps each batch's
+  // lock-hold ~3-4s (under the ceiling, with margin); tune via CTM_ROWS_BACKFILL_MAX_ROWS_PER_SESSION.
+  const maxRowsPerSession = Math.max(0, Number(o.maxRowsPerSession ?? process.env.CTM_ROWS_BACKFILL_MAX_ROWS_PER_SESSION ?? 500));
   const bf = require('./lib/session-content-backfill');
   return bf.runContentRowsBackfillSweep(getDb(), replaceSessionMessageRows, {
     limit, budgetMs, maxRows, maxRowsPerSession,
@@ -10421,7 +10485,7 @@ module.exports = {
   listPermissionLog, addPermissionLog,
   getAlwaysAskTools, setAlwaysAsk,
   importSessionConversation, appendSessionConversation, appendSessionMessageRows, listSessionConversations, getSessionConversation, getSessionConversationMeta, getSessionConversationMessages, updateSessionModel,
-  replaceSessionMessageRows, appendSessionMessageRowsChunk, getSessionMessagesArray, sessionContentRowsAvailable, getSessionMessagesPage, getSessionMessagesTurnPage, countSessionMessageRows,
+  replaceSessionMessageRows, appendSessionMessageRowsChunk, getSessionMessagesArray, sessionContentRowsAvailable, sessionRowsBackfillPending, getSessionMessagesPage, getSessionMessagesTurnPage, countSessionMessageRows,
   runContentRowsBackfillSweep, getSessionRowsStatus, retireLegacyStores, runVacuum, _blobRetirementActive,
   updateSessionTokens, getSessionTokens,
   pickDisplayTitle,

package/template/claude-task-manager/docs/codex-app-server-approvals.md

@@ -64,9 +64,43 @@ The engine is also the right substrate for any **programmatic** Codex consumer (
 Wall-E coding backend), where there is no TUI and driven mode is the *only* mode — there
 the remaining UI item (1) does not even apply.
 
+## Spike re-validation — codex-cli 0.142.3 (2026-06-27)
+
+Triggered by the live `search_graph` strand: the scrape approver missed Codex's new MCP
+approval UI (fixed separately — verb-anchored detection + a drift alarm). That prompted a
+re-validation of the structured channel against the current binary. Findings:
+
+1. **`--remote` removes the old blocker.** 0.142.3 ships `codex --remote <ws://|unix://>`,
+   `codex remote-control`, and `codex app-server proxy` — **a TUI can attach to a running
+   app-server**. So structured approvals no longer cost the interactive terminal, which was
+   the entire "larger product decision" above. The follow-on plan's **step 1** is to stand
+   up an app-server (or `app-server daemon`) and confirm the `--remote` attach UX end to end
+   (CTM hosts; the user's terminal attaches).
+2. **Protocol method names held.** The v2 `item/commandExecution|fileChange|permissions/
+   requestApproval` + legacy `execCommandApproval`/`applyPatchApproval` the bridge maps are
+   still present in the 0.142.3 schema. The existing bridge mapping does not need to change
+   for those.
+3. **GAP — the MCP path the screenshot represents is unhandled.** The schema now has
+   `McpServerElicitationRequest` (params + response) — the `Allow the <server> MCP server to
+   run tool …?` form *is* an MCP elicitation (`mode: form|openai/form|url`, `message` +
+   `requestedSchema` = the rendered `file_pattern`/`limit` fields). The bridge has **no
+   elicitation handler**, and its response is a **form accept**, not a yes/no. Also new and
+   unhandled: `item/tool/requestUserInput` (`ToolRequestUserInputParams`). These are the
+   concrete adds for driven-mode MCP auto-approval.
+4. **Codex now self-reviews.** It emits `item/autoApprovalReview/started|completed`
+   notifications and exposes `approval_policy` / per-server `codex mcp` trust. The follow-on
+   must define how CTM's `decideApproval` **composes** with Codex's native auto-approval
+   (don't double-decide), or push the safe class into Codex config so it never asks.
+
+Net: the structured channel is now the clearly-preferred durable fix (no TUI sacrifice), and
+the remaining work is bounded — wire the driven/remote session type, add the elicitation +
+requestUserInput handlers, and settle the autoApprovalReview composition. This stays a spike
+branch until its own plan is approved.
+
 ## Reference
 
 Exact wire contract (regenerate from the installed binary):
 `codex app-server generate-json-schema --out <dir>` → `ServerRequest.json`,
-`v2/ThreadStartParams.json`, `CommandExecutionRequestApprovalParams.json`, etc.
+`v2/ThreadStartParams.json`, `CommandExecutionRequestApprovalParams.json`,
+`McpServerElicitationRequestParams.json`, etc.
 See memory `ctm-approver-protocol-hook-prototype-validated` for the full protocol notes.