create-walle 0.9.21 → 0.9.23

package/template/claude-task-manager/workers/conversation-import-worker.js

@@ -1,55 +1,16 @@
-// --- Conversation Import Worker Thread ---
-// Runs JSONL -> session_conversations import off the main event loop.
-// Opens its own better-sqlite3 connection; WAL mode supports concurrent access.
+'use strict';
 
-const { parentPort, workerData } = require('worker_threads');
-const path = require('path');
+const { parentPort } = require('node:worker_threads');
 
-const dbModule = require(path.join(workerData.ctmDir, 'db'));
-dbModule.connectDb(workerData.dbPath);
-
-const { runIncrementalConversationImport } = require(path.join(workerData.ctmDir, 'api-prompts'));
-
-let running = false;
-let closing = false;
-let closed = false;
-
-function closeDbAndAck() {
-  if (closed) return;
-  closed = true;
-  try { dbModule.closeDb({ checkpointMode: 'PASSIVE' }); } catch {}
-  parentPort.postMessage({ type: 'closed' });
-  setImmediate(() => process.exit(0));
-}
-
-parentPort.on('message', async (msg) => {
-  if (msg.type === 'close') {
-    closing = true;
-    if (!running) closeDbAndAck();
-    return;
-  }
-  if (msg.type !== 'run') return;
-  if (closing) {
-    parentPort.postMessage({ type: 'error', requestId: msg.requestId, dur: 0, error: 'worker closing' });
+parentPort.on('message', (msg) => {
+  if (msg?.type === 'close') {
+    parentPort.postMessage({ type: 'closed' });
     return;
   }
-  if (running) {
-    parentPort.postMessage({ type: 'done', requestId: msg.requestId, dur: 0, result: { skipped: 'already_running' } });
-    return;
-  }
-  running = true;
-  const t0 = Date.now();
-  try {
-    const result = await runIncrementalConversationImport();
-    parentPort.postMessage({ type: 'done', requestId: msg.requestId, dur: Date.now() - t0, result });
-  } catch (e) {
-    parentPort.postMessage({ type: 'error', requestId: msg.requestId, dur: Date.now() - t0, error: e.message });
-  } finally {
-    running = false;
-    if (closing) closeDbAndAck();
-  }
-});
-
-process.on('exit', () => {
-  try { dbModule.closeDb({ checkpointMode: 'NONE' }); } catch {}
+  parentPort.postMessage({
+    type: 'error',
+    requestId: msg?.requestId,
+    dur: 0,
+    error: 'conversation import DB work is owned by the CTM DB owner worker',
+  });
 });

package/template/claude-task-manager/workers/db-owner-worker.js

@@ -0,0 +1,386 @@
+'use strict';
+
+const { parentPort, workerData } = require('node:worker_threads');
+const { retryOnWriteLockBusy } = require('../../shared/sqlite-write-lock');
+
+const db = require('../db');
+
+// Background writer: transient write-lock contention with the main thread should
+// retry rather than surface (e.g. a pasted image failing). Safe because a busy
+// error means the write never started. This worker thread may async-wait freely.
+const WRITE_LOCK_RETRIES = Math.max(0, Number(process.env.CTM_DB_OWNER_WORKER_BUSY_RETRIES) || 3);
+
+let ready = false;
+let closing = false;
+let active = null;
+let enqueued = 0;
+let completed = 0;
+let failed = 0;
+const queue = [];
+
+function _serializeError(err) {
+  return {
+    message: err && err.message ? err.message : String(err || 'unknown error'),
+    code: err && err.code ? err.code : undefined,
+    stack: err && err.stack ? String(err.stack) : undefined,
+  };
+}
+
+function _status() {
+  return {
+    ready,
+    active,
+    running: !!active,
+    pending: queue.length,
+    enqueued,
+    completed,
+    failed,
+  };
+}
+
+function _postStatus() {
+  try { parentPort.postMessage({ type: 'status', status: _status() }); } catch {}
+}
+
+function _ensureReady() {
+  if (!ready) throw new Error('DB owner worker is not ready');
+}
+
+let apiPrompts = null;
+function _apiPrompts() {
+  if (!apiPrompts) apiPrompts = require('../api-prompts');
+  return apiPrompts;
+}
+
+let promptHarvest = null;
+function _promptHarvest() {
+  if (!promptHarvest) promptHarvest = require('../prompt-harvest');
+  return promptHarvest;
+}
+
+let sessionIntegrity = null;
+function _sessionIntegrity() {
+  if (!sessionIntegrity) sessionIntegrity = require('../session-integrity');
+  return sessionIntegrity;
+}
+
+let transcriptStore = null;
+function _transcriptStore() {
+  if (!transcriptStore) transcriptStore = require('../lib/transcript-store');
+  return transcriptStore;
+}
+
+let sessionUtils = null;
+function _sessionUtils() {
+  if (!sessionUtils) sessionUtils = require('../session-utils');
+  return sessionUtils;
+}
+
+let stripTerminalReplies = null;
+function _stripLeakedTerminalGeneratedColorReplies(value) {
+  if (!stripTerminalReplies) {
+    stripTerminalReplies = require('../lib/terminal-control-sanitize').stripLeakedTerminalGeneratedColorReplies;
+  }
+  return stripTerminalReplies(value);
+}
+
+async function _runOp(op, payload = {}) {
+  _ensureReady();
+  switch (op) {
+    case 'appendScrollbackBatch': {
+      const sessionId = String(payload.sessionId || '');
+      const chunks = Array.isArray(payload.chunks) ? payload.chunks : [];
+      if (!sessionId || chunks.length === 0) return { appended: 0 };
+      db.appendScrollbackBatch(sessionId, chunks);
+      return { appended: chunks.length };
+    }
+    case 'clearScrollback': {
+      const sessionId = String(payload.sessionId || '');
+      if (!sessionId) return { cleared: false };
+      db.clearScrollback(sessionId);
+      return { cleared: true };
+    }
+    case 'prepareScrollbackForSession': {
+      const sessionId = String(payload.sessionId || '');
+      if (!sessionId) return { restoreAllowed: true, cleared: false };
+      return db.prepareScrollbackForSession(sessionId, payload.identity || {}, payload.options || {});
+    }
+    case 'loadRestoreScrollbackTail': {
+      // Offload the heavy restore-scrollback work OFF the main thread:
+      //   1. synchronous better-sqlite3 read (up to ~800 chunks / ~768KB)
+      //   2. chunks.map(c=>c.data).join('') over that whole payload
+      //   3. stripLeakedTerminalGeneratedColorReplies regex over the joined string
+      //   4. tail-cache truncation to tailCacheBytes
+      // This worker has its own WAL reader connection so the read is safe
+      // concurrent with the main process. Returns the final scrollData string
+      // (already stripped + tail-truncated) plus the loaded chunk count.
+      const sessionId = String(payload.sessionId || '');
+      if (!sessionId) return { scrollData: '', chunkCount: 0 };
+      const loadOpts = {};
+      if (Number.isFinite(Number(payload.maxChunks)) && Number(payload.maxChunks) > 0) {
+        loadOpts.maxChunks = Number(payload.maxChunks);
+      }
+      if (Number.isFinite(Number(payload.maxBytes)) && Number(payload.maxBytes) > 0) {
+        loadOpts.maxBytes = Number(payload.maxBytes);
+      }
+      const chunks = typeof db.loadScrollbackTail === 'function'
+        ? db.loadScrollbackTail(sessionId, loadOpts)
+        : db.loadScrollback(sessionId);
+      if (!Array.isArray(chunks) || chunks.length === 0) return { scrollData: '', chunkCount: 0 };
+      const chunkCount = chunks.length;
+      const scrollData = _stripLeakedTerminalGeneratedColorReplies(chunks.map(c => c.data).join(''));
+      if (!scrollData) return { scrollData: '', cacheSnapshot: '', chunkCount };
+      // Mirror _restoreScrollbackCacheFromChunks: the interim bootstrap cache is
+      // tail-truncated to tailCacheBytes, but the FULL (untruncated) scrollData
+      // is fed to background hydration. Return both so the main thread does only
+      // cheap field assignment (no join/regex/big-slice on the hot path).
+      const tailBytes = Number(payload.tailCacheBytes);
+      const cacheSnapshot = (Number.isFinite(tailBytes) && tailBytes > 0 && scrollData.length > tailBytes)
+        ? scrollData.slice(-tailBytes)
+        : scrollData;
+      return { scrollData, cacheSnapshot, chunkCount };
+    }
+    case 'batchRestoreIdentities': {
+      // Fix B Stage 3: precompute per-session restore metadata (identity /
+      // resolved / latest codex agent / title / scrollback seq seed) in ONE
+      // batched read off the main thread. Returns a plain object keyed by
+      // session id (Map is not structured-clone friendly across the worker
+      // boundary in a stable way, so serialize to an object here).
+      const sessionIds = Array.isArray(payload.sessionIds) ? payload.sessionIds : [];
+      if (!sessionIds.length || typeof db.batchRestoreIdentities !== 'function') {
+        return { entries: {} };
+      }
+      const map = db.batchRestoreIdentities(sessionIds);
+      const entries = {};
+      for (const [id, value] of map.entries()) entries[id] = value;
+      return { entries };
+    }
+    case 'retireStartupTask': {
+      const sessionId = String(payload.sessionId || '');
+      if (!sessionId) return { retired: false };
+      if (typeof db.markStartupTaskExited === 'function') db.markStartupTaskExited(sessionId);
+      db.removeStartupTask(sessionId);
+      return { retired: true };
+    }
+    case 'renameSession': {
+      const sessionId = String(payload.sessionId || '');
+      const liveSessionId = String(payload.liveSessionId || sessionId);
+      const title = String(payload.title || '').trim().slice(0, 120);
+      if (!sessionId || !title) return { renamed: false };
+      db.setSessionTitleNew(sessionId, title, true);
+      if (liveSessionId) (db.updateStartupTaskTitle || db.updateStartupTaskLabel)(liveSessionId, title);
+      return { renamed: true };
+    }
+    case 'heartbeatStartupTasks': {
+      const sessionIds = Array.isArray(payload.sessionIds) ? payload.sessionIds : [];
+      return { heartbeat: db.heartbeatStartupTasks(sessionIds) || 0 };
+    }
+    case 'saveImage': {
+      const promptId = Number(payload.promptId || 0);
+      const filename = payload.filename || 'image.png';
+      const mimeType = payload.mimeType || 'image/png';
+      const raw = payload.buffer || Buffer.alloc(0);
+      const buffer = Buffer.isBuffer(raw) ? raw : Buffer.from(raw);
+      return await db.saveImage(promptId, buffer, filename, mimeType);
+    }
+    case 'getWriteLockStats': {
+      // Read-only: this worker thread's own write-lock stats (the 5s-wait writer).
+      return payload && payload.reset ? db.resetWriteLockStats() : db.getWriteLockStats();
+    }
+    case 'getMessageExtractionStats': {
+      // Read-only: append-vs-full-rewrite counters for replaceSessionMessages, which
+      // runs on this worker for the conversation-import path (opportunity D).
+      const stats = db.getMessageExtractionStats ? db.getMessageExtractionStats() : null;
+      if (payload && payload.reset && db.resetMessageExtractionStats) db.resetMessageExtractionStats();
+      return stats;
+    }
+    case 'checkpointWal': {
+      return { rows: db.checkpointWal(payload.mode || 'PASSIVE') || [] };
+    }
+    case 'checkpointWalOrThrow': {
+      return { rows: db.checkpointWalOrThrow(payload.mode || 'PASSIVE') || [] };
+    }
+    case 'runIncrementalConversationImport': {
+      const result = await _apiPrompts().runIncrementalConversationImport();
+      return result || { done: true };
+    }
+    case 'runCursorConversationImport': {
+      const result = await _apiPrompts().runCursorConversationImport();
+      return result || { done: true };
+    }
+    case 'runIncrementalLifecycleRefresh': {
+      return _promptHarvest().runIncrementalLifecycleRefresh() || { done: true };
+    }
+    case 'runSessionIntegrity': {
+      return _sessionIntegrity().runIntegrityCheck({
+        db: db.getDb(),
+        getAllSessionFiles: _sessionUtils().getAllSessionFiles,
+        telemetry: null,
+        autoRecover: true,
+      }) || { checked: true };
+    }
+    case 'recoverSessionIntegrity': {
+      // Apply ONLY the recovery WRITES for issues the off-worker (read-pool) detect already
+      // found, so the heavy full-corpus detection scan never runs on this single serial
+      // worker (which would starve interactive writes → main-thread Dropbox-I/O freezes).
+      // recoverMismatches operates on the passed issues + a cheap file-index enumeration
+      // (not a transcript scan); repairSafeRelinkSwaps re-audits only when cross-links exist.
+      const integ = _sessionIntegrity();
+      const getAll = _sessionUtils().getAllSessionFiles;
+      const recoverable = Array.isArray(payload.recoverable) ? payload.recoverable : [];
+      let recovery = recoverable.length > 0
+        ? integ.recoverMismatches(db.getDb(), recoverable, getAll)
+        : null;
+      if (payload.hasCrossLinks) {
+        const relinkRecovery = integ.repairSafeRelinkSwaps(db.getDb(), null);
+        if (relinkRecovery && (relinkRecovery.fixed > 0 || relinkRecovery.skipped > 0)) {
+          if (!recovery) recovery = { fixed: 0, skipped: 0, actions: [] };
+          recovery.fixed += relinkRecovery.fixed;
+          recovery.skipped += relinkRecovery.skipped;
+          recovery.actions.push(...relinkRecovery.actions);
+        }
+      }
+      return { recovery };
+    }
+    case 'backfillSessionMessagesAsync': {
+      const result = await db.backfillSessionMessagesAsync(payload.options || {});
+      return { extracted: Number(result || 0) };
+    }
+    case 'runContentRowsBackfillSweep': {
+      // Phase 4 migration: populate session_message_rows for pre-migration sessions from
+      // their (small) blobs. Runs here so the chunked writes never block the main loop.
+      return db.runContentRowsBackfillSweep(payload.options || {});
+    }
+    case 'retireLegacyStores': {
+      // De-dup cleanup: drop session_messages + FTS once migration is complete. Cheap (DROP TABLE).
+      return db.retireLegacyStores();
+    }
+    case 'runVacuum': {
+      // Reclaim disk after retirement. EXCLUSIVE + heavy — runs here so the main loop's CPU is free,
+      // though it still serializes DB access for the duration. Operator-triggered only.
+      return db.runVacuum();
+    }
+    case 'pruneApprovalObservations': {
+      return db.pruneApprovalObservations(payload || {});
+    }
+    case 'pruneApprovalAiRefinementRules': {
+      return db.pruneApprovalAiRefinementRules(payload || {});
+    }
+    case 'importSessionConversation': {
+      // Durable conversation-cache write offloaded from apiSessionMessages' main-thread
+      // path: a multi-MB blob upsert + session_messages rewrite that froze the loop on
+      // cold Dropbox. Runs here so the disk I/O never blocks the event loop. Returns a
+      // small summary (no blob).
+      db.importSessionConversation(payload || {});
+      return { ok: true };
+    }
+    case 'catchUpTranscriptFiles': {
+      // Heavy JSONL tail ingest. Runs here (worker thread) so a cold-Dropbox fsync
+      // stall on a single statement can never freeze the main event loop. The main
+      // thread re-reads the freshly-ingested rows from its own connection (WAL) on
+      // the next message poll. Returns the summary only (no messages).
+      return _transcriptStore().catchUpTranscriptFiles(db.getDb(), payload || {});
+    }
+    case 'addApprovalObservation': {
+      return { id: db.addApprovalObservation(payload || {}) };
+    }
+    case 'drain': {
+      return { ok: true, ..._status() };
+    }
+    default: {
+      const err = new Error(`Unsupported DB owner worker op: ${op}`);
+      err.code = 'CTM_DB_OWNER_UNSUPPORTED_OP';
+      throw err;
+    }
+  }
+}
+
+function _pump() {
+  if (active || closing) return;
+  const task = queue.shift();
+  if (!task) {
+    _postStatus();
+    return;
+  }
+  active = task.op;
+  _postStatus();
+  Promise.resolve()
+    .then(() => retryOnWriteLockBusy(() => _runOp(task.op, task.payload), {
+      retries: WRITE_LOCK_RETRIES,
+      backoffMs: 60,
+      onRetry: ({ attempt, retries }) => {
+        console.warn(`[db-owner-worker] write-lock busy on '${task.op}', retry ${attempt}/${retries}`);
+      },
+    }))
+    .then((result) => {
+      completed += 1;
+      parentPort.postMessage({ type: 'response', requestId: task.requestId, result });
+    }, (err) => {
+      failed += 1;
+      parentPort.postMessage({ type: 'response', requestId: task.requestId, error: _serializeError(err) });
+    })
+    .finally(() => {
+      active = null;
+      _pump();
+    });
+}
+
+function _enqueue(msg) {
+  enqueued += 1;
+  queue.push({
+    requestId: msg.requestId,
+    op: msg.op,
+    payload: msg.payload || {},
+  });
+  _pump();
+}
+
+function _close() {
+  closing = true;
+  queue.length = 0;
+  try { db.closeDb({ checkpointMode: 'NONE' }); } catch {}
+  ready = false;
+  try { parentPort.postMessage({ type: 'closed' }); } catch {}
+  setImmediate(() => {
+    try { parentPort.close(); } catch {}
+  });
+}
+
+function _boot() {
+  try {
+    const dbPath = workerData && workerData.dbPath;
+    if (!dbPath) throw new Error('DB owner worker requires dbPath');
+    if (workerData.initDb) db.initDb(dbPath);
+    else db.connectDb(dbPath);
+    ready = true;
+    parentPort.postMessage({ type: 'ready', status: _status() });
+  } catch (err) {
+    parentPort.postMessage({ type: 'fatal', ..._serializeError(err) });
+  }
+}
+
+parentPort.on('message', (msg) => {
+  if (!msg || typeof msg !== 'object') return;
+  if (msg.type === 'request') {
+    _enqueue(msg);
+    return;
+  }
+  if (msg.type === 'status') {
+    _postStatus();
+    return;
+  }
+  if (msg.type === 'close') {
+    if (active) {
+      const interval = setInterval(() => {
+        if (active) return;
+        clearInterval(interval);
+        _close();
+      }, 10);
+      if (typeof interval.unref === 'function') interval.unref();
+    } else {
+      _close();
+    }
+  }
+});
+
+_boot();

package/template/claude-task-manager/workers/harvest-worker.js

@@ -1,61 +1,15 @@
-// --- Harvest Worker Thread ---
-// Runs prompt-harvest's runIncrementalLifecycleRefresh off the main event loop.
-// Opens its own better-sqlite3 connection (WAL mode supports concurrent access).
-// The main thread sends { type: 'run' } messages, receives { type: 'done', result } back.
+'use strict';
 
-const { parentPort, workerData } = require('worker_threads');
-const path = require('path');
-
-// Initialize DB module with the same path as main thread
-const dbModule = require(path.join(workerData.ctmDir, 'db'));
-dbModule.connectDb(workerData.dbPath);
-
-// Load harvest module (it uses dbModule.getDb() internally)
-const harvest = require(path.join(workerData.ctmDir, 'prompt-harvest'));
-
-let running = false;
-let closing = false;
-let closed = false;
-
-function closeDbAndAck() {
-  if (closed) return;
-  closed = true;
-  try { dbModule.closeDb({ checkpointMode: 'PASSIVE' }); } catch {}
-  parentPort.postMessage({ type: 'closed' });
-  setImmediate(() => process.exit(0));
-}
+const { parentPort } = require('node:worker_threads');
 
 parentPort.on('message', (msg) => {
-  if (msg.type === 'close') {
-    closing = true;
-    if (!running) closeDbAndAck();
+  if (msg?.type === 'close') {
+    parentPort.postMessage({ type: 'closed' });
     return;
   }
-  switch (msg.type) {
-    case 'run': {
-      if (closing) {
-        parentPort.postMessage({ type: 'error', dur: 0, error: 'worker closing' });
-        break;
-      }
-      running = true;
-      const t0 = Date.now();
-      try {
-        harvest.runIncrementalLifecycleRefresh();
-        const dur = Date.now() - t0;
-        parentPort.postMessage({ type: 'done', dur });
-      } catch (e) {
-        const dur = Date.now() - t0;
-        parentPort.postMessage({ type: 'error', dur, error: e.message });
-      } finally {
-        running = false;
-        if (closing) closeDbAndAck();
-      }
-      break;
-    }
-  }
-});
-
-// Cleanup on thread exit
-process.on('exit', () => {
-  try { dbModule.closeDb({ checkpointMode: 'NONE' }); } catch {}
+  parentPort.postMessage({
+    type: 'error',
+    dur: 0,
+    error: 'harvest DB work is owned by the CTM DB owner worker',
+  });
 });