create-walle 0.9.28 → 0.9.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/README.md +2 -2
  2. package/bin/create-walle.js +166 -6
  3. package/package.json +1 -1
  4. package/template/bin/ctm-launch.sh +70 -18
  5. package/template/bin/dev.sh +18 -0
  6. package/template/bin/ensure-stable-node.js +11 -0
  7. package/template/bin/node-bin.sh +9 -0
  8. package/template/claude-task-manager/api-prompts.js +214 -23
  9. package/template/claude-task-manager/db.js +884 -50
  10. package/template/claude-task-manager/docs/backfill-incremental-no-main-fallback.md +48 -0
  11. package/template/claude-task-manager/docs/conversation-import-freshness.md +21 -0
  12. package/template/claude-task-manager/docs/conversation-log-redesign.html +587 -0
  13. package/template/claude-task-manager/docs/session-title-authority.md +8 -3
  14. package/template/claude-task-manager/lib/auth-rules.js +13 -0
  15. package/template/claude-task-manager/lib/claude-desktop-sessions.js +63 -0
  16. package/template/claude-task-manager/lib/codex-config-guard.js +124 -0
  17. package/template/claude-task-manager/lib/codex-rollout-snapshot.js +93 -0
  18. package/template/claude-task-manager/lib/coding-agent-models.js +5 -4
  19. package/template/claude-task-manager/lib/db-owner-cooperative-scheduler.js +114 -0
  20. package/template/claude-task-manager/lib/db-owner-task-queue.js +67 -0
  21. package/template/claude-task-manager/lib/db-owner-worker-client.js +5 -1
  22. package/template/claude-task-manager/lib/desktop-fork.js +81 -0
  23. package/template/claude-task-manager/lib/headless-term-service.js +251 -4
  24. package/template/claude-task-manager/lib/message-identity.js +115 -0
  25. package/template/claude-task-manager/lib/mirror-feed-guards.js +25 -0
  26. package/template/claude-task-manager/lib/mirror-feed-sanitize.js +45 -0
  27. package/template/claude-task-manager/lib/path-suggest.js +77 -0
  28. package/template/claude-task-manager/lib/prompt-index-inputs.js +136 -0
  29. package/template/claude-task-manager/lib/real-node.js +36 -4
  30. package/template/claude-task-manager/lib/restore-auto-resume-policy.js +67 -0
  31. package/template/claude-task-manager/lib/restore-resume-batch.js +20 -0
  32. package/template/claude-task-manager/lib/restore-terminal-dims.js +109 -0
  33. package/template/claude-task-manager/lib/resume-cwd.js +124 -3
  34. package/template/claude-task-manager/lib/runtime-approval-recorder.js +152 -0
  35. package/template/claude-task-manager/lib/runtime-context-truth.js +236 -0
  36. package/template/claude-task-manager/lib/runtime-contract.js +195 -0
  37. package/template/claude-task-manager/lib/runtime-history-builder.js +205 -0
  38. package/template/claude-task-manager/lib/runtime-hook-bus.js +98 -0
  39. package/template/claude-task-manager/lib/runtime-input-queue.js +114 -0
  40. package/template/claude-task-manager/lib/runtime-input-recorder.js +156 -0
  41. package/template/claude-task-manager/lib/runtime-lineage.js +189 -0
  42. package/template/claude-task-manager/lib/runtime-registry.js +263 -0
  43. package/template/claude-task-manager/lib/runtime-session-history.js +41 -0
  44. package/template/claude-task-manager/lib/scrollback-snapshot-policy.js +37 -0
  45. package/template/claude-task-manager/lib/server-phase-conditions.js +103 -0
  46. package/template/claude-task-manager/lib/session-content-backfill.js +55 -8
  47. package/template/claude-task-manager/lib/session-db-read-contract.js +67 -0
  48. package/template/claude-task-manager/lib/session-history.js +93 -5
  49. package/template/claude-task-manager/lib/session-host-manager.js +154 -2
  50. package/template/claude-task-manager/lib/session-messages-defer.js +50 -0
  51. package/template/claude-task-manager/lib/session-messages-page.js +13 -0
  52. package/template/claude-task-manager/lib/session-messages-projection.js +48 -29
  53. package/template/claude-task-manager/lib/session-stream.js +80 -17
  54. package/template/claude-task-manager/lib/session-title-signals.js +54 -0
  55. package/template/claude-task-manager/lib/session-token-usage.js +13 -0
  56. package/template/claude-task-manager/lib/state-sync/cell-diff.js +41 -0
  57. package/template/claude-task-manager/lib/state-sync/frame-emitter.js +214 -0
  58. package/template/claude-task-manager/lib/state-sync/frame-rate.js +75 -0
  59. package/template/claude-task-manager/lib/state-sync/row-serializer.js +166 -0
  60. package/template/claude-task-manager/lib/terminal-fingerprint.js +19 -3
  61. package/template/claude-task-manager/lib/transcript-ingest-chunker.js +41 -0
  62. package/template/claude-task-manager/lib/transcript-store.js +99 -7
  63. package/template/claude-task-manager/lib/wal-checkpoint-policy.js +40 -0
  64. package/template/claude-task-manager/lib/walle-session-model-catalog.js +100 -9
  65. package/template/claude-task-manager/lib/worktree-output-binding.js +93 -0
  66. package/template/claude-task-manager/lib/write-coalescer.js +83 -0
  67. package/template/claude-task-manager/public/css/walle-session.css +4 -0
  68. package/template/claude-task-manager/public/css/walle.css +0 -66
  69. package/template/claude-task-manager/public/index.html +1707 -266
  70. package/template/claude-task-manager/public/js/feedback.js +8 -1
  71. package/template/claude-task-manager/public/js/message-renderer.js +72 -2
  72. package/template/claude-task-manager/public/js/session-phase.js +4 -0
  73. package/template/claude-task-manager/public/js/session-status-precedence.js +7 -173
  74. package/template/claude-task-manager/public/js/setup.js +46 -3
  75. package/template/claude-task-manager/public/js/state-sync-client.js +257 -0
  76. package/template/claude-task-manager/public/js/state-sync-predictor.js +41 -0
  77. package/template/claude-task-manager/public/js/stream-view.js +113 -9
  78. package/template/claude-task-manager/public/js/terminal-reconciler.js +24 -4
  79. package/template/claude-task-manager/public/js/walle-session.js +239 -19
  80. package/template/claude-task-manager/public/js/walle.js +32 -119
  81. package/template/claude-task-manager/queue-engine.js +140 -0
  82. package/template/claude-task-manager/server.js +2802 -416
  83. package/template/claude-task-manager/session-integrity.js +16 -1
  84. package/template/claude-task-manager/workers/db-owner-worker.js +23 -6
  85. package/template/claude-task-manager/workers/read-pool-worker.js +55 -1
  86. package/template/claude-task-manager/workers/session-host-pool-process.js +193 -0
  87. package/template/claude-task-manager/workers/session-host-process.js +47 -11
  88. package/template/claude-task-manager/workers/state-detectors/codex.js +33 -0
  89. package/template/package.json +1 -1
  90. package/template/wall-e/agent.js +191 -31
  91. package/template/wall-e/api-walle.js +97 -52
  92. package/template/wall-e/auth/flow-manager.js +78 -1
  93. package/template/wall-e/auth/provider-flows.js +56 -2
  94. package/template/wall-e/bin/walle-mcp-stdio.js +138 -5
  95. package/template/wall-e/brain.js +175 -13
  96. package/template/wall-e/chat.js +46 -1
  97. package/template/wall-e/embeddings.js +70 -0
  98. package/template/wall-e/events/event-bus.js +11 -1
  99. package/template/wall-e/http/auth.js +3 -1
  100. package/template/wall-e/http/model-admin.js +22 -0
  101. package/template/wall-e/lib/brain-owner-worker-client.js +36 -4
  102. package/template/wall-e/lib/diagnostics-flags.js +9 -0
  103. package/template/wall-e/lib/event-loop-monitor.js +84 -5
  104. package/template/wall-e/lib/mcp-scan-lifecycle.js +247 -0
  105. package/template/wall-e/lib/parent-brain-owner-client.js +109 -0
  106. package/template/wall-e/lib/runtime-process-inventory.js +114 -0
  107. package/template/wall-e/lib/runtime-worker-pool.js +214 -23
  108. package/template/wall-e/lib/scheduler-worker-jobs.js +49 -4
  109. package/template/wall-e/lib/scheduler.js +320 -35
  110. package/template/wall-e/lib/slack-identity.js +120 -0
  111. package/template/wall-e/lib/slack-permalink.js +107 -0
  112. package/template/wall-e/lib/slack-web.js +174 -0
  113. package/template/wall-e/lib/worker-thread-pool.js +55 -4
  114. package/template/wall-e/llm/claude-cli.js +21 -3
  115. package/template/wall-e/llm/cli-binary.js +90 -0
  116. package/template/wall-e/llm/codex-cli.js +113 -49
  117. package/template/wall-e/llm/default-fallback.js +10 -4
  118. package/template/wall-e/llm/mlx.js +46 -8
  119. package/template/wall-e/llm/model-catalog.js +129 -17
  120. package/template/wall-e/llm/provider-detector.js +112 -22
  121. package/template/wall-e/loops/backfill.js +32 -16
  122. package/template/wall-e/loops/ingest.js +50 -16
  123. package/template/wall-e/loops/tasks.js +521 -25
  124. package/template/wall-e/mcp-server.js +215 -6
  125. package/template/wall-e/memory/ctm-session-context.js +93 -0
  126. package/template/wall-e/skills/_bundled/google-calendar/run.js +15 -23
  127. package/template/wall-e/skills/_bundled/gws-workspace/gws-router +237 -0
  128. package/template/wall-e/skills/_bundled/gws-workspace/setup.js +112 -1
  129. package/template/wall-e/skills/_bundled/mcp-scan/run.js +265 -41
  130. package/template/wall-e/skills/_bundled/slack-mentions/run.js +434 -93
  131. package/template/wall-e/skills/internal-skill-registry.js +27 -5
  132. package/template/wall-e/skills/mcp-client.js +18 -3
  133. package/template/wall-e/skills/script-skill-runner.js +53 -5
  134. package/template/wall-e/skills/skill-planner.js +5 -26
  135. package/template/wall-e/training/real-trajectory-miner.js +24 -114
  136. package/template/wall-e/utils/dedup.js +165 -66
  137. package/template/wall-e/weather-runtime.js +12 -4
  138. package/template/wall-e/workers/brain-owner-worker.js +68 -0
  139. package/template/wall-e/workers/runtime-worker.js +4 -0
  140. package/template/website/index.html +3 -0
@@ -13,6 +13,7 @@ const permissionMatch = require('./lib/permission-match');
13
13
  // CTM permissions are a standalone global config in the CTM DB.
14
14
  const walleClient = require('./lib/walle-client');
15
15
  const claudeDesktopSessions = require('./lib/claude-desktop-sessions');
16
+ const desktopFork = require('./lib/desktop-fork');
16
17
  const skillAutocomplete = require('./lib/skill-autocomplete');
17
18
  const skillIntentResolver = require('./lib/skill-intent-resolver');
18
19
  const resourceLinks = require('./lib/resource-links');
@@ -48,6 +49,26 @@ const CONVERSATION_IMPORT_MIN_INTERVAL_MS = Math.max(0, Number(process.env.CTM_C
48
49
  const CONVERSATION_IMPORT_FORCE_BYTES = Math.max(0, Number(process.env.CTM_CONVERSATION_IMPORT_FORCE_BYTES ?? 256 * 1024));
49
50
  const _lastConversationImportAt = new Map(); // sessionId → Date.now() of last completed import
50
51
 
52
+ // Event-driven (near-real-time) durable import. The 8s floor above exists to stop a hot session's
53
+ // rapid small appends from re-processing the WHOLE conversation each tick (the giant-transcript
54
+ // write-lock thrash). With blob retirement the common case is now an O(Δ) append (a tiny write on
55
+ // the low-priority lane), so when this is on we track the live tail at a short coalescing window —
56
+ // but ONLY for the cheap append path. A session that recently fell back to a FULL rebuild keeps the
57
+ // long floor so a rewrite-in-progress session can't thrash. Off → exact 8s/batch behavior.
58
+ const CONVERSATION_IMPORT_EVENT_DRIVEN = process.env.CTM_CONVERSATION_IMPORT_EVENT_DRIVEN === '1';
59
+ const CONVERSATION_IMPORT_EVENT_DRIVEN_MIN_INTERVAL_MS = Math.max(0, Number(
60
+ process.env.CTM_CONVERSATION_IMPORT_EVENT_DRIVEN_MIN_INTERVAL_MS ?? 250));
61
+ const _lastConversationFullRebuildAt = new Map(); // sessionId → Date.now() of last FULL rebuild
62
+
63
+ // Pure decision: the debounce floor for a hot session's small-growth (append-eligible) re-import.
64
+ // Extracted so the rebuild-thrash guard is unit-testable without the import machinery.
65
+ function _conversationImportFloorMs({ eventDriven, baseFloorMs, eventFloorMs, lastRebuildAt, now }) {
66
+ if (!eventDriven) return baseFloorMs;
67
+ // Held back to the long floor only while a recent full rebuild is still "warm".
68
+ const rebuiltRecently = lastRebuildAt > 0 && (now - lastRebuildAt) < baseFloorMs;
69
+ return rebuiltRecently ? baseFloorMs : eventFloorMs;
70
+ }
71
+
51
72
  function setDbMaintenanceRunner(fn) {
52
73
  dbMaintenanceRunner = typeof fn === 'function' ? fn : null;
53
74
  }
@@ -192,6 +213,10 @@ function handlePromptApi(req, res, url) {
192
213
  const p = url.pathname;
193
214
  const m = req.method;
194
215
 
216
+ // --- Claude Desktop → Code conversion (snapshot-and-fork) ---
217
+ if (p === '/api/claude-desktop/forks' && m === 'GET') return handleListDesktopForks(req, res);
218
+ if (p === '/api/claude-desktop/convert' && m === 'POST') return handleConvertDesktopSession(req, res);
219
+
195
220
  // --- Prompts ---
196
221
  if (p === '/api/prompts' && m === 'GET') return handleListPrompts(req, res, url);
197
222
  if (p === '/api/prompts' && m === 'POST') return handleCreatePrompt(req, res);
@@ -392,6 +417,70 @@ function handleListPrompts(req, res, url) {
392
417
  jsonResponse(res, 200, db.listPrompts(opts));
393
418
  }
394
419
 
420
+ // Convert a read-only Claude Desktop conversation into a resumable Claude Code session.
421
+ // Thin HTTP wrapper around lib/desktop-fork.convertDesktopConversation; `deps` is injectable
422
+ // for unit tests (defaults to the real Desktop cache + fork orchestrator).
423
+ const DESKTOP_CACHE_ONLY_MESSAGE =
424
+ 'Claude Desktop has only cached this conversation in the recent list — the full transcript ' +
425
+ 'is not in the local cache yet. Open the conversation in Claude Desktop once, then rescan, ' +
426
+ 'before converting it to a Claude Code session.';
427
+
428
+ // List Desktop conversations that already have a live Claude Code fork, so the client can
429
+ // flip "Convert" → "Resume fork" per conversation. Excludes forks whose transcript was
430
+ // deleted (those should re-offer conversion). `deps` injectable for tests.
431
+ function handleListDesktopForks(req, res, deps = {}) {
432
+ const dbm = deps.db || db;
433
+ const fsx = deps.fs || fs;
434
+ let rows;
435
+ try {
436
+ rows = dbm.listDesktopForks() || [];
437
+ } catch (e) {
438
+ return jsonResponse(res, 500, { error: 'list_failed', message: e && e.message || 'failed' });
439
+ }
440
+ const forks = rows
441
+ .filter((f) => f.jsonlPath && fsx.existsSync(f.jsonlPath))
442
+ .map((f) => ({ desktopUuid: f.desktopUuid, forkSessionId: f.forkSessionId, cwd: f.projectPath || '' }));
443
+ return jsonResponse(res, 200, { forks });
444
+ }
445
+
446
+ async function handleConvertDesktopSession(req, res, deps = {}) {
447
+ const desktop = deps.desktop || claudeDesktopSessions;
448
+ const fork = deps.fork || desktopFork;
449
+
450
+ let body;
451
+ try {
452
+ body = await readBody(req);
453
+ } catch {
454
+ return jsonResponse(res, 400, { error: 'invalid_body' });
455
+ }
456
+
457
+ const desktopUuid = String((body && body.desktopUuid) || '').trim();
458
+ const cwd = String((body && body.cwd) || '').trim();
459
+ if (!desktopUuid || !cwd) {
460
+ return jsonResponse(res, 400, { error: 'desktopUuid and cwd are required' });
461
+ }
462
+
463
+ const session = desktop.getSession(desktopUuid);
464
+ if (!session) {
465
+ return jsonResponse(res, 404, { error: 'Claude Desktop conversation not found' });
466
+ }
467
+
468
+ const messages = desktop.getMessages(desktopUuid) || [];
469
+ if (messages.length === 0) {
470
+ return jsonResponse(res, 409, { error: 'cache_only', message: DESKTOP_CACHE_ONLY_MESSAGE });
471
+ }
472
+
473
+ try {
474
+ const result = fork.convertDesktopConversation({ session, cwd });
475
+ return jsonResponse(res, 200, { ok: true, ...result });
476
+ } catch (e) {
477
+ if (/no .*message/i.test(e && e.message || '')) {
478
+ return jsonResponse(res, 409, { error: 'cache_only', message: DESKTOP_CACHE_ONLY_MESSAGE });
479
+ }
480
+ return jsonResponse(res, 500, { error: 'convert_failed', message: e && e.message || 'conversion failed' });
481
+ }
482
+ }
483
+
395
484
  async function handleCreatePrompt(req, res) {
396
485
  try {
397
486
  const data = await readBody(req);
@@ -1053,7 +1142,6 @@ const {
1053
1142
  createCodexUserDeduper,
1054
1143
  parseCodexJsonlFileIntoMessagesAsync,
1055
1144
  parseCodexJsonlFileIntoMessages,
1056
- parseCodexJsonlIntoMessages,
1057
1145
  readCodexRolloutMetadata,
1058
1146
  } = require('./lib/session-history');
1059
1147
  const fsp = require('fs').promises;
@@ -1556,6 +1644,8 @@ async function _importCompactPair(parsed, jsonlPath, bakPath, jsonlSize, bakSize
1556
1644
  model_provider: parsed.modelProvider || (existing && existing.model_provider) || CLAUDE_MODEL_PROVIDER,
1557
1645
  model_id: parsed.modelId || (existing && existing.model_id) || '',
1558
1646
  import_parser_version: DEFAULT_CONVERSATION_IMPORT_PARSER_VERSION,
1647
+ // Chunked-import route (cold/large): defer rows to content-rows-backfill, keep the real blob.
1648
+ ..._chunkedImportFlags(parsed.sessionId, allMessages, totalSize, 'compact-pair'),
1559
1649
  });
1560
1650
  return true;
1561
1651
  }
@@ -1592,17 +1682,6 @@ function _codexUserDeduperFromMessages(messages) {
1592
1682
  return createCodexUserDeduper((Array.isArray(messages) ? messages : []).filter(msg => msg && msg.role === 'user'));
1593
1683
  }
1594
1684
 
1595
- async function _readFileRange(filePath, start, length) {
1596
- const fh = await fsp.open(filePath, 'r');
1597
- try {
1598
- const buf = Buffer.alloc(length);
1599
- const { bytesRead } = await fh.read(buf, 0, length, start);
1600
- return buf.subarray(0, bytesRead).toString('utf8');
1601
- } finally {
1602
- await fh.close();
1603
- }
1604
- }
1605
-
1606
1685
  function _conversationImportIndexRows() {
1607
1686
  // Attribution: two full-table index scans built on every conversation-import
1608
1687
  // tick. They run as the sync prefix of _conversationImportCandidates, which is
@@ -1695,7 +1774,15 @@ async function _conversationImportCandidates(allFiles, lastScanAt) {
1695
1774
  const lastAt = _lastConversationImportAt.get(sessionId) || 0;
1696
1775
  const sinceLast = Date.now() - lastAt;
1697
1776
  const grewBytes = Math.max(0, effectiveSize - existingSize);
1698
- if (lastAt > 0 && sinceLast < CONVERSATION_IMPORT_MIN_INTERVAL_MS && grewBytes < CONVERSATION_IMPORT_FORCE_BYTES) {
1777
+ // Near-real-time floor for the cheap append path; long floor right after a full rebuild.
1778
+ const floorMs = _conversationImportFloorMs({
1779
+ eventDriven: CONVERSATION_IMPORT_EVENT_DRIVEN,
1780
+ baseFloorMs: CONVERSATION_IMPORT_MIN_INTERVAL_MS,
1781
+ eventFloorMs: CONVERSATION_IMPORT_EVENT_DRIVEN_MIN_INTERVAL_MS,
1782
+ lastRebuildAt: _lastConversationFullRebuildAt.get(sessionId) || 0,
1783
+ now: Date.now(),
1784
+ });
1785
+ if (lastAt > 0 && sinceLast < floorMs && grewBytes < CONVERSATION_IMPORT_FORCE_BYTES) {
1699
1786
  continue; // imported very recently and only a little new data — let appends coalesce
1700
1787
  }
1701
1788
  }
@@ -1746,6 +1833,46 @@ function _backgroundTranscriptImportMaxBytes() {
1746
1833
  return Math.max(256 * 1024, Number.isFinite(raw) ? raw : BACKGROUND_TRANSCRIPT_IMPORT_DEFAULT_BYTES);
1747
1834
  }
1748
1835
 
1836
+ // Phase 1.5: a cold/large full-rebuild's inline session_message_rows write (delete-all + insert-all)
1837
+ // can block the db-owner worker for seconds — over the coop slice budget. Above a message-count
1838
+ // threshold, route the import through the CHUNKED path: importSessionConversation writes the blob +
1839
+ // stamps the source-len HWM now (forceBlobWrite + skipMessageRows), then content-rows-backfill fills
1840
+ // rows in windows across slices. Returns the two flags to spread into the importSessionConversation
1841
+ // options. Applied at every large cold-import call site (codex / claude-desktop / generic claude).
1842
+ // Over-cap sessions ignore forceBlobWrite downstream (empty blob, transcript-store source of truth).
1843
+ // Routing counters (per import run) — surfaced in the always-on [auto-import] summary so the
1844
+ // chunked-vs-inline distribution is visible WITHOUT depending on CTM_COOP_LOG reaching the worker
1845
+ // thread. Reset by runIncrementalConversationImport after it logs them.
1846
+ const _chunkedRouteStats = { chunked: 0, inline: 0 };
1847
+ function _readResetChunkedRouteStats() {
1848
+ const s = { ..._chunkedRouteStats };
1849
+ _chunkedRouteStats.chunked = 0; _chunkedRouteStats.inline = 0;
1850
+ return s;
1851
+ }
1852
+
1853
+ function _chunkedImportFlags(sessionId, messages, fileSize, reason) {
1854
+ // Gated on the coop scheduler: deferring rows only helps when the cooperative scheduler is driving
1855
+ // bounded slices (and the content-rows-backfill coop/legacy job fills the deferred rows). With the
1856
+ // flag OFF the import behaves byte-for-byte as legacy (rows written inline) — so merging this branch
1857
+ // is a true no-op on the primary until CTM_COOP_SCHEDULER=1 is set.
1858
+ if (process.env.CTM_COOP_SCHEDULER !== '1') return { skipMessageRows: false, forceBlobWrite: false };
1859
+ const minMsgs = Math.max(1, Number(process.env.CTM_IMPORT_CHUNK_MIN_MSGS || 1000));
1860
+ // ALSO trigger on file size: a giant rollout is tail-bounded at parse time (transcriptMaxBytes),
1861
+ // so `messages.length` here can be small even though the session is huge and its inline row write
1862
+ // would block the worker. Keying on the JSONL byte high-water-mark catches those. Default 2MB.
1863
+ const minBytes = Math.max(1, Number(process.env.CTM_IMPORT_CHUNK_MIN_BYTES || 2 * 1024 * 1024));
1864
+ const n = Array.isArray(messages) ? messages.length : 0;
1865
+ const bytes = Number(fileSize || 0);
1866
+ const useChunked = n > minMsgs || bytes > minBytes;
1867
+ _chunkedRouteStats[useChunked ? 'chunked' : 'inline'] += 1;
1868
+ if (process.env.CTM_COOP_LOG === '1') {
1869
+ const fileMB = (bytes / (1024 * 1024)).toFixed(1);
1870
+ const trig = useChunked ? (n > minMsgs ? 'msgs' : 'bytes') : 'none';
1871
+ console.log(`[coop-import] sid=${String(sessionId || '').slice(0, 8)} msgs=${n} fileMB=${fileMB} path=${useChunked ? 'chunked' : 'inline'} trigger=${trig} reason=${reason}`);
1872
+ }
1873
+ return { skipMessageRows: useChunked, forceBlobWrite: useChunked };
1874
+ }
1875
+
1749
1876
  function _codexImportedFileSize(parsedFileSize, prevFileSize, parsedTail) {
1750
1877
  const fileSize = Math.max(0, Number(parsedFileSize || 0));
1751
1878
  const prev = Math.max(0, Number(prevFileSize || 0));
@@ -1794,8 +1921,15 @@ async function _importCodexSessionFile(parsed, filePath, options = {}) {
1794
1921
  const newMessages = [];
1795
1922
  let parsedTail;
1796
1923
  if (prevFileSize > 0 && parsed.fileSize > prevFileSize) {
1797
- const content = await _readFileRange(filePath, prevFileSize, parsed.fileSize - prevFileSize);
1798
- parsedTail = parseCodexJsonlIntoMessages(content, newMessages, { codexUserDeduper });
1924
+ // Stream the new tail [prevFileSize, EOF) in bounded 1 MiB chunks, line-by-line. Reading the
1925
+ // whole delta in one fs.read (the old _readFileRange path) passed a >2 GiB `length` for a
1926
+ // session that grew >2 GiB since the last import, tripping V8's Int32 assertion in node::fs::Read
1927
+ // and ABORTING the process (uncatchable) → launchd respawn loop → all Codex sessions unusable.
1928
+ parsedTail = await parseCodexJsonlFileIntoMessagesAsync(filePath, newMessages, {
1929
+ codexUserDeduper,
1930
+ startOffset: prevFileSize,
1931
+ yieldAfterMs: options.yieldAfterMs || 25,
1932
+ });
1799
1933
  } else if (options.cooperative) {
1800
1934
  parsedTail = await parseCodexJsonlFileIntoMessagesAsync(filePath, newMessages, {
1801
1935
  codexUserDeduper,
@@ -1857,6 +1991,9 @@ async function _importCodexSessionFile(parsed, filePath, options = {}) {
1857
1991
  model_provider: modelProvider,
1858
1992
  model_id: model || (existing && existing.model_id) || '',
1859
1993
  import_parser_version: parserVersion,
1994
+ // Chunked-import route (cold/large codex rollout — the giant-session case): defer rows to
1995
+ // content-rows-backfill, keep the real blob. Over-cap rollouts ignore forceBlobWrite downstream.
1996
+ ..._chunkedImportFlags(sessionId, allMessages, importedFileSize, 'codex'),
1860
1997
  });
1861
1998
 
1862
1999
  try {
@@ -2055,6 +2192,8 @@ async function importSessionFile(filePath, projectPath, projectEntry, options =
2055
2192
  model_provider: parsed.modelProvider || (existing && existing.model_provider) || CLAUDE_MODEL_PROVIDER,
2056
2193
  model_id: parsed.modelId || (existing && existing.model_id) || '',
2057
2194
  import_parser_version: DEFAULT_CONVERSATION_IMPORT_PARSER_VERSION,
2195
+ // Chunked-import route (cold/large): defer rows to content-rows-backfill, keep the real blob.
2196
+ ..._chunkedImportFlags(parsed.sessionId, messages, parsed.fileSize, 'claude-desktop'),
2058
2197
  });
2059
2198
  return true;
2060
2199
  }
@@ -2136,7 +2275,12 @@ async function importSessionFile(filePath, projectPath, projectEntry, options =
2136
2275
  rename_name: parsedRename || parsed.renameName || '',
2137
2276
  import_parser_version: DEFAULT_CONVERSATION_IMPORT_PARSER_VERSION,
2138
2277
  });
2139
- if (appended && appended.ok) return true;
2278
+ if (appended && appended.ok) {
2279
+ // Cheap O(Δ) append succeeded — clear any rebuild backoff so event-driven mode
2280
+ // tracks this session's live tail at the fast cadence.
2281
+ _lastConversationFullRebuildAt.delete(parsed.sessionId);
2282
+ return true;
2283
+ }
2140
2284
  }
2141
2285
 
2142
2286
  // Full rebuild: cold import, file shrank/rotated, or the append guard refused. Load the base
@@ -2178,6 +2322,11 @@ async function importSessionFile(filePath, projectPath, projectEntry, options =
2178
2322
  // but an empty new parse must not wipe an existing rename.
2179
2323
  const mergedRename = signals.renameName || parsedRename || parsed.renameName || (existing && existing.rename_name) || '';
2180
2324
 
2325
+ // This is the expensive full-rebuild path (cold import, file shrank/rotated, or the append
2326
+ // guard refused a non-clean-prefix). Stamp the rebuild clock so event-driven mode keeps the
2327
+ // long debounce floor for this session until a cheap append succeeds again (no rebuild thrash).
2328
+ try { _lastConversationFullRebuildAt.set(parsed.sessionId, Date.now()); } catch {}
2329
+
2181
2330
  db.importSessionConversation({
2182
2331
  session_id: parsed.sessionId,
2183
2332
  project_path: parsed.cwd || parsed.project,
@@ -2197,6 +2346,8 @@ async function importSessionFile(filePath, projectPath, projectEntry, options =
2197
2346
  model_provider: parsed.modelProvider || (existing && existing.model_provider) || CLAUDE_MODEL_PROVIDER,
2198
2347
  model_id: parsed.modelId || (existing && existing.model_id) || '',
2199
2348
  import_parser_version: DEFAULT_CONVERSATION_IMPORT_PARSER_VERSION,
2349
+ // Chunked-import route (cold/large): defer rows to content-rows-backfill, keep the real blob.
2350
+ ..._chunkedImportFlags(parsed.sessionId, allMessages, parsed.fileSize, 'full-rebuild'),
2200
2351
  });
2201
2352
  return true;
2202
2353
  }
@@ -2214,19 +2365,27 @@ async function handleImportConversations(req, res) {
2214
2365
  // Incremental auto-import: only processes files modified since last scan
2215
2366
  // Uses async filesystem operations to avoid blocking the event loop
2216
2367
  // (critical when ~/.claude/projects is on Dropbox/network-synced FS)
2217
- async function runIncrementalConversationImport() {
2368
+ async function runIncrementalConversationImport(opts = {}) {
2369
+ const _now = typeof opts.now === 'function' ? opts.now : () => Date.now();
2218
2370
  // Capture scanStart BEFORE anything else — we advance lastScanAt even on
2219
2371
  // partial failure so a single bad file doesn't stall the whole job forever.
2220
2372
  // Previously, a top-level failure kept lastScanAt frozen, causing repeated
2221
2373
  // full rescans + repeated failures (stall bug observed in production).
2222
- const scanStart = Date.now();
2374
+ const scanStart = _now();
2223
2375
  let imported = 0;
2224
2376
  let scanned = 0;
2225
2377
  let total = 0;
2226
2378
  let failed = 0;
2227
2379
  let hitLimit = false;
2228
2380
  let hitLimitLabel = '';
2229
- const maxImportedPerRun = Math.max(1, Number(process.env.CTM_CONVERSATION_IMPORT_MAX_PER_RUN || 12));
2381
+ // Wall-clock slice budget: a single giant/cold full-rebuild can overrun the COUNT cap on TIME, so
2382
+ // also stop between files once the budget is spent. 0/absent = no time cap (legacy behavior).
2383
+ const budgetMs = Math.max(0, Number(opts.budgetMs ?? process.env.CTM_CONVERSATION_IMPORT_BUDGET_MS ?? 0));
2384
+ const deadline = budgetMs > 0 ? scanStart + budgetMs : Infinity;
2385
+ // Smaller cap (4) applies only when a slice budget is active (coop driver re-wakes immediately on
2386
+ // hasMore, so smaller slices drain just as fast while keeping each worker op short). Legacy path
2387
+ // (no budget) keeps the original default of 12 so flag-OFF import drain is unchanged.
2388
+ const maxImportedPerRun = Math.max(1, Number(process.env.CTM_CONVERSATION_IMPORT_MAX_PER_RUN || (budgetMs > 0 ? 4 : 12)));
2230
2389
  const maxProcessedPerRun = Math.max(1, Number(process.env.CTM_CONVERSATION_IMPORT_MAX_PROCESSED_PER_RUN || maxImportedPerRun));
2231
2390
  const retryAfterMsRaw = Number(process.env.CTM_CONVERSATION_IMPORT_RETRY_AFTER_MS || CONVERSATION_IMPORT_RETRY_AFTER_MS);
2232
2391
  const retryAfterMs = Number.isFinite(retryAfterMsRaw) && retryAfterMsRaw >= 1000
@@ -2267,6 +2426,11 @@ async function runIncrementalConversationImport() {
2267
2426
  hitLimitLabel = importLimited ? `${maxImportedPerRun} imports` : `${maxProcessedPerRun} files`;
2268
2427
  break;
2269
2428
  }
2429
+ if (_now() >= deadline && scanned < candidates.length) {
2430
+ hitLimit = true;
2431
+ hitLimitLabel = `${budgetMs}ms budget`;
2432
+ break;
2433
+ }
2270
2434
  await new Promise((resolve) => setImmediate(resolve));
2271
2435
  } catch (e) {
2272
2436
  failed++;
@@ -2276,9 +2440,11 @@ async function runIncrementalConversationImport() {
2276
2440
  }
2277
2441
  }
2278
2442
 
2443
+ const _rt = _readResetChunkedRouteStats();
2279
2444
  if (imported > 0 || failed > 0) {
2280
2445
  const suffix = hitLimit ? `, paused after ${hitLimitLabel}` : '';
2281
- console.log(`[auto-import] Imported ${imported}, failed ${failed} (scanned ${scanned}/${total}${suffix})`);
2446
+ const route = (_rt.chunked || _rt.inline) ? ` route=chunked:${_rt.chunked}/inline:${_rt.inline}` : '';
2447
+ console.log(`[auto-import] Imported ${imported}, failed ${failed} (scanned ${scanned}/${total}${suffix})${route}`);
2282
2448
  }
2283
2449
  return {
2284
2450
  imported,
@@ -2287,11 +2453,12 @@ async function runIncrementalConversationImport() {
2287
2453
  candidates: candidates.length,
2288
2454
  failed,
2289
2455
  remaining: hitLimit,
2456
+ hasMore: hitLimit,
2290
2457
  retry_after_ms: hitLimit ? retryAfterMs : undefined,
2291
2458
  };
2292
2459
  } catch (e) {
2293
2460
  console.error('[auto-import] Top-level error:', e.message);
2294
- return { imported, scanned, total, failed, error: e.message };
2461
+ return { imported, scanned, total, failed, error: e.message, hasMore: false };
2295
2462
  } finally {
2296
2463
  // Always advance the timestamp — even on error — so we don't re-scan the
2297
2464
  // same problematic files forever on every tick.
@@ -2384,6 +2551,8 @@ async function runCursorConversationImport({ cursorHome } = {}) {
2384
2551
  hostname: '',
2385
2552
  import_parser_version: CURSOR_IMPORT_PARSER_VERSION,
2386
2553
  rename_name: '',
2554
+ // Chunked-import route (cold/large cursor session): defer rows to content-rows-backfill.
2555
+ ..._chunkedImportFlags(ctmId, fields.messages, 0, 'cursor'),
2387
2556
  });
2388
2557
  if (sig) _cursorImportSignatures.set(ctmId, sig);
2389
2558
  imported += 1;
@@ -2407,6 +2576,16 @@ function handleGetConversation(req, res, url) {
2407
2576
  const sessionId = url.pathname.split('/').pop();
2408
2577
  const conv = db.getSessionConversation(sessionId);
2409
2578
  if (!conv) return jsonResponse(res, 404, { error: 'Not found' });
2579
+ // The stored `messages` blob is retired ('[]') by default — the conversation lives in the
2580
+ // faithful session_message_rows. Hydrate the blob from rows for this cold API read so callers
2581
+ // still receive the turns. O(N) reconstruction, paid only on this rare endpoint (the hot UI
2582
+ // paths page rows directly and never hit this).
2583
+ if ((!conv.messages || conv.messages === '[]') && typeof db.getSessionMessagesArray === 'function') {
2584
+ try {
2585
+ const rows = db.getSessionMessagesArray(sessionId, { fallbackToBlob: false });
2586
+ if (Array.isArray(rows) && rows.length) conv.messages = JSON.stringify(rows);
2587
+ } catch { /* fall through with the empty blob */ }
2588
+ }
2410
2589
  jsonResponse(res, 200, conv);
2411
2590
  }
2412
2591
 
@@ -2926,10 +3105,22 @@ function screenshotResponsibleContext() {
2926
3105
  try { nodeIsBundle = require('./lib/real-node').isBundleExec(node); } catch {}
2927
3106
  const useBridge = disclaimOk && !!node && !nodeIsBundle;
2928
3107
  const responsiblePath = useBridge ? node : process.execPath;
2929
- const responsibleName = responsiblePath ? path.basename(responsiblePath) : 'CTM';
3108
+ const responsibleName = screenshotResponsibleDisplayName(responsiblePath);
2930
3109
  return { realNode: node, disclaim, useBridge, responsiblePath, responsibleName, granted };
2931
3110
  }
2932
3111
 
3112
+ // macOS attributes the Screen Recording grant to the responsible binary's code SIGNATURE, and shows
3113
+ // the enclosing .app's CFBundleName. The adopted Developer-ID-notarized app's vendored node
3114
+ // (~/.walle/notarized-app/Wall-E.app/Contents/Resources/node, create-walle ensureNotarizedBrandedApp)
3115
+ // and the branded daemon bundles list as "Wall-E"/"Coding Task Manager", not the basename "node" —
3116
+ // so the guidance must name the entry the user will actually see, else they grant the wrong thing.
3117
+ function screenshotResponsibleDisplayName(responsiblePath) {
3118
+ if (!responsiblePath) return 'CTM';
3119
+ if (/(?:^|\/)Wall-E\.app\//.test(responsiblePath) || responsiblePath.includes('/.walle/notarized-app/')) return 'Wall-E';
3120
+ if (/(?:^|\/)Coding Task Manager\.app\//.test(responsiblePath)) return 'Coding Task Manager';
3121
+ return path.basename(responsiblePath);
3122
+ }
3123
+
2933
3124
  function screenshotCaptureCommand(tmpFile, context = {}) {
2934
3125
  if (context.useBridge) {
2935
3126
  return {
@@ -5132,4 +5323,4 @@ function safeParse(json, fallback) {
5132
5323
  try { return JSON.parse(json); } catch { return fallback; }
5133
5324
  }
5134
5325
 
5135
- module.exports = { handlePromptApi, queueEngine, runIncrementalConversationImport, runCursorConversationImport, importSessionFile, setUiPrefsBroadcaster, setPromptExecutionsOffThread, setDbMaintenanceRunner, setImageSaveRunner, ensureHotkeyDaemon, hotkeyEnsureAction, screenshotResponsibleContext, screenshotNodeCandidates, probeScreenshotNodeGranted, _ingestPathFromInput, _ingestSourceAllowed, _conversationImportCandidates, _ingestTranscriptStoreForParsedFile, _learnSignatureRules, _appendBlocklistExceptions, _finalizePermIntent, _denyHinted, _lastConversationImportAt };
5326
+ module.exports = { handlePromptApi, handleConvertDesktopSession, handleListDesktopForks, queueEngine, runIncrementalConversationImport, runCursorConversationImport, importSessionFile, _chunkedImportFlags, setUiPrefsBroadcaster, setPromptExecutionsOffThread, setDbMaintenanceRunner, setImageSaveRunner, ensureHotkeyDaemon, hotkeyEnsureAction, screenshotResponsibleContext, screenshotNodeCandidates, probeScreenshotNodeGranted, _ingestPathFromInput, _ingestSourceAllowed, _conversationImportCandidates, _ingestTranscriptStoreForParsedFile, _learnSignatureRules, _appendBlocklistExceptions, _finalizePermIntent, _denyHinted, _lastConversationImportAt, _conversationImportFloorMs, _lastConversationFullRebuildAt };