create-walle 0.9.31 → 0.9.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +1 -1
  2. package/template/claude-task-manager/api-prompts.js +406 -1
  3. package/template/claude-task-manager/api-reviews.js +104 -6
  4. package/template/claude-task-manager/approval-agent.js +293 -27
  5. package/template/claude-task-manager/db.js +77 -13
  6. package/template/claude-task-manager/docs/codex-app-server-approvals.md +35 -1
  7. package/template/claude-task-manager/docs/postgres-concurrency-dgx-spark.html +407 -0
  8. package/template/claude-task-manager/docs/postgres-storage-abstraction.html +732 -0
  9. package/template/claude-task-manager/docs/prompt-manager-redesign-proposal.html +24 -23
  10. package/template/claude-task-manager/docs/unified-auto-approver-proposal.html +520 -0
  11. package/template/claude-task-manager/git-utils.js +164 -26
  12. package/template/claude-task-manager/lib/agent-capabilities.js +26 -5
  13. package/template/claude-task-manager/lib/agent-cli-cache.js +1 -1
  14. package/template/claude-task-manager/lib/agent-presets.js +12 -0
  15. package/template/claude-task-manager/lib/agent-version-service.js +650 -0
  16. package/template/claude-task-manager/lib/approval-drift.js +57 -0
  17. package/template/claude-task-manager/lib/approval-hook.js +13 -0
  18. package/template/claude-task-manager/lib/auth-rules.js +6 -0
  19. package/template/claude-task-manager/lib/broadcast-payload-memo.js +109 -0
  20. package/template/claude-task-manager/lib/coding-agent-models.js +89 -8
  21. package/template/claude-task-manager/lib/document-review.js +75 -1
  22. package/template/claude-task-manager/lib/escalation-review.js +33 -3
  23. package/template/claude-task-manager/lib/headless-term-service.js +71 -9
  24. package/template/claude-task-manager/lib/mcp-risk.js +128 -0
  25. package/template/claude-task-manager/lib/microsoft-dev-tunnel-setup.js +51 -2
  26. package/template/claude-task-manager/lib/native-agent-model-args.js +31 -0
  27. package/template/claude-task-manager/lib/opencode-conversation-store.js +269 -0
  28. package/template/claude-task-manager/lib/permission-match.js +77 -5
  29. package/template/claude-task-manager/lib/read-pool-client.js +15 -6
  30. package/template/claude-task-manager/lib/restore-backoff.js +82 -0
  31. package/template/claude-task-manager/lib/runtime-registry.js +1 -0
  32. package/template/claude-task-manager/lib/scheduled-wake.js +42 -0
  33. package/template/claude-task-manager/lib/session-capture.js +33 -6
  34. package/template/claude-task-manager/lib/session-history.js +56 -2
  35. package/template/claude-task-manager/lib/session-jobs.js +19 -0
  36. package/template/claude-task-manager/lib/session-restore.js +5 -4
  37. package/template/claude-task-manager/lib/session-standup.js +2 -0
  38. package/template/claude-task-manager/lib/session-stream.js +9 -0
  39. package/template/claude-task-manager/lib/state-sync/cell-diff.js +31 -4
  40. package/template/claude-task-manager/lib/state-sync/emit-clamp.js +32 -0
  41. package/template/claude-task-manager/lib/state-sync/frame-emitter.js +1 -1
  42. package/template/claude-task-manager/lib/state-sync/restore-frame-hold.js +39 -0
  43. package/template/claude-task-manager/lib/walle-ctm-history.js +218 -42
  44. package/template/claude-task-manager/lib/walle-session-model-catalog.js +7 -1
  45. package/template/claude-task-manager/lib/walle-token-chip.js +65 -0
  46. package/template/claude-task-manager/package.json +1 -1
  47. package/template/claude-task-manager/providers/codex-mcp-extract.js +33 -0
  48. package/template/claude-task-manager/providers/codex-mcp.js +23 -20
  49. package/template/claude-task-manager/providers/codex.js +65 -1
  50. package/template/claude-task-manager/public/css/prompts.css +345 -0
  51. package/template/claude-task-manager/public/css/reviews.css +16 -40
  52. package/template/claude-task-manager/public/css/walle-session.css +146 -26
  53. package/template/claude-task-manager/public/index.html +1315 -305
  54. package/template/claude-task-manager/public/js/document-review-links.js +215 -9
  55. package/template/claude-task-manager/public/js/message-renderer.js +106 -0
  56. package/template/claude-task-manager/public/js/mobile-review-core.js +54 -0
  57. package/template/claude-task-manager/public/js/prompt-diff.js +55 -0
  58. package/template/claude-task-manager/public/js/prompt-editor-chrome.js +108 -0
  59. package/template/claude-task-manager/public/js/prompt-editor-modes.js +87 -0
  60. package/template/claude-task-manager/public/js/prompt-organize.js +179 -0
  61. package/template/claude-task-manager/public/js/prompt-reuse.js +77 -0
  62. package/template/claude-task-manager/public/js/prompts.js +683 -215
  63. package/template/claude-task-manager/public/js/reviews.js +28 -54
  64. package/template/claude-task-manager/public/js/session-search-utils.js +55 -0
  65. package/template/claude-task-manager/public/js/state-sync-client.js +72 -1
  66. package/template/claude-task-manager/public/js/stream-view.js +6 -0
  67. package/template/claude-task-manager/public/js/walle-session.js +502 -117
  68. package/template/claude-task-manager/public/js/walle.js +21 -9
  69. package/template/claude-task-manager/public/m/app.css +117 -0
  70. package/template/claude-task-manager/public/m/app.js +309 -3
  71. package/template/claude-task-manager/public/m/index.html +56 -1
  72. package/template/claude-task-manager/server.js +695 -306
  73. package/template/claude-task-manager/workers/db-owner-worker.js +8 -0
  74. package/template/claude-task-manager/workers/read-pool-worker.js +4 -0
  75. package/template/claude-task-manager/workers/state-detectors/claude-code.js +10 -1
  76. package/template/claude-task-manager/workers/state-detectors/codex.js +25 -6
  77. package/template/claude-task-manager/workers/state-detectors/index.js +1 -0
  78. package/template/docs/proposals/2026-06-23-local-model-right-path.html +340 -0
  79. package/template/docs/proposals/2026-06-24-qlora-done-right.md +255 -0
  80. package/template/package.json +1 -1
  81. package/template/wall-e/bin/train-gemma-e4b-tooluse.js +27 -4
  82. package/template/wall-e/brain.js +50 -5
  83. package/template/wall-e/chat/force-compact.js +73 -0
  84. package/template/wall-e/chat.js +61 -94
  85. package/template/wall-e/coding/action-memory-policy.js +120 -1
  86. package/template/wall-e/coding/coding-run-controller.js +118 -195
  87. package/template/wall-e/coding/compaction-service.js +101 -13
  88. package/template/wall-e/coding/model-router.js +116 -0
  89. package/template/wall-e/coding/no-progress-guard.js +4 -1
  90. package/template/wall-e/coding/output-boundary.js +94 -0
  91. package/template/wall-e/coding/permission-service.js +15 -0
  92. package/template/wall-e/coding/session-workspaces.js +70 -0
  93. package/template/wall-e/coding/tool-execution-controller.js +54 -3
  94. package/template/wall-e/coding-orchestrator.js +321 -288
  95. package/template/wall-e/coding-prompts.js +63 -9
  96. package/template/wall-e/context/token-counter.js +35 -15
  97. package/template/wall-e/docs/skill-self-heal-design.html +429 -0
  98. package/template/wall-e/llm/client.js +53 -3
  99. package/template/wall-e/llm/default-fallback.js +59 -5
  100. package/template/wall-e/llm/mlx-worker.js +0 -0
  101. package/template/wall-e/llm/mlx.js +46 -12
  102. package/template/wall-e/llm/provider-error.js +2 -2
  103. package/template/wall-e/llm/provider-health-state.js +6 -2
  104. package/template/wall-e/llm/tool-call-validator.js +156 -0
  105. package/template/wall-e/skills/skill-dispatch-decision.js +82 -2
  106. package/template/wall-e/skills/skill-planner.js +72 -5
  107. package/template/wall-e/tools/command-registry.js +34 -0
  108. package/template/wall-e/tools/local-tools.js +45 -17
  109. package/template/wall-e/tools/permission-checker.js +5 -0
  110. package/template/wall-e/training/gemma-e4b-qlora.js +197 -1
  111. package/template/wall-e/training/mlx_lora_launch.py +196 -0
  112. package/template/wall-e/training/training-coexistence.sh +87 -0
  113. package/template/wall-e/training/training-mem-guard.sh +76 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "create-walle",
3
- "version": "0.9.31",
3
+ "version": "0.9.33",
4
4
  "description": "CTM + Wall-E — AI coding dashboard and personal digital twin agent. Multi-agent terminal for Claude Code, Codex, Gemini, Aider, OpenCode, and more, plus prompt editor, task queue, remote phone and tablet access, code/doc review, and an agent that learns from Slack, email & calendar.",
5
5
  "bin": {
6
6
  "create-walle": "bin/create-walle.js"
@@ -61,6 +61,23 @@ const CONVERSATION_IMPORT_EVENT_DRIVEN_MIN_INTERVAL_MS = Math.max(0, Number(
61
61
  process.env.CTM_CONVERSATION_IMPORT_EVENT_DRIVEN_MIN_INTERVAL_MS ?? 250));
62
62
  const _lastConversationFullRebuildAt = new Map(); // sessionId → Date.now() of last FULL rebuild
63
63
 
64
+ // Giant-session RESTART-freeze guard. When a chunked-route session's deferred per-message rows are
65
+ // still backfilling, the O(Δ) append fast-path refuses ('rows-not-appendable') and the caller would
66
+ // fall through to a multi-second full-rebuild blob re-parse — the giant-session restore freeze. We
67
+ // SKIP that rebuild while the rows backfill (the next tick's cheap append succeeds), but bound the
68
+ // deferral with a time window so a stuck/disabled backfill still eventually imports the tail.
69
+ const _fullRebuildDeferSince = new Map(); // sessionId → Date.now() of the FIRST deferred tick
70
+ function _fullRebuildDeferMaxMs() {
71
+ const raw = Number(process.env.CTM_IMPORT_FULL_REBUILD_DEFER_MAX_MS);
72
+ return Number.isFinite(raw) && raw >= 0 ? raw : 120000; // 2 min: covers a normal backfill drain
73
+ }
74
+ function _noteFullRebuildDefer(id, now = Date.now()) { if (id && !_fullRebuildDeferSince.has(id)) _fullRebuildDeferSince.set(id, now); }
75
+ function _clearFullRebuildDefer(id) { if (id) _fullRebuildDeferSince.delete(id); }
76
+ function _fullRebuildDeferExpired(id, now = Date.now()) {
77
+ const t = _fullRebuildDeferSince.get(id);
78
+ return t != null && (now - t) >= _fullRebuildDeferMaxMs();
79
+ }
80
+
64
81
  // Pure decision: the debounce floor for a hot session's small-growth (append-eligible) re-import.
65
82
  // Extracted so the rebuild-thrash guard is unit-testable without the import machinery.
66
83
  function _conversationImportFloorMs({ eventDriven, baseFloorMs, eventFloorMs, lastRebuildAt, now }) {
@@ -1595,6 +1612,62 @@ async function _importCompactPair(parsed, jsonlPath, bakPath, jsonlSize, bakSize
1595
1612
  // Skip re-import if total size unchanged AND model already populated
1596
1613
  if (existing && existing.file_size === totalSize && existing.model_provider) return false;
1597
1614
 
1615
+ // INCREMENTAL FAST PATH (giant compacted-session import freeze): the .bak is byte-stable between
1616
+ // compactions, so once the pair is imported a later .jsonl-only growth is a pure append. Re-reading
1617
+ // + re-parsing the (100MB+) .bak and re-stitching the whole 16k-msg conversation every tick is the
1618
+ // freeze (sliceMs 12-17s, coop-ceiling breach, write-lock starvation on restart). When the prior
1619
+ // stitch recorded the SAME .bak size (proving the bak is unchanged ⇒ the prior .jsonl offset is
1620
+ // exact) and the row base is complete, append ONLY the new .jsonl tail [priorJsonlSize, EOF) and
1621
+ // never touch the .bak. Any mismatch (a re-compaction changed the bak, rows not ready, or the append
1622
+ // guard refuses on a prefix/gap) falls through to the safe full re-stitch below.
1623
+ if (existing && coopSchedulerEnabled()
1624
+ && Number(existing.compact_bak_size) > 0
1625
+ && Number(existing.compact_bak_size) === bakSize) {
1626
+ const priorJsonlSize = Number(existing.file_size) - bakSize; // bak unchanged ⇒ exact prior .jsonl size
1627
+ if (priorJsonlSize >= 0 && jsonlSize > priorJsonlSize) {
1628
+ if (typeof db.sessionRowsBackfillPending === 'function' && db.sessionRowsBackfillPending(parsed.sessionId)) {
1629
+ // Rows still backfilling → defer rather than fall through to the full re-stitch slab (time-boxed).
1630
+ _noteFullRebuildDefer(parsed.sessionId);
1631
+ if (!_fullRebuildDeferExpired(parsed.sessionId)) return false;
1632
+ // window expired (stuck/disabled backfill) → fall through to the full re-stitch so the tail still imports
1633
+ } else if (typeof db.sessionContentRowsAvailable === 'function' && db.sessionContentRowsAvailable(parsed.sessionId)) {
1634
+ let tail = '';
1635
+ let fullyRead = false;
1636
+ const fh = await fsp.open(jsonlPath, 'r');
1637
+ try {
1638
+ const len = jsonlSize - priorJsonlSize;
1639
+ const buf = Buffer.alloc(len);
1640
+ // fileHandle.read() may return fewer bytes than requested (the .jsonl is mid-write, or it
1641
+ // was truncated between the stat and this read). A short read would null-pad the tail and
1642
+ // could advance file_size past un-parsed bytes → a permanent message gap. So only take the
1643
+ // fast path on a FULL read; a short read falls through to the always-correct full re-stitch.
1644
+ const { bytesRead } = await fh.read(buf, 0, len, priorJsonlSize);
1645
+ if (bytesRead === len) { tail = buf.toString('utf8'); fullyRead = true; }
1646
+ } finally { await fh.close(); }
1647
+ const tailParsed = fullyRead ? await _parseConversationContent(tail, { fileSessionId: claudeFileSessionId(jsonlPath), fileDir: path.dirname(jsonlPath) }) : null;
1648
+ if (tailParsed && tailParsed.messages && tailParsed.messages.length > 0) {
1649
+ const appended = db.appendSessionConversation({
1650
+ session_id: parsed.sessionId,
1651
+ new_messages: tailParsed.messages,
1652
+ model_id: parsed.modelId || (existing && existing.model_id) || '',
1653
+ model_provider: parsed.modelProvider || (existing && existing.model_provider) || CLAUDE_MODEL_PROVIDER,
1654
+ file_size: totalSize,
1655
+ last_user_content: tailParsed.lastUserContent || '',
1656
+ project_path: parsed.cwd || parsed.project || '',
1657
+ git_branch: parsed.gitBranch || '',
1658
+ title: parsed.title || '',
1659
+ hostname: parsed.hostname || '',
1660
+ rename_name: tailParsed.renameName || '',
1661
+ import_parser_version: DEFAULT_CONVERSATION_IMPORT_PARSER_VERSION,
1662
+ });
1663
+ if (appended && appended.ok) { _clearFullRebuildDefer(parsed.sessionId); return true; }
1664
+ }
1665
+ // no new messages in the tail, or the append guard refused → fall through to the full re-stitch
1666
+ }
1667
+ }
1668
+ }
1669
+ _clearFullRebuildDefer(parsed.sessionId);
1670
+
1598
1671
  const [bakContent, jsonlContent] = await Promise.all([
1599
1672
  fsp.readFile(bakPath, 'utf8').catch(() => ''),
1600
1673
  fsp.readFile(jsonlPath, 'utf8').catch(() => ''),
@@ -1645,6 +1718,9 @@ async function _importCompactPair(parsed, jsonlPath, bakPath, jsonlSize, bakSize
1645
1718
  model_provider: parsed.modelProvider || (existing && existing.model_provider) || CLAUDE_MODEL_PROVIDER,
1646
1719
  model_id: parsed.modelId || (existing && existing.model_id) || '',
1647
1720
  import_parser_version: DEFAULT_CONVERSATION_IMPORT_PARSER_VERSION,
1721
+ // Record the .bak size so a later .jsonl-only growth takes the incremental fast-path above
1722
+ // (append the new tail) instead of re-stitching the whole pair.
1723
+ compact_bak_size: bakSize,
1648
1724
  // Chunked-import route (cold/large): defer rows to content-rows-backfill, keep the real blob.
1649
1725
  ..._chunkedImportFlags(parsed.sessionId, allMessages, totalSize, 'compact-pair'),
1650
1726
  });
@@ -1851,6 +1927,42 @@ function _readResetChunkedRouteStats() {
1851
1927
  return s;
1852
1928
  }
1853
1929
 
1930
+ // OBSERVE-ONLY Codex prompt-provenance telemetry (step 1 of docs/codex-prompt-provenance-spec.md).
1931
+ // Accumulates the per-import audit summary the Codex parser returns (composer coverage + kept prompts
1932
+ // that still look injected) so the eventual provenance flip is gated on real-corpus data, and new
1933
+ // injected-context leak shapes surface as a METRIC instead of waiting for someone to spot them in the
1934
+ // UI. Cumulative since boot; read via getCodexPromptAuditStats(). No behavior change.
1935
+ const _codexPromptAuditStats = {
1936
+ imports: 0, keptUserPrompts: 0, composerCovered: 0, injectedLookingKept: 0,
1937
+ sessionsWithUncovered: 0, sessionsWithInjectedLooking: 0,
1938
+ };
1939
+ function getCodexPromptAuditStats() {
1940
+ const s = _codexPromptAuditStats;
1941
+ const coverage = s.keptUserPrompts > 0 ? s.composerCovered / s.keptUserPrompts : null;
1942
+ return { ...s, composerCoverage: coverage };
1943
+ }
1944
+ let _lastCodexAuditLogAt = 0;
1945
+ function _recordCodexPromptAudit(sessionId, audit) {
1946
+ if (!audit) return;
1947
+ const kept = Number(audit.keptUserPrompts) || 0;
1948
+ const covered = Number(audit.composerCovered) || 0;
1949
+ const injected = Number(audit.injectedLookingKept) || 0;
1950
+ if (kept === 0 && injected === 0) return; // tail had no user prompts (incremental import) — skip noise
1951
+ const s = _codexPromptAuditStats;
1952
+ s.imports += 1; s.keptUserPrompts += kept; s.composerCovered += covered; s.injectedLookingKept += injected;
1953
+ if (kept > covered) s.sessionsWithUncovered += 1;
1954
+ if (injected > 0) s.sessionsWithInjectedLooking += 1;
1955
+ // Throttled, gated log: always on for a leaked-looking prompt (the actionable signal); otherwise
1956
+ // at most once/30s. CTM_CODEX_PROMPT_AUDIT=0 silences entirely.
1957
+ if (process.env.CTM_CODEX_PROMPT_AUDIT === '0') return;
1958
+ const now = Date.now();
1959
+ if (injected > 0 || now - _lastCodexAuditLogAt > 30000) {
1960
+ _lastCodexAuditLogAt = now;
1961
+ const cov = kept > 0 ? (100 * covered / kept).toFixed(0) + '%' : 'n/a';
1962
+ console.log(`[codex-prompt-audit] sid=${String(sessionId || '').slice(0, 8)} kept=${kept} composerCovered=${covered} (${cov}) injectedLooking=${injected}`);
1963
+ }
1964
+ }
1965
+
1854
1966
  function _chunkedImportFlags(sessionId, messages, fileSize, reason) {
1855
1967
  // Gated on the coop scheduler: deferring rows only helps when the cooperative scheduler is driving
1856
1968
  // bounded slices (and the content-rows-backfill coop/legacy job fills the deferred rows). The feature
@@ -1960,6 +2072,10 @@ async function _importCodexSessionFile(parsed, filePath, options = {}) {
1960
2072
 
1961
2073
  const importedFileSize = _codexImportedFileSize(parsed.fileSize, prevFileSize, parsedTail);
1962
2074
 
2075
+ // Observe-only prompt-provenance telemetry (no behavior change). parsedTail.audit summarizes the
2076
+ // newly-parsed tail (incremental import); cumulative stats gate the eventual provenance flip.
2077
+ try { _recordCodexPromptAudit(sessionId, parsedTail && parsedTail.audit); } catch { /* telemetry must never break import */ }
2078
+
1963
2079
  const fileMeta = readCodexRolloutMetadata(filePath) || {};
1964
2080
  const meta = parsedTail.sessionMeta || fileMeta || {};
1965
2081
  const firstUser = userMessages[0]?.text || userMessages[0]?.content || '';
@@ -1986,6 +2102,10 @@ async function _importCodexSessionFile(parsed, filePath, options = {}) {
1986
2102
  rename_name: existing?.rename_name || '',
1987
2103
  git_branch: meta.git_branch || parsed.gitBranch || '',
1988
2104
  file_size: importedFileSize,
2105
+ // file_size above is the consumed-bytes resume HWM (drives cacheBehind); parse_size_bytes is the
2106
+ // TRUE on-disk size used only for the over-cap decision, so a multi-GB rollout whose importedFileSize
2107
+ // under-reports can't slip under the parse cap and throw at JSON.stringify.
2108
+ parse_size_bytes: parsed.fileSize,
1989
2109
  session_created_at: meta.timestamp || parsed.timestamp || '',
1990
2110
  hostname: parsed.hostname,
1991
2111
  model_provider: modelProvider,
@@ -2063,6 +2183,30 @@ const _ingestCooldown = _ingestCooldownBaseMs > 0
2063
2183
  })
2064
2184
  : null;
2065
2185
 
2186
+ // Per-file FAILURE backoff for the whole-conversation importer (runIncrementalConversationImport):
2187
+ // a file whose importSessionFile() THROWS (e.g. a transiently unreadable file, or — historically — an
2188
+ // over-cap rollout that tripped a >512MB stringify) must not be retried every scan tick. A throw earns
2189
+ // an exponential cooldown; a clean import (no throw) resets it. Same module/pattern as _ingestCooldown
2190
+ // above, but a SEPARATE instance — the two track different concerns on possibly the same path and must
2191
+ // not cross-suppress. Disable via CTM_CONVERSATION_IMPORT_FAILURE_COOLDOWN_MS=0.
2192
+ const _importFailureCooldownBaseMs = Number(process.env.CTM_CONVERSATION_IMPORT_FAILURE_COOLDOWN_MS ?? 30000);
2193
+ const _importFailureCooldown = _importFailureCooldownBaseMs > 0
2194
+ ? createIngestCooldown({
2195
+ baseMs: _importFailureCooldownBaseMs,
2196
+ maxMs: Number(process.env.CTM_CONVERSATION_IMPORT_FAILURE_COOLDOWN_MAX_MS) || undefined,
2197
+ })
2198
+ : null;
2199
+ // Map import outcomes onto the cooldown module's {inserted,bytesRead} contract: a throw is a strike
2200
+ // (bytesRead>0, inserted=0 → exponential cooldown); a clean run resets (inserted>0).
2201
+ function _markImportFailure(filePath) {
2202
+ if (!_importFailureCooldown || !filePath) return null;
2203
+ return _importFailureCooldown.recordResult(filePath, { inserted: 0, bytesRead: 1 });
2204
+ }
2205
+ function _markImportSuccess(filePath) {
2206
+ if (!_importFailureCooldown || !filePath) return;
2207
+ _importFailureCooldown.recordResult(filePath, { inserted: 1 });
2208
+ }
2209
+
2066
2210
  function _ingestTranscriptStoreForParsedFile(filePath, parsed, options = {}) {
2067
2211
  // Claude Desktop sessions use a virtual path (`…#ctm-claude-desktop=<uuid>`) that is never a
2068
2212
  // real file on disk — fs.statSync / ingestJsonlFile always throw ENOENT on it. Their
@@ -2279,8 +2423,27 @@ async function importSessionFile(filePath, projectPath, projectEntry, options =
2279
2423
  // Cheap O(Δ) append succeeded — clear any rebuild backoff so event-driven mode
2280
2424
  // tracks this session's live tail at the fast cadence.
2281
2425
  _lastConversationFullRebuildAt.delete(parsed.sessionId);
2426
+ _clearFullRebuildDefer(parsed.sessionId);
2282
2427
  return true;
2283
2428
  }
2429
+ // RESTART-freeze guard: the append refused ONLY because this chunked-route session's deferred
2430
+ // rows are still backfilling. Do NOT fall through to the multi-second full-rebuild blob re-parse
2431
+ // (the giant-session restore freeze that holds the write lock and starves other writers). Skip
2432
+ // this tick — content-rows-backfill completes the rows, then the next tick's O(Δ) append
2433
+ // succeeds. The existing blob keeps serving reads; the tail lands a tick later. A time-boxed
2434
+ // escape (_fullRebuildDeferExpired) ensures a stuck/disabled backfill still imports the tail.
2435
+ if (appended && appended.reason === 'rows-not-appendable'
2436
+ && coopSchedulerEnabled()
2437
+ && typeof db.sessionRowsBackfillPending === 'function'
2438
+ && db.sessionRowsBackfillPending(parsed.sessionId)) {
2439
+ _noteFullRebuildDefer(parsed.sessionId); // stamp the first deferred tick, then measure the window from it
2440
+ if (!_fullRebuildDeferExpired(parsed.sessionId)) {
2441
+ return false; // within the defer window → skip the full-rebuild slab; re-picked next tick (file_size
2442
+ // unchanged → cacheBehind), debounced by the import floor, after backfill advances
2443
+ }
2444
+ // defer window expired (stuck/disabled backfill) → fall through to the full rebuild so the tail still imports
2445
+ }
2446
+ _clearFullRebuildDefer(parsed.sessionId);
2284
2447
  }
2285
2448
 
2286
2449
  // Full rebuild: cold import, file shrank/rotated, or the append guard refused. Load the base
@@ -2408,6 +2571,10 @@ async function runIncrementalConversationImport(opts = {}) {
2408
2571
  // synchronous SQLite upsert/JSON.stringify at the end, so do not stack
2409
2572
  // several imports in the same event-loop turn.
2410
2573
  for (const { filePath, projectPath, projectEntry, sessionId: candidateSessionId, error } of candidates) {
2574
+ // A file that threw recently is cooling down — skip it WITHOUT counting toward scanned/limits
2575
+ // (so it doesn't starve healthy files) and WITHOUT stamping the per-session debounce. It will be
2576
+ // re-attempted once its time-windowed cooldown elapses (candidate selection re-runs each tick).
2577
+ if (_importFailureCooldown && filePath && _importFailureCooldown.shouldSkip(filePath)) continue;
2411
2578
  try {
2412
2579
  if (error) throw error;
2413
2580
  scanned++;
@@ -2415,6 +2582,8 @@ async function runIncrementalConversationImport(opts = {}) {
2415
2582
  cooperative: true,
2416
2583
  transcriptMaxBytes: _backgroundTranscriptImportMaxBytes(),
2417
2584
  })) imported++;
2585
+ // A clean run (no throw) clears any prior failure strike for this file.
2586
+ _markImportSuccess(filePath);
2418
2587
  // Stamp the debounce clock for this session (whether or not a row was written) so a hot
2419
2588
  // session's rapid small appends coalesce instead of re-importing the whole conversation
2420
2589
  // every tick. Best-effort — keyed by the candidate's resolved session id.
@@ -2437,6 +2606,11 @@ async function runIncrementalConversationImport(opts = {}) {
2437
2606
  // Log individual failures so they're visible (previously swallowed silently).
2438
2607
  // One bad file shouldn't stall the whole job — we continue with others.
2439
2608
  console.error(`[auto-import] Failed to import ${filePath.split('/').pop()}: ${e.message}`);
2609
+ // Earn an exponential cooldown so a file that throws every time stops being retried each tick.
2610
+ const _cd = _markImportFailure(filePath);
2611
+ if (_cd && _cd.cooling && _cd.strikes === 1) {
2612
+ console.warn(`[auto-import] cooling down ${filePath.split('/').pop()} after import failure (cooldown=${_cd.cooldownMs}ms)`);
2613
+ }
2440
2614
  }
2441
2615
  }
2442
2616
 
@@ -2468,6 +2642,78 @@ async function runIncrementalConversationImport(opts = {}) {
2468
2642
  }
2469
2643
  }
2470
2644
 
2645
+ async function importConversationSources(opts = {}) {
2646
+ const requested = Array.isArray(opts.sourceIds) ? opts.sourceIds : [opts.sourceId || opts.sessionId];
2647
+ const sourceIds = [...new Set(requested.map(v => String(v || '').trim()).filter(Boolean))];
2648
+ const maxSources = Math.max(1, Math.min(20, Number(opts.maxSources) || sourceIds.length || 1));
2649
+ const results = [];
2650
+ const seenAgents = new Set();
2651
+ let scanned = 0;
2652
+ let imported = 0;
2653
+ let failed = 0;
2654
+ let missing = 0;
2655
+
2656
+ for (const sourceId of sourceIds.slice(0, maxSources)) {
2657
+ let identity = null;
2658
+ try { identity = db.getSessionIdentity(sourceId); } catch {}
2659
+ let rows = Array.isArray(identity?.agents) ? identity.agents : [];
2660
+ if (identity?.requested_is_agent) {
2661
+ rows = rows.filter(r => r && r.agent_session_id === sourceId);
2662
+ }
2663
+ if (rows.length === 0) {
2664
+ missing++;
2665
+ results.push({ sourceId, imported: false, reason: 'no-agent-session' });
2666
+ continue;
2667
+ }
2668
+
2669
+ for (const row of rows) {
2670
+ const agentId = String(row?.agent_session_id || '').trim();
2671
+ if (!agentId || seenAgents.has(agentId)) continue;
2672
+ seenAgents.add(agentId);
2673
+
2674
+ const configuredPath = String(row?.jsonl_path || '').trim();
2675
+ if (!configuredPath) {
2676
+ missing++;
2677
+ results.push({ sourceId, agentId, imported: false, reason: 'no-jsonl-path' });
2678
+ continue;
2679
+ }
2680
+
2681
+ let importPath = configuredPath;
2682
+ try {
2683
+ if (!fs.existsSync(importPath) && fs.existsSync(`${configuredPath}.bak`)) {
2684
+ importPath = `${configuredPath}.bak`;
2685
+ }
2686
+ if (!fs.existsSync(importPath)) {
2687
+ missing++;
2688
+ results.push({ sourceId, agentId, imported: false, reason: 'missing-file' });
2689
+ continue;
2690
+ }
2691
+ } catch (e) {
2692
+ failed++;
2693
+ results.push({ sourceId, agentId, imported: false, error: e.message });
2694
+ continue;
2695
+ }
2696
+
2697
+ const projectPath = row.project_path || identity?.ctm?.project_path || path.dirname(configuredPath);
2698
+ const projectEntry = path.basename(path.dirname(importPath.replace(/\.bak$/, '')));
2699
+ try {
2700
+ scanned++;
2701
+ const ok = await importSessionFile(importPath, projectPath, projectEntry, {
2702
+ cooperative: true,
2703
+ transcriptMaxBytes: _backgroundTranscriptImportMaxBytes(),
2704
+ });
2705
+ if (ok) imported++;
2706
+ results.push({ sourceId, agentId, imported: !!ok, file: path.basename(importPath) });
2707
+ } catch (e) {
2708
+ failed++;
2709
+ results.push({ sourceId, agentId, imported: false, error: e.message });
2710
+ }
2711
+ }
2712
+ }
2713
+
2714
+ return { imported, scanned, failed, missing, sourceIds: sourceIds.slice(0, maxSources), results };
2715
+ }
2716
+
2471
2717
  // --- Cursor conversation import ----------------------------------------------
2472
2718
  // Cursor stores conversations as a content-addressed blob graph in its own SQLite
2473
2719
  // store (~/.cursor/chats/<ws>/<agent>/store.db), NOT a JSONL append log. Reconstructing
@@ -2572,6 +2818,165 @@ async function runCursorConversationImport({ cursorHome } = {}) {
2572
2818
  }
2573
2819
  }
2574
2820
 
2821
+ const OPENCODE_IMPORT_PARSER_VERSION = 1; // bump to force a full opencode re-import on shape change
2822
+ const _opencodeImportSignatures = new Map(); // ctm_session_id → last-imported per-session signature
2823
+ const _opencodeDbStatSignature = { value: '' }; // whole-DB freshness gate (skip the sweep when idle)
2824
+
2825
+ // Resolve which OpenCode session (ses_…) belongs to a CTM session. OpenCode never surfaces its id
2826
+ // to CTM, so: (1) prefer an already-pinned agent_sessions link (confident exact id), else (2) match
2827
+ // opencode.db `session.directory` to the CTM cwd, nearest time_created ≥ created_at, and PIN that id
2828
+ // into agent_sessions so every later import is an exact-id read. Mirrors cursor's cwd+time fallback.
2829
+ function _resolveOpencodeSessionId(opencodeStore, dbPath, cwd, createdAt, agentSessionIds) {
2830
+ for (const id of agentSessionIds) {
2831
+ if (String(id || '').startsWith('ses_')) return { sessionId: id, pinned: true };
2832
+ }
2833
+ const candidates = opencodeStore.listOpencodeSessionsForCwd(dbPath, cwd);
2834
+ if (!candidates.length) return null;
2835
+ const targetMs = Date.parse(createdAt) || 0;
2836
+ let best = candidates[0]; // newest-first; default to most recent
2837
+ if (targetMs) {
2838
+ const atOrAfter = candidates
2839
+ .filter(c => Number(c.time_created) >= targetMs - 60000)
2840
+ .sort((a, b) => Number(a.time_created) - Number(b.time_created));
2841
+ if (atOrAfter.length) best = atOrAfter[0];
2842
+ }
2843
+ return { sessionId: String(best.id), pinned: false };
2844
+ }
2845
+
2846
+ // Eagerly pin the OpenCode session id (ses_…) for ONE CTM session, synchronously, at restore/resume
2847
+ // time. The full conversation sweep (runOpencodeConversationImport) runs async and may not have pinned
2848
+ // the id yet when a tab is restored — without a pinned ses_ row, restart-restore has nothing to resume
2849
+ // (orderedAgentRows requires a non-empty agent_session_id) and would relaunch the fresh `opencode` splash.
2850
+ // This factors out the importer's resolve+pin step (no message import) so the agent_sessions link exists
2851
+ // before selectRestoreAgentSession runs. Best-effort: any failure (db missing/locked) falls through and
2852
+ // resume degrades to `opencode --continue`. Returns { sessionId, pinned } or null when nothing resolves.
2853
+ function ensureOpencodeSessionPinned(ctmSessionId, { opencodeHome } = {}) {
2854
+ const ctmId = String(ctmSessionId || '').trim();
2855
+ if (!ctmId) return null;
2856
+ let opencodeStore;
2857
+ try {
2858
+ opencodeStore = require('./lib/opencode-conversation-store');
2859
+ } catch {
2860
+ return null;
2861
+ }
2862
+ try {
2863
+ const s = db.getDb().prepare(
2864
+ "SELECT cwd, project_path, created_at FROM ctm_sessions WHERE id = ? AND provider = 'opencode'"
2865
+ ).get(ctmId);
2866
+ if (!s) return null;
2867
+ const cwd = String(s.cwd || s.project_path || '').trim();
2868
+ if (!cwd) return null;
2869
+ const agentRows = db.getDb().prepare(
2870
+ "SELECT agent_session_id FROM agent_sessions WHERE ctm_session_id = ? AND agent_session_id IS NOT NULL AND agent_session_id != ''"
2871
+ ).all(ctmId);
2872
+ const agentSessionIds = agentRows.map(r => String(r.agent_session_id)).filter(Boolean);
2873
+
2874
+ const dbPath = opencodeStore.OPENCODE_DB_PATH(opencodeHome);
2875
+ const resolved = _resolveOpencodeSessionId(opencodeStore, dbPath, cwd, s.created_at, agentSessionIds);
2876
+ if (!resolved) return null;
2877
+ if (resolved.pinned) return resolved; // a ses_… is already linked; nothing to do
2878
+ try {
2879
+ const store = opencodeStore.loadOpencodeSession(dbPath, resolved.sessionId);
2880
+ db.upsertAgentSessionIdentity(resolved.sessionId, {
2881
+ ctmSessionId: ctmId,
2882
+ provider: 'opencode',
2883
+ projectPath: (store && store.directory) || cwd,
2884
+ model: (store && store.model_id) || '',
2885
+ });
2886
+ } catch { /* link is best-effort; cwd+time still resolves on the next pass */ }
2887
+ return resolved;
2888
+ } catch {
2889
+ return null;
2890
+ }
2891
+ }
2892
+
2893
+ // Import OpenCode conversations into session_conversations. OpenCode keeps every session in one
2894
+ // SQLite DB (~/.local/share/opencode/opencode.db) rather than per-session JSONL, so — like cursor —
2895
+ // it uses this dedicated DB-reader path, not the file-stat importer. Runs on the db-owner worker.
2896
+ async function runOpencodeConversationImport({ opencodeHome } = {}) {
2897
+ let imported = 0, scanned = 0, skipped = 0, failed = 0, total = 0;
2898
+ let opencodeStore;
2899
+ try {
2900
+ opencodeStore = require('./lib/opencode-conversation-store');
2901
+ } catch (e) {
2902
+ return { imported, scanned, skipped, failed, total, error: `opencode store unavailable: ${e.message}` };
2903
+ }
2904
+ const dbPath = opencodeStore.OPENCODE_DB_PATH(opencodeHome);
2905
+ // Whole-DB freshness gate: nothing in opencode.db changed since the last sweep → skip entirely.
2906
+ const dbSig = opencodeStore.statSignature(dbPath);
2907
+ if (dbSig && dbSig === _opencodeDbStatSignature.value) {
2908
+ return { imported, scanned, skipped, failed, total, skippedAll: true };
2909
+ }
2910
+ try {
2911
+ const sessions = db.getDb().prepare(
2912
+ "SELECT id, cwd, project_path, created_at, updated_at FROM ctm_sessions WHERE provider = 'opencode'"
2913
+ ).all();
2914
+ total = sessions.length;
2915
+ for (const s of sessions) {
2916
+ try {
2917
+ scanned += 1;
2918
+ const ctmId = String(s.id);
2919
+ const cwd = String(s.cwd || s.project_path || '').trim();
2920
+ if (!cwd) continue;
2921
+ const agentRows = db.getDb().prepare(
2922
+ "SELECT agent_session_id FROM agent_sessions WHERE ctm_session_id = ? AND agent_session_id IS NOT NULL AND agent_session_id != ''"
2923
+ ).all(ctmId);
2924
+ const agentSessionIds = agentRows.map(r => String(r.agent_session_id)).filter(Boolean);
2925
+
2926
+ const resolved = _resolveOpencodeSessionId(opencodeStore, dbPath, cwd, s.created_at, agentSessionIds);
2927
+ if (!resolved) continue;
2928
+ const ocId = resolved.sessionId;
2929
+
2930
+ // Cheap per-session signature gate — skip when this session is unchanged since last import.
2931
+ const sig = opencodeStore.sessionSignature(dbPath, ocId);
2932
+ if (sig && _opencodeImportSignatures.get(ctmId) === sig) { skipped += 1; continue; }
2933
+
2934
+ const store = opencodeStore.loadOpencodeSession(dbPath, ocId);
2935
+ if (!store || !Array.isArray(store.messages) || store.messages.length === 0) continue;
2936
+ const fields = opencodeStore.opencodeSessionToConversationFields(store);
2937
+ if (fields.user_msg_count === 0) continue;
2938
+
2939
+ // Pin the resolved id so future imports are confident exact-id reads (user-selected capture).
2940
+ if (!resolved.pinned) {
2941
+ try {
2942
+ db.upsertAgentSessionIdentity(ocId, {
2943
+ ctmSessionId: ctmId, provider: 'opencode', projectPath: store.directory || cwd,
2944
+ model: store.model_id || '', firstMessage: fields.first_message || '',
2945
+ });
2946
+ } catch { /* link is best-effort; cwd+time still resolves next pass */ }
2947
+ }
2948
+
2949
+ db.importSessionConversation({
2950
+ session_id: ctmId,
2951
+ ...fields,
2952
+ title: store.title || '',
2953
+ git_branch: '',
2954
+ file_size: 0, // opencode has no single jsonl footprint; freshness is gated by `sig`
2955
+ session_created_at: fields.session_created_at || s.created_at || '',
2956
+ hostname: '',
2957
+ import_parser_version: OPENCODE_IMPORT_PARSER_VERSION,
2958
+ rename_name: '',
2959
+ ..._chunkedImportFlags(ctmId, fields.messages, 0, 'opencode'),
2960
+ });
2961
+ if (sig) _opencodeImportSignatures.set(ctmId, sig);
2962
+ imported += 1;
2963
+ await new Promise((resolve) => setImmediate(resolve));
2964
+ } catch (e) {
2965
+ failed += 1;
2966
+ console.error(`[opencode-import] ${String(s.id).slice(0, 8)}: ${e.message}`);
2967
+ }
2968
+ }
2969
+ if (imported || failed) {
2970
+ console.log(`[opencode-import] imported ${imported}, skipped ${skipped}, failed ${failed} (scanned ${scanned}/${total})`);
2971
+ }
2972
+ _opencodeDbStatSignature.value = dbSig;
2973
+ return { imported, scanned, skipped, failed, total };
2974
+ } catch (e) {
2975
+ console.error('[opencode-import] Top-level error:', e.message);
2976
+ return { imported, scanned, skipped, failed, total, error: e.message };
2977
+ }
2978
+ }
2979
+
2575
2980
  function handleGetConversation(req, res, url) {
2576
2981
  const sessionId = url.pathname.split('/').pop();
2577
2982
  const conv = db.getSessionConversation(sessionId);
@@ -5323,4 +5728,4 @@ function safeParse(json, fallback) {
5323
5728
  try { return JSON.parse(json); } catch { return fallback; }
5324
5729
  }
5325
5730
 
5326
- module.exports = { handlePromptApi, handleConvertDesktopSession, handleListDesktopForks, queueEngine, runIncrementalConversationImport, runCursorConversationImport, importSessionFile, _chunkedImportFlags, setUiPrefsBroadcaster, setPromptExecutionsOffThread, setDbMaintenanceRunner, setImageSaveRunner, ensureHotkeyDaemon, hotkeyEnsureAction, screenshotResponsibleContext, screenshotNodeCandidates, probeScreenshotNodeGranted, _ingestPathFromInput, _ingestSourceAllowed, _conversationImportCandidates, _ingestTranscriptStoreForParsedFile, _learnSignatureRules, _appendBlocklistExceptions, _finalizePermIntent, _denyHinted, _lastConversationImportAt, _conversationImportFloorMs, _lastConversationFullRebuildAt };
5731
+ module.exports = { handlePromptApi, handleConvertDesktopSession, handleListDesktopForks, queueEngine, runIncrementalConversationImport, importConversationSources, runCursorConversationImport, runOpencodeConversationImport, ensureOpencodeSessionPinned, importSessionFile, _chunkedImportFlags, setUiPrefsBroadcaster, setPromptExecutionsOffThread, setDbMaintenanceRunner, setImageSaveRunner, ensureHotkeyDaemon, hotkeyEnsureAction, screenshotResponsibleContext, screenshotNodeCandidates, probeScreenshotNodeGranted, _ingestPathFromInput, _ingestSourceAllowed, _conversationImportCandidates, _ingestTranscriptStoreForParsedFile, _learnSignatureRules, _appendBlocklistExceptions, _finalizePermIntent, _denyHinted, _lastConversationImportAt, _conversationImportFloorMs, _lastConversationFullRebuildAt, getCodexPromptAuditStats };