claude-code-session-manager 0.83.0 → 0.85.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/assets/{AgentLibrary-B-OM-4gk.js → AgentLibrary-Bkv-HcP1.js} +1 -1
  2. package/dist/assets/{DataModel-BC6ppcBr.js → DataModel-DRH-Ty20.js} +1 -1
  3. package/dist/assets/{History-D3Qw2BWn.js → History-CfRhT1Im.js} +2 -2
  4. package/dist/assets/{Hooks-OxBQ0nIc.js → Hooks-wEmh_U6c.js} +1 -1
  5. package/dist/assets/{HostBilko-BNFC6bNZ.js → HostBilko-D_t7Rbi7.js} +1 -1
  6. package/dist/assets/{Library-DAMyrUtY.js → Library-CpArQ-OJ.js} +1 -1
  7. package/dist/assets/{ListDetail-BwywrpvE.js → ListDetail-pjaKYs84.js} +1 -1
  8. package/dist/assets/{MarkdownEditor-HEwjN1bp.js → MarkdownEditor-Xc141kjj.js} +1 -1
  9. package/dist/assets/{McpServers-D1VZF7Ix.js → McpServers-ftqaV3kn.js} +1 -1
  10. package/dist/assets/{Memory-CgYJ_QqD.js → Memory-ChMWkNd0.js} +6 -6
  11. package/dist/assets/{Panel-dTNWC59Q.js → Panel-D9Kr40Ai.js} +1 -1
  12. package/dist/assets/{Permissions-qI_NKi1b.js → Permissions-DKoNVgzj.js} +1 -1
  13. package/dist/assets/{Plugins-DZ0_CHiF.js → Plugins-BtChISho.js} +2 -2
  14. package/dist/assets/{ProvenanceBadge-B2i_uxCu.js → ProvenanceBadge-DBA5EcYy.js} +1 -1
  15. package/dist/assets/{SaveBar-4tNMCjUd.js → SaveBar-I0_dWNTX.js} +1 -1
  16. package/dist/assets/{Scheduler-CUen18Gc.js → Scheduler-CbES7MC8.js} +7 -7
  17. package/dist/assets/{ScopeSwitcher-Co3tkLTT.js → ScopeSwitcher-5GTEveb2.js} +1 -1
  18. package/dist/assets/Settings-BX3FElXk.js +3 -0
  19. package/dist/assets/{SkillReferenceGraph-BKcHtMPc.js → SkillReferenceGraph-DNBFGrYE.js} +1 -1
  20. package/dist/assets/{Skills-DHb4INtF.js → Skills-DJB6-bBM.js} +2 -2
  21. package/dist/assets/SystemPrompt-BiDDrJUA.js +1 -0
  22. package/dist/assets/{TagLibrary-xLP5dJvh.js → TagLibrary-_Wrevtop.js} +1 -1
  23. package/dist/assets/{TiptapBody-T-ULTjK4.js → TiptapBody-OWWXdLRy.js} +1 -1
  24. package/dist/assets/{Toggle-B-5M2hGe.js → Toggle-B122N0HL.js} +1 -1
  25. package/dist/assets/{index-CYhdtisq.css → index-CDo9xBR9.css} +1 -1
  26. package/dist/assets/{index-CzAxC432.js → index-DhvuQL4C.js} +316 -314
  27. package/dist/assets/{settingsSchema-C_nsemcF.js → settingsSchema-sGoCTd7J.js} +1 -1
  28. package/dist/index.html +2 -2
  29. package/package.json +4 -1
  30. package/scripts/hooks/guard-destructive-git.cjs +514 -0
  31. package/scripts/hooks/guard-inline-implementation.cjs +219 -0
  32. package/scripts/hooks/guard-prd-writes.cjs +200 -0
  33. package/src/main/__tests__/epicMintTelemetryTap.test.cjs +64 -0
  34. package/src/main/__tests__/health-delegation-chain.test.cjs +2 -1
  35. package/src/main/__tests__/health-queue-dispatch.test.cjs +84 -0
  36. package/src/main/__tests__/health-usage-poller.test.cjs +97 -0
  37. package/src/main/__tests__/health-worktree-cap-blocked.test.cjs +65 -0
  38. package/src/main/__tests__/opsErrorLogTelemetryTap.test.cjs +149 -0
  39. package/src/main/__tests__/pollLoop-dispatch-on-failure.test.cjs +120 -0
  40. package/src/main/__tests__/promptSessionTranscript.test.cjs +0 -0
  41. package/src/main/__tests__/queue-starvation-dispatch-driver.test.cjs +143 -0
  42. package/src/main/__tests__/rateLimitPollerStreak.test.cjs +79 -0
  43. package/src/main/__tests__/scheduleJobTransitionsTelemetryTap.test.cjs +72 -0
  44. package/src/main/__tests__/scheduler-inplace-salvage.test.cjs +74 -0
  45. package/src/main/__tests__/scheduler-job-overrun.test.cjs +58 -0
  46. package/src/main/__tests__/scheduler-notify-originating-tab-transcript.test.cjs +1 -0
  47. package/src/main/__tests__/scheduler-periodic-reverify-guard.test.cjs +134 -2
  48. package/src/main/__tests__/scheduler-reap-dead-running-jobs.test.cjs +33 -0
  49. package/src/main/__tests__/scheduler-stuck-failed-escalation.test.cjs +136 -0
  50. package/src/main/__tests__/telemetryClient.test.cjs +242 -1
  51. package/src/main/__tests__/telemetryContract.test.cjs +930 -0
  52. package/src/main/crashDiagnostics.cjs +29 -1
  53. package/src/main/health.cjs +197 -3
  54. package/src/main/index.cjs +65 -6
  55. package/src/main/ipcSchemas.cjs +19 -2
  56. package/src/main/lib/__tests__/crashTelemetry.test.cjs +103 -0
  57. package/src/main/lib/__tests__/delegationReadiness.test.cjs +302 -4
  58. package/src/main/lib/__tests__/fixtures/scheduler-machine.json.corrupt-1789147548 +34 -0
  59. package/src/main/lib/__tests__/gitWorktree.test.cjs +413 -1
  60. package/src/main/lib/__tests__/jobWorktreeBootLive.test.cjs +71 -0
  61. package/src/main/lib/__tests__/queueStoreAtomicWrite.test.cjs +88 -0
  62. package/src/main/lib/__tests__/queueStoreMachineStateRecovery.test.cjs +123 -0
  63. package/src/main/lib/__tests__/reaperHelpers.test.cjs +58 -0
  64. package/src/main/lib/__tests__/telemetryBacklog.test.cjs +626 -0
  65. package/src/main/lib/__tests__/telemetryBoot.test.cjs +123 -0
  66. package/src/main/lib/__tests__/telemetryConsent.test.cjs +136 -0
  67. package/src/main/lib/__tests__/telemetryCounters.test.cjs +57 -0
  68. package/src/main/lib/__tests__/telemetryCountersMetadataColumn.test.cjs +95 -0
  69. package/src/main/lib/crashTelemetry.cjs +37 -0
  70. package/src/main/lib/delegationReadiness.cjs +119 -1
  71. package/src/main/lib/epicMint.cjs +2 -0
  72. package/src/main/lib/gitWorktree.cjs +427 -17
  73. package/src/main/lib/jobWorktree.cjs +2 -1
  74. package/src/main/lib/jobWorktreeBootLive.cjs +51 -0
  75. package/src/main/lib/jobWorktreeTerminalOrphanLive.cjs +68 -0
  76. package/src/main/lib/opsErrorLog.cjs +78 -25
  77. package/src/main/lib/queueStore.cjs +241 -22
  78. package/src/main/lib/reaperHelpers.cjs +23 -1
  79. package/src/main/lib/scheduleJobSchema.cjs +7 -0
  80. package/src/main/lib/scheduleJobTransitions.cjs +12 -0
  81. package/src/main/lib/telemetryBacklog.cjs +601 -0
  82. package/src/main/lib/telemetryBoot.cjs +76 -0
  83. package/src/main/lib/telemetryClient.cjs +150 -5
  84. package/src/main/lib/telemetryConsent.cjs +34 -0
  85. package/src/main/lib/telemetryCounters.cjs +45 -0
  86. package/src/main/promptSessionTranscript.cjs +0 -0
  87. package/src/main/pty.cjs +2 -0
  88. package/src/main/scheduler.cjs +481 -33
  89. package/src/preload/api.d.ts +84 -4
  90. package/src/preload/index.cjs +9 -0
  91. package/dist/assets/Settings-DXVgKLUx.js +0 -3
  92. package/dist/assets/SystemPrompt-BjqFOHSk.js +0 -1
@@ -44,31 +44,7 @@ function todayFile(cwd) {
44
44
  return path.join(logsDir(cwd), `errors-${yyyy}-${mm}-${dd}.jsonl`);
45
45
  }
46
46
 
47
- /**
48
- * Append one structured error line, tagged for tracing/analysis.
49
- *
50
- * @param {{
51
- * cwd: string, // required — which project's ops root owns this line
52
- * scope: string, // subsystem name, e.g. 'pty', 'chatRunner', 'voice'
53
- * level?: string, // default 'error'; 'warn' also accepted
54
- * tabId?: string, // claudeSessionId of the tab this error belongs to, if any
55
- * epicId?: string, // Epic id, if this error happened inside an Epic-backed run
56
- * tags?: string[], // extra caller-supplied tags, merged with the auto-derived ones
57
- * message: string,
58
- * meta?: unknown,
59
- * }} entry
60
- */
61
- function appendError({ cwd, scope, level = 'error', tabId, epicId, tags = [], message, meta }) {
62
- if (!cwd || typeof cwd !== 'string') return; // no project to attribute this line to — skip
63
- if (isEphemeralCwd(cwd)) {
64
- // A worktree is torn down when its Epic/job ends and os.tmpdir() is
65
- // scratch space either way — never materialize logs there. See
66
- // ephemeralCwd.cjs / queueStore.cjs's projectStateDir for the sibling
67
- // refusal on the scheduler namespace (verified live 2026-09-01 as a
68
- // recreated /tmp/session-manager-operations/logs/ tree).
69
- console.warn(`[opsErrorLog] appendError: refusing ephemeral cwd "${cwd}" (scope=${scope || 'unknown'})`);
70
- return;
71
- }
47
+ function writeLocalLine({ cwd, scope, level, tabId, epicId, tags, message, meta }) {
72
48
  const file = todayFile(cwd);
73
49
  try {
74
50
  assertOpsWrite(file, 'logs');
@@ -102,6 +78,83 @@ function appendError({ cwd, scope, level = 'error', tabId, epicId, tags = [], me
102
78
  const fd = fs.openSync(file, 'a', 0o600);
103
79
  try { fs.writeSync(fd, JSON.stringify(line) + '\n'); } finally { fs.closeSync(fd); }
104
80
  } catch { /* best-effort — a logging failure must never break the caller */ }
81
+ return allTags;
82
+ }
83
+
84
+ /**
85
+ * Mirrors every appendError() line to telemetryClient — the single tap that
86
+ * covers pty/chatRunner/scheduler/logs.cjs/renderer in one place, since they
87
+ * all already funnel through appendError. Runs AFTER the local write and is
88
+ * independently wrapped so a throwing telemetry stub can never prevent or
89
+ * corrupt it. Reported regardless of the local ephemeral-cwd refusal — an
90
+ * Epic/job worktree still has a REAL project behind it, just not one that
91
+ * should ever have ops state materialized inside the worktree itself — so
92
+ * `cwd` is normalized through projectRootOf() before being handed to the
93
+ * client (which converts it to projectHash; the raw path never leaves this
94
+ * process).
95
+ *
96
+ * `message` is caller-supplied free text — it can legitimately contain a
97
+ * fragment of a chat/tool error that itself embeds a prompt, a file path, or
98
+ * other project-identifying content (opsErrorLog is a general-purpose error
99
+ * funnel, not a curated allowlist). It is therefore NEVER put on the wire's
100
+ * `msg`/`name` (which only get path-level redaction upstream) — it travels
101
+ * under a literal `message` key inside `context`/`fields`, which
102
+ * telemetryClient's redactDeep unconditionally rewrites to `[redacted]`
103
+ * (REDACT_KEY matches the key name, not its content). `msg`/`name` carry only
104
+ * `scope`, a small closed vocabulary of subsystem names — structural, never
105
+ * free text.
106
+ */
107
+ function reportToTelemetry({ cwd, scope, level, tabId, epicId, tags, message }) {
108
+ try {
109
+ const telemetryClient = require('./telemetryClient.cjs');
110
+ const { projectRootOf } = require('../../../scripts/lib/activeSessions.cjs');
111
+ const normalizedCwd = projectRootOf(cwd) || cwd;
112
+ const autoTags = [
113
+ `scope:${scope || 'unknown'}`,
114
+ ...(tabId ? [`tab:${tabId}`] : []),
115
+ ...(epicId ? [`epic:${epicId}`] : []),
116
+ ];
117
+ const allTags = Array.from(new Set([...autoTags, ...(tags || [])]));
118
+ const label = `opsError:${scope || 'unknown'}`;
119
+ const context = { scope: scope || 'unknown', tags: allTags, cwd: normalizedCwd, message };
120
+ if (level === 'warn') {
121
+ telemetryClient.logLine({ level: 'warn', msg: label, fields: context });
122
+ } else {
123
+ telemetryClient.reportError({ name: scope || 'opsError', msg: label, context });
124
+ }
125
+ } catch { /* telemetry must never break the caller — this is the sole error funnel */ }
126
+ }
127
+
128
+ /**
129
+ * Append one structured error line, tagged for tracing/analysis.
130
+ *
131
+ * @param {{
132
+ * cwd: string, // required — which project's ops root owns this line
133
+ * scope: string, // subsystem name, e.g. 'pty', 'chatRunner', 'voice'
134
+ * level?: string, // default 'error'; 'warn' also accepted
135
+ * tabId?: string, // claudeSessionId of the tab this error belongs to, if any
136
+ * epicId?: string, // Epic id, if this error happened inside an Epic-backed run
137
+ * tags?: string[], // extra caller-supplied tags, merged with the auto-derived ones
138
+ * message: string,
139
+ * meta?: unknown,
140
+ * }} entry
141
+ */
142
+ function appendError({ cwd, scope, level = 'error', tabId, epicId, tags = [], message, meta }) {
143
+ if (!cwd || typeof cwd !== 'string') return; // no project to attribute this line to — skip
144
+
145
+ if (isEphemeralCwd(cwd)) {
146
+ // A worktree is torn down when its Epic/job ends and os.tmpdir() is
147
+ // scratch space either way — never materialize logs there. See
148
+ // ephemeralCwd.cjs / queueStore.cjs's projectStateDir for the sibling
149
+ // refusal on the scheduler namespace (verified live 2026-09-01 as a
150
+ // recreated /tmp/session-manager-operations/logs/ tree). The telemetry
151
+ // tap below still fires — this refusal is about the LOCAL write only.
152
+ console.warn(`[opsErrorLog] appendError: refusing ephemeral cwd "${cwd}" (scope=${scope || 'unknown'})`);
153
+ } else {
154
+ writeLocalLine({ cwd, scope, level, tabId, epicId, tags, message, meta });
155
+ }
156
+
157
+ reportToTelemetry({ cwd, scope, level, tabId, epicId, tags, message });
105
158
  }
106
159
 
107
160
  module.exports = { appendError, logsDir, todayFile };
@@ -32,6 +32,7 @@ const fs = require('node:fs');
32
32
  const fsp = require('node:fs/promises');
33
33
  const path = require('node:path');
34
34
  const os = require('node:os');
35
+ const crypto = require('node:crypto');
35
36
  const { allProjectCwds, activeProjectCwds } = require('../../../scripts/lib/activeSessions.cjs');
36
37
  const { assertOpsWrite, resolveOpsRoot, OPS_ROOT_DIR } = require('./opsOwnership.cjs');
37
38
  const { ScheduleJobSchema } = require('./scheduleJobSchema.cjs');
@@ -128,20 +129,58 @@ function queuePathOrSkip(cwd, context) {
128
129
  }
129
130
  }
130
131
 
132
+ /**
133
+ * Unique tmp path PER CALL, not per process — matches the pid-ts-rand house
134
+ * style seen elsewhere in this tree (e.g. `admin-api.json.tmp-1065370-
135
+ * 1784409837234-vbz3fd`). A pid-only tmp name lets two overlapping writes in
136
+ * the SAME process share one tmp path: write A (long doc) is mid-flight when
137
+ * write B (short doc) opens the same tmp with O_TRUNC and completes first —
138
+ * A's still-buffered remainder then lands past B's EOF at its own advanced fd
139
+ * offset, and the rename publishes the interleaved bytes (rename is atomic;
140
+ * the shared tmp file never was). This is the exact shape of the
141
+ * `scheduler-machine.json.corrupt-*` tears observed 2026-09-07/10/11: a
142
+ * complete valid object followed by the orphaned tail of a longer one.
143
+ */
144
+ function uniqueTmpPath(file) {
145
+ return `${file}.tmp-${process.pid}-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
146
+ }
147
+
131
148
  function writeJsonAtomicSync(file, value) {
132
149
  assertOpsWrite(file, 'scheduler');
133
150
  fs.mkdirSync(path.dirname(file), { recursive: true });
134
- const tmp = `${file}.tmp-${process.pid}`;
135
- fs.writeFileSync(tmp, JSON.stringify(value, null, 2));
136
- fs.renameSync(tmp, file);
151
+ const tmp = uniqueTmpPath(file);
152
+ try {
153
+ const fd = fs.openSync(tmp, 'w');
154
+ try {
155
+ fs.writeSync(fd, JSON.stringify(value, null, 2));
156
+ fs.fsyncSync(fd);
157
+ } finally {
158
+ fs.closeSync(fd);
159
+ }
160
+ fs.renameSync(tmp, file);
161
+ } catch (e) {
162
+ try { fs.unlinkSync(tmp); } catch { /* tmp may not exist, or rename already consumed it */ }
163
+ throw e;
164
+ }
137
165
  }
138
166
 
139
167
  async function writeJsonAtomic(file, value) {
140
168
  assertOpsWrite(file, 'scheduler');
141
169
  await fsp.mkdir(path.dirname(file), { recursive: true });
142
- const tmp = `${file}.tmp-${process.pid}`;
143
- await fsp.writeFile(tmp, JSON.stringify(value, null, 2));
144
- await fsp.rename(tmp, file);
170
+ const tmp = uniqueTmpPath(file);
171
+ try {
172
+ const handle = await fsp.open(tmp, 'w');
173
+ try {
174
+ await handle.writeFile(JSON.stringify(value, null, 2));
175
+ await handle.sync();
176
+ } finally {
177
+ await handle.close();
178
+ }
179
+ await fsp.rename(tmp, file);
180
+ } catch (e) {
181
+ await fsp.unlink(tmp).catch(() => {}); // tmp may not exist, or rename already consumed it
182
+ throw e;
183
+ }
145
184
  }
146
185
 
147
186
  // ---------- project-cwd enumeration (cached) ----------
@@ -176,12 +215,16 @@ function bustCwdCache() {
176
215
 
177
216
  // ---------- merged read ----------
178
217
 
179
- function shapeMachine(raw) {
180
- const data = raw ? JSON.parse(raw) : {};
218
+ function shapeMachine(data) {
181
219
  return {
182
220
  config: data.config || {},
183
221
  scheduledFor: data.scheduledFor ?? null,
184
222
  lastRunAt: data.lastRunAt ?? null,
223
+ // Distinct from lastRunAt (stamped only when tickQueue actually launches a
224
+ // job): this is stamped every time tickQueue gets far enough to evaluate
225
+ // the queue at all, whether or not that evaluation ends in a launch. See
226
+ // classifyQueueStarvation's header for why the two must never merge.
227
+ lastDispatchAttemptAt: data.lastDispatchAttemptAt ?? null,
185
228
  paused: data.paused ?? null,
186
229
  // Launch circuit breaker (lib/launchFailure.cjs): per-persona blocks and
187
230
  // the degraded-mode env a persona is currently launching with. Machine
@@ -191,6 +234,174 @@ function shapeMachine(raw) {
191
234
  };
192
235
  }
193
236
 
237
+ // ---------- torn machine-state recovery ----------
238
+ //
239
+ // scheduler-machine.json has torn 4 times (Sep 7/10/11 `.corrupt-*`, plus a
240
+ // `.bak-*`) from the pid-only tmp-path bug fixed above. A pre-existing tear
241
+ // (from before this fix, or from any other future writer bug) must not keep
242
+ // poisoning every read with `unreadable` — readMergedSync/readMerged recover
243
+ // what they can and keep the engine dispatching instead.
244
+
245
+ /**
246
+ * findLongestValidJsonPrefix(raw) → { value, prefixLength } for the LONGEST
247
+ * leading substring of `raw` that is itself a complete, valid JSON document,
248
+ * or null if none exists. Scans char-by-char tracking object/array nesting
249
+ * depth (skipping over string contents and escapes so a brace inside a
250
+ * string value never miscounts) and attempts JSON.parse every time depth
251
+ * returns to zero. This is exactly the shape of the observed corruption: a
252
+ * complete, valid object followed by the orphaned tail of a longer one — the
253
+ * tail never balances back to depth 0, so it can never win over the real
254
+ * prefix. O(n) in the length of raw.
255
+ */
256
+ function findLongestValidJsonPrefix(raw) {
257
+ let depth = 0;
258
+ let inString = false;
259
+ let escaped = false;
260
+ let best = null;
261
+ for (let i = 0; i < raw.length; i++) {
262
+ const ch = raw[i];
263
+ if (inString) {
264
+ if (escaped) escaped = false;
265
+ else if (ch === '\\') escaped = true;
266
+ else if (ch === '"') inString = false;
267
+ continue;
268
+ }
269
+ if (ch === '"') { inString = true; continue; }
270
+ if (ch === '{' || ch === '[') { depth++; continue; }
271
+ if (ch === '}' || ch === ']') {
272
+ depth--;
273
+ if (depth === 0) {
274
+ try {
275
+ best = { value: JSON.parse(raw.slice(0, i + 1)), prefixLength: i + 1 };
276
+ } catch { /* balanced at this boundary but not valid JSON — keep scanning */ }
277
+ }
278
+ }
279
+ }
280
+ return best;
281
+ }
282
+
283
+ // Machine state (config/paused/lastRunAt/...) has no owning project cwd of
284
+ // its own — it's a Session-Manager runtime concern (see this file's header).
285
+ // opsErrorLog.appendError requires a cwd to attribute the line to; this
286
+ // project's own ops root is the natural home for a machine-level log entry,
287
+ // mirroring scheduler.cjs's own `job.cwd || DEFAULT_PROJECT_CWD` fallback for
288
+ // cwd-less machine errors (schedulerBatch.cjs's DEFAULT_PROJECT_CWD).
289
+ const MACHINE_STATE_LOG_CWD = path.join(os.homedir(), 'Projects', 'session-manager');
290
+
291
+ // The truncated tail in the real 2026-09-11 tear contained an orphaned
292
+ // `resumeAt` fragment with no way to reconstruct the `paused` object it
293
+ // belonged to (the fragment starts mid-string, missing its own `"paused":`
294
+ // key and opening brace). That data is genuinely unrecoverable — but silently
295
+ // dropping it is what makes recovery unsafe (an active pause could vanish).
296
+ // This can't rebuild the lost value, so instead it makes the loss loud: any
297
+ // discarded tail that still mentions paused/resumeAt gets called out in the
298
+ // log line so a human verifies pause status instead of trusting recovery blindly.
299
+ const PAUSED_HINT_RE = /"paused"|resumeAt/;
300
+
301
+ function logMachineStateRecovery({ level, message, meta }) {
302
+ try {
303
+ const { appendError } = require('./opsErrorLog.cjs');
304
+ appendError({ cwd: MACHINE_STATE_LOG_CWD, scope: 'scheduler', level, message, meta });
305
+ } catch { /* durable logging must never block recovery */ }
306
+ console.error(`[queueStore] ${message}`);
307
+ }
308
+
309
+ /**
310
+ * buildMachineStateRecovery(raw, parseError) → the recovered plain object
311
+ * (never shaped yet) plus what to log. Pure — callers persist it and log it.
312
+ */
313
+ function buildMachineStateRecovery(raw, parseError) {
314
+ const prefix = findLongestValidJsonPrefix(raw);
315
+ if (prefix) {
316
+ const trailing = raw.slice(prefix.prefixLength).trim();
317
+ const pausedHint = Boolean(trailing) && PAUSED_HINT_RE.test(trailing);
318
+ return {
319
+ value: prefix.value,
320
+ mode: 'prefix',
321
+ level: 'warn',
322
+ message:
323
+ `scheduler-machine.json was torn (${parseError?.message}) — recovered the longest valid `
324
+ + `JSON prefix (${prefix.prefixLength}/${raw.length} bytes)`
325
+ + (pausedHint
326
+ ? '; the discarded tail looks like it contained paused/resumeAt data that could not '
327
+ + 'be reconstructed — verify pause status manually'
328
+ : ''),
329
+ meta: { path: MACHINE_STATE_PATH, prefixLength: prefix.prefixLength, totalLength: raw.length, pausedHint },
330
+ };
331
+ }
332
+ // No valid JSON prefix at all: fall back to defaults (shapeMachine({}) —
333
+ // callers merge DEFAULT_CONFIG on top) rather than marking `unreadable`,
334
+ // which would halt tickQueue/runDueJobs machine-wide until a human notices.
335
+ return {
336
+ value: {},
337
+ mode: 'default',
338
+ level: 'error',
339
+ message:
340
+ `scheduler-machine.json unrecoverable (${parseError?.message}) — falling back to defaults `
341
+ + 'so dispatch does not silently halt',
342
+ meta: { path: MACHINE_STATE_PATH, totalLength: raw.length },
343
+ };
344
+ }
345
+
346
+ function recoverTornMachineStateSync(raw, parseError) {
347
+ const recovery = buildMachineStateRecovery(raw, parseError);
348
+ logMachineStateRecovery(recovery);
349
+ try {
350
+ writeJsonAtomicSync(MACHINE_STATE_PATH, recovery.value);
351
+ } catch (e) {
352
+ console.error(`[queueStore] failed to persist recovered machine state: ${e?.message}`);
353
+ }
354
+ return recovery;
355
+ }
356
+
357
+ async function recoverTornMachineState(raw, parseError) {
358
+ const recovery = buildMachineStateRecovery(raw, parseError);
359
+ logMachineStateRecovery(recovery);
360
+ try {
361
+ await writeJsonAtomic(MACHINE_STATE_PATH, recovery.value);
362
+ } catch (e) {
363
+ console.error(`[queueStore] failed to persist recovered machine state: ${e?.message}`);
364
+ }
365
+ return recovery;
366
+ }
367
+
368
+ /**
369
+ * loadMachineStateSync/loadMachineState → { shaped, recovered?, recoveryMode? }
370
+ * or { unreadable, unreadablePath } (ENOENT is neither — first-boot empty).
371
+ * A parse failure recovers instead of poisoning the whole merged read.
372
+ */
373
+ function loadMachineStateSync() {
374
+ let raw;
375
+ try {
376
+ raw = fs.readFileSync(MACHINE_STATE_PATH, 'utf8');
377
+ } catch (e) {
378
+ if (e?.code === 'ENOENT') return {};
379
+ return { unreadable: `machine state unreadable: ${e?.message}`, unreadablePath: MACHINE_STATE_PATH };
380
+ }
381
+ try {
382
+ return { shaped: shapeMachine(JSON.parse(raw)) };
383
+ } catch (parseErr) {
384
+ const recovery = recoverTornMachineStateSync(raw, parseErr);
385
+ return { shaped: shapeMachine(recovery.value), recovered: true, recoveryMode: recovery.mode };
386
+ }
387
+ }
388
+
389
+ async function loadMachineState() {
390
+ let raw;
391
+ try {
392
+ raw = await fsp.readFile(MACHINE_STATE_PATH, 'utf8');
393
+ } catch (e) {
394
+ if (e?.code === 'ENOENT') return {};
395
+ return { unreadable: `machine state unreadable: ${e?.message}`, unreadablePath: MACHINE_STATE_PATH };
396
+ }
397
+ try {
398
+ return { shaped: shapeMachine(JSON.parse(raw)) };
399
+ } catch (parseErr) {
400
+ const recovery = await recoverTornMachineState(raw, parseErr);
401
+ return { shaped: shapeMachine(recovery.value), recovered: true, recoveryMode: recovery.mode };
402
+ }
403
+ }
404
+
194
405
  /**
195
406
  * shapeJobs(raw, file) → { jobs, invalid }.
196
407
  *
@@ -234,15 +445,18 @@ function shapeJobs(raw, file) {
234
445
  * consulted so writeSplit can persist "this project now has zero jobs".
235
446
  */
236
447
  function readMergedSync(opts) {
237
- const out = { config: {}, jobs: [], scheduledFor: null, lastRunAt: null, paused: null, launchBlocks: {}, launchMitigations: {}, invalidJobs: [] };
448
+ const out = { config: {}, jobs: [], scheduledFor: null, lastRunAt: null, lastDispatchAttemptAt: null, paused: null, launchBlocks: {}, launchMitigations: {}, invalidJobs: [] };
238
449
  const sourceCwds = [];
239
- try {
240
- Object.assign(out, shapeMachine(fs.readFileSync(MACHINE_STATE_PATH, 'utf8')));
241
- } catch (e) {
242
- if (e?.code !== 'ENOENT') {
243
- out.unreadable = `machine state unreadable: ${e?.message}`;
244
- out.unreadablePath = MACHINE_STATE_PATH;
450
+ const machine = loadMachineStateSync();
451
+ if (machine.shaped) {
452
+ Object.assign(out, machine.shaped);
453
+ if (machine.recovered) {
454
+ out.machineStateRecovered = true;
455
+ out.machineStateRecoveryMode = machine.recoveryMode;
245
456
  }
457
+ } else if (machine.unreadable) {
458
+ out.unreadable = machine.unreadable;
459
+ out.unreadablePath = machine.unreadablePath;
246
460
  }
247
461
  for (const cwd of stateCwds(opts)) {
248
462
  const file = queuePathOrSkip(cwd, 'read');
@@ -264,15 +478,18 @@ function readMergedSync(opts) {
264
478
 
265
479
  /** Async twin of readMergedSync for IPC hot paths. */
266
480
  async function readMerged(opts) {
267
- const out = { config: {}, jobs: [], scheduledFor: null, lastRunAt: null, paused: null, launchBlocks: {}, launchMitigations: {}, invalidJobs: [] };
481
+ const out = { config: {}, jobs: [], scheduledFor: null, lastRunAt: null, lastDispatchAttemptAt: null, paused: null, launchBlocks: {}, launchMitigations: {}, invalidJobs: [] };
268
482
  const sourceCwds = [];
269
- try {
270
- Object.assign(out, shapeMachine(await fsp.readFile(MACHINE_STATE_PATH, 'utf8')));
271
- } catch (e) {
272
- if (e?.code !== 'ENOENT') {
273
- out.unreadable = `machine state unreadable: ${e?.message}`;
274
- out.unreadablePath = MACHINE_STATE_PATH;
483
+ const machine = await loadMachineState();
484
+ if (machine.shaped) {
485
+ Object.assign(out, machine.shaped);
486
+ if (machine.recovered) {
487
+ out.machineStateRecovered = true;
488
+ out.machineStateRecoveryMode = machine.recoveryMode;
275
489
  }
490
+ } else if (machine.unreadable) {
491
+ out.unreadable = machine.unreadable;
492
+ out.unreadablePath = machine.unreadablePath;
276
493
  }
277
494
  for (const cwd of stateCwds(opts)) {
278
495
  const file = queuePathOrSkip(cwd, 'read');
@@ -313,6 +530,7 @@ async function writeSplit(state, defaultCwd) {
313
530
  config: state.config,
314
531
  scheduledFor: state.scheduledFor ?? null,
315
532
  lastRunAt: state.lastRunAt ?? null,
533
+ lastDispatchAttemptAt: state.lastDispatchAttemptAt ?? null,
316
534
  paused: state.paused ?? null,
317
535
  launchBlocks: state.launchBlocks ?? {},
318
536
  launchMitigations: state.launchMitigations ?? {},
@@ -420,4 +638,5 @@ module.exports = {
420
638
  migrateLegacyGlobalQueue,
421
639
  writeJsonAtomic,
422
640
  writeJsonAtomicSync,
641
+ findLongestValidJsonPrefix,
423
642
  };
@@ -225,6 +225,27 @@ const ORPHAN_REQUEUE_CAP = 5;
225
225
  * `findLiveProcess` (existing callers/tests) preserves prior behaviour
226
226
  * exactly: every pidless row past grace reaps, none are ever recovered.
227
227
  */
228
+ /**
229
+ * Pure, never-throws formatter for the dispatch-phase breadcrumb appended to
230
+ * a pidless reap's reason string. Returns '' when the row carries no
231
+ * breadcrumb at all (an older-build row, or one that never reached the
232
+ * running-stamped mutate) so that case's message stays byte-identical to
233
+ * the pre-breadcrumb text — nothing downstream that matches on it breaks.
234
+ * A present `dispatchPhase` with a missing/unparseable `dispatchPhaseAt`
235
+ * still names the phase, just without the `at <ts>` clause — the reaper
236
+ * must stay a pure decision layer that cannot crash the tick over a
237
+ * malformed timestamp.
238
+ */
239
+ function formatDispatchPhaseSuffix(j) {
240
+ if (!j || typeof j.dispatchPhase !== 'string' || !j.dispatchPhase) return '';
241
+ const at = typeof j.dispatchPhaseAt === 'string' && !Number.isNaN(Date.parse(j.dispatchPhaseAt))
242
+ ? j.dispatchPhaseAt
243
+ : null;
244
+ return at
245
+ ? ` (last dispatch phase: ${j.dispatchPhase} at ${at})`
246
+ : ` (last dispatch phase: ${j.dispatchPhase})`;
247
+ }
248
+
228
249
  function selectReapableJobs(jobs, now, { pidAlive, grace, findLiveProcess } = {}) {
229
250
  const reapable = [];
230
251
  const warnings = [];
@@ -253,7 +274,7 @@ function selectReapableJobs(jobs, now, { pidAlive, grace, findLiveProcess } = {}
253
274
  slug: j.slug,
254
275
  pid: null,
255
276
  pidless: true,
256
- reason: `reaped: no runtime.pid recorded after ${Math.round(grace / 60_000)}m — spawn never completed`,
277
+ reason: `reaped: no runtime.pid recorded after ${Math.round(grace / 60_000)}m — spawn never completed${formatDispatchPhaseSuffix(j)}`,
257
278
  });
258
279
  }
259
280
  return { reapable, warnings, recovered };
@@ -324,4 +345,5 @@ module.exports = {
324
345
  resolvePidlessGateOutcome,
325
346
  isAlreadySatisfiedOnMain,
326
347
  resolveCommitGuardOutcome,
348
+ formatDispatchPhaseSuffix,
327
349
  };
@@ -117,6 +117,13 @@ const ScheduleJobSchema = z
117
117
  // finalizes — never meant to outlive one run.
118
118
  guardBaseline: z.array(z.string()).optional(),
119
119
  guardHeadBefore: z.string().nullable().optional(),
120
+ // Dispatch-phase breadcrumb: how far a 'running' row's own dispatch got
121
+ // through the running-transition → executeJob → onPid region, so a
122
+ // pidless reap can name the step that hung instead of just the symptom.
123
+ // Same lifecycle as heldReason — stamped at each step, deleted at
124
+ // finalize (both the normal completion path and reapDeadRunningJobs).
125
+ dispatchPhase: z.string().optional(),
126
+ dispatchPhaseAt: z.string().optional(),
120
127
  })
121
128
  .passthrough();
122
129
 
@@ -31,6 +31,13 @@
31
31
  'use strict';
32
32
 
33
33
  const { appendAuditEvent } = require('./auditLog.cjs');
34
+ const telemetryCounters = require('./telemetryCounters.cjs');
35
+
36
+ // A run genuinely finished when it leaves 'running' for one of these —
37
+ // distinct from every other legal edge in LEGAL_TRANSITIONS (retries,
38
+ // investigation probes, admin resets), which don't represent a completed
39
+ // execution attempt.
40
+ const FINISH_STATUSES = new Set(['completed', 'failed', 'skipped', 'needs_review']);
34
41
 
35
42
  // Bounded so queue.json (mutation cost, broadcast payload, pickNextBatch
36
43
  // scan) stays small — same rationale as lib/queueHistory.cjs's retention
@@ -183,6 +190,11 @@ function transitionJob(job, toStatus, { reason, source, allowAnyFrom = false } =
183
190
  source: source ?? null,
184
191
  cwd: job.cwd ?? null,
185
192
  });
193
+
194
+ if (from === 'running' && FINISH_STATUSES.has(toStatus)) {
195
+ telemetryCounters.trackSchedulerJobFinish({ status: toStatus });
196
+ }
197
+
186
198
  return true;
187
199
  }
188
200