agent-nuvira 3.5.1 → 3.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/dist/cli/chat.d.ts +42 -0
  2. package/dist/cli/chat.d.ts.map +1 -1
  3. package/dist/cli/chat.js +240 -16
  4. package/dist/cli/chat.js.map +1 -1
  5. package/dist/cli/edit.d.ts.map +1 -1
  6. package/dist/cli/edit.js +3 -2
  7. package/dist/cli/edit.js.map +1 -1
  8. package/dist/cli/execute.d.ts.map +1 -1
  9. package/dist/cli/execute.js +10 -3
  10. package/dist/cli/execute.js.map +1 -1
  11. package/dist/cli/loop-executor.d.ts +16 -0
  12. package/dist/cli/loop-executor.d.ts.map +1 -1
  13. package/dist/cli/loop-executor.js +9 -0
  14. package/dist/cli/loop-executor.js.map +1 -1
  15. package/dist/cli/plan.d.ts.map +1 -1
  16. package/dist/cli/plan.js +18 -6
  17. package/dist/cli/plan.js.map +1 -1
  18. package/dist/cli/run.d.ts.map +1 -1
  19. package/dist/cli/run.js +23 -8
  20. package/dist/cli/run.js.map +1 -1
  21. package/dist/cli/ui/answer-view.d.ts +38 -0
  22. package/dist/cli/ui/answer-view.d.ts.map +1 -0
  23. package/dist/cli/ui/answer-view.js +72 -0
  24. package/dist/cli/ui/answer-view.js.map +1 -0
  25. package/dist/cli/ui/cards.d.ts +61 -0
  26. package/dist/cli/ui/cards.d.ts.map +1 -0
  27. package/dist/cli/ui/cards.js +141 -0
  28. package/dist/cli/ui/cards.js.map +1 -0
  29. package/dist/cli/ui/highlight.d.ts +32 -0
  30. package/dist/cli/ui/highlight.d.ts.map +1 -0
  31. package/dist/cli/ui/highlight.js +261 -0
  32. package/dist/cli/ui/highlight.js.map +1 -0
  33. package/dist/cli/ui/index.d.ts +21 -0
  34. package/dist/cli/ui/index.d.ts.map +1 -0
  35. package/dist/cli/ui/index.js +14 -0
  36. package/dist/cli/ui/index.js.map +1 -0
  37. package/dist/cli/ui/markdown.d.ts +90 -0
  38. package/dist/cli/ui/markdown.d.ts.map +1 -0
  39. package/dist/cli/ui/markdown.js +292 -0
  40. package/dist/cli/ui/markdown.js.map +1 -0
  41. package/dist/cli/ui/presenter.d.ts +147 -0
  42. package/dist/cli/ui/presenter.d.ts.map +1 -0
  43. package/dist/cli/ui/presenter.js +313 -0
  44. package/dist/cli/ui/presenter.js.map +1 -0
  45. package/dist/cli/ui/render.d.ts +52 -0
  46. package/dist/cli/ui/render.d.ts.map +1 -0
  47. package/dist/cli/ui/render.js +384 -0
  48. package/dist/cli/ui/render.js.map +1 -0
  49. package/dist/cli/ui/theme.d.ts +36 -0
  50. package/dist/cli/ui/theme.d.ts.map +1 -0
  51. package/dist/cli/ui/theme.js +92 -0
  52. package/dist/cli/ui/theme.js.map +1 -0
  53. package/dist/inference/model-entitlement.d.ts +10 -0
  54. package/dist/inference/model-entitlement.d.ts.map +1 -1
  55. package/dist/inference/model-entitlement.js +1 -1
  56. package/dist/inference/model-entitlement.js.map +1 -1
  57. package/dist/learning/auto-router.d.ts +10 -1
  58. package/dist/learning/auto-router.d.ts.map +1 -1
  59. package/dist/learning/auto-router.js +63 -11
  60. package/dist/learning/auto-router.js.map +1 -1
  61. package/dist/learning/reasoning-trace.d.ts +21 -1
  62. package/dist/learning/reasoning-trace.d.ts.map +1 -1
  63. package/dist/learning/reasoning-trace.js.map +1 -1
  64. package/dist/learning/resilient-call.d.ts.map +1 -1
  65. package/dist/learning/resilient-call.js +70 -4
  66. package/dist/learning/resilient-call.js.map +1 -1
  67. package/dist/tools/process-liveness.d.ts +227 -0
  68. package/dist/tools/process-liveness.d.ts.map +1 -0
  69. package/dist/tools/process-liveness.js +486 -0
  70. package/dist/tools/process-liveness.js.map +1 -0
  71. package/dist/tools/registry.d.ts.map +1 -1
  72. package/dist/tools/registry.js +4 -2
  73. package/dist/tools/registry.js.map +1 -1
  74. package/dist/tools/run-registry.d.ts +282 -0
  75. package/dist/tools/run-registry.d.ts.map +1 -0
  76. package/dist/tools/run-registry.js +544 -0
  77. package/dist/tools/run-registry.js.map +1 -0
  78. package/dist/tools/run-terminal.d.ts +100 -2
  79. package/dist/tools/run-terminal.d.ts.map +1 -1
  80. package/dist/tools/run-terminal.js +841 -67
  81. package/dist/tools/run-terminal.js.map +1 -1
  82. package/dist/tools/tool-loop.d.ts +46 -0
  83. package/dist/tools/tool-loop.d.ts.map +1 -1
  84. package/dist/tools/tool-loop.js +115 -2
  85. package/dist/tools/tool-loop.js.map +1 -1
  86. package/dist/web-dashboard/chat-console.d.ts +25 -0
  87. package/dist/web-dashboard/chat-console.d.ts.map +1 -1
  88. package/dist/web-dashboard/chat-console.js +16 -1
  89. package/dist/web-dashboard/chat-console.js.map +1 -1
  90. package/dist/web-dashboard/server.d.ts.map +1 -1
  91. package/dist/web-dashboard/server.js +7 -1
  92. package/dist/web-dashboard/server.js.map +1 -1
  93. package/dist/web-dashboard/src/types.d.ts +32 -1
  94. package/dist/web-dashboard/src/types.d.ts.map +1 -1
  95. package/dist/web-dashboard/task-registry.d.ts +98 -0
  96. package/dist/web-dashboard/task-registry.d.ts.map +1 -0
  97. package/dist/web-dashboard/task-registry.js +159 -0
  98. package/dist/web-dashboard/task-registry.js.map +1 -0
  99. package/dist/web-dashboard/task-runner.d.ts +155 -5
  100. package/dist/web-dashboard/task-runner.d.ts.map +1 -1
  101. package/dist/web-dashboard/task-runner.js +505 -14
  102. package/dist/web-dashboard/task-runner.js.map +1 -1
  103. package/package.json +1 -1
  104. package/src/web-dashboard/public/assets/index-DycGBWwy.js +236 -0
  105. package/src/web-dashboard/public/assets/index-DycGBWwy.js.map +1 -0
  106. package/src/web-dashboard/public/index.html +1 -1
  107. package/src/web-dashboard/public/assets/index-Bik91KJh.js +0 -236
  108. package/src/web-dashboard/public/assets/index-Bik91KJh.js.map +0 -1
@@ -32,7 +32,10 @@
32
32
  */
33
33
  import { spawn } from 'node:child_process';
34
34
  import { existsSync } from 'node:fs';
35
+ import { randomUUID } from 'node:crypto';
35
36
  import { applyRemediationAutoFixes, diagnoseFailure, formatAutoApply, formatRemediation, remediationAutoApplyEnabled, } from './remediation-ladder.js';
37
+ import { classifyLiveness, formatSpan, killProcessTree, livenessThresholds, processAlive, processTableSnapshot, sampleProcessTree, tableAlive, lookupProcess, } from './process-liveness.js';
38
+ import { loadRuns, processIdentityCheck, outputWindow, saveRun, spoolBytes, spoolPath, readSpool, sweepSpools, SpoolWriter, } from './run-registry.js';
36
39
  import { maskSenderId } from '../utils/mask.js';
37
40
  import { decideStateChange } from '../learning/autonomy-policy.js';
38
41
  import { envelopeCoversAction } from '../learning/intent-envelope.js';
@@ -52,12 +55,85 @@ const DEFAULT_TIMEOUT_MS = 120_000;
52
55
  * turn (4 minutes of a 5-minute run) while the dashboard console showed nothing,
53
56
  * and then the agent gave up. Two minutes of silence in front of a person is not
54
57
  * patience, it is a bug. Sixty seconds still covers a real test/build on this
55
- * class of project, and an explicit `timeout_ms` (up to MAX_TIMEOUT_MS) still
56
- * buys the model as much time as the work genuinely needs.
58
+ * class of project, and an explicit `timeout_ms` (up to MAX_WAIT_MS) still buys
59
+ * the model as much time as the work genuinely needs.
57
60
  */
58
61
  const INTERACTIVE_DEFAULT_TIMEOUT_MS = 60_000;
59
- /** Hard ceiling — the tool loop bounds the whole turn anyway. */
60
- const MAX_TIMEOUT_MS = 300_000;
62
+ /**
63
+ * The longest this tool WAITS inside one call before handing the command to the
64
+ * monitored (`still running`) path. This is NOT a kill switch any more — see
65
+ * `assessCommandTime` and the detached-run registry below: reaching it means the
66
+ * tool stops blocking and reports, never that the work is destroyed.
67
+ */
68
+ const MAX_WAIT_MS = 1_800_000;
69
+ /**
70
+ * What a command CLASS actually needs, so the wait is chosen from the work
71
+ * instead of a single flat number.
72
+ *
73
+ * The measured defect this replaces: every command that was not a quick read got
74
+ * 60–120s and then a SIGTERM, so `npm test` on a real repo — minutes by nature,
75
+ * and the very command the tool exists for — was killed mid-run, its partial
76
+ * output reported as a failure, and the turn built conclusions on a run that
77
+ * never finished. Freebuff-style behaviour is the opposite: assess the time the
78
+ * work needs, wait for it, and when it outlives the wait, KEEP IT RUNNING and
79
+ * report what it is doing — never destroy it and call that a result.
80
+ *
81
+ * Order matters: the first match wins, so the most specific/expensive class is
82
+ * listed first (a `docker build` is not an `npm install`).
83
+ */
84
+ const LONG_COMMAND_BUDGETS = [
85
+ {
86
+ re: /(^|\s)(docker\s+(build|compose\s+up)|make|cmake\s+--build|gradle(w)?\s|mvn\s)/,
87
+ budgetMs: 900_000,
88
+ why: 'a container/compiler build',
89
+ },
90
+ {
91
+ re: /(^|\s)(npm|pnpm|yarn|bun)\s+(run\s+)?(test|test:\S+|ci|install|i|add)\b|(^|\s)(vitest|jest|pytest|go\s+test|cargo\s+test|ng\s+test)\b/,
92
+ budgetMs: 900_000,
93
+ why: 'a test suite / dependency install',
94
+ },
95
+ {
96
+ re: /(^|\s)(npm|pnpm|yarn|bun)\s+run\s+(build|compile|bundle|dist)\b|(^|\s)(tsc|vite\s+build|webpack|rollup|esbuild|cargo\s+build|go\s+build)\b/,
97
+ budgetMs: 600_000,
98
+ why: 'a build',
99
+ },
100
+ {
101
+ re: /(^|\s)(pip3?|poetry|bundle|composer)\s+(install|add|sync)\b/,
102
+ budgetMs: 600_000,
103
+ why: 'a dependency install',
104
+ },
105
+ ];
106
+ /**
107
+ * The wait budget this command deserves, and the reason (for the message).
108
+ * Exported so the assessment is testable on its own — it is a decision, not an
109
+ * implementation detail.
110
+ */
111
+ export function assessCommandTime(command) {
112
+ const cmd = command.toLowerCase();
113
+ for (const { re, budgetMs, why } of LONG_COMMAND_BUDGETS) {
114
+ if (re.test(cmd))
115
+ return { budgetMs, why };
116
+ }
117
+ return null;
118
+ }
119
+ /**
120
+ * How long this call waits before handing the command to monitoring.
121
+ *
122
+ * Priority, and why in that order:
123
+ * 1. an explicit `timeout_ms` — the model knows something the classifier does not;
124
+ * 2. the ASSESSED class — a full test suite or a build is minutes by nature, and
125
+ * giving it the short default is what produced the kill-mid-run failures;
126
+ * 3. the room — a person watching waits less (P7), an unattended run waits more.
127
+ *
128
+ * Exported because this is a decision, not an implementation detail: it is what
129
+ * `run_terminal` believes the work needs, and it should be asserted directly
130
+ * rather than inferred from a message.
131
+ */
132
+ export function resolveWaitMs(command, explicitMs, interactive) {
133
+ const assessed = assessCommandTime(command);
134
+ const base = interactive ? INTERACTIVE_DEFAULT_TIMEOUT_MS : DEFAULT_TIMEOUT_MS;
135
+ return Math.min(MAX_WAIT_MS, Math.max(1000, Math.floor(explicitMs ?? assessed?.budgetMs ?? base)));
136
+ }
61
137
  /**
62
138
  * P7 — how many times the SAME command (at the SAME timeout) may fail before the
63
139
  * next call is refused.
@@ -114,16 +190,580 @@ function recordCommandFailure(key, now) {
114
190
  function clearCommandFailure(key) {
115
191
  commandFailureStreaks.delete(key);
116
192
  }
117
- /** The guidance a timed-out command gets instead of a bare "timed out". */
118
- export function timeoutGuidance(command, timeoutMs, repeated) {
119
- const seconds = Math.round(timeoutMs / 1000);
120
- return (`\n\n⏱ TIMEOUT — that command was killed after ${seconds}s without finishing. A timeout does NOT mean the project is broken; ` +
121
- `it means this command is too slow or is waiting for something that will not come. ` +
122
- `Do NOT re-run the identical command${repeated ? ' — it has already timed out here' : ''}. Instead: ` +
123
- `(a) narrow it (a single test file, \`--reporter=dot\`, \`-x\`, one target rather than the whole suite), ` +
124
- `(b) make it non-interactive (it may be waiting on a prompt or a server), or ` +
125
- `(c) if the work is genuinely long, re-call with an explicit \`timeout_ms\` up to ${MAX_TIMEOUT_MS} (${Math.round(MAX_TIMEOUT_MS / 1000)}s). ` +
126
- `Tell the user what was slow rather than repeating \`${maskSenderId(command)}\` as-is.`);
193
+ const detachedRuns = new Map();
194
+ /** Bounded: a long session must not accumulate output buffers. */
195
+ const MAX_DETACHED_RUNS = 20;
196
+ /** ─── The liveness watchdog: identifying a HANG, and acting on it ────────────
197
+ *
198
+ * Monitoring without a heartbeat recreates the opposite defect: a command that
199
+ * is genuinely stuck (a prompt with no stdin, a lock, a socket that never
200
+ * answers) runs forever, and "still running" is not a diagnosis. Terminating on
201
+ * a clock is the bug this whole file already removed, so the replacement is:
202
+ *
203
+ * identify — sample the run's process TREE for cumulative CPU every so often;
204
+ * silence ALONE is never a hang (a real suite is quiet for
205
+ * minutes), but silence AND a measurable absence of CPU is;
206
+ * report — a confirmed hang is surfaced on `poll`/`list` with its evidence;
207
+ * act — and only then, after a further grace in which the run shows
208
+ * neither output nor CPU, is it stopped (SIGTERM, then SIGKILL on
209
+ * the whole tree). Any output or any CPU at any sample clears the
210
+ * hang, so a slow-but-working command is never touched.
211
+ *
212
+ * `NUVIRA_HANG_REAP_MS=0` keeps the identification and reporting and disables
213
+ * the stopping (report-only); `NUVIRA_HANG_QUIET_MS` and `NUVIRA_HANG_CPU_MS`
214
+ * tune what counts as a hang; `NUVIRA_HANG_POLL_MS` tunes the sample rate.
215
+ */
216
+ function envMs(name, fallback) {
217
+ const raw = Number.parseInt(String(process.env[name] ?? '').trim(), 10);
218
+ return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
219
+ }
220
+ /** How often a monitored run's process tree is sampled. */
221
+ function hangPollMs() {
222
+ return Math.max(1_000, envMs('NUVIRA_HANG_POLL_MS', 30_000));
223
+ }
224
+ /** How long a CONFIRMED hang is left running before it is stopped (0 = never). */
225
+ export function hangReapMs() {
226
+ return envMs('NUVIRA_HANG_REAP_MS', 300_000);
227
+ }
228
+ let watchdog = null;
229
+ let lastSampleAt = 0;
230
+ function ensureWatchdog() {
231
+ if (watchdog)
232
+ return;
233
+ watchdog = setInterval(() => sampleMonitoredRuns(), hangPollMs());
234
+ // A monitored run must never hold the process open — the CLI exits when the
235
+ // turn ends, and the run's own exit handler is what reports its result.
236
+ watchdog.unref?.();
237
+ }
238
+ function stopWatchdogIfIdle() {
239
+ if (!watchdog)
240
+ return;
241
+ if ([...detachedRuns.values()].some((r) => r.status === 'running'))
242
+ return;
243
+ clearInterval(watchdog);
244
+ watchdog = null;
245
+ }
246
+ /** Stop a run the watchdog has established is hung, and say why. */
247
+ function reapHungRun(run, why, table) {
248
+ if (!maySignal(run)) {
249
+ // A recovered run whose pid could not be re-verified is NEVER signalled — the
250
+ // pid may belong to someone else by now. It stays flagged and reported, and
251
+ // `exitReason` is NOT set: it is still running, and saying otherwise would be
252
+ // the kind of lie the rest of this file exists to avoid.
253
+ run.hangUnstoppable = true;
254
+ return;
255
+ }
256
+ run.reaped = true;
257
+ run.exitReason = why;
258
+ killProcessTree(run.pid ?? -1, 'SIGTERM', run.child, table);
259
+ run.killGrace = setTimeout(() => {
260
+ killProcessTree(run.pid ?? -1, 'SIGKILL', run.child);
261
+ }, 5_000);
262
+ run.killGrace.unref?.();
263
+ // Persisted IMMEDIATELY: the reason a run was stopped must survive the very
264
+ // crash the registry exists for.
265
+ persistRun(run);
266
+ }
267
+ /**
268
+ * One heartbeat over every monitored run.
269
+ *
270
+ * Exported (and takes `now`) so the decision can be driven deterministically in
271
+ * a test instead of waited for.
272
+ */
273
+ export function sampleMonitoredRuns(now = Date.now()) {
274
+ const thresholds = livenessThresholds();
275
+ const reapAfter = hangReapMs();
276
+ const sampleWindowMs = lastSampleAt > 0 ? now - lastSampleAt : hangPollMs();
277
+ lastSampleAt = now;
278
+ // ONE read per tick, shared by every run, by the reconciliation below and by any
279
+ // signalling: on Windows each read is a PowerShell start, and a tick must not
280
+ // cost one per run. The registry is read at most once too, and only when there
281
+ // is a recovered run to reconcile.
282
+ const table = processTableSnapshot();
283
+ const needsRegistry = [...detachedRuns.values()].some((r) => r.reattached && r.status === 'running');
284
+ const recorded = needsRegistry ? loadRuns() : undefined;
285
+ for (const run of [...detachedRuns.values()]) {
286
+ if (run.status !== 'running')
287
+ continue;
288
+ refreshReattachedRun(run, now, { table, ...(recorded ? { recorded } : {}) });
289
+ if (run.status !== 'running')
290
+ continue; // it ended while we were not watching
291
+ const sample = sampleProcessTree(run.pid ?? -1, table);
292
+ // A run this process spawned is alive until its own exit event says otherwise.
293
+ // A recovered one has no such event, so the process table is the only witness
294
+ // — and "cannot tell" (null) must read as alive, never as finished.
295
+ // `!== false` on purpose: a platform that cannot read the process table
296
+ // reports `null`, and "cannot tell" must read as ALIVE (a recovered run is
297
+ // never declared dead by a guess).
298
+ const alive = run.reattached
299
+ ? tableAlive(table, run.pid ?? -1) !== false
300
+ : run.child !== undefined && run.child.exitCode === null && run.child.signalCode === null;
301
+ const verdict = classifyLiveness({
302
+ alive,
303
+ bytes: run.bytes,
304
+ quietMs: now - run.lastOutputAt,
305
+ ageMs: now - run.startedAt,
306
+ cpuMs: sample.cpuMs,
307
+ prevCpuMs: run.cpuMs ?? null,
308
+ sampleWindowMs,
309
+ }, thresholds);
310
+ run.cpuMs = sample.cpuMs;
311
+ run.sampledAt = now;
312
+ run.verdict = verdict.verdict;
313
+ run.verdictWhy = verdict.why;
314
+ if (verdict.verdict === 'hung') {
315
+ // The grace starts once, at the FIRST confirmed hang — and every later
316
+ // sample that still shows a hang keeps it running toward the reap.
317
+ run.hungSince = run.hungSince ?? now;
318
+ if (reapAfter > 0 && now - run.hungSince >= reapAfter && !run.stopRequested) {
319
+ reapHungRun(run, `hung: no output and no CPU for ${formatSpan(now - run.startedAt)} ` +
320
+ `(confirmed hung for ${formatSpan(now - run.hungSince)})`, table);
321
+ }
322
+ }
323
+ else {
324
+ // It spoke or it computed — it is not hung, and the grace starts over.
325
+ run.hungSince = null;
326
+ }
327
+ }
328
+ stopWatchdogIfIdle();
329
+ }
330
+ /**
331
+ * The marker a result carries when the command OUTLIVED its wait budget and is
332
+ * still running under monitoring.
333
+ *
334
+ * It is neither success nor failure, and the loop must read it as neither:
335
+ *
336
+ * - NOT success — a live run has verified nothing yet. Counting it as proof is
337
+ * how a turn ends up claiming "the tests pass" about a suite that is still on
338
+ * its first file.
339
+ * - NOT failure — reporting it as one is the bug this whole change removes: the
340
+ * work was destroyed (or, now, is still going) and the user is told it broke.
341
+ *
342
+ * So it is its own state, and this exported string is the one both the tool and
343
+ * the loop's success classifier read (see `ranOk` in tool-loop.ts).
344
+ */
345
+ export const RUN_TERMINAL_INCOMPLETE_PREFIX = '⏳ run_terminal: still running';
346
+ /**
347
+ * Write a run's record to the durable registry (see `run-registry.ts`).
348
+ *
349
+ * Best-effort by construction: a run must never fail because a state file could
350
+ * not be written. What it buys is that the NEXT process can still answer "what is
351
+ * running, and how did it end?" — which is otherwise lost with this one.
352
+ */
353
+ function persistRun(run) {
354
+ if (run.pid === null || run.pid <= 0)
355
+ return;
356
+ saveRun({
357
+ id: run.id,
358
+ command: run.command,
359
+ cwd: run.cwd,
360
+ pid: run.pid,
361
+ startedAt: run.startedAt,
362
+ status: run.status,
363
+ exitCode: run.exitCode,
364
+ ...(run.status === 'done' ? { endedAt: run.endedAt ?? Date.now() } : {}),
365
+ ...(run.exitReason ? { exitReason: run.exitReason } : {}),
366
+ ...(run.reaped ? { reaped: true } : {}),
367
+ ...(run.stopRequested ? { stopRequested: true } : {}),
368
+ ...(run.osStartedAt ? { osStartedAt: run.osStartedAt } : {}),
369
+ });
370
+ }
371
+ /**
372
+ * Re-verify a recovered run's pid against BOTH witnesses, keeping the evidence.
373
+ * One process-table read, and the reason is reported alongside the verdict.
374
+ */
375
+ function identityOf(entry) {
376
+ const check = processIdentityCheck({ command: entry.command, osStartedAt: entry.osStartedAt ?? null }, lookupProcess(entry.pid));
377
+ return { pidVerified: check.verdict, pidVerifiedWhy: check.why };
378
+ }
379
+ /**
380
+ * Read the OS's start token for a process we are holding a handle to.
381
+ *
382
+ * Called ONCE per run, at hand-off, for a reason: it has to be read while the
383
+ * process is certainly still ours. Reading it later (at re-attach, after a
384
+ * restart) would be asking the recycled pid to identify itself — the very thing
385
+ * this token exists to check. A failed read is `null`, which downgrades the
386
+ * recovered run to the command-line witness alone rather than inventing one.
387
+ */
388
+ function readOsStartToken(pid) {
389
+ if (pid === null || pid <= 0)
390
+ return null;
391
+ try {
392
+ return lookupProcess(pid)?.startedAtText ?? null;
393
+ }
394
+ catch {
395
+ return null;
396
+ }
397
+ }
398
+ /**
399
+ * May this run be signalled?
400
+ *
401
+ * A run this process started — yes, we hold the child handle. A run RECOVERED
402
+ * from the registry — only once its pid has been re-verified as still being that
403
+ * command, because a pid that outlived the process table it came from may now
404
+ * belong to something else, and this tool only ever signals work it started.
405
+ */
406
+ function maySignal(run) {
407
+ return run.reattached !== true || run.pidVerified === true;
408
+ }
409
+ /**
410
+ * Bring a RE-ATTACHED run up to date with the truth on disk.
411
+ *
412
+ * A recovered run has no exit event — the process that would have fired one may
413
+ * be gone — so nothing would ever move it out of `running`: the poll would report
414
+ * "still running" forever about a command that finished minutes ago. Two
415
+ * witnesses settle it: the registry entry (the owning process records the exit
416
+ * code, the stop reason and the reap), and the process table.
417
+ */
418
+ function refreshReattachedRun(run, now = Date.now(),
419
+ /** What a tick already read, so reconciling cost stays O(1) reads, not O(runs). */
420
+ context = {}) {
421
+ if (!run.reattached || run.status !== 'running')
422
+ return;
423
+ const recorded = (context.recorded ?? loadRuns()).find((r) => r.id === run.id);
424
+ if (recorded && recorded.status === 'done') {
425
+ run.status = 'done';
426
+ run.exitCode = recorded.exitCode;
427
+ run.endedAt = recorded.endedAt ?? now;
428
+ if (recorded.exitReason)
429
+ run.exitReason = recorded.exitReason;
430
+ if (recorded.reaped)
431
+ run.reaped = true;
432
+ if (recorded.stopRequested)
433
+ run.stopRequested = true;
434
+ return;
435
+ }
436
+ // The tick's own table when it has one; otherwise (the poll path, a single call)
437
+ // a read of its own. `false` is the only value that means gone.
438
+ const alive = context.table === undefined ? processAlive(run.pid ?? -1) : tableAlive(context.table, run.pid ?? -1);
439
+ if (alive === false) {
440
+ run.status = 'done';
441
+ run.endedAt = now;
442
+ run.exitReason =
443
+ run.exitReason ?? 'the process is gone, and its exit code was not recorded — nothing was watching when it ended';
444
+ }
445
+ }
446
+ let hydrated = false;
447
+ /**
448
+ * Re-attach to the runs a PREVIOUS process left behind.
449
+ *
450
+ * Without this, every restart orphaned the record of work that is still going:
451
+ * the command keeps running, nothing can poll it, and the turn that started it
452
+ * has no way to collect its real verdict. Runs whose process is gone are marked
453
+ * finished (with an honest "its exit code was not recorded" reason), and live
454
+ * ones are re-attached to be polled, sampled and — only after their pid is
455
+ * re-verified — stopped.
456
+ *
457
+ * Returns how many runs were recovered.
458
+ */
459
+ export function hydrateMonitoredRuns() {
460
+ if (hydrated)
461
+ return 0;
462
+ hydrated = true;
463
+ let recovered = 0;
464
+ for (const entry of loadRuns()) {
465
+ if (detachedRuns.has(entry.id))
466
+ continue;
467
+ const alive = processAlive(entry.pid);
468
+ const finished = entry.status === 'done' || alive === false;
469
+ const run = {
470
+ id: entry.id,
471
+ command: entry.command,
472
+ cwd: entry.cwd,
473
+ pid: entry.pid,
474
+ startedAt: entry.startedAt,
475
+ // The silence clock starts NOW, not at the start of the run: we have no
476
+ // output timeline for a recovered run, and running it from `startedAt`
477
+ // would judge a long-quiet-but-working command hung the instant we looked.
478
+ lastOutputAt: entry.endedAt ?? Date.now(),
479
+ bytes: spoolBytes(entry.id),
480
+ out: '',
481
+ err: '',
482
+ status: finished ? 'done' : 'running',
483
+ exitCode: entry.exitCode,
484
+ reattached: true,
485
+ // TWO witnesses, decided by `processIdentityVerdict`: the command line, and
486
+ // the OS start token captured at hand-off. The token is what turns "I cannot
487
+ // prove this is ours, so I will only watch it" into "this IS the process I
488
+ // started, so it can still be stopped" — and, in the other direction, what
489
+ // refuses to adopt a recycled pid running the SAME command line.
490
+ ...(finished
491
+ ? { pidVerified: false }
492
+ : identityOf(entry)),
493
+ osStartedAt: entry.osStartedAt ?? null,
494
+ truncatedOutput: true, // the head lives in the spool, never in memory
495
+ ...(entry.endedAt !== undefined ? { endedAt: entry.endedAt } : {}),
496
+ ...(entry.exitReason ? { exitReason: entry.exitReason } : {}),
497
+ ...(entry.reaped ? { reaped: true } : {}),
498
+ ...(entry.stopRequested ? { stopRequested: true } : {}),
499
+ };
500
+ if (run.status === 'done' && entry.status !== 'done') {
501
+ run.exitReason =
502
+ 'it ended while nothing was watching (this process restarted) — its exit code was not recorded';
503
+ persistRun(run);
504
+ }
505
+ detachedRuns.set(run.id, run);
506
+ recovered += 1;
507
+ }
508
+ if (recovered > 0) {
509
+ pruneDetachedRuns();
510
+ ensureWatchdog();
511
+ }
512
+ // Records are capped; files are not a record. Sweeping here — the one moment
513
+ // the live set is exactly known — is what keeps "bounded state" true of the
514
+ // disk and not just of the JSON.
515
+ sweepSpools([...detachedRuns.keys()]);
516
+ return recovered;
517
+ }
518
+ /** Test/DI seam: forget the monitored runs. Kills nothing (they keep running). */
519
+ export function resetDetachedRuns() {
520
+ detachedRuns.clear();
521
+ if (watchdog)
522
+ clearInterval(watchdog);
523
+ watchdog = null;
524
+ lastSampleAt = 0;
525
+ hydrated = false;
526
+ }
527
+ function pruneDetachedRuns() {
528
+ if (detachedRuns.size <= MAX_DETACHED_RUNS)
529
+ return;
530
+ // Finished first — they are only kept so a late poll can read their result.
531
+ for (const [id, run] of detachedRuns) {
532
+ if (detachedRuns.size <= MAX_DETACHED_RUNS)
533
+ break;
534
+ if (run.status === 'done')
535
+ detachedRuns.delete(id);
536
+ }
537
+ if (detachedRuns.size > MAX_DETACHED_RUNS) {
538
+ // All still running: drop the oldest MONITORING entry. The process is NOT
539
+ // killed — it simply stops being pollable by id.
540
+ const oldest = [...detachedRuns.values()].sort((a, b) => a.startedAt - b.startedAt)[0];
541
+ if (oldest)
542
+ detachedRuns.delete(oldest.id);
543
+ }
544
+ }
545
+ /** The tail of a buffer, capped like the output this tool returns. */
546
+ function outputTail(s) {
547
+ return s.trim().length > MAX_OUTPUT_CHARS ? s.trim().slice(-MAX_OUTPUT_CHARS) : s.trim();
548
+ }
549
+ /** `45s` / `3m 05s` — how long, for a human reading the tool result. */
550
+ function elapsedLabel(ms) {
551
+ const totalSeconds = Math.max(0, Math.round(ms / 1000));
552
+ if (totalSeconds < 60)
553
+ return `${totalSeconds}s`;
554
+ return `${Math.floor(totalSeconds / 60)}m ${String(totalSeconds % 60).padStart(2, '0')}s`;
555
+ }
556
+ /**
557
+ * This run's output, as the model should see it.
558
+ *
559
+ * The in-memory buffer is used when it is COMPLETE — the common case, and the
560
+ * existing behaviour. When the beginning was lost to the buffer's bound, or the
561
+ * run was recovered from disk and there is no buffer at all, the spooled
562
+ * head+tail is read instead: a long run's diagnosis needs the first lines (what
563
+ * it was doing, how it was invoked) as much as the last.
564
+ */
565
+ function runOutputText(run) {
566
+ if (!run.truncatedOutput && !run.reattached && (run.out || run.err))
567
+ return outputTail(run.out || run.err);
568
+ const spooled = readSpool(run.id);
569
+ if (spooled) {
570
+ const window = outputWindow(spooled, MAX_OUTPUT_CHARS);
571
+ if (window.text.trim())
572
+ return window.text.trim();
573
+ }
574
+ return outputTail(run.out || run.err);
575
+ }
576
+ /** The liveness of a monitored run, as a short badge for `list`/`poll`. */
577
+ function livenessBadge(run) {
578
+ if (run.reaped)
579
+ return '🧯 stopped (hung)';
580
+ if (run.status === 'done')
581
+ return `✔ finished (exit ${run.exitCode ?? '?'})`;
582
+ const elapsed = `⏳ running ${elapsedLabel(Date.now() - run.startedAt)}`;
583
+ switch (run.verdict) {
584
+ case 'hung':
585
+ return `${elapsed} · 🚫 HUNG (no output and no CPU) — will be stopped after the grace`;
586
+ case 'stalled':
587
+ return `${elapsed} · ⚠️ quiet — ${run.verdictWhy ?? 'no output recently'}`;
588
+ case 'working':
589
+ return `${elapsed} · ✅ working — ${run.verdictWhy ?? 'making progress'}`;
590
+ default:
591
+ return elapsed;
592
+ }
593
+ }
594
+ /**
595
+ * Everything currently being followed.
596
+ *
597
+ * Hydrates first, so this answers for the runs a PREVIOUS process left behind as
598
+ * well as this one's — otherwise a restart would make the whole list vanish while
599
+ * the commands are still running.
600
+ */
601
+ export function listDetachedRuns() {
602
+ const recovered = hydrateMonitoredRuns();
603
+ if (detachedRuns.size === 0) {
604
+ return 'run_terminal: nothing is being monitored — no command has outlived its wait budget in this process.';
605
+ }
606
+ const lines = [...detachedRuns.values()]
607
+ .sort((a, b) => a.startedAt - b.startedAt)
608
+ .map((r) => `- run_terminal[${r.id}] ${livenessBadge(r)} · ${r.bytes} bytes · \`${maskSenderId(r.command)}\`` +
609
+ (r.reattached ? ' · ↩ recovered from a previous process' : ''));
610
+ return (`run_terminal — monitored commands:\n${lines.join('\n')}\n\n` +
611
+ `Poll one with run_terminal(action:'poll', process_id:'<id>'); stop one with run_terminal(action:'stop', process_id:'<id>').` +
612
+ (recovered > 0
613
+ ? `\n(${recovered} recovered from the registry a previous process wrote — they are still being followed.)`
614
+ : ''));
615
+ }
616
+ /**
617
+ * Stop one monitored run deliberately (the model's own decision after a
618
+ * diagnosis). Bounded on purpose: it can only ever touch a command THIS tool
619
+ * started and is still following — never an arbitrary pid.
620
+ */
621
+ export function stopDetachedRun(id) {
622
+ hydrateMonitoredRuns();
623
+ const run = id ? detachedRuns.get(id) : undefined;
624
+ if (!run)
625
+ return unknownRunMessage(id);
626
+ if (run.status !== 'running') {
627
+ return (`run_terminal[${run.id}] already finished (exit ${run.exitCode ?? '?'}) — nothing to stop. ` +
628
+ (runOutputText(run) ? `Its output:\n${maskSenderId(runOutputText(run))}` : '(no output)'));
629
+ }
630
+ // A RECOVERED run is identified only by a pid, and a pid can be recycled — so
631
+ // it is stopped only once its command line has been re-verified as ours. The
632
+ // alternative is signalling a stranger's process, which this tool must never do.
633
+ if (!maySignal(run)) {
634
+ return (`Error: run_terminal: run_terminal[${run.id}] was recovered from a previous process, and its pid ` +
635
+ `(${run.pid ?? '?'}) could not be re-verified as still being \`${maskSenderId(run.command)}\` — ` +
636
+ `${run.pidVerifiedWhy ?? 'the check could not be completed'}. Refusing to signal it rather than risk ` +
637
+ `stopping an unrelated process. Its output is spooled here: ${spoolPath(run.id)}. ` +
638
+ `If you know it is still yours, stop it by hand with its PID.`);
639
+ }
640
+ run.stopRequested = true;
641
+ run.exitReason = 'stopped on request';
642
+ const signalled = killProcessTree(run.pid ?? -1, 'SIGTERM', run.child);
643
+ run.killGrace = setTimeout(() => {
644
+ killProcessTree(run.pid ?? -1, 'SIGKILL', run.child);
645
+ }, 5_000);
646
+ run.killGrace.unref?.();
647
+ persistRun(run);
648
+ return (`🛑 run_terminal[${run.id}] — SIGTERM sent to ${signalled.length || 'the'} process(es) of \`${maskSenderId(run.command)}\` ` +
649
+ `(the shell AND the real work under it; SIGKILL follows in 5s if it does not exit).\n` +
650
+ `Poll it with run_terminal(action:'poll', process_id:'${run.id}') to get its output and exit code. ` +
651
+ `Nothing else was touched.`);
652
+ }
653
+ /** The one message a poll/stop gives for an id nobody is monitoring. */
654
+ function unknownRunMessage(id) {
655
+ const known = [...detachedRuns.keys()];
656
+ return (`Error: run_terminal: no monitored command with id \`${maskSenderId(id)}\`.` +
657
+ (known.length > 0 ? ` Known: ${known.join(', ')}.` : ' Nothing is being monitored.') +
658
+ ` Use run_terminal(action:'list') to see what is.`);
659
+ }
660
+ /**
661
+ * The state of one monitored command. A finished one keeps its result available
662
+ * (bounded) so the agent can read the exit code and output after doing other
663
+ * work — the poll is how a long command's verdict finally reaches the turn.
664
+ */
665
+ export function pollDetachedRun(id) {
666
+ hydrateMonitoredRuns();
667
+ const run = id ? detachedRuns.get(id) : undefined;
668
+ if (!run)
669
+ return unknownRunMessage(id);
670
+ // A poll is exactly when "how is it doing?" is asked, so it is also when a
671
+ // recovered run must be reconciled — otherwise it reports itself as running
672
+ // forever (see `refreshReattachedRun`).
673
+ refreshReattachedRun(run);
674
+ const elapsed = elapsedLabel(Date.now() - run.startedAt);
675
+ const body = runOutputText(run);
676
+ if (run.status === 'running') {
677
+ const quietMs = Date.now() - run.lastOutputAt;
678
+ return (`${RUN_TERMINAL_INCOMPLETE_PREFIX} — run_terminal[${run.id}], ${elapsed} elapsed, ${run.bytes} bytes of output` +
679
+ (run.reattached ? ' (recovered from a previous process)' : '') +
680
+ (run.bytes === 0
681
+ ? ', and NOTHING produced yet'
682
+ : quietMs >= 60_000
683
+ ? `, silent for the last ${elapsedLabel(quietMs)}`
684
+ : '') +
685
+ `.\nCommand: \`${maskSenderId(run.command)}\`` +
686
+ // The liveness verdict, when the watchdog has one — this is the difference
687
+ // between "it is still running" and "it is running, and here is why I
688
+ // believe it is hung".
689
+ (run.verdict && run.verdict !== 'starting'
690
+ ? `\nLiveness: ${run.verdict.toUpperCase()} — ${run.verdictWhy ?? 'no evidence yet'}.`
691
+ : '') +
692
+ (body ? `\nOutput so far:\n${maskSenderId(body)}` : '') +
693
+ `\nIt was NOT killed. Poll again later with run_terminal(action:'poll', process_id:'${run.id}'), or do other work first.` +
694
+ (run.hangUnstoppable
695
+ ? `\nIt looks HUNG, but it was recovered after a restart and its pid could not be re-verified as this ` +
696
+ `command (${run.pidVerifiedWhy ?? 'the check could not be completed'}) — so it is deliberately NOT ` +
697
+ `being stopped automatically. Its output is spooled here: ${spoolPath(run.id)}.`
698
+ : run.verdict === 'hung'
699
+ ? `\nIf you agree it is hung (it needs input it cannot get, or a lock/port it will never get), ` +
700
+ `stop it with run_terminal(action:'stop', process_id:'${run.id}') and re-run it non-interactively — ` +
701
+ `the watchdog stops it on its own after the grace either way.`
702
+ : ''));
703
+ }
704
+ // A run that FAILED, or that was stopped (by the watchdog or on request), did
705
+ // NOT verify anything — so it must carry the `Error: ` prefix the loop's honest
706
+ // accounting reads (`ranOk` in tool-loop.ts). Without it, collecting a failing
707
+ // suite through `poll` was recorded as a SUCCESSFUL tool call, which feeds the
708
+ // "did you check it" gate: the turn could claim a check it never got a pass
709
+ // from. Only a real exit-0 finish is a success here.
710
+ const ok = run.exitCode === 0 && !run.reaped && !run.stopRequested;
711
+ return (`${ok ? '✅ ' : 'Error: '}run_terminal[${run.id}] ` +
712
+ (run.reaped || run.stopRequested
713
+ ? `was STOPPED (${run.exitReason ?? 'stopped'})`
714
+ : `finished${run.exitCode === 0 ? '' : ` (exit ${run.exitCode ?? '?'})`}`) +
715
+ ` — ${elapsed} total.\n` +
716
+ `Command: \`${maskSenderId(run.command)}\`` +
717
+ // A run recovered as finished has a reason instead of an exit code, and it
718
+ // must be stated: "finished" with no code is not the same fact as "exit 0".
719
+ (run.reattached && run.exitReason ? `\n${run.exitReason}.` : '') +
720
+ (run.reaped
721
+ ? `\nThe work was doing nothing (no output, no CPU) and was stopped deliberately — this is NOT a test ` +
722
+ `failure and NOT a verdict on the code. Fix what it was waiting for and re-run it.`
723
+ : '') +
724
+ (body ? `\nOutput:\n${maskSenderId(body)}` : '\n(no output)'));
725
+ }
726
+ /**
727
+ * Why the command is taking as long as it is — the diagnosis a bare "timed out"
728
+ * never gave. A long run is either PROGRESS (let it finish) or STUCK (a prompt,
729
+ * a lock, a dead network call), and those need opposite responses, so the tool
730
+ * says which one the evidence supports.
731
+ */
732
+ function longRunDiagnosis(bytes, quietMs) {
733
+ if (bytes === 0) {
734
+ return ('NO output at all yet. That is the signature of a command waiting on an interactive prompt ' +
735
+ '(this tool gives it no stdin, so the prompt never gets an answer), or of a first step that is genuinely slow ' +
736
+ '(a cold build, a large download). Make it non-interactive before re-running: `--yes`/`--no-input`/`-y`, ' +
737
+ '`CI=1`, `--non-interactive`, or a flag that skips the prompt.');
738
+ }
739
+ if (quietMs >= 60_000) {
740
+ return (`Output STOPPED ${elapsedLabel(quietMs)} ago after ${bytes} bytes — it is stalled rather than working. ` +
741
+ 'Usual causes: waiting on a prompt, a lock held by another process, a network call with no timeout, or a test ' +
742
+ 'blocked on a server that never came up. Read the tail below for the last thing it said before going quiet.');
743
+ }
744
+ return `It IS still producing output (${bytes} bytes, last one ${elapsedLabel(quietMs)} ago), so it is making progress — let it finish and poll rather than restarting it.`;
745
+ }
746
+ /**
747
+ * What the model is told when a command outlives this call's wait budget.
748
+ * Exported so the wording is testable: the two things it MUST say are that the
749
+ * command was not killed and that re-running it is wrong.
750
+ */
751
+ export function waitBudgetGuidance(command, id, waitMs, assessed, bytes, quietMs, reapAfterMs = 0) {
752
+ const seconds = Math.round(waitMs / 1000);
753
+ return (` — \`${maskSenderId(command)}\` has outlived this call's ${seconds}s wait budget. ` +
754
+ `It was NOT killed and this is NOT a failure: the work is still going, and its output is accumulating.\n` +
755
+ `${assessed ? `It was given an extended wait because it is ${assessed.why} — a class that is minutes by nature.\n` : ''}` +
756
+ `Follow it with \`run_terminal(action:'poll', process_id:'${id}')\` — that returns its status, its liveness ` +
757
+ `(WORKING / STALLED / HUNG, from its output and its CPU), its output tail, and its REAL exit code once it lands; ` +
758
+ `\`run_terminal(action:'list')\` shows everything being monitored. ` +
759
+ `Do NOT re-run the identical command: that would start a second copy of the same work. Do something useful while ` +
760
+ `it finishes (read a file, plan the next step) and poll instead.` +
761
+ (reapAfterMs > 0
762
+ ? `\nA running command is WATCHED: if it stops producing output AND stops using CPU for ` +
763
+ `${Math.round(livenessThresholds().hangQuietMs / 60_000)} minutes it is reported as HUNG, and if it stays that way for a further ` +
764
+ `${Math.round(reapAfterMs / 60_000)} minutes it is stopped and its result reported — a real hang is not left running forever. ` +
765
+ `Output or CPU at any point clears that, so a slow-but-working command is never touched.`
766
+ : ''));
127
767
  }
128
768
  /** ─── Command classification (deny-first) ───────────────────────────────── */
129
769
  /**
@@ -462,6 +1102,18 @@ export function classifyCommand(command) {
462
1102
  return worst;
463
1103
  }
464
1104
  export async function runTerminalTool(args, ctx) {
1105
+ // The follow-up half of a long run: no command is needed to poll or list, and
1106
+ // these must work even when the model has nothing to spawn.
1107
+ // Recover whatever a PREVIOUS process left running, before anything is looked
1108
+ // up. Without this a restart silently orphans the record: the command keeps
1109
+ // running, `poll` reports no such id, and the turn can never collect it.
1110
+ hydrateMonitoredRuns();
1111
+ if (args.action === 'poll')
1112
+ return pollDetachedRun((args.process_id ?? '').trim());
1113
+ if (args.action === 'list')
1114
+ return listDetachedRuns();
1115
+ if (args.action === 'stop')
1116
+ return stopDetachedRun((args.process_id ?? '').trim());
465
1117
  const command = normalize(args.command);
466
1118
  // NOTE: every refusal below is prefixed `Error: ` on purpose. The tool-result
467
1119
  // convention treats that prefix as "this did not run", and the loop's honest
@@ -592,11 +1244,29 @@ export async function runTerminalTool(args, ctx) {
592
1244
  authorization: ctx.writesAuthorized?.reason,
593
1245
  }, 'tool-loop');
594
1246
  }
595
- // P7 — the DEFAULT timeout follows the room: a person watching a chat gets a
596
- // shorter one (see INTERACTIVE_DEFAULT_TIMEOUT_MS). An explicit `timeout_ms`
597
- // always wins, so a genuinely long build still gets its time.
598
- const defaultTimeoutMs = ctx.interactive ? INTERACTIVE_DEFAULT_TIMEOUT_MS : DEFAULT_TIMEOUT_MS;
599
- const timeoutMs = Math.min(MAX_TIMEOUT_MS, Math.max(1000, Math.floor(args.timeout_ms ?? defaultTimeoutMs)));
1247
+ // ── Don't start the same work twice ─────────────────────────────────────────
1248
+ // A long command handed off to monitoring is STILL RUNNING. Re-issuing it
1249
+ // would spawn a second copy (two test suites competing for the same ports and
1250
+ // CPU, two installs over each other). The old code had no such case because
1251
+ // the first copy was already dead; now the honest answer is to follow it.
1252
+ const runningSame = [...detachedRuns.values()].find((r) => r.status === 'running' && r.command === command && r.cwd === (ctx.cwd ?? process.cwd()));
1253
+ if (runningSame) {
1254
+ return (`Error: run_terminal: \`${maskSenderId(command)}\` is ALREADY RUNNING (monitored as ` +
1255
+ `run_terminal[${runningSame.id}], ${elapsedLabel(Date.now() - runningSame.startedAt)} elapsed). ` +
1256
+ `Starting it again would do the same work twice. Follow it instead: ` +
1257
+ `run_terminal(action:'poll', process_id:'${runningSame.id}').`);
1258
+ }
1259
+ // P7 — the wait budget: an explicit `timeout_ms` wins; otherwise the command is
1260
+ // ASSESSED, so a test suite or a build gets the minutes it actually needs
1261
+ // instead of the short default meant for a quick read; otherwise the room
1262
+ // decides (a person watching waits less — see INTERACTIVE_DEFAULT_TIMEOUT_MS).
1263
+ // Reaching the budget now hands the command to monitoring rather than killing
1264
+ // it, so an under-estimate costs a poll, not the work.
1265
+ const assessed = assessCommandTime(command);
1266
+ const waitMs = resolveWaitMs(command, args.timeout_ms, ctx.interactive);
1267
+ // Reported so the model knows the hand-off is WATCHED: a run that goes silent
1268
+ // and idle will be identified as hung and stopped, not left forever.
1269
+ const reapAfterMs = hangReapMs();
600
1270
  // P7 — the IDENTICAL-COMMAND guard. A command that has already failed twice at
601
1271
  // this timeout in this directory will fail a third time the same way; the live
602
1272
  // run spent two 120-second timeouts proving that. Refusing costs milliseconds
@@ -606,7 +1276,7 @@ export async function runTerminalTool(args, ctx) {
606
1276
  // The effective timeout is part of the key on purpose: the guidance a timeout
607
1277
  // carries tells the model to re-call with a longer `timeout_ms` when the work
608
1278
  // is genuinely long, and that advice must not be refused by this guard.
609
- const streakKey = `${ctx.cwd ?? process.cwd()}|${command}|${timeoutMs}`;
1279
+ const streakKey = `${ctx.cwd ?? process.cwd()}|${command}|${waitMs}`;
610
1280
  const priorStreak = commandFailureStreaks.get(streakKey);
611
1281
  if (priorStreak &&
612
1282
  Date.now() - priorStreak.lastAt <= FAILURE_STREAK_WINDOW_MS &&
@@ -617,20 +1287,32 @@ export async function runTerminalTool(args, ctx) {
617
1287
  `or run a different check entirely.`);
618
1288
  }
619
1289
  const startedAt = Date.now();
620
- const output = await execCommand(command, ctx.cwd, timeoutMs, runEnv);
621
- // P7 — a TIMEOUT is its own, specific signal: the command was killed, so
622
- // "it timed out" is not "the project is broken", and the ONE thing the model
623
- // must not do is re-run the identical command. The guidance says what to do
624
- // instead (narrow it, make it non-interactive, or pass a longer timeout_ms).
625
- const timedOut = /run_terminal: `[^`]*` ⏱ timed out after \d+ms/.test(output);
626
- const failureStreak = timedOut
627
- ? recordCommandFailure(streakKey, Date.now())
628
- : output.startsWith('Error:')
629
- ? recordCommandFailure(streakKey, Date.now())
630
- : 0;
631
- if (!timedOut && !output.startsWith('Error:'))
1290
+ const outcome = await execCommand(command, ctx.cwd, waitMs, runEnv);
1291
+ // ── A command that outlived the wait budget is NOT a failure ────────────────
1292
+ // It is still running, its output is still accumulating, and the turn can
1293
+ // carry on: the guidance says how to follow it, what the evidence says about
1294
+ // why it is slow, and that re-running it would duplicate the work. Crucially
1295
+ // it does NOT record a failure streak — counting a live run as a failure is
1296
+ // what made the old behaviour report unfinished work as broken.
1297
+ if (outcome.kind === 'still-running') {
1298
+ ctx.emit?.('run_terminal:monitoring', {
1299
+ tool: 'run_terminal',
1300
+ command,
1301
+ processId: outcome.id,
1302
+ waitMs,
1303
+ bytes: outcome.bytes,
1304
+ }, 'tool-loop');
1305
+ return (`${RUN_TERMINAL_INCOMPLETE_PREFIX}` +
1306
+ waitBudgetGuidance(command, outcome.id, waitMs, assessed, outcome.bytes, outcome.quietMs, reapAfterMs) +
1307
+ `\nMonitored as run_terminal[${outcome.id}].\n` +
1308
+ longRunDiagnosis(outcome.bytes, outcome.quietMs));
1309
+ }
1310
+ const output = outcome.output;
1311
+ // A real non-zero exit is the only thing that counts as a failure now.
1312
+ const failureStreak = output.startsWith('Error:') ? recordCommandFailure(streakKey, Date.now()) : 0;
1313
+ if (!output.startsWith('Error:'))
632
1314
  clearCommandFailure(streakKey);
633
- const outputWithNotes = timedOut ? output + timeoutGuidance(command, timeoutMs, failureStreak > 1) : output;
1315
+ const outputWithNotes = output;
634
1316
  // A4 — a KNOWN toolchain failure gets its known fix appended, so the model
635
1317
  // repairs instead of re-trying the identical command. Advisory only: the note
636
1318
  // names the bounded, project-local step; it never mutates the workspace.
@@ -685,7 +1367,7 @@ export async function runTerminalTool(args, ctx) {
685
1367
  // remembered for this machine (observed), so the same OS-specific command is
686
1368
  // not re-derived. Only with a verb and only on success — a failure is not a
687
1369
  // fact worth remembering, and a verbless command cannot be keyed. Best-effort.
688
- if (args.verb && !timedOut && !output.startsWith('Error:')) {
1370
+ if (args.verb && !output.startsWith('Error:')) {
689
1371
  try {
690
1372
  const { recordLearnedCommand } = await import('../learning/learned-commands.js');
691
1373
  const { leadingBinary } = await import('../learning/command-adaptation.js');
@@ -703,10 +1385,11 @@ export async function runTerminalTool(args, ctx) {
703
1385
  }
704
1386
  // C3 — remember what a SUCCESSFUL run answered (whole command + every fact it
705
1387
  // established), so the same question in the same run is answered from the
706
- // output above instead of a second spawn. A timeout is a failure like any
707
- // other, and `Error:`-prefixed output is a refusal or a crash — neither is a
708
- // fact worth remembering.
709
- if (!timedOut && !output.startsWith('Error:'))
1388
+ // output above instead of a second spawn. `Error:`-prefixed output is a
1389
+ // refusal, a crash, or a real non-zero exit — not a fact worth remembering.
1390
+ // (A command that is merely still running never reaches here: it is followed
1391
+ // with `poll`, and its output is stored when it lands.)
1392
+ if (!output.startsWith('Error:'))
710
1393
  storeMemo(ctx.commandMemo, command, output, memoWhere);
711
1394
  if (!decidedAutonomously)
712
1395
  return outputWithNotes + effectNote + envNote + remediationNote + autoApplyNote + missingNote;
@@ -719,8 +1402,15 @@ export async function runTerminalTool(args, ctx) {
719
1402
  autoApplyNote +
720
1403
  missingNote);
721
1404
  }
722
- /** Spawn the command (shell-interpreted — pipes, &&, substitutions work). */
723
- function execCommand(command, cwd, timeoutMs, envOverride) {
1405
+ /**
1406
+ * Spawn the command (shell-interpreted — pipes, &&, substitutions work).
1407
+ *
1408
+ * `waitMs` is a WAIT budget, not a kill switch. When it runs out the child keeps
1409
+ * running, its output keeps accumulating in the registry, and the caller is told
1410
+ * how to follow it (`pollDetachedRun`). The only thing that changes at the budget
1411
+ * is that THIS CALL stops blocking.
1412
+ */
1413
+ function execCommand(command, cwd, waitMs, envOverride) {
724
1414
  return new Promise((resolve) => {
725
1415
  // On Windows, use Git Bash for cross-platform shell commands (pwd, touch, etc.)
726
1416
  const shellOptions = {
@@ -738,49 +1428,133 @@ function execCommand(command, cwd, timeoutMs, envOverride) {
738
1428
  shellOptions.shell = gitBash;
739
1429
  }
740
1430
  }
1431
+ const id = randomUUID();
1432
+ // Spooling starts at SPAWN, not at hand-off: the beginning of the output is
1433
+ // exactly what an overflowing buffer loses while the wait budget is still
1434
+ // running, and that is the part a diagnosis needs. A run that finishes inside
1435
+ // its budget discards the spool (see `settleRun`).
1436
+ const spool = new SpoolWriter(id);
741
1437
  const child = spawn(command, shellOptions);
742
- let out = '';
743
- let err = '';
1438
+ const startedAt = Date.now();
1439
+ const run = {
1440
+ id,
1441
+ command,
1442
+ cwd: cwd || process.cwd(),
1443
+ child,
1444
+ pid: child.pid ?? null,
1445
+ spool,
1446
+ startedAt,
1447
+ lastOutputAt: startedAt,
1448
+ bytes: 0,
1449
+ out: '',
1450
+ err: '',
1451
+ status: 'running',
1452
+ exitCode: null,
1453
+ };
1454
+ // Whether this call has already answered (so a later exit does not resolve
1455
+ // twice — the registry owns the result from that point on).
1456
+ let answered = false;
744
1457
  const timer = setTimeout(() => {
745
- try {
746
- child.kill('SIGTERM');
1458
+ if (answered)
1459
+ return;
1460
+ answered = true;
1461
+ // HAND OFF, do not kill. See the registry above.
1462
+ detachedRuns.set(run.id, run);
1463
+ pruneDetachedRuns();
1464
+ // Start watching it, so a run that stops making progress is IDENTIFIED
1465
+ // (and, on evidence, stopped) instead of running forever.
1466
+ ensureWatchdog();
1467
+ // …and make it DURABLE, so a restart can still follow and collect it. This
1468
+ // is also what puts the spool on disk: until a run is durable nobody could
1469
+ // read its output back, so writing it earlier only littered the config dir
1470
+ // with a file that a run finishing inside its budget then deleted.
1471
+ run.osStartedAt = readOsStartToken(run.pid);
1472
+ run.spool?.startPersisting();
1473
+ run.spool?.flush();
1474
+ persistRun(run);
1475
+ resolve({
1476
+ kind: 'still-running',
1477
+ id: run.id,
1478
+ bytes: run.bytes,
1479
+ quietMs: Date.now() - run.lastOutputAt,
1480
+ });
1481
+ }, waitMs);
1482
+ const note = (chunk, stream) => {
1483
+ const text = chunk.toString('utf-8');
1484
+ run[stream] += text;
1485
+ run.bytes += chunk.length;
1486
+ run.lastOutputAt = Date.now();
1487
+ // The spool keeps a bounded head AND tail, so what the buffer drops is still
1488
+ // readable later (and after a restart).
1489
+ run.spool?.write(text);
1490
+ // Bounded buffering: a 40MB log must not be held in memory, and the tool
1491
+ // only ever reports the tail.
1492
+ const cap = MAX_OUTPUT_CHARS * 2;
1493
+ if (run[stream].length > cap) {
1494
+ run[stream] = run[stream].slice(-cap);
1495
+ // Once anything is dropped the buffer is missing its BEGINNING, and the
1496
+ // poll must read the spool instead of presenting a tail as the whole story.
1497
+ run.truncatedOutput = true;
747
1498
  }
748
- catch {
749
- /* best-effort */
1499
+ };
1500
+ /**
1501
+ * The run is over. Stop spooling, and keep the durable record only when it can
1502
+ * ever be read: a run handed to monitoring is collected by `poll` (possibly in
1503
+ * another process), while one that finished inside its wait budget already
1504
+ * delivered its result — its record and spool would be litter.
1505
+ */
1506
+ const settleRun = () => {
1507
+ run.endedAt = Date.now();
1508
+ if (detachedRuns.has(run.id)) {
1509
+ run.spool?.close();
1510
+ persistRun(run);
750
1511
  }
751
- }, timeoutMs);
752
- child.stdout?.on('data', (b) => {
753
- out += b.toString('utf-8');
754
- if (out.length > MAX_OUTPUT_CHARS * 2)
755
- out = out.slice(-MAX_OUTPUT_CHARS * 2);
756
- });
757
- child.stderr?.on('data', (b) => {
758
- err += b.toString('utf-8');
759
- if (err.length > MAX_OUTPUT_CHARS)
760
- err = err.slice(-MAX_OUTPUT_CHARS);
761
- });
1512
+ else {
1513
+ run.spool?.discard();
1514
+ }
1515
+ };
1516
+ child.stdout?.on('data', (b) => note(b, 'out'));
1517
+ child.stderr?.on('data', (b) => note(b, 'err'));
762
1518
  child.on('error', (e) => {
763
1519
  clearTimeout(timer);
764
- resolve(`Error: run_terminal: failed to spawn — ${e.message}`);
1520
+ run.status = 'done';
1521
+ run.err += `\n${e.message}`;
1522
+ settleRun();
1523
+ if (answered)
1524
+ return;
1525
+ answered = true;
1526
+ resolve({ kind: 'finished', output: `Error: run_terminal: failed to spawn — ${e.message}` });
765
1527
  });
766
1528
  child.on('exit', (code) => {
767
1529
  clearTimeout(timer);
768
- const body = (out || err).trim().slice(0, MAX_OUTPUT_CHARS);
1530
+ run.status = 'done';
1531
+ run.exitCode = code;
1532
+ settleRun();
1533
+ if (answered)
1534
+ return; // already handed off — the poll reports the result
1535
+ answered = true;
1536
+ const body = (run.out || run.err).trim().slice(0, MAX_OUTPUT_CHARS);
769
1537
  const masked = maskSenderId(body);
770
- const timedOut = code === null;
771
- const ok = !timedOut && code === 0;
772
- const status = timedOut
773
- ? `⏱ timed out after ${timeoutMs}ms`
1538
+ // `code === null` now means the process was terminated by something ELSE
1539
+ // (a signal, the OOM killer, the user) — the tool no longer kills it, so
1540
+ // this is a real, reportable condition rather than its own timeout.
1541
+ const terminated = code === null;
1542
+ const ok = !terminated && code === 0;
1543
+ const status = terminated
1544
+ ? 'was terminated before it finished (a signal, not this tool)'
774
1545
  : ok
775
1546
  ? '✅ succeeded'
776
1547
  : `❌ failed (exit ${code ?? '?'})`;
777
1548
  // Enterprise G1: a command that FAILED is not a verification. The loop's
778
1549
  // honest accounting reads the `Error:` prefix, and the verification gate
779
1550
  // counts a successful run_terminal as proof — so a non-zero exit (or a
780
- // timeout) MUST carry the prefix, or a failing test/build would mark the
781
- // turn "verified". The real output is preserved for diagnosis.
782
- resolve(`${ok ? '' : 'Error: '}run_terminal: \`${maskSenderId(command)}\` ${status}.\n` +
783
- (masked ? `Output:\n${masked}` : '(no output)'));
1551
+ // termination) MUST carry the prefix, or a failing test/build would mark
1552
+ // the turn "verified". The real output is preserved for diagnosis.
1553
+ resolve({
1554
+ kind: 'finished',
1555
+ output: `${ok ? '' : 'Error: '}run_terminal: \`${maskSenderId(command)}\` ${status}.\n` +
1556
+ (masked ? `Output:\n${masked}` : '(no output)'),
1557
+ });
784
1558
  });
785
1559
  });
786
1560
  }