agent-nuvira 3.5.0 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -13
- package/dist/agent-sdk/src/scaffold.d.ts +1 -1
- package/dist/agent-sdk/src/scaffold.js +1 -1
- package/dist/cli/chat.d.ts +159 -6
- package/dist/cli/chat.d.ts.map +1 -1
- package/dist/cli/chat.js +583 -35
- package/dist/cli/chat.js.map +1 -1
- package/dist/cli/duties.d.ts +11 -0
- package/dist/cli/duties.d.ts.map +1 -1
- package/dist/cli/duties.js +31 -0
- package/dist/cli/duties.js.map +1 -1
- package/dist/cli/edit.d.ts.map +1 -1
- package/dist/cli/edit.js +3 -2
- package/dist/cli/edit.js.map +1 -1
- package/dist/cli/execute.d.ts.map +1 -1
- package/dist/cli/execute.js +10 -3
- package/dist/cli/execute.js.map +1 -1
- package/dist/cli/loop-executor.d.ts +16 -0
- package/dist/cli/loop-executor.d.ts.map +1 -1
- package/dist/cli/loop-executor.js +9 -0
- package/dist/cli/loop-executor.js.map +1 -1
- package/dist/cli/models.d.ts.map +1 -1
- package/dist/cli/models.js +55 -0
- package/dist/cli/models.js.map +1 -1
- package/dist/cli/plan.d.ts.map +1 -1
- package/dist/cli/plan.js +18 -6
- package/dist/cli/plan.js.map +1 -1
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +23 -8
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/ui/answer-view.d.ts +38 -0
- package/dist/cli/ui/answer-view.d.ts.map +1 -0
- package/dist/cli/ui/answer-view.js +72 -0
- package/dist/cli/ui/answer-view.js.map +1 -0
- package/dist/cli/ui/cards.d.ts +61 -0
- package/dist/cli/ui/cards.d.ts.map +1 -0
- package/dist/cli/ui/cards.js +141 -0
- package/dist/cli/ui/cards.js.map +1 -0
- package/dist/cli/ui/highlight.d.ts +32 -0
- package/dist/cli/ui/highlight.d.ts.map +1 -0
- package/dist/cli/ui/highlight.js +261 -0
- package/dist/cli/ui/highlight.js.map +1 -0
- package/dist/cli/ui/index.d.ts +21 -0
- package/dist/cli/ui/index.d.ts.map +1 -0
- package/dist/cli/ui/index.js +14 -0
- package/dist/cli/ui/index.js.map +1 -0
- package/dist/cli/ui/markdown.d.ts +90 -0
- package/dist/cli/ui/markdown.d.ts.map +1 -0
- package/dist/cli/ui/markdown.js +292 -0
- package/dist/cli/ui/markdown.js.map +1 -0
- package/dist/cli/ui/presenter.d.ts +147 -0
- package/dist/cli/ui/presenter.d.ts.map +1 -0
- package/dist/cli/ui/presenter.js +313 -0
- package/dist/cli/ui/presenter.js.map +1 -0
- package/dist/cli/ui/render.d.ts +52 -0
- package/dist/cli/ui/render.d.ts.map +1 -0
- package/dist/cli/ui/render.js +384 -0
- package/dist/cli/ui/render.js.map +1 -0
- package/dist/cli/ui/theme.d.ts +36 -0
- package/dist/cli/ui/theme.d.ts.map +1 -0
- package/dist/cli/ui/theme.js +92 -0
- package/dist/cli/ui/theme.js.map +1 -0
- package/dist/inference/model-entitlement.d.ts +10 -0
- package/dist/inference/model-entitlement.d.ts.map +1 -1
- package/dist/inference/model-entitlement.js +1 -1
- package/dist/inference/model-entitlement.js.map +1 -1
- package/dist/learning/auto-router.d.ts +38 -1
- package/dist/learning/auto-router.d.ts.map +1 -1
- package/dist/learning/auto-router.js +139 -8
- package/dist/learning/auto-router.js.map +1 -1
- package/dist/learning/duty-lease.d.ts +163 -0
- package/dist/learning/duty-lease.d.ts.map +1 -0
- package/dist/learning/duty-lease.js +301 -0
- package/dist/learning/duty-lease.js.map +1 -0
- package/dist/learning/model-first-router.d.ts +14 -0
- package/dist/learning/model-first-router.d.ts.map +1 -1
- package/dist/learning/model-first-router.js +54 -17
- package/dist/learning/model-first-router.js.map +1 -1
- package/dist/learning/model-warmup.d.ts +92 -0
- package/dist/learning/model-warmup.d.ts.map +1 -1
- package/dist/learning/model-warmup.js +137 -2
- package/dist/learning/model-warmup.js.map +1 -1
- package/dist/learning/reasoning-trace.d.ts +21 -1
- package/dist/learning/reasoning-trace.d.ts.map +1 -1
- package/dist/learning/reasoning-trace.js.map +1 -1
- package/dist/learning/resilient-call.d.ts.map +1 -1
- package/dist/learning/resilient-call.js +70 -4
- package/dist/learning/resilient-call.js.map +1 -1
- package/dist/learning/retry-patience.d.ts +139 -0
- package/dist/learning/retry-patience.d.ts.map +1 -0
- package/dist/learning/retry-patience.js +169 -0
- package/dist/learning/retry-patience.js.map +1 -0
- package/dist/tools/process-liveness.d.ts +227 -0
- package/dist/tools/process-liveness.d.ts.map +1 -0
- package/dist/tools/process-liveness.js +486 -0
- package/dist/tools/process-liveness.js.map +1 -0
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +4 -2
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/run-registry.d.ts +282 -0
- package/dist/tools/run-registry.d.ts.map +1 -0
- package/dist/tools/run-registry.js +544 -0
- package/dist/tools/run-registry.js.map +1 -0
- package/dist/tools/run-terminal.d.ts +100 -2
- package/dist/tools/run-terminal.d.ts.map +1 -1
- package/dist/tools/run-terminal.js +841 -67
- package/dist/tools/run-terminal.js.map +1 -1
- package/dist/tools/tool-loop.d.ts +87 -3
- package/dist/tools/tool-loop.d.ts.map +1 -1
- package/dist/tools/tool-loop.js +375 -6
- package/dist/tools/tool-loop.js.map +1 -1
- package/dist/web-dashboard/chat-console.d.ts +25 -0
- package/dist/web-dashboard/chat-console.d.ts.map +1 -1
- package/dist/web-dashboard/chat-console.js +16 -1
- package/dist/web-dashboard/chat-console.js.map +1 -1
- package/dist/web-dashboard/server.d.ts.map +1 -1
- package/dist/web-dashboard/server.js +7 -1
- package/dist/web-dashboard/server.js.map +1 -1
- package/dist/web-dashboard/src/types.d.ts +32 -1
- package/dist/web-dashboard/src/types.d.ts.map +1 -1
- package/dist/web-dashboard/task-registry.d.ts +98 -0
- package/dist/web-dashboard/task-registry.d.ts.map +1 -0
- package/dist/web-dashboard/task-registry.js +159 -0
- package/dist/web-dashboard/task-registry.js.map +1 -0
- package/dist/web-dashboard/task-runner.d.ts +155 -5
- package/dist/web-dashboard/task-runner.d.ts.map +1 -1
- package/dist/web-dashboard/task-runner.js +505 -14
- package/dist/web-dashboard/task-runner.js.map +1 -1
- package/package.json +4 -1
- package/src/web-dashboard/public/assets/index-DycGBWwy.js +236 -0
- package/src/web-dashboard/public/assets/index-DycGBWwy.js.map +1 -0
- package/src/web-dashboard/public/index.html +1 -1
- package/src/web-dashboard/public/assets/index-yky0dw5e.js +0 -236
- package/src/web-dashboard/public/assets/index-yky0dw5e.js.map +0 -1
|
@@ -32,7 +32,10 @@
|
|
|
32
32
|
*/
|
|
33
33
|
import { spawn } from 'node:child_process';
|
|
34
34
|
import { existsSync } from 'node:fs';
|
|
35
|
+
import { randomUUID } from 'node:crypto';
|
|
35
36
|
import { applyRemediationAutoFixes, diagnoseFailure, formatAutoApply, formatRemediation, remediationAutoApplyEnabled, } from './remediation-ladder.js';
|
|
37
|
+
import { classifyLiveness, formatSpan, killProcessTree, livenessThresholds, processAlive, processTableSnapshot, sampleProcessTree, tableAlive, lookupProcess, } from './process-liveness.js';
|
|
38
|
+
import { loadRuns, processIdentityCheck, outputWindow, saveRun, spoolBytes, spoolPath, readSpool, sweepSpools, SpoolWriter, } from './run-registry.js';
|
|
36
39
|
import { maskSenderId } from '../utils/mask.js';
|
|
37
40
|
import { decideStateChange } from '../learning/autonomy-policy.js';
|
|
38
41
|
import { envelopeCoversAction } from '../learning/intent-envelope.js';
|
|
@@ -52,12 +55,85 @@ const DEFAULT_TIMEOUT_MS = 120_000;
|
|
|
52
55
|
* turn (4 minutes of a 5-minute run) while the dashboard console showed nothing,
|
|
53
56
|
* and then the agent gave up. Two minutes of silence in front of a person is not
|
|
54
57
|
* patience, it is a bug. Sixty seconds still covers a real test/build on this
|
|
55
|
-
* class of project, and an explicit `timeout_ms` (up to
|
|
56
|
-
*
|
|
58
|
+
* class of project, and an explicit `timeout_ms` (up to MAX_WAIT_MS) still buys
|
|
59
|
+
* the model as much time as the work genuinely needs.
|
|
57
60
|
*/
|
|
58
61
|
const INTERACTIVE_DEFAULT_TIMEOUT_MS = 60_000;
|
|
59
|
-
/**
|
|
60
|
-
|
|
62
|
+
/**
|
|
63
|
+
* The longest this tool WAITS inside one call before handing the command to the
|
|
64
|
+
* monitored (`still running`) path. This is NOT a kill switch any more — see
|
|
65
|
+
* `assessCommandTime` and the detached-run registry below: reaching it means the
|
|
66
|
+
* tool stops blocking and reports, never that the work is destroyed.
|
|
67
|
+
*/
|
|
68
|
+
const MAX_WAIT_MS = 1_800_000;
|
|
69
|
+
/**
|
|
70
|
+
* What a command CLASS actually needs, so the wait is chosen from the work
|
|
71
|
+
* instead of a single flat number.
|
|
72
|
+
*
|
|
73
|
+
* The measured defect this replaces: every command that was not a quick read got
|
|
74
|
+
* 60–120s and then a SIGTERM, so `npm test` on a real repo — minutes by nature,
|
|
75
|
+
* and the very command the tool exists for — was killed mid-run, its partial
|
|
76
|
+
* output reported as a failure, and the turn built conclusions on a run that
|
|
77
|
+
* never finished. Freebuff-style behaviour is the opposite: assess the time the
|
|
78
|
+
* work needs, wait for it, and when it outlives the wait, KEEP IT RUNNING and
|
|
79
|
+
* report what it is doing — never destroy it and call that a result.
|
|
80
|
+
*
|
|
81
|
+
* Order matters: the first match wins, so the most specific/expensive class is
|
|
82
|
+
* listed first (a `docker build` is not an `npm install`).
|
|
83
|
+
*/
|
|
84
|
+
const LONG_COMMAND_BUDGETS = [
|
|
85
|
+
{
|
|
86
|
+
re: /(^|\s)(docker\s+(build|compose\s+up)|make|cmake\s+--build|gradle(w)?\s|mvn\s)/,
|
|
87
|
+
budgetMs: 900_000,
|
|
88
|
+
why: 'a container/compiler build',
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
re: /(^|\s)(npm|pnpm|yarn|bun)\s+(run\s+)?(test|test:\S+|ci|install|i|add)\b|(^|\s)(vitest|jest|pytest|go\s+test|cargo\s+test|ng\s+test)\b/,
|
|
92
|
+
budgetMs: 900_000,
|
|
93
|
+
why: 'a test suite / dependency install',
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
re: /(^|\s)(npm|pnpm|yarn|bun)\s+run\s+(build|compile|bundle|dist)\b|(^|\s)(tsc|vite\s+build|webpack|rollup|esbuild|cargo\s+build|go\s+build)\b/,
|
|
97
|
+
budgetMs: 600_000,
|
|
98
|
+
why: 'a build',
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
re: /(^|\s)(pip3?|poetry|bundle|composer)\s+(install|add|sync)\b/,
|
|
102
|
+
budgetMs: 600_000,
|
|
103
|
+
why: 'a dependency install',
|
|
104
|
+
},
|
|
105
|
+
];
|
|
106
|
+
/**
|
|
107
|
+
* The wait budget this command deserves, and the reason (for the message).
|
|
108
|
+
* Exported so the assessment is testable on its own — it is a decision, not an
|
|
109
|
+
* implementation detail.
|
|
110
|
+
*/
|
|
111
|
+
export function assessCommandTime(command) {
|
|
112
|
+
const cmd = command.toLowerCase();
|
|
113
|
+
for (const { re, budgetMs, why } of LONG_COMMAND_BUDGETS) {
|
|
114
|
+
if (re.test(cmd))
|
|
115
|
+
return { budgetMs, why };
|
|
116
|
+
}
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* How long this call waits before handing the command to monitoring.
|
|
121
|
+
*
|
|
122
|
+
* Priority, and why in that order:
|
|
123
|
+
* 1. an explicit `timeout_ms` — the model knows something the classifier does not;
|
|
124
|
+
* 2. the ASSESSED class — a full test suite or a build is minutes by nature, and
|
|
125
|
+
* giving it the short default is what produced the kill-mid-run failures;
|
|
126
|
+
* 3. the room — a person watching waits less (P7), an unattended run waits more.
|
|
127
|
+
*
|
|
128
|
+
* Exported because this is a decision, not an implementation detail: it is what
|
|
129
|
+
* `run_terminal` believes the work needs, and it should be asserted directly
|
|
130
|
+
* rather than inferred from a message.
|
|
131
|
+
*/
|
|
132
|
+
export function resolveWaitMs(command, explicitMs, interactive) {
|
|
133
|
+
const assessed = assessCommandTime(command);
|
|
134
|
+
const base = interactive ? INTERACTIVE_DEFAULT_TIMEOUT_MS : DEFAULT_TIMEOUT_MS;
|
|
135
|
+
return Math.min(MAX_WAIT_MS, Math.max(1000, Math.floor(explicitMs ?? assessed?.budgetMs ?? base)));
|
|
136
|
+
}
|
|
61
137
|
/**
|
|
62
138
|
* P7 — how many times the SAME command (at the SAME timeout) may fail before the
|
|
63
139
|
* next call is refused.
|
|
@@ -114,16 +190,580 @@ function recordCommandFailure(key, now) {
|
|
|
114
190
|
function clearCommandFailure(key) {
|
|
115
191
|
commandFailureStreaks.delete(key);
|
|
116
192
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
193
|
+
const detachedRuns = new Map();
|
|
194
|
+
/** Bounded: a long session must not accumulate output buffers. */
|
|
195
|
+
const MAX_DETACHED_RUNS = 20;
|
|
196
|
+
/** ─── The liveness watchdog: identifying a HANG, and acting on it ────────────
|
|
197
|
+
*
|
|
198
|
+
* Monitoring without a heartbeat recreates the opposite defect: a command that
|
|
199
|
+
* is genuinely stuck (a prompt with no stdin, a lock, a socket that never
|
|
200
|
+
* answers) runs forever, and "still running" is not a diagnosis. Terminating on
|
|
201
|
+
* a clock is the bug this whole file already removed, so the replacement is:
|
|
202
|
+
*
|
|
203
|
+
* identify — sample the run's process TREE for cumulative CPU every so often;
|
|
204
|
+
* silence ALONE is never a hang (a real suite is quiet for
|
|
205
|
+
* minutes), but silence AND a measurable absence of CPU is;
|
|
206
|
+
* report — a confirmed hang is surfaced on `poll`/`list` with its evidence;
|
|
207
|
+
* act — and only then, after a further grace in which the run shows
|
|
208
|
+
* neither output nor CPU, is it stopped (SIGTERM, then SIGKILL on
|
|
209
|
+
* the whole tree). Any output or any CPU at any sample clears the
|
|
210
|
+
* hang, so a slow-but-working command is never touched.
|
|
211
|
+
*
|
|
212
|
+
* `NUVIRA_HANG_REAP_MS=0` keeps the identification and reporting and disables
|
|
213
|
+
* the stopping (report-only); `NUVIRA_HANG_QUIET_MS` and `NUVIRA_HANG_CPU_MS`
|
|
214
|
+
* tune what counts as a hang; `NUVIRA_HANG_POLL_MS` tunes the sample rate.
|
|
215
|
+
*/
|
|
216
|
+
function envMs(name, fallback) {
|
|
217
|
+
const raw = Number.parseInt(String(process.env[name] ?? '').trim(), 10);
|
|
218
|
+
return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
|
|
219
|
+
}
|
|
220
|
+
/** How often a monitored run's process tree is sampled. */
|
|
221
|
+
function hangPollMs() {
|
|
222
|
+
return Math.max(1_000, envMs('NUVIRA_HANG_POLL_MS', 30_000));
|
|
223
|
+
}
|
|
224
|
+
/** How long a CONFIRMED hang is left running before it is stopped (0 = never). */
|
|
225
|
+
export function hangReapMs() {
|
|
226
|
+
return envMs('NUVIRA_HANG_REAP_MS', 300_000);
|
|
227
|
+
}
|
|
228
|
+
let watchdog = null;
|
|
229
|
+
let lastSampleAt = 0;
|
|
230
|
+
function ensureWatchdog() {
|
|
231
|
+
if (watchdog)
|
|
232
|
+
return;
|
|
233
|
+
watchdog = setInterval(() => sampleMonitoredRuns(), hangPollMs());
|
|
234
|
+
// A monitored run must never hold the process open — the CLI exits when the
|
|
235
|
+
// turn ends, and the run's own exit handler is what reports its result.
|
|
236
|
+
watchdog.unref?.();
|
|
237
|
+
}
|
|
238
|
+
function stopWatchdogIfIdle() {
|
|
239
|
+
if (!watchdog)
|
|
240
|
+
return;
|
|
241
|
+
if ([...detachedRuns.values()].some((r) => r.status === 'running'))
|
|
242
|
+
return;
|
|
243
|
+
clearInterval(watchdog);
|
|
244
|
+
watchdog = null;
|
|
245
|
+
}
|
|
246
|
+
/** Stop a run the watchdog has established is hung, and say why. */
|
|
247
|
+
function reapHungRun(run, why, table) {
|
|
248
|
+
if (!maySignal(run)) {
|
|
249
|
+
// A recovered run whose pid could not be re-verified is NEVER signalled — the
|
|
250
|
+
// pid may belong to someone else by now. It stays flagged and reported, and
|
|
251
|
+
// `exitReason` is NOT set: it is still running, and saying otherwise would be
|
|
252
|
+
// the kind of lie the rest of this file exists to avoid.
|
|
253
|
+
run.hangUnstoppable = true;
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
run.reaped = true;
|
|
257
|
+
run.exitReason = why;
|
|
258
|
+
killProcessTree(run.pid ?? -1, 'SIGTERM', run.child, table);
|
|
259
|
+
run.killGrace = setTimeout(() => {
|
|
260
|
+
killProcessTree(run.pid ?? -1, 'SIGKILL', run.child);
|
|
261
|
+
}, 5_000);
|
|
262
|
+
run.killGrace.unref?.();
|
|
263
|
+
// Persisted IMMEDIATELY: the reason a run was stopped must survive the very
|
|
264
|
+
// crash the registry exists for.
|
|
265
|
+
persistRun(run);
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* One heartbeat over every monitored run.
|
|
269
|
+
*
|
|
270
|
+
* Exported (and takes `now`) so the decision can be driven deterministically in
|
|
271
|
+
* a test instead of waited for.
|
|
272
|
+
*/
|
|
273
|
+
export function sampleMonitoredRuns(now = Date.now()) {
|
|
274
|
+
const thresholds = livenessThresholds();
|
|
275
|
+
const reapAfter = hangReapMs();
|
|
276
|
+
const sampleWindowMs = lastSampleAt > 0 ? now - lastSampleAt : hangPollMs();
|
|
277
|
+
lastSampleAt = now;
|
|
278
|
+
// ONE read per tick, shared by every run, by the reconciliation below and by any
|
|
279
|
+
// signalling: on Windows each read is a PowerShell start, and a tick must not
|
|
280
|
+
// cost one per run. The registry is read at most once too, and only when there
|
|
281
|
+
// is a recovered run to reconcile.
|
|
282
|
+
const table = processTableSnapshot();
|
|
283
|
+
const needsRegistry = [...detachedRuns.values()].some((r) => r.reattached && r.status === 'running');
|
|
284
|
+
const recorded = needsRegistry ? loadRuns() : undefined;
|
|
285
|
+
for (const run of [...detachedRuns.values()]) {
|
|
286
|
+
if (run.status !== 'running')
|
|
287
|
+
continue;
|
|
288
|
+
refreshReattachedRun(run, now, { table, ...(recorded ? { recorded } : {}) });
|
|
289
|
+
if (run.status !== 'running')
|
|
290
|
+
continue; // it ended while we were not watching
|
|
291
|
+
const sample = sampleProcessTree(run.pid ?? -1, table);
|
|
292
|
+
// A run this process spawned is alive until its own exit event says otherwise.
|
|
293
|
+
// A recovered one has no such event, so the process table is the only witness
|
|
294
|
+
// — and "cannot tell" (null) must read as alive, never as finished.
|
|
295
|
+
// `!== false` on purpose: a platform that cannot read the process table
|
|
296
|
+
// reports `null`, and "cannot tell" must read as ALIVE (a recovered run is
|
|
297
|
+
// never declared dead by a guess).
|
|
298
|
+
const alive = run.reattached
|
|
299
|
+
? tableAlive(table, run.pid ?? -1) !== false
|
|
300
|
+
: run.child !== undefined && run.child.exitCode === null && run.child.signalCode === null;
|
|
301
|
+
const verdict = classifyLiveness({
|
|
302
|
+
alive,
|
|
303
|
+
bytes: run.bytes,
|
|
304
|
+
quietMs: now - run.lastOutputAt,
|
|
305
|
+
ageMs: now - run.startedAt,
|
|
306
|
+
cpuMs: sample.cpuMs,
|
|
307
|
+
prevCpuMs: run.cpuMs ?? null,
|
|
308
|
+
sampleWindowMs,
|
|
309
|
+
}, thresholds);
|
|
310
|
+
run.cpuMs = sample.cpuMs;
|
|
311
|
+
run.sampledAt = now;
|
|
312
|
+
run.verdict = verdict.verdict;
|
|
313
|
+
run.verdictWhy = verdict.why;
|
|
314
|
+
if (verdict.verdict === 'hung') {
|
|
315
|
+
// The grace starts once, at the FIRST confirmed hang — and every later
|
|
316
|
+
// sample that still shows a hang keeps it running toward the reap.
|
|
317
|
+
run.hungSince = run.hungSince ?? now;
|
|
318
|
+
if (reapAfter > 0 && now - run.hungSince >= reapAfter && !run.stopRequested) {
|
|
319
|
+
reapHungRun(run, `hung: no output and no CPU for ${formatSpan(now - run.startedAt)} ` +
|
|
320
|
+
`(confirmed hung for ${formatSpan(now - run.hungSince)})`, table);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
else {
|
|
324
|
+
// It spoke or it computed — it is not hung, and the grace starts over.
|
|
325
|
+
run.hungSince = null;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
stopWatchdogIfIdle();
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* The marker a result carries when the command OUTLIVED its wait budget and is
|
|
332
|
+
* still running under monitoring.
|
|
333
|
+
*
|
|
334
|
+
* It is neither success nor failure, and the loop must read it as neither:
|
|
335
|
+
*
|
|
336
|
+
* - NOT success — a live run has verified nothing yet. Counting it as proof is
|
|
337
|
+
* how a turn ends up claiming "the tests pass" about a suite that is still on
|
|
338
|
+
* its first file.
|
|
339
|
+
* - NOT failure — reporting it as one is the bug this whole change removes: the
|
|
340
|
+
* work was destroyed (or, now, is still going) and the user is told it broke.
|
|
341
|
+
*
|
|
342
|
+
* So it is its own state, and this exported string is the one both the tool and
|
|
343
|
+
* the loop's success classifier read (see `ranOk` in tool-loop.ts).
|
|
344
|
+
*/
|
|
345
|
+
export const RUN_TERMINAL_INCOMPLETE_PREFIX = '⏳ run_terminal: still running';
|
|
346
|
+
/**
|
|
347
|
+
* Write a run's record to the durable registry (see `run-registry.ts`).
|
|
348
|
+
*
|
|
349
|
+
* Best-effort by construction: a run must never fail because a state file could
|
|
350
|
+
* not be written. What it buys is that the NEXT process can still answer "what is
|
|
351
|
+
* running, and how did it end?" — which is otherwise lost with this one.
|
|
352
|
+
*/
|
|
353
|
+
function persistRun(run) {
|
|
354
|
+
if (run.pid === null || run.pid <= 0)
|
|
355
|
+
return;
|
|
356
|
+
saveRun({
|
|
357
|
+
id: run.id,
|
|
358
|
+
command: run.command,
|
|
359
|
+
cwd: run.cwd,
|
|
360
|
+
pid: run.pid,
|
|
361
|
+
startedAt: run.startedAt,
|
|
362
|
+
status: run.status,
|
|
363
|
+
exitCode: run.exitCode,
|
|
364
|
+
...(run.status === 'done' ? { endedAt: run.endedAt ?? Date.now() } : {}),
|
|
365
|
+
...(run.exitReason ? { exitReason: run.exitReason } : {}),
|
|
366
|
+
...(run.reaped ? { reaped: true } : {}),
|
|
367
|
+
...(run.stopRequested ? { stopRequested: true } : {}),
|
|
368
|
+
...(run.osStartedAt ? { osStartedAt: run.osStartedAt } : {}),
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Re-verify a recovered run's pid against BOTH witnesses, keeping the evidence.
|
|
373
|
+
* One process-table read, and the reason is reported alongside the verdict.
|
|
374
|
+
*/
|
|
375
|
+
function identityOf(entry) {
|
|
376
|
+
const check = processIdentityCheck({ command: entry.command, osStartedAt: entry.osStartedAt ?? null }, lookupProcess(entry.pid));
|
|
377
|
+
return { pidVerified: check.verdict, pidVerifiedWhy: check.why };
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* Read the OS's start token for a process we are holding a handle to.
|
|
381
|
+
*
|
|
382
|
+
* Called ONCE per run, at hand-off, for a reason: it has to be read while the
|
|
383
|
+
* process is certainly still ours. Reading it later (at re-attach, after a
|
|
384
|
+
* restart) would be asking the recycled pid to identify itself — the very thing
|
|
385
|
+
* this token exists to check. A failed read is `null`, which downgrades the
|
|
386
|
+
* recovered run to the command-line witness alone rather than inventing one.
|
|
387
|
+
*/
|
|
388
|
+
function readOsStartToken(pid) {
|
|
389
|
+
if (pid === null || pid <= 0)
|
|
390
|
+
return null;
|
|
391
|
+
try {
|
|
392
|
+
return lookupProcess(pid)?.startedAtText ?? null;
|
|
393
|
+
}
|
|
394
|
+
catch {
|
|
395
|
+
return null;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
/**
|
|
399
|
+
* May this run be signalled?
|
|
400
|
+
*
|
|
401
|
+
* A run this process started — yes, we hold the child handle. A run RECOVERED
|
|
402
|
+
* from the registry — only once its pid has been re-verified as still being that
|
|
403
|
+
* command, because a pid that outlived the process table it came from may now
|
|
404
|
+
* belong to something else, and this tool only ever signals work it started.
|
|
405
|
+
*/
|
|
406
|
+
function maySignal(run) {
|
|
407
|
+
return run.reattached !== true || run.pidVerified === true;
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* Bring a RE-ATTACHED run up to date with the truth on disk.
|
|
411
|
+
*
|
|
412
|
+
* A recovered run has no exit event — the process that would have fired one may
|
|
413
|
+
* be gone — so nothing would ever move it out of `running`: the poll would report
|
|
414
|
+
* "still running" forever about a command that finished minutes ago. Two
|
|
415
|
+
* witnesses settle it: the registry entry (the owning process records the exit
|
|
416
|
+
* code, the stop reason and the reap), and the process table.
|
|
417
|
+
*/
|
|
418
|
+
function refreshReattachedRun(run, now = Date.now(),
|
|
419
|
+
/** What a tick already read, so reconciling cost stays O(1) reads, not O(runs). */
|
|
420
|
+
context = {}) {
|
|
421
|
+
if (!run.reattached || run.status !== 'running')
|
|
422
|
+
return;
|
|
423
|
+
const recorded = (context.recorded ?? loadRuns()).find((r) => r.id === run.id);
|
|
424
|
+
if (recorded && recorded.status === 'done') {
|
|
425
|
+
run.status = 'done';
|
|
426
|
+
run.exitCode = recorded.exitCode;
|
|
427
|
+
run.endedAt = recorded.endedAt ?? now;
|
|
428
|
+
if (recorded.exitReason)
|
|
429
|
+
run.exitReason = recorded.exitReason;
|
|
430
|
+
if (recorded.reaped)
|
|
431
|
+
run.reaped = true;
|
|
432
|
+
if (recorded.stopRequested)
|
|
433
|
+
run.stopRequested = true;
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
// The tick's own table when it has one; otherwise (the poll path, a single call)
|
|
437
|
+
// a read of its own. `false` is the only value that means gone.
|
|
438
|
+
const alive = context.table === undefined ? processAlive(run.pid ?? -1) : tableAlive(context.table, run.pid ?? -1);
|
|
439
|
+
if (alive === false) {
|
|
440
|
+
run.status = 'done';
|
|
441
|
+
run.endedAt = now;
|
|
442
|
+
run.exitReason =
|
|
443
|
+
run.exitReason ?? 'the process is gone, and its exit code was not recorded — nothing was watching when it ended';
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
let hydrated = false;
|
|
447
|
+
/**
|
|
448
|
+
* Re-attach to the runs a PREVIOUS process left behind.
|
|
449
|
+
*
|
|
450
|
+
* Without this, every restart orphaned the record of work that is still going:
|
|
451
|
+
* the command keeps running, nothing can poll it, and the turn that started it
|
|
452
|
+
* has no way to collect its real verdict. Runs whose process is gone are marked
|
|
453
|
+
* finished (with an honest "its exit code was not recorded" reason), and live
|
|
454
|
+
* ones are re-attached to be polled, sampled and — only after their pid is
|
|
455
|
+
* re-verified — stopped.
|
|
456
|
+
*
|
|
457
|
+
* Returns how many runs were recovered.
|
|
458
|
+
*/
|
|
459
|
+
export function hydrateMonitoredRuns() {
|
|
460
|
+
if (hydrated)
|
|
461
|
+
return 0;
|
|
462
|
+
hydrated = true;
|
|
463
|
+
let recovered = 0;
|
|
464
|
+
for (const entry of loadRuns()) {
|
|
465
|
+
if (detachedRuns.has(entry.id))
|
|
466
|
+
continue;
|
|
467
|
+
const alive = processAlive(entry.pid);
|
|
468
|
+
const finished = entry.status === 'done' || alive === false;
|
|
469
|
+
const run = {
|
|
470
|
+
id: entry.id,
|
|
471
|
+
command: entry.command,
|
|
472
|
+
cwd: entry.cwd,
|
|
473
|
+
pid: entry.pid,
|
|
474
|
+
startedAt: entry.startedAt,
|
|
475
|
+
// The silence clock starts NOW, not at the start of the run: we have no
|
|
476
|
+
// output timeline for a recovered run, and running it from `startedAt`
|
|
477
|
+
// would judge a long-quiet-but-working command hung the instant we looked.
|
|
478
|
+
lastOutputAt: entry.endedAt ?? Date.now(),
|
|
479
|
+
bytes: spoolBytes(entry.id),
|
|
480
|
+
out: '',
|
|
481
|
+
err: '',
|
|
482
|
+
status: finished ? 'done' : 'running',
|
|
483
|
+
exitCode: entry.exitCode,
|
|
484
|
+
reattached: true,
|
|
485
|
+
// TWO witnesses, decided by `processIdentityVerdict`: the command line, and
|
|
486
|
+
// the OS start token captured at hand-off. The token is what turns "I cannot
|
|
487
|
+
// prove this is ours, so I will only watch it" into "this IS the process I
|
|
488
|
+
// started, so it can still be stopped" — and, in the other direction, what
|
|
489
|
+
// refuses to adopt a recycled pid running the SAME command line.
|
|
490
|
+
...(finished
|
|
491
|
+
? { pidVerified: false }
|
|
492
|
+
: identityOf(entry)),
|
|
493
|
+
osStartedAt: entry.osStartedAt ?? null,
|
|
494
|
+
truncatedOutput: true, // the head lives in the spool, never in memory
|
|
495
|
+
...(entry.endedAt !== undefined ? { endedAt: entry.endedAt } : {}),
|
|
496
|
+
...(entry.exitReason ? { exitReason: entry.exitReason } : {}),
|
|
497
|
+
...(entry.reaped ? { reaped: true } : {}),
|
|
498
|
+
...(entry.stopRequested ? { stopRequested: true } : {}),
|
|
499
|
+
};
|
|
500
|
+
if (run.status === 'done' && entry.status !== 'done') {
|
|
501
|
+
run.exitReason =
|
|
502
|
+
'it ended while nothing was watching (this process restarted) — its exit code was not recorded';
|
|
503
|
+
persistRun(run);
|
|
504
|
+
}
|
|
505
|
+
detachedRuns.set(run.id, run);
|
|
506
|
+
recovered += 1;
|
|
507
|
+
}
|
|
508
|
+
if (recovered > 0) {
|
|
509
|
+
pruneDetachedRuns();
|
|
510
|
+
ensureWatchdog();
|
|
511
|
+
}
|
|
512
|
+
// Records are capped; files are not a record. Sweeping here — the one moment
|
|
513
|
+
// the live set is exactly known — is what keeps "bounded state" true of the
|
|
514
|
+
// disk and not just of the JSON.
|
|
515
|
+
sweepSpools([...detachedRuns.keys()]);
|
|
516
|
+
return recovered;
|
|
517
|
+
}
|
|
518
|
+
/** Test/DI seam: forget the monitored runs. Kills nothing (they keep running). */
|
|
519
|
+
export function resetDetachedRuns() {
|
|
520
|
+
detachedRuns.clear();
|
|
521
|
+
if (watchdog)
|
|
522
|
+
clearInterval(watchdog);
|
|
523
|
+
watchdog = null;
|
|
524
|
+
lastSampleAt = 0;
|
|
525
|
+
hydrated = false;
|
|
526
|
+
}
|
|
527
|
+
function pruneDetachedRuns() {
|
|
528
|
+
if (detachedRuns.size <= MAX_DETACHED_RUNS)
|
|
529
|
+
return;
|
|
530
|
+
// Finished first — they are only kept so a late poll can read their result.
|
|
531
|
+
for (const [id, run] of detachedRuns) {
|
|
532
|
+
if (detachedRuns.size <= MAX_DETACHED_RUNS)
|
|
533
|
+
break;
|
|
534
|
+
if (run.status === 'done')
|
|
535
|
+
detachedRuns.delete(id);
|
|
536
|
+
}
|
|
537
|
+
if (detachedRuns.size > MAX_DETACHED_RUNS) {
|
|
538
|
+
// All still running: drop the oldest MONITORING entry. The process is NOT
|
|
539
|
+
// killed — it simply stops being pollable by id.
|
|
540
|
+
const oldest = [...detachedRuns.values()].sort((a, b) => a.startedAt - b.startedAt)[0];
|
|
541
|
+
if (oldest)
|
|
542
|
+
detachedRuns.delete(oldest.id);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
/** The tail of a buffer, capped like the output this tool returns. */
|
|
546
|
+
function outputTail(s) {
|
|
547
|
+
return s.trim().length > MAX_OUTPUT_CHARS ? s.trim().slice(-MAX_OUTPUT_CHARS) : s.trim();
|
|
548
|
+
}
|
|
549
|
+
/** `45s` / `3m 05s` — how long, for a human reading the tool result. */
|
|
550
|
+
function elapsedLabel(ms) {
|
|
551
|
+
const totalSeconds = Math.max(0, Math.round(ms / 1000));
|
|
552
|
+
if (totalSeconds < 60)
|
|
553
|
+
return `${totalSeconds}s`;
|
|
554
|
+
return `${Math.floor(totalSeconds / 60)}m ${String(totalSeconds % 60).padStart(2, '0')}s`;
|
|
555
|
+
}
|
|
556
|
+
/**
|
|
557
|
+
* This run's output, as the model should see it.
|
|
558
|
+
*
|
|
559
|
+
* The in-memory buffer is used when it is COMPLETE — the common case, and the
|
|
560
|
+
* existing behaviour. When the beginning was lost to the buffer's bound, or the
|
|
561
|
+
* run was recovered from disk and there is no buffer at all, the spooled
|
|
562
|
+
* head+tail is read instead: a long run's diagnosis needs the first lines (what
|
|
563
|
+
* it was doing, how it was invoked) as much as the last.
|
|
564
|
+
*/
|
|
565
|
+
function runOutputText(run) {
|
|
566
|
+
if (!run.truncatedOutput && !run.reattached && (run.out || run.err))
|
|
567
|
+
return outputTail(run.out || run.err);
|
|
568
|
+
const spooled = readSpool(run.id);
|
|
569
|
+
if (spooled) {
|
|
570
|
+
const window = outputWindow(spooled, MAX_OUTPUT_CHARS);
|
|
571
|
+
if (window.text.trim())
|
|
572
|
+
return window.text.trim();
|
|
573
|
+
}
|
|
574
|
+
return outputTail(run.out || run.err);
|
|
575
|
+
}
|
|
576
|
+
/** The liveness of a monitored run, as a short badge for `list`/`poll`. */
|
|
577
|
+
function livenessBadge(run) {
|
|
578
|
+
if (run.reaped)
|
|
579
|
+
return '🧯 stopped (hung)';
|
|
580
|
+
if (run.status === 'done')
|
|
581
|
+
return `✔ finished (exit ${run.exitCode ?? '?'})`;
|
|
582
|
+
const elapsed = `⏳ running ${elapsedLabel(Date.now() - run.startedAt)}`;
|
|
583
|
+
switch (run.verdict) {
|
|
584
|
+
case 'hung':
|
|
585
|
+
return `${elapsed} · 🚫 HUNG (no output and no CPU) — will be stopped after the grace`;
|
|
586
|
+
case 'stalled':
|
|
587
|
+
return `${elapsed} · ⚠️ quiet — ${run.verdictWhy ?? 'no output recently'}`;
|
|
588
|
+
case 'working':
|
|
589
|
+
return `${elapsed} · ✅ working — ${run.verdictWhy ?? 'making progress'}`;
|
|
590
|
+
default:
|
|
591
|
+
return elapsed;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* Everything currently being followed.
|
|
596
|
+
*
|
|
597
|
+
* Hydrates first, so this answers for the runs a PREVIOUS process left behind as
|
|
598
|
+
* well as this one's — otherwise a restart would make the whole list vanish while
|
|
599
|
+
* the commands are still running.
|
|
600
|
+
*/
|
|
601
|
+
export function listDetachedRuns() {
|
|
602
|
+
const recovered = hydrateMonitoredRuns();
|
|
603
|
+
if (detachedRuns.size === 0) {
|
|
604
|
+
return 'run_terminal: nothing is being monitored — no command has outlived its wait budget in this process.';
|
|
605
|
+
}
|
|
606
|
+
const lines = [...detachedRuns.values()]
|
|
607
|
+
.sort((a, b) => a.startedAt - b.startedAt)
|
|
608
|
+
.map((r) => `- run_terminal[${r.id}] ${livenessBadge(r)} · ${r.bytes} bytes · \`${maskSenderId(r.command)}\`` +
|
|
609
|
+
(r.reattached ? ' · ↩ recovered from a previous process' : ''));
|
|
610
|
+
return (`run_terminal — monitored commands:\n${lines.join('\n')}\n\n` +
|
|
611
|
+
`Poll one with run_terminal(action:'poll', process_id:'<id>'); stop one with run_terminal(action:'stop', process_id:'<id>').` +
|
|
612
|
+
(recovered > 0
|
|
613
|
+
? `\n(${recovered} recovered from the registry a previous process wrote — they are still being followed.)`
|
|
614
|
+
: ''));
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Stop one monitored run deliberately (the model's own decision after a
|
|
618
|
+
* diagnosis). Bounded on purpose: it can only ever touch a command THIS tool
|
|
619
|
+
* started and is still following — never an arbitrary pid.
|
|
620
|
+
*/
|
|
621
|
+
export function stopDetachedRun(id) {
|
|
622
|
+
hydrateMonitoredRuns();
|
|
623
|
+
const run = id ? detachedRuns.get(id) : undefined;
|
|
624
|
+
if (!run)
|
|
625
|
+
return unknownRunMessage(id);
|
|
626
|
+
if (run.status !== 'running') {
|
|
627
|
+
return (`run_terminal[${run.id}] already finished (exit ${run.exitCode ?? '?'}) — nothing to stop. ` +
|
|
628
|
+
(runOutputText(run) ? `Its output:\n${maskSenderId(runOutputText(run))}` : '(no output)'));
|
|
629
|
+
}
|
|
630
|
+
// A RECOVERED run is identified only by a pid, and a pid can be recycled — so
|
|
631
|
+
// it is stopped only once its command line has been re-verified as ours. The
|
|
632
|
+
// alternative is signalling a stranger's process, which this tool must never do.
|
|
633
|
+
if (!maySignal(run)) {
|
|
634
|
+
return (`Error: run_terminal: run_terminal[${run.id}] was recovered from a previous process, and its pid ` +
|
|
635
|
+
`(${run.pid ?? '?'}) could not be re-verified as still being \`${maskSenderId(run.command)}\` — ` +
|
|
636
|
+
`${run.pidVerifiedWhy ?? 'the check could not be completed'}. Refusing to signal it rather than risk ` +
|
|
637
|
+
`stopping an unrelated process. Its output is spooled here: ${spoolPath(run.id)}. ` +
|
|
638
|
+
`If you know it is still yours, stop it by hand with its PID.`);
|
|
639
|
+
}
|
|
640
|
+
run.stopRequested = true;
|
|
641
|
+
run.exitReason = 'stopped on request';
|
|
642
|
+
const signalled = killProcessTree(run.pid ?? -1, 'SIGTERM', run.child);
|
|
643
|
+
run.killGrace = setTimeout(() => {
|
|
644
|
+
killProcessTree(run.pid ?? -1, 'SIGKILL', run.child);
|
|
645
|
+
}, 5_000);
|
|
646
|
+
run.killGrace.unref?.();
|
|
647
|
+
persistRun(run);
|
|
648
|
+
return (`🛑 run_terminal[${run.id}] — SIGTERM sent to ${signalled.length || 'the'} process(es) of \`${maskSenderId(run.command)}\` ` +
|
|
649
|
+
`(the shell AND the real work under it; SIGKILL follows in 5s if it does not exit).\n` +
|
|
650
|
+
`Poll it with run_terminal(action:'poll', process_id:'${run.id}') to get its output and exit code. ` +
|
|
651
|
+
`Nothing else was touched.`);
|
|
652
|
+
}
|
|
653
|
+
/** The one message a poll/stop gives for an id nobody is monitoring. */
|
|
654
|
+
function unknownRunMessage(id) {
|
|
655
|
+
const known = [...detachedRuns.keys()];
|
|
656
|
+
return (`Error: run_terminal: no monitored command with id \`${maskSenderId(id)}\`.` +
|
|
657
|
+
(known.length > 0 ? ` Known: ${known.join(', ')}.` : ' Nothing is being monitored.') +
|
|
658
|
+
` Use run_terminal(action:'list') to see what is.`);
|
|
659
|
+
}
|
|
660
|
+
/**
|
|
661
|
+
* The state of one monitored command. A finished one keeps its result available
|
|
662
|
+
* (bounded) so the agent can read the exit code and output after doing other
|
|
663
|
+
* work — the poll is how a long command's verdict finally reaches the turn.
|
|
664
|
+
*/
|
|
665
|
+
export function pollDetachedRun(id) {
|
|
666
|
+
hydrateMonitoredRuns();
|
|
667
|
+
const run = id ? detachedRuns.get(id) : undefined;
|
|
668
|
+
if (!run)
|
|
669
|
+
return unknownRunMessage(id);
|
|
670
|
+
// A poll is exactly when "how is it doing?" is asked, so it is also when a
|
|
671
|
+
// recovered run must be reconciled — otherwise it reports itself as running
|
|
672
|
+
// forever (see `refreshReattachedRun`).
|
|
673
|
+
refreshReattachedRun(run);
|
|
674
|
+
const elapsed = elapsedLabel(Date.now() - run.startedAt);
|
|
675
|
+
const body = runOutputText(run);
|
|
676
|
+
if (run.status === 'running') {
|
|
677
|
+
const quietMs = Date.now() - run.lastOutputAt;
|
|
678
|
+
return (`${RUN_TERMINAL_INCOMPLETE_PREFIX} — run_terminal[${run.id}], ${elapsed} elapsed, ${run.bytes} bytes of output` +
|
|
679
|
+
(run.reattached ? ' (recovered from a previous process)' : '') +
|
|
680
|
+
(run.bytes === 0
|
|
681
|
+
? ', and NOTHING produced yet'
|
|
682
|
+
: quietMs >= 60_000
|
|
683
|
+
? `, silent for the last ${elapsedLabel(quietMs)}`
|
|
684
|
+
: '') +
|
|
685
|
+
`.\nCommand: \`${maskSenderId(run.command)}\`` +
|
|
686
|
+
// The liveness verdict, when the watchdog has one — this is the difference
|
|
687
|
+
// between "it is still running" and "it is running, and here is why I
|
|
688
|
+
// believe it is hung".
|
|
689
|
+
(run.verdict && run.verdict !== 'starting'
|
|
690
|
+
? `\nLiveness: ${run.verdict.toUpperCase()} — ${run.verdictWhy ?? 'no evidence yet'}.`
|
|
691
|
+
: '') +
|
|
692
|
+
(body ? `\nOutput so far:\n${maskSenderId(body)}` : '') +
|
|
693
|
+
`\nIt was NOT killed. Poll again later with run_terminal(action:'poll', process_id:'${run.id}'), or do other work first.` +
|
|
694
|
+
(run.hangUnstoppable
|
|
695
|
+
? `\nIt looks HUNG, but it was recovered after a restart and its pid could not be re-verified as this ` +
|
|
696
|
+
`command (${run.pidVerifiedWhy ?? 'the check could not be completed'}) — so it is deliberately NOT ` +
|
|
697
|
+
`being stopped automatically. Its output is spooled here: ${spoolPath(run.id)}.`
|
|
698
|
+
: run.verdict === 'hung'
|
|
699
|
+
? `\nIf you agree it is hung (it needs input it cannot get, or a lock/port it will never get), ` +
|
|
700
|
+
`stop it with run_terminal(action:'stop', process_id:'${run.id}') and re-run it non-interactively — ` +
|
|
701
|
+
`the watchdog stops it on its own after the grace either way.`
|
|
702
|
+
: ''));
|
|
703
|
+
}
|
|
704
|
+
// A run that FAILED, or that was stopped (by the watchdog or on request), did
|
|
705
|
+
// NOT verify anything — so it must carry the `Error: ` prefix the loop's honest
|
|
706
|
+
// accounting reads (`ranOk` in tool-loop.ts). Without it, collecting a failing
|
|
707
|
+
// suite through `poll` was recorded as a SUCCESSFUL tool call, which feeds the
|
|
708
|
+
// "did you check it" gate: the turn could claim a check it never got a pass
|
|
709
|
+
// from. Only a real exit-0 finish is a success here.
|
|
710
|
+
const ok = run.exitCode === 0 && !run.reaped && !run.stopRequested;
|
|
711
|
+
return (`${ok ? '✅ ' : 'Error: '}run_terminal[${run.id}] ` +
|
|
712
|
+
(run.reaped || run.stopRequested
|
|
713
|
+
? `was STOPPED (${run.exitReason ?? 'stopped'})`
|
|
714
|
+
: `finished${run.exitCode === 0 ? '' : ` (exit ${run.exitCode ?? '?'})`}`) +
|
|
715
|
+
` — ${elapsed} total.\n` +
|
|
716
|
+
`Command: \`${maskSenderId(run.command)}\`` +
|
|
717
|
+
// A run recovered as finished has a reason instead of an exit code, and it
|
|
718
|
+
// must be stated: "finished" with no code is not the same fact as "exit 0".
|
|
719
|
+
(run.reattached && run.exitReason ? `\n${run.exitReason}.` : '') +
|
|
720
|
+
(run.reaped
|
|
721
|
+
? `\nThe work was doing nothing (no output, no CPU) and was stopped deliberately — this is NOT a test ` +
|
|
722
|
+
`failure and NOT a verdict on the code. Fix what it was waiting for and re-run it.`
|
|
723
|
+
: '') +
|
|
724
|
+
(body ? `\nOutput:\n${maskSenderId(body)}` : '\n(no output)'));
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* Why the command is taking as long as it is — the diagnosis a bare "timed out"
|
|
728
|
+
* never gave. A long run is either PROGRESS (let it finish) or STUCK (a prompt,
|
|
729
|
+
* a lock, a dead network call), and those need opposite responses, so the tool
|
|
730
|
+
* says which one the evidence supports.
|
|
731
|
+
*/
|
|
732
|
+
function longRunDiagnosis(bytes, quietMs) {
|
|
733
|
+
if (bytes === 0) {
|
|
734
|
+
return ('NO output at all yet. That is the signature of a command waiting on an interactive prompt ' +
|
|
735
|
+
'(this tool gives it no stdin, so the prompt never gets an answer), or of a first step that is genuinely slow ' +
|
|
736
|
+
'(a cold build, a large download). Make it non-interactive before re-running: `--yes`/`--no-input`/`-y`, ' +
|
|
737
|
+
'`CI=1`, `--non-interactive`, or a flag that skips the prompt.');
|
|
738
|
+
}
|
|
739
|
+
if (quietMs >= 60_000) {
|
|
740
|
+
return (`Output STOPPED ${elapsedLabel(quietMs)} ago after ${bytes} bytes — it is stalled rather than working. ` +
|
|
741
|
+
'Usual causes: waiting on a prompt, a lock held by another process, a network call with no timeout, or a test ' +
|
|
742
|
+
'blocked on a server that never came up. Read the tail below for the last thing it said before going quiet.');
|
|
743
|
+
}
|
|
744
|
+
return `It IS still producing output (${bytes} bytes, last one ${elapsedLabel(quietMs)} ago), so it is making progress — let it finish and poll rather than restarting it.`;
|
|
745
|
+
}
|
|
746
|
+
/**
|
|
747
|
+
* What the model is told when a command outlives this call's wait budget.
|
|
748
|
+
* Exported so the wording is testable: the two things it MUST say are that the
|
|
749
|
+
* command was not killed and that re-running it is wrong.
|
|
750
|
+
*/
|
|
751
|
+
export function waitBudgetGuidance(command, id, waitMs, assessed, bytes, quietMs, reapAfterMs = 0) {
|
|
752
|
+
const seconds = Math.round(waitMs / 1000);
|
|
753
|
+
return (` — \`${maskSenderId(command)}\` has outlived this call's ${seconds}s wait budget. ` +
|
|
754
|
+
`It was NOT killed and this is NOT a failure: the work is still going, and its output is accumulating.\n` +
|
|
755
|
+
`${assessed ? `It was given an extended wait because it is ${assessed.why} — a class that is minutes by nature.\n` : ''}` +
|
|
756
|
+
`Follow it with \`run_terminal(action:'poll', process_id:'${id}')\` — that returns its status, its liveness ` +
|
|
757
|
+
`(WORKING / STALLED / HUNG, from its output and its CPU), its output tail, and its REAL exit code once it lands; ` +
|
|
758
|
+
`\`run_terminal(action:'list')\` shows everything being monitored. ` +
|
|
759
|
+
`Do NOT re-run the identical command: that would start a second copy of the same work. Do something useful while ` +
|
|
760
|
+
`it finishes (read a file, plan the next step) and poll instead.` +
|
|
761
|
+
(reapAfterMs > 0
|
|
762
|
+
? `\nA running command is WATCHED: if it stops producing output AND stops using CPU for ` +
|
|
763
|
+
`${Math.round(livenessThresholds().hangQuietMs / 60_000)} minutes it is reported as HUNG, and if it stays that way for a further ` +
|
|
764
|
+
`${Math.round(reapAfterMs / 60_000)} minutes it is stopped and its result reported — a real hang is not left running forever. ` +
|
|
765
|
+
`Output or CPU at any point clears that, so a slow-but-working command is never touched.`
|
|
766
|
+
: ''));
|
|
127
767
|
}
|
|
128
768
|
/** ─── Command classification (deny-first) ───────────────────────────────── */
|
|
129
769
|
/**
|
|
@@ -462,6 +1102,18 @@ export function classifyCommand(command) {
|
|
|
462
1102
|
return worst;
|
|
463
1103
|
}
|
|
464
1104
|
export async function runTerminalTool(args, ctx) {
|
|
1105
|
+
// The follow-up half of a long run: no command is needed to poll or list, and
|
|
1106
|
+
// these must work even when the model has nothing to spawn.
|
|
1107
|
+
// Recover whatever a PREVIOUS process left running, before anything is looked
|
|
1108
|
+
// up. Without this a restart silently orphans the record: the command keeps
|
|
1109
|
+
// running, `poll` reports no such id, and the turn can never collect it.
|
|
1110
|
+
hydrateMonitoredRuns();
|
|
1111
|
+
if (args.action === 'poll')
|
|
1112
|
+
return pollDetachedRun((args.process_id ?? '').trim());
|
|
1113
|
+
if (args.action === 'list')
|
|
1114
|
+
return listDetachedRuns();
|
|
1115
|
+
if (args.action === 'stop')
|
|
1116
|
+
return stopDetachedRun((args.process_id ?? '').trim());
|
|
465
1117
|
const command = normalize(args.command);
|
|
466
1118
|
// NOTE: every refusal below is prefixed `Error: ` on purpose. The tool-result
|
|
467
1119
|
// convention treats that prefix as "this did not run", and the loop's honest
|
|
@@ -592,11 +1244,29 @@ export async function runTerminalTool(args, ctx) {
|
|
|
592
1244
|
authorization: ctx.writesAuthorized?.reason,
|
|
593
1245
|
}, 'tool-loop');
|
|
594
1246
|
}
|
|
595
|
-
//
|
|
596
|
-
//
|
|
597
|
-
//
|
|
598
|
-
|
|
599
|
-
|
|
1247
|
+
// ── Don't start the same work twice ─────────────────────────────────────────
|
|
1248
|
+
// A long command handed off to monitoring is STILL RUNNING. Re-issuing it
|
|
1249
|
+
// would spawn a second copy (two test suites competing for the same ports and
|
|
1250
|
+
// CPU, two installs over each other). The old code had no such case because
|
|
1251
|
+
// the first copy was already dead; now the honest answer is to follow it.
|
|
1252
|
+
const runningSame = [...detachedRuns.values()].find((r) => r.status === 'running' && r.command === command && r.cwd === (ctx.cwd ?? process.cwd()));
|
|
1253
|
+
if (runningSame) {
|
|
1254
|
+
return (`Error: run_terminal: \`${maskSenderId(command)}\` is ALREADY RUNNING (monitored as ` +
|
|
1255
|
+
`run_terminal[${runningSame.id}], ${elapsedLabel(Date.now() - runningSame.startedAt)} elapsed). ` +
|
|
1256
|
+
`Starting it again would do the same work twice. Follow it instead: ` +
|
|
1257
|
+
`run_terminal(action:'poll', process_id:'${runningSame.id}').`);
|
|
1258
|
+
}
|
|
1259
|
+
// P7 — the wait budget: an explicit `timeout_ms` wins; otherwise the command is
|
|
1260
|
+
// ASSESSED, so a test suite or a build gets the minutes it actually needs
|
|
1261
|
+
// instead of the short default meant for a quick read; otherwise the room
|
|
1262
|
+
// decides (a person watching waits less — see INTERACTIVE_DEFAULT_TIMEOUT_MS).
|
|
1263
|
+
// Reaching the budget now hands the command to monitoring rather than killing
|
|
1264
|
+
// it, so an under-estimate costs a poll, not the work.
|
|
1265
|
+
const assessed = assessCommandTime(command);
|
|
1266
|
+
const waitMs = resolveWaitMs(command, args.timeout_ms, ctx.interactive);
|
|
1267
|
+
// Reported so the model knows the hand-off is WATCHED: a run that goes silent
|
|
1268
|
+
// and idle will be identified as hung and stopped, not left forever.
|
|
1269
|
+
const reapAfterMs = hangReapMs();
|
|
600
1270
|
// P7 — the IDENTICAL-COMMAND guard. A command that has already failed twice at
|
|
601
1271
|
// this timeout in this directory will fail a third time the same way; the live
|
|
602
1272
|
// run spent two 120-second timeouts proving that. Refusing costs milliseconds
|
|
@@ -606,7 +1276,7 @@ export async function runTerminalTool(args, ctx) {
|
|
|
606
1276
|
// The effective timeout is part of the key on purpose: the guidance a timeout
|
|
607
1277
|
// carries tells the model to re-call with a longer `timeout_ms` when the work
|
|
608
1278
|
// is genuinely long, and that advice must not be refused by this guard.
|
|
609
|
-
const streakKey = `${ctx.cwd ?? process.cwd()}|${command}|${
|
|
1279
|
+
const streakKey = `${ctx.cwd ?? process.cwd()}|${command}|${waitMs}`;
|
|
610
1280
|
const priorStreak = commandFailureStreaks.get(streakKey);
|
|
611
1281
|
if (priorStreak &&
|
|
612
1282
|
Date.now() - priorStreak.lastAt <= FAILURE_STREAK_WINDOW_MS &&
|
|
@@ -617,20 +1287,32 @@ export async function runTerminalTool(args, ctx) {
|
|
|
617
1287
|
`or run a different check entirely.`);
|
|
618
1288
|
}
|
|
619
1289
|
const startedAt = Date.now();
|
|
620
|
-
const
|
|
621
|
-
//
|
|
622
|
-
//
|
|
623
|
-
//
|
|
624
|
-
//
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
1290
|
+
const outcome = await execCommand(command, ctx.cwd, waitMs, runEnv);
|
|
1291
|
+
// ── A command that outlived the wait budget is NOT a failure ────────────────
|
|
1292
|
+
// It is still running, its output is still accumulating, and the turn can
|
|
1293
|
+
// carry on: the guidance says how to follow it, what the evidence says about
|
|
1294
|
+
// why it is slow, and that re-running it would duplicate the work. Crucially
|
|
1295
|
+
// it does NOT record a failure streak — counting a live run as a failure is
|
|
1296
|
+
// what made the old behaviour report unfinished work as broken.
|
|
1297
|
+
if (outcome.kind === 'still-running') {
|
|
1298
|
+
ctx.emit?.('run_terminal:monitoring', {
|
|
1299
|
+
tool: 'run_terminal',
|
|
1300
|
+
command,
|
|
1301
|
+
processId: outcome.id,
|
|
1302
|
+
waitMs,
|
|
1303
|
+
bytes: outcome.bytes,
|
|
1304
|
+
}, 'tool-loop');
|
|
1305
|
+
return (`${RUN_TERMINAL_INCOMPLETE_PREFIX}` +
|
|
1306
|
+
waitBudgetGuidance(command, outcome.id, waitMs, assessed, outcome.bytes, outcome.quietMs, reapAfterMs) +
|
|
1307
|
+
`\nMonitored as run_terminal[${outcome.id}].\n` +
|
|
1308
|
+
longRunDiagnosis(outcome.bytes, outcome.quietMs));
|
|
1309
|
+
}
|
|
1310
|
+
const output = outcome.output;
|
|
1311
|
+
// A real non-zero exit is the only thing that counts as a failure now.
|
|
1312
|
+
const failureStreak = output.startsWith('Error:') ? recordCommandFailure(streakKey, Date.now()) : 0;
|
|
1313
|
+
if (!output.startsWith('Error:'))
|
|
632
1314
|
clearCommandFailure(streakKey);
|
|
633
|
-
const outputWithNotes =
|
|
1315
|
+
const outputWithNotes = output;
|
|
634
1316
|
// A4 — a KNOWN toolchain failure gets its known fix appended, so the model
|
|
635
1317
|
// repairs instead of re-trying the identical command. Advisory only: the note
|
|
636
1318
|
// names the bounded, project-local step; it never mutates the workspace.
|
|
@@ -685,7 +1367,7 @@ export async function runTerminalTool(args, ctx) {
|
|
|
685
1367
|
// remembered for this machine (observed), so the same OS-specific command is
|
|
686
1368
|
// not re-derived. Only with a verb and only on success — a failure is not a
|
|
687
1369
|
// fact worth remembering, and a verbless command cannot be keyed. Best-effort.
|
|
688
|
-
if (args.verb && !
|
|
1370
|
+
if (args.verb && !output.startsWith('Error:')) {
|
|
689
1371
|
try {
|
|
690
1372
|
const { recordLearnedCommand } = await import('../learning/learned-commands.js');
|
|
691
1373
|
const { leadingBinary } = await import('../learning/command-adaptation.js');
|
|
@@ -703,10 +1385,11 @@ export async function runTerminalTool(args, ctx) {
|
|
|
703
1385
|
}
|
|
704
1386
|
// C3 — remember what a SUCCESSFUL run answered (whole command + every fact it
|
|
705
1387
|
// established), so the same question in the same run is answered from the
|
|
706
|
-
// output above instead of a second spawn.
|
|
707
|
-
//
|
|
708
|
-
//
|
|
709
|
-
|
|
1388
|
+
// output above instead of a second spawn. `Error:`-prefixed output is a
|
|
1389
|
+
// refusal, a crash, or a real non-zero exit — not a fact worth remembering.
|
|
1390
|
+
// (A command that is merely still running never reaches here: it is followed
|
|
1391
|
+
// with `poll`, and its output is stored when it lands.)
|
|
1392
|
+
if (!output.startsWith('Error:'))
|
|
710
1393
|
storeMemo(ctx.commandMemo, command, output, memoWhere);
|
|
711
1394
|
if (!decidedAutonomously)
|
|
712
1395
|
return outputWithNotes + effectNote + envNote + remediationNote + autoApplyNote + missingNote;
|
|
@@ -719,8 +1402,15 @@ export async function runTerminalTool(args, ctx) {
|
|
|
719
1402
|
autoApplyNote +
|
|
720
1403
|
missingNote);
|
|
721
1404
|
}
|
|
722
|
-
/**
|
|
723
|
-
|
|
1405
|
+
/**
|
|
1406
|
+
* Spawn the command (shell-interpreted — pipes, &&, substitutions work).
|
|
1407
|
+
*
|
|
1408
|
+
* `waitMs` is a WAIT budget, not a kill switch. When it runs out the child keeps
|
|
1409
|
+
* running, its output keeps accumulating in the registry, and the caller is told
|
|
1410
|
+
* how to follow it (`pollDetachedRun`). The only thing that changes at the budget
|
|
1411
|
+
* is that THIS CALL stops blocking.
|
|
1412
|
+
*/
|
|
1413
|
+
function execCommand(command, cwd, waitMs, envOverride) {
|
|
724
1414
|
return new Promise((resolve) => {
|
|
725
1415
|
// On Windows, use Git Bash for cross-platform shell commands (pwd, touch, etc.)
|
|
726
1416
|
const shellOptions = {
|
|
@@ -738,49 +1428,133 @@ function execCommand(command, cwd, timeoutMs, envOverride) {
|
|
|
738
1428
|
shellOptions.shell = gitBash;
|
|
739
1429
|
}
|
|
740
1430
|
}
|
|
1431
|
+
const id = randomUUID();
|
|
1432
|
+
// Spooling starts at SPAWN, not at hand-off: the beginning of the output is
|
|
1433
|
+
// exactly what an overflowing buffer loses while the wait budget is still
|
|
1434
|
+
// running, and that is the part a diagnosis needs. A run that finishes inside
|
|
1435
|
+
// its budget discards the spool (see `settleRun`).
|
|
1436
|
+
const spool = new SpoolWriter(id);
|
|
741
1437
|
const child = spawn(command, shellOptions);
|
|
742
|
-
|
|
743
|
-
|
|
1438
|
+
const startedAt = Date.now();
|
|
1439
|
+
const run = {
|
|
1440
|
+
id,
|
|
1441
|
+
command,
|
|
1442
|
+
cwd: cwd || process.cwd(),
|
|
1443
|
+
child,
|
|
1444
|
+
pid: child.pid ?? null,
|
|
1445
|
+
spool,
|
|
1446
|
+
startedAt,
|
|
1447
|
+
lastOutputAt: startedAt,
|
|
1448
|
+
bytes: 0,
|
|
1449
|
+
out: '',
|
|
1450
|
+
err: '',
|
|
1451
|
+
status: 'running',
|
|
1452
|
+
exitCode: null,
|
|
1453
|
+
};
|
|
1454
|
+
// Whether this call has already answered (so a later exit does not resolve
|
|
1455
|
+
// twice — the registry owns the result from that point on).
|
|
1456
|
+
let answered = false;
|
|
744
1457
|
const timer = setTimeout(() => {
|
|
745
|
-
|
|
746
|
-
|
|
1458
|
+
if (answered)
|
|
1459
|
+
return;
|
|
1460
|
+
answered = true;
|
|
1461
|
+
// HAND OFF, do not kill. See the registry above.
|
|
1462
|
+
detachedRuns.set(run.id, run);
|
|
1463
|
+
pruneDetachedRuns();
|
|
1464
|
+
// Start watching it, so a run that stops making progress is IDENTIFIED
|
|
1465
|
+
// (and, on evidence, stopped) instead of running forever.
|
|
1466
|
+
ensureWatchdog();
|
|
1467
|
+
// …and make it DURABLE, so a restart can still follow and collect it. This
|
|
1468
|
+
// is also what puts the spool on disk: until a run is durable nobody could
|
|
1469
|
+
// read its output back, so writing it earlier only littered the config dir
|
|
1470
|
+
// with a file that a run finishing inside its budget then deleted.
|
|
1471
|
+
run.osStartedAt = readOsStartToken(run.pid);
|
|
1472
|
+
run.spool?.startPersisting();
|
|
1473
|
+
run.spool?.flush();
|
|
1474
|
+
persistRun(run);
|
|
1475
|
+
resolve({
|
|
1476
|
+
kind: 'still-running',
|
|
1477
|
+
id: run.id,
|
|
1478
|
+
bytes: run.bytes,
|
|
1479
|
+
quietMs: Date.now() - run.lastOutputAt,
|
|
1480
|
+
});
|
|
1481
|
+
}, waitMs);
|
|
1482
|
+
const note = (chunk, stream) => {
|
|
1483
|
+
const text = chunk.toString('utf-8');
|
|
1484
|
+
run[stream] += text;
|
|
1485
|
+
run.bytes += chunk.length;
|
|
1486
|
+
run.lastOutputAt = Date.now();
|
|
1487
|
+
// The spool keeps a bounded head AND tail, so what the buffer drops is still
|
|
1488
|
+
// readable later (and after a restart).
|
|
1489
|
+
run.spool?.write(text);
|
|
1490
|
+
// Bounded buffering: a 40MB log must not be held in memory, and the tool
|
|
1491
|
+
// only ever reports the tail.
|
|
1492
|
+
const cap = MAX_OUTPUT_CHARS * 2;
|
|
1493
|
+
if (run[stream].length > cap) {
|
|
1494
|
+
run[stream] = run[stream].slice(-cap);
|
|
1495
|
+
// Once anything is dropped the buffer is missing its BEGINNING, and the
|
|
1496
|
+
// poll must read the spool instead of presenting a tail as the whole story.
|
|
1497
|
+
run.truncatedOutput = true;
|
|
747
1498
|
}
|
|
748
|
-
|
|
749
|
-
|
|
1499
|
+
};
|
|
1500
|
+
/**
|
|
1501
|
+
* The run is over. Stop spooling, and keep the durable record only when it can
|
|
1502
|
+
* ever be read: a run handed to monitoring is collected by `poll` (possibly in
|
|
1503
|
+
* another process), while one that finished inside its wait budget already
|
|
1504
|
+
* delivered its result — its record and spool would be litter.
|
|
1505
|
+
*/
|
|
1506
|
+
const settleRun = () => {
|
|
1507
|
+
run.endedAt = Date.now();
|
|
1508
|
+
if (detachedRuns.has(run.id)) {
|
|
1509
|
+
run.spool?.close();
|
|
1510
|
+
persistRun(run);
|
|
750
1511
|
}
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
child.stderr?.on('data', (b) => {
|
|
758
|
-
err += b.toString('utf-8');
|
|
759
|
-
if (err.length > MAX_OUTPUT_CHARS)
|
|
760
|
-
err = err.slice(-MAX_OUTPUT_CHARS);
|
|
761
|
-
});
|
|
1512
|
+
else {
|
|
1513
|
+
run.spool?.discard();
|
|
1514
|
+
}
|
|
1515
|
+
};
|
|
1516
|
+
child.stdout?.on('data', (b) => note(b, 'out'));
|
|
1517
|
+
child.stderr?.on('data', (b) => note(b, 'err'));
|
|
762
1518
|
child.on('error', (e) => {
|
|
763
1519
|
clearTimeout(timer);
|
|
764
|
-
|
|
1520
|
+
run.status = 'done';
|
|
1521
|
+
run.err += `\n${e.message}`;
|
|
1522
|
+
settleRun();
|
|
1523
|
+
if (answered)
|
|
1524
|
+
return;
|
|
1525
|
+
answered = true;
|
|
1526
|
+
resolve({ kind: 'finished', output: `Error: run_terminal: failed to spawn — ${e.message}` });
|
|
765
1527
|
});
|
|
766
1528
|
child.on('exit', (code) => {
|
|
767
1529
|
clearTimeout(timer);
|
|
768
|
-
|
|
1530
|
+
run.status = 'done';
|
|
1531
|
+
run.exitCode = code;
|
|
1532
|
+
settleRun();
|
|
1533
|
+
if (answered)
|
|
1534
|
+
return; // already handed off — the poll reports the result
|
|
1535
|
+
answered = true;
|
|
1536
|
+
const body = (run.out || run.err).trim().slice(0, MAX_OUTPUT_CHARS);
|
|
769
1537
|
const masked = maskSenderId(body);
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
1538
|
+
// `code === null` now means the process was terminated by something ELSE
|
|
1539
|
+
// (a signal, the OOM killer, the user) — the tool no longer kills it, so
|
|
1540
|
+
// this is a real, reportable condition rather than its own timeout.
|
|
1541
|
+
const terminated = code === null;
|
|
1542
|
+
const ok = !terminated && code === 0;
|
|
1543
|
+
const status = terminated
|
|
1544
|
+
? 'was terminated before it finished (a signal, not this tool)'
|
|
774
1545
|
: ok
|
|
775
1546
|
? '✅ succeeded'
|
|
776
1547
|
: `❌ failed (exit ${code ?? '?'})`;
|
|
777
1548
|
// Enterprise G1: a command that FAILED is not a verification. The loop's
|
|
778
1549
|
// honest accounting reads the `Error:` prefix, and the verification gate
|
|
779
1550
|
// counts a successful run_terminal as proof — so a non-zero exit (or a
|
|
780
|
-
//
|
|
781
|
-
// turn "verified". The real output is preserved for diagnosis.
|
|
782
|
-
resolve(
|
|
783
|
-
|
|
1551
|
+
// termination) MUST carry the prefix, or a failing test/build would mark
|
|
1552
|
+
// the turn "verified". The real output is preserved for diagnosis.
|
|
1553
|
+
resolve({
|
|
1554
|
+
kind: 'finished',
|
|
1555
|
+
output: `${ok ? '' : 'Error: '}run_terminal: \`${maskSenderId(command)}\` ${status}.\n` +
|
|
1556
|
+
(masked ? `Output:\n${masked}` : '(no output)'),
|
|
1557
|
+
});
|
|
784
1558
|
});
|
|
785
1559
|
});
|
|
786
1560
|
}
|