@aragon-agent/cli 0.6.31 → 0.6.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +119 -0
- package/README.md +2970 -2826
- package/dist/agent/controller.d.ts +46 -0
- package/dist/agent/controller.js +175 -7
- package/dist/agent/job-notices.d.ts +143 -0
- package/dist/agent/job-notices.js +182 -0
- package/dist/agent/reducer.d.ts +21 -1
- package/dist/agent/reducer.js +7 -0
- package/dist/agent/system-prompt.d.ts +12 -0
- package/dist/agent/system-prompt.js +2 -0
- package/dist/cli.js +236 -8
- package/dist/commands/builtins.js +23 -8
- package/dist/commands/perf.d.ts +77 -1
- package/dist/commands/perf.js +37 -5
- package/dist/config/cli-commands.d.ts +27 -0
- package/dist/config/cli-commands.js +54 -1
- package/dist/config/env.js +38 -0
- package/dist/config/load.js +50 -1
- package/dist/config/schema.d.ts +73 -0
- package/dist/config/schema.js +34 -0
- package/dist/config/store.js +14 -1
- package/dist/exec/events.d.ts +29 -3
- package/dist/exec/events.js +3 -12
- package/dist/exec/runner.d.ts +24 -0
- package/dist/exec/runner.js +79 -0
- package/dist/input/cursor-report.d.ts +61 -0
- package/dist/input/cursor-report.js +97 -0
- package/dist/input/stdin-filter.d.ts +22 -0
- package/dist/input/stdin-filter.js +48 -3
- package/dist/proc/limits.d.ts +76 -2
- package/dist/proc/limits.js +83 -2
- package/dist/proc/log-mirror.d.ts +59 -0
- package/dist/proc/log-mirror.js +203 -0
- package/dist/proc/log-ring.d.ts +22 -0
- package/dist/proc/log-ring.js +20 -1
- package/dist/proc/notices.d.ts +57 -0
- package/dist/proc/notices.js +101 -0
- package/dist/proc/progress.d.ts +40 -0
- package/dist/proc/progress.js +135 -0
- package/dist/proc/prompt.d.ts +24 -2
- package/dist/proc/prompt.js +51 -2
- package/dist/proc/supervisor.d.ts +35 -2
- package/dist/proc/supervisor.js +142 -43
- package/dist/proc/types.d.ts +89 -1
- package/dist/proc/wait.d.ts +57 -0
- package/dist/proc/wait.js +133 -0
- package/dist/session/persist.js +3 -1
- package/dist/session/validate-session.js +17 -0
- package/dist/tools/bash-tool.d.ts +12 -1
- package/dist/tools/bash-tool.js +100 -13
- package/dist/tools/index.d.ts +26 -2
- package/dist/tools/index.js +43 -2
- package/dist/tools/job-tools.d.ts +54 -0
- package/dist/tools/job-tools.js +329 -0
- package/dist/tools/proc-tools.d.ts +1 -1
- package/dist/tools/proc-tools.js +49 -17
- package/dist/ui/ActivityLine.js +7 -0
- package/dist/ui/App.js +98 -12
- package/dist/ui/ScrollbackShell.d.ts +62 -11
- package/dist/ui/ScrollbackShell.js +11 -3
- package/dist/ui/StatusBar.d.ts +5 -0
- package/dist/ui/StatusBar.js +1 -0
- package/dist/ui/Transcript.d.ts +14 -0
- package/dist/ui/Transcript.js +14 -2
- package/dist/ui/entries/ServiceCard.d.ts +17 -1
- package/dist/ui/entries/ServiceCard.js +39 -4
- package/dist/ui/layout/status-layout.d.ts +13 -1
- package/dist/ui/layout/status-layout.js +10 -0
- package/dist/ui/layout/virtual-window.js +14 -1
- package/dist/ui/run-status-row.d.ts +12 -0
- package/dist/ui/screen.d.ts +17 -4
- package/dist/ui/screen.js +14 -6
- package/dist/ui/scrollback/LiveTail.d.ts +79 -0
- package/dist/ui/scrollback/LiveTail.js +32 -0
- package/dist/ui/scrollback/band.d.ts +114 -0
- package/dist/ui/scrollback/band.js +130 -0
- package/dist/ui/scrollback/budget.d.ts +51 -10
- package/dist/ui/scrollback/budget.js +69 -20
- package/dist/ui/scrollback/canvas.d.ts +99 -0
- package/dist/ui/scrollback/canvas.js +113 -0
- package/dist/ui/scrollback/cursor-probe.d.ts +71 -0
- package/dist/ui/scrollback/cursor-probe.js +133 -0
- package/dist/ui/scrollback/limits.d.ts +61 -4
- package/dist/ui/scrollback/limits.js +61 -4
- package/dist/ui/scrollback/live-preview.d.ts +18 -10
- package/dist/ui/scrollback/live-preview.js +14 -11
- package/dist/ui/scrollback/scrollback-writer.d.ts +104 -16
- package/dist/ui/scrollback/scrollback-writer.js +368 -82
- package/dist/ui/scrollback/tail-window.d.ts +60 -0
- package/dist/ui/scrollback/tail-window.js +73 -0
- package/dist/ui/scrollback/use-scrollback-view.d.ts +34 -15
- package/dist/ui/scrollback/use-scrollback-view.js +110 -59
- package/dist/ui/scrollback/writer-shapes.d.ts +68 -2
- package/dist/ui/scrollback/writer-shapes.js +53 -8
- package/package.json +3 -3
package/dist/proc/prompt.js
CHANGED
|
@@ -19,8 +19,57 @@
|
|
|
19
19
|
* `BACKGROUND_SERVICES_BLOCK_VERSION` exists so a change to this wording is
|
|
20
20
|
* greppable from a behaviour report.
|
|
21
21
|
*/
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
/**
|
|
23
|
+
* The `<background_jobs>` system-prompt block (long-task-jobs §12).
|
|
24
|
+
*
|
|
25
|
+
* ASCII ONLY: `src/proc/**` is inside the glyph scanner's scope, and this string
|
|
26
|
+
* also reaches a model running in a `cmd.exe` terminal.
|
|
27
|
+
*
|
|
28
|
+
* SPLICED CONDITIONALLY by `buildSystemPrompt`, which is what preserves I-3: with
|
|
29
|
+
* `jobs.enabled: false` the prompt is BYTE-IDENTICAL to the pre-feature output.
|
|
30
|
+
*
|
|
31
|
+
* EVERY SENTENCE HERE ANSWERS A DECISION THE MODEL ACTUALLY HAS TO MAKE:
|
|
32
|
+
* - "do not poll" is the behaviour this feature exists to replace (six
|
|
33
|
+
* `bash_output` calls per 30 seconds, six LLM turns, six 200-row tails);
|
|
34
|
+
* - "a bounded wait is the only thing that tells you it finished" prevents the
|
|
35
|
+
* opposite failure, a model that starts a job and then guesses;
|
|
36
|
+
* - "`<job_notices>` is the notification, not an error" stops it apologising for
|
|
37
|
+
* a block that is good news;
|
|
38
|
+
* - "`[stalled]` is not a failure" stops it killing a job that is merely quiet.
|
|
39
|
+
*
|
|
40
|
+
* `JOBS_BLOCK_VERSION` exists so a change to this wording is greppable from a
|
|
41
|
+
* behaviour report.
|
|
42
|
+
*/
|
|
43
|
+
import { JOBS_BLOCK_VERSION } from './limits.js';
|
|
44
|
+
export { JOBS_BLOCK_VERSION };
|
|
45
|
+
export function buildBackgroundJobsBlock() {
|
|
46
|
+
return [
|
|
47
|
+
'<background_jobs>',
|
|
48
|
+
'Some commands finish on their own but take far longer than one tool call:',
|
|
49
|
+
'training runs, model downloads, big installs, release builds. Do not poll them',
|
|
50
|
+
'and do not let the tool ceiling kill them.',
|
|
51
|
+
'',
|
|
52
|
+
'Rules:',
|
|
53
|
+
'- Start one with bash({ command, background: true }). You get a job id: `t1` for a',
|
|
54
|
+
' task (it will finish) or `s1` for a service (it will not). The call returns in a',
|
|
55
|
+
' few seconds with the first lines of output.',
|
|
56
|
+
'- Wait with job_wait({ job: "t1" }). Every wait is bounded (default 30s, at most',
|
|
57
|
+
' 60s) and returns one status line, one progress line, and only the output since',
|
|
58
|
+
' your last cursor. Call it again only after deciding something; do other work in',
|
|
59
|
+
' between if you have any.',
|
|
60
|
+
'- Nothing but a bounded wait tells you a job finished. When one finishes while you',
|
|
61
|
+
' are busy, a <job_notices> block appears at the end of a tool result - that is the',
|
|
62
|
+
' notification, not an error.',
|
|
63
|
+
'- `[stalled]` means the job has printed nothing for a while. It has not failed.',
|
|
64
|
+
' Choose: keep waiting, stop it, or read the tail.',
|
|
65
|
+
'- Full output lives in the service log (`bash_output`) and, past the ring, in a',
|
|
66
|
+
' file whose path the truncation note names. Read that file with grep/read_file',
|
|
67
|
+
' instead of asking for the whole log.',
|
|
68
|
+
'- Stop a job with bash_kill({ service: "t1" }) when you are done with it, unless the',
|
|
69
|
+
' user asked for it to stay up. Jobs do not survive this session.',
|
|
70
|
+
'</background_jobs>',
|
|
71
|
+
].join('\n');
|
|
72
|
+
}
|
|
24
73
|
export function buildBackgroundServicesBlock() {
|
|
25
74
|
return [
|
|
26
75
|
'<background_services>',
|
|
@@ -18,10 +18,23 @@
|
|
|
18
18
|
* unconditionally. With the flag off this object holds foreground pids, nothing
|
|
19
19
|
* else, and allocates no timers.
|
|
20
20
|
*/
|
|
21
|
-
import { type ProcEventListener, type ProcSupervisorPort, type ServiceLogPage, type ServiceSnapshot, type ServiceStartRequest, type ServiceStartResult } from './types.js';
|
|
21
|
+
import { type JobLogMirrorPort, type ProcEventListener, type ProcSupervisorPort, type ServiceLogPage, type ServiceSnapshot, type ServiceStartRequest, type ServiceStartResult } from './types.js';
|
|
22
22
|
export interface ProcSupervisorOptions {
|
|
23
23
|
/** `cfg.bash.readyTimeoutMs`, read once per service at spawn. */
|
|
24
24
|
readyTimeoutMs: () => number;
|
|
25
|
+
/**
|
|
26
|
+
* `cfg.jobs.stallSeconds * 1000`, read per snapshot.
|
|
27
|
+
*
|
|
28
|
+
* A CLOSURE RATHER THAN A NUMBER, exactly like `readyTimeoutMs`: a
|
|
29
|
+
* `config set jobs.stallSeconds` has to reach the card of the job already
|
|
30
|
+
* running, and a value captured at construction would not.
|
|
31
|
+
*/
|
|
32
|
+
stallMs?: () => number;
|
|
33
|
+
/**
|
|
34
|
+
* The optional write-through log mirror. `undefined` when `jobs.persistLog` is
|
|
35
|
+
* false, and in every test that does not care about disk.
|
|
36
|
+
*/
|
|
37
|
+
mirror?: JobLogMirrorPort;
|
|
25
38
|
/** Injected by the tests; production uses a real TCP connect. */
|
|
26
39
|
probe?: (port: number) => Promise<boolean>;
|
|
27
40
|
}
|
|
@@ -37,7 +50,15 @@ export declare class ProcSupervisor implements ProcSupervisorPort {
|
|
|
37
50
|
*/
|
|
38
51
|
private readonly foreground;
|
|
39
52
|
private readonly listeners;
|
|
40
|
-
|
|
53
|
+
/**
|
|
54
|
+
* TWO COUNTERS, NEVER ONE (long-task-jobs §4.2 / I-2).
|
|
55
|
+
*
|
|
56
|
+
* `s1` and `t1` are separate id spaces so a `bash_kill({ service: "s1" })` can
|
|
57
|
+
* never reach a task, and so the model can tell "this will finish" from "this
|
|
58
|
+
* will not" from the id alone. Ids are never reused - see `evictTerminal`.
|
|
59
|
+
*/
|
|
60
|
+
private nextServiceId;
|
|
61
|
+
private nextTaskId;
|
|
41
62
|
private disposed;
|
|
42
63
|
constructor(opts: ProcSupervisorOptions);
|
|
43
64
|
trackForeground(pid: number | undefined): () => void;
|
|
@@ -89,6 +110,18 @@ export declare class ProcSupervisor implements ProcSupervisorPort {
|
|
|
89
110
|
private emit;
|
|
90
111
|
/** Coalesce `output` at the SOURCE — `/bg` and `bash_output` read it too. */
|
|
91
112
|
private scheduleOutput;
|
|
113
|
+
/**
|
|
114
|
+
* Recompute the extracted progress line from the tail of the ring.
|
|
115
|
+
*
|
|
116
|
+
* The scan window is `tail(progressScanRows)` SPLIT AROUND THE CARRY: `tail()`
|
|
117
|
+
* appends the not-yet-terminated line as its last element, and
|
|
118
|
+
* `extractProgress` needs to know which one that is, because `source: 'carry'`
|
|
119
|
+
* is how the caller knows a `\r`-rewritten bar (a download, a tqdm run) is
|
|
120
|
+
* moving rather than silent.
|
|
121
|
+
*/
|
|
122
|
+
private refreshProgress;
|
|
123
|
+
/** `now - lastOutputAt` past `jobs.stallSeconds`, on a job that has not ended. */
|
|
124
|
+
private isStalled;
|
|
92
125
|
/** Release every timer, wake every waiter, and announce the terminal state. */
|
|
93
126
|
private finish;
|
|
94
127
|
/**
|
package/dist/proc/supervisor.js
CHANGED
|
@@ -25,6 +25,7 @@ import { extractPortHint } from './classify.js';
|
|
|
25
25
|
import { isAlive, killTree, killTreeSync } from './kill-tree.js';
|
|
26
26
|
import { PROC_LIMITS } from './limits.js';
|
|
27
27
|
import { LogRing } from './log-ring.js';
|
|
28
|
+
import { extractProgress } from './progress.js';
|
|
28
29
|
import { probePort, ReadinessWatcher } from './readiness.js';
|
|
29
30
|
import { isTerminalStatus, } from './types.js';
|
|
30
31
|
export class ProcSupervisor {
|
|
@@ -39,7 +40,15 @@ export class ProcSupervisor {
|
|
|
39
40
|
*/
|
|
40
41
|
foreground = new Set();
|
|
41
42
|
listeners = new Set();
|
|
42
|
-
|
|
43
|
+
/**
|
|
44
|
+
* TWO COUNTERS, NEVER ONE (long-task-jobs §4.2 / I-2).
|
|
45
|
+
*
|
|
46
|
+
* `s1` and `t1` are separate id spaces so a `bash_kill({ service: "s1" })` can
|
|
47
|
+
* never reach a task, and so the model can tell "this will finish" from "this
|
|
48
|
+
* will not" from the id alone. Ids are never reused - see `evictTerminal`.
|
|
49
|
+
*/
|
|
50
|
+
nextServiceId = 1;
|
|
51
|
+
nextTaskId = 1;
|
|
43
52
|
disposed = false;
|
|
44
53
|
constructor(opts) {
|
|
45
54
|
this.opts = opts;
|
|
@@ -80,7 +89,7 @@ export class ProcSupervisor {
|
|
|
80
89
|
if (this.disposed)
|
|
81
90
|
return { ok: false, error: 'The process supervisor is shut down.' };
|
|
82
91
|
this.evictTerminal();
|
|
83
|
-
if (this.services.size >= PROC_LIMITS.
|
|
92
|
+
if (this.services.size >= PROC_LIMITS.maxJobs) {
|
|
84
93
|
// NO EVICTION AND NO SILENT FOREGROUNDING (P2-8). Every record is live, so
|
|
85
94
|
// dropping the oldest would orphan a running server that nothing can any
|
|
86
95
|
// longer name or stop. Naming the ids is what makes the error actionable.
|
|
@@ -91,32 +100,44 @@ export class ProcSupervisor {
|
|
|
91
100
|
.join(', ');
|
|
92
101
|
return {
|
|
93
102
|
ok: false,
|
|
94
|
-
error: `Too many background
|
|
103
|
+
error: `Too many background jobs (${PROC_LIMITS.maxJobs}). ` +
|
|
95
104
|
`Stop one first with bash_kill({ service: "<id>" }) - oldest live: ${live}.`,
|
|
96
105
|
};
|
|
97
106
|
}
|
|
98
|
-
const
|
|
99
|
-
this.
|
|
100
|
-
// THE PRE-FLIGHT PROBE RUNS
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
|
|
107
|
+
const kind = req.kind ?? 'service';
|
|
108
|
+
const id = kind === 'task' ? `t${this.nextTaskId++}` : `s${this.nextServiceId++}`;
|
|
109
|
+
// THE PRE-FLIGHT PROBE RUNS ONLY FOR A SERVICE (R-3), and the reason is the
|
|
110
|
+
// kind itself: a `task` connects to nothing, so probing its (usually
|
|
111
|
+
// non-existent) port hint would spend a socket timeout per launch to learn
|
|
112
|
+
// nothing. For a service, an unrelated process already answering on 3000
|
|
113
|
+
// would make every launch instantly and falsely ready; finding it there
|
|
114
|
+
// disables the probe detector and annotates the card, leaving the URL
|
|
115
|
+
// detector to do the work.
|
|
116
|
+
const hint = kind === 'service' ? extractPortHint(req.command) : undefined;
|
|
105
117
|
const preoccupied = hint === undefined ? false : await (this.opts.probe ?? probePort)(hint).catch(() => false);
|
|
118
|
+
const startedAt = Date.now();
|
|
106
119
|
const ring = new LogRing();
|
|
107
120
|
const record = {
|
|
108
121
|
id,
|
|
122
|
+
kind,
|
|
109
123
|
toolCallId: req.toolCallId,
|
|
110
124
|
command: req.command,
|
|
111
125
|
cwd: req.cwd,
|
|
112
126
|
pid: undefined,
|
|
113
|
-
|
|
114
|
-
|
|
127
|
+
// A TASK IS `running` FROM THE SPAWN (D-14). It has no readiness to wait
|
|
128
|
+
// for: the old behaviour left it `starting` for `readyTimeoutMs` (60s) and
|
|
129
|
+
// told the model "not ready yet - still coming up" about a job that was
|
|
130
|
+
// already training. Note that this changes the STATUS only - the settle
|
|
131
|
+
// window in `runBackground` still applies, via a TERMINAL predicate rather
|
|
132
|
+
// than a `!== 'starting'` one (I-12).
|
|
133
|
+
status: kind === 'task' ? 'running' : 'starting',
|
|
134
|
+
startedAt,
|
|
115
135
|
exitCode: null,
|
|
116
136
|
signal: null,
|
|
117
137
|
rows: [],
|
|
118
138
|
rowsSeen: 0,
|
|
119
139
|
truncated: false,
|
|
140
|
+
lastOutputAt: startedAt,
|
|
120
141
|
...(hint !== undefined ? { port: hint } : {}),
|
|
121
142
|
...(preoccupied ? { portPreoccupied: true } : {}),
|
|
122
143
|
};
|
|
@@ -167,12 +188,24 @@ export class ProcSupervisor {
|
|
|
167
188
|
entry.child = child;
|
|
168
189
|
record.pid = child.pid;
|
|
169
190
|
const onChunk = (chunk) => {
|
|
170
|
-
const
|
|
191
|
+
const text = typeof chunk === 'string' ? chunk : chunk.toString();
|
|
192
|
+
const rows = ring.append(text);
|
|
193
|
+
// EVERY CHUNK, NOT EVERY COMPLETED ROW. A `\r`-rewritten progress bar
|
|
194
|
+
// moves continuously while completing no row at all, so a row-based clock
|
|
195
|
+
// would call a healthy download stalled after `jobs.stallSeconds`.
|
|
196
|
+
record.lastOutputAt = Date.now();
|
|
171
197
|
record.rowsSeen = ring.rowsSeen;
|
|
172
198
|
record.truncated = ring.truncated;
|
|
173
199
|
record.rows = ring.tail(PROC_LIMITS.cardTailRows);
|
|
174
200
|
if (rows.length > 0)
|
|
175
201
|
entry.watcher?.offerRows(rows);
|
|
202
|
+
if (this.opts.mirror) {
|
|
203
|
+
this.opts.mirror.write(id, text);
|
|
204
|
+
// The path exists only once the mirror has actually opened a file, and
|
|
205
|
+
// `bash_output`'s truncation note is useful when it names a real one - so
|
|
206
|
+
// it is read back rather than assumed.
|
|
207
|
+
record.logPath ??= this.opts.mirror.pathOf(id);
|
|
208
|
+
}
|
|
176
209
|
this.scheduleOutput(entry);
|
|
177
210
|
};
|
|
178
211
|
child.stdout?.on('data', onChunk);
|
|
@@ -204,34 +237,41 @@ export class ProcSupervisor {
|
|
|
204
237
|
record.endedAt = Date.now();
|
|
205
238
|
this.finish(entry, asked ? 'stopped' : 'exited');
|
|
206
239
|
});
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
240
|
+
// A TASK HAS NO READINESS TO DETECT (AC-4), and starting a watcher for it
|
|
241
|
+
// would be actively wrong: the watcher's `onTimeout` path would later flip
|
|
242
|
+
// `running` -> `running` and emit a `ready` event for a job that has no
|
|
243
|
+
// address, and a 60s timer would keep firing for the whole life of a task
|
|
244
|
+
// that is already `running`.
|
|
245
|
+
if (kind === 'service') {
|
|
246
|
+
const watcher = new ReadinessWatcher({
|
|
247
|
+
// The hint is dropped when the port was ALREADY answering: probing it
|
|
248
|
+
// would report ready in the first 40 ms for somebody else's server.
|
|
249
|
+
...(hint !== undefined && !preoccupied ? { portHint: hint } : {}),
|
|
250
|
+
readyTimeoutMs: this.opts.readyTimeoutMs(),
|
|
251
|
+
...(this.opts.probe ? { probe: this.opts.probe } : {}),
|
|
252
|
+
onReady: (result) => {
|
|
253
|
+
if (isTerminalStatus(record.status) || record.status === 'ready')
|
|
254
|
+
return;
|
|
255
|
+
record.status = 'ready';
|
|
256
|
+
record.readyAt = Date.now();
|
|
257
|
+
if (result.url)
|
|
258
|
+
record.url = result.url;
|
|
259
|
+
if (result.port !== undefined)
|
|
260
|
+
record.port = result.port;
|
|
261
|
+
record.detectedBy = result.detectedBy;
|
|
262
|
+
this.emit({ type: 'ready', service: this.snapshot(entry) });
|
|
263
|
+
},
|
|
264
|
+
onTimeout: () => {
|
|
265
|
+
if (isTerminalStatus(record.status) || record.status === 'ready')
|
|
266
|
+
return;
|
|
267
|
+
// AN HONEST THIRD STATE (D-4). It is alive and it listens on nothing.
|
|
268
|
+
record.status = 'running';
|
|
269
|
+
this.emit({ type: 'ready', service: this.snapshot(entry) });
|
|
270
|
+
},
|
|
271
|
+
});
|
|
272
|
+
entry.watcher = watcher;
|
|
273
|
+
watcher.start();
|
|
274
|
+
}
|
|
235
275
|
this.emit({ type: 'started', service: this.snapshot(entry) });
|
|
236
276
|
return { ok: true, service: this.snapshot(entry) };
|
|
237
277
|
}
|
|
@@ -346,6 +386,15 @@ export class ProcSupervisor {
|
|
|
346
386
|
clearTimeout(entry.escalation);
|
|
347
387
|
entry.outputTimer = undefined;
|
|
348
388
|
entry.escalation = undefined;
|
|
389
|
+
// THE ONE EMIT THIS METHOD IS ALLOWED (long-task-jobs RV-P1-4). Without it a
|
|
390
|
+
// waiter that subscribed to `ProcEvent` is never woken here: this method
|
|
391
|
+
// sets a terminal status, and `child.on('exit')` then returns EARLY because
|
|
392
|
+
// the status is already terminal - so it does not emit either. A `job_wait`
|
|
393
|
+
// would sleep out its full timeout and only then learn the job was stopped.
|
|
394
|
+
//
|
|
395
|
+
// Safe on a signal-hook path: `emit` wraps every listener in try/catch, so
|
|
396
|
+
// this cannot throw, and it neither awaits nor spawns anything (I-9).
|
|
397
|
+
this.emit({ type: 'stopped', service: this.snapshot(entry) });
|
|
349
398
|
for (const wake of entry.waiters.splice(0))
|
|
350
399
|
wake();
|
|
351
400
|
}
|
|
@@ -399,10 +448,44 @@ export class ProcSupervisor {
|
|
|
399
448
|
return;
|
|
400
449
|
entry.outputTimer = setTimeout(() => {
|
|
401
450
|
entry.outputTimer = undefined;
|
|
451
|
+
// PROGRESS IS COMPUTED HERE, NOT PER CHUNK (long-task-jobs §4.5): a
|
|
452
|
+
// 10 kHz stream of chunks would otherwise run the extraction table 10,000
|
|
453
|
+
// times a second to publish the same `47%`. At most 10 times a second, from
|
|
454
|
+
// the coalesced tick, is also what makes the result a stable value two
|
|
455
|
+
// waits in a row can be compared against.
|
|
456
|
+
this.refreshProgress(entry);
|
|
402
457
|
this.emit({ type: 'output', service: this.snapshot(entry) });
|
|
403
458
|
}, PROC_LIMITS.outputCoalesceMs);
|
|
404
459
|
entry.outputTimer.unref?.();
|
|
405
460
|
}
|
|
461
|
+
/**
|
|
462
|
+
* Recompute the extracted progress line from the tail of the ring.
|
|
463
|
+
*
|
|
464
|
+
* The scan window is `tail(progressScanRows)` SPLIT AROUND THE CARRY: `tail()`
|
|
465
|
+
* appends the not-yet-terminated line as its last element, and
|
|
466
|
+
* `extractProgress` needs to know which one that is, because `source: 'carry'`
|
|
467
|
+
* is how the caller knows a `\r`-rewritten bar (a download, a tqdm run) is
|
|
468
|
+
* moving rather than silent.
|
|
469
|
+
*/
|
|
470
|
+
refreshProgress(entry) {
|
|
471
|
+
const tail = entry.ring.tail(PROC_LIMITS.progressScanRows);
|
|
472
|
+
const carry = entry.ring.carryRow;
|
|
473
|
+
const rows = carry.length > 0 ? tail.slice(0, -1) : tail;
|
|
474
|
+
const progress = extractProgress(rows, carry);
|
|
475
|
+
if (progress)
|
|
476
|
+
entry.record.progress = progress;
|
|
477
|
+
else
|
|
478
|
+
delete entry.record.progress;
|
|
479
|
+
}
|
|
480
|
+
/** `now - lastOutputAt` past `jobs.stallSeconds`, on a job that has not ended. */
|
|
481
|
+
isStalled(record) {
|
|
482
|
+
if (isTerminalStatus(record.status))
|
|
483
|
+
return false;
|
|
484
|
+
const ms = this.opts.stallMs?.();
|
|
485
|
+
if (ms === undefined || ms <= 0)
|
|
486
|
+
return false;
|
|
487
|
+
return Date.now() - record.lastOutputAt > ms;
|
|
488
|
+
}
|
|
406
489
|
/** Release every timer, wake every waiter, and announce the terminal state. */
|
|
407
490
|
finish(entry, type) {
|
|
408
491
|
entry.watcher?.stop();
|
|
@@ -417,6 +500,10 @@ export class ProcSupervisor {
|
|
|
417
500
|
entry.record.rows = entry.ring.tail(PROC_LIMITS.cardTailRows);
|
|
418
501
|
entry.record.rowsSeen = entry.ring.rowsSeen;
|
|
419
502
|
entry.record.truncated = entry.ring.truncated;
|
|
503
|
+
// ONE LAST EXTRACTION, so a job that printed `100%` and exited carries that
|
|
504
|
+
// line into its card, its notice and its `job_wait` result rather than the
|
|
505
|
+
// value from the last coalesced tick 100 ms earlier.
|
|
506
|
+
this.refreshProgress(entry);
|
|
420
507
|
this.emit({ type, service: this.snapshot(entry) });
|
|
421
508
|
for (const wake of entry.waiters.splice(0))
|
|
422
509
|
wake();
|
|
@@ -429,10 +516,10 @@ export class ProcSupervisor {
|
|
|
429
516
|
* something else.
|
|
430
517
|
*/
|
|
431
518
|
evictTerminal() {
|
|
432
|
-
if (this.services.size < PROC_LIMITS.
|
|
519
|
+
if (this.services.size < PROC_LIMITS.maxJobs)
|
|
433
520
|
return;
|
|
434
521
|
for (const [id, entry] of this.services) {
|
|
435
|
-
if (this.services.size < PROC_LIMITS.
|
|
522
|
+
if (this.services.size < PROC_LIMITS.maxJobs)
|
|
436
523
|
break;
|
|
437
524
|
if (isTerminalStatus(entry.record.status))
|
|
438
525
|
this.services.delete(id);
|
|
@@ -442,6 +529,7 @@ export class ProcSupervisor {
|
|
|
442
529
|
const r = entry.record;
|
|
443
530
|
return Object.freeze({
|
|
444
531
|
id: r.id,
|
|
532
|
+
kind: r.kind,
|
|
445
533
|
toolCallId: r.toolCallId,
|
|
446
534
|
command: r.command,
|
|
447
535
|
cwd: r.cwd,
|
|
@@ -460,6 +548,17 @@ export class ProcSupervisor {
|
|
|
460
548
|
rowsSeen: r.rowsSeen,
|
|
461
549
|
truncated: r.truncated,
|
|
462
550
|
...(r.killIncomplete ? { killIncomplete: true } : {}),
|
|
551
|
+
// THE THREE FIELDS THIS HAND-COPIED LITERAL WOULD OTHERWISE LOSE
|
|
552
|
+
// (long-task-jobs RV-P1-8). `kind` and `lastOutputAt` are REQUIRED on
|
|
553
|
+
// `ServiceSnapshot`, so forgetting one is a compile error; `progress` and
|
|
554
|
+
// `logPath` cannot be required (they genuinely may not exist) and are
|
|
555
|
+
// covered by AC-26 / AC-38 instead.
|
|
556
|
+
lastOutputAt: r.lastOutputAt,
|
|
557
|
+
...(r.progress
|
|
558
|
+
? { progress: { text: r.progress.text, kind: r.progress.kind } }
|
|
559
|
+
: {}),
|
|
560
|
+
...(this.isStalled(r) ? { stalled: true } : {}),
|
|
561
|
+
...(r.logPath !== undefined ? { logPath: r.logPath } : {}),
|
|
463
562
|
});
|
|
464
563
|
}
|
|
465
564
|
}
|
package/dist/proc/types.d.ts
CHANGED
|
@@ -9,6 +9,31 @@
|
|
|
9
9
|
* coupling (`no-host-coupling.test.ts`), so a new event member would break the
|
|
10
10
|
* first and teach core about a host shape it has no business defining.
|
|
11
11
|
*/
|
|
12
|
+
import type { JobProgress, ProgressKind } from './progress.js';
|
|
13
|
+
/**
|
|
14
|
+
* WHICH KIND OF JOB A RECORD DESCRIBES (long-task-jobs §4.2, D-1).
|
|
15
|
+
*
|
|
16
|
+
* - `service` never exits on its own and owes the caller a readiness signal;
|
|
17
|
+
* - `task` always exits and is `running` from the spawn.
|
|
18
|
+
*
|
|
19
|
+
* The distinction is not cosmetic. Before it existed, an explicitly backgrounded
|
|
20
|
+
* `python train.py` sat in `starting` for `readyTimeoutMs` (60s) and `bash` told
|
|
21
|
+
* the model "not ready yet - still coming up" about a job that was already
|
|
22
|
+
* working - which is what pushed it into starting a second copy or giving up.
|
|
23
|
+
*/
|
|
24
|
+
export type JobKind = 'service' | 'task';
|
|
25
|
+
/**
|
|
26
|
+
* The progress subset a SNAPSHOT carries: the rendered line and its shape.
|
|
27
|
+
*
|
|
28
|
+
* `source` and `rowIndex` stay on the record. They exist for tests and debugging
|
|
29
|
+
* (which row won, and did it come from a `\r` carry), and a card or a tool result
|
|
30
|
+
* that started depending on them would be depending on an implementation detail
|
|
31
|
+
* of the extraction table.
|
|
32
|
+
*/
|
|
33
|
+
export interface ServiceSnapshotProgress {
|
|
34
|
+
text: string;
|
|
35
|
+
kind: ProgressKind;
|
|
36
|
+
}
|
|
12
37
|
export type ServiceStatus =
|
|
13
38
|
/** Spawned, not yet ready, still alive. */
|
|
14
39
|
'starting'
|
|
@@ -32,8 +57,18 @@ export type ServiceStatus =
|
|
|
32
57
|
/** A status from which nothing further will happen. */
|
|
33
58
|
export declare function isTerminalStatus(status: ServiceStatus): boolean;
|
|
34
59
|
export interface ServiceRecord {
|
|
35
|
-
/** `s1
|
|
60
|
+
/** `s1` (a service), `t3` (a task). Short because the model has to quote it back. */
|
|
36
61
|
id: string;
|
|
62
|
+
/**
|
|
63
|
+
* WHICH KIND OF JOB THIS IS (long-task-jobs §4.2, D-1).
|
|
64
|
+
*
|
|
65
|
+
* A `service` never exits on its own and owes the caller a readiness signal; a
|
|
66
|
+
* `task` always exits and is `running` from the spawn. The distinction is not
|
|
67
|
+
* cosmetic: before it existed, an explicitly backgrounded `python train.py` sat
|
|
68
|
+
* in `starting` for `readyTimeoutMs` (60s) and `bash` told the model "not ready
|
|
69
|
+
* yet - still coming up" about a job that was already working.
|
|
70
|
+
*/
|
|
71
|
+
kind: JobKind;
|
|
37
72
|
/** The `bash` call that started it. */
|
|
38
73
|
toolCallId: string;
|
|
39
74
|
command: string;
|
|
@@ -67,15 +102,35 @@ export interface ServiceRecord {
|
|
|
67
102
|
truncated: boolean;
|
|
68
103
|
/** A stop reported the pid still alive afterwards (R-2 / P2-6). */
|
|
69
104
|
killIncomplete?: boolean;
|
|
105
|
+
/**
|
|
106
|
+
* When output last arrived, updated on EVERY CHUNK rather than per completed
|
|
107
|
+
* row (long-task-jobs §4.5).
|
|
108
|
+
*
|
|
109
|
+
* THE DISTINCTION IS LOAD-BEARING: a `\r`-rewritten progress bar (every
|
|
110
|
+
* download tool there is) moves continuously while completing no row at all,
|
|
111
|
+
* so a "last completed row" clock would declare a healthy download stalled
|
|
112
|
+
* after `jobs.stallSeconds`.
|
|
113
|
+
*/
|
|
114
|
+
lastOutputAt: number;
|
|
115
|
+
/** Latest extracted progress line, recomputed on the coalesced output tick. */
|
|
116
|
+
progress?: JobProgress;
|
|
117
|
+
/** Where the optional on-disk mirror is writing this job's full log. */
|
|
118
|
+
logPath?: string;
|
|
70
119
|
}
|
|
71
120
|
/**
|
|
72
121
|
* The frozen, JSON-serialisable projection listeners receive.
|
|
73
122
|
*
|
|
74
123
|
* LISTENERS GET SNAPSHOTS, NEVER THE RECORD, so a render-side bug cannot mutate
|
|
75
124
|
* supervisor state. `rows` carries only the last `PROC_LIMITS.cardTailRows`.
|
|
125
|
+
*
|
|
126
|
+
* `kind` AND `lastOutputAt` ARE REQUIRED ON PURPOSE (long-task-jobs RV-P1-8).
|
|
127
|
+
* `snapshot()` is a hand-copied object literal, so a new optional field that is
|
|
128
|
+
* forgotten there fails silently in five consumers at once. Required fields turn
|
|
129
|
+
* that omission into a compile error instead.
|
|
76
130
|
*/
|
|
77
131
|
export interface ServiceSnapshot {
|
|
78
132
|
id: string;
|
|
133
|
+
kind: JobKind;
|
|
79
134
|
toolCallId: string;
|
|
80
135
|
command: string;
|
|
81
136
|
cwd: string;
|
|
@@ -94,6 +149,17 @@ export interface ServiceSnapshot {
|
|
|
94
149
|
rowsSeen: number;
|
|
95
150
|
truncated: boolean;
|
|
96
151
|
killIncomplete?: boolean;
|
|
152
|
+
lastOutputAt: number;
|
|
153
|
+
progress?: ServiceSnapshotProgress;
|
|
154
|
+
/**
|
|
155
|
+
* DERIVED, NOT STORED: `now - lastOutputAt > jobs.stallSeconds * 1000` on a
|
|
156
|
+
* job that has not ended.
|
|
157
|
+
*
|
|
158
|
+
* A stored flag would need a timer to flip it, and the whole point is that any
|
|
159
|
+
* render refreshes the judgement for free.
|
|
160
|
+
*/
|
|
161
|
+
stalled?: boolean;
|
|
162
|
+
logPath?: string;
|
|
97
163
|
}
|
|
98
164
|
export type ProcEvent = {
|
|
99
165
|
type: 'started';
|
|
@@ -118,6 +184,11 @@ export interface ServiceStartRequest {
|
|
|
118
184
|
command: string;
|
|
119
185
|
cwd: string;
|
|
120
186
|
toolCallId: string;
|
|
187
|
+
/**
|
|
188
|
+
* Absent means `'service'`, which is what every pre-existing caller means and
|
|
189
|
+
* what keeps `start()` backward compatible without a second method.
|
|
190
|
+
*/
|
|
191
|
+
kind?: JobKind;
|
|
121
192
|
}
|
|
122
193
|
export interface ServiceStartResult {
|
|
123
194
|
ok: boolean;
|
|
@@ -133,6 +204,23 @@ export interface ServiceLogPage {
|
|
|
133
204
|
cursor: number;
|
|
134
205
|
/** Rows were evicted between `since` and what is returned. */
|
|
135
206
|
truncated: boolean;
|
|
207
|
+
/**
|
|
208
|
+
* The not-yet-terminated line among `rows` (empty when the last chunk ended
|
|
209
|
+
* with a newline).
|
|
210
|
+
*
|
|
211
|
+
* THE ONLY WAY TO TELL A REDRAWING PROGRESS BAR FROM A FINISHED ROW: `rows`
|
|
212
|
+
* always ends with the carry line when one exists, so `job_wait` uses this to
|
|
213
|
+
* keep an unchanged carriage-return-rewritten bar out of its incremental row
|
|
214
|
+
* budget (RV-P1-5). Optional so a test double - and `bash_output`, which does
|
|
215
|
+
* not care - can leave it out.
|
|
216
|
+
*/
|
|
217
|
+
carryRow?: string;
|
|
218
|
+
}
|
|
219
|
+
/** What `job_wait` and the supervisor need from a log mirror; no filesystem here. */
|
|
220
|
+
export interface JobLogMirrorPort {
|
|
221
|
+
open(jobId: string): void;
|
|
222
|
+
write(jobId: string, chunk: string): void;
|
|
223
|
+
pathOf(jobId: string): string | undefined;
|
|
136
224
|
}
|
|
137
225
|
/**
|
|
138
226
|
* The port `bash`, the tools, the controller and `/bg` see.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bounded wait primitive (long-task-jobs §4.4.1).
|
|
3
|
+
*
|
|
4
|
+
* ASCII ONLY: `src/proc/**` is inside the glyph scanner's scope.
|
|
5
|
+
*
|
|
6
|
+
* A FREE FUNCTION OVER THREE PORT METHODS, NOT A METHOD ON THE PORT (D-9).
|
|
7
|
+
* `Pick<ProcSupervisorPort, 'get' | 'read' | 'subscribe'>` is enough to answer
|
|
8
|
+
* "has it finished yet, and what did it print since my cursor", so no test double
|
|
9
|
+
* has to grow a `wait()` and the registry keeps owning exactly what it owns.
|
|
10
|
+
*
|
|
11
|
+
* FOUR RULES, and each one closes a way this turns into a hang:
|
|
12
|
+
*
|
|
13
|
+
* 1. TERMINAL FIRST, SUBSCRIBE SECOND. A job that already ended resolves
|
|
14
|
+
* synchronously with `waitedMs: 0` and registers nothing - no listener, no
|
|
15
|
+
* timer. `runBackground` carries the same defence for the same reason: a
|
|
16
|
+
* 1 ms crash is not hypothetical.
|
|
17
|
+
* 2. EVERY TIMER IS `unref`ed AND EVERY LISTENER IS REMOVED at every exit. A
|
|
18
|
+
* wait that leaks a timer is a process that will not exit, which is the
|
|
19
|
+
* failure mode this feature exists to remove.
|
|
20
|
+
* 3. ONE IDEMPOTENT `finish` FUNNEL. Terminal event, timeout and abort race, and
|
|
21
|
+
* exactly one of them may win.
|
|
22
|
+
* 4. IT DOES NOT DEPEND ON A SINGLE WAKE SOURCE. `reapSync()` used to emit
|
|
23
|
+
* nothing at all (the supervisor now emits `stopped` from it), and
|
|
24
|
+
* `dispose()` clears its listener set without emitting; the self-timeout is
|
|
25
|
+
* therefore a real second path, not belt-and-braces.
|
|
26
|
+
*/
|
|
27
|
+
import type { ProcSupervisorPort, ServiceLogPage, ServiceSnapshot } from './types.js';
|
|
28
|
+
/** The port surface this module needs. Deliberately narrower than the full port. */
|
|
29
|
+
export type WaitPort = Pick<ProcSupervisorPort, 'get' | 'read' | 'subscribe'>;
|
|
30
|
+
export interface WaitForJobsRequest {
|
|
31
|
+
/** Ids already resolved by the caller. An empty list returns immediately. */
|
|
32
|
+
targets: readonly string[];
|
|
33
|
+
/** Already clamped by the caller (`jobs.waitSeconds`, I-4 and `WAIT_SECONDS_RANGE`). */
|
|
34
|
+
timeoutMs: number;
|
|
35
|
+
/** The tool call's `ctx.signal`. Aborting resolves the wait with `aborted: true`. */
|
|
36
|
+
signal?: AbortSignal;
|
|
37
|
+
/** Cursor from a previous `job_wait` / `bash_output`; omit for the whole tail. */
|
|
38
|
+
since?: number;
|
|
39
|
+
}
|
|
40
|
+
export interface JobWaitOutcome {
|
|
41
|
+
/** One snapshot per target that still exists, in the requested order. */
|
|
42
|
+
jobs: ServiceSnapshot[];
|
|
43
|
+
/** Every target reached a terminal status. */
|
|
44
|
+
terminal: boolean;
|
|
45
|
+
waitedMs: number;
|
|
46
|
+
timedOut: boolean;
|
|
47
|
+
aborted: boolean;
|
|
48
|
+
/** Incremental page per target, read at settle time. */
|
|
49
|
+
pages: ReadonlyMap<string, ServiceLogPage>;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Wait up to `timeoutMs` for every target to reach a terminal status.
|
|
53
|
+
*
|
|
54
|
+
* Resolves - never rejects - and always after at most `timeoutMs`. `now` is
|
|
55
|
+
* injectable so a test can assert `waitedMs` without depending on wall clock.
|
|
56
|
+
*/
|
|
57
|
+
export declare function waitForJob(port: WaitPort, req: WaitForJobsRequest, now?: () => number): Promise<JobWaitOutcome>;
|