@asrunjs/asrun 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/explore-case.md +11 -6
- package/.claude/commands/gen-spec.md +10 -6
- package/.claude/commands/heal-spec.md +13 -9
- package/.claude/commands/make-case.md +1 -1
- package/.claude/skills/exploring-cases/SKILL.md +84 -63
- package/.claude/skills/exploring-cases/references/assertion-targets.md +8 -4
- package/.claude/skills/exploring-cases/references/case-rules.md +110 -0
- package/.claude/skills/exploring-cases/references/handoff-card.js +73 -11
- package/.claude/skills/exploring-cases/references/notes.md +87 -0
- package/.claude/skills/exploring-cases/references/open-questions.md +58 -6
- package/.claude/skills/exploring-cases/references/readiness.md +7 -5
- package/.claude/skills/exploring-cases/references/tools.md +151 -0
- package/.claude/skills/exploring-cases/references/user-handoff.md +47 -18
- package/.claude/skills/generating-specs/SKILL.md +99 -54
- package/.claude/skills/generating-specs/references/conventions.md +4 -0
- package/.claude/skills/making-cases/SKILL.md +14 -7
- package/.claude/skills/making-cases/references/gaps.md +7 -4
- package/.claude/skills/making-cases/references/normalizing.md +1 -1
- package/AGENTS.md +61 -20
- package/README.md +10 -8
- package/bin/asrun.mjs +2 -0
- package/dist/lib/case-doc.d.ts +41 -4
- package/dist/lib/case-doc.js +109 -17
- package/dist/lib/explore-notes.d.ts +74 -0
- package/dist/lib/explore-notes.js +242 -0
- package/dist/lib/paths.d.ts +2 -0
- package/dist/lib/paths.js +2 -0
- package/dist/scripts/check-cases.js +89 -1
- package/dist/scripts/cli.js +9 -2
- package/dist/scripts/doctor.js +50 -3
- package/dist/scripts/explore-notes.d.ts +1 -0
- package/dist/scripts/explore-notes.js +131 -0
- package/dist/scripts/markers-confirm.d.ts +1 -0
- package/dist/scripts/markers-confirm.js +139 -0
- package/dist/ui/server/actions.js +50 -16
- package/dist/ui/server/app.js +7 -2
- package/dist/ui/server/job-store.d.ts +60 -0
- package/dist/ui/server/job-store.js +198 -0
- package/dist/ui/server/jobs.d.ts +46 -4
- package/dist/ui/server/jobs.js +361 -59
- package/dist/ui/server/process-info.d.ts +17 -0
- package/dist/ui/server/process-info.js +126 -0
- package/dist/ui/server/result-state.d.ts +10 -0
- package/dist/ui/server/result-state.js +106 -0
- package/dist/ui/shared/ai-commands.d.ts +4 -1
- package/dist/ui/shared/ai-commands.js +14 -4
- package/dist/ui/shared/types.d.ts +17 -2
- package/docs/agents/case-format.md +40 -5
- package/docs/agents/framework-dev.md +4 -0
- package/docs/ai-workflow.md +56 -16
- package/docs/cases.md +32 -8
- package/docs/commands.md +17 -1
- package/docs/theme-guide.html +56 -0
- package/docs/troubleshooting.md +40 -0
- package/docs/web-ui.md +59 -15
- package/package.json +3 -1
- package/templates/.claude/commands/explore-case.md +11 -6
- package/templates/.claude/commands/gen-spec.md +10 -5
- package/templates/.claude/commands/heal-spec.md +11 -7
- package/templates/.claude/commands/make-case.md +1 -1
- package/templates/AGENTS.md +9 -6
- package/templates/cases/_template.md +7 -0
- package/templates/package.json +2 -0
- package/ui/dist/assets/{dist-WvvCZeeu.js → dist--BXJInTq.js} +1 -1
- package/ui/dist/assets/{dist-B7HeglDd.js → dist-BMmNFIUH.js} +1 -1
- package/ui/dist/assets/{dist-DevASegc.js → dist-BXNaYHeh.js} +1 -1
- package/ui/dist/assets/{dist-Bb36rNUw.js → dist-ByLbEOlF.js} +1 -1
- package/ui/dist/assets/{dist-Dqc_tfzC.js → dist-C9ra6_BD.js} +1 -1
- package/ui/dist/assets/{dist-DfBpjgZT.js → dist-CK8mow8u.js} +1 -1
- package/ui/dist/assets/{dist-G9KNeh6u.js → dist-CQS0So-i.js} +1 -1
- package/ui/dist/assets/{dist-GvRUDL-2.js → dist-CTrwRc3Q.js} +1 -1
- package/ui/dist/assets/{dist-B9_GXR2G.js → dist-DTYj2vUD.js} +1 -1
- package/ui/dist/assets/{dist-Dq4Vnw5N.js → dist-mV0_nQZe.js} +1 -1
- package/ui/dist/assets/index-BzMqkmQ9.js +348 -0
- package/ui/dist/assets/index-DcJR-8Mt.css +2 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/index-BJ5D2mFK.js +0 -346
- package/ui/dist/assets/index-BXY-eKBn.css +0 -2
package/dist/ui/server/jobs.js
CHANGED
|
@@ -1,33 +1,40 @@
|
|
|
1
|
-
import { spawn
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
2
|
import { EventEmitter } from 'node:events';
|
|
3
3
|
import crypto from 'node:crypto';
|
|
4
4
|
import fs from 'node:fs';
|
|
5
|
-
import path from 'node:path';
|
|
6
5
|
import { StringDecoder } from 'node:string_decoder';
|
|
7
6
|
import { ROOT } from '../../lib/paths.js';
|
|
8
7
|
import { spawnTarget } from '../../lib/command.js';
|
|
9
8
|
import { AiStreamFormatter } from './ai-stream.js';
|
|
10
|
-
|
|
9
|
+
import { parseResultState } from './result-state.js';
|
|
10
|
+
import { appendChunkNow, chunksFile, ensureJobDir, logFile, pruneJobs, readChunks, readJobs, writeRecord, } from './job-store.js';
|
|
11
|
+
import { killProcessTree, probeProcesses } from './process-info.js';
|
|
11
12
|
const MAX_BUFFERED_CHARS = 4_000_000;
|
|
12
|
-
const MAX_FINISHED_JOBS =
|
|
13
|
+
const MAX_FINISHED_JOBS = 50;
|
|
13
14
|
const LOG_RETENTION_MS = 7 * 24 * 3600 * 1000;
|
|
14
15
|
const KILL_GRACE_MS = 5000;
|
|
15
16
|
// An agent that has said nothing for this long gets a "still waiting for <tool call>" line in its log.
|
|
16
17
|
const QUIET_MS = 45_000;
|
|
17
18
|
const HEARTBEAT_MS = 15_000;
|
|
19
|
+
// A leftover process is looked at again this often, so a new AI run stops being blocked once it exits by itself.
|
|
20
|
+
const LEFTOVER_PROBE_MS = 5_000;
|
|
21
|
+
// An AI run that has said nothing at all for this long is given up on: a tool call that never returns (the biggest
|
|
22
|
+
// case: the `playwright-test` MCP server stopped answering) would otherwise leave the job running for ever. A handoff
|
|
23
|
+
// card waiting for a person gets `HANDOFF_GRACE_MS` on top, because that wait is the point of an attended run.
|
|
24
|
+
const STALL_MS = 10 * 60_000;
|
|
25
|
+
const HANDOFF_GRACE_MS = 5 * 60_000;
|
|
26
|
+
const STALL_MESSAGE = 'The agent stopped answering, so this run was stopped. The browser MCP server (`playwright-test`) most likely ' +
|
|
27
|
+
'stopped responding: one interrupted browser call leaves it wedged for every later run in this workspace. Restart ' +
|
|
28
|
+
'OpenCode (that ends its MCP processes) and start the run again; `pnpm asrun doctor` cannot see this, because it ' +
|
|
29
|
+
'starts a fresh server of its own.';
|
|
18
30
|
export class JobConflictError extends Error {
|
|
19
31
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const text = record.chunks
|
|
24
|
-
// Only what the agent said counts: a file it read or a page it dumped may quote the same words.
|
|
32
|
+
/** What the agent itself said, as one text: a file it read or a page it dumped may quote the same words. */
|
|
33
|
+
function resultText(record) {
|
|
34
|
+
return record.chunks
|
|
25
35
|
.filter((chunk) => (chunk.kind ? chunk.kind === 'markdown' : chunk.stream !== 'system'))
|
|
26
36
|
.map((chunk) => chunk.data)
|
|
27
|
-
.join('\n')
|
|
28
|
-
.replace(ANSI, '');
|
|
29
|
-
const matches = [...text.matchAll(/^[\s*`>_-]*RESULT:\s*(ready|needs-input)\b/gm)];
|
|
30
|
-
return matches.at(-1)?.[1] ?? null;
|
|
37
|
+
.join('\n');
|
|
31
38
|
}
|
|
32
39
|
/**
|
|
33
40
|
* A block of an AI run as the log file keeps it: the same text the UI shows, with the timestamp the UI styles written
|
|
@@ -48,31 +55,75 @@ function logFileText(part) {
|
|
|
48
55
|
}
|
|
49
56
|
return `${stamp}${part.data}\n`;
|
|
50
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Whether a process the system reports for a job's pid is really that job. The command line has to hold one of the
|
|
60
|
+
* arguments the job was started with (the script, the case, the profile), and the process must have started around the
|
|
61
|
+
* job: a pid that was reused by an unrelated program fails both.
|
|
62
|
+
*/
|
|
63
|
+
function matchesJob(details, spec, startedAt) {
|
|
64
|
+
if (details.commandLine === '')
|
|
65
|
+
return false;
|
|
66
|
+
const line = details.commandLine.toLowerCase();
|
|
67
|
+
const tokens = spec.args.filter((value) => value.length >= 4).map((value) => value.toLowerCase());
|
|
68
|
+
const spawned = spec.spawnedPath;
|
|
69
|
+
if (spawned && spawned !== process.execPath && /[\\/]/.test(spawned))
|
|
70
|
+
tokens.push(spawned.toLowerCase());
|
|
71
|
+
if (!tokens.some((token) => line.includes(token)))
|
|
72
|
+
return false;
|
|
73
|
+
if (details.startedAt !== null) {
|
|
74
|
+
const started = Date.parse(startedAt);
|
|
75
|
+
if (Number.isFinite(started) && (details.startedAt < started - 60_000 || details.startedAt > started + 180_000)) {
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
51
81
|
class JobManager {
|
|
52
82
|
jobs = new Map();
|
|
83
|
+
leftoverTimer = null;
|
|
53
84
|
/** Emits `job` with a JobInfo on every status change of any job. */
|
|
54
85
|
events = new EventEmitter();
|
|
55
86
|
constructor() {
|
|
56
87
|
this.events.setMaxListeners(0);
|
|
57
|
-
this.
|
|
88
|
+
this.recover();
|
|
58
89
|
}
|
|
59
90
|
list() {
|
|
60
91
|
return [...this.jobs.values()].map((record) => record.info).reverse();
|
|
61
92
|
}
|
|
62
93
|
get(id) {
|
|
63
|
-
|
|
94
|
+
const record = this.jobs.get(id);
|
|
95
|
+
if (record)
|
|
96
|
+
this.ensureChunks(record);
|
|
97
|
+
return record;
|
|
98
|
+
}
|
|
99
|
+
/** The request "Run again" starts, or undefined when this job was not started from one. */
|
|
100
|
+
rerunRequest(id) {
|
|
101
|
+
return this.jobs.get(id)?.spec.rerun;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* True while this job still holds its locks. A job a restart left behind holds them only while its process is
|
|
105
|
+
* confirmed to be alive: starting another agent next to it would wedge the one MCP server for both.
|
|
106
|
+
*/
|
|
107
|
+
holdsLock(info) {
|
|
108
|
+
if (info.status === 'running')
|
|
109
|
+
return true;
|
|
110
|
+
return info.status === 'interrupted' && info.leftover?.state === 'alive';
|
|
64
111
|
}
|
|
65
112
|
start(spec) {
|
|
66
113
|
for (const record of this.jobs.values()) {
|
|
67
|
-
if (record.info
|
|
114
|
+
if (!this.holdsLock(record.info))
|
|
68
115
|
continue;
|
|
69
116
|
const clash = record.spec.locks.find((lock) => spec.locks.includes(lock));
|
|
70
|
-
if (clash)
|
|
117
|
+
if (!clash)
|
|
118
|
+
continue;
|
|
119
|
+
if (record.info.status === 'running') {
|
|
71
120
|
throw new JobConflictError(`Cannot start: "${record.info.command}" is already running (${clash}). Wait for it to finish or cancel it.`);
|
|
72
121
|
}
|
|
122
|
+
throw new JobConflictError(`Cannot start: "${record.info.command}" was interrupted by a UI server restart and its process is still ` +
|
|
123
|
+
`running (${clash}). Stop the leftover process in the Jobs drawer first.`);
|
|
73
124
|
}
|
|
74
125
|
const id = crypto.randomBytes(6).toString('hex');
|
|
75
|
-
|
|
126
|
+
ensureJobDir(id);
|
|
76
127
|
const record = {
|
|
77
128
|
info: {
|
|
78
129
|
id,
|
|
@@ -96,6 +147,7 @@ class JobManager {
|
|
|
96
147
|
resultState: null,
|
|
97
148
|
}),
|
|
98
149
|
...(spec.continuesJobId && { continuesJobId: spec.continuesJobId }),
|
|
150
|
+
...(spec.rerun && { rerunable: true }),
|
|
99
151
|
},
|
|
100
152
|
spec,
|
|
101
153
|
chunks: [],
|
|
@@ -103,20 +155,29 @@ class JobManager {
|
|
|
103
155
|
nextSeq: 0,
|
|
104
156
|
emitter: new EventEmitter(),
|
|
105
157
|
child: null,
|
|
158
|
+
pid: null,
|
|
106
159
|
cancelRequested: false,
|
|
107
|
-
|
|
160
|
+
stalled: false,
|
|
161
|
+
lastActivityAt: Date.now(),
|
|
162
|
+
logStream: fs.createWriteStream(logFile(id)),
|
|
163
|
+
chunkStream: fs.createWriteStream(chunksFile(id)),
|
|
164
|
+
chunksLoaded: true,
|
|
108
165
|
formatter: spec.stream ? new AiStreamFormatter(spec.stream) : null,
|
|
109
166
|
heartbeat: null,
|
|
110
167
|
};
|
|
111
168
|
record.emitter.setMaxListeners(0);
|
|
112
169
|
this.jobs.set(id, record);
|
|
113
170
|
this.push(record, 'system', `$ ${spec.command}\n`);
|
|
171
|
+
// Written before the process starts, so a crash during the start still leaves a record to find.
|
|
172
|
+
this.persist(record);
|
|
114
173
|
// A CLI installed globally is a `.cmd` shim on Windows, which spawn cannot start on its own (see lib/command.ts).
|
|
115
174
|
const target = spawnTarget(spec.file, spec.args);
|
|
116
175
|
if (target === null) {
|
|
117
176
|
this.finish(record, null, null, `Failed to start: "${spec.file}" was not found on PATH. Install it, or choose the other tool.`);
|
|
118
177
|
return record.info;
|
|
119
178
|
}
|
|
179
|
+
record.spec.spawnedPath = target.path;
|
|
180
|
+
this.persist(record);
|
|
120
181
|
let child;
|
|
121
182
|
try {
|
|
122
183
|
child = spawn(target.file, target.args, {
|
|
@@ -135,6 +196,8 @@ class JobManager {
|
|
|
135
196
|
return record.info;
|
|
136
197
|
}
|
|
137
198
|
record.child = child;
|
|
199
|
+
record.pid = child.pid ?? null;
|
|
200
|
+
this.persist(record);
|
|
138
201
|
for (const stream of ['stdout', 'stderr']) {
|
|
139
202
|
const decoder = new StringDecoder('utf8');
|
|
140
203
|
const formatted = stream === 'stdout' ? record.formatter : null;
|
|
@@ -163,6 +226,7 @@ class JobManager {
|
|
|
163
226
|
const line = formatter.waiting(QUIET_MS);
|
|
164
227
|
if (line)
|
|
165
228
|
this.pushPart(record, line);
|
|
229
|
+
this.watchStall(record);
|
|
166
230
|
}, HEARTBEAT_MS);
|
|
167
231
|
record.heartbeat.unref();
|
|
168
232
|
}
|
|
@@ -191,24 +255,215 @@ class JobManager {
|
|
|
191
255
|
}
|
|
192
256
|
cancel(id) {
|
|
193
257
|
const record = this.jobs.get(id);
|
|
194
|
-
|
|
195
|
-
|
|
258
|
+
if (!record)
|
|
259
|
+
return false;
|
|
260
|
+
// A job the previous server left behind: only its own process is ended, and only once it is recognized.
|
|
261
|
+
const leftover = record.info.leftover;
|
|
262
|
+
if (record.info.status === 'interrupted' && leftover?.state === 'alive') {
|
|
263
|
+
const probe = probeProcesses([leftover.pid]).get(leftover.pid) ?? null;
|
|
264
|
+
if (probe === null) {
|
|
265
|
+
record.info.leftover = { ...leftover, state: 'gone' };
|
|
266
|
+
this.persist(record);
|
|
267
|
+
this.events.emit('job', record.info);
|
|
268
|
+
return false;
|
|
269
|
+
}
|
|
270
|
+
if (!matchesJob(probe, record.spec, record.info.startedAt)) {
|
|
271
|
+
record.info.leftover = { ...leftover, state: 'unknown' };
|
|
272
|
+
this.persist(record);
|
|
273
|
+
this.events.emit('job', record.info);
|
|
274
|
+
return false;
|
|
275
|
+
}
|
|
276
|
+
killProcessTree(leftover.pid, 'SIGTERM');
|
|
277
|
+
const leftoverPid = leftover.pid;
|
|
278
|
+
record.info.leftover = { ...leftover, state: 'stopped' };
|
|
279
|
+
this.pushRecovered(record, '\n[stopped: the leftover process tree of an interrupted job was ended]\n');
|
|
280
|
+
this.persist(record);
|
|
281
|
+
this.events.emit('job', record.info);
|
|
282
|
+
// A signal is not proof: if it did not work, the entry goes back to "alive" so the person can try again.
|
|
283
|
+
setTimeout(() => this.verifyLeftoverStopped(record, leftoverPid), KILL_GRACE_MS).unref();
|
|
284
|
+
return true;
|
|
285
|
+
}
|
|
286
|
+
const pid = record.child?.pid ?? record.pid;
|
|
287
|
+
if (record.info.status !== 'running' || !pid)
|
|
196
288
|
return false;
|
|
197
289
|
record.cancelRequested = true;
|
|
198
|
-
|
|
290
|
+
killProcessTree(pid, 'SIGTERM');
|
|
199
291
|
setTimeout(() => {
|
|
200
292
|
if (record.info.status === 'running')
|
|
201
|
-
|
|
293
|
+
killProcessTree(pid, 'SIGKILL');
|
|
202
294
|
}, KILL_GRACE_MS).unref();
|
|
203
295
|
return true;
|
|
204
296
|
}
|
|
205
297
|
/** Stops everything; called when the server shuts down. */
|
|
206
298
|
killAll() {
|
|
207
299
|
for (const record of this.jobs.values()) {
|
|
208
|
-
const pid = record.child?.pid;
|
|
300
|
+
const pid = record.child?.pid ?? record.pid;
|
|
209
301
|
if (record.info.status === 'running' && pid)
|
|
210
|
-
|
|
302
|
+
killProcessTree(pid, 'SIGTERM');
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Reads the jobs of earlier server runs. One that was still running is marked `interrupted`: its output stops where
|
|
307
|
+
* the server died, and its process is either gone or a leftover the person can stop. An AI run that had already
|
|
308
|
+
* written its `RESULT:` line is recovered as succeeded, so its answers can still be answered.
|
|
309
|
+
*/
|
|
310
|
+
recover() {
|
|
311
|
+
pruneJobs(MAX_FINISHED_JOBS, LOG_RETENTION_MS);
|
|
312
|
+
const stored = readJobs();
|
|
313
|
+
const interrupted = stored.filter((job) => job.info.status === 'running');
|
|
314
|
+
const pids = interrupted.map((job) => job.pid).filter((pid) => typeof pid === 'number');
|
|
315
|
+
const probe = probeProcesses(pids);
|
|
316
|
+
const now = new Date().toISOString();
|
|
317
|
+
for (const record of stored) {
|
|
318
|
+
const job = {
|
|
319
|
+
info: record.info,
|
|
320
|
+
spec: { ...record.spec },
|
|
321
|
+
chunks: [],
|
|
322
|
+
bufferedChars: 0,
|
|
323
|
+
nextSeq: 0,
|
|
324
|
+
emitter: new EventEmitter(),
|
|
325
|
+
child: null,
|
|
326
|
+
pid: record.pid ?? null,
|
|
327
|
+
cancelRequested: false,
|
|
328
|
+
stalled: false,
|
|
329
|
+
lastActivityAt: Date.now(),
|
|
330
|
+
logStream: null,
|
|
331
|
+
chunkStream: null,
|
|
332
|
+
chunksLoaded: false,
|
|
333
|
+
formatter: null,
|
|
334
|
+
heartbeat: null,
|
|
335
|
+
};
|
|
336
|
+
job.emitter.setMaxListeners(0);
|
|
337
|
+
if (record.info.status === 'running') {
|
|
338
|
+
// Its result line lives in the chunks, and the note about the restart is appended below.
|
|
339
|
+
this.ensureChunks(job);
|
|
340
|
+
this.recoverInterrupted(job, probe, now);
|
|
341
|
+
}
|
|
342
|
+
this.jobs.set(job.info.id, job);
|
|
343
|
+
}
|
|
344
|
+
// A record from an earlier restart may still say its process is alive; look once before the server answers.
|
|
345
|
+
this.checkLeftovers();
|
|
346
|
+
if ([...this.jobs.values()].some((job) => job.info.leftover?.state === 'alive')) {
|
|
347
|
+
this.startLeftoverProbe();
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
/** Reads a finished job's chunks the first time someone opens it; a job this server runs already has them. */
|
|
351
|
+
ensureChunks(record) {
|
|
352
|
+
if (record.chunksLoaded)
|
|
353
|
+
return;
|
|
354
|
+
const loaded = readChunks(record.info.id, MAX_BUFFERED_CHARS);
|
|
355
|
+
record.chunks = loaded.chunks;
|
|
356
|
+
record.bufferedChars = loaded.bufferedChars;
|
|
357
|
+
record.nextSeq = loaded.nextSeq;
|
|
358
|
+
record.chunksLoaded = true;
|
|
359
|
+
}
|
|
360
|
+
recoverInterrupted(record, probe, now) {
|
|
361
|
+
const info = record.info;
|
|
362
|
+
info.awaitingUser = null;
|
|
363
|
+
info.interruptedAt = now;
|
|
364
|
+
info.endedAt = now;
|
|
365
|
+
const pid = record.pid;
|
|
366
|
+
const found = pid === null ? null : (probe.get(pid) ?? null);
|
|
367
|
+
if (pid === null) {
|
|
368
|
+
info.status = 'interrupted';
|
|
369
|
+
info.leftover = undefined;
|
|
370
|
+
this.pushRecovered(record, '\n[interrupted: the UI server stopped while this job was running. Its process is unknown]\n');
|
|
371
|
+
}
|
|
372
|
+
else if (found === null) {
|
|
373
|
+
info.leftover = { pid, state: 'gone' };
|
|
374
|
+
const state = record.spec.ai ? parseResultState(resultText(record)) : null;
|
|
375
|
+
if (state !== null) {
|
|
376
|
+
// The agent had already finished its turn; the server only died before it could record the outcome.
|
|
377
|
+
info.status = 'succeeded';
|
|
378
|
+
info.resultState = state;
|
|
379
|
+
info.exitCode = null;
|
|
380
|
+
info.signal = null;
|
|
381
|
+
this.pushRecovered(record, `\n[the UI server stopped while this job was running, but the agent had already written ` +
|
|
382
|
+
`RESULT: ${state}. The exit code is unknown]\n`);
|
|
383
|
+
}
|
|
384
|
+
else {
|
|
385
|
+
info.status = 'interrupted';
|
|
386
|
+
info.resultState = null;
|
|
387
|
+
this.pushRecovered(record, '\n[interrupted: the UI server stopped while this job was running. Its process is gone]\n');
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
else if (matchesJob(found, record.spec, info.startedAt)) {
|
|
391
|
+
info.status = 'interrupted';
|
|
392
|
+
info.resultState = null;
|
|
393
|
+
info.leftover = { pid, state: 'alive' };
|
|
394
|
+
this.pushRecovered(record, `\n[interrupted: the UI server stopped while this job was running. Its process (pid ${pid}) is still ` +
|
|
395
|
+
'running; stop it before starting another AI run]\n');
|
|
396
|
+
}
|
|
397
|
+
else {
|
|
398
|
+
info.status = 'interrupted';
|
|
399
|
+
info.resultState = null;
|
|
400
|
+
info.leftover = { pid, state: 'unknown' };
|
|
401
|
+
this.pushRecovered(record, `\n[interrupted: the UI server stopped while this job was running. A process with its pid (${pid}) is ` +
|
|
402
|
+
'running, but it could not be identified as this job, so it was not marked for stopping]\n');
|
|
403
|
+
}
|
|
404
|
+
this.persist(record);
|
|
405
|
+
}
|
|
406
|
+
/** Keeps the leftover state current, so a process that exits by itself stops blocking new jobs. */
|
|
407
|
+
startLeftoverProbe() {
|
|
408
|
+
if (this.leftoverTimer)
|
|
409
|
+
return;
|
|
410
|
+
this.leftoverTimer = setInterval(() => this.checkLeftovers(), LEFTOVER_PROBE_MS);
|
|
411
|
+
this.leftoverTimer.unref();
|
|
412
|
+
}
|
|
413
|
+
checkLeftovers() {
|
|
414
|
+
const records = [...this.jobs.values()].filter((record) => record.info.status === 'interrupted' && record.info.leftover?.state === 'alive');
|
|
415
|
+
if (records.length === 0) {
|
|
416
|
+
if (this.leftoverTimer)
|
|
417
|
+
clearInterval(this.leftoverTimer);
|
|
418
|
+
this.leftoverTimer = null;
|
|
419
|
+
return;
|
|
211
420
|
}
|
|
421
|
+
const probe = probeProcesses(records.map((record) => record.info.leftover.pid));
|
|
422
|
+
for (const record of records) {
|
|
423
|
+
const leftover = record.info.leftover;
|
|
424
|
+
const found = probe.get(leftover.pid) ?? null;
|
|
425
|
+
const state = found === null ? 'gone' : matchesJob(found, record.spec, record.info.startedAt) ? 'alive' : 'unknown';
|
|
426
|
+
if (state === leftover.state)
|
|
427
|
+
continue;
|
|
428
|
+
record.info.leftover = { ...leftover, state };
|
|
429
|
+
// A process that had already written its result and then exited is a finished turn, not an interruption.
|
|
430
|
+
if (state === 'gone' && record.spec.ai && record.info.resultState === null) {
|
|
431
|
+
this.ensureChunks(record);
|
|
432
|
+
const result = parseResultState(resultText(record));
|
|
433
|
+
if (result !== null) {
|
|
434
|
+
record.info.status = 'succeeded';
|
|
435
|
+
record.info.resultState = result;
|
|
436
|
+
this.pushRecovered(record, `\n[the agent had already written RESULT: ${result}, but the UI server did not record the end of the ` +
|
|
437
|
+
'run; the exit code is unknown]\n');
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
this.persist(record);
|
|
441
|
+
this.events.emit('job', record.info);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/** After a stop was requested, make sure the process is really gone; a signal that was ignored is not a stop. */
|
|
445
|
+
verifyLeftoverStopped(record, pid) {
|
|
446
|
+
const found = probeProcesses([pid]).get(pid) ?? null;
|
|
447
|
+
if (found === null)
|
|
448
|
+
return;
|
|
449
|
+
if (!matchesJob(found, record.spec, record.info.startedAt)) {
|
|
450
|
+
record.info.leftover = { ...(record.info.leftover ?? { pid }), state: 'unknown' };
|
|
451
|
+
this.persist(record);
|
|
452
|
+
this.events.emit('job', record.info);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
killProcessTree(pid, 'SIGKILL');
|
|
456
|
+
setTimeout(() => {
|
|
457
|
+
const still = probeProcesses([pid]).get(pid) ?? null;
|
|
458
|
+
if (still === null)
|
|
459
|
+
return;
|
|
460
|
+
const state = matchesJob(still, record.spec, record.info.startedAt) ? 'alive' : 'unknown';
|
|
461
|
+
record.info.leftover = { ...(record.info.leftover ?? { pid }), state };
|
|
462
|
+
this.persist(record);
|
|
463
|
+
this.events.emit('job', record.info);
|
|
464
|
+
if (state === 'alive')
|
|
465
|
+
this.startLeftoverProbe();
|
|
466
|
+
}, 1_000).unref();
|
|
212
467
|
}
|
|
213
468
|
/** Publishes a change of "waiting for the person" (a handoff card) like any other status change. */
|
|
214
469
|
syncAwaiting(record) {
|
|
@@ -222,21 +477,26 @@ class JobManager {
|
|
|
222
477
|
if (now && before && before.since === new Date(now.since).toISOString())
|
|
223
478
|
return;
|
|
224
479
|
info.awaitingUser = now ? { message: now.message, since: new Date(now.since).toISOString() } : null;
|
|
480
|
+
this.persist(record);
|
|
225
481
|
this.events.emit('job', info);
|
|
226
482
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
483
|
+
/**
|
|
484
|
+
* Stops a run whose agent has said nothing for `STALL_MS`: a tool call that never returns leaves the CLI waiting for
|
|
485
|
+
* ever, and the log would only keep repeating "still waiting for ...". The block above the cancel names what to do.
|
|
486
|
+
*/
|
|
487
|
+
watchStall(record) {
|
|
488
|
+
if (record.info.status !== 'running' || record.stalled || record.cancelRequested)
|
|
232
489
|
return;
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
490
|
+
// A handoff card is a wait with a person at the other end, not a hang: the silence is counted from the card's
|
|
491
|
+
// start and gets `HANDOFF_GRACE_MS` on top, so a person has time to act but a card that never returns is caught.
|
|
492
|
+
const awaitingSince = record.formatter?.awaiting?.since ?? 0;
|
|
493
|
+
const since = Math.max(record.lastActivityAt, awaitingSince);
|
|
494
|
+
const limit = awaitingSince ? STALL_MS + HANDOFF_GRACE_MS : STALL_MS;
|
|
495
|
+
if (Date.now() - since < limit)
|
|
496
|
+
return;
|
|
497
|
+
record.stalled = true;
|
|
498
|
+
this.pushPart(record, { kind: 'error', at: Date.now(), data: `\n${STALL_MESSAGE}\n` });
|
|
499
|
+
this.cancel(record.info.id);
|
|
240
500
|
}
|
|
241
501
|
/** A block of an AI run's log, with the time it was produced and how the UI should present it. */
|
|
242
502
|
pushPart(record, part) {
|
|
@@ -258,13 +518,49 @@ class JobManager {
|
|
|
258
518
|
/** Keeps the chunk for the browser and writes it to the job's log file, which is what is left after the run. */
|
|
259
519
|
store(record, chunk, fileText) {
|
|
260
520
|
record.chunks.push(chunk);
|
|
521
|
+
// A heartbeat line reports how long the silence is, so it must not count as activity of its own.
|
|
522
|
+
if (chunk.kind !== 'meta')
|
|
523
|
+
record.lastActivityAt = Date.now();
|
|
261
524
|
record.bufferedChars += chunk.data.length;
|
|
262
525
|
while (record.bufferedChars > MAX_BUFFERED_CHARS && record.chunks.length > 1) {
|
|
263
526
|
record.bufferedChars -= record.chunks.shift().data.length;
|
|
264
527
|
}
|
|
265
|
-
record.logStream
|
|
528
|
+
record.logStream?.write(fileText);
|
|
529
|
+
record.chunkStream?.write(`${JSON.stringify(chunk)}\n`);
|
|
266
530
|
record.emitter.emit('chunk', chunk);
|
|
267
531
|
}
|
|
532
|
+
/** A chunk for a job that no longer runs: appended to both files, never to a stream that is still open. */
|
|
533
|
+
pushRecovered(record, data) {
|
|
534
|
+
this.ensureChunks(record);
|
|
535
|
+
const chunk = { seq: record.nextSeq++, stream: 'system', data };
|
|
536
|
+
record.chunks.push(chunk);
|
|
537
|
+
record.bufferedChars += data.length;
|
|
538
|
+
while (record.bufferedChars > MAX_BUFFERED_CHARS && record.chunks.length > 1) {
|
|
539
|
+
record.bufferedChars -= record.chunks.shift().data.length;
|
|
540
|
+
}
|
|
541
|
+
try {
|
|
542
|
+
appendChunkNow(record.info.id, chunk, data);
|
|
543
|
+
}
|
|
544
|
+
catch (error) {
|
|
545
|
+
console.error('Could not write the job record:', error);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
persist(record) {
|
|
549
|
+
// `onFinish` is a function and is reattached by `buildJobSpec` when the job is started again.
|
|
550
|
+
const { onFinish: _onFinish, ...spec } = record.spec;
|
|
551
|
+
const persisted = {
|
|
552
|
+
version: 1,
|
|
553
|
+
info: record.info,
|
|
554
|
+
spec: spec,
|
|
555
|
+
...(record.pid !== null && { pid: record.pid }),
|
|
556
|
+
};
|
|
557
|
+
try {
|
|
558
|
+
writeRecord(persisted);
|
|
559
|
+
}
|
|
560
|
+
catch (error) {
|
|
561
|
+
console.error('Could not write the job record:', error);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
268
564
|
finish(record, code, signal, note) {
|
|
269
565
|
const { info } = record;
|
|
270
566
|
if (record.heartbeat)
|
|
@@ -273,21 +569,27 @@ class JobManager {
|
|
|
273
569
|
info.exitCode = code;
|
|
274
570
|
info.signal = signal;
|
|
275
571
|
info.endedAt = new Date().toISOString();
|
|
276
|
-
info.status = record.
|
|
277
|
-
? '
|
|
278
|
-
:
|
|
279
|
-
? '
|
|
280
|
-
:
|
|
572
|
+
info.status = record.stalled
|
|
573
|
+
? 'failed'
|
|
574
|
+
: record.cancelRequested
|
|
575
|
+
? 'cancelled'
|
|
576
|
+
: code === 0
|
|
577
|
+
? 'succeeded'
|
|
578
|
+
: 'failed';
|
|
281
579
|
const summary = note ??
|
|
282
|
-
(
|
|
283
|
-
? '[
|
|
284
|
-
:
|
|
285
|
-
?
|
|
286
|
-
:
|
|
580
|
+
(record.stalled
|
|
581
|
+
? '[stopped: the agent did not answer]'
|
|
582
|
+
: info.status === 'cancelled'
|
|
583
|
+
? '[cancelled]'
|
|
584
|
+
: signal
|
|
585
|
+
? `[process killed by ${signal}]`
|
|
586
|
+
: `[process exited with code ${code}]`);
|
|
287
587
|
if (record.spec.ai && info.status === 'succeeded')
|
|
288
|
-
info.resultState = parseResultState(record);
|
|
588
|
+
info.resultState = parseResultState(resultText(record));
|
|
289
589
|
this.push(record, 'system', `\n${summary}\n`);
|
|
290
|
-
record.logStream
|
|
590
|
+
record.logStream?.end();
|
|
591
|
+
record.chunkStream?.end();
|
|
592
|
+
this.persist(record);
|
|
291
593
|
try {
|
|
292
594
|
record.spec.onFinish?.(info);
|
|
293
595
|
}
|
|
@@ -299,18 +601,18 @@ class JobManager {
|
|
|
299
601
|
this.pruneFinished();
|
|
300
602
|
}
|
|
301
603
|
pruneFinished() {
|
|
302
|
-
const
|
|
604
|
+
const live = new Set([...this.jobs.values()].filter((record) => record.info.status === 'running').map((record) => record.info.id));
|
|
605
|
+
const finished = [...this.jobs.values()].filter((record) => record.info.status !== 'running' &&
|
|
606
|
+
record.info.leftover?.state !== 'alive' &&
|
|
607
|
+
record.info.leftover?.state !== 'unknown');
|
|
303
608
|
for (const record of finished.slice(0, Math.max(0, finished.length - MAX_FINISHED_JOBS))) {
|
|
304
609
|
this.jobs.delete(record.info.id);
|
|
305
610
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
const file = path.join(LOG_DIR, name);
|
|
312
|
-
if (Date.now() - fs.statSync(file).mtimeMs > LOG_RETENTION_MS)
|
|
313
|
-
fs.rmSync(file, { force: true });
|
|
611
|
+
try {
|
|
612
|
+
pruneJobs(MAX_FINISHED_JOBS, LOG_RETENTION_MS, live);
|
|
613
|
+
}
|
|
614
|
+
catch (error) {
|
|
615
|
+
console.error('Could not prune the job records:', error);
|
|
314
616
|
}
|
|
315
617
|
}
|
|
316
618
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface ProcessDetails {
|
|
2
|
+
/** The command line the system reports, empty when the platform could not read it. */
|
|
3
|
+
commandLine: string;
|
|
4
|
+
/** Milliseconds since the Unix epoch, or null when the platform could not report it. */
|
|
5
|
+
startedAt: number | null;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* The details of these pids, or `null` for one that is gone. One call for all of them: a restart usually has only a
|
|
9
|
+
* handful of jobs to look at, but probing should not be started once per job.
|
|
10
|
+
*/
|
|
11
|
+
export declare function probeProcesses(pids: number[]): Map<number, ProcessDetails | null>;
|
|
12
|
+
/**
|
|
13
|
+
* Ends a process and everything it started: the agent CLI, the browser and the MCP server below it. Off Windows the
|
|
14
|
+
* process was spawned as the leader of its own group, so the whole group is signalled; Windows has no process groups,
|
|
15
|
+
* and `taskkill /T` walks the tree instead.
|
|
16
|
+
*/
|
|
17
|
+
export declare function killProcessTree(pid: number, signal: NodeJS.Signals): void;
|