@asrunjs/asrun 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/.claude/commands/explore-case.md +11 -6
  2. package/.claude/commands/gen-spec.md +10 -6
  3. package/.claude/commands/heal-spec.md +13 -9
  4. package/.claude/commands/make-case.md +1 -1
  5. package/.claude/skills/exploring-cases/SKILL.md +84 -63
  6. package/.claude/skills/exploring-cases/references/assertion-targets.md +8 -4
  7. package/.claude/skills/exploring-cases/references/case-rules.md +110 -0
  8. package/.claude/skills/exploring-cases/references/handoff-card.js +73 -11
  9. package/.claude/skills/exploring-cases/references/notes.md +87 -0
  10. package/.claude/skills/exploring-cases/references/open-questions.md +58 -6
  11. package/.claude/skills/exploring-cases/references/readiness.md +7 -5
  12. package/.claude/skills/exploring-cases/references/tools.md +151 -0
  13. package/.claude/skills/exploring-cases/references/user-handoff.md +47 -18
  14. package/.claude/skills/generating-specs/SKILL.md +99 -54
  15. package/.claude/skills/generating-specs/references/conventions.md +4 -0
  16. package/.claude/skills/making-cases/SKILL.md +14 -7
  17. package/.claude/skills/making-cases/references/gaps.md +7 -4
  18. package/.claude/skills/making-cases/references/normalizing.md +1 -1
  19. package/AGENTS.md +61 -20
  20. package/README.md +10 -8
  21. package/bin/asrun.mjs +2 -0
  22. package/dist/lib/case-doc.d.ts +41 -4
  23. package/dist/lib/case-doc.js +109 -17
  24. package/dist/lib/explore-notes.d.ts +74 -0
  25. package/dist/lib/explore-notes.js +242 -0
  26. package/dist/lib/paths.d.ts +2 -0
  27. package/dist/lib/paths.js +2 -0
  28. package/dist/scripts/check-cases.js +89 -1
  29. package/dist/scripts/cli.js +9 -2
  30. package/dist/scripts/doctor.js +50 -3
  31. package/dist/scripts/explore-notes.d.ts +1 -0
  32. package/dist/scripts/explore-notes.js +131 -0
  33. package/dist/scripts/markers-confirm.d.ts +1 -0
  34. package/dist/scripts/markers-confirm.js +139 -0
  35. package/dist/ui/server/actions.js +50 -16
  36. package/dist/ui/server/app.js +7 -2
  37. package/dist/ui/server/job-store.d.ts +60 -0
  38. package/dist/ui/server/job-store.js +198 -0
  39. package/dist/ui/server/jobs.d.ts +46 -4
  40. package/dist/ui/server/jobs.js +361 -59
  41. package/dist/ui/server/process-info.d.ts +17 -0
  42. package/dist/ui/server/process-info.js +126 -0
  43. package/dist/ui/server/result-state.d.ts +10 -0
  44. package/dist/ui/server/result-state.js +106 -0
  45. package/dist/ui/shared/ai-commands.d.ts +4 -1
  46. package/dist/ui/shared/ai-commands.js +14 -4
  47. package/dist/ui/shared/types.d.ts +17 -2
  48. package/docs/agents/case-format.md +40 -5
  49. package/docs/agents/framework-dev.md +4 -0
  50. package/docs/ai-workflow.md +56 -16
  51. package/docs/cases.md +32 -8
  52. package/docs/commands.md +17 -1
  53. package/docs/theme-guide.html +56 -0
  54. package/docs/troubleshooting.md +40 -0
  55. package/docs/web-ui.md +59 -15
  56. package/package.json +3 -1
  57. package/templates/.claude/commands/explore-case.md +11 -6
  58. package/templates/.claude/commands/gen-spec.md +10 -5
  59. package/templates/.claude/commands/heal-spec.md +11 -7
  60. package/templates/.claude/commands/make-case.md +1 -1
  61. package/templates/AGENTS.md +9 -6
  62. package/templates/cases/_template.md +7 -0
  63. package/templates/package.json +2 -0
  64. package/ui/dist/assets/{dist-WvvCZeeu.js → dist--BXJInTq.js} +1 -1
  65. package/ui/dist/assets/{dist-B7HeglDd.js → dist-BMmNFIUH.js} +1 -1
  66. package/ui/dist/assets/{dist-DevASegc.js → dist-BXNaYHeh.js} +1 -1
  67. package/ui/dist/assets/{dist-Bb36rNUw.js → dist-ByLbEOlF.js} +1 -1
  68. package/ui/dist/assets/{dist-Dqc_tfzC.js → dist-C9ra6_BD.js} +1 -1
  69. package/ui/dist/assets/{dist-DfBpjgZT.js → dist-CK8mow8u.js} +1 -1
  70. package/ui/dist/assets/{dist-G9KNeh6u.js → dist-CQS0So-i.js} +1 -1
  71. package/ui/dist/assets/{dist-GvRUDL-2.js → dist-CTrwRc3Q.js} +1 -1
  72. package/ui/dist/assets/{dist-B9_GXR2G.js → dist-DTYj2vUD.js} +1 -1
  73. package/ui/dist/assets/{dist-Dq4Vnw5N.js → dist-mV0_nQZe.js} +1 -1
  74. package/ui/dist/assets/index-BzMqkmQ9.js +348 -0
  75. package/ui/dist/assets/index-DcJR-8Mt.css +2 -0
  76. package/ui/dist/index.html +2 -2
  77. package/ui/dist/assets/index-BJ5D2mFK.js +0 -346
  78. package/ui/dist/assets/index-BXY-eKBn.css +0 -2
@@ -1,33 +1,40 @@
1
- import { spawn, spawnSync } from 'node:child_process';
1
+ import { spawn } from 'node:child_process';
2
2
  import { EventEmitter } from 'node:events';
3
3
  import crypto from 'node:crypto';
4
4
  import fs from 'node:fs';
5
- import path from 'node:path';
6
5
  import { StringDecoder } from 'node:string_decoder';
7
6
  import { ROOT } from '../../lib/paths.js';
8
7
  import { spawnTarget } from '../../lib/command.js';
9
8
  import { AiStreamFormatter } from './ai-stream.js';
10
- const LOG_DIR = path.join(ROOT, '.tmp', 'ui-jobs');
9
+ import { parseResultState } from './result-state.js';
10
+ import { appendChunkNow, chunksFile, ensureJobDir, logFile, pruneJobs, readChunks, readJobs, writeRecord, } from './job-store.js';
11
+ import { killProcessTree, probeProcesses } from './process-info.js';
11
12
  const MAX_BUFFERED_CHARS = 4_000_000;
12
- const MAX_FINISHED_JOBS = 30;
13
+ const MAX_FINISHED_JOBS = 50;
13
14
  const LOG_RETENTION_MS = 7 * 24 * 3600 * 1000;
14
15
  const KILL_GRACE_MS = 5000;
15
16
  // An agent that has said nothing for this long gets a "still waiting for <tool call>" line in its log.
16
17
  const QUIET_MS = 45_000;
17
18
  const HEARTBEAT_MS = 15_000;
19
+ // A leftover process is looked at again this often, so a new AI run stops being blocked once it exits by itself.
20
+ const LEFTOVER_PROBE_MS = 5_000;
21
+ // An AI run that has said nothing at all for this long is given up on: a tool call that never returns (the biggest
22
+ // case: the `playwright-test` MCP server stopped answering) would otherwise leave the job running for ever. A handoff
23
+ // card waiting for a person gets `HANDOFF_GRACE_MS` on top, because that wait is the point of an attended run.
24
+ const STALL_MS = 10 * 60_000;
25
+ const HANDOFF_GRACE_MS = 5 * 60_000;
26
+ const STALL_MESSAGE = 'The agent stopped answering, so this run was stopped. The browser MCP server (`playwright-test`) most likely ' +
27
+ 'stopped responding: one interrupted browser call leaves it wedged for every later run in this workspace. Restart ' +
28
+ 'OpenCode (that ends its MCP processes) and start the run again; `pnpm asrun doctor` cannot see this, because it ' +
29
+ 'starts a fresh server of its own.';
18
30
  export class JobConflictError extends Error {
19
31
  }
20
- const ANSI = /\u001b\[[0-9;?]*[ -/]*[@-~]/g;
21
- /** The last `RESULT: ready|needs-input` line the agent said (markdown emphasis around it is fine). */
22
- function parseResultState(record) {
23
- const text = record.chunks
24
- // Only what the agent said counts: a file it read or a page it dumped may quote the same words.
32
+ /** What the agent itself said, as one text: a file it read or a page it dumped may quote the same words. */
33
+ function resultText(record) {
34
+ return record.chunks
25
35
  .filter((chunk) => (chunk.kind ? chunk.kind === 'markdown' : chunk.stream !== 'system'))
26
36
  .map((chunk) => chunk.data)
27
- .join('\n')
28
- .replace(ANSI, '');
29
- const matches = [...text.matchAll(/^[\s*`>_-]*RESULT:\s*(ready|needs-input)\b/gm)];
30
- return matches.at(-1)?.[1] ?? null;
37
+ .join('\n');
31
38
  }
32
39
  /**
33
40
  * A block of an AI run as the log file keeps it: the same text the UI shows, with the timestamp the UI styles written
@@ -48,31 +55,75 @@ function logFileText(part) {
48
55
  }
49
56
  return `${stamp}${part.data}\n`;
50
57
  }
58
+ /**
59
+ * Whether a process the system reports for a job's pid is really that job. The command line has to hold one of the
60
+ * arguments the job was started with (the script, the case, the profile), and the process must have started around the
61
+ * job: a pid that was reused by an unrelated program fails both.
62
+ */
63
+ function matchesJob(details, spec, startedAt) {
64
+ if (details.commandLine === '')
65
+ return false;
66
+ const line = details.commandLine.toLowerCase();
67
+ const tokens = spec.args.filter((value) => value.length >= 4).map((value) => value.toLowerCase());
68
+ const spawned = spec.spawnedPath;
69
+ if (spawned && spawned !== process.execPath && /[\\/]/.test(spawned))
70
+ tokens.push(spawned.toLowerCase());
71
+ if (!tokens.some((token) => line.includes(token)))
72
+ return false;
73
+ if (details.startedAt !== null) {
74
+ const started = Date.parse(startedAt);
75
+ if (Number.isFinite(started) && (details.startedAt < started - 60_000 || details.startedAt > started + 180_000)) {
76
+ return false;
77
+ }
78
+ }
79
+ return true;
80
+ }
51
81
  class JobManager {
52
82
  jobs = new Map();
83
+ leftoverTimer = null;
53
84
  /** Emits `job` with a JobInfo on every status change of any job. */
54
85
  events = new EventEmitter();
55
86
  constructor() {
56
87
  this.events.setMaxListeners(0);
57
- this.pruneOldLogs();
88
+ this.recover();
58
89
  }
59
90
  list() {
60
91
  return [...this.jobs.values()].map((record) => record.info).reverse();
61
92
  }
62
93
  get(id) {
63
- return this.jobs.get(id);
94
+ const record = this.jobs.get(id);
95
+ if (record)
96
+ this.ensureChunks(record);
97
+ return record;
98
+ }
99
+ /** The request "Run again" starts, or undefined when this job was not started from one. */
100
+ rerunRequest(id) {
101
+ return this.jobs.get(id)?.spec.rerun;
102
+ }
103
+ /**
104
+ * True while this job still holds its locks. A job a restart left behind holds them only while its process is
105
+ * confirmed to be alive: starting another agent next to it would wedge the one MCP server for both.
106
+ */
107
+ holdsLock(info) {
108
+ if (info.status === 'running')
109
+ return true;
110
+ return info.status === 'interrupted' && info.leftover?.state === 'alive';
64
111
  }
65
112
  start(spec) {
66
113
  for (const record of this.jobs.values()) {
67
- if (record.info.status !== 'running')
114
+ if (!this.holdsLock(record.info))
68
115
  continue;
69
116
  const clash = record.spec.locks.find((lock) => spec.locks.includes(lock));
70
- if (clash) {
117
+ if (!clash)
118
+ continue;
119
+ if (record.info.status === 'running') {
71
120
  throw new JobConflictError(`Cannot start: "${record.info.command}" is already running (${clash}). Wait for it to finish or cancel it.`);
72
121
  }
122
+ throw new JobConflictError(`Cannot start: "${record.info.command}" was interrupted by a UI server restart and its process is still ` +
123
+ `running (${clash}). Stop the leftover process in the Jobs drawer first.`);
73
124
  }
74
125
  const id = crypto.randomBytes(6).toString('hex');
75
- fs.mkdirSync(LOG_DIR, { recursive: true });
126
+ ensureJobDir(id);
76
127
  const record = {
77
128
  info: {
78
129
  id,
@@ -96,6 +147,7 @@ class JobManager {
96
147
  resultState: null,
97
148
  }),
98
149
  ...(spec.continuesJobId && { continuesJobId: spec.continuesJobId }),
150
+ ...(spec.rerun && { rerunable: true }),
99
151
  },
100
152
  spec,
101
153
  chunks: [],
@@ -103,20 +155,29 @@ class JobManager {
103
155
  nextSeq: 0,
104
156
  emitter: new EventEmitter(),
105
157
  child: null,
158
+ pid: null,
106
159
  cancelRequested: false,
107
- logStream: fs.createWriteStream(path.join(LOG_DIR, `${id}.log`)),
160
+ stalled: false,
161
+ lastActivityAt: Date.now(),
162
+ logStream: fs.createWriteStream(logFile(id)),
163
+ chunkStream: fs.createWriteStream(chunksFile(id)),
164
+ chunksLoaded: true,
108
165
  formatter: spec.stream ? new AiStreamFormatter(spec.stream) : null,
109
166
  heartbeat: null,
110
167
  };
111
168
  record.emitter.setMaxListeners(0);
112
169
  this.jobs.set(id, record);
113
170
  this.push(record, 'system', `$ ${spec.command}\n`);
171
+ // Written before the process starts, so a crash during the start still leaves a record to find.
172
+ this.persist(record);
114
173
  // A CLI installed globally is a `.cmd` shim on Windows, which spawn cannot start on its own (see lib/command.ts).
115
174
  const target = spawnTarget(spec.file, spec.args);
116
175
  if (target === null) {
117
176
  this.finish(record, null, null, `Failed to start: "${spec.file}" was not found on PATH. Install it, or choose the other tool.`);
118
177
  return record.info;
119
178
  }
179
+ record.spec.spawnedPath = target.path;
180
+ this.persist(record);
120
181
  let child;
121
182
  try {
122
183
  child = spawn(target.file, target.args, {
@@ -135,6 +196,8 @@ class JobManager {
135
196
  return record.info;
136
197
  }
137
198
  record.child = child;
199
+ record.pid = child.pid ?? null;
200
+ this.persist(record);
138
201
  for (const stream of ['stdout', 'stderr']) {
139
202
  const decoder = new StringDecoder('utf8');
140
203
  const formatted = stream === 'stdout' ? record.formatter : null;
@@ -163,6 +226,7 @@ class JobManager {
163
226
  const line = formatter.waiting(QUIET_MS);
164
227
  if (line)
165
228
  this.pushPart(record, line);
229
+ this.watchStall(record);
166
230
  }, HEARTBEAT_MS);
167
231
  record.heartbeat.unref();
168
232
  }
@@ -191,24 +255,215 @@ class JobManager {
191
255
  }
192
256
  cancel(id) {
193
257
  const record = this.jobs.get(id);
194
- const pid = record?.child?.pid;
195
- if (!record || record.info.status !== 'running' || !pid)
258
+ if (!record)
259
+ return false;
260
+ // A job the previous server left behind: only its own process is ended, and only once it is recognized.
261
+ const leftover = record.info.leftover;
262
+ if (record.info.status === 'interrupted' && leftover?.state === 'alive') {
263
+ const probe = probeProcesses([leftover.pid]).get(leftover.pid) ?? null;
264
+ if (probe === null) {
265
+ record.info.leftover = { ...leftover, state: 'gone' };
266
+ this.persist(record);
267
+ this.events.emit('job', record.info);
268
+ return false;
269
+ }
270
+ if (!matchesJob(probe, record.spec, record.info.startedAt)) {
271
+ record.info.leftover = { ...leftover, state: 'unknown' };
272
+ this.persist(record);
273
+ this.events.emit('job', record.info);
274
+ return false;
275
+ }
276
+ killProcessTree(leftover.pid, 'SIGTERM');
277
+ const leftoverPid = leftover.pid;
278
+ record.info.leftover = { ...leftover, state: 'stopped' };
279
+ this.pushRecovered(record, '\n[stopped: the leftover process tree of an interrupted job was ended]\n');
280
+ this.persist(record);
281
+ this.events.emit('job', record.info);
282
+ // A signal is not proof: if it did not work, the entry goes back to "alive" so the person can try again.
283
+ setTimeout(() => this.verifyLeftoverStopped(record, leftoverPid), KILL_GRACE_MS).unref();
284
+ return true;
285
+ }
286
+ const pid = record.child?.pid ?? record.pid;
287
+ if (record.info.status !== 'running' || !pid)
196
288
  return false;
197
289
  record.cancelRequested = true;
198
- this.signalGroup(pid, 'SIGTERM');
290
+ killProcessTree(pid, 'SIGTERM');
199
291
  setTimeout(() => {
200
292
  if (record.info.status === 'running')
201
- this.signalGroup(pid, 'SIGKILL');
293
+ killProcessTree(pid, 'SIGKILL');
202
294
  }, KILL_GRACE_MS).unref();
203
295
  return true;
204
296
  }
205
297
  /** Stops everything; called when the server shuts down. */
206
298
  killAll() {
207
299
  for (const record of this.jobs.values()) {
208
- const pid = record.child?.pid;
300
+ const pid = record.child?.pid ?? record.pid;
209
301
  if (record.info.status === 'running' && pid)
210
- this.signalGroup(pid, 'SIGTERM');
302
+ killProcessTree(pid, 'SIGTERM');
303
+ }
304
+ }
305
+ /**
306
+ * Reads the jobs of earlier server runs. One that was still running is marked `interrupted`: its output stops where
307
+ * the server died, and its process is either gone or a leftover the person can stop. An AI run that had already
308
+ * written its `RESULT:` line is recovered as succeeded, so its answers can still be answered.
309
+ */
310
+ recover() {
311
+ pruneJobs(MAX_FINISHED_JOBS, LOG_RETENTION_MS);
312
+ const stored = readJobs();
313
+ const interrupted = stored.filter((job) => job.info.status === 'running');
314
+ const pids = interrupted.map((job) => job.pid).filter((pid) => typeof pid === 'number');
315
+ const probe = probeProcesses(pids);
316
+ const now = new Date().toISOString();
317
+ for (const record of stored) {
318
+ const job = {
319
+ info: record.info,
320
+ spec: { ...record.spec },
321
+ chunks: [],
322
+ bufferedChars: 0,
323
+ nextSeq: 0,
324
+ emitter: new EventEmitter(),
325
+ child: null,
326
+ pid: record.pid ?? null,
327
+ cancelRequested: false,
328
+ stalled: false,
329
+ lastActivityAt: Date.now(),
330
+ logStream: null,
331
+ chunkStream: null,
332
+ chunksLoaded: false,
333
+ formatter: null,
334
+ heartbeat: null,
335
+ };
336
+ job.emitter.setMaxListeners(0);
337
+ if (record.info.status === 'running') {
338
+ // Its result line lives in the chunks, and the note about the restart is appended below.
339
+ this.ensureChunks(job);
340
+ this.recoverInterrupted(job, probe, now);
341
+ }
342
+ this.jobs.set(job.info.id, job);
343
+ }
344
+ // A record from an earlier restart may still say its process is alive; look once before the server answers.
345
+ this.checkLeftovers();
346
+ if ([...this.jobs.values()].some((job) => job.info.leftover?.state === 'alive')) {
347
+ this.startLeftoverProbe();
348
+ }
349
+ }
350
+ /** Reads a finished job's chunks the first time someone opens it; a job this server runs already has them. */
351
+ ensureChunks(record) {
352
+ if (record.chunksLoaded)
353
+ return;
354
+ const loaded = readChunks(record.info.id, MAX_BUFFERED_CHARS);
355
+ record.chunks = loaded.chunks;
356
+ record.bufferedChars = loaded.bufferedChars;
357
+ record.nextSeq = loaded.nextSeq;
358
+ record.chunksLoaded = true;
359
+ }
360
+ recoverInterrupted(record, probe, now) {
361
+ const info = record.info;
362
+ info.awaitingUser = null;
363
+ info.interruptedAt = now;
364
+ info.endedAt = now;
365
+ const pid = record.pid;
366
+ const found = pid === null ? null : (probe.get(pid) ?? null);
367
+ if (pid === null) {
368
+ info.status = 'interrupted';
369
+ info.leftover = undefined;
370
+ this.pushRecovered(record, '\n[interrupted: the UI server stopped while this job was running. Its process is unknown]\n');
371
+ }
372
+ else if (found === null) {
373
+ info.leftover = { pid, state: 'gone' };
374
+ const state = record.spec.ai ? parseResultState(resultText(record)) : null;
375
+ if (state !== null) {
376
+ // The agent had already finished its turn; the server only died before it could record the outcome.
377
+ info.status = 'succeeded';
378
+ info.resultState = state;
379
+ info.exitCode = null;
380
+ info.signal = null;
381
+ this.pushRecovered(record, `\n[the UI server stopped while this job was running, but the agent had already written ` +
382
+ `RESULT: ${state}. The exit code is unknown]\n`);
383
+ }
384
+ else {
385
+ info.status = 'interrupted';
386
+ info.resultState = null;
387
+ this.pushRecovered(record, '\n[interrupted: the UI server stopped while this job was running. Its process is gone]\n');
388
+ }
389
+ }
390
+ else if (matchesJob(found, record.spec, info.startedAt)) {
391
+ info.status = 'interrupted';
392
+ info.resultState = null;
393
+ info.leftover = { pid, state: 'alive' };
394
+ this.pushRecovered(record, `\n[interrupted: the UI server stopped while this job was running. Its process (pid ${pid}) is still ` +
395
+ 'running; stop it before starting another AI run]\n');
396
+ }
397
+ else {
398
+ info.status = 'interrupted';
399
+ info.resultState = null;
400
+ info.leftover = { pid, state: 'unknown' };
401
+ this.pushRecovered(record, `\n[interrupted: the UI server stopped while this job was running. A process with its pid (${pid}) is ` +
402
+ 'running, but it could not be identified as this job, so it was not marked for stopping]\n');
403
+ }
404
+ this.persist(record);
405
+ }
406
+ /** Keeps the leftover state current, so a process that exits by itself stops blocking new jobs. */
407
+ startLeftoverProbe() {
408
+ if (this.leftoverTimer)
409
+ return;
410
+ this.leftoverTimer = setInterval(() => this.checkLeftovers(), LEFTOVER_PROBE_MS);
411
+ this.leftoverTimer.unref();
412
+ }
413
+ checkLeftovers() {
414
+ const records = [...this.jobs.values()].filter((record) => record.info.status === 'interrupted' && record.info.leftover?.state === 'alive');
415
+ if (records.length === 0) {
416
+ if (this.leftoverTimer)
417
+ clearInterval(this.leftoverTimer);
418
+ this.leftoverTimer = null;
419
+ return;
211
420
  }
421
+ const probe = probeProcesses(records.map((record) => record.info.leftover.pid));
422
+ for (const record of records) {
423
+ const leftover = record.info.leftover;
424
+ const found = probe.get(leftover.pid) ?? null;
425
+ const state = found === null ? 'gone' : matchesJob(found, record.spec, record.info.startedAt) ? 'alive' : 'unknown';
426
+ if (state === leftover.state)
427
+ continue;
428
+ record.info.leftover = { ...leftover, state };
429
+ // A process that had already written its result and then exited is a finished turn, not an interruption.
430
+ if (state === 'gone' && record.spec.ai && record.info.resultState === null) {
431
+ this.ensureChunks(record);
432
+ const result = parseResultState(resultText(record));
433
+ if (result !== null) {
434
+ record.info.status = 'succeeded';
435
+ record.info.resultState = result;
436
+ this.pushRecovered(record, `\n[the agent had already written RESULT: ${result}, but the UI server did not record the end of the ` +
437
+ 'run; the exit code is unknown]\n');
438
+ }
439
+ }
440
+ this.persist(record);
441
+ this.events.emit('job', record.info);
442
+ }
443
+ }
444
+ /** After a stop was requested, make sure the process is really gone; a signal that was ignored is not a stop. */
445
+ verifyLeftoverStopped(record, pid) {
446
+ const found = probeProcesses([pid]).get(pid) ?? null;
447
+ if (found === null)
448
+ return;
449
+ if (!matchesJob(found, record.spec, record.info.startedAt)) {
450
+ record.info.leftover = { ...(record.info.leftover ?? { pid }), state: 'unknown' };
451
+ this.persist(record);
452
+ this.events.emit('job', record.info);
453
+ return;
454
+ }
455
+ killProcessTree(pid, 'SIGKILL');
456
+ setTimeout(() => {
457
+ const still = probeProcesses([pid]).get(pid) ?? null;
458
+ if (still === null)
459
+ return;
460
+ const state = matchesJob(still, record.spec, record.info.startedAt) ? 'alive' : 'unknown';
461
+ record.info.leftover = { ...(record.info.leftover ?? { pid }), state };
462
+ this.persist(record);
463
+ this.events.emit('job', record.info);
464
+ if (state === 'alive')
465
+ this.startLeftoverProbe();
466
+ }, 1_000).unref();
212
467
  }
213
468
  /** Publishes a change of "waiting for the person" (a handoff card) like any other status change. */
214
469
  syncAwaiting(record) {
@@ -222,21 +477,26 @@ class JobManager {
222
477
  if (now && before && before.since === new Date(now.since).toISOString())
223
478
  return;
224
479
  info.awaitingUser = now ? { message: now.message, since: new Date(now.since).toISOString() } : null;
480
+ this.persist(record);
225
481
  this.events.emit('job', info);
226
482
  }
227
- signalGroup(pid, signal) {
228
- if (process.platform === 'win32') {
229
- // Windows has no process groups (a negative pid throws ESRCH), so the whole tree is ended at once:
230
- // cmd.exe, the agent CLI it started, and anything below them.
231
- spawnSync('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore', windowsHide: true });
483
+ /**
484
+ * Stops a run whose agent has said nothing for `STALL_MS`: a tool call that never returns leaves the CLI waiting for
485
+ * ever, and the log would only keep repeating "still waiting for ...". The block above the cancel names what to do.
486
+ */
487
+ watchStall(record) {
488
+ if (record.info.status !== 'running' || record.stalled || record.cancelRequested)
232
489
  return;
233
- }
234
- try {
235
- process.kill(-pid, signal);
236
- }
237
- catch {
238
- // Already gone.
239
- }
490
+ // A handoff card is a wait with a person at the other end, not a hang: the silence is counted from the card's
491
+ // start and gets `HANDOFF_GRACE_MS` on top, so a person has time to act but a card that never returns is caught.
492
+ const awaitingSince = record.formatter?.awaiting?.since ?? 0;
493
+ const since = Math.max(record.lastActivityAt, awaitingSince);
494
+ const limit = awaitingSince ? STALL_MS + HANDOFF_GRACE_MS : STALL_MS;
495
+ if (Date.now() - since < limit)
496
+ return;
497
+ record.stalled = true;
498
+ this.pushPart(record, { kind: 'error', at: Date.now(), data: `\n${STALL_MESSAGE}\n` });
499
+ this.cancel(record.info.id);
240
500
  }
241
501
  /** A block of an AI run's log, with the time it was produced and how the UI should present it. */
242
502
  pushPart(record, part) {
@@ -258,13 +518,49 @@ class JobManager {
258
518
  /** Keeps the chunk for the browser and writes it to the job's log file, which is what is left after the run. */
259
519
  store(record, chunk, fileText) {
260
520
  record.chunks.push(chunk);
521
+ // A heartbeat line reports how long the silence is, so it must not count as activity of its own.
522
+ if (chunk.kind !== 'meta')
523
+ record.lastActivityAt = Date.now();
261
524
  record.bufferedChars += chunk.data.length;
262
525
  while (record.bufferedChars > MAX_BUFFERED_CHARS && record.chunks.length > 1) {
263
526
  record.bufferedChars -= record.chunks.shift().data.length;
264
527
  }
265
- record.logStream.write(fileText);
528
+ record.logStream?.write(fileText);
529
+ record.chunkStream?.write(`${JSON.stringify(chunk)}\n`);
266
530
  record.emitter.emit('chunk', chunk);
267
531
  }
532
+ /** A chunk for a job that no longer runs: appended to both files, never to a stream that is still open. */
533
+ pushRecovered(record, data) {
534
+ this.ensureChunks(record);
535
+ const chunk = { seq: record.nextSeq++, stream: 'system', data };
536
+ record.chunks.push(chunk);
537
+ record.bufferedChars += data.length;
538
+ while (record.bufferedChars > MAX_BUFFERED_CHARS && record.chunks.length > 1) {
539
+ record.bufferedChars -= record.chunks.shift().data.length;
540
+ }
541
+ try {
542
+ appendChunkNow(record.info.id, chunk, data);
543
+ }
544
+ catch (error) {
545
+ console.error('Could not write the job record:', error);
546
+ }
547
+ }
548
+ persist(record) {
549
+ // `onFinish` is a function and is reattached by `buildJobSpec` when the job is started again.
550
+ const { onFinish: _onFinish, ...spec } = record.spec;
551
+ const persisted = {
552
+ version: 1,
553
+ info: record.info,
554
+ spec: spec,
555
+ ...(record.pid !== null && { pid: record.pid }),
556
+ };
557
+ try {
558
+ writeRecord(persisted);
559
+ }
560
+ catch (error) {
561
+ console.error('Could not write the job record:', error);
562
+ }
563
+ }
268
564
  finish(record, code, signal, note) {
269
565
  const { info } = record;
270
566
  if (record.heartbeat)
@@ -273,21 +569,27 @@ class JobManager {
273
569
  info.exitCode = code;
274
570
  info.signal = signal;
275
571
  info.endedAt = new Date().toISOString();
276
- info.status = record.cancelRequested
277
- ? 'cancelled'
278
- : code === 0
279
- ? 'succeeded'
280
- : 'failed';
572
+ info.status = record.stalled
573
+ ? 'failed'
574
+ : record.cancelRequested
575
+ ? 'cancelled'
576
+ : code === 0
577
+ ? 'succeeded'
578
+ : 'failed';
281
579
  const summary = note ??
282
- (info.status === 'cancelled'
283
- ? '[cancelled]'
284
- : signal
285
- ? `[process killed by ${signal}]`
286
- : `[process exited with code ${code}]`);
580
+ (record.stalled
581
+ ? '[stopped: the agent did not answer]'
582
+ : info.status === 'cancelled'
583
+ ? '[cancelled]'
584
+ : signal
585
+ ? `[process killed by ${signal}]`
586
+ : `[process exited with code ${code}]`);
287
587
  if (record.spec.ai && info.status === 'succeeded')
288
- info.resultState = parseResultState(record);
588
+ info.resultState = parseResultState(resultText(record));
289
589
  this.push(record, 'system', `\n${summary}\n`);
290
- record.logStream.end();
590
+ record.logStream?.end();
591
+ record.chunkStream?.end();
592
+ this.persist(record);
291
593
  try {
292
594
  record.spec.onFinish?.(info);
293
595
  }
@@ -299,18 +601,18 @@ class JobManager {
299
601
  this.pruneFinished();
300
602
  }
301
603
  pruneFinished() {
302
- const finished = [...this.jobs.values()].filter((record) => record.info.status !== 'running');
604
+ const live = new Set([...this.jobs.values()].filter((record) => record.info.status === 'running').map((record) => record.info.id));
605
+ const finished = [...this.jobs.values()].filter((record) => record.info.status !== 'running' &&
606
+ record.info.leftover?.state !== 'alive' &&
607
+ record.info.leftover?.state !== 'unknown');
303
608
  for (const record of finished.slice(0, Math.max(0, finished.length - MAX_FINISHED_JOBS))) {
304
609
  this.jobs.delete(record.info.id);
305
610
  }
306
- }
307
- pruneOldLogs() {
308
- if (!fs.existsSync(LOG_DIR))
309
- return;
310
- for (const name of fs.readdirSync(LOG_DIR)) {
311
- const file = path.join(LOG_DIR, name);
312
- if (Date.now() - fs.statSync(file).mtimeMs > LOG_RETENTION_MS)
313
- fs.rmSync(file, { force: true });
611
+ try {
612
+ pruneJobs(MAX_FINISHED_JOBS, LOG_RETENTION_MS, live);
613
+ }
614
+ catch (error) {
615
+ console.error('Could not prune the job records:', error);
314
616
  }
315
617
  }
316
618
  }
@@ -0,0 +1,17 @@
1
+ export interface ProcessDetails {
2
+ /** The command line the system reports, empty when the platform could not read it. */
3
+ commandLine: string;
4
+ /** Milliseconds since the Unix epoch, or null when the platform could not report it. */
5
+ startedAt: number | null;
6
+ }
7
+ /**
8
+ * The details of these pids, or `null` for one that is gone. One call for all of them: a restart usually has only a
9
+ * handful of jobs to look at, but probing should not be started once per job.
10
+ */
11
+ export declare function probeProcesses(pids: number[]): Map<number, ProcessDetails | null>;
12
+ /**
13
+ * Ends a process and everything it started: the agent CLI, the browser and the MCP server below it. Off Windows the
14
+ * process was spawned as the leader of its own group, so the whole group is signalled; Windows has no process groups,
15
+ * and `taskkill /T` walks the tree instead.
16
+ */
17
+ export declare function killProcessTree(pid: number, signal: NodeJS.Signals): void;