@xiangyixie/dsh-superagent 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1464 +1,1464 @@
1
- /**
2
- * Run orchestration for dsh-superagent.
3
- *
4
- * A run moves through queued -> planning -> running -> aggregating and ends in
5
- * completed, partial, failed, cancelled, or interrupted. Planning, subtask
6
- * execution, and aggregation all happen in child agents; this module owns
7
- * ordering, retries, quotas, cancellation, and durable status.
8
- *
9
- * A state directory can be shared by more than one host process (two profiles,
10
- * or a synced folder), so every decision about an active run is made from
11
- * persisted evidence — heartbeat, owner pid, and the run lock — and never from
12
- * this process's memory alone.
13
- *
14
- * The module has no host imports: `spawnChild`, `readUsage`, `notify`, and
15
- * `store` are injected, which keeps it testable without a running harness.
16
- *
17
- * @module dsh-superagent/orchestrator
18
- */
19
- import { randomBytes } from 'node:crypto'
20
- import os from 'node:os'
21
- import { ACTIVE_STATUSES, TERMINAL_STATUSES } from './store.js'
22
-
23
- const STATUS_POLL_MS = 250
24
- /** How often the owning process checks for a cancel marker written elsewhere. */
25
- const CANCEL_POLL_MS = 1000
26
- /** How long a cancel from another process waits for the owner to acknowledge. */
27
- const CANCEL_ACK_MS = 5000
28
- /** How often a live run refreshes its heartbeat in the state file. */
29
- const HEARTBEAT_MS = 30 * 1000
30
- /** A run with no heartbeat for this long is treated as orphaned. */
31
- const ORPHAN_MS = 3 * HEARTBEAT_MS
32
- /** A child that ignores abort must not hold a cancel call open forever. */
33
- const DISPOSE_MS = 1500
34
- /** Waking the owning session must not hold a finished run open forever. */
35
- const NOTIFY_MS = 15 * 1000
36
- /** A run lock older than this belongs to a process that never released it. */
37
- const LOCK_STALE_MS = 10 * 60 * 1000
38
-
39
- /**
40
- * Mint a sortable, filesystem-safe run id.
41
- * @returns {string} run id such as `20261114T051233-a1b2`.
42
- */
43
- export function newRunId() {
44
- const stamp = new Date().toISOString().slice(0, 19).replace(/[-:]/g, '')
45
- return `${stamp}-${randomBytes(2).toString('hex')}`
46
- }
47
-
48
- function clamp(value, min, max) {
49
- const number = Number(value)
50
- if (!Number.isFinite(number)) return min
51
- return Math.min(max, Math.max(min, Math.trunc(number)))
52
- }
53
-
54
- /** Join the text blocks of a child result. */
55
- export function textOf(output) {
56
- if (!Array.isArray(output)) return ''
57
- return output
58
- .filter(block => block && block.type === 'text' && typeof block.text === 'string')
59
- .map(block => block.text)
60
- .join('\n')
61
- .trim()
62
- }
63
-
64
- /** Truncate a string to a UTF-8 byte budget without splitting a character. */
65
- export function capText(text, maxBytes) {
66
- if (typeof text !== 'string') return ''
67
- const buffer = Buffer.from(text, 'utf8')
68
- if (buffer.length <= maxBytes) return text
69
- // Cutting at an arbitrary byte can land inside a multi-byte character; the
70
- // decode then ends in U+FFFD. Drop that partial sequence instead of keeping
71
- // a replacement character in the user's report.
72
- const kept = buffer.subarray(0, maxBytes).toString('utf8').replace(/\uFFFD+$/, '')
73
- const omitted = buffer.length - Buffer.byteLength(kept, 'utf8')
74
- return `${kept}\n\n[truncated: ${omitted} bytes omitted]`
75
- }
76
-
77
- /**
78
- * Extract the first JSON object that carries a usable `subtasks` array.
79
- *
80
- * Models often wrap JSON in prose or code fences, so every balanced `{...}`
81
- * region is tried rather than assuming the reply is bare JSON.
82
- *
83
- * @param {string} text - planner reply.
84
- * @returns {Array<{ title: string, prompt: string }>} zero or more subtasks.
85
- */
86
- export function extractPlan(text) {
87
- if (typeof text !== 'string') return []
88
- const candidates = []
89
- for (let start = 0; start < text.length; start++) {
90
- if (text[start] !== '{') continue
91
- let depth = 0
92
- let inString = false
93
- let escaped = false
94
- for (let index = start; index < text.length; index++) {
95
- const char = text[index]
96
- if (inString) {
97
- if (escaped) escaped = false
98
- else if (char === '\\') escaped = true
99
- else if (char === '"') inString = false
100
- continue
101
- }
102
- if (char === '"') inString = true
103
- else if (char === '{') depth++
104
- else if (char === '}') {
105
- depth--
106
- if (depth === 0) {
107
- candidates.push(text.slice(start, index + 1))
108
- start = index
109
- break
110
- }
111
- }
112
- }
113
- }
114
- for (const candidate of candidates) {
115
- let parsed
116
- try {
117
- parsed = JSON.parse(candidate)
118
- } catch {
119
- continue
120
- }
121
- const raw = parsed?.subtasks
122
- if (!Array.isArray(raw)) continue
123
- const subtasks = []
124
- for (const item of raw) {
125
- if (!item || typeof item !== 'object') continue
126
- const prompt = typeof item.prompt === 'string' ? item.prompt.trim() : ''
127
- if (!prompt) continue
128
- const title = typeof item.title === 'string' && item.title.trim()
129
- ? item.title.trim().slice(0, 80)
130
- : prompt.slice(0, 60)
131
- subtasks.push({ title, prompt })
132
- }
133
- if (subtasks.length > 0) return subtasks
134
- }
135
- return []
136
- }
137
-
138
- /** Planner prompt for one goal. */
139
- function plannerPrompt(goal, maxSubtasks) {
140
- return [
141
- 'You are the planner for a long-horizon autonomous run.',
142
- '',
143
- 'Goal:',
144
- goal,
145
- '',
146
- `Decompose the goal into at most ${maxSubtasks} subtasks that can run in parallel because none of them needs another subtask's output.`,
147
- 'Each subtask is executed by a separate agent that sees only the prompt you write for it.',
148
- '',
149
- 'Return ONLY a JSON object, with no prose and no code fences, in exactly this shape:',
150
- '{"subtasks":[{"title":"short title","prompt":"complete self-contained instruction"}]}',
151
- '',
152
- 'Rules:',
153
- `- Use 2 to ${maxSubtasks} subtasks; use 1 only when the goal truly cannot be split.`,
154
- '- Every prompt must restate the context it needs; workers cannot see the goal text or each other.',
155
- '- Prefer concrete research, construction, or verification work over vague summaries.',
156
- ].join('\n')
157
- }
158
-
159
- /** Worker prompt for one subtask. */
160
- function workerPrompt(run, subtask) {
161
- return [
162
- 'You are one worker in a long-horizon autonomous run. Do not ask the user questions and do not wait for input.',
163
- '',
164
- 'Overall goal:',
165
- run.goal,
166
- '',
167
- `Your assigned subtask (${subtask.title}):`,
168
- subtask.prompt,
169
- '',
170
- 'Constraints:',
171
- '- Work only on your subtask; do not attempt the whole goal.',
172
- '- Use the tools you have. Never call a tool whose name starts with "superagent_".',
173
- '- Your final message IS the deliverable. Do not save the report to a file and do not reply with a',
174
- ' summary of what you did; write the findings themselves: concrete facts, numbers, file paths,',
175
- ' commands, and anything unresolved. Under 1500 words, Markdown only.',
176
- ].join('\n')
177
- }
178
-
179
- /** Aggregator prompt over completed subtask reports. */
180
- function aggregatorPrompt(run, completed) {
181
- const sections = completed.map(subtask => [
182
- `### ${subtask.title}`,
183
- capText(subtask.result ?? '', 8000),
184
- ].join('\n')).join('\n\n')
185
- return [
186
- 'You are the aggregator of a long-horizon autonomous run.',
187
- '',
188
- 'Goal:',
189
- run.goal,
190
- '',
191
- 'Subtask reports:',
192
- sections,
193
- '',
194
- 'Write the final report for the user in Markdown as your entire reply:',
195
- '- Lead with the answer or outcome, then supporting detail. If the goal asks for a table, include the table.',
196
- '- Merge duplicate findings; when reports disagree, say so and state which evidence is stronger.',
197
- '- Call out what failed, what is unverified, and what the user should do next.',
198
- '- Do not invent facts that no report contains. Do not say the report is ready — write it. Maximum 1200 words.',
199
- ].join('\n')
200
- }
201
-
202
- /** Deterministic report used when no aggregator model call is available. */
203
- function deterministicJoin(run) {
204
- const completed = (run.subtasks ?? []).filter(subtask => subtask.status === 'completed')
205
- if (completed.length === 0) return null
206
- const parts = [`# ${run.goal}`, '', `_Deterministic merge of ${completed.length} subtask report(s)._`, '']
207
- for (const subtask of completed) {
208
- parts.push(`## ${subtask.title}`, '', subtask.result ?? '', '')
209
- }
210
- return parts.join('\n').trim()
211
- }
212
-
213
- /** Run at most `limit` workers over `items` concurrently. */
214
- async function pool(items, limit, worker) {
215
- if (items.length === 0) return
216
- let cursor = 0
217
- const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
218
- while (cursor < items.length) {
219
- const item = items[cursor++]
220
- await worker(item)
221
- }
222
- })
223
- await Promise.all(runners)
224
- }
225
-
226
- export function sumUsage(entries) {
227
- const total = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, totalTokens: 0, modelCalls: 0, models: {} }
228
- for (const usage of entries) {
229
- if (!usage) continue
230
- total.inputTokens += usage.inputTokens ?? 0
231
- total.outputTokens += usage.outputTokens ?? 0
232
- total.cacheReadTokens += usage.cacheReadTokens ?? 0
233
- total.totalTokens += usage.totalTokens ?? 0
234
- total.modelCalls += usage.modelCalls ?? 0
235
- for (const [model, entry] of Object.entries(usage.models ?? {})) {
236
- const target = total.models[model] ?? (total.models[model] = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, calls: 0 })
237
- target.inputTokens += entry.inputTokens ?? 0
238
- target.outputTokens += entry.outputTokens ?? 0
239
- target.cacheReadTokens += entry.cacheReadTokens ?? 0
240
- target.calls += entry.calls ?? 0
241
- }
242
- }
243
- return total
244
- }
245
-
246
- /** Tokens billed for one usage record. */
247
- function usageTokens(usage) {
248
- return (usage?.inputTokens ?? 0) + (usage?.cacheReadTokens ?? 0) + (usage?.outputTokens ?? 0)
249
- }
250
-
251
- /**
252
- * Price recorded usage with an explicit, per-model table.
253
- *
254
- * Prices are per 1M tokens. Cache reads fall back to the input price when the
255
- * table does not list a separate rate, which makes the estimate an upper bound
256
- * instead of silently under-reporting. Models missing from the table are
257
- * listed in `unpriced` and excluded from the total.
258
- *
259
- * @param {object | null} usage - summed usage for one run.
260
- * @param {Record<string, { input: number, output: number, cacheRead?: number, currency?: string }> | undefined} pricing
261
- * @returns {{ total: number, currency: string, byModel: Record<string, number>, unpriced: string[], complete: boolean } | null}
262
- */
263
- export function computeCost(usage, pricing) {
264
- if (!usage || !usage.modelCalls) return null
265
- const table = pricing && typeof pricing === 'object' ? pricing : {}
266
- const byModel = {}
267
- const unpriced = []
268
- let total = 0
269
- let currency = null
270
- for (const [model, entry] of Object.entries(usage.models ?? {})) {
271
- // The usage log names a route, e.g. `dashscope/qwen-flash`. Accept a bare
272
- // model id too, so a one-route config does not have to guess the prefix;
273
- // an exact `provider/model` key always wins.
274
- const price = table[model] ?? table[model.slice(model.indexOf('/') + 1)]
275
- if (!price || typeof price.input !== 'number' || typeof price.output !== 'number') {
276
- unpriced.push(model)
277
- continue
278
- }
279
- const cacheRead = typeof price.cacheRead === 'number' ? price.cacheRead : price.input
280
- const cost = (
281
- entry.inputTokens * price.input
282
- + (entry.cacheReadTokens ?? 0) * cacheRead
283
- + entry.outputTokens * price.output
284
- ) / 1e6
285
- byModel[model] = cost
286
- total += cost
287
- currency = currency ?? price.currency ?? 'USD'
288
- }
289
- if (Object.keys(byModel).length === 0) return { total: 0, currency: currency ?? 'USD', byModel, unpriced, complete: false }
290
- return { total, currency, byModel, unpriced, complete: unpriced.length === 0 }
291
- }
292
-
293
- /** Total tokens recorded so far for a run, used by the budget guard. */
294
- function spentTokens(run) {
295
- return usageTokens(run.usage)
296
- }
297
-
298
- /**
299
- * Model calls recorded so far for a run, used by the budget guard.
300
- *
301
- * Measured usage is the accurate figure, but it only exists once a child's
302
- * session log can be read. A spawn is at least one model call, so the spawn
303
- * count is a floor: when the log layout changes and every read returns null,
304
- * the call budget still stops the run instead of silently never matching.
305
- */
306
- function spentCalls(run) {
307
- return Math.max(run.usage?.modelCalls ?? 0, run.counters?.spawns ?? 0)
308
- }
309
-
310
- function ensureCounters(run) {
311
- run.counters = { spawns: 0, unreadable: {}, ...(run.counters ?? {}) }
312
- if (!run.counters.unreadable || typeof run.counters.unreadable !== 'object') run.counters.unreadable = {}
313
- return run.counters
314
- }
315
-
316
- /**
317
- * Build the orchestrator.
318
- *
319
- * @param {object} deps - injected collaborators.
320
- * @param {object} deps.store - run store from `createStore`.
321
- * @param {object} deps.config - resolved plugin config.
322
- * @param {(spec: { label: string, prompt: string, parent: object, signal: AbortSignal }) => Promise<{ childId: string, result: Promise<object>, dispose: () => Promise<void> }>} deps.spawnChild
323
- * @param {(childId: string) => Promise<object | null>} deps.readUsage
324
- * @param {(run: object) => object | void | Promise<object | void>} deps.notify - wake the owning session once.
325
- * @param {(message: string) => void} [deps.log]
326
- * @param {() => number} [deps.now]
327
- * @param {(ms: number) => Promise<void>} [deps.sleep]
328
- */
329
- export function createOrchestrator(deps) {
330
- const {
331
- store,
332
- config,
333
- spawnChild,
334
- readUsage,
335
- notify,
336
- log = () => {},
337
- now = () => Date.now(),
338
- sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
339
- } = deps
340
-
341
- /** @type {Map<string, { controller: AbortController, activations: Set<object>, waiters: number, run: object, promise: Promise<void> | null, abortCause: string | null }>} */
342
- const runtime = new Map()
343
- /** Per-run write chain so two subtasks finishing together cannot interleave renames. */
344
- const saveChains = new Map()
345
- let startGate = Promise.resolve()
346
-
347
- // Cancel markers are optional in the store contract so tests can inject a
348
- // memory store without them.
349
- const hasCancelRequest = runId => (typeof store.hasCancelRequest === 'function'
350
- ? store.hasCancelRequest(runId)
351
- : Promise.resolve(false))
352
- const clearCancel = runId => (typeof store.clearCancel === 'function'
353
- ? store.clearCancel(runId)
354
- : Promise.resolve())
355
- const requestCancel = runId => (typeof store.requestCancel === 'function'
356
- ? store.requestCancel(runId)
357
- : Promise.resolve(false))
358
-
359
- function maxActiveRuns() {
360
- return clamp(config.maxActiveRuns ?? 2, 1, 8)
361
- }
362
-
363
- function save(run) {
364
- run.updatedAt = new Date(now()).toISOString()
365
- const previous = saveChains.get(run.id) ?? Promise.resolve()
366
- const next = previous.catch(() => {}).then(() => store.write(run))
367
- saveChains.set(run.id, next)
368
- return next
369
- }
370
-
371
- function liveRun(runId) {
372
- return runtime.get(runId)?.run ?? null
373
- }
374
-
375
- /** Cached machine identity of the state directory. */
376
- let ownerRef = null
377
- async function ownerIdentity() {
378
- if (ownerRef) return ownerRef
379
- const host = os.hostname()
380
- const existing = typeof store.readOwner === 'function' ? await store.readOwner() : null
381
- if (existing && typeof existing.hostname === 'string') {
382
- ownerRef = { hostname: host, local: existing.hostname === host, ownerHostname: existing.hostname }
383
- return ownerRef
384
- }
385
- if (typeof store.writeOwner === 'function') {
386
- await store.writeOwner({ hostname: host, createdAt: new Date(now()).toISOString() })
387
- }
388
- ownerRef = { hostname: host, local: true, ownerHostname: host }
389
- return ownerRef
390
- }
391
-
392
- /**
393
- * @param {unknown} pid - pid recorded by the process that owned the run.
394
- * @returns {boolean | null} true when alive, false when gone, null when unknown.
395
- */
396
- function pidAlive(pid) {
397
- const number = Number(pid)
398
- if (!Number.isInteger(number) || number <= 0) return null
399
- try {
400
- process.kill(number, 0)
401
- return true
402
- } catch (error) {
403
- // EPERM means the process exists but belongs to another user.
404
- return error?.code === 'EPERM'
405
- }
406
- }
407
-
408
- /**
409
- * Decide whether an active run still has a live owner.
410
- *
411
- * Used by recovery, orphan sweeping, and resume, so all three agree: a run
412
- * that somebody can still be working on is never rewritten. The heartbeat
413
- * alone proves ownership on a shared directory; a pid is only evidence when
414
- * the state directory belongs to this machine. A live run lock counts as
415
- * ownership because the holder is the process that may be spending.
416
- *
417
- * @param {object} run - persisted run record.
418
- * @returns {Promise<{ live: boolean, basis: string, heartbeatAt: number, owner: object, pidChecked: boolean }>}
419
- */
420
- async function ownership(run) {
421
- const owner = await ownerIdentity()
422
- const heartbeat = Date.parse(run.heartbeatAt ?? run.updatedAt ?? run.startedAt ?? run.createdAt)
423
- const heartbeatIso = Number.isFinite(heartbeat) ? new Date(heartbeat).toISOString() : 'unknown'
424
- const fresh = Number.isFinite(heartbeat) && heartbeat > now() - ORPHAN_MS
425
- // A pid only means something when the run was written on this machine. On
426
- // a shared directory the run may belong to a host whose pids are unrelated
427
- // to ours, so the heartbeat is the only evidence there.
428
- const localWriter = run.ownerHost ? run.ownerHost === os.hostname() : owner.local
429
- const alive = localWriter ? pidAlive(run.ownerPid) : null
430
-
431
- if (alive === true) {
432
- return { live: true, basis: `owner pid ${run.ownerPid} is still running`, heartbeatAt: heartbeat, owner, pidChecked: true }
433
- }
434
- if (typeof store.readLock === 'function') {
435
- const holder = await store.readLock(run.id).catch(() => null)
436
- if (holder && holder.hostname === os.hostname() && pidAlive(holder.pid) === true) {
437
- return { live: true, basis: `pid ${holder.pid} holds the run lock`, heartbeatAt: heartbeat, owner, pidChecked: true }
438
- }
439
- }
440
- // A heartbeat can be a few seconds old when the host is killed, and waiting
441
- // 90 s for it to expire would strand a run that this machine can prove is
442
- // dead. On the same machine, a missing pid outranks a fresh timestamp.
443
- if (alive === false) {
444
- return {
445
- live: false,
446
- basis: `owner pid ${run.ownerPid} is not running and this state directory is local`,
447
- heartbeatAt: heartbeat,
448
- owner,
449
- pidChecked: true,
450
- }
451
- }
452
- if (fresh) {
453
- return { live: true, basis: `its heartbeat was refreshed at ${heartbeatIso}`, heartbeatAt: heartbeat, owner, pidChecked: false }
454
- }
455
- const basis = owner.local
456
- ? `owner pid ${run.ownerPid ?? 'unknown'} is not running and its heartbeat stopped at ${heartbeatIso}`
457
- : `the state directory is shared with ${owner.ownerHostname}, so a local pid proves nothing; the heartbeat stopped at ${heartbeatIso}`
458
- return { live: false, basis, heartbeatAt: heartbeat, owner, pidChecked: owner.local }
459
- }
460
-
461
- /**
462
- * Rewrite a run whose owner is gone. Never silent: the exact evidence is
463
- * stored on the run and printed by status.
464
- */
465
- function reclaim(run, verdict, cancelled = false) {
466
- run.status = cancelled ? 'cancelled' : 'interrupted'
467
- run.error = cancelled
468
- ? 'cancelled by superagent_cancel while no process was running it'
469
- : 'host process stopped while the run was active'
470
- run.finishedAt = new Date(now()).toISOString()
471
- run.reclaimed = {
472
- at: new Date(now()).toISOString(),
473
- basis: verdict.basis,
474
- heartbeatAt: Number.isFinite(verdict.heartbeatAt) ? new Date(verdict.heartbeatAt).toISOString() : null,
475
- ownerPid: run.ownerPid ?? null,
476
- pidChecked: verdict.pidChecked,
477
- sharedWith: verdict.owner.local ? null : verdict.owner.ownerHostname,
478
- }
479
- for (const subtask of run.subtasks ?? []) {
480
- if (subtask.status === 'running') subtask.status = 'pending'
481
- }
482
- // The owner died before it could settle, so publish what is already known
483
- // about the spend: a reclaimed run must not claim its cost is unknown when
484
- // its child logs were readable.
485
- const entries = Object.values(run.usageByChild ?? {})
486
- if (entries.length > 0) run.usage = sumUsage(entries)
487
- run.cost = computeCost(run.usage, config.pricing)
488
- for (const subtask of run.subtasks ?? []) {
489
- if (subtask.childId && run.usageByChild?.[subtask.childId]) {
490
- subtask.usage = run.usageByChild[subtask.childId]
491
- }
492
- }
493
- }
494
-
495
- async function releaseRunLock(runId) {
496
- if (typeof store.releaseLock !== 'function') return
497
- try {
498
- await store.releaseLock(runId)
499
- } catch (error) {
500
- log(`cannot release the lock for run ${runId}: ${error?.message ?? error}`)
501
- }
502
- }
503
-
504
- /**
505
- * Claim the exclusive right to schedule a run.
506
- *
507
- * Two profiles can point at one state directory, so a resume must not be
508
- * decided from a read alone: both processes would see a terminal run and both
509
- * would schedule it. The lock makes one of them lose.
510
- */
511
- async function takeRunLock(runId) {
512
- if (typeof store.acquireLock !== 'function') return
513
- const holder = { pid: process.pid, hostname: os.hostname(), at: new Date(now()).toISOString() }
514
- let attempt = await store.acquireLock(runId, holder)
515
- if (!attempt.acquired) {
516
- const current = attempt.holder
517
- // A lock left by a process on this machine that no longer exists is
518
- // evidence of a crash, not of a live owner. A lock on another machine has
519
- // no checkable pid, so its age is the only evidence: past LOCK_STALE_MS
520
- // the process that took it is no longer plausibly still scheduling.
521
- const deadHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === false
522
- const aliveHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === true
523
- const age = now() - Date.parse(current?.at ?? '')
524
- const stale = !Number.isFinite(age) || age > LOCK_STALE_MS
525
- // Age is only evidence when the holder cannot be checked: a live process
526
- // on this machine may legitimately schedule a long run for hours.
527
- if (deadHere || (stale && !aliveHere)) {
528
- await releaseRunLock(runId)
529
- attempt = await store.acquireLock(runId, holder)
530
- }
531
- if (!attempt.acquired) {
532
- const held = attempt.holder
533
- const who = held
534
- ? `pid ${held.pid ?? 'unknown'} on ${held.hostname ?? 'an unknown host'}`
535
- : 'another process'
536
- throw new Error(`run ${runId} is already being started by ${who}; wait for it to finish or cancel it`)
537
- }
538
- }
539
- }
540
-
541
- /**
542
- * Reclaim runs whose owning process is gone, so an orphan cannot hold the
543
- * active-run quota forever.
544
- *
545
- * @returns {Promise<string[]>} reclaimed run ids, in list order.
546
- */
547
- async function sweepOrphans() {
548
- const stale = (await store.list()).filter(run => ACTIVE_STATUSES.has(run.status) && !runtime.has(run.id))
549
- if (stale.length === 0) return []
550
- const reclaimed = []
551
- for (const run of stale) {
552
- const verdict = await ownership(run)
553
- if (verdict.live) continue
554
- reclaim(run, verdict)
555
- await save(run)
556
- await releaseRunLock(run.id)
557
- log(`run ${run.id} was active with no live owner; reclaimed its quota (${verdict.basis})`)
558
- reclaimed.push(run.id)
559
- }
560
- return reclaimed
561
- }
562
-
563
- async function activeCount() {
564
- try {
565
- await sweepOrphans()
566
- } catch (error) {
567
- log(`cannot check for orphaned runs: ${error?.message ?? error}`)
568
- }
569
- let count = 0
570
- for (const rt of runtime.values()) {
571
- if (rt.run && ACTIVE_STATUSES.has(rt.run.status)) count++
572
- }
573
- for (const run of await store.list()) {
574
- if (runtime.has(run.id)) continue
575
- if (ACTIVE_STATUSES.has(run.status)) count++
576
- }
577
- return count
578
- }
579
-
580
- function quotaMessage(active) {
581
- return `quota exceeded: ${active} active run(s) and maxActiveRuns is ${maxActiveRuns()}. `
582
- + 'Let one finish with superagent_status or cancel it with superagent_cancel.'
583
- }
584
-
585
- function buildLimits(overrides = {}) {
586
- // A caller may narrow a limit but never widen the operator's cap: the
587
- // config is the spending and blast-radius control, and a model-written tool
588
- // argument must not be able to raise it.
589
- const narrowed = (override, cap) => (
590
- override === undefined || override === null ? cap : Math.min(Number(override), cap)
591
- )
592
- const limits = {
593
- maxSubtasks: clamp(narrowed(overrides.maxSubtasks, config.maxSubtasks), 1, 16),
594
- maxParallel: clamp(narrowed(overrides.maxParallel, config.maxParallel), 1, 8),
595
- deadlineMinutes: clamp(narrowed(overrides.deadlineMinutes, config.deadlineMinutes), 1, 720),
596
- attempts: clamp(config.attempts, 1, 5),
597
- maxModelCalls: clamp(narrowed(overrides.maxModelCalls, config.maxModelCallsPerRun), 0, 10000),
598
- maxTokens: clamp(narrowed(overrides.maxTokens, config.maxTokensPerRun), 0, 1e12),
599
- }
600
- limits.maxParallel = Math.min(limits.maxParallel, limits.maxSubtasks)
601
- return limits
602
- }
603
-
604
- /**
605
- * Report why a run may not start another child, or null when it still can.
606
- * Both budgets are hard stops: they exist so a pathological goal cannot spend
607
- * quietly, so they are checked before a spawn rather than after.
608
- */
609
- function budgetStop(run) {
610
- const calls = spentCalls(run)
611
- const tokens = spentTokens(run)
612
- if (run.limits.maxModelCalls > 0 && calls >= run.limits.maxModelCalls) {
613
- return `model-call budget reached (${calls}/${run.limits.maxModelCalls} calls)`
614
- }
615
- if (run.limits.maxTokens > 0 && tokens >= run.limits.maxTokens) {
616
- return `token budget reached (${tokens}/${run.limits.maxTokens} tokens)`
617
- }
618
- return null
619
- }
620
-
621
- /**
622
- * Spawn one child under the run's abort signal and budget.
623
- * @param {object} run - live run record.
624
- * @param {object} rt - runtime entry.
625
- * @param {{ label: string, prompt: string, parent: object }} spec - child request.
626
- */
627
- async function spawn(run, rt, spec) {
628
- const blocked = budgetStop(run)
629
- if (blocked) {
630
- if (!rt.abortCause) rt.abortCause = 'budget'
631
- if (!run.error) run.error = blocked
632
- if (!rt.controller.signal.aborted) rt.controller.abort(new Error(blocked))
633
- throw new Error(`budget exceeded: ${blocked}; raise maxModelCallsPerRun or maxTokensPerRun if this was intended`)
634
- }
635
- // Counted before the child exists and persisted before it can spend, so a
636
- // crash cannot lose the only evidence of a model call.
637
- ensureCounters(run).spawns += 1
638
- await save(run)
639
- return spawnAbortable(spec, rt.controller.signal)
640
- }
641
-
642
- /**
643
- * Create one child, but stop waiting for it as soon as the run is aborted.
644
- *
645
- * A provider that queues activations can leave `startActivation` pending long
646
- * after the run was cancelled or ran out of time. Without this the run would
647
- * hang in planning/running while its heartbeat kept it looking alive, so no
648
- * recovery path could ever reclaim it. A child that appears after the abort is
649
- * disposed immediately instead of being left to spend.
650
- */
651
- function spawnAbortable(spec, signal) {
652
- const pending = spawnChild({ ...spec, signal })
653
- if (!signal) return pending
654
- if (signal.aborted) {
655
- pending.then(handle => disposeQuietly(handle), () => {})
656
- return Promise.reject(new Error('aborted'))
657
- }
658
- return new Promise((resolve, reject) => {
659
- let done = false
660
- const onAbort = () => {
661
- if (done) return
662
- done = true
663
- // The `pending` handler registered below disposes the handle if it
664
- // arrives late, so the child is not left spending in the background.
665
- reject(new Error('aborted'))
666
- }
667
- signal.addEventListener('abort', onAbort, { once: true })
668
- pending.then(
669
- handle => {
670
- if (done) {
671
- disposeQuietly(handle)
672
- return
673
- }
674
- done = true
675
- signal.removeEventListener('abort', onAbort)
676
- resolve(handle)
677
- },
678
- error => {
679
- if (done) return
680
- done = true
681
- signal.removeEventListener('abort', onAbort)
682
- reject(error)
683
- },
684
- )
685
- })
686
- }
687
-
688
- /**
689
- * Read one run for a tool call, turning the two failure modes a user can hit
690
- * into sentences instead of a null.
691
- * @param {string} runId - run identity.
692
- */
693
- async function loadRun(runId) {
694
- const run = await store.read(runId)
695
- if (run) return run
696
- const quarantined = store.corrupt?.get?.(runId)
697
- if (quarantined) {
698
- throw new Error(
699
- `state file for run ${runId} is not valid JSON, so the run cannot be read. `
700
- + `The original file was kept at ${quarantined}. Inspect it, then delete it; `
701
- + 'the run is skipped by superagent_list until you do.',
702
- )
703
- }
704
- throw new Error(`unknown run ${runId}; call superagent_list to see recorded run ids`)
705
- }
706
-
707
- /**
708
- * Mark runs left active by a previous host process as interrupted.
709
- *
710
- * A run that another live process owns is left completely alone: rewriting it
711
- * here would make two processes fight over the same file while the first one
712
- * keeps spending, and a later resume would then run everything twice.
713
- */
714
- async function recover() {
715
- await store.init()
716
- const recovered = []
717
- for (const run of await store.list()) {
718
- if (!ACTIVE_STATUSES.has(run.status)) {
719
- if (await hasCancelRequest(run.id)) await clearCancel(run.id)
720
- continue
721
- }
722
- if (runtime.has(run.id)) continue
723
- const verdict = await ownership(run)
724
- if (verdict.live) {
725
- log(`run ${run.id} is still owned (${verdict.basis}); leaving it running`)
726
- continue
727
- }
728
- reclaim(run, verdict, await hasCancelRequest(run.id))
729
- await save(run)
730
- await clearCancel(run.id)
731
- await releaseRunLock(run.id)
732
- recovered.push(run.id)
733
- }
734
- return recovered
735
- }
736
-
737
- /**
738
- * Fold one child's usage snapshot into the run. The snapshot is cumulative,
739
- * so the last read for a child replaces the earlier one instead of adding.
740
- * Called at start, on every heartbeat, and on success or failure, which is
741
- * why an unreadable log degrades to the spawn floor instead of to zero.
742
- */
743
- async function recordUsage(run, childId) {
744
- let usage = null
745
- try {
746
- usage = await readUsage(childId)
747
- } catch (error) {
748
- log(`cannot read usage for child ${childId}: ${error?.message ?? error}`)
749
- }
750
- if (!usage) {
751
- ensureCounters(run).unreadable[childId] = true
752
- return null
753
- }
754
- run.usageByChild = run.usageByChild ?? {}
755
- run.usageByChild[childId] = usage
756
- run.usage = sumUsage(Object.values(run.usageByChild))
757
- return usage
758
- }
759
-
760
- /** Refresh usage for children that are still running, from the heartbeat. */
761
- async function refreshUsage(run, rt) {
762
- const handles = [...rt.activations]
763
- for (const handle of handles) await recordUsage(run, handle.childId)
764
- }
765
-
766
- /**
767
- * Wait for a child result, but stop waiting when the run is aborted. Without
768
- * this a child that ignores its abort signal keeps the whole run open.
769
- */
770
- function abortable(promise, signal) {
771
- if (!signal) return promise
772
- return new Promise((resolve, reject) => {
773
- const onAbort = () => {
774
- promise.catch(() => {})
775
- reject(new Error('aborted'))
776
- }
777
- if (signal.aborted) {
778
- onAbort()
779
- return
780
- }
781
- signal.addEventListener('abort', onAbort, { once: true })
782
- promise.then(
783
- value => {
784
- signal.removeEventListener('abort', onAbort)
785
- resolve(value)
786
- },
787
- error => {
788
- signal.removeEventListener('abort', onAbort)
789
- reject(error)
790
- },
791
- )
792
- })
793
- }
794
-
795
- /** Dispose one child activation without letting it block the caller. */
796
- async function disposeQuietly(handle) {
797
- if (!handle || typeof handle.dispose !== 'function') return
798
- try {
799
- await Promise.race([
800
- Promise.resolve().then(() => handle.dispose()),
801
- sleep(DISPOSE_MS),
802
- ])
803
- } catch (error) {
804
- log(`child dispose failed: ${error?.message ?? error}`)
805
- }
806
- }
807
-
808
- /**
809
- * The host's subagent service disappeared while this run was active, which
810
- * in practice means the host is shutting down. That is not a subtask
811
- * failure: the work never got a chance to run. Remember the cause so the run
812
- * settles as `interrupted` (resumable) instead of `failed`, and abort the
813
- * remaining children.
814
- */
815
- function markHostGone(rt, error) {
816
- if (rt.abortCause) return
817
- rt.abortCause = 'host-gone'
818
- if (!rt.controller.signal.aborted) {
819
- rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
820
- }
821
- }
822
-
823
- async function makePlan(run, parent, rt) {
824
- try {
825
- const handle = await spawn(run, rt, {
826
- label: 'superagent-plan',
827
- prompt: plannerPrompt(run.goal, run.limits.maxSubtasks),
828
- parent,
829
- })
830
- rt.activations.add(handle)
831
- run.steps.planner = { childId: handle.childId }
832
- try {
833
- const result = await abortable(handle.result, rt.controller.signal)
834
- const text = textOf(result.output)
835
- const subtasks = extractPlan(text).slice(0, run.limits.maxSubtasks)
836
- if (subtasks.length > 0) return { mode: 'planned', subtasks }
837
- log(`planner reply had no parsable plan; running the goal as one subtask`)
838
- } finally {
839
- await recordUsage(run, handle.childId)
840
- rt.activations.delete(handle)
841
- await disposeQuietly(handle)
842
- }
843
- } catch (error) {
844
- log(`planner failed: ${error?.message ?? error}`)
845
- if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
846
- }
847
- return { mode: 'degraded', subtasks: [{ title: 'Complete the goal', prompt: run.goal }] }
848
- }
849
-
850
- async function runSubtask(run, subtask, parent, rt) {
851
- const signal = rt.controller.signal
852
- for (let attempt = subtask.attempts + 1; attempt <= run.limits.attempts; attempt++) {
853
- if (signal.aborted) break
854
- subtask.status = 'running'
855
- subtask.attempts = attempt
856
- subtask.startedAt = subtask.startedAt ?? new Date(now()).toISOString()
857
- await save(run)
858
-
859
- let handle = null
860
- try {
861
- handle = await spawn(run, rt, {
862
- label: `superagent-${subtask.index}`,
863
- prompt: workerPrompt(run, subtask),
864
- parent,
865
- })
866
- rt.activations.add(handle)
867
- subtask.childId = handle.childId
868
- await save(run)
869
-
870
- const result = await abortable(handle.result, signal)
871
- const text = textOf(result.output)
872
- subtask.stopReason = result.stopReason
873
- if (result.stopReason === 'completed' && text) {
874
- subtask.status = 'completed'
875
- subtask.result = capText(text, config.resultCapBytes)
876
- subtask.error = null
877
- subtask.finishedAt = new Date(now()).toISOString()
878
- await recordUsage(run, handle.childId)
879
- await save(run)
880
- return
881
- }
882
- subtask.error = result.diagnostic || `child stopped with reason "${result.stopReason}"`
883
- log(`subtask ${subtask.index} attempt ${attempt} did not complete: ${subtask.error}`)
884
- } catch (error) {
885
- subtask.error = error?.message ?? String(error)
886
- log(`subtask ${subtask.index} attempt ${attempt} threw: ${subtask.error}`)
887
- if (error?.code === 'subagent-unavailable') {
888
- // The host is going away; leave the subtask as work to do rather
889
- // than burning the remaining attempts on a service that is gone.
890
- subtask.error = null
891
- subtask.stopReason = null
892
- markHostGone(rt, error)
893
- }
894
- } finally {
895
- if (handle) {
896
- // A failed attempt still spent tokens, so read it before disposing.
897
- await recordUsage(run, handle.childId)
898
- rt.activations.delete(handle)
899
- await disposeQuietly(handle)
900
- }
901
- }
902
-
903
- if (signal.aborted) break
904
- if (attempt < run.limits.attempts) {
905
- subtask.status = 'pending'
906
- await save(run)
907
- await sleep(1000 * attempt)
908
- }
909
- }
910
-
911
- if (subtask.status !== 'completed') {
912
- if (signal.aborted) {
913
- // Cancelled or out of time: the subtask did not fail on its own terms,
914
- // so leave it resumable instead of recording a failure.
915
- subtask.status = 'pending'
916
- subtask.error = null
917
- subtask.stopReason = null
918
- } else {
919
- subtask.status = 'failed'
920
- subtask.finishedAt = new Date(now()).toISOString()
921
- }
922
- await save(run)
923
- }
924
- }
925
-
926
- async function aggregate(run, parent, rt) {
927
- const completed = run.subtasks.filter(subtask => subtask.status === 'completed')
928
- if (completed.length === 0) return null
929
- if (rt.controller.signal.aborted) return deterministicJoin(run)
930
- if (completed.length === 1) return `# ${run.goal}\n\n${completed[0].result ?? ''}`.trim()
931
-
932
- try {
933
- const handle = await spawn(run, rt, {
934
- label: 'superagent-aggregate',
935
- prompt: aggregatorPrompt(run, completed),
936
- parent,
937
- })
938
- rt.activations.add(handle)
939
- run.steps.aggregator = { childId: handle.childId }
940
- try {
941
- const result = await abortable(handle.result, rt.controller.signal)
942
- const text = textOf(result.output)
943
- if (result.stopReason === 'completed' && text) return text
944
- log(`aggregator did not complete (${result.stopReason}); using deterministic merge`)
945
- } finally {
946
- await recordUsage(run, handle.childId)
947
- rt.activations.delete(handle)
948
- await disposeQuietly(handle)
949
- }
950
- } catch (error) {
951
- log(`aggregator failed: ${error?.message ?? error}`)
952
- if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
953
- }
954
- return deterministicJoin(run)
955
- }
956
-
957
- function terminalStatus(run, rt) {
958
- if (rt.abortCause === 'cancelled') return 'cancelled'
959
- // The host went away mid-run. Nothing failed on its own terms, so the run
960
- // must stay resumable instead of being buried as `failed`.
961
- if (rt.abortCause === 'host-gone') return 'interrupted'
962
- const subtasks = run.subtasks ?? []
963
- const completed = subtasks.filter(subtask => subtask.status === 'completed').length
964
- if (completed === 0) return 'failed'
965
- if (rt.abortCause === 'deadline') {
966
- return completed === subtasks.length ? 'completed' : 'partial'
967
- }
968
- return completed === subtasks.length ? 'completed' : 'partial'
969
- }
970
-
971
- /**
972
- * Watch for a cancel written by another process. The marker is checked from
973
- * inside the owning process, which is the only place that can abort children;
974
- * that is why cancel is a file and not a state-file rewrite.
975
- */
976
- function watchCancel(run, rt) {
977
- const timer = setInterval(() => {
978
- hasCancelRequest(run.id).then(requested => {
979
- if (!requested || rt.controller.signal.aborted) return
980
- rt.abortCause = 'cancelled'
981
- rt.controller.abort(new Error('cancelled by superagent_cancel from another process'))
982
- // Aborting the signal alone does not stop a child that is already
983
- // running; dispose its activation so it stops spending tokens.
984
- for (const handle of [...rt.activations]) disposeQuietly(handle)
985
- }).catch(() => {})
986
- }, CANCEL_POLL_MS)
987
- timer.unref?.()
988
- return timer
989
- }
990
-
991
- /**
992
- * A caller waiting inside this process will read the terminal result itself,
993
- * so waking the session would be duplicate noise. A caller in *another*
994
- * process is deliberately not treated this way: it can time out just before
995
- * the run settles, in which case suppressing the wake-up would lose the only
996
- * notification the user gets.
997
- */
998
- function watchedHere(rt) {
999
- return rt.waiters > 0
1000
- }
1001
-
1002
- /**
1003
- * Compute the terminal state, decide about notifying, and write it exactly
1004
- * once. Writing the terminal status first and the notify outcome second left
1005
- * a window where readers saw a finished run with no notify line at all.
1006
- */
1007
- async function settle(run, rt) {
1008
- // Usage is stored per child so repeated reads replace rather than add. A
1009
- // run written before that field existed keeps whatever total it had.
1010
- const entries = Object.values(run.usageByChild ?? {})
1011
- if (entries.length > 0) run.usage = sumUsage(entries)
1012
- // The per-subtask figure is read when the child finishes, which can race a
1013
- // log flush; the settle pass has the final snapshot, so publish it here.
1014
- for (const subtask of run.subtasks ?? []) {
1015
- if (subtask.childId && run.usageByChild?.[subtask.childId]) {
1016
- subtask.usage = run.usageByChild[subtask.childId]
1017
- }
1018
- }
1019
- run.cost = computeCost(run.usage, config.pricing)
1020
- const nextStatus = terminalStatus(run, rt)
1021
- const finishedAt = new Date(now()).toISOString()
1022
- let error = run.error
1023
- if (rt.abortCause === 'cancelled') {
1024
- error = 'cancelled by superagent_cancel'
1025
- } else if (rt.abortCause === 'host-gone') {
1026
- error = 'the host shut down while this run was active; run superagent_resume in a new host to finish it'
1027
- } else if (nextStatus === 'failed' && !error) {
1028
- error = run.subtasks.map(subtask => subtask.error).filter(Boolean).slice(0, 3).join('; ') || 'no subtask completed'
1029
- }
1030
- if (rt.abortCause === 'deadline' && !error) error = 'deadline exceeded'
1031
-
1032
- let outcome = null
1033
- if (TERMINAL_STATUSES.has(nextStatus)) {
1034
- if (watchedHere(rt)) {
1035
- outcome = { delivered: false, reason: 'skipped: a caller in this process is reading the run' }
1036
- } else {
1037
- try {
1038
- // `notify` may wait on the host's session loop; cap it so a finished
1039
- // run cannot stay active on disk (and be reclaimed by another process)
1040
- // just because a wake-up is slow.
1041
- outcome = await withTimeout(
1042
- Promise.resolve().then(() => notify({ ...run, status: nextStatus, finishedAt, error })),
1043
- NOTIFY_MS,
1044
- () => ({
1045
- delivered: false,
1046
- reason: `the owning session did not accept the wake-up within ${NOTIFY_MS / 1000}s`,
1047
- }),
1048
- )
1049
- outcome = outcome ?? { delivered: true }
1050
- } catch (notifyError) {
1051
- outcome = { delivered: false, reason: `notification threw: ${notifyError?.message ?? notifyError}` }
1052
- log(`notification failed for ${run.id}: ${notifyError?.message ?? notifyError}`)
1053
- }
1054
- }
1055
- }
1056
-
1057
- // The terminal status and its notify row land in the same write, so a
1058
- // reader never sees a finished run that still claims to be active and never
1059
- // sees a finished run whose notify row is missing.
1060
- run.status = nextStatus
1061
- run.finishedAt = finishedAt
1062
- run.activeMs = (Number.isFinite(run.activeMs) ? run.activeMs : 0) + Math.max(0, now() - rt.activeStart)
1063
- run.error = error
1064
- run.cancelPending = false
1065
- if (outcome) {
1066
- run.notify = {
1067
- at: new Date(now()).toISOString(),
1068
- delivered: outcome.delivered !== false,
1069
- reason: outcome.reason ?? null,
1070
- }
1071
- if (outcome.delivered !== false) run.notified = true
1072
- }
1073
- await save(run)
1074
- // The run is terminal, so a leftover marker would only make the next mount
1075
- // rewrite a finished run; drop it now.
1076
- await clearCancel(run.id)
1077
- await releaseRunLock(run.id)
1078
- }
1079
-
1080
- /**
1081
- * Resolve with `promise`, or with `onTimeout()` after `ms`.
1082
- *
1083
- * The timer is cleared when the promise wins: a stray 15s timer would keep a
1084
- * finished host process (and a test runner) alive for no reason.
1085
- */
1086
- function withTimeout(promise, ms, onTimeout) {
1087
- return new Promise((resolve, reject) => {
1088
- const timer = setTimeout(() => resolve(onTimeout()), ms)
1089
- timer.unref?.()
1090
- promise.then(
1091
- value => {
1092
- clearTimeout(timer)
1093
- resolve(value)
1094
- },
1095
- error => {
1096
- clearTimeout(timer)
1097
- reject(error)
1098
- },
1099
- )
1100
- })
1101
- }
1102
-
1103
- async function execute(run, parent, rt) {
1104
- rt.activeStart = now()
1105
- const remaining = Math.max(0, Date.parse(run.deadlineAt) - now())
1106
- const timer = setTimeout(() => {
1107
- rt.abortCause = 'deadline'
1108
- rt.controller.abort(new Error(`deadline exceeded after ${run.limits.deadlineMinutes} minutes`))
1109
- }, remaining)
1110
- timer.unref?.()
1111
- const cancelTimer = watchCancel(run, rt)
1112
- // The heartbeat is what makes an orphan detectable after a crash on a
1113
- // machine whose pids this process cannot check (shared state directory).
1114
- const beat = setInterval(() => {
1115
- run.heartbeatAt = new Date(now()).toISOString()
1116
- run.ownerPid = process.pid
1117
- refreshUsage(run, rt).then(() => save(run)).catch(() => {})
1118
- }, HEARTBEAT_MS)
1119
- beat.unref?.()
1120
-
1121
- try {
1122
- ensureCounters(run)
1123
- if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
1124
- run.status = 'planning'
1125
- await save(run)
1126
- const plan = await makePlan(run, parent, rt)
1127
- run.plan = { mode: plan.mode, subtasks: plan.subtasks }
1128
- run.subtasks = plan.subtasks.map((entry, index) => ({
1129
- index: index + 1,
1130
- title: entry.title,
1131
- prompt: entry.prompt,
1132
- status: 'pending',
1133
- attempts: 0,
1134
- childId: null,
1135
- stopReason: null,
1136
- error: null,
1137
- result: null,
1138
- usage: null,
1139
- startedAt: null,
1140
- finishedAt: null,
1141
- }))
1142
- await save(run)
1143
- }
1144
-
1145
- const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
1146
- if (open.length > 0) {
1147
- run.status = 'running'
1148
- await save(run)
1149
- await pool(open, run.limits.maxParallel, subtask => runSubtask(run, subtask, parent, rt))
1150
- }
1151
-
1152
- if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
1153
- run.status = 'aggregating'
1154
- await save(run)
1155
- run.report = await aggregate(run, parent, rt)
1156
- } else if (!run.report) {
1157
- run.report = deterministicJoin(run)
1158
- }
1159
- } catch (error) {
1160
- run.error = error?.message ?? String(error)
1161
- log(`run ${run.id} failed: ${run.error}`)
1162
- // Stop any child still running; without this the run would end while its
1163
- // subtasks keep burning tokens.
1164
- rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
1165
- } finally {
1166
- clearTimeout(timer)
1167
- clearInterval(cancelTimer)
1168
- // The heartbeat keeps running until the terminal write lands: waking the
1169
- // session can take a few seconds, and a frozen heartbeat during that
1170
- // window would let another process treat a live run as orphaned.
1171
- try {
1172
- await settle(run, rt)
1173
- } catch (error) {
1174
- // Never leave a run active because finalizing it threw.
1175
- log(`cannot finalize run ${run.id}: ${error?.message ?? error}`)
1176
- if (ACTIVE_STATUSES.has(run.status)) {
1177
- run.status = 'failed'
1178
- run.finishedAt = run.finishedAt ?? new Date(now()).toISOString()
1179
- run.error = run.error ?? `finalizing the run failed: ${error?.message ?? error}`
1180
- }
1181
- try {
1182
- await store.write(run)
1183
- } catch (writeError) {
1184
- log(`cannot persist the terminal state of run ${run.id}: ${writeError?.message ?? writeError}`)
1185
- }
1186
- await releaseRunLock(run.id)
1187
- } finally {
1188
- clearInterval(beat)
1189
- }
1190
- }
1191
- }
1192
-
1193
- function schedule(run, parent) {
1194
- run.ownerPid = process.pid
1195
- run.ownerHost = os.hostname()
1196
- run.heartbeatAt = new Date(now()).toISOString()
1197
- const rt = {
1198
- controller: new AbortController(),
1199
- activations: new Set(),
1200
- waiters: 0,
1201
- run,
1202
- promise: null,
1203
- abortCause: null,
1204
- }
1205
- runtime.set(run.id, rt)
1206
- rt.promise = execute(run, parent, rt)
1207
- .catch(async error => {
1208
- log(`run ${run.id} crashed: ${error?.message ?? error}`)
1209
- if (!ACTIVE_STATUSES.has(run.status)) return
1210
- run.status = 'failed'
1211
- run.finishedAt = new Date(now()).toISOString()
1212
- run.error = error?.message ?? String(error)
1213
- try {
1214
- await store.write(run)
1215
- } catch (writeError) {
1216
- log(`cannot persist the crash state of run ${run.id}: ${writeError?.message ?? writeError}`)
1217
- }
1218
- await releaseRunLock(run.id)
1219
- })
1220
- .then(() => {
1221
- // Finished runs are read from disk from now on; keeping them here would
1222
- // grow these maps for the lifetime of the host.
1223
- runtime.delete(run.id)
1224
- saveChains.delete(run.id)
1225
- })
1226
- return rt
1227
- }
1228
-
1229
- /** Serialize starts and resumes so a quota check cannot interleave. */
1230
- function serialize(task) {
1231
- const result = startGate.then(task, task)
1232
- startGate = result.catch(() => {})
1233
- return result
1234
- }
1235
-
1236
- async function start({ goal, parentSessionId, parent, clientTimeZone, overrides = {} }) {
1237
- const trimmed = typeof goal === 'string' ? goal.trim() : ''
1238
- if (!trimmed) throw new Error('goal is required')
1239
- if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
1240
-
1241
- const limits = buildLimits(overrides)
1242
- // Kept so a resume can rebuild the limits from the *current* config
1243
- // instead of freezing whatever was set when the run first started.
1244
- const keptOverrides = {}
1245
- for (const key of ['maxSubtasks', 'maxParallel', 'deadlineMinutes']) {
1246
- if (overrides[key] !== undefined && overrides[key] !== null) keptOverrides[key] = overrides[key]
1247
- }
1248
- return serialize(async () => {
1249
- const active = await activeCount()
1250
- if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
1251
- const run = {
1252
- id: newRunId(),
1253
- schemaVersion: 1,
1254
- goal: trimmed,
1255
- status: 'queued',
1256
- mode: 'planned',
1257
- parentSessionId,
1258
- clientTimeZone: clientTimeZone ?? null,
1259
- createdAt: new Date(now()).toISOString(),
1260
- updatedAt: null,
1261
- startedAt: new Date(now()).toISOString(),
1262
- deadlineAt: new Date(now() + limits.deadlineMinutes * 60000).toISOString(),
1263
- finishedAt: null,
1264
- resumedAt: null,
1265
- limits,
1266
- overrides: keptOverrides,
1267
- plan: null,
1268
- subtasks: [],
1269
- steps: {},
1270
- report: null,
1271
- error: null,
1272
- usage: null,
1273
- notified: false,
1274
- counters: { spawns: 0, unreadable: {} },
1275
- usageByChild: {},
1276
- }
1277
- await takeRunLock(run.id)
1278
- try {
1279
- await store.write(run)
1280
- } catch (error) {
1281
- await releaseRunLock(run.id)
1282
- throw error
1283
- }
1284
- schedule(run, parent)
1285
- return run
1286
- })
1287
- }
1288
-
1289
- async function resume(runId, parent, clientTimeZone) {
1290
- const id = String(runId ?? '').trim()
1291
- if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
1292
- return serialize(async () => {
1293
- if (runtime.has(id)) {
1294
- throw new Error(`run ${id} is still active in this process; wait for it with superagent_status or cancel it`)
1295
- }
1296
- let run = await loadRun(id)
1297
- if (run.status === 'completed') return run
1298
- if (ACTIVE_STATUSES.has(run.status)) {
1299
- const verdict = await ownership(run)
1300
- if (verdict.live) {
1301
- throw new Error(
1302
- `run ${id} is still owned by a live process (${verdict.basis}); resuming it here would run the same `
1303
- + 'subtasks twice and double the spend. Cancel it with superagent_cancel first, or wait for it to finish.',
1304
- )
1305
- }
1306
- }
1307
-
1308
- await takeRunLock(id)
1309
- let scheduled = false
1310
- try {
1311
- // Re-read under the lock: another process may have started the run
1312
- // between the check above and the claim.
1313
- run = await loadRun(id)
1314
- if (run.status === 'completed') return run
1315
- if (ACTIVE_STATUSES.has(run.status)) {
1316
- const verdict = await ownership(run)
1317
- if (verdict.live) {
1318
- throw new Error(`run ${id} is still owned by a live process (${verdict.basis}); not resuming it here`)
1319
- }
1320
- reclaim(run, verdict, await hasCancelRequest(id))
1321
- await save(run)
1322
- await clearCancel(id)
1323
- }
1324
-
1325
- const active = await activeCount()
1326
- if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
1327
-
1328
- for (const subtask of run.subtasks ?? []) {
1329
- if (subtask.status === 'completed') continue
1330
- subtask.status = 'pending'
1331
- subtask.attempts = 0
1332
- subtask.error = null
1333
- subtask.stopReason = null
1334
- subtask.finishedAt = null
1335
- }
1336
- run.status = 'queued'
1337
- run.report = run.plan ? run.report : null
1338
- run.error = null
1339
- run.finishedAt = null
1340
- run.reclaimed = null
1341
- run.cancelPending = false
1342
- // A resumed run completes a second time, so it must be able to wake the
1343
- // session again; keeping the old outcome would silence it forever.
1344
- run.notified = false
1345
- run.notify = null
1346
- run.resumedAt = new Date(now()).toISOString()
1347
- run.clientTimeZone = clientTimeZone ?? run.clientTimeZone ?? null
1348
- // The limits are rebuilt from the config that is active now, keeping
1349
- // everything already spent: that is what makes "raise the cap, then
1350
- // resume" work. Without this the old caps stayed in force forever.
1351
- run.limits = buildLimits(run.overrides ?? {})
1352
- run.deadlineAt = new Date(now() + run.limits.deadlineMinutes * 60000).toISOString()
1353
- ensureCounters(run)
1354
- await save(run)
1355
- schedule(run, parent)
1356
- scheduled = true
1357
- return run
1358
- } finally {
1359
- if (!scheduled) await releaseRunLock(id)
1360
- }
1361
- })
1362
- }
1363
-
1364
- async function cancel(runId) {
1365
- const rt = runtime.get(runId)
1366
- if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
1367
- rt.abortCause = 'cancelled'
1368
- rt.controller.abort(new Error('cancelled by superagent_cancel'))
1369
- const activations = [...rt.activations]
1370
- // A child that ignores its abort must not hold this call open; the
1371
- // timeout is per child and they are disposed in parallel.
1372
- await Promise.all(activations.map(handle => disposeQuietly(handle)))
1373
- await Promise.race([rt.promise ?? Promise.resolve(), sleep(CANCEL_ACK_MS)])
1374
- if (ACTIVE_STATUSES.has(rt.run.status)) {
1375
- rt.run.cancelPending = true
1376
- await save(rt.run).catch(() => {})
1377
- }
1378
- return rt.run
1379
- }
1380
-
1381
- const run = await loadRun(runId)
1382
- if (!ACTIVE_STATUSES.has(run.status)) return run
1383
-
1384
- // This process does not own the run, so it cannot abort the children. Ask
1385
- // the owner through the marker file and wait for it to rewrite the state.
1386
- // Writing `cancelled` here would be a lie: the owner would overwrite it on
1387
- // its next save and keep spending.
1388
- await requestCancel(runId)
1389
- const endsAt = now() + CANCEL_ACK_MS
1390
- while (now() < endsAt) {
1391
- await sleep(STATUS_POLL_MS)
1392
- const current = await store.read(runId)
1393
- if (!current) return run
1394
- if (!ACTIVE_STATUSES.has(current.status)) return current
1395
- }
1396
- // The owner did not confirm in time. Do not rewrite the state file from
1397
- // here: the owner would overwrite it on its next save. The request marker
1398
- // stays on disk, and `status` reports it as a pending cancel.
1399
- return { ...run, cancelPending: true }
1400
- }
1401
-
1402
- async function status(runId, waitSeconds = 0) {
1403
- const read = async () => liveRun(runId) ?? await store.read(runId)
1404
- let run = await read()
1405
- if (!run) {
1406
- await loadRun(runId)
1407
- throw new Error(`unknown run ${runId}`)
1408
- }
1409
- // A terminal status is only published after the run's final write, so a
1410
- // caller that sees one gets the notify row and usage totals with it.
1411
- const owner = runtime.get(runId)
1412
- if (owner && TERMINAL_STATUSES.has(run.status)) {
1413
- await withTimeout(Promise.resolve(owner.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
1414
- run = (await read()) ?? run
1415
- }
1416
- const limit = clamp(waitSeconds, 0, 3600)
1417
- if (ACTIVE_STATUSES.has(run.status) && !run.cancelPending && await hasCancelRequest(runId)) {
1418
- // The owning process may live elsewhere; the marker is the only evidence
1419
- // that a cancel was asked for and not yet acknowledged.
1420
- run = { ...run, cancelPending: true }
1421
- }
1422
- if (limit <= 0 || TERMINAL_STATUSES.has(run.status)) return run
1423
-
1424
- const rt = runtime.get(runId)
1425
- if (rt) rt.waiters++
1426
- try {
1427
- const endsAt = now() + limit * 1000
1428
- while (now() < endsAt) {
1429
- await sleep(STATUS_POLL_MS)
1430
- run = await read()
1431
- if (!run || TERMINAL_STATUSES.has(run.status)) break
1432
- }
1433
- } finally {
1434
- if (rt) rt.waiters = Math.max(0, rt.waiters - 1)
1435
- }
1436
-
1437
- if (!run) throw new Error(`run ${runId} disappeared from the state directory while waiting for it`)
1438
- const live = runtime.get(runId)
1439
- // Wait for this process's own final write, so a caller that sees a terminal
1440
- // status can immediately trust what is on disk. Bounded, because finalizing
1441
- // includes waking the session and must not turn a status call into a hang.
1442
- if (live && TERMINAL_STATUSES.has(live.run.status)) {
1443
- await withTimeout(Promise.resolve(live.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
1444
- }
1445
- return liveRun(runId) ?? run
1446
- }
1447
-
1448
- async function list() {
1449
- const merged = new Map()
1450
- for (const run of await store.list()) merged.set(run.id, run)
1451
- for (const [runId, rt] of runtime) merged.set(runId, rt.run)
1452
- return [...merged.values()].sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
1453
- }
1454
-
1455
- /**
1456
- * Resolve once every run started by this process has finished. Used by tests
1457
- * instead of sleeping, and by embedders that need a quiescent state.
1458
- */
1459
- async function idle() {
1460
- await Promise.all([...runtime.values()].map(rt => rt.promise?.catch(() => {})))
1461
- }
1462
-
1463
- return { recover, start, resume, cancel, status, list, idle, activeCount, liveRun }
1464
- }
1
+ /**
2
+ * Run orchestration for dsh-superagent.
3
+ *
4
+ * A run moves through queued -> planning -> running -> aggregating and ends in
5
+ * completed, partial, failed, cancelled, or interrupted. Planning, subtask
6
+ * execution, and aggregation all happen in child agents; this module owns
7
+ * ordering, retries, quotas, cancellation, and durable status.
8
+ *
9
+ * A state directory can be shared by more than one host process (two profiles,
10
+ * or a synced folder), so every decision about an active run is made from
11
+ * persisted evidence — heartbeat, owner pid, and the run lock — and never from
12
+ * this process's memory alone.
13
+ *
14
+ * The module has no host imports: `spawnChild`, `readUsage`, `notify`, and
15
+ * `store` are injected, which keeps it testable without a running harness.
16
+ *
17
+ * @module dsh-superagent/orchestrator
18
+ */
19
+ import { randomBytes } from 'node:crypto'
20
+ import os from 'node:os'
21
+ import { ACTIVE_STATUSES, TERMINAL_STATUSES } from './store.js'
22
+
23
+ const STATUS_POLL_MS = 250
24
+ /** How often the owning process checks for a cancel marker written elsewhere. */
25
+ const CANCEL_POLL_MS = 1000
26
+ /** How long a cancel from another process waits for the owner to acknowledge. */
27
+ const CANCEL_ACK_MS = 5000
28
+ /** How often a live run refreshes its heartbeat in the state file. */
29
+ const HEARTBEAT_MS = 30 * 1000
30
+ /** A run with no heartbeat for this long is treated as orphaned. */
31
+ const ORPHAN_MS = 3 * HEARTBEAT_MS
32
+ /** A child that ignores abort must not hold a cancel call open forever. */
33
+ const DISPOSE_MS = 1500
34
+ /** Waking the owning session must not hold a finished run open forever. */
35
+ const NOTIFY_MS = 15 * 1000
36
+ /** A run lock older than this belongs to a process that never released it. */
37
+ const LOCK_STALE_MS = 10 * 60 * 1000
38
+
39
+ /**
40
+ * Mint a sortable, filesystem-safe run id.
41
+ * @returns {string} run id such as `20261114T051233-a1b2`.
42
+ */
43
+ export function newRunId() {
44
+ const stamp = new Date().toISOString().slice(0, 19).replace(/[-:]/g, '')
45
+ return `${stamp}-${randomBytes(2).toString('hex')}`
46
+ }
47
+
48
+ function clamp(value, min, max) {
49
+ const number = Number(value)
50
+ if (!Number.isFinite(number)) return min
51
+ return Math.min(max, Math.max(min, Math.trunc(number)))
52
+ }
53
+
54
+ /** Join the text blocks of a child result. */
55
+ export function textOf(output) {
56
+ if (!Array.isArray(output)) return ''
57
+ return output
58
+ .filter(block => block && block.type === 'text' && typeof block.text === 'string')
59
+ .map(block => block.text)
60
+ .join('\n')
61
+ .trim()
62
+ }
63
+
64
+ /** Truncate a string to a UTF-8 byte budget without splitting a character. */
65
+ export function capText(text, maxBytes) {
66
+ if (typeof text !== 'string') return ''
67
+ const buffer = Buffer.from(text, 'utf8')
68
+ if (buffer.length <= maxBytes) return text
69
+ // Cutting at an arbitrary byte can land inside a multi-byte character; the
70
+ // decode then ends in U+FFFD. Drop that partial sequence instead of keeping
71
+ // a replacement character in the user's report.
72
+ const kept = buffer.subarray(0, maxBytes).toString('utf8').replace(/\uFFFD+$/, '')
73
+ const omitted = buffer.length - Buffer.byteLength(kept, 'utf8')
74
+ return `${kept}\n\n[truncated: ${omitted} bytes omitted]`
75
+ }
76
+
77
+ /**
78
+ * Extract the first JSON object that carries a usable `subtasks` array.
79
+ *
80
+ * Models often wrap JSON in prose or code fences, so every balanced `{...}`
81
+ * region is tried rather than assuming the reply is bare JSON.
82
+ *
83
+ * @param {string} text - planner reply.
84
+ * @returns {Array<{ title: string, prompt: string }>} zero or more subtasks.
85
+ */
86
+ export function extractPlan(text) {
87
+ if (typeof text !== 'string') return []
88
+ const candidates = []
89
+ for (let start = 0; start < text.length; start++) {
90
+ if (text[start] !== '{') continue
91
+ let depth = 0
92
+ let inString = false
93
+ let escaped = false
94
+ for (let index = start; index < text.length; index++) {
95
+ const char = text[index]
96
+ if (inString) {
97
+ if (escaped) escaped = false
98
+ else if (char === '\\') escaped = true
99
+ else if (char === '"') inString = false
100
+ continue
101
+ }
102
+ if (char === '"') inString = true
103
+ else if (char === '{') depth++
104
+ else if (char === '}') {
105
+ depth--
106
+ if (depth === 0) {
107
+ candidates.push(text.slice(start, index + 1))
108
+ start = index
109
+ break
110
+ }
111
+ }
112
+ }
113
+ }
114
+ for (const candidate of candidates) {
115
+ let parsed
116
+ try {
117
+ parsed = JSON.parse(candidate)
118
+ } catch {
119
+ continue
120
+ }
121
+ const raw = parsed?.subtasks
122
+ if (!Array.isArray(raw)) continue
123
+ const subtasks = []
124
+ for (const item of raw) {
125
+ if (!item || typeof item !== 'object') continue
126
+ const prompt = typeof item.prompt === 'string' ? item.prompt.trim() : ''
127
+ if (!prompt) continue
128
+ const title = typeof item.title === 'string' && item.title.trim()
129
+ ? item.title.trim().slice(0, 80)
130
+ : prompt.slice(0, 60)
131
+ subtasks.push({ title, prompt })
132
+ }
133
+ if (subtasks.length > 0) return subtasks
134
+ }
135
+ return []
136
+ }
137
+
138
+ /** Planner prompt for one goal. */
139
+ function plannerPrompt(goal, maxSubtasks) {
140
+ return [
141
+ 'You are the planner for a long-horizon autonomous run.',
142
+ '',
143
+ 'Goal:',
144
+ goal,
145
+ '',
146
+ `Decompose the goal into at most ${maxSubtasks} subtasks that can run in parallel because none of them needs another subtask's output.`,
147
+ 'Each subtask is executed by a separate agent that sees only the prompt you write for it.',
148
+ '',
149
+ 'Return ONLY a JSON object, with no prose and no code fences, in exactly this shape:',
150
+ '{"subtasks":[{"title":"short title","prompt":"complete self-contained instruction"}]}',
151
+ '',
152
+ 'Rules:',
153
+ `- Use 2 to ${maxSubtasks} subtasks; use 1 only when the goal truly cannot be split.`,
154
+ '- Every prompt must restate the context it needs; workers cannot see the goal text or each other.',
155
+ '- Prefer concrete research, construction, or verification work over vague summaries.',
156
+ ].join('\n')
157
+ }
158
+
159
+ /** Worker prompt for one subtask. */
160
+ function workerPrompt(run, subtask) {
161
+ return [
162
+ 'You are one worker in a long-horizon autonomous run. Do not ask the user questions and do not wait for input.',
163
+ '',
164
+ 'Overall goal:',
165
+ run.goal,
166
+ '',
167
+ `Your assigned subtask (${subtask.title}):`,
168
+ subtask.prompt,
169
+ '',
170
+ 'Constraints:',
171
+ '- Work only on your subtask; do not attempt the whole goal.',
172
+ '- Use the tools you have. Never call a tool whose name starts with "superagent_".',
173
+ '- Your final message IS the deliverable. Do not save the report to a file and do not reply with a',
174
+ ' summary of what you did; write the findings themselves: concrete facts, numbers, file paths,',
175
+ ' commands, and anything unresolved. Under 1500 words, Markdown only.',
176
+ ].join('\n')
177
+ }
178
+
179
+ /** Aggregator prompt over completed subtask reports. */
180
+ function aggregatorPrompt(run, completed) {
181
+ const sections = completed.map(subtask => [
182
+ `### ${subtask.title}`,
183
+ capText(subtask.result ?? '', 8000),
184
+ ].join('\n')).join('\n\n')
185
+ return [
186
+ 'You are the aggregator of a long-horizon autonomous run.',
187
+ '',
188
+ 'Goal:',
189
+ run.goal,
190
+ '',
191
+ 'Subtask reports:',
192
+ sections,
193
+ '',
194
+ 'Write the final report for the user in Markdown as your entire reply:',
195
+ '- Lead with the answer or outcome, then supporting detail. If the goal asks for a table, include the table.',
196
+ '- Merge duplicate findings; when reports disagree, say so and state which evidence is stronger.',
197
+ '- Call out what failed, what is unverified, and what the user should do next.',
198
+ '- Do not invent facts that no report contains. Do not say the report is ready — write it. Maximum 1200 words.',
199
+ ].join('\n')
200
+ }
201
+
202
+ /** Deterministic report used when no aggregator model call is available. */
203
+ function deterministicJoin(run) {
204
+ const completed = (run.subtasks ?? []).filter(subtask => subtask.status === 'completed')
205
+ if (completed.length === 0) return null
206
+ const parts = [`# ${run.goal}`, '', `_Deterministic merge of ${completed.length} subtask report(s)._`, '']
207
+ for (const subtask of completed) {
208
+ parts.push(`## ${subtask.title}`, '', subtask.result ?? '', '')
209
+ }
210
+ return parts.join('\n').trim()
211
+ }
212
+
213
+ /** Run at most `limit` workers over `items` concurrently. */
214
+ async function pool(items, limit, worker) {
215
+ if (items.length === 0) return
216
+ let cursor = 0
217
+ const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
218
+ while (cursor < items.length) {
219
+ const item = items[cursor++]
220
+ await worker(item)
221
+ }
222
+ })
223
+ await Promise.all(runners)
224
+ }
225
+
226
+ export function sumUsage(entries) {
227
+ const total = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, totalTokens: 0, modelCalls: 0, models: {} }
228
+ for (const usage of entries) {
229
+ if (!usage) continue
230
+ total.inputTokens += usage.inputTokens ?? 0
231
+ total.outputTokens += usage.outputTokens ?? 0
232
+ total.cacheReadTokens += usage.cacheReadTokens ?? 0
233
+ total.totalTokens += usage.totalTokens ?? 0
234
+ total.modelCalls += usage.modelCalls ?? 0
235
+ for (const [model, entry] of Object.entries(usage.models ?? {})) {
236
+ const target = total.models[model] ?? (total.models[model] = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, calls: 0 })
237
+ target.inputTokens += entry.inputTokens ?? 0
238
+ target.outputTokens += entry.outputTokens ?? 0
239
+ target.cacheReadTokens += entry.cacheReadTokens ?? 0
240
+ target.calls += entry.calls ?? 0
241
+ }
242
+ }
243
+ return total
244
+ }
245
+
246
+ /** Tokens billed for one usage record. */
247
+ function usageTokens(usage) {
248
+ return (usage?.inputTokens ?? 0) + (usage?.cacheReadTokens ?? 0) + (usage?.outputTokens ?? 0)
249
+ }
250
+
251
+ /**
252
+ * Price recorded usage with an explicit, per-model table.
253
+ *
254
+ * Prices are per 1M tokens. Cache reads fall back to the input price when the
255
+ * table does not list a separate rate, which makes the estimate an upper bound
256
+ * instead of silently under-reporting. Models missing from the table are
257
+ * listed in `unpriced` and excluded from the total.
258
+ *
259
+ * @param {object | null} usage - summed usage for one run.
260
+ * @param {Record<string, { input: number, output: number, cacheRead?: number, currency?: string }> | undefined} pricing
261
+ * @returns {{ total: number, currency: string, byModel: Record<string, number>, unpriced: string[], complete: boolean } | null}
262
+ */
263
+ export function computeCost(usage, pricing) {
264
+ if (!usage || !usage.modelCalls) return null
265
+ const table = pricing && typeof pricing === 'object' ? pricing : {}
266
+ const byModel = {}
267
+ const unpriced = []
268
+ let total = 0
269
+ let currency = null
270
+ for (const [model, entry] of Object.entries(usage.models ?? {})) {
271
+ // The usage log names a route, e.g. `dashscope/qwen-flash`. Accept a bare
272
+ // model id too, so a one-route config does not have to guess the prefix;
273
+ // an exact `provider/model` key always wins.
274
+ const price = table[model] ?? table[model.slice(model.indexOf('/') + 1)]
275
+ if (!price || typeof price.input !== 'number' || typeof price.output !== 'number') {
276
+ unpriced.push(model)
277
+ continue
278
+ }
279
+ const cacheRead = typeof price.cacheRead === 'number' ? price.cacheRead : price.input
280
+ const cost = (
281
+ entry.inputTokens * price.input
282
+ + (entry.cacheReadTokens ?? 0) * cacheRead
283
+ + entry.outputTokens * price.output
284
+ ) / 1e6
285
+ byModel[model] = cost
286
+ total += cost
287
+ currency = currency ?? price.currency ?? 'USD'
288
+ }
289
+ if (Object.keys(byModel).length === 0) return { total: 0, currency: currency ?? 'USD', byModel, unpriced, complete: false }
290
+ return { total, currency, byModel, unpriced, complete: unpriced.length === 0 }
291
+ }
292
+
293
+ /** Total tokens recorded so far for a run, used by the budget guard. */
294
+ function spentTokens(run) {
295
+ return usageTokens(run.usage)
296
+ }
297
+
298
+ /**
299
+ * Model calls recorded so far for a run, used by the budget guard.
300
+ *
301
+ * Measured usage is the accurate figure, but it only exists once a child's
302
+ * session log can be read. A spawn is at least one model call, so the spawn
303
+ * count is a floor: when the log layout changes and every read returns null,
304
+ * the call budget still stops the run instead of silently never matching.
305
+ */
306
+ function spentCalls(run) {
307
+ return Math.max(run.usage?.modelCalls ?? 0, run.counters?.spawns ?? 0)
308
+ }
309
+
310
+ function ensureCounters(run) {
311
+ run.counters = { spawns: 0, unreadable: {}, ...(run.counters ?? {}) }
312
+ if (!run.counters.unreadable || typeof run.counters.unreadable !== 'object') run.counters.unreadable = {}
313
+ return run.counters
314
+ }
315
+
316
+ /**
317
+ * Build the orchestrator.
318
+ *
319
+ * @param {object} deps - injected collaborators.
320
+ * @param {object} deps.store - run store from `createStore`.
321
+ * @param {object} deps.config - resolved plugin config.
322
+ * @param {(spec: { label: string, prompt: string, parent: object, signal: AbortSignal }) => Promise<{ childId: string, result: Promise<object>, dispose: () => Promise<void> }>} deps.spawnChild
323
+ * @param {(childId: string) => Promise<object | null>} deps.readUsage
324
+ * @param {(run: object) => object | void | Promise<object | void>} deps.notify - wake the owning session once.
325
+ * @param {(message: string) => void} [deps.log]
326
+ * @param {() => number} [deps.now]
327
+ * @param {(ms: number) => Promise<void>} [deps.sleep]
328
+ */
329
+ export function createOrchestrator(deps) {
330
+ const {
331
+ store,
332
+ config,
333
+ spawnChild,
334
+ readUsage,
335
+ notify,
336
+ log = () => {},
337
+ now = () => Date.now(),
338
+ sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
339
+ } = deps
340
+
341
+ /** @type {Map<string, { controller: AbortController, activations: Set<object>, waiters: number, run: object, promise: Promise<void> | null, abortCause: string | null }>} */
342
+ const runtime = new Map()
343
+ /** Per-run write chain so two subtasks finishing together cannot interleave renames. */
344
+ const saveChains = new Map()
345
+ let startGate = Promise.resolve()
346
+
347
+ // Cancel markers are optional in the store contract so tests can inject a
348
+ // memory store without them.
349
+ const hasCancelRequest = runId => (typeof store.hasCancelRequest === 'function'
350
+ ? store.hasCancelRequest(runId)
351
+ : Promise.resolve(false))
352
+ const clearCancel = runId => (typeof store.clearCancel === 'function'
353
+ ? store.clearCancel(runId)
354
+ : Promise.resolve())
355
+ const requestCancel = runId => (typeof store.requestCancel === 'function'
356
+ ? store.requestCancel(runId)
357
+ : Promise.resolve(false))
358
+
359
+ function maxActiveRuns() {
360
+ return clamp(config.maxActiveRuns ?? 2, 1, 8)
361
+ }
362
+
363
+ function save(run) {
364
+ run.updatedAt = new Date(now()).toISOString()
365
+ const previous = saveChains.get(run.id) ?? Promise.resolve()
366
+ const next = previous.catch(() => {}).then(() => store.write(run))
367
+ saveChains.set(run.id, next)
368
+ return next
369
+ }
370
+
371
+ function liveRun(runId) {
372
+ return runtime.get(runId)?.run ?? null
373
+ }
374
+
375
+ /** Cached machine identity of the state directory. */
376
+ let ownerRef = null
377
+ async function ownerIdentity() {
378
+ if (ownerRef) return ownerRef
379
+ const host = os.hostname()
380
+ const existing = typeof store.readOwner === 'function' ? await store.readOwner() : null
381
+ if (existing && typeof existing.hostname === 'string') {
382
+ ownerRef = { hostname: host, local: existing.hostname === host, ownerHostname: existing.hostname }
383
+ return ownerRef
384
+ }
385
+ if (typeof store.writeOwner === 'function') {
386
+ await store.writeOwner({ hostname: host, createdAt: new Date(now()).toISOString() })
387
+ }
388
+ ownerRef = { hostname: host, local: true, ownerHostname: host }
389
+ return ownerRef
390
+ }
391
+
392
+ /**
393
+ * @param {unknown} pid - pid recorded by the process that owned the run.
394
+ * @returns {boolean | null} true when alive, false when gone, null when unknown.
395
+ */
396
+ function pidAlive(pid) {
397
+ const number = Number(pid)
398
+ if (!Number.isInteger(number) || number <= 0) return null
399
+ try {
400
+ process.kill(number, 0)
401
+ return true
402
+ } catch (error) {
403
+ // EPERM means the process exists but belongs to another user.
404
+ return error?.code === 'EPERM'
405
+ }
406
+ }
407
+
408
+ /**
409
+ * Decide whether an active run still has a live owner.
410
+ *
411
+ * Used by recovery, orphan sweeping, and resume, so all three agree: a run
412
+ * that somebody can still be working on is never rewritten. The heartbeat
413
+ * alone proves ownership on a shared directory; a pid is only evidence when
414
+ * the state directory belongs to this machine. A live run lock counts as
415
+ * ownership because the holder is the process that may be spending.
416
+ *
417
+ * @param {object} run - persisted run record.
418
+ * @returns {Promise<{ live: boolean, basis: string, heartbeatAt: number, owner: object, pidChecked: boolean }>}
419
+ */
420
+ async function ownership(run) {
421
+ const owner = await ownerIdentity()
422
+ const heartbeat = Date.parse(run.heartbeatAt ?? run.updatedAt ?? run.startedAt ?? run.createdAt)
423
+ const heartbeatIso = Number.isFinite(heartbeat) ? new Date(heartbeat).toISOString() : 'unknown'
424
+ const fresh = Number.isFinite(heartbeat) && heartbeat > now() - ORPHAN_MS
425
+ // A pid only means something when the run was written on this machine. On
426
+ // a shared directory the run may belong to a host whose pids are unrelated
427
+ // to ours, so the heartbeat is the only evidence there.
428
+ const localWriter = run.ownerHost ? run.ownerHost === os.hostname() : owner.local
429
+ const alive = localWriter ? pidAlive(run.ownerPid) : null
430
+
431
+ if (alive === true) {
432
+ return { live: true, basis: `owner pid ${run.ownerPid} is still running`, heartbeatAt: heartbeat, owner, pidChecked: true }
433
+ }
434
+ if (typeof store.readLock === 'function') {
435
+ const holder = await store.readLock(run.id).catch(() => null)
436
+ if (holder && holder.hostname === os.hostname() && pidAlive(holder.pid) === true) {
437
+ return { live: true, basis: `pid ${holder.pid} holds the run lock`, heartbeatAt: heartbeat, owner, pidChecked: true }
438
+ }
439
+ }
440
+ // A heartbeat can be a few seconds old when the host is killed, and waiting
441
+ // 90 s for it to expire would strand a run that this machine can prove is
442
+ // dead. On the same machine, a missing pid outranks a fresh timestamp.
443
+ if (alive === false) {
444
+ return {
445
+ live: false,
446
+ basis: `owner pid ${run.ownerPid} is not running and this state directory is local`,
447
+ heartbeatAt: heartbeat,
448
+ owner,
449
+ pidChecked: true,
450
+ }
451
+ }
452
+ if (fresh) {
453
+ return { live: true, basis: `its heartbeat was refreshed at ${heartbeatIso}`, heartbeatAt: heartbeat, owner, pidChecked: false }
454
+ }
455
+ const basis = owner.local
456
+ ? `owner pid ${run.ownerPid ?? 'unknown'} is not running and its heartbeat stopped at ${heartbeatIso}`
457
+ : `the state directory is shared with ${owner.ownerHostname}, so a local pid proves nothing; the heartbeat stopped at ${heartbeatIso}`
458
+ return { live: false, basis, heartbeatAt: heartbeat, owner, pidChecked: owner.local }
459
+ }
460
+
461
+ /**
462
+ * Rewrite a run whose owner is gone. Never silent: the exact evidence is
463
+ * stored on the run and printed by status.
464
+ */
465
+ function reclaim(run, verdict, cancelled = false) {
466
+ run.status = cancelled ? 'cancelled' : 'interrupted'
467
+ run.error = cancelled
468
+ ? 'cancelled by superagent_cancel while no process was running it'
469
+ : 'host process stopped while the run was active'
470
+ run.finishedAt = new Date(now()).toISOString()
471
+ run.reclaimed = {
472
+ at: new Date(now()).toISOString(),
473
+ basis: verdict.basis,
474
+ heartbeatAt: Number.isFinite(verdict.heartbeatAt) ? new Date(verdict.heartbeatAt).toISOString() : null,
475
+ ownerPid: run.ownerPid ?? null,
476
+ pidChecked: verdict.pidChecked,
477
+ sharedWith: verdict.owner.local ? null : verdict.owner.ownerHostname,
478
+ }
479
+ for (const subtask of run.subtasks ?? []) {
480
+ if (subtask.status === 'running') subtask.status = 'pending'
481
+ }
482
+ // The owner died before it could settle, so publish what is already known
483
+ // about the spend: a reclaimed run must not claim its cost is unknown when
484
+ // its child logs were readable.
485
+ const entries = Object.values(run.usageByChild ?? {})
486
+ if (entries.length > 0) run.usage = sumUsage(entries)
487
+ run.cost = computeCost(run.usage, config.pricing)
488
+ for (const subtask of run.subtasks ?? []) {
489
+ if (subtask.childId && run.usageByChild?.[subtask.childId]) {
490
+ subtask.usage = run.usageByChild[subtask.childId]
491
+ }
492
+ }
493
+ }
494
+
495
+ async function releaseRunLock(runId) {
496
+ if (typeof store.releaseLock !== 'function') return
497
+ try {
498
+ await store.releaseLock(runId)
499
+ } catch (error) {
500
+ log(`cannot release the lock for run ${runId}: ${error?.message ?? error}`)
501
+ }
502
+ }
503
+
504
+ /**
505
+ * Claim the exclusive right to schedule a run.
506
+ *
507
+ * Two profiles can point at one state directory, so a resume must not be
508
+ * decided from a read alone: both processes would see a terminal run and both
509
+ * would schedule it. The lock makes one of them lose.
510
+ */
511
+ async function takeRunLock(runId) {
512
+ if (typeof store.acquireLock !== 'function') return
513
+ const holder = { pid: process.pid, hostname: os.hostname(), at: new Date(now()).toISOString() }
514
+ let attempt = await store.acquireLock(runId, holder)
515
+ if (!attempt.acquired) {
516
+ const current = attempt.holder
517
+ // A lock left by a process on this machine that no longer exists is
518
+ // evidence of a crash, not of a live owner. A lock on another machine has
519
+ // no checkable pid, so its age is the only evidence: past LOCK_STALE_MS
520
+ // the process that took it is no longer plausibly still scheduling.
521
+ const deadHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === false
522
+ const aliveHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === true
523
+ const age = now() - Date.parse(current?.at ?? '')
524
+ const stale = !Number.isFinite(age) || age > LOCK_STALE_MS
525
+ // Age is only evidence when the holder cannot be checked: a live process
526
+ // on this machine may legitimately schedule a long run for hours.
527
+ if (deadHere || (stale && !aliveHere)) {
528
+ await releaseRunLock(runId)
529
+ attempt = await store.acquireLock(runId, holder)
530
+ }
531
+ if (!attempt.acquired) {
532
+ const held = attempt.holder
533
+ const who = held
534
+ ? `pid ${held.pid ?? 'unknown'} on ${held.hostname ?? 'an unknown host'}`
535
+ : 'another process'
536
+ throw new Error(`run ${runId} is already being started by ${who}; wait for it to finish or cancel it`)
537
+ }
538
+ }
539
+ }
540
+
541
+ /**
542
+ * Reclaim runs whose owning process is gone, so an orphan cannot hold the
543
+ * active-run quota forever.
544
+ *
545
+ * @returns {Promise<string[]>} reclaimed run ids, in list order.
546
+ */
547
+ async function sweepOrphans() {
548
+ const stale = (await store.list()).filter(run => ACTIVE_STATUSES.has(run.status) && !runtime.has(run.id))
549
+ if (stale.length === 0) return []
550
+ const reclaimed = []
551
+ for (const run of stale) {
552
+ const verdict = await ownership(run)
553
+ if (verdict.live) continue
554
+ reclaim(run, verdict)
555
+ await save(run)
556
+ await releaseRunLock(run.id)
557
+ log(`run ${run.id} was active with no live owner; reclaimed its quota (${verdict.basis})`)
558
+ reclaimed.push(run.id)
559
+ }
560
+ return reclaimed
561
+ }
562
+
563
+ async function activeCount() {
564
+ try {
565
+ await sweepOrphans()
566
+ } catch (error) {
567
+ log(`cannot check for orphaned runs: ${error?.message ?? error}`)
568
+ }
569
+ let count = 0
570
+ for (const rt of runtime.values()) {
571
+ if (rt.run && ACTIVE_STATUSES.has(rt.run.status)) count++
572
+ }
573
+ for (const run of await store.list()) {
574
+ if (runtime.has(run.id)) continue
575
+ if (ACTIVE_STATUSES.has(run.status)) count++
576
+ }
577
+ return count
578
+ }
579
+
580
+ function quotaMessage(active) {
581
+ return `quota exceeded: ${active} active run(s) and maxActiveRuns is ${maxActiveRuns()}. `
582
+ + 'Let one finish with superagent_status or cancel it with superagent_cancel.'
583
+ }
584
+
585
+ function buildLimits(overrides = {}) {
586
+ // A caller may narrow a limit but never widen the operator's cap: the
587
+ // config is the spending and blast-radius control, and a model-written tool
588
+ // argument must not be able to raise it.
589
+ const narrowed = (override, cap) => (
590
+ override === undefined || override === null ? cap : Math.min(Number(override), cap)
591
+ )
592
+ const limits = {
593
+ maxSubtasks: clamp(narrowed(overrides.maxSubtasks, config.maxSubtasks), 1, 16),
594
+ maxParallel: clamp(narrowed(overrides.maxParallel, config.maxParallel), 1, 8),
595
+ deadlineMinutes: clamp(narrowed(overrides.deadlineMinutes, config.deadlineMinutes), 1, 720),
596
+ attempts: clamp(config.attempts, 1, 5),
597
+ maxModelCalls: clamp(narrowed(overrides.maxModelCalls, config.maxModelCallsPerRun), 0, 10000),
598
+ maxTokens: clamp(narrowed(overrides.maxTokens, config.maxTokensPerRun), 0, 1e12),
599
+ }
600
+ limits.maxParallel = Math.min(limits.maxParallel, limits.maxSubtasks)
601
+ return limits
602
+ }
603
+
604
+ /**
605
+ * Report why a run may not start another child, or null when it still can.
606
+ * Both budgets are hard stops: they exist so a pathological goal cannot spend
607
+ * quietly, so they are checked before a spawn rather than after.
608
+ */
609
+ function budgetStop(run) {
610
+ const calls = spentCalls(run)
611
+ const tokens = spentTokens(run)
612
+ if (run.limits.maxModelCalls > 0 && calls >= run.limits.maxModelCalls) {
613
+ return `model-call budget reached (${calls}/${run.limits.maxModelCalls} calls)`
614
+ }
615
+ if (run.limits.maxTokens > 0 && tokens >= run.limits.maxTokens) {
616
+ return `token budget reached (${tokens}/${run.limits.maxTokens} tokens)`
617
+ }
618
+ return null
619
+ }
620
+
621
+ /**
622
+ * Spawn one child under the run's abort signal and budget.
623
+ * @param {object} run - live run record.
624
+ * @param {object} rt - runtime entry.
625
+ * @param {{ label: string, prompt: string, parent: object }} spec - child request.
626
+ */
627
+ async function spawn(run, rt, spec) {
628
+ const blocked = budgetStop(run)
629
+ if (blocked) {
630
+ if (!rt.abortCause) rt.abortCause = 'budget'
631
+ if (!run.error) run.error = blocked
632
+ if (!rt.controller.signal.aborted) rt.controller.abort(new Error(blocked))
633
+ throw new Error(`budget exceeded: ${blocked}; raise maxModelCallsPerRun or maxTokensPerRun if this was intended`)
634
+ }
635
+ // Counted before the child exists and persisted before it can spend, so a
636
+ // crash cannot lose the only evidence of a model call.
637
+ ensureCounters(run).spawns += 1
638
+ await save(run)
639
+ return spawnAbortable(spec, rt.controller.signal)
640
+ }
641
+
642
+ /**
643
+ * Create one child, but stop waiting for it as soon as the run is aborted.
644
+ *
645
+ * A provider that queues activations can leave `startActivation` pending long
646
+ * after the run was cancelled or ran out of time. Without this the run would
647
+ * hang in planning/running while its heartbeat kept it looking alive, so no
648
+ * recovery path could ever reclaim it. A child that appears after the abort is
649
+ * disposed immediately instead of being left to spend.
650
+ */
651
+ function spawnAbortable(spec, signal) {
652
+ const pending = spawnChild({ ...spec, signal })
653
+ if (!signal) return pending
654
+ if (signal.aborted) {
655
+ pending.then(handle => disposeQuietly(handle), () => {})
656
+ return Promise.reject(new Error('aborted'))
657
+ }
658
+ return new Promise((resolve, reject) => {
659
+ let done = false
660
+ const onAbort = () => {
661
+ if (done) return
662
+ done = true
663
+ // The `pending` handler registered below disposes the handle if it
664
+ // arrives late, so the child is not left spending in the background.
665
+ reject(new Error('aborted'))
666
+ }
667
+ signal.addEventListener('abort', onAbort, { once: true })
668
+ pending.then(
669
+ handle => {
670
+ if (done) {
671
+ disposeQuietly(handle)
672
+ return
673
+ }
674
+ done = true
675
+ signal.removeEventListener('abort', onAbort)
676
+ resolve(handle)
677
+ },
678
+ error => {
679
+ if (done) return
680
+ done = true
681
+ signal.removeEventListener('abort', onAbort)
682
+ reject(error)
683
+ },
684
+ )
685
+ })
686
+ }
687
+
688
+ /**
689
+ * Read one run for a tool call, turning the two failure modes a user can hit
690
+ * into sentences instead of a null.
691
+ * @param {string} runId - run identity.
692
+ */
693
+ async function loadRun(runId) {
694
+ const run = await store.read(runId)
695
+ if (run) return run
696
+ const quarantined = store.corrupt?.get?.(runId)
697
+ if (quarantined) {
698
+ throw new Error(
699
+ `state file for run ${runId} is not valid JSON, so the run cannot be read. `
700
+ + `The original file was kept at ${quarantined}. Inspect it, then delete it; `
701
+ + 'the run is skipped by superagent_list until you do.',
702
+ )
703
+ }
704
+ throw new Error(`unknown run ${runId}; call superagent_list to see recorded run ids`)
705
+ }
706
+
707
+ /**
708
+ * Mark runs left active by a previous host process as interrupted.
709
+ *
710
+ * A run that another live process owns is left completely alone: rewriting it
711
+ * here would make two processes fight over the same file while the first one
712
+ * keeps spending, and a later resume would then run everything twice.
713
+ */
714
+ async function recover() {
715
+ await store.init()
716
+ const recovered = []
717
+ for (const run of await store.list()) {
718
+ if (!ACTIVE_STATUSES.has(run.status)) {
719
+ if (await hasCancelRequest(run.id)) await clearCancel(run.id)
720
+ continue
721
+ }
722
+ if (runtime.has(run.id)) continue
723
+ const verdict = await ownership(run)
724
+ if (verdict.live) {
725
+ log(`run ${run.id} is still owned (${verdict.basis}); leaving it running`)
726
+ continue
727
+ }
728
+ reclaim(run, verdict, await hasCancelRequest(run.id))
729
+ await save(run)
730
+ await clearCancel(run.id)
731
+ await releaseRunLock(run.id)
732
+ recovered.push(run.id)
733
+ }
734
+ return recovered
735
+ }
736
+
737
+ /**
738
+ * Fold one child's usage snapshot into the run. The snapshot is cumulative,
739
+ * so the last read for a child replaces the earlier one instead of adding.
740
+ * Called at start, on every heartbeat, and on success or failure, which is
741
+ * why an unreadable log degrades to the spawn floor instead of to zero.
742
+ */
743
+ async function recordUsage(run, childId) {
744
+ let usage = null
745
+ try {
746
+ usage = await readUsage(childId)
747
+ } catch (error) {
748
+ log(`cannot read usage for child ${childId}: ${error?.message ?? error}`)
749
+ }
750
+ if (!usage) {
751
+ ensureCounters(run).unreadable[childId] = true
752
+ return null
753
+ }
754
+ run.usageByChild = run.usageByChild ?? {}
755
+ run.usageByChild[childId] = usage
756
+ run.usage = sumUsage(Object.values(run.usageByChild))
757
+ return usage
758
+ }
759
+
760
+ /** Refresh usage for children that are still running, from the heartbeat. */
761
+ async function refreshUsage(run, rt) {
762
+ const handles = [...rt.activations]
763
+ for (const handle of handles) await recordUsage(run, handle.childId)
764
+ }
765
+
766
+ /**
767
+ * Wait for a child result, but stop waiting when the run is aborted. Without
768
+ * this a child that ignores its abort signal keeps the whole run open.
769
+ */
770
+ function abortable(promise, signal) {
771
+ if (!signal) return promise
772
+ return new Promise((resolve, reject) => {
773
+ const onAbort = () => {
774
+ promise.catch(() => {})
775
+ reject(new Error('aborted'))
776
+ }
777
+ if (signal.aborted) {
778
+ onAbort()
779
+ return
780
+ }
781
+ signal.addEventListener('abort', onAbort, { once: true })
782
+ promise.then(
783
+ value => {
784
+ signal.removeEventListener('abort', onAbort)
785
+ resolve(value)
786
+ },
787
+ error => {
788
+ signal.removeEventListener('abort', onAbort)
789
+ reject(error)
790
+ },
791
+ )
792
+ })
793
+ }
794
+
795
+ /** Dispose one child activation without letting it block the caller. */
796
+ async function disposeQuietly(handle) {
797
+ if (!handle || typeof handle.dispose !== 'function') return
798
+ try {
799
+ await Promise.race([
800
+ Promise.resolve().then(() => handle.dispose()),
801
+ sleep(DISPOSE_MS),
802
+ ])
803
+ } catch (error) {
804
+ log(`child dispose failed: ${error?.message ?? error}`)
805
+ }
806
+ }
807
+
808
+ /**
809
+ * The host's subagent service disappeared while this run was active, which
810
+ * in practice means the host is shutting down. That is not a subtask
811
+ * failure: the work never got a chance to run. Remember the cause so the run
812
+ * settles as `interrupted` (resumable) instead of `failed`, and abort the
813
+ * remaining children.
814
+ */
815
+ function markHostGone(rt, error) {
816
+ if (rt.abortCause) return
817
+ rt.abortCause = 'host-gone'
818
+ if (!rt.controller.signal.aborted) {
819
+ rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
820
+ }
821
+ }
822
+
823
+ async function makePlan(run, parent, rt) {
824
+ try {
825
+ const handle = await spawn(run, rt, {
826
+ label: 'superagent-plan',
827
+ prompt: plannerPrompt(run.goal, run.limits.maxSubtasks),
828
+ parent,
829
+ })
830
+ rt.activations.add(handle)
831
+ run.steps.planner = { childId: handle.childId }
832
+ try {
833
+ const result = await abortable(handle.result, rt.controller.signal)
834
+ const text = textOf(result.output)
835
+ const subtasks = extractPlan(text).slice(0, run.limits.maxSubtasks)
836
+ if (subtasks.length > 0) return { mode: 'planned', subtasks }
837
+ log(`planner reply had no parsable plan; running the goal as one subtask`)
838
+ } finally {
839
+ await recordUsage(run, handle.childId)
840
+ rt.activations.delete(handle)
841
+ await disposeQuietly(handle)
842
+ }
843
+ } catch (error) {
844
+ log(`planner failed: ${error?.message ?? error}`)
845
+ if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
846
+ }
847
+ return { mode: 'degraded', subtasks: [{ title: 'Complete the goal', prompt: run.goal }] }
848
+ }
849
+
850
+ async function runSubtask(run, subtask, parent, rt) {
851
+ const signal = rt.controller.signal
852
+ for (let attempt = subtask.attempts + 1; attempt <= run.limits.attempts; attempt++) {
853
+ if (signal.aborted) break
854
+ subtask.status = 'running'
855
+ subtask.attempts = attempt
856
+ subtask.startedAt = subtask.startedAt ?? new Date(now()).toISOString()
857
+ await save(run)
858
+
859
+ let handle = null
860
+ try {
861
+ handle = await spawn(run, rt, {
862
+ label: `superagent-${subtask.index}`,
863
+ prompt: workerPrompt(run, subtask),
864
+ parent,
865
+ })
866
+ rt.activations.add(handle)
867
+ subtask.childId = handle.childId
868
+ await save(run)
869
+
870
+ const result = await abortable(handle.result, signal)
871
+ const text = textOf(result.output)
872
+ subtask.stopReason = result.stopReason
873
+ if (result.stopReason === 'completed' && text) {
874
+ subtask.status = 'completed'
875
+ subtask.result = capText(text, config.resultCapBytes)
876
+ subtask.error = null
877
+ subtask.finishedAt = new Date(now()).toISOString()
878
+ await recordUsage(run, handle.childId)
879
+ await save(run)
880
+ return
881
+ }
882
+ subtask.error = result.diagnostic || `child stopped with reason "${result.stopReason}"`
883
+ log(`subtask ${subtask.index} attempt ${attempt} did not complete: ${subtask.error}`)
884
+ } catch (error) {
885
+ subtask.error = error?.message ?? String(error)
886
+ log(`subtask ${subtask.index} attempt ${attempt} threw: ${subtask.error}`)
887
+ if (error?.code === 'subagent-unavailable') {
888
+ // The host is going away; leave the subtask as work to do rather
889
+ // than burning the remaining attempts on a service that is gone.
890
+ subtask.error = null
891
+ subtask.stopReason = null
892
+ markHostGone(rt, error)
893
+ }
894
+ } finally {
895
+ if (handle) {
896
+ // A failed attempt still spent tokens, so read it before disposing.
897
+ await recordUsage(run, handle.childId)
898
+ rt.activations.delete(handle)
899
+ await disposeQuietly(handle)
900
+ }
901
+ }
902
+
903
+ if (signal.aborted) break
904
+ if (attempt < run.limits.attempts) {
905
+ subtask.status = 'pending'
906
+ await save(run)
907
+ await sleep(1000 * attempt)
908
+ }
909
+ }
910
+
911
+ if (subtask.status !== 'completed') {
912
+ if (signal.aborted) {
913
+ // Cancelled or out of time: the subtask did not fail on its own terms,
914
+ // so leave it resumable instead of recording a failure.
915
+ subtask.status = 'pending'
916
+ subtask.error = null
917
+ subtask.stopReason = null
918
+ } else {
919
+ subtask.status = 'failed'
920
+ subtask.finishedAt = new Date(now()).toISOString()
921
+ }
922
+ await save(run)
923
+ }
924
+ }
925
+
926
+ async function aggregate(run, parent, rt) {
927
+ const completed = run.subtasks.filter(subtask => subtask.status === 'completed')
928
+ if (completed.length === 0) return null
929
+ if (rt.controller.signal.aborted) return deterministicJoin(run)
930
+ if (completed.length === 1) return `# ${run.goal}\n\n${completed[0].result ?? ''}`.trim()
931
+
932
+ try {
933
+ const handle = await spawn(run, rt, {
934
+ label: 'superagent-aggregate',
935
+ prompt: aggregatorPrompt(run, completed),
936
+ parent,
937
+ })
938
+ rt.activations.add(handle)
939
+ run.steps.aggregator = { childId: handle.childId }
940
+ try {
941
+ const result = await abortable(handle.result, rt.controller.signal)
942
+ const text = textOf(result.output)
943
+ if (result.stopReason === 'completed' && text) return text
944
+ log(`aggregator did not complete (${result.stopReason}); using deterministic merge`)
945
+ } finally {
946
+ await recordUsage(run, handle.childId)
947
+ rt.activations.delete(handle)
948
+ await disposeQuietly(handle)
949
+ }
950
+ } catch (error) {
951
+ log(`aggregator failed: ${error?.message ?? error}`)
952
+ if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
953
+ }
954
+ return deterministicJoin(run)
955
+ }
956
+
957
+ function terminalStatus(run, rt) {
958
+ if (rt.abortCause === 'cancelled') return 'cancelled'
959
+ // The host went away mid-run. Nothing failed on its own terms, so the run
960
+ // must stay resumable instead of being buried as `failed`.
961
+ if (rt.abortCause === 'host-gone') return 'interrupted'
962
+ const subtasks = run.subtasks ?? []
963
+ const completed = subtasks.filter(subtask => subtask.status === 'completed').length
964
+ if (completed === 0) return 'failed'
965
+ if (rt.abortCause === 'deadline') {
966
+ return completed === subtasks.length ? 'completed' : 'partial'
967
+ }
968
+ return completed === subtasks.length ? 'completed' : 'partial'
969
+ }
970
+
971
+ /**
972
+ * Watch for a cancel written by another process. The marker is checked from
973
+ * inside the owning process, which is the only place that can abort children;
974
+ * that is why cancel is a file and not a state-file rewrite.
975
+ */
976
+ function watchCancel(run, rt) {
977
+ const timer = setInterval(() => {
978
+ hasCancelRequest(run.id).then(requested => {
979
+ if (!requested || rt.controller.signal.aborted) return
980
+ rt.abortCause = 'cancelled'
981
+ rt.controller.abort(new Error('cancelled by superagent_cancel from another process'))
982
+ // Aborting the signal alone does not stop a child that is already
983
+ // running; dispose its activation so it stops spending tokens.
984
+ for (const handle of [...rt.activations]) disposeQuietly(handle)
985
+ }).catch(() => {})
986
+ }, CANCEL_POLL_MS)
987
+ timer.unref?.()
988
+ return timer
989
+ }
990
+
991
+ /**
992
+ * A caller waiting inside this process will read the terminal result itself,
993
+ * so waking the session would be duplicate noise. A caller in *another*
994
+ * process is deliberately not treated this way: it can time out just before
995
+ * the run settles, in which case suppressing the wake-up would lose the only
996
+ * notification the user gets.
997
+ */
998
+ function watchedHere(rt) {
999
+ return rt.waiters > 0
1000
+ }
1001
+
1002
+ /**
1003
+ * Compute the terminal state, decide about notifying, and write it exactly
1004
+ * once. Writing the terminal status first and the notify outcome second left
1005
+ * a window where readers saw a finished run with no notify line at all.
1006
+ */
1007
+ async function settle(run, rt) {
1008
+ // Usage is stored per child so repeated reads replace rather than add. A
1009
+ // run written before that field existed keeps whatever total it had.
1010
+ const entries = Object.values(run.usageByChild ?? {})
1011
+ if (entries.length > 0) run.usage = sumUsage(entries)
1012
+ // The per-subtask figure is read when the child finishes, which can race a
1013
+ // log flush; the settle pass has the final snapshot, so publish it here.
1014
+ for (const subtask of run.subtasks ?? []) {
1015
+ if (subtask.childId && run.usageByChild?.[subtask.childId]) {
1016
+ subtask.usage = run.usageByChild[subtask.childId]
1017
+ }
1018
+ }
1019
+ run.cost = computeCost(run.usage, config.pricing)
1020
+ const nextStatus = terminalStatus(run, rt)
1021
+ const finishedAt = new Date(now()).toISOString()
1022
+ let error = run.error
1023
+ if (rt.abortCause === 'cancelled') {
1024
+ error = 'cancelled by superagent_cancel'
1025
+ } else if (rt.abortCause === 'host-gone') {
1026
+ error = 'the host shut down while this run was active; run superagent_resume in a new host to finish it'
1027
+ } else if (nextStatus === 'failed' && !error) {
1028
+ error = run.subtasks.map(subtask => subtask.error).filter(Boolean).slice(0, 3).join('; ') || 'no subtask completed'
1029
+ }
1030
+ if (rt.abortCause === 'deadline' && !error) error = 'deadline exceeded'
1031
+
1032
+ let outcome = null
1033
+ if (TERMINAL_STATUSES.has(nextStatus)) {
1034
+ if (watchedHere(rt)) {
1035
+ outcome = { delivered: false, reason: 'skipped: a caller in this process is reading the run' }
1036
+ } else {
1037
+ try {
1038
+ // `notify` may wait on the host's session loop; cap it so a finished
1039
+ // run cannot stay active on disk (and be reclaimed by another process)
1040
+ // just because a wake-up is slow.
1041
+ outcome = await withTimeout(
1042
+ Promise.resolve().then(() => notify({ ...run, status: nextStatus, finishedAt, error })),
1043
+ NOTIFY_MS,
1044
+ () => ({
1045
+ delivered: false,
1046
+ reason: `the owning session did not accept the wake-up within ${NOTIFY_MS / 1000}s`,
1047
+ }),
1048
+ )
1049
+ outcome = outcome ?? { delivered: true }
1050
+ } catch (notifyError) {
1051
+ outcome = { delivered: false, reason: `notification threw: ${notifyError?.message ?? notifyError}` }
1052
+ log(`notification failed for ${run.id}: ${notifyError?.message ?? notifyError}`)
1053
+ }
1054
+ }
1055
+ }
1056
+
1057
+ // The terminal status and its notify row land in the same write, so a
1058
+ // reader never sees a finished run that still claims to be active and never
1059
+ // sees a finished run whose notify row is missing.
1060
+ run.status = nextStatus
1061
+ run.finishedAt = finishedAt
1062
+ run.activeMs = (Number.isFinite(run.activeMs) ? run.activeMs : 0) + Math.max(0, now() - rt.activeStart)
1063
+ run.error = error
1064
+ run.cancelPending = false
1065
+ if (outcome) {
1066
+ run.notify = {
1067
+ at: new Date(now()).toISOString(),
1068
+ delivered: outcome.delivered !== false,
1069
+ reason: outcome.reason ?? null,
1070
+ }
1071
+ if (outcome.delivered !== false) run.notified = true
1072
+ }
1073
+ await save(run)
1074
+ // The run is terminal, so a leftover marker would only make the next mount
1075
+ // rewrite a finished run; drop it now.
1076
+ await clearCancel(run.id)
1077
+ await releaseRunLock(run.id)
1078
+ }
1079
+
1080
+ /**
1081
+ * Resolve with `promise`, or with `onTimeout()` after `ms`.
1082
+ *
1083
+ * The timer is cleared when the promise wins: a stray 15s timer would keep a
1084
+ * finished host process (and a test runner) alive for no reason.
1085
+ */
1086
+ function withTimeout(promise, ms, onTimeout) {
1087
+ return new Promise((resolve, reject) => {
1088
+ const timer = setTimeout(() => resolve(onTimeout()), ms)
1089
+ timer.unref?.()
1090
+ promise.then(
1091
+ value => {
1092
+ clearTimeout(timer)
1093
+ resolve(value)
1094
+ },
1095
+ error => {
1096
+ clearTimeout(timer)
1097
+ reject(error)
1098
+ },
1099
+ )
1100
+ })
1101
+ }
1102
+
1103
+ async function execute(run, parent, rt) {
1104
+ rt.activeStart = now()
1105
+ const remaining = Math.max(0, Date.parse(run.deadlineAt) - now())
1106
+ const timer = setTimeout(() => {
1107
+ rt.abortCause = 'deadline'
1108
+ rt.controller.abort(new Error(`deadline exceeded after ${run.limits.deadlineMinutes} minutes`))
1109
+ }, remaining)
1110
+ timer.unref?.()
1111
+ const cancelTimer = watchCancel(run, rt)
1112
+ // The heartbeat is what makes an orphan detectable after a crash on a
1113
+ // machine whose pids this process cannot check (shared state directory).
1114
+ const beat = setInterval(() => {
1115
+ run.heartbeatAt = new Date(now()).toISOString()
1116
+ run.ownerPid = process.pid
1117
+ refreshUsage(run, rt).then(() => save(run)).catch(() => {})
1118
+ }, HEARTBEAT_MS)
1119
+ beat.unref?.()
1120
+
1121
+ try {
1122
+ ensureCounters(run)
1123
+ if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
1124
+ run.status = 'planning'
1125
+ await save(run)
1126
+ const plan = await makePlan(run, parent, rt)
1127
+ run.plan = { mode: plan.mode, subtasks: plan.subtasks }
1128
+ run.subtasks = plan.subtasks.map((entry, index) => ({
1129
+ index: index + 1,
1130
+ title: entry.title,
1131
+ prompt: entry.prompt,
1132
+ status: 'pending',
1133
+ attempts: 0,
1134
+ childId: null,
1135
+ stopReason: null,
1136
+ error: null,
1137
+ result: null,
1138
+ usage: null,
1139
+ startedAt: null,
1140
+ finishedAt: null,
1141
+ }))
1142
+ await save(run)
1143
+ }
1144
+
1145
+ const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
1146
+ if (open.length > 0) {
1147
+ run.status = 'running'
1148
+ await save(run)
1149
+ await pool(open, run.limits.maxParallel, subtask => runSubtask(run, subtask, parent, rt))
1150
+ }
1151
+
1152
+ if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
1153
+ run.status = 'aggregating'
1154
+ await save(run)
1155
+ run.report = await aggregate(run, parent, rt)
1156
+ } else if (!run.report) {
1157
+ run.report = deterministicJoin(run)
1158
+ }
1159
+ } catch (error) {
1160
+ run.error = error?.message ?? String(error)
1161
+ log(`run ${run.id} failed: ${run.error}`)
1162
+ // Stop any child still running; without this the run would end while its
1163
+ // subtasks keep burning tokens.
1164
+ rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
1165
+ } finally {
1166
+ clearTimeout(timer)
1167
+ clearInterval(cancelTimer)
1168
+ // The heartbeat keeps running until the terminal write lands: waking the
1169
+ // session can take a few seconds, and a frozen heartbeat during that
1170
+ // window would let another process treat a live run as orphaned.
1171
+ try {
1172
+ await settle(run, rt)
1173
+ } catch (error) {
1174
+ // Never leave a run active because finalizing it threw.
1175
+ log(`cannot finalize run ${run.id}: ${error?.message ?? error}`)
1176
+ if (ACTIVE_STATUSES.has(run.status)) {
1177
+ run.status = 'failed'
1178
+ run.finishedAt = run.finishedAt ?? new Date(now()).toISOString()
1179
+ run.error = run.error ?? `finalizing the run failed: ${error?.message ?? error}`
1180
+ }
1181
+ try {
1182
+ await store.write(run)
1183
+ } catch (writeError) {
1184
+ log(`cannot persist the terminal state of run ${run.id}: ${writeError?.message ?? writeError}`)
1185
+ }
1186
+ await releaseRunLock(run.id)
1187
+ } finally {
1188
+ clearInterval(beat)
1189
+ }
1190
+ }
1191
+ }
1192
+
1193
+ function schedule(run, parent) {
1194
+ run.ownerPid = process.pid
1195
+ run.ownerHost = os.hostname()
1196
+ run.heartbeatAt = new Date(now()).toISOString()
1197
+ const rt = {
1198
+ controller: new AbortController(),
1199
+ activations: new Set(),
1200
+ waiters: 0,
1201
+ run,
1202
+ promise: null,
1203
+ abortCause: null,
1204
+ }
1205
+ runtime.set(run.id, rt)
1206
+ rt.promise = execute(run, parent, rt)
1207
+ .catch(async error => {
1208
+ log(`run ${run.id} crashed: ${error?.message ?? error}`)
1209
+ if (!ACTIVE_STATUSES.has(run.status)) return
1210
+ run.status = 'failed'
1211
+ run.finishedAt = new Date(now()).toISOString()
1212
+ run.error = error?.message ?? String(error)
1213
+ try {
1214
+ await store.write(run)
1215
+ } catch (writeError) {
1216
+ log(`cannot persist the crash state of run ${run.id}: ${writeError?.message ?? writeError}`)
1217
+ }
1218
+ await releaseRunLock(run.id)
1219
+ })
1220
+ .then(() => {
1221
+ // Finished runs are read from disk from now on; keeping them here would
1222
+ // grow these maps for the lifetime of the host.
1223
+ runtime.delete(run.id)
1224
+ saveChains.delete(run.id)
1225
+ })
1226
+ return rt
1227
+ }
1228
+
1229
+ /** Serialize starts and resumes so a quota check cannot interleave. */
1230
+ function serialize(task) {
1231
+ const result = startGate.then(task, task)
1232
+ startGate = result.catch(() => {})
1233
+ return result
1234
+ }
1235
+
1236
+ async function start({ goal, parentSessionId, parent, clientTimeZone, overrides = {} }) {
1237
+ const trimmed = typeof goal === 'string' ? goal.trim() : ''
1238
+ if (!trimmed) throw new Error('goal is required')
1239
+ if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
1240
+
1241
+ const limits = buildLimits(overrides)
1242
+ // Kept so a resume can rebuild the limits from the *current* config
1243
+ // instead of freezing whatever was set when the run first started.
1244
+ const keptOverrides = {}
1245
+ for (const key of ['maxSubtasks', 'maxParallel', 'deadlineMinutes']) {
1246
+ if (overrides[key] !== undefined && overrides[key] !== null) keptOverrides[key] = overrides[key]
1247
+ }
1248
+ return serialize(async () => {
1249
+ const active = await activeCount()
1250
+ if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
1251
+ const run = {
1252
+ id: newRunId(),
1253
+ schemaVersion: 1,
1254
+ goal: trimmed,
1255
+ status: 'queued',
1256
+ mode: 'planned',
1257
+ parentSessionId,
1258
+ clientTimeZone: clientTimeZone ?? null,
1259
+ createdAt: new Date(now()).toISOString(),
1260
+ updatedAt: null,
1261
+ startedAt: new Date(now()).toISOString(),
1262
+ deadlineAt: new Date(now() + limits.deadlineMinutes * 60000).toISOString(),
1263
+ finishedAt: null,
1264
+ resumedAt: null,
1265
+ limits,
1266
+ overrides: keptOverrides,
1267
+ plan: null,
1268
+ subtasks: [],
1269
+ steps: {},
1270
+ report: null,
1271
+ error: null,
1272
+ usage: null,
1273
+ notified: false,
1274
+ counters: { spawns: 0, unreadable: {} },
1275
+ usageByChild: {},
1276
+ }
1277
+ await takeRunLock(run.id)
1278
+ try {
1279
+ await store.write(run)
1280
+ } catch (error) {
1281
+ await releaseRunLock(run.id)
1282
+ throw error
1283
+ }
1284
+ schedule(run, parent)
1285
+ return run
1286
+ })
1287
+ }
1288
+
1289
+ async function resume(runId, parent, clientTimeZone) {
1290
+ const id = String(runId ?? '').trim()
1291
+ if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
1292
+ return serialize(async () => {
1293
+ if (runtime.has(id)) {
1294
+ throw new Error(`run ${id} is still active in this process; wait for it with superagent_status or cancel it`)
1295
+ }
1296
+ let run = await loadRun(id)
1297
+ if (run.status === 'completed') return run
1298
+ if (ACTIVE_STATUSES.has(run.status)) {
1299
+ const verdict = await ownership(run)
1300
+ if (verdict.live) {
1301
+ throw new Error(
1302
+ `run ${id} is still owned by a live process (${verdict.basis}); resuming it here would run the same `
1303
+ + 'subtasks twice and double the spend. Cancel it with superagent_cancel first, or wait for it to finish.',
1304
+ )
1305
+ }
1306
+ }
1307
+
1308
+ await takeRunLock(id)
1309
+ let scheduled = false
1310
+ try {
1311
+ // Re-read under the lock: another process may have started the run
1312
+ // between the check above and the claim.
1313
+ run = await loadRun(id)
1314
+ if (run.status === 'completed') return run
1315
+ if (ACTIVE_STATUSES.has(run.status)) {
1316
+ const verdict = await ownership(run)
1317
+ if (verdict.live) {
1318
+ throw new Error(`run ${id} is still owned by a live process (${verdict.basis}); not resuming it here`)
1319
+ }
1320
+ reclaim(run, verdict, await hasCancelRequest(id))
1321
+ await save(run)
1322
+ await clearCancel(id)
1323
+ }
1324
+
1325
+ const active = await activeCount()
1326
+ if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
1327
+
1328
+ for (const subtask of run.subtasks ?? []) {
1329
+ if (subtask.status === 'completed') continue
1330
+ subtask.status = 'pending'
1331
+ subtask.attempts = 0
1332
+ subtask.error = null
1333
+ subtask.stopReason = null
1334
+ subtask.finishedAt = null
1335
+ }
1336
+ run.status = 'queued'
1337
+ run.report = run.plan ? run.report : null
1338
+ run.error = null
1339
+ run.finishedAt = null
1340
+ run.reclaimed = null
1341
+ run.cancelPending = false
1342
+ // A resumed run completes a second time, so it must be able to wake the
1343
+ // session again; keeping the old outcome would silence it forever.
1344
+ run.notified = false
1345
+ run.notify = null
1346
+ run.resumedAt = new Date(now()).toISOString()
1347
+ run.clientTimeZone = clientTimeZone ?? run.clientTimeZone ?? null
1348
+ // The limits are rebuilt from the config that is active now, keeping
1349
+ // everything already spent: that is what makes "raise the cap, then
1350
+ // resume" work. Without this the old caps stayed in force forever.
1351
+ run.limits = buildLimits(run.overrides ?? {})
1352
+ run.deadlineAt = new Date(now() + run.limits.deadlineMinutes * 60000).toISOString()
1353
+ ensureCounters(run)
1354
+ await save(run)
1355
+ schedule(run, parent)
1356
+ scheduled = true
1357
+ return run
1358
+ } finally {
1359
+ if (!scheduled) await releaseRunLock(id)
1360
+ }
1361
+ })
1362
+ }
1363
+
1364
+ async function cancel(runId) {
1365
+ const rt = runtime.get(runId)
1366
+ if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
1367
+ rt.abortCause = 'cancelled'
1368
+ rt.controller.abort(new Error('cancelled by superagent_cancel'))
1369
+ const activations = [...rt.activations]
1370
+ // A child that ignores its abort must not hold this call open; the
1371
+ // timeout is per child and they are disposed in parallel.
1372
+ await Promise.all(activations.map(handle => disposeQuietly(handle)))
1373
+ await Promise.race([rt.promise ?? Promise.resolve(), sleep(CANCEL_ACK_MS)])
1374
+ if (ACTIVE_STATUSES.has(rt.run.status)) {
1375
+ rt.run.cancelPending = true
1376
+ await save(rt.run).catch(() => {})
1377
+ }
1378
+ return rt.run
1379
+ }
1380
+
1381
+ const run = await loadRun(runId)
1382
+ if (!ACTIVE_STATUSES.has(run.status)) return run
1383
+
1384
+ // This process does not own the run, so it cannot abort the children. Ask
1385
+ // the owner through the marker file and wait for it to rewrite the state.
1386
+ // Writing `cancelled` here would be a lie: the owner would overwrite it on
1387
+ // its next save and keep spending.
1388
+ await requestCancel(runId)
1389
+ const endsAt = now() + CANCEL_ACK_MS
1390
+ while (now() < endsAt) {
1391
+ await sleep(STATUS_POLL_MS)
1392
+ const current = await store.read(runId)
1393
+ if (!current) return run
1394
+ if (!ACTIVE_STATUSES.has(current.status)) return current
1395
+ }
1396
+ // The owner did not confirm in time. Do not rewrite the state file from
1397
+ // here: the owner would overwrite it on its next save. The request marker
1398
+ // stays on disk, and `status` reports it as a pending cancel.
1399
+ return { ...run, cancelPending: true }
1400
+ }
1401
+
1402
+ async function status(runId, waitSeconds = 0) {
1403
+ const read = async () => liveRun(runId) ?? await store.read(runId)
1404
+ let run = await read()
1405
+ if (!run) {
1406
+ await loadRun(runId)
1407
+ throw new Error(`unknown run ${runId}`)
1408
+ }
1409
+ // A terminal status is only published after the run's final write, so a
1410
+ // caller that sees one gets the notify row and usage totals with it.
1411
+ const owner = runtime.get(runId)
1412
+ if (owner && TERMINAL_STATUSES.has(run.status)) {
1413
+ await withTimeout(Promise.resolve(owner.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
1414
+ run = (await read()) ?? run
1415
+ }
1416
+ const limit = clamp(waitSeconds, 0, 3600)
1417
+ if (ACTIVE_STATUSES.has(run.status) && !run.cancelPending && await hasCancelRequest(runId)) {
1418
+ // The owning process may live elsewhere; the marker is the only evidence
1419
+ // that a cancel was asked for and not yet acknowledged.
1420
+ run = { ...run, cancelPending: true }
1421
+ }
1422
+ if (limit <= 0 || TERMINAL_STATUSES.has(run.status)) return run
1423
+
1424
+ const rt = runtime.get(runId)
1425
+ if (rt) rt.waiters++
1426
+ try {
1427
+ const endsAt = now() + limit * 1000
1428
+ while (now() < endsAt) {
1429
+ await sleep(STATUS_POLL_MS)
1430
+ run = await read()
1431
+ if (!run || TERMINAL_STATUSES.has(run.status)) break
1432
+ }
1433
+ } finally {
1434
+ if (rt) rt.waiters = Math.max(0, rt.waiters - 1)
1435
+ }
1436
+
1437
+ if (!run) throw new Error(`run ${runId} disappeared from the state directory while waiting for it`)
1438
+ const live = runtime.get(runId)
1439
+ // Wait for this process's own final write, so a caller that sees a terminal
1440
+ // status can immediately trust what is on disk. Bounded, because finalizing
1441
+ // includes waking the session and must not turn a status call into a hang.
1442
+ if (live && TERMINAL_STATUSES.has(live.run.status)) {
1443
+ await withTimeout(Promise.resolve(live.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
1444
+ }
1445
+ return liveRun(runId) ?? run
1446
+ }
1447
+
1448
+ async function list() {
1449
+ const merged = new Map()
1450
+ for (const run of await store.list()) merged.set(run.id, run)
1451
+ for (const [runId, rt] of runtime) merged.set(runId, rt.run)
1452
+ return [...merged.values()].sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
1453
+ }
1454
+
1455
+ /**
1456
+ * Resolve once every run started by this process has finished. Used by tests
1457
+ * instead of sleeping, and by embedders that need a quiescent state.
1458
+ */
1459
+ async function idle() {
1460
+ await Promise.all([...runtime.values()].map(rt => rt.promise?.catch(() => {})))
1461
+ }
1462
+
1463
+ return { recover, start, resume, cancel, status, list, idle, activeCount, liveRun }
1464
+ }