@xiangyixie/dsh-superagent 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +259 -232
- package/LICENSE +21 -21
- package/README.md +589 -553
- package/bin/report.js +79 -79
- package/bin/watch.js +57 -57
- package/cordis.patch.yml +9 -9
- package/lib/control.js +494 -494
- package/lib/errors.js +199 -199
- package/lib/index.js +822 -822
- package/lib/orchestrator.js +1464 -1464
- package/lib/report.js +733 -733
- package/lib/store.js +385 -385
- package/lib/usage.js +152 -152
- package/lib/watch.js +458 -458
- package/package.json +67 -66
package/lib/orchestrator.js
CHANGED
|
@@ -1,1464 +1,1464 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Run orchestration for dsh-superagent.
|
|
3
|
-
*
|
|
4
|
-
* A run moves through queued -> planning -> running -> aggregating and ends in
|
|
5
|
-
* completed, partial, failed, cancelled, or interrupted. Planning, subtask
|
|
6
|
-
* execution, and aggregation all happen in child agents; this module owns
|
|
7
|
-
* ordering, retries, quotas, cancellation, and durable status.
|
|
8
|
-
*
|
|
9
|
-
* A state directory can be shared by more than one host process (two profiles,
|
|
10
|
-
* or a synced folder), so every decision about an active run is made from
|
|
11
|
-
* persisted evidence — heartbeat, owner pid, and the run lock — and never from
|
|
12
|
-
* this process's memory alone.
|
|
13
|
-
*
|
|
14
|
-
* The module has no host imports: `spawnChild`, `readUsage`, `notify`, and
|
|
15
|
-
* `store` are injected, which keeps it testable without a running harness.
|
|
16
|
-
*
|
|
17
|
-
* @module dsh-superagent/orchestrator
|
|
18
|
-
*/
|
|
19
|
-
import { randomBytes } from 'node:crypto'
|
|
20
|
-
import os from 'node:os'
|
|
21
|
-
import { ACTIVE_STATUSES, TERMINAL_STATUSES } from './store.js'
|
|
22
|
-
|
|
23
|
-
const STATUS_POLL_MS = 250
|
|
24
|
-
/** How often the owning process checks for a cancel marker written elsewhere. */
|
|
25
|
-
const CANCEL_POLL_MS = 1000
|
|
26
|
-
/** How long a cancel from another process waits for the owner to acknowledge. */
|
|
27
|
-
const CANCEL_ACK_MS = 5000
|
|
28
|
-
/** How often a live run refreshes its heartbeat in the state file. */
|
|
29
|
-
const HEARTBEAT_MS = 30 * 1000
|
|
30
|
-
/** A run with no heartbeat for this long is treated as orphaned. */
|
|
31
|
-
const ORPHAN_MS = 3 * HEARTBEAT_MS
|
|
32
|
-
/** A child that ignores abort must not hold a cancel call open forever. */
|
|
33
|
-
const DISPOSE_MS = 1500
|
|
34
|
-
/** Waking the owning session must not hold a finished run open forever. */
|
|
35
|
-
const NOTIFY_MS = 15 * 1000
|
|
36
|
-
/** A run lock older than this belongs to a process that never released it. */
|
|
37
|
-
const LOCK_STALE_MS = 10 * 60 * 1000
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Mint a sortable, filesystem-safe run id.
|
|
41
|
-
* @returns {string} run id such as `20261114T051233-a1b2`.
|
|
42
|
-
*/
|
|
43
|
-
export function newRunId() {
|
|
44
|
-
const stamp = new Date().toISOString().slice(0, 19).replace(/[-:]/g, '')
|
|
45
|
-
return `${stamp}-${randomBytes(2).toString('hex')}`
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function clamp(value, min, max) {
|
|
49
|
-
const number = Number(value)
|
|
50
|
-
if (!Number.isFinite(number)) return min
|
|
51
|
-
return Math.min(max, Math.max(min, Math.trunc(number)))
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/** Join the text blocks of a child result. */
|
|
55
|
-
export function textOf(output) {
|
|
56
|
-
if (!Array.isArray(output)) return ''
|
|
57
|
-
return output
|
|
58
|
-
.filter(block => block && block.type === 'text' && typeof block.text === 'string')
|
|
59
|
-
.map(block => block.text)
|
|
60
|
-
.join('\n')
|
|
61
|
-
.trim()
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
/** Truncate a string to a UTF-8 byte budget without splitting a character. */
|
|
65
|
-
export function capText(text, maxBytes) {
|
|
66
|
-
if (typeof text !== 'string') return ''
|
|
67
|
-
const buffer = Buffer.from(text, 'utf8')
|
|
68
|
-
if (buffer.length <= maxBytes) return text
|
|
69
|
-
// Cutting at an arbitrary byte can land inside a multi-byte character; the
|
|
70
|
-
// decode then ends in U+FFFD. Drop that partial sequence instead of keeping
|
|
71
|
-
// a replacement character in the user's report.
|
|
72
|
-
const kept = buffer.subarray(0, maxBytes).toString('utf8').replace(/\uFFFD+$/, '')
|
|
73
|
-
const omitted = buffer.length - Buffer.byteLength(kept, 'utf8')
|
|
74
|
-
return `${kept}\n\n[truncated: ${omitted} bytes omitted]`
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
/**
|
|
78
|
-
* Extract the first JSON object that carries a usable `subtasks` array.
|
|
79
|
-
*
|
|
80
|
-
* Models often wrap JSON in prose or code fences, so every balanced `{...}`
|
|
81
|
-
* region is tried rather than assuming the reply is bare JSON.
|
|
82
|
-
*
|
|
83
|
-
* @param {string} text - planner reply.
|
|
84
|
-
* @returns {Array<{ title: string, prompt: string }>} zero or more subtasks.
|
|
85
|
-
*/
|
|
86
|
-
export function extractPlan(text) {
|
|
87
|
-
if (typeof text !== 'string') return []
|
|
88
|
-
const candidates = []
|
|
89
|
-
for (let start = 0; start < text.length; start++) {
|
|
90
|
-
if (text[start] !== '{') continue
|
|
91
|
-
let depth = 0
|
|
92
|
-
let inString = false
|
|
93
|
-
let escaped = false
|
|
94
|
-
for (let index = start; index < text.length; index++) {
|
|
95
|
-
const char = text[index]
|
|
96
|
-
if (inString) {
|
|
97
|
-
if (escaped) escaped = false
|
|
98
|
-
else if (char === '\\') escaped = true
|
|
99
|
-
else if (char === '"') inString = false
|
|
100
|
-
continue
|
|
101
|
-
}
|
|
102
|
-
if (char === '"') inString = true
|
|
103
|
-
else if (char === '{') depth++
|
|
104
|
-
else if (char === '}') {
|
|
105
|
-
depth--
|
|
106
|
-
if (depth === 0) {
|
|
107
|
-
candidates.push(text.slice(start, index + 1))
|
|
108
|
-
start = index
|
|
109
|
-
break
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
for (const candidate of candidates) {
|
|
115
|
-
let parsed
|
|
116
|
-
try {
|
|
117
|
-
parsed = JSON.parse(candidate)
|
|
118
|
-
} catch {
|
|
119
|
-
continue
|
|
120
|
-
}
|
|
121
|
-
const raw = parsed?.subtasks
|
|
122
|
-
if (!Array.isArray(raw)) continue
|
|
123
|
-
const subtasks = []
|
|
124
|
-
for (const item of raw) {
|
|
125
|
-
if (!item || typeof item !== 'object') continue
|
|
126
|
-
const prompt = typeof item.prompt === 'string' ? item.prompt.trim() : ''
|
|
127
|
-
if (!prompt) continue
|
|
128
|
-
const title = typeof item.title === 'string' && item.title.trim()
|
|
129
|
-
? item.title.trim().slice(0, 80)
|
|
130
|
-
: prompt.slice(0, 60)
|
|
131
|
-
subtasks.push({ title, prompt })
|
|
132
|
-
}
|
|
133
|
-
if (subtasks.length > 0) return subtasks
|
|
134
|
-
}
|
|
135
|
-
return []
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
/** Planner prompt for one goal. */
|
|
139
|
-
function plannerPrompt(goal, maxSubtasks) {
|
|
140
|
-
return [
|
|
141
|
-
'You are the planner for a long-horizon autonomous run.',
|
|
142
|
-
'',
|
|
143
|
-
'Goal:',
|
|
144
|
-
goal,
|
|
145
|
-
'',
|
|
146
|
-
`Decompose the goal into at most ${maxSubtasks} subtasks that can run in parallel because none of them needs another subtask's output.`,
|
|
147
|
-
'Each subtask is executed by a separate agent that sees only the prompt you write for it.',
|
|
148
|
-
'',
|
|
149
|
-
'Return ONLY a JSON object, with no prose and no code fences, in exactly this shape:',
|
|
150
|
-
'{"subtasks":[{"title":"short title","prompt":"complete self-contained instruction"}]}',
|
|
151
|
-
'',
|
|
152
|
-
'Rules:',
|
|
153
|
-
`- Use 2 to ${maxSubtasks} subtasks; use 1 only when the goal truly cannot be split.`,
|
|
154
|
-
'- Every prompt must restate the context it needs; workers cannot see the goal text or each other.',
|
|
155
|
-
'- Prefer concrete research, construction, or verification work over vague summaries.',
|
|
156
|
-
].join('\n')
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/** Worker prompt for one subtask. */
|
|
160
|
-
function workerPrompt(run, subtask) {
|
|
161
|
-
return [
|
|
162
|
-
'You are one worker in a long-horizon autonomous run. Do not ask the user questions and do not wait for input.',
|
|
163
|
-
'',
|
|
164
|
-
'Overall goal:',
|
|
165
|
-
run.goal,
|
|
166
|
-
'',
|
|
167
|
-
`Your assigned subtask (${subtask.title}):`,
|
|
168
|
-
subtask.prompt,
|
|
169
|
-
'',
|
|
170
|
-
'Constraints:',
|
|
171
|
-
'- Work only on your subtask; do not attempt the whole goal.',
|
|
172
|
-
'- Use the tools you have. Never call a tool whose name starts with "superagent_".',
|
|
173
|
-
'- Your final message IS the deliverable. Do not save the report to a file and do not reply with a',
|
|
174
|
-
' summary of what you did; write the findings themselves: concrete facts, numbers, file paths,',
|
|
175
|
-
' commands, and anything unresolved. Under 1500 words, Markdown only.',
|
|
176
|
-
].join('\n')
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/** Aggregator prompt over completed subtask reports. */
|
|
180
|
-
function aggregatorPrompt(run, completed) {
|
|
181
|
-
const sections = completed.map(subtask => [
|
|
182
|
-
`### ${subtask.title}`,
|
|
183
|
-
capText(subtask.result ?? '', 8000),
|
|
184
|
-
].join('\n')).join('\n\n')
|
|
185
|
-
return [
|
|
186
|
-
'You are the aggregator of a long-horizon autonomous run.',
|
|
187
|
-
'',
|
|
188
|
-
'Goal:',
|
|
189
|
-
run.goal,
|
|
190
|
-
'',
|
|
191
|
-
'Subtask reports:',
|
|
192
|
-
sections,
|
|
193
|
-
'',
|
|
194
|
-
'Write the final report for the user in Markdown as your entire reply:',
|
|
195
|
-
'- Lead with the answer or outcome, then supporting detail. If the goal asks for a table, include the table.',
|
|
196
|
-
'- Merge duplicate findings; when reports disagree, say so and state which evidence is stronger.',
|
|
197
|
-
'- Call out what failed, what is unverified, and what the user should do next.',
|
|
198
|
-
'- Do not invent facts that no report contains. Do not say the report is ready — write it. Maximum 1200 words.',
|
|
199
|
-
].join('\n')
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
/** Deterministic report used when no aggregator model call is available. */
|
|
203
|
-
function deterministicJoin(run) {
|
|
204
|
-
const completed = (run.subtasks ?? []).filter(subtask => subtask.status === 'completed')
|
|
205
|
-
if (completed.length === 0) return null
|
|
206
|
-
const parts = [`# ${run.goal}`, '', `_Deterministic merge of ${completed.length} subtask report(s)._`, '']
|
|
207
|
-
for (const subtask of completed) {
|
|
208
|
-
parts.push(`## ${subtask.title}`, '', subtask.result ?? '', '')
|
|
209
|
-
}
|
|
210
|
-
return parts.join('\n').trim()
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/** Run at most `limit` workers over `items` concurrently. */
|
|
214
|
-
async function pool(items, limit, worker) {
|
|
215
|
-
if (items.length === 0) return
|
|
216
|
-
let cursor = 0
|
|
217
|
-
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
218
|
-
while (cursor < items.length) {
|
|
219
|
-
const item = items[cursor++]
|
|
220
|
-
await worker(item)
|
|
221
|
-
}
|
|
222
|
-
})
|
|
223
|
-
await Promise.all(runners)
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
export function sumUsage(entries) {
|
|
227
|
-
const total = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, totalTokens: 0, modelCalls: 0, models: {} }
|
|
228
|
-
for (const usage of entries) {
|
|
229
|
-
if (!usage) continue
|
|
230
|
-
total.inputTokens += usage.inputTokens ?? 0
|
|
231
|
-
total.outputTokens += usage.outputTokens ?? 0
|
|
232
|
-
total.cacheReadTokens += usage.cacheReadTokens ?? 0
|
|
233
|
-
total.totalTokens += usage.totalTokens ?? 0
|
|
234
|
-
total.modelCalls += usage.modelCalls ?? 0
|
|
235
|
-
for (const [model, entry] of Object.entries(usage.models ?? {})) {
|
|
236
|
-
const target = total.models[model] ?? (total.models[model] = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, calls: 0 })
|
|
237
|
-
target.inputTokens += entry.inputTokens ?? 0
|
|
238
|
-
target.outputTokens += entry.outputTokens ?? 0
|
|
239
|
-
target.cacheReadTokens += entry.cacheReadTokens ?? 0
|
|
240
|
-
target.calls += entry.calls ?? 0
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
return total
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
/** Tokens billed for one usage record. */
|
|
247
|
-
function usageTokens(usage) {
|
|
248
|
-
return (usage?.inputTokens ?? 0) + (usage?.cacheReadTokens ?? 0) + (usage?.outputTokens ?? 0)
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
/**
|
|
252
|
-
* Price recorded usage with an explicit, per-model table.
|
|
253
|
-
*
|
|
254
|
-
* Prices are per 1M tokens. Cache reads fall back to the input price when the
|
|
255
|
-
* table does not list a separate rate, which makes the estimate an upper bound
|
|
256
|
-
* instead of silently under-reporting. Models missing from the table are
|
|
257
|
-
* listed in `unpriced` and excluded from the total.
|
|
258
|
-
*
|
|
259
|
-
* @param {object | null} usage - summed usage for one run.
|
|
260
|
-
* @param {Record<string, { input: number, output: number, cacheRead?: number, currency?: string }> | undefined} pricing
|
|
261
|
-
* @returns {{ total: number, currency: string, byModel: Record<string, number>, unpriced: string[], complete: boolean } | null}
|
|
262
|
-
*/
|
|
263
|
-
export function computeCost(usage, pricing) {
|
|
264
|
-
if (!usage || !usage.modelCalls) return null
|
|
265
|
-
const table = pricing && typeof pricing === 'object' ? pricing : {}
|
|
266
|
-
const byModel = {}
|
|
267
|
-
const unpriced = []
|
|
268
|
-
let total = 0
|
|
269
|
-
let currency = null
|
|
270
|
-
for (const [model, entry] of Object.entries(usage.models ?? {})) {
|
|
271
|
-
// The usage log names a route, e.g. `dashscope/qwen-flash`. Accept a bare
|
|
272
|
-
// model id too, so a one-route config does not have to guess the prefix;
|
|
273
|
-
// an exact `provider/model` key always wins.
|
|
274
|
-
const price = table[model] ?? table[model.slice(model.indexOf('/') + 1)]
|
|
275
|
-
if (!price || typeof price.input !== 'number' || typeof price.output !== 'number') {
|
|
276
|
-
unpriced.push(model)
|
|
277
|
-
continue
|
|
278
|
-
}
|
|
279
|
-
const cacheRead = typeof price.cacheRead === 'number' ? price.cacheRead : price.input
|
|
280
|
-
const cost = (
|
|
281
|
-
entry.inputTokens * price.input
|
|
282
|
-
+ (entry.cacheReadTokens ?? 0) * cacheRead
|
|
283
|
-
+ entry.outputTokens * price.output
|
|
284
|
-
) / 1e6
|
|
285
|
-
byModel[model] = cost
|
|
286
|
-
total += cost
|
|
287
|
-
currency = currency ?? price.currency ?? 'USD'
|
|
288
|
-
}
|
|
289
|
-
if (Object.keys(byModel).length === 0) return { total: 0, currency: currency ?? 'USD', byModel, unpriced, complete: false }
|
|
290
|
-
return { total, currency, byModel, unpriced, complete: unpriced.length === 0 }
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
/** Total tokens recorded so far for a run, used by the budget guard. */
|
|
294
|
-
function spentTokens(run) {
|
|
295
|
-
return usageTokens(run.usage)
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
/**
|
|
299
|
-
* Model calls recorded so far for a run, used by the budget guard.
|
|
300
|
-
*
|
|
301
|
-
* Measured usage is the accurate figure, but it only exists once a child's
|
|
302
|
-
* session log can be read. A spawn is at least one model call, so the spawn
|
|
303
|
-
* count is a floor: when the log layout changes and every read returns null,
|
|
304
|
-
* the call budget still stops the run instead of silently never matching.
|
|
305
|
-
*/
|
|
306
|
-
function spentCalls(run) {
|
|
307
|
-
return Math.max(run.usage?.modelCalls ?? 0, run.counters?.spawns ?? 0)
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
function ensureCounters(run) {
|
|
311
|
-
run.counters = { spawns: 0, unreadable: {}, ...(run.counters ?? {}) }
|
|
312
|
-
if (!run.counters.unreadable || typeof run.counters.unreadable !== 'object') run.counters.unreadable = {}
|
|
313
|
-
return run.counters
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
/**
|
|
317
|
-
* Build the orchestrator.
|
|
318
|
-
*
|
|
319
|
-
* @param {object} deps - injected collaborators.
|
|
320
|
-
* @param {object} deps.store - run store from `createStore`.
|
|
321
|
-
* @param {object} deps.config - resolved plugin config.
|
|
322
|
-
* @param {(spec: { label: string, prompt: string, parent: object, signal: AbortSignal }) => Promise<{ childId: string, result: Promise<object>, dispose: () => Promise<void> }>} deps.spawnChild
|
|
323
|
-
* @param {(childId: string) => Promise<object | null>} deps.readUsage
|
|
324
|
-
* @param {(run: object) => object | void | Promise<object | void>} deps.notify - wake the owning session once.
|
|
325
|
-
* @param {(message: string) => void} [deps.log]
|
|
326
|
-
* @param {() => number} [deps.now]
|
|
327
|
-
* @param {(ms: number) => Promise<void>} [deps.sleep]
|
|
328
|
-
*/
|
|
329
|
-
export function createOrchestrator(deps) {
|
|
330
|
-
const {
|
|
331
|
-
store,
|
|
332
|
-
config,
|
|
333
|
-
spawnChild,
|
|
334
|
-
readUsage,
|
|
335
|
-
notify,
|
|
336
|
-
log = () => {},
|
|
337
|
-
now = () => Date.now(),
|
|
338
|
-
sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
339
|
-
} = deps
|
|
340
|
-
|
|
341
|
-
/** @type {Map<string, { controller: AbortController, activations: Set<object>, waiters: number, run: object, promise: Promise<void> | null, abortCause: string | null }>} */
|
|
342
|
-
const runtime = new Map()
|
|
343
|
-
/** Per-run write chain so two subtasks finishing together cannot interleave renames. */
|
|
344
|
-
const saveChains = new Map()
|
|
345
|
-
let startGate = Promise.resolve()
|
|
346
|
-
|
|
347
|
-
// Cancel markers are optional in the store contract so tests can inject a
|
|
348
|
-
// memory store without them.
|
|
349
|
-
const hasCancelRequest = runId => (typeof store.hasCancelRequest === 'function'
|
|
350
|
-
? store.hasCancelRequest(runId)
|
|
351
|
-
: Promise.resolve(false))
|
|
352
|
-
const clearCancel = runId => (typeof store.clearCancel === 'function'
|
|
353
|
-
? store.clearCancel(runId)
|
|
354
|
-
: Promise.resolve())
|
|
355
|
-
const requestCancel = runId => (typeof store.requestCancel === 'function'
|
|
356
|
-
? store.requestCancel(runId)
|
|
357
|
-
: Promise.resolve(false))
|
|
358
|
-
|
|
359
|
-
function maxActiveRuns() {
|
|
360
|
-
return clamp(config.maxActiveRuns ?? 2, 1, 8)
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
function save(run) {
|
|
364
|
-
run.updatedAt = new Date(now()).toISOString()
|
|
365
|
-
const previous = saveChains.get(run.id) ?? Promise.resolve()
|
|
366
|
-
const next = previous.catch(() => {}).then(() => store.write(run))
|
|
367
|
-
saveChains.set(run.id, next)
|
|
368
|
-
return next
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
function liveRun(runId) {
|
|
372
|
-
return runtime.get(runId)?.run ?? null
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
/** Cached machine identity of the state directory. */
|
|
376
|
-
let ownerRef = null
|
|
377
|
-
async function ownerIdentity() {
|
|
378
|
-
if (ownerRef) return ownerRef
|
|
379
|
-
const host = os.hostname()
|
|
380
|
-
const existing = typeof store.readOwner === 'function' ? await store.readOwner() : null
|
|
381
|
-
if (existing && typeof existing.hostname === 'string') {
|
|
382
|
-
ownerRef = { hostname: host, local: existing.hostname === host, ownerHostname: existing.hostname }
|
|
383
|
-
return ownerRef
|
|
384
|
-
}
|
|
385
|
-
if (typeof store.writeOwner === 'function') {
|
|
386
|
-
await store.writeOwner({ hostname: host, createdAt: new Date(now()).toISOString() })
|
|
387
|
-
}
|
|
388
|
-
ownerRef = { hostname: host, local: true, ownerHostname: host }
|
|
389
|
-
return ownerRef
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
/**
|
|
393
|
-
* @param {unknown} pid - pid recorded by the process that owned the run.
|
|
394
|
-
* @returns {boolean | null} true when alive, false when gone, null when unknown.
|
|
395
|
-
*/
|
|
396
|
-
function pidAlive(pid) {
|
|
397
|
-
const number = Number(pid)
|
|
398
|
-
if (!Number.isInteger(number) || number <= 0) return null
|
|
399
|
-
try {
|
|
400
|
-
process.kill(number, 0)
|
|
401
|
-
return true
|
|
402
|
-
} catch (error) {
|
|
403
|
-
// EPERM means the process exists but belongs to another user.
|
|
404
|
-
return error?.code === 'EPERM'
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
/**
|
|
409
|
-
* Decide whether an active run still has a live owner.
|
|
410
|
-
*
|
|
411
|
-
* Used by recovery, orphan sweeping, and resume, so all three agree: a run
|
|
412
|
-
* that somebody can still be working on is never rewritten. The heartbeat
|
|
413
|
-
* alone proves ownership on a shared directory; a pid is only evidence when
|
|
414
|
-
* the state directory belongs to this machine. A live run lock counts as
|
|
415
|
-
* ownership because the holder is the process that may be spending.
|
|
416
|
-
*
|
|
417
|
-
* @param {object} run - persisted run record.
|
|
418
|
-
* @returns {Promise<{ live: boolean, basis: string, heartbeatAt: number, owner: object, pidChecked: boolean }>}
|
|
419
|
-
*/
|
|
420
|
-
async function ownership(run) {
|
|
421
|
-
const owner = await ownerIdentity()
|
|
422
|
-
const heartbeat = Date.parse(run.heartbeatAt ?? run.updatedAt ?? run.startedAt ?? run.createdAt)
|
|
423
|
-
const heartbeatIso = Number.isFinite(heartbeat) ? new Date(heartbeat).toISOString() : 'unknown'
|
|
424
|
-
const fresh = Number.isFinite(heartbeat) && heartbeat > now() - ORPHAN_MS
|
|
425
|
-
// A pid only means something when the run was written on this machine. On
|
|
426
|
-
// a shared directory the run may belong to a host whose pids are unrelated
|
|
427
|
-
// to ours, so the heartbeat is the only evidence there.
|
|
428
|
-
const localWriter = run.ownerHost ? run.ownerHost === os.hostname() : owner.local
|
|
429
|
-
const alive = localWriter ? pidAlive(run.ownerPid) : null
|
|
430
|
-
|
|
431
|
-
if (alive === true) {
|
|
432
|
-
return { live: true, basis: `owner pid ${run.ownerPid} is still running`, heartbeatAt: heartbeat, owner, pidChecked: true }
|
|
433
|
-
}
|
|
434
|
-
if (typeof store.readLock === 'function') {
|
|
435
|
-
const holder = await store.readLock(run.id).catch(() => null)
|
|
436
|
-
if (holder && holder.hostname === os.hostname() && pidAlive(holder.pid) === true) {
|
|
437
|
-
return { live: true, basis: `pid ${holder.pid} holds the run lock`, heartbeatAt: heartbeat, owner, pidChecked: true }
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
// A heartbeat can be a few seconds old when the host is killed, and waiting
|
|
441
|
-
// 90 s for it to expire would strand a run that this machine can prove is
|
|
442
|
-
// dead. On the same machine, a missing pid outranks a fresh timestamp.
|
|
443
|
-
if (alive === false) {
|
|
444
|
-
return {
|
|
445
|
-
live: false,
|
|
446
|
-
basis: `owner pid ${run.ownerPid} is not running and this state directory is local`,
|
|
447
|
-
heartbeatAt: heartbeat,
|
|
448
|
-
owner,
|
|
449
|
-
pidChecked: true,
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
if (fresh) {
|
|
453
|
-
return { live: true, basis: `its heartbeat was refreshed at ${heartbeatIso}`, heartbeatAt: heartbeat, owner, pidChecked: false }
|
|
454
|
-
}
|
|
455
|
-
const basis = owner.local
|
|
456
|
-
? `owner pid ${run.ownerPid ?? 'unknown'} is not running and its heartbeat stopped at ${heartbeatIso}`
|
|
457
|
-
: `the state directory is shared with ${owner.ownerHostname}, so a local pid proves nothing; the heartbeat stopped at ${heartbeatIso}`
|
|
458
|
-
return { live: false, basis, heartbeatAt: heartbeat, owner, pidChecked: owner.local }
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
/**
|
|
462
|
-
* Rewrite a run whose owner is gone. Never silent: the exact evidence is
|
|
463
|
-
* stored on the run and printed by status.
|
|
464
|
-
*/
|
|
465
|
-
function reclaim(run, verdict, cancelled = false) {
|
|
466
|
-
run.status = cancelled ? 'cancelled' : 'interrupted'
|
|
467
|
-
run.error = cancelled
|
|
468
|
-
? 'cancelled by superagent_cancel while no process was running it'
|
|
469
|
-
: 'host process stopped while the run was active'
|
|
470
|
-
run.finishedAt = new Date(now()).toISOString()
|
|
471
|
-
run.reclaimed = {
|
|
472
|
-
at: new Date(now()).toISOString(),
|
|
473
|
-
basis: verdict.basis,
|
|
474
|
-
heartbeatAt: Number.isFinite(verdict.heartbeatAt) ? new Date(verdict.heartbeatAt).toISOString() : null,
|
|
475
|
-
ownerPid: run.ownerPid ?? null,
|
|
476
|
-
pidChecked: verdict.pidChecked,
|
|
477
|
-
sharedWith: verdict.owner.local ? null : verdict.owner.ownerHostname,
|
|
478
|
-
}
|
|
479
|
-
for (const subtask of run.subtasks ?? []) {
|
|
480
|
-
if (subtask.status === 'running') subtask.status = 'pending'
|
|
481
|
-
}
|
|
482
|
-
// The owner died before it could settle, so publish what is already known
|
|
483
|
-
// about the spend: a reclaimed run must not claim its cost is unknown when
|
|
484
|
-
// its child logs were readable.
|
|
485
|
-
const entries = Object.values(run.usageByChild ?? {})
|
|
486
|
-
if (entries.length > 0) run.usage = sumUsage(entries)
|
|
487
|
-
run.cost = computeCost(run.usage, config.pricing)
|
|
488
|
-
for (const subtask of run.subtasks ?? []) {
|
|
489
|
-
if (subtask.childId && run.usageByChild?.[subtask.childId]) {
|
|
490
|
-
subtask.usage = run.usageByChild[subtask.childId]
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
async function releaseRunLock(runId) {
|
|
496
|
-
if (typeof store.releaseLock !== 'function') return
|
|
497
|
-
try {
|
|
498
|
-
await store.releaseLock(runId)
|
|
499
|
-
} catch (error) {
|
|
500
|
-
log(`cannot release the lock for run ${runId}: ${error?.message ?? error}`)
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
/**
|
|
505
|
-
* Claim the exclusive right to schedule a run.
|
|
506
|
-
*
|
|
507
|
-
* Two profiles can point at one state directory, so a resume must not be
|
|
508
|
-
* decided from a read alone: both processes would see a terminal run and both
|
|
509
|
-
* would schedule it. The lock makes one of them lose.
|
|
510
|
-
*/
|
|
511
|
-
async function takeRunLock(runId) {
|
|
512
|
-
if (typeof store.acquireLock !== 'function') return
|
|
513
|
-
const holder = { pid: process.pid, hostname: os.hostname(), at: new Date(now()).toISOString() }
|
|
514
|
-
let attempt = await store.acquireLock(runId, holder)
|
|
515
|
-
if (!attempt.acquired) {
|
|
516
|
-
const current = attempt.holder
|
|
517
|
-
// A lock left by a process on this machine that no longer exists is
|
|
518
|
-
// evidence of a crash, not of a live owner. A lock on another machine has
|
|
519
|
-
// no checkable pid, so its age is the only evidence: past LOCK_STALE_MS
|
|
520
|
-
// the process that took it is no longer plausibly still scheduling.
|
|
521
|
-
const deadHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === false
|
|
522
|
-
const aliveHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === true
|
|
523
|
-
const age = now() - Date.parse(current?.at ?? '')
|
|
524
|
-
const stale = !Number.isFinite(age) || age > LOCK_STALE_MS
|
|
525
|
-
// Age is only evidence when the holder cannot be checked: a live process
|
|
526
|
-
// on this machine may legitimately schedule a long run for hours.
|
|
527
|
-
if (deadHere || (stale && !aliveHere)) {
|
|
528
|
-
await releaseRunLock(runId)
|
|
529
|
-
attempt = await store.acquireLock(runId, holder)
|
|
530
|
-
}
|
|
531
|
-
if (!attempt.acquired) {
|
|
532
|
-
const held = attempt.holder
|
|
533
|
-
const who = held
|
|
534
|
-
? `pid ${held.pid ?? 'unknown'} on ${held.hostname ?? 'an unknown host'}`
|
|
535
|
-
: 'another process'
|
|
536
|
-
throw new Error(`run ${runId} is already being started by ${who}; wait for it to finish or cancel it`)
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
/**
|
|
542
|
-
* Reclaim runs whose owning process is gone, so an orphan cannot hold the
|
|
543
|
-
* active-run quota forever.
|
|
544
|
-
*
|
|
545
|
-
* @returns {Promise<string[]>} reclaimed run ids, in list order.
|
|
546
|
-
*/
|
|
547
|
-
async function sweepOrphans() {
|
|
548
|
-
const stale = (await store.list()).filter(run => ACTIVE_STATUSES.has(run.status) && !runtime.has(run.id))
|
|
549
|
-
if (stale.length === 0) return []
|
|
550
|
-
const reclaimed = []
|
|
551
|
-
for (const run of stale) {
|
|
552
|
-
const verdict = await ownership(run)
|
|
553
|
-
if (verdict.live) continue
|
|
554
|
-
reclaim(run, verdict)
|
|
555
|
-
await save(run)
|
|
556
|
-
await releaseRunLock(run.id)
|
|
557
|
-
log(`run ${run.id} was active with no live owner; reclaimed its quota (${verdict.basis})`)
|
|
558
|
-
reclaimed.push(run.id)
|
|
559
|
-
}
|
|
560
|
-
return reclaimed
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
async function activeCount() {
|
|
564
|
-
try {
|
|
565
|
-
await sweepOrphans()
|
|
566
|
-
} catch (error) {
|
|
567
|
-
log(`cannot check for orphaned runs: ${error?.message ?? error}`)
|
|
568
|
-
}
|
|
569
|
-
let count = 0
|
|
570
|
-
for (const rt of runtime.values()) {
|
|
571
|
-
if (rt.run && ACTIVE_STATUSES.has(rt.run.status)) count++
|
|
572
|
-
}
|
|
573
|
-
for (const run of await store.list()) {
|
|
574
|
-
if (runtime.has(run.id)) continue
|
|
575
|
-
if (ACTIVE_STATUSES.has(run.status)) count++
|
|
576
|
-
}
|
|
577
|
-
return count
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
function quotaMessage(active) {
|
|
581
|
-
return `quota exceeded: ${active} active run(s) and maxActiveRuns is ${maxActiveRuns()}. `
|
|
582
|
-
+ 'Let one finish with superagent_status or cancel it with superagent_cancel.'
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
function buildLimits(overrides = {}) {
|
|
586
|
-
// A caller may narrow a limit but never widen the operator's cap: the
|
|
587
|
-
// config is the spending and blast-radius control, and a model-written tool
|
|
588
|
-
// argument must not be able to raise it.
|
|
589
|
-
const narrowed = (override, cap) => (
|
|
590
|
-
override === undefined || override === null ? cap : Math.min(Number(override), cap)
|
|
591
|
-
)
|
|
592
|
-
const limits = {
|
|
593
|
-
maxSubtasks: clamp(narrowed(overrides.maxSubtasks, config.maxSubtasks), 1, 16),
|
|
594
|
-
maxParallel: clamp(narrowed(overrides.maxParallel, config.maxParallel), 1, 8),
|
|
595
|
-
deadlineMinutes: clamp(narrowed(overrides.deadlineMinutes, config.deadlineMinutes), 1, 720),
|
|
596
|
-
attempts: clamp(config.attempts, 1, 5),
|
|
597
|
-
maxModelCalls: clamp(narrowed(overrides.maxModelCalls, config.maxModelCallsPerRun), 0, 10000),
|
|
598
|
-
maxTokens: clamp(narrowed(overrides.maxTokens, config.maxTokensPerRun), 0, 1e12),
|
|
599
|
-
}
|
|
600
|
-
limits.maxParallel = Math.min(limits.maxParallel, limits.maxSubtasks)
|
|
601
|
-
return limits
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
/**
|
|
605
|
-
* Report why a run may not start another child, or null when it still can.
|
|
606
|
-
* Both budgets are hard stops: they exist so a pathological goal cannot spend
|
|
607
|
-
* quietly, so they are checked before a spawn rather than after.
|
|
608
|
-
*/
|
|
609
|
-
function budgetStop(run) {
|
|
610
|
-
const calls = spentCalls(run)
|
|
611
|
-
const tokens = spentTokens(run)
|
|
612
|
-
if (run.limits.maxModelCalls > 0 && calls >= run.limits.maxModelCalls) {
|
|
613
|
-
return `model-call budget reached (${calls}/${run.limits.maxModelCalls} calls)`
|
|
614
|
-
}
|
|
615
|
-
if (run.limits.maxTokens > 0 && tokens >= run.limits.maxTokens) {
|
|
616
|
-
return `token budget reached (${tokens}/${run.limits.maxTokens} tokens)`
|
|
617
|
-
}
|
|
618
|
-
return null
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
/**
|
|
622
|
-
* Spawn one child under the run's abort signal and budget.
|
|
623
|
-
* @param {object} run - live run record.
|
|
624
|
-
* @param {object} rt - runtime entry.
|
|
625
|
-
* @param {{ label: string, prompt: string, parent: object }} spec - child request.
|
|
626
|
-
*/
|
|
627
|
-
async function spawn(run, rt, spec) {
|
|
628
|
-
const blocked = budgetStop(run)
|
|
629
|
-
if (blocked) {
|
|
630
|
-
if (!rt.abortCause) rt.abortCause = 'budget'
|
|
631
|
-
if (!run.error) run.error = blocked
|
|
632
|
-
if (!rt.controller.signal.aborted) rt.controller.abort(new Error(blocked))
|
|
633
|
-
throw new Error(`budget exceeded: ${blocked}; raise maxModelCallsPerRun or maxTokensPerRun if this was intended`)
|
|
634
|
-
}
|
|
635
|
-
// Counted before the child exists and persisted before it can spend, so a
|
|
636
|
-
// crash cannot lose the only evidence of a model call.
|
|
637
|
-
ensureCounters(run).spawns += 1
|
|
638
|
-
await save(run)
|
|
639
|
-
return spawnAbortable(spec, rt.controller.signal)
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
/**
|
|
643
|
-
* Create one child, but stop waiting for it as soon as the run is aborted.
|
|
644
|
-
*
|
|
645
|
-
* A provider that queues activations can leave `startActivation` pending long
|
|
646
|
-
* after the run was cancelled or ran out of time. Without this the run would
|
|
647
|
-
* hang in planning/running while its heartbeat kept it looking alive, so no
|
|
648
|
-
* recovery path could ever reclaim it. A child that appears after the abort is
|
|
649
|
-
* disposed immediately instead of being left to spend.
|
|
650
|
-
*/
|
|
651
|
-
function spawnAbortable(spec, signal) {
|
|
652
|
-
const pending = spawnChild({ ...spec, signal })
|
|
653
|
-
if (!signal) return pending
|
|
654
|
-
if (signal.aborted) {
|
|
655
|
-
pending.then(handle => disposeQuietly(handle), () => {})
|
|
656
|
-
return Promise.reject(new Error('aborted'))
|
|
657
|
-
}
|
|
658
|
-
return new Promise((resolve, reject) => {
|
|
659
|
-
let done = false
|
|
660
|
-
const onAbort = () => {
|
|
661
|
-
if (done) return
|
|
662
|
-
done = true
|
|
663
|
-
// The `pending` handler registered below disposes the handle if it
|
|
664
|
-
// arrives late, so the child is not left spending in the background.
|
|
665
|
-
reject(new Error('aborted'))
|
|
666
|
-
}
|
|
667
|
-
signal.addEventListener('abort', onAbort, { once: true })
|
|
668
|
-
pending.then(
|
|
669
|
-
handle => {
|
|
670
|
-
if (done) {
|
|
671
|
-
disposeQuietly(handle)
|
|
672
|
-
return
|
|
673
|
-
}
|
|
674
|
-
done = true
|
|
675
|
-
signal.removeEventListener('abort', onAbort)
|
|
676
|
-
resolve(handle)
|
|
677
|
-
},
|
|
678
|
-
error => {
|
|
679
|
-
if (done) return
|
|
680
|
-
done = true
|
|
681
|
-
signal.removeEventListener('abort', onAbort)
|
|
682
|
-
reject(error)
|
|
683
|
-
},
|
|
684
|
-
)
|
|
685
|
-
})
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
/**
|
|
689
|
-
* Read one run for a tool call, turning the two failure modes a user can hit
|
|
690
|
-
* into sentences instead of a null.
|
|
691
|
-
* @param {string} runId - run identity.
|
|
692
|
-
*/
|
|
693
|
-
async function loadRun(runId) {
|
|
694
|
-
const run = await store.read(runId)
|
|
695
|
-
if (run) return run
|
|
696
|
-
const quarantined = store.corrupt?.get?.(runId)
|
|
697
|
-
if (quarantined) {
|
|
698
|
-
throw new Error(
|
|
699
|
-
`state file for run ${runId} is not valid JSON, so the run cannot be read. `
|
|
700
|
-
+ `The original file was kept at ${quarantined}. Inspect it, then delete it; `
|
|
701
|
-
+ 'the run is skipped by superagent_list until you do.',
|
|
702
|
-
)
|
|
703
|
-
}
|
|
704
|
-
throw new Error(`unknown run ${runId}; call superagent_list to see recorded run ids`)
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
/**
|
|
708
|
-
* Mark runs left active by a previous host process as interrupted.
|
|
709
|
-
*
|
|
710
|
-
* A run that another live process owns is left completely alone: rewriting it
|
|
711
|
-
* here would make two processes fight over the same file while the first one
|
|
712
|
-
* keeps spending, and a later resume would then run everything twice.
|
|
713
|
-
*/
|
|
714
|
-
async function recover() {
|
|
715
|
-
await store.init()
|
|
716
|
-
const recovered = []
|
|
717
|
-
for (const run of await store.list()) {
|
|
718
|
-
if (!ACTIVE_STATUSES.has(run.status)) {
|
|
719
|
-
if (await hasCancelRequest(run.id)) await clearCancel(run.id)
|
|
720
|
-
continue
|
|
721
|
-
}
|
|
722
|
-
if (runtime.has(run.id)) continue
|
|
723
|
-
const verdict = await ownership(run)
|
|
724
|
-
if (verdict.live) {
|
|
725
|
-
log(`run ${run.id} is still owned (${verdict.basis}); leaving it running`)
|
|
726
|
-
continue
|
|
727
|
-
}
|
|
728
|
-
reclaim(run, verdict, await hasCancelRequest(run.id))
|
|
729
|
-
await save(run)
|
|
730
|
-
await clearCancel(run.id)
|
|
731
|
-
await releaseRunLock(run.id)
|
|
732
|
-
recovered.push(run.id)
|
|
733
|
-
}
|
|
734
|
-
return recovered
|
|
735
|
-
}
|
|
736
|
-
|
|
737
|
-
/**
|
|
738
|
-
* Fold one child's usage snapshot into the run. The snapshot is cumulative,
|
|
739
|
-
* so the last read for a child replaces the earlier one instead of adding.
|
|
740
|
-
* Called at start, on every heartbeat, and on success or failure, which is
|
|
741
|
-
* why an unreadable log degrades to the spawn floor instead of to zero.
|
|
742
|
-
*/
|
|
743
|
-
async function recordUsage(run, childId) {
|
|
744
|
-
let usage = null
|
|
745
|
-
try {
|
|
746
|
-
usage = await readUsage(childId)
|
|
747
|
-
} catch (error) {
|
|
748
|
-
log(`cannot read usage for child ${childId}: ${error?.message ?? error}`)
|
|
749
|
-
}
|
|
750
|
-
if (!usage) {
|
|
751
|
-
ensureCounters(run).unreadable[childId] = true
|
|
752
|
-
return null
|
|
753
|
-
}
|
|
754
|
-
run.usageByChild = run.usageByChild ?? {}
|
|
755
|
-
run.usageByChild[childId] = usage
|
|
756
|
-
run.usage = sumUsage(Object.values(run.usageByChild))
|
|
757
|
-
return usage
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
/** Refresh usage for children that are still running, from the heartbeat. */
|
|
761
|
-
async function refreshUsage(run, rt) {
|
|
762
|
-
const handles = [...rt.activations]
|
|
763
|
-
for (const handle of handles) await recordUsage(run, handle.childId)
|
|
764
|
-
}
|
|
765
|
-
|
|
766
|
-
/**
|
|
767
|
-
* Wait for a child result, but stop waiting when the run is aborted. Without
|
|
768
|
-
* this a child that ignores its abort signal keeps the whole run open.
|
|
769
|
-
*/
|
|
770
|
-
function abortable(promise, signal) {
|
|
771
|
-
if (!signal) return promise
|
|
772
|
-
return new Promise((resolve, reject) => {
|
|
773
|
-
const onAbort = () => {
|
|
774
|
-
promise.catch(() => {})
|
|
775
|
-
reject(new Error('aborted'))
|
|
776
|
-
}
|
|
777
|
-
if (signal.aborted) {
|
|
778
|
-
onAbort()
|
|
779
|
-
return
|
|
780
|
-
}
|
|
781
|
-
signal.addEventListener('abort', onAbort, { once: true })
|
|
782
|
-
promise.then(
|
|
783
|
-
value => {
|
|
784
|
-
signal.removeEventListener('abort', onAbort)
|
|
785
|
-
resolve(value)
|
|
786
|
-
},
|
|
787
|
-
error => {
|
|
788
|
-
signal.removeEventListener('abort', onAbort)
|
|
789
|
-
reject(error)
|
|
790
|
-
},
|
|
791
|
-
)
|
|
792
|
-
})
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
/** Dispose one child activation without letting it block the caller. */
|
|
796
|
-
async function disposeQuietly(handle) {
|
|
797
|
-
if (!handle || typeof handle.dispose !== 'function') return
|
|
798
|
-
try {
|
|
799
|
-
await Promise.race([
|
|
800
|
-
Promise.resolve().then(() => handle.dispose()),
|
|
801
|
-
sleep(DISPOSE_MS),
|
|
802
|
-
])
|
|
803
|
-
} catch (error) {
|
|
804
|
-
log(`child dispose failed: ${error?.message ?? error}`)
|
|
805
|
-
}
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
/**
|
|
809
|
-
* The host's subagent service disappeared while this run was active, which
|
|
810
|
-
* in practice means the host is shutting down. That is not a subtask
|
|
811
|
-
* failure: the work never got a chance to run. Remember the cause so the run
|
|
812
|
-
* settles as `interrupted` (resumable) instead of `failed`, and abort the
|
|
813
|
-
* remaining children.
|
|
814
|
-
*/
|
|
815
|
-
function markHostGone(rt, error) {
|
|
816
|
-
if (rt.abortCause) return
|
|
817
|
-
rt.abortCause = 'host-gone'
|
|
818
|
-
if (!rt.controller.signal.aborted) {
|
|
819
|
-
rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
|
|
820
|
-
}
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
async function makePlan(run, parent, rt) {
|
|
824
|
-
try {
|
|
825
|
-
const handle = await spawn(run, rt, {
|
|
826
|
-
label: 'superagent-plan',
|
|
827
|
-
prompt: plannerPrompt(run.goal, run.limits.maxSubtasks),
|
|
828
|
-
parent,
|
|
829
|
-
})
|
|
830
|
-
rt.activations.add(handle)
|
|
831
|
-
run.steps.planner = { childId: handle.childId }
|
|
832
|
-
try {
|
|
833
|
-
const result = await abortable(handle.result, rt.controller.signal)
|
|
834
|
-
const text = textOf(result.output)
|
|
835
|
-
const subtasks = extractPlan(text).slice(0, run.limits.maxSubtasks)
|
|
836
|
-
if (subtasks.length > 0) return { mode: 'planned', subtasks }
|
|
837
|
-
log(`planner reply had no parsable plan; running the goal as one subtask`)
|
|
838
|
-
} finally {
|
|
839
|
-
await recordUsage(run, handle.childId)
|
|
840
|
-
rt.activations.delete(handle)
|
|
841
|
-
await disposeQuietly(handle)
|
|
842
|
-
}
|
|
843
|
-
} catch (error) {
|
|
844
|
-
log(`planner failed: ${error?.message ?? error}`)
|
|
845
|
-
if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
|
|
846
|
-
}
|
|
847
|
-
return { mode: 'degraded', subtasks: [{ title: 'Complete the goal', prompt: run.goal }] }
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
async function runSubtask(run, subtask, parent, rt) {
|
|
851
|
-
const signal = rt.controller.signal
|
|
852
|
-
for (let attempt = subtask.attempts + 1; attempt <= run.limits.attempts; attempt++) {
|
|
853
|
-
if (signal.aborted) break
|
|
854
|
-
subtask.status = 'running'
|
|
855
|
-
subtask.attempts = attempt
|
|
856
|
-
subtask.startedAt = subtask.startedAt ?? new Date(now()).toISOString()
|
|
857
|
-
await save(run)
|
|
858
|
-
|
|
859
|
-
let handle = null
|
|
860
|
-
try {
|
|
861
|
-
handle = await spawn(run, rt, {
|
|
862
|
-
label: `superagent-${subtask.index}`,
|
|
863
|
-
prompt: workerPrompt(run, subtask),
|
|
864
|
-
parent,
|
|
865
|
-
})
|
|
866
|
-
rt.activations.add(handle)
|
|
867
|
-
subtask.childId = handle.childId
|
|
868
|
-
await save(run)
|
|
869
|
-
|
|
870
|
-
const result = await abortable(handle.result, signal)
|
|
871
|
-
const text = textOf(result.output)
|
|
872
|
-
subtask.stopReason = result.stopReason
|
|
873
|
-
if (result.stopReason === 'completed' && text) {
|
|
874
|
-
subtask.status = 'completed'
|
|
875
|
-
subtask.result = capText(text, config.resultCapBytes)
|
|
876
|
-
subtask.error = null
|
|
877
|
-
subtask.finishedAt = new Date(now()).toISOString()
|
|
878
|
-
await recordUsage(run, handle.childId)
|
|
879
|
-
await save(run)
|
|
880
|
-
return
|
|
881
|
-
}
|
|
882
|
-
subtask.error = result.diagnostic || `child stopped with reason "${result.stopReason}"`
|
|
883
|
-
log(`subtask ${subtask.index} attempt ${attempt} did not complete: ${subtask.error}`)
|
|
884
|
-
} catch (error) {
|
|
885
|
-
subtask.error = error?.message ?? String(error)
|
|
886
|
-
log(`subtask ${subtask.index} attempt ${attempt} threw: ${subtask.error}`)
|
|
887
|
-
if (error?.code === 'subagent-unavailable') {
|
|
888
|
-
// The host is going away; leave the subtask as work to do rather
|
|
889
|
-
// than burning the remaining attempts on a service that is gone.
|
|
890
|
-
subtask.error = null
|
|
891
|
-
subtask.stopReason = null
|
|
892
|
-
markHostGone(rt, error)
|
|
893
|
-
}
|
|
894
|
-
} finally {
|
|
895
|
-
if (handle) {
|
|
896
|
-
// A failed attempt still spent tokens, so read it before disposing.
|
|
897
|
-
await recordUsage(run, handle.childId)
|
|
898
|
-
rt.activations.delete(handle)
|
|
899
|
-
await disposeQuietly(handle)
|
|
900
|
-
}
|
|
901
|
-
}
|
|
902
|
-
|
|
903
|
-
if (signal.aborted) break
|
|
904
|
-
if (attempt < run.limits.attempts) {
|
|
905
|
-
subtask.status = 'pending'
|
|
906
|
-
await save(run)
|
|
907
|
-
await sleep(1000 * attempt)
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
if (subtask.status !== 'completed') {
|
|
912
|
-
if (signal.aborted) {
|
|
913
|
-
// Cancelled or out of time: the subtask did not fail on its own terms,
|
|
914
|
-
// so leave it resumable instead of recording a failure.
|
|
915
|
-
subtask.status = 'pending'
|
|
916
|
-
subtask.error = null
|
|
917
|
-
subtask.stopReason = null
|
|
918
|
-
} else {
|
|
919
|
-
subtask.status = 'failed'
|
|
920
|
-
subtask.finishedAt = new Date(now()).toISOString()
|
|
921
|
-
}
|
|
922
|
-
await save(run)
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
async function aggregate(run, parent, rt) {
|
|
927
|
-
const completed = run.subtasks.filter(subtask => subtask.status === 'completed')
|
|
928
|
-
if (completed.length === 0) return null
|
|
929
|
-
if (rt.controller.signal.aborted) return deterministicJoin(run)
|
|
930
|
-
if (completed.length === 1) return `# ${run.goal}\n\n${completed[0].result ?? ''}`.trim()
|
|
931
|
-
|
|
932
|
-
try {
|
|
933
|
-
const handle = await spawn(run, rt, {
|
|
934
|
-
label: 'superagent-aggregate',
|
|
935
|
-
prompt: aggregatorPrompt(run, completed),
|
|
936
|
-
parent,
|
|
937
|
-
})
|
|
938
|
-
rt.activations.add(handle)
|
|
939
|
-
run.steps.aggregator = { childId: handle.childId }
|
|
940
|
-
try {
|
|
941
|
-
const result = await abortable(handle.result, rt.controller.signal)
|
|
942
|
-
const text = textOf(result.output)
|
|
943
|
-
if (result.stopReason === 'completed' && text) return text
|
|
944
|
-
log(`aggregator did not complete (${result.stopReason}); using deterministic merge`)
|
|
945
|
-
} finally {
|
|
946
|
-
await recordUsage(run, handle.childId)
|
|
947
|
-
rt.activations.delete(handle)
|
|
948
|
-
await disposeQuietly(handle)
|
|
949
|
-
}
|
|
950
|
-
} catch (error) {
|
|
951
|
-
log(`aggregator failed: ${error?.message ?? error}`)
|
|
952
|
-
if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
|
|
953
|
-
}
|
|
954
|
-
return deterministicJoin(run)
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
function terminalStatus(run, rt) {
|
|
958
|
-
if (rt.abortCause === 'cancelled') return 'cancelled'
|
|
959
|
-
// The host went away mid-run. Nothing failed on its own terms, so the run
|
|
960
|
-
// must stay resumable instead of being buried as `failed`.
|
|
961
|
-
if (rt.abortCause === 'host-gone') return 'interrupted'
|
|
962
|
-
const subtasks = run.subtasks ?? []
|
|
963
|
-
const completed = subtasks.filter(subtask => subtask.status === 'completed').length
|
|
964
|
-
if (completed === 0) return 'failed'
|
|
965
|
-
if (rt.abortCause === 'deadline') {
|
|
966
|
-
return completed === subtasks.length ? 'completed' : 'partial'
|
|
967
|
-
}
|
|
968
|
-
return completed === subtasks.length ? 'completed' : 'partial'
|
|
969
|
-
}
|
|
970
|
-
|
|
971
|
-
/**
|
|
972
|
-
* Watch for a cancel written by another process. The marker is checked from
|
|
973
|
-
* inside the owning process, which is the only place that can abort children;
|
|
974
|
-
* that is why cancel is a file and not a state-file rewrite.
|
|
975
|
-
*/
|
|
976
|
-
function watchCancel(run, rt) {
|
|
977
|
-
const timer = setInterval(() => {
|
|
978
|
-
hasCancelRequest(run.id).then(requested => {
|
|
979
|
-
if (!requested || rt.controller.signal.aborted) return
|
|
980
|
-
rt.abortCause = 'cancelled'
|
|
981
|
-
rt.controller.abort(new Error('cancelled by superagent_cancel from another process'))
|
|
982
|
-
// Aborting the signal alone does not stop a child that is already
|
|
983
|
-
// running; dispose its activation so it stops spending tokens.
|
|
984
|
-
for (const handle of [...rt.activations]) disposeQuietly(handle)
|
|
985
|
-
}).catch(() => {})
|
|
986
|
-
}, CANCEL_POLL_MS)
|
|
987
|
-
timer.unref?.()
|
|
988
|
-
return timer
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
/**
|
|
992
|
-
* A caller waiting inside this process will read the terminal result itself,
|
|
993
|
-
* so waking the session would be duplicate noise. A caller in *another*
|
|
994
|
-
* process is deliberately not treated this way: it can time out just before
|
|
995
|
-
* the run settles, in which case suppressing the wake-up would lose the only
|
|
996
|
-
* notification the user gets.
|
|
997
|
-
*/
|
|
998
|
-
function watchedHere(rt) {
|
|
999
|
-
return rt.waiters > 0
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
/**
|
|
1003
|
-
* Compute the terminal state, decide about notifying, and write it exactly
|
|
1004
|
-
* once. Writing the terminal status first and the notify outcome second left
|
|
1005
|
-
* a window where readers saw a finished run with no notify line at all.
|
|
1006
|
-
*/
|
|
1007
|
-
async function settle(run, rt) {
|
|
1008
|
-
// Usage is stored per child so repeated reads replace rather than add. A
|
|
1009
|
-
// run written before that field existed keeps whatever total it had.
|
|
1010
|
-
const entries = Object.values(run.usageByChild ?? {})
|
|
1011
|
-
if (entries.length > 0) run.usage = sumUsage(entries)
|
|
1012
|
-
// The per-subtask figure is read when the child finishes, which can race a
|
|
1013
|
-
// log flush; the settle pass has the final snapshot, so publish it here.
|
|
1014
|
-
for (const subtask of run.subtasks ?? []) {
|
|
1015
|
-
if (subtask.childId && run.usageByChild?.[subtask.childId]) {
|
|
1016
|
-
subtask.usage = run.usageByChild[subtask.childId]
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
run.cost = computeCost(run.usage, config.pricing)
|
|
1020
|
-
const nextStatus = terminalStatus(run, rt)
|
|
1021
|
-
const finishedAt = new Date(now()).toISOString()
|
|
1022
|
-
let error = run.error
|
|
1023
|
-
if (rt.abortCause === 'cancelled') {
|
|
1024
|
-
error = 'cancelled by superagent_cancel'
|
|
1025
|
-
} else if (rt.abortCause === 'host-gone') {
|
|
1026
|
-
error = 'the host shut down while this run was active; run superagent_resume in a new host to finish it'
|
|
1027
|
-
} else if (nextStatus === 'failed' && !error) {
|
|
1028
|
-
error = run.subtasks.map(subtask => subtask.error).filter(Boolean).slice(0, 3).join('; ') || 'no subtask completed'
|
|
1029
|
-
}
|
|
1030
|
-
if (rt.abortCause === 'deadline' && !error) error = 'deadline exceeded'
|
|
1031
|
-
|
|
1032
|
-
let outcome = null
|
|
1033
|
-
if (TERMINAL_STATUSES.has(nextStatus)) {
|
|
1034
|
-
if (watchedHere(rt)) {
|
|
1035
|
-
outcome = { delivered: false, reason: 'skipped: a caller in this process is reading the run' }
|
|
1036
|
-
} else {
|
|
1037
|
-
try {
|
|
1038
|
-
// `notify` may wait on the host's session loop; cap it so a finished
|
|
1039
|
-
// run cannot stay active on disk (and be reclaimed by another process)
|
|
1040
|
-
// just because a wake-up is slow.
|
|
1041
|
-
outcome = await withTimeout(
|
|
1042
|
-
Promise.resolve().then(() => notify({ ...run, status: nextStatus, finishedAt, error })),
|
|
1043
|
-
NOTIFY_MS,
|
|
1044
|
-
() => ({
|
|
1045
|
-
delivered: false,
|
|
1046
|
-
reason: `the owning session did not accept the wake-up within ${NOTIFY_MS / 1000}s`,
|
|
1047
|
-
}),
|
|
1048
|
-
)
|
|
1049
|
-
outcome = outcome ?? { delivered: true }
|
|
1050
|
-
} catch (notifyError) {
|
|
1051
|
-
outcome = { delivered: false, reason: `notification threw: ${notifyError?.message ?? notifyError}` }
|
|
1052
|
-
log(`notification failed for ${run.id}: ${notifyError?.message ?? notifyError}`)
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
|
-
// The terminal status and its notify row land in the same write, so a
|
|
1058
|
-
// reader never sees a finished run that still claims to be active and never
|
|
1059
|
-
// sees a finished run whose notify row is missing.
|
|
1060
|
-
run.status = nextStatus
|
|
1061
|
-
run.finishedAt = finishedAt
|
|
1062
|
-
run.activeMs = (Number.isFinite(run.activeMs) ? run.activeMs : 0) + Math.max(0, now() - rt.activeStart)
|
|
1063
|
-
run.error = error
|
|
1064
|
-
run.cancelPending = false
|
|
1065
|
-
if (outcome) {
|
|
1066
|
-
run.notify = {
|
|
1067
|
-
at: new Date(now()).toISOString(),
|
|
1068
|
-
delivered: outcome.delivered !== false,
|
|
1069
|
-
reason: outcome.reason ?? null,
|
|
1070
|
-
}
|
|
1071
|
-
if (outcome.delivered !== false) run.notified = true
|
|
1072
|
-
}
|
|
1073
|
-
await save(run)
|
|
1074
|
-
// The run is terminal, so a leftover marker would only make the next mount
|
|
1075
|
-
// rewrite a finished run; drop it now.
|
|
1076
|
-
await clearCancel(run.id)
|
|
1077
|
-
await releaseRunLock(run.id)
|
|
1078
|
-
}
|
|
1079
|
-
|
|
1080
|
-
/**
|
|
1081
|
-
* Resolve with `promise`, or with `onTimeout()` after `ms`.
|
|
1082
|
-
*
|
|
1083
|
-
* The timer is cleared when the promise wins: a stray 15s timer would keep a
|
|
1084
|
-
* finished host process (and a test runner) alive for no reason.
|
|
1085
|
-
*/
|
|
1086
|
-
function withTimeout(promise, ms, onTimeout) {
|
|
1087
|
-
return new Promise((resolve, reject) => {
|
|
1088
|
-
const timer = setTimeout(() => resolve(onTimeout()), ms)
|
|
1089
|
-
timer.unref?.()
|
|
1090
|
-
promise.then(
|
|
1091
|
-
value => {
|
|
1092
|
-
clearTimeout(timer)
|
|
1093
|
-
resolve(value)
|
|
1094
|
-
},
|
|
1095
|
-
error => {
|
|
1096
|
-
clearTimeout(timer)
|
|
1097
|
-
reject(error)
|
|
1098
|
-
},
|
|
1099
|
-
)
|
|
1100
|
-
})
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
async function execute(run, parent, rt) {
|
|
1104
|
-
rt.activeStart = now()
|
|
1105
|
-
const remaining = Math.max(0, Date.parse(run.deadlineAt) - now())
|
|
1106
|
-
const timer = setTimeout(() => {
|
|
1107
|
-
rt.abortCause = 'deadline'
|
|
1108
|
-
rt.controller.abort(new Error(`deadline exceeded after ${run.limits.deadlineMinutes} minutes`))
|
|
1109
|
-
}, remaining)
|
|
1110
|
-
timer.unref?.()
|
|
1111
|
-
const cancelTimer = watchCancel(run, rt)
|
|
1112
|
-
// The heartbeat is what makes an orphan detectable after a crash on a
|
|
1113
|
-
// machine whose pids this process cannot check (shared state directory).
|
|
1114
|
-
const beat = setInterval(() => {
|
|
1115
|
-
run.heartbeatAt = new Date(now()).toISOString()
|
|
1116
|
-
run.ownerPid = process.pid
|
|
1117
|
-
refreshUsage(run, rt).then(() => save(run)).catch(() => {})
|
|
1118
|
-
}, HEARTBEAT_MS)
|
|
1119
|
-
beat.unref?.()
|
|
1120
|
-
|
|
1121
|
-
try {
|
|
1122
|
-
ensureCounters(run)
|
|
1123
|
-
if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
|
|
1124
|
-
run.status = 'planning'
|
|
1125
|
-
await save(run)
|
|
1126
|
-
const plan = await makePlan(run, parent, rt)
|
|
1127
|
-
run.plan = { mode: plan.mode, subtasks: plan.subtasks }
|
|
1128
|
-
run.subtasks = plan.subtasks.map((entry, index) => ({
|
|
1129
|
-
index: index + 1,
|
|
1130
|
-
title: entry.title,
|
|
1131
|
-
prompt: entry.prompt,
|
|
1132
|
-
status: 'pending',
|
|
1133
|
-
attempts: 0,
|
|
1134
|
-
childId: null,
|
|
1135
|
-
stopReason: null,
|
|
1136
|
-
error: null,
|
|
1137
|
-
result: null,
|
|
1138
|
-
usage: null,
|
|
1139
|
-
startedAt: null,
|
|
1140
|
-
finishedAt: null,
|
|
1141
|
-
}))
|
|
1142
|
-
await save(run)
|
|
1143
|
-
}
|
|
1144
|
-
|
|
1145
|
-
const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
|
|
1146
|
-
if (open.length > 0) {
|
|
1147
|
-
run.status = 'running'
|
|
1148
|
-
await save(run)
|
|
1149
|
-
await pool(open, run.limits.maxParallel, subtask => runSubtask(run, subtask, parent, rt))
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
|
|
1153
|
-
run.status = 'aggregating'
|
|
1154
|
-
await save(run)
|
|
1155
|
-
run.report = await aggregate(run, parent, rt)
|
|
1156
|
-
} else if (!run.report) {
|
|
1157
|
-
run.report = deterministicJoin(run)
|
|
1158
|
-
}
|
|
1159
|
-
} catch (error) {
|
|
1160
|
-
run.error = error?.message ?? String(error)
|
|
1161
|
-
log(`run ${run.id} failed: ${run.error}`)
|
|
1162
|
-
// Stop any child still running; without this the run would end while its
|
|
1163
|
-
// subtasks keep burning tokens.
|
|
1164
|
-
rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
|
|
1165
|
-
} finally {
|
|
1166
|
-
clearTimeout(timer)
|
|
1167
|
-
clearInterval(cancelTimer)
|
|
1168
|
-
// The heartbeat keeps running until the terminal write lands: waking the
|
|
1169
|
-
// session can take a few seconds, and a frozen heartbeat during that
|
|
1170
|
-
// window would let another process treat a live run as orphaned.
|
|
1171
|
-
try {
|
|
1172
|
-
await settle(run, rt)
|
|
1173
|
-
} catch (error) {
|
|
1174
|
-
// Never leave a run active because finalizing it threw.
|
|
1175
|
-
log(`cannot finalize run ${run.id}: ${error?.message ?? error}`)
|
|
1176
|
-
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1177
|
-
run.status = 'failed'
|
|
1178
|
-
run.finishedAt = run.finishedAt ?? new Date(now()).toISOString()
|
|
1179
|
-
run.error = run.error ?? `finalizing the run failed: ${error?.message ?? error}`
|
|
1180
|
-
}
|
|
1181
|
-
try {
|
|
1182
|
-
await store.write(run)
|
|
1183
|
-
} catch (writeError) {
|
|
1184
|
-
log(`cannot persist the terminal state of run ${run.id}: ${writeError?.message ?? writeError}`)
|
|
1185
|
-
}
|
|
1186
|
-
await releaseRunLock(run.id)
|
|
1187
|
-
} finally {
|
|
1188
|
-
clearInterval(beat)
|
|
1189
|
-
}
|
|
1190
|
-
}
|
|
1191
|
-
}
|
|
1192
|
-
|
|
1193
|
-
function schedule(run, parent) {
|
|
1194
|
-
run.ownerPid = process.pid
|
|
1195
|
-
run.ownerHost = os.hostname()
|
|
1196
|
-
run.heartbeatAt = new Date(now()).toISOString()
|
|
1197
|
-
const rt = {
|
|
1198
|
-
controller: new AbortController(),
|
|
1199
|
-
activations: new Set(),
|
|
1200
|
-
waiters: 0,
|
|
1201
|
-
run,
|
|
1202
|
-
promise: null,
|
|
1203
|
-
abortCause: null,
|
|
1204
|
-
}
|
|
1205
|
-
runtime.set(run.id, rt)
|
|
1206
|
-
rt.promise = execute(run, parent, rt)
|
|
1207
|
-
.catch(async error => {
|
|
1208
|
-
log(`run ${run.id} crashed: ${error?.message ?? error}`)
|
|
1209
|
-
if (!ACTIVE_STATUSES.has(run.status)) return
|
|
1210
|
-
run.status = 'failed'
|
|
1211
|
-
run.finishedAt = new Date(now()).toISOString()
|
|
1212
|
-
run.error = error?.message ?? String(error)
|
|
1213
|
-
try {
|
|
1214
|
-
await store.write(run)
|
|
1215
|
-
} catch (writeError) {
|
|
1216
|
-
log(`cannot persist the crash state of run ${run.id}: ${writeError?.message ?? writeError}`)
|
|
1217
|
-
}
|
|
1218
|
-
await releaseRunLock(run.id)
|
|
1219
|
-
})
|
|
1220
|
-
.then(() => {
|
|
1221
|
-
// Finished runs are read from disk from now on; keeping them here would
|
|
1222
|
-
// grow these maps for the lifetime of the host.
|
|
1223
|
-
runtime.delete(run.id)
|
|
1224
|
-
saveChains.delete(run.id)
|
|
1225
|
-
})
|
|
1226
|
-
return rt
|
|
1227
|
-
}
|
|
1228
|
-
|
|
1229
|
-
/** Serialize starts and resumes so a quota check cannot interleave. */
|
|
1230
|
-
function serialize(task) {
|
|
1231
|
-
const result = startGate.then(task, task)
|
|
1232
|
-
startGate = result.catch(() => {})
|
|
1233
|
-
return result
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
async function start({ goal, parentSessionId, parent, clientTimeZone, overrides = {} }) {
|
|
1237
|
-
const trimmed = typeof goal === 'string' ? goal.trim() : ''
|
|
1238
|
-
if (!trimmed) throw new Error('goal is required')
|
|
1239
|
-
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
1240
|
-
|
|
1241
|
-
const limits = buildLimits(overrides)
|
|
1242
|
-
// Kept so a resume can rebuild the limits from the *current* config
|
|
1243
|
-
// instead of freezing whatever was set when the run first started.
|
|
1244
|
-
const keptOverrides = {}
|
|
1245
|
-
for (const key of ['maxSubtasks', 'maxParallel', 'deadlineMinutes']) {
|
|
1246
|
-
if (overrides[key] !== undefined && overrides[key] !== null) keptOverrides[key] = overrides[key]
|
|
1247
|
-
}
|
|
1248
|
-
return serialize(async () => {
|
|
1249
|
-
const active = await activeCount()
|
|
1250
|
-
if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
|
|
1251
|
-
const run = {
|
|
1252
|
-
id: newRunId(),
|
|
1253
|
-
schemaVersion: 1,
|
|
1254
|
-
goal: trimmed,
|
|
1255
|
-
status: 'queued',
|
|
1256
|
-
mode: 'planned',
|
|
1257
|
-
parentSessionId,
|
|
1258
|
-
clientTimeZone: clientTimeZone ?? null,
|
|
1259
|
-
createdAt: new Date(now()).toISOString(),
|
|
1260
|
-
updatedAt: null,
|
|
1261
|
-
startedAt: new Date(now()).toISOString(),
|
|
1262
|
-
deadlineAt: new Date(now() + limits.deadlineMinutes * 60000).toISOString(),
|
|
1263
|
-
finishedAt: null,
|
|
1264
|
-
resumedAt: null,
|
|
1265
|
-
limits,
|
|
1266
|
-
overrides: keptOverrides,
|
|
1267
|
-
plan: null,
|
|
1268
|
-
subtasks: [],
|
|
1269
|
-
steps: {},
|
|
1270
|
-
report: null,
|
|
1271
|
-
error: null,
|
|
1272
|
-
usage: null,
|
|
1273
|
-
notified: false,
|
|
1274
|
-
counters: { spawns: 0, unreadable: {} },
|
|
1275
|
-
usageByChild: {},
|
|
1276
|
-
}
|
|
1277
|
-
await takeRunLock(run.id)
|
|
1278
|
-
try {
|
|
1279
|
-
await store.write(run)
|
|
1280
|
-
} catch (error) {
|
|
1281
|
-
await releaseRunLock(run.id)
|
|
1282
|
-
throw error
|
|
1283
|
-
}
|
|
1284
|
-
schedule(run, parent)
|
|
1285
|
-
return run
|
|
1286
|
-
})
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
async function resume(runId, parent, clientTimeZone) {
|
|
1290
|
-
const id = String(runId ?? '').trim()
|
|
1291
|
-
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
1292
|
-
return serialize(async () => {
|
|
1293
|
-
if (runtime.has(id)) {
|
|
1294
|
-
throw new Error(`run ${id} is still active in this process; wait for it with superagent_status or cancel it`)
|
|
1295
|
-
}
|
|
1296
|
-
let run = await loadRun(id)
|
|
1297
|
-
if (run.status === 'completed') return run
|
|
1298
|
-
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1299
|
-
const verdict = await ownership(run)
|
|
1300
|
-
if (verdict.live) {
|
|
1301
|
-
throw new Error(
|
|
1302
|
-
`run ${id} is still owned by a live process (${verdict.basis}); resuming it here would run the same `
|
|
1303
|
-
+ 'subtasks twice and double the spend. Cancel it with superagent_cancel first, or wait for it to finish.',
|
|
1304
|
-
)
|
|
1305
|
-
}
|
|
1306
|
-
}
|
|
1307
|
-
|
|
1308
|
-
await takeRunLock(id)
|
|
1309
|
-
let scheduled = false
|
|
1310
|
-
try {
|
|
1311
|
-
// Re-read under the lock: another process may have started the run
|
|
1312
|
-
// between the check above and the claim.
|
|
1313
|
-
run = await loadRun(id)
|
|
1314
|
-
if (run.status === 'completed') return run
|
|
1315
|
-
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1316
|
-
const verdict = await ownership(run)
|
|
1317
|
-
if (verdict.live) {
|
|
1318
|
-
throw new Error(`run ${id} is still owned by a live process (${verdict.basis}); not resuming it here`)
|
|
1319
|
-
}
|
|
1320
|
-
reclaim(run, verdict, await hasCancelRequest(id))
|
|
1321
|
-
await save(run)
|
|
1322
|
-
await clearCancel(id)
|
|
1323
|
-
}
|
|
1324
|
-
|
|
1325
|
-
const active = await activeCount()
|
|
1326
|
-
if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
|
|
1327
|
-
|
|
1328
|
-
for (const subtask of run.subtasks ?? []) {
|
|
1329
|
-
if (subtask.status === 'completed') continue
|
|
1330
|
-
subtask.status = 'pending'
|
|
1331
|
-
subtask.attempts = 0
|
|
1332
|
-
subtask.error = null
|
|
1333
|
-
subtask.stopReason = null
|
|
1334
|
-
subtask.finishedAt = null
|
|
1335
|
-
}
|
|
1336
|
-
run.status = 'queued'
|
|
1337
|
-
run.report = run.plan ? run.report : null
|
|
1338
|
-
run.error = null
|
|
1339
|
-
run.finishedAt = null
|
|
1340
|
-
run.reclaimed = null
|
|
1341
|
-
run.cancelPending = false
|
|
1342
|
-
// A resumed run completes a second time, so it must be able to wake the
|
|
1343
|
-
// session again; keeping the old outcome would silence it forever.
|
|
1344
|
-
run.notified = false
|
|
1345
|
-
run.notify = null
|
|
1346
|
-
run.resumedAt = new Date(now()).toISOString()
|
|
1347
|
-
run.clientTimeZone = clientTimeZone ?? run.clientTimeZone ?? null
|
|
1348
|
-
// The limits are rebuilt from the config that is active now, keeping
|
|
1349
|
-
// everything already spent: that is what makes "raise the cap, then
|
|
1350
|
-
// resume" work. Without this the old caps stayed in force forever.
|
|
1351
|
-
run.limits = buildLimits(run.overrides ?? {})
|
|
1352
|
-
run.deadlineAt = new Date(now() + run.limits.deadlineMinutes * 60000).toISOString()
|
|
1353
|
-
ensureCounters(run)
|
|
1354
|
-
await save(run)
|
|
1355
|
-
schedule(run, parent)
|
|
1356
|
-
scheduled = true
|
|
1357
|
-
return run
|
|
1358
|
-
} finally {
|
|
1359
|
-
if (!scheduled) await releaseRunLock(id)
|
|
1360
|
-
}
|
|
1361
|
-
})
|
|
1362
|
-
}
|
|
1363
|
-
|
|
1364
|
-
async function cancel(runId) {
|
|
1365
|
-
const rt = runtime.get(runId)
|
|
1366
|
-
if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
|
|
1367
|
-
rt.abortCause = 'cancelled'
|
|
1368
|
-
rt.controller.abort(new Error('cancelled by superagent_cancel'))
|
|
1369
|
-
const activations = [...rt.activations]
|
|
1370
|
-
// A child that ignores its abort must not hold this call open; the
|
|
1371
|
-
// timeout is per child and they are disposed in parallel.
|
|
1372
|
-
await Promise.all(activations.map(handle => disposeQuietly(handle)))
|
|
1373
|
-
await Promise.race([rt.promise ?? Promise.resolve(), sleep(CANCEL_ACK_MS)])
|
|
1374
|
-
if (ACTIVE_STATUSES.has(rt.run.status)) {
|
|
1375
|
-
rt.run.cancelPending = true
|
|
1376
|
-
await save(rt.run).catch(() => {})
|
|
1377
|
-
}
|
|
1378
|
-
return rt.run
|
|
1379
|
-
}
|
|
1380
|
-
|
|
1381
|
-
const run = await loadRun(runId)
|
|
1382
|
-
if (!ACTIVE_STATUSES.has(run.status)) return run
|
|
1383
|
-
|
|
1384
|
-
// This process does not own the run, so it cannot abort the children. Ask
|
|
1385
|
-
// the owner through the marker file and wait for it to rewrite the state.
|
|
1386
|
-
// Writing `cancelled` here would be a lie: the owner would overwrite it on
|
|
1387
|
-
// its next save and keep spending.
|
|
1388
|
-
await requestCancel(runId)
|
|
1389
|
-
const endsAt = now() + CANCEL_ACK_MS
|
|
1390
|
-
while (now() < endsAt) {
|
|
1391
|
-
await sleep(STATUS_POLL_MS)
|
|
1392
|
-
const current = await store.read(runId)
|
|
1393
|
-
if (!current) return run
|
|
1394
|
-
if (!ACTIVE_STATUSES.has(current.status)) return current
|
|
1395
|
-
}
|
|
1396
|
-
// The owner did not confirm in time. Do not rewrite the state file from
|
|
1397
|
-
// here: the owner would overwrite it on its next save. The request marker
|
|
1398
|
-
// stays on disk, and `status` reports it as a pending cancel.
|
|
1399
|
-
return { ...run, cancelPending: true }
|
|
1400
|
-
}
|
|
1401
|
-
|
|
1402
|
-
async function status(runId, waitSeconds = 0) {
|
|
1403
|
-
const read = async () => liveRun(runId) ?? await store.read(runId)
|
|
1404
|
-
let run = await read()
|
|
1405
|
-
if (!run) {
|
|
1406
|
-
await loadRun(runId)
|
|
1407
|
-
throw new Error(`unknown run ${runId}`)
|
|
1408
|
-
}
|
|
1409
|
-
// A terminal status is only published after the run's final write, so a
|
|
1410
|
-
// caller that sees one gets the notify row and usage totals with it.
|
|
1411
|
-
const owner = runtime.get(runId)
|
|
1412
|
-
if (owner && TERMINAL_STATUSES.has(run.status)) {
|
|
1413
|
-
await withTimeout(Promise.resolve(owner.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
|
|
1414
|
-
run = (await read()) ?? run
|
|
1415
|
-
}
|
|
1416
|
-
const limit = clamp(waitSeconds, 0, 3600)
|
|
1417
|
-
if (ACTIVE_STATUSES.has(run.status) && !run.cancelPending && await hasCancelRequest(runId)) {
|
|
1418
|
-
// The owning process may live elsewhere; the marker is the only evidence
|
|
1419
|
-
// that a cancel was asked for and not yet acknowledged.
|
|
1420
|
-
run = { ...run, cancelPending: true }
|
|
1421
|
-
}
|
|
1422
|
-
if (limit <= 0 || TERMINAL_STATUSES.has(run.status)) return run
|
|
1423
|
-
|
|
1424
|
-
const rt = runtime.get(runId)
|
|
1425
|
-
if (rt) rt.waiters++
|
|
1426
|
-
try {
|
|
1427
|
-
const endsAt = now() + limit * 1000
|
|
1428
|
-
while (now() < endsAt) {
|
|
1429
|
-
await sleep(STATUS_POLL_MS)
|
|
1430
|
-
run = await read()
|
|
1431
|
-
if (!run || TERMINAL_STATUSES.has(run.status)) break
|
|
1432
|
-
}
|
|
1433
|
-
} finally {
|
|
1434
|
-
if (rt) rt.waiters = Math.max(0, rt.waiters - 1)
|
|
1435
|
-
}
|
|
1436
|
-
|
|
1437
|
-
if (!run) throw new Error(`run ${runId} disappeared from the state directory while waiting for it`)
|
|
1438
|
-
const live = runtime.get(runId)
|
|
1439
|
-
// Wait for this process's own final write, so a caller that sees a terminal
|
|
1440
|
-
// status can immediately trust what is on disk. Bounded, because finalizing
|
|
1441
|
-
// includes waking the session and must not turn a status call into a hang.
|
|
1442
|
-
if (live && TERMINAL_STATUSES.has(live.run.status)) {
|
|
1443
|
-
await withTimeout(Promise.resolve(live.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
|
|
1444
|
-
}
|
|
1445
|
-
return liveRun(runId) ?? run
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
async function list() {
|
|
1449
|
-
const merged = new Map()
|
|
1450
|
-
for (const run of await store.list()) merged.set(run.id, run)
|
|
1451
|
-
for (const [runId, rt] of runtime) merged.set(runId, rt.run)
|
|
1452
|
-
return [...merged.values()].sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
|
|
1453
|
-
}
|
|
1454
|
-
|
|
1455
|
-
/**
|
|
1456
|
-
* Resolve once every run started by this process has finished. Used by tests
|
|
1457
|
-
* instead of sleeping, and by embedders that need a quiescent state.
|
|
1458
|
-
*/
|
|
1459
|
-
async function idle() {
|
|
1460
|
-
await Promise.all([...runtime.values()].map(rt => rt.promise?.catch(() => {})))
|
|
1461
|
-
}
|
|
1462
|
-
|
|
1463
|
-
return { recover, start, resume, cancel, status, list, idle, activeCount, liveRun }
|
|
1464
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Run orchestration for dsh-superagent.
|
|
3
|
+
*
|
|
4
|
+
* A run moves through queued -> planning -> running -> aggregating and ends in
|
|
5
|
+
* completed, partial, failed, cancelled, or interrupted. Planning, subtask
|
|
6
|
+
* execution, and aggregation all happen in child agents; this module owns
|
|
7
|
+
* ordering, retries, quotas, cancellation, and durable status.
|
|
8
|
+
*
|
|
9
|
+
* A state directory can be shared by more than one host process (two profiles,
|
|
10
|
+
* or a synced folder), so every decision about an active run is made from
|
|
11
|
+
* persisted evidence — heartbeat, owner pid, and the run lock — and never from
|
|
12
|
+
* this process's memory alone.
|
|
13
|
+
*
|
|
14
|
+
* The module has no host imports: `spawnChild`, `readUsage`, `notify`, and
|
|
15
|
+
* `store` are injected, which keeps it testable without a running harness.
|
|
16
|
+
*
|
|
17
|
+
* @module dsh-superagent/orchestrator
|
|
18
|
+
*/
|
|
19
|
+
import { randomBytes } from 'node:crypto'
|
|
20
|
+
import os from 'node:os'
|
|
21
|
+
import { ACTIVE_STATUSES, TERMINAL_STATUSES } from './store.js'
|
|
22
|
+
|
|
23
|
+
const STATUS_POLL_MS = 250
|
|
24
|
+
/** How often the owning process checks for a cancel marker written elsewhere. */
|
|
25
|
+
const CANCEL_POLL_MS = 1000
|
|
26
|
+
/** How long a cancel from another process waits for the owner to acknowledge. */
|
|
27
|
+
const CANCEL_ACK_MS = 5000
|
|
28
|
+
/** How often a live run refreshes its heartbeat in the state file. */
|
|
29
|
+
const HEARTBEAT_MS = 30 * 1000
|
|
30
|
+
/** A run with no heartbeat for this long is treated as orphaned. */
|
|
31
|
+
const ORPHAN_MS = 3 * HEARTBEAT_MS
|
|
32
|
+
/** A child that ignores abort must not hold a cancel call open forever. */
|
|
33
|
+
const DISPOSE_MS = 1500
|
|
34
|
+
/** Waking the owning session must not hold a finished run open forever. */
|
|
35
|
+
const NOTIFY_MS = 15 * 1000
|
|
36
|
+
/** A run lock older than this belongs to a process that never released it. */
|
|
37
|
+
const LOCK_STALE_MS = 10 * 60 * 1000
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Mint a sortable, filesystem-safe run id.
|
|
41
|
+
* @returns {string} run id such as `20261114T051233-a1b2`.
|
|
42
|
+
*/
|
|
43
|
+
export function newRunId() {
|
|
44
|
+
const stamp = new Date().toISOString().slice(0, 19).replace(/[-:]/g, '')
|
|
45
|
+
return `${stamp}-${randomBytes(2).toString('hex')}`
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function clamp(value, min, max) {
|
|
49
|
+
const number = Number(value)
|
|
50
|
+
if (!Number.isFinite(number)) return min
|
|
51
|
+
return Math.min(max, Math.max(min, Math.trunc(number)))
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Join the text blocks of a child result. */
|
|
55
|
+
export function textOf(output) {
|
|
56
|
+
if (!Array.isArray(output)) return ''
|
|
57
|
+
return output
|
|
58
|
+
.filter(block => block && block.type === 'text' && typeof block.text === 'string')
|
|
59
|
+
.map(block => block.text)
|
|
60
|
+
.join('\n')
|
|
61
|
+
.trim()
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Truncate a string to a UTF-8 byte budget without splitting a character. */
|
|
65
|
+
export function capText(text, maxBytes) {
|
|
66
|
+
if (typeof text !== 'string') return ''
|
|
67
|
+
const buffer = Buffer.from(text, 'utf8')
|
|
68
|
+
if (buffer.length <= maxBytes) return text
|
|
69
|
+
// Cutting at an arbitrary byte can land inside a multi-byte character; the
|
|
70
|
+
// decode then ends in U+FFFD. Drop that partial sequence instead of keeping
|
|
71
|
+
// a replacement character in the user's report.
|
|
72
|
+
const kept = buffer.subarray(0, maxBytes).toString('utf8').replace(/\uFFFD+$/, '')
|
|
73
|
+
const omitted = buffer.length - Buffer.byteLength(kept, 'utf8')
|
|
74
|
+
return `${kept}\n\n[truncated: ${omitted} bytes omitted]`
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Extract the first JSON object that carries a usable `subtasks` array.
|
|
79
|
+
*
|
|
80
|
+
* Models often wrap JSON in prose or code fences, so every balanced `{...}`
|
|
81
|
+
* region is tried rather than assuming the reply is bare JSON.
|
|
82
|
+
*
|
|
83
|
+
* @param {string} text - planner reply.
|
|
84
|
+
* @returns {Array<{ title: string, prompt: string }>} zero or more subtasks.
|
|
85
|
+
*/
|
|
86
|
+
export function extractPlan(text) {
|
|
87
|
+
if (typeof text !== 'string') return []
|
|
88
|
+
const candidates = []
|
|
89
|
+
for (let start = 0; start < text.length; start++) {
|
|
90
|
+
if (text[start] !== '{') continue
|
|
91
|
+
let depth = 0
|
|
92
|
+
let inString = false
|
|
93
|
+
let escaped = false
|
|
94
|
+
for (let index = start; index < text.length; index++) {
|
|
95
|
+
const char = text[index]
|
|
96
|
+
if (inString) {
|
|
97
|
+
if (escaped) escaped = false
|
|
98
|
+
else if (char === '\\') escaped = true
|
|
99
|
+
else if (char === '"') inString = false
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
if (char === '"') inString = true
|
|
103
|
+
else if (char === '{') depth++
|
|
104
|
+
else if (char === '}') {
|
|
105
|
+
depth--
|
|
106
|
+
if (depth === 0) {
|
|
107
|
+
candidates.push(text.slice(start, index + 1))
|
|
108
|
+
start = index
|
|
109
|
+
break
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
for (const candidate of candidates) {
|
|
115
|
+
let parsed
|
|
116
|
+
try {
|
|
117
|
+
parsed = JSON.parse(candidate)
|
|
118
|
+
} catch {
|
|
119
|
+
continue
|
|
120
|
+
}
|
|
121
|
+
const raw = parsed?.subtasks
|
|
122
|
+
if (!Array.isArray(raw)) continue
|
|
123
|
+
const subtasks = []
|
|
124
|
+
for (const item of raw) {
|
|
125
|
+
if (!item || typeof item !== 'object') continue
|
|
126
|
+
const prompt = typeof item.prompt === 'string' ? item.prompt.trim() : ''
|
|
127
|
+
if (!prompt) continue
|
|
128
|
+
const title = typeof item.title === 'string' && item.title.trim()
|
|
129
|
+
? item.title.trim().slice(0, 80)
|
|
130
|
+
: prompt.slice(0, 60)
|
|
131
|
+
subtasks.push({ title, prompt })
|
|
132
|
+
}
|
|
133
|
+
if (subtasks.length > 0) return subtasks
|
|
134
|
+
}
|
|
135
|
+
return []
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Planner prompt for one goal. */
|
|
139
|
+
function plannerPrompt(goal, maxSubtasks) {
|
|
140
|
+
return [
|
|
141
|
+
'You are the planner for a long-horizon autonomous run.',
|
|
142
|
+
'',
|
|
143
|
+
'Goal:',
|
|
144
|
+
goal,
|
|
145
|
+
'',
|
|
146
|
+
`Decompose the goal into at most ${maxSubtasks} subtasks that can run in parallel because none of them needs another subtask's output.`,
|
|
147
|
+
'Each subtask is executed by a separate agent that sees only the prompt you write for it.',
|
|
148
|
+
'',
|
|
149
|
+
'Return ONLY a JSON object, with no prose and no code fences, in exactly this shape:',
|
|
150
|
+
'{"subtasks":[{"title":"short title","prompt":"complete self-contained instruction"}]}',
|
|
151
|
+
'',
|
|
152
|
+
'Rules:',
|
|
153
|
+
`- Use 2 to ${maxSubtasks} subtasks; use 1 only when the goal truly cannot be split.`,
|
|
154
|
+
'- Every prompt must restate the context it needs; workers cannot see the goal text or each other.',
|
|
155
|
+
'- Prefer concrete research, construction, or verification work over vague summaries.',
|
|
156
|
+
].join('\n')
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Worker prompt for one subtask. */
|
|
160
|
+
function workerPrompt(run, subtask) {
|
|
161
|
+
return [
|
|
162
|
+
'You are one worker in a long-horizon autonomous run. Do not ask the user questions and do not wait for input.',
|
|
163
|
+
'',
|
|
164
|
+
'Overall goal:',
|
|
165
|
+
run.goal,
|
|
166
|
+
'',
|
|
167
|
+
`Your assigned subtask (${subtask.title}):`,
|
|
168
|
+
subtask.prompt,
|
|
169
|
+
'',
|
|
170
|
+
'Constraints:',
|
|
171
|
+
'- Work only on your subtask; do not attempt the whole goal.',
|
|
172
|
+
'- Use the tools you have. Never call a tool whose name starts with "superagent_".',
|
|
173
|
+
'- Your final message IS the deliverable. Do not save the report to a file and do not reply with a',
|
|
174
|
+
' summary of what you did; write the findings themselves: concrete facts, numbers, file paths,',
|
|
175
|
+
' commands, and anything unresolved. Under 1500 words, Markdown only.',
|
|
176
|
+
].join('\n')
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Aggregator prompt over completed subtask reports. */
|
|
180
|
+
function aggregatorPrompt(run, completed) {
|
|
181
|
+
const sections = completed.map(subtask => [
|
|
182
|
+
`### ${subtask.title}`,
|
|
183
|
+
capText(subtask.result ?? '', 8000),
|
|
184
|
+
].join('\n')).join('\n\n')
|
|
185
|
+
return [
|
|
186
|
+
'You are the aggregator of a long-horizon autonomous run.',
|
|
187
|
+
'',
|
|
188
|
+
'Goal:',
|
|
189
|
+
run.goal,
|
|
190
|
+
'',
|
|
191
|
+
'Subtask reports:',
|
|
192
|
+
sections,
|
|
193
|
+
'',
|
|
194
|
+
'Write the final report for the user in Markdown as your entire reply:',
|
|
195
|
+
'- Lead with the answer or outcome, then supporting detail. If the goal asks for a table, include the table.',
|
|
196
|
+
'- Merge duplicate findings; when reports disagree, say so and state which evidence is stronger.',
|
|
197
|
+
'- Call out what failed, what is unverified, and what the user should do next.',
|
|
198
|
+
'- Do not invent facts that no report contains. Do not say the report is ready — write it. Maximum 1200 words.',
|
|
199
|
+
].join('\n')
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Deterministic report used when no aggregator model call is available. */
|
|
203
|
+
function deterministicJoin(run) {
|
|
204
|
+
const completed = (run.subtasks ?? []).filter(subtask => subtask.status === 'completed')
|
|
205
|
+
if (completed.length === 0) return null
|
|
206
|
+
const parts = [`# ${run.goal}`, '', `_Deterministic merge of ${completed.length} subtask report(s)._`, '']
|
|
207
|
+
for (const subtask of completed) {
|
|
208
|
+
parts.push(`## ${subtask.title}`, '', subtask.result ?? '', '')
|
|
209
|
+
}
|
|
210
|
+
return parts.join('\n').trim()
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Run at most `limit` workers over `items` concurrently. */
|
|
214
|
+
async function pool(items, limit, worker) {
|
|
215
|
+
if (items.length === 0) return
|
|
216
|
+
let cursor = 0
|
|
217
|
+
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
218
|
+
while (cursor < items.length) {
|
|
219
|
+
const item = items[cursor++]
|
|
220
|
+
await worker(item)
|
|
221
|
+
}
|
|
222
|
+
})
|
|
223
|
+
await Promise.all(runners)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export function sumUsage(entries) {
|
|
227
|
+
const total = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, totalTokens: 0, modelCalls: 0, models: {} }
|
|
228
|
+
for (const usage of entries) {
|
|
229
|
+
if (!usage) continue
|
|
230
|
+
total.inputTokens += usage.inputTokens ?? 0
|
|
231
|
+
total.outputTokens += usage.outputTokens ?? 0
|
|
232
|
+
total.cacheReadTokens += usage.cacheReadTokens ?? 0
|
|
233
|
+
total.totalTokens += usage.totalTokens ?? 0
|
|
234
|
+
total.modelCalls += usage.modelCalls ?? 0
|
|
235
|
+
for (const [model, entry] of Object.entries(usage.models ?? {})) {
|
|
236
|
+
const target = total.models[model] ?? (total.models[model] = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, calls: 0 })
|
|
237
|
+
target.inputTokens += entry.inputTokens ?? 0
|
|
238
|
+
target.outputTokens += entry.outputTokens ?? 0
|
|
239
|
+
target.cacheReadTokens += entry.cacheReadTokens ?? 0
|
|
240
|
+
target.calls += entry.calls ?? 0
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return total
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Tokens billed for one usage record. */
|
|
247
|
+
function usageTokens(usage) {
|
|
248
|
+
return (usage?.inputTokens ?? 0) + (usage?.cacheReadTokens ?? 0) + (usage?.outputTokens ?? 0)
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Price recorded usage with an explicit, per-model table.
|
|
253
|
+
*
|
|
254
|
+
* Prices are per 1M tokens. Cache reads fall back to the input price when the
|
|
255
|
+
* table does not list a separate rate, which makes the estimate an upper bound
|
|
256
|
+
* instead of silently under-reporting. Models missing from the table are
|
|
257
|
+
* listed in `unpriced` and excluded from the total.
|
|
258
|
+
*
|
|
259
|
+
* @param {object | null} usage - summed usage for one run.
|
|
260
|
+
* @param {Record<string, { input: number, output: number, cacheRead?: number, currency?: string }> | undefined} pricing
|
|
261
|
+
* @returns {{ total: number, currency: string, byModel: Record<string, number>, unpriced: string[], complete: boolean } | null}
|
|
262
|
+
*/
|
|
263
|
+
export function computeCost(usage, pricing) {
|
|
264
|
+
if (!usage || !usage.modelCalls) return null
|
|
265
|
+
const table = pricing && typeof pricing === 'object' ? pricing : {}
|
|
266
|
+
const byModel = {}
|
|
267
|
+
const unpriced = []
|
|
268
|
+
let total = 0
|
|
269
|
+
let currency = null
|
|
270
|
+
for (const [model, entry] of Object.entries(usage.models ?? {})) {
|
|
271
|
+
// The usage log names a route, e.g. `dashscope/qwen-flash`. Accept a bare
|
|
272
|
+
// model id too, so a one-route config does not have to guess the prefix;
|
|
273
|
+
// an exact `provider/model` key always wins.
|
|
274
|
+
const price = table[model] ?? table[model.slice(model.indexOf('/') + 1)]
|
|
275
|
+
if (!price || typeof price.input !== 'number' || typeof price.output !== 'number') {
|
|
276
|
+
unpriced.push(model)
|
|
277
|
+
continue
|
|
278
|
+
}
|
|
279
|
+
const cacheRead = typeof price.cacheRead === 'number' ? price.cacheRead : price.input
|
|
280
|
+
const cost = (
|
|
281
|
+
entry.inputTokens * price.input
|
|
282
|
+
+ (entry.cacheReadTokens ?? 0) * cacheRead
|
|
283
|
+
+ entry.outputTokens * price.output
|
|
284
|
+
) / 1e6
|
|
285
|
+
byModel[model] = cost
|
|
286
|
+
total += cost
|
|
287
|
+
currency = currency ?? price.currency ?? 'USD'
|
|
288
|
+
}
|
|
289
|
+
if (Object.keys(byModel).length === 0) return { total: 0, currency: currency ?? 'USD', byModel, unpriced, complete: false }
|
|
290
|
+
return { total, currency, byModel, unpriced, complete: unpriced.length === 0 }
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** Total tokens recorded so far for a run, used by the budget guard. */
|
|
294
|
+
function spentTokens(run) {
|
|
295
|
+
return usageTokens(run.usage)
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Model calls recorded so far for a run, used by the budget guard.
|
|
300
|
+
*
|
|
301
|
+
* Measured usage is the accurate figure, but it only exists once a child's
|
|
302
|
+
* session log can be read. A spawn is at least one model call, so the spawn
|
|
303
|
+
* count is a floor: when the log layout changes and every read returns null,
|
|
304
|
+
* the call budget still stops the run instead of silently never matching.
|
|
305
|
+
*/
|
|
306
|
+
function spentCalls(run) {
|
|
307
|
+
return Math.max(run.usage?.modelCalls ?? 0, run.counters?.spawns ?? 0)
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function ensureCounters(run) {
|
|
311
|
+
run.counters = { spawns: 0, unreadable: {}, ...(run.counters ?? {}) }
|
|
312
|
+
if (!run.counters.unreadable || typeof run.counters.unreadable !== 'object') run.counters.unreadable = {}
|
|
313
|
+
return run.counters
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Build the orchestrator.
|
|
318
|
+
*
|
|
319
|
+
* @param {object} deps - injected collaborators.
|
|
320
|
+
* @param {object} deps.store - run store from `createStore`.
|
|
321
|
+
* @param {object} deps.config - resolved plugin config.
|
|
322
|
+
* @param {(spec: { label: string, prompt: string, parent: object, signal: AbortSignal }) => Promise<{ childId: string, result: Promise<object>, dispose: () => Promise<void> }>} deps.spawnChild
|
|
323
|
+
* @param {(childId: string) => Promise<object | null>} deps.readUsage
|
|
324
|
+
* @param {(run: object) => object | void | Promise<object | void>} deps.notify - wake the owning session once.
|
|
325
|
+
* @param {(message: string) => void} [deps.log]
|
|
326
|
+
* @param {() => number} [deps.now]
|
|
327
|
+
* @param {(ms: number) => Promise<void>} [deps.sleep]
|
|
328
|
+
*/
|
|
329
|
+
export function createOrchestrator(deps) {
|
|
330
|
+
const {
|
|
331
|
+
store,
|
|
332
|
+
config,
|
|
333
|
+
spawnChild,
|
|
334
|
+
readUsage,
|
|
335
|
+
notify,
|
|
336
|
+
log = () => {},
|
|
337
|
+
now = () => Date.now(),
|
|
338
|
+
sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
339
|
+
} = deps
|
|
340
|
+
|
|
341
|
+
/** @type {Map<string, { controller: AbortController, activations: Set<object>, waiters: number, run: object, promise: Promise<void> | null, abortCause: string | null }>} */
|
|
342
|
+
const runtime = new Map()
|
|
343
|
+
/** Per-run write chain so two subtasks finishing together cannot interleave renames. */
|
|
344
|
+
const saveChains = new Map()
|
|
345
|
+
let startGate = Promise.resolve()
|
|
346
|
+
|
|
347
|
+
// Cancel markers are optional in the store contract so tests can inject a
|
|
348
|
+
// memory store without them.
|
|
349
|
+
const hasCancelRequest = runId => (typeof store.hasCancelRequest === 'function'
|
|
350
|
+
? store.hasCancelRequest(runId)
|
|
351
|
+
: Promise.resolve(false))
|
|
352
|
+
const clearCancel = runId => (typeof store.clearCancel === 'function'
|
|
353
|
+
? store.clearCancel(runId)
|
|
354
|
+
: Promise.resolve())
|
|
355
|
+
const requestCancel = runId => (typeof store.requestCancel === 'function'
|
|
356
|
+
? store.requestCancel(runId)
|
|
357
|
+
: Promise.resolve(false))
|
|
358
|
+
|
|
359
|
+
function maxActiveRuns() {
|
|
360
|
+
return clamp(config.maxActiveRuns ?? 2, 1, 8)
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function save(run) {
|
|
364
|
+
run.updatedAt = new Date(now()).toISOString()
|
|
365
|
+
const previous = saveChains.get(run.id) ?? Promise.resolve()
|
|
366
|
+
const next = previous.catch(() => {}).then(() => store.write(run))
|
|
367
|
+
saveChains.set(run.id, next)
|
|
368
|
+
return next
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function liveRun(runId) {
|
|
372
|
+
return runtime.get(runId)?.run ?? null
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/** Cached machine identity of the state directory. */
|
|
376
|
+
let ownerRef = null
|
|
377
|
+
async function ownerIdentity() {
|
|
378
|
+
if (ownerRef) return ownerRef
|
|
379
|
+
const host = os.hostname()
|
|
380
|
+
const existing = typeof store.readOwner === 'function' ? await store.readOwner() : null
|
|
381
|
+
if (existing && typeof existing.hostname === 'string') {
|
|
382
|
+
ownerRef = { hostname: host, local: existing.hostname === host, ownerHostname: existing.hostname }
|
|
383
|
+
return ownerRef
|
|
384
|
+
}
|
|
385
|
+
if (typeof store.writeOwner === 'function') {
|
|
386
|
+
await store.writeOwner({ hostname: host, createdAt: new Date(now()).toISOString() })
|
|
387
|
+
}
|
|
388
|
+
ownerRef = { hostname: host, local: true, ownerHostname: host }
|
|
389
|
+
return ownerRef
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* @param {unknown} pid - pid recorded by the process that owned the run.
|
|
394
|
+
* @returns {boolean | null} true when alive, false when gone, null when unknown.
|
|
395
|
+
*/
|
|
396
|
+
function pidAlive(pid) {
|
|
397
|
+
const number = Number(pid)
|
|
398
|
+
if (!Number.isInteger(number) || number <= 0) return null
|
|
399
|
+
try {
|
|
400
|
+
process.kill(number, 0)
|
|
401
|
+
return true
|
|
402
|
+
} catch (error) {
|
|
403
|
+
// EPERM means the process exists but belongs to another user.
|
|
404
|
+
return error?.code === 'EPERM'
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Decide whether an active run still has a live owner.
|
|
410
|
+
*
|
|
411
|
+
* Used by recovery, orphan sweeping, and resume, so all three agree: a run
|
|
412
|
+
* that somebody can still be working on is never rewritten. The heartbeat
|
|
413
|
+
* alone proves ownership on a shared directory; a pid is only evidence when
|
|
414
|
+
* the state directory belongs to this machine. A live run lock counts as
|
|
415
|
+
* ownership because the holder is the process that may be spending.
|
|
416
|
+
*
|
|
417
|
+
* @param {object} run - persisted run record.
|
|
418
|
+
* @returns {Promise<{ live: boolean, basis: string, heartbeatAt: number, owner: object, pidChecked: boolean }>}
|
|
419
|
+
*/
|
|
420
|
+
async function ownership(run) {
|
|
421
|
+
const owner = await ownerIdentity()
|
|
422
|
+
const heartbeat = Date.parse(run.heartbeatAt ?? run.updatedAt ?? run.startedAt ?? run.createdAt)
|
|
423
|
+
const heartbeatIso = Number.isFinite(heartbeat) ? new Date(heartbeat).toISOString() : 'unknown'
|
|
424
|
+
const fresh = Number.isFinite(heartbeat) && heartbeat > now() - ORPHAN_MS
|
|
425
|
+
// A pid only means something when the run was written on this machine. On
|
|
426
|
+
// a shared directory the run may belong to a host whose pids are unrelated
|
|
427
|
+
// to ours, so the heartbeat is the only evidence there.
|
|
428
|
+
const localWriter = run.ownerHost ? run.ownerHost === os.hostname() : owner.local
|
|
429
|
+
const alive = localWriter ? pidAlive(run.ownerPid) : null
|
|
430
|
+
|
|
431
|
+
if (alive === true) {
|
|
432
|
+
return { live: true, basis: `owner pid ${run.ownerPid} is still running`, heartbeatAt: heartbeat, owner, pidChecked: true }
|
|
433
|
+
}
|
|
434
|
+
if (typeof store.readLock === 'function') {
|
|
435
|
+
const holder = await store.readLock(run.id).catch(() => null)
|
|
436
|
+
if (holder && holder.hostname === os.hostname() && pidAlive(holder.pid) === true) {
|
|
437
|
+
return { live: true, basis: `pid ${holder.pid} holds the run lock`, heartbeatAt: heartbeat, owner, pidChecked: true }
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
// A heartbeat can be a few seconds old when the host is killed, and waiting
|
|
441
|
+
// 90 s for it to expire would strand a run that this machine can prove is
|
|
442
|
+
// dead. On the same machine, a missing pid outranks a fresh timestamp.
|
|
443
|
+
if (alive === false) {
|
|
444
|
+
return {
|
|
445
|
+
live: false,
|
|
446
|
+
basis: `owner pid ${run.ownerPid} is not running and this state directory is local`,
|
|
447
|
+
heartbeatAt: heartbeat,
|
|
448
|
+
owner,
|
|
449
|
+
pidChecked: true,
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
if (fresh) {
|
|
453
|
+
return { live: true, basis: `its heartbeat was refreshed at ${heartbeatIso}`, heartbeatAt: heartbeat, owner, pidChecked: false }
|
|
454
|
+
}
|
|
455
|
+
const basis = owner.local
|
|
456
|
+
? `owner pid ${run.ownerPid ?? 'unknown'} is not running and its heartbeat stopped at ${heartbeatIso}`
|
|
457
|
+
: `the state directory is shared with ${owner.ownerHostname}, so a local pid proves nothing; the heartbeat stopped at ${heartbeatIso}`
|
|
458
|
+
return { live: false, basis, heartbeatAt: heartbeat, owner, pidChecked: owner.local }
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Rewrite a run whose owner is gone. Never silent: the exact evidence is
|
|
463
|
+
* stored on the run and printed by status.
|
|
464
|
+
*/
|
|
465
|
+
function reclaim(run, verdict, cancelled = false) {
|
|
466
|
+
run.status = cancelled ? 'cancelled' : 'interrupted'
|
|
467
|
+
run.error = cancelled
|
|
468
|
+
? 'cancelled by superagent_cancel while no process was running it'
|
|
469
|
+
: 'host process stopped while the run was active'
|
|
470
|
+
run.finishedAt = new Date(now()).toISOString()
|
|
471
|
+
run.reclaimed = {
|
|
472
|
+
at: new Date(now()).toISOString(),
|
|
473
|
+
basis: verdict.basis,
|
|
474
|
+
heartbeatAt: Number.isFinite(verdict.heartbeatAt) ? new Date(verdict.heartbeatAt).toISOString() : null,
|
|
475
|
+
ownerPid: run.ownerPid ?? null,
|
|
476
|
+
pidChecked: verdict.pidChecked,
|
|
477
|
+
sharedWith: verdict.owner.local ? null : verdict.owner.ownerHostname,
|
|
478
|
+
}
|
|
479
|
+
for (const subtask of run.subtasks ?? []) {
|
|
480
|
+
if (subtask.status === 'running') subtask.status = 'pending'
|
|
481
|
+
}
|
|
482
|
+
// The owner died before it could settle, so publish what is already known
|
|
483
|
+
// about the spend: a reclaimed run must not claim its cost is unknown when
|
|
484
|
+
// its child logs were readable.
|
|
485
|
+
const entries = Object.values(run.usageByChild ?? {})
|
|
486
|
+
if (entries.length > 0) run.usage = sumUsage(entries)
|
|
487
|
+
run.cost = computeCost(run.usage, config.pricing)
|
|
488
|
+
for (const subtask of run.subtasks ?? []) {
|
|
489
|
+
if (subtask.childId && run.usageByChild?.[subtask.childId]) {
|
|
490
|
+
subtask.usage = run.usageByChild[subtask.childId]
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
async function releaseRunLock(runId) {
|
|
496
|
+
if (typeof store.releaseLock !== 'function') return
|
|
497
|
+
try {
|
|
498
|
+
await store.releaseLock(runId)
|
|
499
|
+
} catch (error) {
|
|
500
|
+
log(`cannot release the lock for run ${runId}: ${error?.message ?? error}`)
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* Claim the exclusive right to schedule a run.
|
|
506
|
+
*
|
|
507
|
+
* Two profiles can point at one state directory, so a resume must not be
|
|
508
|
+
* decided from a read alone: both processes would see a terminal run and both
|
|
509
|
+
* would schedule it. The lock makes one of them lose.
|
|
510
|
+
*/
|
|
511
|
+
async function takeRunLock(runId) {
|
|
512
|
+
if (typeof store.acquireLock !== 'function') return
|
|
513
|
+
const holder = { pid: process.pid, hostname: os.hostname(), at: new Date(now()).toISOString() }
|
|
514
|
+
let attempt = await store.acquireLock(runId, holder)
|
|
515
|
+
if (!attempt.acquired) {
|
|
516
|
+
const current = attempt.holder
|
|
517
|
+
// A lock left by a process on this machine that no longer exists is
|
|
518
|
+
// evidence of a crash, not of a live owner. A lock on another machine has
|
|
519
|
+
// no checkable pid, so its age is the only evidence: past LOCK_STALE_MS
|
|
520
|
+
// the process that took it is no longer plausibly still scheduling.
|
|
521
|
+
const deadHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === false
|
|
522
|
+
const aliveHere = current && current.hostname === os.hostname() && pidAlive(current.pid) === true
|
|
523
|
+
const age = now() - Date.parse(current?.at ?? '')
|
|
524
|
+
const stale = !Number.isFinite(age) || age > LOCK_STALE_MS
|
|
525
|
+
// Age is only evidence when the holder cannot be checked: a live process
|
|
526
|
+
// on this machine may legitimately schedule a long run for hours.
|
|
527
|
+
if (deadHere || (stale && !aliveHere)) {
|
|
528
|
+
await releaseRunLock(runId)
|
|
529
|
+
attempt = await store.acquireLock(runId, holder)
|
|
530
|
+
}
|
|
531
|
+
if (!attempt.acquired) {
|
|
532
|
+
const held = attempt.holder
|
|
533
|
+
const who = held
|
|
534
|
+
? `pid ${held.pid ?? 'unknown'} on ${held.hostname ?? 'an unknown host'}`
|
|
535
|
+
: 'another process'
|
|
536
|
+
throw new Error(`run ${runId} is already being started by ${who}; wait for it to finish or cancel it`)
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/**
|
|
542
|
+
* Reclaim runs whose owning process is gone, so an orphan cannot hold the
|
|
543
|
+
* active-run quota forever.
|
|
544
|
+
*
|
|
545
|
+
* @returns {Promise<string[]>} reclaimed run ids, in list order.
|
|
546
|
+
*/
|
|
547
|
+
async function sweepOrphans() {
|
|
548
|
+
const stale = (await store.list()).filter(run => ACTIVE_STATUSES.has(run.status) && !runtime.has(run.id))
|
|
549
|
+
if (stale.length === 0) return []
|
|
550
|
+
const reclaimed = []
|
|
551
|
+
for (const run of stale) {
|
|
552
|
+
const verdict = await ownership(run)
|
|
553
|
+
if (verdict.live) continue
|
|
554
|
+
reclaim(run, verdict)
|
|
555
|
+
await save(run)
|
|
556
|
+
await releaseRunLock(run.id)
|
|
557
|
+
log(`run ${run.id} was active with no live owner; reclaimed its quota (${verdict.basis})`)
|
|
558
|
+
reclaimed.push(run.id)
|
|
559
|
+
}
|
|
560
|
+
return reclaimed
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
async function activeCount() {
|
|
564
|
+
try {
|
|
565
|
+
await sweepOrphans()
|
|
566
|
+
} catch (error) {
|
|
567
|
+
log(`cannot check for orphaned runs: ${error?.message ?? error}`)
|
|
568
|
+
}
|
|
569
|
+
let count = 0
|
|
570
|
+
for (const rt of runtime.values()) {
|
|
571
|
+
if (rt.run && ACTIVE_STATUSES.has(rt.run.status)) count++
|
|
572
|
+
}
|
|
573
|
+
for (const run of await store.list()) {
|
|
574
|
+
if (runtime.has(run.id)) continue
|
|
575
|
+
if (ACTIVE_STATUSES.has(run.status)) count++
|
|
576
|
+
}
|
|
577
|
+
return count
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
function quotaMessage(active) {
|
|
581
|
+
return `quota exceeded: ${active} active run(s) and maxActiveRuns is ${maxActiveRuns()}. `
|
|
582
|
+
+ 'Let one finish with superagent_status or cancel it with superagent_cancel.'
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
function buildLimits(overrides = {}) {
|
|
586
|
+
// A caller may narrow a limit but never widen the operator's cap: the
|
|
587
|
+
// config is the spending and blast-radius control, and a model-written tool
|
|
588
|
+
// argument must not be able to raise it.
|
|
589
|
+
const narrowed = (override, cap) => (
|
|
590
|
+
override === undefined || override === null ? cap : Math.min(Number(override), cap)
|
|
591
|
+
)
|
|
592
|
+
const limits = {
|
|
593
|
+
maxSubtasks: clamp(narrowed(overrides.maxSubtasks, config.maxSubtasks), 1, 16),
|
|
594
|
+
maxParallel: clamp(narrowed(overrides.maxParallel, config.maxParallel), 1, 8),
|
|
595
|
+
deadlineMinutes: clamp(narrowed(overrides.deadlineMinutes, config.deadlineMinutes), 1, 720),
|
|
596
|
+
attempts: clamp(config.attempts, 1, 5),
|
|
597
|
+
maxModelCalls: clamp(narrowed(overrides.maxModelCalls, config.maxModelCallsPerRun), 0, 10000),
|
|
598
|
+
maxTokens: clamp(narrowed(overrides.maxTokens, config.maxTokensPerRun), 0, 1e12),
|
|
599
|
+
}
|
|
600
|
+
limits.maxParallel = Math.min(limits.maxParallel, limits.maxSubtasks)
|
|
601
|
+
return limits
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Report why a run may not start another child, or null when it still can.
|
|
606
|
+
* Both budgets are hard stops: they exist so a pathological goal cannot spend
|
|
607
|
+
* quietly, so they are checked before a spawn rather than after.
|
|
608
|
+
*/
|
|
609
|
+
function budgetStop(run) {
|
|
610
|
+
const calls = spentCalls(run)
|
|
611
|
+
const tokens = spentTokens(run)
|
|
612
|
+
if (run.limits.maxModelCalls > 0 && calls >= run.limits.maxModelCalls) {
|
|
613
|
+
return `model-call budget reached (${calls}/${run.limits.maxModelCalls} calls)`
|
|
614
|
+
}
|
|
615
|
+
if (run.limits.maxTokens > 0 && tokens >= run.limits.maxTokens) {
|
|
616
|
+
return `token budget reached (${tokens}/${run.limits.maxTokens} tokens)`
|
|
617
|
+
}
|
|
618
|
+
return null
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* Spawn one child under the run's abort signal and budget.
|
|
623
|
+
* @param {object} run - live run record.
|
|
624
|
+
* @param {object} rt - runtime entry.
|
|
625
|
+
* @param {{ label: string, prompt: string, parent: object }} spec - child request.
|
|
626
|
+
*/
|
|
627
|
+
async function spawn(run, rt, spec) {
|
|
628
|
+
const blocked = budgetStop(run)
|
|
629
|
+
if (blocked) {
|
|
630
|
+
if (!rt.abortCause) rt.abortCause = 'budget'
|
|
631
|
+
if (!run.error) run.error = blocked
|
|
632
|
+
if (!rt.controller.signal.aborted) rt.controller.abort(new Error(blocked))
|
|
633
|
+
throw new Error(`budget exceeded: ${blocked}; raise maxModelCallsPerRun or maxTokensPerRun if this was intended`)
|
|
634
|
+
}
|
|
635
|
+
// Counted before the child exists and persisted before it can spend, so a
|
|
636
|
+
// crash cannot lose the only evidence of a model call.
|
|
637
|
+
ensureCounters(run).spawns += 1
|
|
638
|
+
await save(run)
|
|
639
|
+
return spawnAbortable(spec, rt.controller.signal)
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
/**
|
|
643
|
+
* Create one child, but stop waiting for it as soon as the run is aborted.
|
|
644
|
+
*
|
|
645
|
+
* A provider that queues activations can leave `startActivation` pending long
|
|
646
|
+
* after the run was cancelled or ran out of time. Without this the run would
|
|
647
|
+
* hang in planning/running while its heartbeat kept it looking alive, so no
|
|
648
|
+
* recovery path could ever reclaim it. A child that appears after the abort is
|
|
649
|
+
* disposed immediately instead of being left to spend.
|
|
650
|
+
*/
|
|
651
|
+
function spawnAbortable(spec, signal) {
|
|
652
|
+
const pending = spawnChild({ ...spec, signal })
|
|
653
|
+
if (!signal) return pending
|
|
654
|
+
if (signal.aborted) {
|
|
655
|
+
pending.then(handle => disposeQuietly(handle), () => {})
|
|
656
|
+
return Promise.reject(new Error('aborted'))
|
|
657
|
+
}
|
|
658
|
+
return new Promise((resolve, reject) => {
|
|
659
|
+
let done = false
|
|
660
|
+
const onAbort = () => {
|
|
661
|
+
if (done) return
|
|
662
|
+
done = true
|
|
663
|
+
// The `pending` handler registered below disposes the handle if it
|
|
664
|
+
// arrives late, so the child is not left spending in the background.
|
|
665
|
+
reject(new Error('aborted'))
|
|
666
|
+
}
|
|
667
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
668
|
+
pending.then(
|
|
669
|
+
handle => {
|
|
670
|
+
if (done) {
|
|
671
|
+
disposeQuietly(handle)
|
|
672
|
+
return
|
|
673
|
+
}
|
|
674
|
+
done = true
|
|
675
|
+
signal.removeEventListener('abort', onAbort)
|
|
676
|
+
resolve(handle)
|
|
677
|
+
},
|
|
678
|
+
error => {
|
|
679
|
+
if (done) return
|
|
680
|
+
done = true
|
|
681
|
+
signal.removeEventListener('abort', onAbort)
|
|
682
|
+
reject(error)
|
|
683
|
+
},
|
|
684
|
+
)
|
|
685
|
+
})
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
/**
|
|
689
|
+
* Read one run for a tool call, turning the two failure modes a user can hit
|
|
690
|
+
* into sentences instead of a null.
|
|
691
|
+
* @param {string} runId - run identity.
|
|
692
|
+
*/
|
|
693
|
+
async function loadRun(runId) {
|
|
694
|
+
const run = await store.read(runId)
|
|
695
|
+
if (run) return run
|
|
696
|
+
const quarantined = store.corrupt?.get?.(runId)
|
|
697
|
+
if (quarantined) {
|
|
698
|
+
throw new Error(
|
|
699
|
+
`state file for run ${runId} is not valid JSON, so the run cannot be read. `
|
|
700
|
+
+ `The original file was kept at ${quarantined}. Inspect it, then delete it; `
|
|
701
|
+
+ 'the run is skipped by superagent_list until you do.',
|
|
702
|
+
)
|
|
703
|
+
}
|
|
704
|
+
throw new Error(`unknown run ${runId}; call superagent_list to see recorded run ids`)
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/**
|
|
708
|
+
* Mark runs left active by a previous host process as interrupted.
|
|
709
|
+
*
|
|
710
|
+
* A run that another live process owns is left completely alone: rewriting it
|
|
711
|
+
* here would make two processes fight over the same file while the first one
|
|
712
|
+
* keeps spending, and a later resume would then run everything twice.
|
|
713
|
+
*/
|
|
714
|
+
async function recover() {
|
|
715
|
+
await store.init()
|
|
716
|
+
const recovered = []
|
|
717
|
+
for (const run of await store.list()) {
|
|
718
|
+
if (!ACTIVE_STATUSES.has(run.status)) {
|
|
719
|
+
if (await hasCancelRequest(run.id)) await clearCancel(run.id)
|
|
720
|
+
continue
|
|
721
|
+
}
|
|
722
|
+
if (runtime.has(run.id)) continue
|
|
723
|
+
const verdict = await ownership(run)
|
|
724
|
+
if (verdict.live) {
|
|
725
|
+
log(`run ${run.id} is still owned (${verdict.basis}); leaving it running`)
|
|
726
|
+
continue
|
|
727
|
+
}
|
|
728
|
+
reclaim(run, verdict, await hasCancelRequest(run.id))
|
|
729
|
+
await save(run)
|
|
730
|
+
await clearCancel(run.id)
|
|
731
|
+
await releaseRunLock(run.id)
|
|
732
|
+
recovered.push(run.id)
|
|
733
|
+
}
|
|
734
|
+
return recovered
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
/**
|
|
738
|
+
* Fold one child's usage snapshot into the run. The snapshot is cumulative,
|
|
739
|
+
* so the last read for a child replaces the earlier one instead of adding.
|
|
740
|
+
* Called at start, on every heartbeat, and on success or failure, which is
|
|
741
|
+
* why an unreadable log degrades to the spawn floor instead of to zero.
|
|
742
|
+
*/
|
|
743
|
+
async function recordUsage(run, childId) {
|
|
744
|
+
let usage = null
|
|
745
|
+
try {
|
|
746
|
+
usage = await readUsage(childId)
|
|
747
|
+
} catch (error) {
|
|
748
|
+
log(`cannot read usage for child ${childId}: ${error?.message ?? error}`)
|
|
749
|
+
}
|
|
750
|
+
if (!usage) {
|
|
751
|
+
ensureCounters(run).unreadable[childId] = true
|
|
752
|
+
return null
|
|
753
|
+
}
|
|
754
|
+
run.usageByChild = run.usageByChild ?? {}
|
|
755
|
+
run.usageByChild[childId] = usage
|
|
756
|
+
run.usage = sumUsage(Object.values(run.usageByChild))
|
|
757
|
+
return usage
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/** Refresh usage for children that are still running, from the heartbeat. */
|
|
761
|
+
async function refreshUsage(run, rt) {
|
|
762
|
+
const handles = [...rt.activations]
|
|
763
|
+
for (const handle of handles) await recordUsage(run, handle.childId)
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/**
|
|
767
|
+
* Wait for a child result, but stop waiting when the run is aborted. Without
|
|
768
|
+
* this a child that ignores its abort signal keeps the whole run open.
|
|
769
|
+
*/
|
|
770
|
+
function abortable(promise, signal) {
|
|
771
|
+
if (!signal) return promise
|
|
772
|
+
return new Promise((resolve, reject) => {
|
|
773
|
+
const onAbort = () => {
|
|
774
|
+
promise.catch(() => {})
|
|
775
|
+
reject(new Error('aborted'))
|
|
776
|
+
}
|
|
777
|
+
if (signal.aborted) {
|
|
778
|
+
onAbort()
|
|
779
|
+
return
|
|
780
|
+
}
|
|
781
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
782
|
+
promise.then(
|
|
783
|
+
value => {
|
|
784
|
+
signal.removeEventListener('abort', onAbort)
|
|
785
|
+
resolve(value)
|
|
786
|
+
},
|
|
787
|
+
error => {
|
|
788
|
+
signal.removeEventListener('abort', onAbort)
|
|
789
|
+
reject(error)
|
|
790
|
+
},
|
|
791
|
+
)
|
|
792
|
+
})
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** Dispose one child activation without letting it block the caller. */
|
|
796
|
+
async function disposeQuietly(handle) {
|
|
797
|
+
if (!handle || typeof handle.dispose !== 'function') return
|
|
798
|
+
try {
|
|
799
|
+
await Promise.race([
|
|
800
|
+
Promise.resolve().then(() => handle.dispose()),
|
|
801
|
+
sleep(DISPOSE_MS),
|
|
802
|
+
])
|
|
803
|
+
} catch (error) {
|
|
804
|
+
log(`child dispose failed: ${error?.message ?? error}`)
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
/**
|
|
809
|
+
* The host's subagent service disappeared while this run was active, which
|
|
810
|
+
* in practice means the host is shutting down. That is not a subtask
|
|
811
|
+
* failure: the work never got a chance to run. Remember the cause so the run
|
|
812
|
+
* settles as `interrupted` (resumable) instead of `failed`, and abort the
|
|
813
|
+
* remaining children.
|
|
814
|
+
*/
|
|
815
|
+
function markHostGone(rt, error) {
|
|
816
|
+
if (rt.abortCause) return
|
|
817
|
+
rt.abortCause = 'host-gone'
|
|
818
|
+
if (!rt.controller.signal.aborted) {
|
|
819
|
+
rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
async function makePlan(run, parent, rt) {
|
|
824
|
+
try {
|
|
825
|
+
const handle = await spawn(run, rt, {
|
|
826
|
+
label: 'superagent-plan',
|
|
827
|
+
prompt: plannerPrompt(run.goal, run.limits.maxSubtasks),
|
|
828
|
+
parent,
|
|
829
|
+
})
|
|
830
|
+
rt.activations.add(handle)
|
|
831
|
+
run.steps.planner = { childId: handle.childId }
|
|
832
|
+
try {
|
|
833
|
+
const result = await abortable(handle.result, rt.controller.signal)
|
|
834
|
+
const text = textOf(result.output)
|
|
835
|
+
const subtasks = extractPlan(text).slice(0, run.limits.maxSubtasks)
|
|
836
|
+
if (subtasks.length > 0) return { mode: 'planned', subtasks }
|
|
837
|
+
log(`planner reply had no parsable plan; running the goal as one subtask`)
|
|
838
|
+
} finally {
|
|
839
|
+
await recordUsage(run, handle.childId)
|
|
840
|
+
rt.activations.delete(handle)
|
|
841
|
+
await disposeQuietly(handle)
|
|
842
|
+
}
|
|
843
|
+
} catch (error) {
|
|
844
|
+
log(`planner failed: ${error?.message ?? error}`)
|
|
845
|
+
if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
|
|
846
|
+
}
|
|
847
|
+
return { mode: 'degraded', subtasks: [{ title: 'Complete the goal', prompt: run.goal }] }
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
async function runSubtask(run, subtask, parent, rt) {
|
|
851
|
+
const signal = rt.controller.signal
|
|
852
|
+
for (let attempt = subtask.attempts + 1; attempt <= run.limits.attempts; attempt++) {
|
|
853
|
+
if (signal.aborted) break
|
|
854
|
+
subtask.status = 'running'
|
|
855
|
+
subtask.attempts = attempt
|
|
856
|
+
subtask.startedAt = subtask.startedAt ?? new Date(now()).toISOString()
|
|
857
|
+
await save(run)
|
|
858
|
+
|
|
859
|
+
let handle = null
|
|
860
|
+
try {
|
|
861
|
+
handle = await spawn(run, rt, {
|
|
862
|
+
label: `superagent-${subtask.index}`,
|
|
863
|
+
prompt: workerPrompt(run, subtask),
|
|
864
|
+
parent,
|
|
865
|
+
})
|
|
866
|
+
rt.activations.add(handle)
|
|
867
|
+
subtask.childId = handle.childId
|
|
868
|
+
await save(run)
|
|
869
|
+
|
|
870
|
+
const result = await abortable(handle.result, signal)
|
|
871
|
+
const text = textOf(result.output)
|
|
872
|
+
subtask.stopReason = result.stopReason
|
|
873
|
+
if (result.stopReason === 'completed' && text) {
|
|
874
|
+
subtask.status = 'completed'
|
|
875
|
+
subtask.result = capText(text, config.resultCapBytes)
|
|
876
|
+
subtask.error = null
|
|
877
|
+
subtask.finishedAt = new Date(now()).toISOString()
|
|
878
|
+
await recordUsage(run, handle.childId)
|
|
879
|
+
await save(run)
|
|
880
|
+
return
|
|
881
|
+
}
|
|
882
|
+
subtask.error = result.diagnostic || `child stopped with reason "${result.stopReason}"`
|
|
883
|
+
log(`subtask ${subtask.index} attempt ${attempt} did not complete: ${subtask.error}`)
|
|
884
|
+
} catch (error) {
|
|
885
|
+
subtask.error = error?.message ?? String(error)
|
|
886
|
+
log(`subtask ${subtask.index} attempt ${attempt} threw: ${subtask.error}`)
|
|
887
|
+
if (error?.code === 'subagent-unavailable') {
|
|
888
|
+
// The host is going away; leave the subtask as work to do rather
|
|
889
|
+
// than burning the remaining attempts on a service that is gone.
|
|
890
|
+
subtask.error = null
|
|
891
|
+
subtask.stopReason = null
|
|
892
|
+
markHostGone(rt, error)
|
|
893
|
+
}
|
|
894
|
+
} finally {
|
|
895
|
+
if (handle) {
|
|
896
|
+
// A failed attempt still spent tokens, so read it before disposing.
|
|
897
|
+
await recordUsage(run, handle.childId)
|
|
898
|
+
rt.activations.delete(handle)
|
|
899
|
+
await disposeQuietly(handle)
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
if (signal.aborted) break
|
|
904
|
+
if (attempt < run.limits.attempts) {
|
|
905
|
+
subtask.status = 'pending'
|
|
906
|
+
await save(run)
|
|
907
|
+
await sleep(1000 * attempt)
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
if (subtask.status !== 'completed') {
|
|
912
|
+
if (signal.aborted) {
|
|
913
|
+
// Cancelled or out of time: the subtask did not fail on its own terms,
|
|
914
|
+
// so leave it resumable instead of recording a failure.
|
|
915
|
+
subtask.status = 'pending'
|
|
916
|
+
subtask.error = null
|
|
917
|
+
subtask.stopReason = null
|
|
918
|
+
} else {
|
|
919
|
+
subtask.status = 'failed'
|
|
920
|
+
subtask.finishedAt = new Date(now()).toISOString()
|
|
921
|
+
}
|
|
922
|
+
await save(run)
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
async function aggregate(run, parent, rt) {
|
|
927
|
+
const completed = run.subtasks.filter(subtask => subtask.status === 'completed')
|
|
928
|
+
if (completed.length === 0) return null
|
|
929
|
+
if (rt.controller.signal.aborted) return deterministicJoin(run)
|
|
930
|
+
if (completed.length === 1) return `# ${run.goal}\n\n${completed[0].result ?? ''}`.trim()
|
|
931
|
+
|
|
932
|
+
try {
|
|
933
|
+
const handle = await spawn(run, rt, {
|
|
934
|
+
label: 'superagent-aggregate',
|
|
935
|
+
prompt: aggregatorPrompt(run, completed),
|
|
936
|
+
parent,
|
|
937
|
+
})
|
|
938
|
+
rt.activations.add(handle)
|
|
939
|
+
run.steps.aggregator = { childId: handle.childId }
|
|
940
|
+
try {
|
|
941
|
+
const result = await abortable(handle.result, rt.controller.signal)
|
|
942
|
+
const text = textOf(result.output)
|
|
943
|
+
if (result.stopReason === 'completed' && text) return text
|
|
944
|
+
log(`aggregator did not complete (${result.stopReason}); using deterministic merge`)
|
|
945
|
+
} finally {
|
|
946
|
+
await recordUsage(run, handle.childId)
|
|
947
|
+
rt.activations.delete(handle)
|
|
948
|
+
await disposeQuietly(handle)
|
|
949
|
+
}
|
|
950
|
+
} catch (error) {
|
|
951
|
+
log(`aggregator failed: ${error?.message ?? error}`)
|
|
952
|
+
if (error?.code === 'subagent-unavailable') markHostGone(rt, error)
|
|
953
|
+
}
|
|
954
|
+
return deterministicJoin(run)
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
function terminalStatus(run, rt) {
|
|
958
|
+
if (rt.abortCause === 'cancelled') return 'cancelled'
|
|
959
|
+
// The host went away mid-run. Nothing failed on its own terms, so the run
|
|
960
|
+
// must stay resumable instead of being buried as `failed`.
|
|
961
|
+
if (rt.abortCause === 'host-gone') return 'interrupted'
|
|
962
|
+
const subtasks = run.subtasks ?? []
|
|
963
|
+
const completed = subtasks.filter(subtask => subtask.status === 'completed').length
|
|
964
|
+
if (completed === 0) return 'failed'
|
|
965
|
+
if (rt.abortCause === 'deadline') {
|
|
966
|
+
return completed === subtasks.length ? 'completed' : 'partial'
|
|
967
|
+
}
|
|
968
|
+
return completed === subtasks.length ? 'completed' : 'partial'
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
/**
|
|
972
|
+
* Watch for a cancel written by another process. The marker is checked from
|
|
973
|
+
* inside the owning process, which is the only place that can abort children;
|
|
974
|
+
* that is why cancel is a file and not a state-file rewrite.
|
|
975
|
+
*/
|
|
976
|
+
function watchCancel(run, rt) {
|
|
977
|
+
const timer = setInterval(() => {
|
|
978
|
+
hasCancelRequest(run.id).then(requested => {
|
|
979
|
+
if (!requested || rt.controller.signal.aborted) return
|
|
980
|
+
rt.abortCause = 'cancelled'
|
|
981
|
+
rt.controller.abort(new Error('cancelled by superagent_cancel from another process'))
|
|
982
|
+
// Aborting the signal alone does not stop a child that is already
|
|
983
|
+
// running; dispose its activation so it stops spending tokens.
|
|
984
|
+
for (const handle of [...rt.activations]) disposeQuietly(handle)
|
|
985
|
+
}).catch(() => {})
|
|
986
|
+
}, CANCEL_POLL_MS)
|
|
987
|
+
timer.unref?.()
|
|
988
|
+
return timer
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
/**
|
|
992
|
+
* A caller waiting inside this process will read the terminal result itself,
|
|
993
|
+
* so waking the session would be duplicate noise. A caller in *another*
|
|
994
|
+
* process is deliberately not treated this way: it can time out just before
|
|
995
|
+
* the run settles, in which case suppressing the wake-up would lose the only
|
|
996
|
+
* notification the user gets.
|
|
997
|
+
*/
|
|
998
|
+
function watchedHere(rt) {
|
|
999
|
+
return rt.waiters > 0
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/**
|
|
1003
|
+
* Compute the terminal state, decide about notifying, and write it exactly
|
|
1004
|
+
* once. Writing the terminal status first and the notify outcome second left
|
|
1005
|
+
* a window where readers saw a finished run with no notify line at all.
|
|
1006
|
+
*/
|
|
1007
|
+
async function settle(run, rt) {
|
|
1008
|
+
// Usage is stored per child so repeated reads replace rather than add. A
|
|
1009
|
+
// run written before that field existed keeps whatever total it had.
|
|
1010
|
+
const entries = Object.values(run.usageByChild ?? {})
|
|
1011
|
+
if (entries.length > 0) run.usage = sumUsage(entries)
|
|
1012
|
+
// The per-subtask figure is read when the child finishes, which can race a
|
|
1013
|
+
// log flush; the settle pass has the final snapshot, so publish it here.
|
|
1014
|
+
for (const subtask of run.subtasks ?? []) {
|
|
1015
|
+
if (subtask.childId && run.usageByChild?.[subtask.childId]) {
|
|
1016
|
+
subtask.usage = run.usageByChild[subtask.childId]
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
run.cost = computeCost(run.usage, config.pricing)
|
|
1020
|
+
const nextStatus = terminalStatus(run, rt)
|
|
1021
|
+
const finishedAt = new Date(now()).toISOString()
|
|
1022
|
+
let error = run.error
|
|
1023
|
+
if (rt.abortCause === 'cancelled') {
|
|
1024
|
+
error = 'cancelled by superagent_cancel'
|
|
1025
|
+
} else if (rt.abortCause === 'host-gone') {
|
|
1026
|
+
error = 'the host shut down while this run was active; run superagent_resume in a new host to finish it'
|
|
1027
|
+
} else if (nextStatus === 'failed' && !error) {
|
|
1028
|
+
error = run.subtasks.map(subtask => subtask.error).filter(Boolean).slice(0, 3).join('; ') || 'no subtask completed'
|
|
1029
|
+
}
|
|
1030
|
+
if (rt.abortCause === 'deadline' && !error) error = 'deadline exceeded'
|
|
1031
|
+
|
|
1032
|
+
let outcome = null
|
|
1033
|
+
if (TERMINAL_STATUSES.has(nextStatus)) {
|
|
1034
|
+
if (watchedHere(rt)) {
|
|
1035
|
+
outcome = { delivered: false, reason: 'skipped: a caller in this process is reading the run' }
|
|
1036
|
+
} else {
|
|
1037
|
+
try {
|
|
1038
|
+
// `notify` may wait on the host's session loop; cap it so a finished
|
|
1039
|
+
// run cannot stay active on disk (and be reclaimed by another process)
|
|
1040
|
+
// just because a wake-up is slow.
|
|
1041
|
+
outcome = await withTimeout(
|
|
1042
|
+
Promise.resolve().then(() => notify({ ...run, status: nextStatus, finishedAt, error })),
|
|
1043
|
+
NOTIFY_MS,
|
|
1044
|
+
() => ({
|
|
1045
|
+
delivered: false,
|
|
1046
|
+
reason: `the owning session did not accept the wake-up within ${NOTIFY_MS / 1000}s`,
|
|
1047
|
+
}),
|
|
1048
|
+
)
|
|
1049
|
+
outcome = outcome ?? { delivered: true }
|
|
1050
|
+
} catch (notifyError) {
|
|
1051
|
+
outcome = { delivered: false, reason: `notification threw: ${notifyError?.message ?? notifyError}` }
|
|
1052
|
+
log(`notification failed for ${run.id}: ${notifyError?.message ?? notifyError}`)
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
// The terminal status and its notify row land in the same write, so a
|
|
1058
|
+
// reader never sees a finished run that still claims to be active and never
|
|
1059
|
+
// sees a finished run whose notify row is missing.
|
|
1060
|
+
run.status = nextStatus
|
|
1061
|
+
run.finishedAt = finishedAt
|
|
1062
|
+
run.activeMs = (Number.isFinite(run.activeMs) ? run.activeMs : 0) + Math.max(0, now() - rt.activeStart)
|
|
1063
|
+
run.error = error
|
|
1064
|
+
run.cancelPending = false
|
|
1065
|
+
if (outcome) {
|
|
1066
|
+
run.notify = {
|
|
1067
|
+
at: new Date(now()).toISOString(),
|
|
1068
|
+
delivered: outcome.delivered !== false,
|
|
1069
|
+
reason: outcome.reason ?? null,
|
|
1070
|
+
}
|
|
1071
|
+
if (outcome.delivered !== false) run.notified = true
|
|
1072
|
+
}
|
|
1073
|
+
await save(run)
|
|
1074
|
+
// The run is terminal, so a leftover marker would only make the next mount
|
|
1075
|
+
// rewrite a finished run; drop it now.
|
|
1076
|
+
await clearCancel(run.id)
|
|
1077
|
+
await releaseRunLock(run.id)
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* Resolve with `promise`, or with `onTimeout()` after `ms`.
|
|
1082
|
+
*
|
|
1083
|
+
* The timer is cleared when the promise wins: a stray 15s timer would keep a
|
|
1084
|
+
* finished host process (and a test runner) alive for no reason.
|
|
1085
|
+
*/
|
|
1086
|
+
function withTimeout(promise, ms, onTimeout) {
|
|
1087
|
+
return new Promise((resolve, reject) => {
|
|
1088
|
+
const timer = setTimeout(() => resolve(onTimeout()), ms)
|
|
1089
|
+
timer.unref?.()
|
|
1090
|
+
promise.then(
|
|
1091
|
+
value => {
|
|
1092
|
+
clearTimeout(timer)
|
|
1093
|
+
resolve(value)
|
|
1094
|
+
},
|
|
1095
|
+
error => {
|
|
1096
|
+
clearTimeout(timer)
|
|
1097
|
+
reject(error)
|
|
1098
|
+
},
|
|
1099
|
+
)
|
|
1100
|
+
})
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
async function execute(run, parent, rt) {
|
|
1104
|
+
rt.activeStart = now()
|
|
1105
|
+
const remaining = Math.max(0, Date.parse(run.deadlineAt) - now())
|
|
1106
|
+
const timer = setTimeout(() => {
|
|
1107
|
+
rt.abortCause = 'deadline'
|
|
1108
|
+
rt.controller.abort(new Error(`deadline exceeded after ${run.limits.deadlineMinutes} minutes`))
|
|
1109
|
+
}, remaining)
|
|
1110
|
+
timer.unref?.()
|
|
1111
|
+
const cancelTimer = watchCancel(run, rt)
|
|
1112
|
+
// The heartbeat is what makes an orphan detectable after a crash on a
|
|
1113
|
+
// machine whose pids this process cannot check (shared state directory).
|
|
1114
|
+
const beat = setInterval(() => {
|
|
1115
|
+
run.heartbeatAt = new Date(now()).toISOString()
|
|
1116
|
+
run.ownerPid = process.pid
|
|
1117
|
+
refreshUsage(run, rt).then(() => save(run)).catch(() => {})
|
|
1118
|
+
}, HEARTBEAT_MS)
|
|
1119
|
+
beat.unref?.()
|
|
1120
|
+
|
|
1121
|
+
try {
|
|
1122
|
+
ensureCounters(run)
|
|
1123
|
+
if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
|
|
1124
|
+
run.status = 'planning'
|
|
1125
|
+
await save(run)
|
|
1126
|
+
const plan = await makePlan(run, parent, rt)
|
|
1127
|
+
run.plan = { mode: plan.mode, subtasks: plan.subtasks }
|
|
1128
|
+
run.subtasks = plan.subtasks.map((entry, index) => ({
|
|
1129
|
+
index: index + 1,
|
|
1130
|
+
title: entry.title,
|
|
1131
|
+
prompt: entry.prompt,
|
|
1132
|
+
status: 'pending',
|
|
1133
|
+
attempts: 0,
|
|
1134
|
+
childId: null,
|
|
1135
|
+
stopReason: null,
|
|
1136
|
+
error: null,
|
|
1137
|
+
result: null,
|
|
1138
|
+
usage: null,
|
|
1139
|
+
startedAt: null,
|
|
1140
|
+
finishedAt: null,
|
|
1141
|
+
}))
|
|
1142
|
+
await save(run)
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
|
|
1146
|
+
if (open.length > 0) {
|
|
1147
|
+
run.status = 'running'
|
|
1148
|
+
await save(run)
|
|
1149
|
+
await pool(open, run.limits.maxParallel, subtask => runSubtask(run, subtask, parent, rt))
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
|
|
1153
|
+
run.status = 'aggregating'
|
|
1154
|
+
await save(run)
|
|
1155
|
+
run.report = await aggregate(run, parent, rt)
|
|
1156
|
+
} else if (!run.report) {
|
|
1157
|
+
run.report = deterministicJoin(run)
|
|
1158
|
+
}
|
|
1159
|
+
} catch (error) {
|
|
1160
|
+
run.error = error?.message ?? String(error)
|
|
1161
|
+
log(`run ${run.id} failed: ${run.error}`)
|
|
1162
|
+
// Stop any child still running; without this the run would end while its
|
|
1163
|
+
// subtasks keep burning tokens.
|
|
1164
|
+
rt.controller.abort(error instanceof Error ? error : new Error(String(error)))
|
|
1165
|
+
} finally {
|
|
1166
|
+
clearTimeout(timer)
|
|
1167
|
+
clearInterval(cancelTimer)
|
|
1168
|
+
// The heartbeat keeps running until the terminal write lands: waking the
|
|
1169
|
+
// session can take a few seconds, and a frozen heartbeat during that
|
|
1170
|
+
// window would let another process treat a live run as orphaned.
|
|
1171
|
+
try {
|
|
1172
|
+
await settle(run, rt)
|
|
1173
|
+
} catch (error) {
|
|
1174
|
+
// Never leave a run active because finalizing it threw.
|
|
1175
|
+
log(`cannot finalize run ${run.id}: ${error?.message ?? error}`)
|
|
1176
|
+
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1177
|
+
run.status = 'failed'
|
|
1178
|
+
run.finishedAt = run.finishedAt ?? new Date(now()).toISOString()
|
|
1179
|
+
run.error = run.error ?? `finalizing the run failed: ${error?.message ?? error}`
|
|
1180
|
+
}
|
|
1181
|
+
try {
|
|
1182
|
+
await store.write(run)
|
|
1183
|
+
} catch (writeError) {
|
|
1184
|
+
log(`cannot persist the terminal state of run ${run.id}: ${writeError?.message ?? writeError}`)
|
|
1185
|
+
}
|
|
1186
|
+
await releaseRunLock(run.id)
|
|
1187
|
+
} finally {
|
|
1188
|
+
clearInterval(beat)
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
function schedule(run, parent) {
|
|
1194
|
+
run.ownerPid = process.pid
|
|
1195
|
+
run.ownerHost = os.hostname()
|
|
1196
|
+
run.heartbeatAt = new Date(now()).toISOString()
|
|
1197
|
+
const rt = {
|
|
1198
|
+
controller: new AbortController(),
|
|
1199
|
+
activations: new Set(),
|
|
1200
|
+
waiters: 0,
|
|
1201
|
+
run,
|
|
1202
|
+
promise: null,
|
|
1203
|
+
abortCause: null,
|
|
1204
|
+
}
|
|
1205
|
+
runtime.set(run.id, rt)
|
|
1206
|
+
rt.promise = execute(run, parent, rt)
|
|
1207
|
+
.catch(async error => {
|
|
1208
|
+
log(`run ${run.id} crashed: ${error?.message ?? error}`)
|
|
1209
|
+
if (!ACTIVE_STATUSES.has(run.status)) return
|
|
1210
|
+
run.status = 'failed'
|
|
1211
|
+
run.finishedAt = new Date(now()).toISOString()
|
|
1212
|
+
run.error = error?.message ?? String(error)
|
|
1213
|
+
try {
|
|
1214
|
+
await store.write(run)
|
|
1215
|
+
} catch (writeError) {
|
|
1216
|
+
log(`cannot persist the crash state of run ${run.id}: ${writeError?.message ?? writeError}`)
|
|
1217
|
+
}
|
|
1218
|
+
await releaseRunLock(run.id)
|
|
1219
|
+
})
|
|
1220
|
+
.then(() => {
|
|
1221
|
+
// Finished runs are read from disk from now on; keeping them here would
|
|
1222
|
+
// grow these maps for the lifetime of the host.
|
|
1223
|
+
runtime.delete(run.id)
|
|
1224
|
+
saveChains.delete(run.id)
|
|
1225
|
+
})
|
|
1226
|
+
return rt
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
/** Serialize starts and resumes so a quota check cannot interleave. */
|
|
1230
|
+
function serialize(task) {
|
|
1231
|
+
const result = startGate.then(task, task)
|
|
1232
|
+
startGate = result.catch(() => {})
|
|
1233
|
+
return result
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
async function start({ goal, parentSessionId, parent, clientTimeZone, overrides = {} }) {
|
|
1237
|
+
const trimmed = typeof goal === 'string' ? goal.trim() : ''
|
|
1238
|
+
if (!trimmed) throw new Error('goal is required')
|
|
1239
|
+
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
1240
|
+
|
|
1241
|
+
const limits = buildLimits(overrides)
|
|
1242
|
+
// Kept so a resume can rebuild the limits from the *current* config
|
|
1243
|
+
// instead of freezing whatever was set when the run first started.
|
|
1244
|
+
const keptOverrides = {}
|
|
1245
|
+
for (const key of ['maxSubtasks', 'maxParallel', 'deadlineMinutes']) {
|
|
1246
|
+
if (overrides[key] !== undefined && overrides[key] !== null) keptOverrides[key] = overrides[key]
|
|
1247
|
+
}
|
|
1248
|
+
return serialize(async () => {
|
|
1249
|
+
const active = await activeCount()
|
|
1250
|
+
if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
|
|
1251
|
+
const run = {
|
|
1252
|
+
id: newRunId(),
|
|
1253
|
+
schemaVersion: 1,
|
|
1254
|
+
goal: trimmed,
|
|
1255
|
+
status: 'queued',
|
|
1256
|
+
mode: 'planned',
|
|
1257
|
+
parentSessionId,
|
|
1258
|
+
clientTimeZone: clientTimeZone ?? null,
|
|
1259
|
+
createdAt: new Date(now()).toISOString(),
|
|
1260
|
+
updatedAt: null,
|
|
1261
|
+
startedAt: new Date(now()).toISOString(),
|
|
1262
|
+
deadlineAt: new Date(now() + limits.deadlineMinutes * 60000).toISOString(),
|
|
1263
|
+
finishedAt: null,
|
|
1264
|
+
resumedAt: null,
|
|
1265
|
+
limits,
|
|
1266
|
+
overrides: keptOverrides,
|
|
1267
|
+
plan: null,
|
|
1268
|
+
subtasks: [],
|
|
1269
|
+
steps: {},
|
|
1270
|
+
report: null,
|
|
1271
|
+
error: null,
|
|
1272
|
+
usage: null,
|
|
1273
|
+
notified: false,
|
|
1274
|
+
counters: { spawns: 0, unreadable: {} },
|
|
1275
|
+
usageByChild: {},
|
|
1276
|
+
}
|
|
1277
|
+
await takeRunLock(run.id)
|
|
1278
|
+
try {
|
|
1279
|
+
await store.write(run)
|
|
1280
|
+
} catch (error) {
|
|
1281
|
+
await releaseRunLock(run.id)
|
|
1282
|
+
throw error
|
|
1283
|
+
}
|
|
1284
|
+
schedule(run, parent)
|
|
1285
|
+
return run
|
|
1286
|
+
})
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
async function resume(runId, parent, clientTimeZone) {
|
|
1290
|
+
const id = String(runId ?? '').trim()
|
|
1291
|
+
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
1292
|
+
return serialize(async () => {
|
|
1293
|
+
if (runtime.has(id)) {
|
|
1294
|
+
throw new Error(`run ${id} is still active in this process; wait for it with superagent_status or cancel it`)
|
|
1295
|
+
}
|
|
1296
|
+
let run = await loadRun(id)
|
|
1297
|
+
if (run.status === 'completed') return run
|
|
1298
|
+
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1299
|
+
const verdict = await ownership(run)
|
|
1300
|
+
if (verdict.live) {
|
|
1301
|
+
throw new Error(
|
|
1302
|
+
`run ${id} is still owned by a live process (${verdict.basis}); resuming it here would run the same `
|
|
1303
|
+
+ 'subtasks twice and double the spend. Cancel it with superagent_cancel first, or wait for it to finish.',
|
|
1304
|
+
)
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
await takeRunLock(id)
|
|
1309
|
+
let scheduled = false
|
|
1310
|
+
try {
|
|
1311
|
+
// Re-read under the lock: another process may have started the run
|
|
1312
|
+
// between the check above and the claim.
|
|
1313
|
+
run = await loadRun(id)
|
|
1314
|
+
if (run.status === 'completed') return run
|
|
1315
|
+
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1316
|
+
const verdict = await ownership(run)
|
|
1317
|
+
if (verdict.live) {
|
|
1318
|
+
throw new Error(`run ${id} is still owned by a live process (${verdict.basis}); not resuming it here`)
|
|
1319
|
+
}
|
|
1320
|
+
reclaim(run, verdict, await hasCancelRequest(id))
|
|
1321
|
+
await save(run)
|
|
1322
|
+
await clearCancel(id)
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
const active = await activeCount()
|
|
1326
|
+
if (active >= maxActiveRuns()) throw new Error(quotaMessage(active))
|
|
1327
|
+
|
|
1328
|
+
for (const subtask of run.subtasks ?? []) {
|
|
1329
|
+
if (subtask.status === 'completed') continue
|
|
1330
|
+
subtask.status = 'pending'
|
|
1331
|
+
subtask.attempts = 0
|
|
1332
|
+
subtask.error = null
|
|
1333
|
+
subtask.stopReason = null
|
|
1334
|
+
subtask.finishedAt = null
|
|
1335
|
+
}
|
|
1336
|
+
run.status = 'queued'
|
|
1337
|
+
run.report = run.plan ? run.report : null
|
|
1338
|
+
run.error = null
|
|
1339
|
+
run.finishedAt = null
|
|
1340
|
+
run.reclaimed = null
|
|
1341
|
+
run.cancelPending = false
|
|
1342
|
+
// A resumed run completes a second time, so it must be able to wake the
|
|
1343
|
+
// session again; keeping the old outcome would silence it forever.
|
|
1344
|
+
run.notified = false
|
|
1345
|
+
run.notify = null
|
|
1346
|
+
run.resumedAt = new Date(now()).toISOString()
|
|
1347
|
+
run.clientTimeZone = clientTimeZone ?? run.clientTimeZone ?? null
|
|
1348
|
+
// The limits are rebuilt from the config that is active now, keeping
|
|
1349
|
+
// everything already spent: that is what makes "raise the cap, then
|
|
1350
|
+
// resume" work. Without this the old caps stayed in force forever.
|
|
1351
|
+
run.limits = buildLimits(run.overrides ?? {})
|
|
1352
|
+
run.deadlineAt = new Date(now() + run.limits.deadlineMinutes * 60000).toISOString()
|
|
1353
|
+
ensureCounters(run)
|
|
1354
|
+
await save(run)
|
|
1355
|
+
schedule(run, parent)
|
|
1356
|
+
scheduled = true
|
|
1357
|
+
return run
|
|
1358
|
+
} finally {
|
|
1359
|
+
if (!scheduled) await releaseRunLock(id)
|
|
1360
|
+
}
|
|
1361
|
+
})
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
async function cancel(runId) {
|
|
1365
|
+
const rt = runtime.get(runId)
|
|
1366
|
+
if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
|
|
1367
|
+
rt.abortCause = 'cancelled'
|
|
1368
|
+
rt.controller.abort(new Error('cancelled by superagent_cancel'))
|
|
1369
|
+
const activations = [...rt.activations]
|
|
1370
|
+
// A child that ignores its abort must not hold this call open; the
|
|
1371
|
+
// timeout is per child and they are disposed in parallel.
|
|
1372
|
+
await Promise.all(activations.map(handle => disposeQuietly(handle)))
|
|
1373
|
+
await Promise.race([rt.promise ?? Promise.resolve(), sleep(CANCEL_ACK_MS)])
|
|
1374
|
+
if (ACTIVE_STATUSES.has(rt.run.status)) {
|
|
1375
|
+
rt.run.cancelPending = true
|
|
1376
|
+
await save(rt.run).catch(() => {})
|
|
1377
|
+
}
|
|
1378
|
+
return rt.run
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
const run = await loadRun(runId)
|
|
1382
|
+
if (!ACTIVE_STATUSES.has(run.status)) return run
|
|
1383
|
+
|
|
1384
|
+
// This process does not own the run, so it cannot abort the children. Ask
|
|
1385
|
+
// the owner through the marker file and wait for it to rewrite the state.
|
|
1386
|
+
// Writing `cancelled` here would be a lie: the owner would overwrite it on
|
|
1387
|
+
// its next save and keep spending.
|
|
1388
|
+
await requestCancel(runId)
|
|
1389
|
+
const endsAt = now() + CANCEL_ACK_MS
|
|
1390
|
+
while (now() < endsAt) {
|
|
1391
|
+
await sleep(STATUS_POLL_MS)
|
|
1392
|
+
const current = await store.read(runId)
|
|
1393
|
+
if (!current) return run
|
|
1394
|
+
if (!ACTIVE_STATUSES.has(current.status)) return current
|
|
1395
|
+
}
|
|
1396
|
+
// The owner did not confirm in time. Do not rewrite the state file from
|
|
1397
|
+
// here: the owner would overwrite it on its next save. The request marker
|
|
1398
|
+
// stays on disk, and `status` reports it as a pending cancel.
|
|
1399
|
+
return { ...run, cancelPending: true }
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
async function status(runId, waitSeconds = 0) {
|
|
1403
|
+
const read = async () => liveRun(runId) ?? await store.read(runId)
|
|
1404
|
+
let run = await read()
|
|
1405
|
+
if (!run) {
|
|
1406
|
+
await loadRun(runId)
|
|
1407
|
+
throw new Error(`unknown run ${runId}`)
|
|
1408
|
+
}
|
|
1409
|
+
// A terminal status is only published after the run's final write, so a
|
|
1410
|
+
// caller that sees one gets the notify row and usage totals with it.
|
|
1411
|
+
const owner = runtime.get(runId)
|
|
1412
|
+
if (owner && TERMINAL_STATUSES.has(run.status)) {
|
|
1413
|
+
await withTimeout(Promise.resolve(owner.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
|
|
1414
|
+
run = (await read()) ?? run
|
|
1415
|
+
}
|
|
1416
|
+
const limit = clamp(waitSeconds, 0, 3600)
|
|
1417
|
+
if (ACTIVE_STATUSES.has(run.status) && !run.cancelPending && await hasCancelRequest(runId)) {
|
|
1418
|
+
// The owning process may live elsewhere; the marker is the only evidence
|
|
1419
|
+
// that a cancel was asked for and not yet acknowledged.
|
|
1420
|
+
run = { ...run, cancelPending: true }
|
|
1421
|
+
}
|
|
1422
|
+
if (limit <= 0 || TERMINAL_STATUSES.has(run.status)) return run
|
|
1423
|
+
|
|
1424
|
+
const rt = runtime.get(runId)
|
|
1425
|
+
if (rt) rt.waiters++
|
|
1426
|
+
try {
|
|
1427
|
+
const endsAt = now() + limit * 1000
|
|
1428
|
+
while (now() < endsAt) {
|
|
1429
|
+
await sleep(STATUS_POLL_MS)
|
|
1430
|
+
run = await read()
|
|
1431
|
+
if (!run || TERMINAL_STATUSES.has(run.status)) break
|
|
1432
|
+
}
|
|
1433
|
+
} finally {
|
|
1434
|
+
if (rt) rt.waiters = Math.max(0, rt.waiters - 1)
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
if (!run) throw new Error(`run ${runId} disappeared from the state directory while waiting for it`)
|
|
1438
|
+
const live = runtime.get(runId)
|
|
1439
|
+
// Wait for this process's own final write, so a caller that sees a terminal
|
|
1440
|
+
// status can immediately trust what is on disk. Bounded, because finalizing
|
|
1441
|
+
// includes waking the session and must not turn a status call into a hang.
|
|
1442
|
+
if (live && TERMINAL_STATUSES.has(live.run.status)) {
|
|
1443
|
+
await withTimeout(Promise.resolve(live.promise).catch(() => undefined), NOTIFY_MS + 5000, () => undefined)
|
|
1444
|
+
}
|
|
1445
|
+
return liveRun(runId) ?? run
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
async function list() {
|
|
1449
|
+
const merged = new Map()
|
|
1450
|
+
for (const run of await store.list()) merged.set(run.id, run)
|
|
1451
|
+
for (const [runId, rt] of runtime) merged.set(runId, rt.run)
|
|
1452
|
+
return [...merged.values()].sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
/**
|
|
1456
|
+
* Resolve once every run started by this process has finished. Used by tests
|
|
1457
|
+
* instead of sleeping, and by embedders that need a quiescent state.
|
|
1458
|
+
*/
|
|
1459
|
+
async function idle() {
|
|
1460
|
+
await Promise.all([...runtime.values()].map(rt => rt.promise?.catch(() => {})))
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
return { recover, start, resume, cancel, status, list, idle, activeCount, liveRun }
|
|
1464
|
+
}
|