@xiangyixie/dsh-superagent 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +286 -88
- package/README.md +571 -39
- package/bin/archive.js +185 -0
- package/bin/doctor.js +84 -0
- package/bin/sync.js +187 -0
- package/bin/workbench.js +98 -0
- package/docs/policy.example.json +38 -0
- package/lib/audit.js +555 -0
- package/lib/cost.js +267 -0
- package/lib/doctor.js +878 -0
- package/lib/guard.js +841 -0
- package/lib/index.js +2031 -1104
- package/lib/inject.js +138 -0
- package/lib/ledger.js +270 -0
- package/lib/memory.js +736 -503
- package/lib/orchestrator.js +1950 -217
- package/lib/policy.js +739 -0
- package/lib/report.js +1068 -733
- package/lib/store.js +873 -402
- package/lib/sync.js +456 -0
- package/lib/templates.js +690 -0
- package/lib/watch.js +810 -458
- package/lib/workbench.js +1119 -0
- package/lib/workspace.js +433 -0
- package/package.json +92 -70
- package/public/workbench.html +836 -0
package/lib/orchestrator.js
CHANGED
|
@@ -20,8 +20,10 @@ import { createHash, randomBytes } from 'node:crypto'
|
|
|
20
20
|
import fsp from 'node:fs/promises'
|
|
21
21
|
import os from 'node:os'
|
|
22
22
|
import path from 'node:path'
|
|
23
|
+
import { applyCost, computeCost } from './cost.js'
|
|
23
24
|
import { extractMemoryBlock } from './memory.js'
|
|
24
25
|
import { ACTIVE_STATUSES, GOAL_GUARD_PREFIX, TERMINAL_STATUSES } from './store.js'
|
|
26
|
+
import { createWorkspaceManager } from './workspace.js'
|
|
25
27
|
|
|
26
28
|
const STATUS_POLL_MS = 250
|
|
27
29
|
/** How often the owning process checks for a cancel marker written elsewhere. */
|
|
@@ -51,6 +53,39 @@ const RETRY_CAP_MS = 60 * 1000
|
|
|
51
53
|
/** Fenced block a worker or the aggregator uses to ask for a decision. */
|
|
52
54
|
const ATTENTION_FENCE = /```attention[^\n]*\n([\s\S]*?)```/i
|
|
53
55
|
|
|
56
|
+
/** Instructions kept per run; older ones are dropped so a long dialog stays bounded. */
|
|
57
|
+
const FOLLOWUP_MAX = 8
|
|
58
|
+
/** Characters kept from one instruction or answer. */
|
|
59
|
+
const FOLLOWUP_MAX_CHARS = 1000
|
|
60
|
+
/** Live injection receipts kept per run; each may target several workers. */
|
|
61
|
+
const INJECTION_MAX = 12
|
|
62
|
+
/** Poll period for the measurement of one step boundary, in milliseconds. */
|
|
63
|
+
const INJECT_POLL_MS = 25
|
|
64
|
+
/** How many expired attention batches stay readable for a late answer. */
|
|
65
|
+
const ATTENTION_HISTORY_MAX = 5
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Which pipeline a run uses.
|
|
69
|
+
*
|
|
70
|
+
* `planned` is the default: a planner splits the goal, workers run in parallel,
|
|
71
|
+
* and an aggregator merges their reports. `direct` is for one self-contained
|
|
72
|
+
* task: a single worker does it, there is no plan and no merge, and its reply
|
|
73
|
+
* is the report verbatim. Direct saves two model calls and the coordination
|
|
74
|
+
* they need, so it is right for a single lookup, edit, query, or short piece of
|
|
75
|
+
* writing — and wrong for a goal with parts that can be worked in parallel.
|
|
76
|
+
*
|
|
77
|
+
* @param {unknown} value - requested mode.
|
|
78
|
+
* @returns {'planned' | 'direct'} normalized mode.
|
|
79
|
+
*/
|
|
80
|
+
export function normalizeMode(value) {
|
|
81
|
+
if (value === undefined || value === null || value === '') return 'planned'
|
|
82
|
+
const text = String(value).trim().toLowerCase()
|
|
83
|
+
if (text === 'planned' || text === 'direct') return text
|
|
84
|
+
throw new Error(
|
|
85
|
+
`unknown mode ${JSON.stringify(value)}; use "planned" (multi-worker) or "direct" (one worker, no planning or aggregation)`,
|
|
86
|
+
)
|
|
87
|
+
}
|
|
88
|
+
|
|
54
89
|
/**
|
|
55
90
|
* Canonical form of a goal: case and runs of whitespace are formatting, not a
|
|
56
91
|
* different request, so they must not defeat duplicate detection.
|
|
@@ -188,7 +223,8 @@ export function capText(text, maxBytes) {
|
|
|
188
223
|
* region is tried rather than assuming the reply is bare JSON.
|
|
189
224
|
*
|
|
190
225
|
* @param {string} text - planner reply.
|
|
191
|
-
* @returns {Array<{ title: string, prompt: string
|
|
226
|
+
* @returns {Array<{ title: string, prompt: string, id?: string, dependsOn?: string[] }>}
|
|
227
|
+
* zero or more subtasks; `id` and `dependsOn` appear only when the planner set them.
|
|
192
228
|
*/
|
|
193
229
|
export function extractPlan(text) {
|
|
194
230
|
if (typeof text !== 'string') return []
|
|
@@ -235,21 +271,206 @@ export function extractPlan(text) {
|
|
|
235
271
|
const title = typeof item.title === 'string' && item.title.trim()
|
|
236
272
|
? item.title.trim().slice(0, 80)
|
|
237
273
|
: prompt.slice(0, 60)
|
|
238
|
-
|
|
274
|
+
const entry = { title, prompt }
|
|
275
|
+
const id = oneLine(item.id, 32)
|
|
276
|
+
if (/^[A-Za-z0-9][A-Za-z0-9._:-]{0,31}$/.test(id)) entry.id = id
|
|
277
|
+
if (Array.isArray(item.dependsOn)) {
|
|
278
|
+
// References are kept verbatim here; whether each one names a real
|
|
279
|
+
// subtask (by id or title) is decided once the whole plan is known.
|
|
280
|
+
const dependsOn = [...new Set(item.dependsOn.map(ref => oneLine(ref, 80)).filter(Boolean))]
|
|
281
|
+
if (dependsOn.length > 0) entry.dependsOn = dependsOn
|
|
282
|
+
}
|
|
283
|
+
subtasks.push(entry)
|
|
239
284
|
}
|
|
240
285
|
if (subtasks.length > 0) return subtasks
|
|
241
286
|
}
|
|
242
287
|
return []
|
|
243
288
|
}
|
|
244
289
|
|
|
290
|
+
/** Collapse arbitrary text into at most `maxChars` characters. */
|
|
291
|
+
function capChars(value, maxChars) {
|
|
292
|
+
return [...String(value ?? '').trim()].slice(0, maxChars).join('')
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Where an instruction written against this run will actually land.
|
|
297
|
+
*
|
|
298
|
+
* Exported so the tool reply and `superagent_status` state the same boundary:
|
|
299
|
+
* a child that is already running cannot be reached, and claiming otherwise
|
|
300
|
+
* would be the one lie this feature must not tell.
|
|
301
|
+
*
|
|
302
|
+
* @param {object} run - run record, live or persisted.
|
|
303
|
+
* @returns {{ boundary: string, detail: string }} next boundary and how to read it.
|
|
304
|
+
*/
|
|
305
|
+
export function followupEffect(run) {
|
|
306
|
+
const subtasks = Array.isArray(run?.subtasks) ? run.subtasks : []
|
|
307
|
+
const open = subtasks.filter(subtask => subtask.status !== 'completed')
|
|
308
|
+
// `planning` covers two windows: before the planner is spawned, and while it
|
|
309
|
+
// runs. Only the first can still change the planner prompt; the spawn is
|
|
310
|
+
// recorded on `steps.planner`, so this does not promise a prompt that exists.
|
|
311
|
+
if (run?.status === 'planning' && subtasks.length === 0 && run.steps?.planner) {
|
|
312
|
+
return {
|
|
313
|
+
boundary: 'next-subtask-or-aggregator',
|
|
314
|
+
detail: 'the planner prompt is already built, so this reaches the first worker prompt or the aggregation, '
|
|
315
|
+
+ 'whichever comes first',
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
if (run?.status === 'queued' || run?.status === 'planning') {
|
|
319
|
+
if (subtasks.length === 0) {
|
|
320
|
+
return { boundary: 'planner', detail: 'the planner prompt, before any subtask starts' }
|
|
321
|
+
}
|
|
322
|
+
if (open.length === 0) {
|
|
323
|
+
return { boundary: 'aggregator', detail: 'the aggregation of the already completed subtasks' }
|
|
324
|
+
}
|
|
325
|
+
return {
|
|
326
|
+
boundary: 'next-subtask-or-aggregator',
|
|
327
|
+
detail: 'the next worker spawn or the aggregation, whichever comes first',
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (run?.status === 'running') {
|
|
331
|
+
return {
|
|
332
|
+
boundary: 'next-subtask-or-aggregator',
|
|
333
|
+
detail: 'the next worker spawn or the aggregation, whichever comes first; subtasks already running or '
|
|
334
|
+
+ 'finished keep their results and are not interrupted',
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (run?.status === 'aggregating') {
|
|
338
|
+
return {
|
|
339
|
+
boundary: 'next-round',
|
|
340
|
+
detail: 'the aggregation already running is not interrupted, so this applies to the next round or resume, '
|
|
341
|
+
+ 'not to the report being written now',
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
// A paused run has a real next round: resume resets its pending subtasks and
|
|
345
|
+
// builds fresh prompts, so an instruction queued now is not lost.
|
|
346
|
+
if (['interrupted', 'failed', 'partial', 'cancelled'].includes(run?.status)) {
|
|
347
|
+
return {
|
|
348
|
+
boundary: 'next-resume',
|
|
349
|
+
detail: 'the next round, which begins when you call superagent_resume',
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return { boundary: 'none', detail: 'the run is not active, so nothing will pick this up' }
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* The instructions a prompt built now can carry. The list is bounded at both
|
|
357
|
+
* ends: the newest eight instructions, and a cap on each text, so a long dialog
|
|
358
|
+
* cannot crowd out the goal itself.
|
|
359
|
+
*
|
|
360
|
+
* @param {object} run - run record.
|
|
361
|
+
* @returns {object[]} entries, oldest first.
|
|
362
|
+
*/
|
|
363
|
+
function carriedEntries(run) {
|
|
364
|
+
return Array.isArray(run?.followups) ? run.followups.slice(-FOLLOWUP_MAX) : []
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* True when an answer was recorded but no prompt has carried it yet. Such a
|
|
369
|
+
* run is not really finished even if its status says `completed`.
|
|
370
|
+
*
|
|
371
|
+
* @param {object} run - run record.
|
|
372
|
+
* @returns {boolean} whether a resume must reopen it.
|
|
373
|
+
*/
|
|
374
|
+
function hasUnappliedAnswer(run) {
|
|
375
|
+
return (Array.isArray(run?.followups) ? run.followups : [])
|
|
376
|
+
.some(entry => entry.kind === 'answer' && !entry.appliedAt)
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Render exactly the entries a prompt will carry. Taking the entries rather
|
|
381
|
+
* than the run keeps the "applied" mark and the prompt text in agreement: an
|
|
382
|
+
* instruction that arrives during the spawn is not claimed to be in it.
|
|
383
|
+
*
|
|
384
|
+
* @param {object[]} entries - from `carriedEntries`.
|
|
385
|
+
* @returns {string[]} prompt lines, empty when there is nothing to carry.
|
|
386
|
+
*/
|
|
387
|
+
function instructionLines(entries) {
|
|
388
|
+
if (entries.length === 0) return []
|
|
389
|
+
return [
|
|
390
|
+
'Instructions the user added while this run was in progress. They take precedence over the wording above:',
|
|
391
|
+
...entries.map(entry => `- ${entry.text}`),
|
|
392
|
+
'',
|
|
393
|
+
]
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Resolve each subtask's `dependsOn` references and report dependency cycles.
|
|
398
|
+
*
|
|
399
|
+
* A reference may name another subtask by id or by title, because a planner
|
|
400
|
+
* writes one or the other; anything that names nothing is reported instead of
|
|
401
|
+
* being dropped, so the run can say why a subtask never started.
|
|
402
|
+
*
|
|
403
|
+
* @param {Array<{ key: string, title: string, dependsOn?: string[] }>} subtasks - plan entries with ids assigned.
|
|
404
|
+
* @returns {{ edges: Map<string, string[]>, unknown: Map<string, string[]>, cycle: string[] | null }}
|
|
405
|
+
* resolved edges per task, unresolved references per task, and one cycle
|
|
406
|
+
* (as keys, first repeated at the end) when the graph has one.
|
|
407
|
+
*/
|
|
408
|
+
export function linkSubtasks(subtasks) {
|
|
409
|
+
const byKey = new Map()
|
|
410
|
+
const byTitle = new Map()
|
|
411
|
+
for (const subtask of subtasks) {
|
|
412
|
+
byKey.set(subtask.key, subtask)
|
|
413
|
+
const title = String(subtask.title ?? '').trim().toLowerCase()
|
|
414
|
+
if (title && !byTitle.has(title)) byTitle.set(title, subtask.key)
|
|
415
|
+
}
|
|
416
|
+
const edges = new Map()
|
|
417
|
+
const unknown = new Map()
|
|
418
|
+
for (const subtask of subtasks) {
|
|
419
|
+
const resolved = []
|
|
420
|
+
const missing = []
|
|
421
|
+
for (const ref of subtask.dependsOn ?? []) {
|
|
422
|
+
const key = byKey.has(ref) ? ref : byTitle.get(String(ref).trim().toLowerCase())
|
|
423
|
+
if (!key) missing.push(ref)
|
|
424
|
+
else if (!resolved.includes(key)) resolved.push(key)
|
|
425
|
+
}
|
|
426
|
+
edges.set(subtask.key, resolved)
|
|
427
|
+
if (missing.length > 0) unknown.set(subtask.key, missing)
|
|
428
|
+
}
|
|
429
|
+
return { edges, unknown, cycle: findCycle([...byKey.keys()], edges) }
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Depth-first search for one dependency cycle, kept small because a plan has at
|
|
434
|
+
* most a handful of subtasks and the message must name the tasks involved.
|
|
435
|
+
* @param {string[]} keys - every subtask key.
|
|
436
|
+
* @param {Map<string, string[]>} edges - resolved edges.
|
|
437
|
+
* @returns {string[] | null} cycle as `w1 -> w2 -> w1`, or null.
|
|
438
|
+
*/
|
|
439
|
+
function findCycle(keys, edges) {
|
|
440
|
+
const state = new Map()
|
|
441
|
+
const stack = []
|
|
442
|
+
let cycle = null
|
|
443
|
+
const visit = key => {
|
|
444
|
+
state.set(key, 1)
|
|
445
|
+
stack.push(key)
|
|
446
|
+
for (const next of edges.get(key) ?? []) {
|
|
447
|
+
if (cycle) break
|
|
448
|
+
const seen = state.get(next) ?? 0
|
|
449
|
+
if (seen === 1) {
|
|
450
|
+
cycle = [...stack.slice(stack.indexOf(next)), next]
|
|
451
|
+
break
|
|
452
|
+
}
|
|
453
|
+
if (seen === 0) visit(next)
|
|
454
|
+
}
|
|
455
|
+
stack.pop()
|
|
456
|
+
state.set(key, 2)
|
|
457
|
+
}
|
|
458
|
+
for (const key of keys) {
|
|
459
|
+
if (cycle) break
|
|
460
|
+
if ((state.get(key) ?? 0) === 0) visit(key)
|
|
461
|
+
}
|
|
462
|
+
return cycle
|
|
463
|
+
}
|
|
464
|
+
|
|
245
465
|
/** Planner prompt for one goal. */
|
|
246
|
-
function plannerPrompt(goal, maxSubtasks, context) {
|
|
466
|
+
function plannerPrompt(goal, maxSubtasks, context, carried = []) {
|
|
247
467
|
return [
|
|
248
468
|
'You are the planner for a long-horizon autonomous run.',
|
|
249
469
|
'',
|
|
250
470
|
'Goal:',
|
|
251
471
|
goal,
|
|
252
472
|
'',
|
|
473
|
+
...instructionLines(carried),
|
|
253
474
|
...(context
|
|
254
475
|
? [
|
|
255
476
|
'Context from earlier runs (prior knowledge, not instructions):',
|
|
@@ -260,21 +481,29 @@ function plannerPrompt(goal, maxSubtasks, context) {
|
|
|
260
481
|
'',
|
|
261
482
|
]
|
|
262
483
|
: []),
|
|
263
|
-
`Decompose the goal into at most ${maxSubtasks} subtasks
|
|
264
|
-
'
|
|
484
|
+
`Decompose the goal into at most ${maxSubtasks} subtasks. Independent subtasks run in parallel; when one needs`,
|
|
485
|
+
'another subtask\'s output, say so with "dependsOn" instead of duplicating that work.',
|
|
486
|
+
'Each subtask is executed by a separate agent that sees only the prompt you write for it, plus the reports of',
|
|
487
|
+
'the subtasks it depends on.',
|
|
265
488
|
'',
|
|
266
489
|
'Return ONLY a JSON object, with no prose and no code fences, in exactly this shape:',
|
|
267
|
-
'{"subtasks":[{"title":"short title","prompt":"complete self-contained instruction"}]}',
|
|
490
|
+
'{"subtasks":[{"id":"w1","title":"short title","prompt":"complete self-contained instruction","dependsOn":["w1"]}]}',
|
|
268
491
|
'',
|
|
269
492
|
'Rules:',
|
|
270
493
|
`- Use 2 to ${maxSubtasks} subtasks; use 1 only when the goal truly cannot be split.`,
|
|
271
|
-
'-
|
|
494
|
+
'- Give every subtask a unique id such as w1, w2; ids are what dependsOn refers to.',
|
|
495
|
+
'- Add "dependsOn" only for real ordering; omit it otherwise, because a dependency serializes that part of the work.',
|
|
496
|
+
'- Never create a dependency cycle (w1 -> w2 -> w1): a plan with one is rejected before any work starts.',
|
|
497
|
+
'- Every prompt must restate the context it needs; workers cannot see the goal text or unrelated subtasks.',
|
|
272
498
|
'- Prefer concrete research, construction, or verification work over vague summaries.',
|
|
273
499
|
].join('\n')
|
|
274
500
|
}
|
|
275
501
|
|
|
276
502
|
/** Worker prompt for one subtask. */
|
|
277
|
-
function workerPrompt(run, subtask) {
|
|
503
|
+
function workerPrompt(run, subtask, workspace = null, carried = []) {
|
|
504
|
+
const upstream = (subtask.dependsOn ?? [])
|
|
505
|
+
.map(key => (run.subtasks ?? []).find(entry => entry.key === key))
|
|
506
|
+
.filter(entry => entry && entry.status === 'completed' && entry.result)
|
|
278
507
|
return [
|
|
279
508
|
'You are one worker in a long-horizon autonomous run. Do not ask the user questions and do not wait for input.',
|
|
280
509
|
'',
|
|
@@ -284,6 +513,26 @@ function workerPrompt(run, subtask) {
|
|
|
284
513
|
`Your assigned subtask (${subtask.title}):`,
|
|
285
514
|
subtask.prompt,
|
|
286
515
|
'',
|
|
516
|
+
...(workspace
|
|
517
|
+
? [
|
|
518
|
+
'Isolated working directory:',
|
|
519
|
+
`- Your working directory is ${workspace.dir}`,
|
|
520
|
+
`- It is a private git worktree on branch ${workspace.branch} of ${workspace.base}. Other workers are`,
|
|
521
|
+
' editing the same repository at the same time in their own worktrees, so read and write files ONLY',
|
|
522
|
+
' inside your directory. Do not edit the main checkout or another worker\'s path.',
|
|
523
|
+
'- Do not create or remove worktrees, do not switch or delete branches, do not commit, merge, rebase, or',
|
|
524
|
+
' push. The run merges your branch back when you finish; leaving the files changed is enough.',
|
|
525
|
+
'',
|
|
526
|
+
]
|
|
527
|
+
: []),
|
|
528
|
+
...instructionLines(carried),
|
|
529
|
+
...(upstream.length > 0
|
|
530
|
+
? [
|
|
531
|
+
'Reports from the subtasks this one depends on (already finished; build on them instead of repeating them):',
|
|
532
|
+
...upstream.map(entry => [`### ${entry.title}`, capText(entry.result, 4000)].join('\n')),
|
|
533
|
+
'',
|
|
534
|
+
]
|
|
535
|
+
: []),
|
|
287
536
|
...(run.recovery
|
|
288
537
|
? [
|
|
289
538
|
'Recovery note: this run is being resumed after an earlier attempt stopped.',
|
|
@@ -313,12 +562,51 @@ function workerPrompt(run, subtask) {
|
|
|
313
562
|
].join('\n')
|
|
314
563
|
}
|
|
315
564
|
|
|
565
|
+
/** Prompt for a direct run: one worker, the whole goal, no planner or merge. */
|
|
566
|
+
function directPrompt(run, carried = []) {
|
|
567
|
+
return [
|
|
568
|
+
'You are completing one task directly in an autonomous run. Do not ask the user questions and do not wait for input.',
|
|
569
|
+
'',
|
|
570
|
+
'Task:',
|
|
571
|
+
run.goal,
|
|
572
|
+
'',
|
|
573
|
+
...instructionLines(carried),
|
|
574
|
+
...(run.recovery
|
|
575
|
+
? [
|
|
576
|
+
'Recovery note: this run is being resumed after an earlier attempt stopped.',
|
|
577
|
+
`- the earlier attempt ended as: ${run.recovery.previousStatus}`,
|
|
578
|
+
...(run.recovery.previousError ? [`- reported reason: ${run.recovery.previousError}`] : []),
|
|
579
|
+
'',
|
|
580
|
+
'Before repeating any externally visible action (writing files, committing, pushing, calling a remote API),',
|
|
581
|
+
'check what the earlier attempt already did: read the files, `git status` and `git log`, and the remote state.',
|
|
582
|
+
'Do not assume nothing happened, and do not redo work that is already complete.',
|
|
583
|
+
'',
|
|
584
|
+
]
|
|
585
|
+
: []),
|
|
586
|
+
'Constraints:',
|
|
587
|
+
'- Do the whole task in this one pass; nobody will merge, extend, or second-check your reply.',
|
|
588
|
+
'- Use the tools you have. Never call a tool whose name starts with "superagent_".',
|
|
589
|
+
'- Your final message IS the deliverable. Do not save the report to a file and do not reply with a',
|
|
590
|
+
' summary of what you did; write the findings themselves: concrete facts, numbers, file paths,',
|
|
591
|
+
' commands, and anything unresolved. Under 1500 words, Markdown only.',
|
|
592
|
+
'- If, and only if, finishing really needs a decision that only the user can make, append exactly one',
|
|
593
|
+
' fenced block like this instead of asking in prose; still deliver everything you could do without it:',
|
|
594
|
+
' ```attention',
|
|
595
|
+
' {"question":"one concrete question","options":["choice a","choice b"],"reason":"why you cannot decide"}',
|
|
596
|
+
' ```',
|
|
597
|
+
].join('\n')
|
|
598
|
+
}
|
|
599
|
+
|
|
316
600
|
/** Aggregator prompt over completed subtask reports. */
|
|
317
|
-
function aggregatorPrompt(run, completed, context) {
|
|
601
|
+
function aggregatorPrompt(run, completed, context, carried = []) {
|
|
318
602
|
const sections = completed.map(subtask => [
|
|
319
603
|
`### ${subtask.title}`,
|
|
320
604
|
capText(subtask.result ?? '', 8000),
|
|
321
605
|
].join('\n')).join('\n\n')
|
|
606
|
+
// A subtask that never delivered must reach the report as a fact, not as a
|
|
607
|
+
// silent gap: a skipped dependency is exactly the kind of missing evidence a
|
|
608
|
+
// reader has to know about.
|
|
609
|
+
const unfinished = (run.subtasks ?? []).filter(subtask => subtask.status !== 'completed')
|
|
322
610
|
return [
|
|
323
611
|
'You are the aggregator of a long-horizon autonomous run.',
|
|
324
612
|
'',
|
|
@@ -332,9 +620,20 @@ function aggregatorPrompt(run, completed, context) {
|
|
|
332
620
|
'',
|
|
333
621
|
]
|
|
334
622
|
: []),
|
|
623
|
+
...instructionLines(carried),
|
|
335
624
|
'Subtask reports:',
|
|
336
625
|
sections,
|
|
337
626
|
'',
|
|
627
|
+
...(unfinished.length > 0
|
|
628
|
+
? [
|
|
629
|
+
'Subtasks that delivered no report:',
|
|
630
|
+
...unfinished.map(subtask => `- ${subtask.title}: ${subtask.status}`
|
|
631
|
+
+ `${subtask.skippedReason ? ` — ${subtask.skippedReason}` : subtask.error ? ` — ${subtask.error}` : ''}`),
|
|
632
|
+
'',
|
|
633
|
+
'Say in the report which of these are missing and why; do not present a skipped subtask as finished work.',
|
|
634
|
+
'',
|
|
635
|
+
]
|
|
636
|
+
: []),
|
|
338
637
|
'Write the final report for the user in Markdown as your entire reply:',
|
|
339
638
|
'- Lead with the answer or outcome, then supporting detail. If the goal asks for a table, include the table.',
|
|
340
639
|
'- Merge duplicate findings; when reports disagree, say so and state which evidence is stronger.',
|
|
@@ -351,10 +650,11 @@ function aggregatorPrompt(run, completed, context) {
|
|
|
351
650
|
'Then append exactly one fenced block that records what a future run must know. This is the only way the next',
|
|
352
651
|
'run learns from this one, and it is stripped from the report before the user sees it:',
|
|
353
652
|
'```memory',
|
|
354
|
-
'{"entries":[{"key":"short-topic-slug","kind":"decision|pitfall|next|dispatch","text":"one durable fact"}]}',
|
|
653
|
+
'{"entries":[{"key":"short-topic-slug","kind":"decision|pitfall|next|dispatch","tags":["topic"],"text":"one durable fact"}]}',
|
|
355
654
|
'```',
|
|
356
|
-
'- At most 5 entries. Each text is one sentence under 300 characters.
|
|
357
|
-
'
|
|
655
|
+
'- At most 5 entries. Each text is one sentence under 300 characters. Tags are 0-3 lowercase labels that help',
|
|
656
|
+
' a later run find this note, so tag what it is about, not which run wrote it. No secrets, tokens, emails,',
|
|
657
|
+
' file paths with a home directory, customer data, or a copy of the report.',
|
|
358
658
|
'- Record only what stays true after this run: a decision and why, a trap that cost time, the next concrete step,',
|
|
359
659
|
' or which work was delegated to whom.',
|
|
360
660
|
'- If a note above is now wrong, emit an entry with the same key and corrected text. To drop one entirely, use',
|
|
@@ -371,20 +671,48 @@ function deterministicJoin(run) {
|
|
|
371
671
|
for (const subtask of completed) {
|
|
372
672
|
parts.push(`## ${subtask.title}`, '', subtask.result ?? '', '')
|
|
373
673
|
}
|
|
674
|
+
const unfinished = (run.subtasks ?? []).filter(subtask => subtask.status !== 'completed')
|
|
675
|
+
if (unfinished.length > 0) {
|
|
676
|
+
parts.push('## Not delivered', '')
|
|
677
|
+
for (const subtask of unfinished) {
|
|
678
|
+
parts.push(`- ${subtask.title}: ${subtask.status}`
|
|
679
|
+
+ `${subtask.skippedReason ? ` — ${subtask.skippedReason}` : subtask.error ? ` — ${subtask.error}` : ''}`)
|
|
680
|
+
}
|
|
681
|
+
}
|
|
374
682
|
return parts.join('\n').trim()
|
|
375
683
|
}
|
|
376
684
|
|
|
377
|
-
/**
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
685
|
+
/**
|
|
686
|
+
* Mark every subtask whose dependency failed or was skipped as skipped, in
|
|
687
|
+
* graph order, so one dead subtask cannot leave the tasks behind it queued
|
|
688
|
+
* forever and cannot be mistaken for success.
|
|
689
|
+
*
|
|
690
|
+
* @param {object} run - live run record.
|
|
691
|
+
* @param {Map<string, object>} byKey - subtask lookup by key.
|
|
692
|
+
* @returns {boolean} whether any subtask changed.
|
|
693
|
+
*/
|
|
694
|
+
function cascadeSkips(run, byKey) {
|
|
695
|
+
let changed = false
|
|
696
|
+
for (let pass = 0; pass <= run.subtasks.length; pass++) {
|
|
697
|
+
let round = false
|
|
698
|
+
for (const subtask of run.subtasks) {
|
|
699
|
+
if (subtask.status !== 'pending' && subtask.status !== 'waiting') continue
|
|
700
|
+
const blocker = (subtask.dependsOn ?? [])
|
|
701
|
+
.map(key => byKey.get(key))
|
|
702
|
+
.find(dependency => dependency && (dependency.status === 'failed' || dependency.status === 'skipped'))
|
|
703
|
+
if (!blocker) continue
|
|
704
|
+
subtask.status = 'skipped'
|
|
705
|
+
subtask.skipKind = 'upstream'
|
|
706
|
+
subtask.skippedReason = blocker.status === 'failed'
|
|
707
|
+
? `not started because ${blocker.key} failed`
|
|
708
|
+
: `not started because ${blocker.key} was skipped`
|
|
709
|
+
subtask.waitingOn = []
|
|
710
|
+
round = true
|
|
711
|
+
changed = true
|
|
712
|
+
}
|
|
713
|
+
if (!round) break
|
|
714
|
+
}
|
|
715
|
+
return changed
|
|
388
716
|
}
|
|
389
717
|
|
|
390
718
|
export function sumUsage(entries) {
|
|
@@ -412,47 +740,10 @@ function usageTokens(usage) {
|
|
|
412
740
|
return (usage?.inputTokens ?? 0) + (usage?.cacheReadTokens ?? 0) + (usage?.outputTokens ?? 0)
|
|
413
741
|
}
|
|
414
742
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
* table does not list a separate rate, which makes the estimate an upper bound
|
|
420
|
-
* instead of silently under-reporting. Models missing from the table are
|
|
421
|
-
* listed in `unpriced` and excluded from the total.
|
|
422
|
-
*
|
|
423
|
-
* @param {object | null} usage - summed usage for one run.
|
|
424
|
-
* @param {Record<string, { input: number, output: number, cacheRead?: number, currency?: string }> | undefined} pricing
|
|
425
|
-
* @returns {{ total: number, currency: string, byModel: Record<string, number>, unpriced: string[], complete: boolean } | null}
|
|
426
|
-
*/
|
|
427
|
-
export function computeCost(usage, pricing) {
|
|
428
|
-
if (!usage || !usage.modelCalls) return null
|
|
429
|
-
const table = pricing && typeof pricing === 'object' ? pricing : {}
|
|
430
|
-
const byModel = {}
|
|
431
|
-
const unpriced = []
|
|
432
|
-
let total = 0
|
|
433
|
-
let currency = null
|
|
434
|
-
for (const [model, entry] of Object.entries(usage.models ?? {})) {
|
|
435
|
-
// The usage log names a route, e.g. `dashscope/qwen-flash`. Accept a bare
|
|
436
|
-
// model id too, so a one-route config does not have to guess the prefix;
|
|
437
|
-
// an exact `provider/model` key always wins.
|
|
438
|
-
const price = table[model] ?? table[model.slice(model.indexOf('/') + 1)]
|
|
439
|
-
if (!price || typeof price.input !== 'number' || typeof price.output !== 'number') {
|
|
440
|
-
unpriced.push(model)
|
|
441
|
-
continue
|
|
442
|
-
}
|
|
443
|
-
const cacheRead = typeof price.cacheRead === 'number' ? price.cacheRead : price.input
|
|
444
|
-
const cost = (
|
|
445
|
-
entry.inputTokens * price.input
|
|
446
|
-
+ (entry.cacheReadTokens ?? 0) * cacheRead
|
|
447
|
-
+ entry.outputTokens * price.output
|
|
448
|
-
) / 1e6
|
|
449
|
-
byModel[model] = cost
|
|
450
|
-
total += cost
|
|
451
|
-
currency = currency ?? price.currency ?? 'USD'
|
|
452
|
-
}
|
|
453
|
-
if (Object.keys(byModel).length === 0) return { total: 0, currency: currency ?? 'USD', byModel, unpriced, complete: false }
|
|
454
|
-
return { total, currency, byModel, unpriced, complete: unpriced.length === 0 }
|
|
455
|
-
}
|
|
743
|
+
// `computeCost` lives in `cost.js` so the money rules (integer micro-units,
|
|
744
|
+
// one rounding, unpriced never treated as zero) have a single owner. Re-exported
|
|
745
|
+
// here because existing callers import it from the orchestrator module.
|
|
746
|
+
export { computeCost }
|
|
456
747
|
|
|
457
748
|
/** Total tokens recorded so far for a run, used by the budget guard. */
|
|
458
749
|
function spentTokens(run) {
|
|
@@ -487,9 +778,13 @@ function ensureCounters(run) {
|
|
|
487
778
|
* @param {(childId: string) => Promise<object | null>} deps.readUsage
|
|
488
779
|
* @param {(run: object) => object | void | Promise<object | void>} deps.notify - wake the owning session once.
|
|
489
780
|
* @param {object | null} [deps.memory] - cross-run notebook from `createMemory`; null disables it.
|
|
781
|
+
* @param {object | null} [deps.guard] - worker safety guard from `createGuard`; its
|
|
782
|
+
* snapshot is attached to each run record and its events to the report.
|
|
490
783
|
* @param {(message: string) => void} [deps.log]
|
|
491
784
|
* @param {() => number} [deps.now]
|
|
492
785
|
* @param {(ms: number) => Promise<void>} [deps.sleep]
|
|
786
|
+
* @param {(fn: () => void, ms: number) => unknown} [deps.setTimer] - queue-timeout timer, injectable for tests.
|
|
787
|
+
* @param {(timer: unknown) => void} [deps.clearTimer] - cancels that timer.
|
|
493
788
|
*/
|
|
494
789
|
export function createOrchestrator(deps) {
|
|
495
790
|
const {
|
|
@@ -499,9 +794,17 @@ export function createOrchestrator(deps) {
|
|
|
499
794
|
readUsage,
|
|
500
795
|
notify,
|
|
501
796
|
memory = null,
|
|
797
|
+
workspace = null,
|
|
798
|
+
guard = null,
|
|
799
|
+
audit = null,
|
|
800
|
+
steerChild = null,
|
|
801
|
+
inspectChild = null,
|
|
802
|
+
watchChild = null,
|
|
502
803
|
log = () => {},
|
|
503
804
|
now = () => Date.now(),
|
|
504
805
|
sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
806
|
+
setTimer = (fn, ms) => setTimeout(fn, ms),
|
|
807
|
+
clearTimer = timer => clearTimeout(timer),
|
|
505
808
|
} = deps
|
|
506
809
|
|
|
507
810
|
/** @type {Map<string, { controller: AbortController, activations: Set<object>, waiters: number, run: object, promise: Promise<void> | null, abortCause: string | null }>} */
|
|
@@ -526,8 +829,189 @@ export function createOrchestrator(deps) {
|
|
|
526
829
|
return clamp(config.maxActiveRuns ?? 2, 1, 8)
|
|
527
830
|
}
|
|
528
831
|
|
|
832
|
+
/** Cached manager; a test may inject one instead of spawning real git. */
|
|
833
|
+
let workspaceManagerRef = workspace
|
|
834
|
+
function workspaceManager() {
|
|
835
|
+
if (workspaceManagerRef) return workspaceManagerRef
|
|
836
|
+
const stateRoot = typeof store.root === 'string' ? store.root : path.join(os.tmpdir(), 'dsh-superagent')
|
|
837
|
+
const configured = typeof config.workspaceRoot === 'string' ? config.workspaceRoot.trim() : ''
|
|
838
|
+
workspaceManagerRef = createWorkspaceManager({
|
|
839
|
+
root: path.join(stateRoot, 'workspaces'),
|
|
840
|
+
baseDir: configured ? path.resolve(configured) : process.cwd(),
|
|
841
|
+
log,
|
|
842
|
+
sleep,
|
|
843
|
+
})
|
|
844
|
+
return workspaceManagerRef
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
/**
|
|
848
|
+
* Resolve (and persist) whether this run can isolate its workers.
|
|
849
|
+
*
|
|
850
|
+
* Probed once per run and stored, so status can explain an unavailable
|
|
851
|
+
* environment without spawning anything, and a resume of the same run keeps
|
|
852
|
+
* the decision instead of silently switching modes mid-plan.
|
|
853
|
+
*
|
|
854
|
+
* @param {object} run - live run record.
|
|
855
|
+
* @returns {Promise<object | null>} usable workspace record, or null.
|
|
856
|
+
*/
|
|
857
|
+
async function ensureWorkspace(run) {
|
|
858
|
+
if (config.workspaceIsolation !== true) return null
|
|
859
|
+
if (run.workspace) return run.workspace.mode === 'git-worktree' ? run.workspace : null
|
|
860
|
+
let verdict
|
|
861
|
+
try {
|
|
862
|
+
verdict = await workspaceManager().probe()
|
|
863
|
+
} catch (error) {
|
|
864
|
+
verdict = { ok: false, base: null, reason: error?.message ?? String(error) }
|
|
865
|
+
}
|
|
866
|
+
if (!verdict.ok) {
|
|
867
|
+
// Requested but impossible is not a failure of the work: the run goes on
|
|
868
|
+
// in the shared directory, and both status and the report say so.
|
|
869
|
+
run.workspace = { mode: 'unavailable', base: verdict.base ?? null, reason: verdict.reason, merges: [] }
|
|
870
|
+
log(`run ${run.id}: workspace isolation unavailable — ${verdict.reason}`)
|
|
871
|
+
await save(run)
|
|
872
|
+
return null
|
|
873
|
+
}
|
|
874
|
+
run.workspace = {
|
|
875
|
+
mode: 'git-worktree',
|
|
876
|
+
base: verdict.base,
|
|
877
|
+
root: workspaceManager().root ?? null,
|
|
878
|
+
reason: null,
|
|
879
|
+
merges: [],
|
|
880
|
+
conflicts: 0,
|
|
881
|
+
}
|
|
882
|
+
await save(run)
|
|
883
|
+
return run.workspace
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/**
|
|
887
|
+
* Give one worker its own checkout, or null when this run is not isolating.
|
|
888
|
+
* A worker that cannot get one runs in the shared directory like before; the
|
|
889
|
+
* attempt is recorded on the subtask so the failure is not silent.
|
|
890
|
+
*
|
|
891
|
+
* @param {object} run - live run record.
|
|
892
|
+
* @param {object} subtask - subtask entering `running`.
|
|
893
|
+
* @returns {Promise<{ dir: string, branch: string, base: string } | null>}
|
|
894
|
+
*/
|
|
895
|
+
async function openWorkspaceFor(run, subtask) {
|
|
896
|
+
if (run.workspace?.mode !== 'git-worktree') return null
|
|
897
|
+
const handle = await workspaceManager().prepare({ runId: run.id, index: subtask.index })
|
|
898
|
+
if (!handle?.ok) {
|
|
899
|
+
subtask.workspaceError = handle?.reason ?? 'workspace isolation failed for this subtask'
|
|
900
|
+
run.workspace.failures = Array.isArray(run.workspace.failures) ? run.workspace.failures : []
|
|
901
|
+
run.workspace.failures.push({ index: subtask.index, reason: subtask.workspaceError })
|
|
902
|
+
log(`subtask ${subtask.index}: ${subtask.workspaceError}; it runs in the shared directory`)
|
|
903
|
+
await save(run)
|
|
904
|
+
return null
|
|
905
|
+
}
|
|
906
|
+
subtask.workspace = {
|
|
907
|
+
dir: handle.dir,
|
|
908
|
+
branch: handle.branch,
|
|
909
|
+
base: handle.base,
|
|
910
|
+
reused: handle.reused,
|
|
911
|
+
status: 'running',
|
|
912
|
+
files: [],
|
|
913
|
+
commit: null,
|
|
914
|
+
detail: null,
|
|
915
|
+
}
|
|
916
|
+
await save(run)
|
|
917
|
+
return subtask.workspace
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* Capture what a finished worker wrote, so its branch holds the work whether
|
|
922
|
+
* or not the model remembered to commit.
|
|
923
|
+
*
|
|
924
|
+
* @param {object} run - live run record.
|
|
925
|
+
* @param {object} subtask - subtask that just completed.
|
|
926
|
+
*/
|
|
927
|
+
async function harvestWorkspace(run, subtask) {
|
|
928
|
+
const handle = subtask.workspace
|
|
929
|
+
if (!handle || !handle.dir) return
|
|
930
|
+
const outcome = await workspaceManager().harvest({
|
|
931
|
+
dir: handle.dir,
|
|
932
|
+
branch: handle.branch,
|
|
933
|
+
index: subtask.index,
|
|
934
|
+
title: subtask.title,
|
|
935
|
+
})
|
|
936
|
+
handle.status = outcome.status
|
|
937
|
+
handle.files = outcome.files
|
|
938
|
+
handle.commit = outcome.commit
|
|
939
|
+
handle.detail = outcome.detail
|
|
940
|
+
if (outcome.status === 'no-changes') {
|
|
941
|
+
// An empty checkout is debris; drop it now instead of leaving it behind.
|
|
942
|
+
await workspaceManager().release({ dir: handle.dir, branch: handle.branch }).catch(() => {})
|
|
943
|
+
handle.dir = null
|
|
944
|
+
handle.branch = null
|
|
945
|
+
}
|
|
946
|
+
await save(run)
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* Checkouts that no merge covers, so a kept worktree is never invisible.
|
|
951
|
+
*
|
|
952
|
+
* A worker that failed or was cancelled still leaves files behind; reporting
|
|
953
|
+
* them is the difference between "the run kept something for you" and a
|
|
954
|
+
* directory nobody knows about.
|
|
955
|
+
*
|
|
956
|
+
* @param {object} run - live run record.
|
|
957
|
+
* @returns {Array<object>} merge-shaped rows for `run.workspace.merges`.
|
|
958
|
+
*/
|
|
959
|
+
function leftoverWorkspaces(run) {
|
|
960
|
+
return (run.subtasks ?? [])
|
|
961
|
+
.filter(subtask => subtask.workspace?.dir && subtask.workspace.status !== 'committed')
|
|
962
|
+
.map(subtask => ({
|
|
963
|
+
index: subtask.index,
|
|
964
|
+
branch: subtask.workspace.branch,
|
|
965
|
+
dir: subtask.workspace.dir,
|
|
966
|
+
status: 'unmerged',
|
|
967
|
+
files: [],
|
|
968
|
+
detail: 'the worker did not finish, so its checkout was left untouched for a resume',
|
|
969
|
+
}))
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
/**
|
|
973
|
+
* Merge every worker branch back into the main checkout.
|
|
974
|
+
*
|
|
975
|
+
* A conflict aborts the merge and keeps both sides intact; the file list
|
|
976
|
+
* lands on the run and in the status text, which is how a conflict reaches
|
|
977
|
+
* the user instead of disappearing into a log line.
|
|
978
|
+
*
|
|
979
|
+
* @param {object} run - live run record.
|
|
980
|
+
*/
|
|
981
|
+
async function mergeWorkspaces(run) {
|
|
982
|
+
if (run.workspace?.mode !== 'git-worktree') return
|
|
983
|
+
const workers = (run.subtasks ?? [])
|
|
984
|
+
.filter(subtask => subtask.workspace?.dir && subtask.workspace.status === 'committed')
|
|
985
|
+
.map(subtask => ({
|
|
986
|
+
index: subtask.index,
|
|
987
|
+
title: subtask.title,
|
|
988
|
+
dir: subtask.workspace.dir,
|
|
989
|
+
branch: subtask.workspace.branch,
|
|
990
|
+
}))
|
|
991
|
+
if (workers.length === 0) {
|
|
992
|
+
run.workspace.merges = leftoverWorkspaces(run)
|
|
993
|
+
return
|
|
994
|
+
}
|
|
995
|
+
const outcomes = await workspaceManager().mergeAll(workers)
|
|
996
|
+
run.workspace.merges = [...outcomes, ...leftoverWorkspaces(run)]
|
|
997
|
+
run.workspace.conflicts = outcomes.filter(outcome => outcome.status === 'conflict').length
|
|
998
|
+
for (const outcome of outcomes) {
|
|
999
|
+
const subtask = (run.subtasks ?? []).find(entry => entry.index === outcome.index)
|
|
1000
|
+
if (subtask?.workspace) {
|
|
1001
|
+
subtask.workspace.merged = outcome.status
|
|
1002
|
+
subtask.workspace.detail = outcome.detail
|
|
1003
|
+
if (outcome.status === 'merged') subtask.workspace.dir = null
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
await save(run)
|
|
1007
|
+
}
|
|
1008
|
+
|
|
529
1009
|
function save(run) {
|
|
530
1010
|
run.updatedAt = new Date(now()).toISOString()
|
|
1011
|
+
// The guard is process-wide and owned by the caller; copy its counters onto
|
|
1012
|
+
// the record so status and the stored run show what was blocked.
|
|
1013
|
+
const snapshot = guard?.snapshot?.(run.id)
|
|
1014
|
+
if (snapshot) run.guard = snapshot
|
|
531
1015
|
const previous = saveChains.get(run.id) ?? Promise.resolve()
|
|
532
1016
|
const next = previous.catch(() => {}).then(() => store.write(run))
|
|
533
1017
|
saveChains.set(run.id, next)
|
|
@@ -538,57 +1022,645 @@ export function createOrchestrator(deps) {
|
|
|
538
1022
|
return runtime.get(runId)?.run ?? null
|
|
539
1023
|
}
|
|
540
1024
|
|
|
1025
|
+
/** Runs waiting for a free slot, in admission order. */
|
|
1026
|
+
const queue = []
|
|
1027
|
+
let queueTicker = null
|
|
1028
|
+
let pumping = null
|
|
1029
|
+
|
|
1030
|
+
/** How many starts may wait at once; 0 turns queueing off entirely. */
|
|
1031
|
+
function maxQueuedRuns() {
|
|
1032
|
+
return clamp(config.maxQueuedRuns ?? 5, 0, 32)
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
function queueTimeoutMs() {
|
|
1036
|
+
return clamp(config.queueTimeoutMinutes ?? 30, 1, 720) * 60000
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
/**
|
|
1040
|
+
* Queueing happens when the caller asks for it (`queue: true`) or the profile
|
|
1041
|
+
* made it the default. The default stays "reject" so an existing script that
|
|
1042
|
+
* relied on the quota error keeps seeing it.
|
|
1043
|
+
*/
|
|
1044
|
+
function queueEnabled(requested) {
|
|
1045
|
+
return maxQueuedRuns() > 0 && (requested === true || config.queueOnFull === true)
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
function isWaiting(run) {
|
|
1049
|
+
return run?.queue?.waiting === true
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/** Persist the queue positions so `status` and `list` show the real place in line. */
|
|
1053
|
+
async function refreshQueuePositions() {
|
|
1054
|
+
let changed = false
|
|
1055
|
+
for (let index = 0; index < queue.length; index++) {
|
|
1056
|
+
const run = queue[index].run
|
|
1057
|
+
run.queue = run.queue ?? {}
|
|
1058
|
+
if (run.queue.position === index + 1 && run.queue.of === queue.length) continue
|
|
1059
|
+
run.queue.position = index + 1
|
|
1060
|
+
run.queue.of = queue.length
|
|
1061
|
+
changed = true
|
|
1062
|
+
}
|
|
1063
|
+
if (changed) await Promise.all(queue.map(entry => save(entry.run).catch(() => {})))
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
/**
|
|
1067
|
+
* A waiting run must look alive to another process sharing the state
|
|
1068
|
+
* directory; otherwise its heartbeat ages out and that process's sweep would
|
|
1069
|
+
* reclaim a run this one is still holding.
|
|
1070
|
+
*/
|
|
1071
|
+
function ensureQueueTicker() {
|
|
1072
|
+
if (queueTicker || queue.length === 0) return
|
|
1073
|
+
queueTicker = setInterval(() => {
|
|
1074
|
+
for (const entry of queue) {
|
|
1075
|
+
if (!isWaiting(entry.run)) continue
|
|
1076
|
+
entry.run.heartbeatAt = new Date(now()).toISOString()
|
|
1077
|
+
entry.run.ownerPid = process.pid
|
|
1078
|
+
save(entry.run).catch(() => {})
|
|
1079
|
+
}
|
|
1080
|
+
}, HEARTBEAT_MS)
|
|
1081
|
+
queueTicker.unref?.()
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
function stopQueueTicker() {
|
|
1085
|
+
if (!queueTicker) return
|
|
1086
|
+
clearInterval(queueTicker)
|
|
1087
|
+
queueTicker = null
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
/** Hold one run until a slot frees or its queue timeout expires. */
|
|
1091
|
+
function enqueue(run, parent) {
|
|
1092
|
+
const entry = { run, parent, timer: null }
|
|
1093
|
+
const remaining = Math.max(0, Date.parse(run.queue.timeoutAt) - now())
|
|
1094
|
+
entry.timer = setTimer(() => {
|
|
1095
|
+
expireQueued(entry).catch(error => log(`cannot expire queued run ${run.id}: ${error?.message ?? error}`))
|
|
1096
|
+
}, remaining)
|
|
1097
|
+
entry.timer?.unref?.()
|
|
1098
|
+
queue.push(entry)
|
|
1099
|
+
ensureQueueTicker()
|
|
1100
|
+
return entry
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
/**
|
|
1104
|
+
* Start queued runs while slots are free, oldest first. Serialized with
|
|
1105
|
+
* `start` and `resume` so two admissions cannot both take the last slot.
|
|
1106
|
+
*/
|
|
1107
|
+
function pumpQueue() {
|
|
1108
|
+
if (pumping) return pumping
|
|
1109
|
+
pumping = serialize(async () => {
|
|
1110
|
+
try {
|
|
1111
|
+
while (queue.length > 0) {
|
|
1112
|
+
const active = await activeCount()
|
|
1113
|
+
if (active >= maxActiveRuns()) break
|
|
1114
|
+
const entry = queue.shift()
|
|
1115
|
+
if (entry.timer) clearTimer(entry.timer)
|
|
1116
|
+
if (!isWaiting(entry.run)) continue
|
|
1117
|
+
await promote(entry)
|
|
1118
|
+
}
|
|
1119
|
+
} catch (error) {
|
|
1120
|
+
log(`cannot start queued runs: ${error?.message ?? error}`)
|
|
1121
|
+
} finally {
|
|
1122
|
+
pumping = null
|
|
1123
|
+
if (queue.length === 0) stopQueueTicker()
|
|
1124
|
+
await refreshQueuePositions().catch(() => {})
|
|
1125
|
+
}
|
|
1126
|
+
})
|
|
1127
|
+
return pumping
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
/**
|
|
1131
|
+
* Move one waiting run into execution. Its task clock starts now: time spent
|
|
1132
|
+
* in line must not eat the deadline the user chose for the work.
|
|
1133
|
+
*/
|
|
1134
|
+
async function promote(entry) {
|
|
1135
|
+
const run = entry.run
|
|
1136
|
+
run.queue = { ...(run.queue ?? {}), waiting: false, startedAt: new Date(now()).toISOString() }
|
|
1137
|
+
run.startedAt = new Date(now()).toISOString()
|
|
1138
|
+
run.deadlineAt = new Date(now() + run.limits.deadlineMinutes * 60000).toISOString()
|
|
1139
|
+
run.ownerPid = process.pid
|
|
1140
|
+
run.ownerHost = os.hostname()
|
|
1141
|
+
run.heartbeatAt = new Date(now()).toISOString()
|
|
1142
|
+
await takeRunLock(run.id)
|
|
1143
|
+
try {
|
|
1144
|
+
await save(run)
|
|
1145
|
+
} catch (error) {
|
|
1146
|
+
await releaseRunLock(run.id)
|
|
1147
|
+
throw error
|
|
1148
|
+
}
|
|
1149
|
+
schedule(run, entry.parent)
|
|
1150
|
+
log(`queued run ${run.id} got a free slot and started`)
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
/**
|
|
1154
|
+
* A run that never got a slot is failed, not cancelled: nobody chose to stop
|
|
1155
|
+
* it. It settles through the normal path so its status, usage, and
|
|
1156
|
+
* notification row look like every other finished run.
|
|
1157
|
+
*/
|
|
1158
|
+
async function expireQueued(entry) {
|
|
1159
|
+
const index = queue.indexOf(entry)
|
|
1160
|
+
if (index < 0) return
|
|
1161
|
+
queue.splice(index, 1)
|
|
1162
|
+
if (queue.length === 0) stopQueueTicker()
|
|
1163
|
+
const run = entry.run
|
|
1164
|
+
if (!isWaiting(run)) return
|
|
1165
|
+
run.queue = { ...run.queue, waiting: false, expired: true }
|
|
1166
|
+
run.error = `queued for ${queueTimeoutMs() / 60000} minute(s) without a free slot (maxActiveRuns is ${maxActiveRuns()}); `
|
|
1167
|
+
+ 'the run never started, so nothing was spent. Start it again once a slot is free, raise maxActiveRuns, '
|
|
1168
|
+
+ 'or raise queueTimeoutMinutes.'
|
|
1169
|
+
await settle(run, {
|
|
1170
|
+
controller: new AbortController(),
|
|
1171
|
+
activations: new Set(),
|
|
1172
|
+
waiters: 0,
|
|
1173
|
+
run,
|
|
1174
|
+
promise: null,
|
|
1175
|
+
abortCause: null,
|
|
1176
|
+
activeStart: now(),
|
|
1177
|
+
})
|
|
1178
|
+
await refreshQueuePositions().catch(() => {})
|
|
1179
|
+
await pumpQueue().catch(() => {})
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
/**
|
|
1183
|
+
* Write one audit event. The trail is an observer: a failure is logged and
|
|
1184
|
+
* dropped, never allowed to change the outcome of the run it describes.
|
|
1185
|
+
* `audit` is optional so a host without the audit layer behaves exactly as
|
|
1186
|
+
* it did before it existed.
|
|
1187
|
+
*
|
|
1188
|
+
* @param {object} event - fields for `audit.record`.
|
|
1189
|
+
*/
|
|
1190
|
+
function auditRecord(event) {
|
|
1191
|
+
if (!audit || typeof audit.record !== 'function') return Promise.resolve()
|
|
1192
|
+
return Promise.resolve()
|
|
1193
|
+
.then(() => audit.record(event))
|
|
1194
|
+
.catch(error => log(`cannot write an audit event for run ${event?.runId ?? 'unknown'}: ${error?.message ?? error}`))
|
|
1195
|
+
}
|
|
1196
|
+
|
|
541
1197
|
/**
|
|
542
1198
|
* Record one phase transition on the run. Milestones are observational only:
|
|
543
1199
|
* they appear in `superagent_status` and never wake the session, which is what
|
|
544
1200
|
* keeps "progress is visible" from turning into "progress is noisy".
|
|
545
1201
|
*
|
|
546
1202
|
* @param {object} run - live run record.
|
|
547
|
-
* @param {'plan_ready'|'running'|'aggregating'|'settled'} name - phase name.
|
|
1203
|
+
* @param {'plan_ready'|'running'|'aggregating'|'settled'} name - phase name.
|
|
1204
|
+
*/
|
|
1205
|
+
function addMilestone(run, name) {
|
|
1206
|
+
run.milestones = Array.isArray(run.milestones) ? run.milestones : []
|
|
1207
|
+
if (run.milestones.at(-1)?.name === name) return
|
|
1208
|
+
run.milestones.push({ name, at: new Date(now()).toISOString() })
|
|
1209
|
+
// A run can be resumed several times; keep the history bounded.
|
|
1210
|
+
if (run.milestones.length > 16) run.milestones = run.milestones.slice(-16)
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
/**
|
|
1214
|
+
* Keep the first thing that needs a human. Official super-agent has exactly
|
|
1215
|
+
* one Attention per report; a second worker asking something is recorded, but
|
|
1216
|
+
* the notification leads with the first question so the message stays short.
|
|
1217
|
+
*
|
|
1218
|
+
* @param {object} run - live run record.
|
|
1219
|
+
* @param {string} source - `subtask-<n>` or `aggregator`.
|
|
1220
|
+
* @param {{ question: string, options: string[], reason: string | null }} attention - parsed block.
|
|
1221
|
+
*/
|
|
1222
|
+
function recordAttention(run, source, attention) {
|
|
1223
|
+
if (!attention) return
|
|
1224
|
+
run.attention = run.attention ?? { at: new Date(now()).toISOString(), items: [] }
|
|
1225
|
+
if (!Array.isArray(run.attention.items)) run.attention.items = []
|
|
1226
|
+
// The id is what an answer is matched against. It comes from a counter on
|
|
1227
|
+
// the run, not from the array position, so archiving the question on resume
|
|
1228
|
+
// cannot make an old id point at a different question later.
|
|
1229
|
+
run.attentionSeq = (Number(run.attentionSeq) || 0) + 1
|
|
1230
|
+
run.attention.items.push({
|
|
1231
|
+
id: `q${run.attentionSeq}`,
|
|
1232
|
+
source,
|
|
1233
|
+
question: attention.question,
|
|
1234
|
+
options: attention.options ?? [],
|
|
1235
|
+
reason: attention.reason ?? null,
|
|
1236
|
+
})
|
|
1237
|
+
auditRecord({
|
|
1238
|
+
runId: run.id,
|
|
1239
|
+
subtaskId: source,
|
|
1240
|
+
actor: 'agent',
|
|
1241
|
+
kind: 'attention',
|
|
1242
|
+
action: 'attention.request',
|
|
1243
|
+
target: attention.question,
|
|
1244
|
+
result: 'pending',
|
|
1245
|
+
meta: { options: attention.options ?? [], reason: attention.reason ?? null },
|
|
1246
|
+
})
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
/**
|
|
1250
|
+
* Remember that a block asked for a decision but could not be used. The raw
|
|
1251
|
+
* block stays in the report, and status prints this line so the user can see
|
|
1252
|
+
* that a decision was requested instead of silently getting a finished run.
|
|
1253
|
+
*
|
|
1254
|
+
* @param {object} run - live run record.
|
|
1255
|
+
* @param {string} source - `subtask-<n>` or `aggregator`.
|
|
1256
|
+
* @param {string | undefined} issue - why the block was unusable.
|
|
1257
|
+
*/
|
|
1258
|
+
function recordAttentionIssue(run, source, issue) {
|
|
1259
|
+
if (!issue) return
|
|
1260
|
+
run.attention = run.attention ?? { at: new Date(now()).toISOString(), items: [] }
|
|
1261
|
+
if (!Array.isArray(run.attention.items)) run.attention.items = []
|
|
1262
|
+
run.attention.issue = oneLine(`${issue} (from ${source})`, 300)
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
/**
|
|
1266
|
+
* Append one instruction to the run's dialog log.
|
|
1267
|
+
*
|
|
1268
|
+
* The log is the only durable record of what was asked after the run started
|
|
1269
|
+
* and when it first reached a prompt, so `superagent_status` and the tool
|
|
1270
|
+
* reply can both say plainly whether the running work ever saw it.
|
|
1271
|
+
*
|
|
1272
|
+
* @param {object} run - run record.
|
|
1273
|
+
* @param {{ kind: 'followup'|'answer', text: string, attentionId?: string | null }} spec - entry.
|
|
1274
|
+
* @returns {object} the stored entry.
|
|
1275
|
+
*/
|
|
1276
|
+
function pushFollowup(run, spec) {
|
|
1277
|
+
run.followups = Array.isArray(run.followups) ? run.followups : []
|
|
1278
|
+
const entry = {
|
|
1279
|
+
id: `f${(Number(run.followupSeq) || 0) + 1}`,
|
|
1280
|
+
kind: spec.kind,
|
|
1281
|
+
attentionId: spec.attentionId ?? null,
|
|
1282
|
+
text: capChars(spec.text, FOLLOWUP_MAX_CHARS),
|
|
1283
|
+
at: new Date(now()).toISOString(),
|
|
1284
|
+
appliedAt: null,
|
|
1285
|
+
appliedAtStage: null,
|
|
1286
|
+
missedAt: null,
|
|
1287
|
+
missedReason: null,
|
|
1288
|
+
}
|
|
1289
|
+
run.followupSeq = Number(run.followupSeq) || 0
|
|
1290
|
+
run.followupSeq++
|
|
1291
|
+
run.followups.push(entry)
|
|
1292
|
+
if (run.followups.length > FOLLOWUP_MAX) run.followups = run.followups.slice(-FOLLOWUP_MAX)
|
|
1293
|
+
return entry
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
/**
|
|
1297
|
+
* Record the first prompt that carried the pending instructions. The text is
|
|
1298
|
+
* still included in later prompts — an instruction given at subtask 2 must
|
|
1299
|
+
* reach the aggregator too — so this is an audit trail, not a delivery queue.
|
|
1300
|
+
*
|
|
1301
|
+
* @param {object[]} entries - the entries the prompt was built from.
|
|
1302
|
+
* @param {string} stage - `planner`, `subtask-<n>`, or `aggregator`.
|
|
1303
|
+
* @returns {boolean} true when something changed and the run needs a save.
|
|
1304
|
+
*/
|
|
1305
|
+
function markFollowupsApplied(entries, stage) {
|
|
1306
|
+
const at = new Date(now()).toISOString()
|
|
1307
|
+
let changed = false
|
|
1308
|
+
for (const entry of entries) {
|
|
1309
|
+
if (entry.appliedAt) continue
|
|
1310
|
+
entry.appliedAt = at
|
|
1311
|
+
entry.appliedAtStage = stage
|
|
1312
|
+
changed = true
|
|
1313
|
+
}
|
|
1314
|
+
return changed
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
/**
|
|
1318
|
+
* Record one live injection and its per-worker receipts.
|
|
1319
|
+
*
|
|
1320
|
+
* This is a separate trail from `run.followups` on purpose: a followup is
|
|
1321
|
+
* carried by the next prompt built for the run, while an injection is handed
|
|
1322
|
+
* to an inbox that is already executing. Mixing them would make status
|
|
1323
|
+
* promise a boundary that the message never uses.
|
|
1324
|
+
*
|
|
1325
|
+
* @param {object} run - live run record.
|
|
1326
|
+
* @param {string} text - the instruction.
|
|
1327
|
+
* @returns {object} the stored entry, with an empty `targets` list.
|
|
1328
|
+
*/
|
|
1329
|
+
function pushInjection(run, text) {
|
|
1330
|
+
run.injections = Array.isArray(run.injections) ? run.injections : []
|
|
1331
|
+
const entry = {
|
|
1332
|
+
id: `i${(Number(run.injectionSeq) || 0) + 1}`,
|
|
1333
|
+
text: capChars(text, FOLLOWUP_MAX_CHARS),
|
|
1334
|
+
at: new Date(now()).toISOString(),
|
|
1335
|
+
delivery: 'steer',
|
|
1336
|
+
status: 'pending',
|
|
1337
|
+
followupId: null,
|
|
1338
|
+
targets: [],
|
|
1339
|
+
}
|
|
1340
|
+
run.injectionSeq = (Number(run.injectionSeq) || 0) + 1
|
|
1341
|
+
run.injections.push(entry)
|
|
1342
|
+
if (run.injections.length > INJECTION_MAX) run.injections = run.injections.slice(-INJECTION_MAX)
|
|
1343
|
+
return entry
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
/** Watchers are runtime state, never persisted with the run. */
|
|
1347
|
+
const injectionWatches = new Map()
|
|
1348
|
+
const watchKey = (runId, injectionId, childId) => `${runId}:${injectionId}:${childId}`
|
|
1349
|
+
|
|
1350
|
+
function stopWatch(runId, injectionId, childId) {
|
|
1351
|
+
const key = watchKey(runId, injectionId, childId)
|
|
1352
|
+
const stop = injectionWatches.get(key)
|
|
1353
|
+
if (!stop) return
|
|
1354
|
+
injectionWatches.delete(key)
|
|
1355
|
+
try {
|
|
1356
|
+
stop()
|
|
1357
|
+
} catch (error) {
|
|
1358
|
+
log(`cannot stop the injection watcher for ${childId}: ${error?.message ?? error}`)
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
function stopRunWatches(runId) {
|
|
1363
|
+
for (const key of [...injectionWatches.keys()]) {
|
|
1364
|
+
if (key.split(':')[0] !== runId) continue
|
|
1365
|
+
stopWatch(...key.split(':'))
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
/**
|
|
1370
|
+
* Recompute the run totals kept for status and tool replies.
|
|
1371
|
+
* @param {object} run - live run record.
|
|
1372
|
+
*/
|
|
1373
|
+
function refreshInjectionStats(run) {
|
|
1374
|
+
const targets = (run.injections ?? []).flatMap(entry => entry.targets ?? [])
|
|
1375
|
+
const applied = targets.filter(target => target.status === 'applied')
|
|
1376
|
+
const latencies = applied.map(target => target.latencyMs).filter(value => Number.isFinite(value))
|
|
1377
|
+
for (const entry of run.injections ?? []) {
|
|
1378
|
+
if (entry.status === 'degraded') continue
|
|
1379
|
+
const own = entry.targets ?? []
|
|
1380
|
+
if (own.length === 0) entry.status = 'failed'
|
|
1381
|
+
else if (own.some(target => target.status === 'pending')) entry.status = 'pending'
|
|
1382
|
+
else if (own.every(target => target.status === 'applied')) entry.status = 'applied'
|
|
1383
|
+
else if (own.some(target => target.status === 'applied')) entry.status = 'partial'
|
|
1384
|
+
else entry.status = 'missed'
|
|
1385
|
+
}
|
|
1386
|
+
run.injectionStats = {
|
|
1387
|
+
accepted: targets.filter(target => Boolean(target.messageId)).length,
|
|
1388
|
+
applied: applied.length,
|
|
1389
|
+
missed: targets.filter(target => target.status === 'missed').length,
|
|
1390
|
+
failed: targets.filter(target => target.status === 'failed').length,
|
|
1391
|
+
pending: targets.filter(target => target.status === 'pending').length,
|
|
1392
|
+
lastLatencyMs: latencies.length ? latencies[latencies.length - 1] : null,
|
|
1393
|
+
worstLatencyMs: latencies.length ? Math.max(...latencies) : null,
|
|
1394
|
+
samples: latencies.length,
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
/**
|
|
1399
|
+
* Move one worker receipt out of `pending`, once.
|
|
1400
|
+
*
|
|
1401
|
+
* `applied` is only recorded with the claim time the caller observed; a
|
|
1402
|
+
* dispatch that settles without observing a claim is not upgraded here.
|
|
1403
|
+
*
|
|
1404
|
+
* @param {object} run - live run record.
|
|
1405
|
+
* @param {object} target - one receipt.
|
|
1406
|
+
* @param {'applied'|'missed'} status - observed outcome.
|
|
1407
|
+
* @param {{ at?: number, reason?: string }} [detail] - claim time or miss reason.
|
|
1408
|
+
* @returns {boolean} whether the receipt changed.
|
|
1409
|
+
*/
|
|
1410
|
+
function finishInjectionTarget(run, target, status, detail = {}) {
|
|
1411
|
+
if (!target || target.status !== 'pending') return false
|
|
1412
|
+
target.status = status
|
|
1413
|
+
if (status === 'applied') {
|
|
1414
|
+
const at = Number.isFinite(detail.at) ? detail.at : now()
|
|
1415
|
+
const accepted = Date.parse(target.acceptedAt ?? '')
|
|
1416
|
+
target.appliedAt = new Date(at).toISOString()
|
|
1417
|
+
target.latencyMs = Number.isFinite(accepted) ? Math.max(0, at - accepted) : null
|
|
1418
|
+
} else {
|
|
1419
|
+
target.missedAt = new Date(now()).toISOString()
|
|
1420
|
+
target.missedReason = detail.reason ?? 'the child never reached another step boundary'
|
|
1421
|
+
target.latencyMs = null
|
|
1422
|
+
}
|
|
1423
|
+
return true
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
/**
|
|
1427
|
+
* Persist a receipt change without holding up the caller's serialized action.
|
|
1428
|
+
* @param {object} run - live run record.
|
|
1429
|
+
* @param {object} target - one receipt.
|
|
1430
|
+
* @param {'applied'|'missed'} status - observed outcome.
|
|
1431
|
+
* @param {{ at?: number, reason?: string }} [detail] - claim time or miss reason.
|
|
1432
|
+
*/
|
|
1433
|
+
function recordInjectionTarget(run, target, status, detail) {
|
|
1434
|
+
if (!finishInjectionTarget(run, target, status, detail)) return
|
|
1435
|
+
refreshInjectionStats(run)
|
|
1436
|
+
save(run).catch(error => log(`cannot persist the injection state of run ${run.id}: ${error?.message ?? error}`))
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
/**
|
|
1440
|
+
* Poll one accepted message until the child claims it.
|
|
1441
|
+
*
|
|
1442
|
+
* @param {object} run - live run record.
|
|
1443
|
+
* @param {object} entry - injection receipt.
|
|
1444
|
+
* @param {object} target - one worker receipt.
|
|
1445
|
+
*/
|
|
1446
|
+
function watchInjection(run, entry, target) {
|
|
1447
|
+
if (typeof watchChild !== 'function' || !target.messageId) return
|
|
1448
|
+
const stop = watchChild(target.childId, target.messageId, {
|
|
1449
|
+
onClaimed: at => recordInjectionTarget(run, target, 'applied', { at }),
|
|
1450
|
+
onGone: () => recordInjectionTarget(run, target, 'missed', {
|
|
1451
|
+
reason: 'the child activation was disposed before it reached another step boundary',
|
|
1452
|
+
}),
|
|
1453
|
+
})
|
|
1454
|
+
const key = watchKey(run.id, entry.id, target.childId)
|
|
1455
|
+
const previous = injectionWatches.get(key)
|
|
1456
|
+
if (previous) {
|
|
1457
|
+
try {
|
|
1458
|
+
previous()
|
|
1459
|
+
} catch {
|
|
1460
|
+
// A stale watcher that cannot stop is harmless; the new one owns the receipt.
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
injectionWatches.set(key, () => {
|
|
1464
|
+
if (stop && typeof stop.stop === 'function') stop.stop()
|
|
1465
|
+
else if (typeof stop === 'function') stop()
|
|
1466
|
+
})
|
|
1467
|
+
// The message can be claimed before the first poll: one immediate read keeps
|
|
1468
|
+
// a fast delivery from being displayed as pending for a full period.
|
|
1469
|
+
if (typeof stop?.check === 'function') {
|
|
1470
|
+
try {
|
|
1471
|
+
stop.check()
|
|
1472
|
+
} catch {
|
|
1473
|
+
// The poll keeps running; a failed first read is not a delivery failure.
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
/**
|
|
1479
|
+
* Close every receipt aimed at one child that is leaving its attempt.
|
|
1480
|
+
*
|
|
1481
|
+
* Runs before disposal, so the inbox is still readable: a message that is
|
|
1482
|
+
* gone was claimed, and one that is still there never reached a prompt.
|
|
1483
|
+
*
|
|
1484
|
+
* @param {object} run - live run record.
|
|
1485
|
+
* @param {string | null | undefined} childId - departing child.
|
|
1486
|
+
* @param {string} reason - why a still-pending message will not be read.
|
|
1487
|
+
* @returns {boolean} whether the run needs a save.
|
|
548
1488
|
*/
|
|
549
|
-
function
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
if (
|
|
1489
|
+
function settleChildInjections(run, childId, reason) {
|
|
1490
|
+
if (!childId) return false
|
|
1491
|
+
const targets = (run.injections ?? [])
|
|
1492
|
+
.flatMap(entry => (entry.targets ?? []).map(target => ({ entry, target })))
|
|
1493
|
+
.filter(({ target }) => target.childId === childId && target.status === 'pending')
|
|
1494
|
+
if (targets.length === 0) return false
|
|
1495
|
+
let inspected = null
|
|
1496
|
+
try {
|
|
1497
|
+
inspected = typeof inspectChild === 'function' ? inspectChild(childId) : null
|
|
1498
|
+
} catch {
|
|
1499
|
+
inspected = null
|
|
1500
|
+
}
|
|
1501
|
+
let changed = false
|
|
1502
|
+
for (const { entry, target } of targets) {
|
|
1503
|
+
stopWatch(run.id, entry.id, target.childId)
|
|
1504
|
+
const live = Boolean(inspected?.live)
|
|
1505
|
+
const stillPending = live && inspected?.pending?.has?.(String(target.messageId))
|
|
1506
|
+
changed = finishInjectionTarget(run, target, live && !stillPending ? 'applied' : 'missed', {
|
|
1507
|
+
reason: live
|
|
1508
|
+
? reason
|
|
1509
|
+
: 'the child was gone before the message reached another step boundary',
|
|
1510
|
+
}) || changed
|
|
1511
|
+
}
|
|
1512
|
+
if (changed) refreshInjectionStats(run)
|
|
1513
|
+
return changed
|
|
555
1514
|
}
|
|
556
1515
|
|
|
557
1516
|
/**
|
|
558
|
-
*
|
|
559
|
-
* one Attention per report; a second worker asking something is recorded, but
|
|
560
|
-
* the notification leads with the first question so the message stays short.
|
|
1517
|
+
* Deliver one instruction into the workers that are running now.
|
|
561
1518
|
*
|
|
562
|
-
*
|
|
563
|
-
*
|
|
564
|
-
*
|
|
1519
|
+
* Delivery is a steer: `agent-loop` claims it at the next pre-step, so it is
|
|
1520
|
+
* part of the child's next model request instead of interrupting the one in
|
|
1521
|
+
* flight. A child that had already decided to stop runs one more turn when
|
|
1522
|
+
* the inbox is non-empty, which is why a live, uncancelled child consumes it.
|
|
1523
|
+
* The latency is measured from acceptance to that claim and written on the
|
|
1524
|
+
* receipt; the theoretical bound is the remainder of the running step.
|
|
1525
|
+
*
|
|
1526
|
+
* @param {string} runId - run identity.
|
|
1527
|
+
* @param {string} text - the instruction.
|
|
1528
|
+
* @param {object} parent - live agent that owns the run.
|
|
1529
|
+
* @param {{ subtaskIndex?: number | string | null }} [options] - narrow to one worker.
|
|
1530
|
+
* @returns {Promise<{ run: object, injection: object, degraded: object | null, effect: object }>}
|
|
565
1531
|
*/
|
|
566
|
-
function
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
1532
|
+
async function inject(runId, text, parent, options = {}) {
|
|
1533
|
+
const id = String(runId ?? '').trim()
|
|
1534
|
+
if (!id) throw new Error('run_id is required')
|
|
1535
|
+
const message = capChars(text, FOLLOWUP_MAX_CHARS)
|
|
1536
|
+
if (!message) throw new Error('message is required; pass the instruction the running workers should read')
|
|
1537
|
+
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
1538
|
+
const rawIndex = options.subtaskIndex
|
|
1539
|
+
const wanted = rawIndex === undefined || rawIndex === null || rawIndex === ''
|
|
1540
|
+
? null
|
|
1541
|
+
: Number(rawIndex)
|
|
1542
|
+
if (wanted !== null && !Number.isInteger(wanted)) {
|
|
1543
|
+
throw new Error('subtask_index must be a whole number from superagent_status')
|
|
1544
|
+
}
|
|
1545
|
+
return serialize(async () => {
|
|
1546
|
+
const rt = runtime.get(id)
|
|
1547
|
+
const run = rt?.run ?? await loadRun(id)
|
|
1548
|
+
assertResumeOwner(run, parent)
|
|
1549
|
+
if (!ACTIVE_STATUSES.has(run.status)) {
|
|
1550
|
+
throw new Error(
|
|
1551
|
+
`run ${id} is ${run.status}, so no child is running to receive a live instruction. `
|
|
1552
|
+
+ 'Use superagent_followup to queue it for the next prompt.',
|
|
1553
|
+
)
|
|
1554
|
+
}
|
|
1555
|
+
if (!rt) {
|
|
1556
|
+
throw new Error(
|
|
1557
|
+
`run ${id} is active but owned by another process; a live instruction must come from the process that owns `
|
|
1558
|
+
+ 'its children, because only that process holds their inboxes.',
|
|
1559
|
+
)
|
|
1560
|
+
}
|
|
1561
|
+
const running = (run.subtasks ?? []).filter(subtask => subtask.status === 'running' && subtask.childId)
|
|
1562
|
+
const selected = wanted === null ? running : running.filter(subtask => subtask.index === wanted)
|
|
1563
|
+
// A running subtask can be between attempts, where its old child id no
|
|
1564
|
+
// longer resolves. Steering a dead inbox would look like delivery, so the
|
|
1565
|
+
// live ones are selected here and the reply names the gap otherwise.
|
|
1566
|
+
const live = typeof inspectChild === 'function'
|
|
1567
|
+
? selected.filter(subtask => {
|
|
1568
|
+
try {
|
|
1569
|
+
return Boolean(inspectChild(subtask.childId)?.live)
|
|
1570
|
+
} catch {
|
|
1571
|
+
return false
|
|
1572
|
+
}
|
|
1573
|
+
})
|
|
1574
|
+
: selected
|
|
1575
|
+
const targets = live
|
|
1576
|
+
if (targets.length === 0) {
|
|
1577
|
+
const who = wanted === null ? 'no worker is live' : `worker ${wanted} is not live`
|
|
1578
|
+
const activity = (run.subtasks ?? [])
|
|
1579
|
+
.filter(subtask => subtask.status === 'running')
|
|
1580
|
+
.map(subtask => `#${subtask.index}`)
|
|
1581
|
+
.join(', ')
|
|
1582
|
+
throw new Error(
|
|
1583
|
+
`${who} on run ${id} (status ${run.status}${activity ? `, attempts in progress: ${activity}` : ''}), so `
|
|
1584
|
+
+ 'there is no inbox to steer right now. superagent_followup queues the instruction for the next prompt '
|
|
1585
|
+
+ 'instead.',
|
|
1586
|
+
)
|
|
1587
|
+
}
|
|
1588
|
+
const entry = pushInjection(run, message)
|
|
1589
|
+
let unsupported = null
|
|
1590
|
+
for (const subtask of targets) {
|
|
1591
|
+
const target = {
|
|
1592
|
+
subtaskIndex: subtask.index,
|
|
1593
|
+
childId: subtask.childId,
|
|
1594
|
+
messageId: null,
|
|
1595
|
+
via: null,
|
|
1596
|
+
acceptedAt: new Date(now()).toISOString(),
|
|
1597
|
+
status: 'pending',
|
|
1598
|
+
appliedAt: null,
|
|
1599
|
+
latencyMs: null,
|
|
1600
|
+
missedAt: null,
|
|
1601
|
+
missedReason: null,
|
|
1602
|
+
error: null,
|
|
1603
|
+
}
|
|
1604
|
+
entry.targets.push(target)
|
|
1605
|
+
if (typeof steerChild !== 'function') {
|
|
1606
|
+
target.status = 'failed'
|
|
1607
|
+
target.error = 'this host exposes no live channel into a running child'
|
|
1608
|
+
unsupported = target.error
|
|
1609
|
+
continue
|
|
1610
|
+
}
|
|
1611
|
+
try {
|
|
1612
|
+
const receipt = await steerChild({
|
|
1613
|
+
parent,
|
|
1614
|
+
childId: target.childId,
|
|
1615
|
+
text: message,
|
|
1616
|
+
signal: rt.controller.signal,
|
|
1617
|
+
})
|
|
1618
|
+
target.messageId = String(receipt?.messageId ?? '')
|
|
1619
|
+
target.via = receipt?.via ?? null
|
|
1620
|
+
watchInjection(run, entry, target)
|
|
1621
|
+
} catch (error) {
|
|
1622
|
+
target.status = 'failed'
|
|
1623
|
+
target.error = error?.message ?? String(error)
|
|
1624
|
+
if (error?.code === 'inject-unsupported') unsupported = target.error
|
|
1625
|
+
else if (error?.code === 'inject-child-gone') target.error = `${target.error}; the worker already settled`
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
let degraded = null
|
|
1629
|
+
if (unsupported && entry.targets.every(target => target.status === 'failed')) {
|
|
1630
|
+
// No live channel at all. Do not pretend the instruction was delivered:
|
|
1631
|
+
// queue it exactly where superagent_followup would, and label the
|
|
1632
|
+
// receipt as degraded so status cannot read as a live delivery.
|
|
1633
|
+
const followup = pushFollowup(run, { kind: 'followup', text: message })
|
|
1634
|
+
entry.status = 'degraded'
|
|
1635
|
+
entry.followupId = followup.id
|
|
1636
|
+
degraded = { followup, effect: followupEffect(run), reason: unsupported }
|
|
1637
|
+
}
|
|
1638
|
+
refreshInjectionStats(run)
|
|
1639
|
+
await save(run)
|
|
1640
|
+
return { run, injection: entry, degraded, effect: followupEffect(run) }
|
|
575
1641
|
})
|
|
576
1642
|
}
|
|
577
1643
|
|
|
578
1644
|
/**
|
|
579
|
-
*
|
|
580
|
-
*
|
|
581
|
-
*
|
|
1645
|
+
* Wait for the receipts of one injection to leave `pending`.
|
|
1646
|
+
*
|
|
1647
|
+
* The claim is observed by a poll, so the reported latency has that
|
|
1648
|
+
* resolution; waiting does not change delivery, only what the reply can say.
|
|
582
1649
|
*
|
|
583
1650
|
* @param {object} run - live run record.
|
|
584
|
-
* @param {
|
|
585
|
-
* @param {
|
|
1651
|
+
* @param {object} entry - injection receipt.
|
|
1652
|
+
* @param {number} [seconds] - upper bound on the wait.
|
|
1653
|
+
* @returns {Promise<object>} the same entry, updated in place.
|
|
586
1654
|
*/
|
|
587
|
-
function
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
1655
|
+
async function waitForInjection(run, entry, seconds) {
|
|
1656
|
+
const limit = Math.max(0, Number(seconds) || 0) * 1000
|
|
1657
|
+
if (limit <= 0) return entry
|
|
1658
|
+
const started = now()
|
|
1659
|
+
while (now() - started < limit) {
|
|
1660
|
+
if (!(entry.targets ?? []).some(target => target.status === 'pending')) break
|
|
1661
|
+
await sleep(Math.min(INJECT_POLL_MS * 2, Math.max(5, limit - (now() - started))))
|
|
1662
|
+
}
|
|
1663
|
+
return entry
|
|
592
1664
|
}
|
|
593
1665
|
|
|
594
1666
|
/**
|
|
@@ -735,6 +1807,7 @@ export function createOrchestrator(deps) {
|
|
|
735
1807
|
? 'cancelled by superagent_cancel while no process was running it'
|
|
736
1808
|
: 'host process stopped while the run was active'
|
|
737
1809
|
run.finishedAt = new Date(now()).toISOString()
|
|
1810
|
+
if (run.queue?.waiting) run.queue = { ...run.queue, waiting: false, lost: true }
|
|
738
1811
|
run.reclaimed = {
|
|
739
1812
|
at: new Date(now()).toISOString(),
|
|
740
1813
|
basis: verdict.basis,
|
|
@@ -751,12 +1824,12 @@ export function createOrchestrator(deps) {
|
|
|
751
1824
|
// its child logs were readable.
|
|
752
1825
|
const entries = Object.values(run.usageByChild ?? {})
|
|
753
1826
|
if (entries.length > 0) run.usage = sumUsage(entries)
|
|
754
|
-
run.cost = computeCost(run.usage, config.pricing)
|
|
755
1827
|
for (const subtask of run.subtasks ?? []) {
|
|
756
1828
|
if (subtask.childId && run.usageByChild?.[subtask.childId]) {
|
|
757
1829
|
subtask.usage = run.usageByChild[subtask.childId]
|
|
758
1830
|
}
|
|
759
1831
|
}
|
|
1832
|
+
applyCost(run, config.pricing)
|
|
760
1833
|
}
|
|
761
1834
|
|
|
762
1835
|
async function releaseRunLock(runId) {
|
|
@@ -923,17 +1996,20 @@ export function createOrchestrator(deps) {
|
|
|
923
1996
|
}
|
|
924
1997
|
let count = 0
|
|
925
1998
|
for (const rt of runtime.values()) {
|
|
926
|
-
|
|
1999
|
+
// A run waiting for a slot is not spending anything yet, so it must not
|
|
2000
|
+
// consume the quota that decides when it may start.
|
|
2001
|
+
if (rt.run && ACTIVE_STATUSES.has(rt.run.status) && !isWaiting(rt.run)) count++
|
|
927
2002
|
}
|
|
928
2003
|
for (const run of await store.list()) {
|
|
929
2004
|
if (runtime.has(run.id)) continue
|
|
930
|
-
if (ACTIVE_STATUSES.has(run.status)) count++
|
|
2005
|
+
if (ACTIVE_STATUSES.has(run.status) && !isWaiting(run)) count++
|
|
931
2006
|
}
|
|
932
2007
|
return count
|
|
933
2008
|
}
|
|
934
2009
|
|
|
935
|
-
function quotaMessage(active) {
|
|
936
|
-
|
|
2010
|
+
function quotaMessage(active, queued = 0) {
|
|
2011
|
+
const waiting = queued > 0 ? ` ${queued} run(s) are already queued, so this start would wait behind them.` : ''
|
|
2012
|
+
return `quota exceeded: ${active} active run(s) and maxActiveRuns is ${maxActiveRuns()}.${waiting} `
|
|
937
2013
|
+ 'Let one finish with superagent_status or cancel it with superagent_cancel.'
|
|
938
2014
|
}
|
|
939
2015
|
|
|
@@ -978,8 +2054,9 @@ export function createOrchestrator(deps) {
|
|
|
978
2054
|
* before run notes: a rule must not be drowned out by recent observations.
|
|
979
2055
|
*
|
|
980
2056
|
* @param {string | null} currentId - run being started or resumed.
|
|
2057
|
+
* @param {string} goal - the run's goal, used to recall the notes that matter.
|
|
981
2058
|
*/
|
|
982
|
-
async function memoryContext(currentId) {
|
|
2059
|
+
async function memoryContext(currentId, goal = '') {
|
|
983
2060
|
if (!memory) return null
|
|
984
2061
|
const settings = memorySettings()
|
|
985
2062
|
const layers = []
|
|
@@ -1002,6 +2079,9 @@ export function createOrchestrator(deps) {
|
|
|
1002
2079
|
}
|
|
1003
2080
|
const notes = await memory.digest({
|
|
1004
2081
|
scope: settings.scope,
|
|
2082
|
+
// Rank notes against this goal instead of only taking the newest ones,
|
|
2083
|
+
// so a run with many predecessors still finds the relevant experience.
|
|
2084
|
+
query: goal,
|
|
1005
2085
|
maxEntries: settings.digestEntries,
|
|
1006
2086
|
maxChars: settings.digestChars,
|
|
1007
2087
|
maxAgeDays: settings.maxAgeDays,
|
|
@@ -1100,7 +2180,8 @@ export function createOrchestrator(deps) {
|
|
|
1100
2180
|
// crash cannot lose the only evidence of a model call.
|
|
1101
2181
|
ensureCounters(run).spawns += 1
|
|
1102
2182
|
await save(run)
|
|
1103
|
-
|
|
2183
|
+
// runId lets the caller bind the child to this run's guard counters.
|
|
2184
|
+
return spawnAbortable({ ...spec, runId: run.id }, rt.controller.signal)
|
|
1104
2185
|
}
|
|
1105
2186
|
|
|
1106
2187
|
/**
|
|
@@ -1286,11 +2367,15 @@ export function createOrchestrator(deps) {
|
|
|
1286
2367
|
|
|
1287
2368
|
async function makePlan(run, parent, rt) {
|
|
1288
2369
|
try {
|
|
2370
|
+
const carried = carriedEntries(run)
|
|
1289
2371
|
const handle = await spawn(run, rt, {
|
|
1290
2372
|
label: 'superagent-plan',
|
|
1291
|
-
prompt: plannerPrompt(run.goal, run.limits.maxSubtasks, run.memory?.digest ?? ''),
|
|
2373
|
+
prompt: plannerPrompt(run.goal, run.limits.maxSubtasks, run.memory?.digest ?? '', carried),
|
|
1292
2374
|
parent,
|
|
1293
2375
|
})
|
|
2376
|
+
// The planner prompt carried them, so record where they landed before
|
|
2377
|
+
// the run moves on; status must not claim they are still waiting.
|
|
2378
|
+
if (carried.length > 0 && markFollowupsApplied(carried, 'planner')) await save(run)
|
|
1294
2379
|
rt.activations.add(handle)
|
|
1295
2380
|
run.steps.planner = { childId: handle.childId }
|
|
1296
2381
|
try {
|
|
@@ -1318,13 +2403,22 @@ export function createOrchestrator(deps) {
|
|
|
1318
2403
|
subtask.status = 'running'
|
|
1319
2404
|
subtask.attempts = attempt
|
|
1320
2405
|
subtask.startedAt = subtask.startedAt ?? new Date(now()).toISOString()
|
|
2406
|
+
// An instruction that arrived after this subtask was queued reaches it
|
|
2407
|
+
// here, at the last moment before the prompt is built.
|
|
2408
|
+
const carried = carriedEntries(run)
|
|
2409
|
+
markFollowupsApplied(carried, `subtask-${subtask.index}`)
|
|
1321
2410
|
await save(run)
|
|
1322
2411
|
|
|
1323
2412
|
let handle = null
|
|
2413
|
+
// A retry reuses the checkout from the previous attempt, so work the
|
|
2414
|
+
// earlier attempt already wrote is not thrown away.
|
|
2415
|
+
const workspace = await openWorkspaceFor(run, subtask)
|
|
1324
2416
|
try {
|
|
1325
2417
|
handle = await spawn(run, rt, {
|
|
1326
|
-
label: `superagent-${subtask.index}`,
|
|
1327
|
-
prompt:
|
|
2418
|
+
label: run.mode === 'direct' ? 'superagent-direct' : `superagent-${subtask.index}`,
|
|
2419
|
+
prompt: run.mode === 'direct'
|
|
2420
|
+
? directPrompt(run, carried)
|
|
2421
|
+
: workerPrompt(run, subtask, workspace, carried),
|
|
1328
2422
|
parent,
|
|
1329
2423
|
model,
|
|
1330
2424
|
})
|
|
@@ -1353,6 +2447,7 @@ export function createOrchestrator(deps) {
|
|
|
1353
2447
|
subtask.error = null
|
|
1354
2448
|
subtask.finishedAt = new Date(now()).toISOString()
|
|
1355
2449
|
await recordUsage(run, handle.childId)
|
|
2450
|
+
await harvestWorkspace(run, subtask)
|
|
1356
2451
|
await save(run)
|
|
1357
2452
|
return
|
|
1358
2453
|
}
|
|
@@ -1372,6 +2467,13 @@ export function createOrchestrator(deps) {
|
|
|
1372
2467
|
if (handle) {
|
|
1373
2468
|
// A failed attempt still spent tokens, so read it before disposing.
|
|
1374
2469
|
await recordUsage(run, handle.childId)
|
|
2470
|
+
// Receipts are resolved while the inbox is still readable: a message
|
|
2471
|
+
// still waiting here never entered a prompt for this attempt.
|
|
2472
|
+
if (settleChildInjections(
|
|
2473
|
+
run,
|
|
2474
|
+
handle.childId,
|
|
2475
|
+
'the child finished its turn before the instruction reached another step boundary',
|
|
2476
|
+
)) await save(run)
|
|
1375
2477
|
rt.activations.delete(handle)
|
|
1376
2478
|
await disposeQuietly(handle)
|
|
1377
2479
|
}
|
|
@@ -1407,16 +2509,104 @@ export function createOrchestrator(deps) {
|
|
|
1407
2509
|
}
|
|
1408
2510
|
}
|
|
1409
2511
|
|
|
2512
|
+
/**
|
|
2513
|
+
* Run the subtask graph: a task starts as soon as every task it depends on has
|
|
2514
|
+
* completed, and at most `maxParallel` execute at once. A task that is only
|
|
2515
|
+
* held back by a dependency is marked `waiting` with the keys it waits for, so
|
|
2516
|
+
* status tells it apart from one that is actually running.
|
|
2517
|
+
*
|
|
2518
|
+
* A task whose dependency failed or was skipped is marked `skipped` with the
|
|
2519
|
+
* reason and never runs. Those skips are revived on the next call (self-heal,
|
|
2520
|
+
* resume) if the dependency completes then, so a retry can still unblock them.
|
|
2521
|
+
*
|
|
2522
|
+
* @param {object} run - live run record.
|
|
2523
|
+
* @param {object} parent - calling session.
|
|
2524
|
+
* @param {object} rt - runtime record with the abort controller.
|
|
2525
|
+
* @param {{ model?: string | null }} [options] - child model override.
|
|
2526
|
+
*/
|
|
2527
|
+
async function runGraph(run, parent, rt, { model = null } = {}) {
|
|
2528
|
+
const signal = rt.controller.signal
|
|
2529
|
+
const byKey = new Map(run.subtasks.map(subtask => [subtask.key, subtask]))
|
|
2530
|
+
let revived = false
|
|
2531
|
+
for (const subtask of run.subtasks) {
|
|
2532
|
+
if (subtask.status === 'skipped' && subtask.skipKind === 'upstream') {
|
|
2533
|
+
subtask.status = 'pending'
|
|
2534
|
+
subtask.skipKind = null
|
|
2535
|
+
subtask.skippedReason = null
|
|
2536
|
+
subtask.waitingOn = []
|
|
2537
|
+
revived = true
|
|
2538
|
+
}
|
|
2539
|
+
}
|
|
2540
|
+
if (revived) await save(run)
|
|
2541
|
+
|
|
2542
|
+
const inFlight = new Map()
|
|
2543
|
+
while (!signal.aborted) {
|
|
2544
|
+
let dirty = cascadeSkips(run, byKey)
|
|
2545
|
+
for (const subtask of run.subtasks) {
|
|
2546
|
+
if (signal.aborted) break
|
|
2547
|
+
if (subtask.status !== 'pending' && subtask.status !== 'waiting') continue
|
|
2548
|
+
if (inFlight.has(subtask.key)) continue
|
|
2549
|
+
const outstanding = (subtask.dependsOn ?? [])
|
|
2550
|
+
.map(key => byKey.get(key))
|
|
2551
|
+
.filter(dependency => dependency && dependency.status !== 'completed')
|
|
2552
|
+
if (outstanding.length > 0) {
|
|
2553
|
+
const waitingOn = outstanding.map(dependency => dependency.key)
|
|
2554
|
+
if (subtask.status !== 'waiting' || subtask.waitingOn.join() !== waitingOn.join()) {
|
|
2555
|
+
subtask.waitingOn = waitingOn
|
|
2556
|
+
if (subtask.status !== 'waiting') subtask.status = 'waiting'
|
|
2557
|
+
dirty = true
|
|
2558
|
+
}
|
|
2559
|
+
continue
|
|
2560
|
+
}
|
|
2561
|
+
if (subtask.status === 'waiting') {
|
|
2562
|
+
subtask.status = 'pending'
|
|
2563
|
+
subtask.waitingOn = []
|
|
2564
|
+
dirty = true
|
|
2565
|
+
}
|
|
2566
|
+
if (inFlight.size >= run.limits.maxParallel) break
|
|
2567
|
+
const task = runSubtask(run, subtask, parent, rt, { model })
|
|
2568
|
+
.catch(error => log(`subtask ${subtask.index} graph runner threw: ${error?.message ?? error}`))
|
|
2569
|
+
.finally(() => inFlight.delete(subtask.key))
|
|
2570
|
+
inFlight.set(subtask.key, task)
|
|
2571
|
+
}
|
|
2572
|
+
if (dirty) await save(run)
|
|
2573
|
+
// Nothing is executing, so nothing can finish and no dependency can clear.
|
|
2574
|
+
if (inFlight.size === 0) break
|
|
2575
|
+
await Promise.race(inFlight.values())
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
// Leaving a subtask as `waiting` after the graph stopped would claim it is
|
|
2579
|
+
// still held back by live work. An aborted run keeps it resumable instead.
|
|
2580
|
+
let leftover = false
|
|
2581
|
+
for (const subtask of run.subtasks) {
|
|
2582
|
+
if (subtask.status === 'waiting') {
|
|
2583
|
+
subtask.status = 'pending'
|
|
2584
|
+
subtask.waitingOn = []
|
|
2585
|
+
leftover = true
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
if (leftover) await save(run)
|
|
2589
|
+
}
|
|
2590
|
+
|
|
1410
2591
|
async function aggregate(run, parent, rt) {
|
|
1411
2592
|
const completed = run.subtasks.filter(subtask => subtask.status === 'completed')
|
|
1412
2593
|
if (completed.length === 0) return null
|
|
1413
2594
|
if (rt.controller.signal.aborted) return deterministicJoin(run)
|
|
1414
|
-
|
|
2595
|
+
// One report needs no merging, but it does need a prompt when the user has
|
|
2596
|
+
// since added an instruction or answered a question: the shortcut would
|
|
2597
|
+
// return the old text and silently drop the decision.
|
|
2598
|
+
const hasInstructions = (run.followups ?? []).length > 0
|
|
2599
|
+
if (completed.length === 1 && !hasInstructions) return `# ${run.goal}\n\n${completed[0].result ?? ''}`.trim()
|
|
1415
2600
|
|
|
1416
2601
|
try {
|
|
2602
|
+
// Instructions that arrived while the workers ran land here, and this is
|
|
2603
|
+
// the last boundary a terminal run has: the report can honor them even
|
|
2604
|
+
// when no worker will start again.
|
|
2605
|
+
const carried = carriedEntries(run)
|
|
2606
|
+
if (carried.length > 0 && markFollowupsApplied(carried, 'aggregator')) await save(run)
|
|
1417
2607
|
const handle = await spawn(run, rt, {
|
|
1418
2608
|
label: 'superagent-aggregate',
|
|
1419
|
-
prompt: aggregatorPrompt(run, completed, run.memory?.digest ?? ''),
|
|
2609
|
+
prompt: aggregatorPrompt(run, completed, run.memory?.digest ?? '', carried),
|
|
1420
2610
|
parent,
|
|
1421
2611
|
})
|
|
1422
2612
|
rt.activations.add(handle)
|
|
@@ -1500,10 +2690,34 @@ export function createOrchestrator(deps) {
|
|
|
1500
2690
|
subtask.usage = run.usageByChild[subtask.childId]
|
|
1501
2691
|
}
|
|
1502
2692
|
}
|
|
1503
|
-
|
|
2693
|
+
applyCost(run, config.pricing)
|
|
1504
2694
|
const nextStatus = terminalStatus(run, rt)
|
|
1505
2695
|
const finishedAt = new Date(now()).toISOString()
|
|
1506
2696
|
|
|
2697
|
+
// An instruction that arrived in the last moments of a run never reached a
|
|
2698
|
+
// prompt. Record that instead of leaving it looking delivered: the user
|
|
2699
|
+
// has to know it did not shape anything.
|
|
2700
|
+
for (const entry of run.followups ?? []) {
|
|
2701
|
+
if (entry.appliedAt || entry.missedAt) continue
|
|
2702
|
+
entry.missedAt = finishedAt
|
|
2703
|
+
entry.missedReason = 'the run settled before another boundary ran; start a new run to apply it'
|
|
2704
|
+
}
|
|
2705
|
+
|
|
2706
|
+
// A live injection has no boundary of its own: once the run is over, an
|
|
2707
|
+
// untouched receipt is a miss, and the queue that carried it is gone.
|
|
2708
|
+
let injectionsChanged = false
|
|
2709
|
+
for (const injection of run.injections ?? []) {
|
|
2710
|
+
for (const target of injection.targets ?? []) {
|
|
2711
|
+
if (target.status !== 'pending') continue
|
|
2712
|
+
stopWatch(run.id, injection.id, target.childId)
|
|
2713
|
+
finishInjectionTarget(run, target, 'missed', {
|
|
2714
|
+
reason: 'the run settled before the child reached another step boundary',
|
|
2715
|
+
})
|
|
2716
|
+
injectionsChanged = true
|
|
2717
|
+
}
|
|
2718
|
+
}
|
|
2719
|
+
if (injectionsChanged) refreshInjectionStats(run)
|
|
2720
|
+
|
|
1507
2721
|
// The aggregator appends a fenced notebook block to its report. Strip it in
|
|
1508
2722
|
// every case so the user never sees plugin syntax, then persist it when the
|
|
1509
2723
|
// notebook is enabled. A write failure is recorded on the run but never
|
|
@@ -1557,17 +2771,24 @@ export function createOrchestrator(deps) {
|
|
|
1557
2771
|
}
|
|
1558
2772
|
}
|
|
1559
2773
|
}
|
|
1560
|
-
// The
|
|
1561
|
-
//
|
|
1562
|
-
|
|
1563
|
-
|
|
2774
|
+
// The guard's event list belongs in the user-facing report, not only in the
|
|
2775
|
+
// status line: a blocked command is part of what happened during the run.
|
|
2776
|
+
const guardSection = guard?.reportSection?.(run.id)
|
|
2777
|
+
if (guardSection && typeof run.report === 'string' && run.report) {
|
|
2778
|
+
run.report = `${run.report.trimEnd()}\n\n${guardSection}`
|
|
2779
|
+
}
|
|
2780
|
+
// The report gets a durable address before the run is announced, so a
|
|
2781
|
+
// notification or status line can point at a file that already exists.
|
|
2782
|
+
if (TERMINAL_STATUSES.has(nextStatus)) await writeReportArtifact(run)
|
|
2783
|
+
addMilestone(run, 'settled')
|
|
1564
2784
|
let error = run.error
|
|
1565
2785
|
if (rt.abortCause === 'cancelled') {
|
|
1566
2786
|
error = 'cancelled by superagent_cancel'
|
|
1567
2787
|
} else if (rt.abortCause === 'host-gone') {
|
|
1568
2788
|
error = 'the host shut down while this run was active; run superagent_resume in a new host to finish it'
|
|
1569
2789
|
} else if (nextStatus === 'failed' && !error) {
|
|
1570
|
-
error = run.subtasks.map(subtask => subtask.error).filter(Boolean).slice(0, 3).join('; ')
|
|
2790
|
+
error = run.subtasks.map(subtask => subtask.error ?? subtask.skippedReason).filter(Boolean).slice(0, 3).join('; ')
|
|
2791
|
+
|| 'no subtask completed'
|
|
1571
2792
|
}
|
|
1572
2793
|
if (rt.abortCause === 'deadline' && !error) error = 'deadline exceeded'
|
|
1573
2794
|
|
|
@@ -1613,6 +2834,24 @@ export function createOrchestrator(deps) {
|
|
|
1613
2834
|
if (outcome.delivered !== false) run.notified = true
|
|
1614
2835
|
}
|
|
1615
2836
|
await save(run)
|
|
2837
|
+
// Awaited: the trail must contain the outcome before the run is published
|
|
2838
|
+
// as terminal, or a crash right here would leave a finished run with no
|
|
2839
|
+
// record of how it finished. A failure is still only logged.
|
|
2840
|
+
await auditRecord({
|
|
2841
|
+
runId: run.id,
|
|
2842
|
+
actor: `session:${run.parentSessionId}`,
|
|
2843
|
+
kind: 'run.finish',
|
|
2844
|
+
action: 'settle',
|
|
2845
|
+
target: oneLine(run.goal, 200),
|
|
2846
|
+
result: nextStatus,
|
|
2847
|
+
durationMs: run.activeMs,
|
|
2848
|
+
meta: {
|
|
2849
|
+
subtasks: run.subtasks.length,
|
|
2850
|
+
completed: run.subtasks.filter(subtask => subtask.status === 'completed').length,
|
|
2851
|
+
attention: Boolean(run.attention?.items?.length),
|
|
2852
|
+
error: error ? oneLine(error, 300) : null,
|
|
2853
|
+
},
|
|
2854
|
+
})
|
|
1616
2855
|
// The run is terminal, so a leftover marker would only make the next mount
|
|
1617
2856
|
// rewrite a finished run; drop it now.
|
|
1618
2857
|
await clearCancel(run.id)
|
|
@@ -1642,6 +2881,160 @@ export function createOrchestrator(deps) {
|
|
|
1642
2881
|
})
|
|
1643
2882
|
}
|
|
1644
2883
|
|
|
2884
|
+
/** One subtask record, shared by the planned pipeline and direct mode. */
|
|
2885
|
+
function makeSubtask(entry, index) {
|
|
2886
|
+
return {
|
|
2887
|
+
index: index + 1,
|
|
2888
|
+
title: entry.title,
|
|
2889
|
+
prompt: entry.prompt,
|
|
2890
|
+
status: 'pending',
|
|
2891
|
+
attempts: 0,
|
|
2892
|
+
childId: null,
|
|
2893
|
+
stopReason: null,
|
|
2894
|
+
error: null,
|
|
2895
|
+
result: null,
|
|
2896
|
+
usage: null,
|
|
2897
|
+
startedAt: null,
|
|
2898
|
+
finishedAt: null,
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
|
|
2902
|
+
/** Planner, parallel workers, self-heal, then one merged report. */
|
|
2903
|
+
async function runPlanned(run, parent, rt) {
|
|
2904
|
+
if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
|
|
2905
|
+
run.status = 'planning'
|
|
2906
|
+
await save(run)
|
|
2907
|
+
const plan = await makePlan(run, parent, rt)
|
|
2908
|
+
run.plan = { mode: plan.mode, subtasks: plan.subtasks }
|
|
2909
|
+
const entries = plan.subtasks.map((entry, index) => ({
|
|
2910
|
+
key: entry.id ?? `w${index + 1}`,
|
|
2911
|
+
title: entry.title,
|
|
2912
|
+
dependsOn: entry.dependsOn ?? [],
|
|
2913
|
+
}))
|
|
2914
|
+
const keys = new Set(entries.map(entry => entry.key))
|
|
2915
|
+
if (keys.size !== entries.length) {
|
|
2916
|
+
throw new Error(
|
|
2917
|
+
'plan rejected: duplicate subtask ids; dependsOn would be ambiguous, so replan with unique ids',
|
|
2918
|
+
)
|
|
2919
|
+
}
|
|
2920
|
+
const linked = linkSubtasks(entries)
|
|
2921
|
+
if (linked.cycle) {
|
|
2922
|
+
// A cycle cannot be scheduled, and picking an arbitrary edge to drop
|
|
2923
|
+
// would change what the planner asked for. Reject the plan and name
|
|
2924
|
+
// the tasks involved so the next attempt can be fixed.
|
|
2925
|
+
throw new Error(
|
|
2926
|
+
`plan rejected: dependency cycle ${linked.cycle.join(' -> ')}; `
|
|
2927
|
+
+ 'no subtask was started, so replan without the cycle',
|
|
2928
|
+
)
|
|
2929
|
+
}
|
|
2930
|
+
run.subtasks = plan.subtasks.map((entry, index) => {
|
|
2931
|
+
const key = entries[index].key
|
|
2932
|
+
const missing = linked.unknown.get(key) ?? []
|
|
2933
|
+
return {
|
|
2934
|
+
...makeSubtask(entry, index),
|
|
2935
|
+
key,
|
|
2936
|
+
dependsOn: linked.edges.get(key) ?? [],
|
|
2937
|
+
waitingOn: [],
|
|
2938
|
+
status: missing.length > 0 ? 'skipped' : 'pending',
|
|
2939
|
+
skipKind: missing.length > 0 ? 'missing' : null,
|
|
2940
|
+
skippedReason: missing.length > 0
|
|
2941
|
+
? `depends on unknown subtask ${missing.map(ref => `"${ref}"`).join(', ')}`
|
|
2942
|
+
: null,
|
|
2943
|
+
}
|
|
2944
|
+
})
|
|
2945
|
+
await save(run)
|
|
2946
|
+
addMilestone(run, 'plan_ready')
|
|
2947
|
+
await save(run)
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2950
|
+
const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
|
|
2951
|
+
if (open.length > 0) {
|
|
2952
|
+
run.status = 'running'
|
|
2953
|
+
addMilestone(run, 'running')
|
|
2954
|
+
await save(run)
|
|
2955
|
+
await runGraph(run, parent, rt)
|
|
2956
|
+
}
|
|
2957
|
+
|
|
2958
|
+
// Self-heal before reporting: a failed subtask gets another full round
|
|
2959
|
+
// while the run is not aborted and the budget still allows it. A failure
|
|
2960
|
+
// that looks transient (quota, rate limit, timeout) may switch to the
|
|
2961
|
+
// configured fallback child model on that round. Only what still fails
|
|
2962
|
+
// after this is reported to the user, with what was tried.
|
|
2963
|
+
const rounds = selfHealRounds()
|
|
2964
|
+
const fallbackModel = typeof config.childModelFallback === 'string' && config.childModelFallback.trim()
|
|
2965
|
+
? config.childModelFallback.trim()
|
|
2966
|
+
: null
|
|
2967
|
+
for (let round = 1; round <= rounds; round++) {
|
|
2968
|
+
if (rt.controller.signal.aborted) break
|
|
2969
|
+
// Only a failure that looks transient is retried as a whole round.
|
|
2970
|
+
// Re-running a refusal or a permanent error would double the spend
|
|
2971
|
+
// without changing the outcome, which is the opposite of self-healing.
|
|
2972
|
+
const retryable = run.subtasks.filter(subtask => subtask.status === 'failed' && looksTransient(subtask.error))
|
|
2973
|
+
if (retryable.length === 0) break
|
|
2974
|
+
if (budgetStop(run)) {
|
|
2975
|
+
log(`not retrying ${retryable.length} failed subtask(s): ${budgetStop(run)}`)
|
|
2976
|
+
break
|
|
2977
|
+
}
|
|
2978
|
+
const useFallback = Boolean(fallbackModel && retryable.some(subtask => looksTransient(subtask.error)))
|
|
2979
|
+
run.selfHeal = {
|
|
2980
|
+
rounds: round,
|
|
2981
|
+
retried: retryable.length,
|
|
2982
|
+
recovered: 0,
|
|
2983
|
+
model: useFallback ? fallbackModel : null,
|
|
2984
|
+
}
|
|
2985
|
+
for (const subtask of retryable) {
|
|
2986
|
+
subtask.status = 'pending'
|
|
2987
|
+
subtask.attempts = 0
|
|
2988
|
+
subtask.error = null
|
|
2989
|
+
subtask.stopReason = null
|
|
2990
|
+
subtask.finishedAt = null
|
|
2991
|
+
}
|
|
2992
|
+
log(
|
|
2993
|
+
`self-heal round ${round}: retrying ${retryable.length} failed subtask(s)`
|
|
2994
|
+
+ (useFallback ? ` on ${fallbackModel}` : ''),
|
|
2995
|
+
)
|
|
2996
|
+
await save(run)
|
|
2997
|
+
// The graph re-runs the retried task and revives the downstream tasks
|
|
2998
|
+
// that were skipped because of it, in dependency order.
|
|
2999
|
+
await runGraph(run, parent, rt, { model: useFallback ? fallbackModel : null })
|
|
3000
|
+
run.selfHeal.recovered = retryable.filter(subtask => subtask.status === 'completed').length
|
|
3001
|
+
await save(run)
|
|
3002
|
+
}
|
|
3003
|
+
|
|
3004
|
+
if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
|
|
3005
|
+
run.status = 'aggregating'
|
|
3006
|
+
addMilestone(run, 'aggregating')
|
|
3007
|
+
await save(run)
|
|
3008
|
+
run.report = await aggregate(run, parent, rt)
|
|
3009
|
+
} else if (!run.report) {
|
|
3010
|
+
run.report = deterministicJoin(run)
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
|
|
3014
|
+
/**
|
|
3015
|
+
* Direct mode: one worker does the whole goal. There is no planner and no
|
|
3016
|
+
* aggregator, so the child's reply becomes the report verbatim; the run
|
|
3017
|
+
* record, quota, deadline, cancel, budget, and usage accounting are the same
|
|
3018
|
+
* as for a planned run.
|
|
3019
|
+
*/
|
|
3020
|
+
async function runDirect(run, parent, rt) {
|
|
3021
|
+
if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
|
|
3022
|
+
const entry = { title: 'Complete the goal', prompt: run.goal }
|
|
3023
|
+
run.plan = { mode: 'direct', subtasks: [entry] }
|
|
3024
|
+
run.subtasks = [makeSubtask(entry, 0)]
|
|
3025
|
+
await save(run)
|
|
3026
|
+
}
|
|
3027
|
+
const subtask = run.subtasks.find(item => item.status !== 'completed') ?? run.subtasks[run.subtasks.length - 1]
|
|
3028
|
+
if (subtask && subtask.status !== 'completed') {
|
|
3029
|
+
run.status = 'running'
|
|
3030
|
+
addMilestone(run, 'running')
|
|
3031
|
+
await save(run)
|
|
3032
|
+
await runSubtask(run, subtask, parent, rt)
|
|
3033
|
+
}
|
|
3034
|
+
if (subtask?.status === 'completed') run.report = subtask.result ?? null
|
|
3035
|
+
else if (!run.report) run.report = deterministicJoin(run)
|
|
3036
|
+
}
|
|
3037
|
+
|
|
1645
3038
|
async function execute(run, parent, rt) {
|
|
1646
3039
|
rt.activeStart = now()
|
|
1647
3040
|
const remaining = Math.max(0, Date.parse(run.deadlineAt) - now())
|
|
@@ -1662,90 +3055,24 @@ export function createOrchestrator(deps) {
|
|
|
1662
3055
|
|
|
1663
3056
|
try {
|
|
1664
3057
|
ensureCounters(run)
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
finishedAt: null,
|
|
1683
|
-
}))
|
|
1684
|
-
await save(run)
|
|
1685
|
-
addMilestone(run, 'plan_ready')
|
|
1686
|
-
await save(run)
|
|
1687
|
-
}
|
|
1688
|
-
|
|
1689
|
-
const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
|
|
1690
|
-
if (open.length > 0) {
|
|
1691
|
-
run.status = 'running'
|
|
1692
|
-
addMilestone(run, 'running')
|
|
1693
|
-
await save(run)
|
|
1694
|
-
await pool(open, run.limits.maxParallel, subtask => runSubtask(run, subtask, parent, rt))
|
|
1695
|
-
}
|
|
1696
|
-
|
|
1697
|
-
// Self-heal before reporting: a failed subtask gets another full round
|
|
1698
|
-
// while the run is not aborted and the budget still allows it. A failure
|
|
1699
|
-
// that looks transient (quota, rate limit, timeout) may switch to the
|
|
1700
|
-
// configured fallback child model on that round. Only what still fails
|
|
1701
|
-
// after this is reported to the user, with what was tried.
|
|
1702
|
-
const rounds = selfHealRounds()
|
|
1703
|
-
const fallbackModel = typeof config.childModelFallback === 'string' && config.childModelFallback.trim()
|
|
1704
|
-
? config.childModelFallback.trim()
|
|
1705
|
-
: null
|
|
1706
|
-
for (let round = 1; round <= rounds; round++) {
|
|
1707
|
-
if (rt.controller.signal.aborted) break
|
|
1708
|
-
// Only a failure that looks transient is retried as a whole round.
|
|
1709
|
-
// Re-running a refusal or a permanent error would double the spend
|
|
1710
|
-
// without changing the outcome, which is the opposite of self-healing.
|
|
1711
|
-
const retryable = run.subtasks.filter(subtask => subtask.status === 'failed' && looksTransient(subtask.error))
|
|
1712
|
-
if (retryable.length === 0) break
|
|
1713
|
-
if (budgetStop(run)) {
|
|
1714
|
-
log(`not retrying ${retryable.length} failed subtask(s): ${budgetStop(run)}`)
|
|
1715
|
-
break
|
|
1716
|
-
}
|
|
1717
|
-
const useFallback = Boolean(fallbackModel && retryable.some(subtask => looksTransient(subtask.error)))
|
|
1718
|
-
run.selfHeal = {
|
|
1719
|
-
rounds: round,
|
|
1720
|
-
retried: retryable.length,
|
|
1721
|
-
recovered: 0,
|
|
1722
|
-
model: useFallback ? fallbackModel : null,
|
|
1723
|
-
}
|
|
1724
|
-
for (const subtask of retryable) {
|
|
1725
|
-
subtask.status = 'pending'
|
|
1726
|
-
subtask.attempts = 0
|
|
1727
|
-
subtask.error = null
|
|
1728
|
-
subtask.stopReason = null
|
|
1729
|
-
subtask.finishedAt = null
|
|
3058
|
+
// Decided once, before any child runs, so the whole plan uses one mode and
|
|
3059
|
+
// status can already say whether isolation is on.
|
|
3060
|
+
await ensureWorkspace(run)
|
|
3061
|
+
if (run.mode === 'direct') await runDirect(run, parent, rt)
|
|
3062
|
+
else await runPlanned(run, parent, rt)
|
|
3063
|
+
|
|
3064
|
+
// Merge before the report: a conflict is part of what happened, and the
|
|
3065
|
+
// report is written from the run, so it must exist by now. A run that
|
|
3066
|
+
// stopped early keeps its branches instead of merging half-finished work;
|
|
3067
|
+
// a resume picks the same worktrees up again. Direct mode has a single
|
|
3068
|
+
// worker, but it is isolated the same way, so it merges here too.
|
|
3069
|
+
if (run.workspace?.mode === 'git-worktree') {
|
|
3070
|
+
if (rt.controller.signal.aborted) {
|
|
3071
|
+
run.workspace.deferred = 'the run stopped before merging; the worker branches are kept for a resume'
|
|
3072
|
+
await save(run)
|
|
3073
|
+
} else {
|
|
3074
|
+
await mergeWorkspaces(run)
|
|
1730
3075
|
}
|
|
1731
|
-
log(
|
|
1732
|
-
`self-heal round ${round}: retrying ${retryable.length} failed subtask(s)`
|
|
1733
|
-
+ (useFallback ? ` on ${fallbackModel}` : ''),
|
|
1734
|
-
)
|
|
1735
|
-
await save(run)
|
|
1736
|
-
await pool(retryable, run.limits.maxParallel,
|
|
1737
|
-
subtask => runSubtask(run, subtask, parent, rt, { model: useFallback ? fallbackModel : null }))
|
|
1738
|
-
run.selfHeal.recovered = retryable.filter(subtask => subtask.status === 'completed').length
|
|
1739
|
-
await save(run)
|
|
1740
|
-
}
|
|
1741
|
-
|
|
1742
|
-
if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
|
|
1743
|
-
run.status = 'aggregating'
|
|
1744
|
-
addMilestone(run, 'aggregating')
|
|
1745
|
-
await save(run)
|
|
1746
|
-
run.report = await aggregate(run, parent, rt)
|
|
1747
|
-
} else if (!run.report) {
|
|
1748
|
-
run.report = deterministicJoin(run)
|
|
1749
3076
|
}
|
|
1750
3077
|
} catch (error) {
|
|
1751
3078
|
run.error = error?.message ?? String(error)
|
|
@@ -1811,8 +3138,11 @@ export function createOrchestrator(deps) {
|
|
|
1811
3138
|
.then(() => {
|
|
1812
3139
|
// Finished runs are read from disk from now on; keeping them here would
|
|
1813
3140
|
// grow these maps for the lifetime of the host.
|
|
3141
|
+
stopRunWatches(run.id)
|
|
1814
3142
|
runtime.delete(run.id)
|
|
1815
3143
|
saveChains.delete(run.id)
|
|
3144
|
+
// The slot this run held may let a waiting run start.
|
|
3145
|
+
pumpQueue().catch(() => {})
|
|
1816
3146
|
})
|
|
1817
3147
|
return rt
|
|
1818
3148
|
}
|
|
@@ -1824,10 +3154,21 @@ export function createOrchestrator(deps) {
|
|
|
1824
3154
|
return result
|
|
1825
3155
|
}
|
|
1826
3156
|
|
|
1827
|
-
async function start({
|
|
3157
|
+
async function start({
|
|
3158
|
+
goal,
|
|
3159
|
+
parentSessionId,
|
|
3160
|
+
parent,
|
|
3161
|
+
clientTimeZone,
|
|
3162
|
+
overrides = {},
|
|
3163
|
+
allowDuplicate = false,
|
|
3164
|
+
queue: wantsQueue = false,
|
|
3165
|
+
mode: requestedMode,
|
|
3166
|
+
approvePolicy = false,
|
|
3167
|
+
}) {
|
|
1828
3168
|
const trimmed = typeof goal === 'string' ? goal.trim() : ''
|
|
1829
3169
|
if (!trimmed) throw new Error('goal is required')
|
|
1830
3170
|
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
3171
|
+
const mode = normalizeMode(requestedMode)
|
|
1831
3172
|
|
|
1832
3173
|
const limits = buildLimits(overrides)
|
|
1833
3174
|
// Kept so a resume can rebuild the limits from the *current* config
|
|
@@ -1852,18 +3193,63 @@ export function createOrchestrator(deps) {
|
|
|
1852
3193
|
return { ...existing, deduped: true }
|
|
1853
3194
|
}
|
|
1854
3195
|
}
|
|
1855
|
-
|
|
3196
|
+
const waiting = queue.length
|
|
3197
|
+
// A run already in line is first in line: a new start must not jump it.
|
|
3198
|
+
const atCapacity = active >= maxActiveRuns() || waiting > 0
|
|
3199
|
+
if (atCapacity) {
|
|
3200
|
+
if (!queueEnabled(wantsQueue)) throw new Error(quotaMessage(active, waiting))
|
|
3201
|
+
if (waiting >= maxQueuedRuns()) {
|
|
3202
|
+
throw new Error(
|
|
3203
|
+
`quota exceeded: ${active} active run(s), maxActiveRuns is ${maxActiveRuns()}, and the queue is full `
|
|
3204
|
+
+ `(${waiting} waiting, maxQueuedRuns is ${maxQueuedRuns()}). Let one finish with superagent_status or `
|
|
3205
|
+
+ 'cancel one with superagent_cancel.',
|
|
3206
|
+
)
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
// A permission policy decides whether this start may exist at all. The
|
|
3210
|
+
// verdict is recorded before the run is written, so a refused start still
|
|
3211
|
+
// leaves evidence of what was asked for and why it was stopped.
|
|
3212
|
+
const runId = newRunId()
|
|
3213
|
+
if (audit && typeof audit.gate === 'function') {
|
|
3214
|
+
const verdict = await audit.gate({
|
|
3215
|
+
runId,
|
|
3216
|
+
kind: 'run.start',
|
|
3217
|
+
action: 'superagent_start',
|
|
3218
|
+
actor: `session:${parentSessionId}`,
|
|
3219
|
+
target: oneLine(trimmed, 200),
|
|
3220
|
+
meta: { goalChars: trimmed.length },
|
|
3221
|
+
})
|
|
3222
|
+
const trail = verdict.eventId ? ` (audit event ${verdict.eventId})` : ''
|
|
3223
|
+
if (verdict.decision === 'deny') {
|
|
3224
|
+
throw new Error(`refused by the permission policy: ${verdict.reason}${trail}`)
|
|
3225
|
+
}
|
|
3226
|
+
if (verdict.decision === 'confirm') {
|
|
3227
|
+
if (!approvePolicy) {
|
|
3228
|
+
throw new Error(
|
|
3229
|
+
`the permission policy holds this start for confirmation: ${verdict.reason}${trail}; `
|
|
3230
|
+
+ 'call superagent_start again with approve_policy: true to release it',
|
|
3231
|
+
)
|
|
3232
|
+
}
|
|
3233
|
+
await audit.approve(verdict.eventId, {
|
|
3234
|
+
actor: `session:${parentSessionId}`,
|
|
3235
|
+
note: 'approve_policy on superagent_start',
|
|
3236
|
+
runId,
|
|
3237
|
+
kind: 'run.start',
|
|
3238
|
+
})
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
1856
3241
|
// Read once, here: the digest is part of the run record, so a status call
|
|
1857
3242
|
// can show exactly what the planner was told, and a resumed run does not
|
|
1858
3243
|
// silently change the context its plan was based on.
|
|
1859
|
-
const context = await memoryContext(null)
|
|
3244
|
+
const context = await memoryContext(null, trimmed)
|
|
1860
3245
|
const run = {
|
|
1861
|
-
id:
|
|
3246
|
+
id: runId,
|
|
1862
3247
|
schemaVersion: 1,
|
|
1863
3248
|
goal: trimmed,
|
|
1864
3249
|
dedupeKey,
|
|
1865
3250
|
status: 'queued',
|
|
1866
|
-
mode
|
|
3251
|
+
mode,
|
|
3252
|
+
queue: null,
|
|
1867
3253
|
parentSessionId,
|
|
1868
3254
|
clientTimeZone: clientTimeZone ?? null,
|
|
1869
3255
|
createdAt: new Date(now()).toISOString(),
|
|
@@ -1879,6 +3265,13 @@ export function createOrchestrator(deps) {
|
|
|
1879
3265
|
steps: {},
|
|
1880
3266
|
report: null,
|
|
1881
3267
|
attention: null,
|
|
3268
|
+
// Questions move here when a resume retires them, so a late answer can
|
|
3269
|
+
// still be told apart from an id that never existed.
|
|
3270
|
+
attentionHistory: [],
|
|
3271
|
+
attentionSeq: 0,
|
|
3272
|
+
// Instructions and answers given to this run after it started.
|
|
3273
|
+
followups: [],
|
|
3274
|
+
followupSeq: 0,
|
|
1882
3275
|
milestones: [],
|
|
1883
3276
|
artifacts: [],
|
|
1884
3277
|
memory: context,
|
|
@@ -1890,6 +3283,25 @@ export function createOrchestrator(deps) {
|
|
|
1890
3283
|
counters: { spawns: 0, unreadable: {} },
|
|
1891
3284
|
usageByChild: {},
|
|
1892
3285
|
}
|
|
3286
|
+
if (atCapacity) {
|
|
3287
|
+
// Waiting runs are persisted with an owner and a heartbeat so another
|
|
3288
|
+
// process sharing this directory can tell a live queue from a dead one.
|
|
3289
|
+
run.queue = {
|
|
3290
|
+
waiting: true,
|
|
3291
|
+
position: waiting + 1,
|
|
3292
|
+
of: waiting + 1,
|
|
3293
|
+
enqueuedAt: new Date(now()).toISOString(),
|
|
3294
|
+
timeoutAt: new Date(now() + queueTimeoutMs()).toISOString(),
|
|
3295
|
+
}
|
|
3296
|
+
run.ownerPid = process.pid
|
|
3297
|
+
run.ownerHost = os.hostname()
|
|
3298
|
+
run.heartbeatAt = new Date(now()).toISOString()
|
|
3299
|
+
await store.write(run)
|
|
3300
|
+
enqueue(run, parent)
|
|
3301
|
+
await refreshQueuePositions()
|
|
3302
|
+
log(`run ${run.id} is queued at position ${run.queue.position} (${active} active run(s))`)
|
|
3303
|
+
return run
|
|
3304
|
+
}
|
|
1893
3305
|
await takeRunLock(run.id)
|
|
1894
3306
|
try {
|
|
1895
3307
|
await store.write(run)
|
|
@@ -1912,20 +3324,33 @@ export function createOrchestrator(deps) {
|
|
|
1912
3324
|
}
|
|
1913
3325
|
}
|
|
1914
3326
|
|
|
1915
|
-
// A resume only finishes work that already exists, so it is never counted
|
|
1916
|
-
// against `maxActiveRuns`: a person asking to finish an interrupted run is
|
|
1917
|
-
// always let through.
|
|
1918
|
-
// `force`
|
|
1919
|
-
|
|
3327
|
+
// A resume only finishes work that already exists, so it is never counted
|
|
3328
|
+
// against `maxActiveRuns`: a person asking to finish an interrupted run is
|
|
3329
|
+
// always let through. An options bag is still accepted so an older caller
|
|
3330
|
+
// that passes `force` keeps working; it no longer changes the outcome.
|
|
3331
|
+
/**
|
|
3332
|
+
* Resume under the start gate. Split from `resume` so answering a question
|
|
3333
|
+
* can record the answer and reopen the run in one serialized step instead of
|
|
3334
|
+
* waiting on a nested gate (which would deadlock on the promise it is holding).
|
|
3335
|
+
*
|
|
3336
|
+
* `reopen` lets `superagent_answer` continue a `completed` run: its subtasks
|
|
3337
|
+
* stay done and only the aggregation runs again, now with the answer.
|
|
3338
|
+
*/
|
|
3339
|
+
async function resumeLocked(runId, parent, clientTimeZone, { reopen = false, approvePolicy = false } = {}) {
|
|
1920
3340
|
const id = String(runId ?? '').trim()
|
|
1921
3341
|
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
1922
|
-
|
|
3342
|
+
{
|
|
1923
3343
|
if (runtime.has(id)) {
|
|
1924
3344
|
throw new Error(`run ${id} is still active in this process; wait for it with superagent_status or cancel it`)
|
|
1925
3345
|
}
|
|
1926
3346
|
let run = await loadRun(id)
|
|
1927
3347
|
assertResumeOwner(run, parent)
|
|
1928
|
-
|
|
3348
|
+
// A completed run is normally finished for good. The exception is an
|
|
3349
|
+
// answer that was recorded but never applied because the reopen failed
|
|
3350
|
+
// (quota, lock): then resume is the recovery path and must actually run
|
|
3351
|
+
// the aggregation again, not return the old report.
|
|
3352
|
+
let reopenCompleted = reopen || hasUnappliedAnswer(run)
|
|
3353
|
+
if (run.status === 'completed' && !reopenCompleted) return run
|
|
1929
3354
|
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1930
3355
|
const verdict = await ownership(run)
|
|
1931
3356
|
if (verdict.live) {
|
|
@@ -1943,7 +3368,38 @@ export function createOrchestrator(deps) {
|
|
|
1943
3368
|
// between the check above and the claim.
|
|
1944
3369
|
run = await loadRun(id)
|
|
1945
3370
|
assertResumeOwner(run, parent)
|
|
1946
|
-
|
|
3371
|
+
reopenCompleted = reopen || hasUnappliedAnswer(run)
|
|
3372
|
+
if (run.status === 'completed' && !reopenCompleted) return run
|
|
3373
|
+
// A resume spends the remaining budget again, so a per-run deny applies
|
|
3374
|
+
// to it as well; otherwise a policy added mid-run could be bypassed by
|
|
3375
|
+
// resuming the run instead of starting a new one.
|
|
3376
|
+
if (audit && typeof audit.gate === 'function') {
|
|
3377
|
+
const verdict = await audit.gate({
|
|
3378
|
+
runId: run.id,
|
|
3379
|
+
kind: 'run.resume',
|
|
3380
|
+
action: 'superagent_resume',
|
|
3381
|
+
actor: `session:${parent.id}`,
|
|
3382
|
+
target: oneLine(run.goal, 200),
|
|
3383
|
+
})
|
|
3384
|
+
const trail = verdict.eventId ? ` (audit event ${verdict.eventId})` : ''
|
|
3385
|
+
if (verdict.decision === 'deny') {
|
|
3386
|
+
throw new Error(`refused by the permission policy: ${verdict.reason}${trail}`)
|
|
3387
|
+
}
|
|
3388
|
+
if (verdict.decision === 'confirm') {
|
|
3389
|
+
if (!approvePolicy) {
|
|
3390
|
+
throw new Error(
|
|
3391
|
+
`the permission policy holds this resume for confirmation: ${verdict.reason}${trail}; `
|
|
3392
|
+
+ 'call superagent_resume again with approve_policy: true to release it',
|
|
3393
|
+
)
|
|
3394
|
+
}
|
|
3395
|
+
await audit.approve(verdict.eventId, {
|
|
3396
|
+
actor: `session:${parent.id}`,
|
|
3397
|
+
note: 'approve_policy on superagent_resume',
|
|
3398
|
+
runId: run.id,
|
|
3399
|
+
kind: 'run.resume',
|
|
3400
|
+
})
|
|
3401
|
+
}
|
|
3402
|
+
}
|
|
1947
3403
|
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1948
3404
|
const verdict = await ownership(run)
|
|
1949
3405
|
if (verdict.live) {
|
|
@@ -1970,18 +3426,29 @@ export function createOrchestrator(deps) {
|
|
|
1970
3426
|
|
|
1971
3427
|
for (const subtask of run.subtasks ?? []) {
|
|
1972
3428
|
if (subtask.status === 'completed') continue
|
|
3429
|
+
// A dependency on a subtask the plan never contained cannot be fixed
|
|
3430
|
+
// by resuming: nothing would ever clear it.
|
|
3431
|
+
if (subtask.skipKind === 'missing') continue
|
|
1973
3432
|
subtask.status = 'pending'
|
|
1974
3433
|
subtask.attempts = 0
|
|
1975
3434
|
subtask.error = null
|
|
1976
3435
|
subtask.stopReason = null
|
|
1977
3436
|
subtask.finishedAt = null
|
|
3437
|
+
subtask.skipKind = null
|
|
3438
|
+
subtask.skippedReason = null
|
|
3439
|
+
subtask.waitingOn = []
|
|
1978
3440
|
}
|
|
1979
3441
|
run.status = 'queued'
|
|
1980
|
-
|
|
3442
|
+
// A reopened run has nothing left to execute, so the old report must go:
|
|
3443
|
+
// keeping it would return the report that ignored the answer.
|
|
3444
|
+
run.report = reopenCompleted ? null : (run.plan ? run.report : null)
|
|
1981
3445
|
run.error = null
|
|
1982
3446
|
run.finishedAt = null
|
|
1983
3447
|
run.reclaimed = null
|
|
1984
3448
|
run.cancelPending = false
|
|
3449
|
+
// A queued marker from an earlier attempt would keep this run out of the
|
|
3450
|
+
// active count after it is scheduled again.
|
|
3451
|
+
run.queue = null
|
|
1985
3452
|
// A resumed run completes a second time, so it must be able to wake the
|
|
1986
3453
|
// session again; keeping the old outcome would silence it forever.
|
|
1987
3454
|
run.notified = false
|
|
@@ -1991,13 +3458,28 @@ export function createOrchestrator(deps) {
|
|
|
1991
3458
|
run.selfHeal = null
|
|
1992
3459
|
run.memoryWrite = null
|
|
1993
3460
|
// A question from the previous attempt was already reported; keeping it
|
|
1994
|
-
// would attach an old decision to the new outcome.
|
|
3461
|
+
// would attach an old decision to the new outcome. It is archived rather
|
|
3462
|
+
// than dropped, so a late answer is refused as expired instead of
|
|
3463
|
+
// looking like an id that never existed.
|
|
3464
|
+
if (run.attention && ((run.attention.items ?? []).length > 0 || run.attention.issue)) {
|
|
3465
|
+
run.attentionHistory = Array.isArray(run.attentionHistory) ? run.attentionHistory : []
|
|
3466
|
+
run.attentionHistory.push({ ...run.attention, expiredAt: new Date(now()).toISOString() })
|
|
3467
|
+
if (run.attentionHistory.length > ATTENTION_HISTORY_MAX) {
|
|
3468
|
+
run.attentionHistory = run.attentionHistory.slice(-ATTENTION_HISTORY_MAX)
|
|
3469
|
+
}
|
|
3470
|
+
}
|
|
1995
3471
|
run.attention = null
|
|
3472
|
+
// An instruction that missed the previous attempt gets another chance:
|
|
3473
|
+
// the new round builds fresh prompts.
|
|
3474
|
+
for (const entry of run.followups ?? []) {
|
|
3475
|
+
entry.missedAt = null
|
|
3476
|
+
entry.missedReason = null
|
|
3477
|
+
}
|
|
1996
3478
|
run.dedupeKey = run.dedupeKey ?? dedupeKeyOf(run.goal)
|
|
1997
|
-
run.resumedAt = new Date(now()).toISOString()
|
|
1998
|
-
// A resume replans nothing, but the aggregator still runs, so refresh
|
|
1999
|
-
// the digest the notes have advanced since the run started.
|
|
2000
|
-
run.memory = await memoryContext(run.id)
|
|
3479
|
+
run.resumedAt = new Date(now()).toISOString()
|
|
3480
|
+
// A resume replans nothing, but the aggregator still runs, so refresh
|
|
3481
|
+
// the digest the notes have advanced since the run started.
|
|
3482
|
+
run.memory = await memoryContext(run.id, run.goal)
|
|
2001
3483
|
run.clientTimeZone = clientTimeZone ?? run.clientTimeZone ?? null
|
|
2002
3484
|
// The limits are rebuilt from the config that is active now, keeping
|
|
2003
3485
|
// everything already spent: that is what makes "raise the cap, then
|
|
@@ -2007,15 +3489,236 @@ export function createOrchestrator(deps) {
|
|
|
2007
3489
|
ensureCounters(run)
|
|
2008
3490
|
await save(run)
|
|
2009
3491
|
schedule(run, parent)
|
|
3492
|
+
auditRecord({
|
|
3493
|
+
runId: run.id,
|
|
3494
|
+
actor: `session:${parent.id}`,
|
|
3495
|
+
kind: 'run.resume',
|
|
3496
|
+
action: 'superagent_resume',
|
|
3497
|
+
target: oneLine(run.goal, 200),
|
|
3498
|
+
result: 'queued',
|
|
3499
|
+
meta: { previousStatus: run.recovery?.previousStatus ?? null },
|
|
3500
|
+
})
|
|
2010
3501
|
scheduled = true
|
|
2011
3502
|
return run
|
|
2012
3503
|
} finally {
|
|
2013
3504
|
if (!scheduled) await releaseRunLock(id)
|
|
2014
3505
|
}
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
3508
|
+
|
|
3509
|
+
/** Serialize resumes; `resumeLocked` assumes the start gate is already held. */
|
|
3510
|
+
function resume(runId, parent, clientTimeZone, options) {
|
|
3511
|
+
return serialize(() => resumeLocked(runId, parent, clientTimeZone, options))
|
|
3512
|
+
}
|
|
3513
|
+
|
|
3514
|
+
/**
|
|
3515
|
+
* Add an instruction to a run that is already executing.
|
|
3516
|
+
*
|
|
3517
|
+
* The host exposes no way to append a message to a child activation, so this
|
|
3518
|
+
* is deliberately not a live channel: the instruction is queued on the run
|
|
3519
|
+
* and the next prompt built for it carries it. The reply says which prompt
|
|
3520
|
+
* that will be. A run owned by another process is refused rather than written:
|
|
3521
|
+
* that owner's next save would overwrite the change and the instruction would
|
|
3522
|
+
* silently vanish.
|
|
3523
|
+
*
|
|
3524
|
+
* @param {string} runId - run identity.
|
|
3525
|
+
* @param {string} text - the instruction.
|
|
3526
|
+
* @param {object} parent - live agent that owns the run.
|
|
3527
|
+
* @returns {Promise<{ run: object, followup: object, effect: { boundary: string, detail: string } }>}
|
|
3528
|
+
*/
|
|
3529
|
+
async function followup(runId, text, parent) {
|
|
3530
|
+
const id = String(runId ?? '').trim()
|
|
3531
|
+
if (!id) throw new Error('run_id is required')
|
|
3532
|
+
const message = capChars(text, FOLLOWUP_MAX_CHARS)
|
|
3533
|
+
if (!message) throw new Error('message is required; pass the instruction to add to the run')
|
|
3534
|
+
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
3535
|
+
return serialize(async () => {
|
|
3536
|
+
const rt = runtime.get(id)
|
|
3537
|
+
if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
|
|
3538
|
+
const run = rt.run
|
|
3539
|
+
assertResumeOwner(run, parent)
|
|
3540
|
+
const entry = pushFollowup(run, { kind: 'followup', text: message })
|
|
3541
|
+
await save(run)
|
|
3542
|
+
return { run, followup: entry, effect: followupEffect(run) }
|
|
3543
|
+
}
|
|
3544
|
+
const run = await loadRun(id)
|
|
3545
|
+
assertResumeOwner(run, parent)
|
|
3546
|
+
if (ACTIVE_STATUSES.has(run.status)) {
|
|
3547
|
+
throw new Error(
|
|
3548
|
+
`run ${id} is active but owned by another process; an instruction written here would be overwritten by `
|
|
3549
|
+
+ 'that process\'s next save. Send it from the session that owns the run, or cancel the run and resume it '
|
|
3550
|
+
+ 'here first.',
|
|
3551
|
+
)
|
|
3552
|
+
}
|
|
3553
|
+
if (run.status !== 'completed') {
|
|
3554
|
+
// A paused run will build fresh prompts when it is resumed, so queueing
|
|
3555
|
+
// the instruction now is a real delivery, not a courtesy.
|
|
3556
|
+
const entry = pushFollowup(run, { kind: 'followup', text: message })
|
|
3557
|
+
await save(run)
|
|
3558
|
+
return { run, followup: entry, effect: followupEffect(run) }
|
|
3559
|
+
}
|
|
3560
|
+
throw new Error(
|
|
3561
|
+
`run ${id} is completed, so it is not running and cannot take an instruction. `
|
|
3562
|
+
+ 'Start a new run, or answer a pending question with superagent_answer.',
|
|
3563
|
+
)
|
|
3564
|
+
})
|
|
3565
|
+
}
|
|
3566
|
+
|
|
3567
|
+
/**
|
|
3568
|
+
* Answer one structured attention item and let the run continue.
|
|
3569
|
+
*
|
|
3570
|
+
* The answer is matched by id, never by array position, and is recorded on
|
|
3571
|
+
* the item it answers. An active run picks it up at its next boundary; a
|
|
3572
|
+
* terminal run is reopened so its aggregation runs again with the decision —
|
|
3573
|
+
* that is the difference between continuing this run and starting another one.
|
|
3574
|
+
*
|
|
3575
|
+
* @param {string} runId - run identity.
|
|
3576
|
+
* @param {string | null | undefined} attentionId - `q1` from status; optional only when one question is open.
|
|
3577
|
+
* @param {string} text - the decision.
|
|
3578
|
+
* @param {object} parent - live agent that owns the run.
|
|
3579
|
+
* @param {string | undefined} clientTimeZone - forwarded to a resume.
|
|
3580
|
+
* @returns {Promise<object>} run plus which question was answered and whether the run restarted.
|
|
3581
|
+
*/
|
|
3582
|
+
async function answer(runId, attentionId, text, parent, clientTimeZone) {
|
|
3583
|
+
const id = String(runId ?? '').trim()
|
|
3584
|
+
if (!id) throw new Error('run_id is required')
|
|
3585
|
+
const value = capChars(text, FOLLOWUP_MAX_CHARS)
|
|
3586
|
+
if (!value) throw new Error('answer is required; pass the decision the run should use')
|
|
3587
|
+
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
3588
|
+
return serialize(async () => {
|
|
3589
|
+
const rt = runtime.get(id)
|
|
3590
|
+
const run = rt?.run ?? await loadRun(id)
|
|
3591
|
+
assertResumeOwner(run, parent)
|
|
3592
|
+
const picked = selectAttention(run, attentionId, value)
|
|
3593
|
+
const at = new Date(now()).toISOString()
|
|
3594
|
+
picked.item.answer = value
|
|
3595
|
+
picked.item.answeredAt = at
|
|
3596
|
+
const entry = pushFollowup(run, {
|
|
3597
|
+
kind: 'answer',
|
|
3598
|
+
attentionId: picked.id,
|
|
3599
|
+
text: `Answer to question ${picked.id} ("${picked.item.question}"): ${value}. `
|
|
3600
|
+
+ 'Apply this decision and continue the work.',
|
|
3601
|
+
})
|
|
3602
|
+
await save(run)
|
|
3603
|
+
|
|
3604
|
+
if (ACTIVE_STATUSES.has(run.status)) {
|
|
3605
|
+
return { run, attention: picked.item, attentionId: picked.id, followup: entry, resumed: false, effect: followupEffect(run) }
|
|
3606
|
+
}
|
|
3607
|
+
try {
|
|
3608
|
+
const resumed = await resumeLocked(id, parent, clientTimeZone, { reopen: true })
|
|
3609
|
+
return { run: resumed, attention: picked.item, attentionId: picked.id, followup: entry, resumed: true, effect: followupEffect(resumed) }
|
|
3610
|
+
} catch (error) {
|
|
3611
|
+
// The answer is durable; only the restart failed. Saying so is better
|
|
3612
|
+
// than pretending the decision was applied or losing it.
|
|
3613
|
+
return {
|
|
3614
|
+
run: (await store.read(id)) ?? run,
|
|
3615
|
+
attention: picked.item,
|
|
3616
|
+
attentionId: picked.id,
|
|
3617
|
+
followup: entry,
|
|
3618
|
+
resumed: false,
|
|
3619
|
+
reason: error?.message ?? String(error),
|
|
3620
|
+
}
|
|
3621
|
+
}
|
|
2015
3622
|
})
|
|
2016
3623
|
}
|
|
2017
3624
|
|
|
3625
|
+
/**
|
|
3626
|
+
* Find the question an answer belongs to, and refuse the three ways an answer
|
|
3627
|
+
* can miss: an id that was never asked, a question retired by a resume, and a
|
|
3628
|
+
* question already answered. A free-form answer is allowed — options are
|
|
3629
|
+
* suggestions — but an answer that matches another question's option is
|
|
3630
|
+
* treated as a misdirected answer and named as such.
|
|
3631
|
+
*
|
|
3632
|
+
* @param {object} run - run record.
|
|
3633
|
+
* @param {unknown} requested - `attention_id` as given.
|
|
3634
|
+
* @param {string} value - the answer text.
|
|
3635
|
+
* @returns {{ item: object, id: string }} the matched question.
|
|
3636
|
+
*/
|
|
3637
|
+
function selectAttention(run, requested, value) {
|
|
3638
|
+
const items = Array.isArray(run.attention?.items) ? run.attention.items : []
|
|
3639
|
+
const idOf = (item, index) => item?.id ?? `q${index + 1}`
|
|
3640
|
+
const idIn = item => idOf(item, items.indexOf(item))
|
|
3641
|
+
const open = items.filter(item => !item.answer)
|
|
3642
|
+
const wanted = String(requested ?? '').trim()
|
|
3643
|
+
const alreadyAnswered = item => `question ${idIn(item)} was already answered at ${item.answeredAt}: `
|
|
3644
|
+
+ `${item.answer}. Answering again would replace a decision the run already acted on.`
|
|
3645
|
+
const describeOpen = () => (open.length === 0
|
|
3646
|
+
? 'this run is not waiting for an answer'
|
|
3647
|
+
: `open question(s): ${open.map(idIn).join(', ')}`)
|
|
3648
|
+
|
|
3649
|
+
if (!wanted) {
|
|
3650
|
+
// An id is optional only when there is exactly one question it could mean.
|
|
3651
|
+
if (open.length === 1) return { item: open[0], id: idIn(open[0]) }
|
|
3652
|
+
if (open.length > 1) {
|
|
3653
|
+
throw new Error(
|
|
3654
|
+
`run ${run.id} has ${open.length} open questions, so an answer needs attention_id to say which one: `
|
|
3655
|
+
+ `${open.map(idIn).join(', ')}.`,
|
|
3656
|
+
)
|
|
3657
|
+
}
|
|
3658
|
+
if (items.length > 0) throw new Error(alreadyAnswered(items[0]))
|
|
3659
|
+
throw new Error(`run ${run.id} is not waiting for an answer (${describeOpen()}).`)
|
|
3660
|
+
}
|
|
3661
|
+
|
|
3662
|
+
const match = open.find(item => idIn(item) === wanted)
|
|
3663
|
+
if (match) return { item: match, id: wanted }
|
|
3664
|
+
const answeredItem = items.find(item => idIn(item) === wanted)
|
|
3665
|
+
if (answeredItem) throw new Error(alreadyAnswered(answeredItem))
|
|
3666
|
+
|
|
3667
|
+
for (const batch of Array.isArray(run.attentionHistory) ? run.attentionHistory : []) {
|
|
3668
|
+
const retired = (batch.items ?? []).find((item, position) => idOf(item, position) === wanted)
|
|
3669
|
+
if (!retired) continue
|
|
3670
|
+
if (retired.answer) {
|
|
3671
|
+
throw new Error(
|
|
3672
|
+
`question ${wanted} was already answered at ${retired.answeredAt}: ${retired.answer}. `
|
|
3673
|
+
+ 'It is no longer open, so there is nothing to change.',
|
|
3674
|
+
)
|
|
3675
|
+
}
|
|
3676
|
+
throw new Error(
|
|
3677
|
+
`question ${wanted} expired at ${batch.expiredAt}: the run was resumed after asking it, so that question no `
|
|
3678
|
+
+ 'longer blocks anything and an answer now would attach to work that is not waiting for it. '
|
|
3679
|
+
+ 'Read superagent_status for the current question.',
|
|
3680
|
+
)
|
|
3681
|
+
}
|
|
3682
|
+
|
|
3683
|
+
// The id is wrong, but the text is exactly another question's option: that
|
|
3684
|
+
// is a misdirected answer, and naming the right id is more useful than
|
|
3685
|
+
// "no such question".
|
|
3686
|
+
const misdirected = items.findIndex((item, position) => idOf(item, position) !== wanted
|
|
3687
|
+
&& (item.options ?? []).some(option => String(option).trim().toLowerCase() === value.toLowerCase()))
|
|
3688
|
+
if (misdirected >= 0) {
|
|
3689
|
+
const other = items[misdirected]
|
|
3690
|
+
throw new Error(
|
|
3691
|
+
`"${value}" is one of the options of question ${idOf(other, misdirected)} ("${other.question}"), not of `
|
|
3692
|
+
+ `${wanted}. Pass attention_id="${idOf(other, misdirected)}" if that is the question you mean.`,
|
|
3693
|
+
)
|
|
3694
|
+
}
|
|
3695
|
+
throw new Error(`no question ${wanted} on run ${run.id}; ${describeOpen()}.`)
|
|
3696
|
+
}
|
|
3697
|
+
|
|
2018
3698
|
async function cancel(runId) {
|
|
3699
|
+
// A run that never got a slot has no owner process to ask: remove it from
|
|
3700
|
+
// the line here, so no timer can start it afterwards.
|
|
3701
|
+
const queuedIndex = queue.findIndex(entry => entry.run.id === runId)
|
|
3702
|
+
if (queuedIndex >= 0) {
|
|
3703
|
+
const entry = queue.splice(queuedIndex, 1)[0]
|
|
3704
|
+
if (entry.timer) clearTimer(entry.timer)
|
|
3705
|
+
if (queue.length === 0) stopQueueTicker()
|
|
3706
|
+
const run = entry.run
|
|
3707
|
+
if (isWaiting(run)) {
|
|
3708
|
+
run.queue = { ...run.queue, waiting: false, cancelled: true }
|
|
3709
|
+
run.status = 'cancelled'
|
|
3710
|
+
run.error = 'cancelled before a free slot was available; nothing was spent'
|
|
3711
|
+
run.finishedAt = new Date(now()).toISOString()
|
|
3712
|
+
run.activeMs = Number.isFinite(run.activeMs) ? run.activeMs : 0
|
|
3713
|
+
await save(run)
|
|
3714
|
+
await clearCancel(run.id)
|
|
3715
|
+
await releaseRunLock(run.id)
|
|
3716
|
+
}
|
|
3717
|
+
await refreshQueuePositions().catch(() => {})
|
|
3718
|
+
pumpQueue().catch(() => {})
|
|
3719
|
+
return run
|
|
3720
|
+
}
|
|
3721
|
+
|
|
2019
3722
|
const rt = runtime.get(runId)
|
|
2020
3723
|
if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
|
|
2021
3724
|
rt.abortCause = 'cancelled'
|
|
@@ -2029,6 +3732,14 @@ export function createOrchestrator(deps) {
|
|
|
2029
3732
|
rt.run.cancelPending = true
|
|
2030
3733
|
await save(rt.run).catch(() => {})
|
|
2031
3734
|
}
|
|
3735
|
+
auditRecord({
|
|
3736
|
+
runId,
|
|
3737
|
+
actor: 'session',
|
|
3738
|
+
kind: 'run.cancel',
|
|
3739
|
+
action: 'superagent_cancel',
|
|
3740
|
+
result: ACTIVE_STATUSES.has(rt.run.status) ? 'requested' : 'cancelled',
|
|
3741
|
+
meta: { ownedHere: true },
|
|
3742
|
+
})
|
|
2032
3743
|
return rt.run
|
|
2033
3744
|
}
|
|
2034
3745
|
|
|
@@ -2040,6 +3751,14 @@ export function createOrchestrator(deps) {
|
|
|
2040
3751
|
// Writing `cancelled` here would be a lie: the owner would overwrite it on
|
|
2041
3752
|
// its next save and keep spending.
|
|
2042
3753
|
await requestCancel(runId)
|
|
3754
|
+
auditRecord({
|
|
3755
|
+
runId,
|
|
3756
|
+
actor: 'session',
|
|
3757
|
+
kind: 'run.cancel',
|
|
3758
|
+
action: 'superagent_cancel',
|
|
3759
|
+
result: 'requested',
|
|
3760
|
+
meta: { ownedHere: false },
|
|
3761
|
+
})
|
|
2043
3762
|
const endsAt = now() + CANCEL_ACK_MS
|
|
2044
3763
|
while (now() < endsAt) {
|
|
2045
3764
|
await sleep(STATUS_POLL_MS)
|
|
@@ -2114,5 +3833,19 @@ export function createOrchestrator(deps) {
|
|
|
2114
3833
|
await Promise.all([...runtime.values()].map(rt => rt.promise?.catch(() => {})))
|
|
2115
3834
|
}
|
|
2116
3835
|
|
|
2117
|
-
return {
|
|
3836
|
+
return {
|
|
3837
|
+
recover,
|
|
3838
|
+
start,
|
|
3839
|
+
resume,
|
|
3840
|
+
cancel,
|
|
3841
|
+
followup,
|
|
3842
|
+
answer,
|
|
3843
|
+
inject,
|
|
3844
|
+
waitForInjection,
|
|
3845
|
+
status,
|
|
3846
|
+
list,
|
|
3847
|
+
idle,
|
|
3848
|
+
activeCount,
|
|
3849
|
+
liveRun,
|
|
3850
|
+
}
|
|
2118
3851
|
}
|