@xiangyixie/dsh-superagent 0.9.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/LICENSE +21 -21
- package/README.md +422 -42
- package/bin/archive.js +185 -0
- package/bin/doctor.js +84 -0
- package/bin/report.js +79 -79
- package/bin/watch.js +57 -57
- package/cordis.patch.yml +9 -9
- package/lib/control.js +495 -494
- package/lib/cost.js +267 -0
- package/lib/doctor.js +876 -0
- package/lib/errors.js +205 -199
- package/lib/index.js +767 -42
- package/lib/ledger.js +270 -0
- package/lib/lock.js +107 -0
- package/lib/memory.js +736 -0
- package/lib/notify.js +257 -0
- package/lib/orchestrator.js +1975 -127
- package/lib/report.js +350 -15
- package/lib/store.js +503 -15
- package/lib/templates.js +690 -0
- package/lib/usage.js +152 -152
- package/lib/watch.js +422 -70
- package/lib/workspace.js +433 -0
- package/package.json +16 -3
package/lib/orchestrator.js
CHANGED
|
@@ -16,9 +16,14 @@
|
|
|
16
16
|
*
|
|
17
17
|
* @module dsh-superagent/orchestrator
|
|
18
18
|
*/
|
|
19
|
-
import { randomBytes } from 'node:crypto'
|
|
19
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
20
|
+
import fsp from 'node:fs/promises'
|
|
20
21
|
import os from 'node:os'
|
|
21
|
-
import
|
|
22
|
+
import path from 'node:path'
|
|
23
|
+
import { applyCost, computeCost } from './cost.js'
|
|
24
|
+
import { extractMemoryBlock } from './memory.js'
|
|
25
|
+
import { ACTIVE_STATUSES, GOAL_GUARD_PREFIX, TERMINAL_STATUSES } from './store.js'
|
|
26
|
+
import { createWorkspaceManager } from './workspace.js'
|
|
22
27
|
|
|
23
28
|
const STATUS_POLL_MS = 250
|
|
24
29
|
/** How often the owning process checks for a cancel marker written elsewhere. */
|
|
@@ -35,6 +40,139 @@ const DISPOSE_MS = 1500
|
|
|
35
40
|
const NOTIFY_MS = 15 * 1000
|
|
36
41
|
/** A run lock older than this belongs to a process that never released it. */
|
|
37
42
|
const LOCK_STALE_MS = 10 * 60 * 1000
|
|
43
|
+
/** How long a start waits for another process that is claiming the same goal. */
|
|
44
|
+
const GOAL_GUARD_WAIT_MS = 5000
|
|
45
|
+
/** How often that wait re-tries the goal guard. */
|
|
46
|
+
const GOAL_GUARD_POLL_MS = 50
|
|
47
|
+
|
|
48
|
+
/** First retry delay for a failure that looks transient; it doubles per attempt. */
|
|
49
|
+
const RETRY_BASE_MS = 2000
|
|
50
|
+
/** A retry delay never grows past this, so a long run cannot stall for hours. */
|
|
51
|
+
const RETRY_CAP_MS = 60 * 1000
|
|
52
|
+
|
|
53
|
+
/** Fenced block a worker or the aggregator uses to ask for a decision. */
|
|
54
|
+
const ATTENTION_FENCE = /```attention[^\n]*\n([\s\S]*?)```/i
|
|
55
|
+
|
|
56
|
+
/** Instructions kept per run; older ones are dropped so a long dialog stays bounded. */
|
|
57
|
+
const FOLLOWUP_MAX = 8
|
|
58
|
+
/** Characters kept from one instruction or answer. */
|
|
59
|
+
const FOLLOWUP_MAX_CHARS = 1000
|
|
60
|
+
/** How many expired attention batches stay readable for a late answer. */
|
|
61
|
+
const ATTENTION_HISTORY_MAX = 5
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Which pipeline a run uses.
|
|
65
|
+
*
|
|
66
|
+
* `planned` is the default: a planner splits the goal, workers run in parallel,
|
|
67
|
+
* and an aggregator merges their reports. `direct` is for one self-contained
|
|
68
|
+
* task: a single worker does it, there is no plan and no merge, and its reply
|
|
69
|
+
* is the report verbatim. Direct saves two model calls and the coordination
|
|
70
|
+
* they need, so it is right for a single lookup, edit, query, or short piece of
|
|
71
|
+
* writing — and wrong for a goal with parts that can be worked in parallel.
|
|
72
|
+
*
|
|
73
|
+
* @param {unknown} value - requested mode.
|
|
74
|
+
* @returns {'planned' | 'direct'} normalized mode.
|
|
75
|
+
*/
|
|
76
|
+
export function normalizeMode(value) {
|
|
77
|
+
if (value === undefined || value === null || value === '') return 'planned'
|
|
78
|
+
const text = String(value).trim().toLowerCase()
|
|
79
|
+
if (text === 'planned' || text === 'direct') return text
|
|
80
|
+
throw new Error(
|
|
81
|
+
`unknown mode ${JSON.stringify(value)}; use "planned" (multi-worker) or "direct" (one worker, no planning or aggregation)`,
|
|
82
|
+
)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Canonical form of a goal: case and runs of whitespace are formatting, not a
|
|
87
|
+
* different request, so they must not defeat duplicate detection.
|
|
88
|
+
*
|
|
89
|
+
* @param {unknown} goal - raw goal text.
|
|
90
|
+
* @returns {string} normalized goal, or '' when there is no goal.
|
|
91
|
+
*/
|
|
92
|
+
export function normalizeGoal(goal) {
|
|
93
|
+
if (typeof goal !== 'string') return ''
|
|
94
|
+
return goal.replace(/\s+/g, ' ').trim().toLowerCase()
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Stable identity of one goal, used to dispatch the same work only once.
|
|
99
|
+
* @param {unknown} goal - raw goal text.
|
|
100
|
+
* @returns {string | null} sha256 hex of the normalized goal, or null when empty.
|
|
101
|
+
*/
|
|
102
|
+
export function dedupeKeyOf(goal) {
|
|
103
|
+
const normalized = normalizeGoal(goal)
|
|
104
|
+
if (!normalized) return null
|
|
105
|
+
return createHash('sha256').update(normalized, 'utf8').digest('hex')
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Collapse arbitrary text into one bounded line.
|
|
110
|
+
*
|
|
111
|
+
* A question, option, or reason comes from a model, so it can contain newlines
|
|
112
|
+
* and even a line that looks like a plugin message. A status line and a
|
|
113
|
+
* notification lead are single lines; keeping the raw text would let the block
|
|
114
|
+
* forge both. Counting by code points keeps a multi-byte character intact.
|
|
115
|
+
*
|
|
116
|
+
* @param {unknown} value - source text.
|
|
117
|
+
* @param {number} maxChars - maximum number of characters.
|
|
118
|
+
* @returns {string} one line, at most `maxChars` characters.
|
|
119
|
+
*/
|
|
120
|
+
function oneLine(value, maxChars) {
|
|
121
|
+
const text = String(value ?? '').replace(/\s+/g, ' ').trim()
|
|
122
|
+
if (!text) return ''
|
|
123
|
+
return [...text].slice(0, maxChars).join('')
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Pull the fenced `attention` block out of a worker or aggregator reply.
|
|
128
|
+
*
|
|
129
|
+
* A valid block is removed from the report: the user asked for a report, not
|
|
130
|
+
* for plugin syntax. An invalid block is *kept* in the text, because stripping
|
|
131
|
+
* it would silently erase the fact that a decision was requested; the caller
|
|
132
|
+
* records the error so status still says so.
|
|
133
|
+
*
|
|
134
|
+
* @param {unknown} text - child reply or final report.
|
|
135
|
+
* @returns {{ text: string, attention: { question: string, options: string[], reason: string | null } | null, found: boolean, error?: string }}
|
|
136
|
+
*/
|
|
137
|
+
export function extractAttentionBlock(text) {
|
|
138
|
+
if (typeof text !== 'string' || !text) {
|
|
139
|
+
return { text: typeof text === 'string' ? text : '', attention: null, found: false }
|
|
140
|
+
}
|
|
141
|
+
const match = ATTENTION_FENCE.exec(text)
|
|
142
|
+
if (!match) return { text, attention: null, found: false }
|
|
143
|
+
const rest = `${text.slice(0, match.index)}${text.slice(match.index + match[0].length)}`
|
|
144
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
145
|
+
.trim()
|
|
146
|
+
let parsed
|
|
147
|
+
try {
|
|
148
|
+
parsed = JSON.parse(match[1].trim())
|
|
149
|
+
} catch {
|
|
150
|
+
return {
|
|
151
|
+
text,
|
|
152
|
+
attention: null,
|
|
153
|
+
found: true,
|
|
154
|
+
error: 'the attention block was not valid JSON; the raw block is kept in the report',
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const question = oneLine(parsed?.question ?? parsed?.text, 500)
|
|
158
|
+
if (!question) {
|
|
159
|
+
return {
|
|
160
|
+
text,
|
|
161
|
+
attention: null,
|
|
162
|
+
found: true,
|
|
163
|
+
error: 'the attention block had no question; the raw block is kept in the report',
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const options = Array.isArray(parsed?.options)
|
|
167
|
+
? parsed.options.map(option => oneLine(option, 200)).filter(Boolean).slice(0, 6)
|
|
168
|
+
: []
|
|
169
|
+
const reason = oneLine(parsed?.reason, 300) || null
|
|
170
|
+
return {
|
|
171
|
+
text: rest,
|
|
172
|
+
attention: { question, options, reason },
|
|
173
|
+
found: true,
|
|
174
|
+
}
|
|
175
|
+
}
|
|
38
176
|
|
|
39
177
|
/**
|
|
40
178
|
* Mint a sortable, filesystem-safe run id.
|
|
@@ -81,7 +219,8 @@ export function capText(text, maxBytes) {
|
|
|
81
219
|
* region is tried rather than assuming the reply is bare JSON.
|
|
82
220
|
*
|
|
83
221
|
* @param {string} text - planner reply.
|
|
84
|
-
* @returns {Array<{ title: string, prompt: string
|
|
222
|
+
* @returns {Array<{ title: string, prompt: string, id?: string, dependsOn?: string[] }>}
|
|
223
|
+
* zero or more subtasks; `id` and `dependsOn` appear only when the planner set them.
|
|
85
224
|
*/
|
|
86
225
|
export function extractPlan(text) {
|
|
87
226
|
if (typeof text !== 'string') return []
|
|
@@ -128,36 +267,239 @@ export function extractPlan(text) {
|
|
|
128
267
|
const title = typeof item.title === 'string' && item.title.trim()
|
|
129
268
|
? item.title.trim().slice(0, 80)
|
|
130
269
|
: prompt.slice(0, 60)
|
|
131
|
-
|
|
270
|
+
const entry = { title, prompt }
|
|
271
|
+
const id = oneLine(item.id, 32)
|
|
272
|
+
if (/^[A-Za-z0-9][A-Za-z0-9._:-]{0,31}$/.test(id)) entry.id = id
|
|
273
|
+
if (Array.isArray(item.dependsOn)) {
|
|
274
|
+
// References are kept verbatim here; whether each one names a real
|
|
275
|
+
// subtask (by id or title) is decided once the whole plan is known.
|
|
276
|
+
const dependsOn = [...new Set(item.dependsOn.map(ref => oneLine(ref, 80)).filter(Boolean))]
|
|
277
|
+
if (dependsOn.length > 0) entry.dependsOn = dependsOn
|
|
278
|
+
}
|
|
279
|
+
subtasks.push(entry)
|
|
132
280
|
}
|
|
133
281
|
if (subtasks.length > 0) return subtasks
|
|
134
282
|
}
|
|
135
283
|
return []
|
|
136
284
|
}
|
|
137
285
|
|
|
286
|
+
/** Collapse arbitrary text into at most `maxChars` characters. */
|
|
287
|
+
function capChars(value, maxChars) {
|
|
288
|
+
return [...String(value ?? '').trim()].slice(0, maxChars).join('')
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Where an instruction written against this run will actually land.
|
|
293
|
+
*
|
|
294
|
+
* Exported so the tool reply and `superagent_status` state the same boundary:
|
|
295
|
+
* a child that is already running cannot be reached, and claiming otherwise
|
|
296
|
+
* would be the one lie this feature must not tell.
|
|
297
|
+
*
|
|
298
|
+
* @param {object} run - run record, live or persisted.
|
|
299
|
+
* @returns {{ boundary: string, detail: string }} next boundary and how to read it.
|
|
300
|
+
*/
|
|
301
|
+
export function followupEffect(run) {
|
|
302
|
+
const subtasks = Array.isArray(run?.subtasks) ? run.subtasks : []
|
|
303
|
+
const open = subtasks.filter(subtask => subtask.status !== 'completed')
|
|
304
|
+
// `planning` covers two windows: before the planner is spawned, and while it
|
|
305
|
+
// runs. Only the first can still change the planner prompt; the spawn is
|
|
306
|
+
// recorded on `steps.planner`, so this does not promise a prompt that exists.
|
|
307
|
+
if (run?.status === 'planning' && subtasks.length === 0 && run.steps?.planner) {
|
|
308
|
+
return {
|
|
309
|
+
boundary: 'next-subtask-or-aggregator',
|
|
310
|
+
detail: 'the planner prompt is already built, so this reaches the first worker prompt or the aggregation, '
|
|
311
|
+
+ 'whichever comes first',
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
if (run?.status === 'queued' || run?.status === 'planning') {
|
|
315
|
+
if (subtasks.length === 0) {
|
|
316
|
+
return { boundary: 'planner', detail: 'the planner prompt, before any subtask starts' }
|
|
317
|
+
}
|
|
318
|
+
if (open.length === 0) {
|
|
319
|
+
return { boundary: 'aggregator', detail: 'the aggregation of the already completed subtasks' }
|
|
320
|
+
}
|
|
321
|
+
return {
|
|
322
|
+
boundary: 'next-subtask-or-aggregator',
|
|
323
|
+
detail: 'the next worker spawn or the aggregation, whichever comes first',
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
if (run?.status === 'running') {
|
|
327
|
+
return {
|
|
328
|
+
boundary: 'next-subtask-or-aggregator',
|
|
329
|
+
detail: 'the next worker spawn or the aggregation, whichever comes first; subtasks already running or '
|
|
330
|
+
+ 'finished keep their results and are not interrupted',
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
if (run?.status === 'aggregating') {
|
|
334
|
+
return {
|
|
335
|
+
boundary: 'next-round',
|
|
336
|
+
detail: 'the aggregation already running is not interrupted, so this applies to the next round or resume, '
|
|
337
|
+
+ 'not to the report being written now',
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
// A paused run has a real next round: resume resets its pending subtasks and
|
|
341
|
+
// builds fresh prompts, so an instruction queued now is not lost.
|
|
342
|
+
if (['interrupted', 'failed', 'partial', 'cancelled'].includes(run?.status)) {
|
|
343
|
+
return {
|
|
344
|
+
boundary: 'next-resume',
|
|
345
|
+
detail: 'the next round, which begins when you call superagent_resume',
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return { boundary: 'none', detail: 'the run is not active, so nothing will pick this up' }
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* The instructions a prompt built now can carry. The list is bounded at both
|
|
353
|
+
* ends: the newest eight instructions, and a cap on each text, so a long dialog
|
|
354
|
+
* cannot crowd out the goal itself.
|
|
355
|
+
*
|
|
356
|
+
* @param {object} run - run record.
|
|
357
|
+
* @returns {object[]} entries, oldest first.
|
|
358
|
+
*/
|
|
359
|
+
function carriedEntries(run) {
|
|
360
|
+
return Array.isArray(run?.followups) ? run.followups.slice(-FOLLOWUP_MAX) : []
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* True when an answer was recorded but no prompt has carried it yet. Such a
|
|
365
|
+
* run is not really finished even if its status says `completed`.
|
|
366
|
+
*
|
|
367
|
+
* @param {object} run - run record.
|
|
368
|
+
* @returns {boolean} whether a resume must reopen it.
|
|
369
|
+
*/
|
|
370
|
+
function hasUnappliedAnswer(run) {
|
|
371
|
+
return (Array.isArray(run?.followups) ? run.followups : [])
|
|
372
|
+
.some(entry => entry.kind === 'answer' && !entry.appliedAt)
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Render exactly the entries a prompt will carry. Taking the entries rather
|
|
377
|
+
* than the run keeps the "applied" mark and the prompt text in agreement: an
|
|
378
|
+
* instruction that arrives during the spawn is not claimed to be in it.
|
|
379
|
+
*
|
|
380
|
+
* @param {object[]} entries - from `carriedEntries`.
|
|
381
|
+
* @returns {string[]} prompt lines, empty when there is nothing to carry.
|
|
382
|
+
*/
|
|
383
|
+
function instructionLines(entries) {
|
|
384
|
+
if (entries.length === 0) return []
|
|
385
|
+
return [
|
|
386
|
+
'Instructions the user added while this run was in progress. They take precedence over the wording above:',
|
|
387
|
+
...entries.map(entry => `- ${entry.text}`),
|
|
388
|
+
'',
|
|
389
|
+
]
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Resolve each subtask's `dependsOn` references and report dependency cycles.
|
|
394
|
+
*
|
|
395
|
+
* A reference may name another subtask by id or by title, because a planner
|
|
396
|
+
* writes one or the other; anything that names nothing is reported instead of
|
|
397
|
+
* being dropped, so the run can say why a subtask never started.
|
|
398
|
+
*
|
|
399
|
+
* @param {Array<{ key: string, title: string, dependsOn?: string[] }>} subtasks - plan entries with ids assigned.
|
|
400
|
+
* @returns {{ edges: Map<string, string[]>, unknown: Map<string, string[]>, cycle: string[] | null }}
|
|
401
|
+
* resolved edges per task, unresolved references per task, and one cycle
|
|
402
|
+
* (as keys, first repeated at the end) when the graph has one.
|
|
403
|
+
*/
|
|
404
|
+
export function linkSubtasks(subtasks) {
|
|
405
|
+
const byKey = new Map()
|
|
406
|
+
const byTitle = new Map()
|
|
407
|
+
for (const subtask of subtasks) {
|
|
408
|
+
byKey.set(subtask.key, subtask)
|
|
409
|
+
const title = String(subtask.title ?? '').trim().toLowerCase()
|
|
410
|
+
if (title && !byTitle.has(title)) byTitle.set(title, subtask.key)
|
|
411
|
+
}
|
|
412
|
+
const edges = new Map()
|
|
413
|
+
const unknown = new Map()
|
|
414
|
+
for (const subtask of subtasks) {
|
|
415
|
+
const resolved = []
|
|
416
|
+
const missing = []
|
|
417
|
+
for (const ref of subtask.dependsOn ?? []) {
|
|
418
|
+
const key = byKey.has(ref) ? ref : byTitle.get(String(ref).trim().toLowerCase())
|
|
419
|
+
if (!key) missing.push(ref)
|
|
420
|
+
else if (!resolved.includes(key)) resolved.push(key)
|
|
421
|
+
}
|
|
422
|
+
edges.set(subtask.key, resolved)
|
|
423
|
+
if (missing.length > 0) unknown.set(subtask.key, missing)
|
|
424
|
+
}
|
|
425
|
+
return { edges, unknown, cycle: findCycle([...byKey.keys()], edges) }
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Depth-first search for one dependency cycle, kept small because a plan has at
|
|
430
|
+
* most a handful of subtasks and the message must name the tasks involved.
|
|
431
|
+
* @param {string[]} keys - every subtask key.
|
|
432
|
+
* @param {Map<string, string[]>} edges - resolved edges.
|
|
433
|
+
* @returns {string[] | null} cycle as `w1 -> w2 -> w1`, or null.
|
|
434
|
+
*/
|
|
435
|
+
function findCycle(keys, edges) {
|
|
436
|
+
const state = new Map()
|
|
437
|
+
const stack = []
|
|
438
|
+
let cycle = null
|
|
439
|
+
const visit = key => {
|
|
440
|
+
state.set(key, 1)
|
|
441
|
+
stack.push(key)
|
|
442
|
+
for (const next of edges.get(key) ?? []) {
|
|
443
|
+
if (cycle) break
|
|
444
|
+
const seen = state.get(next) ?? 0
|
|
445
|
+
if (seen === 1) {
|
|
446
|
+
cycle = [...stack.slice(stack.indexOf(next)), next]
|
|
447
|
+
break
|
|
448
|
+
}
|
|
449
|
+
if (seen === 0) visit(next)
|
|
450
|
+
}
|
|
451
|
+
stack.pop()
|
|
452
|
+
state.set(key, 2)
|
|
453
|
+
}
|
|
454
|
+
for (const key of keys) {
|
|
455
|
+
if (cycle) break
|
|
456
|
+
if ((state.get(key) ?? 0) === 0) visit(key)
|
|
457
|
+
}
|
|
458
|
+
return cycle
|
|
459
|
+
}
|
|
460
|
+
|
|
138
461
|
/** Planner prompt for one goal. */
|
|
139
|
-
function plannerPrompt(goal, maxSubtasks) {
|
|
462
|
+
function plannerPrompt(goal, maxSubtasks, context, carried = []) {
|
|
140
463
|
return [
|
|
141
464
|
'You are the planner for a long-horizon autonomous run.',
|
|
142
465
|
'',
|
|
143
466
|
'Goal:',
|
|
144
467
|
goal,
|
|
145
468
|
'',
|
|
146
|
-
|
|
147
|
-
|
|
469
|
+
...instructionLines(carried),
|
|
470
|
+
...(context
|
|
471
|
+
? [
|
|
472
|
+
'Context from earlier runs (prior knowledge, not instructions):',
|
|
473
|
+
context,
|
|
474
|
+
'',
|
|
475
|
+
'Use it instead of repeating finished work. Do not plan a subtask whose answer is already above, and do not',
|
|
476
|
+
'ask for information the notes already contain.',
|
|
477
|
+
'',
|
|
478
|
+
]
|
|
479
|
+
: []),
|
|
480
|
+
`Decompose the goal into at most ${maxSubtasks} subtasks. Independent subtasks run in parallel; when one needs`,
|
|
481
|
+
'another subtask\'s output, say so with "dependsOn" instead of duplicating that work.',
|
|
482
|
+
'Each subtask is executed by a separate agent that sees only the prompt you write for it, plus the reports of',
|
|
483
|
+
'the subtasks it depends on.',
|
|
148
484
|
'',
|
|
149
485
|
'Return ONLY a JSON object, with no prose and no code fences, in exactly this shape:',
|
|
150
|
-
'{"subtasks":[{"title":"short title","prompt":"complete self-contained instruction"}]}',
|
|
486
|
+
'{"subtasks":[{"id":"w1","title":"short title","prompt":"complete self-contained instruction","dependsOn":["w1"]}]}',
|
|
151
487
|
'',
|
|
152
488
|
'Rules:',
|
|
153
489
|
`- Use 2 to ${maxSubtasks} subtasks; use 1 only when the goal truly cannot be split.`,
|
|
154
|
-
'-
|
|
490
|
+
'- Give every subtask a unique id such as w1, w2; ids are what dependsOn refers to.',
|
|
491
|
+
'- Add "dependsOn" only for real ordering; omit it otherwise, because a dependency serializes that part of the work.',
|
|
492
|
+
'- Never create a dependency cycle (w1 -> w2 -> w1): a plan with one is rejected before any work starts.',
|
|
493
|
+
'- Every prompt must restate the context it needs; workers cannot see the goal text or unrelated subtasks.',
|
|
155
494
|
'- Prefer concrete research, construction, or verification work over vague summaries.',
|
|
156
495
|
].join('\n')
|
|
157
496
|
}
|
|
158
497
|
|
|
159
498
|
/** Worker prompt for one subtask. */
|
|
160
|
-
function workerPrompt(run, subtask) {
|
|
499
|
+
function workerPrompt(run, subtask, workspace = null, carried = []) {
|
|
500
|
+
const upstream = (subtask.dependsOn ?? [])
|
|
501
|
+
.map(key => (run.subtasks ?? []).find(entry => entry.key === key))
|
|
502
|
+
.filter(entry => entry && entry.status === 'completed' && entry.result)
|
|
161
503
|
return [
|
|
162
504
|
'You are one worker in a long-horizon autonomous run. Do not ask the user questions and do not wait for input.',
|
|
163
505
|
'',
|
|
@@ -167,35 +509,153 @@ function workerPrompt(run, subtask) {
|
|
|
167
509
|
`Your assigned subtask (${subtask.title}):`,
|
|
168
510
|
subtask.prompt,
|
|
169
511
|
'',
|
|
512
|
+
...(workspace
|
|
513
|
+
? [
|
|
514
|
+
'Isolated working directory:',
|
|
515
|
+
`- Your working directory is ${workspace.dir}`,
|
|
516
|
+
`- It is a private git worktree on branch ${workspace.branch} of ${workspace.base}. Other workers are`,
|
|
517
|
+
' editing the same repository at the same time in their own worktrees, so read and write files ONLY',
|
|
518
|
+
' inside your directory. Do not edit the main checkout or another worker\'s path.',
|
|
519
|
+
'- Do not create or remove worktrees, do not switch or delete branches, do not commit, merge, rebase, or',
|
|
520
|
+
' push. The run merges your branch back when you finish; leaving the files changed is enough.',
|
|
521
|
+
'',
|
|
522
|
+
]
|
|
523
|
+
: []),
|
|
524
|
+
...instructionLines(carried),
|
|
525
|
+
...(upstream.length > 0
|
|
526
|
+
? [
|
|
527
|
+
'Reports from the subtasks this one depends on (already finished; build on them instead of repeating them):',
|
|
528
|
+
...upstream.map(entry => [`### ${entry.title}`, capText(entry.result, 4000)].join('\n')),
|
|
529
|
+
'',
|
|
530
|
+
]
|
|
531
|
+
: []),
|
|
532
|
+
...(run.recovery
|
|
533
|
+
? [
|
|
534
|
+
'Recovery note: this run is being resumed after an earlier attempt stopped.',
|
|
535
|
+
`- the earlier attempt ended as: ${run.recovery.previousStatus}`,
|
|
536
|
+
...(run.recovery.previousError ? [`- reported reason: ${run.recovery.previousError}`] : []),
|
|
537
|
+
...(run.recovery.failed?.length
|
|
538
|
+
? [`- subtasks that had failed: ${run.recovery.failed.join('; ')}`]
|
|
539
|
+
: []),
|
|
540
|
+
'',
|
|
541
|
+
'Before repeating any externally visible action (writing files, committing, pushing, calling a remote API),',
|
|
542
|
+
'check what the earlier attempt already did: read the files, `git status` and `git log`, and the remote state.',
|
|
543
|
+
'Do not assume nothing happened, and do not redo work that is already complete.',
|
|
544
|
+
'',
|
|
545
|
+
]
|
|
546
|
+
: []),
|
|
170
547
|
'Constraints:',
|
|
171
548
|
'- Work only on your subtask; do not attempt the whole goal.',
|
|
172
549
|
'- Use the tools you have. Never call a tool whose name starts with "superagent_".',
|
|
173
550
|
'- Your final message IS the deliverable. Do not save the report to a file and do not reply with a',
|
|
174
551
|
' summary of what you did; write the findings themselves: concrete facts, numbers, file paths,',
|
|
175
552
|
' commands, and anything unresolved. Under 1500 words, Markdown only.',
|
|
553
|
+
'- If, and only if, finishing really needs a decision that only the user can make, append exactly one',
|
|
554
|
+
' fenced block like this instead of asking in prose; still deliver everything you could do without it:',
|
|
555
|
+
' ```attention',
|
|
556
|
+
' {"question":"one concrete question","options":["choice a","choice b"],"reason":"why you cannot decide"}',
|
|
557
|
+
' ```',
|
|
558
|
+
].join('\n')
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/** Prompt for a direct run: one worker, the whole goal, no planner or merge. */
|
|
562
|
+
function directPrompt(run, carried = []) {
|
|
563
|
+
return [
|
|
564
|
+
'You are completing one task directly in an autonomous run. Do not ask the user questions and do not wait for input.',
|
|
565
|
+
'',
|
|
566
|
+
'Task:',
|
|
567
|
+
run.goal,
|
|
568
|
+
'',
|
|
569
|
+
...instructionLines(carried),
|
|
570
|
+
...(run.recovery
|
|
571
|
+
? [
|
|
572
|
+
'Recovery note: this run is being resumed after an earlier attempt stopped.',
|
|
573
|
+
`- the earlier attempt ended as: ${run.recovery.previousStatus}`,
|
|
574
|
+
...(run.recovery.previousError ? [`- reported reason: ${run.recovery.previousError}`] : []),
|
|
575
|
+
'',
|
|
576
|
+
'Before repeating any externally visible action (writing files, committing, pushing, calling a remote API),',
|
|
577
|
+
'check what the earlier attempt already did: read the files, `git status` and `git log`, and the remote state.',
|
|
578
|
+
'Do not assume nothing happened, and do not redo work that is already complete.',
|
|
579
|
+
'',
|
|
580
|
+
]
|
|
581
|
+
: []),
|
|
582
|
+
'Constraints:',
|
|
583
|
+
'- Do the whole task in this one pass; nobody will merge, extend, or second-check your reply.',
|
|
584
|
+
'- Use the tools you have. Never call a tool whose name starts with "superagent_".',
|
|
585
|
+
'- Your final message IS the deliverable. Do not save the report to a file and do not reply with a',
|
|
586
|
+
' summary of what you did; write the findings themselves: concrete facts, numbers, file paths,',
|
|
587
|
+
' commands, and anything unresolved. Under 1500 words, Markdown only.',
|
|
588
|
+
'- If, and only if, finishing really needs a decision that only the user can make, append exactly one',
|
|
589
|
+
' fenced block like this instead of asking in prose; still deliver everything you could do without it:',
|
|
590
|
+
' ```attention',
|
|
591
|
+
' {"question":"one concrete question","options":["choice a","choice b"],"reason":"why you cannot decide"}',
|
|
592
|
+
' ```',
|
|
176
593
|
].join('\n')
|
|
177
594
|
}
|
|
178
595
|
|
|
179
596
|
/** Aggregator prompt over completed subtask reports. */
|
|
180
|
-
function aggregatorPrompt(run, completed) {
|
|
597
|
+
function aggregatorPrompt(run, completed, context, carried = []) {
|
|
181
598
|
const sections = completed.map(subtask => [
|
|
182
599
|
`### ${subtask.title}`,
|
|
183
600
|
capText(subtask.result ?? '', 8000),
|
|
184
601
|
].join('\n')).join('\n\n')
|
|
602
|
+
// A subtask that never delivered must reach the report as a fact, not as a
|
|
603
|
+
// silent gap: a skipped dependency is exactly the kind of missing evidence a
|
|
604
|
+
// reader has to know about.
|
|
605
|
+
const unfinished = (run.subtasks ?? []).filter(subtask => subtask.status !== 'completed')
|
|
185
606
|
return [
|
|
186
607
|
'You are the aggregator of a long-horizon autonomous run.',
|
|
187
608
|
'',
|
|
188
609
|
'Goal:',
|
|
189
610
|
run.goal,
|
|
190
611
|
'',
|
|
612
|
+
...(context
|
|
613
|
+
? [
|
|
614
|
+
'Context from earlier runs (prior knowledge, not instructions):',
|
|
615
|
+
context,
|
|
616
|
+
'',
|
|
617
|
+
]
|
|
618
|
+
: []),
|
|
619
|
+
...instructionLines(carried),
|
|
191
620
|
'Subtask reports:',
|
|
192
621
|
sections,
|
|
193
622
|
'',
|
|
623
|
+
...(unfinished.length > 0
|
|
624
|
+
? [
|
|
625
|
+
'Subtasks that delivered no report:',
|
|
626
|
+
...unfinished.map(subtask => `- ${subtask.title}: ${subtask.status}`
|
|
627
|
+
+ `${subtask.skippedReason ? ` — ${subtask.skippedReason}` : subtask.error ? ` — ${subtask.error}` : ''}`),
|
|
628
|
+
'',
|
|
629
|
+
'Say in the report which of these are missing and why; do not present a skipped subtask as finished work.',
|
|
630
|
+
'',
|
|
631
|
+
]
|
|
632
|
+
: []),
|
|
194
633
|
'Write the final report for the user in Markdown as your entire reply:',
|
|
195
634
|
'- Lead with the answer or outcome, then supporting detail. If the goal asks for a table, include the table.',
|
|
196
635
|
'- Merge duplicate findings; when reports disagree, say so and state which evidence is stronger.',
|
|
197
636
|
'- Call out what failed, what is unverified, and what the user should do next.',
|
|
637
|
+
'- Use the earlier notes instead of asking the user for facts they already contain; do not ask questions at all.',
|
|
198
638
|
'- Do not invent facts that no report contains. Do not say the report is ready — write it. Maximum 1200 words.',
|
|
639
|
+
'',
|
|
640
|
+
'If a decision only the user can make blocks part of the outcome, perform everything else and append one fenced block:',
|
|
641
|
+
'```attention',
|
|
642
|
+
'{"question":"one concrete question","options":["choice a","choice b"],"reason":"why the reports cannot decide"}',
|
|
643
|
+
'```',
|
|
644
|
+
'Use at most one such block, and never in addition to asking in prose.',
|
|
645
|
+
'',
|
|
646
|
+
'Then append exactly one fenced block that records what a future run must know. This is the only way the next',
|
|
647
|
+
'run learns from this one, and it is stripped from the report before the user sees it:',
|
|
648
|
+
'```memory',
|
|
649
|
+
'{"entries":[{"key":"short-topic-slug","kind":"decision|pitfall|next|dispatch","tags":["topic"],"text":"one durable fact"}]}',
|
|
650
|
+
'```',
|
|
651
|
+
'- At most 5 entries. Each text is one sentence under 300 characters. Tags are 0-3 lowercase labels that help',
|
|
652
|
+
' a later run find this note, so tag what it is about, not which run wrote it. No secrets, tokens, emails,',
|
|
653
|
+
' file paths with a home directory, customer data, or a copy of the report.',
|
|
654
|
+
'- Record only what stays true after this run: a decision and why, a trap that cost time, the next concrete step,',
|
|
655
|
+
' or which work was delegated to whom.',
|
|
656
|
+
'- If a note above is now wrong, emit an entry with the same key and corrected text. To drop one entirely, use',
|
|
657
|
+
' {"key":"the-key","status":"retracted"}.',
|
|
658
|
+
'- If nothing is worth keeping, emit {"entries":[]}.',
|
|
199
659
|
].join('\n')
|
|
200
660
|
}
|
|
201
661
|
|
|
@@ -207,20 +667,48 @@ function deterministicJoin(run) {
|
|
|
207
667
|
for (const subtask of completed) {
|
|
208
668
|
parts.push(`## ${subtask.title}`, '', subtask.result ?? '', '')
|
|
209
669
|
}
|
|
670
|
+
const unfinished = (run.subtasks ?? []).filter(subtask => subtask.status !== 'completed')
|
|
671
|
+
if (unfinished.length > 0) {
|
|
672
|
+
parts.push('## Not delivered', '')
|
|
673
|
+
for (const subtask of unfinished) {
|
|
674
|
+
parts.push(`- ${subtask.title}: ${subtask.status}`
|
|
675
|
+
+ `${subtask.skippedReason ? ` — ${subtask.skippedReason}` : subtask.error ? ` — ${subtask.error}` : ''}`)
|
|
676
|
+
}
|
|
677
|
+
}
|
|
210
678
|
return parts.join('\n').trim()
|
|
211
679
|
}
|
|
212
680
|
|
|
213
|
-
/**
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
681
|
+
/**
|
|
682
|
+
* Mark every subtask whose dependency failed or was skipped as skipped, in
|
|
683
|
+
* graph order, so one dead subtask cannot leave the tasks behind it queued
|
|
684
|
+
* forever and cannot be mistaken for success.
|
|
685
|
+
*
|
|
686
|
+
* @param {object} run - live run record.
|
|
687
|
+
* @param {Map<string, object>} byKey - subtask lookup by key.
|
|
688
|
+
* @returns {boolean} whether any subtask changed.
|
|
689
|
+
*/
|
|
690
|
+
function cascadeSkips(run, byKey) {
|
|
691
|
+
let changed = false
|
|
692
|
+
for (let pass = 0; pass <= run.subtasks.length; pass++) {
|
|
693
|
+
let round = false
|
|
694
|
+
for (const subtask of run.subtasks) {
|
|
695
|
+
if (subtask.status !== 'pending' && subtask.status !== 'waiting') continue
|
|
696
|
+
const blocker = (subtask.dependsOn ?? [])
|
|
697
|
+
.map(key => byKey.get(key))
|
|
698
|
+
.find(dependency => dependency && (dependency.status === 'failed' || dependency.status === 'skipped'))
|
|
699
|
+
if (!blocker) continue
|
|
700
|
+
subtask.status = 'skipped'
|
|
701
|
+
subtask.skipKind = 'upstream'
|
|
702
|
+
subtask.skippedReason = blocker.status === 'failed'
|
|
703
|
+
? `not started because ${blocker.key} failed`
|
|
704
|
+
: `not started because ${blocker.key} was skipped`
|
|
705
|
+
subtask.waitingOn = []
|
|
706
|
+
round = true
|
|
707
|
+
changed = true
|
|
708
|
+
}
|
|
709
|
+
if (!round) break
|
|
710
|
+
}
|
|
711
|
+
return changed
|
|
224
712
|
}
|
|
225
713
|
|
|
226
714
|
export function sumUsage(entries) {
|
|
@@ -248,47 +736,10 @@ function usageTokens(usage) {
|
|
|
248
736
|
return (usage?.inputTokens ?? 0) + (usage?.cacheReadTokens ?? 0) + (usage?.outputTokens ?? 0)
|
|
249
737
|
}
|
|
250
738
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
* table does not list a separate rate, which makes the estimate an upper bound
|
|
256
|
-
* instead of silently under-reporting. Models missing from the table are
|
|
257
|
-
* listed in `unpriced` and excluded from the total.
|
|
258
|
-
*
|
|
259
|
-
* @param {object | null} usage - summed usage for one run.
|
|
260
|
-
* @param {Record<string, { input: number, output: number, cacheRead?: number, currency?: string }> | undefined} pricing
|
|
261
|
-
* @returns {{ total: number, currency: string, byModel: Record<string, number>, unpriced: string[], complete: boolean } | null}
|
|
262
|
-
*/
|
|
263
|
-
export function computeCost(usage, pricing) {
|
|
264
|
-
if (!usage || !usage.modelCalls) return null
|
|
265
|
-
const table = pricing && typeof pricing === 'object' ? pricing : {}
|
|
266
|
-
const byModel = {}
|
|
267
|
-
const unpriced = []
|
|
268
|
-
let total = 0
|
|
269
|
-
let currency = null
|
|
270
|
-
for (const [model, entry] of Object.entries(usage.models ?? {})) {
|
|
271
|
-
// The usage log names a route, e.g. `dashscope/qwen-flash`. Accept a bare
|
|
272
|
-
// model id too, so a one-route config does not have to guess the prefix;
|
|
273
|
-
// an exact `provider/model` key always wins.
|
|
274
|
-
const price = table[model] ?? table[model.slice(model.indexOf('/') + 1)]
|
|
275
|
-
if (!price || typeof price.input !== 'number' || typeof price.output !== 'number') {
|
|
276
|
-
unpriced.push(model)
|
|
277
|
-
continue
|
|
278
|
-
}
|
|
279
|
-
const cacheRead = typeof price.cacheRead === 'number' ? price.cacheRead : price.input
|
|
280
|
-
const cost = (
|
|
281
|
-
entry.inputTokens * price.input
|
|
282
|
-
+ (entry.cacheReadTokens ?? 0) * cacheRead
|
|
283
|
-
+ entry.outputTokens * price.output
|
|
284
|
-
) / 1e6
|
|
285
|
-
byModel[model] = cost
|
|
286
|
-
total += cost
|
|
287
|
-
currency = currency ?? price.currency ?? 'USD'
|
|
288
|
-
}
|
|
289
|
-
if (Object.keys(byModel).length === 0) return { total: 0, currency: currency ?? 'USD', byModel, unpriced, complete: false }
|
|
290
|
-
return { total, currency, byModel, unpriced, complete: unpriced.length === 0 }
|
|
291
|
-
}
|
|
739
|
+
// `computeCost` lives in `cost.js` so the money rules (integer micro-units,
|
|
740
|
+
// one rounding, unpriced never treated as zero) have a single owner. Re-exported
|
|
741
|
+
// here because existing callers import it from the orchestrator module.
|
|
742
|
+
export { computeCost }
|
|
292
743
|
|
|
293
744
|
/** Total tokens recorded so far for a run, used by the budget guard. */
|
|
294
745
|
function spentTokens(run) {
|
|
@@ -322,9 +773,12 @@ function ensureCounters(run) {
|
|
|
322
773
|
* @param {(spec: { label: string, prompt: string, parent: object, signal: AbortSignal }) => Promise<{ childId: string, result: Promise<object>, dispose: () => Promise<void> }>} deps.spawnChild
|
|
323
774
|
* @param {(childId: string) => Promise<object | null>} deps.readUsage
|
|
324
775
|
* @param {(run: object) => object | void | Promise<object | void>} deps.notify - wake the owning session once.
|
|
776
|
+
* @param {object | null} [deps.memory] - cross-run notebook from `createMemory`; null disables it.
|
|
325
777
|
* @param {(message: string) => void} [deps.log]
|
|
326
778
|
* @param {() => number} [deps.now]
|
|
327
779
|
* @param {(ms: number) => Promise<void>} [deps.sleep]
|
|
780
|
+
* @param {(fn: () => void, ms: number) => unknown} [deps.setTimer] - queue-timeout timer, injectable for tests.
|
|
781
|
+
* @param {(timer: unknown) => void} [deps.clearTimer] - cancels that timer.
|
|
328
782
|
*/
|
|
329
783
|
export function createOrchestrator(deps) {
|
|
330
784
|
const {
|
|
@@ -333,9 +787,13 @@ export function createOrchestrator(deps) {
|
|
|
333
787
|
spawnChild,
|
|
334
788
|
readUsage,
|
|
335
789
|
notify,
|
|
790
|
+
memory = null,
|
|
791
|
+
workspace = null,
|
|
336
792
|
log = () => {},
|
|
337
793
|
now = () => Date.now(),
|
|
338
794
|
sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
795
|
+
setTimer = (fn, ms) => setTimeout(fn, ms),
|
|
796
|
+
clearTimer = timer => clearTimeout(timer),
|
|
339
797
|
} = deps
|
|
340
798
|
|
|
341
799
|
/** @type {Map<string, { controller: AbortController, activations: Set<object>, waiters: number, run: object, promise: Promise<void> | null, abortCause: string | null }>} */
|
|
@@ -360,6 +818,183 @@ export function createOrchestrator(deps) {
|
|
|
360
818
|
return clamp(config.maxActiveRuns ?? 2, 1, 8)
|
|
361
819
|
}
|
|
362
820
|
|
|
821
|
+
/** Cached manager; a test may inject one instead of spawning real git. */
|
|
822
|
+
let workspaceManagerRef = workspace
|
|
823
|
+
function workspaceManager() {
|
|
824
|
+
if (workspaceManagerRef) return workspaceManagerRef
|
|
825
|
+
const stateRoot = typeof store.root === 'string' ? store.root : path.join(os.tmpdir(), 'dsh-superagent')
|
|
826
|
+
const configured = typeof config.workspaceRoot === 'string' ? config.workspaceRoot.trim() : ''
|
|
827
|
+
workspaceManagerRef = createWorkspaceManager({
|
|
828
|
+
root: path.join(stateRoot, 'workspaces'),
|
|
829
|
+
baseDir: configured ? path.resolve(configured) : process.cwd(),
|
|
830
|
+
log,
|
|
831
|
+
sleep,
|
|
832
|
+
})
|
|
833
|
+
return workspaceManagerRef
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/**
|
|
837
|
+
* Resolve (and persist) whether this run can isolate its workers.
|
|
838
|
+
*
|
|
839
|
+
* Probed once per run and stored, so status can explain an unavailable
|
|
840
|
+
* environment without spawning anything, and a resume of the same run keeps
|
|
841
|
+
* the decision instead of silently switching modes mid-plan.
|
|
842
|
+
*
|
|
843
|
+
* @param {object} run - live run record.
|
|
844
|
+
* @returns {Promise<object | null>} usable workspace record, or null.
|
|
845
|
+
*/
|
|
846
|
+
async function ensureWorkspace(run) {
|
|
847
|
+
if (config.workspaceIsolation !== true) return null
|
|
848
|
+
if (run.workspace) return run.workspace.mode === 'git-worktree' ? run.workspace : null
|
|
849
|
+
let verdict
|
|
850
|
+
try {
|
|
851
|
+
verdict = await workspaceManager().probe()
|
|
852
|
+
} catch (error) {
|
|
853
|
+
verdict = { ok: false, base: null, reason: error?.message ?? String(error) }
|
|
854
|
+
}
|
|
855
|
+
if (!verdict.ok) {
|
|
856
|
+
// Requested but impossible is not a failure of the work: the run goes on
|
|
857
|
+
// in the shared directory, and both status and the report say so.
|
|
858
|
+
run.workspace = { mode: 'unavailable', base: verdict.base ?? null, reason: verdict.reason, merges: [] }
|
|
859
|
+
log(`run ${run.id}: workspace isolation unavailable — ${verdict.reason}`)
|
|
860
|
+
await save(run)
|
|
861
|
+
return null
|
|
862
|
+
}
|
|
863
|
+
run.workspace = {
|
|
864
|
+
mode: 'git-worktree',
|
|
865
|
+
base: verdict.base,
|
|
866
|
+
root: workspaceManager().root ?? null,
|
|
867
|
+
reason: null,
|
|
868
|
+
merges: [],
|
|
869
|
+
conflicts: 0,
|
|
870
|
+
}
|
|
871
|
+
await save(run)
|
|
872
|
+
return run.workspace
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
/**
|
|
876
|
+
* Give one worker its own checkout, or null when this run is not isolating.
|
|
877
|
+
* A worker that cannot get one runs in the shared directory like before; the
|
|
878
|
+
* attempt is recorded on the subtask so the failure is not silent.
|
|
879
|
+
*
|
|
880
|
+
* @param {object} run - live run record.
|
|
881
|
+
* @param {object} subtask - subtask entering `running`.
|
|
882
|
+
* @returns {Promise<{ dir: string, branch: string, base: string } | null>}
|
|
883
|
+
*/
|
|
884
|
+
async function openWorkspaceFor(run, subtask) {
|
|
885
|
+
if (run.workspace?.mode !== 'git-worktree') return null
|
|
886
|
+
const handle = await workspaceManager().prepare({ runId: run.id, index: subtask.index })
|
|
887
|
+
if (!handle?.ok) {
|
|
888
|
+
subtask.workspaceError = handle?.reason ?? 'workspace isolation failed for this subtask'
|
|
889
|
+
run.workspace.failures = Array.isArray(run.workspace.failures) ? run.workspace.failures : []
|
|
890
|
+
run.workspace.failures.push({ index: subtask.index, reason: subtask.workspaceError })
|
|
891
|
+
log(`subtask ${subtask.index}: ${subtask.workspaceError}; it runs in the shared directory`)
|
|
892
|
+
await save(run)
|
|
893
|
+
return null
|
|
894
|
+
}
|
|
895
|
+
subtask.workspace = {
|
|
896
|
+
dir: handle.dir,
|
|
897
|
+
branch: handle.branch,
|
|
898
|
+
base: handle.base,
|
|
899
|
+
reused: handle.reused,
|
|
900
|
+
status: 'running',
|
|
901
|
+
files: [],
|
|
902
|
+
commit: null,
|
|
903
|
+
detail: null,
|
|
904
|
+
}
|
|
905
|
+
await save(run)
|
|
906
|
+
return subtask.workspace
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/**
|
|
910
|
+
* Capture what a finished worker wrote, so its branch holds the work whether
|
|
911
|
+
* or not the model remembered to commit.
|
|
912
|
+
*
|
|
913
|
+
* @param {object} run - live run record.
|
|
914
|
+
* @param {object} subtask - subtask that just completed.
|
|
915
|
+
*/
|
|
916
|
+
async function harvestWorkspace(run, subtask) {
|
|
917
|
+
const handle = subtask.workspace
|
|
918
|
+
if (!handle || !handle.dir) return
|
|
919
|
+
const outcome = await workspaceManager().harvest({
|
|
920
|
+
dir: handle.dir,
|
|
921
|
+
branch: handle.branch,
|
|
922
|
+
index: subtask.index,
|
|
923
|
+
title: subtask.title,
|
|
924
|
+
})
|
|
925
|
+
handle.status = outcome.status
|
|
926
|
+
handle.files = outcome.files
|
|
927
|
+
handle.commit = outcome.commit
|
|
928
|
+
handle.detail = outcome.detail
|
|
929
|
+
if (outcome.status === 'no-changes') {
|
|
930
|
+
// An empty checkout is debris; drop it now instead of leaving it behind.
|
|
931
|
+
await workspaceManager().release({ dir: handle.dir, branch: handle.branch }).catch(() => {})
|
|
932
|
+
handle.dir = null
|
|
933
|
+
handle.branch = null
|
|
934
|
+
}
|
|
935
|
+
await save(run)
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/**
|
|
939
|
+
* Checkouts that no merge covers, so a kept worktree is never invisible.
|
|
940
|
+
*
|
|
941
|
+
* A worker that failed or was cancelled still leaves files behind; reporting
|
|
942
|
+
* them is the difference between "the run kept something for you" and a
|
|
943
|
+
* directory nobody knows about.
|
|
944
|
+
*
|
|
945
|
+
* @param {object} run - live run record.
|
|
946
|
+
* @returns {Array<object>} merge-shaped rows for `run.workspace.merges`.
|
|
947
|
+
*/
|
|
948
|
+
function leftoverWorkspaces(run) {
|
|
949
|
+
return (run.subtasks ?? [])
|
|
950
|
+
.filter(subtask => subtask.workspace?.dir && subtask.workspace.status !== 'committed')
|
|
951
|
+
.map(subtask => ({
|
|
952
|
+
index: subtask.index,
|
|
953
|
+
branch: subtask.workspace.branch,
|
|
954
|
+
dir: subtask.workspace.dir,
|
|
955
|
+
status: 'unmerged',
|
|
956
|
+
files: [],
|
|
957
|
+
detail: 'the worker did not finish, so its checkout was left untouched for a resume',
|
|
958
|
+
}))
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
/**
|
|
962
|
+
* Merge every worker branch back into the main checkout.
|
|
963
|
+
*
|
|
964
|
+
* A conflict aborts the merge and keeps both sides intact; the file list
|
|
965
|
+
* lands on the run and in the status text, which is how a conflict reaches
|
|
966
|
+
* the user instead of disappearing into a log line.
|
|
967
|
+
*
|
|
968
|
+
* @param {object} run - live run record.
|
|
969
|
+
*/
|
|
970
|
+
async function mergeWorkspaces(run) {
|
|
971
|
+
if (run.workspace?.mode !== 'git-worktree') return
|
|
972
|
+
const workers = (run.subtasks ?? [])
|
|
973
|
+
.filter(subtask => subtask.workspace?.dir && subtask.workspace.status === 'committed')
|
|
974
|
+
.map(subtask => ({
|
|
975
|
+
index: subtask.index,
|
|
976
|
+
title: subtask.title,
|
|
977
|
+
dir: subtask.workspace.dir,
|
|
978
|
+
branch: subtask.workspace.branch,
|
|
979
|
+
}))
|
|
980
|
+
if (workers.length === 0) {
|
|
981
|
+
run.workspace.merges = leftoverWorkspaces(run)
|
|
982
|
+
return
|
|
983
|
+
}
|
|
984
|
+
const outcomes = await workspaceManager().mergeAll(workers)
|
|
985
|
+
run.workspace.merges = [...outcomes, ...leftoverWorkspaces(run)]
|
|
986
|
+
run.workspace.conflicts = outcomes.filter(outcome => outcome.status === 'conflict').length
|
|
987
|
+
for (const outcome of outcomes) {
|
|
988
|
+
const subtask = (run.subtasks ?? []).find(entry => entry.index === outcome.index)
|
|
989
|
+
if (subtask?.workspace) {
|
|
990
|
+
subtask.workspace.merged = outcome.status
|
|
991
|
+
subtask.workspace.detail = outcome.detail
|
|
992
|
+
if (outcome.status === 'merged') subtask.workspace.dir = null
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
await save(run)
|
|
996
|
+
}
|
|
997
|
+
|
|
363
998
|
function save(run) {
|
|
364
999
|
run.updatedAt = new Date(now()).toISOString()
|
|
365
1000
|
const previous = saveChains.get(run.id) ?? Promise.resolve()
|
|
@@ -372,6 +1007,321 @@ export function createOrchestrator(deps) {
|
|
|
372
1007
|
return runtime.get(runId)?.run ?? null
|
|
373
1008
|
}
|
|
374
1009
|
|
|
1010
|
+
/** Runs waiting for a free slot, in admission order. */
|
|
1011
|
+
const queue = []
|
|
1012
|
+
let queueTicker = null
|
|
1013
|
+
let pumping = null
|
|
1014
|
+
|
|
1015
|
+
/** How many starts may wait at once; 0 turns queueing off entirely. */
|
|
1016
|
+
function maxQueuedRuns() {
|
|
1017
|
+
return clamp(config.maxQueuedRuns ?? 5, 0, 32)
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
function queueTimeoutMs() {
|
|
1021
|
+
return clamp(config.queueTimeoutMinutes ?? 30, 1, 720) * 60000
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
/**
|
|
1025
|
+
* Queueing happens when the caller asks for it (`queue: true`) or the profile
|
|
1026
|
+
* made it the default. The default stays "reject" so an existing script that
|
|
1027
|
+
* relied on the quota error keeps seeing it.
|
|
1028
|
+
*/
|
|
1029
|
+
function queueEnabled(requested) {
|
|
1030
|
+
return maxQueuedRuns() > 0 && (requested === true || config.queueOnFull === true)
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
function isWaiting(run) {
|
|
1034
|
+
return run?.queue?.waiting === true
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
/** Persist the queue positions so `status` and `list` show the real place in line. */
|
|
1038
|
+
async function refreshQueuePositions() {
|
|
1039
|
+
let changed = false
|
|
1040
|
+
for (let index = 0; index < queue.length; index++) {
|
|
1041
|
+
const run = queue[index].run
|
|
1042
|
+
run.queue = run.queue ?? {}
|
|
1043
|
+
if (run.queue.position === index + 1 && run.queue.of === queue.length) continue
|
|
1044
|
+
run.queue.position = index + 1
|
|
1045
|
+
run.queue.of = queue.length
|
|
1046
|
+
changed = true
|
|
1047
|
+
}
|
|
1048
|
+
if (changed) await Promise.all(queue.map(entry => save(entry.run).catch(() => {})))
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* A waiting run must look alive to another process sharing the state
|
|
1053
|
+
* directory; otherwise its heartbeat ages out and that process's sweep would
|
|
1054
|
+
* reclaim a run this one is still holding.
|
|
1055
|
+
*/
|
|
1056
|
+
function ensureQueueTicker() {
|
|
1057
|
+
if (queueTicker || queue.length === 0) return
|
|
1058
|
+
queueTicker = setInterval(() => {
|
|
1059
|
+
for (const entry of queue) {
|
|
1060
|
+
if (!isWaiting(entry.run)) continue
|
|
1061
|
+
entry.run.heartbeatAt = new Date(now()).toISOString()
|
|
1062
|
+
entry.run.ownerPid = process.pid
|
|
1063
|
+
save(entry.run).catch(() => {})
|
|
1064
|
+
}
|
|
1065
|
+
}, HEARTBEAT_MS)
|
|
1066
|
+
queueTicker.unref?.()
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
function stopQueueTicker() {
|
|
1070
|
+
if (!queueTicker) return
|
|
1071
|
+
clearInterval(queueTicker)
|
|
1072
|
+
queueTicker = null
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
/** Hold one run until a slot frees or its queue timeout expires. */
|
|
1076
|
+
function enqueue(run, parent) {
|
|
1077
|
+
const entry = { run, parent, timer: null }
|
|
1078
|
+
const remaining = Math.max(0, Date.parse(run.queue.timeoutAt) - now())
|
|
1079
|
+
entry.timer = setTimer(() => {
|
|
1080
|
+
expireQueued(entry).catch(error => log(`cannot expire queued run ${run.id}: ${error?.message ?? error}`))
|
|
1081
|
+
}, remaining)
|
|
1082
|
+
entry.timer?.unref?.()
|
|
1083
|
+
queue.push(entry)
|
|
1084
|
+
ensureQueueTicker()
|
|
1085
|
+
return entry
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
/**
|
|
1089
|
+
* Start queued runs while slots are free, oldest first. Serialized with
|
|
1090
|
+
* `start` and `resume` so two admissions cannot both take the last slot.
|
|
1091
|
+
*/
|
|
1092
|
+
function pumpQueue() {
|
|
1093
|
+
if (pumping) return pumping
|
|
1094
|
+
pumping = serialize(async () => {
|
|
1095
|
+
try {
|
|
1096
|
+
while (queue.length > 0) {
|
|
1097
|
+
const active = await activeCount()
|
|
1098
|
+
if (active >= maxActiveRuns()) break
|
|
1099
|
+
const entry = queue.shift()
|
|
1100
|
+
if (entry.timer) clearTimer(entry.timer)
|
|
1101
|
+
if (!isWaiting(entry.run)) continue
|
|
1102
|
+
await promote(entry)
|
|
1103
|
+
}
|
|
1104
|
+
} catch (error) {
|
|
1105
|
+
log(`cannot start queued runs: ${error?.message ?? error}`)
|
|
1106
|
+
} finally {
|
|
1107
|
+
pumping = null
|
|
1108
|
+
if (queue.length === 0) stopQueueTicker()
|
|
1109
|
+
await refreshQueuePositions().catch(() => {})
|
|
1110
|
+
}
|
|
1111
|
+
})
|
|
1112
|
+
return pumping
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
/**
|
|
1116
|
+
* Move one waiting run into execution. Its task clock starts now: time spent
|
|
1117
|
+
* in line must not eat the deadline the user chose for the work.
|
|
1118
|
+
*/
|
|
1119
|
+
async function promote(entry) {
|
|
1120
|
+
const run = entry.run
|
|
1121
|
+
run.queue = { ...(run.queue ?? {}), waiting: false, startedAt: new Date(now()).toISOString() }
|
|
1122
|
+
run.startedAt = new Date(now()).toISOString()
|
|
1123
|
+
run.deadlineAt = new Date(now() + run.limits.deadlineMinutes * 60000).toISOString()
|
|
1124
|
+
run.ownerPid = process.pid
|
|
1125
|
+
run.ownerHost = os.hostname()
|
|
1126
|
+
run.heartbeatAt = new Date(now()).toISOString()
|
|
1127
|
+
await takeRunLock(run.id)
|
|
1128
|
+
try {
|
|
1129
|
+
await save(run)
|
|
1130
|
+
} catch (error) {
|
|
1131
|
+
await releaseRunLock(run.id)
|
|
1132
|
+
throw error
|
|
1133
|
+
}
|
|
1134
|
+
schedule(run, entry.parent)
|
|
1135
|
+
log(`queued run ${run.id} got a free slot and started`)
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
/**
|
|
1139
|
+
* A run that never got a slot is failed, not cancelled: nobody chose to stop
|
|
1140
|
+
* it. It settles through the normal path so its status, usage, and
|
|
1141
|
+
* notification row look like every other finished run.
|
|
1142
|
+
*/
|
|
1143
|
+
async function expireQueued(entry) {
|
|
1144
|
+
const index = queue.indexOf(entry)
|
|
1145
|
+
if (index < 0) return
|
|
1146
|
+
queue.splice(index, 1)
|
|
1147
|
+
if (queue.length === 0) stopQueueTicker()
|
|
1148
|
+
const run = entry.run
|
|
1149
|
+
if (!isWaiting(run)) return
|
|
1150
|
+
run.queue = { ...run.queue, waiting: false, expired: true }
|
|
1151
|
+
run.error = `queued for ${queueTimeoutMs() / 60000} minute(s) without a free slot (maxActiveRuns is ${maxActiveRuns()}); `
|
|
1152
|
+
+ 'the run never started, so nothing was spent. Start it again once a slot is free, raise maxActiveRuns, '
|
|
1153
|
+
+ 'or raise queueTimeoutMinutes.'
|
|
1154
|
+
await settle(run, {
|
|
1155
|
+
controller: new AbortController(),
|
|
1156
|
+
activations: new Set(),
|
|
1157
|
+
waiters: 0,
|
|
1158
|
+
run,
|
|
1159
|
+
promise: null,
|
|
1160
|
+
abortCause: null,
|
|
1161
|
+
activeStart: now(),
|
|
1162
|
+
})
|
|
1163
|
+
await refreshQueuePositions().catch(() => {})
|
|
1164
|
+
await pumpQueue().catch(() => {})
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
/**
|
|
1168
|
+
* Record one phase transition on the run. Milestones are observational only:
|
|
1169
|
+
* they appear in `superagent_status` and never wake the session, which is what
|
|
1170
|
+
* keeps "progress is visible" from turning into "progress is noisy".
|
|
1171
|
+
*
|
|
1172
|
+
* @param {object} run - live run record.
|
|
1173
|
+
* @param {'plan_ready'|'running'|'aggregating'|'settled'} name - phase name.
|
|
1174
|
+
*/
|
|
1175
|
+
function addMilestone(run, name) {
|
|
1176
|
+
run.milestones = Array.isArray(run.milestones) ? run.milestones : []
|
|
1177
|
+
if (run.milestones.at(-1)?.name === name) return
|
|
1178
|
+
run.milestones.push({ name, at: new Date(now()).toISOString() })
|
|
1179
|
+
// A run can be resumed several times; keep the history bounded.
|
|
1180
|
+
if (run.milestones.length > 16) run.milestones = run.milestones.slice(-16)
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
/**
|
|
1184
|
+
* Keep the first thing that needs a human. Official super-agent has exactly
|
|
1185
|
+
* one Attention per report; a second worker asking something is recorded, but
|
|
1186
|
+
* the notification leads with the first question so the message stays short.
|
|
1187
|
+
*
|
|
1188
|
+
* @param {object} run - live run record.
|
|
1189
|
+
* @param {string} source - `subtask-<n>` or `aggregator`.
|
|
1190
|
+
* @param {{ question: string, options: string[], reason: string | null }} attention - parsed block.
|
|
1191
|
+
*/
|
|
1192
|
+
function recordAttention(run, source, attention) {
|
|
1193
|
+
if (!attention) return
|
|
1194
|
+
run.attention = run.attention ?? { at: new Date(now()).toISOString(), items: [] }
|
|
1195
|
+
if (!Array.isArray(run.attention.items)) run.attention.items = []
|
|
1196
|
+
// The id is what an answer is matched against. It comes from a counter on
|
|
1197
|
+
// the run, not from the array position, so archiving the question on resume
|
|
1198
|
+
// cannot make an old id point at a different question later.
|
|
1199
|
+
run.attentionSeq = (Number(run.attentionSeq) || 0) + 1
|
|
1200
|
+
run.attention.items.push({
|
|
1201
|
+
id: `q${run.attentionSeq}`,
|
|
1202
|
+
source,
|
|
1203
|
+
question: attention.question,
|
|
1204
|
+
options: attention.options ?? [],
|
|
1205
|
+
reason: attention.reason ?? null,
|
|
1206
|
+
})
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
/**
|
|
1210
|
+
* Remember that a block asked for a decision but could not be used. The raw
|
|
1211
|
+
* block stays in the report, and status prints this line so the user can see
|
|
1212
|
+
* that a decision was requested instead of silently getting a finished run.
|
|
1213
|
+
*
|
|
1214
|
+
* @param {object} run - live run record.
|
|
1215
|
+
* @param {string} source - `subtask-<n>` or `aggregator`.
|
|
1216
|
+
* @param {string | undefined} issue - why the block was unusable.
|
|
1217
|
+
*/
|
|
1218
|
+
function recordAttentionIssue(run, source, issue) {
|
|
1219
|
+
if (!issue) return
|
|
1220
|
+
run.attention = run.attention ?? { at: new Date(now()).toISOString(), items: [] }
|
|
1221
|
+
if (!Array.isArray(run.attention.items)) run.attention.items = []
|
|
1222
|
+
run.attention.issue = oneLine(`${issue} (from ${source})`, 300)
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
/**
|
|
1226
|
+
* Append one instruction to the run's dialog log.
|
|
1227
|
+
*
|
|
1228
|
+
* The log is the only durable record of what was asked after the run started
|
|
1229
|
+
* and when it first reached a prompt, so `superagent_status` and the tool
|
|
1230
|
+
* reply can both say plainly whether the running work ever saw it.
|
|
1231
|
+
*
|
|
1232
|
+
* @param {object} run - run record.
|
|
1233
|
+
* @param {{ kind: 'followup'|'answer', text: string, attentionId?: string | null }} spec - entry.
|
|
1234
|
+
* @returns {object} the stored entry.
|
|
1235
|
+
*/
|
|
1236
|
+
function pushFollowup(run, spec) {
|
|
1237
|
+
run.followups = Array.isArray(run.followups) ? run.followups : []
|
|
1238
|
+
const entry = {
|
|
1239
|
+
id: `f${(Number(run.followupSeq) || 0) + 1}`,
|
|
1240
|
+
kind: spec.kind,
|
|
1241
|
+
attentionId: spec.attentionId ?? null,
|
|
1242
|
+
text: capChars(spec.text, FOLLOWUP_MAX_CHARS),
|
|
1243
|
+
at: new Date(now()).toISOString(),
|
|
1244
|
+
appliedAt: null,
|
|
1245
|
+
appliedAtStage: null,
|
|
1246
|
+
missedAt: null,
|
|
1247
|
+
missedReason: null,
|
|
1248
|
+
}
|
|
1249
|
+
run.followupSeq = Number(run.followupSeq) || 0
|
|
1250
|
+
run.followupSeq++
|
|
1251
|
+
run.followups.push(entry)
|
|
1252
|
+
if (run.followups.length > FOLLOWUP_MAX) run.followups = run.followups.slice(-FOLLOWUP_MAX)
|
|
1253
|
+
return entry
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/**
|
|
1257
|
+
* Record the first prompt that carried the pending instructions. The text is
|
|
1258
|
+
* still included in later prompts — an instruction given at subtask 2 must
|
|
1259
|
+
* reach the aggregator too — so this is an audit trail, not a delivery queue.
|
|
1260
|
+
*
|
|
1261
|
+
* @param {object[]} entries - the entries the prompt was built from.
|
|
1262
|
+
* @param {string} stage - `planner`, `subtask-<n>`, or `aggregator`.
|
|
1263
|
+
* @returns {boolean} true when something changed and the run needs a save.
|
|
1264
|
+
*/
|
|
1265
|
+
function markFollowupsApplied(entries, stage) {
|
|
1266
|
+
const at = new Date(now()).toISOString()
|
|
1267
|
+
let changed = false
|
|
1268
|
+
for (const entry of entries) {
|
|
1269
|
+
if (entry.appliedAt) continue
|
|
1270
|
+
entry.appliedAt = at
|
|
1271
|
+
entry.appliedAtStage = stage
|
|
1272
|
+
changed = true
|
|
1273
|
+
}
|
|
1274
|
+
return changed
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
/**
|
|
1278
|
+
* Write the final report next to the run state, so the outcome has an address
|
|
1279
|
+
* the user can open or attach, not only a paragraph in a tool reply. The
|
|
1280
|
+
* directory carries its own `.gitignore` for the same reason the memory
|
|
1281
|
+
* directory does: a stateDir inside a repository must not leak run output.
|
|
1282
|
+
*
|
|
1283
|
+
* @param {object} run - terminal run with `report`.
|
|
1284
|
+
*/
|
|
1285
|
+
async function writeReportArtifact(run) {
|
|
1286
|
+
const root = typeof store.root === 'string' ? store.root : null
|
|
1287
|
+
if (!root || typeof run.report !== 'string' || !run.report.trim()) return
|
|
1288
|
+
const dir = path.join(root, 'reports')
|
|
1289
|
+
const file = path.join(dir, `${run.id}.md`)
|
|
1290
|
+
try {
|
|
1291
|
+
await fsp.mkdir(dir, { recursive: true })
|
|
1292
|
+
const ignore = path.join(dir, '.gitignore')
|
|
1293
|
+
try {
|
|
1294
|
+
await fsp.access(ignore)
|
|
1295
|
+
} catch {
|
|
1296
|
+
await fsp.writeFile(ignore, '*\n', 'utf8')
|
|
1297
|
+
}
|
|
1298
|
+
const body = `${run.report.trim()}\n`
|
|
1299
|
+
const tmp = path.join(dir, `.${run.id}.${process.pid}.tmp`)
|
|
1300
|
+
await fsp.writeFile(tmp, body, 'utf8')
|
|
1301
|
+
for (let attempt = 1; ; attempt++) {
|
|
1302
|
+
try {
|
|
1303
|
+
await fsp.rename(tmp, file)
|
|
1304
|
+
break
|
|
1305
|
+
} catch (error) {
|
|
1306
|
+
if (attempt >= 5 || !['EPERM', 'EACCES', 'EBUSY'].includes(error?.code)) throw error
|
|
1307
|
+
await sleep(20 * attempt)
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
run.artifacts = (Array.isArray(run.artifacts) ? run.artifacts : [])
|
|
1311
|
+
.filter(artifact => artifact?.path !== file)
|
|
1312
|
+
run.artifacts.push({
|
|
1313
|
+
kind: 'report',
|
|
1314
|
+
path: file,
|
|
1315
|
+
bytes: Buffer.byteLength(body, 'utf8'),
|
|
1316
|
+
at: new Date(now()).toISOString(),
|
|
1317
|
+
})
|
|
1318
|
+
run.artifactError = null
|
|
1319
|
+
} catch (error) {
|
|
1320
|
+
run.artifactError = `cannot write the report file: ${error?.message ?? error}`
|
|
1321
|
+
log(`run ${run.id}: ${run.artifactError}`)
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
|
|
375
1325
|
/** Cached machine identity of the state directory. */
|
|
376
1326
|
let ownerRef = null
|
|
377
1327
|
async function ownerIdentity() {
|
|
@@ -468,6 +1418,7 @@ export function createOrchestrator(deps) {
|
|
|
468
1418
|
? 'cancelled by superagent_cancel while no process was running it'
|
|
469
1419
|
: 'host process stopped while the run was active'
|
|
470
1420
|
run.finishedAt = new Date(now()).toISOString()
|
|
1421
|
+
if (run.queue?.waiting) run.queue = { ...run.queue, waiting: false, lost: true }
|
|
471
1422
|
run.reclaimed = {
|
|
472
1423
|
at: new Date(now()).toISOString(),
|
|
473
1424
|
basis: verdict.basis,
|
|
@@ -484,12 +1435,12 @@ export function createOrchestrator(deps) {
|
|
|
484
1435
|
// its child logs were readable.
|
|
485
1436
|
const entries = Object.values(run.usageByChild ?? {})
|
|
486
1437
|
if (entries.length > 0) run.usage = sumUsage(entries)
|
|
487
|
-
run.cost = computeCost(run.usage, config.pricing)
|
|
488
1438
|
for (const subtask of run.subtasks ?? []) {
|
|
489
1439
|
if (subtask.childId && run.usageByChild?.[subtask.childId]) {
|
|
490
1440
|
subtask.usage = run.usageByChild[subtask.childId]
|
|
491
1441
|
}
|
|
492
1442
|
}
|
|
1443
|
+
applyCost(run, config.pricing)
|
|
493
1444
|
}
|
|
494
1445
|
|
|
495
1446
|
async function releaseRunLock(runId) {
|
|
@@ -538,6 +1489,94 @@ export function createOrchestrator(deps) {
|
|
|
538
1489
|
}
|
|
539
1490
|
}
|
|
540
1491
|
|
|
1492
|
+
/** The lock id used for one goal; it fits the id pattern the store validates. */
|
|
1493
|
+
function goalGuardId(key) {
|
|
1494
|
+
return `${GOAL_GUARD_PREFIX}${key.slice(0, 40)}`
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
/**
|
|
1498
|
+
* The active run that already covers one goal, if any.
|
|
1499
|
+
*
|
|
1500
|
+
* The list on disk is the authority, because another host may own the run; a
|
|
1501
|
+
* run this process is finishing keeps its in-memory status until the terminal
|
|
1502
|
+
* write lands, so the runtime copy wins when both exist.
|
|
1503
|
+
*
|
|
1504
|
+
* @param {string} key - `dedupeKeyOf(goal)` of the goal being started.
|
|
1505
|
+
* @param {string} parentSessionId - session that would own the new run.
|
|
1506
|
+
* @returns {Promise<object | null>} the newest active run for that goal in this session.
|
|
1507
|
+
*/
|
|
1508
|
+
async function activeRunForGoal(key, parentSessionId) {
|
|
1509
|
+
const runs = new Map()
|
|
1510
|
+
for (const run of await store.list()) runs.set(run.id, run)
|
|
1511
|
+
for (const [runId, rt] of runtime) if (rt.run) runs.set(runId, rt.run)
|
|
1512
|
+
const hits = [...runs.values()]
|
|
1513
|
+
// A goal is only the same work inside the session that asked for it: two
|
|
1514
|
+
// sessions may legitimately run the same generic goal at the same time.
|
|
1515
|
+
.filter(run => run.parentSessionId === parentSessionId
|
|
1516
|
+
&& ACTIVE_STATUSES.has(run.status)
|
|
1517
|
+
&& (run.dedupeKey ?? dedupeKeyOf(run.goal)) === key)
|
|
1518
|
+
if (hits.length === 0) return null
|
|
1519
|
+
hits.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
|
|
1520
|
+
return hits[0]
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
/**
|
|
1524
|
+
* Run `task` while holding the claim on one goal.
|
|
1525
|
+
*
|
|
1526
|
+
* Two processes sharing a state directory would otherwise both list the runs,
|
|
1527
|
+
* both see nothing active, and both create a run for the same goal. The claim
|
|
1528
|
+
* is keyed by the goal rather than by a run id, which does not exist yet. The
|
|
1529
|
+
* winner writes the run record before it releases the claim, so the loser's
|
|
1530
|
+
* own check under the lock finds that record and joins it instead.
|
|
1531
|
+
*
|
|
1532
|
+
* A guard whose owner is gone is broken exactly like a run lock. A merely
|
|
1533
|
+
* slow owner is waited for, because a duplicate costs money; past the wait a
|
|
1534
|
+
* real start proceeds without the guard, since refusing it because a file was
|
|
1535
|
+
* left behind would be the worse failure.
|
|
1536
|
+
*
|
|
1537
|
+
* @template T
|
|
1538
|
+
* @param {string} key - `dedupeKeyOf(goal)`.
|
|
1539
|
+
* @param {() => Promise<T>} task - runs while the claim is held.
|
|
1540
|
+
* @returns {Promise<T>} the task's result.
|
|
1541
|
+
*/
|
|
1542
|
+
async function withGoalGuard(key, task) {
|
|
1543
|
+
if (typeof store.acquireLock !== 'function' || typeof store.releaseLock !== 'function') return task()
|
|
1544
|
+
const id = goalGuardId(key)
|
|
1545
|
+
const holder = { pid: process.pid, hostname: os.hostname(), at: new Date(now()).toISOString() }
|
|
1546
|
+
const endsAt = now() + GOAL_GUARD_WAIT_MS
|
|
1547
|
+
let locked = false
|
|
1548
|
+
while (!locked && now() < endsAt) {
|
|
1549
|
+
const attempt = await store.acquireLock(id, holder)
|
|
1550
|
+
if (attempt.acquired) {
|
|
1551
|
+
locked = true
|
|
1552
|
+
break
|
|
1553
|
+
}
|
|
1554
|
+
const current = attempt.holder
|
|
1555
|
+
const local = Boolean(current) && current.hostname === os.hostname()
|
|
1556
|
+
const age = now() - Date.parse(current?.at ?? '')
|
|
1557
|
+
const stale = !Number.isFinite(age) || age > LOCK_STALE_MS
|
|
1558
|
+
const aliveHere = local && pidAlive(current.pid) === true
|
|
1559
|
+
// A lock on this machine whose pid is gone is evidence of a crash; an
|
|
1560
|
+
// uncheckable lock is broken only once it is old.
|
|
1561
|
+
if ((local && pidAlive(current.pid) === false) || (stale && !aliveHere)) {
|
|
1562
|
+
await store.releaseLock(id)
|
|
1563
|
+
continue
|
|
1564
|
+
}
|
|
1565
|
+
await sleep(GOAL_GUARD_POLL_MS)
|
|
1566
|
+
}
|
|
1567
|
+
try {
|
|
1568
|
+
return await task()
|
|
1569
|
+
} finally {
|
|
1570
|
+
if (locked) {
|
|
1571
|
+
try {
|
|
1572
|
+
await store.releaseLock(id)
|
|
1573
|
+
} catch (error) {
|
|
1574
|
+
log(`cannot release the goal guard ${id}: ${error?.message ?? error}`)
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
|
|
541
1580
|
/**
|
|
542
1581
|
* Reclaim runs whose owning process is gone, so an orphan cannot hold the
|
|
543
1582
|
* active-run quota forever.
|
|
@@ -568,17 +1607,20 @@ export function createOrchestrator(deps) {
|
|
|
568
1607
|
}
|
|
569
1608
|
let count = 0
|
|
570
1609
|
for (const rt of runtime.values()) {
|
|
571
|
-
|
|
1610
|
+
// A run waiting for a slot is not spending anything yet, so it must not
|
|
1611
|
+
// consume the quota that decides when it may start.
|
|
1612
|
+
if (rt.run && ACTIVE_STATUSES.has(rt.run.status) && !isWaiting(rt.run)) count++
|
|
572
1613
|
}
|
|
573
1614
|
for (const run of await store.list()) {
|
|
574
1615
|
if (runtime.has(run.id)) continue
|
|
575
|
-
if (ACTIVE_STATUSES.has(run.status)) count++
|
|
1616
|
+
if (ACTIVE_STATUSES.has(run.status) && !isWaiting(run)) count++
|
|
576
1617
|
}
|
|
577
1618
|
return count
|
|
578
1619
|
}
|
|
579
1620
|
|
|
580
|
-
function quotaMessage(active) {
|
|
581
|
-
|
|
1621
|
+
function quotaMessage(active, queued = 0) {
|
|
1622
|
+
const waiting = queued > 0 ? ` ${queued} run(s) are already queued, so this start would wait behind them.` : ''
|
|
1623
|
+
return `quota exceeded: ${active} active run(s) and maxActiveRuns is ${maxActiveRuns()}.${waiting} `
|
|
582
1624
|
+ 'Let one finish with superagent_status or cancel it with superagent_cancel.'
|
|
583
1625
|
}
|
|
584
1626
|
|
|
@@ -601,6 +1643,119 @@ export function createOrchestrator(deps) {
|
|
|
601
1643
|
return limits
|
|
602
1644
|
}
|
|
603
1645
|
|
|
1646
|
+
/** Memory limits, clamped once so a bad config cannot remove the cap. */
|
|
1647
|
+
function memorySettings() {
|
|
1648
|
+
return {
|
|
1649
|
+
scope: typeof config.memoryScope === 'string' && config.memoryScope ? config.memoryScope : 'default',
|
|
1650
|
+
maxEntries: clamp(config.memoryMaxEntries ?? 60, 1, 500),
|
|
1651
|
+
maxAgeDays: clamp(config.memoryMaxAgeDays ?? 30, 1, 3650),
|
|
1652
|
+
digestEntries: clamp(config.memoryDigestEntries ?? 12, 1, 50),
|
|
1653
|
+
digestChars: clamp(config.memoryDigestChars ?? 1600, 200, 8000),
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
/**
|
|
1658
|
+
* Build the bounded context carried into planning and aggregation: exact run
|
|
1659
|
+
* history from the store plus the curated notes. Returns null when there is
|
|
1660
|
+
* nothing to carry, so a first run's prompts stay unchanged.
|
|
1661
|
+
*
|
|
1662
|
+
* Notes are layered: `rules` holds project rules, `profile` holds user
|
|
1663
|
+
* preferences, and `memoryScope` holds what earlier runs learned. The first
|
|
1664
|
+
* two are curated by hand (the `superagent_memory` tool), so they always come
|
|
1665
|
+
* before run notes: a rule must not be drowned out by recent observations.
|
|
1666
|
+
*
|
|
1667
|
+
* @param {string | null} currentId - run being started or resumed.
|
|
1668
|
+
* @param {string} goal - the run's goal, used to recall the notes that matter.
|
|
1669
|
+
*/
|
|
1670
|
+
async function memoryContext(currentId, goal = '') {
|
|
1671
|
+
if (!memory) return null
|
|
1672
|
+
const settings = memorySettings()
|
|
1673
|
+
const layers = []
|
|
1674
|
+
let noteCount = 0
|
|
1675
|
+
try {
|
|
1676
|
+
for (const [label, scope] of [
|
|
1677
|
+
['Project rules', 'rules'],
|
|
1678
|
+
['User profile', 'profile'],
|
|
1679
|
+
]) {
|
|
1680
|
+
const text = await memory.digest({
|
|
1681
|
+
scope,
|
|
1682
|
+
maxEntries: settings.digestEntries,
|
|
1683
|
+
maxChars: Math.floor(settings.digestChars / 3),
|
|
1684
|
+
maxAgeDays: settings.maxAgeDays,
|
|
1685
|
+
})
|
|
1686
|
+
if (!text) continue
|
|
1687
|
+
const read = await memory.read({ scope, maxAgeDays: settings.maxAgeDays })
|
|
1688
|
+
noteCount += read.entries.length
|
|
1689
|
+
layers.push(`${label} (scope "${scope}"):\n${text}`)
|
|
1690
|
+
}
|
|
1691
|
+
const notes = await memory.digest({
|
|
1692
|
+
scope: settings.scope,
|
|
1693
|
+
// Rank notes against this goal instead of only taking the newest ones,
|
|
1694
|
+
// so a run with many predecessors still finds the relevant experience.
|
|
1695
|
+
query: goal,
|
|
1696
|
+
maxEntries: settings.digestEntries,
|
|
1697
|
+
maxChars: settings.digestChars,
|
|
1698
|
+
maxAgeDays: settings.maxAgeDays,
|
|
1699
|
+
})
|
|
1700
|
+
if (notes) {
|
|
1701
|
+
const read = await memory.read({ scope: settings.scope, maxAgeDays: settings.maxAgeDays })
|
|
1702
|
+
noteCount += read.entries.length
|
|
1703
|
+
layers.push(notes)
|
|
1704
|
+
}
|
|
1705
|
+
} catch (error) {
|
|
1706
|
+
log(`cannot read cross-run memory: ${error?.message ?? error}`)
|
|
1707
|
+
}
|
|
1708
|
+
const notes = layers.join('\n\n')
|
|
1709
|
+
|
|
1710
|
+
let history = ''
|
|
1711
|
+
let runCount = 0
|
|
1712
|
+
try {
|
|
1713
|
+
const runs = (await store.list())
|
|
1714
|
+
.filter(run => run.id !== currentId)
|
|
1715
|
+
.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
|
|
1716
|
+
.slice(0, 6)
|
|
1717
|
+
runCount = runs.length
|
|
1718
|
+
history = runs.map(run => {
|
|
1719
|
+
const subtasks = run.subtasks ?? []
|
|
1720
|
+
const done = subtasks.filter(subtask => subtask.status === 'completed').length
|
|
1721
|
+
const goal = [...String(run.goal ?? '').replace(/\s+/g, ' ').trim()].slice(0, 90).join('')
|
|
1722
|
+
return `- ${String(run.createdAt ?? '').slice(0, 10)} ${run.status} ${done}/${subtasks.length} ${goal} (run ${run.id})`
|
|
1723
|
+
}).join('\n')
|
|
1724
|
+
} catch (error) {
|
|
1725
|
+
log(`cannot read earlier runs for the memory context: ${error?.message ?? error}`)
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
if (!notes && !history) return null
|
|
1729
|
+
const parts = []
|
|
1730
|
+
if (history) parts.push('Earlier runs in this state directory (newest first):', history)
|
|
1731
|
+
if (notes) {
|
|
1732
|
+
parts.push(
|
|
1733
|
+
'Notes left by earlier runs and the operator (may be stale; verify before relying on them; correct a wrong '
|
|
1734
|
+
+ 'one by writing a new entry with the same key):',
|
|
1735
|
+
notes,
|
|
1736
|
+
)
|
|
1737
|
+
}
|
|
1738
|
+
const digest = parts.join('\n\n').slice(0, 2600)
|
|
1739
|
+
return {
|
|
1740
|
+
digest,
|
|
1741
|
+
notes: noteCount,
|
|
1742
|
+
runs: runCount,
|
|
1743
|
+
digestChars: digest.length,
|
|
1744
|
+
scopes: ['rules', 'profile', settings.scope],
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
/** A failure that a retry on another attempt or route can plausibly fix. */
|
|
1749
|
+
function looksTransient(text) {
|
|
1750
|
+
return /quota|rate.?limit|\b429\b|overload|capacity|timed?\s?out|timeout|unavailable|ECONNRESET|ETIMEDOUT|\b50[0234]\b/i
|
|
1751
|
+
.test(String(text ?? ''))
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
/** Extra rounds that retry failed subtasks automatically; 0 disables. */
|
|
1755
|
+
function selfHealRounds() {
|
|
1756
|
+
return clamp(config.autoResumeRounds ?? 1, 0, 3)
|
|
1757
|
+
}
|
|
1758
|
+
|
|
604
1759
|
/**
|
|
605
1760
|
* Report why a run may not start another child, or null when it still can.
|
|
606
1761
|
* Both budgets are hard stops: they exist so a pathological goal cannot spend
|
|
@@ -822,11 +1977,15 @@ export function createOrchestrator(deps) {
|
|
|
822
1977
|
|
|
823
1978
|
async function makePlan(run, parent, rt) {
|
|
824
1979
|
try {
|
|
1980
|
+
const carried = carriedEntries(run)
|
|
825
1981
|
const handle = await spawn(run, rt, {
|
|
826
1982
|
label: 'superagent-plan',
|
|
827
|
-
prompt: plannerPrompt(run.goal, run.limits.maxSubtasks),
|
|
1983
|
+
prompt: plannerPrompt(run.goal, run.limits.maxSubtasks, run.memory?.digest ?? '', carried),
|
|
828
1984
|
parent,
|
|
829
1985
|
})
|
|
1986
|
+
// The planner prompt carried them, so record where they landed before
|
|
1987
|
+
// the run moves on; status must not claim they are still waiting.
|
|
1988
|
+
if (carried.length > 0 && markFollowupsApplied(carried, 'planner')) await save(run)
|
|
830
1989
|
rt.activations.add(handle)
|
|
831
1990
|
run.steps.planner = { childId: handle.childId }
|
|
832
1991
|
try {
|
|
@@ -847,35 +2006,58 @@ export function createOrchestrator(deps) {
|
|
|
847
2006
|
return { mode: 'degraded', subtasks: [{ title: 'Complete the goal', prompt: run.goal }] }
|
|
848
2007
|
}
|
|
849
2008
|
|
|
850
|
-
async function runSubtask(run, subtask, parent, rt) {
|
|
2009
|
+
async function runSubtask(run, subtask, parent, rt, { model = null } = {}) {
|
|
851
2010
|
const signal = rt.controller.signal
|
|
852
2011
|
for (let attempt = subtask.attempts + 1; attempt <= run.limits.attempts; attempt++) {
|
|
853
2012
|
if (signal.aborted) break
|
|
854
2013
|
subtask.status = 'running'
|
|
855
2014
|
subtask.attempts = attempt
|
|
856
2015
|
subtask.startedAt = subtask.startedAt ?? new Date(now()).toISOString()
|
|
2016
|
+
// An instruction that arrived after this subtask was queued reaches it
|
|
2017
|
+
// here, at the last moment before the prompt is built.
|
|
2018
|
+
const carried = carriedEntries(run)
|
|
2019
|
+
markFollowupsApplied(carried, `subtask-${subtask.index}`)
|
|
857
2020
|
await save(run)
|
|
858
2021
|
|
|
859
2022
|
let handle = null
|
|
2023
|
+
// A retry reuses the checkout from the previous attempt, so work the
|
|
2024
|
+
// earlier attempt already wrote is not thrown away.
|
|
2025
|
+
const workspace = await openWorkspaceFor(run, subtask)
|
|
860
2026
|
try {
|
|
861
2027
|
handle = await spawn(run, rt, {
|
|
862
|
-
label: `superagent-${subtask.index}`,
|
|
863
|
-
prompt:
|
|
2028
|
+
label: run.mode === 'direct' ? 'superagent-direct' : `superagent-${subtask.index}`,
|
|
2029
|
+
prompt: run.mode === 'direct'
|
|
2030
|
+
? directPrompt(run, carried)
|
|
2031
|
+
: workerPrompt(run, subtask, workspace, carried),
|
|
864
2032
|
parent,
|
|
2033
|
+
model,
|
|
865
2034
|
})
|
|
866
2035
|
rt.activations.add(handle)
|
|
867
2036
|
subtask.childId = handle.childId
|
|
868
2037
|
await save(run)
|
|
869
2038
|
|
|
870
2039
|
const result = await abortable(handle.result, signal)
|
|
871
|
-
const
|
|
2040
|
+
const raw = textOf(result.output)
|
|
872
2041
|
subtask.stopReason = result.stopReason
|
|
873
|
-
if (result.stopReason === 'completed' &&
|
|
2042
|
+
if (result.stopReason === 'completed' && raw) {
|
|
2043
|
+
// A worker may append one fenced block to ask for a decision. Strip it
|
|
2044
|
+
// before the report is stored: the delivered result must be the
|
|
2045
|
+
// findings, while the question becomes a structured attention item.
|
|
2046
|
+
const cleaned = extractAttentionBlock(raw)
|
|
2047
|
+
if (cleaned.found) {
|
|
2048
|
+
if (cleaned.attention) recordAttention(run, `subtask-${subtask.index}`, cleaned.attention)
|
|
2049
|
+
else recordAttentionIssue(run, `subtask-${subtask.index}`, cleaned.error)
|
|
2050
|
+
if (cleaned.error) log(`subtask ${subtask.index}: ${cleaned.error}`)
|
|
2051
|
+
}
|
|
2052
|
+
// A valid block is stripped; an unusable one stays in the worker result
|
|
2053
|
+
// so the report the user reads still contains what was asked.
|
|
2054
|
+
const text = cleaned.attention ? cleaned.text : raw
|
|
874
2055
|
subtask.status = 'completed'
|
|
875
2056
|
subtask.result = capText(text, config.resultCapBytes)
|
|
876
2057
|
subtask.error = null
|
|
877
2058
|
subtask.finishedAt = new Date(now()).toISOString()
|
|
878
2059
|
await recordUsage(run, handle.childId)
|
|
2060
|
+
await harvestWorkspace(run, subtask)
|
|
879
2061
|
await save(run)
|
|
880
2062
|
return
|
|
881
2063
|
}
|
|
@@ -904,7 +2086,14 @@ export function createOrchestrator(deps) {
|
|
|
904
2086
|
if (attempt < run.limits.attempts) {
|
|
905
2087
|
subtask.status = 'pending'
|
|
906
2088
|
await save(run)
|
|
907
|
-
|
|
2089
|
+
// Retrying a rate limit immediately burns the next attempt against the
|
|
2090
|
+
// same limit. A transient failure therefore waits 2s, 4s, 8s... while
|
|
2091
|
+
// any other failure keeps the short, existing delay: it may be fixed by
|
|
2092
|
+
// a different model answer rather than by time.
|
|
2093
|
+
const delay = looksTransient(subtask.error)
|
|
2094
|
+
? Math.min(RETRY_CAP_MS, RETRY_BASE_MS * 2 ** Math.max(0, attempt - 1))
|
|
2095
|
+
: 1000 * attempt
|
|
2096
|
+
await sleep(delay)
|
|
908
2097
|
}
|
|
909
2098
|
}
|
|
910
2099
|
|
|
@@ -923,16 +2112,104 @@ export function createOrchestrator(deps) {
|
|
|
923
2112
|
}
|
|
924
2113
|
}
|
|
925
2114
|
|
|
2115
|
+
/**
|
|
2116
|
+
* Run the subtask graph: a task starts as soon as every task it depends on has
|
|
2117
|
+
* completed, and at most `maxParallel` execute at once. A task that is only
|
|
2118
|
+
* held back by a dependency is marked `waiting` with the keys it waits for, so
|
|
2119
|
+
* status tells it apart from one that is actually running.
|
|
2120
|
+
*
|
|
2121
|
+
* A task whose dependency failed or was skipped is marked `skipped` with the
|
|
2122
|
+
* reason and never runs. Those skips are revived on the next call (self-heal,
|
|
2123
|
+
* resume) if the dependency completes then, so a retry can still unblock them.
|
|
2124
|
+
*
|
|
2125
|
+
* @param {object} run - live run record.
|
|
2126
|
+
* @param {object} parent - calling session.
|
|
2127
|
+
* @param {object} rt - runtime record with the abort controller.
|
|
2128
|
+
* @param {{ model?: string | null }} [options] - child model override.
|
|
2129
|
+
*/
|
|
2130
|
+
async function runGraph(run, parent, rt, { model = null } = {}) {
|
|
2131
|
+
const signal = rt.controller.signal
|
|
2132
|
+
const byKey = new Map(run.subtasks.map(subtask => [subtask.key, subtask]))
|
|
2133
|
+
let revived = false
|
|
2134
|
+
for (const subtask of run.subtasks) {
|
|
2135
|
+
if (subtask.status === 'skipped' && subtask.skipKind === 'upstream') {
|
|
2136
|
+
subtask.status = 'pending'
|
|
2137
|
+
subtask.skipKind = null
|
|
2138
|
+
subtask.skippedReason = null
|
|
2139
|
+
subtask.waitingOn = []
|
|
2140
|
+
revived = true
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
if (revived) await save(run)
|
|
2144
|
+
|
|
2145
|
+
const inFlight = new Map()
|
|
2146
|
+
while (!signal.aborted) {
|
|
2147
|
+
let dirty = cascadeSkips(run, byKey)
|
|
2148
|
+
for (const subtask of run.subtasks) {
|
|
2149
|
+
if (signal.aborted) break
|
|
2150
|
+
if (subtask.status !== 'pending' && subtask.status !== 'waiting') continue
|
|
2151
|
+
if (inFlight.has(subtask.key)) continue
|
|
2152
|
+
const outstanding = (subtask.dependsOn ?? [])
|
|
2153
|
+
.map(key => byKey.get(key))
|
|
2154
|
+
.filter(dependency => dependency && dependency.status !== 'completed')
|
|
2155
|
+
if (outstanding.length > 0) {
|
|
2156
|
+
const waitingOn = outstanding.map(dependency => dependency.key)
|
|
2157
|
+
if (subtask.status !== 'waiting' || subtask.waitingOn.join() !== waitingOn.join()) {
|
|
2158
|
+
subtask.waitingOn = waitingOn
|
|
2159
|
+
if (subtask.status !== 'waiting') subtask.status = 'waiting'
|
|
2160
|
+
dirty = true
|
|
2161
|
+
}
|
|
2162
|
+
continue
|
|
2163
|
+
}
|
|
2164
|
+
if (subtask.status === 'waiting') {
|
|
2165
|
+
subtask.status = 'pending'
|
|
2166
|
+
subtask.waitingOn = []
|
|
2167
|
+
dirty = true
|
|
2168
|
+
}
|
|
2169
|
+
if (inFlight.size >= run.limits.maxParallel) break
|
|
2170
|
+
const task = runSubtask(run, subtask, parent, rt, { model })
|
|
2171
|
+
.catch(error => log(`subtask ${subtask.index} graph runner threw: ${error?.message ?? error}`))
|
|
2172
|
+
.finally(() => inFlight.delete(subtask.key))
|
|
2173
|
+
inFlight.set(subtask.key, task)
|
|
2174
|
+
}
|
|
2175
|
+
if (dirty) await save(run)
|
|
2176
|
+
// Nothing is executing, so nothing can finish and no dependency can clear.
|
|
2177
|
+
if (inFlight.size === 0) break
|
|
2178
|
+
await Promise.race(inFlight.values())
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
// Leaving a subtask as `waiting` after the graph stopped would claim it is
|
|
2182
|
+
// still held back by live work. An aborted run keeps it resumable instead.
|
|
2183
|
+
let leftover = false
|
|
2184
|
+
for (const subtask of run.subtasks) {
|
|
2185
|
+
if (subtask.status === 'waiting') {
|
|
2186
|
+
subtask.status = 'pending'
|
|
2187
|
+
subtask.waitingOn = []
|
|
2188
|
+
leftover = true
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
if (leftover) await save(run)
|
|
2192
|
+
}
|
|
2193
|
+
|
|
926
2194
|
async function aggregate(run, parent, rt) {
|
|
927
2195
|
const completed = run.subtasks.filter(subtask => subtask.status === 'completed')
|
|
928
2196
|
if (completed.length === 0) return null
|
|
929
2197
|
if (rt.controller.signal.aborted) return deterministicJoin(run)
|
|
930
|
-
|
|
2198
|
+
// One report needs no merging, but it does need a prompt when the user has
|
|
2199
|
+
// since added an instruction or answered a question: the shortcut would
|
|
2200
|
+
// return the old text and silently drop the decision.
|
|
2201
|
+
const hasInstructions = (run.followups ?? []).length > 0
|
|
2202
|
+
if (completed.length === 1 && !hasInstructions) return `# ${run.goal}\n\n${completed[0].result ?? ''}`.trim()
|
|
931
2203
|
|
|
932
2204
|
try {
|
|
2205
|
+
// Instructions that arrived while the workers ran land here, and this is
|
|
2206
|
+
// the last boundary a terminal run has: the report can honor them even
|
|
2207
|
+
// when no worker will start again.
|
|
2208
|
+
const carried = carriedEntries(run)
|
|
2209
|
+
if (carried.length > 0 && markFollowupsApplied(carried, 'aggregator')) await save(run)
|
|
933
2210
|
const handle = await spawn(run, rt, {
|
|
934
2211
|
label: 'superagent-aggregate',
|
|
935
|
-
prompt: aggregatorPrompt(run, completed),
|
|
2212
|
+
prompt: aggregatorPrompt(run, completed, run.memory?.digest ?? '', carried),
|
|
936
2213
|
parent,
|
|
937
2214
|
})
|
|
938
2215
|
rt.activations.add(handle)
|
|
@@ -1016,16 +2293,84 @@ export function createOrchestrator(deps) {
|
|
|
1016
2293
|
subtask.usage = run.usageByChild[subtask.childId]
|
|
1017
2294
|
}
|
|
1018
2295
|
}
|
|
1019
|
-
|
|
2296
|
+
applyCost(run, config.pricing)
|
|
1020
2297
|
const nextStatus = terminalStatus(run, rt)
|
|
1021
2298
|
const finishedAt = new Date(now()).toISOString()
|
|
2299
|
+
|
|
2300
|
+
// An instruction that arrived in the last moments of a run never reached a
|
|
2301
|
+
// prompt. Record that instead of leaving it looking delivered: the user
|
|
2302
|
+
// has to know it did not shape anything.
|
|
2303
|
+
for (const entry of run.followups ?? []) {
|
|
2304
|
+
if (entry.appliedAt || entry.missedAt) continue
|
|
2305
|
+
entry.missedAt = finishedAt
|
|
2306
|
+
entry.missedReason = 'the run settled before another boundary ran; start a new run to apply it'
|
|
2307
|
+
}
|
|
2308
|
+
|
|
2309
|
+
// The aggregator appends a fenced notebook block to its report. Strip it in
|
|
2310
|
+
// every case so the user never sees plugin syntax, then persist it when the
|
|
2311
|
+
// notebook is enabled. A write failure is recorded on the run but never
|
|
2312
|
+
// allowed to fail the run itself: memory is an optimisation, not a result.
|
|
2313
|
+
if (typeof run.report === 'string' && run.report) {
|
|
2314
|
+
// An attention block is the aggregator's way of separating "here is the
|
|
2315
|
+
// outcome" from "you must decide this". It is stripped first, so the
|
|
2316
|
+
// report and the artifact never contain the fence.
|
|
2317
|
+
const attention = extractAttentionBlock(run.report)
|
|
2318
|
+
if (attention.found) {
|
|
2319
|
+
if (attention.attention) {
|
|
2320
|
+
run.report = attention.text
|
|
2321
|
+
recordAttention(run, 'aggregator', attention.attention)
|
|
2322
|
+
} else {
|
|
2323
|
+
// Keep the raw block in the report and record why it could not be used:
|
|
2324
|
+
// a malformed decision request is still a decision request.
|
|
2325
|
+
recordAttentionIssue(run, 'aggregator', attention.error)
|
|
2326
|
+
}
|
|
2327
|
+
if (attention.error) log(`run ${run.id}: ${attention.error}`)
|
|
2328
|
+
}
|
|
2329
|
+
const extracted = extractMemoryBlock(run.report)
|
|
2330
|
+
if (extracted.found) {
|
|
2331
|
+
// Assign even when the stripped report is empty: a report that was only
|
|
2332
|
+
// a notebook block must not survive into the user-facing result.
|
|
2333
|
+
run.report = extracted.report
|
|
2334
|
+
if (memory && TERMINAL_STATUSES.has(nextStatus)) {
|
|
2335
|
+
const settings = memorySettings()
|
|
2336
|
+
try {
|
|
2337
|
+
const result = await memory.write({
|
|
2338
|
+
scope: settings.scope,
|
|
2339
|
+
runId: run.id,
|
|
2340
|
+
entries: extracted.entries,
|
|
2341
|
+
maxEntries: settings.maxEntries,
|
|
2342
|
+
maxAgeDays: settings.maxAgeDays,
|
|
2343
|
+
})
|
|
2344
|
+
run.memoryWrite = {
|
|
2345
|
+
at: new Date(now()).toISOString(),
|
|
2346
|
+
written: result.written,
|
|
2347
|
+
reason: extracted.error ?? result.reason ?? null,
|
|
2348
|
+
}
|
|
2349
|
+
} catch (writeError) {
|
|
2350
|
+
log(`cannot write cross-run memory for run ${run.id}: ${writeError?.message ?? writeError}`)
|
|
2351
|
+
run.memoryWrite = {
|
|
2352
|
+
at: new Date(now()).toISOString(),
|
|
2353
|
+
written: 0,
|
|
2354
|
+
reason: `write failed: ${writeError?.message ?? writeError}`,
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
} else if (extracted.error) {
|
|
2358
|
+
run.memoryWrite = { at: new Date(now()).toISOString(), written: 0, reason: extracted.error }
|
|
2359
|
+
}
|
|
2360
|
+
}
|
|
2361
|
+
}
|
|
2362
|
+
// The report gets a durable address before the run is announced, so a
|
|
2363
|
+
// notification or status line can point at a file that already exists.
|
|
2364
|
+
if (TERMINAL_STATUSES.has(nextStatus)) await writeReportArtifact(run)
|
|
2365
|
+
addMilestone(run, 'settled')
|
|
1022
2366
|
let error = run.error
|
|
1023
2367
|
if (rt.abortCause === 'cancelled') {
|
|
1024
2368
|
error = 'cancelled by superagent_cancel'
|
|
1025
2369
|
} else if (rt.abortCause === 'host-gone') {
|
|
1026
2370
|
error = 'the host shut down while this run was active; run superagent_resume in a new host to finish it'
|
|
1027
2371
|
} else if (nextStatus === 'failed' && !error) {
|
|
1028
|
-
error = run.subtasks.map(subtask => subtask.error).filter(Boolean).slice(0, 3).join('; ')
|
|
2372
|
+
error = run.subtasks.map(subtask => subtask.error ?? subtask.skippedReason).filter(Boolean).slice(0, 3).join('; ')
|
|
2373
|
+
|| 'no subtask completed'
|
|
1029
2374
|
}
|
|
1030
2375
|
if (rt.abortCause === 'deadline' && !error) error = 'deadline exceeded'
|
|
1031
2376
|
|
|
@@ -1100,6 +2445,160 @@ export function createOrchestrator(deps) {
|
|
|
1100
2445
|
})
|
|
1101
2446
|
}
|
|
1102
2447
|
|
|
2448
|
+
/** One subtask record, shared by the planned pipeline and direct mode. */
|
|
2449
|
+
function makeSubtask(entry, index) {
|
|
2450
|
+
return {
|
|
2451
|
+
index: index + 1,
|
|
2452
|
+
title: entry.title,
|
|
2453
|
+
prompt: entry.prompt,
|
|
2454
|
+
status: 'pending',
|
|
2455
|
+
attempts: 0,
|
|
2456
|
+
childId: null,
|
|
2457
|
+
stopReason: null,
|
|
2458
|
+
error: null,
|
|
2459
|
+
result: null,
|
|
2460
|
+
usage: null,
|
|
2461
|
+
startedAt: null,
|
|
2462
|
+
finishedAt: null,
|
|
2463
|
+
}
|
|
2464
|
+
}
|
|
2465
|
+
|
|
2466
|
+
/** Planner, parallel workers, self-heal, then one merged report. */
|
|
2467
|
+
async function runPlanned(run, parent, rt) {
|
|
2468
|
+
if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
|
|
2469
|
+
run.status = 'planning'
|
|
2470
|
+
await save(run)
|
|
2471
|
+
const plan = await makePlan(run, parent, rt)
|
|
2472
|
+
run.plan = { mode: plan.mode, subtasks: plan.subtasks }
|
|
2473
|
+
const entries = plan.subtasks.map((entry, index) => ({
|
|
2474
|
+
key: entry.id ?? `w${index + 1}`,
|
|
2475
|
+
title: entry.title,
|
|
2476
|
+
dependsOn: entry.dependsOn ?? [],
|
|
2477
|
+
}))
|
|
2478
|
+
const keys = new Set(entries.map(entry => entry.key))
|
|
2479
|
+
if (keys.size !== entries.length) {
|
|
2480
|
+
throw new Error(
|
|
2481
|
+
'plan rejected: duplicate subtask ids; dependsOn would be ambiguous, so replan with unique ids',
|
|
2482
|
+
)
|
|
2483
|
+
}
|
|
2484
|
+
const linked = linkSubtasks(entries)
|
|
2485
|
+
if (linked.cycle) {
|
|
2486
|
+
// A cycle cannot be scheduled, and picking an arbitrary edge to drop
|
|
2487
|
+
// would change what the planner asked for. Reject the plan and name
|
|
2488
|
+
// the tasks involved so the next attempt can be fixed.
|
|
2489
|
+
throw new Error(
|
|
2490
|
+
`plan rejected: dependency cycle ${linked.cycle.join(' -> ')}; `
|
|
2491
|
+
+ 'no subtask was started, so replan without the cycle',
|
|
2492
|
+
)
|
|
2493
|
+
}
|
|
2494
|
+
run.subtasks = plan.subtasks.map((entry, index) => {
|
|
2495
|
+
const key = entries[index].key
|
|
2496
|
+
const missing = linked.unknown.get(key) ?? []
|
|
2497
|
+
return {
|
|
2498
|
+
...makeSubtask(entry, index),
|
|
2499
|
+
key,
|
|
2500
|
+
dependsOn: linked.edges.get(key) ?? [],
|
|
2501
|
+
waitingOn: [],
|
|
2502
|
+
status: missing.length > 0 ? 'skipped' : 'pending',
|
|
2503
|
+
skipKind: missing.length > 0 ? 'missing' : null,
|
|
2504
|
+
skippedReason: missing.length > 0
|
|
2505
|
+
? `depends on unknown subtask ${missing.map(ref => `"${ref}"`).join(', ')}`
|
|
2506
|
+
: null,
|
|
2507
|
+
}
|
|
2508
|
+
})
|
|
2509
|
+
await save(run)
|
|
2510
|
+
addMilestone(run, 'plan_ready')
|
|
2511
|
+
await save(run)
|
|
2512
|
+
}
|
|
2513
|
+
|
|
2514
|
+
const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
|
|
2515
|
+
if (open.length > 0) {
|
|
2516
|
+
run.status = 'running'
|
|
2517
|
+
addMilestone(run, 'running')
|
|
2518
|
+
await save(run)
|
|
2519
|
+
await runGraph(run, parent, rt)
|
|
2520
|
+
}
|
|
2521
|
+
|
|
2522
|
+
// Self-heal before reporting: a failed subtask gets another full round
|
|
2523
|
+
// while the run is not aborted and the budget still allows it. A failure
|
|
2524
|
+
// that looks transient (quota, rate limit, timeout) may switch to the
|
|
2525
|
+
// configured fallback child model on that round. Only what still fails
|
|
2526
|
+
// after this is reported to the user, with what was tried.
|
|
2527
|
+
const rounds = selfHealRounds()
|
|
2528
|
+
const fallbackModel = typeof config.childModelFallback === 'string' && config.childModelFallback.trim()
|
|
2529
|
+
? config.childModelFallback.trim()
|
|
2530
|
+
: null
|
|
2531
|
+
for (let round = 1; round <= rounds; round++) {
|
|
2532
|
+
if (rt.controller.signal.aborted) break
|
|
2533
|
+
// Only a failure that looks transient is retried as a whole round.
|
|
2534
|
+
// Re-running a refusal or a permanent error would double the spend
|
|
2535
|
+
// without changing the outcome, which is the opposite of self-healing.
|
|
2536
|
+
const retryable = run.subtasks.filter(subtask => subtask.status === 'failed' && looksTransient(subtask.error))
|
|
2537
|
+
if (retryable.length === 0) break
|
|
2538
|
+
if (budgetStop(run)) {
|
|
2539
|
+
log(`not retrying ${retryable.length} failed subtask(s): ${budgetStop(run)}`)
|
|
2540
|
+
break
|
|
2541
|
+
}
|
|
2542
|
+
const useFallback = Boolean(fallbackModel && retryable.some(subtask => looksTransient(subtask.error)))
|
|
2543
|
+
run.selfHeal = {
|
|
2544
|
+
rounds: round,
|
|
2545
|
+
retried: retryable.length,
|
|
2546
|
+
recovered: 0,
|
|
2547
|
+
model: useFallback ? fallbackModel : null,
|
|
2548
|
+
}
|
|
2549
|
+
for (const subtask of retryable) {
|
|
2550
|
+
subtask.status = 'pending'
|
|
2551
|
+
subtask.attempts = 0
|
|
2552
|
+
subtask.error = null
|
|
2553
|
+
subtask.stopReason = null
|
|
2554
|
+
subtask.finishedAt = null
|
|
2555
|
+
}
|
|
2556
|
+
log(
|
|
2557
|
+
`self-heal round ${round}: retrying ${retryable.length} failed subtask(s)`
|
|
2558
|
+
+ (useFallback ? ` on ${fallbackModel}` : ''),
|
|
2559
|
+
)
|
|
2560
|
+
await save(run)
|
|
2561
|
+
// The graph re-runs the retried task and revives the downstream tasks
|
|
2562
|
+
// that were skipped because of it, in dependency order.
|
|
2563
|
+
await runGraph(run, parent, rt, { model: useFallback ? fallbackModel : null })
|
|
2564
|
+
run.selfHeal.recovered = retryable.filter(subtask => subtask.status === 'completed').length
|
|
2565
|
+
await save(run)
|
|
2566
|
+
}
|
|
2567
|
+
|
|
2568
|
+
if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
|
|
2569
|
+
run.status = 'aggregating'
|
|
2570
|
+
addMilestone(run, 'aggregating')
|
|
2571
|
+
await save(run)
|
|
2572
|
+
run.report = await aggregate(run, parent, rt)
|
|
2573
|
+
} else if (!run.report) {
|
|
2574
|
+
run.report = deterministicJoin(run)
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
/**
|
|
2579
|
+
* Direct mode: one worker does the whole goal. There is no planner and no
|
|
2580
|
+
* aggregator, so the child's reply becomes the report verbatim; the run
|
|
2581
|
+
* record, quota, deadline, cancel, budget, and usage accounting are the same
|
|
2582
|
+
* as for a planned run.
|
|
2583
|
+
*/
|
|
2584
|
+
async function runDirect(run, parent, rt) {
|
|
2585
|
+
if (!Array.isArray(run.subtasks) || run.subtasks.length === 0) {
|
|
2586
|
+
const entry = { title: 'Complete the goal', prompt: run.goal }
|
|
2587
|
+
run.plan = { mode: 'direct', subtasks: [entry] }
|
|
2588
|
+
run.subtasks = [makeSubtask(entry, 0)]
|
|
2589
|
+
await save(run)
|
|
2590
|
+
}
|
|
2591
|
+
const subtask = run.subtasks.find(item => item.status !== 'completed') ?? run.subtasks[run.subtasks.length - 1]
|
|
2592
|
+
if (subtask && subtask.status !== 'completed') {
|
|
2593
|
+
run.status = 'running'
|
|
2594
|
+
addMilestone(run, 'running')
|
|
2595
|
+
await save(run)
|
|
2596
|
+
await runSubtask(run, subtask, parent, rt)
|
|
2597
|
+
}
|
|
2598
|
+
if (subtask?.status === 'completed') run.report = subtask.result ?? null
|
|
2599
|
+
else if (!run.report) run.report = deterministicJoin(run)
|
|
2600
|
+
}
|
|
2601
|
+
|
|
1103
2602
|
async function execute(run, parent, rt) {
|
|
1104
2603
|
rt.activeStart = now()
|
|
1105
2604
|
const remaining = Math.max(0, Date.parse(run.deadlineAt) - now())
|
|
@@ -1120,41 +2619,24 @@ export function createOrchestrator(deps) {
|
|
|
1120
2619
|
|
|
1121
2620
|
try {
|
|
1122
2621
|
ensureCounters(run)
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
}))
|
|
1142
|
-
await save(run)
|
|
1143
|
-
}
|
|
1144
|
-
|
|
1145
|
-
const open = run.subtasks.filter(subtask => subtask.status !== 'completed')
|
|
1146
|
-
if (open.length > 0) {
|
|
1147
|
-
run.status = 'running'
|
|
1148
|
-
await save(run)
|
|
1149
|
-
await pool(open, run.limits.maxParallel, subtask => runSubtask(run, subtask, parent, rt))
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
if (run.subtasks.some(subtask => subtask.status === 'completed') && !rt.controller.signal.aborted) {
|
|
1153
|
-
run.status = 'aggregating'
|
|
1154
|
-
await save(run)
|
|
1155
|
-
run.report = await aggregate(run, parent, rt)
|
|
1156
|
-
} else if (!run.report) {
|
|
1157
|
-
run.report = deterministicJoin(run)
|
|
2622
|
+
// Decided once, before any child runs, so the whole plan uses one mode and
|
|
2623
|
+
// status can already say whether isolation is on.
|
|
2624
|
+
await ensureWorkspace(run)
|
|
2625
|
+
if (run.mode === 'direct') await runDirect(run, parent, rt)
|
|
2626
|
+
else await runPlanned(run, parent, rt)
|
|
2627
|
+
|
|
2628
|
+
// Merge before the report: a conflict is part of what happened, and the
|
|
2629
|
+
// report is written from the run, so it must exist by now. A run that
|
|
2630
|
+
// stopped early keeps its branches instead of merging half-finished work;
|
|
2631
|
+
// a resume picks the same worktrees up again. Direct mode has a single
|
|
2632
|
+
// worker, but it is isolated the same way, so it merges here too.
|
|
2633
|
+
if (run.workspace?.mode === 'git-worktree') {
|
|
2634
|
+
if (rt.controller.signal.aborted) {
|
|
2635
|
+
run.workspace.deferred = 'the run stopped before merging; the worker branches are kept for a resume'
|
|
2636
|
+
await save(run)
|
|
2637
|
+
} else {
|
|
2638
|
+
await mergeWorkspaces(run)
|
|
2639
|
+
}
|
|
1158
2640
|
}
|
|
1159
2641
|
} catch (error) {
|
|
1160
2642
|
run.error = error?.message ?? String(error)
|
|
@@ -1222,6 +2704,8 @@ export function createOrchestrator(deps) {
|
|
|
1222
2704
|
// grow these maps for the lifetime of the host.
|
|
1223
2705
|
runtime.delete(run.id)
|
|
1224
2706
|
saveChains.delete(run.id)
|
|
2707
|
+
// The slot this run held may let a waiting run start.
|
|
2708
|
+
pumpQueue().catch(() => {})
|
|
1225
2709
|
})
|
|
1226
2710
|
return rt
|
|
1227
2711
|
}
|
|
@@ -1233,10 +2717,20 @@ export function createOrchestrator(deps) {
|
|
|
1233
2717
|
return result
|
|
1234
2718
|
}
|
|
1235
2719
|
|
|
1236
|
-
async function start({
|
|
2720
|
+
async function start({
|
|
2721
|
+
goal,
|
|
2722
|
+
parentSessionId,
|
|
2723
|
+
parent,
|
|
2724
|
+
clientTimeZone,
|
|
2725
|
+
overrides = {},
|
|
2726
|
+
allowDuplicate = false,
|
|
2727
|
+
queue: wantsQueue = false,
|
|
2728
|
+
mode: requestedMode,
|
|
2729
|
+
}) {
|
|
1237
2730
|
const trimmed = typeof goal === 'string' ? goal.trim() : ''
|
|
1238
2731
|
if (!trimmed) throw new Error('goal is required')
|
|
1239
2732
|
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
2733
|
+
const mode = normalizeMode(requestedMode)
|
|
1240
2734
|
|
|
1241
2735
|
const limits = buildLimits(overrides)
|
|
1242
2736
|
// Kept so a resume can rebuild the limits from the *current* config
|
|
@@ -1245,15 +2739,47 @@ export function createOrchestrator(deps) {
|
|
|
1245
2739
|
for (const key of ['maxSubtasks', 'maxParallel', 'deadlineMinutes']) {
|
|
1246
2740
|
if (overrides[key] !== undefined && overrides[key] !== null) keptOverrides[key] = overrides[key]
|
|
1247
2741
|
}
|
|
1248
|
-
|
|
2742
|
+
const dedupeKey = dedupeKeyOf(trimmed)
|
|
2743
|
+
// The goal claim is held across "list, check, write" so two processes
|
|
2744
|
+
// sharing a state directory cannot both conclude that nothing is running.
|
|
2745
|
+
return serialize(() => withGoalGuard(dedupeKey, async () => {
|
|
1249
2746
|
const active = await activeCount()
|
|
1250
|
-
|
|
2747
|
+
// The same goal twice is much more often a retry than a second piece of
|
|
2748
|
+
// work: the caller lost the run id, or a model re-issued the call. Return
|
|
2749
|
+
// the active run instead of paying for a second plan. `allowDuplicate` is
|
|
2750
|
+
// the escape hatch for a genuinely repeated task.
|
|
2751
|
+
if (!allowDuplicate) {
|
|
2752
|
+
const existing = await activeRunForGoal(dedupeKey, parentSessionId)
|
|
2753
|
+
if (existing) {
|
|
2754
|
+
log(`run ${existing.id} already has this goal; returning it instead of starting a duplicate`)
|
|
2755
|
+
return { ...existing, deduped: true }
|
|
2756
|
+
}
|
|
2757
|
+
}
|
|
2758
|
+
const waiting = queue.length
|
|
2759
|
+
// A run already in line is first in line: a new start must not jump it.
|
|
2760
|
+
const atCapacity = active >= maxActiveRuns() || waiting > 0
|
|
2761
|
+
if (atCapacity) {
|
|
2762
|
+
if (!queueEnabled(wantsQueue)) throw new Error(quotaMessage(active, waiting))
|
|
2763
|
+
if (waiting >= maxQueuedRuns()) {
|
|
2764
|
+
throw new Error(
|
|
2765
|
+
`quota exceeded: ${active} active run(s), maxActiveRuns is ${maxActiveRuns()}, and the queue is full `
|
|
2766
|
+
+ `(${waiting} waiting, maxQueuedRuns is ${maxQueuedRuns()}). Let one finish with superagent_status or `
|
|
2767
|
+
+ 'cancel one with superagent_cancel.',
|
|
2768
|
+
)
|
|
2769
|
+
}
|
|
2770
|
+
}
|
|
2771
|
+
// Read once, here: the digest is part of the run record, so a status call
|
|
2772
|
+
// can show exactly what the planner was told, and a resumed run does not
|
|
2773
|
+
// silently change the context its plan was based on.
|
|
2774
|
+
const context = await memoryContext(null, trimmed)
|
|
1251
2775
|
const run = {
|
|
1252
2776
|
id: newRunId(),
|
|
1253
2777
|
schemaVersion: 1,
|
|
1254
2778
|
goal: trimmed,
|
|
2779
|
+
dedupeKey,
|
|
1255
2780
|
status: 'queued',
|
|
1256
|
-
mode
|
|
2781
|
+
mode,
|
|
2782
|
+
queue: null,
|
|
1257
2783
|
parentSessionId,
|
|
1258
2784
|
clientTimeZone: clientTimeZone ?? null,
|
|
1259
2785
|
createdAt: new Date(now()).toISOString(),
|
|
@@ -1268,12 +2794,44 @@ export function createOrchestrator(deps) {
|
|
|
1268
2794
|
subtasks: [],
|
|
1269
2795
|
steps: {},
|
|
1270
2796
|
report: null,
|
|
2797
|
+
attention: null,
|
|
2798
|
+
// Questions move here when a resume retires them, so a late answer can
|
|
2799
|
+
// still be told apart from an id that never existed.
|
|
2800
|
+
attentionHistory: [],
|
|
2801
|
+
attentionSeq: 0,
|
|
2802
|
+
// Instructions and answers given to this run after it started.
|
|
2803
|
+
followups: [],
|
|
2804
|
+
followupSeq: 0,
|
|
2805
|
+
milestones: [],
|
|
2806
|
+
artifacts: [],
|
|
2807
|
+
memory: context,
|
|
2808
|
+
memoryWrite: null,
|
|
2809
|
+
selfHeal: null,
|
|
1271
2810
|
error: null,
|
|
1272
2811
|
usage: null,
|
|
1273
2812
|
notified: false,
|
|
1274
2813
|
counters: { spawns: 0, unreadable: {} },
|
|
1275
2814
|
usageByChild: {},
|
|
1276
2815
|
}
|
|
2816
|
+
if (atCapacity) {
|
|
2817
|
+
// Waiting runs are persisted with an owner and a heartbeat so another
|
|
2818
|
+
// process sharing this directory can tell a live queue from a dead one.
|
|
2819
|
+
run.queue = {
|
|
2820
|
+
waiting: true,
|
|
2821
|
+
position: waiting + 1,
|
|
2822
|
+
of: waiting + 1,
|
|
2823
|
+
enqueuedAt: new Date(now()).toISOString(),
|
|
2824
|
+
timeoutAt: new Date(now() + queueTimeoutMs()).toISOString(),
|
|
2825
|
+
}
|
|
2826
|
+
run.ownerPid = process.pid
|
|
2827
|
+
run.ownerHost = os.hostname()
|
|
2828
|
+
run.heartbeatAt = new Date(now()).toISOString()
|
|
2829
|
+
await store.write(run)
|
|
2830
|
+
enqueue(run, parent)
|
|
2831
|
+
await refreshQueuePositions()
|
|
2832
|
+
log(`run ${run.id} is queued at position ${run.queue.position} (${active} active run(s))`)
|
|
2833
|
+
return run
|
|
2834
|
+
}
|
|
1277
2835
|
await takeRunLock(run.id)
|
|
1278
2836
|
try {
|
|
1279
2837
|
await store.write(run)
|
|
@@ -1283,18 +2841,46 @@ export function createOrchestrator(deps) {
|
|
|
1283
2841
|
}
|
|
1284
2842
|
schedule(run, parent)
|
|
1285
2843
|
return run
|
|
1286
|
-
})
|
|
2844
|
+
}))
|
|
2845
|
+
}
|
|
2846
|
+
|
|
2847
|
+
/** A run may only be resumed by the session that started it. */
|
|
2848
|
+
function assertResumeOwner(run, parent) {
|
|
2849
|
+
if (run.parentSessionId && parent?.id && run.parentSessionId !== parent.id) {
|
|
2850
|
+
throw new Error(
|
|
2851
|
+
`run ${run.id} belongs to session ${run.parentSessionId}, not ${parent.id}; `
|
|
2852
|
+
+ 'resume it from that session, or start a new run from this one',
|
|
2853
|
+
)
|
|
2854
|
+
}
|
|
1287
2855
|
}
|
|
1288
2856
|
|
|
1289
|
-
|
|
2857
|
+
// A resume only finishes work that already exists, so it is never counted
|
|
2858
|
+
// against `maxActiveRuns`: a person asking to finish an interrupted run is
|
|
2859
|
+
// always let through. An options bag is still accepted so an older caller
|
|
2860
|
+
// that passes `force` keeps working; it no longer changes the outcome.
|
|
2861
|
+
/**
|
|
2862
|
+
* Resume under the start gate. Split from `resume` so answering a question
|
|
2863
|
+
* can record the answer and reopen the run in one serialized step instead of
|
|
2864
|
+
* waiting on a nested gate (which would deadlock on the promise it is holding).
|
|
2865
|
+
*
|
|
2866
|
+
* `reopen` lets `superagent_answer` continue a `completed` run: its subtasks
|
|
2867
|
+
* stay done and only the aggregation runs again, now with the answer.
|
|
2868
|
+
*/
|
|
2869
|
+
async function resumeLocked(runId, parent, clientTimeZone, { reopen = false } = {}) {
|
|
1290
2870
|
const id = String(runId ?? '').trim()
|
|
1291
2871
|
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
1292
|
-
|
|
2872
|
+
{
|
|
1293
2873
|
if (runtime.has(id)) {
|
|
1294
2874
|
throw new Error(`run ${id} is still active in this process; wait for it with superagent_status or cancel it`)
|
|
1295
2875
|
}
|
|
1296
2876
|
let run = await loadRun(id)
|
|
1297
|
-
|
|
2877
|
+
assertResumeOwner(run, parent)
|
|
2878
|
+
// A completed run is normally finished for good. The exception is an
|
|
2879
|
+
// answer that was recorded but never applied because the reopen failed
|
|
2880
|
+
// (quota, lock): then resume is the recovery path and must actually run
|
|
2881
|
+
// the aggregation again, not return the old report.
|
|
2882
|
+
let reopenCompleted = reopen || hasUnappliedAnswer(run)
|
|
2883
|
+
if (run.status === 'completed' && !reopenCompleted) return run
|
|
1298
2884
|
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1299
2885
|
const verdict = await ownership(run)
|
|
1300
2886
|
if (verdict.live) {
|
|
@@ -1311,7 +2897,9 @@ export function createOrchestrator(deps) {
|
|
|
1311
2897
|
// Re-read under the lock: another process may have started the run
|
|
1312
2898
|
// between the check above and the claim.
|
|
1313
2899
|
run = await loadRun(id)
|
|
1314
|
-
|
|
2900
|
+
assertResumeOwner(run, parent)
|
|
2901
|
+
reopenCompleted = reopen || hasUnappliedAnswer(run)
|
|
2902
|
+
if (run.status === 'completed' && !reopenCompleted) return run
|
|
1315
2903
|
if (ACTIVE_STATUSES.has(run.status)) {
|
|
1316
2904
|
const verdict = await ownership(run)
|
|
1317
2905
|
if (verdict.live) {
|
|
@@ -1322,28 +2910,76 @@ export function createOrchestrator(deps) {
|
|
|
1322
2910
|
await clearCancel(id)
|
|
1323
2911
|
}
|
|
1324
2912
|
|
|
1325
|
-
|
|
1326
|
-
|
|
2913
|
+
// A resume is a person's action on a run that already exists, so it is
|
|
2914
|
+
// not gated by `maxActiveRuns`; only a start creates new spend. The
|
|
2915
|
+
// sweep still runs, so an orphan cannot hide behind a resumed run.
|
|
2916
|
+
const failedBefore = (run.subtasks ?? [])
|
|
2917
|
+
.filter(subtask => subtask.status !== 'completed' && subtask.error)
|
|
2918
|
+
.map(subtask => `${subtask.title}: ${String(subtask.error).replace(/\s+/g, ' ').slice(0, 160)}`)
|
|
2919
|
+
.slice(0, 3)
|
|
2920
|
+
run.recovery = {
|
|
2921
|
+
at: new Date(now()).toISOString(),
|
|
2922
|
+
previousStatus: run.status,
|
|
2923
|
+
previousError: run.error ? String(run.error).replace(/\s+/g, ' ').slice(0, 300) : null,
|
|
2924
|
+
failed: failedBefore,
|
|
2925
|
+
}
|
|
1327
2926
|
|
|
1328
2927
|
for (const subtask of run.subtasks ?? []) {
|
|
1329
2928
|
if (subtask.status === 'completed') continue
|
|
2929
|
+
// A dependency on a subtask the plan never contained cannot be fixed
|
|
2930
|
+
// by resuming: nothing would ever clear it.
|
|
2931
|
+
if (subtask.skipKind === 'missing') continue
|
|
1330
2932
|
subtask.status = 'pending'
|
|
1331
2933
|
subtask.attempts = 0
|
|
1332
2934
|
subtask.error = null
|
|
1333
2935
|
subtask.stopReason = null
|
|
1334
2936
|
subtask.finishedAt = null
|
|
2937
|
+
subtask.skipKind = null
|
|
2938
|
+
subtask.skippedReason = null
|
|
2939
|
+
subtask.waitingOn = []
|
|
1335
2940
|
}
|
|
1336
2941
|
run.status = 'queued'
|
|
1337
|
-
|
|
2942
|
+
// A reopened run has nothing left to execute, so the old report must go:
|
|
2943
|
+
// keeping it would return the report that ignored the answer.
|
|
2944
|
+
run.report = reopenCompleted ? null : (run.plan ? run.report : null)
|
|
1338
2945
|
run.error = null
|
|
1339
2946
|
run.finishedAt = null
|
|
1340
2947
|
run.reclaimed = null
|
|
1341
2948
|
run.cancelPending = false
|
|
2949
|
+
// A queued marker from an earlier attempt would keep this run out of the
|
|
2950
|
+
// active count after it is scheduled again.
|
|
2951
|
+
run.queue = null
|
|
1342
2952
|
// A resumed run completes a second time, so it must be able to wake the
|
|
1343
2953
|
// session again; keeping the old outcome would silence it forever.
|
|
1344
2954
|
run.notified = false
|
|
1345
2955
|
run.notify = null
|
|
2956
|
+
// These describe the previous settle. Keeping them would report an old
|
|
2957
|
+
// self-heal round or memory write against the new attempt.
|
|
2958
|
+
run.selfHeal = null
|
|
2959
|
+
run.memoryWrite = null
|
|
2960
|
+
// A question from the previous attempt was already reported; keeping it
|
|
2961
|
+
// would attach an old decision to the new outcome. It is archived rather
|
|
2962
|
+
// than dropped, so a late answer is refused as expired instead of
|
|
2963
|
+
// looking like an id that never existed.
|
|
2964
|
+
if (run.attention && ((run.attention.items ?? []).length > 0 || run.attention.issue)) {
|
|
2965
|
+
run.attentionHistory = Array.isArray(run.attentionHistory) ? run.attentionHistory : []
|
|
2966
|
+
run.attentionHistory.push({ ...run.attention, expiredAt: new Date(now()).toISOString() })
|
|
2967
|
+
if (run.attentionHistory.length > ATTENTION_HISTORY_MAX) {
|
|
2968
|
+
run.attentionHistory = run.attentionHistory.slice(-ATTENTION_HISTORY_MAX)
|
|
2969
|
+
}
|
|
2970
|
+
}
|
|
2971
|
+
run.attention = null
|
|
2972
|
+
// An instruction that missed the previous attempt gets another chance:
|
|
2973
|
+
// the new round builds fresh prompts.
|
|
2974
|
+
for (const entry of run.followups ?? []) {
|
|
2975
|
+
entry.missedAt = null
|
|
2976
|
+
entry.missedReason = null
|
|
2977
|
+
}
|
|
2978
|
+
run.dedupeKey = run.dedupeKey ?? dedupeKeyOf(run.goal)
|
|
1346
2979
|
run.resumedAt = new Date(now()).toISOString()
|
|
2980
|
+
// A resume replans nothing, but the aggregator still runs, so refresh
|
|
2981
|
+
// the digest the notes have advanced since the run started.
|
|
2982
|
+
run.memory = await memoryContext(run.id, run.goal)
|
|
1347
2983
|
run.clientTimeZone = clientTimeZone ?? run.clientTimeZone ?? null
|
|
1348
2984
|
// The limits are rebuilt from the config that is active now, keeping
|
|
1349
2985
|
// everything already spent: that is what makes "raise the cap, then
|
|
@@ -1358,10 +2994,222 @@ export function createOrchestrator(deps) {
|
|
|
1358
2994
|
} finally {
|
|
1359
2995
|
if (!scheduled) await releaseRunLock(id)
|
|
1360
2996
|
}
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
|
|
3000
|
+
/** Serialize resumes; `resumeLocked` assumes the start gate is already held. */
|
|
3001
|
+
function resume(runId, parent, clientTimeZone, options) {
|
|
3002
|
+
return serialize(() => resumeLocked(runId, parent, clientTimeZone, options))
|
|
3003
|
+
}
|
|
3004
|
+
|
|
3005
|
+
/**
|
|
3006
|
+
* Add an instruction to a run that is already executing.
|
|
3007
|
+
*
|
|
3008
|
+
* The host exposes no way to append a message to a child activation, so this
|
|
3009
|
+
* is deliberately not a live channel: the instruction is queued on the run
|
|
3010
|
+
* and the next prompt built for it carries it. The reply says which prompt
|
|
3011
|
+
* that will be. A run owned by another process is refused rather than written:
|
|
3012
|
+
* that owner's next save would overwrite the change and the instruction would
|
|
3013
|
+
* silently vanish.
|
|
3014
|
+
*
|
|
3015
|
+
* @param {string} runId - run identity.
|
|
3016
|
+
* @param {string} text - the instruction.
|
|
3017
|
+
* @param {object} parent - live agent that owns the run.
|
|
3018
|
+
* @returns {Promise<{ run: object, followup: object, effect: { boundary: string, detail: string } }>}
|
|
3019
|
+
*/
|
|
3020
|
+
async function followup(runId, text, parent) {
|
|
3021
|
+
const id = String(runId ?? '').trim()
|
|
3022
|
+
if (!id) throw new Error('run_id is required')
|
|
3023
|
+
const message = capChars(text, FOLLOWUP_MAX_CHARS)
|
|
3024
|
+
if (!message) throw new Error('message is required; pass the instruction to add to the run')
|
|
3025
|
+
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
3026
|
+
return serialize(async () => {
|
|
3027
|
+
const rt = runtime.get(id)
|
|
3028
|
+
if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
|
|
3029
|
+
const run = rt.run
|
|
3030
|
+
assertResumeOwner(run, parent)
|
|
3031
|
+
const entry = pushFollowup(run, { kind: 'followup', text: message })
|
|
3032
|
+
await save(run)
|
|
3033
|
+
return { run, followup: entry, effect: followupEffect(run) }
|
|
3034
|
+
}
|
|
3035
|
+
const run = await loadRun(id)
|
|
3036
|
+
assertResumeOwner(run, parent)
|
|
3037
|
+
if (ACTIVE_STATUSES.has(run.status)) {
|
|
3038
|
+
throw new Error(
|
|
3039
|
+
`run ${id} is active but owned by another process; an instruction written here would be overwritten by `
|
|
3040
|
+
+ 'that process\'s next save. Send it from the session that owns the run, or cancel the run and resume it '
|
|
3041
|
+
+ 'here first.',
|
|
3042
|
+
)
|
|
3043
|
+
}
|
|
3044
|
+
if (run.status !== 'completed') {
|
|
3045
|
+
// A paused run will build fresh prompts when it is resumed, so queueing
|
|
3046
|
+
// the instruction now is a real delivery, not a courtesy.
|
|
3047
|
+
const entry = pushFollowup(run, { kind: 'followup', text: message })
|
|
3048
|
+
await save(run)
|
|
3049
|
+
return { run, followup: entry, effect: followupEffect(run) }
|
|
3050
|
+
}
|
|
3051
|
+
throw new Error(
|
|
3052
|
+
`run ${id} is completed, so it is not running and cannot take an instruction. `
|
|
3053
|
+
+ 'Start a new run, or answer a pending question with superagent_answer.',
|
|
3054
|
+
)
|
|
1361
3055
|
})
|
|
1362
3056
|
}
|
|
1363
3057
|
|
|
3058
|
+
/**
|
|
3059
|
+
* Answer one structured attention item and let the run continue.
|
|
3060
|
+
*
|
|
3061
|
+
* The answer is matched by id, never by array position, and is recorded on
|
|
3062
|
+
* the item it answers. An active run picks it up at its next boundary; a
|
|
3063
|
+
* terminal run is reopened so its aggregation runs again with the decision —
|
|
3064
|
+
* that is the difference between continuing this run and starting another one.
|
|
3065
|
+
*
|
|
3066
|
+
* @param {string} runId - run identity.
|
|
3067
|
+
* @param {string | null | undefined} attentionId - `q1` from status; optional only when one question is open.
|
|
3068
|
+
* @param {string} text - the decision.
|
|
3069
|
+
* @param {object} parent - live agent that owns the run.
|
|
3070
|
+
* @param {string | undefined} clientTimeZone - forwarded to a resume.
|
|
3071
|
+
* @returns {Promise<object>} run plus which question was answered and whether the run restarted.
|
|
3072
|
+
*/
|
|
3073
|
+
async function answer(runId, attentionId, text, parent, clientTimeZone) {
|
|
3074
|
+
const id = String(runId ?? '').trim()
|
|
3075
|
+
if (!id) throw new Error('run_id is required')
|
|
3076
|
+
const value = capChars(text, FOLLOWUP_MAX_CHARS)
|
|
3077
|
+
if (!value) throw new Error('answer is required; pass the decision the run should use')
|
|
3078
|
+
if (!parent || typeof parent !== 'object') throw new Error('superagent requires a live parent agent')
|
|
3079
|
+
return serialize(async () => {
|
|
3080
|
+
const rt = runtime.get(id)
|
|
3081
|
+
const run = rt?.run ?? await loadRun(id)
|
|
3082
|
+
assertResumeOwner(run, parent)
|
|
3083
|
+
const picked = selectAttention(run, attentionId, value)
|
|
3084
|
+
const at = new Date(now()).toISOString()
|
|
3085
|
+
picked.item.answer = value
|
|
3086
|
+
picked.item.answeredAt = at
|
|
3087
|
+
const entry = pushFollowup(run, {
|
|
3088
|
+
kind: 'answer',
|
|
3089
|
+
attentionId: picked.id,
|
|
3090
|
+
text: `Answer to question ${picked.id} ("${picked.item.question}"): ${value}. `
|
|
3091
|
+
+ 'Apply this decision and continue the work.',
|
|
3092
|
+
})
|
|
3093
|
+
await save(run)
|
|
3094
|
+
|
|
3095
|
+
if (ACTIVE_STATUSES.has(run.status)) {
|
|
3096
|
+
return { run, attention: picked.item, attentionId: picked.id, followup: entry, resumed: false, effect: followupEffect(run) }
|
|
3097
|
+
}
|
|
3098
|
+
try {
|
|
3099
|
+
const resumed = await resumeLocked(id, parent, clientTimeZone, { reopen: true })
|
|
3100
|
+
return { run: resumed, attention: picked.item, attentionId: picked.id, followup: entry, resumed: true, effect: followupEffect(resumed) }
|
|
3101
|
+
} catch (error) {
|
|
3102
|
+
// The answer is durable; only the restart failed. Saying so is better
|
|
3103
|
+
// than pretending the decision was applied or losing it.
|
|
3104
|
+
return {
|
|
3105
|
+
run: (await store.read(id)) ?? run,
|
|
3106
|
+
attention: picked.item,
|
|
3107
|
+
attentionId: picked.id,
|
|
3108
|
+
followup: entry,
|
|
3109
|
+
resumed: false,
|
|
3110
|
+
reason: error?.message ?? String(error),
|
|
3111
|
+
}
|
|
3112
|
+
}
|
|
3113
|
+
})
|
|
3114
|
+
}
|
|
3115
|
+
|
|
3116
|
+
/**
|
|
3117
|
+
* Find the question an answer belongs to, and refuse the three ways an answer
|
|
3118
|
+
* can miss: an id that was never asked, a question retired by a resume, and a
|
|
3119
|
+
* question already answered. A free-form answer is allowed — options are
|
|
3120
|
+
* suggestions — but an answer that matches another question's option is
|
|
3121
|
+
* treated as a misdirected answer and named as such.
|
|
3122
|
+
*
|
|
3123
|
+
* @param {object} run - run record.
|
|
3124
|
+
* @param {unknown} requested - `attention_id` as given.
|
|
3125
|
+
* @param {string} value - the answer text.
|
|
3126
|
+
* @returns {{ item: object, id: string }} the matched question.
|
|
3127
|
+
*/
|
|
3128
|
+
function selectAttention(run, requested, value) {
|
|
3129
|
+
const items = Array.isArray(run.attention?.items) ? run.attention.items : []
|
|
3130
|
+
const idOf = (item, index) => item?.id ?? `q${index + 1}`
|
|
3131
|
+
const idIn = item => idOf(item, items.indexOf(item))
|
|
3132
|
+
const open = items.filter(item => !item.answer)
|
|
3133
|
+
const wanted = String(requested ?? '').trim()
|
|
3134
|
+
const alreadyAnswered = item => `question ${idIn(item)} was already answered at ${item.answeredAt}: `
|
|
3135
|
+
+ `${item.answer}. Answering again would replace a decision the run already acted on.`
|
|
3136
|
+
const describeOpen = () => (open.length === 0
|
|
3137
|
+
? 'this run is not waiting for an answer'
|
|
3138
|
+
: `open question(s): ${open.map(idIn).join(', ')}`)
|
|
3139
|
+
|
|
3140
|
+
if (!wanted) {
|
|
3141
|
+
// An id is optional only when there is exactly one question it could mean.
|
|
3142
|
+
if (open.length === 1) return { item: open[0], id: idIn(open[0]) }
|
|
3143
|
+
if (open.length > 1) {
|
|
3144
|
+
throw new Error(
|
|
3145
|
+
`run ${run.id} has ${open.length} open questions, so an answer needs attention_id to say which one: `
|
|
3146
|
+
+ `${open.map(idIn).join(', ')}.`,
|
|
3147
|
+
)
|
|
3148
|
+
}
|
|
3149
|
+
if (items.length > 0) throw new Error(alreadyAnswered(items[0]))
|
|
3150
|
+
throw new Error(`run ${run.id} is not waiting for an answer (${describeOpen()}).`)
|
|
3151
|
+
}
|
|
3152
|
+
|
|
3153
|
+
const match = open.find(item => idIn(item) === wanted)
|
|
3154
|
+
if (match) return { item: match, id: wanted }
|
|
3155
|
+
const answeredItem = items.find(item => idIn(item) === wanted)
|
|
3156
|
+
if (answeredItem) throw new Error(alreadyAnswered(answeredItem))
|
|
3157
|
+
|
|
3158
|
+
for (const batch of Array.isArray(run.attentionHistory) ? run.attentionHistory : []) {
|
|
3159
|
+
const retired = (batch.items ?? []).find((item, position) => idOf(item, position) === wanted)
|
|
3160
|
+
if (!retired) continue
|
|
3161
|
+
if (retired.answer) {
|
|
3162
|
+
throw new Error(
|
|
3163
|
+
`question ${wanted} was already answered at ${retired.answeredAt}: ${retired.answer}. `
|
|
3164
|
+
+ 'It is no longer open, so there is nothing to change.',
|
|
3165
|
+
)
|
|
3166
|
+
}
|
|
3167
|
+
throw new Error(
|
|
3168
|
+
`question ${wanted} expired at ${batch.expiredAt}: the run was resumed after asking it, so that question no `
|
|
3169
|
+
+ 'longer blocks anything and an answer now would attach to work that is not waiting for it. '
|
|
3170
|
+
+ 'Read superagent_status for the current question.',
|
|
3171
|
+
)
|
|
3172
|
+
}
|
|
3173
|
+
|
|
3174
|
+
// The id is wrong, but the text is exactly another question's option: that
|
|
3175
|
+
// is a misdirected answer, and naming the right id is more useful than
|
|
3176
|
+
// "no such question".
|
|
3177
|
+
const misdirected = items.findIndex((item, position) => idOf(item, position) !== wanted
|
|
3178
|
+
&& (item.options ?? []).some(option => String(option).trim().toLowerCase() === value.toLowerCase()))
|
|
3179
|
+
if (misdirected >= 0) {
|
|
3180
|
+
const other = items[misdirected]
|
|
3181
|
+
throw new Error(
|
|
3182
|
+
`"${value}" is one of the options of question ${idOf(other, misdirected)} ("${other.question}"), not of `
|
|
3183
|
+
+ `${wanted}. Pass attention_id="${idOf(other, misdirected)}" if that is the question you mean.`,
|
|
3184
|
+
)
|
|
3185
|
+
}
|
|
3186
|
+
throw new Error(`no question ${wanted} on run ${run.id}; ${describeOpen()}.`)
|
|
3187
|
+
}
|
|
3188
|
+
|
|
1364
3189
|
async function cancel(runId) {
|
|
3190
|
+
// A run that never got a slot has no owner process to ask: remove it from
|
|
3191
|
+
// the line here, so no timer can start it afterwards.
|
|
3192
|
+
const queuedIndex = queue.findIndex(entry => entry.run.id === runId)
|
|
3193
|
+
if (queuedIndex >= 0) {
|
|
3194
|
+
const entry = queue.splice(queuedIndex, 1)[0]
|
|
3195
|
+
if (entry.timer) clearTimer(entry.timer)
|
|
3196
|
+
if (queue.length === 0) stopQueueTicker()
|
|
3197
|
+
const run = entry.run
|
|
3198
|
+
if (isWaiting(run)) {
|
|
3199
|
+
run.queue = { ...run.queue, waiting: false, cancelled: true }
|
|
3200
|
+
run.status = 'cancelled'
|
|
3201
|
+
run.error = 'cancelled before a free slot was available; nothing was spent'
|
|
3202
|
+
run.finishedAt = new Date(now()).toISOString()
|
|
3203
|
+
run.activeMs = Number.isFinite(run.activeMs) ? run.activeMs : 0
|
|
3204
|
+
await save(run)
|
|
3205
|
+
await clearCancel(run.id)
|
|
3206
|
+
await releaseRunLock(run.id)
|
|
3207
|
+
}
|
|
3208
|
+
await refreshQueuePositions().catch(() => {})
|
|
3209
|
+
pumpQueue().catch(() => {})
|
|
3210
|
+
return run
|
|
3211
|
+
}
|
|
3212
|
+
|
|
1365
3213
|
const rt = runtime.get(runId)
|
|
1366
3214
|
if (rt && ACTIVE_STATUSES.has(rt.run.status)) {
|
|
1367
3215
|
rt.abortCause = 'cancelled'
|
|
@@ -1460,5 +3308,5 @@ export function createOrchestrator(deps) {
|
|
|
1460
3308
|
await Promise.all([...runtime.values()].map(rt => rt.promise?.catch(() => {})))
|
|
1461
3309
|
}
|
|
1462
3310
|
|
|
1463
|
-
return { recover, start, resume, cancel, status, list, idle, activeCount, liveRun }
|
|
3311
|
+
return { recover, start, resume, cancel, followup, answer, status, list, idle, activeCount, liveRun }
|
|
1464
3312
|
}
|