agents-relay 2.0.207 → 2.0.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.js +2 -1
- package/dist/agent-shortcut.js +113 -41
- package/dist/audit-tail.js +52 -0
- package/dist/browser-worker-runtime.js +7 -2
- package/dist/chatgpt-platform-fallback.js +30 -0
- package/dist/cli-shortcut-local.js +49 -0
- package/dist/cli-shortcut.js +81 -19
- package/dist/cli.js +8 -3
- package/dist/decision-audit.js +143 -1
- package/dist/execution-routing.js +8 -2
- package/dist/force-restate-validation.js +3 -1
- package/dist/job-execution-slot.js +4 -1
- package/dist/model-routing.js +8 -0
- package/dist/reconciler-review.js +39 -13
- package/dist/reconciler-runtime.js +46 -52
- package/dist/reconciler.js +12 -2
- package/dist/relay-job-snapshot-reads.js +5 -2
- package/dist/relay-workflow-operations.js +1 -1
- package/dist/relayd.js +1 -1
- package/dist/route-outcome-recording.js +51 -0
- package/dist/runtime-status.js +11 -4
- package/dist/shortcut-node-worker.js +128 -0
- package/dist/shortcut-stage-contract.js +50 -0
- package/dist/shortcut-worker-runner.js +73 -0
- package/dist/task-actions.js +4 -13
- package/dist/task-inactivity-watchdog.js +7 -0
- package/dist/terminal-event-recovery.js +13 -0
- package/dist/worker-failure-recovery.js +89 -0
- package/dist/workflow/contracts.js +4 -0
- package/dist/workflow/definitions-v2.js +2 -0
- package/dist/workflow/runtime.js +30 -0
- package/dist/workflow/task-resolution.js +1 -4
- package/package.json +4 -3
- package/skills/agents-relay/agents/planner.agent.md +14 -4
- package/skills/agents-relay/agents/reviewer.agent.md +2 -0
- package/skills/agents-relay/contracts/orchestrator-workflows.md +6 -0
- package/skills/agents-relay/contracts/task-job-contract.md +4 -0
- package/workflow/force-restate-validation.mmd +1 -1
- package/workflow/review-lifecycle.mmd +1 -1
- package/workflow/task-lifecycle.mmd +13 -12
- package/workflow/task-resolution.mmd +1 -6
package/dist/adapters.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { terminalRecoveryPrompt } from './terminal-event-recovery.js';
|
|
1
2
|
import { access, readFile, realpath } from 'node:fs/promises';
|
|
2
3
|
import { relative, resolve, sep, basename } from 'node:path';
|
|
3
4
|
import { DEFAULT_PROMPT_REFERENCE_DIRECTORY } from './agent-prompt-builder.js';
|
|
@@ -298,7 +299,7 @@ function browserWorkerRunner() {
|
|
|
298
299
|
} : { message, temporary: false } }),
|
|
299
300
|
recover: async (threadId) => ({
|
|
300
301
|
thread_id: threadId, mode: 'sync',
|
|
301
|
-
message:
|
|
302
|
+
message: terminalRecoveryPrompt(request.execution?.executionId ?? 'unknown'),
|
|
302
303
|
}),
|
|
303
304
|
}, signal);
|
|
304
305
|
};
|
package/dist/agent-shortcut.js
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
3
|
import { readFile } from 'node:fs/promises';
|
|
4
|
-
import { resolve, relative, isAbsolute, sep } from 'node:path';
|
|
4
|
+
import { resolve, relative, isAbsolute, sep, dirname } from 'node:path';
|
|
5
5
|
import { realpath } from 'node:fs/promises';
|
|
6
6
|
import { once } from 'node:events';
|
|
7
7
|
import { pathToFileURL } from 'node:url';
|
|
8
8
|
import { spawnOwnedProcess } from './owned-process.js';
|
|
9
|
+
import { validateShortcutStageGate } from './shortcut-stage-contract.js';
|
|
9
10
|
import { parseAgentGraph } from './agent-graph.js';
|
|
10
11
|
import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
|
|
11
12
|
import { dispatchJobLifecycleSync } from './workflow/lifecycle.js';
|
|
12
|
-
import { dependencyReady } from './task-transitions.js';
|
|
13
|
+
import { dependencyReady, tryTransitionTaskEvent } from './task-transitions.js';
|
|
13
14
|
const digest = (value) => createHash('sha256').update(value).digest('hex');
|
|
14
15
|
const digestBytes = (value) => createHash('sha256').update(value).digest('hex');
|
|
15
16
|
function canonical(value) {
|
|
@@ -116,7 +117,25 @@ async function discoverShortcut(agent, projectPath, neoRoot) {
|
|
|
116
117
|
throw new Error('Shared shortcut symlink escapes shortcut root');
|
|
117
118
|
return actual;
|
|
118
119
|
}
|
|
120
|
+
export async function missingShortcutNodes(agentGraph, projectPath, neoRoot) {
|
|
121
|
+
const graph = parseAgentGraph(agentGraph);
|
|
122
|
+
const result = await Promise.all(graph.nodes.map(async (node) => ({
|
|
123
|
+
id: node.id, available: Boolean(await discoverShortcut(node.agent, projectPath, neoRoot)),
|
|
124
|
+
})));
|
|
125
|
+
return result.filter(item => !item.available).map(item => item.id);
|
|
126
|
+
}
|
|
119
127
|
function inside(root, path) { const rel = relative(root, path); return rel === '' || (rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel)); }
|
|
128
|
+
function raceAbort(work, signal) {
|
|
129
|
+
if (signal.aborted)
|
|
130
|
+
return Promise.reject(signal.reason ?? new Error('Shortcut execution interrupted'));
|
|
131
|
+
let rejectAbort;
|
|
132
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
133
|
+
rejectAbort = () => reject(signal.reason ?? new Error('Shortcut execution interrupted'));
|
|
134
|
+
signal.addEventListener('abort', rejectAbort, { once: true });
|
|
135
|
+
});
|
|
136
|
+
return Promise.race([work, aborted]).finally(() => { if (rejectAbort)
|
|
137
|
+
signal.removeEventListener('abort', rejectAbort); });
|
|
138
|
+
}
|
|
120
139
|
// Only Vlog's Visioner evidence follows the source-manifest/index artifact schema.
|
|
121
140
|
// Distribution Visioner has a separate domain contract and cannot reuse it.
|
|
122
141
|
function requiresVlogVisionerEvidence(agent) {
|
|
@@ -144,10 +163,10 @@ async function validateVisionerResult(result, projectRoot) {
|
|
|
144
163
|
artifactFiles.set(name, { bytes, parsed: parsed });
|
|
145
164
|
}
|
|
146
165
|
const evidenceFile = artifactFiles.get('vision-evidence.json'), indexFile = artifactFiles.get('vision-segment-index.json'), metadataFile = artifactFiles.get('metadata.json');
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
||
|
|
150
|
-
throw new Error('Visioner evidence
|
|
166
|
+
// Stage authority is the source-linked structure and original media IDs, not repeated checksum validation.
|
|
167
|
+
if (evidenceFile.parsed.status !== 'EVIDENCE_READY_UNREVIEWED' || indexFile.parsed.status !== 'SOURCE_LINKED_UNREVIEWED'
|
|
168
|
+
|| !metadataFile.parsed)
|
|
169
|
+
throw new Error('Visioner structured evidence does not satisfy the Stage contract');
|
|
151
170
|
}
|
|
152
171
|
export async function runAgentShortcut(options) {
|
|
153
172
|
const { store, job, task, projectPath, neoRoot } = options;
|
|
@@ -177,6 +196,10 @@ export async function runAgentShortcut(options) {
|
|
|
177
196
|
throw new Error('Shortcut invocation Task snapshot is stale; reload the durable Task before execution or receipt consumption');
|
|
178
197
|
if (currentJob.executionMode !== 'shortcut' || currentJob.contentOnly !== true)
|
|
179
198
|
throw new Error('Shortcut execution requires durable shortcut and content-only Job policy');
|
|
199
|
+
if (currentJob.shortcutExecutionDeadlineAt && !Number.isFinite(Date.parse(currentJob.shortcutExecutionDeadlineAt)))
|
|
200
|
+
throw new Error('Shortcut execution budget metadata is invalid');
|
|
201
|
+
if (currentJob.shortcutExecutionStartedAt && !Number.isFinite(Date.parse(currentJob.shortcutExecutionStartedAt)))
|
|
202
|
+
throw new Error('Shortcut execution budget metadata is invalid');
|
|
180
203
|
const existingShortcut = currentTask.result?.data?.shortcut;
|
|
181
204
|
if (existingShortcut?.workerFallback === true)
|
|
182
205
|
throw new Error('Task is durably routed to the original Worker; shortcut execution is not authorized');
|
|
@@ -185,6 +208,13 @@ export async function runAgentShortcut(options) {
|
|
|
185
208
|
if (currentTask.state === 'QUEUED' && !dependencyReady(currentTask, new Map(currentJob.tasks.map(item => [item.id, item]))))
|
|
186
209
|
throw new Error(`Task ${task.id} dependencies are not satisfied`);
|
|
187
210
|
let state = checkpoint(currentTask);
|
|
211
|
+
// A Task Lifecycle-authorized retry retains verified completed nodes and
|
|
212
|
+
// execution history, but its previous attempt's terminal verdict must not
|
|
213
|
+
// contaminate the new run's eventual success receipt.
|
|
214
|
+
if (currentTask.state === 'QUEUED' && currentTask.attempt > 0 &&
|
|
215
|
+
currentTask.executionId === null && currentTask.leaseOwner === null &&
|
|
216
|
+
'terminal' in state)
|
|
217
|
+
delete state.terminal;
|
|
188
218
|
if (state.graphHash && (state.version !== 1 || state.graphHash !== graphHash || state.inputHash !== inputHash || state.shortcutHash !== shortcutHash))
|
|
189
219
|
throw new Error('Shortcut checkpoint fingerprint changed; refusing to reuse prior node output');
|
|
190
220
|
state.version = 1;
|
|
@@ -218,6 +248,9 @@ export async function runAgentShortcut(options) {
|
|
|
218
248
|
throw new Error(`Checkpoint artifact has no completed node: ${nodeId}`);
|
|
219
249
|
}
|
|
220
250
|
if (currentTask.state === 'QUEUED') {
|
|
251
|
+
const launch = tryTransitionTaskEvent(currentTask, 'dispatch', { dispatchable: true, shortcutExecution: true, shortcutFallback: false, shortcutHasGraph: true, shortcutDirectInvocation: true });
|
|
252
|
+
if (!launch.accepted || !launch.actions.includes('effect.launchShortcut'))
|
|
253
|
+
throw new Error('Task Lifecycle graph did not select direct shortcut execution');
|
|
221
254
|
await store.mutateJob(job.id, fresh => {
|
|
222
255
|
const t = fresh.tasks.find(item => item.id === task.id);
|
|
223
256
|
if (fresh.state !== 'ACTIVE' || fresh.pause || !t || t.state !== 'QUEUED' || t.executionId !== null || t.agentGraph !== task.agentGraph || t.input !== task.input
|
|
@@ -227,6 +260,10 @@ export async function runAgentShortcut(options) {
|
|
|
227
260
|
t.executionId = randomUUID();
|
|
228
261
|
t.leaseOwner = owner;
|
|
229
262
|
t.leaseExpiresAt = new Date(Date.now() + leaseMs).toISOString();
|
|
263
|
+
if (!fresh.shortcutExecutionStartedAt)
|
|
264
|
+
fresh.shortcutExecutionStartedAt = new Date().toISOString();
|
|
265
|
+
if (!fresh.shortcutExecutionDeadlineAt)
|
|
266
|
+
fresh.shortcutExecutionDeadlineAt = new Date(Date.parse(fresh.shortcutExecutionStartedAt) + 20 * 60 * 1000).toISOString();
|
|
230
267
|
t.executionHistory = [...(t.executionHistory ?? []), { id: t.executionId, attempt: t.attempt, startedAt: new Date().toISOString() }].slice(-8);
|
|
231
268
|
return fresh;
|
|
232
269
|
});
|
|
@@ -285,6 +322,8 @@ export async function runAgentShortcut(options) {
|
|
|
285
322
|
const nodeAgent = graph.nodes.find(node => node.id === nodeId)?.agent;
|
|
286
323
|
if (requiresVlogVisionerEvidence(nodeAgent?.toLowerCase()))
|
|
287
324
|
await validateVisionerResult(output, projectReal);
|
|
325
|
+
if (nodeAgent && task.output?.kind === 'file')
|
|
326
|
+
await validateShortcutStageGate({ projectPath: projectReal, agent: nodeAgent, runPath: dirname(task.output.path), execute: (args, cwd) => runCommand(args, cwd, new AbortController().signal) });
|
|
288
327
|
state.nodes[nodeId] = structuredClone(output);
|
|
289
328
|
state.artifactHashes[nodeId] = artifactHashes;
|
|
290
329
|
delete state.waits[nodeId];
|
|
@@ -292,7 +331,28 @@ export async function runAgentShortcut(options) {
|
|
|
292
331
|
}
|
|
293
332
|
if (Object.keys(state.nodes).length > Object.keys(checkpoint(currentTask).nodes).length)
|
|
294
333
|
await persist(state);
|
|
334
|
+
currentJob = await store.load(job.id);
|
|
335
|
+
if (!currentJob.shortcutExecutionStartedAt)
|
|
336
|
+
throw new Error('Direct shortcut Job has no durable first-execution timestamp; refusing to start or reset its budget');
|
|
337
|
+
const startedAt = Date.parse(currentJob.shortcutExecutionStartedAt);
|
|
338
|
+
const deadlineAt = currentJob.shortcutExecutionDeadlineAt ? Date.parse(currentJob.shortcutExecutionDeadlineAt) : startedAt + 20 * 60 * 1000;
|
|
339
|
+
if (!Number.isFinite(startedAt) || !Number.isFinite(deadlineAt) || deadlineAt !== startedAt + 20 * 60 * 1000)
|
|
340
|
+
throw new Error('Shortcut execution budget metadata is invalid');
|
|
341
|
+
if (!currentJob.shortcutExecutionDeadlineAt) {
|
|
342
|
+
await store.mutateJob(job.id, fresh => {
|
|
343
|
+
if (fresh.shortcutExecutionStartedAt !== currentJob.shortcutExecutionStartedAt)
|
|
344
|
+
throw new Error('Shortcut execution budget origin changed');
|
|
345
|
+
fresh.shortcutExecutionDeadlineAt = new Date(deadlineAt).toISOString();
|
|
346
|
+
return fresh;
|
|
347
|
+
});
|
|
348
|
+
}
|
|
295
349
|
const executionAbort = new AbortController();
|
|
350
|
+
const abortFromCaller = () => executionAbort.abort(options.signal?.reason ?? new Error('Shortcut execution interrupted'));
|
|
351
|
+
if (options.signal?.aborted)
|
|
352
|
+
abortFromCaller();
|
|
353
|
+
else
|
|
354
|
+
options.signal?.addEventListener('abort', abortFromCaller, { once: true });
|
|
355
|
+
const budgetTimer = setTimeout(() => executionAbort.abort(new Error('Shortcut execution budget exceeded (20 minutes)')), Math.max(0, deadlineAt - Date.now()));
|
|
296
356
|
let leaseHeartbeatError;
|
|
297
357
|
let heartbeatInFlight = false;
|
|
298
358
|
const heartbeat = setInterval(() => {
|
|
@@ -309,18 +369,17 @@ export async function runAgentShortcut(options) {
|
|
|
309
369
|
}).catch(error => { leaseHeartbeatError = error; executionAbort.abort(error); }).finally(() => { heartbeatInFlight = false; });
|
|
310
370
|
}, heartbeatMs);
|
|
311
371
|
heartbeat.unref();
|
|
312
|
-
const stopHeartbeat = () => clearInterval(heartbeat);
|
|
313
|
-
|
|
314
|
-
let shortcutModuleEntered = false;
|
|
372
|
+
const stopHeartbeat = () => { clearInterval(heartbeat); clearTimeout(budgetTimer); options.signal?.removeEventListener('abort', abortFromCaller); };
|
|
373
|
+
const emit = (event) => options.onNodeEvent?.({ ...event, timestamp: new Date().toISOString() });
|
|
315
374
|
try {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
//
|
|
375
|
+
if (Date.now() >= deadlineAt)
|
|
376
|
+
return await fail('Shortcut execution budget exceeded (20 minutes)');
|
|
377
|
+
// The existing Graph executor owns node readiness for both code and model
|
|
378
|
+
// nodes. Model nodes execute using Relay's existing durable Worker adapter,
|
|
379
|
+
// never by routing the entire Agent Graph to one Worker.
|
|
321
380
|
const missing = shortcuts.filter(item => !item.path);
|
|
322
|
-
if (missing.length)
|
|
323
|
-
return await
|
|
381
|
+
if (missing.length && !options.runModelNode)
|
|
382
|
+
return await fail(`Incomplete Agent Graph shortcut coverage: ${missing.map(item => item.nodeId).join(', ')}`);
|
|
324
383
|
const incoming = new Map(graph.nodes.map(node => [node.id, graph.edges.filter(edge => edge.to === node.id).map(edge => edge.from)]));
|
|
325
384
|
const max = Math.max(1, Math.min(8, options.concurrency ?? 4));
|
|
326
385
|
const shortcutByNode = new Map(shortcuts.map(item => [item.nodeId, item.path]));
|
|
@@ -328,27 +387,29 @@ export async function runAgentShortcut(options) {
|
|
|
328
387
|
const ready = wave.filter(id => !state.nodes[id] && !state.waits[id] && (incoming.get(id) ?? []).every(parent => Boolean(state.nodes[parent])));
|
|
329
388
|
for (let offset = 0; offset < ready.length; offset += max) {
|
|
330
389
|
const batch = ready.slice(offset, offset + max);
|
|
331
|
-
for (const id of batch)
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
390
|
+
for (const id of batch) {
|
|
391
|
+
state.inFlight[id] = currentTask.executionId;
|
|
392
|
+
emit({ nodeId: id, type: 'started', data: { executionId: currentTask.executionId, agent: graph.nodes.find(node => node.id === id)?.agent } });
|
|
393
|
+
}
|
|
394
|
+
if (batch.length)
|
|
335
395
|
await persist(state);
|
|
336
396
|
const outcomes = await Promise.all(batch.map(async (id) => {
|
|
337
397
|
const node = graph.nodes.find(item => item.id === id);
|
|
338
398
|
const shortcut = shortcutByNode.get(id) ?? null;
|
|
339
|
-
if (!shortcut)
|
|
340
|
-
return { id, value: null };
|
|
341
399
|
try {
|
|
342
|
-
shortcutModuleEntered = true;
|
|
343
|
-
const module = await import(pathToFileURL(shortcut).href);
|
|
344
400
|
const predecessorOutputs = Object.fromEntries((incoming.get(id) ?? []).map(parent => [parent, state.nodes[parent]?.output]));
|
|
345
401
|
const trackedRunCommand = async (args, cwd) => {
|
|
346
402
|
if (executionAbort.signal.aborted)
|
|
347
403
|
throw new Error(`Shortcut execution was cancelled: ${String(leaseHeartbeatError ?? 'lease lost')}`);
|
|
348
|
-
effectsStarted = true;
|
|
349
404
|
return runCommand(args, cwd, executionAbort.signal);
|
|
350
405
|
};
|
|
351
|
-
|
|
406
|
+
emit({ nodeId: id, type: 'progress', data: { phase: shortcut ? 'shortcut.invoke' : 'worker.invoke' } });
|
|
407
|
+
const raw = await raceAbort(shortcut
|
|
408
|
+
? (async () => {
|
|
409
|
+
const module = await import(pathToFileURL(shortcut).href);
|
|
410
|
+
return module.run({ input: { ...taskShortcutInput(task.input), projectPath, jobId: job.id, taskId: task.id, nodeId: id, agent: node.agent, predecessorOutputs }, signal: executionAbort.signal, runCommand: trackedRunCommand });
|
|
411
|
+
})()
|
|
412
|
+
: options.runModelNode({ id, agent: node.agent, predecessorOutputs, executionId: currentTask.executionId, signal: executionAbort.signal }), executionAbort.signal);
|
|
352
413
|
if (leaseHeartbeatError)
|
|
353
414
|
throw new Error(`Shortcut lease renewal failed: ${String(leaseHeartbeatError)}`);
|
|
354
415
|
if (!raw || typeof raw !== 'object')
|
|
@@ -367,6 +428,8 @@ export async function runAgentShortcut(options) {
|
|
|
367
428
|
const projectReal = await realpath(projectPath);
|
|
368
429
|
if (requiresVlogVisionerEvidence(node.agent.toLowerCase()))
|
|
369
430
|
await validateVisionerResult(result, projectReal);
|
|
431
|
+
if (task.output?.kind === 'file')
|
|
432
|
+
await validateShortcutStageGate({ projectPath: projectReal, agent: node.agent, runPath: dirname(task.output.path), execute: trackedRunCommand });
|
|
370
433
|
const artifactHashes = {};
|
|
371
434
|
for (const artifact of artifactPaths) {
|
|
372
435
|
if (!artifact)
|
|
@@ -390,26 +453,33 @@ export async function runAgentShortcut(options) {
|
|
|
390
453
|
return { id, value: null, error: error instanceof Error ? error.message : String(error) };
|
|
391
454
|
}
|
|
392
455
|
}));
|
|
456
|
+
const failed = outcomes.find(result => !result.value);
|
|
393
457
|
for (const result of outcomes) {
|
|
394
|
-
if (!result.value)
|
|
395
|
-
|
|
458
|
+
if (!result.value) {
|
|
459
|
+
delete state.inFlight[result.id];
|
|
460
|
+
continue;
|
|
461
|
+
}
|
|
396
462
|
if (result.value.status === 'WAITING_EVENT') {
|
|
397
463
|
const evt = result.value.event;
|
|
398
464
|
if (evt.kind !== 'nonhuman_result')
|
|
399
|
-
return await
|
|
465
|
+
return await fail('Shortcut event wait must be a nonhuman result; human approval must use Relay hostInvocation');
|
|
400
466
|
if (typeof evt.id !== 'string' || typeof evt.revision !== 'string' || typeof evt.contentHash !== 'string'
|
|
401
467
|
|| (evt.nodeId !== undefined && evt.nodeId !== result.id) || !/^[0-9a-f]{64}$/i.test(evt.contentHash))
|
|
402
|
-
return await
|
|
468
|
+
return await fail(`Invalid event wait from ${result.id}`);
|
|
403
469
|
delete state.inFlight[result.id];
|
|
404
470
|
state.waits[result.id] = { kind: 'nonhuman_result', id: evt.id, revision: evt.revision, contentHash: evt.contentHash, executionId: currentTask.executionId };
|
|
471
|
+
emit({ nodeId: result.id, type: 'waiting', data: { waitId: evt.id, revision: evt.revision, executionId: currentTask.executionId } });
|
|
405
472
|
}
|
|
406
473
|
else {
|
|
407
474
|
delete state.inFlight[result.id];
|
|
408
475
|
state.nodes[result.id] = result.value;
|
|
409
476
|
state.artifactHashes[result.id] = result.artifactHashes ?? {};
|
|
477
|
+
emit({ nodeId: result.id, type: 'completed', data: { executionId: currentTask.executionId, output: result.value.output } });
|
|
410
478
|
}
|
|
411
479
|
}
|
|
412
480
|
await persist(state);
|
|
481
|
+
if (failed)
|
|
482
|
+
return await fail(`Shortcut node ${failed.id} failed: ${failed.error ?? 'no valid result'}`);
|
|
413
483
|
}
|
|
414
484
|
}
|
|
415
485
|
if (Object.keys(state.waits).length) {
|
|
@@ -456,25 +526,27 @@ export async function runAgentShortcut(options) {
|
|
|
456
526
|
}
|
|
457
527
|
async function findTask() { const fresh = await store.load(job.id); const found = fresh.tasks.find(item => item.id === task.id); if (!found)
|
|
458
528
|
throw new Error('Task disappeared'); return found; }
|
|
459
|
-
async function
|
|
460
|
-
const
|
|
529
|
+
async function fail(reason) {
|
|
530
|
+
const timedOut = reason.includes('budget exceeded') || Date.now() >= deadlineAt;
|
|
531
|
+
const interrupted = options.signal?.aborted === true && !timedOut;
|
|
532
|
+
const failure = timedOut ? `Shortcut execution timed out after 20 minutes: ${reason}` : interrupted ? `Shortcut execution interrupted: ${reason}` : reason;
|
|
533
|
+
for (const nodeId of Object.keys(state.inFlight ?? {}))
|
|
534
|
+
emit({ nodeId, type: 'failed', data: { executionId: currentTask.executionId, reason: failure } });
|
|
461
535
|
stopHeartbeat();
|
|
462
536
|
const result = await transitions.apply(job.id, task.id, 'shortcut.failed', {
|
|
463
537
|
eligible: (_j, t) => t.state === 'RUNNING' && t.leaseOwner === owner,
|
|
464
|
-
|
|
465
|
-
mutate: (_j, t, decision) => {
|
|
466
|
-
const workerFallback = decision.actions.includes('mutation.markShortcutWorkerFallback');
|
|
467
|
-
const failure = workerFallback ? reason : `Shortcut stopped after possible partial effects; Troubleshooter recovery required: ${reason}`;
|
|
538
|
+
mutate: (_j, t) => {
|
|
468
539
|
t.error = failure;
|
|
540
|
+
t.leaseOwner = null;
|
|
541
|
+
t.leaseExpiresAt = null;
|
|
542
|
+
t.executionId = null;
|
|
469
543
|
const prior = t.result ?? { summary: 'Shortcut execution failed', data: {} };
|
|
470
|
-
t.result = { ...prior, data: { ...(prior.data ?? {}), shortcut: { ...state,
|
|
544
|
+
t.result = { ...prior, summary: failure, data: { ...(prior.data ?? {}), shortcut: { ...state, terminal: { status: timedOut ? 'TIMED_OUT' : interrupted ? 'INTERRUPTED' : 'FAILED', reason: failure, at: new Date().toISOString(), executionId: currentTask.executionId } } } };
|
|
471
545
|
},
|
|
472
546
|
});
|
|
473
547
|
if (!result?.decision.accepted)
|
|
474
|
-
throw new Error(`Task lifecycle graph rejected shortcut failure: ${
|
|
475
|
-
|
|
476
|
-
const failure = result.task.error ?? reason;
|
|
477
|
-
return { outcome: { status: 'FAILED', reason: failure }, task: await findTask(), fallback: workerFallback };
|
|
548
|
+
throw new Error(`Task lifecycle graph rejected shortcut failure: ${failure}`);
|
|
549
|
+
return { outcome: { status: 'FAILED', reason: failure }, task: await findTask(), fallback: false };
|
|
478
550
|
}
|
|
479
551
|
}
|
|
480
552
|
export function shortcutResumeEvent(_value) {
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { open } from 'node:fs/promises';
|
|
2
|
+
/** Reverse-read newline-delimited records without loading the complete audit file. */
|
|
3
|
+
export async function* readJsonlNewestFirst(path) {
|
|
4
|
+
let file;
|
|
5
|
+
try {
|
|
6
|
+
file = await open(path, 'r');
|
|
7
|
+
}
|
|
8
|
+
catch (error) {
|
|
9
|
+
if (error.code === 'ENOENT')
|
|
10
|
+
return;
|
|
11
|
+
throw error;
|
|
12
|
+
}
|
|
13
|
+
try {
|
|
14
|
+
const { size } = await file.stat();
|
|
15
|
+
let offset = size;
|
|
16
|
+
let prefix = Buffer.alloc(0);
|
|
17
|
+
while (offset > 0) {
|
|
18
|
+
const length = Math.min(64 * 1024, offset);
|
|
19
|
+
offset -= length;
|
|
20
|
+
const chunk = Buffer.allocUnsafe(length);
|
|
21
|
+
const { bytesRead } = await file.read(chunk, 0, length, offset);
|
|
22
|
+
if (bytesRead !== length)
|
|
23
|
+
throw new Error('Short reverse audit read');
|
|
24
|
+
const block = Buffer.concat([chunk, prefix]);
|
|
25
|
+
let end = block.length;
|
|
26
|
+
for (let i = block.length - 1; i >= 0; i--) {
|
|
27
|
+
if (block[i] !== 10)
|
|
28
|
+
continue;
|
|
29
|
+
const line = block.subarray(i + 1, end);
|
|
30
|
+
end = i;
|
|
31
|
+
if (!line.length)
|
|
32
|
+
continue;
|
|
33
|
+
try {
|
|
34
|
+
yield JSON.parse(line.toString('utf8'));
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
// A concurrent writer may leave an incomplete trailing line. Ignore it.
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
prefix = block.subarray(0, end);
|
|
41
|
+
}
|
|
42
|
+
if (prefix.length) {
|
|
43
|
+
try {
|
|
44
|
+
yield JSON.parse(prefix.toString('utf8'));
|
|
45
|
+
}
|
|
46
|
+
catch { }
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
finally {
|
|
50
|
+
await file.close();
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
|
+
import { terminalRecoveryEligible } from './terminal-event-recovery.js';
|
|
3
4
|
import { spawnOwnedProcess } from './owned-process.js';
|
|
4
5
|
import { BrowserWorkerPoolManager, BrowserPoolAdmissionError } from './browser-worker-pool.js';
|
|
5
6
|
import { monitorPoolDecision, progressPoolDecision } from './browser-worker-pool-graph.js';
|
|
@@ -179,8 +180,12 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
179
180
|
// after its last_used_at expires without falsely freeing the Relay slot.
|
|
180
181
|
void stop().catch(() => { });
|
|
181
182
|
};
|
|
183
|
+
const canRecoverTerminal = () => terminalRecoveryEligible({
|
|
184
|
+
hasSession: !!sessionId, hasThread: !!threadId, terminalObserved: false, recoveryAttempts: recoveries,
|
|
185
|
+
outcomeUncertain: recoveryOutcomeUncertain, cancelled: stopped || released || uncertain,
|
|
186
|
+
});
|
|
182
187
|
const sendRecovery = async () => {
|
|
183
|
-
if (
|
|
188
|
+
if (!request.recover || !sessionId || !threadId || !canRecoverTerminal())
|
|
184
189
|
return;
|
|
185
190
|
recoveries++;
|
|
186
191
|
const cfg = await request.recover(threadId);
|
|
@@ -225,7 +230,7 @@ export function pooledWebHelmRun(request, signal) {
|
|
|
225
230
|
await request.onActivity?.('Browser Worker produced new assistant output');
|
|
226
231
|
}
|
|
227
232
|
const due = now - observedAt >= Math.max(request.recoveryMinMs ?? 60_000, Math.round(request.timeoutMs * .6)) &&
|
|
228
|
-
!observation.state?.generating &&
|
|
233
|
+
!observation.state?.generating && canRecoverTerminal() && !!request.recover;
|
|
229
234
|
if (monitorPoolDecision(due) === 'recoverTurn' && request.recover && threadId) {
|
|
230
235
|
// Count before the send. An ambiguous response must never cause automatic retransmission.
|
|
231
236
|
observedAt = now;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { eligibleWorkerCandidates, workerCandidates } from './worker-routing.js';
|
|
2
2
|
import { routeExecution } from './execution-routing.js';
|
|
3
|
+
import { configuredCodexDefault } from './model-routing.js';
|
|
3
4
|
import { eventFor } from './events.js';
|
|
4
5
|
export function codexFallbackRouting(task, adapters, registered, usage) {
|
|
5
6
|
const requirements = task.workerRequirements ?? { capabilities: [] };
|
|
@@ -54,3 +55,32 @@ export function persistChatGptPlatformFailure(task, original, executionId, class
|
|
|
54
55
|
}];
|
|
55
56
|
}
|
|
56
57
|
}
|
|
58
|
+
/** One deterministic, verified configured-default Codex route for an
|
|
59
|
+
* unstarted failed ChatGPT/Z Worker generation. Never use catalog defaults. */
|
|
60
|
+
export async function codexNoStartFallbackRoute(task, adapters, registered, usage) {
|
|
61
|
+
if ((!['chatgpt', 'z'].includes(task.adapter) && !task.noStartFallback) || task.chatgptThreadDelivery)
|
|
62
|
+
return null;
|
|
63
|
+
const configured = await configuredCodexDefault();
|
|
64
|
+
if (!configured)
|
|
65
|
+
return null;
|
|
66
|
+
const requirements = task.workerRequirements ?? { capabilities: [] };
|
|
67
|
+
const { candidates, excludedRoutes } = codexFallbackRouting(task, adapters, registered, usage);
|
|
68
|
+
if (!candidates.length)
|
|
69
|
+
return null;
|
|
70
|
+
try {
|
|
71
|
+
const decision = await routeExecution(candidates, requirements, {
|
|
72
|
+
registered, usage, excludedRoutes, decisionMode: 'off',
|
|
73
|
+
routingConstraints: { provider: configured.provider, model: configured.model },
|
|
74
|
+
taskFacts: { task: { id: task.id, fallback: 'no-start' } },
|
|
75
|
+
});
|
|
76
|
+
if (decision.candidate.adapter !== 'codex'
|
|
77
|
+
|| decision.candidate.routing.provider !== configured.provider
|
|
78
|
+
|| decision.candidate.routing.model !== configured.model
|
|
79
|
+
|| decision.model.profile !== undefined)
|
|
80
|
+
return null;
|
|
81
|
+
return decision;
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { DisabledGitHubClient } from './no-github.js';
|
|
2
|
+
import { ensureManagedGitHubJob } from './cli-job-create.js';
|
|
3
|
+
import { workerTaskFromArgs } from './cli-worker-task.js';
|
|
4
|
+
import { arg, hasArg, repository, storeFor, textOption } from './cli-support.js';
|
|
5
|
+
import { relayStoreDirectory } from './relay-store.js';
|
|
6
|
+
/** Private foreground shortcut bootstrap. This must never contact relayd.
|
|
7
|
+
* Only content-only Shortcut Jobs use this path; the conventional CLI stays
|
|
8
|
+
* daemon-owned. Atomic SQLite records remain authoritative for both paths. */
|
|
9
|
+
export async function runLocalShortcutBinding(kind, action, args) {
|
|
10
|
+
if (kind === 'job' && action === 'create') {
|
|
11
|
+
if (!hasArg(args, '--shortcut-execution') || hasArg(args, '--initial-plan'))
|
|
12
|
+
throw new Error('local shortcut create requires --shortcut-execution without --initial-plan');
|
|
13
|
+
const objective = await textOption(args, '--objective', '--objective-file');
|
|
14
|
+
if (!objective.trim())
|
|
15
|
+
throw new Error('local shortcut create requires objective');
|
|
16
|
+
const { job } = await ensureManagedGitHubJob(new DisabledGitHubClient(), repository(args), new Set(), {
|
|
17
|
+
title: arg(args, '--title') || undefined, objective,
|
|
18
|
+
projectPath: arg(args, '--project-path') || undefined,
|
|
19
|
+
executionMode: 'shortcut', storeDirectory: arg(args, '--store-dir') || relayStoreDirectory(),
|
|
20
|
+
noGitHub: true,
|
|
21
|
+
});
|
|
22
|
+
console.log(JSON.stringify({ repository: job.repository, job: job.id, state: job.state }));
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
if (kind === 'job' && action === 'show' || kind === 'task' && action === 'list') {
|
|
26
|
+
const { job } = await storeFor(args);
|
|
27
|
+
if (job.repository !== repository(args) || job.executionMode !== 'shortcut' || job.contentOnly !== true)
|
|
28
|
+
throw new Error('local shortcut Job identity mismatch');
|
|
29
|
+
console.log(JSON.stringify(kind === 'job' ? job : job.tasks, null, 2));
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (kind === 'task' && action === 'create') {
|
|
33
|
+
const { store, job, localFile } = await storeFor(args);
|
|
34
|
+
if (localFile !== null || !store.mutateJob || job.repository !== repository(args) || job.executionMode !== 'shortcut' || job.contentOnly !== true || job.state !== 'DRAFT')
|
|
35
|
+
throw new Error('local shortcut Task creation requires a DRAFT content-only SQLite Job');
|
|
36
|
+
const task = await workerTaskFromArgs(job, args, new Date().toISOString());
|
|
37
|
+
if (task.id !== arg(args, '--task-id') || !task.agentGraph || task.output?.kind !== 'file')
|
|
38
|
+
throw new Error('local shortcut Task must have explicit ID, Agent Graph and file output');
|
|
39
|
+
await store.mutateJob(job.id, current => {
|
|
40
|
+
if (current.state !== 'DRAFT' || current.tasks.some(existing => existing.id === task.id))
|
|
41
|
+
throw new Error('local shortcut Task already exists or Job is no longer DRAFT');
|
|
42
|
+
current.tasks.push(structuredClone(task));
|
|
43
|
+
return current;
|
|
44
|
+
});
|
|
45
|
+
console.log(JSON.stringify(task, null, 2));
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
throw new Error('unsupported local shortcut command; only create, show and list bindings are allowed');
|
|
49
|
+
}
|
package/dist/cli-shortcut.js
CHANGED
|
@@ -1,16 +1,37 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
|
-
import { recordShortcutReceipt, runAgentShortcut, shortcutResumeEvent } from './agent-shortcut.js';
|
|
3
|
+
import { missingShortcutNodes, recordShortcutReceipt, runAgentShortcut, shortcutResumeEvent } from './agent-shortcut.js';
|
|
4
4
|
import { parseAgentGraph } from './agent-graph.js';
|
|
5
|
+
import { tryTransitionTaskEvent } from './task-transitions.js';
|
|
6
|
+
import { dispatchJobLifecycleSync } from './workflow/lifecycle.js';
|
|
7
|
+
import { runShortcutModelNode } from './shortcut-node-worker.js';
|
|
5
8
|
import { noGitHubMode } from './no-github.js';
|
|
6
|
-
import { dispatchTaskAction } from './task-actions.js';
|
|
7
9
|
import { resolveManagedProjectPath, resolveRepositoryProjectPath, resolveWorkspaceRoot } from './workspace.js';
|
|
8
10
|
import { arg, hasArg, integerArg, publishWake, saveLocal, storeFor, textOption, withEventBus } from './cli-support.js';
|
|
9
11
|
function assertSqliteShortcutArgs(args) {
|
|
10
12
|
if (args.some(value => value === '--file' || value.startsWith('--file=')))
|
|
11
13
|
throw new Error('Shortcut commands require the authoritative SQLite Relay store; --file is unsupported');
|
|
12
14
|
}
|
|
13
|
-
|
|
15
|
+
/** Apply the canonical Job Graph activate edge before the first direct run.
|
|
16
|
+
* The Job is created DRAFT so creation alone cannot launch any work. */
|
|
17
|
+
export async function activateShortcutJobForDirectRun(store, jobId) {
|
|
18
|
+
if (!store.mutateJob)
|
|
19
|
+
throw new Error('Shortcut activation requires atomic SQLite Job mutation');
|
|
20
|
+
await store.mutateJob(jobId, current => {
|
|
21
|
+
if (current.executionMode !== 'shortcut' || current.contentOnly !== true)
|
|
22
|
+
throw new Error('Only a content-only Shortcut Job may be activated by direct run');
|
|
23
|
+
if (current.state === 'DRAFT') {
|
|
24
|
+
const decision = dispatchJobLifecycleSync(current, 'activate');
|
|
25
|
+
if (!decision.accepted || decision.target !== 'ACTIVE')
|
|
26
|
+
throw new Error('Job Lifecycle graph rejected shortcut activation');
|
|
27
|
+
}
|
|
28
|
+
else if (current.state !== 'ACTIVE') {
|
|
29
|
+
throw new Error(`Shortcut Job ${jobId} is not runnable (${current.state})`);
|
|
30
|
+
}
|
|
31
|
+
return current;
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
export async function runShortcutJobCommand(args, id, repo, makeRunner) {
|
|
14
35
|
assertSqliteShortcutArgs(args);
|
|
15
36
|
if (arg(args, '--mode') !== 'shortcut')
|
|
16
37
|
throw new Error('job run requires --mode shortcut');
|
|
@@ -26,11 +47,20 @@ export async function runShortcutJobCommand(args, id, repo) {
|
|
|
26
47
|
let task = job.tasks.find(item => item.id === taskId);
|
|
27
48
|
if (!task)
|
|
28
49
|
throw new Error(`Task ${taskId} not found`);
|
|
29
|
-
if (
|
|
30
|
-
|
|
31
|
-
throw new Error('Task is durably routed to the original Worker; let the Reconciler dispatch it instead of running shortcut mode');
|
|
50
|
+
if (hasArg(args, '--event-json'))
|
|
51
|
+
shortcutResumeEvent(arg(args, '--event-json'));
|
|
32
52
|
if (!task.agentGraph && !hasArg(args, '--agent-graph-file'))
|
|
33
53
|
throw new Error(`Task ${taskId} has no Agent Graph; set it on the Task or pass --agent-graph-file`);
|
|
54
|
+
let resumeReceiptSelected = false;
|
|
55
|
+
if (task.state === 'RUNNING' && task.leaseOwner === null && task.executionId !== null) {
|
|
56
|
+
const checkpoint = task.result?.data?.shortcut;
|
|
57
|
+
if (Object.keys(checkpoint?.receipts ?? {}).length > 0) {
|
|
58
|
+
const decision = tryTransitionTaskEvent(task, 'shortcut.result', { shortcutResultCorrelated: true });
|
|
59
|
+
if (!decision.accepted || !decision.actions.includes('effect.resumeShortcut'))
|
|
60
|
+
throw new Error('Task Lifecycle graph did not select direct receipt resumption');
|
|
61
|
+
resumeReceiptSelected = true;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
34
64
|
const graphOverride = hasArg(args, '--agent-graph-file') ? await textOption(args, '--agent-graph', '--agent-graph-file') : undefined;
|
|
35
65
|
if (graphOverride) {
|
|
36
66
|
parseAgentGraph(graphOverride);
|
|
@@ -52,24 +82,56 @@ export async function runShortcutJobCommand(args, id, repo) {
|
|
|
52
82
|
if (!task)
|
|
53
83
|
throw new Error(`Task ${taskId} disappeared`);
|
|
54
84
|
}
|
|
85
|
+
await activateShortcutJobForDirectRun(loaded.store, id);
|
|
86
|
+
loaded.job = await loaded.store.load(id);
|
|
55
87
|
const workspaceRoot = await resolveWorkspaceRoot(arg(args, '--workspace-root') || undefined);
|
|
56
88
|
const projectRoot = await resolveManagedProjectPath(noGitHubMode(args) ? undefined : repo, arg(args, '--project-path') || loaded.job.projectPath || '', workspaceRoot);
|
|
57
89
|
const projectPath = resolveRepositoryProjectPath(projectRoot, loaded.job.projectPath);
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
90
|
+
const missing = await missingShortcutNodes(task.agentGraph, projectPath, arg(args, '--neo-root') || join(workspaceRoot, 'neo'));
|
|
91
|
+
const previous = task.result?.data?.shortcut;
|
|
92
|
+
if (previous?.workerFallback === true)
|
|
93
|
+
throw new Error('Legacy whole-Graph Worker fallback is not compatible with per-node Shortcut execution; retain its original recovery ownership');
|
|
94
|
+
const controller = new AbortController();
|
|
95
|
+
const onInterrupt = () => controller.abort(new Error('Interrupted by operator'));
|
|
96
|
+
process.once('SIGINT', onInterrupt);
|
|
97
|
+
process.once('SIGTERM', onInterrupt);
|
|
98
|
+
const execute = async (bus) => {
|
|
99
|
+
const workerReconciler = missing.length && bus && makeRunner ? makeRunner(loaded.store, bus) : undefined;
|
|
100
|
+
const unwatch = workerReconciler ? await workerReconciler.watch(id) : undefined;
|
|
101
|
+
try {
|
|
102
|
+
const result = await runAgentShortcut({ store: loaded.store, job: loaded.job, task, projectPath,
|
|
103
|
+
neoRoot: arg(args, '--neo-root') || join(workspaceRoot, 'neo'),
|
|
104
|
+
concurrency: integerArg(args, '--concurrency', 4), signal: controller.signal,
|
|
105
|
+
...(resumeReceiptSelected ? { resumeReceiptSelected: true } : {}),
|
|
106
|
+
...(missing.length ? { runModelNode: node => {
|
|
107
|
+
if (!workerReconciler)
|
|
108
|
+
throw new Error('Model-backed node requires a reachable EventBus and existing Relay Worker Adapter');
|
|
109
|
+
return runShortcutModelNode({ store: loaded.store, reconciler: workerReconciler, jobId: id, parentTaskId: task.id, node });
|
|
110
|
+
} } : {}),
|
|
111
|
+
onNodeEvent: event => process.stderr.write(`shortcut.node ${JSON.stringify(event)}\n`) });
|
|
112
|
+
console.log(JSON.stringify({ outcome: result.outcome, fallbackQueued: false, task: result.task }, null, 2));
|
|
113
|
+
if (result.outcome.status === 'FAILED') {
|
|
114
|
+
const checkpoint = result.task.result?.data?.shortcut;
|
|
115
|
+
process.exitCode = checkpoint?.terminal?.status === 'TIMED_OUT' ? 4 : checkpoint?.terminal?.status === 'INTERRUPTED' ? 5 : 1;
|
|
116
|
+
}
|
|
117
|
+
else if (result.outcome.status === 'WAITING_EVENT')
|
|
118
|
+
process.exitCode = 3;
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
await unwatch?.();
|
|
122
|
+
await workerReconciler?.close();
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
try {
|
|
126
|
+
if (missing.length)
|
|
127
|
+
await withEventBus(args, execute);
|
|
128
|
+
else
|
|
129
|
+
await execute(undefined);
|
|
68
130
|
}
|
|
69
|
-
|
|
70
|
-
|
|
131
|
+
finally {
|
|
132
|
+
process.removeListener('SIGINT', onInterrupt);
|
|
133
|
+
process.removeListener('SIGTERM', onInterrupt);
|
|
71
134
|
}
|
|
72
|
-
console.log(JSON.stringify({ outcome: result.outcome, fallbackQueued: result.fallback, task: outputTask }, null, 2));
|
|
73
135
|
}
|
|
74
136
|
export async function recordShortcutReceiptCommand(args, id) {
|
|
75
137
|
assertSqliteShortcutArgs(args);
|