agents-relay 2.0.207 → 2.0.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.js +2 -1
- package/dist/agent-shortcut.js +113 -41
- package/dist/audit-tail.js +52 -0
- package/dist/browser-worker-runtime.js +7 -2
- package/dist/chatgpt-platform-fallback.js +30 -0
- package/dist/cli-shortcut-local.js +49 -0
- package/dist/cli-shortcut.js +81 -19
- package/dist/cli.js +8 -3
- package/dist/decision-audit.js +143 -1
- package/dist/execution-routing.js +8 -2
- package/dist/force-restate-validation.js +3 -1
- package/dist/job-execution-slot.js +4 -1
- package/dist/model-routing.js +8 -0
- package/dist/reconciler-review.js +39 -13
- package/dist/reconciler-runtime.js +46 -52
- package/dist/reconciler.js +12 -2
- package/dist/relay-job-snapshot-reads.js +5 -2
- package/dist/relay-workflow-operations.js +1 -1
- package/dist/relayd.js +1 -1
- package/dist/route-outcome-recording.js +51 -0
- package/dist/runtime-status.js +11 -4
- package/dist/shortcut-node-worker.js +128 -0
- package/dist/shortcut-stage-contract.js +50 -0
- package/dist/shortcut-worker-runner.js +73 -0
- package/dist/task-actions.js +4 -13
- package/dist/task-inactivity-watchdog.js +7 -0
- package/dist/terminal-event-recovery.js +13 -0
- package/dist/worker-failure-recovery.js +89 -0
- package/dist/workflow/contracts.js +4 -0
- package/dist/workflow/definitions-v2.js +2 -0
- package/dist/workflow/runtime.js +30 -0
- package/dist/workflow/task-resolution.js +1 -4
- package/package.json +4 -3
- package/skills/agents-relay/agents/planner.agent.md +14 -4
- package/skills/agents-relay/agents/reviewer.agent.md +2 -0
- package/skills/agents-relay/contracts/orchestrator-workflows.md +6 -0
- package/skills/agents-relay/contracts/task-job-contract.md +4 -0
- package/workflow/force-restate-validation.mmd +1 -1
- package/workflow/review-lifecycle.mmd +1 -1
- package/workflow/task-lifecycle.mmd +13 -12
- package/workflow/task-resolution.mmd +1 -6
|
@@ -86,7 +86,7 @@ export class SQLiteWorkflowOperations {
|
|
|
86
86
|
const job = JSON.parse(row.json);
|
|
87
87
|
const task = job.tasks.find(item => item.id === identity.taskId);
|
|
88
88
|
const shortcut = task?.result?.data?.shortcut;
|
|
89
|
-
const decision = dispatchWorkflowGraphSync(taskLifecycleV2, { initialState: 'QUEUED', input: { transport: 'event', id: 'dispatch', payload: {} }, durableFacts: { dispatchable: true, shortcutExecution: job.executionMode === 'shortcut' && job.contentOnly === true, shortcutFallback: shortcut?.workerFallback === true, shortcutHasGraph: Boolean(task?.agentGraph) } });
|
|
89
|
+
const decision = dispatchWorkflowGraphSync(taskLifecycleV2, { initialState: 'QUEUED', input: { transport: 'event', id: 'dispatch', payload: {} }, durableFacts: { dispatchable: true, shortcutExecution: job.executionMode === 'shortcut' && job.contentOnly === true && shortcut?.workerFallback !== true && Boolean(task?.agentGraph), shortcutFallback: shortcut?.workerFallback === true, shortcutHasGraph: Boolean(task?.agentGraph), shortcutDirectInvocation: job.executionMode === 'shortcut' && job.contentOnly === true } });
|
|
90
90
|
if (!decision.accepted)
|
|
91
91
|
throw new Error('canonical Task Graph did not authorize launch');
|
|
92
92
|
const actionId = decision.actions.find(action => action.id === 'effect.launchTask' || action.id === 'effect.launchShortcut')?.id;
|
package/dist/relayd.js
CHANGED
|
@@ -504,7 +504,7 @@ export async function runDaemon(argv) {
|
|
|
504
504
|
storySize: task.storySize ?? 'M',
|
|
505
505
|
},
|
|
506
506
|
usage,
|
|
507
|
-
chatgptRunning: runtime.workers.chatgpt,
|
|
507
|
+
chatgptRunning: runtime.workers.chatgpt, historyStore: decisionAuditStore,
|
|
508
508
|
});
|
|
509
509
|
return {
|
|
510
510
|
selected: {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { outcomeForDecision } from './decision-audit.js';
|
|
2
|
+
/** Preserve the source classification except when the failure text gives clear evidence. */
|
|
3
|
+
export function classifyRoutingFailure(message, fallback) {
|
|
4
|
+
if (/invalid (?:api|public) key|(?:authentication|authorization) failed|401 unauthorized/i.test(message))
|
|
5
|
+
return 'auth_config_failure';
|
|
6
|
+
if (/stream disconnected|response\.completed|rate[_ .-]?limit(?:[_ .-]?exceeded)?|quota exceeded|provider[_ .-]?unavailable|model[_ .-]?(?:unavailable|not[_ .-]?found)|service unavailable/i.test(message))
|
|
7
|
+
return 'provider_failure';
|
|
8
|
+
if (/timeout|timed out|deadline exceeded/i.test(message))
|
|
9
|
+
return 'timeout';
|
|
10
|
+
return fallback;
|
|
11
|
+
}
|
|
12
|
+
/** One best-effort audit effect after an authoritative Graph/SQLite transition. */
|
|
13
|
+
export async function recordRouteOutcome(store, task, kind, reason, evidence = {}) {
|
|
14
|
+
if (!store || !task.routing?.decisionId)
|
|
15
|
+
return;
|
|
16
|
+
try {
|
|
17
|
+
await store.append(outcomeForDecision(task.routing.decisionId, kind, reason, {
|
|
18
|
+
job_id: task.jobId, task_id: task.id, attempt: task.attempt,
|
|
19
|
+
route: {
|
|
20
|
+
worker: task.workerSelection?.workerId ?? task.adapter,
|
|
21
|
+
adapter: task.adapter,
|
|
22
|
+
provider: task.routing.provider,
|
|
23
|
+
model: task.routing.model,
|
|
24
|
+
profile: task.routing.profile ?? null,
|
|
25
|
+
reasoning: task.routing.reasoning ?? null,
|
|
26
|
+
},
|
|
27
|
+
...evidence,
|
|
28
|
+
}));
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
// A failed audit append must not undo an already committed SQLite transition.
|
|
32
|
+
process.stderr.write('[agents-relay] route outcome audit unavailable: ' + (error instanceof Error ? error.message : String(error)) + '\n');
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** Accepted terminal Worker event; reviewer/human gates remain distinct outcomes. */
|
|
36
|
+
export async function recordTerminalRouteOutcome(store, task, event, executionId) {
|
|
37
|
+
const kind = event.type === 'task.completed' ? 'success'
|
|
38
|
+
: event.type === 'task.cancelled' ? 'cancelled' : event.type === 'task.blocked' ? 'blocked'
|
|
39
|
+
: classifyRoutingFailure([event.message, event.data?.reason].join(' '), 'task_failure');
|
|
40
|
+
const reason = event.type === 'task.completed' ? 'task.completed'
|
|
41
|
+
: [typeof event.data?.reason === 'string' ? event.data.reason : null, event.message].filter(Boolean).join(': ') || event.type;
|
|
42
|
+
await recordRouteOutcome(store, task, kind, reason, { execution_id: executionId, event_type: event.type });
|
|
43
|
+
}
|
|
44
|
+
/** Launcher Graph accepted a failure or safe pre-submission fallback. */
|
|
45
|
+
export async function recordLaunchRouteOutcome(store, task, message, phase, executionId, unknownOutcome) {
|
|
46
|
+
await recordRouteOutcome(store, task, unknownOutcome ? 'outcome_unknown' : classifyRoutingFailure(message, phase === 'launch' ? 'launch_failure' : 'runtime_failure'), 'worker_' + phase + '_failure: ' + message, { phase, execution_id: executionId, target_state: task.state, outcome_unknown: unknownOutcome });
|
|
47
|
+
}
|
|
48
|
+
/** Watchdog Graph applies its durable state before emitting a non-authoritative event. */
|
|
49
|
+
export async function recordWatchdogRouteOutcome(store, task, evidence) {
|
|
50
|
+
await recordRouteOutcome(store, task, evidence.failedExit ? classifyRoutingFailure(evidence.rawError, 'runtime_failure') : 'timeout', (evidence.failureReason ?? 'watchdog_failed') + ': ' + evidence.failureMessage, { execution_id: evidence.executionId, watchdog: true, exit_code: evidence.exitCode });
|
|
51
|
+
}
|
package/dist/runtime-status.js
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
|
-
function isSuperseded(task) {
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
export function isSuperseded(task, byId) {
|
|
2
|
+
if (task.state === 'CANCELLED' && task.resolution?.path === 'cancel_supersede' && task.resolution.state === 'resolved')
|
|
3
|
+
return true;
|
|
4
|
+
const classification = task.resolutionClassification;
|
|
5
|
+
return task.state === 'FAILED' && classification?.kind === 'superseded'
|
|
6
|
+
&& typeof classification.targetTaskId === 'string' && Boolean(classification.targetTaskId.trim())
|
|
7
|
+
&& typeof classification.evidence === 'string' && Boolean(classification.evidence.trim())
|
|
8
|
+
&& typeof classification.reason === 'string' && Boolean(classification.reason.trim())
|
|
9
|
+
&& byId.get(classification.targetTaskId)?.state === 'SUCCEEDED';
|
|
4
10
|
}
|
|
5
11
|
export function implementationTasks(job) {
|
|
12
|
+
const byId = new Map(job.tasks.map(task => [task.id, task]));
|
|
6
13
|
return job.tasks.filter(task => task.kind !== 'planner'
|
|
7
14
|
&& task.agentName !== 'reviewer'
|
|
8
15
|
&& task.agentName !== 'troubleshooter'
|
|
9
|
-
&& !isSuperseded(task)
|
|
16
|
+
&& !isSuperseded(task, byId)
|
|
10
17
|
&& task.state !== 'DRAFT');
|
|
11
18
|
}
|
|
12
19
|
export function implementationComplete(job) {
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { mkdir, readFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
3
|
+
import { createTaskEntity } from './workflow/entity-creation.js';
|
|
4
|
+
/** A model node stays within its original Shortcut Job. Its durable child Task
|
|
5
|
+
* uses the existing Relay Worker adapter, lifecycle Graph and Troubleshooter.
|
|
6
|
+
* The parent Shortcut Graph, not the Worker Reconciler, selects DAG readiness. */
|
|
7
|
+
export async function runShortcutModelNode(options) {
|
|
8
|
+
const { store, reconciler, jobId, parentTaskId, node } = options;
|
|
9
|
+
if (!store.mutateJob)
|
|
10
|
+
throw new Error('Model nodes require the durable Relay SQLite store');
|
|
11
|
+
if (!/^[A-Za-z_][\w.-]*$/.test(node.id))
|
|
12
|
+
throw new Error('Invalid shortcut model node ID');
|
|
13
|
+
const taskId = `shortcut-${parentTaskId}-${node.id}`;
|
|
14
|
+
const initial = await store.load(jobId);
|
|
15
|
+
const parent = initial.tasks.find(t => t.id === parentTaskId);
|
|
16
|
+
if (!parent || parent.state !== 'RUNNING' || parent.executionId !== node.executionId || !parent.leaseOwner)
|
|
17
|
+
throw new Error('Model node parent execution is not leased by Shortcut Runner');
|
|
18
|
+
if (parent.output?.kind !== 'file' || !isAbsolute(parent.output.path))
|
|
19
|
+
throw new Error('Model nodes require a canonical absolute Task output directory');
|
|
20
|
+
const runRoot = dirname(parent.output.path);
|
|
21
|
+
const declaredRun = parent.input.match(/^- Canonical run:\s*(\/[^\r\n]+)$/m)?.[1]?.trim();
|
|
22
|
+
if (declaredRun && resolve(declaredRun) !== resolve(runRoot))
|
|
23
|
+
throw new Error('Shortcut Worker declared canonical run does not match the bound parent Task output directory');
|
|
24
|
+
const sourceRef = parent.input.match(/^- Private input evidence:\s*(\/[^\r\n]+)$/m)?.[1]?.trim();
|
|
25
|
+
if (sourceRef && (!declaredRun || dirname(resolve(sourceRef)) !== resolve(runRoot)))
|
|
26
|
+
throw new Error('Shortcut Worker source evidence is not confined to the declared canonical run');
|
|
27
|
+
const outputPath = join(runRoot, '.shortcut-node-results', `${taskId}.json`);
|
|
28
|
+
const started = initial.shortcutExecutionStartedAt && Date.parse(initial.shortcutExecutionStartedAt);
|
|
29
|
+
if (!started || !Number.isFinite(started))
|
|
30
|
+
throw new Error('Model node has no immutable Shortcut deadline origin');
|
|
31
|
+
const deadline = started + 20 * 60_000;
|
|
32
|
+
if (initial.shortcutExecutionDeadlineAt && Date.parse(initial.shortcutExecutionDeadlineAt) !== deadline)
|
|
33
|
+
throw new Error('Model node deadline differs from the original execution budget');
|
|
34
|
+
if (Date.now() >= deadline)
|
|
35
|
+
throw new Error('Shortcut execution budget exceeded before model node dispatch');
|
|
36
|
+
await mkdir(dirname(outputPath), { recursive: true, mode: 0o700 });
|
|
37
|
+
await store.mutateJob(jobId, current => {
|
|
38
|
+
const active = current.tasks.find(t => t.id === parentTaskId);
|
|
39
|
+
if (current.state !== 'ACTIVE' || current.executionMode !== 'shortcut' || current.contentOnly !== true || current.pause ||
|
|
40
|
+
!active || active.executionId !== node.executionId || active.state !== 'RUNNING' || !active.leaseOwner)
|
|
41
|
+
throw new Error('Shortcut model worker claim invalidated by Job lifecycle');
|
|
42
|
+
const cp = active.result?.data?.shortcut;
|
|
43
|
+
if (!cp || typeof cp !== 'object')
|
|
44
|
+
throw new Error('Shortcut model worker requires a persisted Graph checkpoint');
|
|
45
|
+
const state = cp;
|
|
46
|
+
if (state.nodes?.[node.id] || state.inFlight?.[node.id] !== node.executionId)
|
|
47
|
+
throw new Error('Model node is no longer pending in the current Graph execution');
|
|
48
|
+
state.workerNodes ??= {};
|
|
49
|
+
if (state.workerNodes[node.id] && state.workerNodes[node.id] !== taskId)
|
|
50
|
+
throw new Error('Conflicting durable Worker node mapping');
|
|
51
|
+
state.workerNodes[node.id] = taskId;
|
|
52
|
+
const existing = current.tasks.find(t => t.id === taskId);
|
|
53
|
+
if (existing) {
|
|
54
|
+
if (existing.parentTaskId !== parentTaskId || existing.agentName !== node.agent || existing.output?.kind !== 'file' || existing.output.path !== outputPath)
|
|
55
|
+
throw new Error('Durable Worker node identity mismatch');
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
const now = new Date().toISOString();
|
|
59
|
+
const body = JSON.stringify({
|
|
60
|
+
source: 'shortcut-graph-node', jobId, parentTaskId, nodeId: node.id, agent: node.agent,
|
|
61
|
+
predecessorOutputs: node.predecessorOutputs,
|
|
62
|
+
originalInput: active.input,
|
|
63
|
+
...(declaredRun ? { canonicalRunPath: declaredRun } : {}),
|
|
64
|
+
...(sourceRef ? { sourceEvidencePath: sourceRef } : {}),
|
|
65
|
+
requiredResult: { status: 'COMPLETED', output: {}, artifacts: [] },
|
|
66
|
+
outputPath,
|
|
67
|
+
instruction: 'Execute only this Agent role for this DAG node. The ONLY structured DAG predecessor values are predecessorOutputs; never infer an undeclared sibling dependency. If canonicalRunPath is declared, read the authorized original source manifest and prior stage artifact files there as required by the project Agent/Stage contract; inspect and validate their real contents, source provenance, hashes and use rights. These files are allowed canonical run evidence, not an alternative DAG predecessor result and not proof of human approval. Do not claim they are absent merely because they are not repeated in predecessorOutputs. Write {status:"COMPLETED",output:{...},artifacts:[]} to outputPath only after meeting stage acceptance and send a correlated task.completed event. If a real prerequisite is unavailable, report the specific verified blocker and task.failed; do not fabricate, publish, approve or infer new permissions.'
|
|
68
|
+
});
|
|
69
|
+
current.tasks.push(createTaskEntity(current, {
|
|
70
|
+
jobId, id: taskId, title: `Shortcut model node ${node.id}`, kind: 'work', parentTaskId,
|
|
71
|
+
agentName: node.agent, dependencies: [], capabilities: ['model'], workerRequirements: { capabilities: ['model'] },
|
|
72
|
+
adapter: 'codex', input: body, output: { kind: 'file', path: outputPath },
|
|
73
|
+
attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null,
|
|
74
|
+
result: null, error: null, timeoutMs: Math.max(1, deadline - Date.now()), createdAt: now, updatedAt: now,
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
return current;
|
|
78
|
+
});
|
|
79
|
+
while (true) {
|
|
80
|
+
if (node.signal.aborted)
|
|
81
|
+
throw node.signal.reason ?? new Error('Shortcut Worker node interrupted');
|
|
82
|
+
if (Date.now() >= deadline)
|
|
83
|
+
throw new Error('Shortcut execution budget exceeded while waiting for model node');
|
|
84
|
+
const current = await store.load(jobId);
|
|
85
|
+
if (current.state !== 'ACTIVE' || current.pause)
|
|
86
|
+
throw new Error('Shortcut Job cancelled/paused while Worker node was pending');
|
|
87
|
+
const task = current.tasks.find(t => t.id === taskId);
|
|
88
|
+
if (!task)
|
|
89
|
+
throw new Error('Worker node disappeared from the durable Job');
|
|
90
|
+
if (task.state === 'SUCCEEDED') {
|
|
91
|
+
const buffer = await readFile(outputPath);
|
|
92
|
+
const value = JSON.parse(buffer.toString('utf8'));
|
|
93
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
94
|
+
throw new Error('Worker Task reported success without contract-valid node output');
|
|
95
|
+
const record = value;
|
|
96
|
+
if (record.status !== 'COMPLETED' || !record.output || typeof record.output !== 'object' || Array.isArray(record.output))
|
|
97
|
+
throw new Error('Worker Task reported success without contract-valid node output');
|
|
98
|
+
// Some existing Worker roles report artifact records as {path,type}.
|
|
99
|
+
// Convert ONLY exact file records to the Shortcut contract's path list;
|
|
100
|
+
// the Graph executor still verifies realpath containment and SHA-256.
|
|
101
|
+
if (Array.isArray(record.artifacts)) {
|
|
102
|
+
const files = record.artifacts.map((item) => {
|
|
103
|
+
if (typeof item === 'string' && item.trim())
|
|
104
|
+
return item;
|
|
105
|
+
if (item && typeof item === 'object' && !Array.isArray(item)) {
|
|
106
|
+
const ref = item;
|
|
107
|
+
const fields = Object.keys(ref);
|
|
108
|
+
if (fields.length === 2 && fields.includes('path')
|
|
109
|
+
&& ((fields.includes('type') && ref.type === 'file') || (fields.includes('kind') && ref.kind === 'file'))
|
|
110
|
+
&& typeof ref.path === 'string' && ref.path.trim())
|
|
111
|
+
return ref.path;
|
|
112
|
+
}
|
|
113
|
+
throw new Error('Worker node returned unsupported artifact entry; expected a file path, {type:"file",path} or {kind:"file",path}');
|
|
114
|
+
});
|
|
115
|
+
return { ...record, artifacts: files };
|
|
116
|
+
}
|
|
117
|
+
if (!record.artifacts || typeof record.artifacts !== 'object')
|
|
118
|
+
throw new Error('Worker Task reported success without contract-valid artifact output');
|
|
119
|
+
return record;
|
|
120
|
+
}
|
|
121
|
+
if (task.state === 'CANCELLED')
|
|
122
|
+
throw new Error('Worker node cancelled');
|
|
123
|
+
if (task.state === 'BLOCKED')
|
|
124
|
+
throw new Error(`Worker node requires escalation: ${task.failureReason ?? task.error ?? task.id}`);
|
|
125
|
+
await reconciler.reconcile(jobId);
|
|
126
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/** Optional project-declared Stage completion gate. Runs after any code/model
|
|
2
|
+
* node reports COMPLETED and before the canonical DAG accepts its result. */
|
|
3
|
+
import { readFile, realpath } from 'node:fs/promises';
|
|
4
|
+
import { resolve, relative, isAbsolute, sep, join } from 'node:path';
|
|
5
|
+
const within = (root, target) => { const rel = relative(root, target); return rel === '' || (rel !== '..' && !rel.startsWith('..' + sep) && !isAbsolute(rel)); };
|
|
6
|
+
export async function validateShortcutStageGate(options) {
|
|
7
|
+
const root = await realpath(options.projectPath);
|
|
8
|
+
let manifest;
|
|
9
|
+
try {
|
|
10
|
+
manifest = JSON.parse(await readFile(join(root, 'project.run.json'), 'utf8'));
|
|
11
|
+
}
|
|
12
|
+
catch (error) {
|
|
13
|
+
if (error.code === 'ENOENT')
|
|
14
|
+
return;
|
|
15
|
+
throw error;
|
|
16
|
+
}
|
|
17
|
+
const config = manifest?.stageValidation;
|
|
18
|
+
if (!config || typeof config !== 'object' || Array.isArray(config))
|
|
19
|
+
return;
|
|
20
|
+
const declaration = config;
|
|
21
|
+
if (!declaration.agents || typeof declaration.agents !== 'object' || Array.isArray(declaration.agents))
|
|
22
|
+
throw new Error('Invalid Stage validator agent mapping');
|
|
23
|
+
const stage = declaration.agents[options.agent];
|
|
24
|
+
if (stage === undefined)
|
|
25
|
+
return;
|
|
26
|
+
if (typeof stage !== 'string' || !/^[-a-z]+$/.test(stage))
|
|
27
|
+
throw new Error('Invalid declared Stage ID');
|
|
28
|
+
const script = declaration.script;
|
|
29
|
+
if (typeof script !== 'string' || !/^scripts\/[a-zA-Z0-9._-]+\.mjs$/.test(script))
|
|
30
|
+
throw new Error('Invalid Stage validator script');
|
|
31
|
+
const file = await realpath(resolve(root, script));
|
|
32
|
+
if (!within(root, file))
|
|
33
|
+
throw new Error('Stage validator escaped project root');
|
|
34
|
+
const run = await realpath(options.runPath);
|
|
35
|
+
if (!within(await realpath(resolve(root, 'runs')), run) || run === root)
|
|
36
|
+
throw new Error('Stage validator run escaped project runs');
|
|
37
|
+
const response = await options.execute([process.execPath, file, stage, run], root);
|
|
38
|
+
let outcome;
|
|
39
|
+
try {
|
|
40
|
+
outcome = JSON.parse(response.stdout);
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
throw new Error('Stage validator did not return a structured contract result');
|
|
44
|
+
}
|
|
45
|
+
const report = outcome;
|
|
46
|
+
if (response.code !== 0 || report.stage !== stage || report.status !== 'PASS' || (Array.isArray(report.errors) && report.errors.length > 0)) {
|
|
47
|
+
const detail = Array.isArray(report.errors) ? JSON.stringify(report.errors).slice(0, 1200) : response.stderr.slice(0, 1200);
|
|
48
|
+
throw new Error('STAGE_CONTRACT_FAILED ' + stage + ': ' + detail);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { dispatchJobLifecycleSync } from './workflow/lifecycle.js';
|
|
2
|
+
/** One foreground owner drives all Worker fallback and Troubleshooter Tasks.
|
|
3
|
+
* No background scheduler may mutate a Shortcut Job. */
|
|
4
|
+
export async function driveShortcutWorkers(options) {
|
|
5
|
+
const { store, jobId, reconciler, taskId } = options;
|
|
6
|
+
if (!store.mutateJob)
|
|
7
|
+
throw new Error('Foreground Worker fallback requires atomic Relay SQLite');
|
|
8
|
+
const initial = await store.mutateJob(jobId, job => {
|
|
9
|
+
if (job.executionMode !== 'shortcut' || job.contentOnly !== true || job.state !== 'ACTIVE')
|
|
10
|
+
throw new Error('Not an active Shortcut Job');
|
|
11
|
+
if (!job.tasks.find(task => task.id === taskId))
|
|
12
|
+
throw new Error('Shortcut parent Task missing');
|
|
13
|
+
const now = Date.now();
|
|
14
|
+
if (!job.shortcutExecutionStartedAt)
|
|
15
|
+
job.shortcutExecutionStartedAt = new Date(now).toISOString();
|
|
16
|
+
const first = Date.parse(job.shortcutExecutionStartedAt);
|
|
17
|
+
if (!Number.isFinite(first))
|
|
18
|
+
throw new Error('Invalid original Shortcut start time');
|
|
19
|
+
const deadline = new Date(first + 20 * 60_000).toISOString();
|
|
20
|
+
if (job.shortcutExecutionDeadlineAt && job.shortcutExecutionDeadlineAt !== deadline)
|
|
21
|
+
throw new Error('Shortcut deadline changed');
|
|
22
|
+
job.shortcutExecutionDeadlineAt = deadline;
|
|
23
|
+
return job;
|
|
24
|
+
});
|
|
25
|
+
const deadline = Date.parse(initial.shortcutExecutionDeadlineAt);
|
|
26
|
+
const last = new Map();
|
|
27
|
+
const stopWatching = await reconciler.watch(jobId);
|
|
28
|
+
const observe = (job) => {
|
|
29
|
+
for (const task of job.tasks) {
|
|
30
|
+
const key = `${task.state}:${task.attempt}:${task.error ?? ''}`;
|
|
31
|
+
if (last.get(task.id) !== key) {
|
|
32
|
+
last.set(task.id, key);
|
|
33
|
+
options.onTask?.(task);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
try {
|
|
38
|
+
while (true) {
|
|
39
|
+
if (options.signal?.aborted)
|
|
40
|
+
return { status: 'INTERRUPTED', job: await store.load(jobId) };
|
|
41
|
+
if (Date.now() >= deadline)
|
|
42
|
+
return { status: 'TIMED_OUT', job: await store.load(jobId) };
|
|
43
|
+
await reconciler.reconcile(jobId);
|
|
44
|
+
let job = await store.load(jobId);
|
|
45
|
+
observe(job);
|
|
46
|
+
if (job.state === 'COMPLETED')
|
|
47
|
+
return { status: 'COMPLETED', job };
|
|
48
|
+
if (job.state === 'CANCELLED')
|
|
49
|
+
return { status: 'FAILED', job };
|
|
50
|
+
if (job.tasks.every(task => task.state === 'SUCCEEDED')) {
|
|
51
|
+
job = await store.mutateJob(jobId, current => {
|
|
52
|
+
if (current.state !== 'ACTIVE' || !current.tasks.length || !current.tasks.every(task => task.state === 'SUCCEEDED')
|
|
53
|
+
|| current.prNumber !== 0 || current.localMerge || current.autoMerge !== false || current.reviewEnabled !== false || current.review || current.plans?.length)
|
|
54
|
+
throw new Error('Shortcut completion requires all durable Tasks and content-only policy');
|
|
55
|
+
const decision = dispatchJobLifecycleSync(current, 'shortcut.completed', { shortcutCompletionAllowed: true });
|
|
56
|
+
if (!decision.accepted || decision.target !== 'COMPLETED')
|
|
57
|
+
throw new Error('Job Graph rejected Shortcut completion');
|
|
58
|
+
return current;
|
|
59
|
+
});
|
|
60
|
+
return { status: 'COMPLETED', job };
|
|
61
|
+
}
|
|
62
|
+
if (job.tasks.some(task => task.state === 'BLOCKED'))
|
|
63
|
+
return { status: 'WAITING_EVENT', job };
|
|
64
|
+
if (job.tasks.some(task => task.agentName === 'troubleshooter' && task.state === 'FAILED' && task.attempt >= task.maxAttempts))
|
|
65
|
+
return { status: 'FAILED', job };
|
|
66
|
+
await new Promise(resolve => setTimeout(resolve, 350));
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
await stopWatching();
|
|
71
|
+
await reconciler.close();
|
|
72
|
+
}
|
|
73
|
+
}
|
package/dist/task-actions.js
CHANGED
|
@@ -47,11 +47,6 @@ function clearExecution(task) {
|
|
|
47
47
|
task.leaseExpiresAt = null;
|
|
48
48
|
task.executionId = null;
|
|
49
49
|
}
|
|
50
|
-
function markShortcutWorkerFallback(task) {
|
|
51
|
-
const data = task.result?.data ?? {};
|
|
52
|
-
const shortcut = data.shortcut && typeof data.shortcut === 'object' ? data.shortcut : {};
|
|
53
|
-
task.result = { summary: task.result?.summary ?? 'Troubleshooter selected original Worker recovery', data: { ...data, shortcut: { ...shortcut, workerFallback: true } } };
|
|
54
|
-
}
|
|
55
50
|
function applyTaskRetrySnapshot(task, policy = {}) {
|
|
56
51
|
if (task.kind === 'planner')
|
|
57
52
|
throw new Error(`Legacy planner Task ${task.id} is read-only; retry the Plan instead`);
|
|
@@ -227,11 +222,9 @@ export async function dispatchTaskAction(store, request) {
|
|
|
227
222
|
if (!recovery.executionId)
|
|
228
223
|
throw new Error(`Troubleshooter ${recovery.id} has no current execution for a fresh resolution`);
|
|
229
224
|
const proposal = { path: 'exact_original_retry', targetTaskId: task.id, recoveryExecutionId: recovery.executionId, rootCause: task.failureReason ?? task.error ?? 'durable failure', solution: `Recover exact original Task via ${request.recoveryTaskId}` };
|
|
230
|
-
const activateRecovery = (
|
|
231
|
-
markShortcutWorkerFallback(task); recordRecoveryRouting(task, request); task.resolution = { ...materializeResolution(task, proposal, 'troubleshooter'), state: 'resolved' }; };
|
|
225
|
+
const activateRecovery = () => { applyTaskRetrySnapshot(task, { extendBudget: true }); recordRecoveryRouting(task, request); task.resolution = { ...materializeResolution(task, proposal, 'troubleshooter'), state: 'resolved' }; };
|
|
232
226
|
const decision = dispatchTaskResolutionSync(task, proposal, 'troubleshooter', {
|
|
233
|
-
activateRecovery
|
|
234
|
-
activateShortcutWorkerFallback: () => activateRecovery(true),
|
|
227
|
+
activateRecovery,
|
|
235
228
|
});
|
|
236
229
|
if (!decision.accepted)
|
|
237
230
|
throw new Error(`Task resolution graph rejected exact-original recovery for ${task.id}`);
|
|
@@ -268,11 +261,9 @@ export async function dispatchTaskAction(store, request) {
|
|
|
268
261
|
if (!recovery.executionId)
|
|
269
262
|
throw new Error(`Troubleshooter ${recovery.id} has no current execution for a fresh resolution`);
|
|
270
263
|
const proposal = { path: 'exact_original_retry', targetTaskId: task.id, recoveryExecutionId: recovery.executionId, rootCause: task.failureReason ?? task.error ?? 'durable failure', solution: `Recover exact original Task via ${request.recoveryTaskId}` };
|
|
271
|
-
const activateRecovery = (
|
|
272
|
-
markShortcutWorkerFallback(task); recordRecoveryRouting(task, request); task.resolution = { ...materializeResolution(task, proposal, 'troubleshooter'), state: 'resolved' }; };
|
|
264
|
+
const activateRecovery = () => { applyTaskRetrySnapshot(task, { extendBudget: true }); recordRecoveryRouting(task, request); task.resolution = { ...materializeResolution(task, proposal, 'troubleshooter'), state: 'resolved' }; };
|
|
273
265
|
const decision = dispatchTaskResolutionSync(task, proposal, 'troubleshooter', {
|
|
274
|
-
activateRecovery
|
|
275
|
-
activateShortcutWorkerFallback: () => activateRecovery(true),
|
|
266
|
+
activateRecovery,
|
|
276
267
|
});
|
|
277
268
|
if (!decision.accepted)
|
|
278
269
|
throw new Error(`Task resolution graph rejected exact-original recovery for ${task.id}`);
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import { executionEventSource, eventSourceId } from './execution-event-identity.js';
|
|
2
|
+
/** A foreground Shortcut Graph parent is not a Worker: its elapsed budget
|
|
3
|
+
* belongs to the direct runner. Real Worker child Tasks keep standard timeouts. */
|
|
4
|
+
export function requiresWorkerInactivityWatchdog(job, task) {
|
|
5
|
+
const shortcut = task.result?.data?.shortcut;
|
|
6
|
+
return !(job.executionMode === 'shortcut' && job.contentOnly === true
|
|
7
|
+
&& Boolean(task.agentGraph) && shortcut?.workerFallback !== true);
|
|
8
|
+
}
|
|
2
9
|
export function isRecoverableWatchdogFailure(task) {
|
|
3
10
|
if (['worker_start_timeout', 'terminal_event_timeout', 'worker_exit_before_start', 'worker_exit_without_terminal_event', 'worker_delivery_after_start'].includes(task.failureReason ?? ''))
|
|
4
11
|
return true;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Shared, adapter-neutral terminal-event recovery policy.
|
|
2
|
+
* A recovery turn is not a Task retry, progress event, or completion proof.
|
|
3
|
+
* Only an existing resumable conversation for the ORIGINAL execution qualifies.
|
|
4
|
+
*/
|
|
5
|
+
export const MAX_TERMINAL_RECOVERY_TURNS = 1;
|
|
6
|
+
export function terminalRecoveryEligible(facts) {
|
|
7
|
+
return facts.hasSession && facts.hasThread && !facts.terminalObserved
|
|
8
|
+
&& !facts.outcomeUncertain && !facts.cancelled
|
|
9
|
+
&& facts.recoveryAttempts < MAX_TERMINAL_RECOVERY_TURNS;
|
|
10
|
+
}
|
|
11
|
+
export function terminalRecoveryPrompt(executionId) {
|
|
12
|
+
return `[Relay terminal-event recovery: ${executionId}]\nThis is ONE reminder for the ORIGINAL Task execution, not a new task or authorization to restart work. Inspect the authoritative Task context, Neo event history, and actual output. If a correlated terminal event already exists, do not publish a duplicate. If work is complete with verified evidence, publish task.completed with the ORIGINAL execution identity. If unfinished or unverified, publish task.failed with the real reason; do not claim success. Follow the canonical event contract. Do not emit task.progress merely because this reminder arrived.`;
|
|
13
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { hasSafeAlternativeWorker } from './browser-worker-routing.js';
|
|
2
|
+
import { codexNoStartFallbackRoute, persistChatGptPlatformFailure } from './chatgpt-platform-fallback.js';
|
|
3
|
+
import { classifyChatGptPlatformFailure } from './worker-rejection.js';
|
|
4
|
+
import { recordLaunchRouteOutcome } from './route-outcome-recording.js';
|
|
5
|
+
import { eventFor } from './events.js';
|
|
6
|
+
export async function handleWorkerLaunchFailure(ports, jobId, taskId, executionId, failure, phase = 'launch') {
|
|
7
|
+
const message = failure instanceof Error ? failure.message : String(failure);
|
|
8
|
+
const currentJob = await ports.store.load(jobId);
|
|
9
|
+
const currentTask = currentJob.tasks.find(task => task.id === taskId);
|
|
10
|
+
const agents = await ports.store.listAgents();
|
|
11
|
+
const usage = ports.usageSnapshotProvider ? await ports.usageSnapshotProvider() : undefined;
|
|
12
|
+
const alternative = hasSafeAlternativeWorker(currentTask, ports.adapters, agents, usage);
|
|
13
|
+
const activeExecution = ports.live(taskId);
|
|
14
|
+
const noStarted = currentTask?.state === 'QUEUED'
|
|
15
|
+
&& currentTask.executionId === executionId
|
|
16
|
+
&& !currentTask.lastProgressEventAt
|
|
17
|
+
&& !activeExecution?.workerStarted;
|
|
18
|
+
const canRetryNoStart = noStarted && currentJob.executionMode !== 'shortcut'
|
|
19
|
+
&& (currentTask?.adapter === 'chatgpt' || currentTask?.adapter === 'z')
|
|
20
|
+
&& currentTask.agentName !== 'reviewer'
|
|
21
|
+
&& currentTask.agentName !== 'troubleshooter' && currentTask.kind !== 'planner'
|
|
22
|
+
&& !currentTask.noStartFallback && currentTask.attempt < currentTask.maxAttempts;
|
|
23
|
+
const codexFallback = canRetryNoStart
|
|
24
|
+
? await codexNoStartFallbackRoute(currentTask, ports.adapters, agents, usage)
|
|
25
|
+
: null;
|
|
26
|
+
const noStartFallbackAllowed = Boolean(canRetryNoStart && codexFallback);
|
|
27
|
+
const platformFailure = (currentTask?.adapter === 'chatgpt' || currentTask?.adapter === 'z')
|
|
28
|
+
? classifyChatGptPlatformFailure(failure, phase === 'process' || phase === 'terminal' ? 'delivery' : phase)
|
|
29
|
+
: undefined;
|
|
30
|
+
const unknownOutcome = phase === 'start-ack' || ['outcome_unknown', 'start_ack_timeout'].includes(platformFailure?.classification ?? '')
|
|
31
|
+
|| (phase === 'launch' && activeExecution?.executionId === executionId)
|
|
32
|
+
|| (phase === 'delivery' && activeExecution?.executionId === executionId && activeExecution.workerStarted);
|
|
33
|
+
const failureReason = phase === 'start-ack' ? 'worker_start_timeout'
|
|
34
|
+
: /WEBHELM_UNAVAILABLE|webhelm runtime dependency unavailable|webhelm session start returned no session_id|no close frame received or sent/i.test(message)
|
|
35
|
+
? 'browser_workspace_unavailable'
|
|
36
|
+
: undefined;
|
|
37
|
+
const claim = executionId ?? `${jobId}:${taskId}`;
|
|
38
|
+
if (ports.launchFailureClaims.has(claim))
|
|
39
|
+
return;
|
|
40
|
+
ports.launchFailureClaims.add(claim);
|
|
41
|
+
const cleanup = setTimeout(() => ports.launchFailureClaims.delete(claim), 60000);
|
|
42
|
+
cleanup.unref();
|
|
43
|
+
const workflowEvent = noStartFallbackAllowed ? 'worker.no_start_failed'
|
|
44
|
+
: platformFailure ? 'worker.platform_failure'
|
|
45
|
+
: phase === 'start-ack' ? 'worker.start_ack_failed' : 'worker.delivery_failed';
|
|
46
|
+
const transition = await ports.taskTransitions.apply(jobId, taskId, workflowEvent, {
|
|
47
|
+
eligible: (_job, task) => task.executionId === executionId,
|
|
48
|
+
facts: () => ({
|
|
49
|
+
platformFallbackSafe: platformFailure?.safeToReroute === true && alternative,
|
|
50
|
+
noStartFallbackAllowed: noStartFallbackAllowed && !currentTask?.noStartFallback,
|
|
51
|
+
}),
|
|
52
|
+
mutate: (_job, task, decision) => {
|
|
53
|
+
if (platformFailure && workflowEvent !== 'worker.no_start_failed')
|
|
54
|
+
persistChatGptPlatformFailure(task, currentTask, executionId, platformFailure.classification, platformFailure.evidence, platformFailure.safeToReroute);
|
|
55
|
+
if (workflowEvent === 'worker.no_start_failed' && task.noStartFallback)
|
|
56
|
+
task.noStartFallback.reason = message;
|
|
57
|
+
task.error = decision.target === 'QUEUED' ? null : `Worker ${phase} failed: ${message}`;
|
|
58
|
+
task.failureReason = decision.target === 'QUEUED' ? undefined : unknownOutcome
|
|
59
|
+
? 'worker_outcome_unknown' : platformFailure ? 'worker_platform_failure' : failureReason;
|
|
60
|
+
task.leaseOwner = null;
|
|
61
|
+
task.leaseExpiresAt = null;
|
|
62
|
+
task.executionId = null;
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
if (!transition)
|
|
66
|
+
return;
|
|
67
|
+
if (!transition.decision.accepted) {
|
|
68
|
+
await ports.emit(eventFor(jobId, taskId, transition.task.parentTaskId, 'task.process.delivery_failed', 'failed', `Worker ${phase} reported after lifecycle moved to ${transition.from}: ${message}`, 'orchestrator', { phase, authoritative: false, executionId, workflowEvent }, { agent: 'relay', component: 'agents-relay' }));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
await recordLaunchRouteOutcome(ports.decisionAuditStore, currentTask ?? transition.task, message, phase === 'process' || phase === 'terminal' ? 'delivery' : phase, executionId, unknownOutcome);
|
|
72
|
+
if (executionId) {
|
|
73
|
+
const live = ports.live(taskId);
|
|
74
|
+
if (live?.executionId === executionId) {
|
|
75
|
+
live.controller.abort();
|
|
76
|
+
live.execution.cancel();
|
|
77
|
+
ports.clearExecution(taskId, executionId);
|
|
78
|
+
}
|
|
79
|
+
// Browser after-terminal waits must not outlive this failed generation:
|
|
80
|
+
// otherwise they could reject a legitimate terminal event from Codex.
|
|
81
|
+
ports.clearTerminalWaiter(taskId, executionId);
|
|
82
|
+
}
|
|
83
|
+
const event = eventFor(jobId, taskId, transition.task.parentTaskId, transition.task.state === 'QUEUED' && workflowEvent === 'worker.no_start_failed'
|
|
84
|
+
? 'worker.no_start_fallback' : transition.task.state === 'QUEUED' ? 'worker.pre_submission_rejected' : 'task.failed', transition.task.state === 'QUEUED' ? 'queued' : 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason ?? failureReason, workflowEvent,
|
|
85
|
+
...(transition.task.noStartFallback ? { noStartFallback: transition.task.noStartFallback } : {}),
|
|
86
|
+
...(transition.task.platformFailure ? { platformFailure: transition.task.platformFailure } : {}) }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
87
|
+
ports.detach(ports.emit(event), jobId, taskId, 'worker failure event');
|
|
88
|
+
ports.wake(jobId, taskId, `worker-${phase}-failure`);
|
|
89
|
+
}
|
|
@@ -50,6 +50,7 @@ export const WORKFLOW_EVENT_NAMES = [
|
|
|
50
50
|
'worker.delivery_failed',
|
|
51
51
|
'worker.start_ack_failed',
|
|
52
52
|
'worker.platform_failure',
|
|
53
|
+
'worker.no_start_failed',
|
|
53
54
|
];
|
|
54
55
|
export const WORKFLOW_FACT_NAMES = [
|
|
55
56
|
'dependenciesSatisfied',
|
|
@@ -59,6 +60,7 @@ export const WORKFLOW_FACT_NAMES = [
|
|
|
59
60
|
'shortcutExecution',
|
|
60
61
|
'shortcutFallback',
|
|
61
62
|
'shortcutHasGraph',
|
|
63
|
+
'shortcutDirectInvocation',
|
|
62
64
|
'shortcutEffectsSafe',
|
|
63
65
|
'shortcutResultCorrelated',
|
|
64
66
|
'notMerged',
|
|
@@ -74,6 +76,7 @@ export const WORKFLOW_FACT_NAMES = [
|
|
|
74
76
|
'headAvailable',
|
|
75
77
|
'mergeCapable',
|
|
76
78
|
'platformFallbackSafe',
|
|
79
|
+
'noStartFallbackAllowed',
|
|
77
80
|
];
|
|
78
81
|
export const WORKFLOW_MUTATION_NAMES = [
|
|
79
82
|
'mutation.clearError',
|
|
@@ -87,6 +90,7 @@ export const WORKFLOW_MUTATION_NAMES = [
|
|
|
87
90
|
'mutation.markShortcutWorkerFallback',
|
|
88
91
|
'mutation.markShortcutRecoveryRequired',
|
|
89
92
|
'mutation.preparePlatformFallback',
|
|
93
|
+
'mutation.prepareNoStartFallback',
|
|
90
94
|
'mutation.quarantinePlatformFailure',
|
|
91
95
|
];
|
|
92
96
|
export const WORKFLOW_EFFECT_NAMES = [
|
|
@@ -32,6 +32,7 @@ export const taskLifecycleV2 = load('task-lifecycle', TASK_STATES, {
|
|
|
32
32
|
shortcutExecution: facts => facts.durableFacts.shortcutExecution === true,
|
|
33
33
|
shortcutFallback: facts => facts.durableFacts.shortcutFallback === true,
|
|
34
34
|
shortcutHasGraph: facts => facts.durableFacts.shortcutHasGraph === true,
|
|
35
|
+
shortcutDirectInvocation: facts => facts.durableFacts.shortcutDirectInvocation === true,
|
|
35
36
|
shortcutEffectsSafe: facts => facts.durableFacts.shortcutEffectsSafe === true,
|
|
36
37
|
shortcutResultCorrelated: facts => facts.durableFacts.shortcutResultCorrelated === true,
|
|
37
38
|
recoverableExecution: boolGuard('recoverableExecution'),
|
|
@@ -40,6 +41,7 @@ export const taskLifecycleV2 = load('task-lifecycle', TASK_STATES, {
|
|
|
40
41
|
awaitingInteractiveDecision: facts => facts.durableFacts.awaitingInteractiveDecision === true,
|
|
41
42
|
retryAvailable: boolGuard('retryAvailable'),
|
|
42
43
|
platformFallbackSafe: facts => facts.durableFacts.platformFallbackSafe === true,
|
|
44
|
+
noStartFallbackAllowed: facts => facts.durableFacts.noStartFallbackAllowed === true,
|
|
43
45
|
});
|
|
44
46
|
export const jobLifecycleV2 = load('job-lifecycle', JOB_STATES, {
|
|
45
47
|
notMerged: boolGuard('notMerged'),
|
package/dist/workflow/runtime.js
CHANGED
|
@@ -47,6 +47,36 @@ export function applyTaskMutation(task, mutation) {
|
|
|
47
47
|
if (mutation === 'mutation.markShortcutRecoveryRequired') {
|
|
48
48
|
task.failureReason = 'shortcut_requires_recovery';
|
|
49
49
|
}
|
|
50
|
+
if (mutation === 'mutation.prepareNoStartFallback') {
|
|
51
|
+
const route = task.routing;
|
|
52
|
+
if (task.adapter !== 'chatgpt' && task.adapter !== 'z')
|
|
53
|
+
throw new Error('No-start fallback is only legal for ChatGPT/Z Workers');
|
|
54
|
+
const originalAdapter = task.adapter;
|
|
55
|
+
const workerId = task.workerSelection?.workerId ?? originalAdapter;
|
|
56
|
+
const failedAt = new Date().toISOString();
|
|
57
|
+
task.noStartFallback = {
|
|
58
|
+
failedExecutionId: task.executionId ?? 'missing',
|
|
59
|
+
failedAttempt: task.attempt,
|
|
60
|
+
previous: { adapter: originalAdapter, workerId,
|
|
61
|
+
provider: route?.provider ?? 'unknown', model: route?.model ?? 'unknown',
|
|
62
|
+
...(route?.profile ? { profile: route.profile } : {}) },
|
|
63
|
+
reason: task.error ?? 'Worker failed without an accepted task.started event',
|
|
64
|
+
at: failedAt,
|
|
65
|
+
};
|
|
66
|
+
task.routingExclusions = [...(task.routingExclusions ?? []), {
|
|
67
|
+
adapter: originalAdapter, workerId, provider: route?.provider ?? 'unknown',
|
|
68
|
+
model: route?.model ?? 'unknown',
|
|
69
|
+
...(route?.profile ? { profile: route.profile } : {}),
|
|
70
|
+
reason: 'no_started_event', at: failedAt,
|
|
71
|
+
}];
|
|
72
|
+
task.error = null;
|
|
73
|
+
task.failureReason = undefined;
|
|
74
|
+
task.routing = undefined;
|
|
75
|
+
task.workerSelection = undefined;
|
|
76
|
+
task.leaseOwner = null;
|
|
77
|
+
task.leaseExpiresAt = null;
|
|
78
|
+
task.executionId = null;
|
|
79
|
+
}
|
|
50
80
|
if (mutation === 'mutation.preparePlatformFallback') {
|
|
51
81
|
task.error = null;
|
|
52
82
|
task.failureReason = undefined;
|
|
@@ -11,7 +11,6 @@ export const taskResolutionV2 = new GraphRuntime(source, {
|
|
|
11
11
|
guards: {
|
|
12
12
|
recoveryOwnerAvailable: boolGuard('recoveryOwnerAvailable'), plannerOwnerAvailable: boolGuard('plannerOwnerAvailable'),
|
|
13
13
|
exactOriginalRetry: boolGuard('exactOriginalRetry'), sameJobDependency: boolGuard('sameJobDependency'),
|
|
14
|
-
shortcutInterrupted: boolGuard('shortcutInterrupted'),
|
|
15
14
|
blockerJobDependency: boolGuard('blockerJobDependency'), escalationGate: boolGuard('escalationGate'), genuinelyObsolete: boolGuard('genuinelyObsolete'),
|
|
16
15
|
freshRecoveryExecution: boolGuard('freshRecoveryExecution'),
|
|
17
16
|
classifiableFailure: boolGuard('classifiableFailure'),
|
|
@@ -65,9 +64,7 @@ export function validateResolutionProposal(job, original, proposal) {
|
|
|
65
64
|
export function resolutionFacts(task, proposal, owner) {
|
|
66
65
|
const freshRecoveryExecution = owner !== 'troubleshooter' || proposal.path !== 'exact_original_retry' || (Boolean(proposal.recoveryExecutionId)
|
|
67
66
|
&& proposal.recoveryExecutionId !== task.resolution?.recoveryExecutionId);
|
|
68
|
-
|
|
69
|
-
const shortcutInterrupted = owner === 'troubleshooter' && Boolean(shortcut && typeof shortcut === 'object' && Object.keys(shortcut.inFlight ?? {}).length > 0);
|
|
70
|
-
return { recoveryOwnerAvailable: owner === 'troubleshooter', plannerOwnerAvailable: owner !== null && owner !== 'troubleshooter', exactOriginalRetry: proposal.path === 'exact_original_retry' && task.state === 'FAILED', shortcutInterrupted, sameJobDependency: proposal.path === 'same_job_dependency' && owner !== null && proposal.targetTaskId !== task.id, blockerJobDependency: proposal.path === 'blocker_job_dependency' && owner !== null && Boolean(proposal.targetJobId && proposal.mode), escalationGate: proposal.path === 'escalation_gate' && owner === 'troubleshooter' && Boolean(proposal.rootCause && proposal.solution), genuinelyObsolete: proposal.path === 'cancel_supersede' && Boolean(proposal.rootCause), freshRecoveryExecution, unresolved: unresolvedTask(task) };
|
|
67
|
+
return { recoveryOwnerAvailable: owner === 'troubleshooter', plannerOwnerAvailable: owner !== null && owner !== 'troubleshooter', exactOriginalRetry: proposal.path === 'exact_original_retry' && task.state === 'FAILED', sameJobDependency: proposal.path === 'same_job_dependency' && owner !== null && proposal.targetTaskId !== task.id, blockerJobDependency: proposal.path === 'blocker_job_dependency' && owner !== null && Boolean(proposal.targetJobId && proposal.mode), escalationGate: proposal.path === 'escalation_gate' && owner === 'troubleshooter' && Boolean(proposal.rootCause && proposal.solution), genuinelyObsolete: proposal.path === 'cancel_supersede' && Boolean(proposal.rootCause), freshRecoveryExecution, unresolved: unresolvedTask(task) };
|
|
71
68
|
}
|
|
72
69
|
export function decideTaskResolution(task, proposal, owner) {
|
|
73
70
|
return checkedResolutionDecision(dispatchWorkflowGraphSync(taskResolutionV2, { initialState: 'unresolved', input: { transport: 'event', id: 'resolution.propose', payload: proposal }, durableFacts: resolutionFacts(task, proposal, owner), graphName: 'task-resolution', isTransientState: state => ['check_owner', 'check_path', 'check_fresh_execution'].includes(state), maxSteps: 8 }));
|