agents-relay 2.0.207 → 2.0.209

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/adapters.js +2 -1
  2. package/dist/agent-shortcut.js +113 -41
  3. package/dist/audit-tail.js +52 -0
  4. package/dist/browser-worker-runtime.js +7 -2
  5. package/dist/chatgpt-platform-fallback.js +30 -0
  6. package/dist/cli-shortcut-local.js +49 -0
  7. package/dist/cli-shortcut.js +81 -19
  8. package/dist/cli.js +8 -3
  9. package/dist/decision-audit.js +143 -1
  10. package/dist/execution-routing.js +8 -2
  11. package/dist/force-restate-validation.js +3 -1
  12. package/dist/job-execution-slot.js +4 -1
  13. package/dist/model-routing.js +8 -0
  14. package/dist/reconciler-review.js +39 -13
  15. package/dist/reconciler-runtime.js +46 -52
  16. package/dist/reconciler.js +12 -2
  17. package/dist/relay-job-snapshot-reads.js +5 -2
  18. package/dist/relay-workflow-operations.js +1 -1
  19. package/dist/relayd.js +1 -1
  20. package/dist/route-outcome-recording.js +51 -0
  21. package/dist/runtime-status.js +11 -4
  22. package/dist/shortcut-node-worker.js +128 -0
  23. package/dist/shortcut-stage-contract.js +50 -0
  24. package/dist/shortcut-worker-runner.js +73 -0
  25. package/dist/task-actions.js +4 -13
  26. package/dist/task-inactivity-watchdog.js +7 -0
  27. package/dist/terminal-event-recovery.js +13 -0
  28. package/dist/worker-failure-recovery.js +89 -0
  29. package/dist/workflow/contracts.js +4 -0
  30. package/dist/workflow/definitions-v2.js +2 -0
  31. package/dist/workflow/runtime.js +30 -0
  32. package/dist/workflow/task-resolution.js +1 -4
  33. package/package.json +4 -3
  34. package/skills/agents-relay/agents/planner.agent.md +14 -4
  35. package/skills/agents-relay/agents/reviewer.agent.md +2 -0
  36. package/skills/agents-relay/contracts/orchestrator-workflows.md +6 -0
  37. package/skills/agents-relay/contracts/task-job-contract.md +4 -0
  38. package/workflow/force-restate-validation.mmd +1 -1
  39. package/workflow/review-lifecycle.mmd +1 -1
  40. package/workflow/task-lifecycle.mmd +13 -12
  41. package/workflow/task-resolution.mmd +1 -6
package/dist/cli.js CHANGED
@@ -50,6 +50,7 @@ import { requestTroubleshooterEscalation, resumeTroubleshooter } from './task-re
50
50
  import { pauseJob, resumeJob } from './job-pause.js';
51
51
  export { ensureManagedGitHubJob } from './cli-job-create.js';
52
52
  import { recordShortcutReceiptCommand, runShortcutJobCommand } from './cli-shortcut.js';
53
+ import { runLocalShortcutBinding } from './cli-shortcut-local.js';
53
54
  import { runContextCommand } from './cli-context.js';
54
55
  async function runManagedTaskCommand(action, args) {
55
56
  const jobId = arg(args, '--id');
@@ -79,6 +80,8 @@ async function runManagedTaskCommand(action, args) {
79
80
  console.log(JSON.stringify(task, null, 2));
80
81
  }
81
82
  async function runTaskCommand(action, args) {
83
+ if (hasArg(args, '--shortcut-local'))
84
+ return runLocalShortcutBinding('task', action, args);
82
85
  if (!['create', 'update', 'show', 'context', 'events', 'comment', 'comments', 'list', 'retry', 'recover', 'unblock', 'escalate', 'resume-troubleshooter', 'cancel', 'supersede', 'classify'].includes(action))
83
86
  throw new Error(usage);
84
87
  if (action === 'context' || action === 'events') {
@@ -381,6 +384,8 @@ async function runTaskCommand(action, args) {
381
384
  console.log(JSON.stringify(task, null, 2));
382
385
  }
383
386
  async function runJobCommand(action, args) {
387
+ if (hasArg(args, '--shortcut-local'))
388
+ return runLocalShortcutBinding('job', action, args);
384
389
  if (!['create', 'adopt', 'update', 'show', 'list', 'merge', 'unblock', 'cancel', 'pause', 'resume', 'run', 'shortcut-receipt'].includes(action))
385
390
  throw new Error(usage);
386
391
  const repo = repository(args);
@@ -452,7 +457,7 @@ async function runJobCommand(action, args) {
452
457
  return;
453
458
  }
454
459
  if (action === 'run') {
455
- await runShortcutJobCommand(args, id, repo);
460
+ await runShortcutJobCommand(args, id, repo, (store, bus) => runtime(store, args, bus, codexAndZaiUsageRegistry(), true));
456
461
  return;
457
462
  }
458
463
  if (action === 'shortcut-receipt') {
@@ -540,12 +545,12 @@ async function runJobCommand(action, args) {
540
545
  console.log(JSON.stringify(await relayControlCommand(args, { command: 'job.update', jobId: id, patch }), null, 2));
541
546
  }
542
547
  export function runtimePlanner(args) { const plannerCommand = arg([...args], '--planner-command'); const modelRuntime = arg([...args], '--codex', 'codex'); return plannerCommand ? new CommandObjectivePlanner(plannerCommand) : new AgentObjectivePlanner(modelRuntime); }
543
- export function runtime(store, args, bus, usageRegistry = codexAndZaiUsageRegistry()) {
548
+ export function runtime(store, args, bus, usageRegistry = codexAndZaiUsageRegistry(), shortcutRunnerOwner = false) {
544
549
  const emit = (event) => { process.stderr.write(`${JSON.stringify(event)}\n`); };
545
550
  const modelRuntime = arg(args, '--codex', 'codex');
546
551
  const policy = chatGptTabClosePolicy(args);
547
552
  const planner = runtimePlanner(args);
548
- return new Reconciler(store, { noGitHub: noGitHubMode(args), owner: arg(args, '--owner', `cli-${process.pid}`), maxConcurrent: Number(arg(args, '--concurrency', '4')), leaseMs: Number(arg(args, '--lease-ms', '300000')), adapters: [new CodexAdapter(modelRuntime), new ChatGptAdapter({ tabClosePolicy: policy }), new ZAdapter()], continuations: [new CodexThreadContinuation(modelRuntime), new CommandContinuation(), new WebhookContinuation()], planner, usageSnapshotProvider: () => usageRegistry.snapshot(), decisionEngine: executionDecisionEngine(), decisionMode: decisionMode(), reviewer: runtimeReviewer(args), eventBus: bus, emit, chatgptTabClosePolicy: policy, executionContextRoots: { workspaceRoot: arg(args, '--workspace-root') || process.env.AGENTS_RELAY_WORKSPACE_ROOT }, projectPathForJob: async (job, task) => {
553
+ return new Reconciler(store, { shortcutRunnerOwner, noGitHub: noGitHubMode(args), owner: arg(args, '--owner', `cli-${process.pid}`), maxConcurrent: Number(arg(args, '--concurrency', '4')), leaseMs: Number(arg(args, '--lease-ms', '300000')), adapters: [new CodexAdapter(modelRuntime), new ChatGptAdapter({ tabClosePolicy: policy }), new ZAdapter()], continuations: [new CodexThreadContinuation(modelRuntime), new CommandContinuation(), new WebhookContinuation()], planner, usageSnapshotProvider: () => usageRegistry.snapshot(), decisionEngine: executionDecisionEngine(), decisionMode: decisionMode(), reviewer: runtimeReviewer(args), eventBus: bus, emit, chatgptTabClosePolicy: policy, executionContextRoots: { workspaceRoot: arg(args, '--workspace-root') || process.env.AGENTS_RELAY_WORKSPACE_ROOT }, projectPathForJob: async (job, task) => {
549
554
  const workspaceRoot = await resolveWorkspaceRoot(arg(args, '--workspace-root') || undefined);
550
555
  const canonical = await resolveManagedProjectPath(noGitHubMode(args) ? undefined : job.repository, arg(args, '--project-path'), workspaceRoot);
551
556
  if (!(task ? taskRequiresManagedWorktree(job, task) : Boolean(job.localMerge?.sourceRef)))
@@ -1,13 +1,155 @@
1
1
  import { appendFile, mkdir, readFile } from 'node:fs/promises';
2
2
  import { dirname } from 'node:path';
3
+ import { readJsonlNewestFirst } from './audit-tail.js';
3
4
  import { randomUUID, createHash } from 'node:crypto';
4
5
  import { modelChoiceId } from './model-routing.js';
5
6
  export const DECISION_AUDIT_SCHEMA_VERSION = 5;
7
+ function textField(value) {
8
+ return typeof value === 'string' && value.trim() ? value : null;
9
+ }
10
+ /** Avoid passing long worker logs, inline credentials or private task content to the decision engine. */
11
+ function routeReason(value) {
12
+ return value.replace(/\s+/g, ' ')
13
+ .replace(/\b(Bearer\s+)[A-Za-z0-9._~-]+/gi, '$1[redacted]')
14
+ .replace(/\b((?:api[_ -]?key|secret|password|access[_ -]?token|token|credential|authorization)\s*[:=]\s*)[^\s,;]+/gi, '$1[redacted]')
15
+ .slice(0, 240);
16
+ }
17
+ /**
18
+ * Router facts cross Job boundaries. Never copy arbitrary worker error messages
19
+ * into a later Job's Laya prompt: only deterministic, content-free reason codes
20
+ * may leave this private audit store.
21
+ */
22
+ function routerReasonCode(kind, summary) {
23
+ if (kind === 'success')
24
+ return 'task_completed';
25
+ if (kind === 'cancelled')
26
+ return 'task_cancelled';
27
+ if (kind === 'blocked')
28
+ return 'task_blocked';
29
+ if (kind === 'outcome_unknown')
30
+ return 'execution_outcome_unknown';
31
+ if (kind === 'auth_config_failure')
32
+ return 'authentication_rejected';
33
+ if (kind === 'timeout')
34
+ return 'worker_timeout';
35
+ if (kind === 'launch_failure')
36
+ return 'worker_launch_failed';
37
+ if (kind === 'provider_failure') {
38
+ if (/stream disconnected|response\.completed/i.test(summary))
39
+ return 'provider_stream_disconnected';
40
+ if (/rate[_ .-]?limit(?:[_ .-]?exceeded)?|HTTP 429/i.test(summary))
41
+ return 'provider_rate_limited';
42
+ if (/quota|credits? exhausted/i.test(summary))
43
+ return 'provider_quota_exhausted';
44
+ return 'provider_failure';
45
+ }
46
+ if (kind === 'runtime_failure')
47
+ return 'worker_runtime_failure';
48
+ if (kind === 'semantic_failure')
49
+ return 'task_semantic_failure';
50
+ return 'task_reported_failure';
51
+ }
52
+ function routeResult(record, route) {
53
+ const worker = textField(route.worker);
54
+ const adapter = textField(route.adapter);
55
+ const provider = textField(route.provider);
56
+ const model = textField(route.model);
57
+ if (!worker || !adapter || !provider || !model)
58
+ return null;
59
+ return {
60
+ timestamp: record.timestamp,
61
+ worker, adapter, provider, model,
62
+ profile: textField(route.profile),
63
+ reasoning: textField(route.reasoning),
64
+ result: record.kind === 'success' ? 'success' : record.kind === 'cancelled' ? 'cancelled'
65
+ : record.kind === 'blocked' ? 'blocked' : record.kind === 'outcome_unknown' ? 'unknown' : 'failure',
66
+ failure_kind: ['success', 'cancelled', 'blocked', 'outcome_unknown'].includes(record.kind) ? null : record.kind,
67
+ reason: routerReasonCode(record.kind, record.summary),
68
+ };
69
+ }
70
+ function outcomeAttemptKey(record) {
71
+ const evidence = record.evidence ?? {};
72
+ const attempt = typeof evidence.attempt === 'number' && Number.isSafeInteger(evidence.attempt) ? String(evidence.attempt) : null;
73
+ const execution = textField(evidence.execution_id);
74
+ return JSON.stringify([record.decision_id, execution ?? (attempt ? 'attempt:' + attempt : 'legacy')]);
75
+ }
76
+ /**
77
+ * Join durable decision and outcome records by decision ID, retaining only the
78
+ * latest terminal outcome per execution attempt. History is global across Jobs and sorted
79
+ * by outcome time, not by when a route was chosen. Only completed attempts count.
80
+ */
81
+ export async function recentExecutionRouteResults(store, limit = 10) {
82
+ if (store.readRecentRoutes)
83
+ return store.readRecentRoutes(limit);
84
+ const records = await store.read();
85
+ const decisions = new Map();
86
+ const outcomes = new Map();
87
+ records.forEach((record, order) => {
88
+ if (record.record_type === 'decision') {
89
+ decisions.set(record.decision_id, record);
90
+ }
91
+ else if (record.kind !== 'task_started') {
92
+ // Reused decisions from an authorized retry are separate executions.
93
+ const key = outcomeAttemptKey(record);
94
+ const current = outcomes.get(key);
95
+ if (!current || record.timestamp > current.record.timestamp || (record.timestamp === current.record.timestamp && order > current.order)) {
96
+ outcomes.set(key, { record, order });
97
+ }
98
+ }
99
+ });
100
+ const results = [];
101
+ for (const { record, order } of outcomes.values()) {
102
+ const decision = decisions.get(record.decision_id);
103
+ const route = decision?.selected;
104
+ const worker = route?.worker && typeof route.worker === 'object' ? route.worker : undefined;
105
+ const model = route?.model && typeof route.model === 'object' ? route.model : undefined;
106
+ const result = routeResult(record, {
107
+ worker: worker?.id, adapter: worker?.adapter, provider: model?.provider, model: model?.model,
108
+ profile: model?.profile, reasoning: model?.reasoning,
109
+ });
110
+ if (result)
111
+ results.push({ ...result, order });
112
+ }
113
+ return results.sort((a, b) => b.timestamp.localeCompare(a.timestamp) || b.order - a.order)
114
+ .slice(0, Math.max(0, Math.min(10, Math.trunc(limit))))
115
+ .map(({ order: _order, ...result }) => result);
116
+ }
6
117
  export class JsonlDecisionAuditStore {
7
118
  path;
8
119
  constructor(path) {
9
120
  this.path = path;
10
121
  }
122
+ /** The normal path reads backward only until ten distinct completed attempts.
123
+ * Legacy outcomes without embedded routing facts use a one-time compatibility
124
+ * fallback to the full audit reader; new outcomes are self-contained. */
125
+ async readRecentRoutes(limit = 10) {
126
+ const cap = Math.max(0, Math.min(10, Math.trunc(limit)));
127
+ if (cap === 0)
128
+ return [];
129
+ const results = [];
130
+ const seen = new Set();
131
+ let needsLegacyJoin = false;
132
+ for await (const record of readJsonlNewestFirst(this.path)) {
133
+ if (record.record_type !== 'outcome' || record.kind === 'task_started')
134
+ continue;
135
+ const key = outcomeAttemptKey(record);
136
+ if (seen.has(key))
137
+ continue;
138
+ seen.add(key);
139
+ const route = record.evidence?.route;
140
+ const result = route && typeof route === 'object' && !Array.isArray(route)
141
+ ? routeResult(record, route) : null;
142
+ if (result)
143
+ results.push(result);
144
+ else
145
+ needsLegacyJoin = true;
146
+ if (seen.size >= cap)
147
+ break;
148
+ }
149
+ if (needsLegacyJoin)
150
+ return recentExecutionRouteResults({ read: () => this.read() }, cap);
151
+ return results.sort((a, b) => b.timestamp.localeCompare(a.timestamp)).slice(0, cap);
152
+ }
11
153
  async append(record) { await mkdir(dirname(this.path), { recursive: true }); await appendFile(this.path, `${JSON.stringify(sanitize(record))}\n`, 'utf8'); }
12
154
  async read() {
13
155
  try {
@@ -29,7 +171,7 @@ function sanitize(value, key = '') {
29
171
  }
30
172
  export function decisionAuditId() { return randomUUID(); }
31
173
  export function auditInputHash(record) { return createHash('sha256').update(JSON.stringify(record)).digest('hex'); }
32
- export function outcomeForDecision(decisionId, kind, summary, evidence) { return { record_type: 'outcome', schema_version: DECISION_AUDIT_SCHEMA_VERSION, outcome_id: randomUUID(), decision_id: decisionId, timestamp: new Date().toISOString(), kind, summary, evidence }; }
174
+ export function outcomeForDecision(decisionId, kind, summary, evidence) { return { record_type: 'outcome', schema_version: DECISION_AUDIT_SCHEMA_VERSION, outcome_id: randomUUID(), decision_id: decisionId, timestamp: new Date().toISOString(), kind, summary: routeReason(summary), evidence }; }
33
175
  export function taskAuditFacts(task) {
34
176
  if (!task)
35
177
  return {};
@@ -1,4 +1,4 @@
1
- import { makeExecutionDecisionAudit } from './decision-audit.js';
1
+ import { makeExecutionDecisionAudit, recentExecutionRouteResults } from './decision-audit.js';
2
2
  import { discoverModelCandidates } from './model-routing.js';
3
3
  import { loadExecutionRouterRules, loadRoutingPolicy, policyAllows } from './routing-policy.js';
4
4
  import { DEFAULT_CHATGPT_PREFERRED_CONCURRENCY } from './worker-routing.js';
@@ -274,6 +274,10 @@ export async function routeExecution(workers, requirements, options = {}) {
274
274
  const chatgptRunning = options.chatgptRunning ?? 0;
275
275
  const taskFacts = options.taskFacts ?? {};
276
276
  const taskShape = executionTaskShape(taskFacts);
277
+ const historyStore = options.historyStore ?? options.auditStore;
278
+ const recentRouteResults = historyStore
279
+ ? await recentExecutionRouteResults(historyStore).catch(() => []) // Advisory history cannot block execution.
280
+ : [];
277
281
  const discoveredDefaults = new Map();
278
282
  for (const worker of workers) {
279
283
  const discovered = await discoverModelCandidates(worker, options.registered ?? [], options.usage, undefined, options.rulesPath);
@@ -318,8 +322,9 @@ export async function routeExecution(workers, requirements, options = {}) {
318
322
  chatgpt_running: chatgptRunning,
319
323
  preferred_chatgpt_concurrency: preferredChatgptConcurrency,
320
324
  },
325
+ recent_route_results: recentRouteResults,
321
326
  }));
322
- const question = appendRoutingInstruction('User routing instructions have highest priority over default preferences, task size, runtime and resource signals. Candidate order is arbitrary and conveys no priority, ranking or preference. Do not favor earlier choices. Select an offered route that obeys the user instructions.\nUser routing instructions:\n' + routingInstruction, 'For vision, image generation, visual review, and final review tasks, prefer ChatGPT Worker when available and within capacity; for coding, CLI and filesystem tasks, prefer Codex. These are soft preferences subordinate to user rules. Choose exactly one listed route; do not invent routes.');
327
+ const question = appendRoutingInstruction('User routing instructions have highest priority over default preferences, task size, runtime and resource signals. Candidate order is arbitrary and conveys no priority, ranking or preference. Do not favor earlier choices. Select an offered route that obeys the user instructions.\nUser routing instructions:\n' + routingInstruction, 'Use recent_route_results (up to the last 10 confirmed execution outcomes) to weigh route reliability, including success, failure and the recorded reason. Distinguish provider/runtime faults from task-specific failures and cancellations; historical failures are evidence, not automatic hard exclusions. For vision, image generation, visual review, and final review tasks, prefer ChatGPT Worker when available and within capacity; for coding, CLI and filesystem tasks, prefer Codex. These are soft preferences subordinate to user rules. Choose exactly one listed route; do not invent routes.');
323
328
  let selected = fallback;
324
329
  let evidence = { mode: options.decisionMode ?? 'off', fallback: true };
325
330
  const decisionStages = [];
@@ -380,6 +385,7 @@ export async function routeExecution(workers, requirements, options = {}) {
380
385
  candidateRoutes: routes.map(route => ({ id: route.id, workerId: route.worker.workerId, adapter: route.worker.adapter })),
381
386
  effectiveChoices,
382
387
  routingInstruction,
388
+ recentRouteResults,
383
389
  },
384
390
  policy,
385
391
  routing,
@@ -1,6 +1,7 @@
1
1
  import { classifyTaskResolution } from './workflow/task-resolution.js';
2
2
  import { tryTransitionTaskEvent } from './task-transitions.js';
3
3
  import { satisfiesWorker } from './worker-routing.js';
4
+ import { isSuperseded } from './runtime-status.js';
4
5
  function dependencySatisfied(task, byId) {
5
6
  return task.dependencies.every(id => {
6
7
  const dependency = byId.get(id);
@@ -74,8 +75,9 @@ export function objectiveSatisfiedForReview(job) {
74
75
  const latest = (job.plans ?? []).at(-1);
75
76
  if (latest?.status !== 'SUCCEEDED' || latest.objectiveStatus !== 'satisfied')
76
77
  return false;
78
+ const byId = new Map(job.tasks.map(task => [task.id, task]));
77
79
  const implementation = job.tasks.filter(task => task.kind !== 'planner' && task.agentName !== 'reviewer' && task.agentName !== 'troubleshooter'
78
- && task.resolutionClassification?.kind !== 'superseded');
80
+ && !isSuperseded(task, byId));
79
81
  return implementation.length > 0 && implementation.every(task => task.state === 'SUCCEEDED');
80
82
  }
81
83
  /** Apply Planner-proposed history dispositions to a private candidate snapshot.
@@ -8,11 +8,14 @@ export function jobHasActiveExecution(job, except = {}) {
8
8
  if (task.id === except.taskId)
9
9
  return false;
10
10
  const shortcut = task.result?.data?.shortcut;
11
+ const delegatedWorker = job.executionMode === 'shortcut' && job.contentOnly === true && task.state === 'RUNNING'
12
+ && task.result?.data?.shortcut && typeof task.result.data.shortcut === 'object'
13
+ && Object.keys(task.result.data.shortcut.workerNodes ?? {}).length > 0;
11
14
  const durableWait = job.executionMode === 'shortcut' && job.contentOnly === true && task.state === 'RUNNING'
12
15
  && task.leaseOwner === null && shortcut && typeof shortcut === 'object'
13
16
  && shortcut.waits
14
17
  && Object.keys(shortcut.waits).length > 0;
15
- return !durableWait && (task.state === 'RUNNING' || (task.executionId !== null && !['SUCCEEDED', 'FAILED', 'CANCELLED'].includes(task.state)));
18
+ return !durableWait && !delegatedWorker && (task.state === 'RUNNING' || (task.executionId !== null && !['SUCCEEDED', 'FAILED', 'CANCELLED'].includes(task.state)));
16
19
  });
17
20
  const planner = (job.plans ?? []).some(plan => plan.id !== except.planId
18
21
  && plan.executionId !== null && plan.status === 'PLANNING');
@@ -71,6 +71,14 @@ async function readConfig(path) {
71
71
  return null;
72
72
  }
73
73
  }
74
+ /** Only a real top-level Codex config model counts as a configured default. */
75
+ export async function configuredCodexDefault(codexHome = process.env.CODEX_HOME ?? join(homedir(), '.codex')) {
76
+ const config = await readConfig(join(codexHome, 'config.toml'));
77
+ return config?.model
78
+ ? { provider: config.provider ?? 'openai', model: config.model,
79
+ ...(config.reasoning ? { reasoning: config.reasoning } : {}) }
80
+ : null;
81
+ }
74
82
  function withResource(candidate, usage) {
75
83
  const resource = usageForProvider(usage, candidate.provider);
76
84
  return { ...candidate, resourceHealthy: resource.healthy, resourceUsedPercent: resource.usedPercent, resourceRemainingPercent: resource.remainingPercent, resourceResetAt: resource.resetAt, resourceResetInMinutes: resource.resetInMinutes, resourceWindows: resource.windows };
@@ -5,6 +5,7 @@ import { implementationComplete, implementationTasks } from './runtime-status.js
5
5
  import { eventFor } from './events.js';
6
6
  import { workerCandidates } from './worker-routing.js';
7
7
  import { routeExecution } from './execution-routing.js';
8
+ import { recordRouteOutcome } from './route-outcome-recording.js';
8
9
  import { ReconcilerRuntimeBase } from './reconciler-runtime.js';
9
10
  import { dispatchReviewLifecycle, reviewOutcome } from './workflow/review-v2.js';
10
11
  import { dispatchTaskLifecycleSync } from './workflow/lifecycle.js';
@@ -12,7 +13,7 @@ import { createTaskEntity } from './workflow/entity-creation.js';
12
13
  import { localSourceHead, localWorkingHead } from './local-git-merge.js';
13
14
  import { chatgptThreadRequirements, resolveReviewerThreadDestination, resolveImplementationTaskThreadDestination, prepareReviewerThreadDelivery } from './reviewer-thread.js';
14
15
  import { reviewGenerationMatches } from './review-generation-fence.js';
15
- import { quarantineUnavailableCodexFallback } from './chatgpt-platform-fallback.js';
16
+ import { codexNoStartFallbackRoute, quarantineUnavailableCodexFallback } from './chatgpt-platform-fallback.js';
16
17
  export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
17
18
  async currentHeadSha(job) {
18
19
  // Tests may replace the external Git effect entirely; in that case its
@@ -260,6 +261,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
260
261
  registered,
261
262
  chatgptRunning: runtimeSummary?.chatgptRunning ?? 0,
262
263
  routingConstraints: configured?.routing,
264
+ auditStore: this.options.decisionAuditStore,
263
265
  });
264
266
  return {
265
267
  candidate: routed.candidate,
@@ -313,18 +315,22 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
313
315
  : [];
314
316
  const taskExcludedRoutes = (task.routingExclusions ?? []).map(item => ({ provider: item.provider, model: item.model, profile: item.profile }));
315
317
  const excludedRoutes = [...taskExcludedRoutes, ...parentExcludedRoutes];
316
- const routed = await routeExecution(candidates, requirements, {
317
- decisionEngine: this.options.decisionEngine,
318
- decisionMode: this.options.decisionMode,
319
- taskFacts: { ...taskFacts, recovery: task.recoveryContext ?? null, recovery_parent_failed_route: excludedRoutes[0] ?? null },
320
- usage,
321
- registered,
322
- chatgptRunning: runtimeSummary?.chatgptRunning ?? 0,
323
- routingConstraints,
324
- excludedRoutes,
325
- auditStore: this.options.decisionAuditStore,
326
- auditTask: task,
327
- });
318
+ const routed = task.noStartFallback
319
+ ? await codexNoStartFallbackRoute(task, this.options.adapters, registered, usage)
320
+ : await routeExecution(candidates, requirements, {
321
+ decisionEngine: this.options.decisionEngine,
322
+ decisionMode: this.options.decisionMode,
323
+ taskFacts: { ...taskFacts, recovery: task.recoveryContext ?? null, recovery_parent_failed_route: excludedRoutes[0] ?? null },
324
+ usage,
325
+ registered,
326
+ chatgptRunning: runtimeSummary?.chatgptRunning ?? 0,
327
+ routingConstraints,
328
+ excludedRoutes,
329
+ auditStore: this.options.decisionAuditStore,
330
+ auditTask: task,
331
+ });
332
+ if (!routed)
333
+ throw new Error('Codex configured default unavailable for no-start recovery');
328
334
  task.workerSelection = routed.selection;
329
335
  task.adapter = routed.candidate.adapter;
330
336
  task.routing = routed.candidate.routing;
@@ -340,6 +346,21 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
340
346
  return await this.routeTaskForExecution(job, task);
341
347
  }
342
348
  catch (error) {
349
+ if (task.noStartFallback) {
350
+ const message = error instanceof Error ? error.message : String(error);
351
+ const transition = await this.taskTransitions.apply(job.id, task.id, 'fail', {
352
+ eligible: (_job, current) => current.state === 'QUEUED'
353
+ && current.executionId === null
354
+ && current.noStartFallback?.failedExecutionId === task.noStartFallback?.failedExecutionId,
355
+ mutate: (_job, current) => {
356
+ current.error = 'Codex default fallback route unavailable: ' + message;
357
+ current.failureReason = 'codex_default_fallback_unavailable';
358
+ },
359
+ });
360
+ if (transition?.decision.accepted)
361
+ await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? message, 'user', { reason: 'codex_default_fallback_unavailable' }, { agent: 'relay', component: 'agents-relay' }));
362
+ return undefined;
363
+ }
343
364
  if (!['chatgpt', 'z'].includes(task.adapter) || !task.platformFailure?.safeToReroute)
344
365
  throw error;
345
366
  await quarantineUnavailableCodexFallback(this.taskTransitions, event => this.emit(event), job.id, task.id, error);
@@ -479,6 +500,11 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
479
500
  if (outcome === 'Failed')
480
501
  throw Error('reviewer start postcondition violated');
481
502
  }
503
+ // A changes-requested verdict is a successfully delivered review, not a failed model route.
504
+ if (event.type === 'review.approved' || event.type === 'review.changes_required') {
505
+ const committedReviewer = job.tasks.find(item => item.id === task.id) ?? task;
506
+ await recordRouteOutcome(this.options.decisionAuditStore, committedReviewer, 'success', 'review.' + outcome, { event_type: event.type });
507
+ }
482
508
  // Review approval records evidence only. The subsequent Graph reconcile
483
509
  // drives durable local Git merge delivery and terminalization.
484
510
  if (outcome === 'approved')
@@ -1,12 +1,13 @@
1
- import { hasSafeAlternativeWorker } from './browser-worker-routing.js';
1
+ import { handleWorkerLaunchFailure } from './worker-failure-recovery.js';
2
2
  import { normalizeReviewerTerminalEvent } from './review-terminal-normalization.js';
3
3
  import { executionEventSource, eventSourceId, executionIdFromSource, claimTerminalExecution, terminalExecutionClaimed } from './execution-event-identity.js';
4
4
  export { executionEventSource } from './execution-event-identity.js';
5
5
  import { applyJobEffectOutcomes, applyTaskEffectOutcomes } from './workflow-effect-failures.js';
6
- import { classifyChatGptPlatformFailure, classifyWorkerRejection, reroutableWorkerRejections } from './worker-rejection.js';
6
+ import { classifyWorkerRejection, reroutableWorkerRejections } from './worker-rejection.js';
7
7
  import { applyTaskRetryAction } from './task-actions.js';
8
8
  import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
9
9
  import { eventFor } from './events.js';
10
+ import { recordTerminalRouteOutcome, recordWatchdogRouteOutcome } from './route-outcome-recording.js';
10
11
  import { WorkflowEffectExecutor } from './workflow/effects.js';
11
12
  import { ReconcileCoordinator } from './reconcile-coordinator.js';
12
13
  import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
@@ -14,7 +15,6 @@ import { createWorkerStartAck } from './worker-start-ack.js';
14
15
  import { monitorExecutionProcess } from './execution-process-tracker.js';
15
16
  import { HostInteractiveInvocationCoordinator, hostInvocationPorts } from './interactive-invocation.js';
16
17
  import { resumeInteractiveTask } from './interactive-resume.js';
17
- import { persistChatGptPlatformFailure } from './chatgpt-platform-fallback.js';
18
18
  import { acceptTaskProgress, armInactivityWatchdog, inactivityDeadline, isCorrelatedWorkerEvent, isRecoverableWatchdogFailure } from './task-inactivity-watchdog.js';
19
19
  const activeExecutionOwners = new Map();
20
20
  import { withExecutionBoundary } from './execution-boundary.js';
@@ -255,56 +255,26 @@ export class ReconcilerRuntimeBase {
255
255
  return true;
256
256
  }
257
257
  async failLaunch(jobId, taskId, executionId, failure, phase = 'launch') {
258
- const message = failure instanceof Error ? failure.message : String(failure);
259
- const currentJob = await this.store.load(jobId);
260
- const currentTask = currentJob.tasks.find(task => task.id === taskId);
261
- const alternative = hasSafeAlternativeWorker(currentTask, this.options.adapters, await this.store.listAgents(), this.options.usageSnapshotProvider ? await this.options.usageSnapshotProvider() : undefined);
262
- const activeExecution = this.live.get(taskId);
263
- const platformFailure = (currentTask?.adapter === 'chatgpt' || currentTask?.adapter === 'z') ? classifyChatGptPlatformFailure(failure, phase) : undefined;
264
- const unknownOutcome = phase === 'start-ack' || ['outcome_unknown', 'start_ack_timeout'].includes(platformFailure?.classification ?? '')
265
- || (phase === 'delivery' && activeExecution?.executionId === executionId && activeExecution.workerStarted);
266
- const failureReason = phase === 'start-ack' ? 'worker_start_timeout'
267
- : /WEBHELM_UNAVAILABLE|webhelm runtime dependency unavailable|webhelm session start returned no session_id|no close frame received or sent/i.test(message)
268
- ? 'browser_workspace_unavailable'
269
- : undefined;
270
- const claim = executionId ?? `${jobId}:${taskId}`;
271
- if (this.launchFailureClaims.has(claim))
272
- return;
273
- this.launchFailureClaims.add(claim);
274
- const cleanup = setTimeout(() => this.launchFailureClaims.delete(claim), 60000);
275
- cleanup.unref();
276
- const workflowEvent = platformFailure ? 'worker.platform_failure' : phase === 'start-ack' ? 'worker.start_ack_failed' : 'worker.delivery_failed';
277
- const transition = await this.taskTransitions.apply(jobId, taskId, platformFailure ? 'worker.platform_failure' : workflowEvent, {
278
- eligible: (_job, task) => task.executionId === executionId,
279
- facts: () => ({ platformFallbackSafe: platformFailure?.safeToReroute === true && alternative }),
280
- mutate: (_job, task, decision) => {
281
- if (platformFailure)
282
- persistChatGptPlatformFailure(task, currentTask, executionId, platformFailure.classification, platformFailure.evidence, platformFailure.safeToReroute);
283
- task.error = decision.target === 'QUEUED' ? null : `Worker ${phase} failed: ${message}`;
284
- task.failureReason = decision.target === 'QUEUED' ? undefined : unknownOutcome
285
- ? 'worker_outcome_unknown' : platformFailure ? 'worker_platform_failure' : failureReason;
286
- task.leaseOwner = null;
287
- task.leaseExpiresAt = null;
288
- task.executionId = null;
258
+ await handleWorkerLaunchFailure({
259
+ store: this.store,
260
+ adapters: this.options.adapters,
261
+ usageSnapshotProvider: this.options.usageSnapshotProvider,
262
+ decisionAuditStore: this.options.decisionAuditStore,
263
+ taskTransitions: this.taskTransitions,
264
+ live: id => this.live.get(id),
265
+ launchFailureClaims: this.launchFailureClaims,
266
+ clearExecution: (id, execution) => this.clearExecution(id, execution),
267
+ clearTerminalWaiter: (id, execution) => {
268
+ const waiter = this.terminalWaiters.get(id);
269
+ if (waiter?.executionId === execution) {
270
+ waiter.resolve();
271
+ this.terminalWaiters.delete(id);
272
+ }
289
273
  },
290
- });
291
- if (!transition)
292
- return;
293
- if (!transition.decision.accepted) {
294
- await this.emit(eventFor(jobId, taskId, transition.task.parentTaskId, 'task.process.delivery_failed', 'failed', `Worker ${phase} reported after lifecycle moved to ${transition.from}: ${message}`, 'orchestrator', { phase, authoritative: false, executionId, workflowEvent }, { agent: 'relay', component: 'agents-relay' }));
295
- return;
296
- }
297
- if (executionId) {
298
- const live = this.live.get(taskId);
299
- if (live?.executionId === executionId) {
300
- live.controller.abort();
301
- live.execution.cancel();
302
- this.clearExecution(taskId, executionId);
303
- }
304
- }
305
- const event = eventFor(jobId, taskId, transition.task.parentTaskId, transition.task.state === 'QUEUED' ? 'worker.pre_submission_rejected' : 'task.failed', transition.task.state === 'QUEUED' ? 'queued' : 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason ?? failureReason, workflowEvent, ...(transition.task.platformFailure ? { platformFailure: transition.task.platformFailure } : {}) }, { agent: 'relay', component: 'agents-relay', watchdog: true });
306
- this.detach(this.emit(event), jobId, taskId, 'worker failure event');
307
- this.detach(this.eventCoordinator.accept({ jobId, reason: `worker-${phase}-failure` }), jobId, taskId, 'worker failure reconcile wake');
274
+ emit: event => this.emit(event),
275
+ detach: (promise, job, task, context) => this.detach(promise, job, task, context),
276
+ wake: (job, task, reason) => this.detach(this.eventCoordinator.accept({ jobId: job, reason }), job, task, 'worker failure reconcile wake'),
277
+ }, jobId, taskId, executionId, failure, phase);
308
278
  }
309
279
  async runtimeSettled(jobId, taskId, executionId, result, error) {
310
280
  const live = this.live.get(taskId);
@@ -316,6 +286,12 @@ export class ReconcilerRuntimeBase {
316
286
  const task = job.tasks.find(item => item.id === taskId);
317
287
  if (!task)
318
288
  return;
289
+ if (error && ['chatgpt', 'z'].includes(task.adapter)
290
+ && task.executionId === executionId && task.state === 'QUEUED'
291
+ && !live?.workerStarted) {
292
+ await this.failLaunch(jobId, taskId, executionId, error, 'process');
293
+ return;
294
+ }
319
295
  if (task.adapter !== 'chatgpt' && task.adapter !== 'z') {
320
296
  const exited = await this.taskTransitions.updateFact(jobId, taskId, current => current.executionId === executionId && ['RUNNING', 'QUEUED'].includes(current.state), current => { current.processExitedAt = new Date().toISOString(); });
321
297
  if (exited)
@@ -339,6 +315,11 @@ export class ReconcilerRuntimeBase {
339
315
  return;
340
316
  }
341
317
  const live = this.live.get(taskId);
318
+ if ((current.adapter === 'chatgpt' || current.adapter === 'z')
319
+ && current.state === 'QUEUED' && !live?.workerStarted) {
320
+ await this.failLaunch(jobId, taskId, executionId, new Error('Worker failed to publish task.started before the inactivity deadline'), 'start-ack');
321
+ return;
322
+ }
342
323
  // Do not cancel before the serialized watchdog transition rechecks the
343
324
  // deadline: a concurrent accepted progress event may have extended it.
344
325
  const exit = live?.executionId === executionId && live.processSettled ? live.execution.processExit : undefined;
@@ -362,6 +343,7 @@ export class ReconcilerRuntimeBase {
362
343
  this.armInactivityWatchdog(latest, executionId);
363
344
  return;
364
345
  }
346
+ await recordWatchdogRouteOutcome(this.options.decisionAuditStore, transition.task, { executionId, failedExit, rawError, failureReason, failureMessage, exitCode: exit?.exitCode ?? null });
365
347
  if (live?.executionId === executionId) {
366
348
  live.controller.abort();
367
349
  live.execution.cancel();
@@ -397,6 +379,16 @@ export class ReconcilerRuntimeBase {
397
379
  }
398
380
  }
399
381
  async applyTerminalEvent(event, trustedInternal = false) {
382
+ if (event.type === 'task.failed') {
383
+ const job = await this.store.load(event.job_id);
384
+ const original = job.tasks.find(task => task.id === event.task_id);
385
+ if (original && ['chatgpt', 'z'].includes(original.adapter)
386
+ && original.state === 'QUEUED' && original.executionId
387
+ && isCorrelatedWorkerEvent(original, event)) {
388
+ await this.failLaunch(event.job_id, event.task_id, original.executionId, new Error(event.message || 'Worker reported failure before task.started'), 'terminal');
389
+ return { applied: true, reconcile: false, reason: 'task.failed.no-start-fallback' };
390
+ }
391
+ }
400
392
  const liveAtTerminal = this.live.get(event.task_id);
401
393
  const terminalEventExecutionId = executionIdFromSource(eventSourceId(event), event.job_id, event.task_id);
402
394
  const lifecycleEvent = event.type === 'task.blocked'
@@ -477,6 +469,8 @@ export class ReconcilerRuntimeBase {
477
469
  }));
478
470
  if (!transition?.decision.accepted)
479
471
  return { applied: false, reconcile: false, reason: `${event.type}.ignored` };
472
+ if (lifecycleEvent !== 'interactive.waiting')
473
+ await recordTerminalRouteOutcome(this.options.decisionAuditStore, transition.task, event, terminalEventExecutionId ?? null);
480
474
  if (transition.task.adapter === 'codex' && liveAtTerminal && !liveAtTerminal.processSettled && liveAtTerminal.jobId === event.job_id && (!terminalEventExecutionId || liveAtTerminal.executionId === terminalEventExecutionId)) {
481
475
  liveAtTerminal.controller.abort();
482
476
  liveAtTerminal.execution.cancel();
@@ -27,6 +27,7 @@ import { exhaustedTroubleshooterRecoveries, pendingTroubleshooterTasks, pendingT
27
27
  import { fallbackFailedTroubleshooters, escalateExhaustedTroubleshooters } from './troubleshooter-fallback.js';
28
28
  import { cleanupMergedJobWorktree } from './job-worktree-cleanup.js';
29
29
  import { jobHasActiveExecution } from './job-execution-slot.js';
30
+ import { requiresWorkerInactivityWatchdog } from './task-inactivity-watchdog.js';
30
31
  import { managedWorkerInput } from './managed-worker-input.js';
31
32
  import { launchShortcutEffect, resumeShortcutEffect, resumeShortcutEventWaits } from './reconciler-shortcut.js';
32
33
  export { managedWorkerInput } from './managed-worker-input.js';
@@ -58,6 +59,10 @@ export class Reconciler extends ReconcilerReviewBase {
58
59
  }
59
60
  async dispatchExecutableTasks(jobId) {
60
61
  const scheduled = refreshTaskDependencies(await this.store.load(jobId));
62
+ // Ordinary daemon/queue never owns Shortcut Jobs, including recovery Tasks.
63
+ // Only the explicit foreground runner may use existing Worker Adapters.
64
+ if (scheduled.executionMode === 'shortcut' && !this.options.shortcutRunnerOwner)
65
+ return scheduled;
61
66
  if (scheduled.pause)
62
67
  return scheduled;
63
68
  for (const task of scheduled.tasks.filter(item => item.state === 'SUCCEEDED' && (item.continuation || scheduled.continuation) && !item.continuationDeliveredAt)) {
@@ -74,7 +79,7 @@ export class Reconciler extends ReconcilerReviewBase {
74
79
  .slice(0, Math.min(capacity, 1));
75
80
  for (const task of dispatchable) {
76
81
  const shortcut = task.result?.data?.shortcut;
77
- await transitionTaskEvent(task, 'dispatch', { dispatchable: true, shortcutExecution: scheduled.executionMode === 'shortcut' && scheduled.contentOnly === true, shortcutFallback: shortcut?.workerFallback === true, shortcutHasGraph: Boolean(task.agentGraph) }, async (actionId) => {
82
+ await transitionTaskEvent(task, 'dispatch', { dispatchable: true, shortcutExecution: scheduled.executionMode === 'shortcut' && scheduled.contentOnly === true && shortcut?.workerFallback !== true && Boolean(task.agentGraph), shortcutFallback: shortcut?.workerFallback === true, shortcutHasGraph: Boolean(task.agentGraph) }, async (actionId) => {
78
83
  const outcomes = await this.workflowEffects.execute([actionId], { jobId: scheduled.id, taskId: task.id, transitionKey: `dispatch:${task.attempt}`, data: { attempt: task.attempt } });
79
84
  await this.applyTaskEffectOutcomes(scheduled, task, outcomes);
80
85
  }, async () => { await this.store.saveTask(task); });
@@ -87,6 +92,10 @@ export class Reconciler extends ReconcilerReviewBase {
87
92
  this.reconciling = true;
88
93
  try {
89
94
  let job = await this.store.load(jobId);
95
+ // The same durable Job flag covers children, retries, and Troubleshooters.
96
+ // Do not let a background Reconciler mutate or dispatch any of them.
97
+ if (job.executionMode === 'shortcut' && !this.options.shortcutRunnerOwner)
98
+ return job;
90
99
  await drainProductionWorkflowOutbox(this.store, this.options);
91
100
  await deliverPendingEscalations(this.store, jobId, this.options.escalationNotifier);
92
101
  job = await this.store.load(jobId);
@@ -178,7 +187,8 @@ export class Reconciler extends ReconcilerReviewBase {
178
187
  const taskStatesBeforeSchedule = new Map(beforeSchedule.tasks.map(task => [task.id, task.state]));
179
188
  let scheduled = refreshTaskDependencies(beforeSchedule);
180
189
  for (const task of scheduled.tasks)
181
- if (task.executionId && ['QUEUED', 'RUNNING'].includes(task.state))
190
+ if (task.executionId && ['QUEUED', 'RUNNING'].includes(task.state)
191
+ && requiresWorkerInactivityWatchdog(scheduled, task))
182
192
  this.armInactivityWatchdog(task, task.executionId);
183
193
  for (const task of scheduled.tasks) {
184
194
  if (taskStatesBeforeSchedule.get(task.id) !== task.state)
@@ -1,8 +1,11 @@
1
1
  import { normalizeLegacyTaskLifecycle } from './types.js';
2
2
  import { normalizeLegacyJobPrNumber } from './job-pr-number-migration.js';
3
3
  export function currentJobMetadata(job) {
4
- const { schedule: _schedule, ...current } = job;
5
- return current;
4
+ // Retired historical modes must not survive reads or writes. Only the
5
+ // supported opt-in Direct Shortcut mode is part of the active Job contract.
6
+ const { schedule: _schedule, executionMode, ...current } = job;
7
+ return (executionMode === 'shortcut'
8
+ ? { ...current, executionMode: 'shortcut' } : current);
6
9
  }
7
10
  export function normalizeRelayJob(json) {
8
11
  const job = currentJobMetadata(normalizeLegacyJobPrNumber(JSON.parse(json)));