agents-relay 2.0.67 → 2.0.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +56 -20
  2. package/dist/adapters.js +39 -3
  3. package/dist/blockers.js +17 -4
  4. package/dist/cli-blocker.js +12 -30
  5. package/dist/cli-help.js +22 -17
  6. package/dist/cli-job-create.js +77 -0
  7. package/dist/cli-local-merge-config.js +11 -0
  8. package/dist/cli-projection.js +23 -55
  9. package/dist/cli-review.js +7 -24
  10. package/dist/cli-support.js +81 -8
  11. package/dist/cli-worker-task.js +63 -0
  12. package/dist/cli.js +223 -185
  13. package/dist/dashboard-html.js +15 -12
  14. package/dist/dashboard.js +27 -2
  15. package/dist/e2e-injection.js +76 -0
  16. package/dist/github-auth.js +4 -3
  17. package/dist/github-binding-order.js +15 -0
  18. package/dist/github-client.js +7 -2
  19. package/dist/github-pr-archive.js +22 -0
  20. package/dist/github-webhook.js +43 -10
  21. package/dist/job-actions.js +29 -0
  22. package/dist/local-authority-store.js +288 -0
  23. package/dist/local-commit-events.js +2 -0
  24. package/dist/local-git-merge.js +124 -0
  25. package/dist/local-job-path.js +7 -0
  26. package/dist/local-job-registry.js +29 -0
  27. package/dist/local-managed-store.js +21 -0
  28. package/dist/local-merge-delivery.js +53 -0
  29. package/dist/local-merge-lock.js +103 -0
  30. package/dist/local-merge-projection.js +60 -0
  31. package/dist/managed-worker-input.js +13 -6
  32. package/dist/manual-plan.js +29 -7
  33. package/dist/no-github.js +11 -0
  34. package/dist/planned-review-head.js +7 -8
  35. package/dist/planner-runtime.js +182 -41
  36. package/dist/planner.js +35 -2
  37. package/dist/pool.js +89 -136
  38. package/dist/projected-store.js +20 -3
  39. package/dist/projection-outbox.js +3 -1
  40. package/dist/publication-store.js +37 -0
  41. package/dist/reconciler-cancellation.js +21 -0
  42. package/dist/reconciler-review.js +99 -29
  43. package/dist/reconciler-runtime.js +72 -35
  44. package/dist/reconciler.js +124 -85
  45. package/dist/relay-store.js +108 -6
  46. package/dist/relayd-control.js +184 -0
  47. package/dist/relayd-dashboard-actions.js +108 -0
  48. package/dist/relayd-utilities.js +107 -0
  49. package/dist/relayd.js +304 -299
  50. package/dist/review-generation-fence.js +12 -0
  51. package/dist/runtime-status-events.js +21 -0
  52. package/dist/runtime-status.js +21 -7
  53. package/dist/scheduled-jobs.js +43 -36
  54. package/dist/service-external-actions.js +27 -0
  55. package/dist/service-publication.js +29 -0
  56. package/dist/store-github.js +17 -6
  57. package/dist/task-actions.js +59 -1
  58. package/dist/task-update.js +1 -1
  59. package/dist/troubleshooter.js +21 -2
  60. package/dist/unblock.js +47 -15
  61. package/dist/worker-rejection.js +11 -0
  62. package/dist/workflow/contracts.js +10 -1
  63. package/dist/workflow/definitions-v2.js +2 -0
  64. package/dist/workflow/job-reconcile-v2.js +9 -60
  65. package/dist/workflow/runtime.js +18 -3
  66. package/dist/workflow-effect-failures.js +30 -0
  67. package/dist/workspace.js +95 -6
  68. package/package.json +6 -3
  69. package/skills/agents-relay/agents/planner.agent.md +3 -0
  70. package/skills/agents-relay/contracts/orchestrator-workflows.md +4 -7
  71. package/workflow/job-lifecycle.mmd +19 -60
  72. package/workflow/review-lifecycle.mmd +5 -8
@@ -1,22 +1,37 @@
1
1
  import { storySizeTimeoutMs, taskOverallTimeoutMs } from './types.js';
2
- import { completionGateSatisfied, schedule, transitionTaskEvent, transitionJobEvent, tryTransitionTaskEvent } from './scheduler.js';
2
+ import { completionGateSatisfied, schedule, transitionTaskEvent, tryTransitionTaskEvent } from './scheduler.js';
3
3
  import { implementationComplete, implementationTasks } from './runtime-status.js';
4
4
  import { eventFor } from './events.js';
5
5
  import { workerCandidates } from './worker-routing.js';
6
6
  import { routeExecution } from './execution-routing.js';
7
7
  import { ReconcilerRuntimeBase } from './reconciler-runtime.js';
8
8
  import { dispatchReviewLifecycle, reviewOutcome } from './workflow/review-v2.js';
9
+ import { dispatchTaskLifecycleSync } from './workflow/lifecycle.js';
9
10
  import { createTaskEntity } from './workflow/entity-creation.js';
11
+ import { localSourceHead, localWorkingHead } from './local-git-merge.js';
10
12
  import { chatgptThreadRequirements, resolveReviewerThreadDestination } from './reviewer-thread.js';
13
+ import { reviewGenerationMatches } from './review-generation-fence.js';
11
14
  export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
12
- async currentHeadSha(job) { return await this.store.headSha?.() ?? job.headSha ?? job.review?.headSha ?? null; }
13
- reviewFacts(job, prState, headSha) {
15
+ async currentHeadSha(job) {
16
+ // GitHub PR head facts are audit-only. Source identity is local Git.
17
+ if (!this.options.projectPathForJob)
18
+ return job.headSha ?? job.review?.headSha ?? null;
19
+ try {
20
+ const project = await this.options.projectPathForJob(job);
21
+ if (job.localMerge?.sourceRef)
22
+ return localSourceHead(project, job.localMerge.sourceRef);
23
+ return localWorkingHead(project);
24
+ }
25
+ catch {
26
+ return job.headSha ?? job.review?.headSha ?? null;
27
+ }
28
+ }
29
+ reviewFacts(job, headSha) {
14
30
  const reviewTasks = job.tasks.filter(task => task.agentName === 'reviewer' && task.reviewHeadSha === headSha);
15
31
  const currentReviewTask = job.review?.taskId ? job.tasks.find(task => task.id === job.review?.taskId) : undefined;
16
32
  return {
17
33
  unscheduled: !job.schedule,
18
34
  reviewEnabled: job.reviewEnabled !== false,
19
- prOpen: prState === 'OPEN',
20
35
  jobActive: job.state === 'ACTIVE',
21
36
  implementationComplete: implementationComplete(job),
22
37
  headAvailable: Boolean(headSha),
@@ -105,8 +120,12 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
105
120
  }
106
121
  throw new Error('Too many final review rounds for ' + headSha);
107
122
  }
123
+ externalReviewPrNumber(job) {
124
+ return job.externalPrNumber && job.externalPrNumber > 0 ? job.externalPrNumber : job.prNumber;
125
+ }
108
126
  async createFinalReviewTask(job, headSha) {
109
127
  const id = this.nextFinalReviewTaskId(job, headSha);
128
+ const reviewPrNumber = this.externalReviewPrNumber(job);
110
129
  const now = new Date().toISOString();
111
130
  const reviewGateTasks = implementationTasks(job);
112
131
  const reviewThread = resolveReviewerThreadDestination(job, this.options.reviewer?.threadId);
@@ -117,13 +136,13 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
117
136
  threadSource: reviewThread.source,
118
137
  requestId: `${job.id}:${id}:${headSha}`,
119
138
  jobId: job.id,
120
- prNumber: job.prNumber,
139
+ prNumber: reviewPrNumber,
121
140
  reviewHeadSha: headSha,
122
141
  projectId: reviewerRouting.candidate.routing.projectId ?? null,
123
142
  } : undefined;
124
143
  const input = [
125
144
  `Job ID: ${job.id}`,
126
- `Review PR #${job.prNumber} at exactly head SHA ${headSha}.`,
145
+ `Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.`,
127
146
  `Review head SHA: ${headSha}`,
128
147
  'Job objective: ' + (job.objective ?? job.description ?? job.title),
129
148
  'Use the selected reviewer agent contract as the authoritative review protocol.',
@@ -146,9 +165,9 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
146
165
  await this.emit(eventFor(job.id, id, null, 'review.created', 'queued', 'Final review task ' + id + ' created for ' + headSha + ' using ' + reviewerRouting.reason, 'orchestrator', { headSha, agentName: 'reviewer', reviewerRouting: { workerId: reviewerRouting.selection.workerId, reason: reviewerRouting.reason, selectedBy: reviewerRouting.selection.selectedBy } }));
147
166
  return id;
148
167
  }
149
- async ensureFinalReview(job, prState, headSha) {
168
+ async ensureFinalReview(job, headSha) {
150
169
  let reviewTaskId = null;
151
- const decision = await dispatchReviewLifecycle(job.review?.outcome, 'reconcile', this.reviewFacts(job, prState, headSha), async (actionId) => {
170
+ const decision = await dispatchReviewLifecycle(job.review?.outcome, 'reconcile', this.reviewFacts(job, headSha), async (actionId) => {
152
171
  if (actionId !== 'review.create')
153
172
  throw new Error('Unsupported review reconcile action ' + actionId);
154
173
  if (!headSha)
@@ -234,7 +253,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
234
253
  id: job.id, title: job.title, objective: job.objective ?? null,
235
254
  description: job.description ?? null, priority: job.priority ?? null,
236
255
  executionMode: job.executionMode, repository: job.repository,
237
- prNumber: job.prNumber, state: job.state,
256
+ prNumber: this.externalReviewPrNumber(job), state: job.state,
238
257
  },
239
258
  task: {
240
259
  id: task.id, kind: task.kind ?? 'work', storySize: task.storySize ?? 'M',
@@ -251,9 +270,11 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
251
270
  ? job.tasks.find(item => item.id === task.parentTaskId)
252
271
  : undefined;
253
272
  const parentRouteFailedBeforeStart = Boolean(parent?.routing && /exited before publishing task\.started|worker_exit_before_start|launch failed/i.test(parent.error ?? parent.failureReason ?? ''));
254
- const excludedRoutes = parentRouteFailedBeforeStart && parent?.routing
273
+ const parentExcludedRoutes = parentRouteFailedBeforeStart && parent?.routing
255
274
  ? [{ provider: parent.routing.provider, model: parent.routing.model, profile: parent.routing.profile }]
256
275
  : [];
276
+ const taskExcludedRoutes = (task.routingExclusions ?? []).map(item => ({ provider: item.provider, model: item.model, profile: item.profile }));
277
+ const excludedRoutes = [...taskExcludedRoutes, ...parentExcludedRoutes];
257
278
  const routed = await routeExecution(candidates, requirements, {
258
279
  decisionEngine: this.options.decisionEngine,
259
280
  decisionMode: this.options.decisionMode,
@@ -287,26 +308,30 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
287
308
  const headSha = normalizedEvidence.headSha;
288
309
  if (task.reviewHeadSha !== headSha)
289
310
  return { applied: false, reconcile: false, reason: event.type + '.stale' };
290
- const currentHeadSha = await this.store.headSha?.();
311
+ const currentHeadSha = await this.currentHeadSha(job);
291
312
  if ((currentHeadSha && currentHeadSha !== headSha) || (job.headSha && job.headSha !== headSha))
292
313
  return { applied: false, reconcile: false, reason: event.type + '.stale' };
293
314
  // Planner-created reviewer tasks predate the final-review Job gate. Bind
294
315
  // their first trustworthy review event to PENDING before dispatching it.
295
316
  if (!job.review && task.planningDecisionId) {
296
317
  job.review = { outcome: 'pending', headSha, taskId: task.id, evidence: null, updatedAt: event.timestamp || new Date().toISOString() };
297
- await this.store.saveJob(job);
298
318
  }
299
- const taskDecision = tryTransitionTaskEvent(task, event.type);
319
+ const taskDecision = this.store.mutateJob
320
+ ? dispatchTaskLifecycleSync(structuredClone(task), event.type)
321
+ : tryTransitionTaskEvent(task, event.type);
300
322
  if (!taskDecision.accepted || !taskDecision.target)
301
323
  return { applied: false, reconcile: false, reason: event.type + '.ignored' };
302
- task.error = null;
303
- if (taskDecision.target !== 'RUNNING') {
304
- task.leaseOwner = null;
305
- task.leaseExpiresAt = null;
306
- task.executionId = null;
324
+ if (!this.store.mutateJob && taskDecision.target !== 'RUNNING' && !this.finalizeReviewExecution(job.id, task, event)) {
325
+ return { applied: false, reconcile: false, reason: event.type + '.stale-execution' };
326
+ }
327
+ if (!this.store.mutateJob) {
328
+ task.error = null;
329
+ if (taskDecision.target !== 'RUNNING')
330
+ task.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
331
+ await this.store.saveTask(task);
307
332
  }
308
- await this.store.saveTask(task);
309
333
  let needsReconcile = false;
334
+ let pendingFixTask = null;
310
335
  const reviewDecision = await dispatchReviewLifecycle(job.review?.outcome, event.type, { autoMerge: job.autoMerge !== false }, async (actionId) => {
311
336
  if (actionId === 'review.createFix') {
312
337
  needsReconcile = true;
@@ -316,18 +341,14 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
316
341
  const now = event.timestamp || new Date().toISOString();
317
342
  const id = 'review-fix-' + headSha.slice(0, 8);
318
343
  if (!job.tasks.some(item => item.id === id)) {
319
- const input = 'Address all reviewer findings for PR #' + job.prNumber + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
344
+ const input = 'Address all reviewer findings for PR #' + this.externalReviewPrNumber(job) + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
320
345
  const fixTask = createTaskEntity(job, { jobId: job.id, id, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
321
- await this.store.appendTask(fixTask);
322
- await this.emit(eventFor(job.id, id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + id + ' for ' + issues.length + ' reviewer finding(s)', 'orchestrator', { headSha, issues }, { agent: 'relay', component: 'agents-relay' }));
346
+ pendingFixTask = fixTask;
323
347
  }
324
348
  return {};
325
349
  }
326
350
  if (actionId === 'review.requestMerge') {
327
- await transitionJobEvent(job, 'review.approved', { autoMerge: true }, async (effectId) => {
328
- const outcomes = await this.workflowEffects.execute([effectId], { jobId: job.id, taskId: task.id, transitionKey: 'review.approved:' + headSha, data: { headSha } });
329
- await this.applyJobEffectOutcomes(job, outcomes);
330
- }, async () => { await this.store.saveJob(job); });
351
+ // Approval is a local fact; only subsequent local Git delivery may complete the Job.
331
352
  return {};
332
353
  }
333
354
  throw new Error('Unsupported review event action ' + actionId);
@@ -342,9 +363,58 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
342
363
  const evidence = { summary: event.message || 'Review ' + outcome, data: {
343
364
  ...(event.data ?? {}), review: { ...(reviewData ?? {}), headSha }, headSha, outcome, source: event.source,
344
365
  } };
345
- job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: event.timestamp || new Date().toISOString() };
346
- job.updatedAt = job.review.updatedAt;
347
- await this.store.saveJob(job);
366
+ const reviewUpdatedAt = event.timestamp || new Date().toISOString();
367
+ if (this.store.mutateJob) {
368
+ let committed = false;
369
+ let terminalExecutionId = null;
370
+ const updated = await this.store.mutateJob(job.id, current => {
371
+ const freshTask = current.tasks.find(item => item.id === task.id);
372
+ if (!freshTask || !reviewGenerationMatches(current, job, task, headSha))
373
+ return current;
374
+ const freshDecision = tryTransitionTaskEvent(freshTask, event.type);
375
+ if (!freshDecision.accepted || !freshDecision.target)
376
+ return current;
377
+ if (freshDecision.target !== 'RUNNING') {
378
+ if (!this.finalizeReviewExecution(job.id, freshTask, event, false))
379
+ return current;
380
+ terminalExecutionId = freshTask.executionId;
381
+ }
382
+ freshTask.error = null;
383
+ if (freshDecision.target !== 'RUNNING')
384
+ freshTask.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
385
+ if (pendingFixTask && !current.tasks.some(item => item.id === pendingFixTask.id))
386
+ current.tasks.push(structuredClone(pendingFixTask));
387
+ current.review = { outcome, headSha, taskId: freshTask.id, evidence, updatedAt: reviewUpdatedAt };
388
+ current.updatedAt = reviewUpdatedAt;
389
+ committed = true;
390
+ return current;
391
+ });
392
+ if (!committed)
393
+ return { applied: false, reconcile: false, reason: event.type + '.stale-execution' };
394
+ Object.assign(job, structuredClone(updated));
395
+ // Live waiter cleanup/terminal claims are irreversible; run after commit.
396
+ if (terminalExecutionId) {
397
+ this.finalizeReviewExecution(job.id, { ...task, executionId: terminalExecutionId }, event);
398
+ }
399
+ const committedFixTask = pendingFixTask;
400
+ if (committedFixTask)
401
+ await this.emit(eventFor(job.id, committedFixTask.id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + committedFixTask.id + ' for reviewer findings', 'orchestrator', { headSha }, { agent: 'relay', component: 'agents-relay' }));
402
+ }
403
+ else {
404
+ const legacyFixTask = pendingFixTask;
405
+ if (legacyFixTask) {
406
+ await this.store.appendTask(legacyFixTask);
407
+ await this.emit(eventFor(job.id, legacyFixTask.id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + legacyFixTask.id + ' for reviewer findings', 'orchestrator', { headSha }, { agent: 'relay', component: 'agents-relay' }));
408
+ }
409
+ job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: reviewUpdatedAt };
410
+ job.updatedAt = reviewUpdatedAt;
411
+ await this.store.saveTask(task);
412
+ await this.store.saveJob(job);
413
+ }
414
+ // Review approval records evidence only. The subsequent Graph reconcile
415
+ // drives durable local Git merge delivery and terminalization.
416
+ if (outcome === 'approved')
417
+ needsReconcile = true;
348
418
  return { applied: true, reconcile: needsReconcile, reason: 'review.' + outcome, reconcileMode: needsReconcile ? 'detached' : undefined };
349
419
  }
350
420
  }
@@ -1,9 +1,9 @@
1
+ import { applyJobEffectOutcomes, applyTaskEffectOutcomes } from './workflow-effect-failures.js';
2
+ import { classifyWorkerRejection, reroutableWorkerRejections } from './worker-rejection.js';
1
3
  import { applyTaskRetryAction } from './task-actions.js';
2
- import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
3
4
  import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
4
5
  import { eventFor } from './events.js';
5
6
  import { WorkflowEffectExecutor } from './workflow/effects.js';
6
- import { workflowEffectFailureEvent } from './workflow/contracts.js';
7
7
  import { ReconcileCoordinator } from './reconcile-coordinator.js';
8
8
  import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
9
9
  import { rearmTerminalEventDeadline } from './terminal-event-watchdog.js';
@@ -77,31 +77,10 @@ export class ReconcilerRuntimeBase {
77
77
  });
78
78
  }
79
79
  async applyJobEffectOutcomes(job, outcomes) {
80
- for (const outcome of outcomes.filter(item => item.status === 'failed')) {
81
- const eventName = workflowEffectFailureEvent(outcome.effect);
82
- try {
83
- await transitionJobEvent(job, eventName);
84
- await this.store.saveJob(job);
85
- }
86
- catch {
87
- // Some effects are observational for a given state; failure is still emitted below.
88
- }
89
- await this.emit(eventFor(job.id, 'job', null, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
90
- }
80
+ await applyJobEffectOutcomes(this.store, event => this.emit(event), job, outcomes);
91
81
  }
92
82
  async applyTaskEffectOutcomes(job, task, outcomes) {
93
- for (const outcome of outcomes.filter(item => item.status === 'failed')) {
94
- const eventName = workflowEffectFailureEvent(outcome.effect);
95
- try {
96
- await transitionTaskEvent(task, eventName);
97
- task.error = outcome.error ?? `${outcome.effect} failed`;
98
- await this.store.saveTask(task);
99
- }
100
- catch {
101
- // The effect failure remains observable even if this state has no feedback transition.
102
- }
103
- await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
104
- }
83
+ await applyTaskEffectOutcomes(this.store, event => this.emit(event), job, task, outcomes);
105
84
  }
106
85
  async applyEvent(event) {
107
86
  const sourceId = eventSourceId(event);
@@ -122,7 +101,9 @@ export class ReconcilerRuntimeBase {
122
101
  }
123
102
  if (['review.started', 'review.approved', 'review.changes_required', 'review.blocked'].includes(event.type))
124
103
  return this.applyReviewEvent(event);
125
- if (event.type === 'job.wake' || event.type === 'github.webhook')
104
+ if (event.type === 'github.webhook')
105
+ return { applied: true, reconcile: false, reason: 'github.audit_only' };
106
+ if (event.type === 'job.wake')
126
107
  return { applied: true, reconcile: true, reason: event.type };
127
108
  return { applied: false, reconcile: false, reason: event.type };
128
109
  }
@@ -254,6 +235,28 @@ export class ReconcilerRuntimeBase {
254
235
  isCorrelatedWorkerEvent(task, event) {
255
236
  return task.executionId !== null && eventSourceId(event) === executionEventSource(event.job_id, task.id, task.executionId);
256
237
  }
238
+ finalizeReviewExecution(jobId, task, event, afterCommit = true) {
239
+ if (!task.executionId)
240
+ return true;
241
+ const sourceId = eventSourceId(event);
242
+ const expected = executionEventSource(jobId, task.id, task.executionId);
243
+ if ((task.adapter === 'codex' || task.adapter === 'chatgpt') && sourceId !== expected)
244
+ return false;
245
+ if (afterCommit)
246
+ claimTerminalExecution(expected);
247
+ const executionId = task.executionId;
248
+ if (afterCommit)
249
+ this.clearExecution(task.id, executionId);
250
+ task.leaseOwner = null;
251
+ task.leaseExpiresAt = null;
252
+ task.executionId = null;
253
+ task.updatedAt = new Date().toISOString();
254
+ if (afterCommit) {
255
+ this.terminalWaiters.get(task.id)?.resolve();
256
+ this.terminalWaiters.delete(task.id);
257
+ }
258
+ return true;
259
+ }
257
260
  async failLaunch(jobId, taskId, executionId, message, phase = 'launch') {
258
261
  const claim = executionId ?? `${jobId}:${taskId}`;
259
262
  if (this.launchFailureClaims.has(claim))
@@ -280,6 +283,11 @@ export class ReconcilerRuntimeBase {
280
283
  }
281
284
  const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason, workflowEvent }, { agent: 'relay', component: 'agents-relay', watchdog: true });
282
285
  this.detach(this.emit(event), jobId, taskId, 'worker failure event');
286
+ // failLaunch already committed the authoritative FAILED state. The emitted
287
+ // task.failed observation may be ignored as a duplicate, so explicitly wake
288
+ // graph reconciliation from the durable failure fact. This is required for
289
+ // automatic Troubleshooter recovery, including reviewer start-ack failures.
290
+ this.detach(this.eventCoordinator.accept({ jobId, reason: `worker-${phase}-failure` }), jobId, taskId, 'worker failure reconcile wake');
283
291
  }
284
292
  async runtimeSettled(jobId, taskId, executionId, result, error) {
285
293
  const live = this.live.get(taskId);
@@ -289,7 +297,15 @@ export class ReconcilerRuntimeBase {
289
297
  }
290
298
  const job = await this.store.load(jobId);
291
299
  const task = job.tasks.find(item => item.id === taskId);
292
- if (terminalExecutionClaimed(jobId, taskId, executionId) || !task || task.executionId !== executionId)
300
+ // Async transport settlement is an independent, non-authoritative observation.
301
+ // A worker terminal event may already have claimed the Task; do not suppress
302
+ // transport-exit evidence merely because business completion won the race.
303
+ if (!task)
304
+ return;
305
+ const correlated = task.executionId === executionId || task.executionHistory?.some(item => item.id === executionId);
306
+ if (!correlated)
307
+ return;
308
+ if (task.adapter !== 'chatgpt' && (task.executionId !== executionId || terminalExecutionClaimed(jobId, taskId, executionId)))
293
309
  return;
294
310
  if (task.adapter !== 'chatgpt' && task.executionId === executionId && live?.executionId === executionId)
295
311
  rearmTerminalEventDeadline(this.terminalTimers, taskId, task.timeoutMs - (Date.now() - live.launchBarrier), this.options.postExitTerminalGraceMs ?? 15_000, () => this.detach(this.timeoutTask(jobId, taskId, executionId), jobId, taskId, 'post-exit terminal event timeout'));
@@ -305,7 +321,7 @@ export class ReconcilerRuntimeBase {
305
321
  const exit = live?.executionId === executionId && live.processSettled ? live.execution.processExit : undefined;
306
322
  const rawError = live?.executionId === executionId && live.processSettled ? (live.processError || exit?.stderr || '').trim() : '';
307
323
  const failedExit = Boolean(exit && (exit.exitCode !== 0 || exit.signal));
308
- const failureReason = failedExit ? (/rate.limit.exceeded|usage limit reached/i.test(rawError) ? 'rate_limit_exceeded' : 'worker_process_failed') : 'terminal_event_timeout';
324
+ const failureReason = failedExit ? classifyWorkerRejection(rawError) : 'terminal_event_timeout';
309
325
  const failureMessage = failedExit ? (rawError || `Worker exited with code ${exit?.exitCode ?? 'null'}${exit?.signal ? `, signal ${exit.signal}` : ''}`) : `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
310
326
  const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
311
327
  eligible: (_job, task) => task.executionId === executionId,
@@ -321,21 +337,38 @@ export class ReconcilerRuntimeBase {
321
337
  this.clearExecution(taskId, executionId);
322
338
  return;
323
339
  }
324
- const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? failureMessage, 'user', { reason: failureReason, exitCode: exit?.exitCode ?? null, signal: exit?.signal ?? null, timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed', executionId }, { agent: 'relay', component: 'agents-relay', watchdog: true });
325
- await this.emit(event);
326
- // Provider quota failures are recoverable. Drop the stale selection so the
327
- // next launch runs the normal worker/model router with fresh usage data.
328
- if (failureReason === 'rate_limit_exceeded' && transition.task.attempt < transition.task.maxAttempts) {
340
+ let reroutedTask = null;
341
+ // Provider quota failures are recoverable. Commit the retry + route exclusion
342
+ // before publishing task.failed so event-driven reconciliation can never see
343
+ // an intermediate FAILED state and spawn a Troubleshooter for quota exhaustion.
344
+ if (reroutableWorkerRejections.has(failureReason) && transition.task.attempt < transition.task.maxAttempts) {
329
345
  const current = (await this.store.load(jobId)).tasks.find(task => task.id === taskId);
330
346
  if (current?.state === 'FAILED' && current.failureReason === failureReason && current.attempt === transition.task.attempt) {
347
+ const failedRoute = current.routing;
331
348
  await applyTaskRetryAction(current);
349
+ if (failedRoute) {
350
+ const duplicate = current.routingExclusions?.some(item => item.provider === failedRoute.provider && item.model === failedRoute.model && item.profile === failedRoute.profile);
351
+ if (!duplicate)
352
+ current.routingExclusions = [...(current.routingExclusions ?? []), {
353
+ provider: failedRoute.provider,
354
+ model: failedRoute.model,
355
+ ...(failedRoute.profile ? { profile: failedRoute.profile } : {}),
356
+ reason: 'rate_limit_exceeded',
357
+ at: new Date().toISOString(),
358
+ }];
359
+ }
332
360
  current.routing = undefined;
333
361
  current.workerSelection = undefined;
334
362
  await this.store.saveTask(current);
335
- await this.emit(eventFor(jobId, taskId, current.parentTaskId, 'task.rerouting', 'queued', 'Provider rate limit reached; retrying with fresh routing and usage data', 'orchestrator', { previousFailure: failureReason, attempt: current.attempt, maxAttempts: current.maxAttempts }));
336
- await this.eventCoordinator.accept({ jobId, reason: 'rate-limit-reroute' });
363
+ reroutedTask = current;
337
364
  }
338
365
  }
366
+ const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? failureMessage, 'user', { reason: failureReason, exitCode: exit?.exitCode ?? null, signal: exit?.signal ?? null, timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed', executionId }, { agent: 'relay', component: 'agents-relay', watchdog: true });
367
+ await this.emit(event);
368
+ if (reroutedTask) {
369
+ await this.emit(eventFor(jobId, taskId, reroutedTask.parentTaskId, 'task.rerouting', 'queued', `Worker rejection ${failureReason}; retrying with fresh routing and usage data`, 'orchestrator', { previousFailure: failureReason, attempt: reroutedTask.attempt, maxAttempts: reroutedTask.maxAttempts }));
370
+ await this.eventCoordinator.accept({ jobId, reason: 'worker-rejection-reroute' });
371
+ }
339
372
  }
340
373
  async applyTerminalEvent(event, trustedInternal = false) {
341
374
  const liveAtTerminal = this.live.get(event.task_id);
@@ -344,6 +377,10 @@ export class ReconcilerRuntimeBase {
344
377
  eligible: (_job, task) => {
345
378
  if (!['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
346
379
  return false;
380
+ // Reviewer business terminal state is owned by review.approved / review.changes_required / review.blocked.
381
+ // A generic task.completed may arrive before the review decision and must not consume the execution generation.
382
+ if (task.agentName === 'reviewer' && event.type === 'task.completed')
383
+ return false;
347
384
  const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
348
385
  if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event)) {
349
386
  const eventExecutionId = executionIdFromSource(eventSourceId(event), _job.id, task.id);