agents-relay 2.0.67 → 2.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -20
- package/dist/adapters.js +39 -3
- package/dist/blockers.js +17 -4
- package/dist/cli-blocker.js +12 -30
- package/dist/cli-help.js +22 -17
- package/dist/cli-job-create.js +77 -0
- package/dist/cli-local-merge-config.js +11 -0
- package/dist/cli-projection.js +23 -55
- package/dist/cli-review.js +7 -24
- package/dist/cli-support.js +81 -8
- package/dist/cli-worker-task.js +63 -0
- package/dist/cli.js +223 -185
- package/dist/dashboard-html.js +15 -12
- package/dist/dashboard.js +27 -2
- package/dist/e2e-injection.js +76 -0
- package/dist/github-auth.js +4 -3
- package/dist/github-binding-order.js +15 -0
- package/dist/github-client.js +7 -2
- package/dist/github-pr-archive.js +22 -0
- package/dist/github-webhook.js +43 -10
- package/dist/job-actions.js +29 -0
- package/dist/local-authority-store.js +288 -0
- package/dist/local-commit-events.js +2 -0
- package/dist/local-git-merge.js +124 -0
- package/dist/local-job-path.js +7 -0
- package/dist/local-job-registry.js +29 -0
- package/dist/local-managed-store.js +21 -0
- package/dist/local-merge-delivery.js +53 -0
- package/dist/local-merge-lock.js +103 -0
- package/dist/local-merge-projection.js +60 -0
- package/dist/managed-worker-input.js +13 -6
- package/dist/manual-plan.js +29 -7
- package/dist/no-github.js +11 -0
- package/dist/planned-review-head.js +7 -8
- package/dist/planner-runtime.js +182 -41
- package/dist/planner.js +35 -2
- package/dist/pool.js +89 -136
- package/dist/projected-store.js +20 -3
- package/dist/projection-outbox.js +3 -1
- package/dist/publication-store.js +37 -0
- package/dist/reconciler-cancellation.js +21 -0
- package/dist/reconciler-review.js +99 -29
- package/dist/reconciler-runtime.js +72 -35
- package/dist/reconciler.js +124 -85
- package/dist/relay-store.js +108 -6
- package/dist/relayd-control.js +184 -0
- package/dist/relayd-dashboard-actions.js +108 -0
- package/dist/relayd-utilities.js +107 -0
- package/dist/relayd.js +304 -299
- package/dist/review-generation-fence.js +12 -0
- package/dist/runtime-status-events.js +21 -0
- package/dist/runtime-status.js +21 -7
- package/dist/scheduled-jobs.js +43 -36
- package/dist/service-external-actions.js +27 -0
- package/dist/service-publication.js +29 -0
- package/dist/store-github.js +17 -6
- package/dist/task-actions.js +59 -1
- package/dist/task-update.js +1 -1
- package/dist/troubleshooter.js +21 -2
- package/dist/unblock.js +47 -15
- package/dist/worker-rejection.js +11 -0
- package/dist/workflow/contracts.js +10 -1
- package/dist/workflow/definitions-v2.js +2 -0
- package/dist/workflow/job-reconcile-v2.js +9 -60
- package/dist/workflow/runtime.js +18 -3
- package/dist/workflow-effect-failures.js +30 -0
- package/dist/workspace.js +95 -6
- package/package.json +6 -3
- package/skills/agents-relay/agents/planner.agent.md +3 -0
- package/skills/agents-relay/contracts/orchestrator-workflows.md +4 -7
- package/workflow/job-lifecycle.mmd +19 -60
- package/workflow/review-lifecycle.mmd +5 -8
|
@@ -1,22 +1,37 @@
|
|
|
1
1
|
import { storySizeTimeoutMs, taskOverallTimeoutMs } from './types.js';
|
|
2
|
-
import { completionGateSatisfied, schedule, transitionTaskEvent,
|
|
2
|
+
import { completionGateSatisfied, schedule, transitionTaskEvent, tryTransitionTaskEvent } from './scheduler.js';
|
|
3
3
|
import { implementationComplete, implementationTasks } from './runtime-status.js';
|
|
4
4
|
import { eventFor } from './events.js';
|
|
5
5
|
import { workerCandidates } from './worker-routing.js';
|
|
6
6
|
import { routeExecution } from './execution-routing.js';
|
|
7
7
|
import { ReconcilerRuntimeBase } from './reconciler-runtime.js';
|
|
8
8
|
import { dispatchReviewLifecycle, reviewOutcome } from './workflow/review-v2.js';
|
|
9
|
+
import { dispatchTaskLifecycleSync } from './workflow/lifecycle.js';
|
|
9
10
|
import { createTaskEntity } from './workflow/entity-creation.js';
|
|
11
|
+
import { localSourceHead, localWorkingHead } from './local-git-merge.js';
|
|
10
12
|
import { chatgptThreadRequirements, resolveReviewerThreadDestination } from './reviewer-thread.js';
|
|
13
|
+
import { reviewGenerationMatches } from './review-generation-fence.js';
|
|
11
14
|
export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
12
|
-
async currentHeadSha(job) {
|
|
13
|
-
|
|
15
|
+
async currentHeadSha(job) {
|
|
16
|
+
// GitHub PR head facts are audit-only. Source identity is local Git.
|
|
17
|
+
if (!this.options.projectPathForJob)
|
|
18
|
+
return job.headSha ?? job.review?.headSha ?? null;
|
|
19
|
+
try {
|
|
20
|
+
const project = await this.options.projectPathForJob(job);
|
|
21
|
+
if (job.localMerge?.sourceRef)
|
|
22
|
+
return localSourceHead(project, job.localMerge.sourceRef);
|
|
23
|
+
return localWorkingHead(project);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return job.headSha ?? job.review?.headSha ?? null;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
reviewFacts(job, headSha) {
|
|
14
30
|
const reviewTasks = job.tasks.filter(task => task.agentName === 'reviewer' && task.reviewHeadSha === headSha);
|
|
15
31
|
const currentReviewTask = job.review?.taskId ? job.tasks.find(task => task.id === job.review?.taskId) : undefined;
|
|
16
32
|
return {
|
|
17
33
|
unscheduled: !job.schedule,
|
|
18
34
|
reviewEnabled: job.reviewEnabled !== false,
|
|
19
|
-
prOpen: prState === 'OPEN',
|
|
20
35
|
jobActive: job.state === 'ACTIVE',
|
|
21
36
|
implementationComplete: implementationComplete(job),
|
|
22
37
|
headAvailable: Boolean(headSha),
|
|
@@ -105,8 +120,12 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
105
120
|
}
|
|
106
121
|
throw new Error('Too many final review rounds for ' + headSha);
|
|
107
122
|
}
|
|
123
|
+
externalReviewPrNumber(job) {
|
|
124
|
+
return job.externalPrNumber && job.externalPrNumber > 0 ? job.externalPrNumber : job.prNumber;
|
|
125
|
+
}
|
|
108
126
|
async createFinalReviewTask(job, headSha) {
|
|
109
127
|
const id = this.nextFinalReviewTaskId(job, headSha);
|
|
128
|
+
const reviewPrNumber = this.externalReviewPrNumber(job);
|
|
110
129
|
const now = new Date().toISOString();
|
|
111
130
|
const reviewGateTasks = implementationTasks(job);
|
|
112
131
|
const reviewThread = resolveReviewerThreadDestination(job, this.options.reviewer?.threadId);
|
|
@@ -117,13 +136,13 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
117
136
|
threadSource: reviewThread.source,
|
|
118
137
|
requestId: `${job.id}:${id}:${headSha}`,
|
|
119
138
|
jobId: job.id,
|
|
120
|
-
prNumber:
|
|
139
|
+
prNumber: reviewPrNumber,
|
|
121
140
|
reviewHeadSha: headSha,
|
|
122
141
|
projectId: reviewerRouting.candidate.routing.projectId ?? null,
|
|
123
142
|
} : undefined;
|
|
124
143
|
const input = [
|
|
125
144
|
`Job ID: ${job.id}`,
|
|
126
|
-
`Review PR #${
|
|
145
|
+
`Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.`,
|
|
127
146
|
`Review head SHA: ${headSha}`,
|
|
128
147
|
'Job objective: ' + (job.objective ?? job.description ?? job.title),
|
|
129
148
|
'Use the selected reviewer agent contract as the authoritative review protocol.',
|
|
@@ -146,9 +165,9 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
146
165
|
await this.emit(eventFor(job.id, id, null, 'review.created', 'queued', 'Final review task ' + id + ' created for ' + headSha + ' using ' + reviewerRouting.reason, 'orchestrator', { headSha, agentName: 'reviewer', reviewerRouting: { workerId: reviewerRouting.selection.workerId, reason: reviewerRouting.reason, selectedBy: reviewerRouting.selection.selectedBy } }));
|
|
147
166
|
return id;
|
|
148
167
|
}
|
|
149
|
-
async ensureFinalReview(job,
|
|
168
|
+
async ensureFinalReview(job, headSha) {
|
|
150
169
|
let reviewTaskId = null;
|
|
151
|
-
const decision = await dispatchReviewLifecycle(job.review?.outcome, 'reconcile', this.reviewFacts(job,
|
|
170
|
+
const decision = await dispatchReviewLifecycle(job.review?.outcome, 'reconcile', this.reviewFacts(job, headSha), async (actionId) => {
|
|
152
171
|
if (actionId !== 'review.create')
|
|
153
172
|
throw new Error('Unsupported review reconcile action ' + actionId);
|
|
154
173
|
if (!headSha)
|
|
@@ -234,7 +253,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
234
253
|
id: job.id, title: job.title, objective: job.objective ?? null,
|
|
235
254
|
description: job.description ?? null, priority: job.priority ?? null,
|
|
236
255
|
executionMode: job.executionMode, repository: job.repository,
|
|
237
|
-
prNumber: job
|
|
256
|
+
prNumber: this.externalReviewPrNumber(job), state: job.state,
|
|
238
257
|
},
|
|
239
258
|
task: {
|
|
240
259
|
id: task.id, kind: task.kind ?? 'work', storySize: task.storySize ?? 'M',
|
|
@@ -251,9 +270,11 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
251
270
|
? job.tasks.find(item => item.id === task.parentTaskId)
|
|
252
271
|
: undefined;
|
|
253
272
|
const parentRouteFailedBeforeStart = Boolean(parent?.routing && /exited before publishing task\.started|worker_exit_before_start|launch failed/i.test(parent.error ?? parent.failureReason ?? ''));
|
|
254
|
-
const
|
|
273
|
+
const parentExcludedRoutes = parentRouteFailedBeforeStart && parent?.routing
|
|
255
274
|
? [{ provider: parent.routing.provider, model: parent.routing.model, profile: parent.routing.profile }]
|
|
256
275
|
: [];
|
|
276
|
+
const taskExcludedRoutes = (task.routingExclusions ?? []).map(item => ({ provider: item.provider, model: item.model, profile: item.profile }));
|
|
277
|
+
const excludedRoutes = [...taskExcludedRoutes, ...parentExcludedRoutes];
|
|
257
278
|
const routed = await routeExecution(candidates, requirements, {
|
|
258
279
|
decisionEngine: this.options.decisionEngine,
|
|
259
280
|
decisionMode: this.options.decisionMode,
|
|
@@ -287,26 +308,30 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
287
308
|
const headSha = normalizedEvidence.headSha;
|
|
288
309
|
if (task.reviewHeadSha !== headSha)
|
|
289
310
|
return { applied: false, reconcile: false, reason: event.type + '.stale' };
|
|
290
|
-
const currentHeadSha = await this.
|
|
311
|
+
const currentHeadSha = await this.currentHeadSha(job);
|
|
291
312
|
if ((currentHeadSha && currentHeadSha !== headSha) || (job.headSha && job.headSha !== headSha))
|
|
292
313
|
return { applied: false, reconcile: false, reason: event.type + '.stale' };
|
|
293
314
|
// Planner-created reviewer tasks predate the final-review Job gate. Bind
|
|
294
315
|
// their first trustworthy review event to PENDING before dispatching it.
|
|
295
316
|
if (!job.review && task.planningDecisionId) {
|
|
296
317
|
job.review = { outcome: 'pending', headSha, taskId: task.id, evidence: null, updatedAt: event.timestamp || new Date().toISOString() };
|
|
297
|
-
await this.store.saveJob(job);
|
|
298
318
|
}
|
|
299
|
-
const taskDecision =
|
|
319
|
+
const taskDecision = this.store.mutateJob
|
|
320
|
+
? dispatchTaskLifecycleSync(structuredClone(task), event.type)
|
|
321
|
+
: tryTransitionTaskEvent(task, event.type);
|
|
300
322
|
if (!taskDecision.accepted || !taskDecision.target)
|
|
301
323
|
return { applied: false, reconcile: false, reason: event.type + '.ignored' };
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
task.
|
|
324
|
+
if (!this.store.mutateJob && taskDecision.target !== 'RUNNING' && !this.finalizeReviewExecution(job.id, task, event)) {
|
|
325
|
+
return { applied: false, reconcile: false, reason: event.type + '.stale-execution' };
|
|
326
|
+
}
|
|
327
|
+
if (!this.store.mutateJob) {
|
|
328
|
+
task.error = null;
|
|
329
|
+
if (taskDecision.target !== 'RUNNING')
|
|
330
|
+
task.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
|
|
331
|
+
await this.store.saveTask(task);
|
|
307
332
|
}
|
|
308
|
-
await this.store.saveTask(task);
|
|
309
333
|
let needsReconcile = false;
|
|
334
|
+
let pendingFixTask = null;
|
|
310
335
|
const reviewDecision = await dispatchReviewLifecycle(job.review?.outcome, event.type, { autoMerge: job.autoMerge !== false }, async (actionId) => {
|
|
311
336
|
if (actionId === 'review.createFix') {
|
|
312
337
|
needsReconcile = true;
|
|
@@ -316,18 +341,14 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
316
341
|
const now = event.timestamp || new Date().toISOString();
|
|
317
342
|
const id = 'review-fix-' + headSha.slice(0, 8);
|
|
318
343
|
if (!job.tasks.some(item => item.id === id)) {
|
|
319
|
-
const input = 'Address all reviewer findings for PR #' + job
|
|
344
|
+
const input = 'Address all reviewer findings for PR #' + this.externalReviewPrNumber(job) + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
|
|
320
345
|
const fixTask = createTaskEntity(job, { jobId: job.id, id, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
|
|
321
|
-
|
|
322
|
-
await this.emit(eventFor(job.id, id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + id + ' for ' + issues.length + ' reviewer finding(s)', 'orchestrator', { headSha, issues }, { agent: 'relay', component: 'agents-relay' }));
|
|
346
|
+
pendingFixTask = fixTask;
|
|
323
347
|
}
|
|
324
348
|
return {};
|
|
325
349
|
}
|
|
326
350
|
if (actionId === 'review.requestMerge') {
|
|
327
|
-
|
|
328
|
-
const outcomes = await this.workflowEffects.execute([effectId], { jobId: job.id, taskId: task.id, transitionKey: 'review.approved:' + headSha, data: { headSha } });
|
|
329
|
-
await this.applyJobEffectOutcomes(job, outcomes);
|
|
330
|
-
}, async () => { await this.store.saveJob(job); });
|
|
351
|
+
// Approval is a local fact; only subsequent local Git delivery may complete the Job.
|
|
331
352
|
return {};
|
|
332
353
|
}
|
|
333
354
|
throw new Error('Unsupported review event action ' + actionId);
|
|
@@ -342,9 +363,58 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
|
|
|
342
363
|
const evidence = { summary: event.message || 'Review ' + outcome, data: {
|
|
343
364
|
...(event.data ?? {}), review: { ...(reviewData ?? {}), headSha }, headSha, outcome, source: event.source,
|
|
344
365
|
} };
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
366
|
+
const reviewUpdatedAt = event.timestamp || new Date().toISOString();
|
|
367
|
+
if (this.store.mutateJob) {
|
|
368
|
+
let committed = false;
|
|
369
|
+
let terminalExecutionId = null;
|
|
370
|
+
const updated = await this.store.mutateJob(job.id, current => {
|
|
371
|
+
const freshTask = current.tasks.find(item => item.id === task.id);
|
|
372
|
+
if (!freshTask || !reviewGenerationMatches(current, job, task, headSha))
|
|
373
|
+
return current;
|
|
374
|
+
const freshDecision = tryTransitionTaskEvent(freshTask, event.type);
|
|
375
|
+
if (!freshDecision.accepted || !freshDecision.target)
|
|
376
|
+
return current;
|
|
377
|
+
if (freshDecision.target !== 'RUNNING') {
|
|
378
|
+
if (!this.finalizeReviewExecution(job.id, freshTask, event, false))
|
|
379
|
+
return current;
|
|
380
|
+
terminalExecutionId = freshTask.executionId;
|
|
381
|
+
}
|
|
382
|
+
freshTask.error = null;
|
|
383
|
+
if (freshDecision.target !== 'RUNNING')
|
|
384
|
+
freshTask.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
|
|
385
|
+
if (pendingFixTask && !current.tasks.some(item => item.id === pendingFixTask.id))
|
|
386
|
+
current.tasks.push(structuredClone(pendingFixTask));
|
|
387
|
+
current.review = { outcome, headSha, taskId: freshTask.id, evidence, updatedAt: reviewUpdatedAt };
|
|
388
|
+
current.updatedAt = reviewUpdatedAt;
|
|
389
|
+
committed = true;
|
|
390
|
+
return current;
|
|
391
|
+
});
|
|
392
|
+
if (!committed)
|
|
393
|
+
return { applied: false, reconcile: false, reason: event.type + '.stale-execution' };
|
|
394
|
+
Object.assign(job, structuredClone(updated));
|
|
395
|
+
// Live waiter cleanup/terminal claims are irreversible; run after commit.
|
|
396
|
+
if (terminalExecutionId) {
|
|
397
|
+
this.finalizeReviewExecution(job.id, { ...task, executionId: terminalExecutionId }, event);
|
|
398
|
+
}
|
|
399
|
+
const committedFixTask = pendingFixTask;
|
|
400
|
+
if (committedFixTask)
|
|
401
|
+
await this.emit(eventFor(job.id, committedFixTask.id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + committedFixTask.id + ' for reviewer findings', 'orchestrator', { headSha }, { agent: 'relay', component: 'agents-relay' }));
|
|
402
|
+
}
|
|
403
|
+
else {
|
|
404
|
+
const legacyFixTask = pendingFixTask;
|
|
405
|
+
if (legacyFixTask) {
|
|
406
|
+
await this.store.appendTask(legacyFixTask);
|
|
407
|
+
await this.emit(eventFor(job.id, legacyFixTask.id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + legacyFixTask.id + ' for reviewer findings', 'orchestrator', { headSha }, { agent: 'relay', component: 'agents-relay' }));
|
|
408
|
+
}
|
|
409
|
+
job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: reviewUpdatedAt };
|
|
410
|
+
job.updatedAt = reviewUpdatedAt;
|
|
411
|
+
await this.store.saveTask(task);
|
|
412
|
+
await this.store.saveJob(job);
|
|
413
|
+
}
|
|
414
|
+
// Review approval records evidence only. The subsequent Graph reconcile
|
|
415
|
+
// drives durable local Git merge delivery and terminalization.
|
|
416
|
+
if (outcome === 'approved')
|
|
417
|
+
needsReconcile = true;
|
|
348
418
|
return { applied: true, reconcile: needsReconcile, reason: 'review.' + outcome, reconcileMode: needsReconcile ? 'detached' : undefined };
|
|
349
419
|
}
|
|
350
420
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
import { applyJobEffectOutcomes, applyTaskEffectOutcomes } from './workflow-effect-failures.js';
|
|
2
|
+
import { classifyWorkerRejection, reroutableWorkerRejections } from './worker-rejection.js';
|
|
1
3
|
import { applyTaskRetryAction } from './task-actions.js';
|
|
2
|
-
import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
|
|
3
4
|
import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
|
|
4
5
|
import { eventFor } from './events.js';
|
|
5
6
|
import { WorkflowEffectExecutor } from './workflow/effects.js';
|
|
6
|
-
import { workflowEffectFailureEvent } from './workflow/contracts.js';
|
|
7
7
|
import { ReconcileCoordinator } from './reconcile-coordinator.js';
|
|
8
8
|
import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
|
|
9
9
|
import { rearmTerminalEventDeadline } from './terminal-event-watchdog.js';
|
|
@@ -77,31 +77,10 @@ export class ReconcilerRuntimeBase {
|
|
|
77
77
|
});
|
|
78
78
|
}
|
|
79
79
|
async applyJobEffectOutcomes(job, outcomes) {
|
|
80
|
-
|
|
81
|
-
const eventName = workflowEffectFailureEvent(outcome.effect);
|
|
82
|
-
try {
|
|
83
|
-
await transitionJobEvent(job, eventName);
|
|
84
|
-
await this.store.saveJob(job);
|
|
85
|
-
}
|
|
86
|
-
catch {
|
|
87
|
-
// Some effects are observational for a given state; failure is still emitted below.
|
|
88
|
-
}
|
|
89
|
-
await this.emit(eventFor(job.id, 'job', null, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
|
|
90
|
-
}
|
|
80
|
+
await applyJobEffectOutcomes(this.store, event => this.emit(event), job, outcomes);
|
|
91
81
|
}
|
|
92
82
|
async applyTaskEffectOutcomes(job, task, outcomes) {
|
|
93
|
-
|
|
94
|
-
const eventName = workflowEffectFailureEvent(outcome.effect);
|
|
95
|
-
try {
|
|
96
|
-
await transitionTaskEvent(task, eventName);
|
|
97
|
-
task.error = outcome.error ?? `${outcome.effect} failed`;
|
|
98
|
-
await this.store.saveTask(task);
|
|
99
|
-
}
|
|
100
|
-
catch {
|
|
101
|
-
// The effect failure remains observable even if this state has no feedback transition.
|
|
102
|
-
}
|
|
103
|
-
await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
|
|
104
|
-
}
|
|
83
|
+
await applyTaskEffectOutcomes(this.store, event => this.emit(event), job, task, outcomes);
|
|
105
84
|
}
|
|
106
85
|
async applyEvent(event) {
|
|
107
86
|
const sourceId = eventSourceId(event);
|
|
@@ -122,7 +101,9 @@ export class ReconcilerRuntimeBase {
|
|
|
122
101
|
}
|
|
123
102
|
if (['review.started', 'review.approved', 'review.changes_required', 'review.blocked'].includes(event.type))
|
|
124
103
|
return this.applyReviewEvent(event);
|
|
125
|
-
if (event.type === '
|
|
104
|
+
if (event.type === 'github.webhook')
|
|
105
|
+
return { applied: true, reconcile: false, reason: 'github.audit_only' };
|
|
106
|
+
if (event.type === 'job.wake')
|
|
126
107
|
return { applied: true, reconcile: true, reason: event.type };
|
|
127
108
|
return { applied: false, reconcile: false, reason: event.type };
|
|
128
109
|
}
|
|
@@ -254,6 +235,28 @@ export class ReconcilerRuntimeBase {
|
|
|
254
235
|
isCorrelatedWorkerEvent(task, event) {
|
|
255
236
|
return task.executionId !== null && eventSourceId(event) === executionEventSource(event.job_id, task.id, task.executionId);
|
|
256
237
|
}
|
|
238
|
+
finalizeReviewExecution(jobId, task, event, afterCommit = true) {
|
|
239
|
+
if (!task.executionId)
|
|
240
|
+
return true;
|
|
241
|
+
const sourceId = eventSourceId(event);
|
|
242
|
+
const expected = executionEventSource(jobId, task.id, task.executionId);
|
|
243
|
+
if ((task.adapter === 'codex' || task.adapter === 'chatgpt') && sourceId !== expected)
|
|
244
|
+
return false;
|
|
245
|
+
if (afterCommit)
|
|
246
|
+
claimTerminalExecution(expected);
|
|
247
|
+
const executionId = task.executionId;
|
|
248
|
+
if (afterCommit)
|
|
249
|
+
this.clearExecution(task.id, executionId);
|
|
250
|
+
task.leaseOwner = null;
|
|
251
|
+
task.leaseExpiresAt = null;
|
|
252
|
+
task.executionId = null;
|
|
253
|
+
task.updatedAt = new Date().toISOString();
|
|
254
|
+
if (afterCommit) {
|
|
255
|
+
this.terminalWaiters.get(task.id)?.resolve();
|
|
256
|
+
this.terminalWaiters.delete(task.id);
|
|
257
|
+
}
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
257
260
|
async failLaunch(jobId, taskId, executionId, message, phase = 'launch') {
|
|
258
261
|
const claim = executionId ?? `${jobId}:${taskId}`;
|
|
259
262
|
if (this.launchFailureClaims.has(claim))
|
|
@@ -280,6 +283,11 @@ export class ReconcilerRuntimeBase {
|
|
|
280
283
|
}
|
|
281
284
|
const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason, workflowEvent }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
282
285
|
this.detach(this.emit(event), jobId, taskId, 'worker failure event');
|
|
286
|
+
// failLaunch already committed the authoritative FAILED state. The emitted
|
|
287
|
+
// task.failed observation may be ignored as a duplicate, so explicitly wake
|
|
288
|
+
// graph reconciliation from the durable failure fact. This is required for
|
|
289
|
+
// automatic Troubleshooter recovery, including reviewer start-ack failures.
|
|
290
|
+
this.detach(this.eventCoordinator.accept({ jobId, reason: `worker-${phase}-failure` }), jobId, taskId, 'worker failure reconcile wake');
|
|
283
291
|
}
|
|
284
292
|
async runtimeSettled(jobId, taskId, executionId, result, error) {
|
|
285
293
|
const live = this.live.get(taskId);
|
|
@@ -289,7 +297,15 @@ export class ReconcilerRuntimeBase {
|
|
|
289
297
|
}
|
|
290
298
|
const job = await this.store.load(jobId);
|
|
291
299
|
const task = job.tasks.find(item => item.id === taskId);
|
|
292
|
-
|
|
300
|
+
// Async transport settlement is an independent, non-authoritative observation.
|
|
301
|
+
// A worker terminal event may already have claimed the Task; do not suppress
|
|
302
|
+
// transport-exit evidence merely because business completion won the race.
|
|
303
|
+
if (!task)
|
|
304
|
+
return;
|
|
305
|
+
const correlated = task.executionId === executionId || task.executionHistory?.some(item => item.id === executionId);
|
|
306
|
+
if (!correlated)
|
|
307
|
+
return;
|
|
308
|
+
if (task.adapter !== 'chatgpt' && (task.executionId !== executionId || terminalExecutionClaimed(jobId, taskId, executionId)))
|
|
293
309
|
return;
|
|
294
310
|
if (task.adapter !== 'chatgpt' && task.executionId === executionId && live?.executionId === executionId)
|
|
295
311
|
rearmTerminalEventDeadline(this.terminalTimers, taskId, task.timeoutMs - (Date.now() - live.launchBarrier), this.options.postExitTerminalGraceMs ?? 15_000, () => this.detach(this.timeoutTask(jobId, taskId, executionId), jobId, taskId, 'post-exit terminal event timeout'));
|
|
@@ -305,7 +321,7 @@ export class ReconcilerRuntimeBase {
|
|
|
305
321
|
const exit = live?.executionId === executionId && live.processSettled ? live.execution.processExit : undefined;
|
|
306
322
|
const rawError = live?.executionId === executionId && live.processSettled ? (live.processError || exit?.stderr || '').trim() : '';
|
|
307
323
|
const failedExit = Boolean(exit && (exit.exitCode !== 0 || exit.signal));
|
|
308
|
-
const failureReason = failedExit ? (
|
|
324
|
+
const failureReason = failedExit ? classifyWorkerRejection(rawError) : 'terminal_event_timeout';
|
|
309
325
|
const failureMessage = failedExit ? (rawError || `Worker exited with code ${exit?.exitCode ?? 'null'}${exit?.signal ? `, signal ${exit.signal}` : ''}`) : `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
|
|
310
326
|
const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
|
|
311
327
|
eligible: (_job, task) => task.executionId === executionId,
|
|
@@ -321,21 +337,38 @@ export class ReconcilerRuntimeBase {
|
|
|
321
337
|
this.clearExecution(taskId, executionId);
|
|
322
338
|
return;
|
|
323
339
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
//
|
|
327
|
-
//
|
|
328
|
-
if (failureReason
|
|
340
|
+
let reroutedTask = null;
|
|
341
|
+
// Provider quota failures are recoverable. Commit the retry + route exclusion
|
|
342
|
+
// before publishing task.failed so event-driven reconciliation can never see
|
|
343
|
+
// an intermediate FAILED state and spawn a Troubleshooter for quota exhaustion.
|
|
344
|
+
if (reroutableWorkerRejections.has(failureReason) && transition.task.attempt < transition.task.maxAttempts) {
|
|
329
345
|
const current = (await this.store.load(jobId)).tasks.find(task => task.id === taskId);
|
|
330
346
|
if (current?.state === 'FAILED' && current.failureReason === failureReason && current.attempt === transition.task.attempt) {
|
|
347
|
+
const failedRoute = current.routing;
|
|
331
348
|
await applyTaskRetryAction(current);
|
|
349
|
+
if (failedRoute) {
|
|
350
|
+
const duplicate = current.routingExclusions?.some(item => item.provider === failedRoute.provider && item.model === failedRoute.model && item.profile === failedRoute.profile);
|
|
351
|
+
if (!duplicate)
|
|
352
|
+
current.routingExclusions = [...(current.routingExclusions ?? []), {
|
|
353
|
+
provider: failedRoute.provider,
|
|
354
|
+
model: failedRoute.model,
|
|
355
|
+
...(failedRoute.profile ? { profile: failedRoute.profile } : {}),
|
|
356
|
+
reason: 'rate_limit_exceeded',
|
|
357
|
+
at: new Date().toISOString(),
|
|
358
|
+
}];
|
|
359
|
+
}
|
|
332
360
|
current.routing = undefined;
|
|
333
361
|
current.workerSelection = undefined;
|
|
334
362
|
await this.store.saveTask(current);
|
|
335
|
-
|
|
336
|
-
await this.eventCoordinator.accept({ jobId, reason: 'rate-limit-reroute' });
|
|
363
|
+
reroutedTask = current;
|
|
337
364
|
}
|
|
338
365
|
}
|
|
366
|
+
const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? failureMessage, 'user', { reason: failureReason, exitCode: exit?.exitCode ?? null, signal: exit?.signal ?? null, timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed', executionId }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
367
|
+
await this.emit(event);
|
|
368
|
+
if (reroutedTask) {
|
|
369
|
+
await this.emit(eventFor(jobId, taskId, reroutedTask.parentTaskId, 'task.rerouting', 'queued', `Worker rejection ${failureReason}; retrying with fresh routing and usage data`, 'orchestrator', { previousFailure: failureReason, attempt: reroutedTask.attempt, maxAttempts: reroutedTask.maxAttempts }));
|
|
370
|
+
await this.eventCoordinator.accept({ jobId, reason: 'worker-rejection-reroute' });
|
|
371
|
+
}
|
|
339
372
|
}
|
|
340
373
|
async applyTerminalEvent(event, trustedInternal = false) {
|
|
341
374
|
const liveAtTerminal = this.live.get(event.task_id);
|
|
@@ -344,6 +377,10 @@ export class ReconcilerRuntimeBase {
|
|
|
344
377
|
eligible: (_job, task) => {
|
|
345
378
|
if (!['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
|
|
346
379
|
return false;
|
|
380
|
+
// Reviewer business terminal state is owned by review.approved / review.changes_required / review.blocked.
|
|
381
|
+
// A generic task.completed may arrive before the review decision and must not consume the execution generation.
|
|
382
|
+
if (task.agentName === 'reviewer' && event.type === 'task.completed')
|
|
383
|
+
return false;
|
|
347
384
|
const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
|
|
348
385
|
if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event)) {
|
|
349
386
|
const eventExecutionId = executionIdFromSource(eventSourceId(event), _job.id, task.id);
|