agents-relay 1.0.68 → 1.0.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/cli.js +21 -20
- package/dist/dashboard.js +6 -0
- package/dist/markers.js +2 -2
- package/dist/pool.js +4 -4
- package/dist/reconciler.js +126 -74
- package/dist/scheduler.js +29 -8
- package/dist/types.js +3 -6
- package/dist/workflow/definitions.js +14 -0
- package/dist/workflow/mermaid.js +180 -0
- package/dist/workflow/rollout.js +171 -0
- package/package.json +2 -1
- package/skills/agents-relay/contracts/task-job-contract.md +6 -0
- package/workflow/job-lifecycle.mmd +15 -0
- package/workflow/task-lifecycle.mmd +32 -0
package/README.md
CHANGED
|
@@ -49,6 +49,10 @@ npx agents-relay task update --repo OWNER/REPO --pr 12 --id job-1 --task-id chil
|
|
|
49
49
|
|
|
50
50
|
Jobs separate durable lifecycle from derived runtime status. Durable lifecycle is `DRAFT | ACTIVE | COMPLETED | CANCELLED`: a newly created unscheduled Job is DRAFT, creating executable work activates it, and scheduled Jobs are ACTIVE because the scheduler is allowed to advance them. Runtime status (`IDLE`, `WAITING`, `QUEUED`, `RUNNING`, `BLOCKED`, `FAILED`, `REVIEWING`, `MERGE_PENDING`, `MERGED`, or `CANCELLED`) is computed from Tasks, Review, PR, and schedule facts and is never persisted as lifecycle truth.
|
|
51
51
|
|
|
52
|
+
Lifecycle transition authority lives in the directly renderable canonical sources [`workflow/job-lifecycle.mmd`](workflow/job-lifecycle.mmd) and [`workflow/task-lifecycle.mmd`](workflow/task-lifecycle.mmd); see [`docs/workflow-engine.md`](docs/workflow-engine.md) for the executable Mermaid profile and runtime boundary.
|
|
53
|
+
|
|
54
|
+
Graph-engine rollout is confidence-gated with `AGENTS_RELAY_WORKFLOW_MODE=legacy|shadow|graph` (default `legacy`). `shadow` keeps legacy authoritative while comparing the Mermaid decision and exposing `{mode,matches,mismatches}` at `/api/workflow-rollout`; `graph` is explicit opt-in until the shadow cutover gate is approved.
|
|
55
|
+
|
|
52
56
|
Job objectives and Task inputs are Markdown. Prefer `--objective-file` / `--input-file` (use `-` for stdin) for durable long-form context; inline `--objective` / `--input` remain for short/simple text and are mutually exclusive with the file variants. When a diagram materially clarifies the work, put Mermaid first and then Markdown detail. A newly created Job must capture enough context to finish independently of the originating chat; its objective becomes the default PR body and objective updates keep the PR body synchronized.
|
|
53
57
|
|
|
54
58
|
Jobs default to `fixed`. `job create` persists only the durable job and never
|
package/dist/cli.js
CHANGED
|
@@ -15,7 +15,7 @@ import { serveDashboard } from './dashboard.js';
|
|
|
15
15
|
import { codexAndZaiUsageRegistry } from './usage.js';
|
|
16
16
|
import { discoverAgents } from './registry.js';
|
|
17
17
|
import { AgentObjectivePlanner, CommandObjectivePlanner } from './planner.js';
|
|
18
|
-
import { extendRetryBudget, parseJobSchedule,
|
|
18
|
+
import { extendRetryBudget, parseJobSchedule, transitionJobEvent, transitionTaskEvent } from './scheduler.js';
|
|
19
19
|
import { dueBuckets, processDueScheduledJobs, syncScheduleLabels } from './scheduled-jobs.js';
|
|
20
20
|
import { resolveExecutionContext } from './execution-context.js';
|
|
21
21
|
import { defaultWorkspaceRoot, resolveManagedProjectPath } from './workspace.js';
|
|
@@ -419,8 +419,7 @@ async function runTaskCommand(action, args) {
|
|
|
419
419
|
await store.appendTask(task);
|
|
420
420
|
const refreshed = await store.load(job.id);
|
|
421
421
|
if (refreshed.state === 'DRAFT') {
|
|
422
|
-
refreshed
|
|
423
|
-
refreshed.updatedAt = now;
|
|
422
|
+
await transitionJobEvent(refreshed, 'activate');
|
|
424
423
|
await store.saveJob(refreshed);
|
|
425
424
|
}
|
|
426
425
|
const durableTask = (await store.load(job.id)).tasks.find(item => item.id === task.id) ?? task;
|
|
@@ -439,13 +438,14 @@ async function runTaskCommand(action, args) {
|
|
|
439
438
|
if (action === 'retry' || action === 'cancel') {
|
|
440
439
|
if (action === 'retry')
|
|
441
440
|
task.maxAttempts = extendRetryBudget(task);
|
|
442
|
-
task
|
|
441
|
+
await transitionTaskEvent(task, action === 'retry' ? 'retry' : 'cancel');
|
|
443
442
|
task.error = action === 'retry' ? null : 'Cancelled by user';
|
|
444
|
-
task.
|
|
443
|
+
task.leaseOwner = null;
|
|
444
|
+
task.leaseExpiresAt = null;
|
|
445
|
+
task.executionId = null;
|
|
445
446
|
await store.saveTask(task);
|
|
446
447
|
if (action === 'retry' && job.state === 'DRAFT') {
|
|
447
|
-
job
|
|
448
|
-
job.updatedAt = task.updatedAt;
|
|
448
|
+
await transitionJobEvent(job, 'activate');
|
|
449
449
|
await store.saveJob(job);
|
|
450
450
|
}
|
|
451
451
|
await saveLocal(localFile, job);
|
|
@@ -486,13 +486,16 @@ async function runTaskCommand(action, args) {
|
|
|
486
486
|
task.maxAttempts = integerArg(args, '--max-attempts', task.maxAttempts);
|
|
487
487
|
if (hasArg(args, '--draft') && hasArg(args, '--queue'))
|
|
488
488
|
throw new Error('--draft and --queue are mutually exclusive');
|
|
489
|
-
if (hasArg(args, '--draft'))
|
|
490
|
-
|
|
489
|
+
if (hasArg(args, '--draft') && task.state === 'QUEUED')
|
|
490
|
+
await transitionTaskEvent(task, 'hold');
|
|
491
491
|
if (hasArg(args, '--queue')) {
|
|
492
492
|
const taskMap = new Map(job.tasks.map(item => [item.id, item]));
|
|
493
493
|
if (!task.dependencies.every(dep => taskMap.get(dep)?.state === 'SUCCEEDED'))
|
|
494
494
|
throw new Error('Task dependencies must succeed before queueing');
|
|
495
|
-
|
|
495
|
+
if (task.state === 'DRAFT')
|
|
496
|
+
await transitionTaskEvent(task, 'release');
|
|
497
|
+
else if (task.state === 'FAILED' || task.state === 'BLOCKED')
|
|
498
|
+
await transitionTaskEvent(task, 'retry');
|
|
496
499
|
}
|
|
497
500
|
if (hasArg(args, '--max-attempts') && task.maxAttempts < task.attempt)
|
|
498
501
|
throw new Error('--max-attempts cannot be less than the current attempt count');
|
|
@@ -603,8 +606,7 @@ async function runJobCommand(action, args) {
|
|
|
603
606
|
throw new Error(`Cannot merge cancelled job ${job.id}`);
|
|
604
607
|
const headSha = await store.headSha();
|
|
605
608
|
await store.mergePullRequest(headSha ?? undefined);
|
|
606
|
-
job
|
|
607
|
-
job.updatedAt = new Date().toISOString();
|
|
609
|
+
await transitionJobEvent(job, 'pr.merged');
|
|
608
610
|
await store.saveJob(job);
|
|
609
611
|
console.log(JSON.stringify(await store.load(id), null, 2));
|
|
610
612
|
return;
|
|
@@ -613,15 +615,15 @@ async function runJobCommand(action, args) {
|
|
|
613
615
|
if (job.state === 'COMPLETED')
|
|
614
616
|
throw new Error(`Cannot cancel completed job ${job.id}`);
|
|
615
617
|
for (const task of job.tasks.filter(item => !['SUCCEEDED', 'CANCELLED'].includes(item.state))) {
|
|
616
|
-
|
|
618
|
+
await transitionTaskEvent(task, 'cancel');
|
|
617
619
|
task.error = 'Job cancelled';
|
|
618
620
|
task.leaseOwner = null;
|
|
619
621
|
task.leaseExpiresAt = null;
|
|
622
|
+
task.executionId = null;
|
|
620
623
|
await store.saveTask(task);
|
|
621
624
|
}
|
|
622
625
|
await store.closePullRequest();
|
|
623
|
-
job
|
|
624
|
-
job.updatedAt = new Date().toISOString();
|
|
626
|
+
await transitionJobEvent(job, 'cancel');
|
|
625
627
|
await store.saveJob(job);
|
|
626
628
|
console.log(JSON.stringify(await store.load(id), null, 2));
|
|
627
629
|
return;
|
|
@@ -674,23 +676,22 @@ export function dashboardJobAction(store, reconciler, action) {
|
|
|
674
676
|
if (action === 'start') {
|
|
675
677
|
if (job.state !== 'DRAFT')
|
|
676
678
|
throw new Error(`Job ${job.id} is not startable (${job.state})`);
|
|
677
|
-
job
|
|
678
|
-
job.updatedAt = new Date().toISOString();
|
|
679
|
+
await transitionJobEvent(job, 'activate');
|
|
679
680
|
await store.saveJob(job);
|
|
680
681
|
}
|
|
681
682
|
else if (action === 'cancel') {
|
|
682
683
|
if (job.state === 'COMPLETED' || job.state === 'CANCELLED')
|
|
683
684
|
throw new Error(`Job ${job.id} is not cancellable (${job.state})`);
|
|
684
685
|
for (const task of job.tasks.filter(item => !['SUCCEEDED', 'CANCELLED'].includes(item.state))) {
|
|
685
|
-
|
|
686
|
+
await transitionTaskEvent(task, 'cancel');
|
|
686
687
|
task.error = 'Job cancelled';
|
|
687
688
|
task.leaseOwner = null;
|
|
688
689
|
task.leaseExpiresAt = null;
|
|
690
|
+
task.executionId = null;
|
|
689
691
|
await store.saveTask(task);
|
|
690
692
|
}
|
|
691
693
|
await store.closePullRequest?.();
|
|
692
|
-
job
|
|
693
|
-
job.updatedAt = new Date().toISOString();
|
|
694
|
+
await transitionJobEvent(job, 'cancel');
|
|
694
695
|
await store.saveJob(job);
|
|
695
696
|
return await store.load(job.id);
|
|
696
697
|
}
|
package/dist/dashboard.js
CHANGED
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import { readUsageSnapshot } from './usage.js';
|
|
6
6
|
import { deriveJobRuntimeStatus } from './runtime-status.js';
|
|
7
7
|
import { DeliveryDeduper, parseGitHubWebhook, verifyGitHubWebhook } from './github-webhook.js';
|
|
8
|
+
import { workflowRolloutSnapshot } from './workflow/rollout.js';
|
|
8
9
|
const GITHUB_REPOSITORY_PATTERN = /^([A-Za-z0-9](?:[A-Za-z0-9-]{0,36}[A-Za-z0-9])?)\/([A-Za-z0-9._-]{1,100})$/;
|
|
9
10
|
const SSE_HEARTBEAT_MS = 15000;
|
|
10
11
|
const EVENTS_HISTORY_FILE = process.env.EVENTS_BUS_DATA_DIR ? join(process.env.EVENTS_BUS_DATA_DIR, 'events.jsonl') : join(homedir(), 'Library', 'Application Support', 'EventsBus', 'events.jsonl');
|
|
@@ -496,6 +497,11 @@ export function serveDashboard(store, port = 8787, bus, refreshMs = 300000, jobI
|
|
|
496
497
|
res.end(JSON.stringify(events));
|
|
497
498
|
return;
|
|
498
499
|
}
|
|
500
|
+
if (req.url === '/api/workflow-rollout') {
|
|
501
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
502
|
+
res.end(JSON.stringify(workflowRolloutSnapshot()));
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
499
505
|
if (req.url === '/api/status') {
|
|
500
506
|
const status = statusProvider ? statusProvider() : { limited: false, cooldownUntil: null };
|
|
501
507
|
res.writeHead(200, { 'content-type': 'application/json' });
|
package/dist/markers.js
CHANGED
|
@@ -34,7 +34,7 @@ function contentBeforeDelimiter(body, heading, end) {
|
|
|
34
34
|
}
|
|
35
35
|
export function renderJobComment(job) {
|
|
36
36
|
const objective = job.objective?.trim() || '_No objective recorded._';
|
|
37
|
-
const { objective: _objective, tasks: _tasks, ...metadata } = job;
|
|
37
|
+
const { objective: _objective, description: _description, tasks: _tasks, ...metadata } = job;
|
|
38
38
|
return `${marker(JOB_MARKER, metadata)}
|
|
39
39
|
# Agents Relay job: ${job.title}
|
|
40
40
|
|
|
@@ -70,7 +70,7 @@ export function parseJob(body) {
|
|
|
70
70
|
const job = parseMarker(body, JOB_MARKER);
|
|
71
71
|
if (!job)
|
|
72
72
|
return null;
|
|
73
|
-
const objective = job.objective ?? contentBeforeDelimiter(body, '\n## Objective\n\n', JOB_OBJECTIVE_END) ?? markdownSection(body, '\n## Objective\n');
|
|
73
|
+
const objective = job.objective ?? job.description ?? contentBeforeDelimiter(body, '\n## Objective\n\n', JOB_OBJECTIVE_END) ?? markdownSection(body, '\n## Objective\n');
|
|
74
74
|
const parsed = { ...job, objective };
|
|
75
75
|
if (parsed.executionMode === undefined)
|
|
76
76
|
return { ...parsed, executionMode: 'fixed' };
|
package/dist/pool.js
CHANGED
|
@@ -7,9 +7,9 @@ export function needsLifecycleReconciliation(job, now = Date.now()) {
|
|
|
7
7
|
// DRAFT create-only Jobs stay dormant until a caller explicitly creates work.
|
|
8
8
|
if (job.tasks.length === 0)
|
|
9
9
|
return false;
|
|
10
|
-
if (job.tasks.some(task => task.state === 'QUEUED' || (task.state === 'DRAFT' && task.dependencies.length > 0)))
|
|
10
|
+
if (job.tasks.some(task => (task.state === 'QUEUED' && task.executionId === null) || (task.state === 'DRAFT' && task.dependencies.length > 0)))
|
|
11
11
|
return true;
|
|
12
|
-
return job.tasks.some(task => task.state
|
|
12
|
+
return job.tasks.some(task => ['RUNNING', 'QUEUED'].includes(task.state) && task.executionId !== null && task.leaseExpiresAt !== null && new Date(task.leaseExpiresAt).getTime() <= now);
|
|
13
13
|
}
|
|
14
14
|
export function terminalManagedJobs(items) { return items.filter(item => item.githubState !== 'OPEN' && ((item.githubState === 'MERGED' && item.job.state !== 'COMPLETED') || (item.githubState === 'CLOSED' && item.job.state !== 'CANCELLED'))).map(item => item.job).sort(compareJobPriority); }
|
|
15
15
|
export async function aggregateManagedGitHubJobs(client, repositories, trustedAuthors) {
|
|
@@ -48,8 +48,8 @@ export class RepositoryWorkerPool {
|
|
|
48
48
|
if (capacity <= 0)
|
|
49
49
|
break;
|
|
50
50
|
const store = new GitHubStore(this.client, job.repository, job.prNumber, this.trustedAuthors);
|
|
51
|
-
const ready = job.tasks.filter(task => task.state === 'QUEUED' || (task.state === 'DRAFT' && task.dependencies.length > 0)).length;
|
|
52
|
-
const expiredRunning = job.tasks.some(task => task.state
|
|
51
|
+
const ready = job.tasks.filter(task => (task.state === 'QUEUED' && task.executionId === null) || (task.state === 'DRAFT' && task.dependencies.length > 0)).length;
|
|
52
|
+
const expiredRunning = job.tasks.some(task => ['RUNNING', 'QUEUED'].includes(task.state) && task.executionId !== null && task.leaseExpiresAt !== null && new Date(task.leaseExpiresAt).getTime() <= Date.now());
|
|
53
53
|
if (!needsLifecycleReconciliation(job))
|
|
54
54
|
continue;
|
|
55
55
|
const allowance = Math.max(1, Math.min(capacity, ready || (expiredRunning ? 1 : 0) || 1));
|
package/dist/reconciler.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { storySizeTimeoutMs, taskOverallTimeoutMs, chatGptStartAckTimeoutMs } from './types.js';
|
|
3
|
-
import { dependencyReady, schedule,
|
|
3
|
+
import { dependencyReady, schedule, transitionJobEvent, transitionTaskEvent } from './scheduler.js';
|
|
4
4
|
import { implementationComplete } from './runtime-status.js';
|
|
5
5
|
import { LaunchFailure, StartAckTimeout } from './adapters.js';
|
|
6
6
|
import { eventFor } from './events.js';
|
|
@@ -10,11 +10,16 @@ import { selectReviewer } from './reviewer-routing.js';
|
|
|
10
10
|
import { selectWorker, usageForProvider, workerCandidates } from './worker-routing.js';
|
|
11
11
|
const PRIORITY_RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
12
12
|
export function compareTaskPriority(a, b, jobPriority = 'P2') { const ar = PRIORITY_RANK[a.priority ?? jobPriority]; const br = PRIORITY_RANK[b.priority ?? jobPriority]; return ar - br || a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); }
|
|
13
|
+
export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
|
|
14
|
+
function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
|
|
13
15
|
export function managedWorkerInput(job, task, context) {
|
|
14
16
|
if (!['codex', 'chatgpt'].includes(task.adapter) || !job.repository || job.prNumber <= 0)
|
|
15
17
|
return task.input;
|
|
16
18
|
if (!task.output)
|
|
17
19
|
throw new Error(`Task ${task.id} requires an explicit output contract`);
|
|
20
|
+
if (!task.executionId)
|
|
21
|
+
throw new Error(`Task ${task.id} requires an executionId before worker input is built`);
|
|
22
|
+
const sourceId = executionEventSource(job.id, task.id, task.executionId);
|
|
18
23
|
const prUrl = `https://github.com/${job.repository}/pull/${job.prNumber}`;
|
|
19
24
|
const taskUrl = task.taskUrl ?? `${prUrl}#issuecomment-${task.id}`;
|
|
20
25
|
const output = task.output.kind === 'file'
|
|
@@ -36,6 +41,8 @@ ${output}
|
|
|
36
41
|
|
|
37
42
|
[Execution event contract — mandatory]
|
|
38
43
|
Use the canonical Neo events-bus with exactly Job ${job.id} and Task ${task.id}.
|
|
44
|
+
Your exact event source identity is: ${sourceId}
|
|
45
|
+
Every lifecycle event for this execution MUST set source.id to that exact string. Do not derive, rename, or replace it.
|
|
39
46
|
Follow the events-bus skill/protocol. For sandboxed Codex or hosted ChatGPT,
|
|
40
47
|
publish through the federated MCP tool events__publish; do not open NATS directly.
|
|
41
48
|
For a direct non-sandbox local worker, use NEO_EVENTS_EMIT when the caller provides it.
|
|
@@ -125,16 +132,23 @@ export class Reconciler {
|
|
|
125
132
|
for (const task of job.tasks.filter(item => !['SUCCEEDED', 'FAILED', 'CANCELLED'].includes(item.state))) {
|
|
126
133
|
this.live.get(task.id)?.execution.cancel();
|
|
127
134
|
this.plannerLive.get(task.id)?.controller.abort();
|
|
128
|
-
|
|
135
|
+
await transitionTaskEvent(task, 'cancel');
|
|
129
136
|
task.error = reason;
|
|
130
137
|
task.leaseOwner = null;
|
|
131
138
|
task.leaseExpiresAt = null;
|
|
139
|
+
task.executionId = null;
|
|
132
140
|
await this.store.saveTask(task);
|
|
133
141
|
}
|
|
134
142
|
const terminalState = prState === 'MERGED' ? 'COMPLETED' : 'CANCELLED';
|
|
135
143
|
if (job.state !== terminalState) {
|
|
136
|
-
job.state
|
|
137
|
-
|
|
144
|
+
if (!['DRAFT', 'ACTIVE', 'COMPLETED', 'CANCELLED'].includes(job.state)) {
|
|
145
|
+
// Compatibility repair for legacy records that persisted derived runtime status as lifecycle.
|
|
146
|
+
job.state = terminalState;
|
|
147
|
+
job.updatedAt = new Date().toISOString();
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
await transitionJobEvent(job, prState === 'MERGED' ? 'pr.merged' : 'pr.closed', prState === 'CLOSED' ? { notMerged: true } : {});
|
|
151
|
+
}
|
|
138
152
|
await this.store.saveJob(job);
|
|
139
153
|
}
|
|
140
154
|
return this.store.load(jobId);
|
|
@@ -167,11 +181,11 @@ export class Reconciler {
|
|
|
167
181
|
for (const task of scheduled.tasks.filter(item => item.state === 'SUCCEEDED' && (item.continuation || scheduled.continuation) && !item.continuationDeliveredAt))
|
|
168
182
|
this.detach(this.deliverContinuation(scheduled, task), scheduled.id, task.id, 'continuation');
|
|
169
183
|
const capacity = Math.max(0, (this.options.maxConcurrent ?? 4) - this.live.size - this.plannerLive.size);
|
|
170
|
-
for (const task of scheduled.tasks.filter(t => t.state === 'QUEUED' && dependencyReady(t, new Map(scheduled.tasks.map(item => [item.id, item])))).sort((a, b) => compareTaskPriority(a, b, scheduled.priority ?? 'P2')).slice(0, capacity)) {
|
|
184
|
+
for (const task of scheduled.tasks.filter(t => t.state === 'QUEUED' && t.executionId === null && dependencyReady(t, new Map(scheduled.tasks.map(item => [item.id, item])))).sort((a, b) => compareTaskPriority(a, b, scheduled.priority ?? 'P2')).slice(0, capacity)) {
|
|
171
185
|
if (task.kind === 'planner')
|
|
172
186
|
await this.launchPlanner(scheduled, task);
|
|
173
187
|
else if (task.adapter === 'orchestrator') {
|
|
174
|
-
|
|
188
|
+
await transitionTaskEvent(task, 'task.started');
|
|
175
189
|
task.leaseOwner = null;
|
|
176
190
|
task.leaseExpiresAt = null;
|
|
177
191
|
task.executionId = null;
|
|
@@ -204,7 +218,7 @@ export class Reconciler {
|
|
|
204
218
|
const staleTask = job.review.taskId ? job.tasks.find(task => task.id === job.review?.taskId) : undefined;
|
|
205
219
|
if (staleTask && ['DRAFT', 'QUEUED', 'RUNNING'].includes(staleTask.state)) {
|
|
206
220
|
this.live.get(staleTask.id)?.execution.cancel();
|
|
207
|
-
|
|
221
|
+
await transitionTaskEvent(staleTask, 'cancel');
|
|
208
222
|
staleTask.error = 'Review evidence became stale after the PR head changed';
|
|
209
223
|
staleTask.leaseOwner = null;
|
|
210
224
|
staleTask.leaseExpiresAt = null;
|
|
@@ -229,7 +243,7 @@ export class Reconciler {
|
|
|
229
243
|
const now = new Date().toISOString();
|
|
230
244
|
const implementationTasks = job.tasks.filter(item => item.agentName !== 'reviewer' && !this.isSupersededTask(item));
|
|
231
245
|
const reviewerRouting = await this.selectReviewer(implementationTasks);
|
|
232
|
-
const task = { jobId: job.id, id, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'reviewer', parentTaskId: null, dependencies: implementationTasks.map(item => item.id), capabilities: ['review'], workerSelection: reviewerRouting.selection, adapter: reviewerRouting.candidate.adapter, input: `Review PR #${job.prNumber} at head SHA ${headSha}. Objective: ${job.objective ?? job.description ?? job.title}. First publish task.started
|
|
246
|
+
const task = { jobId: job.id, id, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'reviewer', parentTaskId: null, dependencies: implementationTasks.map(item => item.id), capabilities: ['review'], workerSelection: reviewerRouting.selection, adapter: reviewerRouting.candidate.adapter, input: `Review PR #${job.prNumber} at head SHA ${headSha}. Objective: ${job.objective ?? job.description ?? job.title}. First publish task.started using the caller-provided execution source identity, then verify the deterministic pre-review gates have passed and inspect the completed work. Emit review.started, then exactly one of review.approved, review.changes_required, or review.blocked with data.review.headSha set to ${headSha}. For review.changes_required, include every actionable finding in data.review.issues as an array of concise issue strings; Relay will create one durable fix task containing all findings. Do not modify code. When auto-merge is enabled, Relay will merge this exact reviewed head after review.approved.`, output: { kind: 'task_pr' }, routing: reviewerRouting.candidate.routing, continuation: null, continuationDeliveredAt: null, reviewHeadSha: headSha, state: 'QUEUED', attempt: 0, maxAttempts: this.options.reviewer?.maxAttempts ?? 2, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: this.options.reviewer?.timeoutMs ?? storySizeTimeoutMs('M'), createdAt: now, updatedAt: now };
|
|
233
247
|
await this.store.appendTask(task);
|
|
234
248
|
job.review = { outcome: 'pending', headSha, taskId: id, evidence: null, updatedAt: now };
|
|
235
249
|
job.updatedAt = now;
|
|
@@ -303,15 +317,6 @@ export class Reconciler {
|
|
|
303
317
|
return;
|
|
304
318
|
const evidence = { summary: event.message || `Review ${outcome}`, data: { ...(event.data ?? {}), review: reviewData ?? { headSha }, headSha, outcome } };
|
|
305
319
|
job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: event.timestamp || new Date().toISOString() };
|
|
306
|
-
if (outcome !== 'started') {
|
|
307
|
-
task.state = 'SUCCEEDED';
|
|
308
|
-
task.result = evidence;
|
|
309
|
-
task.error = null;
|
|
310
|
-
task.leaseOwner = null;
|
|
311
|
-
task.leaseExpiresAt = null;
|
|
312
|
-
task.updatedAt = job.review.updatedAt;
|
|
313
|
-
await this.store.saveTask(task);
|
|
314
|
-
}
|
|
315
320
|
if (outcome === 'changes_required') {
|
|
316
321
|
const rawIssues = Array.isArray(reviewData?.issues) ? reviewData.issues : [];
|
|
317
322
|
const issues = rawIssues.map(item => {
|
|
@@ -335,7 +340,7 @@ export class Reconciler {
|
|
|
335
340
|
agentName: 'review-fix', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'],
|
|
336
341
|
adapter: 'codex', input, output: { kind: 'task_pr' },
|
|
337
342
|
routing: { provider: 'openai', model: 'gpt-5.6-luna', reasoning: 'medium', decidedBy: 'review-fix', decidedAt: now },
|
|
338
|
-
continuation: null, continuationDeliveredAt: null, state: '
|
|
343
|
+
continuation: null, continuationDeliveredAt: null, state: 'DRAFT', attempt: 0, maxAttempts: 3,
|
|
339
344
|
leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null,
|
|
340
345
|
error: null, timeoutMs: storySizeTimeoutMs('M'), createdAt: now, updatedAt: now
|
|
341
346
|
};
|
|
@@ -353,7 +358,7 @@ export class Reconciler {
|
|
|
353
358
|
}
|
|
354
359
|
}
|
|
355
360
|
async recoverLostExecutions(job, now) {
|
|
356
|
-
for (const task of job.tasks.filter(item => item.state === 'RUNNING')) {
|
|
361
|
+
for (const task of job.tasks.filter(item => item.state === 'RUNNING' || (item.state === 'QUEUED' && item.executionId !== null))) {
|
|
357
362
|
const expired = task.leaseExpiresAt !== null && new Date(task.leaseExpiresAt).getTime() <= now;
|
|
358
363
|
const ownerLost = task.leaseOwner !== null && task.leaseOwner !== this.options.owner;
|
|
359
364
|
if (!expired && !ownerLost)
|
|
@@ -365,17 +370,18 @@ export class Reconciler {
|
|
|
365
370
|
planner.controller.abort();
|
|
366
371
|
this.plannerLive.delete(task.id);
|
|
367
372
|
}
|
|
373
|
+
const failedEvent = task.state === 'RUNNING' ? 'task.failed' : 'fail';
|
|
374
|
+
await transitionTaskEvent(task, failedEvent);
|
|
375
|
+
task.error = expired ? 'Lease expired' : 'Execution owner restarted';
|
|
376
|
+
task.leaseOwner = null;
|
|
377
|
+
task.leaseExpiresAt = null;
|
|
378
|
+
task.executionId = null;
|
|
368
379
|
if (task.attempt < task.maxAttempts) {
|
|
369
|
-
|
|
370
|
-
task.leaseOwner = null;
|
|
371
|
-
task.leaseExpiresAt = null;
|
|
380
|
+
await transitionTaskEvent(task, 'retry');
|
|
372
381
|
await this.store.saveTask(task);
|
|
373
382
|
}
|
|
374
383
|
else {
|
|
375
|
-
task.error
|
|
376
|
-
task.leaseOwner = null;
|
|
377
|
-
task.leaseExpiresAt = null;
|
|
378
|
-
transitionTask(task, 'FAILED');
|
|
384
|
+
task.error += ' after maximum attempts';
|
|
379
385
|
await this.store.saveTask(task);
|
|
380
386
|
await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.failed', 'failed', task.error, 'user', { reason: expired ? 'lease_expired' : 'execution_owner_lost' }));
|
|
381
387
|
}
|
|
@@ -445,26 +451,24 @@ export class Reconciler {
|
|
|
445
451
|
async launch(job, task) {
|
|
446
452
|
const adapter = this.options.adapters.find(x => x.name === task.adapter);
|
|
447
453
|
if (!adapter) {
|
|
448
|
-
|
|
454
|
+
await transitionTaskEvent(task, 'fail');
|
|
449
455
|
task.error = `No adapter ${task.adapter}`;
|
|
450
456
|
await this.store.saveTask(task);
|
|
451
457
|
return;
|
|
452
458
|
}
|
|
453
459
|
if (['codex', 'chatgpt'].includes(task.adapter) && !this.options.eventBus) {
|
|
454
|
-
|
|
460
|
+
await transitionTaskEvent(task, 'block');
|
|
455
461
|
task.error = 'Model-backed tasks require an event bus for authoritative lifecycle state';
|
|
456
462
|
await this.store.saveTask(task);
|
|
457
463
|
return;
|
|
458
464
|
}
|
|
459
|
-
let workerInput;
|
|
460
465
|
let executionContext;
|
|
461
466
|
try {
|
|
462
467
|
const role = task.agentName === 'reviewer' ? 'reviewer' : task.parentTaskId ? 'nested-worker' : 'worker';
|
|
463
468
|
executionContext = await this.resolvedContext(job, task, role);
|
|
464
|
-
workerInput = managedWorkerInput(job, task, executionContext);
|
|
465
469
|
}
|
|
466
470
|
catch (error) {
|
|
467
|
-
|
|
471
|
+
await transitionTaskEvent(task, 'block');
|
|
468
472
|
task.error = error instanceof Error ? error.message : String(error);
|
|
469
473
|
await this.store.saveTask(task);
|
|
470
474
|
return;
|
|
@@ -481,7 +485,8 @@ export class Reconciler {
|
|
|
481
485
|
return;
|
|
482
486
|
const leaseMs = Math.max(this.options.leaseMs ?? 300000, freshTask.timeoutMs);
|
|
483
487
|
const startedAt = (this.options.now ?? new Date()).getTime();
|
|
484
|
-
|
|
488
|
+
if (freshTask.executionId !== null)
|
|
489
|
+
return;
|
|
485
490
|
freshTask.attempt += 1;
|
|
486
491
|
freshTask.leaseOwner = this.options.owner;
|
|
487
492
|
freshTask.leaseExpiresAt = new Date(startedAt + leaseMs).toISOString();
|
|
@@ -499,8 +504,10 @@ export class Reconciler {
|
|
|
499
504
|
const durableExecutionId = task.executionId;
|
|
500
505
|
const controller = new AbortController();
|
|
501
506
|
let execution;
|
|
507
|
+
const sourceId = executionEventSource(job.id, task.id, durableExecutionId);
|
|
508
|
+
const workerInput = managedWorkerInput(job, task, executionContext);
|
|
502
509
|
const workerTask = { ...task, input: workerInput, routing: task.routing ? { ...task.routing, cwd: task.routing.cwd ?? executionContext.project.path } : task.routing };
|
|
503
|
-
const startAckSetup = task.adapter === 'chatgpt' ? await this.startAck(job.id, task.id, Date.now(), this.options.startAckTimeoutMs ?? chatGptStartAckTimeoutMs(task.timeoutMs)) : null;
|
|
510
|
+
const startAckSetup = task.adapter === 'chatgpt' ? await this.startAck(job.id, task.id, sourceId, Date.now(), this.options.startAckTimeoutMs ?? chatGptStartAckTimeoutMs(task.timeoutMs)) : null;
|
|
504
511
|
const closePolicy = task.chatgptTabClosePolicy ?? this.options.chatgptTabClosePolicy ?? adapter.tabClosePolicy ?? 'after-start';
|
|
505
512
|
const terminalWait = task.adapter === 'chatgpt' && closePolicy === 'after-terminal'
|
|
506
513
|
? this.terminalWait(task.id, durableExecutionId, startedAt - 1)
|
|
@@ -533,7 +540,7 @@ export class Reconciler {
|
|
|
533
540
|
? this.failLaunch(job.id, task.id, durableExecutionId, error.message, 'delivery')
|
|
534
541
|
: this.runtimeSettled(job.id, task.id, durableExecutionId, null, error instanceof Error ? error.message : String(error))), job.id, task.id, 'worker runtime completion');
|
|
535
542
|
}
|
|
536
|
-
async startAck(jobId, taskId, launchBarrier, timeoutMs = this.options.startAckTimeoutMs ?? 60000) {
|
|
543
|
+
async startAck(jobId, taskId, sourceId, launchBarrier, timeoutMs = this.options.startAckTimeoutMs ?? 60000) {
|
|
537
544
|
if (!this.options.eventBus)
|
|
538
545
|
throw new Error('Worker start acknowledgement requires an event bus');
|
|
539
546
|
let resolveAck;
|
|
@@ -544,10 +551,7 @@ export class Reconciler {
|
|
|
544
551
|
const unsubscribe = await this.options.eventBus.subscribe(jobId, async (event) => {
|
|
545
552
|
if (settled || event.type !== 'task.started' || event.job_id !== jobId || event.task_id !== taskId)
|
|
546
553
|
return;
|
|
547
|
-
|
|
548
|
-
const sourceComponent = typeof event.source?.component === 'string' ? event.source.component : '';
|
|
549
|
-
const workerOwned = event.data?.worker_owned === true || ['codex', 'chatgpt'].includes(sourceAgent);
|
|
550
|
-
if (!workerOwned || sourceAgent === 'relay' || sourceComponent === 'agents-relay')
|
|
554
|
+
if (eventSourceId(event) !== sourceId)
|
|
551
555
|
return;
|
|
552
556
|
if (!event.timestamp || new Date(event.timestamp).getTime() < launchBarrier)
|
|
553
557
|
return;
|
|
@@ -574,13 +578,13 @@ export class Reconciler {
|
|
|
574
578
|
if (!planner) {
|
|
575
579
|
task.attempt = task.maxAttempts;
|
|
576
580
|
task.error = 'No objective planner configured';
|
|
577
|
-
|
|
581
|
+
await transitionTaskEvent(task, 'fail');
|
|
578
582
|
await this.store.saveTask(task);
|
|
579
583
|
await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'planner.failed', 'failed', `Planner ${task.id}: ${task.error}`, 'user'));
|
|
580
584
|
return;
|
|
581
585
|
}
|
|
582
586
|
const startedAt = (this.options.now ?? new Date()).getTime();
|
|
583
|
-
|
|
587
|
+
await transitionTaskEvent(task, 'task.started');
|
|
584
588
|
task.attempt += 1;
|
|
585
589
|
task.leaseOwner = this.options.owner;
|
|
586
590
|
task.leaseExpiresAt = new Date(startedAt + Math.max(this.options.leaseMs ?? 300000, task.timeoutMs)).toISOString();
|
|
@@ -618,7 +622,10 @@ export class Reconciler {
|
|
|
618
622
|
task.error = error.message;
|
|
619
623
|
task.leaseOwner = null;
|
|
620
624
|
task.leaseExpiresAt = null;
|
|
621
|
-
|
|
625
|
+
task.executionId = null;
|
|
626
|
+
await transitionTaskEvent(task, 'task.failed');
|
|
627
|
+
if (task.attempt < task.maxAttempts)
|
|
628
|
+
await transitionTaskEvent(task, 'retry');
|
|
622
629
|
await this.store.saveTask(task);
|
|
623
630
|
await this.emit(eventFor(job.id, task.id, task.parentTaskId, task.state === 'FAILED' ? 'planner.failed' : 'planner.retry', task.state === 'FAILED' ? 'failed' : 'queued', `Planner ${task.id}: ${error.message}`, 'user'));
|
|
624
631
|
this.detach(this.reconcile(jobId), jobId, task.id, 'planner retry reconciliation');
|
|
@@ -638,7 +645,8 @@ export class Reconciler {
|
|
|
638
645
|
task.error = null;
|
|
639
646
|
task.leaseOwner = null;
|
|
640
647
|
task.leaseExpiresAt = null;
|
|
641
|
-
|
|
648
|
+
task.executionId = null;
|
|
649
|
+
await transitionTaskEvent(task, 'task.completed');
|
|
642
650
|
await this.store.saveTask(task);
|
|
643
651
|
for (const child of children)
|
|
644
652
|
if (!job.tasks.some(item => item.id === child.id))
|
|
@@ -656,9 +664,12 @@ export class Reconciler {
|
|
|
656
664
|
task.error = message;
|
|
657
665
|
task.leaseOwner = null;
|
|
658
666
|
task.leaseExpiresAt = null;
|
|
659
|
-
|
|
667
|
+
task.executionId = null;
|
|
668
|
+
await transitionTaskEvent(task, 'task.failed');
|
|
669
|
+
if (task.attempt < task.maxAttempts)
|
|
670
|
+
await transitionTaskEvent(task, 'retry');
|
|
660
671
|
await this.store.saveTask(task);
|
|
661
|
-
await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'planner.failed', 'failed', `Planner ${task.id}: ${message}`, 'user'));
|
|
672
|
+
await this.emit(eventFor(job.id, task.id, task.parentTaskId, task.state === 'FAILED' ? 'planner.failed' : 'planner.retry', task.state === 'FAILED' ? 'failed' : 'queued', `Planner ${task.id}: ${message}`, 'user'));
|
|
662
673
|
}
|
|
663
674
|
}
|
|
664
675
|
this.detach(this.reconcile(jobId), jobId, task.id, 'planner follow-up reconciliation');
|
|
@@ -681,24 +692,30 @@ export class Reconciler {
|
|
|
681
692
|
this.terminalExitGraceTimers.delete(taskId);
|
|
682
693
|
}
|
|
683
694
|
async markWorkerStarted(event) {
|
|
684
|
-
const sourceAgent = typeof event.source?.agent === 'string' ? event.source.agent : '';
|
|
685
|
-
const sourceComponent = typeof event.source?.component === 'string' ? event.source.component : '';
|
|
686
|
-
if (sourceAgent === 'relay' || sourceComponent === 'agents-relay')
|
|
687
|
-
return;
|
|
688
695
|
const live = this.live.get(event.task_id);
|
|
689
|
-
if (live && event.job_id === live.jobId)
|
|
690
|
-
if (event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
|
|
691
|
-
return;
|
|
692
|
-
live.workerStarted = true;
|
|
696
|
+
if (live && event.job_id === live.jobId && event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
|
|
693
697
|
return;
|
|
694
|
-
}
|
|
695
698
|
const job = await this.store.load(event.job_id);
|
|
696
699
|
const task = job.tasks.find(item => item.id === event.task_id);
|
|
697
|
-
if (!task || task.
|
|
700
|
+
if (!task || !task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
|
|
701
|
+
return;
|
|
702
|
+
const claimedExecution = task.state === 'QUEUED' && task.executionId !== null && task.leaseOwner !== null;
|
|
703
|
+
const exactLiveExecution = live && event.job_id === live.jobId && task.executionId === live.executionId;
|
|
704
|
+
if ((exactLiveExecution || claimedExecution) && task.state === 'QUEUED') {
|
|
705
|
+
await transitionTaskEvent(task, 'task.started');
|
|
706
|
+
task.failureReason = undefined;
|
|
707
|
+
task.error = null;
|
|
708
|
+
task.updatedAt = event.timestamp || new Date().toISOString();
|
|
709
|
+
await this.store.saveTask(task);
|
|
710
|
+
if (live)
|
|
711
|
+
live.workerStarted = true;
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
if (task.state !== 'FAILED' || !this.isRecoverableWatchdogFailure(task))
|
|
698
715
|
return;
|
|
699
716
|
if (event.timestamp && new Date(event.timestamp).getTime() < new Date(task.updatedAt).getTime())
|
|
700
717
|
return;
|
|
701
|
-
|
|
718
|
+
await transitionTaskEvent(task, 'reconcile.started');
|
|
702
719
|
task.failureReason = undefined;
|
|
703
720
|
task.error = null;
|
|
704
721
|
task.updatedAt = event.timestamp || new Date().toISOString();
|
|
@@ -714,10 +731,8 @@ export class Reconciler {
|
|
|
714
731
|
|| error.includes('exited before publishing task.started')
|
|
715
732
|
|| error.includes('exited after task.started without publishing a terminal task event');
|
|
716
733
|
}
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
const sourceComponent = typeof event.source?.component === 'string' ? event.source.component : '';
|
|
720
|
-
return sourceAgent !== 'relay' && sourceComponent !== 'agents-relay';
|
|
734
|
+
isCorrelatedWorkerEvent(task, event) {
|
|
735
|
+
return task.executionId !== null && eventSourceId(event) === executionEventSource(event.job_id, task.id, task.executionId);
|
|
721
736
|
}
|
|
722
737
|
async failLaunch(jobId, taskId, executionId, message, phase = 'launch') {
|
|
723
738
|
const claim = executionId ?? `${jobId}:${taskId}`;
|
|
@@ -728,23 +743,35 @@ export class Reconciler {
|
|
|
728
743
|
cleanup.unref();
|
|
729
744
|
const job = await this.store.load(jobId);
|
|
730
745
|
const task = job.tasks.find(item => item.id === taskId);
|
|
731
|
-
if (!task || task.executionId !== executionId ||
|
|
746
|
+
if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state))
|
|
732
747
|
return;
|
|
733
748
|
const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Worker ${phase} failed: ${message}`, 'user', { phase, reason: phase === 'start-ack' ? 'worker_start_timeout' : undefined }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
734
|
-
|
|
749
|
+
if (task.state === 'QUEUED') {
|
|
750
|
+
this.clearExecution(task.id, executionId);
|
|
751
|
+
task.error = event.message;
|
|
752
|
+
task.failureReason = phase === 'start-ack' ? 'worker_start_timeout' : undefined;
|
|
753
|
+
task.leaseOwner = null;
|
|
754
|
+
task.leaseExpiresAt = null;
|
|
755
|
+
await transitionTaskEvent(task, 'fail');
|
|
756
|
+
await this.store.saveTask(task);
|
|
757
|
+
}
|
|
758
|
+
else {
|
|
759
|
+
await this.applyTerminalEvent(event, true);
|
|
760
|
+
}
|
|
735
761
|
await this.emit(event);
|
|
736
762
|
}
|
|
737
763
|
async runtimeSettled(jobId, taskId, executionId, result, error) {
|
|
738
764
|
const live = this.live.get(taskId);
|
|
739
|
-
const
|
|
765
|
+
const liveWorkerStarted = live?.executionId === executionId && live.workerStarted;
|
|
740
766
|
if (live?.executionId === executionId)
|
|
741
767
|
this.live.delete(taskId);
|
|
742
768
|
const job = await this.store.load(jobId);
|
|
743
769
|
const task = job.tasks.find(item => item.id === taskId);
|
|
770
|
+
const workerStarted = liveWorkerStarted || task?.state === 'RUNNING';
|
|
744
771
|
if (!task || task.executionId !== executionId)
|
|
745
772
|
return;
|
|
746
773
|
await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', error ? 'failed' : 'succeeded', error ?? result?.summary ?? 'Worker runtime exited', 'orchestrator', { authoritative: false, waitingForTerminalEvent: true, processError: error }, { agent: 'relay', component: 'agents-relay' }));
|
|
747
|
-
if (
|
|
774
|
+
if (['QUEUED', 'RUNNING'].includes(task.state) && task.adapter === 'codex')
|
|
748
775
|
this.scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted === true);
|
|
749
776
|
}
|
|
750
777
|
scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted) {
|
|
@@ -758,14 +785,25 @@ export class Reconciler {
|
|
|
758
785
|
async failUnexpectedExit(jobId, taskId, executionId, workerStarted) {
|
|
759
786
|
const job = await this.store.load(jobId);
|
|
760
787
|
const task = job.tasks.find(item => item.id === taskId);
|
|
761
|
-
if (!task || task.executionId !== executionId ||
|
|
788
|
+
if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state))
|
|
762
789
|
return;
|
|
763
790
|
const reason = workerStarted ? 'worker_exit_without_terminal_event' : 'worker_exit_before_start';
|
|
764
791
|
const message = workerStarted
|
|
765
792
|
? `Worker ${taskId} exited after task.started without publishing a terminal task event`
|
|
766
793
|
: `Worker ${taskId} exited before publishing task.started`;
|
|
767
794
|
const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', message, 'user', { reason, terminalGraceMs: this.options.terminalExitGraceMs ?? 1000 }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
768
|
-
|
|
795
|
+
if (task.state === 'QUEUED') {
|
|
796
|
+
this.clearExecution(task.id, executionId);
|
|
797
|
+
task.error = message;
|
|
798
|
+
task.failureReason = reason;
|
|
799
|
+
task.leaseOwner = null;
|
|
800
|
+
task.leaseExpiresAt = null;
|
|
801
|
+
await transitionTaskEvent(task, 'fail');
|
|
802
|
+
await this.store.saveTask(task);
|
|
803
|
+
}
|
|
804
|
+
else {
|
|
805
|
+
await this.applyTerminalEvent(event, true);
|
|
806
|
+
}
|
|
769
807
|
await this.emit(event);
|
|
770
808
|
}
|
|
771
809
|
async timeoutTask(jobId, taskId, executionId) {
|
|
@@ -776,23 +814,37 @@ export class Reconciler {
|
|
|
776
814
|
}
|
|
777
815
|
const job = await this.store.load(jobId);
|
|
778
816
|
const task = job.tasks.find(item => item.id === taskId);
|
|
779
|
-
if (!task || task.executionId !== executionId ||
|
|
817
|
+
if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state)) {
|
|
780
818
|
this.clearExecution(taskId, executionId);
|
|
781
819
|
return;
|
|
782
820
|
}
|
|
783
821
|
const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: task.timeoutMs });
|
|
784
|
-
|
|
822
|
+
if (task.state === 'QUEUED') {
|
|
823
|
+
task.error = event.message;
|
|
824
|
+
task.failureReason = 'terminal_event_timeout';
|
|
825
|
+
task.leaseOwner = null;
|
|
826
|
+
task.leaseExpiresAt = null;
|
|
827
|
+
await transitionTaskEvent(task, 'fail');
|
|
828
|
+
await this.store.saveTask(task);
|
|
829
|
+
this.clearExecution(task.id, executionId);
|
|
830
|
+
}
|
|
831
|
+
else {
|
|
832
|
+
await this.applyTerminalEvent(event, true);
|
|
833
|
+
}
|
|
785
834
|
await this.emit(event);
|
|
786
835
|
}
|
|
787
|
-
async applyTerminalEvent(event) {
|
|
836
|
+
async applyTerminalEvent(event, trustedInternal = false) {
|
|
788
837
|
const job = await this.store.load(event.job_id);
|
|
789
838
|
const task = job.tasks.find(item => item.id === event.task_id);
|
|
790
839
|
if (!task || !['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
|
|
791
840
|
return;
|
|
792
|
-
|
|
841
|
+
const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
|
|
842
|
+
if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event))
|
|
843
|
+
return;
|
|
844
|
+
if (task.state === 'FAILED' && this.isRecoverableWatchdogFailure(task) && (trustedInternal || !modelBacked || this.isCorrelatedWorkerEvent(task, event))) {
|
|
793
845
|
if (event.timestamp && new Date(event.timestamp).getTime() < new Date(task.updatedAt).getTime())
|
|
794
846
|
return;
|
|
795
|
-
|
|
847
|
+
await transitionTaskEvent(task, 'reconcile.started');
|
|
796
848
|
task.failureReason = undefined;
|
|
797
849
|
task.error = null;
|
|
798
850
|
}
|
|
@@ -811,24 +863,24 @@ export class Reconciler {
|
|
|
811
863
|
task.result = { summary: event.message || `Task ${task.id} completed`, data: event.data };
|
|
812
864
|
task.error = null;
|
|
813
865
|
task.failureReason = undefined;
|
|
814
|
-
|
|
866
|
+
await transitionTaskEvent(task, 'task.completed');
|
|
815
867
|
await this.store.saveTask(task);
|
|
816
868
|
await this.deliverContinuation(job, task);
|
|
817
869
|
}
|
|
818
870
|
else if (event.type === 'task.failed') {
|
|
819
871
|
task.error = event.message || `Task ${task.id} failed`;
|
|
820
872
|
task.failureReason = typeof event.data?.reason === 'string' ? event.data.reason : undefined;
|
|
821
|
-
|
|
873
|
+
await transitionTaskEvent(task, 'task.failed');
|
|
822
874
|
await this.store.saveTask(task);
|
|
823
875
|
}
|
|
824
876
|
else if (event.type === 'task.blocked') {
|
|
825
877
|
task.error = event.message || `Task ${task.id} blocked`;
|
|
826
|
-
|
|
878
|
+
await transitionTaskEvent(task, 'task.blocked');
|
|
827
879
|
await this.store.saveTask(task);
|
|
828
880
|
}
|
|
829
881
|
else if (event.type === 'task.cancelled') {
|
|
830
882
|
task.error = event.message || `Task ${task.id} cancelled`;
|
|
831
|
-
|
|
883
|
+
await transitionTaskEvent(task, 'task.cancelled');
|
|
832
884
|
await this.store.saveTask(task);
|
|
833
885
|
}
|
|
834
886
|
this.detach(this.reconcile(job.id), job.id, task.id, 'terminal-event reconciliation');
|
package/dist/scheduler.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { decideJobLifecycle, decideTaskLifecycle } from './workflow/rollout.js';
|
|
2
2
|
export function dependencyReady(task, tasks) { return task.dependencies.every(id => tasks.get(id)?.state === 'SUCCEEDED'); }
|
|
3
3
|
export function detectCycles(tasks) { const map = new Map(tasks.map(t => [t.id, t])); const visiting = new Set(); const done = new Set(); const visit = (id) => { if (visiting.has(id))
|
|
4
4
|
throw new Error(`Dependency cycle at ${id}`); if (done.has(id))
|
|
@@ -25,12 +25,16 @@ export function schedule(job) {
|
|
|
25
25
|
const map = new Map(job.tasks.map(t => [t.id, t]));
|
|
26
26
|
for (const task of scopedTasks) {
|
|
27
27
|
const ready = dependencyReady(task, map);
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
28
|
+
const facts = {
|
|
29
|
+
dependenciesSatisfied: task.dependencies.length > 0 && ready,
|
|
30
|
+
dependenciesUnsatisfied: task.dependencies.length > 0 && !ready,
|
|
31
|
+
runnable: ready,
|
|
32
|
+
};
|
|
33
|
+
const decision = decideTaskLifecycle(task, 'reconcile', facts);
|
|
34
|
+
if (decision.reason === 'ambiguous')
|
|
35
|
+
throw new Error(`Ambiguous automatic task transition from ${task.state}`);
|
|
36
|
+
if (decision.accepted && decision.target)
|
|
37
|
+
applyTaskState(task, decision.target);
|
|
34
38
|
}
|
|
35
39
|
if (job.schedule) {
|
|
36
40
|
const planners = scopedTasks.filter(task => task.kind === 'planner');
|
|
@@ -52,8 +56,25 @@ export function extendRetryBudget(task, windowSize = 3) {
|
|
|
52
56
|
const usedOrBudgeted = Math.max(task.maxAttempts, task.attempt);
|
|
53
57
|
return Math.max(windowSize, (Math.floor(usedOrBudgeted / windowSize) + 1) * windowSize);
|
|
54
58
|
}
|
|
55
|
-
|
|
59
|
+
function applyTaskState(task, state) { task.state = state; if (['QUEUED', 'RUNNING', 'SUCCEEDED'].includes(state))
|
|
56
60
|
task.error = null; task.updatedAt = new Date().toISOString(); }
|
|
61
|
+
export async function transitionTaskEvent(task, event, context = {}) {
|
|
62
|
+
const decision = decideTaskLifecycle(task, event, context);
|
|
63
|
+
if (!decision.accepted || !decision.target)
|
|
64
|
+
throw new Error(decision.reason === 'ambiguous'
|
|
65
|
+
? `Ambiguous workflow transition from ${task.state} on ${event}`
|
|
66
|
+
: `No enabled workflow transition from ${task.state} on ${event}`);
|
|
67
|
+
applyTaskState(task, decision.target);
|
|
68
|
+
}
|
|
69
|
+
export async function transitionJobEvent(job, event, context = {}) {
|
|
70
|
+
const decision = decideJobLifecycle(job, event, context);
|
|
71
|
+
if (!decision.accepted || !decision.target)
|
|
72
|
+
throw new Error(decision.reason === 'ambiguous'
|
|
73
|
+
? `Ambiguous workflow transition from ${job.state} on ${event}`
|
|
74
|
+
: `No enabled workflow transition from ${job.state} on ${event}`);
|
|
75
|
+
job.state = decision.target;
|
|
76
|
+
job.updatedAt = new Date().toISOString();
|
|
77
|
+
}
|
|
57
78
|
export const SCHEDULE_RESOLUTION_MS = 600_000;
|
|
58
79
|
export const RUN_BUCKET_LABEL_COUNT = 144;
|
|
59
80
|
const CRON_PART = /^(?:\*|(\d+)(?:-(\d+))?)(?:\/(\d+))?$/;
|
package/dist/types.js
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
|
+
import { jobLifecycleWorkflow, taskLifecycleWorkflow } from './workflow/definitions.js';
|
|
1
2
|
export const STORY_SIZE_TIMEOUT_MS = { S: 300000, M: 900000, L: 2700000 };
|
|
2
3
|
export function storySizeTimeoutMs(size) { return STORY_SIZE_TIMEOUT_MS[size]; }
|
|
3
4
|
export function taskOverallTimeoutMs(size, reasoning) { return storySizeTimeoutMs(size) * (reasoning?.toLowerCase() === 'high' ? 2 : 1); }
|
|
4
5
|
export function chatGptStartAckTimeoutMs(overallTimeoutMs) { return Math.min(600000, Math.max(120000, Math.floor(overallTimeoutMs / 3))); }
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
export function assertJobLifecycleTransition(from, to) { if (from !== to && !jobLifecycleTransitions[from].includes(to))
|
|
8
|
-
throw new Error(`Invalid job transition ${from} -> ${to}`); }
|
|
9
|
-
export function assertTaskTransition(from, to) { if (from !== to && !taskTransitions[from].includes(to))
|
|
10
|
-
throw new Error(`Invalid task transition ${from} -> ${to}`); }
|
|
6
|
+
export function assertJobLifecycleTransition(from, to) { jobLifecycleWorkflow.assertStateTransition(from, to, 'job'); }
|
|
7
|
+
export function assertTaskTransition(from, to) { taskLifecycleWorkflow.assertStateTransition(from, to, 'task'); }
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { parseExecutableMermaid, WorkflowEngine } from './mermaid.js';
|
|
3
|
+
function load(name, guards = {}, actions = {}) {
|
|
4
|
+
const url = new URL(`../../workflow/${name}.mmd`, import.meta.url);
|
|
5
|
+
return new WorkflowEngine(parseExecutableMermaid(readFileSync(url, 'utf8')), guards, actions);
|
|
6
|
+
}
|
|
7
|
+
const booleanGuard = (name) => context => context[name] === true;
|
|
8
|
+
const lifecycleAction = () => { };
|
|
9
|
+
export const jobLifecycleWorkflow = load('job-lifecycle', { notMerged: booleanGuard('notMerged') }, { cancelJob: lifecycleAction });
|
|
10
|
+
export const taskLifecycleWorkflow = load('task-lifecycle', {
|
|
11
|
+
dependenciesSatisfied: booleanGuard('dependenciesSatisfied'),
|
|
12
|
+
dependenciesUnsatisfied: booleanGuard('dependenciesUnsatisfied'),
|
|
13
|
+
runnable: booleanGuard('runnable'),
|
|
14
|
+
}, { release: lifecycleAction, hold: lifecycleAction });
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
const STATE = String.raw `(?:[A-Za-z_][A-Za-z0-9_.-]*|\[\*\])`;
|
|
2
|
+
const TRANSITION = new RegExp(`^(${STATE})\\s*-->\\s*(${STATE})(?:\\s*:\\s*(.+))?$`);
|
|
3
|
+
const NAME = /^[A-Za-z_][A-Za-z0-9_.-]*$/;
|
|
4
|
+
function parseLabel(label, line, terminal) {
|
|
5
|
+
if (!label?.trim()) {
|
|
6
|
+
if (terminal)
|
|
7
|
+
return { event: '', guard: null, actions: [] };
|
|
8
|
+
throw new Error(`Executable Mermaid transition at line ${line} requires an event label`);
|
|
9
|
+
}
|
|
10
|
+
const pieces = label.split('/');
|
|
11
|
+
if (pieces.length > 2)
|
|
12
|
+
throw new Error(`Executable Mermaid label at line ${line} may contain at most one '/' action separator`);
|
|
13
|
+
const left = pieces[0].trim();
|
|
14
|
+
const guardMatch = left.match(/^(.*?)(?:\s+\[([A-Za-z_][A-Za-z0-9_.-]*)\])?$/);
|
|
15
|
+
if (!guardMatch)
|
|
16
|
+
throw new Error(`Invalid executable Mermaid label at line ${line}`);
|
|
17
|
+
const event = guardMatch[1].trim();
|
|
18
|
+
const guard = guardMatch[2] ?? null;
|
|
19
|
+
if (!event || !NAME.test(event))
|
|
20
|
+
throw new Error(`Invalid event '${event}' at line ${line}`);
|
|
21
|
+
const actions = pieces[1]?.split(',').map(value => value.trim()).filter(Boolean) ?? [];
|
|
22
|
+
for (const action of actions)
|
|
23
|
+
if (!NAME.test(action))
|
|
24
|
+
throw new Error(`Invalid action '${action}' at line ${line}`);
|
|
25
|
+
return { event, guard, actions };
|
|
26
|
+
}
|
|
27
|
+
export function parseExecutableMermaid(source) {
|
|
28
|
+
const lines = source.split(/\r?\n/);
|
|
29
|
+
let headerSeen = false;
|
|
30
|
+
const states = new Set();
|
|
31
|
+
const finals = new Set();
|
|
32
|
+
const transitions = [];
|
|
33
|
+
const initials = [];
|
|
34
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
35
|
+
const lineNumber = index + 1;
|
|
36
|
+
const line = lines[index].trim();
|
|
37
|
+
if (!line || line.startsWith('%%'))
|
|
38
|
+
continue;
|
|
39
|
+
if (!headerSeen) {
|
|
40
|
+
if (line !== 'stateDiagram-v2')
|
|
41
|
+
throw new Error(`Executable Mermaid must start with stateDiagram-v2 (line ${lineNumber})`);
|
|
42
|
+
headerSeen = true;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const match = line.match(TRANSITION);
|
|
46
|
+
if (!match)
|
|
47
|
+
throw new Error(`Unsupported executable Mermaid syntax at line ${lineNumber}: ${line}`);
|
|
48
|
+
const [, from, to, label] = match;
|
|
49
|
+
const fromTerminal = from === '[*]';
|
|
50
|
+
const toTerminal = to === '[*]';
|
|
51
|
+
if (fromTerminal && toTerminal)
|
|
52
|
+
throw new Error(`Invalid [*] -> [*] transition at line ${lineNumber}`);
|
|
53
|
+
if (!fromTerminal)
|
|
54
|
+
states.add(from);
|
|
55
|
+
if (!toTerminal)
|
|
56
|
+
states.add(to);
|
|
57
|
+
if (fromTerminal) {
|
|
58
|
+
if (label?.trim())
|
|
59
|
+
throw new Error(`Initial transition at line ${lineNumber} must not have an event label`);
|
|
60
|
+
initials.push(to);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (toTerminal) {
|
|
64
|
+
if (label?.trim())
|
|
65
|
+
throw new Error(`Final transition at line ${lineNumber} must not have an event label`);
|
|
66
|
+
finals.add(from);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
transitions.push({ from, to, ...parseLabel(label, lineNumber, false), line: lineNumber });
|
|
70
|
+
}
|
|
71
|
+
if (!headerSeen)
|
|
72
|
+
throw new Error('Executable Mermaid is empty');
|
|
73
|
+
if (initials.length !== 1)
|
|
74
|
+
throw new Error(`Executable Mermaid requires exactly one [*] initial transition; found ${initials.length}`);
|
|
75
|
+
if (transitions.length === 0)
|
|
76
|
+
throw new Error('Executable Mermaid requires at least one executable transition');
|
|
77
|
+
const seen = new Set();
|
|
78
|
+
const deterministicKeys = new Set();
|
|
79
|
+
for (const transition of transitions) {
|
|
80
|
+
const key = `${transition.from}\u0000${transition.event}\u0000${transition.guard ?? ''}\u0000${transition.to}`;
|
|
81
|
+
if (seen.has(key))
|
|
82
|
+
throw new Error(`Duplicate executable transition at line ${transition.line}: ${transition.from} --${transition.event}--> ${transition.to}`);
|
|
83
|
+
seen.add(key);
|
|
84
|
+
const deterministicKey = `${transition.from}\u0000${transition.event}\u0000${transition.guard ?? ''}`;
|
|
85
|
+
if (deterministicKeys.has(deterministicKey))
|
|
86
|
+
throw new Error(`Ambiguous executable transition at line ${transition.line}: duplicate source/event/guard ${transition.from} --${transition.event} [${transition.guard ?? 'unguarded'}]`);
|
|
87
|
+
deterministicKeys.add(deterministicKey);
|
|
88
|
+
}
|
|
89
|
+
return { source, states, initial: initials[0], finals, transitions };
|
|
90
|
+
}
|
|
91
|
+
export class WorkflowEngine {
|
|
92
|
+
graph;
|
|
93
|
+
guards;
|
|
94
|
+
actions;
|
|
95
|
+
constructor(graph, guards = {}, actions = {}) {
|
|
96
|
+
this.graph = graph;
|
|
97
|
+
this.guards = guards;
|
|
98
|
+
this.actions = actions;
|
|
99
|
+
this.validateRegistryBindings();
|
|
100
|
+
}
|
|
101
|
+
validateRegistryBindings() {
|
|
102
|
+
for (const transition of this.graph.transitions) {
|
|
103
|
+
if (transition.guard && !this.guards[transition.guard])
|
|
104
|
+
throw new Error(`Unknown workflow guard ${transition.guard} referenced at line ${transition.line}`);
|
|
105
|
+
for (const action of transition.actions)
|
|
106
|
+
if (!this.actions[action])
|
|
107
|
+
throw new Error(`Unknown workflow action ${action} referenced at line ${transition.line}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
transitionsFrom(state) {
|
|
111
|
+
if (!this.graph.states.has(state))
|
|
112
|
+
throw new Error(`Unknown workflow state ${state}`);
|
|
113
|
+
return this.graph.transitions.filter(transition => transition.from === state);
|
|
114
|
+
}
|
|
115
|
+
assertStateTransition(from, to, kind = 'workflow') {
|
|
116
|
+
if (from === to)
|
|
117
|
+
return;
|
|
118
|
+
if (!this.graph.states.has(from) || !this.graph.states.has(to))
|
|
119
|
+
throw new Error(`Unknown ${kind} state ${!this.graph.states.has(from) ? from : to}`);
|
|
120
|
+
if (!this.graph.transitions.some(transition => transition.from === from && transition.to === to))
|
|
121
|
+
throw new Error(`Invalid ${kind} transition ${from} -> ${to}`);
|
|
122
|
+
}
|
|
123
|
+
enabledSync(state, event, context = {}) {
|
|
124
|
+
const candidates = this.transitionsFrom(state).filter(transition => transition.event === event);
|
|
125
|
+
const enabled = [];
|
|
126
|
+
for (const transition of candidates) {
|
|
127
|
+
if (!transition.guard) {
|
|
128
|
+
enabled.push(transition);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const value = this.guards[transition.guard](context);
|
|
132
|
+
if (value instanceof Promise)
|
|
133
|
+
throw new Error(`Workflow guard ${transition.guard} is asynchronous and cannot be used in a synchronous transition`);
|
|
134
|
+
if (value)
|
|
135
|
+
enabled.push(transition);
|
|
136
|
+
}
|
|
137
|
+
return enabled;
|
|
138
|
+
}
|
|
139
|
+
transitionSync(state, event, context = {}) {
|
|
140
|
+
const enabled = this.enabledSync(state, event, context);
|
|
141
|
+
if (enabled.length === 0)
|
|
142
|
+
throw new Error(`No enabled workflow transition from ${state} on ${event}`);
|
|
143
|
+
if (enabled.length > 1)
|
|
144
|
+
throw new Error(`Ambiguous workflow transition from ${state} on ${event}`);
|
|
145
|
+
const transition = enabled[0];
|
|
146
|
+
for (const actionName of transition.actions) {
|
|
147
|
+
const value = this.actions[actionName](context, transition);
|
|
148
|
+
if (value instanceof Promise)
|
|
149
|
+
throw new Error(`Workflow action ${actionName} is asynchronous and cannot be used in a synchronous transition`);
|
|
150
|
+
}
|
|
151
|
+
return transition.to;
|
|
152
|
+
}
|
|
153
|
+
async enabled(state, event, context = {}) {
|
|
154
|
+
const candidates = this.transitionsFrom(state).filter(transition => transition.event === event);
|
|
155
|
+
const enabled = [];
|
|
156
|
+
for (const transition of candidates) {
|
|
157
|
+
if (!transition.guard) {
|
|
158
|
+
enabled.push(transition);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
const guard = this.guards[transition.guard];
|
|
162
|
+
if (await guard(context))
|
|
163
|
+
enabled.push(transition);
|
|
164
|
+
}
|
|
165
|
+
return enabled;
|
|
166
|
+
}
|
|
167
|
+
async transition(state, event, context = {}) {
|
|
168
|
+
const enabled = await this.enabled(state, event, context);
|
|
169
|
+
if (enabled.length === 0)
|
|
170
|
+
throw new Error(`No enabled workflow transition from ${state} on ${event}`);
|
|
171
|
+
if (enabled.length > 1)
|
|
172
|
+
throw new Error(`Ambiguous workflow transition from ${state} on ${event}`);
|
|
173
|
+
const transition = enabled[0];
|
|
174
|
+
for (const actionName of transition.actions) {
|
|
175
|
+
const action = this.actions[actionName];
|
|
176
|
+
await action(context, transition);
|
|
177
|
+
}
|
|
178
|
+
return transition.to;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { jobLifecycleWorkflow, taskLifecycleWorkflow } from './definitions.js';
|
|
2
|
+
let modeOverride = null;
|
|
3
|
+
let reporter = null;
|
|
4
|
+
let matches = 0;
|
|
5
|
+
let mismatches = 0;
|
|
6
|
+
export function parseWorkflowRolloutMode(value) {
|
|
7
|
+
if (value === undefined || value === '')
|
|
8
|
+
return 'legacy';
|
|
9
|
+
if (value === 'legacy' || value === 'shadow' || value === 'graph')
|
|
10
|
+
return value;
|
|
11
|
+
throw new Error(`Invalid AGENTS_RELAY_WORKFLOW_MODE '${value}'; expected legacy, shadow, or graph`);
|
|
12
|
+
}
|
|
13
|
+
export function workflowRolloutMode() {
|
|
14
|
+
return modeOverride ?? parseWorkflowRolloutMode(process.env.AGENTS_RELAY_WORKFLOW_MODE);
|
|
15
|
+
}
|
|
16
|
+
export function setWorkflowRolloutMode(mode) {
|
|
17
|
+
modeOverride = mode;
|
|
18
|
+
}
|
|
19
|
+
export function setWorkflowShadowReporter(value) {
|
|
20
|
+
reporter = value;
|
|
21
|
+
}
|
|
22
|
+
export function resetWorkflowRolloutDiagnostics() {
|
|
23
|
+
matches = 0;
|
|
24
|
+
mismatches = 0;
|
|
25
|
+
}
|
|
26
|
+
export function workflowRolloutSnapshot() {
|
|
27
|
+
return { mode: workflowRolloutMode(), matches, mismatches };
|
|
28
|
+
}
|
|
29
|
+
function graphDecision(engine, from, event, context) {
|
|
30
|
+
const enabled = engine.enabledSync(from, event, context);
|
|
31
|
+
if (enabled.length === 0)
|
|
32
|
+
return { accepted: false, target: null, guard: null, actions: [], reason: 'no_transition' };
|
|
33
|
+
if (enabled.length > 1)
|
|
34
|
+
return { accepted: false, target: null, guard: null, actions: [], reason: 'ambiguous' };
|
|
35
|
+
const transition = enabled[0];
|
|
36
|
+
return {
|
|
37
|
+
accepted: true,
|
|
38
|
+
target: transition.to,
|
|
39
|
+
guard: transition.guard,
|
|
40
|
+
actions: [...transition.actions],
|
|
41
|
+
reason: 'accepted',
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
const taskLegacy = {
|
|
45
|
+
DRAFT: {
|
|
46
|
+
release: { target: 'QUEUED' },
|
|
47
|
+
block: { target: 'BLOCKED' },
|
|
48
|
+
cancel: { target: 'CANCELLED' },
|
|
49
|
+
},
|
|
50
|
+
QUEUED: {
|
|
51
|
+
hold: { target: 'DRAFT' },
|
|
52
|
+
'task.started': { target: 'RUNNING' },
|
|
53
|
+
block: { target: 'BLOCKED' },
|
|
54
|
+
fail: { target: 'FAILED' },
|
|
55
|
+
cancel: { target: 'CANCELLED' },
|
|
56
|
+
},
|
|
57
|
+
RUNNING: {
|
|
58
|
+
'task.blocked': { target: 'BLOCKED' },
|
|
59
|
+
'task.completed': { target: 'SUCCEEDED' },
|
|
60
|
+
'task.failed': { target: 'FAILED' },
|
|
61
|
+
'task.cancelled': { target: 'CANCELLED' },
|
|
62
|
+
},
|
|
63
|
+
BLOCKED: {
|
|
64
|
+
retry: { target: 'QUEUED' },
|
|
65
|
+
cancel: { target: 'CANCELLED' },
|
|
66
|
+
},
|
|
67
|
+
SUCCEEDED: {},
|
|
68
|
+
FAILED: {
|
|
69
|
+
retry: { target: 'QUEUED' },
|
|
70
|
+
'reconcile.started': { target: 'RUNNING' },
|
|
71
|
+
'reconcile.completed': { target: 'SUCCEEDED' },
|
|
72
|
+
'reconcile.blocked': { target: 'BLOCKED' },
|
|
73
|
+
cancel: { target: 'CANCELLED' },
|
|
74
|
+
},
|
|
75
|
+
CANCELLED: {},
|
|
76
|
+
};
|
|
77
|
+
const jobLegacy = {
|
|
78
|
+
DRAFT: {
|
|
79
|
+
activate: { target: 'ACTIVE' },
|
|
80
|
+
'pr.merged': { target: 'COMPLETED' },
|
|
81
|
+
cancel: { target: 'CANCELLED' },
|
|
82
|
+
},
|
|
83
|
+
ACTIVE: {
|
|
84
|
+
'pr.merged': { target: 'COMPLETED' },
|
|
85
|
+
cancel: { target: 'CANCELLED' },
|
|
86
|
+
},
|
|
87
|
+
COMPLETED: {},
|
|
88
|
+
CANCELLED: {
|
|
89
|
+
'pr.merged': { target: 'COMPLETED' },
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
function taskLegacyDecision(from, event, context) {
|
|
93
|
+
if (event === 'reconcile') {
|
|
94
|
+
if (from === 'DRAFT' && context.dependenciesSatisfied === true)
|
|
95
|
+
return { accepted: true, target: 'QUEUED', guard: 'dependenciesSatisfied', actions: ['release'], reason: 'accepted' };
|
|
96
|
+
if (from === 'QUEUED' && context.dependenciesUnsatisfied === true)
|
|
97
|
+
return { accepted: true, target: 'DRAFT', guard: 'dependenciesUnsatisfied', actions: ['hold'], reason: 'accepted' };
|
|
98
|
+
if (from === 'QUEUED' && context.runnable === true)
|
|
99
|
+
return { accepted: true, target: 'QUEUED', guard: 'runnable', actions: ['release'], reason: 'accepted' };
|
|
100
|
+
return { accepted: false, target: null, guard: null, actions: [], reason: 'no_transition' };
|
|
101
|
+
}
|
|
102
|
+
const transition = taskLegacy[from][event];
|
|
103
|
+
if (!transition)
|
|
104
|
+
return { accepted: false, target: null, guard: null, actions: [], reason: 'no_transition' };
|
|
105
|
+
return { accepted: true, target: transition.target, guard: transition.guard ?? null, actions: transition.actions ?? [], reason: 'accepted' };
|
|
106
|
+
}
|
|
107
|
+
function jobLegacyDecision(from, event, context) {
|
|
108
|
+
if (event === 'pr.closed') {
|
|
109
|
+
if ((from === 'DRAFT' || from === 'ACTIVE') && context.notMerged === true) {
|
|
110
|
+
return { accepted: true, target: 'CANCELLED', guard: 'notMerged', actions: ['cancelJob'], reason: 'accepted' };
|
|
111
|
+
}
|
|
112
|
+
return { accepted: false, target: null, guard: null, actions: [], reason: 'no_transition' };
|
|
113
|
+
}
|
|
114
|
+
const transition = jobLegacy[from][event];
|
|
115
|
+
if (!transition)
|
|
116
|
+
return { accepted: false, target: null, guard: null, actions: [], reason: 'no_transition' };
|
|
117
|
+
return { accepted: true, target: transition.target, guard: transition.guard ?? null, actions: transition.actions ?? [], reason: 'accepted' };
|
|
118
|
+
}
|
|
119
|
+
function sameDecision(a, b) {
|
|
120
|
+
return a.accepted === b.accepted
|
|
121
|
+
&& a.target === b.target
|
|
122
|
+
&& a.guard === b.guard
|
|
123
|
+
&& a.reason === b.reason
|
|
124
|
+
&& a.actions.length === b.actions.length
|
|
125
|
+
&& a.actions.every((value, index) => value === b.actions[index]);
|
|
126
|
+
}
|
|
127
|
+
function report(kind, jobId, taskId, event, from, facts, legacy, graph) {
|
|
128
|
+
const match = sameDecision(legacy, graph);
|
|
129
|
+
if (match)
|
|
130
|
+
matches += 1;
|
|
131
|
+
else
|
|
132
|
+
mismatches += 1;
|
|
133
|
+
const diagnostic = {
|
|
134
|
+
type: match ? 'workflow.shadow.match' : 'workflow.shadow.mismatch',
|
|
135
|
+
kind,
|
|
136
|
+
jobId,
|
|
137
|
+
...(taskId ? { taskId } : {}),
|
|
138
|
+
event,
|
|
139
|
+
from,
|
|
140
|
+
facts: { ...facts },
|
|
141
|
+
legacy,
|
|
142
|
+
graph,
|
|
143
|
+
};
|
|
144
|
+
reporter?.(diagnostic);
|
|
145
|
+
if (!match && !reporter)
|
|
146
|
+
console.warn(JSON.stringify(diagnostic));
|
|
147
|
+
}
|
|
148
|
+
export function decideTaskLifecycle(task, event, context = {}) {
|
|
149
|
+
const legacy = taskLegacyDecision(task.state, event, context);
|
|
150
|
+
const mode = workflowRolloutMode();
|
|
151
|
+
if (mode === 'legacy')
|
|
152
|
+
return legacy;
|
|
153
|
+
const graph = graphDecision(taskLifecycleWorkflow, task.state, event, context);
|
|
154
|
+
if (mode === 'shadow') {
|
|
155
|
+
report('task', task.jobId ?? 'unknown', task.id, event, task.state, context, legacy, graph);
|
|
156
|
+
return legacy;
|
|
157
|
+
}
|
|
158
|
+
return graph;
|
|
159
|
+
}
|
|
160
|
+
export function decideJobLifecycle(job, event, context = {}) {
|
|
161
|
+
const legacy = jobLegacyDecision(job.state, event, context);
|
|
162
|
+
const mode = workflowRolloutMode();
|
|
163
|
+
if (mode === 'legacy')
|
|
164
|
+
return legacy;
|
|
165
|
+
const graph = graphDecision(jobLifecycleWorkflow, job.state, event, context);
|
|
166
|
+
if (mode === 'shadow') {
|
|
167
|
+
report('job', job.id, undefined, event, job.state, context, legacy, graph);
|
|
168
|
+
return legacy;
|
|
169
|
+
}
|
|
170
|
+
return graph;
|
|
171
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agents-relay",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.70",
|
|
4
4
|
"description": "Durable async agent jobs coordinated through GitHub pull requests",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"files": [
|
|
25
25
|
"dist",
|
|
26
26
|
"skills",
|
|
27
|
+
"workflow",
|
|
27
28
|
"README.md",
|
|
28
29
|
"LICENSE"
|
|
29
30
|
]
|
|
@@ -47,6 +47,8 @@ Raw `timeoutMs` is a compatibility/debug/hotfix override, not the normal task-pr
|
|
|
47
47
|
|
|
48
48
|
## Task readiness states
|
|
49
49
|
|
|
50
|
+
Canonical Job/Task lifecycle legality is defined by the directly renderable runtime sources `workflow/job-lifecycle.mmd` and `workflow/task-lifecycle.mmd`. Producers, runtime code, docs, and agents must treat those files as the transition source of truth; the prose below explains semantics and must not become a second transition table.
|
|
51
|
+
|
|
50
52
|
Task readiness is explicit and must use these semantics:
|
|
51
53
|
|
|
52
54
|
- `DRAFT`: durable Task exists but is not executable now. A DRAFT Task with dependencies is promoted to `QUEUED` when all dependencies succeed. A DRAFT Task with no dependencies remains held until an explicit release to `QUEUED`.
|
|
@@ -61,8 +63,12 @@ Do not use `WAITING` as a Task state or durable Job lifecycle state. `WAITING` i
|
|
|
61
63
|
|
|
62
64
|
Relay owns process lifecycle. Workers own semantic task lifecycle. A worker must publish canonical `task.started` as its first execution action, then exactly one of `task.completed`, `task.failed`, `task.blocked`, or `task.cancelled`. Process launch/exit events do not substitute for worker semantic events.
|
|
63
65
|
|
|
66
|
+
Relay allocates the durable `executionId` before launching model-backed work and gives that worker one exact event source identity: `job/<job-id>/task/<task-id>/execution/<execution-id>`. Every semantic lifecycle event from that execution must copy the identity verbatim in `source.id`. Relay correlates model-worker lifecycle only by this execution identity; `source.agent`, `source.component`, provider/adapter names, and `data.worker_owned` are diagnostic/legacy metadata and MUST NOT authorize or correlate lifecycle state. A retry receives a new execution identity, so events from an older execution are ignored.
|
|
67
|
+
|
|
64
68
|
## Job lifecycle and runtime status
|
|
65
69
|
|
|
70
|
+
The canonical durable lifecycle transition graphs are `workflow/job-lifecycle.mmd` and `workflow/task-lifecycle.mmd`. They use Relay's minimal executable Mermaid `stateDiagram-v2` profile and are the single source of truth for transition legality; do not maintain a parallel JSON/YAML transition table. Runtime status is not part of those lifecycle graphs.
|
|
71
|
+
|
|
66
72
|
Durable Job lifecycle is intentionally small: `DRAFT | ACTIVE | COMPLETED | CANCELLED`.
|
|
67
73
|
|
|
68
74
|
- `DRAFT`: durable objective exists, but the execution engine must not run it.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
stateDiagram-v2
|
|
2
|
+
[*] --> DRAFT
|
|
3
|
+
|
|
4
|
+
DRAFT --> ACTIVE: activate
|
|
5
|
+
DRAFT --> COMPLETED: pr.merged
|
|
6
|
+
DRAFT --> CANCELLED: cancel
|
|
7
|
+
DRAFT --> CANCELLED: pr.closed [notMerged] / cancelJob
|
|
8
|
+
|
|
9
|
+
ACTIVE --> COMPLETED: pr.merged
|
|
10
|
+
ACTIVE --> CANCELLED: cancel
|
|
11
|
+
ACTIVE --> CANCELLED: pr.closed [notMerged] / cancelJob
|
|
12
|
+
|
|
13
|
+
CANCELLED --> COMPLETED: pr.merged
|
|
14
|
+
|
|
15
|
+
COMPLETED --> [*]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
stateDiagram-v2
|
|
2
|
+
[*] --> DRAFT
|
|
3
|
+
|
|
4
|
+
DRAFT --> QUEUED: release
|
|
5
|
+
DRAFT --> QUEUED: reconcile [dependenciesSatisfied] / release
|
|
6
|
+
DRAFT --> BLOCKED: block
|
|
7
|
+
DRAFT --> CANCELLED: cancel
|
|
8
|
+
|
|
9
|
+
QUEUED --> DRAFT: hold
|
|
10
|
+
QUEUED --> DRAFT: reconcile [dependenciesUnsatisfied] / hold
|
|
11
|
+
QUEUED --> QUEUED: reconcile [runnable] / release
|
|
12
|
+
QUEUED --> RUNNING: task.started
|
|
13
|
+
QUEUED --> BLOCKED: block
|
|
14
|
+
QUEUED --> FAILED: fail
|
|
15
|
+
QUEUED --> CANCELLED: cancel
|
|
16
|
+
|
|
17
|
+
RUNNING --> BLOCKED: task.blocked
|
|
18
|
+
RUNNING --> SUCCEEDED: task.completed
|
|
19
|
+
RUNNING --> FAILED: task.failed
|
|
20
|
+
RUNNING --> CANCELLED: task.cancelled
|
|
21
|
+
|
|
22
|
+
BLOCKED --> QUEUED: retry
|
|
23
|
+
BLOCKED --> CANCELLED: cancel
|
|
24
|
+
|
|
25
|
+
FAILED --> QUEUED: retry
|
|
26
|
+
FAILED --> RUNNING: reconcile.started
|
|
27
|
+
FAILED --> SUCCEEDED: reconcile.completed
|
|
28
|
+
FAILED --> BLOCKED: reconcile.blocked
|
|
29
|
+
FAILED --> CANCELLED: cancel
|
|
30
|
+
|
|
31
|
+
SUCCEEDED --> [*]
|
|
32
|
+
CANCELLED --> [*]
|