agents-relay 1.0.102 → 1.0.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-graph.js +14 -0
- package/dist/cli-help.js +3 -1
- package/dist/cli-support.js +10 -0
- package/dist/cli.js +11 -1
- package/dist/dashboard-html.js +40 -7
- package/dist/dashboard.js +42 -0
- package/dist/decision-audit.js +58 -0
- package/dist/execution-context.js +1 -0
- package/dist/model-redecision.js +41 -0
- package/dist/model-routing.js +43 -17
- package/dist/planner.js +8 -1
- package/dist/reconciler-review.js +1 -1
- package/dist/reconciler-runtime.js +117 -127
- package/dist/reconciler.js +12 -6
- package/dist/relayd.js +3 -1
- package/dist/routing-policy.js +75 -0
- package/dist/runtime-status.js +2 -0
- package/dist/scheduler.js +10 -4
- package/dist/troubleshooter.js +1 -1
- package/dist/worker-routing.js +36 -2
- package/dist/workflow/contracts.js +4 -3
- package/dist/workflow/definitions.js +1 -0
- package/dist/workflow/durable-transition.js +45 -0
- package/dist/workflow/runtime.js +7 -0
- package/package.json +1 -1
- package/skills/agents-relay/agents/implementation.agent.md +24 -0
- package/skills/agents-relay/agents/planner.agent.md +2 -0
- package/workflow/task-lifecycle.mmd +9 -3
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
|
|
2
|
+
import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
|
|
2
3
|
import { StartAckTimeout } from './adapters.js';
|
|
3
4
|
import { eventFor } from './events.js';
|
|
4
5
|
import { WorkflowEffectExecutor } from './workflow/effects.js';
|
|
@@ -24,11 +25,13 @@ export class ReconcilerRuntimeBase {
|
|
|
24
25
|
continuationLive = new Set();
|
|
25
26
|
launchFailureClaims = new Set();
|
|
26
27
|
terminalWaiters = new Map();
|
|
28
|
+
taskTransitions;
|
|
27
29
|
workflowEffects = new WorkflowEffectExecutor();
|
|
28
30
|
reconciling = false;
|
|
29
31
|
constructor(store, options) {
|
|
30
32
|
this.store = store;
|
|
31
33
|
this.options = options;
|
|
34
|
+
this.taskTransitions = new DurableTaskTransitionRuntime(store);
|
|
32
35
|
}
|
|
33
36
|
async applyJobEffectOutcomes(job, outcomes) {
|
|
34
37
|
for (const outcome of outcomes.filter(item => item.status === 'failed')) {
|
|
@@ -158,32 +161,28 @@ export class ReconcilerRuntimeBase {
|
|
|
158
161
|
const live = this.live.get(event.task_id);
|
|
159
162
|
if (live && event.job_id === live.jobId && event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
|
|
160
163
|
return;
|
|
161
|
-
const
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
task
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
177
|
-
if (
|
|
178
|
-
return;
|
|
179
|
-
if (event.timestamp && new Date(event.timestamp).getTime() < new Date(task.updatedAt).getTime())
|
|
164
|
+
const transition = await this.taskTransitions.apply(event.job_id, event.task_id, 'task.started', {
|
|
165
|
+
eligible: (job, task) => {
|
|
166
|
+
if (!task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
|
|
167
|
+
return false;
|
|
168
|
+
const claimedExecution = task.executionId !== null && task.leaseOwner !== null;
|
|
169
|
+
const exactLiveExecution = live && event.job_id === live.jobId && task.executionId === live.executionId;
|
|
170
|
+
return Boolean(exactLiveExecution || claimedExecution || this.isRecoverableWatchdogFailure(task));
|
|
171
|
+
},
|
|
172
|
+
facts: (_job, task) => ({
|
|
173
|
+
recoverableExecution: this.isRecoverableWatchdogFailure(task)
|
|
174
|
+
&& (!event.timestamp || new Date(event.timestamp).getTime() >= new Date(task.updatedAt).getTime()),
|
|
175
|
+
}),
|
|
176
|
+
mutate: (_job, task) => {
|
|
177
|
+
task.updatedAt = event.timestamp || new Date().toISOString();
|
|
178
|
+
},
|
|
179
|
+
});
|
|
180
|
+
if (!transition?.decision.accepted)
|
|
180
181
|
return;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
await this.store.saveTask(task);
|
|
186
|
-
this.detach(this.reconcile(job.id), job.id, task.id, 'late worker-start reconciliation');
|
|
182
|
+
if (live?.executionId === transition.task.executionId)
|
|
183
|
+
live.workerStarted = true;
|
|
184
|
+
if (transition.from === 'FAILED')
|
|
185
|
+
this.detach(this.reconcile(event.job_id), event.job_id, event.task_id, 'late worker-start reconciliation');
|
|
187
186
|
}
|
|
188
187
|
isRecoverableWatchdogFailure(task) {
|
|
189
188
|
if (['worker_start_timeout', 'terminal_event_timeout', 'worker_exit_before_start', 'worker_exit_without_terminal_event', 'worker_delivery_after_start'].includes(task.failureReason ?? ''))
|
|
@@ -205,23 +204,25 @@ export class ReconcilerRuntimeBase {
|
|
|
205
204
|
this.launchFailureClaims.add(claim);
|
|
206
205
|
const cleanup = setTimeout(() => this.launchFailureClaims.delete(claim), 60000);
|
|
207
206
|
cleanup.unref();
|
|
208
|
-
const
|
|
209
|
-
const
|
|
210
|
-
|
|
207
|
+
const workflowEvent = phase === 'start-ack' ? 'worker.start_ack_failed' : 'worker.delivery_failed';
|
|
208
|
+
const transition = await this.taskTransitions.apply(jobId, taskId, workflowEvent, {
|
|
209
|
+
eligible: (_job, task) => task.executionId === executionId,
|
|
210
|
+
mutate: (_job, task) => {
|
|
211
|
+
this.clearExecution(task.id, executionId);
|
|
212
|
+
task.error = `Worker ${phase} failed: ${message}`;
|
|
213
|
+
task.failureReason = phase === 'start-ack' ? 'worker_start_timeout' : undefined;
|
|
214
|
+
task.leaseOwner = null;
|
|
215
|
+
task.leaseExpiresAt = null;
|
|
216
|
+
},
|
|
217
|
+
});
|
|
218
|
+
if (!transition)
|
|
211
219
|
return;
|
|
212
|
-
if (
|
|
213
|
-
await this.emit(eventFor(jobId, taskId, task.parentTaskId, 'task.process.delivery_failed', 'failed', `Worker ${phase} reported after
|
|
220
|
+
if (!transition.decision.accepted) {
|
|
221
|
+
await this.emit(eventFor(jobId, taskId, transition.task.parentTaskId, 'task.process.delivery_failed', 'failed', `Worker ${phase} reported after lifecycle moved to ${transition.from}: ${message}`, 'orchestrator', { phase, authoritative: false, executionId, workflowEvent }, { agent: 'relay', component: 'agents-relay' }));
|
|
214
222
|
return;
|
|
215
223
|
}
|
|
216
|
-
const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Worker ${phase} failed: ${message}`, 'user', { phase, reason:
|
|
217
|
-
this.
|
|
218
|
-
task.error = event.message;
|
|
219
|
-
task.failureReason = phase === 'start-ack' ? 'worker_start_timeout' : undefined;
|
|
220
|
-
task.leaseOwner = null;
|
|
221
|
-
task.leaseExpiresAt = null;
|
|
222
|
-
await transitionTaskEvent(task, 'fail');
|
|
223
|
-
await this.store.saveTask(task);
|
|
224
|
-
await this.emit(event);
|
|
224
|
+
const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason, workflowEvent }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
225
|
+
this.detach(this.emit(event), jobId, taskId, 'worker failure event');
|
|
225
226
|
}
|
|
226
227
|
async runtimeSettled(jobId, taskId, executionId, result, error) {
|
|
227
228
|
const live = this.live.get(taskId);
|
|
@@ -234,7 +235,7 @@ export class ReconcilerRuntimeBase {
|
|
|
234
235
|
if (!task || task.executionId !== executionId)
|
|
235
236
|
return;
|
|
236
237
|
await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', error ? 'failed' : 'succeeded', error ?? result?.summary ?? 'Worker runtime exited', 'orchestrator', { authoritative: false, waitingForTerminalEvent: true, processError: error }, { agent: 'relay', component: 'agents-relay' }));
|
|
237
|
-
if (
|
|
238
|
+
if (task.adapter === 'codex' && !terminalExecutionClaimed(jobId, taskId, executionId))
|
|
238
239
|
this.scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted === true);
|
|
239
240
|
}
|
|
240
241
|
scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted) {
|
|
@@ -248,28 +249,24 @@ export class ReconcilerRuntimeBase {
|
|
|
248
249
|
async failUnexpectedExit(jobId, taskId, executionId, workerStarted) {
|
|
249
250
|
if (terminalExecutionClaimed(jobId, taskId, executionId))
|
|
250
251
|
return;
|
|
251
|
-
const job = await this.store.load(jobId);
|
|
252
|
-
const task = job.tasks.find(item => item.id === taskId);
|
|
253
|
-
if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state) || terminalExecutionClaimed(jobId, taskId, executionId))
|
|
254
|
-
return;
|
|
255
252
|
const reason = workerStarted ? 'worker_exit_without_terminal_event' : 'worker_exit_before_start';
|
|
256
253
|
const message = workerStarted
|
|
257
254
|
? `Worker ${taskId} exited after task.started without publishing a terminal task event`
|
|
258
255
|
: `Worker ${taskId} exited before publishing task.started`;
|
|
259
|
-
const
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
}
|
|
272
|
-
|
|
256
|
+
const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
|
|
257
|
+
eligible: (_job, task) => task.executionId === executionId && !terminalExecutionClaimed(jobId, taskId, executionId),
|
|
258
|
+
mutate: (_job, task) => {
|
|
259
|
+
this.clearExecution(task.id, executionId);
|
|
260
|
+
task.error = message;
|
|
261
|
+
task.failureReason = reason;
|
|
262
|
+
task.leaseOwner = null;
|
|
263
|
+
task.leaseExpiresAt = null;
|
|
264
|
+
},
|
|
265
|
+
});
|
|
266
|
+
if (!transition?.decision.accepted)
|
|
267
|
+
return;
|
|
268
|
+
const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', message, 'user', { reason, terminalGraceMs: this.options.terminalExitGraceMs ?? 1000, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
269
|
+
this.detach(this.emit(event), jobId, taskId, 'watchdog failure event');
|
|
273
270
|
}
|
|
274
271
|
async timeoutTask(jobId, taskId, executionId) {
|
|
275
272
|
const live = this.live.get(taskId);
|
|
@@ -277,81 +274,74 @@ export class ReconcilerRuntimeBase {
|
|
|
277
274
|
live.controller.abort();
|
|
278
275
|
live.execution.cancel();
|
|
279
276
|
}
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
|
|
277
|
+
const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
|
|
278
|
+
eligible: (_job, task) => task.executionId === executionId,
|
|
279
|
+
mutate: (_job, task) => {
|
|
280
|
+
this.clearExecution(task.id, executionId);
|
|
281
|
+
task.error = `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
|
|
282
|
+
task.failureReason = 'terminal_event_timeout';
|
|
283
|
+
task.leaseOwner = null;
|
|
284
|
+
task.leaseExpiresAt = null;
|
|
285
|
+
},
|
|
286
|
+
});
|
|
287
|
+
if (!transition?.decision.accepted) {
|
|
283
288
|
this.clearExecution(taskId, executionId);
|
|
284
289
|
return;
|
|
285
290
|
}
|
|
286
|
-
const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: task.timeoutMs });
|
|
287
|
-
|
|
288
|
-
task.error = event.message;
|
|
289
|
-
task.failureReason = 'terminal_event_timeout';
|
|
290
|
-
task.leaseOwner = null;
|
|
291
|
-
task.leaseExpiresAt = null;
|
|
292
|
-
await transitionTaskEvent(task, 'fail');
|
|
293
|
-
await this.store.saveTask(task);
|
|
294
|
-
this.clearExecution(task.id, executionId);
|
|
295
|
-
}
|
|
296
|
-
else {
|
|
297
|
-
await this.applyTerminalEvent(event, true);
|
|
298
|
-
}
|
|
299
|
-
await this.emit(event);
|
|
291
|
+
const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
|
|
292
|
+
this.detach(this.emit(event), jobId, taskId, 'timeout failure event');
|
|
300
293
|
}
|
|
301
294
|
async applyTerminalEvent(event, trustedInternal = false) {
|
|
302
|
-
const
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
295
|
+
const transition = await this.taskTransitions.apply(event.job_id, event.task_id, event.type, {
|
|
296
|
+
eligible: (_job, task) => {
|
|
297
|
+
if (!['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
|
|
298
|
+
return false;
|
|
299
|
+
const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
|
|
300
|
+
if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event))
|
|
301
|
+
return false;
|
|
302
|
+
const terminalWaiter = this.terminalWaiters.get(task.id);
|
|
303
|
+
return !terminalWaiter || Boolean(event.timestamp && new Date(event.timestamp).getTime() >= terminalWaiter.launchBarrier);
|
|
304
|
+
},
|
|
305
|
+
facts: (_job, task) => {
|
|
306
|
+
const processExitedAfterPublishedTerminal = task.failureReason === 'worker_exit_without_terminal_event';
|
|
307
|
+
const freshEnough = processExitedAfterPublishedTerminal
|
|
308
|
+
|| !event.timestamp
|
|
309
|
+
|| new Date(event.timestamp).getTime() >= new Date(task.updatedAt).getTime();
|
|
310
|
+
return { recoverableExecution: this.isRecoverableWatchdogFailure(task) && freshEnough };
|
|
311
|
+
},
|
|
312
|
+
mutate: (job, task) => {
|
|
313
|
+
const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
|
|
314
|
+
const acceptedExecutionId = task.executionId;
|
|
315
|
+
if (modelBacked && acceptedExecutionId)
|
|
316
|
+
claimTerminalExecution(executionEventSource(job.id, task.id, acceptedExecutionId));
|
|
317
|
+
this.clearExecution(task.id, acceptedExecutionId);
|
|
318
|
+
task.leaseOwner = null;
|
|
319
|
+
task.leaseExpiresAt = null;
|
|
320
|
+
task.updatedAt = event.timestamp || new Date().toISOString();
|
|
321
|
+
this.terminalWaiters.get(task.id)?.resolve();
|
|
322
|
+
this.terminalWaiters.delete(task.id);
|
|
323
|
+
if (event.type === 'task.completed') {
|
|
324
|
+
task.result = { summary: event.message || `Task ${task.id} completed`, data: event.data };
|
|
325
|
+
task.error = null;
|
|
326
|
+
task.failureReason = undefined;
|
|
327
|
+
}
|
|
328
|
+
else if (event.type === 'task.failed') {
|
|
329
|
+
task.error = event.message || `Task ${task.id} failed`;
|
|
330
|
+
task.failureReason = typeof event.data?.reason === 'string' ? event.data.reason : undefined;
|
|
331
|
+
}
|
|
332
|
+
else if (event.type === 'task.blocked') {
|
|
333
|
+
task.error = event.message || `Task ${task.id} blocked`;
|
|
334
|
+
}
|
|
335
|
+
else if (event.type === 'task.cancelled') {
|
|
336
|
+
task.error = event.message || `Task ${task.id} cancelled`;
|
|
337
|
+
}
|
|
338
|
+
},
|
|
339
|
+
});
|
|
340
|
+
if (!transition?.decision.accepted)
|
|
320
341
|
return;
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
this.clearExecution(task.id, acceptedExecutionId);
|
|
325
|
-
task.leaseOwner = null;
|
|
326
|
-
task.leaseExpiresAt = null;
|
|
327
|
-
task.updatedAt = event.timestamp || new Date().toISOString();
|
|
328
|
-
this.terminalWaiters.get(task.id)?.resolve();
|
|
329
|
-
this.terminalWaiters.delete(task.id);
|
|
330
|
-
if (event.type === 'task.completed') {
|
|
331
|
-
task.result = { summary: event.message || `Task ${task.id} completed`, data: event.data };
|
|
332
|
-
task.error = null;
|
|
333
|
-
task.failureReason = undefined;
|
|
334
|
-
await transitionTaskEvent(task, 'task.completed');
|
|
335
|
-
await this.store.saveTask(task);
|
|
336
|
-
await this.deliverContinuation(job, task);
|
|
337
|
-
}
|
|
338
|
-
else if (event.type === 'task.failed') {
|
|
339
|
-
task.error = event.message || `Task ${task.id} failed`;
|
|
340
|
-
task.failureReason = typeof event.data?.reason === 'string' ? event.data.reason : undefined;
|
|
341
|
-
await transitionTaskEvent(task, 'task.failed');
|
|
342
|
-
await this.store.saveTask(task);
|
|
343
|
-
}
|
|
344
|
-
else if (event.type === 'task.blocked') {
|
|
345
|
-
task.error = event.message || `Task ${task.id} blocked`;
|
|
346
|
-
await transitionTaskEvent(task, 'task.blocked');
|
|
347
|
-
await this.store.saveTask(task);
|
|
348
|
-
}
|
|
349
|
-
else if (event.type === 'task.cancelled') {
|
|
350
|
-
task.error = event.message || `Task ${task.id} cancelled`;
|
|
351
|
-
await transitionTaskEvent(task, 'task.cancelled');
|
|
352
|
-
await this.store.saveTask(task);
|
|
353
|
-
}
|
|
354
|
-
this.detach(this.reconcile(job.id), job.id, task.id, 'terminal-event reconciliation');
|
|
342
|
+
if (event.type === 'task.completed')
|
|
343
|
+
await this.deliverContinuation(transition.job, transition.task);
|
|
344
|
+
this.detach(this.reconcile(transition.job.id), transition.job.id, transition.task.id, 'terminal-event reconciliation');
|
|
355
345
|
}
|
|
356
346
|
terminalWait(taskId, executionId, launchBarrier) {
|
|
357
347
|
let resolve;
|
package/dist/reconciler.js
CHANGED
|
@@ -6,11 +6,12 @@ import { LaunchFailure, StartAckTimeout } from './adapters.js';
|
|
|
6
6
|
import { eventFor } from './events.js';
|
|
7
7
|
import { parsePlannerResult, plannerInput } from './planner.js';
|
|
8
8
|
import { agentContractPrompt, executionContextPrompt, resolveExecutionContext } from './execution-context.js';
|
|
9
|
-
import { agentGraphPrompt } from './agent-graph.js';
|
|
9
|
+
import { agentGraphPrompt, validateAgentSelection } from './agent-graph.js';
|
|
10
10
|
import { ReconcilerReviewBase } from './reconciler-review.js';
|
|
11
11
|
import { executionEventSource } from './reconciler-runtime.js';
|
|
12
12
|
export { executionEventSource } from './reconciler-runtime.js';
|
|
13
13
|
import { CallbackWorkflowEffectHandler } from './workflow/effects.js';
|
|
14
|
+
import { applyHistoricalJobCompatibilityRepair } from './workflow/runtime.js';
|
|
14
15
|
import { pendingTroubleshooterTasks } from './troubleshooter.js';
|
|
15
16
|
import { prepareManagedExecutionWorktree } from './workspace.js';
|
|
16
17
|
const PRIORITY_RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
@@ -114,8 +115,7 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
114
115
|
if (job.state !== terminalState) {
|
|
115
116
|
if (!['DRAFT', 'ACTIVE', 'COMPLETED', 'CANCELLED'].includes(job.state)) {
|
|
116
117
|
// Compatibility repair boundary for historical records that persisted runtime status as lifecycle.
|
|
117
|
-
job
|
|
118
|
-
job.updatedAt = new Date().toISOString();
|
|
118
|
+
applyHistoricalJobCompatibilityRepair(job, terminalState);
|
|
119
119
|
await this.store.saveJob(job);
|
|
120
120
|
const outcomes = await this.workflowEffects.execute(['effect.cancelActiveTasks'], { jobId: job.id, transitionKey: `compatibility:${prState}`, data: { reason } });
|
|
121
121
|
await this.applyJobEffectOutcomes(job, outcomes);
|
|
@@ -289,8 +289,12 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
289
289
|
return this.store.load(job.id);
|
|
290
290
|
}
|
|
291
291
|
async recoverPlannerChildren(job) {
|
|
292
|
+
const contextTask = job.tasks.find(task => task.kind === 'planner') ?? job.tasks[0];
|
|
293
|
+
if (!contextTask)
|
|
294
|
+
return;
|
|
295
|
+
const executionContext = await this.resolvedContext(job, contextTask, 'planner');
|
|
292
296
|
for (const planner of job.tasks.filter(task => task.kind === 'planner' && task.state === 'SUCCEEDED' && task.plannerResult)) {
|
|
293
|
-
const children = planner.plannerResult?.next_tasks.map(spec => this.plannerChild(job, planner, spec)) ?? [];
|
|
297
|
+
const children = planner.plannerResult?.next_tasks.map(spec => this.plannerChild(job, planner, spec, executionContext)) ?? [];
|
|
294
298
|
for (const child of children) {
|
|
295
299
|
const existing = job.tasks.find(item => item.id === child.id);
|
|
296
300
|
if (existing) {
|
|
@@ -475,7 +479,8 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
475
479
|
}
|
|
476
480
|
try {
|
|
477
481
|
const result = parsePlannerResult(raw);
|
|
478
|
-
const
|
|
482
|
+
const executionContext = await this.resolvedContext(job, task, 'planner');
|
|
483
|
+
const children = result.next_tasks.map(spec => this.plannerChild(job, task, spec, executionContext));
|
|
479
484
|
for (const child of children) {
|
|
480
485
|
const existing = job.tasks.find(item => item.id === child.id);
|
|
481
486
|
if (!existing)
|
|
@@ -516,7 +521,8 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
516
521
|
}
|
|
517
522
|
this.detach(this.reconcile(jobId), jobId, task.id, 'planner follow-up reconciliation');
|
|
518
523
|
}
|
|
519
|
-
plannerChild(job, planner, spec) {
|
|
524
|
+
plannerChild(job, planner, spec, executionContext) {
|
|
525
|
+
validateAgentSelection(executionContext, spec.agentName, spec.agentGraph);
|
|
520
526
|
const now = new Date().toISOString();
|
|
521
527
|
return { jobId: job.id, id: spec.id, kind: 'work', priority: spec.priority ?? job.priority, storySize: spec.storySize ?? 'M', projectName: spec.projectName, agentName: spec.agentName, agentGraph: spec.agentGraph, parentTaskId: planner.id, dependencies: spec.dependencies ?? [], capabilities: spec.capabilities ?? [], workerRequirements: spec.workerRequirements, adapter: 'codex', input: spec.input, output: spec.output, continuation: spec.continuation ?? null, continuationDeliveredAt: null, state: (spec.dependencies?.length ?? 0) > 0 ? 'DRAFT' : 'QUEUED', attempt: 0, maxAttempts: spec.maxAttempts ?? 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: spec.timeoutMs ?? taskOverallTimeoutMs(spec.storySize ?? 'M'), createdAt: now, updatedAt: now, planningDecisionId: planner.id, planningSource: planner.planningSource ?? 'model-planner', planningRound: planner.planningRound ?? 1 };
|
|
522
528
|
}
|
package/dist/relayd.js
CHANGED
|
@@ -16,6 +16,7 @@ import { codexAndZaiUsageRegistry } from './usage.js';
|
|
|
16
16
|
import { defaultWorkspaceRoot, discoverWorkspaceRepositories, resolveManagedProjectPath } from './workspace.js';
|
|
17
17
|
import { deriveJobRuntimeStatus } from './runtime-status.js';
|
|
18
18
|
import { decisionMode, LayaCliDecisionEngine } from './decision.js';
|
|
19
|
+
import { JsonlDecisionAuditStore } from './decision-audit.js';
|
|
19
20
|
export const daemonHelp = `Agents Relay daemon runs the dashboard and worker pool.
|
|
20
21
|
|
|
21
22
|
Usage:
|
|
@@ -72,7 +73,8 @@ export async function runDaemon(argv) {
|
|
|
72
73
|
const repositoryPaths = new Map(discoveredRepositories.map(item => [item.repository, item.path]));
|
|
73
74
|
const explicitProjectPath = value(argv, '--project-path');
|
|
74
75
|
const usageRegistry = codexAndZaiUsageRegistry();
|
|
75
|
-
const
|
|
76
|
+
const decisionAuditStore = new JsonlDecisionAuditStore(process.env.AGENTS_RELAY_DECISION_AUDIT_PATH ?? join(homedir(), '.agents-relay', 'decision-audit.jsonl'));
|
|
77
|
+
const makeReconciler = (store, maxConcurrent) => new Reconciler(store, { owner: `relayd-${process.pid}`, maxConcurrent, leaseMs: Number(value(argv, '--lease-ms', '300000')), adapters: [new CodexAdapter(value(argv, '--codex', 'codex')), new ChatGptAdapter()], continuations: [new CodexThreadContinuation(value(argv, '--codex', 'codex')), new CommandContinuation(), new WebhookContinuation()], planner: runtimePlanner(argv), usageSnapshotProvider: () => usageRegistry.snapshot(), decisionEngine: new LayaCliDecisionEngine(), decisionMode: decisionMode(), decisionAuditStore, eventBus: bus, reviewer: runtimeReviewer(), projectPathForJob: job => repositoryPaths.get(job.repository) ?? resolveManagedProjectPath(job.repository, explicitProjectPath, workspaceRoot) });
|
|
76
78
|
if (repositories.length === 0)
|
|
77
79
|
throw new Error(`No Git repositories with GitHub origins found under ${normalized.workspaceRoot ?? defaultWorkspaceRoot()}`);
|
|
78
80
|
const pool = new RepositoryWorkerPool(client, repositories, trusted, concurrency, makeReconciler);
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
4
|
+
const EMPTY_POLICY = {};
|
|
5
|
+
function safePolicy(value) {
|
|
6
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7
|
+
return {};
|
|
8
|
+
const source = value;
|
|
9
|
+
const strings = (item) => Array.isArray(item) && item.every(value => typeof value === 'string') ? [...item] : undefined;
|
|
10
|
+
const result = {};
|
|
11
|
+
result.disabledProfiles = strings(source.disabledProfiles);
|
|
12
|
+
result.disabledProviders = strings(source.disabledProviders);
|
|
13
|
+
result.preferredProviders = strings(source.preferredProviders);
|
|
14
|
+
result.preferredProfiles = strings(source.preferredProfiles);
|
|
15
|
+
if (source.qualityByTaskKind && typeof source.qualityByTaskKind === 'object' && !Array.isArray(source.qualityByTaskKind)) {
|
|
16
|
+
result.qualityByTaskKind = {};
|
|
17
|
+
for (const [key, value] of Object.entries(source.qualityByTaskKind)) {
|
|
18
|
+
const parsed = strings(value);
|
|
19
|
+
if (parsed)
|
|
20
|
+
result.qualityByTaskKind[key] = parsed;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if (source.recovery && typeof source.recovery === 'object' && !Array.isArray(source.recovery)) {
|
|
24
|
+
const penalize = source.recovery.penalizePriorRoute;
|
|
25
|
+
if (typeof penalize === 'boolean')
|
|
26
|
+
result.recovery = { penalizePriorRoute: penalize };
|
|
27
|
+
}
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
30
|
+
export function policyPath(explicit) {
|
|
31
|
+
return explicit ?? process.env.AGENTS_RELAY_ROUTING_POLICY ?? join(process.cwd(), '.agents-relay-routing-policy.json');
|
|
32
|
+
}
|
|
33
|
+
export function modelRouterRulesPath(explicit) {
|
|
34
|
+
return explicit ?? process.env.AGENTS_RELAY_MODEL_ROUTER_RULES ?? join(process.cwd(), '.agents-relay-model-router-rules.md');
|
|
35
|
+
}
|
|
36
|
+
export async function loadModelRouterRules(explicit) {
|
|
37
|
+
const path = modelRouterRulesPath(explicit);
|
|
38
|
+
try {
|
|
39
|
+
const text = await readFile(path, 'utf8');
|
|
40
|
+
return { path, hash: createHash('sha256').update(text).digest('hex'), text };
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return { path, hash: 'none', text: '' };
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
export async function saveModelRouterRules(text, explicit) {
|
|
47
|
+
const path = modelRouterRulesPath(explicit);
|
|
48
|
+
await mkdir(dirname(path), { recursive: true });
|
|
49
|
+
await writeFile(path, text, 'utf8');
|
|
50
|
+
return { path, hash: createHash('sha256').update(text).digest('hex'), text };
|
|
51
|
+
}
|
|
52
|
+
export async function loadRoutingPolicy(explicit, explicitRules) {
|
|
53
|
+
const path = policyPath(explicit);
|
|
54
|
+
const rules = await loadModelRouterRules(explicitRules);
|
|
55
|
+
try {
|
|
56
|
+
const text = await readFile(path, 'utf8');
|
|
57
|
+
const policy = safePolicy(JSON.parse(text));
|
|
58
|
+
return { path, hash: createHash('sha256').update(text).digest('hex'), policy, snapshot: JSON.parse(JSON.stringify(policy)), rules };
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return { path: null, hash: 'none', policy: EMPTY_POLICY, snapshot: {}, rules };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export function policyAllows(candidate, policy) {
|
|
65
|
+
if (policy.disabledProviders?.includes(candidate.provider))
|
|
66
|
+
return false;
|
|
67
|
+
if (candidate.profile && policy.disabledProfiles?.includes(candidate.profile))
|
|
68
|
+
return false;
|
|
69
|
+
return true;
|
|
70
|
+
}
|
|
71
|
+
export function policyRank(candidate, taskKind, policy) {
|
|
72
|
+
const preferred = new Set([...(policy.preferredProviders ?? []), ...(policy.preferredProfiles ?? [])]);
|
|
73
|
+
const quality = taskKind ? new Set(policy.qualityByTaskKind?.[taskKind] ?? []) : new Set();
|
|
74
|
+
return (preferred.has(candidate.provider) || (candidate.profile && preferred.has(candidate.profile)) ? 2 : 0) + (quality.has(candidate.id) ? 1 : 0);
|
|
75
|
+
}
|
package/dist/runtime-status.js
CHANGED
|
@@ -46,6 +46,8 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
|
|
|
46
46
|
return 'FAILED';
|
|
47
47
|
if (job.tasks.some(task => task.state === 'DRAFT'))
|
|
48
48
|
return 'WAITING';
|
|
49
|
+
if (job.executionMode === 'autonomous' && job.autoMerge === false && job.reviewEnabled === false && implementationComplete(job) && prState === 'OPEN')
|
|
50
|
+
return 'WAITING';
|
|
49
51
|
if (job.reviewEnabled === false && implementationComplete(job) && prState === 'OPEN')
|
|
50
52
|
return 'MERGE_PENDING';
|
|
51
53
|
return 'IDLE';
|
package/dist/scheduler.js
CHANGED
|
@@ -35,13 +35,19 @@ export function extendRetryBudget(task, windowSize = 3) {
|
|
|
35
35
|
const usedOrBudgeted = Math.max(task.maxAttempts, task.attempt);
|
|
36
36
|
return Math.max(windowSize, (Math.floor(usedOrBudgeted / windowSize) + 1) * windowSize);
|
|
37
37
|
}
|
|
38
|
-
export
|
|
38
|
+
export function tryTransitionTaskEvent(task, event, context = {}) {
|
|
39
39
|
const decision = decideTaskLifecycle(task, event, context);
|
|
40
|
+
if (decision.accepted && decision.target)
|
|
41
|
+
applyTaskWorkflowDecision(task, decision);
|
|
42
|
+
return decision;
|
|
43
|
+
}
|
|
44
|
+
export async function transitionTaskEvent(task, event, context = {}) {
|
|
45
|
+
const from = task.state;
|
|
46
|
+
const decision = tryTransitionTaskEvent(task, event, context);
|
|
40
47
|
if (!decision.accepted || !decision.target)
|
|
41
48
|
throw new Error(decision.reason === 'ambiguous'
|
|
42
|
-
? `Ambiguous workflow transition from ${
|
|
43
|
-
: `No enabled workflow transition from ${
|
|
44
|
-
applyTaskWorkflowDecision(task, decision);
|
|
49
|
+
? `Ambiguous workflow transition from ${from} on ${event}`
|
|
50
|
+
: `No enabled workflow transition from ${from} on ${event}`);
|
|
45
51
|
return decision;
|
|
46
52
|
}
|
|
47
53
|
export async function transitionJobEvent(job, event, context = {}) {
|
package/dist/troubleshooter.js
CHANGED
|
@@ -24,7 +24,7 @@ Retrieve this task's lifecycle event history before classifying the root cause o
|
|
|
24
24
|
GET http://127.0.0.1:8787/api/events?job=${encodeURIComponent(job.id)}&task=${encodeURIComponent(task.id)}&limit=200
|
|
25
25
|
Inspect event order, type/status, source, data/phase, duplicate events, missing expected lifecycle events, late events, and execution identifiers. If the event stream points to delivery/runtime failure, inspect the corresponding Relay/worker/tool logs too. Do not treat the final task.error string as the complete root cause.
|
|
26
26
|
|
|
27
|
-
Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
|
|
27
|
+
Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Only model/provider/runtime evidence may call the narrow model re-decision action; that action gathers fresh usage, reloads current policy, updates this exact Task's routing, and does not retry it. Troubleshooter must then reactivate/retry this exact original task and verify progress. Do not manually choose a provider/model or create a replacement task. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
|
|
28
28
|
}
|
|
29
29
|
function belongsToCurrentRun(job, task) {
|
|
30
30
|
const slot = job.schedule?.currentRunSlot;
|
package/dist/worker-routing.js
CHANGED
|
@@ -1,9 +1,31 @@
|
|
|
1
1
|
import { chooseWithDecision } from './decision.js';
|
|
2
|
-
export function usageForProvider(snapshot, provider) {
|
|
2
|
+
export function usageForProvider(snapshot, provider, now = Date.now()) {
|
|
3
3
|
const usage = snapshot?.providers.find(item => item.id === provider)?.usage;
|
|
4
4
|
if (usage?.available) {
|
|
5
5
|
const usedPercent = Math.max(usage.fiveHour?.usedPercent ?? 0, usage.weekly?.usedPercent ?? 0);
|
|
6
|
-
|
|
6
|
+
const resourceWindow = (window) => {
|
|
7
|
+
if (!window)
|
|
8
|
+
return undefined;
|
|
9
|
+
const resetAt = window.resetsAt;
|
|
10
|
+
return {
|
|
11
|
+
remainingPercent: 100 - window.usedPercent,
|
|
12
|
+
resetAt,
|
|
13
|
+
resetInMinutes: resetAt === null ? null : Math.max(0, Math.ceil((resetAt * 1000 - now) / 60_000)),
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
const windows = { fiveHour: resourceWindow(usage.fiveHour), weekly: resourceWindow(usage.weekly) };
|
|
17
|
+
const resetWindow = [usage.fiveHour, usage.weekly]
|
|
18
|
+
.filter((window) => window?.resetsAt !== null && window?.resetsAt !== undefined)
|
|
19
|
+
.filter(window => window.resetsAt * 1000 > now)
|
|
20
|
+
.sort((left, right) => left.resetsAt - right.resetsAt)[0];
|
|
21
|
+
return {
|
|
22
|
+
healthy: usedPercent < 100,
|
|
23
|
+
usedPercent,
|
|
24
|
+
remainingPercent: resetWindow ? 100 - resetWindow.usedPercent : undefined,
|
|
25
|
+
resetAt: resetWindow?.resetsAt ?? undefined,
|
|
26
|
+
resetInMinutes: resetWindow?.resetsAt ? Math.max(0, Math.ceil((resetWindow.resetsAt * 1000 - now) / 60_000)) : undefined,
|
|
27
|
+
windows,
|
|
28
|
+
};
|
|
7
29
|
}
|
|
8
30
|
const credits = snapshot?.credits.find(item => item.id === provider)?.credits;
|
|
9
31
|
if (credits?.available) {
|
|
@@ -34,6 +56,10 @@ export function workerCandidates(adapters, registered, snapshot) {
|
|
|
34
56
|
resourceHealthy: resource.healthy,
|
|
35
57
|
resourceProvider: provider,
|
|
36
58
|
resourceUsedPercent: resource.usedPercent,
|
|
59
|
+
resourceRemainingPercent: resource.remainingPercent,
|
|
60
|
+
resourceResetAt: resource.resetAt,
|
|
61
|
+
resourceResetInMinutes: resource.resetInMinutes,
|
|
62
|
+
resourceWindows: resource.windows,
|
|
37
63
|
};
|
|
38
64
|
});
|
|
39
65
|
});
|
|
@@ -76,6 +102,10 @@ function selection(selected, selectedBy, decisionEvidence) {
|
|
|
76
102
|
resourceHealthy: selected.resourceHealthy,
|
|
77
103
|
resourceProvider: selected.resourceProvider,
|
|
78
104
|
resourceUsedPercent: selected.resourceUsedPercent,
|
|
105
|
+
resourceRemainingPercent: selected.resourceRemainingPercent,
|
|
106
|
+
resourceResetAt: selected.resourceResetAt,
|
|
107
|
+
resourceResetInMinutes: selected.resourceResetInMinutes,
|
|
108
|
+
resourceWindows: selected.resourceWindows,
|
|
79
109
|
decisionEvidence,
|
|
80
110
|
};
|
|
81
111
|
}
|
|
@@ -108,6 +138,10 @@ export async function routeWorker(candidates, requirements, options = {}) {
|
|
|
108
138
|
resourceHealthy: item.resourceHealthy ?? null,
|
|
109
139
|
resourceProvider: item.resourceProvider ?? null,
|
|
110
140
|
resourceUsedPercent: item.resourceUsedPercent ?? null,
|
|
141
|
+
resourceRemainingPercent: item.resourceRemainingPercent ?? null,
|
|
142
|
+
resourceResetAt: item.resourceResetAt ?? null,
|
|
143
|
+
resourceResetInMinutes: item.resourceResetInMinutes ?? null,
|
|
144
|
+
resourceWindows: item.resourceWindows ?? null,
|
|
111
145
|
})),
|
|
112
146
|
}));
|
|
113
147
|
const choice = await chooseWithDecision(options.decisionEngine, options.decisionMode ?? 'off', { facts }, { type: 'choice', question: 'Which eligible worker is best suited to execute this task?', choices: eligible.map(item => item.workerId) }, () => deterministic.workerId);
|
|
@@ -12,9 +12,6 @@ export const WORKFLOW_EVENT_NAMES = [
|
|
|
12
12
|
'pr.closed',
|
|
13
13
|
'pr.merged',
|
|
14
14
|
'reconcile',
|
|
15
|
-
'reconcile.blocked',
|
|
16
|
-
'reconcile.completed',
|
|
17
|
-
'reconcile.started',
|
|
18
15
|
'release',
|
|
19
16
|
'retry',
|
|
20
17
|
'review.approved',
|
|
@@ -23,6 +20,9 @@ export const WORKFLOW_EVENT_NAMES = [
|
|
|
23
20
|
'task.completed',
|
|
24
21
|
'task.failed',
|
|
25
22
|
'task.started',
|
|
23
|
+
'watchdog.failed',
|
|
24
|
+
'worker.delivery_failed',
|
|
25
|
+
'worker.start_ack_failed',
|
|
26
26
|
];
|
|
27
27
|
export const WORKFLOW_FACT_NAMES = [
|
|
28
28
|
'dependenciesSatisfied',
|
|
@@ -31,6 +31,7 @@ export const WORKFLOW_FACT_NAMES = [
|
|
|
31
31
|
'dispatchable',
|
|
32
32
|
'notMerged',
|
|
33
33
|
'autoMerge',
|
|
34
|
+
'recoverableExecution',
|
|
34
35
|
];
|
|
35
36
|
export const WORKFLOW_MUTATION_NAMES = [
|
|
36
37
|
'mutation.clearError',
|
|
@@ -33,4 +33,5 @@ export const taskLifecycleWorkflow = load('task-lifecycle', {
|
|
|
33
33
|
dependenciesUnsatisfied: booleanGuard('dependenciesUnsatisfied'),
|
|
34
34
|
runnable: booleanGuard('runnable'),
|
|
35
35
|
dispatchable: booleanGuard('dispatchable'),
|
|
36
|
+
recoverableExecution: booleanGuard('recoverableExecution'),
|
|
36
37
|
});
|