agents-relay 1.0.102 → 1.0.104

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
2
+ import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
2
3
  import { StartAckTimeout } from './adapters.js';
3
4
  import { eventFor } from './events.js';
4
5
  import { WorkflowEffectExecutor } from './workflow/effects.js';
@@ -24,11 +25,13 @@ export class ReconcilerRuntimeBase {
24
25
  continuationLive = new Set();
25
26
  launchFailureClaims = new Set();
26
27
  terminalWaiters = new Map();
28
+ taskTransitions;
27
29
  workflowEffects = new WorkflowEffectExecutor();
28
30
  reconciling = false;
29
31
  constructor(store, options) {
30
32
  this.store = store;
31
33
  this.options = options;
34
+ this.taskTransitions = new DurableTaskTransitionRuntime(store);
32
35
  }
33
36
  async applyJobEffectOutcomes(job, outcomes) {
34
37
  for (const outcome of outcomes.filter(item => item.status === 'failed')) {
@@ -158,32 +161,28 @@ export class ReconcilerRuntimeBase {
158
161
  const live = this.live.get(event.task_id);
159
162
  if (live && event.job_id === live.jobId && event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
160
163
  return;
161
- const job = await this.store.load(event.job_id);
162
- const task = job.tasks.find(item => item.id === event.task_id);
163
- if (!task || !task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
164
- return;
165
- const claimedExecution = task.state === 'QUEUED' && task.executionId !== null && task.leaseOwner !== null;
166
- const exactLiveExecution = live && event.job_id === live.jobId && task.executionId === live.executionId;
167
- if ((exactLiveExecution || claimedExecution) && task.state === 'QUEUED') {
168
- await transitionTaskEvent(task, 'task.started');
169
- task.failureReason = undefined;
170
- task.error = null;
171
- task.updatedAt = event.timestamp || new Date().toISOString();
172
- await this.store.saveTask(task);
173
- if (live)
174
- live.workerStarted = true;
175
- return;
176
- }
177
- if (task.state !== 'FAILED' || !this.isRecoverableWatchdogFailure(task))
178
- return;
179
- if (event.timestamp && new Date(event.timestamp).getTime() < new Date(task.updatedAt).getTime())
164
+ const transition = await this.taskTransitions.apply(event.job_id, event.task_id, 'task.started', {
165
+ eligible: (job, task) => {
166
+ if (!task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
167
+ return false;
168
+ const claimedExecution = task.executionId !== null && task.leaseOwner !== null;
169
+ const exactLiveExecution = live && event.job_id === live.jobId && task.executionId === live.executionId;
170
+ return Boolean(exactLiveExecution || claimedExecution || this.isRecoverableWatchdogFailure(task));
171
+ },
172
+ facts: (_job, task) => ({
173
+ recoverableExecution: this.isRecoverableWatchdogFailure(task)
174
+ && (!event.timestamp || new Date(event.timestamp).getTime() >= new Date(task.updatedAt).getTime()),
175
+ }),
176
+ mutate: (_job, task) => {
177
+ task.updatedAt = event.timestamp || new Date().toISOString();
178
+ },
179
+ });
180
+ if (!transition?.decision.accepted)
180
181
  return;
181
- await transitionTaskEvent(task, 'reconcile.started');
182
- task.failureReason = undefined;
183
- task.error = null;
184
- task.updatedAt = event.timestamp || new Date().toISOString();
185
- await this.store.saveTask(task);
186
- this.detach(this.reconcile(job.id), job.id, task.id, 'late worker-start reconciliation');
182
+ if (live?.executionId === transition.task.executionId)
183
+ live.workerStarted = true;
184
+ if (transition.from === 'FAILED')
185
+ this.detach(this.reconcile(event.job_id), event.job_id, event.task_id, 'late worker-start reconciliation');
187
186
  }
188
187
  isRecoverableWatchdogFailure(task) {
189
188
  if (['worker_start_timeout', 'terminal_event_timeout', 'worker_exit_before_start', 'worker_exit_without_terminal_event', 'worker_delivery_after_start'].includes(task.failureReason ?? ''))
@@ -205,23 +204,25 @@ export class ReconcilerRuntimeBase {
205
204
  this.launchFailureClaims.add(claim);
206
205
  const cleanup = setTimeout(() => this.launchFailureClaims.delete(claim), 60000);
207
206
  cleanup.unref();
208
- const job = await this.store.load(jobId);
209
- const task = job.tasks.find(item => item.id === taskId);
210
- if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state))
207
+ const workflowEvent = phase === 'start-ack' ? 'worker.start_ack_failed' : 'worker.delivery_failed';
208
+ const transition = await this.taskTransitions.apply(jobId, taskId, workflowEvent, {
209
+ eligible: (_job, task) => task.executionId === executionId,
210
+ mutate: (_job, task) => {
211
+ this.clearExecution(task.id, executionId);
212
+ task.error = `Worker ${phase} failed: ${message}`;
213
+ task.failureReason = phase === 'start-ack' ? 'worker_start_timeout' : undefined;
214
+ task.leaseOwner = null;
215
+ task.leaseExpiresAt = null;
216
+ },
217
+ });
218
+ if (!transition)
211
219
  return;
212
- if (task.state === 'RUNNING') {
213
- await this.emit(eventFor(jobId, taskId, task.parentTaskId, 'task.process.delivery_failed', 'failed', `Worker ${phase} reported after task.started: ${message}`, 'orchestrator', { phase, authoritative: false, executionId }, { agent: 'relay', component: 'agents-relay' }));
220
+ if (!transition.decision.accepted) {
221
+ await this.emit(eventFor(jobId, taskId, transition.task.parentTaskId, 'task.process.delivery_failed', 'failed', `Worker ${phase} reported after lifecycle moved to ${transition.from}: ${message}`, 'orchestrator', { phase, authoritative: false, executionId, workflowEvent }, { agent: 'relay', component: 'agents-relay' }));
214
222
  return;
215
223
  }
216
- const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Worker ${phase} failed: ${message}`, 'user', { phase, reason: phase === 'start-ack' ? 'worker_start_timeout' : undefined }, { agent: 'relay', component: 'agents-relay', watchdog: true });
217
- this.clearExecution(task.id, executionId);
218
- task.error = event.message;
219
- task.failureReason = phase === 'start-ack' ? 'worker_start_timeout' : undefined;
220
- task.leaseOwner = null;
221
- task.leaseExpiresAt = null;
222
- await transitionTaskEvent(task, 'fail');
223
- await this.store.saveTask(task);
224
- await this.emit(event);
224
+ const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason, workflowEvent }, { agent: 'relay', component: 'agents-relay', watchdog: true });
225
+ this.detach(this.emit(event), jobId, taskId, 'worker failure event');
225
226
  }
226
227
  async runtimeSettled(jobId, taskId, executionId, result, error) {
227
228
  const live = this.live.get(taskId);
@@ -234,7 +235,7 @@ export class ReconcilerRuntimeBase {
234
235
  if (!task || task.executionId !== executionId)
235
236
  return;
236
237
  await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', error ? 'failed' : 'succeeded', error ?? result?.summary ?? 'Worker runtime exited', 'orchestrator', { authoritative: false, waitingForTerminalEvent: true, processError: error }, { agent: 'relay', component: 'agents-relay' }));
237
- if (['QUEUED', 'RUNNING'].includes(task.state) && task.adapter === 'codex' && !terminalExecutionClaimed(jobId, taskId, executionId))
238
+ if (task.adapter === 'codex' && !terminalExecutionClaimed(jobId, taskId, executionId))
238
239
  this.scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted === true);
239
240
  }
240
241
  scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted) {
@@ -248,28 +249,24 @@ export class ReconcilerRuntimeBase {
248
249
  async failUnexpectedExit(jobId, taskId, executionId, workerStarted) {
249
250
  if (terminalExecutionClaimed(jobId, taskId, executionId))
250
251
  return;
251
- const job = await this.store.load(jobId);
252
- const task = job.tasks.find(item => item.id === taskId);
253
- if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state) || terminalExecutionClaimed(jobId, taskId, executionId))
254
- return;
255
252
  const reason = workerStarted ? 'worker_exit_without_terminal_event' : 'worker_exit_before_start';
256
253
  const message = workerStarted
257
254
  ? `Worker ${taskId} exited after task.started without publishing a terminal task event`
258
255
  : `Worker ${taskId} exited before publishing task.started`;
259
- const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', message, 'user', { reason, terminalGraceMs: this.options.terminalExitGraceMs ?? 1000 }, { agent: 'relay', component: 'agents-relay', watchdog: true });
260
- if (task.state === 'QUEUED') {
261
- this.clearExecution(task.id, executionId);
262
- task.error = message;
263
- task.failureReason = reason;
264
- task.leaseOwner = null;
265
- task.leaseExpiresAt = null;
266
- await transitionTaskEvent(task, 'fail');
267
- await this.store.saveTask(task);
268
- }
269
- else {
270
- await this.applyTerminalEvent(event, true);
271
- }
272
- await this.emit(event);
256
+ const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
257
+ eligible: (_job, task) => task.executionId === executionId && !terminalExecutionClaimed(jobId, taskId, executionId),
258
+ mutate: (_job, task) => {
259
+ this.clearExecution(task.id, executionId);
260
+ task.error = message;
261
+ task.failureReason = reason;
262
+ task.leaseOwner = null;
263
+ task.leaseExpiresAt = null;
264
+ },
265
+ });
266
+ if (!transition?.decision.accepted)
267
+ return;
268
+ const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', message, 'user', { reason, terminalGraceMs: this.options.terminalExitGraceMs ?? 1000, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
269
+ this.detach(this.emit(event), jobId, taskId, 'watchdog failure event');
273
270
  }
274
271
  async timeoutTask(jobId, taskId, executionId) {
275
272
  const live = this.live.get(taskId);
@@ -277,81 +274,74 @@ export class ReconcilerRuntimeBase {
277
274
  live.controller.abort();
278
275
  live.execution.cancel();
279
276
  }
280
- const job = await this.store.load(jobId);
281
- const task = job.tasks.find(item => item.id === taskId);
282
- if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state)) {
277
+ const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
278
+ eligible: (_job, task) => task.executionId === executionId,
279
+ mutate: (_job, task) => {
280
+ this.clearExecution(task.id, executionId);
281
+ task.error = `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
282
+ task.failureReason = 'terminal_event_timeout';
283
+ task.leaseOwner = null;
284
+ task.leaseExpiresAt = null;
285
+ },
286
+ });
287
+ if (!transition?.decision.accepted) {
283
288
  this.clearExecution(taskId, executionId);
284
289
  return;
285
290
  }
286
- const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: task.timeoutMs });
287
- if (task.state === 'QUEUED') {
288
- task.error = event.message;
289
- task.failureReason = 'terminal_event_timeout';
290
- task.leaseOwner = null;
291
- task.leaseExpiresAt = null;
292
- await transitionTaskEvent(task, 'fail');
293
- await this.store.saveTask(task);
294
- this.clearExecution(task.id, executionId);
295
- }
296
- else {
297
- await this.applyTerminalEvent(event, true);
298
- }
299
- await this.emit(event);
291
+ const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
292
+ this.detach(this.emit(event), jobId, taskId, 'timeout failure event');
300
293
  }
301
294
  async applyTerminalEvent(event, trustedInternal = false) {
302
- const job = await this.store.load(event.job_id);
303
- const task = job.tasks.find(item => item.id === event.task_id);
304
- if (!task || !['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
305
- return;
306
- const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
307
- if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event))
308
- return;
309
- if (task.state === 'FAILED' && this.isRecoverableWatchdogFailure(task) && (trustedInternal || !modelBacked || this.isCorrelatedWorkerEvent(task, event))) {
310
- if (event.timestamp && new Date(event.timestamp).getTime() < new Date(task.updatedAt).getTime())
311
- return;
312
- await transitionTaskEvent(task, 'reconcile.started');
313
- task.failureReason = undefined;
314
- task.error = null;
315
- }
316
- if (task.state !== 'RUNNING')
317
- return;
318
- const terminalWaiter = this.terminalWaiters.get(task.id);
319
- if (terminalWaiter && (!event.timestamp || new Date(event.timestamp).getTime() < terminalWaiter.launchBarrier))
295
+ const transition = await this.taskTransitions.apply(event.job_id, event.task_id, event.type, {
296
+ eligible: (_job, task) => {
297
+ if (!['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
298
+ return false;
299
+ const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
300
+ if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event))
301
+ return false;
302
+ const terminalWaiter = this.terminalWaiters.get(task.id);
303
+ return !terminalWaiter || Boolean(event.timestamp && new Date(event.timestamp).getTime() >= terminalWaiter.launchBarrier);
304
+ },
305
+ facts: (_job, task) => {
306
+ const processExitedAfterPublishedTerminal = task.failureReason === 'worker_exit_without_terminal_event';
307
+ const freshEnough = processExitedAfterPublishedTerminal
308
+ || !event.timestamp
309
+ || new Date(event.timestamp).getTime() >= new Date(task.updatedAt).getTime();
310
+ return { recoverableExecution: this.isRecoverableWatchdogFailure(task) && freshEnough };
311
+ },
312
+ mutate: (job, task) => {
313
+ const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
314
+ const acceptedExecutionId = task.executionId;
315
+ if (modelBacked && acceptedExecutionId)
316
+ claimTerminalExecution(executionEventSource(job.id, task.id, acceptedExecutionId));
317
+ this.clearExecution(task.id, acceptedExecutionId);
318
+ task.leaseOwner = null;
319
+ task.leaseExpiresAt = null;
320
+ task.updatedAt = event.timestamp || new Date().toISOString();
321
+ this.terminalWaiters.get(task.id)?.resolve();
322
+ this.terminalWaiters.delete(task.id);
323
+ if (event.type === 'task.completed') {
324
+ task.result = { summary: event.message || `Task ${task.id} completed`, data: event.data };
325
+ task.error = null;
326
+ task.failureReason = undefined;
327
+ }
328
+ else if (event.type === 'task.failed') {
329
+ task.error = event.message || `Task ${task.id} failed`;
330
+ task.failureReason = typeof event.data?.reason === 'string' ? event.data.reason : undefined;
331
+ }
332
+ else if (event.type === 'task.blocked') {
333
+ task.error = event.message || `Task ${task.id} blocked`;
334
+ }
335
+ else if (event.type === 'task.cancelled') {
336
+ task.error = event.message || `Task ${task.id} cancelled`;
337
+ }
338
+ },
339
+ });
340
+ if (!transition?.decision.accepted)
320
341
  return;
321
- const acceptedExecutionId = task.executionId;
322
- if (modelBacked && acceptedExecutionId)
323
- claimTerminalExecution(executionEventSource(job.id, task.id, acceptedExecutionId));
324
- this.clearExecution(task.id, acceptedExecutionId);
325
- task.leaseOwner = null;
326
- task.leaseExpiresAt = null;
327
- task.updatedAt = event.timestamp || new Date().toISOString();
328
- this.terminalWaiters.get(task.id)?.resolve();
329
- this.terminalWaiters.delete(task.id);
330
- if (event.type === 'task.completed') {
331
- task.result = { summary: event.message || `Task ${task.id} completed`, data: event.data };
332
- task.error = null;
333
- task.failureReason = undefined;
334
- await transitionTaskEvent(task, 'task.completed');
335
- await this.store.saveTask(task);
336
- await this.deliverContinuation(job, task);
337
- }
338
- else if (event.type === 'task.failed') {
339
- task.error = event.message || `Task ${task.id} failed`;
340
- task.failureReason = typeof event.data?.reason === 'string' ? event.data.reason : undefined;
341
- await transitionTaskEvent(task, 'task.failed');
342
- await this.store.saveTask(task);
343
- }
344
- else if (event.type === 'task.blocked') {
345
- task.error = event.message || `Task ${task.id} blocked`;
346
- await transitionTaskEvent(task, 'task.blocked');
347
- await this.store.saveTask(task);
348
- }
349
- else if (event.type === 'task.cancelled') {
350
- task.error = event.message || `Task ${task.id} cancelled`;
351
- await transitionTaskEvent(task, 'task.cancelled');
352
- await this.store.saveTask(task);
353
- }
354
- this.detach(this.reconcile(job.id), job.id, task.id, 'terminal-event reconciliation');
342
+ if (event.type === 'task.completed')
343
+ await this.deliverContinuation(transition.job, transition.task);
344
+ this.detach(this.reconcile(transition.job.id), transition.job.id, transition.task.id, 'terminal-event reconciliation');
355
345
  }
356
346
  terminalWait(taskId, executionId, launchBarrier) {
357
347
  let resolve;
@@ -6,11 +6,12 @@ import { LaunchFailure, StartAckTimeout } from './adapters.js';
6
6
  import { eventFor } from './events.js';
7
7
  import { parsePlannerResult, plannerInput } from './planner.js';
8
8
  import { agentContractPrompt, executionContextPrompt, resolveExecutionContext } from './execution-context.js';
9
- import { agentGraphPrompt } from './agent-graph.js';
9
+ import { agentGraphPrompt, validateAgentSelection } from './agent-graph.js';
10
10
  import { ReconcilerReviewBase } from './reconciler-review.js';
11
11
  import { executionEventSource } from './reconciler-runtime.js';
12
12
  export { executionEventSource } from './reconciler-runtime.js';
13
13
  import { CallbackWorkflowEffectHandler } from './workflow/effects.js';
14
+ import { applyHistoricalJobCompatibilityRepair } from './workflow/runtime.js';
14
15
  import { pendingTroubleshooterTasks } from './troubleshooter.js';
15
16
  import { prepareManagedExecutionWorktree } from './workspace.js';
16
17
  const PRIORITY_RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
@@ -114,8 +115,7 @@ export class Reconciler extends ReconcilerReviewBase {
114
115
  if (job.state !== terminalState) {
115
116
  if (!['DRAFT', 'ACTIVE', 'COMPLETED', 'CANCELLED'].includes(job.state)) {
116
117
  // Compatibility repair boundary for historical records that persisted runtime status as lifecycle.
117
- job.state = terminalState;
118
- job.updatedAt = new Date().toISOString();
118
+ applyHistoricalJobCompatibilityRepair(job, terminalState);
119
119
  await this.store.saveJob(job);
120
120
  const outcomes = await this.workflowEffects.execute(['effect.cancelActiveTasks'], { jobId: job.id, transitionKey: `compatibility:${prState}`, data: { reason } });
121
121
  await this.applyJobEffectOutcomes(job, outcomes);
@@ -289,8 +289,12 @@ export class Reconciler extends ReconcilerReviewBase {
289
289
  return this.store.load(job.id);
290
290
  }
291
291
  async recoverPlannerChildren(job) {
292
+ const contextTask = job.tasks.find(task => task.kind === 'planner') ?? job.tasks[0];
293
+ if (!contextTask)
294
+ return;
295
+ const executionContext = await this.resolvedContext(job, contextTask, 'planner');
292
296
  for (const planner of job.tasks.filter(task => task.kind === 'planner' && task.state === 'SUCCEEDED' && task.plannerResult)) {
293
- const children = planner.plannerResult?.next_tasks.map(spec => this.plannerChild(job, planner, spec)) ?? [];
297
+ const children = planner.plannerResult?.next_tasks.map(spec => this.plannerChild(job, planner, spec, executionContext)) ?? [];
294
298
  for (const child of children) {
295
299
  const existing = job.tasks.find(item => item.id === child.id);
296
300
  if (existing) {
@@ -475,7 +479,8 @@ export class Reconciler extends ReconcilerReviewBase {
475
479
  }
476
480
  try {
477
481
  const result = parsePlannerResult(raw);
478
- const children = result.next_tasks.map(spec => this.plannerChild(job, task, spec));
482
+ const executionContext = await this.resolvedContext(job, task, 'planner');
483
+ const children = result.next_tasks.map(spec => this.plannerChild(job, task, spec, executionContext));
479
484
  for (const child of children) {
480
485
  const existing = job.tasks.find(item => item.id === child.id);
481
486
  if (!existing)
@@ -516,7 +521,8 @@ export class Reconciler extends ReconcilerReviewBase {
516
521
  }
517
522
  this.detach(this.reconcile(jobId), jobId, task.id, 'planner follow-up reconciliation');
518
523
  }
519
- plannerChild(job, planner, spec) {
524
+ plannerChild(job, planner, spec, executionContext) {
525
+ validateAgentSelection(executionContext, spec.agentName, spec.agentGraph);
520
526
  const now = new Date().toISOString();
521
527
  return { jobId: job.id, id: spec.id, kind: 'work', priority: spec.priority ?? job.priority, storySize: spec.storySize ?? 'M', projectName: spec.projectName, agentName: spec.agentName, agentGraph: spec.agentGraph, parentTaskId: planner.id, dependencies: spec.dependencies ?? [], capabilities: spec.capabilities ?? [], workerRequirements: spec.workerRequirements, adapter: 'codex', input: spec.input, output: spec.output, continuation: spec.continuation ?? null, continuationDeliveredAt: null, state: (spec.dependencies?.length ?? 0) > 0 ? 'DRAFT' : 'QUEUED', attempt: 0, maxAttempts: spec.maxAttempts ?? 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: spec.timeoutMs ?? taskOverallTimeoutMs(spec.storySize ?? 'M'), createdAt: now, updatedAt: now, planningDecisionId: planner.id, planningSource: planner.planningSource ?? 'model-planner', planningRound: planner.planningRound ?? 1 };
522
528
  }
package/dist/relayd.js CHANGED
@@ -16,6 +16,7 @@ import { codexAndZaiUsageRegistry } from './usage.js';
16
16
  import { defaultWorkspaceRoot, discoverWorkspaceRepositories, resolveManagedProjectPath } from './workspace.js';
17
17
  import { deriveJobRuntimeStatus } from './runtime-status.js';
18
18
  import { decisionMode, LayaCliDecisionEngine } from './decision.js';
19
+ import { JsonlDecisionAuditStore } from './decision-audit.js';
19
20
  export const daemonHelp = `Agents Relay daemon runs the dashboard and worker pool.
20
21
 
21
22
  Usage:
@@ -72,7 +73,8 @@ export async function runDaemon(argv) {
72
73
  const repositoryPaths = new Map(discoveredRepositories.map(item => [item.repository, item.path]));
73
74
  const explicitProjectPath = value(argv, '--project-path');
74
75
  const usageRegistry = codexAndZaiUsageRegistry();
75
- const makeReconciler = (store, maxConcurrent) => new Reconciler(store, { owner: `relayd-${process.pid}`, maxConcurrent, leaseMs: Number(value(argv, '--lease-ms', '300000')), adapters: [new CodexAdapter(value(argv, '--codex', 'codex')), new ChatGptAdapter()], continuations: [new CodexThreadContinuation(value(argv, '--codex', 'codex')), new CommandContinuation(), new WebhookContinuation()], planner: runtimePlanner(argv), usageSnapshotProvider: () => usageRegistry.snapshot(), decisionEngine: new LayaCliDecisionEngine(), decisionMode: decisionMode(), eventBus: bus, reviewer: runtimeReviewer(), projectPathForJob: job => repositoryPaths.get(job.repository) ?? resolveManagedProjectPath(job.repository, explicitProjectPath, workspaceRoot) });
76
+ const decisionAuditStore = new JsonlDecisionAuditStore(process.env.AGENTS_RELAY_DECISION_AUDIT_PATH ?? join(homedir(), '.agents-relay', 'decision-audit.jsonl'));
77
+ const makeReconciler = (store, maxConcurrent) => new Reconciler(store, { owner: `relayd-${process.pid}`, maxConcurrent, leaseMs: Number(value(argv, '--lease-ms', '300000')), adapters: [new CodexAdapter(value(argv, '--codex', 'codex')), new ChatGptAdapter()], continuations: [new CodexThreadContinuation(value(argv, '--codex', 'codex')), new CommandContinuation(), new WebhookContinuation()], planner: runtimePlanner(argv), usageSnapshotProvider: () => usageRegistry.snapshot(), decisionEngine: new LayaCliDecisionEngine(), decisionMode: decisionMode(), decisionAuditStore, eventBus: bus, reviewer: runtimeReviewer(), projectPathForJob: job => repositoryPaths.get(job.repository) ?? resolveManagedProjectPath(job.repository, explicitProjectPath, workspaceRoot) });
76
78
  if (repositories.length === 0)
77
79
  throw new Error(`No Git repositories with GitHub origins found under ${normalized.workspaceRoot ?? defaultWorkspaceRoot()}`);
78
80
  const pool = new RepositoryWorkerPool(client, repositories, trusted, concurrency, makeReconciler);
@@ -0,0 +1,75 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { mkdir, readFile, writeFile } from 'node:fs/promises';
3
+ import { dirname, join } from 'node:path';
4
+ const EMPTY_POLICY = {};
5
+ function safePolicy(value) {
6
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7
+ return {};
8
+ const source = value;
9
+ const strings = (item) => Array.isArray(item) && item.every(value => typeof value === 'string') ? [...item] : undefined;
10
+ const result = {};
11
+ result.disabledProfiles = strings(source.disabledProfiles);
12
+ result.disabledProviders = strings(source.disabledProviders);
13
+ result.preferredProviders = strings(source.preferredProviders);
14
+ result.preferredProfiles = strings(source.preferredProfiles);
15
+ if (source.qualityByTaskKind && typeof source.qualityByTaskKind === 'object' && !Array.isArray(source.qualityByTaskKind)) {
16
+ result.qualityByTaskKind = {};
17
+ for (const [key, value] of Object.entries(source.qualityByTaskKind)) {
18
+ const parsed = strings(value);
19
+ if (parsed)
20
+ result.qualityByTaskKind[key] = parsed;
21
+ }
22
+ }
23
+ if (source.recovery && typeof source.recovery === 'object' && !Array.isArray(source.recovery)) {
24
+ const penalize = source.recovery.penalizePriorRoute;
25
+ if (typeof penalize === 'boolean')
26
+ result.recovery = { penalizePriorRoute: penalize };
27
+ }
28
+ return result;
29
+ }
30
+ export function policyPath(explicit) {
31
+ return explicit ?? process.env.AGENTS_RELAY_ROUTING_POLICY ?? join(process.cwd(), '.agents-relay-routing-policy.json');
32
+ }
33
+ export function modelRouterRulesPath(explicit) {
34
+ return explicit ?? process.env.AGENTS_RELAY_MODEL_ROUTER_RULES ?? join(process.cwd(), '.agents-relay-model-router-rules.md');
35
+ }
36
+ export async function loadModelRouterRules(explicit) {
37
+ const path = modelRouterRulesPath(explicit);
38
+ try {
39
+ const text = await readFile(path, 'utf8');
40
+ return { path, hash: createHash('sha256').update(text).digest('hex'), text };
41
+ }
42
+ catch {
43
+ return { path, hash: 'none', text: '' };
44
+ }
45
+ }
46
+ export async function saveModelRouterRules(text, explicit) {
47
+ const path = modelRouterRulesPath(explicit);
48
+ await mkdir(dirname(path), { recursive: true });
49
+ await writeFile(path, text, 'utf8');
50
+ return { path, hash: createHash('sha256').update(text).digest('hex'), text };
51
+ }
52
+ export async function loadRoutingPolicy(explicit, explicitRules) {
53
+ const path = policyPath(explicit);
54
+ const rules = await loadModelRouterRules(explicitRules);
55
+ try {
56
+ const text = await readFile(path, 'utf8');
57
+ const policy = safePolicy(JSON.parse(text));
58
+ return { path, hash: createHash('sha256').update(text).digest('hex'), policy, snapshot: JSON.parse(JSON.stringify(policy)), rules };
59
+ }
60
+ catch {
61
+ return { path: null, hash: 'none', policy: EMPTY_POLICY, snapshot: {}, rules };
62
+ }
63
+ }
64
+ export function policyAllows(candidate, policy) {
65
+ if (policy.disabledProviders?.includes(candidate.provider))
66
+ return false;
67
+ if (candidate.profile && policy.disabledProfiles?.includes(candidate.profile))
68
+ return false;
69
+ return true;
70
+ }
71
+ export function policyRank(candidate, taskKind, policy) {
72
+ const preferred = new Set([...(policy.preferredProviders ?? []), ...(policy.preferredProfiles ?? [])]);
73
+ const quality = taskKind ? new Set(policy.qualityByTaskKind?.[taskKind] ?? []) : new Set();
74
+ return (preferred.has(candidate.provider) || (candidate.profile && preferred.has(candidate.profile)) ? 2 : 0) + (quality.has(candidate.id) ? 1 : 0);
75
+ }
@@ -46,6 +46,8 @@ export function deriveJobRuntimeStatus(job, prState = 'OPEN') {
46
46
  return 'FAILED';
47
47
  if (job.tasks.some(task => task.state === 'DRAFT'))
48
48
  return 'WAITING';
49
+ if (job.executionMode === 'autonomous' && job.autoMerge === false && job.reviewEnabled === false && implementationComplete(job) && prState === 'OPEN')
50
+ return 'WAITING';
49
51
  if (job.reviewEnabled === false && implementationComplete(job) && prState === 'OPEN')
50
52
  return 'MERGE_PENDING';
51
53
  return 'IDLE';
package/dist/scheduler.js CHANGED
@@ -35,13 +35,19 @@ export function extendRetryBudget(task, windowSize = 3) {
35
35
  const usedOrBudgeted = Math.max(task.maxAttempts, task.attempt);
36
36
  return Math.max(windowSize, (Math.floor(usedOrBudgeted / windowSize) + 1) * windowSize);
37
37
  }
38
- export async function transitionTaskEvent(task, event, context = {}) {
38
+ export function tryTransitionTaskEvent(task, event, context = {}) {
39
39
  const decision = decideTaskLifecycle(task, event, context);
40
+ if (decision.accepted && decision.target)
41
+ applyTaskWorkflowDecision(task, decision);
42
+ return decision;
43
+ }
44
+ export async function transitionTaskEvent(task, event, context = {}) {
45
+ const from = task.state;
46
+ const decision = tryTransitionTaskEvent(task, event, context);
40
47
  if (!decision.accepted || !decision.target)
41
48
  throw new Error(decision.reason === 'ambiguous'
42
- ? `Ambiguous workflow transition from ${task.state} on ${event}`
43
- : `No enabled workflow transition from ${task.state} on ${event}`);
44
- applyTaskWorkflowDecision(task, decision);
49
+ ? `Ambiguous workflow transition from ${from} on ${event}`
50
+ : `No enabled workflow transition from ${from} on ${event}`);
45
51
  return decision;
46
52
  }
47
53
  export async function transitionJobEvent(job, event, context = {}) {
@@ -24,7 +24,7 @@ Retrieve this task's lifecycle event history before classifying the root cause o
24
24
  GET http://127.0.0.1:8787/api/events?job=${encodeURIComponent(job.id)}&task=${encodeURIComponent(task.id)}&limit=200
25
25
  Inspect event order, type/status, source, data/phase, duplicate events, missing expected lifecycle events, late events, and execution identifiers. If the event stream points to delivery/runtime failure, inspect the corresponding Relay/worker/tool logs too. Do not treat the final task.error string as the complete root cause.
26
26
 
27
- Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
27
+ Follow the selected Troubleshooter agent contract. Diagnose root cause from durable task state + event history + relevant runtime logs before retry. The final task.error string alone is never sufficient. Classify the evidence as model/provider/runtime, worker/tool/infrastructure, semantic task failure, or timeout/unknown. Only model/provider/runtime evidence may call the narrow model re-decision action; that action gathers fresh usage, reloads current policy, updates this exact Task's routing, and does not retry it. Troubleshooter must then reactivate/retry this exact original task and verify progress. Do not manually choose a provider/model or create a replacement task. If the blocker is a shared worker/runtime/tool defect, create and directly complete a P0/self-job in the owning repository, including required release/deploy verification. Retry this exact original task only after the blocker is removed, reconcile the original job, and verify durable progress.`;
28
28
  }
29
29
  function belongsToCurrentRun(job, task) {
30
30
  const slot = job.schedule?.currentRunSlot;
@@ -1,9 +1,31 @@
1
1
  import { chooseWithDecision } from './decision.js';
2
- export function usageForProvider(snapshot, provider) {
2
+ export function usageForProvider(snapshot, provider, now = Date.now()) {
3
3
  const usage = snapshot?.providers.find(item => item.id === provider)?.usage;
4
4
  if (usage?.available) {
5
5
  const usedPercent = Math.max(usage.fiveHour?.usedPercent ?? 0, usage.weekly?.usedPercent ?? 0);
6
- return { healthy: usedPercent < 100, usedPercent };
6
+ const resourceWindow = (window) => {
7
+ if (!window)
8
+ return undefined;
9
+ const resetAt = window.resetsAt;
10
+ return {
11
+ remainingPercent: 100 - window.usedPercent,
12
+ resetAt,
13
+ resetInMinutes: resetAt === null ? null : Math.max(0, Math.ceil((resetAt * 1000 - now) / 60_000)),
14
+ };
15
+ };
16
+ const windows = { fiveHour: resourceWindow(usage.fiveHour), weekly: resourceWindow(usage.weekly) };
17
+ const resetWindow = [usage.fiveHour, usage.weekly]
18
+ .filter((window) => window?.resetsAt !== null && window?.resetsAt !== undefined)
19
+ .filter(window => window.resetsAt * 1000 > now)
20
+ .sort((left, right) => left.resetsAt - right.resetsAt)[0];
21
+ return {
22
+ healthy: usedPercent < 100,
23
+ usedPercent,
24
+ remainingPercent: resetWindow ? 100 - resetWindow.usedPercent : undefined,
25
+ resetAt: resetWindow?.resetsAt ?? undefined,
26
+ resetInMinutes: resetWindow?.resetsAt ? Math.max(0, Math.ceil((resetWindow.resetsAt * 1000 - now) / 60_000)) : undefined,
27
+ windows,
28
+ };
7
29
  }
8
30
  const credits = snapshot?.credits.find(item => item.id === provider)?.credits;
9
31
  if (credits?.available) {
@@ -34,6 +56,10 @@ export function workerCandidates(adapters, registered, snapshot) {
34
56
  resourceHealthy: resource.healthy,
35
57
  resourceProvider: provider,
36
58
  resourceUsedPercent: resource.usedPercent,
59
+ resourceRemainingPercent: resource.remainingPercent,
60
+ resourceResetAt: resource.resetAt,
61
+ resourceResetInMinutes: resource.resetInMinutes,
62
+ resourceWindows: resource.windows,
37
63
  };
38
64
  });
39
65
  });
@@ -76,6 +102,10 @@ function selection(selected, selectedBy, decisionEvidence) {
76
102
  resourceHealthy: selected.resourceHealthy,
77
103
  resourceProvider: selected.resourceProvider,
78
104
  resourceUsedPercent: selected.resourceUsedPercent,
105
+ resourceRemainingPercent: selected.resourceRemainingPercent,
106
+ resourceResetAt: selected.resourceResetAt,
107
+ resourceResetInMinutes: selected.resourceResetInMinutes,
108
+ resourceWindows: selected.resourceWindows,
79
109
  decisionEvidence,
80
110
  };
81
111
  }
@@ -108,6 +138,10 @@ export async function routeWorker(candidates, requirements, options = {}) {
108
138
  resourceHealthy: item.resourceHealthy ?? null,
109
139
  resourceProvider: item.resourceProvider ?? null,
110
140
  resourceUsedPercent: item.resourceUsedPercent ?? null,
141
+ resourceRemainingPercent: item.resourceRemainingPercent ?? null,
142
+ resourceResetAt: item.resourceResetAt ?? null,
143
+ resourceResetInMinutes: item.resourceResetInMinutes ?? null,
144
+ resourceWindows: item.resourceWindows ?? null,
111
145
  })),
112
146
  }));
113
147
  const choice = await chooseWithDecision(options.decisionEngine, options.decisionMode ?? 'off', { facts }, { type: 'choice', question: 'Which eligible worker is best suited to execute this task?', choices: eligible.map(item => item.workerId) }, () => deterministic.workerId);
@@ -12,9 +12,6 @@ export const WORKFLOW_EVENT_NAMES = [
12
12
  'pr.closed',
13
13
  'pr.merged',
14
14
  'reconcile',
15
- 'reconcile.blocked',
16
- 'reconcile.completed',
17
- 'reconcile.started',
18
15
  'release',
19
16
  'retry',
20
17
  'review.approved',
@@ -23,6 +20,9 @@ export const WORKFLOW_EVENT_NAMES = [
23
20
  'task.completed',
24
21
  'task.failed',
25
22
  'task.started',
23
+ 'watchdog.failed',
24
+ 'worker.delivery_failed',
25
+ 'worker.start_ack_failed',
26
26
  ];
27
27
  export const WORKFLOW_FACT_NAMES = [
28
28
  'dependenciesSatisfied',
@@ -31,6 +31,7 @@ export const WORKFLOW_FACT_NAMES = [
31
31
  'dispatchable',
32
32
  'notMerged',
33
33
  'autoMerge',
34
+ 'recoverableExecution',
34
35
  ];
35
36
  export const WORKFLOW_MUTATION_NAMES = [
36
37
  'mutation.clearError',
@@ -33,4 +33,5 @@ export const taskLifecycleWorkflow = load('task-lifecycle', {
33
33
  dependenciesUnsatisfied: booleanGuard('dependenciesUnsatisfied'),
34
34
  runnable: booleanGuard('runnable'),
35
35
  dispatchable: booleanGuard('dispatchable'),
36
+ recoverableExecution: booleanGuard('recoverableExecution'),
36
37
  });