agents-relay 2.0.60 → 2.0.62

package/dist/pool.js

@@ -138,6 +138,13 @@ export class RepositoryWorkerPool {
                 continue;
             }
             const result = await withGitHubRequestContext(this.client, `pool.coordinator-apply:${signal.reason}`, () => reconciler.applyEvent(signal.event));
+            if (!result.applied)
+                observability().record({
+                    stage: 'transition', event: 'lifecycle.event_rejected', level: 'warn', outcome: 'skipped',
+                    message: `Lifecycle event ${signal.event.type} not applied: ${result.reason}`,
+                    correlation: { repository: target.repository, prNumber: target.prNumber, jobId, taskId: signal.event.task_id },
+                    data: { eventId: signal.event.event_id, sourceId: signal.event.source?.id, reason: result.reason, eventTimestamp: signal.event.timestamp },
+                });
             shouldReconcile ||= result.reconcile;
         }
         if (!shouldReconcile) {

package/dist/reconciler-runtime.js

@@ -1,12 +1,13 @@
+import { applyTaskRetryAction } from './task-actions.js';
 import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
 import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
-import { StartAckTimeout } from './adapters.js';
 import { eventFor } from './events.js';
 import { WorkflowEffectExecutor } from './workflow/effects.js';
 import { workflowEffectFailureEvent } from './workflow/contracts.js';
 import { ReconcileCoordinator } from './reconcile-coordinator.js';
 import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
 import { rearmTerminalEventDeadline } from './terminal-event-watchdog.js';
+import { createWorkerStartAck } from './worker-start-ack.js';
 import { monitorExecutionProcess } from './execution-process-tracker.js';
 export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
 function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
@@ -183,46 +184,7 @@ export class ReconcilerRuntimeBase {
     async startAck(jobId, taskId, sourceId, launchBarrier, timeoutMs = this.options.startAckTimeoutMs ?? 60000) {
         if (!this.options.eventBus)
             throw new Error('Worker start acknowledgement requires an event bus');
-        let resolveAck;
-        let rejectAck;
-        let settled = false;
-        let timer;
-        const ack = new Promise((resolve, reject) => { resolveAck = resolve; rejectAck = reject; });
-        const unsubscribe = await this.options.eventBus.subscribe(jobId, async (event) => {
-            if (settled || event.type !== 'task.started' || event.job_id !== jobId || event.task_id !== taskId)
-                return;
-            if (eventSourceId(event) !== sourceId)
-                return;
-            if (!event.timestamp || new Date(event.timestamp).getTime() < launchBarrier)
-                return;
-            settled = true;
-            if (timer)
-                clearTimeout(timer);
-            resolveAck();
-        });
-        timer = setTimeout(() => {
-            if (settled)
-                return;
-            settled = true;
-            void unsubscribe();
-            rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_timeout: no worker task.started start-ack`));
-        }, timeoutMs);
-        let stopped = false;
-        const stop = async () => {
-            if (stopped)
-                return;
-            stopped = true;
-            if (timer)
-                clearTimeout(timer);
-            if (!settled) {
-                settled = true;
-                rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_cancelled: start-ack owner closed`));
-            }
-            this.startAckCleanups.delete(stop);
-            await unsubscribe();
-        };
-        this.startAckCleanups.add(stop);
-        return { ack, stop };
+        return createWorkerStartAck(this.options.eventBus, jobId, taskId, sourceId, timeoutMs, this.startAckCleanups);
     }
     registerExecutionOwner(jobId, taskId, executionId) {
         activeExecutionOwners.set(executionEventSource(jobId, taskId, executionId), this);
@@ -269,8 +231,7 @@ export class ReconcilerRuntimeBase {
     }
     async markWorkerStarted(event) {
         const live = this.live.get(event.task_id);
-        if (live && event.job_id === live.jobId && event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
-            return { applied: false, reconcile: false, reason: 'task.started.stale' };
+        // Event timestamps are worker-authored and may be wrong; exact execution identity is authoritative.
         const transition = await this.taskTransitions.apply(event.job_id, event.task_id, 'task.started', {
             eligible: (job, task) => {
                 if (!task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
@@ -281,10 +242,10 @@ export class ReconcilerRuntimeBase {
             },
             facts: (_job, task) => ({
                 recoverableExecution: this.isRecoverableWatchdogFailure(task)
-                    && (!event.timestamp || new Date(event.timestamp).getTime() >= new Date(task.updatedAt).getTime()),
+                    && Boolean(task.executionId && eventSourceId(event) === executionEventSource(event.job_id, task.id, task.executionId)),
             }),
             mutate: (_job, task) => {
-                task.updatedAt = event.timestamp || new Date().toISOString();
+                task.updatedAt = new Date().toISOString();
             },
         });
         if (!transition?.decision.accepted)
@@ -335,8 +296,10 @@ export class ReconcilerRuntimeBase {
     }
     async runtimeSettled(jobId, taskId, executionId, result, error) {
         const live = this.live.get(taskId);
-        if (live?.executionId === executionId)
+        if (live?.executionId === executionId) {
             live.processSettled = true;
+            live.processError = error;
+        }
         const job = await this.store.load(jobId);
         const task = job.tasks.find(item => item.id === taskId);
         if (!task || (task.executionId !== executionId && !terminalExecutionClaimed(jobId, taskId, executionId)))
@@ -352,12 +315,17 @@ export class ReconcilerRuntimeBase {
             live.controller.abort();
             live.execution.cancel();
         }
+        const exit = live?.executionId === executionId && live.processSettled ? live.execution.processExit : undefined;
+        const rawError = live?.executionId === executionId && live.processSettled ? (live.processError || exit?.stderr || '').trim() : '';
+        const failedExit = Boolean(exit && (exit.exitCode !== 0 || exit.signal));
+        const failureReason = failedExit ? (/rate.limit.exceeded|usage limit reached/i.test(rawError) ? 'rate_limit_exceeded' : 'worker_process_failed') : 'terminal_event_timeout';
+        const failureMessage = failedExit ? (rawError || `Worker exited with code ${exit?.exitCode ?? 'null'}${exit?.signal ? `, signal ${exit.signal}` : ''}`) : `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
         const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
             eligible: (_job, task) => task.executionId === executionId,
             mutate: (_job, task) => {
                 this.clearExecution(task.id, executionId);
-                task.error = `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
-                task.failureReason = 'terminal_event_timeout';
+                task.error = failureMessage;
+                task.failureReason = failureReason;
                 task.leaseOwner = null;
                 task.leaseExpiresAt = null;
             },
@@ -366,8 +334,21 @@ export class ReconcilerRuntimeBase {
             this.clearExecution(taskId, executionId);
             return;
         }
-        const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
-        this.detach(this.emit(event), jobId, taskId, 'timeout failure event');
+        const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? failureMessage, 'user', { reason: failureReason, exitCode: exit?.exitCode ?? null, signal: exit?.signal ?? null, timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
+        await this.emit(event);
+        // Provider quota failures are recoverable. Drop the stale selection so the
+        // next launch runs the normal worker/model router with fresh usage data.
+        if (failureReason === 'rate_limit_exceeded' && transition.task.attempt < transition.task.maxAttempts) {
+            const current = (await this.store.load(jobId)).tasks.find(task => task.id === taskId);
+            if (current?.state === 'FAILED' && current.failureReason === failureReason && current.attempt === transition.task.attempt) {
+                await applyTaskRetryAction(current);
+                current.routing = undefined;
+                current.workerSelection = undefined;
+                await this.store.saveTask(current);
+                await this.emit(eventFor(jobId, taskId, current.parentTaskId, 'task.rerouting', 'queued', 'Provider rate limit reached; retrying with fresh routing and usage data', 'orchestrator', { previousFailure: failureReason, attempt: current.attempt, maxAttempts: current.maxAttempts }));
+                await this.eventCoordinator.accept({ jobId, reason: 'rate-limit-reroute' });
+            }
+        }
     }
     async applyTerminalEvent(event, trustedInternal = false) {
         const liveAtTerminal = this.live.get(event.task_id);
@@ -388,7 +369,7 @@ export class ReconcilerRuntimeBase {
                         return false;
                 }
                 const terminalWaiter = this.terminalWaiters.get(task.id);
-                return !terminalWaiter || Boolean(event.timestamp && new Date(event.timestamp).getTime() >= terminalWaiter.launchBarrier);
+                return !terminalWaiter || Boolean(task.executionId && terminalWaiter.executionId === task.executionId && this.isCorrelatedWorkerEvent(task, event));
             },
             facts: (_job, task) => {
                 const processExitedAfterPublishedTerminal = task.failureReason === 'worker_exit_without_terminal_event';
@@ -411,7 +392,7 @@ export class ReconcilerRuntimeBase {
                 task.leaseOwner = null;
                 task.leaseExpiresAt = null;
                 task.executionId = null;
-                task.updatedAt = event.timestamp || new Date().toISOString();
+                task.updatedAt = new Date().toISOString();
                 this.terminalWaiters.get(task.id)?.resolve();
                 this.terminalWaiters.delete(task.id);
                 const reportedTag = acceptedReportedTag(job, event.data);

package/dist/worker-start-ack.js

@@ -0,0 +1,42 @@
+import { StartAckTimeout } from './adapters.js';
+/** A worker start ack is scoped to its server-issued execution source, never its client clock. */
+export async function createWorkerStartAck(bus, jobId, taskId, sourceId, timeoutMs, cleanups) {
+    let resolveAck;
+    let rejectAck;
+    let settled = false;
+    let timer;
+    const ack = new Promise((resolve, reject) => { resolveAck = resolve; rejectAck = reject; });
+    const unsubscribe = await bus.subscribe(jobId, async (event) => {
+        if (settled || event.type !== 'task.started' || event.job_id !== jobId || event.task_id !== taskId)
+            return;
+        if (event.source?.id !== sourceId)
+            return;
+        settled = true;
+        if (timer)
+            clearTimeout(timer);
+        resolveAck();
+    });
+    timer = setTimeout(() => {
+        if (settled)
+            return;
+        settled = true;
+        void unsubscribe();
+        rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_timeout: no worker task.started start-ack`));
+    }, timeoutMs);
+    let stopped = false;
+    const stop = async () => {
+        if (stopped)
+            return;
+        stopped = true;
+        if (timer)
+            clearTimeout(timer);
+        if (!settled) {
+            settled = true;
+            rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_cancelled: start-ack owner closed`));
+        }
+        cleanups.delete(stop);
+        await unsubscribe();
+    };
+    cleanups.add(stop);
+    return { ack, stop };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agents-relay",
-  "version": "2.0.60",
+  "version": "2.0.62",
   "description": "Durable async agent jobs coordinated through GitHub pull requests",
   "type": "module",
   "bin": {