agents-relay 2.0.61 → 2.0.63

package/dist/github-webhook.js

@@ -1,4 +1,25 @@
 import { createHmac, timingSafeEqual } from 'node:crypto';
+export const PACKAGE_FAILED_MARKER = 'agents-relay:package-publish-failed:v1';
+export function parsePackagePublishFailureMarker(body) {
+    if (typeof body !== 'string')
+        return null;
+    const prefix = '<!-- ' + PACKAGE_FAILED_MARKER + ' ';
+    const start = body.indexOf(prefix);
+    if (start < 0)
+        return null;
+    const end = body.indexOf(' -->', start + prefix.length);
+    if (end < 0)
+        return null;
+    try {
+        const data = JSON.parse(body.slice(start + prefix.length, end));
+        if (typeof data.runId !== 'string' || !/^\d+$/.test(data.runId) || !Number.isInteger(data.attempt) || Number(data.attempt) < 1 || typeof data.sha !== 'string' || !/^[a-f0-9]{7,40}$/i.test(data.sha) || typeof data.stage !== 'string' || !data.stage)
+            return null;
+        return { runId: data.runId, attempt: Number(data.attempt), sha: data.sha, stage: data.stage };
+    }
+    catch {
+        return null;
+    }
+}
 export const PACKAGE_PUBLISHED_MARKER = 'agents-relay:package-published:v1';
 export function parsePackagePublicationMarker(body) {
     if (typeof body !== 'string')
@@ -71,9 +92,11 @@ export function parseGitHubWebhook(headers, body) {
     if (event === 'issue_comment') {
         const comment = payload.comment;
         const publication = parsePackagePublicationMarker(comment?.body);
-        if (!publication)
+        const publishFailure = comment?.user?.login === 'github-actions[bot]'
+            ? parsePackagePublishFailureMarker(comment?.body) : null;
+        if (!publication && !publishFailure)
             return null;
-        return { deliveryId, event, action, repository: repositoryName, pullRequest: number, publication, pullRequestFacts: null };
+        return { deliveryId, event, action, repository: repositoryName, pullRequest: number, publication, ...(publishFailure ? { publishFailure } : {}), pullRequestFacts: null };
     }
     const facts = pullRequestFacts(payload, action);
     if (!facts)

package/dist/pool.js

@@ -138,6 +138,13 @@ export class RepositoryWorkerPool {
                 continue;
             }
             const result = await withGitHubRequestContext(this.client, `pool.coordinator-apply:${signal.reason}`, () => reconciler.applyEvent(signal.event));
+            if (!result.applied)
+                observability().record({
+                    stage: 'transition', event: 'lifecycle.event_rejected', level: 'warn', outcome: 'skipped',
+                    message: `Lifecycle event ${signal.event.type} not applied: ${result.reason}`,
+                    correlation: { repository: target.repository, prNumber: target.prNumber, jobId, taskId: signal.event.task_id },
+                    data: { eventId: signal.event.event_id, sourceId: signal.event.source?.id, reason: result.reason, eventTimestamp: signal.event.timestamp },
+                });
             shouldReconcile ||= result.reconcile;
         }
         if (!shouldReconcile) {
@@ -304,6 +311,9 @@ export class RepositoryWorkerPool {
                     sha: wake.publication.sha,
                 }, source));
             }
+            if (wake.publishFailure) {
+                await bus.publish(eventFor(job.id, 'release', null, 'package.publish_failed', 'failed', `Publish workflow ${wake.publishFailure.runId} failed: ${wake.publishFailure.stage}`, 'user', { ...wake.publishFailure, repository: job.repository, prNumber: job.prNumber, deliveryId: wake.deliveryId }, source));
+            }
             await bus.publish(eventFor(job.id, 'github', null, 'github.webhook', 'queued', `GitHub webhook ${wake.event}${wake.action ? ':' + wake.action : ''}`, 'orchestrator', { deliveryId: wake.deliveryId, event: wake.event, action: wake.action, repository: job.repository, prNumber: job.prNumber }, source));
             return;
         }

package/dist/reconciler-runtime.js

@@ -1,12 +1,13 @@
+import { applyTaskRetryAction } from './task-actions.js';
 import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
 import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
-import { StartAckTimeout } from './adapters.js';
 import { eventFor } from './events.js';
 import { WorkflowEffectExecutor } from './workflow/effects.js';
 import { workflowEffectFailureEvent } from './workflow/contracts.js';
 import { ReconcileCoordinator } from './reconcile-coordinator.js';
 import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
 import { rearmTerminalEventDeadline } from './terminal-event-watchdog.js';
+import { createWorkerStartAck } from './worker-start-ack.js';
 import { monitorExecutionProcess } from './execution-process-tracker.js';
 export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
 function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
@@ -183,46 +184,7 @@ export class ReconcilerRuntimeBase {
     async startAck(jobId, taskId, sourceId, launchBarrier, timeoutMs = this.options.startAckTimeoutMs ?? 60000) {
         if (!this.options.eventBus)
             throw new Error('Worker start acknowledgement requires an event bus');
-        let resolveAck;
-        let rejectAck;
-        let settled = false;
-        let timer;
-        const ack = new Promise((resolve, reject) => { resolveAck = resolve; rejectAck = reject; });
-        const unsubscribe = await this.options.eventBus.subscribe(jobId, async (event) => {
-            if (settled || event.type !== 'task.started' || event.job_id !== jobId || event.task_id !== taskId)
-                return;
-            if (eventSourceId(event) !== sourceId)
-                return;
-            if (!event.timestamp || new Date(event.timestamp).getTime() < launchBarrier)
-                return;
-            settled = true;
-            if (timer)
-                clearTimeout(timer);
-            resolveAck();
-        });
-        timer = setTimeout(() => {
-            if (settled)
-                return;
-            settled = true;
-            void unsubscribe();
-            rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_timeout: no worker task.started start-ack`));
-        }, timeoutMs);
-        let stopped = false;
-        const stop = async () => {
-            if (stopped)
-                return;
-            stopped = true;
-            if (timer)
-                clearTimeout(timer);
-            if (!settled) {
-                settled = true;
-                rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_cancelled: start-ack owner closed`));
-            }
-            this.startAckCleanups.delete(stop);
-            await unsubscribe();
-        };
-        this.startAckCleanups.add(stop);
-        return { ack, stop };
+        return createWorkerStartAck(this.options.eventBus, jobId, taskId, sourceId, timeoutMs, this.startAckCleanups);
     }
     registerExecutionOwner(jobId, taskId, executionId) {
         activeExecutionOwners.set(executionEventSource(jobId, taskId, executionId), this);
@@ -269,8 +231,7 @@ export class ReconcilerRuntimeBase {
     }
     async markWorkerStarted(event) {
         const live = this.live.get(event.task_id);
-        if (live && event.job_id === live.jobId && event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
-            return { applied: false, reconcile: false, reason: 'task.started.stale' };
+        // Event timestamps are worker-authored and may be wrong; exact execution identity is authoritative.
         const transition = await this.taskTransitions.apply(event.job_id, event.task_id, 'task.started', {
             eligible: (job, task) => {
                 if (!task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
@@ -281,10 +242,10 @@ export class ReconcilerRuntimeBase {
             },
             facts: (_job, task) => ({
                 recoverableExecution: this.isRecoverableWatchdogFailure(task)
-                    && (!event.timestamp || new Date(event.timestamp).getTime() >= new Date(task.updatedAt).getTime()),
+                    && Boolean(task.executionId && eventSourceId(event) === executionEventSource(event.job_id, task.id, task.executionId)),
             }),
             mutate: (_job, task) => {
-                task.updatedAt = event.timestamp || new Date().toISOString();
+                task.updatedAt = new Date().toISOString();
             },
         });
         if (!transition?.decision.accepted)
@@ -335,8 +296,10 @@ export class ReconcilerRuntimeBase {
     }
     async runtimeSettled(jobId, taskId, executionId, result, error) {
         const live = this.live.get(taskId);
-        if (live?.executionId === executionId)
+        if (live?.executionId === executionId) {
             live.processSettled = true;
+            live.processError = error;
+        }
         const job = await this.store.load(jobId);
         const task = job.tasks.find(item => item.id === taskId);
         if (!task || (task.executionId !== executionId && !terminalExecutionClaimed(jobId, taskId, executionId)))
@@ -352,12 +315,17 @@ export class ReconcilerRuntimeBase {
             live.controller.abort();
             live.execution.cancel();
         }
+        const exit = live?.executionId === executionId && live.processSettled ? live.execution.processExit : undefined;
+        const rawError = live?.executionId === executionId && live.processSettled ? (live.processError || exit?.stderr || '').trim() : '';
+        const failedExit = Boolean(exit && (exit.exitCode !== 0 || exit.signal));
+        const failureReason = failedExit ? (/rate.limit.exceeded|usage limit reached/i.test(rawError) ? 'rate_limit_exceeded' : 'worker_process_failed') : 'terminal_event_timeout';
+        const failureMessage = failedExit ? (rawError || `Worker exited with code ${exit?.exitCode ?? 'null'}${exit?.signal ? `, signal ${exit.signal}` : ''}`) : `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
         const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
             eligible: (_job, task) => task.executionId === executionId,
             mutate: (_job, task) => {
                 this.clearExecution(task.id, executionId);
-                task.error = `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
-                task.failureReason = 'terminal_event_timeout';
+                task.error = failureMessage;
+                task.failureReason = failureReason;
                 task.leaseOwner = null;
                 task.leaseExpiresAt = null;
             },
@@ -366,8 +334,21 @@ export class ReconcilerRuntimeBase {
             this.clearExecution(taskId, executionId);
             return;
         }
-        const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
-        this.detach(this.emit(event), jobId, taskId, 'timeout failure event');
+        const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? failureMessage, 'user', { reason: failureReason, exitCode: exit?.exitCode ?? null, signal: exit?.signal ?? null, timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed' }, { agent: 'relay', component: 'agents-relay', watchdog: true });
+        await this.emit(event);
+        // Provider quota failures are recoverable. Drop the stale selection so the
+        // next launch runs the normal worker/model router with fresh usage data.
+        if (failureReason === 'rate_limit_exceeded' && transition.task.attempt < transition.task.maxAttempts) {
+            const current = (await this.store.load(jobId)).tasks.find(task => task.id === taskId);
+            if (current?.state === 'FAILED' && current.failureReason === failureReason && current.attempt === transition.task.attempt) {
+                await applyTaskRetryAction(current);
+                current.routing = undefined;
+                current.workerSelection = undefined;
+                await this.store.saveTask(current);
+                await this.emit(eventFor(jobId, taskId, current.parentTaskId, 'task.rerouting', 'queued', 'Provider rate limit reached; retrying with fresh routing and usage data', 'orchestrator', { previousFailure: failureReason, attempt: current.attempt, maxAttempts: current.maxAttempts }));
+                await this.eventCoordinator.accept({ jobId, reason: 'rate-limit-reroute' });
+            }
+        }
     }
     async applyTerminalEvent(event, trustedInternal = false) {
         const liveAtTerminal = this.live.get(event.task_id);
@@ -388,7 +369,7 @@ export class ReconcilerRuntimeBase {
                         return false;
                 }
                 const terminalWaiter = this.terminalWaiters.get(task.id);
-                return !terminalWaiter || Boolean(event.timestamp && new Date(event.timestamp).getTime() >= terminalWaiter.launchBarrier);
+                return !terminalWaiter || Boolean(task.executionId && terminalWaiter.executionId === task.executionId && this.isCorrelatedWorkerEvent(task, event));
             },
             facts: (_job, task) => {
                 const processExitedAfterPublishedTerminal = task.failureReason === 'worker_exit_without_terminal_event';
@@ -411,7 +392,7 @@ export class ReconcilerRuntimeBase {
                 task.leaseOwner = null;
                 task.leaseExpiresAt = null;
                 task.executionId = null;
-                task.updatedAt = event.timestamp || new Date().toISOString();
+                task.updatedAt = new Date().toISOString();
                 this.terminalWaiters.get(task.id)?.resolve();
                 this.terminalWaiters.delete(task.id);
                 const reportedTag = acceptedReportedTag(job, event.data);

package/dist/worker-start-ack.js

@@ -0,0 +1,42 @@
+import { StartAckTimeout } from './adapters.js';
+/** A worker start ack is scoped to its server-issued execution source, never its client clock. */
+export async function createWorkerStartAck(bus, jobId, taskId, sourceId, timeoutMs, cleanups) {
+    let resolveAck;
+    let rejectAck;
+    let settled = false;
+    let timer;
+    const ack = new Promise((resolve, reject) => { resolveAck = resolve; rejectAck = reject; });
+    const unsubscribe = await bus.subscribe(jobId, async (event) => {
+        if (settled || event.type !== 'task.started' || event.job_id !== jobId || event.task_id !== taskId)
+            return;
+        if (event.source?.id !== sourceId)
+            return;
+        settled = true;
+        if (timer)
+            clearTimeout(timer);
+        resolveAck();
+    });
+    timer = setTimeout(() => {
+        if (settled)
+            return;
+        settled = true;
+        void unsubscribe();
+        rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_timeout: no worker task.started start-ack`));
+    }, timeoutMs);
+    let stopped = false;
+    const stop = async () => {
+        if (stopped)
+            return;
+        stopped = true;
+        if (timer)
+            clearTimeout(timer);
+        if (!settled) {
+            settled = true;
+            rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_cancelled: start-ack owner closed`));
+        }
+        cleanups.delete(stop);
+        await unsubscribe();
+    };
+    cleanups.add(stop);
+    return { ack, stop };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agents-relay",
-  "version": "2.0.61",
+  "version": "2.0.63",
   "description": "Durable async agent jobs coordinated through GitHub pull requests",
   "type": "module",
   "bin": {