agents-relay 1.0.90 → 1.0.92

package/dist/reconciler-runtime.js

@@ -0,0 +1,404 @@
+import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
+import { StartAckTimeout } from './adapters.js';
+import { eventFor } from './events.js';
+import { WorkflowEffectExecutor } from './workflow/effects.js';
+import { workflowEffectFailureEvent } from './workflow/contracts.js';
+export function executionEventSource(jobId, taskId, executionId) { return `job/${jobId}/task/${taskId}/execution/${executionId}`; }
+function eventSourceId(event) { return typeof event.source?.id === 'string' ? event.source.id : undefined; }
+const terminalExecutionClaims = new Set();
+function claimTerminalExecution(sourceId) {
+    terminalExecutionClaims.add(sourceId);
+    const cleanup = setTimeout(() => terminalExecutionClaims.delete(sourceId), 60000);
+    cleanup.unref();
+}
+function terminalExecutionClaimed(jobId, taskId, executionId) {
+    return terminalExecutionClaims.has(executionEventSource(jobId, taskId, executionId));
+}
+export class ReconcilerRuntimeBase {
+    store;
+    options;
+    live = new Map();
+    plannerLive = new Map();
+    terminalTimers = new Map();
+    terminalExitGraceTimers = new Map();
+    continuationLive = new Set();
+    launchFailureClaims = new Set();
+    terminalWaiters = new Map();
+    workflowEffects = new WorkflowEffectExecutor();
+    reconciling = false;
+    constructor(store, options) {
+        this.store = store;
+        this.options = options;
+    }
+    async applyJobEffectOutcomes(job, outcomes) {
+        for (const outcome of outcomes.filter(item => item.status === 'failed')) {
+            const eventName = workflowEffectFailureEvent(outcome.effect);
+            try {
+                await transitionJobEvent(job, eventName);
+                await this.store.saveJob(job);
+            }
+            catch {
+                // Some effects are observational for a given state; failure is still emitted below.
+            }
+            await this.emit(eventFor(job.id, 'job', null, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
+        }
+    }
+    async applyTaskEffectOutcomes(job, task, outcomes) {
+        for (const outcome of outcomes.filter(item => item.status === 'failed')) {
+            const eventName = workflowEffectFailureEvent(outcome.effect);
+            try {
+                await transitionTaskEvent(task, eventName);
+                task.error = outcome.error ?? `${outcome.effect} failed`;
+                await this.store.saveTask(task);
+            }
+            catch {
+                // The effect failure remains observable even if this state has no feedback transition.
+            }
+            await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
+        }
+    }
+    async handleEvent(event) {
+        if (event.type === 'task.started') {
+            await this.markWorkerStarted(event);
+            return;
+        }
+        if (['task.completed', 'task.failed', 'task.blocked', 'task.cancelled'].includes(event.type)) {
+            await this.applyTerminalEvent(event);
+            return;
+        }
+        if (['review.started', 'review.approved', 'review.changes_required', 'review.blocked'].includes(event.type)) {
+            await this.applyReviewEvent(event);
+            return;
+        }
+        if (event.type === 'job.wake' || event.type === 'github.webhook')
+            await this.reconcile(event.job_id);
+    }
+    async watch(jobId) {
+        if (!this.options.eventBus)
+            return async () => { };
+        try {
+            return await this.options.eventBus.subscribe(jobId, event => this.handleEvent(event));
+        }
+        catch (error) {
+            await this.reportTransportFailure(error);
+            return async () => { };
+        }
+    }
+    async idle() { while (this.live.size > 0 || this.plannerLive.size > 0 || this.continuationLive.size > 0 || this.reconciling)
+        await new Promise(resolve => setTimeout(resolve, 25)); }
+    async wake(jobId) { return this.reconcile(jobId); }
+    async startAck(jobId, taskId, sourceId, launchBarrier, timeoutMs = this.options.startAckTimeoutMs ?? 60000) {
+        if (!this.options.eventBus)
+            throw new Error('Worker start acknowledgement requires an event bus');
+        let resolveAck;
+        let rejectAck;
+        let settled = false;
+        let timer;
+        const ack = new Promise((resolve, reject) => { resolveAck = resolve; rejectAck = reject; });
+        const unsubscribe = await this.options.eventBus.subscribe(jobId, async (event) => {
+            if (settled || event.type !== 'task.started' || event.job_id !== jobId || event.task_id !== taskId)
+                return;
+            if (eventSourceId(event) !== sourceId)
+                return;
+            if (!event.timestamp || new Date(event.timestamp).getTime() < launchBarrier)
+                return;
+            settled = true;
+            if (timer)
+                clearTimeout(timer);
+            resolveAck();
+        });
+        timer = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            void unsubscribe();
+            rejectAck(new StartAckTimeout(`Task ${taskId} worker_start_timeout: no worker task.started start-ack`));
+        }, timeoutMs);
+        let stopped = false;
+        const stop = async () => { if (stopped)
+            return; stopped = true; if (timer)
+            clearTimeout(timer); await unsubscribe(); };
+        return { ack, stop };
+    }
+    clearExecution(taskId, executionId) {
+        const live = this.live.get(taskId);
+        if (!executionId || live?.executionId === executionId)
+            this.live.delete(taskId);
+        const timer = this.terminalTimers.get(taskId);
+        if (timer)
+            clearTimeout(timer);
+        this.terminalTimers.delete(taskId);
+        const graceTimer = this.terminalExitGraceTimers.get(taskId);
+        if (graceTimer)
+            clearTimeout(graceTimer);
+        this.terminalExitGraceTimers.delete(taskId);
+    }
+    async markWorkerStarted(event) {
+        const live = this.live.get(event.task_id);
+        if (live && event.job_id === live.jobId && event.timestamp && new Date(event.timestamp).getTime() < live.launchBarrier)
+            return;
+        const job = await this.store.load(event.job_id);
+        const task = job.tasks.find(item => item.id === event.task_id);
+        if (!task || !task.executionId || eventSourceId(event) !== executionEventSource(job.id, task.id, task.executionId))
+            return;
+        const claimedExecution = task.state === 'QUEUED' && task.executionId !== null && task.leaseOwner !== null;
+        const exactLiveExecution = live && event.job_id === live.jobId && task.executionId === live.executionId;
+        if ((exactLiveExecution || claimedExecution) && task.state === 'QUEUED') {
+            await transitionTaskEvent(task, 'task.started');
+            task.failureReason = undefined;
+            task.error = null;
+            task.updatedAt = event.timestamp || new Date().toISOString();
+            await this.store.saveTask(task);
+            if (live)
+                live.workerStarted = true;
+            return;
+        }
+        if (task.state !== 'FAILED' || !this.isRecoverableWatchdogFailure(task))
+            return;
+        if (event.timestamp && new Date(event.timestamp).getTime() < new Date(task.updatedAt).getTime())
+            return;
+        await transitionTaskEvent(task, 'reconcile.started');
+        task.failureReason = undefined;
+        task.error = null;
+        task.updatedAt = event.timestamp || new Date().toISOString();
+        await this.store.saveTask(task);
+        this.detach(this.reconcile(job.id), job.id, task.id, 'late worker-start reconciliation');
+    }
+    isRecoverableWatchdogFailure(task) {
+        if (['worker_start_timeout', 'terminal_event_timeout', 'worker_exit_before_start', 'worker_exit_without_terminal_event'].includes(task.failureReason ?? ''))
+            return true;
+        const error = task.error ?? '';
+        return error.includes('worker_start_timeout')
+            || error.includes('timed out waiting for the mandatory terminal worker event')
+            || error.includes('exited before publishing task.started')
+            || error.includes('exited after task.started without publishing a terminal task event');
+    }
+    isCorrelatedWorkerEvent(task, event) {
+        return task.executionId !== null && eventSourceId(event) === executionEventSource(event.job_id, task.id, task.executionId);
+    }
+    async failLaunch(jobId, taskId, executionId, message, phase = 'launch') {
+        const claim = executionId ?? `${jobId}:${taskId}`;
+        if (this.launchFailureClaims.has(claim))
+            return;
+        this.launchFailureClaims.add(claim);
+        const cleanup = setTimeout(() => this.launchFailureClaims.delete(claim), 60000);
+        cleanup.unref();
+        const job = await this.store.load(jobId);
+        const task = job.tasks.find(item => item.id === taskId);
+        if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state))
+            return;
+        const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Worker ${phase} failed: ${message}`, 'user', { phase, reason: phase === 'start-ack' ? 'worker_start_timeout' : undefined }, { agent: 'relay', component: 'agents-relay', watchdog: true });
+        if (task.state === 'QUEUED') {
+            this.clearExecution(task.id, executionId);
+            task.error = event.message;
+            task.failureReason = phase === 'start-ack' ? 'worker_start_timeout' : undefined;
+            task.leaseOwner = null;
+            task.leaseExpiresAt = null;
+            await transitionTaskEvent(task, 'fail');
+            await this.store.saveTask(task);
+        }
+        else {
+            await this.applyTerminalEvent(event, true);
+        }
+        await this.emit(event);
+    }
+    async runtimeSettled(jobId, taskId, executionId, result, error) {
+        const live = this.live.get(taskId);
+        const liveWorkerStarted = live?.executionId === executionId && live.workerStarted;
+        if (live?.executionId === executionId)
+            this.live.delete(taskId);
+        const job = await this.store.load(jobId);
+        const task = job.tasks.find(item => item.id === taskId);
+        const workerStarted = liveWorkerStarted || task?.state === 'RUNNING';
+        if (!task || task.executionId !== executionId)
+            return;
+        await this.emit(eventFor(jobId, taskId, task.parentTaskId, task.adapter === 'chatgpt' ? 'task.process.async_exited' : 'task.process.exited', error ? 'failed' : 'succeeded', error ?? result?.summary ?? 'Worker runtime exited', 'orchestrator', { authoritative: false, waitingForTerminalEvent: true, processError: error }, { agent: 'relay', component: 'agents-relay' }));
+        if (['QUEUED', 'RUNNING'].includes(task.state) && task.adapter === 'codex' && !terminalExecutionClaimed(jobId, taskId, executionId))
+            this.scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted === true);
+    }
+    scheduleUnexpectedExit(jobId, taskId, executionId, workerStarted) {
+        const graceMs = Math.max(1, this.options.terminalExitGraceMs ?? 1000);
+        const timer = setTimeout(() => {
+            this.terminalExitGraceTimers.delete(taskId);
+            this.detach(this.failUnexpectedExit(jobId, taskId, executionId, workerStarted), jobId, taskId, 'unexpected worker process exit');
+        }, graceMs);
+        this.terminalExitGraceTimers.set(taskId, timer);
+    }
+    async failUnexpectedExit(jobId, taskId, executionId, workerStarted) {
+        if (terminalExecutionClaimed(jobId, taskId, executionId))
+            return;
+        const job = await this.store.load(jobId);
+        const task = job.tasks.find(item => item.id === taskId);
+        if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state) || terminalExecutionClaimed(jobId, taskId, executionId))
+            return;
+        const reason = workerStarted ? 'worker_exit_without_terminal_event' : 'worker_exit_before_start';
+        const message = workerStarted
+            ? `Worker ${taskId} exited after task.started without publishing a terminal task event`
+            : `Worker ${taskId} exited before publishing task.started`;
+        const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', message, 'user', { reason, terminalGraceMs: this.options.terminalExitGraceMs ?? 1000 }, { agent: 'relay', component: 'agents-relay', watchdog: true });
+        if (task.state === 'QUEUED') {
+            this.clearExecution(task.id, executionId);
+            task.error = message;
+            task.failureReason = reason;
+            task.leaseOwner = null;
+            task.leaseExpiresAt = null;
+            await transitionTaskEvent(task, 'fail');
+            await this.store.saveTask(task);
+        }
+        else {
+            await this.applyTerminalEvent(event, true);
+        }
+        await this.emit(event);
+    }
+    async timeoutTask(jobId, taskId, executionId) {
+        const live = this.live.get(taskId);
+        if (live?.executionId === executionId) {
+            live.controller.abort();
+            live.execution.cancel();
+        }
+        const job = await this.store.load(jobId);
+        const task = job.tasks.find(item => item.id === taskId);
+        if (!task || task.executionId !== executionId || !['QUEUED', 'RUNNING'].includes(task.state)) {
+            this.clearExecution(taskId, executionId);
+            return;
+        }
+        const event = eventFor(jobId, taskId, task.parentTaskId, 'task.failed', 'failed', `Task ${taskId} timed out waiting for the mandatory terminal worker event`, 'user', { reason: 'terminal_event_timeout', timeoutMs: task.timeoutMs });
+        if (task.state === 'QUEUED') {
+            task.error = event.message;
+            task.failureReason = 'terminal_event_timeout';
+            task.leaseOwner = null;
+            task.leaseExpiresAt = null;
+            await transitionTaskEvent(task, 'fail');
+            await this.store.saveTask(task);
+            this.clearExecution(task.id, executionId);
+        }
+        else {
+            await this.applyTerminalEvent(event, true);
+        }
+        await this.emit(event);
+    }
+    async applyTerminalEvent(event, trustedInternal = false) {
+        const job = await this.store.load(event.job_id);
+        const task = job.tasks.find(item => item.id === event.task_id);
+        if (!task || !['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
+            return;
+        const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
+        if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event))
+            return;
+        if (task.state === 'FAILED' && this.isRecoverableWatchdogFailure(task) && (trustedInternal || !modelBacked || this.isCorrelatedWorkerEvent(task, event))) {
+            if (event.timestamp && new Date(event.timestamp).getTime() < new Date(task.updatedAt).getTime())
+                return;
+            await transitionTaskEvent(task, 'reconcile.started');
+            task.failureReason = undefined;
+            task.error = null;
+        }
+        if (task.state !== 'RUNNING')
+            return;
+        const terminalWaiter = this.terminalWaiters.get(task.id);
+        if (terminalWaiter && (!event.timestamp || new Date(event.timestamp).getTime() < terminalWaiter.launchBarrier))
+            return;
+        const acceptedExecutionId = task.executionId;
+        if (modelBacked && acceptedExecutionId)
+            claimTerminalExecution(executionEventSource(job.id, task.id, acceptedExecutionId));
+        this.clearExecution(task.id, acceptedExecutionId);
+        task.leaseOwner = null;
+        task.leaseExpiresAt = null;
+        task.updatedAt = event.timestamp || new Date().toISOString();
+        this.terminalWaiters.get(task.id)?.resolve();
+        this.terminalWaiters.delete(task.id);
+        if (event.type === 'task.completed') {
+            task.result = { summary: event.message || `Task ${task.id} completed`, data: event.data };
+            task.error = null;
+            task.failureReason = undefined;
+            await transitionTaskEvent(task, 'task.completed');
+            await this.store.saveTask(task);
+            await this.deliverContinuation(job, task);
+        }
+        else if (event.type === 'task.failed') {
+            task.error = event.message || `Task ${task.id} failed`;
+            task.failureReason = typeof event.data?.reason === 'string' ? event.data.reason : undefined;
+            await transitionTaskEvent(task, 'task.failed');
+            await this.store.saveTask(task);
+        }
+        else if (event.type === 'task.blocked') {
+            task.error = event.message || `Task ${task.id} blocked`;
+            await transitionTaskEvent(task, 'task.blocked');
+            await this.store.saveTask(task);
+        }
+        else if (event.type === 'task.cancelled') {
+            task.error = event.message || `Task ${task.id} cancelled`;
+            await transitionTaskEvent(task, 'task.cancelled');
+            await this.store.saveTask(task);
+        }
+        this.detach(this.reconcile(job.id), job.id, task.id, 'terminal-event reconciliation');
+    }
+    terminalWait(taskId, executionId, launchBarrier) {
+        let resolve;
+        const promise = new Promise(resolver => { resolve = resolver; });
+        this.terminalWaiters.set(taskId, { executionId, launchBarrier, resolve });
+        return {
+            promise,
+            cancel: () => {
+                const current = this.terminalWaiters.get(taskId);
+                if (current?.executionId === executionId)
+                    this.terminalWaiters.delete(taskId);
+            },
+        };
+    }
+    async deliverContinuation(job, task) {
+        const continuation = task.continuation ?? job.continuation;
+        if (!continuation || task.continuationDeliveredAt || this.continuationLive.has(task.id))
+            return;
+        const adapter = this.options.continuations?.find(candidate => candidate.kind === continuation.kind);
+        if (!adapter) {
+            task.error = `No continuation adapter for ${continuation.kind}`;
+            await this.store.saveTask(task);
+            return;
+        }
+        this.continuationLive.add(task.id);
+        try {
+            await adapter.continue(task.result?.summary ?? '', continuation);
+            const fresh = await this.store.load(job.id);
+            const durable = fresh.tasks.find(item => item.id === task.id);
+            if (durable?.state === 'SUCCEEDED' && !durable.continuationDeliveredAt) {
+                durable.continuationDeliveredAt = new Date().toISOString();
+                await this.store.saveTask(durable);
+            }
+        }
+        catch (error) {
+            await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'continuation.failed', 'failed', `Continuation failed: ${error instanceof Error ? error.message : String(error)}`, 'user'));
+        }
+        finally {
+            this.continuationLive.delete(task.id);
+        }
+    }
+    async emit(event) { try {
+        if (this.options.emit)
+            await this.options.emit(event);
+    }
+    catch { /* observability cannot block durable execution */ } try {
+        if (this.options.eventBus)
+            await this.options.eventBus.publish(event);
+    }
+    catch (error) {
+        await this.reportTransportFailure(error);
+    } }
+    detach(promise, jobId, taskId, context) {
+        void promise.catch(error => this.reportDetachedFailure(jobId, taskId, context, error));
+    }
+    async reportDetachedFailure(jobId, taskId, context, error) {
+        if (!this.options.emit)
+            return;
+        const message = error instanceof Error ? error.message : String(error);
+        try {
+            await this.options.emit(eventFor(jobId, taskId, null, 'runtime.persistence.failed', 'failed', `${context} failed: ${message}`, 'orchestrator'));
+        }
+        catch { /* detached failure reporting must never crash the daemon */ }
+    }
+    async reportTransportFailure(error) { if (this.options.emit) {
+        try {
+            await this.options.emit(eventFor('system', 'event-bus', null, 'event.transport.failed', 'failed', `Event transport unavailable: ${error instanceof Error ? error.message : String(error)}`, 'orchestrator'));
+        }
+        catch { /* reporting is best effort */ }
+    } }
+}