agents-relay 2.0.130 → 2.0.131

package/dist/reconciler.js

@@ -21,6 +21,7 @@ import { executionEventSource, withExecutionBoundary } from './reconciler-runtim
 export { executionEventSource } from './reconciler-runtime.js';
 import { CallbackWorkflowEffectHandler } from './workflow/effects.js';
 import { exhaustedTroubleshooterRecoveries, pendingTroubleshooterTasks, pendingTroubleshooterReopens, refreshTroubleshooterInput } from './troubleshooter.js';
+import { fallbackFailedTroubleshooters, escalateExhaustedTroubleshooters } from './troubleshooter-fallback.js';
 import { taskUsesCanonicalProjectRun } from './workspace.js';
 import { cleanupMergedJobWorktree } from './job-worktree-cleanup.js';
 import { jobHasActiveExecution } from './job-execution-slot.js';
@@ -145,6 +146,10 @@ export class Reconciler extends ReconcilerReviewBase {
                     await this.store.saveTask(recovery);
                     await this.emit(eventFor(job.id, recovery.id, original.id, 'troubleshooter.retry_queued', 'queued', `Retrying failed Troubleshooter ${recovery.id} (attempt ${recovery.attempt + 1}/${recovery.maxAttempts})`, 'orchestrator', { originalTaskId: original.id, attempt: recovery.attempt + 1, maxAttempts: recovery.maxAttempts }));
                 }
+                for (const id of await fallbackFailedTroubleshooters(this.store, jobId))
+                    await this.emit(eventFor(job.id, id, null, 'troubleshooter.retry_queued', 'queued', `Alternate model fallback scheduled for ${id}`, 'orchestrator'));
+                for (const id of await escalateExhaustedTroubleshooters(this.store, jobId))
+                    await this.emit(eventFor(job.id, id, null, 'task.blocked', 'blocked', `Fallback exhausted: human action required for ${id}`, 'orchestrator'));
                 const exhaustedJob = await this.store.load(jobId);
                 for (const exhausted of exhaustedTroubleshooterRecoveries(exhaustedJob)) {
                     const at = new Date().toISOString();

package/dist/task-recovery.js

@@ -36,7 +36,7 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
             result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
             return job;
         }
-        if (!['QUEUED', 'RUNNING'].includes(recovery.state))
+        if (!['QUEUED', 'RUNNING', 'FAILED'].includes(recovery.state))
             throw new Error(`Troubleshooter ${recoveryTaskId} cannot escalate from ${recovery.state}`);
         const proposal = { path: 'escalation_gate', targetTaskId: original.id, rootCause: request.context.trim(), solution: request.requiredAction.trim() };
         const decision = dispatchTaskResolutionSync(original, proposal, 'troubleshooter', {
@@ -44,7 +44,7 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
         });
         if (!decision.accepted)
             throw new Error(`Task resolution graph rejected escalation for ${original.id}`);
-        if (!tryTransitionTaskEvent(recovery, 'task.blocked').accepted)
+        if (!tryTransitionTaskEvent(recovery, 'task.blocked', { recoverableExecution: true }).accepted)
             throw new Error(`Troubleshooter ${recoveryTaskId} cannot be blocked`);
         const escalation = {
             id: randomUUID(), originalTaskId: original.id, originalExecutionId: original.executionId ?? undefined,

package/dist/troubleshooter-fallback.js

@@ -0,0 +1,62 @@
+import { tryTransitionTaskEvent } from './scheduler.js';
+import { requestTroubleshooterEscalation } from './task-recovery.js';
+const MARKER = '[Relay execution fallback attempted]';
+const RUNTIME_FAILURE = /worker_process_failed|worker_start_timeout|terminal_event_timeout|model.*not supported|unsupported.model|delivery|process.exited|provider/i;
+/** One bounded Codex-config-default retry after normal recovery attempts. */
+export async function fallbackFailedTroubleshooters(store, jobId) {
+    const job = await store.load(jobId);
+    const queued = [];
+    for (const recovery of job.tasks.filter(task => task.agentName === 'troubleshooter' && task.state === 'FAILED' && task.attempt >= task.maxAttempts && !task.input.includes(MARKER))) {
+        const original = job.tasks.find(task => task.id === recovery.parentTaskId);
+        if (!original || original.state !== 'FAILED' || !RUNTIME_FAILURE.test(`${recovery.failureReason ?? ''} ${recovery.error ?? ''}`))
+            continue;
+        const update = (task) => {
+            if (task.state !== 'FAILED' || task.input.includes(MARKER))
+                return false;
+            if (!tryTransitionTaskEvent(task, 'retry').accepted)
+                throw new Error(`Fallback retry rejected for ${task.id}`);
+            task.maxAttempts = Math.max(task.maxAttempts, task.attempt + 1);
+            task.input += `\n\n${MARKER} Retry diagnosis with Codex config default model; report valid resolution or explicit blocker.`;
+            task.routing = { provider: '', model: '', decidedBy: 'troubleshooter-fallback:codex-config-default', decidedAt: new Date().toISOString() };
+            task.workerSelection = undefined;
+            task.leaseOwner = null;
+            task.leaseExpiresAt = null;
+            task.executionId = null;
+            return true;
+        };
+        if (store.mutateJob) {
+            await store.mutateJob(jobId, current => {
+                const task = current.tasks.find(item => item.id === recovery.id);
+                if (task && update(task))
+                    queued.push(task.id);
+                return current;
+            });
+        }
+        else {
+            const task = (await store.load(jobId)).tasks.find(item => item.id === recovery.id);
+            if (task && update(task)) {
+                await store.saveTask(task);
+                queued.push(task.id);
+            }
+        }
+    }
+    return queued;
+}
+/** The Core, not the crashed Agent, creates a durable human action gate. */
+export async function escalateExhaustedTroubleshooters(store, jobId) {
+    const job = await store.load(jobId);
+    const escalated = [];
+    for (const recovery of job.tasks.filter(task => task.agentName === 'troubleshooter' && task.state === 'FAILED' && task.attempt >= task.maxAttempts && !task.escalation)) {
+        const original = job.tasks.find(task => task.id === recovery.parentTaskId);
+        if (!original || original.state !== 'FAILED')
+            continue;
+        const context = `Automatic Troubleshooter ${recovery.id} could not produce a verified resolution after ${recovery.attempt} attempts. Last failure: ${recovery.error ?? recovery.failureReason ?? 'unknown'}. Original Task ${original.id}: ${original.error ?? original.failureReason ?? 'failed'}.`;
+        const result = await requestTroubleshooterEscalation(store, jobId, recovery.id, {
+            requiredAction: `Inspect Task ${original.id} and Troubleshooter ${recovery.id}; correct provider/runtime or supply a concrete recovery decision and supporting evidence.`,
+            resolutionCondition: `User supplies a corrective action or verified recovery instructions for Task ${original.id}.`, context,
+        });
+        if (!result.duplicate)
+            escalated.push(recovery.id);
+    }
+    return escalated;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agents-relay",
-  "version": "2.0.130",
+  "version": "2.0.131",
   "description": "Durable async agent jobs coordinated through GitHub pull requests",
   "repository": {
     "type": "git",