agents-relay 2.0.68 → 2.0.70

package/dist/reconciler-review.js

@@ -6,9 +6,11 @@ import { workerCandidates } from './worker-routing.js';
 import { routeExecution } from './execution-routing.js';
 import { ReconcilerRuntimeBase } from './reconciler-runtime.js';
 import { dispatchReviewLifecycle, reviewOutcome } from './workflow/review-v2.js';
+import { dispatchTaskLifecycleSync } from './workflow/lifecycle.js';
 import { createTaskEntity } from './workflow/entity-creation.js';
 import { localSourceHead, localWorkingHead } from './local-git-merge.js';
 import { chatgptThreadRequirements, resolveReviewerThreadDestination } from './reviewer-thread.js';
+import { reviewGenerationMatches } from './review-generation-fence.js';
 export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
     async currentHeadSha(job) {
         // GitHub PR head facts are audit-only. Source identity is local Git.
@@ -118,8 +120,12 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
         }
         throw new Error('Too many final review rounds for ' + headSha);
     }
+    externalReviewPrNumber(job) {
+        return job.externalPrNumber && job.externalPrNumber > 0 ? job.externalPrNumber : job.prNumber;
+    }
     async createFinalReviewTask(job, headSha) {
         const id = this.nextFinalReviewTaskId(job, headSha);
+        const reviewPrNumber = this.externalReviewPrNumber(job);
         const now = new Date().toISOString();
         const reviewGateTasks = implementationTasks(job);
         const reviewThread = resolveReviewerThreadDestination(job, this.options.reviewer?.threadId);
@@ -130,13 +136,13 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
             threadSource: reviewThread.source,
             requestId: `${job.id}:${id}:${headSha}`,
             jobId: job.id,
-            prNumber: job.prNumber,
+            prNumber: reviewPrNumber,
             reviewHeadSha: headSha,
             projectId: reviewerRouting.candidate.routing.projectId ?? null,
         } : undefined;
         const input = [
             `Job ID: ${job.id}`,
-            `Review PR #${job.prNumber} at exactly head SHA ${headSha}.`,
+            `Review PR #${reviewPrNumber} at exactly head SHA ${headSha}.`,
             `Review head SHA: ${headSha}`,
             'Job objective: ' + (job.objective ?? job.description ?? job.title),
             'Use the selected reviewer agent contract as the authoritative review protocol.',
@@ -247,7 +253,7 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
                 id: job.id, title: job.title, objective: job.objective ?? null,
                 description: job.description ?? null, priority: job.priority ?? null,
                 executionMode: job.executionMode, repository: job.repository,
-                prNumber: job.prNumber, state: job.state,
+                prNumber: this.externalReviewPrNumber(job), state: job.state,
             },
             task: {
                 id: task.id, kind: task.kind ?? 'work', storySize: task.storySize ?? 'M',
@@ -264,9 +270,11 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
             ? job.tasks.find(item => item.id === task.parentTaskId)
             : undefined;
         const parentRouteFailedBeforeStart = Boolean(parent?.routing && /exited before publishing task\.started|worker_exit_before_start|launch failed/i.test(parent.error ?? parent.failureReason ?? ''));
-        const excludedRoutes = parentRouteFailedBeforeStart && parent?.routing
+        const parentExcludedRoutes = parentRouteFailedBeforeStart && parent?.routing
             ? [{ provider: parent.routing.provider, model: parent.routing.model, profile: parent.routing.profile }]
             : [];
+        const taskExcludedRoutes = (task.routingExclusions ?? []).map(item => ({ provider: item.provider, model: item.model, profile: item.profile }));
+        const excludedRoutes = [...taskExcludedRoutes, ...parentExcludedRoutes];
         const routed = await routeExecution(candidates, requirements, {
             decisionEngine: this.options.decisionEngine,
             decisionMode: this.options.decisionMode,
@@ -307,19 +315,23 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
         // their first trustworthy review event to PENDING before dispatching it.
         if (!job.review && task.planningDecisionId) {
             job.review = { outcome: 'pending', headSha, taskId: task.id, evidence: null, updatedAt: event.timestamp || new Date().toISOString() };
-            await this.store.saveJob(job);
         }
-        const taskDecision = tryTransitionTaskEvent(task, event.type);
+        const taskDecision = this.store.mutateJob
+            ? dispatchTaskLifecycleSync(structuredClone(task), event.type)
+            : tryTransitionTaskEvent(task, event.type);
         if (!taskDecision.accepted || !taskDecision.target)
             return { applied: false, reconcile: false, reason: event.type + '.ignored' };
-        task.error = null;
-        if (taskDecision.target !== 'RUNNING') {
-            task.leaseOwner = null;
-            task.leaseExpiresAt = null;
-            task.executionId = null;
+        if (!this.store.mutateJob && taskDecision.target !== 'RUNNING' && !this.finalizeReviewExecution(job.id, task, event)) {
+            return { applied: false, reconcile: false, reason: event.type + '.stale-execution' };
+        }
+        if (!this.store.mutateJob) {
+            task.error = null;
+            if (taskDecision.target !== 'RUNNING')
+                task.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
+            await this.store.saveTask(task);
         }
-        await this.store.saveTask(task);
         let needsReconcile = false;
+        let pendingFixTask = null;
         const reviewDecision = await dispatchReviewLifecycle(job.review?.outcome, event.type, { autoMerge: job.autoMerge !== false }, async (actionId) => {
             if (actionId === 'review.createFix') {
                 needsReconcile = true;
@@ -329,10 +341,9 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
                 const now = event.timestamp || new Date().toISOString();
                 const id = 'review-fix-' + headSha.slice(0, 8);
                 if (!job.tasks.some(item => item.id === id)) {
-                    const input = 'Address all reviewer findings for PR #' + job.prNumber + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
+                    const input = 'Address all reviewer findings for PR #' + this.externalReviewPrNumber(job) + ' at ' + headSha + ':\n\n' + issues.map((issue, index) => (index + 1) + '. ' + issue).join('\n');
                     const fixTask = createTaskEntity(job, { jobId: job.id, id, kind: 'work', priority: job.priority, storySize: 'M', projectName: job.title, agentName: 'implementation', parentTaskId: task.id, dependencies: [task.id], capabilities: ['implementation'], workerRequirements: { capabilities: [] }, adapter: 'codex', input, output: { kind: 'task_pr' }, continuation: null, continuationDeliveredAt: null, attempt: 0, maxAttempts: 3, leaseOwner: null, leaseExpiresAt: null, executionId: null, threadId: null, result: null, plannerResult: null, error: null, timeoutMs: 0, createdAt: now, updatedAt: now });
-                    await this.store.appendTask(fixTask);
-                    await this.emit(eventFor(job.id, id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + id + ' for ' + issues.length + ' reviewer finding(s)', 'orchestrator', { headSha, issues }, { agent: 'relay', component: 'agents-relay' }));
+                    pendingFixTask = fixTask;
                 }
                 return {};
             }
@@ -352,9 +363,54 @@ export class ReconcilerReviewBase extends ReconcilerRuntimeBase {
         const evidence = { summary: event.message || 'Review ' + outcome, data: {
                 ...(event.data ?? {}), review: { ...(reviewData ?? {}), headSha }, headSha, outcome, source: event.source,
             } };
-        job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: event.timestamp || new Date().toISOString() };
-        job.updatedAt = job.review.updatedAt;
-        await this.store.saveJob(job);
+        const reviewUpdatedAt = event.timestamp || new Date().toISOString();
+        if (this.store.mutateJob) {
+            let committed = false;
+            let terminalExecutionId = null;
+            const updated = await this.store.mutateJob(job.id, current => {
+                const freshTask = current.tasks.find(item => item.id === task.id);
+                if (!freshTask || !reviewGenerationMatches(current, job, task, headSha))
+                    return current;
+                const freshDecision = tryTransitionTaskEvent(freshTask, event.type);
+                if (!freshDecision.accepted || !freshDecision.target)
+                    return current;
+                if (freshDecision.target !== 'RUNNING') {
+                    if (!this.finalizeReviewExecution(job.id, freshTask, event, false))
+                        return current;
+                    terminalExecutionId = freshTask.executionId;
+                }
+                freshTask.error = null;
+                if (freshDecision.target !== 'RUNNING')
+                    freshTask.result = { summary: event.message || `Review ${event.type}`, data: event.data ?? {} };
+                if (pendingFixTask && !current.tasks.some(item => item.id === pendingFixTask.id))
+                    current.tasks.push(structuredClone(pendingFixTask));
+                current.review = { outcome, headSha, taskId: freshTask.id, evidence, updatedAt: reviewUpdatedAt };
+                current.updatedAt = reviewUpdatedAt;
+                committed = true;
+                return current;
+            });
+            if (!committed)
+                return { applied: false, reconcile: false, reason: event.type + '.stale-execution' };
+            Object.assign(job, structuredClone(updated));
+            // Live waiter cleanup/terminal claims are irreversible; run after commit.
+            if (terminalExecutionId) {
+                this.finalizeReviewExecution(job.id, { ...task, executionId: terminalExecutionId }, event);
+            }
+            const committedFixTask = pendingFixTask;
+            if (committedFixTask)
+                await this.emit(eventFor(job.id, committedFixTask.id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + committedFixTask.id + ' for reviewer findings', 'orchestrator', { headSha }, { agent: 'relay', component: 'agents-relay' }));
+        }
+        else {
+            const legacyFixTask = pendingFixTask;
+            if (legacyFixTask) {
+                await this.store.appendTask(legacyFixTask);
+                await this.emit(eventFor(job.id, legacyFixTask.id, task.id, 'review.fix_task_created', 'queued', 'Created one fix task ' + legacyFixTask.id + ' for reviewer findings', 'orchestrator', { headSha }, { agent: 'relay', component: 'agents-relay' }));
+            }
+            job.review = { outcome, headSha, taskId: task.id, evidence, updatedAt: reviewUpdatedAt };
+            job.updatedAt = reviewUpdatedAt;
+            await this.store.saveTask(task);
+            await this.store.saveJob(job);
+        }
         // Review approval records evidence only. The subsequent Graph reconcile
         // drives durable local Git merge delivery and terminalization.
         if (outcome === 'approved')

package/dist/reconciler-runtime.js

@@ -1,9 +1,9 @@
+import { applyJobEffectOutcomes, applyTaskEffectOutcomes } from './workflow-effect-failures.js';
+import { classifyWorkerRejection, reroutableWorkerRejections } from './worker-rejection.js';
 import { applyTaskRetryAction } from './task-actions.js';
-import { transitionJobEvent, transitionTaskEvent } from './scheduler.js';
 import { DurableTaskTransitionRuntime } from './workflow/durable-transition.js';
 import { eventFor } from './events.js';
 import { WorkflowEffectExecutor } from './workflow/effects.js';
-import { workflowEffectFailureEvent } from './workflow/contracts.js';
 import { ReconcileCoordinator } from './reconcile-coordinator.js';
 import { acceptedReportedTag, eventDataWithoutTag } from './task-tag.js';
 import { rearmTerminalEventDeadline } from './terminal-event-watchdog.js';
@@ -77,31 +77,10 @@ export class ReconcilerRuntimeBase {
         });
     }
     async applyJobEffectOutcomes(job, outcomes) {
-        for (const outcome of outcomes.filter(item => item.status === 'failed')) {
-            const eventName = workflowEffectFailureEvent(outcome.effect);
-            try {
-                await transitionJobEvent(job, eventName);
-                await this.store.saveJob(job);
-            }
-            catch {
-                // Some effects are observational for a given state; failure is still emitted below.
-            }
-            await this.emit(eventFor(job.id, 'job', null, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
-        }
+        await applyJobEffectOutcomes(this.store, event => this.emit(event), job, outcomes);
     }
     async applyTaskEffectOutcomes(job, task, outcomes) {
-        for (const outcome of outcomes.filter(item => item.status === 'failed')) {
-            const eventName = workflowEffectFailureEvent(outcome.effect);
-            try {
-                await transitionTaskEvent(task, eventName);
-                task.error = outcome.error ?? `${outcome.effect} failed`;
-                await this.store.saveTask(task);
-            }
-            catch {
-                // The effect failure remains observable even if this state has no feedback transition.
-            }
-            await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'workflow.effect.failed', 'failed', `${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`, 'orchestrator', { effect: outcome.effect, idempotencyKey: outcome.key, workflowEvent: eventName }, { agent: 'relay', component: 'agents-relay' }));
-        }
+        await applyTaskEffectOutcomes(this.store, event => this.emit(event), job, task, outcomes);
     }
     async applyEvent(event) {
         const sourceId = eventSourceId(event);
@@ -256,6 +235,28 @@ export class ReconcilerRuntimeBase {
     isCorrelatedWorkerEvent(task, event) {
         return task.executionId !== null && eventSourceId(event) === executionEventSource(event.job_id, task.id, task.executionId);
     }
+    finalizeReviewExecution(jobId, task, event, afterCommit = true) {
+        if (!task.executionId)
+            return true;
+        const sourceId = eventSourceId(event);
+        const expected = executionEventSource(jobId, task.id, task.executionId);
+        if ((task.adapter === 'codex' || task.adapter === 'chatgpt') && sourceId !== expected)
+            return false;
+        if (afterCommit)
+            claimTerminalExecution(expected);
+        const executionId = task.executionId;
+        if (afterCommit)
+            this.clearExecution(task.id, executionId);
+        task.leaseOwner = null;
+        task.leaseExpiresAt = null;
+        task.executionId = null;
+        task.updatedAt = new Date().toISOString();
+        if (afterCommit) {
+            this.terminalWaiters.get(task.id)?.resolve();
+            this.terminalWaiters.delete(task.id);
+        }
+        return true;
+    }
     async failLaunch(jobId, taskId, executionId, message, phase = 'launch') {
         const claim = executionId ?? `${jobId}:${taskId}`;
         if (this.launchFailureClaims.has(claim))
@@ -282,6 +283,11 @@ export class ReconcilerRuntimeBase {
         }
         const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? `Worker ${phase} failed: ${message}`, 'user', { phase, reason: transition.task.failureReason, workflowEvent }, { agent: 'relay', component: 'agents-relay', watchdog: true });
         this.detach(this.emit(event), jobId, taskId, 'worker failure event');
+        // failLaunch already committed the authoritative FAILED state. The emitted
+        // task.failed observation may be ignored as a duplicate, so explicitly wake
+        // graph reconciliation from the durable failure fact. This is required for
+        // automatic Troubleshooter recovery, including reviewer start-ack failures.
+        this.detach(this.eventCoordinator.accept({ jobId, reason: `worker-${phase}-failure` }), jobId, taskId, 'worker failure reconcile wake');
     }
     async runtimeSettled(jobId, taskId, executionId, result, error) {
         const live = this.live.get(taskId);
@@ -291,7 +297,15 @@ export class ReconcilerRuntimeBase {
         }
         const job = await this.store.load(jobId);
         const task = job.tasks.find(item => item.id === taskId);
-        if (terminalExecutionClaimed(jobId, taskId, executionId) || !task || task.executionId !== executionId)
+        // Async transport settlement is an independent, non-authoritative observation.
+        // A worker terminal event may already have claimed the Task; do not suppress
+        // transport-exit evidence merely because business completion won the race.
+        if (!task)
+            return;
+        const correlated = task.executionId === executionId || task.executionHistory?.some(item => item.id === executionId);
+        if (!correlated)
+            return;
+        if (task.adapter !== 'chatgpt' && (task.executionId !== executionId || terminalExecutionClaimed(jobId, taskId, executionId)))
             return;
         if (task.adapter !== 'chatgpt' && task.executionId === executionId && live?.executionId === executionId)
             rearmTerminalEventDeadline(this.terminalTimers, taskId, task.timeoutMs - (Date.now() - live.launchBarrier), this.options.postExitTerminalGraceMs ?? 15_000, () => this.detach(this.timeoutTask(jobId, taskId, executionId), jobId, taskId, 'post-exit terminal event timeout'));
@@ -307,7 +321,7 @@ export class ReconcilerRuntimeBase {
         const exit = live?.executionId === executionId && live.processSettled ? live.execution.processExit : undefined;
         const rawError = live?.executionId === executionId && live.processSettled ? (live.processError || exit?.stderr || '').trim() : '';
         const failedExit = Boolean(exit && (exit.exitCode !== 0 || exit.signal));
-        const failureReason = failedExit ? (/rate.limit.exceeded|usage limit reached/i.test(rawError) ? 'rate_limit_exceeded' : 'worker_process_failed') : 'terminal_event_timeout';
+        const failureReason = failedExit ? classifyWorkerRejection(rawError) : 'terminal_event_timeout';
         const failureMessage = failedExit ? (rawError || `Worker exited with code ${exit?.exitCode ?? 'null'}${exit?.signal ? `, signal ${exit.signal}` : ''}`) : `Task ${taskId} timed out waiting for the mandatory terminal worker event`;
         const transition = await this.taskTransitions.apply(jobId, taskId, 'watchdog.failed', {
             eligible: (_job, task) => task.executionId === executionId,
@@ -323,21 +337,38 @@ export class ReconcilerRuntimeBase {
             this.clearExecution(taskId, executionId);
             return;
         }
-        const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? failureMessage, 'user', { reason: failureReason, exitCode: exit?.exitCode ?? null, signal: exit?.signal ?? null, timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed', executionId }, { agent: 'relay', component: 'agents-relay', watchdog: true });
-        await this.emit(event);
-        // Provider quota failures are recoverable. Drop the stale selection so the
-        // next launch runs the normal worker/model router with fresh usage data.
-        if (failureReason === 'rate_limit_exceeded' && transition.task.attempt < transition.task.maxAttempts) {
+        let reroutedTask = null;
+        // Provider quota failures are recoverable. Commit the retry + route exclusion
+        // before publishing task.failed so event-driven reconciliation can never see
+        // an intermediate FAILED state and spawn a Troubleshooter for quota exhaustion.
+        if (reroutableWorkerRejections.has(failureReason) && transition.task.attempt < transition.task.maxAttempts) {
             const current = (await this.store.load(jobId)).tasks.find(task => task.id === taskId);
             if (current?.state === 'FAILED' && current.failureReason === failureReason && current.attempt === transition.task.attempt) {
+                const failedRoute = current.routing;
                 await applyTaskRetryAction(current);
+                if (failedRoute) {
+                    const duplicate = current.routingExclusions?.some(item => item.provider === failedRoute.provider && item.model === failedRoute.model && item.profile === failedRoute.profile);
+                    if (!duplicate)
+                        current.routingExclusions = [...(current.routingExclusions ?? []), {
+                                provider: failedRoute.provider,
+                                model: failedRoute.model,
+                                ...(failedRoute.profile ? { profile: failedRoute.profile } : {}),
+                                reason: 'rate_limit_exceeded',
+                                at: new Date().toISOString(),
+                            }];
+                }
                 current.routing = undefined;
                 current.workerSelection = undefined;
                 await this.store.saveTask(current);
-                await this.emit(eventFor(jobId, taskId, current.parentTaskId, 'task.rerouting', 'queued', 'Provider rate limit reached; retrying with fresh routing and usage data', 'orchestrator', { previousFailure: failureReason, attempt: current.attempt, maxAttempts: current.maxAttempts }));
-                await this.eventCoordinator.accept({ jobId, reason: 'rate-limit-reroute' });
+                reroutedTask = current;
             }
         }
+        const event = eventFor(jobId, taskId, transition.task.parentTaskId, 'task.failed', 'failed', transition.task.error ?? failureMessage, 'user', { reason: failureReason, exitCode: exit?.exitCode ?? null, signal: exit?.signal ?? null, timeoutMs: transition.task.timeoutMs, workflowEvent: 'watchdog.failed', executionId }, { agent: 'relay', component: 'agents-relay', watchdog: true });
+        await this.emit(event);
+        if (reroutedTask) {
+            await this.emit(eventFor(jobId, taskId, reroutedTask.parentTaskId, 'task.rerouting', 'queued', `Worker rejection ${failureReason}; retrying with fresh routing and usage data`, 'orchestrator', { previousFailure: failureReason, attempt: reroutedTask.attempt, maxAttempts: reroutedTask.maxAttempts }));
+            await this.eventCoordinator.accept({ jobId, reason: 'worker-rejection-reroute' });
+        }
     }
     async applyTerminalEvent(event, trustedInternal = false) {
         const liveAtTerminal = this.live.get(event.task_id);
@@ -346,6 +377,10 @@ export class ReconcilerRuntimeBase {
             eligible: (_job, task) => {
                 if (!['codex', 'chatgpt', 'orchestrator'].includes(task.adapter))
                     return false;
+                // Reviewer business terminal state is owned by review.approved / review.changes_required / review.blocked.
+                // A generic task.completed may arrive before the review decision and must not consume the execution generation.
+                if (task.agentName === 'reviewer' && event.type === 'task.completed')
+                    return false;
                 const modelBacked = task.adapter === 'codex' || task.adapter === 'chatgpt';
                 if (!trustedInternal && modelBacked && !this.isCorrelatedWorkerEvent(task, event)) {
                     const eventExecutionId = executionIdFromSource(eventSourceId(event), _job.id, task.id);

package/dist/reconciler.js

@@ -1,12 +1,15 @@
+import { cancelActiveTasks } from './reconciler-cancellation.js';
+import { authorizedInjection } from './e2e-injection.js';
 import { implementationComplete } from './runtime-status.js';
 import { bindPlannedReviewHead } from './planned-review-head.js';
 import { randomUUID } from 'node:crypto';
 import { storySizeTimeoutMs, chatGptStartAckTimeoutMs } from './types.js';
 import { dependencyReady, schedule, transitionTaskEvent } from './scheduler.js';
-import { applyTaskCancelAction } from './task-actions.js';
+import { applyTaskRetryAction } from './task-actions.js';
 import { LaunchFailure, StartAckTimeout } from './adapters.js';
 import { eventFor } from './events.js';
 import { createPlanRound, materializePlannerChild, pendingPlan } from './planner.js';
+import { retryPendingPlannerUnblocks } from './planner-runtime.js';
 import { dispatchPlannerOrchestration, plannerOrchestrationFacts } from './workflow/planner-orchestration-v2.js';
 import { resolveExecutionContext } from './execution-context.js';
 import { ReconcilerReviewBase } from './reconciler-review.js';
@@ -15,7 +18,7 @@ import { executeManualPlan } from './manual-plan.js';
 import { executionEventSource, withExecutionBoundary } from './reconciler-runtime.js';
 export { executionEventSource } from './reconciler-runtime.js';
 import { CallbackWorkflowEffectHandler } from './workflow/effects.js';
-import { pendingTroubleshooterTasks } from './troubleshooter.js';
+import { exhaustedTroubleshooterRecoveries, pendingTroubleshooterTasks } from './troubleshooter.js';
 import { taskUsesCanonicalProjectRun } from './workspace.js';
 import { cleanupMergedJobWorktree } from './job-worktree-cleanup.js';
 import { jobHasActiveExecution } from './job-execution-slot.js';
@@ -98,10 +101,49 @@ export class Reconciler extends ReconcilerReviewBase {
                 await this.emit(eventFor(job.id, task.id, task.parentTaskId, 'task.orchestrator_rescued', 'queued', `Stale orchestrator task ${task.id} converted to ChatGPT execution`, 'orchestrator', { ageMs: rescue.ageMs, adapter: 'chatgpt' }, { agent: 'relay', component: 'agents-relay' }));
             }
             await this.recoverPlannerChildren(job);
+            await retryPendingPlannerUnblocks({
+                store: this.store,
+                options: this.options,
+                live: this.plannerLive,
+                plannerContext: (currentJob, currentPlan) => this.plannerContext(currentJob, currentPlan),
+                emit: event => this.emit(event),
+                detach: (promise, currentJobId, planId, context) => this.detach(promise, currentJobId, planId, context),
+                reconcile: currentJobId => this.reconcile(currentJobId),
+            }, jobId);
             for (const recovery of pendingTroubleshooterTasks(await this.store.load(jobId))) {
                 await this.store.appendTask(recovery);
                 await this.emit(eventFor(job.id, recovery.id, null, "troubleshooter.created", "queued", "Troubleshooter task " + recovery.id + " created for failed/blocked work", "orchestrator", { agentName: "troubleshooter" }));
             }
+            {
+                const recoveryJob = await this.store.load(jobId);
+                // A failed recovery worker is itself recoverable: retry the SAME Task
+                // within its budget. No recursive Troubleshooter is generated.
+                for (const recovery of recoveryJob.tasks) {
+                    if (recovery.agentName !== 'troubleshooter' || recovery.state !== 'FAILED' || recovery.attempt >= recovery.maxAttempts)
+                        continue;
+                    const original = recoveryJob.tasks.find(task => task.id === recovery.parentTaskId);
+                    if (!original || !['FAILED', 'BLOCKED'].includes(original.state))
+                        continue;
+                    await applyTaskRetryAction(recovery);
+                    recovery.routing = undefined;
+                    recovery.workerSelection = undefined;
+                    await this.store.saveTask(recovery);
+                    await this.emit(eventFor(job.id, recovery.id, original.id, 'troubleshooter.retry_queued', 'queued', `Retrying failed Troubleshooter ${recovery.id} (attempt ${recovery.attempt + 1}/${recovery.maxAttempts})`, 'orchestrator', { originalTaskId: original.id, attempt: recovery.attempt + 1, maxAttempts: recovery.maxAttempts }));
+                }
+                const exhaustedJob = await this.store.load(jobId);
+                for (const exhausted of exhaustedTroubleshooterRecoveries(exhaustedJob)) {
+                    const at = new Date().toISOString();
+                    exhaustedJob.recoveryFailures = [...(exhaustedJob.recoveryFailures ?? []), {
+                            taskId: exhausted.original.id,
+                            recoveryTaskId: exhausted.recovery.id,
+                            reason: exhausted.reason,
+                            at,
+                        }];
+                    exhaustedJob.updatedAt = at;
+                    await this.store.saveJob(exhaustedJob);
+                    await this.emit(eventFor(job.id, exhausted.recovery.id, exhausted.original.id, 'recovery.exhausted', 'failed', `Recovery exhausted for ${exhausted.original.id}: ${exhausted.reason}`, 'user', { originalTaskId: exhausted.original.id, recoveryTaskId: exhausted.recovery.id, reason: exhausted.reason }, { agent: 'relay', component: 'agents-relay' }));
+                }
+            }
             for (const task of job.tasks)
                 if (task.state === 'CANCELLED') {
                     this.cancelExecutionGeneration(job.id, task.id, task.executionId ?? task.executionHistory?.at(-1)?.id);
@@ -133,7 +175,7 @@ export class Reconciler extends ReconcilerReviewBase {
             await this.ensureFinalReview(finalState, headSha);
             await this.dispatchExecutableTasks(jobId);
             finalState = await this.store.load(jobId);
-            if (await reconcileLocalGitCompletion(this.store, finalState, headSha, true, this.options.projectPathForJob))
+            if (await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob))
                 return this.store.load(jobId);
             await dispatchJobReconcile(finalState, { jobActive: finalState.state === 'ACTIVE', implementationComplete: implementationComplete(finalState), headAvailable: Boolean(headSha), deliveryReady: finalState.localMerge?.status === 'succeeded' }, async (actionId) => {
                 const outcomes = await this.workflowEffects.execute([actionId], { jobId: finalState.id, transitionKey: `reconcile:${headSha ?? 'no-head'}`, data: { headSha } });
@@ -172,29 +214,11 @@ export class Reconciler extends ReconcilerReviewBase {
         }
     }
     async cancelActiveTasks(jobId, reason) {
-        const job = await this.store.load(jobId);
-        for (const plan of (job.plans ?? []).filter(item => item.status === 'PLANNING')) {
-            const live = this.plannerLive.get(plan.id);
-            if (live) {
-                clearTimeout(live.timer);
-                live.controller.abort();
-                this.plannerLive.delete(plan.id);
-            }
-            plan.status = 'FAILED';
-            plan.error = reason;
-            plan.assessment = `Planning failed: ${reason}`;
-            plan.leaseOwner = null;
-            plan.leaseExpiresAt = null;
-            plan.executionId = null;
-            plan.updatedAt = new Date().toISOString();
-            await this.store.savePlan(plan);
-        }
-        for (const task of job.tasks.filter(item => item.kind !== 'planner' && !['SUCCEEDED', 'FAILED', 'UNBLOCKED', 'CANCELLED'].includes(item.state))) {
-            this.cancelExecutionGeneration(job.id, task.id, task.executionId ?? task.executionHistory?.at(-1)?.id);
-            this.plannerLive.get(task.id)?.controller.abort();
-            await applyTaskCancelAction(task, reason);
-            await this.store.saveTask(task);
-        }
+        return cancelActiveTasks(this.store, jobId, reason, taskId => { const live = this.plannerLive.get(taskId); if (live) {
+            clearTimeout(live.timer);
+            live.controller.abort();
+            this.plannerLive.delete(taskId);
+        } }, (job, taskId, executionId) => { this.cancelExecutionGeneration(job, taskId, executionId); });
     }
     executionRecoveryPorts(job) {
         return {
@@ -339,6 +363,31 @@ export class Reconciler extends ReconcilerReviewBase {
                     attempt: freshTask.attempt,
                     startedAt: new Date(startedAt).toISOString(),
                 }].slice(-8);
+            if (this.store.mutateJob) {
+                let claimed;
+                await this.store.mutateJob(job.id, current => {
+                    const durable = current.tasks.find(item => item.id === freshTask.id);
+                    if (!durable || durable.state !== 'QUEUED' || durable.executionId !== null
+                        || !dependencyReady(durable, new Map(current.tasks.map(item => [item.id, item]))))
+                        return current;
+                    // Routing is a pre-claim fact; copy only that fact into the fresh
+                    // authoritative snapshot, then claim the generation atomically.
+                    if (freshTask.routing)
+                        durable.routing = structuredClone(freshTask.routing);
+                    if (freshTask.workerSelection)
+                        durable.workerSelection = structuredClone(freshTask.workerSelection);
+                    durable.attempt += 1;
+                    durable.leaseOwner = this.options.owner;
+                    durable.leaseExpiresAt = new Date(startedAt + leaseMs).toISOString();
+                    durable.executionId = randomUUID();
+                    durable.executionHistory = [...(durable.executionHistory ?? []), {
+                            id: durable.executionId, attempt: durable.attempt, startedAt: new Date(startedAt).toISOString(),
+                        }].slice(-8);
+                    claimed = structuredClone(durable);
+                    return current;
+                });
+                return claimed;
+            }
             await this.store.saveTask(freshTask);
             return freshTask;
         }) ?? null;
@@ -380,7 +429,24 @@ export class Reconciler extends ReconcilerReviewBase {
         const startAckSetup = task.adapter === 'chatgpt' ? await this.startAck(job.id, task.id, sourceId, Date.now(), this.options.startAckTimeoutMs ?? chatGptStartAckTimeoutMs(task.timeoutMs)) : null;
         const closePolicy = task.chatgptTabClosePolicy ?? this.options.chatgptTabClosePolicy ?? adapter.tabClosePolicy ?? 'after-start';
         const terminalWait = task.adapter === 'chatgpt' && closePolicy === 'after-terminal' ? this.terminalWait(task.id, durableExecutionId, startedAt - 1) : null;
+        // Consume E2E injection before execution: a retry will not repeat it.
+        // Production repositories ignore the option entirely.
+        const declaration = authorizedInjection(job.repository, task.testInjection);
+        const testInjection = declaration && ((task.adapter === 'codex' && declaration.method === 'worker.codex.launch') || (task.adapter === 'chatgpt' && declaration.method === 'browserWorkspace.command')) ? declaration : undefined;
+        if (testInjection) {
+            const durable = (await this.store.load(job.id)).tasks.find(item => item.id === task.id);
+            if (!durable || durable.executionId !== durableExecutionId)
+                return;
+            delete durable.testInjection;
+            durable.updatedAt = new Date(Math.max(Date.now(), Date.parse(durable.updatedAt) + 1)).toISOString();
+            await this.store.saveTask(durable);
+            const consumed = (await this.store.load(job.id)).tasks.find(item => item.id === task.id);
+            if (!consumed || consumed.executionId !== durableExecutionId || consumed.testInjection)
+                throw new Error(`E2E injection consumption was not durably committed for ${task.id}`);
+            delete task.testInjection;
+        }
         const launchContext = {
+            ...(testInjection ? { testInjection } : {}),
             ...(startAckSetup ? { waitForWorkerStart: startAckSetup.ack } : {}),
             ...(terminalWait ? { waitForTerminal: terminalWait.promise } : {}),
             executionContext,

package/dist/relay-store.js

@@ -367,7 +367,7 @@ export class SQLiteRelayStore {
 export function relayStorePath(directory, repository, prNumber) {
     return directory + '/' + `${repository.replace('/', '_')}-${prNumber}.db`;
 }
-export { relayJobStorePath } from './local-job-path.js';
+export { relayJobStorePath, relayStoreDirectory } from './local-job-path.js';
 /**
  * SQLite is the only job registry. Inventory queries must be strictly read-only:
  * opening a writable RelayStore here would reset in-flight projection claims