agents-relay 2.0.141 → 2.0.143

package/dist/reconciler.js

@@ -1,3 +1,7 @@
+import { deliverPendingEscalations } from './escalation-notifier.js';
+import { createFailedPlanRecovery } from './planner-recovery.js';
+import { recoverPlannerChildren } from './planner-child-recovery.js';
+import { drainProductionWorkflowOutbox } from './reconciler-workflow-outbox.js';
 import { cancelPausedGenerations } from './reconciler-pause.js';
 import { taskCommentContext } from './task-comments.js';
 import { cancelActiveTasks } from './reconciler-cancellation.js';
@@ -11,7 +15,7 @@ import { applyTaskRetryAction } from './task-actions.js';
 import { LaunchFailure, StartAckTimeout } from './adapters.js';
 import { eventFor } from './events.js';
 import { PreSubmissionRejection } from './worker-rejection.js';
-import { createPlanRound, materializePlannerChild, pendingPlan } from './planner.js';
+import { createPlanRound, pendingPlan } from './planner.js';
 import { retryPendingPlannerUnblocks } from './planner-runtime.js';
 import { dispatchPlannerOrchestration, plannerOrchestrationFacts } from './workflow/planner-orchestration-v2.js';
 import { resolveExecutionContext } from './execution-context.js';
@@ -27,7 +31,6 @@ import { taskUsesCanonicalProjectRun } from './workspace.js';
 import { cleanupMergedJobWorktree } from './job-worktree-cleanup.js';
 import { jobHasActiveExecution } from './job-execution-slot.js';
 import { managedWorkerInput } from './managed-worker-input.js';
-import { attachResolutionOwnership } from './workflow/task-resolution.js';
 export { managedWorkerInput } from './managed-worker-input.js';
 import { recoverExpiredExecutions, recoverStartupOrphans as recoverStartupExecutionOrphans } from './execution-recovery.js';
 import { dispatchJobReconcile } from './workflow/job-reconcile-v2.js';
@@ -81,7 +84,10 @@ export class Reconciler extends ReconcilerReviewBase {
             return this.store.load(jobId);
         this.reconciling = true;
         try {
-            const job = await this.store.load(jobId);
+            let job = await this.store.load(jobId);
+            await drainProductionWorkflowOutbox(this.store, this.options);
+            await deliverPendingEscalations(this.store, jobId, this.options.escalationNotifier);
+            job = await this.store.load(jobId);
             for (const live of [...this.live.values()])
                 if (live.jobId === jobId && job.tasks.some(task => task.id === live.taskId && task.state === 'BLOCKED'))
                     this.cancelExecutionGeneration(jobId, live.taskId, live.executionId);
@@ -196,7 +202,9 @@ export class Reconciler extends ReconcilerReviewBase {
             await this.ensureFinalReview(finalState, headSha);
             await this.dispatchExecutableTasks(jobId);
             finalState = await this.store.load(jobId);
-            if (await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event)))
+            const localMergeCompleted = await reconcileLocalGitCompletion(this.store, finalState, headSha, this.options.noGitHub === true, this.options.projectPathForJob, event => this.emit(event), this.options.localMergeEffect, () => (this.options.now ?? new Date()).getTime());
+            await drainProductionWorkflowOutbox(this.store, this.options);
+            if (localMergeCompleted)
                 return this.store.load(jobId);
             await dispatchJobReconcile(finalState, { jobActive: finalState.state === 'ACTIVE', implementationComplete: implementationComplete(finalState), headAvailable: Boolean(headSha), deliveryReady: finalState.localMerge?.status === 'succeeded' }, async (actionId) => {
                 const outcomes = await this.workflowEffects.execute([actionId], { jobId: finalState.id, transitionKey: `reconcile:${headSha ?? 'no-head'}`, data: { headSha } });
@@ -279,6 +287,16 @@ export class Reconciler extends ReconcilerReviewBase {
                 changed = true;
                 return {};
             }
+            if (actionId === 'planner.createRecovery') {
+                const failed = (job.plans ?? []).at(-1);
+                if (!failed || failed.status !== 'FAILED')
+                    throw Error('Graph selected missing FAILED Plan recovery');
+                const recovery = createFailedPlanRecovery(job, failed);
+                await this.store.appendTask(recovery);
+                await this.emit(eventFor(job.id, recovery.id, null, 'troubleshooter.created', 'queued', `Troubleshooter recovery for failed Plan ${failed.id}`, 'orchestrator', { agentName: 'troubleshooter', planId: failed.id }));
+                changed = true;
+                return {};
+            }
             if (actionId === 'planner.createRound') {
                 const plan = createPlanRound(job, { timeoutMs: this.options.plannerTimeoutMs ?? storySizeTimeoutMs('M') });
                 await this.store.appendPlan(plan);
@@ -293,47 +311,7 @@ export class Reconciler extends ReconcilerReviewBase {
             return job;
         return this.store.load(job.id);
     }
-    async recoverPlannerChildren(job) {
-        for (const plan of (job.plans ?? []).filter(item => item.status === 'SUCCEEDED' && item.plannerResult)) {
-            const executionContext = await this.plannerContext(job, plan);
-            // Historical plans were validated on commit. Rechecking against later Task failures
-            // blocks recovery before the Troubleshooter graph can run.
-            const children = plan.plannerResult.next_tasks.map(spec => materializePlannerChild(job, plan, spec, executionContext));
-            for (const child of children) {
-                const existing = job.tasks.find(item => item.id === child.id);
-                if (existing)
-                    continue;
-                const spec = plan.plannerResult.next_tasks.find(item => item.id === child.id);
-                if (spec?.resolution)
-                    await this.store.saveTask(attachResolutionOwnership(job, child, spec.resolution, 'planner'));
-                await this.store.appendTask(child);
-                job.tasks.push(child);
-            }
-        }
-        const historicalPlanners = job.tasks.filter(task => task.kind === 'planner' && task.state === 'SUCCEEDED' && task.plannerResult);
-        if (historicalPlanners.length === 0)
-            return;
-        const executionContext = await this.plannerContext(job, historicalPlanners[0]);
-        for (const planner of historicalPlanners) {
-            const result = planner.plannerResult;
-            const plan = { jobId: job.id, id: planner.id, round: planner.planningRound ?? 1, source: planner.planningSource ?? 'model-planner', previousPlanId: planner.previousPlannerTaskId ?? null, trigger: planner.planTrigger ?? { source: 'continuation' }, status: 'SUCCEEDED', guidance: planner.plannerGuidance, objectiveStatus: result.objective_status, plannerResult: result, assessment: result.assessment, generatedTaskIds: result.next_tasks.map(item => item.id), headSha: planner.planTrigger?.currentHeadSha ?? job.headSha, attempt: planner.attempt, maxAttempts: planner.maxAttempts, leaseOwner: null, leaseExpiresAt: null, executionId: null, timeoutMs: planner.timeoutMs, manualPlanning: planner.manualPlanning, createdAt: planner.createdAt, updatedAt: planner.updatedAt };
-            if (!(job.plans ?? []).some(item => item.id === plan.id)) {
-                await this.store.appendPlan(plan);
-                (job.plans ??= []).push(plan);
-            }
-            else
-                continue;
-            const children = result.next_tasks.map(spec => materializePlannerChild(job, plan, spec, executionContext));
-            for (const child of children)
-                if (!job.tasks.some(item => item.id === child.id)) {
-                    const spec = result.next_tasks.find(item => item.id === child.id);
-                    if (spec?.resolution)
-                        await this.store.saveTask(attachResolutionOwnership(job, child, spec.resolution, 'planner'));
-                    await this.store.appendTask(child);
-                    job.tasks.push(child);
-                }
-        }
-    }
+    async recoverPlannerChildren(job) { return recoverPlannerChildren(this.store, job, (j, p) => this.plannerContext(j, p)); }
     async resolvedContext(job, task, role, projectPathOverride) {
         const projectPath = projectPathOverride ?? (this.options.projectPathForJob ? await this.options.projectPathForJob(job) : task.routing?.cwd ?? process.cwd());
         return resolveExecutionContext(projectPath, role, {
@@ -443,6 +421,14 @@ export class Reconciler extends ReconcilerReviewBase {
         if (!claimedTask)
             return;
         task = claimedTask;
+        // After the durable execution claim, fence this Graph-selected launch
+        // before invoking the adapter. An Unknown previous attempt must never be
+        // re-launched speculatively. Legacy test stores have no SQLite ledger.
+        if (this.store.reserveWorkflowLaunch && task.executionId && task.leaseOwner) {
+            const reserved = await this.store.reserveWorkflowLaunch({ jobId: job.id, taskId: task.id, executionId: task.executionId, attempt: task.attempt, leaseOwner: task.leaseOwner });
+            if (!reserved)
+                throw new Error('already reserved launch attempt: reconcile existing generation');
+        }
         const adapter = this.options.adapters.find(x => x.name === task.adapter);
         if (!adapter) {
             await this.failLaunch(job.id, task.id, task.executionId, 'No adapter ' + task.adapter);

package/dist/relay-job-snapshot-reads.js

@@ -0,0 +1,11 @@
+import { normalizeLegacyTaskLifecycle } from './types.js';
+import { normalizeLegacyJobPrNumber } from './job-pr-number-migration.js';
+function normalize(json) { const job = normalizeLegacyJobPrNumber(JSON.parse(json)); job.tasks = job.tasks.map(t => normalizeLegacyTaskLifecycle(t, job.executionMode)); return job; }
+export function loadRelayJob(db, jobId) {
+    const row = db.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(jobId);
+    return row?.json ? normalize(row.json) : null;
+}
+export function listRelayJobs(db, scopeJobId) {
+    const rows = db.prepare(scopeJobId ? 'SELECT json FROM job_snapshot WHERE job_id=? ORDER BY job_id' : 'SELECT json FROM job_snapshot ORDER BY job_id').all(...(scopeJobId ? [scopeJobId] : []));
+    return rows.map(row => normalize(row.json));
+}

package/dist/relay-store.js

@@ -1,15 +1,16 @@
+import { loadRelayJob, listRelayJobs } from './relay-job-snapshot-reads.js';
+import { SQLiteWorkflowOperations } from './relay-workflow-operations.js';
 import { preserveTaskComments } from './task-comments.js';
 import { DatabaseSync } from 'node:sqlite';
 import { dirname } from 'node:path';
 import { mkdirSync } from 'node:fs';
-import { normalizeLegacyTaskLifecycle } from './types.js';
 import { recordPublicationInDatabase, loadPublicationFromDatabase, listPublicationEventsFromDatabase } from './publication-store.js';
 import { observability } from './observability.js';
 import { createJobIfAbsent } from './relay-create.js';
 import { ensureUnifiedJobRegistry, createUnifiedJobIfAbsent } from './unified-job-registry.js';
 import { migrateUnifiedRelaySchema } from './unified-relay-schema.js';
 import { saveUnifiedPullRequestFacts } from './unified-pr-facts.js';
-import { migrateLegacyJobPrNumbers, normalizeLegacyJobPrNumber } from './job-pr-number-migration.js';
+import { migrateLegacyJobPrNumbers } from './job-pr-number-migration.js';
 import { hostInvocationTablesSql, recordHostInvocationEvent, hasHostInvocationDecision, recordHostInvocationDecision } from './host-invocation-store.js';
 import { ProjectionOutbox, SCHEMA_VERSION, logicalOperationKey, } from './projection-outbox.js';
 function newerRecord(local, remote) {
@@ -144,19 +145,8 @@ export class SQLiteRelayStore {
     async listPublicationEvents(jobId) {
         return listPublicationEventsFromDatabase(this.db, jobId);
     }
-    async loadJob(jobId) {
-        this.assertScope(jobId);
-        const row = this.db.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(jobId);
-        if (!row?.json)
-            return null;
-        const job = normalizeLegacyJobPrNumber(JSON.parse(row.json));
-        job.tasks = job.tasks.map(task => normalizeLegacyTaskLifecycle(task, job.executionMode));
-        return job;
-    }
-    async listJobs() {
-        const rows = this.db.prepare(this.scopeJobId ? 'SELECT json FROM job_snapshot WHERE job_id=? ORDER BY job_id' : 'SELECT json FROM job_snapshot ORDER BY job_id').all(...(this.scopeJobId ? [this.scopeJobId] : []));
-        return rows.map(row => { const job = normalizeLegacyJobPrNumber(JSON.parse(row.json)); job.tasks = job.tasks.map(task => normalizeLegacyTaskLifecycle(task, job.executionMode)); return job; });
-    }
+    async loadJob(jobId) { this.assertScope(jobId); return loadRelayJob(this.db, jobId); }
+    async listJobs() { return listRelayJobs(this.db, this.scopeJobId); }
     async saveJob(job) {
         if (!this.scopeJobId) {
             this.writeJobSnapshot(job, new Date().toISOString());
@@ -172,6 +162,28 @@ export class SQLiteRelayStore {
             throw error;
         }
     }
+    workflowOperations() {
+        return new SQLiteWorkflowOperations(this.db, (job, at) => this.writeJobSnapshot(job, at));
+    }
+    async commitWorkflowMutationAndEvents(jobId, mutation, events) {
+        this.assertScope(jobId);
+        return this.workflowOperations().commitWorkflowMutationAndEvents(jobId, mutation, events);
+    }
+    async commitWorkflowMutationWithEvents(jobId, mutation) {
+        this.assertScope(jobId);
+        return this.workflowOperations().commitWorkflowMutationWithEvents(jobId, mutation);
+    }
+    async reserveWorkflowLaunch(identity) {
+        this.assertScope(identity.jobId);
+        return this.workflowOperations().reserveWorkflowLaunch(identity);
+    }
+    async verifyWorkflowLaunch(jobId, taskId, executionId) {
+        this.assertScope(jobId);
+        return this.workflowOperations().verifyWorkflowLaunch(jobId, taskId, executionId);
+    }
+    async drainWorkflowEvents(publish) {
+        return this.workflowOperations().drainWorkflowEvents(publish);
+    }
     async createJobIfAbsent(job) {
         this.assertScope(job.id);
         return this.scopeJobId
@@ -181,28 +193,7 @@ export class SQLiteRelayStore {
     /** Atomic read/modify/write for local authoritative Task and Plan updates. */
     async mutateJob(jobId, mutation) {
         this.assertScope(jobId);
-        this.db.exec('BEGIN IMMEDIATE');
-        try {
-            const row = this.db.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(jobId);
-            if (!row?.json)
-                throw new Error(`Local authoritative Job ${jobId} not found`);
-            const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
-            current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
-            const before = JSON.stringify(current);
-            const next = mutation(current);
-            if (next.id !== jobId)
-                throw new Error('Cannot change authoritative Job identity');
-            if (JSON.stringify(next) !== before && next.updatedAt <= JSON.parse(before).updatedAt) {
-                next.updatedAt = monotonicJobTimestamp(JSON.parse(before).updatedAt);
-            }
-            this.writeJobSnapshot(next, new Date().toISOString());
-            this.db.exec('COMMIT');
-            return structuredClone(next);
-        }
-        catch (error) {
-            this.db.exec('ROLLBACK');
-            throw error;
-        }
+        return this.workflowOperations().mutateJob(jobId, mutation);
     }
     async ingestJobSnapshot(job) {
         this.assertScope(job.id);

package/dist/relay-workflow-operations.js

@@ -0,0 +1,104 @@
+import { normalizeLegacyTaskLifecycle } from './types.js';
+import { normalizeLegacyJobPrNumber } from './job-pr-number-migration.js';
+import { taskLifecycleV2 } from './workflow/definitions-v2.js';
+import { dispatchWorkflowGraphSync } from './workflow/dispatcher.js';
+import { WorkflowActionLedger } from './workflow/action-ledger.js';
+import { ConditionRegistry, booleanCondition } from './workflow/conditions.js';
+import { DurableWorkflowPublication } from './workflow/durable-publication.js';
+function monotonicJobTimestamp(previous, candidate = new Date().toISOString()) { const a = Date.parse(previous), b = Date.parse(candidate); return new Date(Math.max(Number.isFinite(a) ? a + 1 : Date.now(), Number.isFinite(b) ? b : Date.now())).toISOString(); }
+export class SQLiteWorkflowOperations {
+    db;
+    write;
+    constructor(db, write) {
+        this.db = db;
+        this.write = write;
+    }
+    async commitWorkflowMutationAndEvents(jobId, mutation, events) {
+        let committed;
+        new DurableWorkflowPublication(this.db).commitTransitionAndEvents(() => {
+            const row = this.db.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(jobId);
+            if (!row?.json)
+                throw new Error(`Unknown Job ${jobId}`);
+            const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
+            current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
+            const next = mutation(current);
+            if (next.id !== jobId)
+                throw new Error('Cannot change authoritative Job identity');
+            if (next.state !== current.state && ['COMPLETED', 'CANCELLED'].includes(current.state))
+                throw new Error('Cannot overwrite terminal authoritative state');
+            const beforeTime = Date.parse(current.updatedAt);
+            if (Number.isFinite(beforeTime) && !(Date.parse(next.updatedAt) > beforeTime))
+                next.updatedAt = new Date(beforeTime + 1).toISOString();
+            this.write(next, new Date().toISOString());
+            committed = structuredClone(next);
+        }, () => events.map(item => ({ key: item.key, eventId: item.event.type, payload: item.event })));
+        return committed;
+    }
+    async commitWorkflowMutationWithEvents(jobId, mutation) {
+        const emitted = [];
+        return this.commitWorkflowMutationAndEvents(jobId, current => {
+            const result = mutation(current);
+            emitted.push(...result.events);
+            return result.next;
+        }, emitted);
+    }
+    async reserveWorkflowLaunch(identity) {
+        const decision = dispatchWorkflowGraphSync(taskLifecycleV2, { initialState: 'QUEUED', input: { transport: 'event', id: 'dispatch', payload: {} }, durableFacts: { dispatchable: true } });
+        if (!decision.accepted || !decision.actions.some(action => action.id === 'effect.launchTask'))
+            throw new Error('canonical Task Graph did not authorize launch');
+        return new WorkflowActionLedger(this.db).reserveForClaimedDispatch({
+            ...identity, generation: identity.attempt, graph: 'task-lifecycle.mmd', actionId: 'effect.launchTask',
+            attemptKey: `${identity.jobId}/${identity.taskId}/${identity.executionId}/launch`,
+        }, decision, 'worker.started');
+    }
+    async verifyWorkflowLaunch(jobId, taskId, executionId) {
+        const ledger = new WorkflowActionLedger(this.db);
+        const key = `${jobId}/${taskId}/${executionId}/launch`;
+        if (!ledger.trace(key))
+            return 'Unknown';
+        const row = this.db.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(jobId);
+        if (!row?.json)
+            return 'Unknown';
+        const job = JSON.parse(row.json);
+        const task = job.tasks.find(t => t.id === taskId);
+        // Only the canonical task.started transition persists RUNNING. The action
+        // did not set RUNNING at claim-time; therefore this state is a durable
+        // postcondition correlated to the exact execution generation.
+        const started = task?.state === 'RUNNING' && task.executionId === executionId
+            && !!task.leaseOwner && task.executionHistory?.some(e => e.id === executionId && e.attempt === task.attempt);
+        const registry = new ConditionRegistry([booleanCondition('worker.started')]);
+        return ledger.verify(key, registry, { 'worker.started': started ? true : undefined }).outcome;
+    }
+    async drainWorkflowEvents(publish) {
+        return new DurableWorkflowPublication(this.db).replay(async (item) => {
+            const event = item.payload;
+            if (!event || event.event_id !== item.key || event.type !== item.eventId)
+                throw new Error('Outbox event identity mismatch');
+            await publish(event);
+        });
+    }
+    async mutateJob(jobId, mutation) {
+        this.db.exec('BEGIN IMMEDIATE');
+        try {
+            const row = this.db.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(jobId);
+            if (!row?.json)
+                throw new Error(`Local authoritative Job ${jobId} not found`);
+            const current = normalizeLegacyJobPrNumber(JSON.parse(row.json));
+            current.tasks = current.tasks.map(task => normalizeLegacyTaskLifecycle(task, current.executionMode));
+            const before = JSON.stringify(current);
+            const next = mutation(current);
+            if (next.id !== jobId)
+                throw new Error('Cannot change authoritative Job identity');
+            if (JSON.stringify(next) !== before && next.updatedAt <= JSON.parse(before).updatedAt) {
+                next.updatedAt = monotonicJobTimestamp(JSON.parse(before).updatedAt);
+            }
+            this.write(next, new Date().toISOString());
+            this.db.exec('COMMIT');
+            return structuredClone(next);
+        }
+        catch (error) {
+            this.db.exec('ROLLBACK');
+            throw error;
+        }
+    }
+}

package/dist/relayd.js

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import { escalationNotifierFromEnvironment } from './escalation-notifier.js';
 import { consumeMethodInjection, withTestInjection } from './e2e-injection.js';
 import { once } from 'node:events';
 import { spawnOwnedProcess } from './owned-process.js';
@@ -151,7 +152,7 @@ export async function runDaemon(argv) {
         return dashboardManagedJobs(entries.filter(item => !['COMPLETED', 'CANCELLED'].includes(item.job.state) || counts.get(item.job.id) !== 1));
     };
     cachedOverviews = await inventoryFromLocal();
-    const makeReconciler = (store, maxConcurrent) => new Reconciler(store, { noGitHub, owner: `relayd-${process.pid}`, maxConcurrent, leaseMs: Number(value(argv, '--lease-ms', '300000')), adapters: [new CodexAdapter(value(argv, '--codex', 'codex')), new ChatGptAdapter()], continuations: [new CodexThreadContinuation(value(argv, '--codex', 'codex')), new CommandContinuation(), new WebhookContinuation()], planner: runtimePlanner(argv), usageSnapshotProvider: () => usageRegistry.snapshot(), runtimeSummaryProvider: () => ({ chatgptRunning: summarizeRuntime(cachedOverviews.map(item => item.job)).workers.chatgpt }), decisionEngine: executionDecisionEngine(), decisionMode: decisionMode(), decisionAuditStore, eventBus: bus, reviewer: runtimeReviewer(argv), projectPathForJob: async (job) => {
+    const makeReconciler = (store, maxConcurrent) => new Reconciler(store, { noGitHub, owner: `relayd-${process.pid}`, maxConcurrent, leaseMs: Number(value(argv, '--lease-ms', '300000')), adapters: [new CodexAdapter(value(argv, '--codex', 'codex')), new ChatGptAdapter()], continuations: [new CodexThreadContinuation(value(argv, '--codex', 'codex')), new CommandContinuation(), new WebhookContinuation()], planner: runtimePlanner(argv), usageSnapshotProvider: () => usageRegistry.snapshot(), runtimeSummaryProvider: () => ({ chatgptRunning: summarizeRuntime(cachedOverviews.map(item => item.job)).workers.chatgpt }), decisionEngine: executionDecisionEngine(), decisionMode: decisionMode(), decisionAuditStore, eventBus: bus, escalationNotifier: escalationNotifierFromEnvironment(), reviewer: runtimeReviewer(argv), projectPathForJob: async (job) => {
             const canonical = repositoryPaths.get(job.repository) ?? await resolveManagedProjectPath(job.repository, explicitProjectPath, workspaceRoot);
             if (!jobRequiresManagedWorktree(job))
                 return resolveRepositoryProjectPath(canonical, job.projectPath);

package/dist/task-recovery.js

@@ -1,4 +1,5 @@
 import { randomUUID } from 'node:crypto';
+import { eventFor } from './events.js';
 import { tryTransitionTaskEvent } from './scheduler.js';
 import { dispatchTaskResolutionSync, materializeResolution } from './workflow/task-resolution.js';
 function activeRecovery(job, original) {
@@ -23,35 +24,47 @@ function validateRequest(request) {
 export async function requestTroubleshooterEscalation(store, jobId, recoveryTaskId, request, now = new Date().toISOString()) {
     validateRequest(request);
     let result;
+    let requestedEvent;
     const apply = (job) => {
         const recovery = job.tasks.find(task => task.id === recoveryTaskId);
-        if (!recovery || recovery.agentName !== 'troubleshooter' || !recovery.parentTaskId)
+        if (!recovery || recovery.agentName !== 'troubleshooter' || (!recovery.parentTaskId && !recovery.recoveryPlanId))
             throw new Error(`Task ${recoveryTaskId} is not a Troubleshooter recovery`);
         if (job.state === 'CANCELLED' || job.state === 'COMPLETED')
             throw new Error(`Job ${job.id} is terminal`);
-        const original = job.tasks.find(task => task.id === recovery.parentTaskId);
-        if (!original || original.state !== 'FAILED')
-            throw new Error(`Troubleshooter ${recoveryTaskId} has no current FAILED original Task`);
+        const original = recovery.parentTaskId ? job.tasks.find(task => task.id === recovery.parentTaskId) : undefined;
+        const originalPlan = recovery.recoveryPlanId ? job.plans?.find(plan => plan.id === recovery.recoveryPlanId) : undefined;
+        if ((!original || original.state !== 'FAILED') && (!originalPlan || originalPlan.status !== 'FAILED'))
+            throw new Error(`Troubleshooter ${recoveryTaskId} has no current FAILED original Task or Plan`);
         if (recovery.escalation?.status === 'pending' && recovery.state === 'BLOCKED') {
             result = { task: structuredClone(recovery), escalation: structuredClone(recovery.escalation), duplicate: true };
             return job;
         }
         if (!['QUEUED', 'RUNNING', 'FAILED'].includes(recovery.state))
             throw new Error(`Troubleshooter ${recoveryTaskId} cannot escalate from ${recovery.state}`);
-        const proposal = { path: 'escalation_gate', targetTaskId: original.id, rootCause: request.context.trim(), solution: request.requiredAction.trim() };
-        const decision = dispatchTaskResolutionSync(original, proposal, 'troubleshooter', {
-            createEscalationGate: () => { original.resolution = materializeResolution(original, proposal, 'troubleshooter', now); },
-        });
-        if (!decision.accepted)
-            throw new Error(`Task resolution graph rejected escalation for ${original.id}`);
-        if (!tryTransitionTaskEvent(recovery, 'task.blocked', { recoverableExecution: true }).accepted)
+        if (original) {
+            const proposal = { path: 'escalation_gate', targetTaskId: original.id, rootCause: request.context.trim(), solution: request.requiredAction.trim() };
+            const decision = dispatchTaskResolutionSync(original, proposal, 'troubleshooter', {
+                createEscalationGate: () => { original.resolution = materializeResolution(original, proposal, 'troubleshooter', now); },
+            });
+            if (!decision.accepted)
+                throw new Error(`Task resolution graph rejected escalation for ${original.id}`);
+        }
+        // A Plan is an independent durable entity, never a synthetic Task. The
+        // Plan-failure guard is owned by planner-orchestration; only the real
+        // Troubleshooter Task undergoes the task-lifecycle BLOCKED transition.
+        if (!tryTransitionTaskEvent(recovery, recovery.state === 'QUEUED' ? 'block' : 'task.blocked', { recoverableExecution: true }).accepted)
             throw new Error(`Troubleshooter ${recoveryTaskId} cannot be blocked`);
         const escalation = {
-            id: randomUUID(), originalTaskId: original.id, originalExecutionId: original.executionId ?? undefined,
+            id: randomUUID(), originalTaskId: original?.id, originalPlanId: originalPlan?.id,
+            originalExecutionId: original?.executionId ?? undefined,
             recoveryExecutionId: recovery.executionId ?? undefined, requiredAction: request.requiredAction.trim(),
             resolutionCondition: request.resolutionCondition.trim(), context: request.context.trim(), status: 'pending',
-            requestedAt: now, notification: request.notification ?? null,
+            requestedAt: now, notification: { status: 'pending', channel: typeof request.notification?.channel === 'string' ? request.notification.channel : 'relay-event', requestedAt: now },
         };
+        const event = eventFor(job.id, recovery.id, original?.id ?? null, 'escalation.requested', 'blocked', `Human action required: ${escalation.requiredAction}`, 'user', { escalationId: escalation.id, originalTaskId: original?.id, originalPlanId: originalPlan?.id, requiredAction: escalation.requiredAction,
+            resolutionCondition: escalation.resolutionCondition, notificationStatus: 'pending' }, { agent: 'relay', component: 'agents-relay' });
+        event.event_id = `escalation/${escalation.id}`;
+        requestedEvent = event;
         recovery.escalation = escalation;
         recovery.error = `Waiting for human action: ${escalation.requiredAction}`;
         recovery.failureReason = 'human_action_required';
@@ -61,8 +74,14 @@ export async function requestTroubleshooterEscalation(store, jobId, recoveryTask
         job.updatedAt = now;
         return job;
     };
-    const job = store.mutateJob ? await store.mutateJob(jobId, apply) : await store.load(jobId).then(async (current) => { apply(current); for (const task of current.tasks)
-        await store.saveTask(task); await store.saveJob(current); return current; });
+    const job = store.commitWorkflowMutationWithEvents
+        ? await store.commitWorkflowMutationWithEvents(jobId, current => {
+            const next = apply(current);
+            return { next, events: requestedEvent ? [{ key: requestedEvent.event_id, event: requestedEvent }] : [] };
+        })
+        : store.mutateJob ? await store.mutateJob(jobId, apply)
+            : await store.load(jobId).then(async (current) => { apply(current); for (const task of current.tasks)
+                await store.saveTask(task); await store.saveJob(current); return current; });
     if (!result)
         throw new Error('Escalation did not produce a durable result');
     return { job, ...result };
@@ -103,3 +122,34 @@ export function hasPendingTroubleshooterEscalation(job, original) {
     const recovery = activeRecovery(job, original);
     return recovery?.state === 'BLOCKED' && recovery.escalation?.status === 'pending';
 }
+/** Trusted notification adapter receipt. Publishing to the bus does NOT count as
+ * delivery to a human. Only an actual downstream delivery ID can set delivered. */
+export async function recordEscalationNotificationDelivery(store, jobId, recoveryTaskId, escalationId, receipt) {
+    if (!receipt.channel.trim() || !receipt.deliveryId.trim() || !Number.isFinite(Date.parse(receipt.deliveredAt)))
+        throw Error('Escalation delivery requires a real channel, provider receipt ID and timestamp');
+    if (!store.mutateJob)
+        throw Error('Escalation receipt requires authoritative SQLite mutation');
+    let confirmed;
+    await store.mutateJob(jobId, job => {
+        const recovery = job.tasks.find(task => task.id === recoveryTaskId && task.agentName === 'troubleshooter');
+        const escalation = recovery?.escalation;
+        if (!escalation || escalation.id !== escalationId)
+            throw Error('Unknown escalation delivery identity');
+        if (escalation.status !== 'pending')
+            throw Error('Escalation no longer pending');
+        const previous = escalation.notification;
+        if (previous?.status === 'delivered') {
+            if (previous.deliveryId !== receipt.deliveryId || previous.channel !== receipt.channel)
+                throw Error('Escalation delivery receipt conflict');
+            confirmed = structuredClone(escalation);
+            return job;
+        }
+        escalation.notification = { status: 'delivered', channel: receipt.channel, deliveryId: receipt.deliveryId, deliveredAt: receipt.deliveredAt };
+        recovery.updatedAt = receipt.deliveredAt;
+        confirmed = structuredClone(escalation);
+        return job;
+    });
+    if (!confirmed)
+        throw Error('Escalation delivery receipt not committed');
+    return confirmed;
+}

package/dist/troubleshooter-fallback.js

@@ -8,7 +8,10 @@ export async function fallbackFailedTroubleshooters(store, jobId) {
     const queued = [];
     for (const recovery of job.tasks.filter(task => task.agentName === 'troubleshooter' && task.state === 'FAILED' && task.attempt >= task.maxAttempts && !task.input.includes(MARKER))) {
         const original = job.tasks.find(task => task.id === recovery.parentTaskId);
-        if (!original || original.state !== 'FAILED' || (original.resolution?.path === 'same_job_dependency' && ['waiting', 'active'].includes(original.resolution.state)) || !RUNTIME_FAILURE.test(`${recovery.failureReason ?? ''} ${recovery.error ?? ''}`))
+        const plan = job.plans?.find(item => item.id === recovery.recoveryPlanId);
+        if ((!original || original.state !== 'FAILED') && (!plan || plan.status !== 'FAILED'))
+            continue;
+        if ((original?.resolution?.path === 'same_job_dependency' && ['waiting', 'active'].includes(original.resolution.state)) || !RUNTIME_FAILURE.test(`${recovery.failureReason ?? ''} ${recovery.error ?? ''}`))
             continue;
         const update = (task) => {
             if (task.state !== 'FAILED' || task.input.includes(MARKER))
@@ -48,12 +51,17 @@ export async function escalateExhaustedTroubleshooters(store, jobId) {
     const escalated = [];
     for (const recovery of job.tasks.filter(task => task.agentName === 'troubleshooter' && task.state === 'FAILED' && task.attempt >= task.maxAttempts && !task.escalation)) {
         const original = job.tasks.find(task => task.id === recovery.parentTaskId);
-        if (!original || original.state !== 'FAILED' || (original.resolution?.path === 'same_job_dependency' && ['waiting', 'active'].includes(original.resolution.state)))
+        const plan = job.plans?.find(item => item.id === recovery.recoveryPlanId);
+        if ((!original || original.state !== 'FAILED') && (!plan || plan.status !== 'FAILED'))
+            continue;
+        if (original?.resolution?.path === 'same_job_dependency' && ['waiting', 'active'].includes(original.resolution.state))
             continue;
-        const context = `Automatic Troubleshooter ${recovery.id} could not produce a verified resolution after ${recovery.attempt} attempts. Last failure: ${recovery.error ?? recovery.failureReason ?? 'unknown'}. Original Task ${original.id}: ${original.error ?? original.failureReason ?? 'failed'}.`;
+        const subject = original ? `Task ${original.id}` : `Plan ${plan.id}`;
+        const failure = original?.error ?? original?.failureReason ?? plan?.error ?? 'unknown';
+        const context = `Automatic Troubleshooter ${recovery.id} could not produce a verified resolution after ${recovery.attempt} attempts. Last failure: ${recovery.error ?? recovery.failureReason ?? 'unknown'}. Original ${subject}: ${failure}.`;
         const result = await requestTroubleshooterEscalation(store, jobId, recovery.id, {
-            requiredAction: `Inspect Task ${original.id} and Troubleshooter ${recovery.id}; correct provider/runtime or supply a concrete recovery decision and supporting evidence.`,
-            resolutionCondition: `User supplies a corrective action or verified recovery instructions for Task ${original.id}.`, context,
+            requiredAction: `Inspect ${subject} and Troubleshooter ${recovery.id}; correct provider/runtime or supply a concrete recovery decision and supporting evidence.`,
+            resolutionCondition: `User supplies a corrective action or verified recovery instructions for ${subject}.`, context,
         });
         if (!result.duplicate)
             escalated.push(recovery.id);

package/dist/workflow/action-ledger.js

@@ -0,0 +1,75 @@
+export class WorkflowActionLedger {
+    database;
+    constructor(database) {
+        this.database = database;
+        database.exec(`CREATE TABLE IF NOT EXISTS workflow_action_trace (
+      attempt_key TEXT PRIMARY KEY, execution_id TEXT NOT NULL, generation INTEGER NOT NULL,
+      graph TEXT NOT NULL, action_id TEXT NOT NULL, condition_id TEXT NOT NULL,
+      outcome TEXT NOT NULL CHECK(outcome IN ('Unknown','Verified','Failed')),
+      observed_at TEXT NOT NULL
+    )`);
+    }
+    reserve(identity, decision, conditionId) {
+        if (!identity.executionId || !identity.attemptKey || !conditionId || !Number.isSafeInteger(identity.generation) || identity.generation < 0)
+            throw new Error('invalid action identity');
+        if (!decision.accepted || !decision.actions.some(action => action.id === identity.actionId))
+            throw new Error('action not authorized by graph decision');
+        const statement = this.database.prepare(`INSERT OR IGNORE INTO workflow_action_trace
+      (attempt_key,execution_id,generation,graph,action_id,condition_id,outcome,observed_at)
+      VALUES (?,?,?,?,?,?,'Unknown',?)`);
+        const inserted = statement.run(identity.attemptKey, identity.executionId, identity.generation, identity.graph, identity.actionId, conditionId, new Date().toISOString()).changes === 1;
+        if (!inserted) {
+            const old = this.trace(identity.attemptKey);
+            if (!old || old.executionId !== identity.executionId || old.generation !== identity.generation || old.graph !== identity.graph || old.actionId !== identity.actionId || old.conditionId !== conditionId)
+                throw new Error('attempt key collision: inconsistent action identity');
+        }
+        return inserted;
+    }
+    /** Authoritative, fail-closed reservation against the durable job_snapshot row. */
+    reserveForRunningTask(identity, decision, conditionId, now = new Date().toISOString(), allowedState = 'RUNNING') {
+        this.database.exec('BEGIN IMMEDIATE');
+        try {
+            const row = this.database.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(identity.jobId);
+            if (!row?.json)
+                throw new Error('authoritative job snapshot unavailable');
+            const job = JSON.parse(row.json);
+            const task = job.tasks?.find(item => item.id === identity.taskId);
+            if (!task || task.state !== allowedState || task.executionId !== identity.executionId || task.leaseOwner !== identity.leaseOwner || task.attempt !== identity.attempt || task.attempt !== identity.generation || !task.executionHistory?.some(entry => entry.id === identity.executionId && entry.attempt === identity.generation) || Boolean(job.pause) || !task.leaseExpiresAt || !Number.isFinite(Date.parse(task.leaseExpiresAt)) || !Number.isFinite(Date.parse(now)) || Date.parse(task.leaseExpiresAt) <= Date.parse(now)) {
+                throw new Error('stale execution, owner, attempt or lease: action fenced');
+            }
+            const created = this.reserve(identity, decision, conditionId);
+            this.database.exec('COMMIT');
+            return created;
+        }
+        catch (error) {
+            this.database.exec('ROLLBACK');
+            throw error;
+        }
+    }
+    /** A dispatch launch is authorized while the Task is QUEUED with a claimed
+     * execution lease; task.started later transitions it to RUNNING. */
+    reserveForClaimedDispatch(identity, decision, conditionId) {
+        if (!decision.accepted || decision.source !== 'QUEUED' || decision.target !== 'QUEUED'
+            || identity.actionId !== 'effect.launchTask')
+            throw Error('dispatch reservation requires canonical QUEUED Graph launch decision');
+        return this.reserveForRunningTask(identity, decision, conditionId, new Date().toISOString(), 'QUEUED');
+    }
+    trace(attemptKey) {
+        const row = this.database.prepare('SELECT * FROM workflow_action_trace WHERE attempt_key=?').get(attemptKey);
+        if (!row)
+            return null;
+        return { attemptKey: String(row.attempt_key), executionId: String(row.execution_id), generation: Number(row.generation), graph: String(row.graph), actionId: String(row.action_id), conditionId: String(row.condition_id), outcome: row.outcome, observedAt: String(row.observed_at) };
+    }
+    verify(attemptKey, conditions, facts) {
+        const existing = this.trace(attemptKey);
+        if (!existing)
+            throw new Error('unreserved action attempt');
+        if (existing.outcome !== 'Unknown')
+            return existing;
+        const outcome = conditions.check(existing.conditionId, facts);
+        // Unknown is intentionally retained: no second executor may treat it as a new attempt.
+        this.database.prepare('UPDATE workflow_action_trace SET outcome=?, observed_at=? WHERE attempt_key=? AND outcome=?')
+            .run(outcome, new Date().toISOString(), attemptKey, 'Unknown');
+        return this.trace(attemptKey);
+    }
+}

package/dist/workflow/authoritative-conditions.js

@@ -0,0 +1,38 @@
+import { completionGateSatisfied } from '../scheduler.js';
+export function verifyAuthoritativeCondition(db, jobId, predicate, taskId, executionId) {
+    if (!jobId)
+        throw new Error('job ID required');
+    const row = db.prepare('SELECT json FROM job_snapshot WHERE job_id=?').get(jobId);
+    if (!row?.json)
+        return 'Unknown';
+    const job = JSON.parse(row.json);
+    if (predicate === 'job.objective-verified')
+        return completionGateSatisfied(job) ? 'Verified' : 'Failed';
+    if (predicate === 'job.completed')
+        return job.state === 'COMPLETED' && completionGateSatisfied(job) ? 'Verified' : 'Failed';
+    const task = job.tasks.find(item => item.id === taskId);
+    if (!task)
+        return 'Unknown';
+    switch (predicate) {
+        case 'task.queued': return task.state === 'QUEUED' ? 'Verified' : 'Failed';
+        case 'task.succeeded': return task.state === 'SUCCEEDED' ? 'Verified' : 'Failed';
+        case 'task.failed': return task.state === 'FAILED' ? 'Verified' : 'Failed';
+        case 'task.routing-cleared': return task.executionId === null && !task.leaseOwner ? 'Verified' : 'Failed';
+        case 'task.execution-stopped': {
+            if (!executionId)
+                return 'Unknown';
+            if (task.executionId === executionId)
+                return 'Failed';
+            const history = task.executionHistory?.find(entry => entry.id === executionId);
+            if (!history)
+                return 'Unknown';
+            // A changed execution identity cannot alone prove process termination.
+            // Require authoritative ended process evidence, not just a lease replacement.
+            const table = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='execution_process'").get();
+            if (!table)
+                return 'Unknown';
+            const record = db.prepare('SELECT ended_at FROM execution_process WHERE job_id=? AND task_id=? AND execution_id=?').get(jobId, task.id, executionId);
+            return record?.ended_at ? 'Verified' : 'Unknown';
+        }
+    }
+}