@pennixrv/trellis 0.7.1-beta.14 → 0.7.1-beta.16

package/dist/commands/teamwork/coordinator.js

@@ -1,8 +1,10 @@
+import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
+import { isDeepStrictEqual } from "node:util";
 import { isTaskPhaseId, parseTaskWorkflowState, } from "@pennixrv/trellis-core/task";
 import { assertCoordinationTransition, readTeamworkEvents, readCoordinationState, recordCoordinationEvent, } from "@pennixrv/trellis-core/teamwork";
-import { cleanupSettledTeamworkArtifacts, readTeammateReport, writeTeammateQueueFact, writeTeammateClaim, writeTeammateDisposition, validateTeamworkPlan, } from "./artifact.js";
+import { TeammateReportReadError, cleanupSettledTeamworkArtifacts, readTeammateReport, writeTeammateQueueFact, writeTeammateClaim, writeTeammateDisposition, validateTeamworkPlan, } from "./artifact.js";
 import { assertTeammateAdapterContract, assertTeammateHostCapability, } from "./teammate-adapters.js";
 import { effectiveTeammateCapacity, resolveTeammateRuntimeConfig, } from "./teammate-config.js";
 export function assertTeamworkPlanCurrent(plan, taskRoot) {
@@ -35,6 +37,165 @@ export function assertTeamworkPlanCurrent(plan, taskRoot) {
         throw new Error("Teamwork plan does not match the current active task phase and revisions");
     }
 }
+function terminalReceiptFromProjection(plan, projection) {
+    const mismatch = (reason) => {
+        throw new Error(`batch '${plan.batchId}' ${reason}`);
+    };
+    if (projection.issues.length > 0)
+        mismatch("has coordination projection issues");
+    const batch = projection.batches.find((entry) => entry.batchId === plan.batchId);
+    if (!batch)
+        return mismatch("does not match the frozen plan");
+    if (batch.planHash !== plan.planHash)
+        return mismatch("does not match the frozen plan");
+    if (batch.state !== "ALL_ACCEPTED" && batch.state !== "SETTLED_WITH_FAILURE")
+        mismatch("is not terminal");
+    const effectiveCapacity = batch.effectiveCapacity;
+    if (typeof effectiveCapacity !== "number" ||
+        !Number.isSafeInteger(effectiveCapacity) ||
+        effectiveCapacity < 1)
+        return mismatch("has no valid effective capacity");
+    if (projection.leases.some((lease) => lease.batchId === plan.batchId))
+        mismatch("still owns capacity leases");
+    const projectedAssignments = projection.assignments.filter((assignment) => assignment.batchId === plan.batchId);
+    if (projectedAssignments.length !== plan.assignments.length ||
+        new Set(projectedAssignments.map((assignment) => assignment.assignmentId))
+            .size !== plan.assignments.length)
+        mismatch("does not contain the exact frozen assignment set");
+    const assignmentById = new Map(projectedAssignments.map((assignment) => [
+        assignment.assignmentId,
+        assignment,
+    ]));
+    const planAssignmentIds = new Set(plan.assignments.map((assignment) => assignment.assignmentId));
+    const attempts = projection.attempts.filter((attempt) => attempt.batchId === plan.batchId);
+    const teammates = projection.teammates.filter((teammate) => teammate.batchId === plan.batchId);
+    if (attempts.length !== teammates.length)
+        mismatch("has unmatched provider attempts and teammates");
+    for (const attempt of attempts) {
+        if (!attempt.assignmentId ||
+            !planAssignmentIds.has(attempt.assignmentId) ||
+            !attempt.teammateId ||
+            attempt.terminal !== true ||
+            !["REPORT_SUBMITTED", "FAILED", "FINAL_FAILED"].includes(attempt.state))
+            mismatch("has an unresolved provider attempt");
+        const teammate = teammates.find((entry) => entry.attemptId === attempt.attemptId &&
+            entry.teammateId === attempt.teammateId);
+        if (teammate?.terminal !== true ||
+            (teammate?.state !== "REPORT_SUBMITTED" && teammate?.state !== "FAILED"))
+            mismatch("has an unresolved teammate terminal fact");
+    }
+    const assignments = plan.assignments.map((assignment) => {
+        const projected = assignmentById.get(assignment.assignmentId);
+        if (!projected ||
+            (projected.state !== "ACCEPTED" && projected.state !== "FINAL_FAILED"))
+            return mismatch(`assignment '${assignment.assignmentId}' is not settled`);
+        if (projected.state === "ACCEPTED") {
+            const acceptedAttempt = attempts.find((attempt) => attempt.attemptId === projected.attemptId &&
+                attempt.assignmentId === assignment.assignmentId);
+            if (acceptedAttempt?.state !== "REPORT_SUBMITTED")
+                return mismatch(`assignment '${assignment.assignmentId}' has no accepted attempt`);
+        }
+        return {
+            assignmentId: assignment.assignmentId,
+            status: projected.state,
+        };
+    });
+    const status = assignments.every((item) => item.status === "ACCEPTED")
+        ? "ALL_ACCEPTED"
+        : "SETTLED_WITH_FAILURE";
+    if (batch.state !== status)
+        mismatch("terminal state disagrees with its assignment dispositions");
+    const retryCounts = Object.fromEntries(plan.assignments.flatMap((assignment) => {
+        const count = assignmentById.get(assignment.assignmentId)?.retryCount;
+        if (count === undefined)
+            return mismatch(`assignment '${assignment.assignmentId}' is missing`);
+        const started = attempts.some((attempt) => attempt.assignmentId === assignment.assignmentId);
+        return started || count > 0 ? [[assignment.assignmentId, count]] : [];
+    }));
+    const evaluationGroups = [
+        ...new Set(plan.assignments
+            .map((assignment) => assignment.evaluationGroup?.id)
+            .filter((id) => id !== undefined)),
+    ].map((id) => ({
+        id,
+        status: plan.assignments
+            .filter((assignment) => assignment.evaluationGroup?.id === id)
+            .every((assignment) => assignmentById.get(assignment.assignmentId)?.state === "ACCEPTED")
+            ? "COMPLETE"
+            : "INCOMPLETE",
+    }));
+    return {
+        schemaVersion: 1,
+        batchId: plan.batchId,
+        status,
+        assignments,
+        capacity: effectiveCapacity,
+        retryCounts,
+        evaluationGroups,
+    };
+}
+function reconciliationReceipt(plan, projection, capacity) {
+    return {
+        schemaVersion: 1,
+        batchId: plan.batchId,
+        status: "RECONCILIATION_REQUIRED",
+        assignments: plan.assignments.map((item) => {
+            const current = projection.assignments.find((entry) => entry.batchId === plan.batchId &&
+                entry.assignmentId === item.assignmentId);
+            return {
+                assignmentId: item.assignmentId,
+                status: current?.state === "ACCEPTED" || current?.state === "FINAL_FAILED"
+                    ? current.state
+                    : "UNKNOWN",
+            };
+        }),
+        capacity: projection.batches.find((entry) => entry.batchId === plan.batchId)
+            ?.effectiveCapacity ?? capacity,
+        retryCounts: Object.fromEntries(projection.assignments
+            .filter((entry) => entry.batchId === plan.batchId)
+            .map((entry) => [entry.assignmentId, entry.retryCount])),
+    };
+}
+async function recordRecoveryReconciliation(input) {
+    const { plan, cwd, by, existingState, projection, capacity, reason } = input;
+    if (existingState !== "RECONCILIATION_REQUIRED" &&
+        existingState !== "ALL_ACCEPTED" &&
+        existingState !== "SETTLED_WITH_FAILURE") {
+        await appendTransition({
+            sessionId: plan.sessionId,
+            cwd,
+            by,
+            batchId: plan.batchId,
+            entity: "batch",
+            id: plan.batchId,
+            state: "RECONCILIATION_REQUIRED",
+            states: new Map([[plan.batchId, existingState]]),
+            planHash: plan.planHash,
+            payload: { recoveredAfterRestart: true },
+        });
+    }
+    const receipt = reconciliationReceipt(plan, projection, capacity);
+    await appendAction({
+        sessionId: plan.sessionId,
+        cwd,
+        by,
+        batchId: plan.batchId,
+        entity: "batch",
+        action: "reconciliation_required",
+        reason,
+    });
+    await appendAction({
+        sessionId: plan.sessionId,
+        cwd,
+        by,
+        batchId: plan.batchId,
+        entity: "batch",
+        action: "receipt_recorded",
+        id: `${plan.batchId}:RECONCILIATION_REQUIRED`,
+        payload: { ...receipt, reason },
+    });
+    return receipt;
+}
 async function recoverExistingBatch(input) {
     const projection = await readCoordinationState({
         sessionId: input.plan.sessionId,
@@ -50,64 +211,54 @@ async function recoverExistingBatch(input) {
         sessionId: input.plan.sessionId,
         cwd: input.cwd,
     });
-    const saved = events.find((event) => event.kind === "coordination" &&
-        event.batchId === input.plan.batchId &&
-        event.action === "receipt_recorded" &&
-        event.payload &&
-        (event.payload.status === "ALL_ACCEPTED" ||
-            event.payload.status === "SETTLED_WITH_FAILURE"));
-    if (saved?.kind === "coordination" && saved.payload) {
-        return saved.payload;
-    }
-    const states = new Map([
-        [input.plan.batchId, existing.state],
-    ]);
-    if (existing.state !== "RECONCILIATION_REQUIRED") {
-        await appendTransition({
-            sessionId: input.plan.sessionId,
-            by: input.by,
-            batchId: input.plan.batchId,
-            entity: "batch",
-            id: input.plan.batchId,
-            state: "RECONCILIATION_REQUIRED",
-            states,
-            planHash: input.plan.planHash,
-            payload: { recoveredAfterRestart: true },
-        });
+    if (existing.state === "ALL_ACCEPTED" ||
+        existing.state === "SETTLED_WITH_FAILURE") {
+        let receipt;
+        try {
+            receipt = terminalReceiptFromProjection(input.plan, projection);
+        }
+        catch (error) {
+            return recordRecoveryReconciliation({
+                ...input,
+                existingState: existing.state,
+                projection,
+                reason: error instanceof Error ? error.message : String(error),
+            });
+        }
+        const receipts = events.filter((event) => event.kind === "coordination" &&
+            event.batchId === input.plan.batchId &&
+            event.action === "receipt_recorded");
+        if (receipts.length > 1 ||
+            (receipts.length === 1 &&
+                !isDeepStrictEqual(receipts[0]?.payload, receipt))) {
+            return recordRecoveryReconciliation({
+                ...input,
+                existingState: existing.state,
+                projection,
+                reason: "saved terminal receipt disagrees with durable coordination facts",
+            });
+        }
+        if (receipts.length === 0) {
+            await appendAction({
+                sessionId: input.plan.sessionId,
+                cwd: input.cwd,
+                by: input.by,
+                batchId: input.plan.batchId,
+                entity: "batch",
+                action: "receipt_recorded",
+                id: `${input.plan.batchId}:${receipt.status}`,
+                payload: { ...receipt },
+            });
+        }
+        await assertTeamworkReceiptDurable(input.plan, receipt, input.cwd);
+        return receipt;
     }
-    const assignments = input.plan.assignments.map((item) => {
-        const current = projection.assignments.find((entry) => entry.batchId === input.plan.batchId &&
-            entry.assignmentId === item.assignmentId);
-        return {
-            assignmentId: item.assignmentId,
-            status: current?.state === "ACCEPTED" || current?.state === "FINAL_FAILED"
-                ? current.state
-                : "UNKNOWN",
-        };
-    });
-    const retryCounts = Object.fromEntries(projection.assignments
-        .filter((entry) => entry.batchId === input.plan.batchId)
-        .map((entry) => [entry.assignmentId, entry.retryCount]));
-    const receipt = {
-        schemaVersion: 1,
-        batchId: input.plan.batchId,
-        status: "RECONCILIATION_REQUIRED",
-        assignments,
-        capacity: existing.effectiveCapacity ?? input.capacity,
-        retryCounts,
-    };
-    await recordCoordinationEvent({
-        sessionId: input.plan.sessionId,
-        cwd: input.cwd,
-        by: input.by,
-        batchId: input.plan.batchId,
-        entity: "batch",
-        action: "receipt_recorded",
-        receiptId: `${input.plan.batchId}:RECONCILIATION_REQUIRED`,
-        payload: { ...receipt, reason: "recovered batch requires reconciliation" },
-        idempotencyKey: `${input.plan.batchId}:RECONCILIATION_REQUIRED:receipt`,
+    return recordRecoveryReconciliation({
+        ...input,
+        existingState: existing.state,
+        projection,
+        reason: "recovered nonterminal batch requires reconciliation",
     });
-    return receipt;
 }
 function transitionState(states, key, next, entity) {
     assertCoordinationTransition(entity, states.get(key), next);
@@ -117,6 +268,7 @@ async function appendTransition(input) {
     transitionState(input.states, input.id, input.state, input.entity);
     await recordCoordinationEvent({
         sessionId: input.sessionId,
+        ...(input.cwd ? { cwd: input.cwd } : {}),
         by: input.by,
         batchId: input.batchId,
         entity: input.entity,
@@ -139,6 +291,7 @@ async function appendTransition(input) {
 async function appendAction(input) {
     await recordCoordinationEvent({
         sessionId: input.sessionId,
+        ...(input.cwd ? { cwd: input.cwd } : {}),
         by: input.by,
         batchId: input.batchId,
         entity: input.entity,
@@ -203,19 +356,8 @@ export async function runTeamworkCoordinator(input) {
         capacity,
     });
     let guardHeld = true;
-    let recovered;
-    try {
-        recovered = await recoverExistingBatch({ plan, cwd, by, capacity });
-    }
-    catch (error) {
-        await input.guard.release({
-            batchId: plan.batchId,
-            planHash: plan.planHash,
-            capacity,
-        });
-        guardHeld = false;
-        throw error;
-    }
+    // Failed recovery cannot prove that an older provider process exited.
+    const recovered = await recoverExistingBatch({ plan, cwd, by, capacity });
     if (recovered) {
         if (recovered.status !== "RECONCILIATION_REQUIRED") {
             await input.guard.release({
@@ -324,8 +466,9 @@ export async function runTeamworkCoordinator(input) {
     const active = new Map();
     const waits = new Map();
     const final = new Map();
-    const acceptedAttempts = new Map();
+    const retainedReports = new Map();
     const retryCounts = {};
+    let admissionClosed = false;
     async function reconcile(reason) {
         await appendAction({
             sessionId: plan.sessionId,
@@ -368,9 +511,39 @@ export async function runTeamworkCoordinator(input) {
         });
         return receipt;
     }
+    async function failPending(reason) {
+        admissionClosed = true;
+        while (pending.length > 0) {
+            const assignment = pending.shift();
+            if (!assignment)
+                continue;
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "assignment",
+                id: assignment.assignmentId,
+                state: "FINAL_FAILED",
+                states: assignmentStates,
+                assignmentId: assignment.assignmentId,
+                retryCount: retryCounts[assignment.assignmentId] ?? 0,
+                payload: { reason, dispatched: false },
+            });
+            final.set(assignment.assignmentId, "FINAL_FAILED");
+        }
+    }
     async function dispatch(slot, assignment, retryCount) {
-        const attemptId = `${assignment.assignmentId}.attempt.${retryCount}`;
-        const teammateId = `${assignment.assignmentId}.teammate.${retryCount}`;
+        const boundedId = (kind) => {
+            const candidate = `${assignment.assignmentId}.${kind}.${retryCount}`;
+            return candidate.length <= 64
+                ? candidate
+                : crypto
+                    .createHash("sha256")
+                    .update(`${assignment.assignmentId}:${kind}:${retryCount}`)
+                    .digest("hex");
+        };
+        const attemptId = boundedId("attempt");
+        const teammateId = boundedId("teammate");
         const frozen = frozenLaunches.get(assignment.assignmentId);
         if (!frozen)
             throw new Error(`missing frozen launch for '${assignment.assignmentId}'`);
@@ -506,6 +679,7 @@ export async function runTeamworkCoordinator(input) {
             teammateId,
             slot,
             retryCount,
+            transitionKey: `attempt:${attemptId}`,
             ...(context.providerSessionId
                 ? { providerSessionId: context.providerSessionId }
                 : {}),
@@ -558,7 +732,63 @@ export async function runTeamworkCoordinator(input) {
         });
         guardHeld = false;
     }
+    async function resumeFromWait(current) {
+        if (assignmentStates.get(current.assignment.assignmentId) !==
+            "WAITING_FOR_MAIN_COMMAND")
+            return;
+        const transitionKey = `continuation:${current.continuations}`;
+        await appendTransition({
+            sessionId: plan.sessionId,
+            by,
+            batchId: plan.batchId,
+            entity: "attempt",
+            id: current.context.attemptId,
+            state: "RUNNING",
+            states: attemptStates,
+            assignmentId: current.assignment.assignmentId,
+            attemptId: current.context.attemptId,
+            teammateId: current.context.teammateId,
+            retryCount: current.retryCount,
+            ...(current.providerSessionId
+                ? { providerSessionId: current.providerSessionId }
+                : {}),
+            transitionKey,
+        });
+        await appendTransition({
+            sessionId: plan.sessionId,
+            by,
+            batchId: plan.batchId,
+            entity: "teammate",
+            id: current.context.teammateId,
+            state: "RUNNING",
+            states: teammateStates,
+            attemptId: current.context.attemptId,
+            teammateId: current.context.teammateId,
+            ...(current.providerSessionId
+                ? { providerSessionId: current.providerSessionId }
+                : {}),
+            transitionKey,
+        });
+        await appendTransition({
+            sessionId: plan.sessionId,
+            by,
+            batchId: plan.batchId,
+            entity: "assignment",
+            id: current.assignment.assignmentId,
+            state: "EXECUTING",
+            states: assignmentStates,
+            assignmentId: current.assignment.assignmentId,
+            attemptId: current.context.attemptId,
+            teammateId: current.context.teammateId,
+            retryCount: current.retryCount,
+            ...(current.providerSessionId
+                ? { providerSessionId: current.providerSessionId }
+                : {}),
+            transitionKey,
+        });
+    }
     async function cancelAndDrain(reason, terminal) {
+        await failPending(reason);
         if (batchStates.get(plan.batchId) === "RUNNING") {
             await appendTransition({
                 sessionId: plan.sessionId,
@@ -571,52 +801,68 @@ export async function runTeamworkCoordinator(input) {
                 payload: { reason },
             });
         }
-        if (!input.runtime.cancel && active.size > 0 && !terminal) {
-            return reconcile(`host cannot confirm cancellation: ${reason}`);
-        }
-        for (const current of [...active.values()]) {
-            let result;
-            if (terminal?.current === current) {
-                result = terminal.result;
+        const cancel = input.runtime.cancel;
+        const outcomes = await Promise.all([...active.values()].map(async (current) => {
+            if (terminal?.current === current)
+                return { current, result: terminal.result };
+            if (!cancel)
+                return {
+                    current,
+                    error: new Error(`host cannot cancel active attempt: ${reason}`),
+                };
+            try {
+                return { current, result: await cancel(current.context) };
             }
-            else {
-                try {
-                    if (!input.runtime.cancel)
-                        return reconcile(`host cannot cancel active attempt: ${reason}`);
-                    result = await input.runtime.cancel(current.context);
-                }
-                catch (error) {
-                    return reconcile(error instanceof Error ? error.message : String(error));
-                }
+            catch (error) {
+                return { current, error };
+            }
+        }));
+        let cancellationFailure;
+        for (const outcome of outcomes) {
+            const { current } = outcome;
+            if ("error" in outcome) {
+                cancellationFailure ??=
+                    outcome.error instanceof Error
+                        ? outcome.error.message
+                        : String(outcome.error);
+                continue;
             }
+            const { result } = outcome;
             if (result.kind === "reconciliation" ||
                 result.kind === "timeout" ||
                 result.terminal !== true) {
-                return reconcile(`attempt cancellation is not terminal: ${reason}`);
+                cancellationFailure ??= `attempt cancellation is not terminal: ${reason}`;
+                continue;
             }
             if (result.providerSessionId &&
                 current.providerSessionId &&
                 result.providerSessionId !== current.providerSessionId) {
-                return reconcile("cancellation returned a different host session");
+                cancellationFailure ??=
+                    "cancellation returned a different host session";
+                continue;
             }
             if (result.providerSessionId) {
                 current.providerSessionId = result.providerSessionId;
                 current.context.providerSessionId = result.providerSessionId;
             }
             try {
-                if (result.kind === "report")
+                await resumeFromWait(current);
+                if (result.kind === "report") {
                     await complete(current, false);
+                }
                 else
                     await fail(current, result.reason ?? "cancelled by coordinator", false);
             }
             catch (error) {
-                return reconcile(error instanceof Error ? error.message : String(error));
+                cancellationFailure ??=
+                    error instanceof Error ? error.message : String(error);
+                continue;
             }
             active.delete(current.slot);
             waits.delete(current.slot);
         }
-        if (pending.length > 0)
-            return reconcile(`queued assignment remains after cancellation: ${reason}`);
+        if (cancellationFailure)
+            return reconcile(cancellationFailure);
         return settle();
     }
     async function fail(activeWork, reason, allowRetry = true) {
@@ -653,7 +899,7 @@ export async function runTeamworkCoordinator(input) {
                 : {}),
             terminal: true,
         });
-        if (allowRetry && retryCount < assignment.maxRetries) {
+        if (allowRetry && !admissionClosed && retryCount < assignment.maxRetries) {
             await release(activeWork);
             retryCounts[assignment.assignmentId] = retryCount + 1;
             pending.unshift(assignment);
@@ -739,7 +985,7 @@ export async function runTeamworkCoordinator(input) {
         });
         await release(activeWork);
         final.set(assignment.assignmentId, "ACCEPTED");
-        acceptedAttempts.set(assignment.assignmentId, context.attemptId);
+        retainedReports.set(assignment.assignmentId, context.attemptId);
     }
     async function complete(activeWork, allowRetry = true) {
         const { assignment, context, retryCount } = activeWork;
@@ -778,9 +1024,20 @@ export async function runTeamworkCoordinator(input) {
             });
         }
         catch (error) {
+            if (error instanceof TeammateReportReadError && error.kind !== "missing")
+                throw error;
             await fail(activeWork, error instanceof Error ? error.message : String(error), allowRetry);
             return;
         }
+        if (report.status === "blocked") {
+            retainedReports.set(assignment.assignmentId, context.attemptId);
+            await fail(activeWork, `teammate reported blocked: ${report.evidence[0]?.summary ?? "blocker evidence missing"}`, false);
+            return;
+        }
+        if (report.status !== "complete") {
+            await fail(activeWork, `teammate reported ${report.status}`, allowRetry);
+            return;
+        }
         if (input.runtime.validateReport &&
             !(await input.runtime.validateReport(context, report))) {
             await fail(activeWork, "report rejected by acceptance contract", allowRetry);
@@ -866,7 +1123,7 @@ export async function runTeamworkCoordinator(input) {
         const cleanupFailures = cleanupSettledTeamworkArtifacts({
             taskRoot: input.taskRoot,
             assignments: plan.assignments,
-            acceptedAttempts,
+            retainedReports,
         });
         if (cleanupFailures.length > 0) {
             return reconcile(`Teamwork batch cleanup failed: ${cleanupFailures.join("; ")}`);
@@ -914,12 +1171,25 @@ export async function runTeamworkCoordinator(input) {
             id: `${plan.batchId}:${status}`,
             payload: { ...receipt },
         });
-        await assertTeamworkReceiptDurable(plan.sessionId, plan.batchId, plan.planHash, cwd);
+        await assertTeamworkReceiptDurable(plan, receipt, cwd);
         await releaseGuard();
         return receipt;
     }
     async function fill() {
+        if (admissionClosed)
+            return;
         while (active.size < capacity && pending.length > 0) {
+            if (Date.parse(plan.deadline) <= Date.now()) {
+                await failPending("batch deadline elapsed before dispatch");
+                return;
+            }
+            try {
+                assertTeamworkPlanCurrent(plan, input.taskRoot);
+            }
+            catch (error) {
+                await failPending(`task phase changed before dispatch: ${error instanceof Error ? error.message : String(error)}`);
+                return;
+            }
             const assignment = pending.shift();
             if (!assignment)
                 return;
@@ -987,162 +1257,116 @@ export async function runTeamworkCoordinator(input) {
                 result: next.result,
             });
         }
+        try {
+            await resumeFromWait(current);
+        }
+        catch (error) {
+            return reconcile(error instanceof Error ? error.message : String(error));
+        }
         let releaseSlot = true;
         if (next.result.kind === "timeout") {
             current.continuations += 1;
-            if (current.continuations > 8) {
-                return reconcile("continuation limit exceeded");
+            if (Date.now() >= Date.parse(plan.deadline))
+                return cancelAndDrain("batch deadline elapsed");
+            const transitionKey = `continuation:${current.continuations}`;
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "attempt",
+                id: current.context.attemptId,
+                state: "TIMEOUT_YIELDED",
+                states: attemptStates,
+                assignmentId: current.assignment.assignmentId,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                retryCount: current.retryCount,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "teammate",
+                id: current.context.teammateId,
+                state: "TIMEOUT_YIELDED",
+                states: teammateStates,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "assignment",
+                id: current.assignment.assignmentId,
+                state: "WAITING_FOR_MAIN_COMMAND",
+                states: assignmentStates,
+                assignmentId: current.assignment.assignmentId,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                retryCount: current.retryCount,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "attempt",
+                id: current.context.attemptId,
+                state: "WAITING_FOR_MAIN_COMMAND",
+                states: attemptStates,
+                assignmentId: current.assignment.assignmentId,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                retryCount: current.retryCount,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "teammate",
+                id: current.context.teammateId,
+                state: "WAITING_FOR_MAIN_COMMAND",
+                states: teammateStates,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            let continued;
+            try {
+                continued = input.runtime.continue(current.context);
             }
-            else {
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "attempt",
-                    id: current.context.attemptId,
-                    state: "TIMEOUT_YIELDED",
-                    states: attemptStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "teammate",
-                    id: current.context.teammateId,
-                    state: "TIMEOUT_YIELDED",
-                    states: teammateStates,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "assignment",
-                    id: current.assignment.assignmentId,
-                    state: "WAITING_FOR_MAIN_COMMAND",
-                    states: assignmentStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "attempt",
-                    id: current.context.attemptId,
-                    state: "WAITING_FOR_MAIN_COMMAND",
-                    states: attemptStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "teammate",
-                    id: current.context.teammateId,
-                    state: "WAITING_FOR_MAIN_COMMAND",
-                    states: teammateStates,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                let continued;
-                try {
-                    continued = await input.runtime.continue(current.context);
-                }
-                catch (error) {
-                    return reconcile(error instanceof Error ? error.message : String(error));
-                }
-                if (continued.providerSessionId &&
-                    current.providerSessionId &&
-                    continued.providerSessionId !== current.providerSessionId) {
-                    return reconcile("continuation returned a different host session");
-                }
-                if (continued.providerSessionId) {
-                    current.providerSessionId = continued.providerSessionId;
-                    current.context.providerSessionId = continued.providerSessionId;
-                }
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "attempt",
-                    id: current.context.attemptId,
-                    state: "RUNNING",
-                    states: attemptStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "teammate",
-                    id: current.context.teammateId,
-                    state: "RUNNING",
-                    states: teammateStates,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "assignment",
-                    id: current.assignment.assignmentId,
-                    state: "EXECUTING",
-                    states: assignmentStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                waits.set(next.slot, Promise.resolve({ slot: next.slot, result: continued }));
-                releaseSlot = false;
+            catch (error) {
+                return reconcile(error instanceof Error ? error.message : String(error));
             }
+            waits.set(next.slot, Promise.resolve(continued)
+                .then((result) => ({ slot: next.slot, result }))
+                .catch((error) => ({
+                slot: next.slot,
+                result: {
+                    kind: "reconciliation",
+                    reason: error instanceof Error ? error.message : String(error),
+                },
+            })));
+            releaseSlot = false;
         }
         else if (next.result.kind === "report") {
             try {
@@ -1173,22 +1397,25 @@ export async function runTeamworkCoordinator(input) {
     }
     return settle();
 }
-export async function assertTeamworkReceiptDurable(sessionId, batchId, planHash, cwd) {
+export async function assertTeamworkReceiptDurable(plan, receipt, cwd) {
     const [state, events] = await Promise.all([
-        readCoordinationState({ sessionId, ...(cwd ? { cwd } : {}) }),
-        readTeamworkEvents({ sessionId, ...(cwd ? { cwd } : {}) }),
+        readCoordinationState({
+            sessionId: plan.sessionId,
+            ...(cwd ? { cwd } : {}),
+        }),
+        readTeamworkEvents({
+            sessionId: plan.sessionId,
+            ...(cwd ? { cwd } : {}),
+        }),
     ]);
-    const batch = state.batches.find((entry) => entry.batchId === batchId);
-    if (batch?.planHash !== planHash ||
-        (batch.state !== "ALL_ACCEPTED" && batch.state !== "SETTLED_WITH_FAILURE")) {
-        throw new Error(`batch '${batchId}' has no durable terminal receipt`);
-    }
-    const receiptRecorded = events.some((event) => event.kind === "coordination" &&
-        event.batchId === batchId &&
-        event.action === "receipt_recorded" &&
-        event.payload?.status === batch.state);
-    if (!receiptRecorded) {
-        throw new Error(`batch '${batchId}' has no matching durable receipt event`);
-    }
+    const expected = terminalReceiptFromProjection(plan, state);
+    if (!isDeepStrictEqual(receipt, expected))
+        throw new Error(`batch '${plan.batchId}' receipt disagrees with durable facts`);
+    const receipts = events.filter((event) => event.kind === "coordination" &&
+        event.batchId === plan.batchId &&
+        event.action === "receipt_recorded");
+    if (receipts.length !== 1 ||
+        !isDeepStrictEqual(receipts[0]?.payload, expected))
+        throw new Error(`batch '${plan.batchId}' has no matching durable receipt event`);
 }
 //# sourceMappingURL=coordinator.js.map