@pennixrv/trellis 0.7.1-beta.9 → 0.7.2-beta.0

package/dist/commands/teamwork/coordinator.js

@@ -1,12 +1,13 @@
+import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
+import { isDeepStrictEqual } from "node:util";
 import { isTaskPhaseId, parseTaskWorkflowState, } from "@pennixrv/trellis-core/task";
 import { assertCoordinationTransition, readTeamworkEvents, readCoordinationState, recordCoordinationEvent, } from "@pennixrv/trellis-core/teamwork";
-import { cleanupSettledTeamworkArtifacts, readTeammateReport, writeTeammateQueueFact, writeTeammateClaim, writeTeammateDisposition, validateTeamworkPlan, } from "./artifact.js";
-import { assertTeammateAdapterContract, } from "./teammate-adapters.js";
+import { TeammateReportReadError, cleanupSettledTeamworkArtifacts, readTeammateReport, writeTeammateQueueFact, writeTeammateClaim, writeTeammateDisposition, validateTeamworkPlan, } from "./artifact.js";
+import { assertTeammateAdapterContract, assertTeammateHostCapability, } from "./teammate-adapters.js";
 import { effectiveTeammateCapacity, resolveTeammateRuntimeConfig, } from "./teammate-config.js";
-import { resolveTeammateProfile } from "./profiles.js";
-import { buildTeammateLaunchSpec, } from "./teammate-launch.js";
+import { readProcessIdentity } from "./process-identity.js";
 export function assertTeamworkPlanCurrent(plan, taskRoot) {
     const taskFile = path.join(taskRoot, "task.json");
     const stat = fs.lstatSync(taskFile);
@@ -37,6 +38,165 @@ export function assertTeamworkPlanCurrent(plan, taskRoot) {
         throw new Error("Teamwork plan does not match the current active task phase and revisions");
     }
 }
+function terminalReceiptFromProjection(plan, projection) {
+    const mismatch = (reason) => {
+        throw new Error(`batch '${plan.batchId}' ${reason}`);
+    };
+    if (projection.issues.length > 0)
+        mismatch("has coordination projection issues");
+    const batch = projection.batches.find((entry) => entry.batchId === plan.batchId);
+    if (!batch)
+        return mismatch("does not match the frozen plan");
+    if (batch.planHash !== plan.planHash)
+        return mismatch("does not match the frozen plan");
+    if (batch.state !== "ALL_ACCEPTED" && batch.state !== "SETTLED_WITH_FAILURE")
+        mismatch("is not terminal");
+    const effectiveCapacity = batch.effectiveCapacity;
+    if (typeof effectiveCapacity !== "number" ||
+        !Number.isSafeInteger(effectiveCapacity) ||
+        effectiveCapacity < 1)
+        return mismatch("has no valid effective capacity");
+    if (projection.leases.some((lease) => lease.batchId === plan.batchId))
+        mismatch("still owns capacity leases");
+    const projectedAssignments = projection.assignments.filter((assignment) => assignment.batchId === plan.batchId);
+    if (projectedAssignments.length !== plan.assignments.length ||
+        new Set(projectedAssignments.map((assignment) => assignment.assignmentId))
+            .size !== plan.assignments.length)
+        mismatch("does not contain the exact frozen assignment set");
+    const assignmentById = new Map(projectedAssignments.map((assignment) => [
+        assignment.assignmentId,
+        assignment,
+    ]));
+    const planAssignmentIds = new Set(plan.assignments.map((assignment) => assignment.assignmentId));
+    const attempts = projection.attempts.filter((attempt) => attempt.batchId === plan.batchId);
+    const teammates = projection.teammates.filter((teammate) => teammate.batchId === plan.batchId);
+    if (attempts.length !== teammates.length)
+        mismatch("has unmatched provider attempts and teammates");
+    for (const attempt of attempts) {
+        if (!attempt.assignmentId ||
+            !planAssignmentIds.has(attempt.assignmentId) ||
+            !attempt.teammateId ||
+            attempt.terminal !== true ||
+            !["REPORT_SUBMITTED", "FAILED", "FINAL_FAILED"].includes(attempt.state))
+            mismatch("has an unresolved provider attempt");
+        const teammate = teammates.find((entry) => entry.attemptId === attempt.attemptId &&
+            entry.teammateId === attempt.teammateId);
+        if (teammate?.terminal !== true ||
+            (teammate?.state !== "REPORT_SUBMITTED" && teammate?.state !== "FAILED"))
+            mismatch("has an unresolved teammate terminal fact");
+    }
+    const assignments = plan.assignments.map((assignment) => {
+        const projected = assignmentById.get(assignment.assignmentId);
+        if (!projected ||
+            (projected.state !== "ACCEPTED" && projected.state !== "FINAL_FAILED"))
+            return mismatch(`assignment '${assignment.assignmentId}' is not settled`);
+        if (projected.state === "ACCEPTED") {
+            const acceptedAttempt = attempts.find((attempt) => attempt.attemptId === projected.attemptId &&
+                attempt.assignmentId === assignment.assignmentId);
+            if (acceptedAttempt?.state !== "REPORT_SUBMITTED")
+                return mismatch(`assignment '${assignment.assignmentId}' has no accepted attempt`);
+        }
+        return {
+            assignmentId: assignment.assignmentId,
+            status: projected.state,
+        };
+    });
+    const status = assignments.every((item) => item.status === "ACCEPTED")
+        ? "ALL_ACCEPTED"
+        : "SETTLED_WITH_FAILURE";
+    if (batch.state !== status)
+        mismatch("terminal state disagrees with its assignment dispositions");
+    const retryCounts = Object.fromEntries(plan.assignments.flatMap((assignment) => {
+        const count = assignmentById.get(assignment.assignmentId)?.retryCount;
+        if (count === undefined)
+            return mismatch(`assignment '${assignment.assignmentId}' is missing`);
+        const started = attempts.some((attempt) => attempt.assignmentId === assignment.assignmentId);
+        return started || count > 0 ? [[assignment.assignmentId, count]] : [];
+    }));
+    const evaluationGroups = [
+        ...new Set(plan.assignments
+            .map((assignment) => assignment.evaluationGroup?.id)
+            .filter((id) => id !== undefined)),
+    ].map((id) => ({
+        id,
+        status: plan.assignments
+            .filter((assignment) => assignment.evaluationGroup?.id === id)
+            .every((assignment) => assignmentById.get(assignment.assignmentId)?.state === "ACCEPTED")
+            ? "COMPLETE"
+            : "INCOMPLETE",
+    }));
+    return {
+        schemaVersion: 1,
+        batchId: plan.batchId,
+        status,
+        assignments,
+        capacity: effectiveCapacity,
+        retryCounts,
+        evaluationGroups,
+    };
+}
+function reconciliationReceipt(plan, projection, capacity) {
+    return {
+        schemaVersion: 1,
+        batchId: plan.batchId,
+        status: "RECONCILIATION_REQUIRED",
+        assignments: plan.assignments.map((item) => {
+            const current = projection.assignments.find((entry) => entry.batchId === plan.batchId &&
+                entry.assignmentId === item.assignmentId);
+            return {
+                assignmentId: item.assignmentId,
+                status: current?.state === "ACCEPTED" || current?.state === "FINAL_FAILED"
+                    ? current.state
+                    : "UNKNOWN",
+            };
+        }),
+        capacity: projection.batches.find((entry) => entry.batchId === plan.batchId)
+            ?.effectiveCapacity ?? capacity,
+        retryCounts: Object.fromEntries(projection.assignments
+            .filter((entry) => entry.batchId === plan.batchId)
+            .map((entry) => [entry.assignmentId, entry.retryCount])),
+    };
+}
+async function recordRecoveryReconciliation(input) {
+    const { plan, cwd, by, existingState, projection, capacity, reason } = input;
+    if (existingState !== "RECONCILIATION_REQUIRED" &&
+        existingState !== "ALL_ACCEPTED" &&
+        existingState !== "SETTLED_WITH_FAILURE") {
+        await appendTransition({
+            sessionId: plan.sessionId,
+            cwd,
+            by,
+            batchId: plan.batchId,
+            entity: "batch",
+            id: plan.batchId,
+            state: "RECONCILIATION_REQUIRED",
+            states: new Map([[plan.batchId, existingState]]),
+            planHash: plan.planHash,
+            payload: { recoveredAfterRestart: true },
+        });
+    }
+    const receipt = reconciliationReceipt(plan, projection, capacity);
+    await appendAction({
+        sessionId: plan.sessionId,
+        cwd,
+        by,
+        batchId: plan.batchId,
+        entity: "batch",
+        action: "reconciliation_required",
+        reason,
+    });
+    await appendAction({
+        sessionId: plan.sessionId,
+        cwd,
+        by,
+        batchId: plan.batchId,
+        entity: "batch",
+        action: "receipt_recorded",
+        id: `${plan.batchId}:RECONCILIATION_REQUIRED`,
+        payload: { ...receipt, reason },
+    });
+    return receipt;
+}
 async function recoverExistingBatch(input) {
     const projection = await readCoordinationState({
         sessionId: input.plan.sessionId,
@@ -52,64 +212,55 @@ async function recoverExistingBatch(input) {
         sessionId: input.plan.sessionId,
         cwd: input.cwd,
     });
-    const saved = events.find((event) => event.kind === "coordination" &&
-        event.batchId === input.plan.batchId &&
-        event.action === "receipt_recorded" &&
-        event.payload &&
-        (event.payload.status === "ALL_ACCEPTED" ||
-            event.payload.status === "SETTLED_WITH_FAILURE"));
-    if (saved?.kind === "coordination" && saved.payload) {
-        return saved.payload;
-    }
-    const states = new Map([
-        [input.plan.batchId, existing.state],
-    ]);
-    if (existing.state !== "RECONCILIATION_REQUIRED") {
-        await appendTransition({
-            sessionId: input.plan.sessionId,
-            by: input.by,
-            batchId: input.plan.batchId,
-            entity: "batch",
-            id: input.plan.batchId,
-            state: "RECONCILIATION_REQUIRED",
-            states,
-            planHash: input.plan.planHash,
-            payload: { recoveredAfterRestart: true },
-        });
+    if (existing.state === "ALL_ACCEPTED" ||
+        existing.state === "SETTLED_WITH_FAILURE") {
+        let receipt;
+        try {
+            receipt = terminalReceiptFromProjection(input.plan, projection);
+        }
+        catch (error) {
+            return recordRecoveryReconciliation({
+                ...input,
+                existingState: existing.state,
+                projection,
+                reason: error instanceof Error ? error.message : String(error),
+            });
+        }
+        const receipts = events.filter((event) => event.kind === "coordination" &&
+            event.batchId === input.plan.batchId &&
+            event.action === "receipt_recorded" &&
+            event.payload?.status !== "RECONCILIATION_REQUIRED");
+        if (receipts.length > 1 ||
+            (receipts.length === 1 &&
+                !isDeepStrictEqual(receipts[0]?.payload, receipt))) {
+            return recordRecoveryReconciliation({
+                ...input,
+                existingState: existing.state,
+                projection,
+                reason: "saved terminal receipt disagrees with durable coordination facts",
+            });
+        }
+        if (receipts.length === 0) {
+            await appendAction({
+                sessionId: input.plan.sessionId,
+                cwd: input.cwd,
+                by: input.by,
+                batchId: input.plan.batchId,
+                entity: "batch",
+                action: "receipt_recorded",
+                id: `${input.plan.batchId}:${receipt.status}`,
+                payload: { ...receipt },
+            });
+        }
+        await assertTeamworkReceiptDurable(input.plan, receipt, input.cwd);
+        return receipt;
     }
-    const assignments = input.plan.assignments.map((item) => {
-        const current = projection.assignments.find((entry) => entry.batchId === input.plan.batchId &&
-            entry.assignmentId === item.assignmentId);
-        return {
-            assignmentId: item.assignmentId,
-            status: current?.state === "ACCEPTED" || current?.state === "FINAL_FAILED"
-                ? current.state
-                : "UNKNOWN",
-        };
-    });
-    const retryCounts = Object.fromEntries(projection.assignments
-        .filter((entry) => entry.batchId === input.plan.batchId)
-        .map((entry) => [entry.assignmentId, entry.retryCount]));
-    const receipt = {
-        schemaVersion: 1,
-        batchId: input.plan.batchId,
-        status: "RECONCILIATION_REQUIRED",
-        assignments,
-        capacity: existing.effectiveCapacity ?? input.capacity,
-        retryCounts,
-    };
-    await recordCoordinationEvent({
-        sessionId: input.plan.sessionId,
-        cwd: input.cwd,
-        by: input.by,
-        batchId: input.plan.batchId,
-        entity: "batch",
-        action: "receipt_recorded",
-        receiptId: `${input.plan.batchId}:RECONCILIATION_REQUIRED`,
-        payload: { ...receipt, reason: "recovered batch requires reconciliation" },
-        idempotencyKey: `${input.plan.batchId}:RECONCILIATION_REQUIRED:receipt`,
+    return recordRecoveryReconciliation({
+        ...input,
+        existingState: existing.state,
+        projection,
+        reason: "recovered nonterminal batch requires reconciliation",
     });
-    return receipt;
 }
 function transitionState(states, key, next, entity) {
     assertCoordinationTransition(entity, states.get(key), next);
@@ -119,6 +270,7 @@ async function appendTransition(input) {
     transitionState(input.states, input.id, input.state, input.entity);
     await recordCoordinationEvent({
         sessionId: input.sessionId,
+        ...(input.cwd ? { cwd: input.cwd } : {}),
         by: input.by,
         batchId: input.batchId,
         entity: input.entity,
@@ -141,6 +293,7 @@ async function appendTransition(input) {
 async function appendAction(input) {
     await recordCoordinationEvent({
         sessionId: input.sessionId,
+        ...(input.cwd ? { cwd: input.cwd } : {}),
         by: input.by,
         batchId: input.batchId,
         entity: input.entity,
@@ -164,47 +317,60 @@ async function appendAction(input) {
         idempotencyKey: `${input.batchId}:${input.action}:${input.id ?? input.leaseId ?? input.attemptId ?? "event"}`,
     });
 }
-function readPlanFile(planPath) {
+export function readPlanFile(planPath) {
+    const stat = fs.lstatSync(planPath);
+    if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 2 * 1024 * 1024)
+        throw new Error("Teamwork plan must be a regular file under 2097152 bytes");
     const raw = fs.readFileSync(planPath, "utf8");
     return JSON.parse(raw);
 }
 export function loadTeamworkPlan(planPath, taskRoot) {
-    const plan = validateTeamworkPlan(readPlanFile(planPath), taskRoot);
+    const plan = validateTeamworkPlan(readPlanFile(planPath), taskRoot, undefined, { allowExpired: true });
     assertTeamworkPlanCurrent(plan, taskRoot);
     return plan;
 }
 export async function runTeamworkCoordinator(input) {
-    const plan = validateTeamworkPlan(input.plan, input.taskRoot);
+    const plan = validateTeamworkPlan(input.plan, input.taskRoot, undefined, {
+        allowExpired: true,
+    });
     assertTeamworkPlanCurrent(plan, input.taskRoot);
     const cwd = input.cwd ?? input.taskRoot;
     const frozenLaunches = new Map();
     for (const assignment of plan.assignments) {
         assertTeammateAdapterContract(assignment.provider);
-        const profile = resolveTeammateProfile({
-            cwd,
-            profile: assignment.profile,
-            provider: assignment.provider,
-            model: assignment.model,
-        });
-        frozenLaunches.set(assignment.assignmentId, {
-            profile,
-            launch: buildTeammateLaunchSpec({
-                cwd,
-                profile,
-                host: input.host,
-                contextFiles: assignment.contextFiles,
-                contextManifests: assignment.contextManifests,
-                protectedTargets: plan.protectedTargets,
-            }),
-        });
+        assertTeammateHostCapability(assignment.provider, input.host, assignment.execution.profile.options);
+        frozenLaunches.set(assignment.assignmentId, assignment.execution);
     }
-    const config = input.config ?? resolveTeammateRuntimeConfig(cwd);
+    const config = input.config ??
+        resolveTeammateRuntimeConfig(cwd, process.env, plan.coordinationMode);
     const capacity = effectiveTeammateCapacity({
         requested: plan.requestedCapacity,
+        assignmentCount: plan.assignments.length,
         config,
         verifiedHostCap: input.host.verifiedHostCap,
     });
     const by = input.by ?? "main";
+    const existing = await readCoordinationState({
+        sessionId: plan.sessionId,
+        cwd,
+    });
+    const settled = existing.batches.find((batch) => batch.batchId === plan.batchId);
+    if (settled &&
+        ["ALL_ACCEPTED", "SETTLED_WITH_FAILURE"].includes(settled.state)) {
+        let terminal = false;
+        try {
+            terminalReceiptFromProjection(plan, existing);
+            terminal = true;
+        }
+        catch {
+            // Inconsistent terminal facts still require the protected recovery path.
+        }
+        if (terminal) {
+            const recovered = await recoverExistingBatch({ plan, cwd, by, capacity });
+            if (recovered)
+                return recovered;
+        }
+    }
     const batchStates = new Map();
     const assignmentStates = new Map();
     const attemptStates = new Map();
@@ -215,19 +381,8 @@ export async function runTeamworkCoordinator(input) {
         capacity,
     });
     let guardHeld = true;
-    let recovered;
-    try {
-        recovered = await recoverExistingBatch({ plan, cwd, by, capacity });
-    }
-    catch (error) {
-        await input.guard.release({
-            batchId: plan.batchId,
-            planHash: plan.planHash,
-            capacity,
-        });
-        guardHeld = false;
-        throw error;
-    }
+    // Failed recovery cannot prove that an older provider process exited.
+    const recovered = await recoverExistingBatch({ plan, cwd, by, capacity });
     if (recovered) {
         if (recovered.status !== "RECONCILIATION_REQUIRED") {
             await input.guard.release({
@@ -239,6 +394,14 @@ export async function runTeamworkCoordinator(input) {
         }
         return recovered;
     }
+    if (Date.parse(plan.deadline) <= Date.now()) {
+        await input.guard.release({
+            batchId: plan.batchId,
+            planHash: plan.planHash,
+            capacity,
+        });
+        throw new Error("deadline elapsed before batch dispatch");
+    }
     try {
         for (const [queueIndex, assignment] of plan.assignments.entries()) {
             writeTeammateQueueFact({
@@ -328,8 +491,8 @@ export async function runTeamworkCoordinator(input) {
     const active = new Map();
     const waits = new Map();
     const final = new Map();
-    const acceptedAttempts = new Map();
     const retryCounts = {};
+    let admissionClosed = false;
     async function reconcile(reason) {
         await appendAction({
             sessionId: plan.sessionId,
@@ -372,9 +535,39 @@ export async function runTeamworkCoordinator(input) {
         });
         return receipt;
     }
+    async function failPending(reason) {
+        admissionClosed = true;
+        while (pending.length > 0) {
+            const assignment = pending.shift();
+            if (!assignment)
+                continue;
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "assignment",
+                id: assignment.assignmentId,
+                state: "FINAL_FAILED",
+                states: assignmentStates,
+                assignmentId: assignment.assignmentId,
+                retryCount: retryCounts[assignment.assignmentId] ?? 0,
+                payload: { reason, dispatched: false },
+            });
+            final.set(assignment.assignmentId, "FINAL_FAILED");
+        }
+    }
     async function dispatch(slot, assignment, retryCount) {
-        const attemptId = `${assignment.assignmentId}.attempt.${retryCount}`;
-        const teammateId = `${assignment.assignmentId}.teammate.${retryCount}`;
+        const boundedId = (kind) => {
+            const candidate = `${assignment.assignmentId}.${kind}.${retryCount}`;
+            return candidate.length <= 64
+                ? candidate
+                : crypto
+                    .createHash("sha256")
+                    .update(`${assignment.assignmentId}:${kind}:${retryCount}`)
+                    .digest("hex");
+        };
+        const attemptId = boundedId("attempt");
+        const teammateId = boundedId("teammate");
         const frozen = frozenLaunches.get(assignment.assignmentId);
         if (!frozen)
             throw new Error(`missing frozen launch for '${assignment.assignmentId}'`);
@@ -388,6 +581,7 @@ export async function runTeamworkCoordinator(input) {
             profile,
             launch,
         };
+        input.runtime.prepare?.(context);
         await appendTransition({
             sessionId: plan.sessionId,
             by,
@@ -510,6 +704,7 @@ export async function runTeamworkCoordinator(input) {
             teammateId,
             slot,
             retryCount,
+            transitionKey: `attempt:${attemptId}`,
             ...(context.providerSessionId
                 ? { providerSessionId: context.providerSessionId }
                 : {}),
@@ -562,7 +757,63 @@ export async function runTeamworkCoordinator(input) {
         });
         guardHeld = false;
     }
+    async function resumeFromWait(current) {
+        if (assignmentStates.get(current.assignment.assignmentId) !==
+            "WAITING_FOR_MAIN_COMMAND")
+            return;
+        const transitionKey = `continuation:${current.continuations}`;
+        await appendTransition({
+            sessionId: plan.sessionId,
+            by,
+            batchId: plan.batchId,
+            entity: "attempt",
+            id: current.context.attemptId,
+            state: "RUNNING",
+            states: attemptStates,
+            assignmentId: current.assignment.assignmentId,
+            attemptId: current.context.attemptId,
+            teammateId: current.context.teammateId,
+            retryCount: current.retryCount,
+            ...(current.providerSessionId
+                ? { providerSessionId: current.providerSessionId }
+                : {}),
+            transitionKey,
+        });
+        await appendTransition({
+            sessionId: plan.sessionId,
+            by,
+            batchId: plan.batchId,
+            entity: "teammate",
+            id: current.context.teammateId,
+            state: "RUNNING",
+            states: teammateStates,
+            attemptId: current.context.attemptId,
+            teammateId: current.context.teammateId,
+            ...(current.providerSessionId
+                ? { providerSessionId: current.providerSessionId }
+                : {}),
+            transitionKey,
+        });
+        await appendTransition({
+            sessionId: plan.sessionId,
+            by,
+            batchId: plan.batchId,
+            entity: "assignment",
+            id: current.assignment.assignmentId,
+            state: "EXECUTING",
+            states: assignmentStates,
+            assignmentId: current.assignment.assignmentId,
+            attemptId: current.context.attemptId,
+            teammateId: current.context.teammateId,
+            retryCount: current.retryCount,
+            ...(current.providerSessionId
+                ? { providerSessionId: current.providerSessionId }
+                : {}),
+            transitionKey,
+        });
+    }
     async function cancelAndDrain(reason, terminal) {
+        await failPending(reason);
         if (batchStates.get(plan.batchId) === "RUNNING") {
             await appendTransition({
                 sessionId: plan.sessionId,
@@ -575,52 +826,68 @@ export async function runTeamworkCoordinator(input) {
                 payload: { reason },
             });
         }
-        if (!input.runtime.cancel && active.size > 0 && !terminal) {
-            return reconcile(`host cannot confirm cancellation: ${reason}`);
-        }
-        for (const current of [...active.values()]) {
-            let result;
-            if (terminal?.current === current) {
-                result = terminal.result;
+        const cancel = input.runtime.cancel;
+        const outcomes = await Promise.all([...active.values()].map(async (current) => {
+            if (terminal?.current === current)
+                return { current, result: terminal.result };
+            if (!cancel)
+                return {
+                    current,
+                    error: new Error(`host cannot cancel active attempt: ${reason}`),
+                };
+            try {
+                return { current, result: await cancel(current.context) };
             }
-            else {
-                try {
-                    if (!input.runtime.cancel)
-                        return reconcile(`host cannot cancel active attempt: ${reason}`);
-                    result = await input.runtime.cancel(current.context);
-                }
-                catch (error) {
-                    return reconcile(error instanceof Error ? error.message : String(error));
-                }
+            catch (error) {
+                return { current, error };
             }
+        }));
+        let cancellationFailure;
+        for (const outcome of outcomes) {
+            const { current } = outcome;
+            if ("error" in outcome) {
+                cancellationFailure ??=
+                    outcome.error instanceof Error
+                        ? outcome.error.message
+                        : String(outcome.error);
+                continue;
+            }
+            const { result } = outcome;
             if (result.kind === "reconciliation" ||
                 result.kind === "timeout" ||
                 result.terminal !== true) {
-                return reconcile(`attempt cancellation is not terminal: ${reason}`);
+                cancellationFailure ??= `attempt cancellation is not terminal: ${reason}`;
+                continue;
             }
             if (result.providerSessionId &&
                 current.providerSessionId &&
                 result.providerSessionId !== current.providerSessionId) {
-                return reconcile("cancellation returned a different host session");
+                cancellationFailure ??=
+                    "cancellation returned a different host session";
+                continue;
             }
             if (result.providerSessionId) {
                 current.providerSessionId = result.providerSessionId;
                 current.context.providerSessionId = result.providerSessionId;
             }
             try {
-                if (result.kind === "report")
+                await resumeFromWait(current);
+                if (result.kind === "report") {
                     await complete(current, false);
+                }
                 else
                     await fail(current, result.reason ?? "cancelled by coordinator", false);
             }
             catch (error) {
-                return reconcile(error instanceof Error ? error.message : String(error));
+                cancellationFailure ??=
+                    error instanceof Error ? error.message : String(error);
+                continue;
             }
             active.delete(current.slot);
             waits.delete(current.slot);
         }
-        if (pending.length > 0)
-            return reconcile(`queued assignment remains after cancellation: ${reason}`);
+        if (cancellationFailure)
+            return reconcile(cancellationFailure);
         return settle();
     }
     async function fail(activeWork, reason, allowRetry = true) {
@@ -657,7 +924,7 @@ export async function runTeamworkCoordinator(input) {
                 : {}),
             terminal: true,
         });
-        if (allowRetry && retryCount < assignment.maxRetries) {
+        if (allowRetry && !admissionClosed && retryCount < assignment.maxRetries) {
             await release(activeWork);
             retryCounts[assignment.assignmentId] = retryCount + 1;
             pending.unshift(assignment);
@@ -743,7 +1010,6 @@ export async function runTeamworkCoordinator(input) {
         });
         await release(activeWork);
         final.set(assignment.assignmentId, "ACCEPTED");
-        acceptedAttempts.set(assignment.assignmentId, context.attemptId);
     }
     async function complete(activeWork, allowRetry = true) {
         const { assignment, context, retryCount } = activeWork;
@@ -782,9 +1048,23 @@ export async function runTeamworkCoordinator(input) {
             });
         }
         catch (error) {
+            if (error instanceof TeammateReportReadError && error.kind !== "missing")
+                throw error;
             await fail(activeWork, error instanceof Error ? error.message : String(error), allowRetry);
             return;
         }
+        if (context.launch.reportVersion === 2 && report.schemaVersion !== 2) {
+            await fail(activeWork, "report schema_version must match the frozen result contract (2)", allowRetry);
+            return;
+        }
+        if (report.status === "blocked") {
+            await fail(activeWork, `teammate reported blocked: ${report.evidence[0]?.summary ?? "blocker evidence missing"}`, false);
+            return;
+        }
+        if (report.status !== "complete") {
+            await fail(activeWork, `teammate reported ${report.status}`, allowRetry);
+            return;
+        }
         if (input.runtime.validateReport &&
             !(await input.runtime.validateReport(context, report))) {
             await fail(activeWork, "report rejected by acceptance contract", allowRetry);
@@ -870,7 +1150,6 @@ export async function runTeamworkCoordinator(input) {
         const cleanupFailures = cleanupSettledTeamworkArtifacts({
             taskRoot: input.taskRoot,
             assignments: plan.assignments,
-            acceptedAttempts,
         });
         if (cleanupFailures.length > 0) {
             return reconcile(`Teamwork batch cleanup failed: ${cleanupFailures.join("; ")}`);
@@ -918,20 +1197,44 @@ export async function runTeamworkCoordinator(input) {
             id: `${plan.batchId}:${status}`,
             payload: { ...receipt },
         });
-        await assertTeamworkReceiptDurable(plan.sessionId, plan.batchId, plan.planHash, cwd);
+        await assertTeamworkReceiptDurable(plan, receipt, cwd);
         await releaseGuard();
         return receipt;
     }
     async function fill() {
+        if (admissionClosed)
+            return;
         while (active.size < capacity && pending.length > 0) {
-            const assignment = pending.shift();
+            if (Date.parse(plan.deadline) <= Date.now()) {
+                await failPending("batch deadline elapsed before dispatch");
+                return;
+            }
+            try {
+                assertTeamworkPlanCurrent(plan, input.taskRoot);
+            }
+            catch (error) {
+                await failPending(`task phase changed before dispatch: ${error instanceof Error ? error.message : String(error)}`);
+                return;
+            }
+            const assignment = pending[0];
             if (!assignment)
                 return;
             const retryCount = retryCounts[assignment.assignmentId] ?? 0;
             let slot = 0;
             while (active.has(slot))
                 slot += 1;
-            await dispatch(slot, assignment, retryCount);
+            try {
+                await dispatch(slot, assignment, retryCount);
+                pending.shift();
+            }
+            catch (error) {
+                if (!assignmentStates.has(assignment.assignmentId) ||
+                    assignmentStates.get(assignment.assignmentId) === "QUEUED") {
+                    await failPending(`launch preflight failed before dispatch: ${error instanceof Error ? error.message : String(error)}`);
+                    return;
+                }
+                throw error;
+            }
         }
     }
     try {
@@ -991,162 +1294,116 @@ export async function runTeamworkCoordinator(input) {
                 result: next.result,
             });
         }
+        try {
+            await resumeFromWait(current);
+        }
+        catch (error) {
+            return reconcile(error instanceof Error ? error.message : String(error));
+        }
         let releaseSlot = true;
         if (next.result.kind === "timeout") {
             current.continuations += 1;
-            if (current.continuations > 8) {
-                return reconcile("continuation limit exceeded");
+            if (Date.now() >= Date.parse(plan.deadline))
+                return cancelAndDrain("batch deadline elapsed");
+            const transitionKey = `continuation:${current.continuations}`;
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "attempt",
+                id: current.context.attemptId,
+                state: "TIMEOUT_YIELDED",
+                states: attemptStates,
+                assignmentId: current.assignment.assignmentId,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                retryCount: current.retryCount,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "teammate",
+                id: current.context.teammateId,
+                state: "TIMEOUT_YIELDED",
+                states: teammateStates,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "assignment",
+                id: current.assignment.assignmentId,
+                state: "WAITING_FOR_MAIN_COMMAND",
+                states: assignmentStates,
+                assignmentId: current.assignment.assignmentId,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                retryCount: current.retryCount,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "attempt",
+                id: current.context.attemptId,
+                state: "WAITING_FOR_MAIN_COMMAND",
+                states: attemptStates,
+                assignmentId: current.assignment.assignmentId,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                retryCount: current.retryCount,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            await appendTransition({
+                sessionId: plan.sessionId,
+                by,
+                batchId: plan.batchId,
+                entity: "teammate",
+                id: current.context.teammateId,
+                state: "WAITING_FOR_MAIN_COMMAND",
+                states: teammateStates,
+                attemptId: current.context.attemptId,
+                teammateId: current.context.teammateId,
+                ...(current.providerSessionId
+                    ? { providerSessionId: current.providerSessionId }
+                    : {}),
+                transitionKey,
+            });
+            let continued;
+            try {
+                continued = input.runtime.continue(current.context);
             }
-            else {
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "attempt",
-                    id: current.context.attemptId,
-                    state: "TIMEOUT_YIELDED",
-                    states: attemptStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "teammate",
-                    id: current.context.teammateId,
-                    state: "TIMEOUT_YIELDED",
-                    states: teammateStates,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "assignment",
-                    id: current.assignment.assignmentId,
-                    state: "WAITING_FOR_MAIN_COMMAND",
-                    states: assignmentStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "attempt",
-                    id: current.context.attemptId,
-                    state: "WAITING_FOR_MAIN_COMMAND",
-                    states: attemptStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "teammate",
-                    id: current.context.teammateId,
-                    state: "WAITING_FOR_MAIN_COMMAND",
-                    states: teammateStates,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                let continued;
-                try {
-                    continued = await input.runtime.continue(current.context);
-                }
-                catch (error) {
-                    return reconcile(error instanceof Error ? error.message : String(error));
-                }
-                if (continued.providerSessionId &&
-                    current.providerSessionId &&
-                    continued.providerSessionId !== current.providerSessionId) {
-                    return reconcile("continuation returned a different host session");
-                }
-                if (continued.providerSessionId) {
-                    current.providerSessionId = continued.providerSessionId;
-                    current.context.providerSessionId = continued.providerSessionId;
-                }
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "attempt",
-                    id: current.context.attemptId,
-                    state: "RUNNING",
-                    states: attemptStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "teammate",
-                    id: current.context.teammateId,
-                    state: "RUNNING",
-                    states: teammateStates,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                await appendTransition({
-                    sessionId: plan.sessionId,
-                    by,
-                    batchId: plan.batchId,
-                    entity: "assignment",
-                    id: current.assignment.assignmentId,
-                    state: "EXECUTING",
-                    states: assignmentStates,
-                    assignmentId: current.assignment.assignmentId,
-                    attemptId: current.context.attemptId,
-                    teammateId: current.context.teammateId,
-                    retryCount: current.retryCount,
-                    ...(current.providerSessionId
-                        ? { providerSessionId: current.providerSessionId }
-                        : {}),
-                    transitionKey: `continuation:${current.continuations}`,
-                });
-                waits.set(next.slot, Promise.resolve({ slot: next.slot, result: continued }));
-                releaseSlot = false;
+            catch (error) {
+                return reconcile(error instanceof Error ? error.message : String(error));
             }
+            waits.set(next.slot, Promise.resolve(continued)
+                .then((result) => ({ slot: next.slot, result }))
+                .catch((error) => ({
+                slot: next.slot,
+                result: {
+                    kind: "reconciliation",
+                    reason: error instanceof Error ? error.message : String(error),
+                },
+            })));
+            releaseSlot = false;
         }
         else if (next.result.kind === "report") {
             try {
@@ -1177,22 +1434,248 @@ export async function runTeamworkCoordinator(input) {
     }
     return settle();
 }
-export async function assertTeamworkReceiptDurable(sessionId, batchId, planHash, cwd) {
+export async function assertTeamworkReceiptDurable(plan, receipt, cwd) {
     const [state, events] = await Promise.all([
-        readCoordinationState({ sessionId, ...(cwd ? { cwd } : {}) }),
-        readTeamworkEvents({ sessionId, ...(cwd ? { cwd } : {}) }),
+        readCoordinationState({
+            sessionId: plan.sessionId,
+            ...(cwd ? { cwd } : {}),
+        }),
+        readTeamworkEvents({
+            sessionId: plan.sessionId,
+            ...(cwd ? { cwd } : {}),
+        }),
     ]);
-    const batch = state.batches.find((entry) => entry.batchId === batchId);
-    if (batch?.planHash !== planHash ||
-        (batch.state !== "ALL_ACCEPTED" && batch.state !== "SETTLED_WITH_FAILURE")) {
-        throw new Error(`batch '${batchId}' has no durable terminal receipt`);
-    }
-    const receiptRecorded = events.some((event) => event.kind === "coordination" &&
-        event.batchId === batchId &&
+    const expected = terminalReceiptFromProjection(plan, state);
+    if (!isDeepStrictEqual(receipt, expected))
+        throw new Error(`batch '${plan.batchId}' receipt disagrees with durable facts`);
+    const receipts = events.filter((event) => event.kind === "coordination" &&
+        event.batchId === plan.batchId &&
         event.action === "receipt_recorded" &&
-        event.payload?.status === batch.state);
-    if (!receiptRecorded) {
-        throw new Error(`batch '${batchId}' has no matching durable receipt event`);
+        event.payload?.status !== "RECONCILIATION_REQUIRED");
+    if (receipts.length !== 1 ||
+        !isDeepStrictEqual(receipts[0]?.payload, expected))
+        throw new Error(`batch '${plan.batchId}' has no matching durable receipt event`);
+}
+/** Retire uncertain work only from associated terminal or process-birth evidence. */
+export async function reconcileTeamworkBatch(input) {
+    const { plan, cwd } = input;
+    const projection = await readCoordinationState({
+        sessionId: plan.sessionId,
+        cwd,
+    });
+    const batch = projection.batches.find((entry) => entry.batchId === plan.batchId);
+    if (batch?.planHash !== plan.planHash)
+        throw new Error("Batch does not match the frozen plan");
+    const events = await readTeamworkEvents({ sessionId: plan.sessionId, cwd });
+    const resolving = events.some((event) => event.kind === "coordination" &&
+        event.batchId === plan.batchId &&
+        event.action === "reconciliation_resolved");
+    if (["ALL_ACCEPTED", "SETTLED_WITH_FAILURE"].includes(batch.state)) {
+        const receipt = terminalReceiptFromProjection(plan, projection);
+        if (input.execute) {
+            const recovered = await recoverExistingBatch({
+                plan,
+                cwd,
+                by: "main",
+                capacity: receipt.capacity,
+            });
+            if (!recovered)
+                throw new Error("Settled batch disappeared during reconciliation");
+            return recovered;
+        }
+        return receipt;
+    }
+    if (batch.state !== "RECONCILIATION_REQUIRED" &&
+        !(resolving && batch.state === "SETTLING"))
+        throw new Error("Recover the original batch into RECONCILIATION_REQUIRED first");
+    const attempts = projection.attempts.filter((entry) => entry.batchId === plan.batchId);
+    const blockers = projection.issues.filter((issue) => !issue.includes(": reconciliation required"));
+    for (const attempt of attempts) {
+        if (attempt.terminal)
+            continue;
+        const facts = events.filter((event) => event.kind === "coordination" &&
+            event.batchId === plan.batchId &&
+            event.attemptId === attempt.attemptId &&
+            event.planHash === plan.planHash);
+        if (facts.some((event) => event.kind === "coordination" && event.action === "process_exited"))
+            continue;
+        const started = facts
+            .filter((event) => event.kind === "coordination" && event.action === "process_started")
+            .pop();
+        const pid = started?.kind === "coordination" ? started.payload?.pid : undefined;
+        const birth = started?.kind === "coordination"
+            ? started.payload?.processIdentity
+            : undefined;
+        let exited = false;
+        if (typeof pid === "number" &&
+            Number.isSafeInteger(pid) &&
+            pid > 0 &&
+            typeof birth === "string" &&
+            birth.startsWith(`${pid}:`)) {
+            const current = readProcessIdentity(pid);
+            if (current)
+                exited = current !== birth;
+            else {
+                try {
+                    process.kill(pid, 0);
+                }
+                catch (error) {
+                    exited = error.code === "ESRCH";
+                }
+            }
+        }
+        if (!exited)
+            blockers.push(`Attempt ${attempt.attemptId} lacks confirmed process exit; preserve its lease`);
     }
+    if (!input.execute)
+        return { status: blockers.length ? "blocked" : "ready", blockers };
+    if (blockers.length)
+        throw new Error(blockers.join("; "));
+    if (!input.reason?.trim())
+        throw new Error("Reconciliation requires an explicit disposition reason");
+    const by = "main";
+    for (const attempt of attempts) {
+        await recordCoordinationEvent({
+            sessionId: plan.sessionId,
+            cwd,
+            by,
+            batchId: plan.batchId,
+            entity: "attempt",
+            action: "terminal_observed",
+            attemptId: attempt.attemptId,
+            teammateId: attempt.teammateId,
+            terminal: true,
+            reason: input.reason,
+            idempotencyKey: `${plan.batchId}:reconciled-terminal:${attempt.attemptId}`,
+        });
+        if (!["FINAL_FAILED", "REPORT_SUBMITTED"].includes(attempt.state))
+            await appendTransition({
+                sessionId: plan.sessionId,
+                cwd,
+                by,
+                batchId: plan.batchId,
+                entity: "attempt",
+                id: attempt.attemptId,
+                attemptId: attempt.attemptId,
+                assignmentId: attempt.assignmentId,
+                teammateId: attempt.teammateId,
+                retryCount: attempt.retryCount,
+                state: "FINAL_FAILED",
+                terminal: true,
+                states: new Map([[attempt.attemptId, attempt.state]]),
+                transitionKey: "reconcile",
+            });
+        const teammate = projection.teammates.find((entry) => entry.batchId === plan.batchId &&
+            entry.teammateId === attempt.teammateId);
+        if (teammate && !["FAILED", "REPORT_SUBMITTED"].includes(teammate.state))
+            await appendTransition({
+                sessionId: plan.sessionId,
+                cwd,
+                by,
+                batchId: plan.batchId,
+                entity: "teammate",
+                id: teammate.teammateId,
+                teammateId: teammate.teammateId,
+                attemptId: attempt.attemptId,
+                state: "FAILED",
+                terminal: true,
+                states: new Map([[teammate.teammateId, teammate.state]]),
+                transitionKey: "reconcile",
+            });
+    }
+    for (const assignment of projection.assignments.filter((entry) => entry.batchId === plan.batchId)) {
+        if (!["ACCEPTED", "FINAL_FAILED"].includes(assignment.state))
+            await appendTransition({
+                sessionId: plan.sessionId,
+                cwd,
+                by,
+                batchId: plan.batchId,
+                entity: "assignment",
+                id: assignment.assignmentId,
+                assignmentId: assignment.assignmentId,
+                attemptId: assignment.attemptId,
+                teammateId: assignment.teammateId,
+                retryCount: assignment.retryCount,
+                state: "FINAL_FAILED",
+                states: new Map([[assignment.assignmentId, assignment.state]]),
+                transitionKey: "reconcile",
+                payload: { reason: input.reason },
+            });
+    }
+    for (const lease of projection.leases.filter((entry) => entry.batchId === plan.batchId))
+        await recordCoordinationEvent({
+            sessionId: plan.sessionId,
+            cwd,
+            by,
+            batchId: plan.batchId,
+            entity: "attempt",
+            action: "lease_released",
+            leaseId: lease.leaseId,
+            protected: false,
+            reason: input.reason,
+            idempotencyKey: `${plan.batchId}:reconciled-release:${lease.leaseId}`,
+        });
+    const accepted = projection.assignments
+        .filter((entry) => entry.batchId === plan.batchId)
+        .every((entry) => entry.state === "ACCEPTED");
+    const states = new Map([
+        [plan.batchId, batch.state],
+    ]);
+    await recordCoordinationEvent({
+        sessionId: plan.sessionId,
+        cwd,
+        by,
+        batchId: plan.batchId,
+        entity: "batch",
+        action: "reconciliation_resolved",
+        reason: input.reason,
+        idempotencyKey: `${plan.batchId}:reconciled`,
+    });
+    if (batch.state !== "SETTLING")
+        await appendTransition({
+            sessionId: plan.sessionId,
+            cwd,
+            by,
+            batchId: plan.batchId,
+            entity: "batch",
+            id: plan.batchId,
+            state: "SETTLING",
+            states,
+            transitionKey: "reconcile",
+        });
+    await appendTransition({
+        sessionId: plan.sessionId,
+        cwd,
+        by,
+        batchId: plan.batchId,
+        entity: "batch",
+        id: plan.batchId,
+        state: accepted ? "ALL_ACCEPTED" : "SETTLED_WITH_FAILURE",
+        states,
+        transitionKey: "reconcile",
+    });
+    const receipt = terminalReceiptFromProjection(plan, await readCoordinationState({ sessionId: plan.sessionId, cwd }));
+    await appendAction({
+        sessionId: plan.sessionId,
+        cwd,
+        by,
+        batchId: plan.batchId,
+        entity: "batch",
+        action: "receipt_recorded",
+        id: `${plan.batchId}:${receipt.status}`,
+        payload: { ...receipt },
+    });
+    await assertTeamworkReceiptDurable(plan, receipt, cwd);
+    return receipt;
+}
+/** Read settled facts without admitting work or consulting current resource policy. */
+export async function readSettledTeamworkReceipt(plan, cwd) {
+    const state = await readCoordinationState({ sessionId: plan.sessionId, cwd });
+    const batch = state.batches.find((entry) => entry.batchId === plan.batchId);
+    if (!batch || !["ALL_ACCEPTED", "SETTLED_WITH_FAILURE"].includes(batch.state))
+        return undefined;
+    const receipt = terminalReceiptFromProjection(plan, state);
+    await assertTeamworkReceiptDurable(plan, receipt, cwd);
+    return receipt;
 }
 //# sourceMappingURL=coordinator.js.map