@sema-agent/core 7.5.0 → 7.5.2

package/dist/core/runner/prepare-suspend-saga.js

@@ -0,0 +1,308 @@
+import { defaultTaskRegistry } from "../task-registry.js";
+import { isRemoteExecutionEnv, isSuspendable, RemoteExecutionError } from "../remote-env.js";
+import { constraintChainDigest, constraintChainEntryOfLayer } from "../tool-policy.js";
+import { raceSettlementAgainstSignal } from "./abort-race.js";
+import { placementValueOrAbsent } from "./checkpoint-scope.js";
+import { serializeAnnouncedListings } from "./announce-once-ledger.js";
+import { remoteEnvFailureNote, restoreWorkspaceWithRetry } from "./remote-env-retry.js";
+const PARK_COMPENSATION_TIMEOUT_MS = 30_000;
+export async function compensateUnparkedPause(remoteEnv, snapshotId, io) {
+    const boundCtl = new AbortController();
+    const boundTimer = setTimeout(() => boundCtl.abort(), io.boundMs);
+    const bound = boundCtl.signal;
+    let phase = "resumeVM";
+    let restoreAttempts = 1;
+    let vmRunning = false;
+    const work = (async () => {
+        const { outcome: back, attempts: backAttempts } = await restoreWorkspaceWithRetry(remoteEnv, snapshotId, { abortSignal: bound }, (attempt) => (restoreAttempts = attempt));
+        if (!back.ok) {
+            io.noteFailure(remoteEnvFailureNote("resumeVM", back.error, backAttempts));
+            io.disclose(back.error);
+            return { ok: false, reason: `resumeVM failed (${back.error.code}) while compensating an unparked pause` };
+        }
+        vmRunning = true;
+        if (bound.aborted) {
+            return { ok: false, reason: "resumeVM settled only after the decision bound" };
+        }
+        phase = "postResumeInit";
+        const init = await remoteEnv.postResumeInit({ abortSignal: bound });
+        if (!init.ok) {
+            io.noteFailure(remoteEnvFailureNote("postResumeInit", init.error, 1));
+            io.disclose(init.error);
+            return { ok: false, reason: `postResumeInit failed (${init.error.code}) while compensating an unparked pause` };
+        }
+        return { ok: true };
+    })();
+    const raced = await raceSettlementAgainstSignal(work, bound);
+    clearTimeout(boundTimer);
+    if (raced.tag === "value")
+        return raced.value;
+    if (raced.tag === "threw") {
+        io.noteFailure(remoteEnvFailureNote(phase, raced.error instanceof RemoteExecutionError
+            ? raced.error
+            : new RemoteExecutionError("unknown", `the pause compensation's ${phase} call threw: ${raced.error instanceof Error ? raced.error.message : String(raced.error)}`), phase === "resumeVM" ? restoreAttempts : 1));
+        io.disclose(raced.error);
+        return {
+            ok: false,
+            reason: "the pause compensation threw (adapter contract violation; the deployment's error face carries the exception)",
+        };
+    }
+    const timedOut = new RemoteExecutionError("timeout", `the pause compensation's ${phase} call did not settle within the ${io.boundMs}ms decision bound`);
+    io.noteFailure(remoteEnvFailureNote(phase, timedOut, phase === "resumeVM" ? restoreAttempts : 1));
+    io.disclose(timedOut);
+    const destroyOwnerlessLate = () => {
+        io.disclose(new Error(`park compensation settled after its ${io.boundMs}ms decision bound: the VM is running with no run ` +
+            `left to own it — destroying the env (an ownerless running VM must not linger; the run already took ` +
+            `the fail-closed arm when the bound fired)`));
+        void Promise.resolve()
+            .then(() => remoteEnv.destroy())
+            .catch((destroyErr) => io.disclose(destroyErr));
+    };
+    void work.then((late) => {
+        if (vmRunning) {
+            destroyOwnerlessLate();
+            return;
+        }
+        void late;
+    }, (lateErr) => {
+        io.disclose(lateErr);
+        if (vmRunning)
+            destroyOwnerlessLate();
+    });
+    return {
+        ok: false,
+        reason: `the pause compensation did not settle within ${io.boundMs}ms — treated as failed (a late-restored VM is destroyed; a late restore failure is disclosed)`,
+    };
+}
+function stampPlacementRootSessionId(placementRootResolved) {
+    return placementValueOrAbsent(placementRootResolved);
+}
+function autoModeLatchHealthy(d) {
+    try {
+        return d.breakerOpen() !== true && typeof d.consecutiveFailures === "function" && d.consecutiveFailures() === 0;
+    }
+    catch {
+        return false;
+    }
+}
+export function prepareSuspendSaga(input) {
+    const { gateMachineryActive, abortController, harness, checkpointStore, deps, sessionId, hostTaskId, taskScope, liveSpendRef, resume, nestedStats, internals, activeTools, outputRef, readFileStateForCheckpoint, faceCheckpointSection, reminderMark, handsCwdRef, worktreeSessionRef, inheritedParentConstraints, seedInheritedGate, inheritedAncestorRules, inheritedShellGate, autoModeIntent, autoModeDecider, inheritedAdmittedOrgScopes, ownOrgVerdictRef, orgGovernedProvenance, announcedListingsRef, gitStatusRef, hookIdentity, placementRootResolved, externalContentTargetActive, remoteEnvFailures, memoryEngineSession, suspendLoopRef, ownedEnv, incompleteSuspendAdapter } = input;
+    if (!gateMachineryActive)
+        return { saga: undefined };
+    const inFlightSpendMicroUsd = () => {
+        const own = liveSpendRef.get?.().costMicroUsd ?? 0;
+        const seededNested = resume?.seed.nestedStats.costMicroUsd;
+        const nestedDelta = nestedStats.costMicroUsd - (typeof seededNested === "number" && Number.isFinite(seededNested) ? seededNested : 0);
+        return (Number.isFinite(own) ? own : 0) + Math.max(0, Number.isFinite(nestedDelta) ? nestedDelta : 0);
+    };
+    const repairBundleForCheckpoint = (parkedSpendMicroUsd) => {
+        const carried = internals?.repairBundle !== undefined
+            ? structuredClone(internals.repairBundle)
+            : resume?.seed.repairBundle !== undefined
+                ? structuredClone(resume.seed.repairBundle)
+                : undefined;
+        if (carried === undefined)
+            return undefined;
+        if (parkedSpendMicroUsd === undefined || !Number.isFinite(parkedSpendMicroUsd) || parkedSpendMicroUsd <= 0)
+            return carried;
+        const prior = typeof carried.spentMicroUsd === "number" && Number.isFinite(carried.spentMicroUsd) ? Math.max(0, carried.spentMicroUsd) : 0;
+        carried.spentMicroUsd = prior + parkedSpendMicroUsd;
+        return carried;
+    };
+    const screeningParkDisclosedRef = { done: false };
+    const serializeCheckpointState = (workspaceHandle, parkedSpendMicroUsd) => ({
+        activeTools: [...activeTools],
+        outputRef: { value: outputRef.value, set: outputRef.set },
+        nestedStats: { ...nestedStats },
+        consolidationNotes: undefined,
+        readFileState: readFileStateForCheckpoint ? [...readFileStateForCheckpoint.entries()] : undefined,
+        readFace: faceCheckpointSection(),
+        reminderMark,
+        repairBundle: repairBundleForCheckpoint(parkedSpendMicroUsd),
+        workspaceHandle,
+        handsCwd: handsCwdRef?.current,
+        activeWorktree: worktreeSessionRef?.current ? { ...worktreeSessionRef.current } : undefined,
+        runningBackgroundTasks: (() => {
+            const live = defaultTaskRegistry
+                .list({ owner: hostTaskId, scope: taskScope, sessionId })
+                .filter((t) => t.status === "pending" || t.status === "running")
+                .map((t) => ({ id: t.task_id, ...(t.description !== undefined ? { description: t.description } : {}) }));
+            return live.length > 0 ? live : undefined;
+        })(),
+        pendingSteer: undefined,
+        pendingSteerQueue: undefined,
+        inheritedGate: (() => {
+            const requiresParentConstraint = (inheritedParentConstraints?.length ?? 0) > 0 || seedInheritedGate?.requiresParentConstraint === true;
+            const parentConstraintCount = (inheritedParentConstraints?.length ?? 0) > 0 ? inheritedParentConstraints.length : seedInheritedGate?.parentConstraintCount;
+            const constraintChain = (inheritedParentConstraints?.length ?? 0) > 0
+                ? inheritedParentConstraints.map((pc) => constraintChainEntryOfLayer(pc))
+                : seedInheritedGate?.constraintChain;
+            const constraintDigest = (inheritedParentConstraints?.length ?? 0) > 0
+                ? constraintChainDigest(constraintChain)
+                : seedInheritedGate?.constraintDigest;
+            const autoModeIntentCarried = autoModeIntent && (autoModeDecider === undefined || autoModeLatchHealthy(autoModeDecider));
+            return inheritedAncestorRules !== undefined ||
+                inheritedShellGate !== undefined ||
+                autoModeIntentCarried ||
+                inheritedAdmittedOrgScopes !== undefined ||
+                ownOrgVerdictRef.current !== undefined ||
+                orgGovernedProvenance ||
+                requiresParentConstraint
+                ? {
+                    ...(inheritedAncestorRules !== undefined ? { ancestorRules: structuredClone(inheritedAncestorRules) } : {}),
+                    ...(inheritedShellGate !== undefined ? { shellGate: inheritedShellGate } : {}),
+                    ...(autoModeIntentCarried ? { autoModeRequested: true } : {}),
+                    ...(inheritedAdmittedOrgScopes !== undefined ? { admittedOrgScopes: [...inheritedAdmittedOrgScopes] } : {}),
+                    ...(ownOrgVerdictRef.current !== undefined
+                        ? { ownAdmittedOrgScopes: [...ownOrgVerdictRef.current.scopes], ownAdmittedOrgWriteScope: ownOrgVerdictRef.current.writeScope }
+                        : {}),
+                    ...(orgGovernedProvenance ? { orgAdmissionGoverned: true } : {}),
+                    requiresParentConstraint,
+                    ...(requiresParentConstraint && parentConstraintCount !== undefined ? { parentConstraintCount } : {}),
+                    ...(requiresParentConstraint && constraintChain !== undefined
+                        ? { constraintChain: structuredClone(constraintChain), ...(constraintDigest !== undefined ? { constraintDigest } : {}) }
+                        : {}),
+                }
+                : undefined;
+        })(),
+        announcedListings: serializeAnnouncedListings(announcedListingsRef),
+        gitAnnouncement: gitStatusRef.announced !== undefined ? { ...gitStatusRef.announced } : undefined,
+        delegationProvenance: internals?.delegationProvenance !== undefined ? { ...internals.delegationProvenance.ref.current } : undefined,
+        isDelegatedChild: hookIdentity.isDelegatedChild ? true : undefined,
+        placementRootSessionId: stampPlacementRootSessionId(placementRootResolved),
+        externalContentTarget: externalContentTargetActive ? true : undefined,
+    });
+    const compensatePausedVM = (remoteEnv, snapshotId) => compensateUnparkedPause(remoteEnv, snapshotId, {
+        boundMs: PARK_COMPENSATION_TIMEOUT_MS,
+        noteFailure: (note) => remoteEnvFailures.push(note),
+        disclose: (err) => {
+            try {
+                deps.onError?.(err, { phase: "config", sessionId });
+            }
+            catch {
+            }
+        },
+    });
+    const commitSuspendSaga = async (token, cp, remoteEnv, remoteHandle, cutSignal) => {
+        if (!checkpointStore)
+            return { tag: "absent" };
+        if (memoryEngineSession)
+            await memoryEngineSession.harvest("checkpoint");
+        if (cutSignal?.aborted) {
+            if (remoteEnv !== undefined && remoteHandle?.snapshotId !== undefined) {
+                const back = await compensatePausedVM(remoteEnv, remoteHandle.snapshotId);
+                if (!back.ok) {
+                    abortController.abort();
+                    void harness.abort();
+                    return { tag: "compensation_failed", reason: back.reason };
+                }
+            }
+            return { tag: "cut" };
+        }
+        const announceCommittedScreeningPark = () => {
+            const committedCount = cp.state.inheritedGate?.parentConstraintCount;
+            if (cp.state.inheritedGate?.requiresParentConstraint === true &&
+                !screeningParkDisclosedRef.done &&
+                (inheritedParentConstraints ?? []).some((pc) => pc.preToolUse !== undefined)) {
+                screeningParkDisclosedRef.done = true;
+                try {
+                    deps.onError?.(new Error(`durable park under an inherited PreToolUse SCREENING constraint: this checkpoint records ` +
+                        `requiresParentConstraint with parentConstraintCount=${committedCount ?? "(unrecorded)"}, and the live ` +
+                        `closures cannot be persisted. A resume on THIS Runner re-supplies them automatically; a resume on a fresh ` +
+                        `Runner (restart / another replica) must hand back the whole chain via resumeStream(..., internals), ` +
+                        `rebuilding the screening entry with createPreToolUseConstraintPolicy(hook, env) — otherwise the row stays ` +
+                        `pending. If the face only observes, declare Hooks.preToolUseObservational and it stops entering the chain.`), { phase: "degraded", sessionId, classification: "screening-constraint-in-durable-chain" });
+                }
+                catch {
+                }
+            }
+        };
+        const confirmPutOutcome = async (putErr) => {
+            if (putErr?.code === "checkpoint.already_exists")
+                return "absent";
+            try {
+                const row = await checkpointStore.get(token);
+                if (row == null)
+                    return "absent";
+                const sameCall = (a, b) => a.kind === b.kind && (a.kind !== "tool_approval" || b.kind !== "tool_approval" || a.toolCallId === b.toolCallId);
+                const isOurs = row.scope === cp.scope && row.sessionId === cp.sessionId && row.leafId === cp.leafId && sameCall(row.pendingAction, cp.pendingAction);
+                return isOurs ? "committed" : "absent";
+            }
+            catch (readErr) {
+                deps.onError?.(readErr, { phase: "config", sessionId });
+                return "unknown";
+            }
+        };
+        try {
+            await checkpointStore.put(token, cp);
+            announceCommittedScreeningPark();
+            return { tag: "committed" };
+        }
+        catch (putErr) {
+            deps.onError?.(putErr, { phase: "config", sessionId });
+            const confirmed = await confirmPutOutcome(putErr);
+            if (confirmed === "committed") {
+                try {
+                    deps.onError?.(new Error(`durable approval checkpoint: the store's write reported a failure but the row EXISTS — the commit ` +
+                        `landed and its acknowledgement was lost (a network backend can complete the INSERT and drop the ` +
+                        `connection before replying). The run suspends on that row rather than continuing, which would leave ` +
+                        `a redeemable pending checkpoint behind. This backend's acknowledgement path is lossy — the operator ` +
+                        `face carries the store's own message.`), { phase: "degraded", sessionId, classification: "checkpoint-put-confirmation-lost" });
+                }
+                catch {
+                }
+                announceCommittedScreeningPark();
+                return { tag: "committed" };
+            }
+            if (confirmed === "unknown") {
+                const unknownReason = "the approval checkpoint's state cannot be established (the store rejected the write and then could not be read back; a committed-but-unacknowledged row may exist) — the run is stopped rather than continued past an approval whose durable record is unknown";
+                try {
+                    deps.onError?.(new Error(`durable approval checkpoint: ${unknownReason}`), {
+                        phase: "degraded",
+                        sessionId,
+                        classification: "checkpoint-put-outcome-unknown",
+                    });
+                }
+                catch {
+                }
+                abortController.abort();
+                void harness.abort();
+                return { tag: "unknown", reason: unknownReason };
+            }
+            const reason = "the approval checkpoint could not be persisted (the store rejected the write; the deployment's error face carries the store's own message)";
+            if (remoteEnv !== undefined && remoteHandle?.snapshotId !== undefined) {
+                const back = await compensatePausedVM(remoteEnv, remoteHandle.snapshotId);
+                if (back.ok)
+                    return { tag: "absent", reason };
+                abortController.abort();
+                void harness.abort();
+                return { tag: "compensation_failed", reason };
+            }
+            return { tag: "absent", reason };
+        }
+    };
+    const suspendLoopCapHit = (count, cap, detail) => {
+        if (cap === undefined)
+            return false;
+        if (count + 1 <= cap)
+            return false;
+        suspendLoopRef.hit = true;
+        abortController.abort();
+        void harness.abort();
+        try {
+            deps.onError?.(new Error(`suspend loop: task already suspended ${count} time(s) (max ${cap}); refusing to suspend again${detail}`), { phase: "config", sessionId });
+        }
+        catch {
+        }
+        return true;
+    };
+    const suspendableEnv = ownedEnv !== undefined && isSuspendable(ownedEnv) ? ownedEnv : undefined;
+    const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && incompleteSuspendAdapter === undefined
+        ? ownedEnv
+        : undefined;
+    const parkOnlyHandle = (env) => {
+        const { snapshotId: _lineage, ...identity } = env.workspaceHandle();
+        return { ...identity, restoreMode: "park_only" };
+    };
+    return { saga: { inFlightSpendMicroUsd, serializeCheckpointState, compensatePausedVM, commitSuspendSaga, suspendLoopCapHit, suspendableEnv, parkOnlyRemoteEnv, parkOnlyHandle } };
+}