@inflexa-ai/harness 0.46.2 → 0.47.0

package/dist/sandbox/leases.js

@@ -0,0 +1,519 @@
+/**
+ * The sandbox module (sandbox-leases spec). A consumer holds a sandbox through a lease, and the module owns each
+ * machine: the admission, the spawn, the replacement, the idle removal, and the teardown.
+ *
+ * Each generation of a lease has one lease workflow. A consumer enqueues it through a `DBOSClient`, thus the lease
+ * workflow is not a child of the consumer: a cancel, a pause, or a crash of the consumer does not stop it, and its
+ * teardown step can always run. The consumer waits for the events of the lease workflow with the same client,
+ * because DBOS refuses `recv` and `getEvent` inside a step, and each tool call is a step.
+ *
+ * A machine of a lease is replaceable. The state of a lease is its files, the exec records in its records
+ * directory, and the checkpoints of its consumer. Thus the module removes the machine of an idle lease, and the
+ * next exec gets a new one.
+ */
+import { rm } from "node:fs/promises";
+import { setTimeout as sleep } from "node:timers/promises";
+import { DBOS, Error as DBOSErrors, StatusString } from "@dbos-inc/dbos-sdk";
+import { err, ok } from "neverthrow";
+import { z } from "zod";
+import { clampResources } from "../config/resource-limits.js";
+import { createNoopLogger } from "../lib/console-logger.js";
+import { recordLeaseWait, recordMachineReplaced, recordMachineStart, trackLeaseWorkflow } from "../lib/metrics.js";
+import { unwrapOrThrow } from "../lib/result.js";
+import { errorMessageOf } from "../lib/thrown.js";
+import { textOf } from "../lib/unknown-text.js";
+import { queryPausesOfWork } from "../state/paused-work.js";
+import { activeLease, clearIdleMachine, leasesOwnedBy, clearMachine, latestGeneration, oldestWaiting, readLease, releaseLease, requestLease, setMachine, touchLease, tryGrant, } from "../state/sandbox-leases.js";
+import { pauseAtGate, workPause } from "../workflows/pause.js";
+import { tailWritePrefix } from "../workspace/paths.js";
+import { noteExecOutcome, summarizeExec } from "./exec-outcome.js";
+import { saltedStepExecId } from "./exec.js";
+import { keyedSandboxIdentity } from "./identity.js";
+import { leaseRecordTail, leaseWorkflowIdOf } from "./lease-names.js";
+import { registerSandboxReconciler } from "./reconciler.js";
+import { SandboxFailure } from "./sandbox-error.js";
+export const LEASE_WORKFLOW = "sandbox-lease";
+export const LEASE_QUEUE = "sandbox-leases";
+const LEASE_TOPIC = "lease";
+const GRANTED_EVENT = "granted";
+/** The tick of a lease workflow: each tick reads the owner and tests the lease for idleness. */
+const DEFAULT_TICK_SECONDS = 60;
+/** One slice of a wait of a consumer. Between two slices the wait reads the cancel state and mirrors a pause. */
+const WAIT_SLICE_SECONDS = 10;
+/** The poll of a wait for an event. A client outside the engine reads the event table, and no notification wakes it. */
+const EVENT_POLL_MS = 500;
+export const DEFAULT_IDLE_PERIOD_MS = 15 * 60_000;
+/** The timeout of a command that gives none. A step has no deadline of its own, thus each command has one. */
+export const DEFAULT_COMMAND_TIMEOUT_SECONDS = 3600;
+/** The time that the poll of an exec waits past the timeout of its command, for the last result. */
+const POLL_GRACE_MS = 120_000;
+const HEALTH_TIMEOUT_MS = 10_000;
+/**
+ * The health answer of sandbox-server. An image that predates the count answers with no count, and the module then
+ * reads the machine as busy: an idle removal on a guess could stop a command that runs.
+ */
+const HealthSchema = z.object({ runningExecs: z.number().int().nonnegative().optional() });
+const TERMINAL_STATUSES = new Set([StatusString.SUCCESS, StatusString.ERROR, StatusString.CANCELLED, StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED]);
+function machineEventKey(generation) {
+    return `machine:${generation}`;
+}
+function sameSize(a, b) {
+    return a.cpu === b.cpu && a.memoryGb === b.memoryGb && (a.gpu?.count ?? 0) === (b.gpu?.count ?? 0);
+}
+/** Run `fn` as a step when the caller is a workflow body, and inline when the caller is a step or no workflow. */
+function durably(name, fn) {
+    return DBOS.isInWorkflow() ? DBOS.runStep(fn, { name }) : fn();
+}
+/** True once the workflow of the caller is canceled. Outside a workflow, nothing can cancel the caller. */
+async function callerCanceled() {
+    const workflowId = DBOS.workflowID;
+    if (workflowId === undefined)
+        return false;
+    const status = await DBOS.getWorkflowStatus(workflowId);
+    return status?.status === StatusString.CANCELLED;
+}
+export function createSandboxLeases(config) {
+    const logger = (config.logger ?? createNoopLogger()).named("sandbox-leases");
+    const { pool, client } = config;
+    const idlePeriodMs = config.idlePeriodMs ?? DEFAULT_IDLE_PERIOD_MS;
+    const fetchImpl = config.fetch ?? fetch;
+    const engine = config.dbosClient;
+    const tickSeconds = config.tickSeconds ?? DEFAULT_TICK_SECONDS;
+    let queueRegistered;
+    const registerQueue = (dbos) => {
+        queueRegistered ??= dbos.registerQueue(LEASE_QUEUE, { onConflict: "never_update" }).catch((cause) => {
+            queueRegistered = undefined;
+            throw cause;
+        });
+        return queueRegistered;
+    };
+    registerSandboxReconciler({
+        pool,
+        sandboxClient: client,
+        resolveWorkspaceRoot: config.resolveWorkspaceRoot,
+        cancel: async (workflowId) => (await engine()).cancelWorkflow(workflowId),
+        logger: config.logger,
+    });
+    const removeRecords = async (analysisId, leaseKey) => {
+        const dir = tailWritePrefix({ workspaceRoot: await config.resolveWorkspaceRoot(analysisId), tail: leaseRecordTail(leaseKey) });
+        await rm(dir, { recursive: true, force: true });
+    };
+    const teardownQuietly = async (ref, cause) => {
+        try {
+            await client.teardown(ref);
+        }
+        catch (failure) {
+            // The reconciler removes a machine whose lease ended, thus a failed teardown leaks nothing for long.
+            logger.warn("the teardown of a machine failed", { sandboxId: ref.sandboxId, ...logger.errorFields(failure) });
+        }
+        recordMachineReplaced({ cause });
+    };
+    // ── The lease workflow ────────────────────────────────────────────
+    const runLease = async (input) => {
+        const log = logger.with({ leaseWorkflowId: input.leaseWorkflowId, ownerWorkflowId: input.ownerWorkflowId });
+        const id = input.leaseWorkflowId;
+        const pause = workPause({
+            pool,
+            workId: input.work.id,
+            workKind: input.work.kind,
+            analysisId: input.analysisId,
+            // A host reads the streams of the work, not of the lease workflow. The wait of the consumer mirrors the pause.
+            emit: () => Promise.resolve(),
+            retryDelaysMs: config.pauseRetryDelaysMs,
+            run: { session: input.session, authorizer: config.runAuthorizer },
+            logger: log,
+        });
+        const ownerEnded = () => DBOS.runStep(async () => {
+            const status = await DBOS.getWorkflowStatus(input.ownerWorkflowId);
+            return status === null || TERMINAL_STATUSES.has(status.status);
+        }, { name: "lease.owner" });
+        const wakeNext = async () => {
+            const next = await DBOS.runStep(async () => unwrapOrThrow(await oldestWaiting(pool)), { name: "lease.next" });
+            if (next !== undefined && next !== id)
+                await DBOS.send(next, { kind: "capacity" }, LEASE_TOPIC);
+        };
+        const end = async (reason) => {
+            await DBOS.runStep(async () => {
+                const released = unwrapOrThrow(await releaseLease(pool, id, reason));
+                if (released?.machine !== undefined)
+                    await teardownQuietly(released.machine, "release");
+                await removeRecords(input.analysisId, input.leaseKey);
+            }, { name: "lease.release" });
+            log.info("lease released", { reason });
+            await wakeNext();
+        };
+        /** `grantedAtMs`: the time of the grant, given only for the first machine, whose start ends the wait of the request. */
+        const spawn = (machineGeneration, grantedAtMs) => DBOS.runStep(async () => {
+            const identity = keyedSandboxIdentity(input.session.runFrame.runId, `${id}#${machineGeneration}`);
+            const startedAt = Date.now();
+            const created = await client.createSandbox(input.session, {
+                ownerWorkflowId: id,
+                recordTail: leaseRecordTail(input.leaseKey),
+                resources: input.size,
+                image: input.spawn.image,
+                extraEnv: input.spawn.extraEnv === undefined ? undefined : { ...input.spawn.extraEnv },
+                readOnly: input.spawn.readOnly,
+                writableTail: input.spawn.writableTail,
+            }, identity);
+            if (created.isErr()) {
+                const failure = created.error;
+                if (failure.type === "labels_refused") {
+                    return failure.suspend ? { kind: "suspended", reason: failure.reason } : { kind: "refused", reason: failure.reason };
+                }
+                return { kind: "failed", detail: new SandboxFailure(failure).message };
+            }
+            unwrapOrThrow(await setMachine(pool, id, created.value, machineGeneration));
+            recordMachineStart({ backend: created.value.backend, startMs: Date.now() - startedAt });
+            if (grantedAtMs !== undefined)
+                recordLeaseWait({ backend: created.value.backend, waitMs: grantedAtMs - input.requestedAtMs });
+            return { kind: "ready", ref: created.value };
+        }, { name: "lease.spawn" });
+        /** Start a machine. A gate refusal with the suspend flag waits in a pause under the work of the consumer. */
+        const startMachine = async (machineGeneration, grantedAtMs) => {
+            for (;;) {
+                const outcome = await spawn(machineGeneration, grantedAtMs);
+                if (outcome.kind === "suspended") {
+                    await pauseAtGate(pause, outcome.reason);
+                    if (await ownerEnded())
+                        return { kind: "refused", reason: "owner-ended" };
+                    continue;
+                }
+                const event = outcome.kind === "ready" ? outcome : { kind: "failed", detail: outcome.kind === "refused" ? outcome.reason : outcome.detail };
+                await DBOS.setEvent(machineEventKey(machineGeneration), event);
+                return outcome;
+            }
+        };
+        // 1. The grant.
+        let size;
+        let grantedAtMs = 0;
+        while (size === undefined) {
+            const grant = await DBOS.runStep(async () => ({ outcome: unwrapOrThrow(await tryGrant(pool, id, config.resourcePolicy?.budget)), atMs: Date.now() }), { name: "lease.grant" });
+            if (grant.outcome.kind === "granted") {
+                size = grant.outcome.size;
+                grantedAtMs = grant.atMs;
+                break;
+            }
+            if (grant.outcome.kind === "released") {
+                await DBOS.setEvent(GRANTED_EVENT, { kind: "refused", reason: "released" });
+                return;
+            }
+            const message = await DBOS.recv(LEASE_TOPIC, tickSeconds);
+            if (message?.kind === "release" || (await ownerEnded())) {
+                await DBOS.setEvent(GRANTED_EVENT, { kind: "refused", reason: "released" });
+                await end(message?.kind === "release" ? message.reason : "owner-ended");
+                return;
+            }
+        }
+        // A free capacity can admit the next request too.
+        await wakeNext();
+        // 2. The first machine.
+        let machineGeneration = 1;
+        const first = await startMachine(machineGeneration, grantedAtMs);
+        if (first.kind !== "ready") {
+            const outcome = first.kind === "failed"
+                ? { kind: "failed", detail: first.detail }
+                : { kind: "refused", reason: first.kind === "refused" ? first.reason : "suspended" };
+            await DBOS.setEvent(GRANTED_EVENT, outcome);
+            await end(outcome.kind === "failed" ? "spawn-failed" : `refused:${outcome.reason}`);
+            return;
+        }
+        await DBOS.setEvent(GRANTED_EVENT, { kind: "granted", size });
+        // 3. The life of the lease.
+        let reason = "released";
+        for (;;) {
+            const message = await DBOS.recv(LEASE_TOPIC, tickSeconds);
+            if (message === null) {
+                const generation = machineGeneration;
+                const tick = await DBOS.runStep(() => tickOnce(id, generation, input.ownerWorkflowId), { name: "lease.tick" });
+                if (tick === "owner-ended") {
+                    reason = "owner-ended";
+                    break;
+                }
+                continue;
+            }
+            if (message.kind === "release") {
+                reason = message.reason;
+                break;
+            }
+            if (message.kind === "machine") {
+                // A request names the generation that its exec found empty. A later generation already answers it.
+                if (message.after < machineGeneration)
+                    continue;
+                const next = await startMachine(machineGeneration + 1);
+                if (next.kind !== "ready") {
+                    reason = next.kind === "failed" ? "spawn-failed" : `refused:${next.kind === "refused" ? next.reason : "suspended"}`;
+                    break;
+                }
+                machineGeneration += 1;
+                continue;
+            }
+            if (message.kind === "machine-lost") {
+                // The exec that saw the death cleared the machine from the row. The teardown removes what remains.
+                await DBOS.runStep(() => teardownQuietly(message.machine, message.oomKilled ? "oom" : "dead"), { name: "lease.lost" });
+            }
+        }
+        await end(reason);
+    };
+    const leaseWorkflow = DBOS.registerWorkflow(async (input) => {
+        const untrack = trackLeaseWorkflow();
+        try {
+            await runLease(input);
+        }
+        finally {
+            untrack();
+        }
+    }, { name: LEASE_WORKFLOW });
+    /** One tick of a granted lease: the end of the owner, a dead machine, and the idle removal. */
+    const tickOnce = async (id, machineGeneration, ownerWorkflowId) => {
+        const owner = await DBOS.getWorkflowStatus(ownerWorkflowId);
+        if (owner === null || TERMINAL_STATUSES.has(owner.status))
+            return "owner-ended";
+        const lease = unwrapOrThrow(await readLease(pool, id));
+        const machine = lease?.machine;
+        if (machine === undefined || lease?.machineGeneration !== machineGeneration)
+            return "none";
+        const running = await runningExecs(machine);
+        if (running === "unreachable") {
+            const liveness = await client.isAlive(machine).catch(() => undefined);
+            if (liveness?.alive !== false)
+                return "none";
+            const lost = unwrapOrThrow(await clearMachine(pool, id, machineGeneration));
+            if (lost !== undefined)
+                await teardownQuietly(lost, liveness.oomKilled ? "oom" : "dead");
+            return "dead-removed";
+        }
+        // A command that runs keeps its machine, also when no host polls it, for example during a restart.
+        if (running > 0) {
+            unwrapOrThrow(await touchLease(pool, id));
+            return "busy";
+        }
+        const idle = unwrapOrThrow(await clearIdleMachine(pool, id, machineGeneration, idlePeriodMs));
+        if (idle === undefined)
+            return "none";
+        await teardownQuietly(idle, "idle");
+        return "idle-removed";
+    };
+    const runningExecs = async (machine) => {
+        try {
+            const response = await fetchImpl(`http://${machine.host}:${machine.port}/health`, { signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) });
+            if (!response.ok)
+                return "unreachable";
+            const health = HealthSchema.safeParse(await response.json());
+            return health.success ? (health.data.runningExecs ?? 1) : 1;
+        }
+        catch {
+            return "unreachable";
+        }
+    };
+    // ── The waits of a consumer ───────────────────────────────────────
+    /**
+     * Wait for an event of a lease workflow. Between two slices, the wait reads the cancel state of its caller and
+     * mirrors the pause of the lease workflow into the stream of the consumer. `ended`: the lease workflow ended with
+     * no event under the key.
+     */
+    const waitForEvent = async (lease, key, mirror) => {
+        const dbos = await engine();
+        let mirrored;
+        for (;;) {
+            const value = await dbos.getEvent(lease.leaseWorkflowId, key, { timeoutSeconds: WAIT_SLICE_SECONDS, pollingIntervalMs: EVENT_POLL_MS });
+            if (value !== null) {
+                if (mirrored !== undefined && mirror !== undefined) {
+                    await mirror({ type: "data-pause", id: lease.leaseWorkflowId, workId: lease.workId, workKind: lease.workKind, state: "running" });
+                }
+                return value;
+            }
+            if (await callerCanceled())
+                throw new DBOSErrors.DBOSWorkflowCancelledError(DBOS.workflowID ?? lease.leaseWorkflowId);
+            const status = await dbos.getWorkflow(lease.leaseWorkflowId);
+            if (status !== undefined && TERMINAL_STATUSES.has(status.status)) {
+                const last = await dbos.getEvent(lease.leaseWorkflowId, key, { timeoutSeconds: 0 });
+                return last ?? "ended";
+            }
+            if (mirror !== undefined)
+                mirrored = await mirrorPause(lease, mirror, mirrored);
+        }
+    };
+    const mirrorPause = async (lease, mirror, mirrored) => {
+        const pauses = unwrapOrThrow(await queryPausesOfWork(pool, lease.analysisId, lease.workId));
+        const pause = pauses.find((row) => row.workflowId === lease.leaseWorkflowId);
+        const state = pause === undefined ? undefined : `${pause.state}:${pause.attempt}`;
+        if (state === mirrored)
+            return mirrored;
+        if (pause === undefined) {
+            await mirror({ type: "data-pause", id: lease.leaseWorkflowId, workId: lease.workId, workKind: lease.workKind, state: "running" });
+            return undefined;
+        }
+        await mirror({
+            type: "data-pause",
+            id: lease.leaseWorkflowId,
+            workId: lease.workId,
+            workKind: lease.workKind,
+            state: pause.state,
+            cause: pause.cause,
+            attempt: pause.attempt,
+            retryAt: pause.retryAt,
+        });
+        return state;
+    };
+    const enqueue = async (binding, lease, size) => {
+        const dbos = await engine();
+        await registerQueue(dbos);
+        const input = {
+            leaseWorkflowId: lease.leaseWorkflowId,
+            leaseKey: lease.leaseKey,
+            analysisId: lease.analysisId,
+            ownerWorkflowId: lease.ownerWorkflowId,
+            work: binding.work,
+            session: binding.session,
+            spawn: binding.spawn ?? {},
+            size,
+            requestedAtMs: Date.now(),
+        };
+        // DBOS gives back the existing workflow for an id that it holds, thus a rerun starts nothing new.
+        await dbos.enqueue({ workflowName: leaseWorkflow.name, queueName: LEASE_QUEUE, workflowID: lease.leaseWorkflowId, appVersion: DBOS.applicationVersion }, input);
+    };
+    /** Tell the lease workflow to end. A lease workflow in a pause reads no message, thus this ends it here. */
+    const stopLease = async (lease) => {
+        const dbos = await engine();
+        const pauses = unwrapOrThrow(await queryPausesOfWork(pool, lease.analysisId, lease.workId));
+        if (pauses.some((row) => row.workflowId === lease.leaseWorkflowId)) {
+            // A lease workflow that waits in a pause holds no machine, and it reads no message of the lease until a
+            // resume. Thus the release ends it here.
+            await dbos.cancelWorkflow(lease.leaseWorkflowId);
+            unwrapOrThrow(await releaseLease(pool, lease.leaseWorkflowId, "released"));
+            await removeRecords(lease.analysisId, lease.leaseKey);
+            const next = unwrapOrThrow(await oldestWaiting(pool));
+            if (next !== undefined)
+                await dbos.send(next, { kind: "capacity" }, LEASE_TOPIC);
+            return "ended";
+        }
+        await dbos.send(lease.leaseWorkflowId, { kind: "release", reason: "released" }, LEASE_TOPIC, `${lease.leaseWorkflowId}:release`);
+        return "told";
+    };
+    const release = async (leaseKey) => {
+        const lease = unwrapOrThrow(await activeLease(pool, leaseKey));
+        if (lease === undefined)
+            return;
+        if ((await stopLease(lease)) === "ended")
+            return;
+        const dbos = await engine();
+        for (;;) {
+            const handle = await dbos.getWorkflow(lease.leaseWorkflowId);
+            if (handle === undefined || TERMINAL_STATUSES.has(handle.status))
+                return;
+            const row = unwrapOrThrow(await readLease(pool, lease.leaseWorkflowId));
+            if (row?.state === "released")
+                return;
+            if (await callerCanceled())
+                throw new DBOSErrors.DBOSWorkflowCancelledError(DBOS.workflowID ?? lease.leaseWorkflowId);
+            await sleep(1_000);
+        }
+    };
+    const acquire = async (binding, requested) => {
+        const size = clampResources(requested, client.resourceLimits);
+        const ownerWorkflowId = DBOS.workflowID;
+        if (ownerWorkflowId === undefined)
+            throw new Error("a sandbox lease is acquired inside a DBOS workflow, and none is active");
+        for (;;) {
+            let lease = unwrapOrThrow(await activeLease(pool, binding.leaseKey));
+            if (lease !== undefined && !sameSize(lease.requested, size)) {
+                await release(binding.leaseKey);
+                lease = undefined;
+            }
+            if (lease === undefined) {
+                const generation = unwrapOrThrow(await latestGeneration(pool, binding.leaseKey)) + 1;
+                unwrapOrThrow(await requestLease(pool, {
+                    leaseWorkflowId: leaseWorkflowIdOf(binding.session.scope.analysisId, binding.leaseKey, generation),
+                    leaseKey: binding.leaseKey,
+                    generation,
+                    analysisId: binding.session.scope.analysisId,
+                    ownerWorkflowId,
+                    workKind: binding.work.kind,
+                    workId: binding.work.id,
+                    requested: size,
+                }));
+                lease = unwrapOrThrow(await activeLease(pool, binding.leaseKey));
+                // A concurrent request of the same key won the insert. The loop reads its lease again.
+                if (lease === undefined || !sameSize(lease.requested, size))
+                    continue;
+            }
+            await enqueue(binding, lease, size);
+            const outcome = await waitForEvent(lease, GRANTED_EVENT, binding.emit);
+            if (outcome !== "ended")
+                return outcome;
+            const row = unwrapOrThrow(await readLease(pool, lease.leaseWorkflowId));
+            return { kind: "failed", detail: `the lease ended before its grant (${row?.releaseReason ?? "no reason"})` };
+        }
+    };
+    /** Get a machine for an exec that found none: ask the lease workflow, and wait for the next machine generation. */
+    const requestMachine = async (lease, after, mirror) => {
+        const dbos = await engine();
+        const key = DBOS.workflowID === undefined ? undefined : `${DBOS.workflowID}:${String(DBOS.stepID)}:machine:${after}`;
+        await dbos.send(lease.leaseWorkflowId, { kind: "machine", after }, LEASE_TOPIC, key);
+        const event = await waitForEvent(lease, machineEventKey(after + 1), mirror);
+        if (event === "ended")
+            return err({ kind: "no_lease", detail: "the lease ended" });
+        if (event.kind === "failed")
+            return err({ kind: "no_lease", detail: `the sandbox did not start again: ${event.detail}` });
+        return ok(event.ref);
+    };
+    const exec = (leaseKey, request, emit, mirror) => durably("sandbox.exec", async () => {
+        const lease = unwrapOrThrow(await activeLease(pool, leaseKey));
+        if (lease?.state !== "granted")
+            return err({ kind: "no_lease" });
+        // The use is written before the submit, thus the idle removal cannot take the machine of this exec.
+        const touched = unwrapOrThrow(await touchLease(pool, lease.leaseWorkflowId));
+        if (touched === undefined)
+            return err({ kind: "no_lease" });
+        let machine = touched.machine;
+        let machineGeneration = touched.machineGeneration;
+        if (machine === undefined) {
+            const started = await requestMachine(lease, machineGeneration, mirror);
+            if (started.isErr())
+                return err(started.error);
+            machine = started.value;
+            machineGeneration += 1;
+        }
+        const timeoutSeconds = request.timeoutSeconds ?? DEFAULT_COMMAND_TIMEOUT_SECONDS;
+        const result = await client.exec(machine, { ...request, timeoutSeconds }, emit, Date.now() + timeoutSeconds * 1000 + POLL_GRACE_MS, saltedStepExecId(lease.execSalt));
+        noteExecOutcome(leaseKey, summarizeExec(result));
+        const failure = result.syntheticFailure;
+        if (failure === undefined)
+            return ok(result);
+        if (failure.reason === "sandbox-dead" || failure.reason === "sandbox-oom-killed") {
+            // The row forgets the dead machine before the exec returns, thus the next exec asks for a new one.
+            const lost = unwrapOrThrow(await clearMachine(pool, lease.leaseWorkflowId, machineGeneration));
+            if (lost !== undefined) {
+                const dbos = await engine();
+                await dbos.send(lease.leaseWorkflowId, { kind: "machine-lost", machine: lost, oomKilled: failure.reason === "sandbox-oom-killed" }, LEASE_TOPIC, `${result.execId}:lost`);
+            }
+        }
+        return ok({ ...result, syntheticFailure: { ...failure, memoryLimitGb: lease.granted?.memoryGb } });
+    });
+    const leases = {
+        toolchainSource: client.toolchainSource,
+        acquire: (binding, size) => durably("sandbox.acquire", async () => {
+            try {
+                return await acquire(binding, size);
+            }
+            catch (failure) {
+                if (failure instanceof DBOSErrors.DBOSWorkflowCancelledError)
+                    throw failure;
+                logger.error("an acquire failed", { leaseKey: binding.leaseKey, ...logger.errorFields(failure) });
+                return { kind: "failed", detail: errorMessageOf(failure) ?? textOf(failure) };
+            }
+        }),
+        release: (leaseKey) => durably("sandbox.release", () => release(leaseKey)),
+        releaseOwnedBy: async (ownerWorkflowIds) => {
+            for (const lease of unwrapOrThrow(await leasesOwnedBy(pool, ownerWorkflowIds)))
+                await stopLease(lease);
+        },
+        exec,
+        bind: (binding) => ({
+            acquire: (size) => leases.acquire(binding, size),
+            release: () => leases.release(binding.leaseKey),
+            exec: (request, emit) => exec(binding.leaseKey, request, emit, binding.emit),
+        }),
+    };
+    return leases;
+}

package/dist/sandbox/liveness.d.ts

@@ -44,8 +44,11 @@ export declare function createEscalationPolicy(threshold?: number): {
  * caller resumes polling.
  */
 export declare function probeLiveness(isAlive: (ref: SandboxRef) => Promise<SandboxLiveness>, ref: SandboxRef): Promise<ProbeVerdict>;
-/** The closed set of reasons a synthetic failure carries. */
-export type SyntheticFailureReason = "sandbox-oom-killed" | "sandbox-dead";
+/**
+ * The closed set of reasons a synthetic failure carries. `sandbox-replaced`: a new machine of the lease found the
+ * exec cut by the end of the earlier machine.
+ */
+export type SyntheticFailureReason = "sandbox-oom-killed" | "sandbox-dead" | "sandbox-replaced";
 /**
  * An OOM-killed machine gets a distinguishable reason so the step failure
  * reads "exceeded its memory limit", not a mystery death.

package/dist/sandbox/mount-plan.d.ts

@@ -36,6 +36,7 @@ export declare const STEP_SUBDIRS: readonly ["output", "scripts", "figures", "lo
  * store the way the sandbox does configures nothing further.
  */
 export declare const LIBS_CONTAINER_PATH = "/mnt/libs";
+export declare const EXEC_RECORDS_CONTAINER_PATH = "/mnt/exec-records";
 export declare const REFS_CONTAINER_PATH = "/mnt/refs";
 /** Container path of the per-analysis read-write cache, when a farm resolution carries a cache location. */
 export declare const FARM_CACHE_CONTAINER_PATH = "/mnt/libs/cache";
@@ -64,8 +65,10 @@ export type MountPlanCoords = {
      * contradiction, and the builders refuse it.
      */
     writableTail?: string;
+    /** The workspace-relative directory of the exec records of the lease, mounted read-write in each machine of the lease. */
+    recordTail: string;
 };
-export declare function mountCoordsOf(session: SpawnSession, spec: Pick<SandboxSpec, "readOnly" | "writableTail">): MountPlanCoords;
+export declare function mountCoordsOf(session: SpawnSession, spec: Pick<SandboxSpec, "readOnly" | "writableTail" | "recordTail">): MountPlanCoords;
 export type MountPlanStores = {
     /** Lib store is mounted at `/mnt/libs`, with the farm nested inside it, and its env is emitted. */
     libs: boolean;
@@ -101,6 +104,10 @@ export type MountPlan = {
     cachePath?: string;
     /** Container path of the ref store, present only when `refs`. */
     refsPath?: string;
+    /** Container path of the exec records of the lease. */
+    recordsPath: string;
+    /** The workspace-relative source of the records mount. */
+    recordTail: string;
     /** Pre-created subdirectories under the writable mount. Empty when read-only
      *  (nothing to pre-create) and when a declared tail replaces the step tail. */
     stepSubdirs: readonly string[];
@@ -113,6 +120,8 @@ export type SessionSubPaths = {
     readonly ro: string;
     /** Read-write mount of the step's artifact dir, or of the declared tail; absent when read-only. */
     readonly rw?: string;
+    /** Read-write mount of the exec records of the lease. */
+    readonly records: string;
 };
 /** The one writable mount of a sandbox: its workspace-relative tail, and what the host pre-makes under it. */
 export type SandboxWriteTail = {

package/dist/sandbox/mount-plan.js

@@ -35,6 +35,7 @@ export const STEP_SUBDIRS = ["output", "scripts", "figures", "logs", "notebooks"
  * store the way the sandbox does configures nothing further.
  */
 export const LIBS_CONTAINER_PATH = "/mnt/libs";
+export const EXEC_RECORDS_CONTAINER_PATH = "/mnt/exec-records";
 export const REFS_CONTAINER_PATH = "/mnt/refs";
 /** Container path of the per-analysis read-write cache, when a farm resolution carries a cache location. */
 export const FARM_CACHE_CONTAINER_PATH = "/mnt/libs/cache";
@@ -54,6 +55,7 @@ export function mountCoordsOf(session, spec) {
         stepId: session.runFrame.stepId,
         readOnly: spec.readOnly,
         writableTail: spec.writableTail,
+        recordTail: spec.recordTail,
     };
 }
 /**
@@ -111,6 +113,7 @@ export function buildSessionSubPaths(coords, workspaceSubPath) {
     return {
         ro: workspaceSubPath,
         rw: write === undefined ? undefined : `${workspaceSubPath}/${write.tail}`,
+        records: `${workspaceSubPath}/${assertSafeTail(coords.recordTail, "recordTail").join("/")}`,
     };
 }
 /**
@@ -155,7 +158,8 @@ export function buildMountPlan(coords, stores) {
     const write = sandboxWriteTail(coords);
     const writableStepPath = write === undefined ? undefined : `${readonlyTreePath}/${write.tail}`;
     const cache = stores.libs && stores.cache === true;
-    const env = { PROVENANCE_WATCH_DIRS: readonlyTreePath };
+    const recordTail = assertSafeTail(coords.recordTail, "recordTail").join("/");
+    const env = { PROVENANCE_WATCH_DIRS: readonlyTreePath, EXEC_RECORD_DIR: EXEC_RECORDS_CONTAINER_PATH };
     if (stores.libs)
         Object.assign(env, libStoreEnv(stores.toolchainSource, cache));
     return {
@@ -166,6 +170,8 @@ export function buildMountPlan(coords, stores) {
         farmPath: stores.libs ? farmContainerPath(stores.toolchainSource) : undefined,
         cachePath: cache ? FARM_CACHE_CONTAINER_PATH : undefined,
         refsPath: stores.refs ? REFS_CONTAINER_PATH : undefined,
+        recordsPath: EXEC_RECORDS_CONTAINER_PATH,
+        recordTail,
         stepSubdirs: write?.subdirs ?? [],
         env,
     };