@inflexa-ai/harness 0.46.2 → 0.47.0

package/dist/sandbox/client.d.ts

@@ -19,6 +19,7 @@
  */
 import type { ResultAsync } from "neverthrow";
 import type { SpawnSession } from "../auth/types.js";
+import type { ResourceLimits } from "../config/resource-limits.js";
 import type { SandboxError } from "./sandbox-error.js";
 import type { ExecEmit, ExecRequest, ExecResult, ManagedSandbox, SandboxIdentity, SandboxLiveness, SandboxRef, SandboxSpec, ToolchainSource } from "./types.js";
 export type SandboxClient = {
@@ -31,6 +32,8 @@ export type SandboxClient = {
      * optional field would carry two meanings, undeclared and unknown.
      */
     readonly toolchainSource: ToolchainSource;
+    /** The per-machine ceilings that the client clamps each spawn to. The sandbox module clamps a lease request to the same values. */
+    readonly resourceLimits: ResourceLimits;
     /**
      * DBOS step (`sandbox.create`) — the spawn half of the two-step create
      * (see the harness-sandbox-exec spec). Launches the sandbox-base container/Job under the
@@ -59,9 +62,8 @@ export type SandboxClient = {
      * same step gets the same id, and the sandbox gives it the result of the
      * first. Outside a workflow it throws.
      *
-     * `execId` replaces that id. A caller that runs more than one exec in one
-     * step, or an exec outside a workflow, gives its own id
-     * (`sandbox/step-sandbox.ts`).
+     * `execId` replaces that id. The sandbox leases give an id under the salt
+     * of the lease (`sandbox/leases.ts`).
      *
      * A recovered step submits again under the same id. The sandbox dedups the
      * submit on the id, thus the poll attaches to the exec that already ran or

package/dist/sandbox/create-sandbox.d.ts

@@ -2,17 +2,11 @@
  * Backend-selected `SandboxClient` factory.
  *
  * Docker for local dev, K8s for production — the single place the
- * `SANDBOX_BACKEND` env decision lives. Injected at the composition root
- * as a construction-time dependency (see the harness-durable-runtime spec); callers do NOT import a
- * backend module directly.
- *
- * Registry write/clear callbacks are wired here so the per-backend
- * `createSandbox` / `teardown` implementations stay free of state-layer
- * coupling (they just call the closures we hand them). `exec` is
+ * `SANDBOX_BACKEND` env decision lives. The embedder makes the client and gives
+ * it to `assembleCoreRuntime`, and only the sandbox module calls it. `exec` is
  * backend-agnostic — it only needs the SandboxRef.
  */
 import type { V1Toleration } from "@kubernetes/client-node";
-import type { Querier } from "../state/db.js";
 import type { Logger } from "../lib/logger.js";
 import { type ResourceLimits } from "../config/resource-limits.js";
 import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
@@ -30,8 +24,6 @@ export type SandboxBackendConfig = {
     readonly namespace: string;
 };
 export type CreateSandboxClientConfig = {
-    /** App pool — used for registry writes inside the create/teardown steps. */
-    pool: Querier;
     /** Backend default + namespace fallback source. */
     env: SandboxBackendConfig;
     /** Default sandbox-base image when the workflow doesn't override per-step. */
@@ -174,6 +166,9 @@ export declare function precreateStepTree(deps: {
     resolveWorkspaceRoot: ResolveWorkspaceRoot;
     stepTreeAccess?: "world-writable";
 }, coords: MountPlanCoords): Promise<void>;
+/** Pre-create the directory of the exec records of the lease, which each machine of the lease mounts read-write. */
+export declare function precreateRecordDir(deps: {
+    resolveWorkspaceRoot: ResolveWorkspaceRoot;
+    stepTreeAccess?: "world-writable";
+}, coords: MountPlanCoords): Promise<void>;
 export declare function createSandboxClient(config: CreateSandboxClientConfig): SandboxClient;
-/** Re-exports so consumers import once from this module. */
-export { clearSandboxRef } from "../state/index.js";

package/dist/sandbox/create-sandbox.js

@@ -2,13 +2,8 @@
  * Backend-selected `SandboxClient` factory.
  *
  * Docker for local dev, K8s for production — the single place the
- * `SANDBOX_BACKEND` env decision lives. Injected at the composition root
- * as a construction-time dependency (see the harness-durable-runtime spec); callers do NOT import a
- * backend module directly.
- *
- * Registry write/clear callbacks are wired here so the per-backend
- * `createSandbox` / `teardown` implementations stay free of state-layer
- * coupling (they just call the closures we hand them). `exec` is
+ * `SANDBOX_BACKEND` env decision lives. The embedder makes the client and gives
+ * it to `assembleCoreRuntime`, and only the sandbox module calls it. `exec` is
  * backend-agnostic — it only needs the SandboxRef.
  */
 import { chmod, mkdir } from "node:fs/promises";
@@ -20,13 +15,11 @@ import { recordSandboxExec } from "../lib/metrics.js";
 import { ATTR_INFLEXA_EXEC_ID, stableSpan } from "../lib/otel-spans.js";
 import { clampResources } from "../config/resource-limits.js";
 import { tailWritePrefix } from "../workspace/paths.js";
-import { tryMutation } from "../lib/db-result.js";
 import { unwrapOrThrow } from "../lib/result.js";
-import { setSandboxRef } from "../state/index.js";
 import { capExecStreams, EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
 import { createDockerSandboxOps } from "./docker-client.js";
 import { runExec, stepExecId } from "./exec.js";
-import { noteExecOutcome, sandboxExecOutcomeOf, summarizeExec } from "./exec-outcome.js";
+import { sandboxExecOutcomeOf } from "./exec-outcome.js";
 import { createK8sSandboxOps } from "./k8s-client.js";
 import { mountCoordsOf, sandboxWriteTail } from "./mount-plan.js";
 import { SandboxFailure } from "./sandbox-error.js";
@@ -65,6 +58,13 @@ export async function precreateStepTree(deps, coords) {
         await Promise.all(write.subdirs.map((sub) => chmod(join(writeDir, sub), 0o777)));
     }
 }
+/** Pre-create the directory of the exec records of the lease, which each machine of the lease mounts read-write. */
+export async function precreateRecordDir(deps, coords) {
+    const recordDir = tailWritePrefix({ workspaceRoot: await deps.resolveWorkspaceRoot(coords.analysisId), tail: coords.recordTail });
+    await mkdir(recordDir, { recursive: true });
+    if (deps.stepTreeAccess === "world-writable")
+        await chmod(recordDir, 0o777);
+}
 export function createSandboxClient(config) {
     const backend = config.backend ?? config.env.backend;
     // Normalized once, here, and handed to the backends and to the client
@@ -100,9 +100,6 @@ export function createSandboxClient(config) {
      * unwraps below stay on the plain bridge: they carry a `DbError`.
      */
     const failing = (e) => new SandboxFailure(e);
-    const registerSandbox = async (session, spec, ref) => {
-        unwrapOrThrow(await setSandboxRef(config.pool, session.runFrame.runId, session.runFrame.stepId, ref));
-    };
     const ops = backend === "k8s"
         ? createK8sSandboxOps({
             image: config.image,
@@ -120,7 +117,6 @@ export function createSandboxClient(config) {
             tolerations: config.tolerations,
             runtimeClassName: config.runtimeClassName,
             logger: config.logger,
-            registerSandbox,
         })
         : createDockerSandboxOps({
             image: config.image,
@@ -133,27 +129,9 @@ export function createSandboxClient(config) {
             platform: config.platform,
             engineSocketPath: config.engineSocketPath,
             logger: config.logger,
-            registerSandbox,
         });
-    // `teardown` needs the registry-clear closure but the SandboxRef alone
-    // doesn't carry runId/stepId. Wrap the backend impl so callers pass the
-    // ref + step coordinates and the wrapper does the clear after the
-    // backend-level removal.
     const teardown = async (ref) => {
         unwrapOrThrow((await ops.teardown(ref)).mapErr(failing));
-        // Sandboxes are 1-per-step (see `sandbox-step.ts` body — each child
-        // workflow calls `createSandbox` once then `teardown` once), so the
-        // broad WHERE clause matches exactly one row. If that invariant ever
-        // changes (e.g. a step reuses a sibling's sandbox), this clear must
-        // be scoped to (run, step) to avoid wiping prior steps' provenance.
-        unwrapOrThrow(await tryMutation("createSandbox.teardownClearSandboxRef", async () => {
-            await config.pool.query({
-                text: `UPDATE cortex_step_executions
-            SET sandbox_ref = NULL
-            WHERE sandbox_ref->>'sandboxId' = $1`,
-                values: [ref.sandboxId],
-            });
-        }));
     };
     // One arbiter for both surfaces: the client's `isAlive` method and the
     // poll loop's escalation probe are the same backend inspect.
@@ -185,10 +163,12 @@ export function createSandboxClient(config) {
         // The config states an engine fact (binds preserve host ownership);
         // which remediation that fact demands — today, a world-writable step
         // write tree — is harness-owned and chosen here, not by the embedder.
-        const precreated = await ResultAsync.fromPromise(precreateStepTree({
+        const treeDeps = {
             resolveWorkspaceRoot: config.resolveWorkspaceRoot,
             stepTreeAccess: config.engineBindOwnership === "host-preserved" ? "world-writable" : undefined,
-        }, mountCoordsOf(session, spec)), createFailed("createSandbox.precreateStepTree"));
+        };
+        const coords = mountCoordsOf(session, spec);
+        const precreated = await ResultAsync.fromPromise(Promise.all([precreateStepTree(treeDeps, coords), precreateRecordDir(treeDeps, coords)]), createFailed("createSandbox.precreateStepTree"));
         if (precreated.isErr())
             return err(precreated.error);
         // Clamp to cluster ceilings so the pod is always quota-admissible.
@@ -197,6 +177,7 @@ export function createSandboxClient(config) {
     };
     return {
         toolchainSource,
+        resourceLimits: config.resourceLimits,
         createSandbox: (session, spec, identity) => new ResultAsync(createSandbox(session, spec, identity)),
         // The one place an ExecResult crosses from the wire into the process, so
         // the cap lands here rather than at each consumer. Every downstream use —
@@ -223,7 +204,6 @@ export function createSandboxClient(config) {
                 peakMemoryBytes: result.usage?.peakMemoryBytes,
                 cpuMillis: result.usage?.cpuMillis,
             });
-            noteExecOutcome(ref.sandboxId, summarizeExec(result));
             return result;
         }, { name: "sandbox.exec" }),
         isAlive,
@@ -233,5 +213,3 @@ export function createSandboxClient(config) {
         listManagedSandboxes: async () => unwrapOrThrow((await ops.listManagedSandboxes()).mapErr(failing)),
     };
 }
-/** Re-exports so consumers import once from this module. */
-export { clearSandboxRef } from "../state/index.js";

package/dist/sandbox/docker-client.d.ts

@@ -143,11 +143,9 @@ export type DockerClientConfig = {
     /**
      * Optional logger so a store degradation (a resolved farm whose `inflexa.lock`
      * is absent or invalid by sandbox-create time) is observable instead of a
-     * silent libs-mount drop. Matches the `reaper`/`watchdog` logger seam.
+     * silent libs-mount drop.
      */
     logger?: Logger;
-    /** Hook called after the registry row is written. */
-    registerSandbox: (session: SpawnSession, spec: SandboxSpec, ref: SandboxRef) => Promise<void>;
 };
 /**
  * dockerode connection options for a configured engine socket, or `undefined`

package/dist/sandbox/docker-client.js

@@ -319,6 +319,7 @@ export function createDockerSandboxOps(config) {
                     ...(libsMounted && farm && plan.farmPath ? [`${farm.farmPath}:${plan.farmPath}:ro`] : []),
                     ...(libsMounted && cacheBindable && farm?.cachePath && plan.cachePath ? [`${farm.cachePath}:${plan.cachePath}:rw`] : []),
                     ...(refsMounted && config.refStorePath ? [`${config.refStorePath}:${String(plan.refsPath)}:ro`] : []),
+                    `${tailWritePrefix({ workspaceRoot: hostTreePath, tail: plan.recordTail })}:${plan.recordsPath}:rw`,
                     ...cpuBinds,
                 ];
                 const createFailed = (status, cause) => ({
@@ -348,7 +349,7 @@ export function createDockerSandboxOps(config) {
                     Env: env,
                     User: SANDBOX_ROOT_USER,
                     WorkingDir: plan.workingDir,
-                    Labels: mergeLabels(hostLabels, { ...harnessLabels(session, sandboxId), [OWNER_WORKFLOW_LABEL]: spec.childWorkflowId }),
+                    Labels: mergeLabels(hostLabels, { ...harnessLabels(session, sandboxId), [OWNER_WORKFLOW_LABEL]: spec.ownerWorkflowId }),
                     ExposedPorts: { [`${SANDBOX_SERVER_PORT}/tcp`]: {} },
                     HostConfig: {
                         Binds: binds,
@@ -378,7 +379,7 @@ export function createDockerSandboxOps(config) {
                 };
                 if (config.platform !== undefined)
                     createOpts.platform = config.platform;
-                const sandbox = await createOrAdopt(docker, createOpts, sandboxId, spec.childWorkflowId, createFailed, logger);
+                const sandbox = await createOrAdopt(docker, createOpts, sandboxId, spec.ownerWorkflowId, createFailed, logger);
                 if (sandbox.isErr())
                     return err(sandbox.error);
                 if (!sandbox.value.alreadyRunning) {
@@ -407,9 +408,6 @@ export function createDockerSandboxOps(config) {
                     port,
                     backend: "docker",
                 };
-                const registered = await trySandbox(() => config.registerSandbox(session, spec, ref), createFailed);
-                if (registered.isErr())
-                    return err(registered.error);
                 return ok(ref);
             })());
         },

package/dist/sandbox/exec-outcome.d.ts

@@ -8,13 +8,13 @@
  * keeps that text on the sandbox side of the boundary. A summary gives the
  * SIZE of stderr and never a byte of it.
  *
- * The summary of the last exec of a sandbox lives in one process-local cell,
- * keyed by sandbox id. A sandbox belongs to one step (see the teardown of
- * `create-sandbox.ts`), thus the cell is step-scoped in effect and the step
- * body reads it without a new dependency through the agent factory. A replay
- * re-runs the await loop and writes the cell again, thus a recovered body
- * reads the summary of the same exec. The map is capped, so a sandbox whose
- * step never reaches the read site cannot hold memory.
+ * The summary of the last exec of a lease lives in one process-local cell,
+ * keyed by the lease key. A step holds one lease key, thus the cell is
+ * step-scoped in effect and the step body reads it without a new dependency
+ * through the agent factory. The cell keeps the summary across a machine
+ * replacement. A replay re-runs the await loop and writes the cell again, thus
+ * a recovered body reads the summary of the same exec. The map is capped, so a
+ * lease whose step never reaches the read site cannot hold memory.
  */
 import type { SandboxExecOutcome } from "../lib/metrics.js";
 import type { ExecResult } from "./types.js";
@@ -39,7 +39,7 @@ export type ExecOutcomeSummary = {
 };
 /** Reduce a result to its summary. */
 export declare function summarizeExec(result: ExecResult): ExecOutcomeSummary;
-/** Keep `summary` as the last known exec of `sandboxId`. */
-export declare function noteExecOutcome(sandboxId: string, summary: ExecOutcomeSummary): void;
-/** The last known exec of `sandboxId`, or `undefined` when it ran none. */
-export declare function lastExecOutcome(sandboxId: string): ExecOutcomeSummary | undefined;
+/** Keep `summary` as the last known exec of the lease `leaseKey`. */
+export declare function noteExecOutcome(leaseKey: string, summary: ExecOutcomeSummary): void;
+/** The last known exec of the lease `leaseKey`, or `undefined` when it ran none. */
+export declare function lastExecOutcome(leaseKey: string): ExecOutcomeSummary | undefined;

package/dist/sandbox/exec-outcome.js

@@ -8,13 +8,13 @@
  * keeps that text on the sandbox side of the boundary. A summary gives the
  * SIZE of stderr and never a byte of it.
  *
- * The summary of the last exec of a sandbox lives in one process-local cell,
- * keyed by sandbox id. A sandbox belongs to one step (see the teardown of
- * `create-sandbox.ts`), thus the cell is step-scoped in effect and the step
- * body reads it without a new dependency through the agent factory. A replay
- * re-runs the await loop and writes the cell again, thus a recovered body
- * reads the summary of the same exec. The map is capped, so a sandbox whose
- * step never reaches the read site cannot hold memory.
+ * The summary of the last exec of a lease lives in one process-local cell,
+ * keyed by the lease key. A step holds one lease key, thus the cell is
+ * step-scoped in effect and the step body reads it without a new dependency
+ * through the agent factory. The cell keeps the summary across a machine
+ * replacement. A replay re-runs the await loop and writes the cell again, thus
+ * a recovered body reads the summary of the same exec. The map is capped, so a
+ * lease whose step never reaches the read site cannot hold memory.
  */
 /** The reason that names a machine the kernel killed for its memory limit. */
 const OOM_KILLED = "sandbox-oom-killed";
@@ -41,25 +41,25 @@ export function summarizeExec(result) {
     };
 }
 /**
- * How many sandboxes keep a summary. A host runs far fewer sandboxes at once
- * than this, thus the cap only drops the summary of a step that ended long ago.
+ * How many leases keep a summary. A host holds far fewer leases at once than
+ * this, thus the cap only drops the summary of a step that ended long ago.
  */
-const MAX_TRACKED_SANDBOXES = 256;
-const lastExecBySandbox = new Map();
-/** Keep `summary` as the last known exec of `sandboxId`. */
-export function noteExecOutcome(sandboxId, summary) {
+const MAX_TRACKED_LEASES = 256;
+const lastExecByLease = new Map();
+/** Keep `summary` as the last known exec of the lease `leaseKey`. */
+export function noteExecOutcome(leaseKey, summary) {
     // Delete before set so the key moves to the end of the insertion order and
-    // the eviction below drops the least recently written sandbox.
-    lastExecBySandbox.delete(sandboxId);
-    lastExecBySandbox.set(sandboxId, summary);
-    while (lastExecBySandbox.size > MAX_TRACKED_SANDBOXES) {
-        const oldest = lastExecBySandbox.keys().next();
+    // the eviction below drops the least recently written lease.
+    lastExecByLease.delete(leaseKey);
+    lastExecByLease.set(leaseKey, summary);
+    while (lastExecByLease.size > MAX_TRACKED_LEASES) {
+        const oldest = lastExecByLease.keys().next();
         if (oldest.done)
             break;
-        lastExecBySandbox.delete(oldest.value);
+        lastExecByLease.delete(oldest.value);
     }
 }
-/** The last known exec of `sandboxId`, or `undefined` when it ran none. */
-export function lastExecOutcome(sandboxId) {
-    return lastExecBySandbox.get(sandboxId);
+/** The last known exec of the lease `leaseKey`, or `undefined` when it ran none. */
+export function lastExecOutcome(leaseKey) {
+    return lastExecByLease.get(leaseKey);
 }

package/dist/sandbox/exec.d.ts

@@ -52,6 +52,11 @@ export type ExecDeps = {
  * makes the pair unique, thus no caller mints an exec id.
  */
 export declare function stepExecId(): string;
+/**
+ * The exec id of a step under a salt that no machine receives. The id is the same on each run of the step, but the
+ * ids that a command can read in the exec records do not give the id of a later exec.
+ */
+export declare function saltedStepExecId(salt: string): string;
 /**
  * Run one exec to its terminal result.
  *

package/dist/sandbox/exec.js

@@ -14,6 +14,7 @@
  */
 import { errorMessageOf } from "../lib/thrown.js";
 import { textOf } from "../lib/unknown-text.js";
+import { createHash } from "node:crypto";
 import { setTimeout as sleep } from "node:timers/promises";
 import { DBOS, Error as DBOSErrors, StatusString } from "@dbos-inc/dbos-sdk";
 import { untracedFetch } from "../lib/otel-spans.js";
@@ -62,6 +63,13 @@ export function stepExecId() {
     }
     return `${workflowId}:${stepId}`;
 }
+/**
+ * The exec id of a step under a salt that no machine receives. The id is the same on each run of the step, but the
+ * ids that a command can read in the exec records do not give the id of a later exec.
+ */
+export function saltedStepExecId(salt) {
+    return createHash("sha256").update(`${salt}:${stepExecId()}`).digest("hex");
+}
 /**
  * Run one exec to its terminal result.
  *
@@ -177,7 +185,7 @@ async function pollExec(ref, execId, emit, deadlineMs, deps) {
             if (resp.cursor > cursor)
                 cursor = resp.cursor;
             if (resp.result !== undefined)
-                return resp.result;
+                return resp.result.lost === undefined ? resp.result : syntheticFailureResult(execId, "sandbox-replaced");
         }
         if (deps.isAlive && escalation.onPoll(polled.kind === "ok" ? "ok" : "unavailable")) {
             const verdict = await probeLiveness(deps.isAlive, ref);

package/dist/sandbox/identity.d.ts

@@ -1,22 +1,10 @@
 /**
- * Sandbox identity minting — the durable half of the two-step create (ADR
- * 0016). Backend-agnostic: the `sandboxId` is minted here, checkpointed by the
- * `sandbox.mint` DBOS step, then handed to the backend's spawn step. Because the
- * identity is durable before any machine exists, a crash between spawn and the
- * spawn-step checkpoint is recoverable — the re-run adopts the existing machine
- * under the same name.
+ * The name of a sandbox machine. It comes from a key: the id of the lease workflow and the machine generation. Thus
+ * a rerun of the spawn step adopts the machine of the earlier attempt, and no step checkpoints a name before a spawn.
  */
 import type { SandboxIdentity } from "./types.js";
-/** `sbx-{run8}-{rand8}` — run prefix aids `kubectl` triage; the 8-hex random
- *  suffix makes within-run collisions across a run's concurrent steps
- *  negligible. Names are NOT the idempotency mechanism (the checkpoint is), and
- *  adoption is owner-label-guarded, but a unique name keeps a collision from
- *  ever reaching that guard. */
-export declare function mintSandboxIdentity(runId: string): SandboxIdentity;
 /**
- * `sbx-{run8}-{hash12}` — the name of a sandbox whose identity comes from a
- * key, not from a checkpoint. The same key gives the same name, thus a rerun
- * of a step adopts the machine of the earlier attempt. The key is hashed,
- * because a workflow id can hold characters that a container name refuses.
+ * `sbx-{run8}-{hash12}`. The run prefix aids `kubectl` triage. The same key gives the same name, and the key is
+ * hashed, because a workflow id can hold characters that a container name refuses.
  */
 export declare function keyedSandboxIdentity(runId: string, key: string): SandboxIdentity;

package/dist/sandbox/identity.js

@@ -1,27 +1,11 @@
 /**
- * Sandbox identity minting — the durable half of the two-step create (ADR
- * 0016). Backend-agnostic: the `sandboxId` is minted here, checkpointed by the
- * `sandbox.mint` DBOS step, then handed to the backend's spawn step. Because the
- * identity is durable before any machine exists, a crash between spawn and the
- * spawn-step checkpoint is recoverable — the re-run adopts the existing machine
- * under the same name.
+ * The name of a sandbox machine. It comes from a key: the id of the lease workflow and the machine generation. Thus
+ * a rerun of the spawn step adopts the machine of the earlier attempt, and no step checkpoints a name before a spawn.
  */
-import { createHash, randomUUID } from "node:crypto";
-/** `sbx-{run8}-{rand8}` — run prefix aids `kubectl` triage; the 8-hex random
- *  suffix makes within-run collisions across a run's concurrent steps
- *  negligible. Names are NOT the idempotency mechanism (the checkpoint is), and
- *  adoption is owner-label-guarded, but a unique name keeps a collision from
- *  ever reaching that guard. */
-export function mintSandboxIdentity(runId) {
-    const run8 = runId.replaceAll("-", "").slice(0, 8);
-    const suffix = randomUUID().replaceAll("-", "").slice(0, 8);
-    return { sandboxId: `sbx-${run8}-${suffix}` };
-}
+import { createHash } from "node:crypto";
 /**
- * `sbx-{run8}-{hash12}` — the name of a sandbox whose identity comes from a
- * key, not from a checkpoint. The same key gives the same name, thus a rerun
- * of a step adopts the machine of the earlier attempt. The key is hashed,
- * because a workflow id can hold characters that a container name refuses.
+ * `sbx-{run8}-{hash12}`. The run prefix aids `kubectl` triage. The same key gives the same name, and the key is
+ * hashed, because a workflow id can hold characters that a container name refuses.
  */
 export function keyedSandboxIdentity(runId, key) {
     const run8 = runId.replaceAll("-", "").slice(0, 8);

package/dist/sandbox/k8s-client.d.ts

@@ -105,7 +105,6 @@ export type K8sClientConfig = {
     /** Injected for tests. */
     batchApi?: BatchJobApi;
     coreApi?: CorePodApi;
-    registerSandbox: (session: SpawnSession, spec: SandboxSpec, ref: SandboxRef) => Promise<void>;
 };
 /** Clock and sleep seams, so a test can drive the slow-pod warning without waiting a minute. */
 export type WaitForPodReadyDeps = {

package/dist/sandbox/k8s-client.js

@@ -171,6 +171,12 @@ async function buildJobSpec(session, sandboxSpec, hostLabels, config, identity,
                 readOnly: false,
             });
         }
+        volumeMounts.push({
+            name: SESSION_VOLUME_NAME,
+            mountPath: plan.recordsPath,
+            subPath: subPaths.records,
+            readOnly: false,
+        });
     }
     if (config.libStorePvc && plan.libsPath) {
         const cacheSubPath = farm?.cachePath !== undefined && plan.cachePath ? pvcSubPath(farm.cachePath, "the farm cache") : undefined;
@@ -263,7 +269,7 @@ async function buildJobSpec(session, sandboxSpec, hostLabels, config, identity,
             name: sandboxId,
             namespace: config.namespace,
             annotations: {
-                [OWNER_WORKFLOW_ANNOTATION]: sandboxSpec.childWorkflowId,
+                [OWNER_WORKFLOW_ANNOTATION]: sandboxSpec.ownerWorkflowId,
             },
             labels,
         },
@@ -493,7 +499,7 @@ export function createK8sSandboxOps(config) {
                     }
                 }
                 const job = await buildJobSpec(session, spec, hostLabels, config, identity, farm, libsMounted);
-                const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, spec.childWorkflowId, job);
+                const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, spec.ownerWorkflowId, job);
                 if (adopted.isErr())
                     return err(adopted.error);
                 // A Job whose pod never schedules (quota rejection, no nodes, spot
@@ -510,17 +516,6 @@ export function createK8sSandboxOps(config) {
                         port: SANDBOX_SERVER_PORT,
                         backend: "k8s",
                     };
-                    const registered = await trySandbox(() => config.registerSandbox(session, spec, ref), (status, cause) => ({
-                        type: "container_create_failed",
-                        op: "k8s.registerSandbox",
-                        sandboxId,
-                        status,
-                        cause,
-                    }));
-                    if (registered.isErr()) {
-                        await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
-                        return err(registered.error);
-                    }
                     return ok(ref);
                 }
                 await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());

package/dist/sandbox/lease-names.d.ts

@@ -0,0 +1,6 @@
+/** The names that the lease module, the reconciler, and the purge share. */
+/** The prefix of each lease workflow id. A purge finds the lease workflows of an analysis by it. */
+export declare const LEASE_WORKFLOW_PREFIX = "sandboxlease";
+export declare function leaseWorkflowIdOf(analysisId: string, leaseKey: string, generation: number): string;
+/** The workspace-relative records directory of a lease key. Each generation of the key uses it. */
+export declare function leaseRecordTail(leaseKey: string): string;

package/dist/sandbox/lease-names.js

@@ -0,0 +1,12 @@
+/** The names that the lease module, the reconciler, and the purge share. */
+import { createHash } from "node:crypto";
+import { HARNESS_PRIVATE_DIR } from "../workspace/paths.js";
+/** The prefix of each lease workflow id. A purge finds the lease workflows of an analysis by it. */
+export const LEASE_WORKFLOW_PREFIX = "sandboxlease";
+export function leaseWorkflowIdOf(analysisId, leaseKey, generation) {
+    return `${LEASE_WORKFLOW_PREFIX}:${analysisId}:${leaseKey}:${generation}`;
+}
+/** The workspace-relative records directory of a lease key. Each generation of the key uses it. */
+export function leaseRecordTail(leaseKey) {
+    return `${HARNESS_PRIVATE_DIR}/exec-records/${createHash("sha256").update(leaseKey, "utf8").digest("hex").slice(0, 16)}`;
+}

package/dist/sandbox/leases.d.ts

@@ -0,0 +1,105 @@
+/**
+ * The sandbox module (sandbox-leases spec). A consumer holds a sandbox through a lease, and the module owns each
+ * machine: the admission, the spawn, the replacement, the idle removal, and the teardown.
+ *
+ * Each generation of a lease has one lease workflow. A consumer enqueues it through a `DBOSClient`, thus the lease
+ * workflow is not a child of the consumer: a cancel, a pause, or a crash of the consumer does not stop it, and its
+ * teardown step can always run. The consumer waits for the events of the lease workflow with the same client,
+ * because DBOS refuses `recv` and `getEvent` inside a step, and each tool call is a step.
+ *
+ * A machine of a lease is replaceable. The state of a lease is its files, the exec records in its records
+ * directory, and the checkpoints of its consumer. Thus the module removes the machine of an idle lease, and the
+ * next exec gets a new one.
+ */
+import { type DBOSClient } from "@dbos-inc/dbos-sdk";
+import { type Result } from "neverthrow";
+import type { SpawnSession } from "../auth/types.js";
+import { type ResourcePolicy, type ResourceSpec } from "../config/resource-limits.js";
+import type { PausePart, WorkKind } from "../contracts/chat-parts.js";
+import type { RunAuthorizer } from "../execution/run-authorizer.js";
+import type { Connectable } from "../lib/db-result.js";
+import type { Logger } from "../lib/logger.js";
+import type { Querier } from "../state/db.js";
+import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
+import type { SandboxClient } from "./client.js";
+import type { ExecEmit, ExecRequest, ExecResult, ToolchainSource } from "./types.js";
+export declare const LEASE_WORKFLOW = "sandbox-lease";
+export declare const LEASE_QUEUE = "sandbox-leases";
+export declare const DEFAULT_IDLE_PERIOD_MS: number;
+/** The timeout of a command that gives none. A step has no deadline of its own, thus each command has one. */
+export declare const DEFAULT_COMMAND_TIMEOUT_SECONDS = 3600;
+/** The spawn settings of a consumer, beside the size. */
+export type LeaseSpawnSpec = {
+    readonly image?: string;
+    readonly extraEnv?: Readonly<Record<string, string>>;
+    readonly readOnly?: boolean;
+    readonly writableTail?: string;
+};
+/** Who holds a lease, and where its pause and its parts go. */
+export type LeaseBinding = {
+    /** The key of the lease. A tool gives the id of its workflow. */
+    readonly leaseKey: string;
+    /** The spawn session of a run step or of a data profile. It goes into the input of the lease workflow. */
+    readonly session: SpawnSession;
+    /** The work of the consumer. A pause of the lease workflow belongs to it. */
+    readonly work: {
+        readonly kind: WorkKind;
+        readonly id: string;
+    };
+    readonly spawn?: LeaseSpawnSpec;
+    /** Writes a part to the stream of the consumer. A wait mirrors the pause of the lease workflow through it. */
+    readonly emit?: (part: PausePart) => Promise<void>;
+};
+export type AcquireOutcome = {
+    readonly kind: "granted";
+    readonly size: ResourceSpec;
+} | {
+    readonly kind: "refused";
+    readonly reason: string;
+} | {
+    readonly kind: "failed";
+    readonly detail: string;
+};
+/** An exec found no lease that can run it. `detail` says why, for a lease that ended. */
+export type NoLease = {
+    readonly kind: "no_lease";
+    readonly detail?: string;
+};
+/** A lease bound to its consumer. A tool holds one, and gives the size and the command only. */
+export type BoundLease = {
+    acquire(size: ResourceSpec): Promise<AcquireOutcome>;
+    release(): Promise<void>;
+    exec(request: ExecRequest, emit: ExecEmit): Promise<Result<ExecResult, NoLease>>;
+};
+export type SandboxLeases = {
+    readonly toolchainSource: ToolchainSource;
+    acquire(binding: LeaseBinding, size: ResourceSpec): Promise<AcquireOutcome>;
+    release(leaseKey: string): Promise<void>;
+    /**
+     * End each lease of the given owner workflows, and do not wait for the lease workflows. A host that cancels
+     * work calls it, thus the machines go now and not at the next tick of each lease workflow.
+     */
+    releaseOwnedBy(ownerWorkflowIds: readonly string[]): Promise<void>;
+    exec(leaseKey: string, request: ExecRequest, emit: ExecEmit, mirror?: LeaseBinding["emit"]): Promise<Result<ExecResult, NoLease>>;
+    bind(binding: LeaseBinding): BoundLease;
+};
+export type SandboxLeasesConfig = {
+    /** The application pool, which holds the lease table and the pause rows. */
+    readonly pool: Querier & Connectable;
+    readonly client: SandboxClient;
+    /** The budget of the grant. Absent: each lease gets its grant at once. */
+    readonly resourcePolicy?: ResourcePolicy;
+    readonly resolveWorkspaceRoot: ResolveWorkspaceRoot;
+    readonly runAuthorizer: RunAuthorizer;
+    /** Default {@link DEFAULT_IDLE_PERIOD_MS}. */
+    readonly idlePeriodMs?: number;
+    readonly pauseRetryDelaysMs?: readonly number[];
+    readonly logger?: Logger;
+    /** The DBOS client of the enqueue, the sends, and the waits of a consumer. */
+    readonly dbosClient: () => Promise<DBOSClient>;
+    /** The fetch of the health probe. Defaults to `globalThis.fetch`. */
+    readonly fetch?: (url: string, init?: RequestInit) => Promise<Response>;
+    /** The tick of a lease workflow, in seconds. Default 60. */
+    readonly tickSeconds?: number;
+};
+export declare function createSandboxLeases(config: SandboxLeasesConfig): SandboxLeases;