@inflexa-ai/harness 0.46.2 → 0.48.0

package/dist/runtime/assemble.js

@@ -28,9 +28,11 @@ import { workPause } from "../workflows/pause.js";
 import { hasBrowserUrl } from "../lib/chrome.js";
 import { createStaticEyes } from "../lib/eyes.js";
 import { registerExecuteAnalysis } from "../workflows/execute-analysis.js";
+import { createSandboxLeases } from "../sandbox/leases.js";
+import { createNoopLogger } from "../lib/console-logger.js";
+import { dbosClientFor } from "../execution/dbos-workflow-purger.js";
 import { registerSandboxStep } from "../workflows/sandbox-step.js";
 import { registerDataProfileWorkflow, stopDataProfile } from "../tasks/data-profile.js";
-import { createExtractionArm, runExtraction } from "../tasks/extract-values.js";
 import { createCitationResolver } from "../citations/resolve.js";
 import { createThreadStore } from "../memory/thread-store.js";
 import { createWorkingMemory } from "../memory/working-memory.js";
@@ -38,7 +40,7 @@ import { createArtifactReadStore, createProductionResolver } from "../report-mod
 import { createReportVersionStore } from "../state/report-versions.js";
 import { createToolOutputStore } from "../state/tool-outputs.js";
 import { createAskGateway } from "../tools/approval/gateway.js";
-import { createLiteratureReviewerTool } from "../tools/research/literature-reviewer.js";
+import { createSynthesisLookupTools } from "../tools/research/synthesis-lookups.js";
 const _assertDomainError = true;
 /**
  * Wrap a type→agent registry as the resolution surface. Held apart from
@@ -83,7 +85,7 @@ export function resolveCompositionEyes(seam, chrome) {
     return undefined;
 }
 export function assembleCoreRuntime(deps) {
-    const { conversation, workflows: wf, resourcePolicy } = deps;
+    const { conversation, workflows: wf } = deps;
     const usageRecorder = deps.usageRecorder ?? createNoopUsageRecorder();
     const citationResolver = createCitationResolver({
         ...deps.citationResolverConfig,
@@ -102,8 +104,21 @@ export function assembleCoreRuntime(deps) {
         logger: conversation.logger,
     }).afterFailure;
     const registerPausingSubAgent = (name, body) => registerSubAgent(name, body, subAgentPauses);
+    // One sandbox module for each runtime: one budget, one lease workflow, and one reconciler.
+    const leases = createSandboxLeases({
+        pool: conversation.pool,
+        client: deps.sandbox.client,
+        resourcePolicy: deps.resourcePolicy,
+        resolveWorkspaceRoot: conversation.resolveWorkspaceRoot,
+        runAuthorizer: conversation.runAuthorizer,
+        idlePeriodMs: deps.sandbox.idlePeriodMs,
+        pauseRetryDelaysMs,
+        logger: conversation.logger,
+        dbosClient: () => dbosClientFor(conversation.pool, conversation.logger ?? createNoopLogger()),
+    });
     const sandboxStep = registerSandboxStep({
         ...wf.sandboxStep,
+        leases,
         citationResolver,
         usageRecorder,
         toolOutputStore,
@@ -111,24 +126,18 @@ export function assembleCoreRuntime(deps) {
         runAuthorizer: conversation.runAuthorizer,
     });
     const { bioKeys: synthesisBioKeys, ...executeAnalysisDeps } = wf.buildExecuteAnalysis(sandboxStep);
-    const literatureReviewer = createLiteratureReviewerTool({
-        provider: executeAnalysisDeps.provider,
-        model: executeAnalysisDeps.synthesisModel,
-        bioKeys: synthesisBioKeys,
-        citationResolver,
-        usageRecorder,
-        toolOutputStore,
-        logger: executeAnalysisDeps.logger,
-    }, registerPausingSubAgent);
+    const synthesisLookups = createSynthesisLookupTools({ bioKeys: synthesisBioKeys, citationResolver, logger: executeAnalysisDeps.logger });
     const executeAnalysis = registerExecuteAnalysis({
         ...executeAnalysisDeps,
-        literatureReviewer,
+        synthesisLookups,
+        scriptRuntime: conversation.scriptRuntime,
+        workspaceFs: conversation.workspaceFs,
         usageRecorder,
         toolOutputStore,
         workingMemory: createWorkingMemory(conversation.pool),
         pauseRetryDelaysMs,
     });
-    const dataProfile = registerDataProfileWorkflow({ ...wf.dataProfile, usageRecorder, toolOutputStore, pauseRetryDelaysMs });
+    const dataProfile = registerDataProfileWorkflow({ ...wf.dataProfile, leases, usageRecorder, toolOutputStore, pauseRetryDelaysMs });
     // The session runtime binds the per-session state to the thread behind the
     // tool boundary, thus one instance serves every report thread. It comes before
     // the conversation agent, because `start_report_session` takes its anchor
@@ -147,7 +156,6 @@ export function assembleCoreRuntime(deps) {
         ...conversation,
         executeAnalysisWorkflow: executeAnalysis,
         anchorReportSession: reportSession.ensureSessionState.bind(reportSession),
-        resourcePolicy,
         usageRecorder,
         toolOutputStore,
         citationResolver,
@@ -155,6 +163,7 @@ export function assembleCoreRuntime(deps) {
         // The start tool passes it to the spawn, which emits the creation of a
         // report session. The same seam reaches the report agent below.
         provenance: deps.provenance,
+        syncSessionFile: deps.syncSessionFile,
         registerSubAgent: registerPausingSubAgent,
     });
     // The report agent is a singleton over the conversation deps, the same way the
@@ -164,21 +173,16 @@ export function assembleCoreRuntime(deps) {
     const reportThreads = createThreadStore(conversation.pool);
     // The read cache of the production report path, one for each recent analysis. A preview
     // and the record that follows it are two tool calls over one draft, thus they meet the
-    // same artifacts. The store carries the streamed hash and the extracted rows between
-    // them, and each resolver still binds its own extraction arm below. Thus the auth of a
-    // call never outlives the call, and one unchanged file costs one hash and one container.
+    // same artifacts. The store carries the streamed hash and the parsed rows between them,
+    // thus one unchanged file costs one hash and one parse.
     const reportArtifactReads = createArtifactReadStore();
     // The reference resolver of the report path, and its production wiring. A resolver reads
     // the pinned artifacts of one analysis, thus the factory binds one analysis and one caller
-    // for each tool call. The extraction arm falls through to an extraction pass on the sandbox
-    // rail of the profile for an over-cap file or a host parse fault. The pass runs inside the
-    // step of the tool call. The cap gates the host read, and the resolver defaults it to 16 MiB.
+    // for each tool call. The cap gates the host read, and the resolver defaults it to 16 MiB.
     const makeProductionReportResolver = async (scope) => {
-        const arm = createExtractionArm((requests) => runExtraction({ sandboxClient: wf.dataProfile.sandboxClient, logger: wf.dataProfile.logger }, scope, requests));
         return createProductionResolver({
             workspaceRoot: await conversation.resolveWorkspaceRoot(scope.analysisId),
             analysisId: scope.analysisId,
-            extractionArm: arm,
             readCache: reportArtifactReads.forAnalysis(scope.analysisId),
             cap: deps.reportHostReadCapBytes,
         });
@@ -192,21 +196,19 @@ export function assembleCoreRuntime(deps) {
         gateway: reportSession.gateway,
         resolveWorkspaceRoot: conversation.resolveWorkspaceRoot,
         store: reportVersionStore,
+        derivations: reportSession.derivations,
         threads: reportThreads,
         chrome: conversation.chrome,
-        // The derivation tool draws the rail of the value tier: the same sandbox client. Thus a
-        // session derivation and an extraction run on one substrate, and neither one mints a run
-        // of the analysis.
-        derivations: reportSession.derivations,
-        sandboxClient: wf.dataProfile.sandboxClient,
         makeResolver: makeReportResolver,
         eyes,
         resolvePageAsset: deps.resolveReportPageAsset,
         makeSessionPages: deps.makeSessionPagePublisher,
         resolvePageUrl: deps.resolveReportPageUrl,
         provenance: deps.provenance,
+        syncSessionFile: deps.syncSessionFile,
         logger: conversation.logger,
         toolOutputStore,
+        scriptRuntime: conversation.scriptRuntime,
     });
     // The single registration point for every typed agent. A future thread type's
     // agent plugs in here at assembly with no embedder change. `conversation` is
@@ -232,8 +234,18 @@ export function assembleCoreRuntime(deps) {
         credentialFor: deps.credentialFor,
         pauseRetryDelaysMs,
     });
+    const runCanceler = createRunCanceler({
+        pool: conversation.pool,
+        runCharge: executeAnalysisDeps.runCharge,
+        runAuthorizer: conversation.runAuthorizer,
+        // The end of a canceled run goes where the run engine emits the rest of the run.
+        provenance: executeAnalysisDeps.provenance,
+        logger: conversation.logger,
+        releaseLeasesOf: (workflowIds) => leases.releaseOwnedBy(workflowIds),
+    });
     return {
         agents: agentResolver,
+        runCanceler,
         askGateway,
         chatTurns,
         work: createWork({
@@ -241,15 +253,15 @@ export function assembleCoreRuntime(deps) {
             runAuthorizer: conversation.runAuthorizer,
             logger: conversation.logger,
             stopTurn: (turnId) => chatTurns.stop(turnId),
-            runCanceler: createRunCanceler({
-                pool: conversation.pool,
-                runCharge: executeAnalysisDeps.runCharge,
-                runAuthorizer: conversation.runAuthorizer,
-                // The end of a canceled run goes where the run engine emits the rest of the run.
-                provenance: executeAnalysisDeps.provenance,
-                logger: conversation.logger,
-            }),
-            stopProfile: (analysisId, workflowId) => stopDataProfile({ pool: conversation.pool, logger: conversation.logger, notifyDataProfileSettled: wf.dataProfile.notifyDataProfileSettled }, analysisId, workflowId),
+            runCanceler,
+            stopProfile: async (analysisId, workflowId) => {
+                await stopDataProfile({ pool: conversation.pool, logger: conversation.logger, notifyDataProfileSettled: wf.dataProfile.notifyDataProfileSettled }, analysisId, workflowId);
+                // A canceled body runs no teardown, thus the lease of the profile ends here and not at its owner check.
+                await leases.releaseOwnedBy([workflowId]).catch((failure) => {
+                    const logger = conversation.logger ?? createNoopLogger();
+                    logger.error("the sandbox lease of a stopped profile did not end", { analysisId, ...logger.errorFields(failure) });
+                });
+            },
         }),
         reportSession: { ensureSessionState: reportSession.ensureSessionState.bind(reportSession) },
         workflows: {
@@ -263,7 +275,6 @@ export function assembleCoreRuntime(deps) {
             executeAnalysisWorkflow: executeAnalysis,
             runAuthorizer: conversation.runAuthorizer,
             runLauncher: conversation.runLauncher,
-            resourcePolicy,
             extendAnalysisFarm: conversation.extendAnalysisFarm,
             logger: conversation.logger,
             provenance: deps.provenance,

package/dist/sandbox/client.d.ts

@@ -19,6 +19,7 @@
  */
 import type { ResultAsync } from "neverthrow";
 import type { SpawnSession } from "../auth/types.js";
+import type { ResourceLimits } from "../config/resource-limits.js";
 import type { SandboxError } from "./sandbox-error.js";
 import type { ExecEmit, ExecRequest, ExecResult, ManagedSandbox, SandboxIdentity, SandboxLiveness, SandboxRef, SandboxSpec, ToolchainSource } from "./types.js";
 export type SandboxClient = {
@@ -31,6 +32,8 @@ export type SandboxClient = {
      * optional field would carry two meanings, undeclared and unknown.
      */
     readonly toolchainSource: ToolchainSource;
+    /** The per-machine ceilings that the client clamps each spawn to. The sandbox module clamps a lease request to the same values. */
+    readonly resourceLimits: ResourceLimits;
     /**
      * DBOS step (`sandbox.create`) — the spawn half of the two-step create
      * (see the harness-sandbox-exec spec). Launches the sandbox-base container/Job under the
@@ -59,9 +62,8 @@ export type SandboxClient = {
      * same step gets the same id, and the sandbox gives it the result of the
      * first. Outside a workflow it throws.
      *
-     * `execId` replaces that id. A caller that runs more than one exec in one
-     * step, or an exec outside a workflow, gives its own id
-     * (`sandbox/step-sandbox.ts`).
+     * `execId` replaces that id. The sandbox leases give an id under the salt
+     * of the lease (`sandbox/leases.ts`).
      *
      * A recovered step submits again under the same id. The sandbox dedups the
      * submit on the id, thus the poll attaches to the exec that already ran or

package/dist/sandbox/create-sandbox.d.ts

@@ -2,17 +2,11 @@
  * Backend-selected `SandboxClient` factory.
  *
  * Docker for local dev, K8s for production — the single place the
- * `SANDBOX_BACKEND` env decision lives. Injected at the composition root
- * as a construction-time dependency (see the harness-durable-runtime spec); callers do NOT import a
- * backend module directly.
- *
- * Registry write/clear callbacks are wired here so the per-backend
- * `createSandbox` / `teardown` implementations stay free of state-layer
- * coupling (they just call the closures we hand them). `exec` is
+ * `SANDBOX_BACKEND` env decision lives. The embedder makes the client and gives
+ * it to `assembleCoreRuntime`, and only the sandbox module calls it. `exec` is
  * backend-agnostic — it only needs the SandboxRef.
  */
 import type { V1Toleration } from "@kubernetes/client-node";
-import type { Querier } from "../state/db.js";
 import type { Logger } from "../lib/logger.js";
 import { type ResourceLimits } from "../config/resource-limits.js";
 import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
@@ -30,8 +24,6 @@ export type SandboxBackendConfig = {
     readonly namespace: string;
 };
 export type CreateSandboxClientConfig = {
-    /** App pool — used for registry writes inside the create/teardown steps. */
-    pool: Querier;
     /** Backend default + namespace fallback source. */
     env: SandboxBackendConfig;
     /** Default sandbox-base image when the workflow doesn't override per-step. */
@@ -174,6 +166,9 @@ export declare function precreateStepTree(deps: {
     resolveWorkspaceRoot: ResolveWorkspaceRoot;
     stepTreeAccess?: "world-writable";
 }, coords: MountPlanCoords): Promise<void>;
+/** Pre-create the directory of the exec records of the lease, which each machine of the lease mounts read-write. */
+export declare function precreateRecordDir(deps: {
+    resolveWorkspaceRoot: ResolveWorkspaceRoot;
+    stepTreeAccess?: "world-writable";
+}, coords: MountPlanCoords): Promise<void>;
 export declare function createSandboxClient(config: CreateSandboxClientConfig): SandboxClient;
-/** Re-exports so consumers import once from this module. */
-export { clearSandboxRef } from "../state/index.js";

package/dist/sandbox/create-sandbox.js

@@ -2,13 +2,8 @@
  * Backend-selected `SandboxClient` factory.
  *
  * Docker for local dev, K8s for production — the single place the
- * `SANDBOX_BACKEND` env decision lives. Injected at the composition root
- * as a construction-time dependency (see the harness-durable-runtime spec); callers do NOT import a
- * backend module directly.
- *
- * Registry write/clear callbacks are wired here so the per-backend
- * `createSandbox` / `teardown` implementations stay free of state-layer
- * coupling (they just call the closures we hand them). `exec` is
+ * `SANDBOX_BACKEND` env decision lives. The embedder makes the client and gives
+ * it to `assembleCoreRuntime`, and only the sandbox module calls it. `exec` is
  * backend-agnostic — it only needs the SandboxRef.
  */
 import { chmod, mkdir } from "node:fs/promises";
@@ -20,13 +15,11 @@ import { recordSandboxExec } from "../lib/metrics.js";
 import { ATTR_INFLEXA_EXEC_ID, stableSpan } from "../lib/otel-spans.js";
 import { clampResources } from "../config/resource-limits.js";
 import { tailWritePrefix } from "../workspace/paths.js";
-import { tryMutation } from "../lib/db-result.js";
 import { unwrapOrThrow } from "../lib/result.js";
-import { setSandboxRef } from "../state/index.js";
 import { capExecStreams, EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
 import { createDockerSandboxOps } from "./docker-client.js";
 import { runExec, stepExecId } from "./exec.js";
-import { noteExecOutcome, sandboxExecOutcomeOf, summarizeExec } from "./exec-outcome.js";
+import { sandboxExecOutcomeOf } from "./exec-outcome.js";
 import { createK8sSandboxOps } from "./k8s-client.js";
 import { mountCoordsOf, sandboxWriteTail } from "./mount-plan.js";
 import { SandboxFailure } from "./sandbox-error.js";
@@ -65,6 +58,13 @@ export async function precreateStepTree(deps, coords) {
         await Promise.all(write.subdirs.map((sub) => chmod(join(writeDir, sub), 0o777)));
     }
 }
+/** Pre-create the directory of the exec records of the lease, which each machine of the lease mounts read-write. */
+export async function precreateRecordDir(deps, coords) {
+    const recordDir = tailWritePrefix({ workspaceRoot: await deps.resolveWorkspaceRoot(coords.analysisId), tail: coords.recordTail });
+    await mkdir(recordDir, { recursive: true });
+    if (deps.stepTreeAccess === "world-writable")
+        await chmod(recordDir, 0o777);
+}
 export function createSandboxClient(config) {
     const backend = config.backend ?? config.env.backend;
     // Normalized once, here, and handed to the backends and to the client
@@ -100,9 +100,6 @@ export function createSandboxClient(config) {
      * unwraps below stay on the plain bridge: they carry a `DbError`.
      */
     const failing = (e) => new SandboxFailure(e);
-    const registerSandbox = async (session, spec, ref) => {
-        unwrapOrThrow(await setSandboxRef(config.pool, session.runFrame.runId, session.runFrame.stepId, ref));
-    };
     const ops = backend === "k8s"
         ? createK8sSandboxOps({
             image: config.image,
@@ -120,7 +117,6 @@ export function createSandboxClient(config) {
             tolerations: config.tolerations,
             runtimeClassName: config.runtimeClassName,
             logger: config.logger,
-            registerSandbox,
         })
         : createDockerSandboxOps({
             image: config.image,
@@ -133,27 +129,9 @@ export function createSandboxClient(config) {
             platform: config.platform,
             engineSocketPath: config.engineSocketPath,
             logger: config.logger,
-            registerSandbox,
         });
-    // `teardown` needs the registry-clear closure but the SandboxRef alone
-    // doesn't carry runId/stepId. Wrap the backend impl so callers pass the
-    // ref + step coordinates and the wrapper does the clear after the
-    // backend-level removal.
     const teardown = async (ref) => {
         unwrapOrThrow((await ops.teardown(ref)).mapErr(failing));
-        // Sandboxes are 1-per-step (see `sandbox-step.ts` body — each child
-        // workflow calls `createSandbox` once then `teardown` once), so the
-        // broad WHERE clause matches exactly one row. If that invariant ever
-        // changes (e.g. a step reuses a sibling's sandbox), this clear must
-        // be scoped to (run, step) to avoid wiping prior steps' provenance.
-        unwrapOrThrow(await tryMutation("createSandbox.teardownClearSandboxRef", async () => {
-            await config.pool.query({
-                text: `UPDATE cortex_step_executions
-            SET sandbox_ref = NULL
-            WHERE sandbox_ref->>'sandboxId' = $1`,
-                values: [ref.sandboxId],
-            });
-        }));
     };
     // One arbiter for both surfaces: the client's `isAlive` method and the
     // poll loop's escalation probe are the same backend inspect.
@@ -185,10 +163,12 @@ export function createSandboxClient(config) {
         // The config states an engine fact (binds preserve host ownership);
         // which remediation that fact demands — today, a world-writable step
         // write tree — is harness-owned and chosen here, not by the embedder.
-        const precreated = await ResultAsync.fromPromise(precreateStepTree({
+        const treeDeps = {
             resolveWorkspaceRoot: config.resolveWorkspaceRoot,
             stepTreeAccess: config.engineBindOwnership === "host-preserved" ? "world-writable" : undefined,
-        }, mountCoordsOf(session, spec)), createFailed("createSandbox.precreateStepTree"));
+        };
+        const coords = mountCoordsOf(session, spec);
+        const precreated = await ResultAsync.fromPromise(Promise.all([precreateStepTree(treeDeps, coords), precreateRecordDir(treeDeps, coords)]), createFailed("createSandbox.precreateStepTree"));
         if (precreated.isErr())
             return err(precreated.error);
         // Clamp to cluster ceilings so the pod is always quota-admissible.
@@ -197,6 +177,7 @@ export function createSandboxClient(config) {
     };
     return {
         toolchainSource,
+        resourceLimits: config.resourceLimits,
         createSandbox: (session, spec, identity) => new ResultAsync(createSandbox(session, spec, identity)),
         // The one place an ExecResult crosses from the wire into the process, so
         // the cap lands here rather than at each consumer. Every downstream use —
@@ -223,7 +204,6 @@ export function createSandboxClient(config) {
                 peakMemoryBytes: result.usage?.peakMemoryBytes,
                 cpuMillis: result.usage?.cpuMillis,
             });
-            noteExecOutcome(ref.sandboxId, summarizeExec(result));
             return result;
         }, { name: "sandbox.exec" }),
         isAlive,
@@ -233,5 +213,3 @@ export function createSandboxClient(config) {
         listManagedSandboxes: async () => unwrapOrThrow((await ops.listManagedSandboxes()).mapErr(failing)),
     };
 }
-/** Re-exports so consumers import once from this module. */
-export { clearSandboxRef } from "../state/index.js";

package/dist/sandbox/docker-client.d.ts

@@ -143,11 +143,9 @@ export type DockerClientConfig = {
     /**
      * Optional logger so a store degradation (a resolved farm whose `inflexa.lock`
      * is absent or invalid by sandbox-create time) is observable instead of a
-     * silent libs-mount drop. Matches the `reaper`/`watchdog` logger seam.
+     * silent libs-mount drop.
      */
     logger?: Logger;
-    /** Hook called after the registry row is written. */
-    registerSandbox: (session: SpawnSession, spec: SandboxSpec, ref: SandboxRef) => Promise<void>;
 };
 /**
  * dockerode connection options for a configured engine socket, or `undefined`

package/dist/sandbox/docker-client.js

@@ -319,6 +319,7 @@ export function createDockerSandboxOps(config) {
                     ...(libsMounted && farm && plan.farmPath ? [`${farm.farmPath}:${plan.farmPath}:ro`] : []),
                     ...(libsMounted && cacheBindable && farm?.cachePath && plan.cachePath ? [`${farm.cachePath}:${plan.cachePath}:rw`] : []),
                     ...(refsMounted && config.refStorePath ? [`${config.refStorePath}:${String(plan.refsPath)}:ro`] : []),
+                    `${tailWritePrefix({ workspaceRoot: hostTreePath, tail: plan.recordTail })}:${plan.recordsPath}:rw`,
                     ...cpuBinds,
                 ];
                 const createFailed = (status, cause) => ({
@@ -348,7 +349,7 @@ export function createDockerSandboxOps(config) {
                     Env: env,
                     User: SANDBOX_ROOT_USER,
                     WorkingDir: plan.workingDir,
-                    Labels: mergeLabels(hostLabels, { ...harnessLabels(session, sandboxId), [OWNER_WORKFLOW_LABEL]: spec.childWorkflowId }),
+                    Labels: mergeLabels(hostLabels, { ...harnessLabels(session, sandboxId), [OWNER_WORKFLOW_LABEL]: spec.ownerWorkflowId }),
                     ExposedPorts: { [`${SANDBOX_SERVER_PORT}/tcp`]: {} },
                     HostConfig: {
                         Binds: binds,
@@ -378,7 +379,7 @@ export function createDockerSandboxOps(config) {
                 };
                 if (config.platform !== undefined)
                     createOpts.platform = config.platform;
-                const sandbox = await createOrAdopt(docker, createOpts, sandboxId, spec.childWorkflowId, createFailed, logger);
+                const sandbox = await createOrAdopt(docker, createOpts, sandboxId, spec.ownerWorkflowId, createFailed, logger);
                 if (sandbox.isErr())
                     return err(sandbox.error);
                 if (!sandbox.value.alreadyRunning) {
@@ -407,9 +408,6 @@ export function createDockerSandboxOps(config) {
                     port,
                     backend: "docker",
                 };
-                const registered = await trySandbox(() => config.registerSandbox(session, spec, ref), createFailed);
-                if (registered.isErr())
-                    return err(registered.error);
                 return ok(ref);
             })());
         },

package/dist/sandbox/exec-outcome.d.ts

@@ -8,13 +8,13 @@
  * keeps that text on the sandbox side of the boundary. A summary gives the
  * SIZE of stderr and never a byte of it.
  *
- * The summary of the last exec of a sandbox lives in one process-local cell,
- * keyed by sandbox id. A sandbox belongs to one step (see the teardown of
- * `create-sandbox.ts`), thus the cell is step-scoped in effect and the step
- * body reads it without a new dependency through the agent factory. A replay
- * re-runs the await loop and writes the cell again, thus a recovered body
- * reads the summary of the same exec. The map is capped, so a sandbox whose
- * step never reaches the read site cannot hold memory.
+ * The summary of the last exec of a lease lives in one process-local cell,
+ * keyed by the lease key. A step holds one lease key, thus the cell is
+ * step-scoped in effect and the step body reads it without a new dependency
+ * through the agent factory. The cell keeps the summary across a machine
+ * replacement. A replay re-runs the await loop and writes the cell again, thus
+ * a recovered body reads the summary of the same exec. The map is capped, so a
+ * lease whose step never reaches the read site cannot hold memory.
  */
 import type { SandboxExecOutcome } from "../lib/metrics.js";
 import type { ExecResult } from "./types.js";
@@ -39,7 +39,7 @@ export type ExecOutcomeSummary = {
 };
 /** Reduce a result to its summary. */
 export declare function summarizeExec(result: ExecResult): ExecOutcomeSummary;
-/** Keep `summary` as the last known exec of `sandboxId`. */
-export declare function noteExecOutcome(sandboxId: string, summary: ExecOutcomeSummary): void;
-/** The last known exec of `sandboxId`, or `undefined` when it ran none. */
-export declare function lastExecOutcome(sandboxId: string): ExecOutcomeSummary | undefined;
+/** Keep `summary` as the last known exec of the lease `leaseKey`. */
+export declare function noteExecOutcome(leaseKey: string, summary: ExecOutcomeSummary): void;
+/** The last known exec of the lease `leaseKey`, or `undefined` when it ran none. */
+export declare function lastExecOutcome(leaseKey: string): ExecOutcomeSummary | undefined;

package/dist/sandbox/exec-outcome.js

@@ -8,13 +8,13 @@
  * keeps that text on the sandbox side of the boundary. A summary gives the
  * SIZE of stderr and never a byte of it.
  *
- * The summary of the last exec of a sandbox lives in one process-local cell,
- * keyed by sandbox id. A sandbox belongs to one step (see the teardown of
- * `create-sandbox.ts`), thus the cell is step-scoped in effect and the step
- * body reads it without a new dependency through the agent factory. A replay
- * re-runs the await loop and writes the cell again, thus a recovered body
- * reads the summary of the same exec. The map is capped, so a sandbox whose
- * step never reaches the read site cannot hold memory.
+ * The summary of the last exec of a lease lives in one process-local cell,
+ * keyed by the lease key. A step holds one lease key, thus the cell is
+ * step-scoped in effect and the step body reads it without a new dependency
+ * through the agent factory. The cell keeps the summary across a machine
+ * replacement. A replay re-runs the await loop and writes the cell again, thus
+ * a recovered body reads the summary of the same exec. The map is capped, so a
+ * lease whose step never reaches the read site cannot hold memory.
  */
 /** The reason that names a machine the kernel killed for its memory limit. */
 const OOM_KILLED = "sandbox-oom-killed";
@@ -41,25 +41,25 @@ export function summarizeExec(result) {
     };
 }
 /**
- * How many sandboxes keep a summary. A host runs far fewer sandboxes at once
- * than this, thus the cap only drops the summary of a step that ended long ago.
+ * How many leases keep a summary. A host holds far fewer leases at once than
+ * this, thus the cap only drops the summary of a step that ended long ago.
  */
-const MAX_TRACKED_SANDBOXES = 256;
-const lastExecBySandbox = new Map();
-/** Keep `summary` as the last known exec of `sandboxId`. */
-export function noteExecOutcome(sandboxId, summary) {
+const MAX_TRACKED_LEASES = 256;
+const lastExecByLease = new Map();
+/** Keep `summary` as the last known exec of the lease `leaseKey`. */
+export function noteExecOutcome(leaseKey, summary) {
     // Delete before set so the key moves to the end of the insertion order and
-    // the eviction below drops the least recently written sandbox.
-    lastExecBySandbox.delete(sandboxId);
-    lastExecBySandbox.set(sandboxId, summary);
-    while (lastExecBySandbox.size > MAX_TRACKED_SANDBOXES) {
-        const oldest = lastExecBySandbox.keys().next();
+    // the eviction below drops the least recently written lease.
+    lastExecByLease.delete(leaseKey);
+    lastExecByLease.set(leaseKey, summary);
+    while (lastExecByLease.size > MAX_TRACKED_LEASES) {
+        const oldest = lastExecByLease.keys().next();
         if (oldest.done)
             break;
-        lastExecBySandbox.delete(oldest.value);
+        lastExecByLease.delete(oldest.value);
     }
 }
-/** The last known exec of `sandboxId`, or `undefined` when it ran none. */
-export function lastExecOutcome(sandboxId) {
-    return lastExecBySandbox.get(sandboxId);
+/** The last known exec of the lease `leaseKey`, or `undefined` when it ran none. */
+export function lastExecOutcome(leaseKey) {
+    return lastExecByLease.get(leaseKey);
 }

package/dist/sandbox/exec.d.ts

@@ -52,6 +52,11 @@ export type ExecDeps = {
  * makes the pair unique, thus no caller mints an exec id.
  */
 export declare function stepExecId(): string;
+/**
+ * The exec id of a step under a salt that no machine receives. The id is the same on each run of the step, but the
+ * ids that a command can read in the exec records do not give the id of a later exec.
+ */
+export declare function saltedStepExecId(salt: string): string;
 /**
  * Run one exec to its terminal result.
  *

package/dist/sandbox/exec.js

@@ -14,6 +14,7 @@
  */
 import { errorMessageOf } from "../lib/thrown.js";
 import { textOf } from "../lib/unknown-text.js";
+import { createHash } from "node:crypto";
 import { setTimeout as sleep } from "node:timers/promises";
 import { DBOS, Error as DBOSErrors, StatusString } from "@dbos-inc/dbos-sdk";
 import { untracedFetch } from "../lib/otel-spans.js";
@@ -62,6 +63,13 @@ export function stepExecId() {
     }
     return `${workflowId}:${stepId}`;
 }
+/**
+ * The exec id of a step under a salt that no machine receives. The id is the same on each run of the step, but the
+ * ids that a command can read in the exec records do not give the id of a later exec.
+ */
+export function saltedStepExecId(salt) {
+    return createHash("sha256").update(`${salt}:${stepExecId()}`).digest("hex");
+}
 /**
  * Run one exec to its terminal result.
  *
@@ -177,7 +185,7 @@ async function pollExec(ref, execId, emit, deadlineMs, deps) {
             if (resp.cursor > cursor)
                 cursor = resp.cursor;
             if (resp.result !== undefined)
-                return resp.result;
+                return resp.result.lost === undefined ? resp.result : syntheticFailureResult(execId, "sandbox-replaced");
         }
         if (deps.isAlive && escalation.onPoll(polled.kind === "ok" ? "ok" : "unavailable")) {
             const verdict = await probeLiveness(deps.isAlive, ref);

package/dist/sandbox/identity.d.ts

@@ -1,22 +1,10 @@
 /**
- * Sandbox identity minting — the durable half of the two-step create (ADR
- * 0016). Backend-agnostic: the `sandboxId` is minted here, checkpointed by the
- * `sandbox.mint` DBOS step, then handed to the backend's spawn step. Because the
- * identity is durable before any machine exists, a crash between spawn and the
- * spawn-step checkpoint is recoverable — the re-run adopts the existing machine
- * under the same name.
+ * The name of a sandbox machine. It comes from a key: the id of the lease workflow and the machine generation. Thus
+ * a rerun of the spawn step adopts the machine of the earlier attempt, and no step checkpoints a name before a spawn.
  */
 import type { SandboxIdentity } from "./types.js";
-/** `sbx-{run8}-{rand8}` — run prefix aids `kubectl` triage; the 8-hex random
- *  suffix makes within-run collisions across a run's concurrent steps
- *  negligible. Names are NOT the idempotency mechanism (the checkpoint is), and
- *  adoption is owner-label-guarded, but a unique name keeps a collision from
- *  ever reaching that guard. */
-export declare function mintSandboxIdentity(runId: string): SandboxIdentity;
 /**
- * `sbx-{run8}-{hash12}` — the name of a sandbox whose identity comes from a
- * key, not from a checkpoint. The same key gives the same name, thus a rerun
- * of a step adopts the machine of the earlier attempt. The key is hashed,
- * because a workflow id can hold characters that a container name refuses.
+ * `sbx-{run8}-{hash12}`. The run prefix aids `kubectl` triage. The same key gives the same name, and the key is
+ * hashed, because a workflow id can hold characters that a container name refuses.
  */
 export declare function keyedSandboxIdentity(runId: string, key: string): SandboxIdentity;