@inflexa-ai/harness 0.1.1

package/dist/sandbox/mount-plan.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Backend-agnostic mount model — the single source of truth for the
+ * container-side paths, writable step subdirectories, and lib-store env
+ * that both the Docker and K8s clients translate into their own mount
+ * mechanism (`HostConfig.Binds` vs `volumes`/`volumeMounts`).
+ *
+ * The storage contract (CLAUDE.md "Storage Layout"): a flat read-only
+ * mount of the analysis tree at `/{resourceId}`, a nested read-write
+ * mount at `/{resourceId}/runs/{runId}/{stepId}` for the step's
+ * artifacts, lib store read-only at `/mnt/libs`, ref store read-only at
+ * `/mnt/refs`.
+ *
+ * Container paths are a function of `resourceId` alone — they never carry the
+ * host location of the tree. Where that tree physically lives is the embedder's
+ * (`resolveWorkspaceRoot`); the two are reconciled per backend: Docker binds the
+ * resolved root directly, K8s addresses it as a `subPath` into the session PVC
+ * (see {@link buildSessionSubPaths}).
+ */
+export declare const STEP_SUBDIRS: readonly ["output", "scripts", "figures", "logs", "notebooks"];
+export interface MountPlanCoords {
+    analysisId: string;
+    runId: string;
+    stepId: string;
+    /**
+     * Enforced read-only: emit no read-write step mount. The container sees only
+     * the read-only analysis tree (plus container-local `/tmp`). Used by the
+     * ephemeral executor, whose prompt promises it physically cannot write files.
+     */
+    readOnly?: boolean;
+}
+export interface MountPlanStores {
+    /** Lib store is mounted at `/mnt/libs` and its env is emitted. */
+    libs: boolean;
+    /** Ref store is mounted at `/mnt/refs`. */
+    refs: boolean;
+}
+export interface MountPlan {
+    /** Flat read-only mount of the whole analysis tree. */
+    readonlyTreePath: string;
+    /**
+     * Nested read-write mount for this step's artifacts, also the WorkingDir.
+     * Undefined for a read-only sandbox — no writable mount exists; the
+     * WorkingDir falls back to the read-only tree root.
+     */
+    writableStepPath?: string;
+    /** Container WorkingDir: the writable step path, or the RO tree root in
+     *  read-only mode. */
+    workingDir: string;
+    /** Container path of the lib store, present only when `libs`. */
+    libsPath?: string;
+    /** Container path of the ref store, present only when `refs`. */
+    refsPath?: string;
+    /** Pre-created subdirectories under the writable step path. Empty when
+     *  read-only (nothing to pre-create). */
+    stepSubdirs: readonly string[];
+    /** Env merged into the sandbox container: provenance + lib-store vars. */
+    env: Record<string, string>;
+}
+/** K8s `subPath` strings into the session PVC. No leading or trailing slash. */
+export interface SessionSubPaths {
+    /** Read-only mount of the whole analysis tree. */
+    readonly ro: string;
+    /** Read-write mount of the step's artifact dir; absent when read-only. */
+    readonly rw?: string;
+}
+/**
+ * K8s `subPath`s for the session-PVC mounts.
+ *
+ * `workspaceSubPath` is the analysis's workspace root expressed relative to the PVC root —
+ * i.e. `relative(pvcRoot, resolveWorkspaceRoot(analysisId))`. Deriving the subPaths from the
+ * same resolver that `precreateStepTree` mkdirs under is what makes the directory the harness
+ * creates and the directory the pod mounts provably the same one. Hardcoding `{analysisId}`
+ * here instead would silently mount elsewhere for any embedder whose roots are not laid out
+ * as `{pvcRoot}/{analysisId}` — a coupling nothing in the type system would catch.
+ */
+export declare function buildSessionSubPaths(coords: MountPlanCoords, workspaceSubPath: string): SessionSubPaths;
+export declare function buildMountPlan(coords: MountPlanCoords, stores: MountPlanStores): MountPlan;

package/dist/sandbox/mount-plan.js

@@ -0,0 +1,84 @@
+/**
+ * Backend-agnostic mount model — the single source of truth for the
+ * container-side paths, writable step subdirectories, and lib-store env
+ * that both the Docker and K8s clients translate into their own mount
+ * mechanism (`HostConfig.Binds` vs `volumes`/`volumeMounts`).
+ *
+ * The storage contract (CLAUDE.md "Storage Layout"): a flat read-only
+ * mount of the analysis tree at `/{resourceId}`, a nested read-write
+ * mount at `/{resourceId}/runs/{runId}/{stepId}` for the step's
+ * artifacts, lib store read-only at `/mnt/libs`, ref store read-only at
+ * `/mnt/refs`.
+ *
+ * Container paths are a function of `resourceId` alone — they never carry the
+ * host location of the tree. Where that tree physically lives is the embedder's
+ * (`resolveWorkspaceRoot`); the two are reconciled per backend: Docker binds the
+ * resolved root directly, K8s addresses it as a `subPath` into the session PVC
+ * (see {@link buildSessionSubPaths}).
+ */
+import { assertSafeId } from "../workspace/paths.js";
+export const STEP_SUBDIRS = ["output", "scripts", "figures", "logs", "notebooks"];
+const LIBS_CONTAINER_PATH = "/mnt/libs";
+const REFS_CONTAINER_PATH = "/mnt/refs";
+/**
+ * The step tail beneath an analysis's workspace root, in both container and
+ * PVC-relative space. The harness owns this layout (workspace-layout spec).
+ */
+function stepTail(runId, stepId) {
+    // The single builder feeding both the container RW path (`buildMountPlan`)
+    // and the K8s RW subPath (`buildSessionSubPaths`); validate here so a crafted
+    // `stepId` cannot inject a `..`/`/` into either.
+    assertSafeId(runId, "runId");
+    assertSafeId(stepId, "stepId");
+    return `runs/${runId}/${stepId}`;
+}
+/**
+ * K8s `subPath`s for the session-PVC mounts.
+ *
+ * `workspaceSubPath` is the analysis's workspace root expressed relative to the PVC root —
+ * i.e. `relative(pvcRoot, resolveWorkspaceRoot(analysisId))`. Deriving the subPaths from the
+ * same resolver that `precreateStepTree` mkdirs under is what makes the directory the harness
+ * creates and the directory the pod mounts provably the same one. Hardcoding `{analysisId}`
+ * here instead would silently mount elsewhere for any embedder whose roots are not laid out
+ * as `{pvcRoot}/{analysisId}` — a coupling nothing in the type system would catch.
+ */
+export function buildSessionSubPaths(coords, workspaceSubPath) {
+    if (workspaceSubPath.length === 0 || workspaceSubPath.startsWith("/") || workspaceSubPath.split("/").includes("..")) {
+        throw new Error(`buildSessionSubPaths: workspaceSubPath must be a non-empty PVC-root-relative path without '..' (got ${JSON.stringify(workspaceSubPath)})`);
+    }
+    return {
+        ro: workspaceSubPath,
+        rw: coords.readOnly ? undefined : `${workspaceSubPath}/${stepTail(coords.runId, coords.stepId)}`,
+    };
+}
+/**
+ * Lib-store package-discovery env. PYTHONPATH is intentionally omitted —
+ * system Python resolves via a `.pth` file in the lib store.
+ */
+function libStoreEnv() {
+    return {
+        R_LIBS_SITE: "/mnt/libs/current/r/github:/mnt/libs/current/r/bioconductor:/mnt/libs/current/r/cran",
+        NODE_PATH: "/mnt/libs/current/node/node_modules",
+        PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/libs/current/conda/bin",
+    };
+}
+export function buildMountPlan(coords, stores) {
+    const { analysisId, runId, stepId, readOnly } = coords;
+    // `analysisId` becomes the RO mount point `/{analysisId}` even in read-only
+    // mode (where `stepTail` — which validates runId/stepId — is not reached).
+    assertSafeId(analysisId, "analysisId");
+    const readonlyTreePath = `/${analysisId}`;
+    const writableStepPath = readOnly ? undefined : `${readonlyTreePath}/${stepTail(runId, stepId)}`;
+    return {
+        readonlyTreePath,
+        writableStepPath,
+        workingDir: writableStepPath ?? readonlyTreePath,
+        libsPath: stores.libs ? LIBS_CONTAINER_PATH : undefined,
+        refsPath: stores.refs ? REFS_CONTAINER_PATH : undefined,
+        stepSubdirs: readOnly ? [] : STEP_SUBDIRS,
+        env: {
+            PROVENANCE_WATCH_DIRS: readonlyTreePath,
+            ...(stores.libs ? libStoreEnv() : {}),
+        },
+    };
+}

package/dist/sandbox/notification-sweep.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Notification-cleanup sweep — clears unconsumed `dbos.notifications`
+ * rows whose owning workflow has reached a terminal status (SUCCESS,
+ * ERROR, CANCELLED).
+ *
+ * Background: a `DBOS.send` to an already-completed workflow accumulates
+ * forever as `consumed=f`. The watchdog's
+ * synthetic-failure send guarantees this happens at the protocol level —
+ * a real `complete` may arrive microseconds before the watchdog and the
+ * loser becomes a stuck notification.
+ *
+ * Cadence is separate from the liveness watchdog (~5 minutes vs ~1
+ * minute). Bounded delete batch keeps the lock window short.
+ *
+ * Production reads the DBOS system DB via the app pool (DBOS by default
+ * shares the database, only the pool is separate). The query function is
+ * injectable so tests can drive the sweep against a stub.
+ */
+import type { Pool } from "pg";
+import type pino from "pino";
+export interface SweepDeps {
+    /**
+     * Execute the bounded DELETE. Returns the row count cleared. Injected
+     * for tests; production wires the app pool.
+     */
+    deleteStale: (limit: number) => Promise<number>;
+    logger?: Pick<pino.Logger, "info" | "warn" | "error">;
+}
+export declare function sweepStaleNotifications(deps: SweepDeps): Promise<number>;
+/**
+ * Default `deleteStale` for production — issues the bounded DELETE
+ * against the shared application pool, which DBOS by default connects
+ * to. The CTE form keeps the join + the LIMIT in one statement.
+ */
+export declare function makeDefaultDeleteStale(pool: Pool): (limit: number) => Promise<number>;
+export interface RegisterNotificationSweepDeps {
+    pool: Pool;
+    logger?: Pick<pino.Logger, "info" | "warn" | "error">;
+    /** Override the DELETE driver — production defaults to the app pool. */
+    deleteStale?: (limit: number) => Promise<number>;
+}
+export declare function registerNotificationSweep(deps: RegisterNotificationSweepDeps): void;

package/dist/sandbox/notification-sweep.js

@@ -0,0 +1,74 @@
+/**
+ * Notification-cleanup sweep — clears unconsumed `dbos.notifications`
+ * rows whose owning workflow has reached a terminal status (SUCCESS,
+ * ERROR, CANCELLED).
+ *
+ * Background: a `DBOS.send` to an already-completed workflow accumulates
+ * forever as `consumed=f`. The watchdog's
+ * synthetic-failure send guarantees this happens at the protocol level —
+ * a real `complete` may arrive microseconds before the watchdog and the
+ * loser becomes a stuck notification.
+ *
+ * Cadence is separate from the liveness watchdog (~5 minutes vs ~1
+ * minute). Bounded delete batch keeps the lock window short.
+ *
+ * Production reads the DBOS system DB via the app pool (DBOS by default
+ * shares the database, only the pool is separate). The query function is
+ * injectable so tests can drive the sweep against a stub.
+ */
+import { DBOS } from "@dbos-inc/dbos-sdk";
+const SWEEP_CRON = "0 */5 * * * *";
+const DELETE_BATCH_LIMIT = 10_000;
+export async function sweepStaleNotifications(deps) {
+    try {
+        const cleared = await deps.deleteStale(DELETE_BATCH_LIMIT);
+        deps.logger?.info({ rowsCleared: cleared }, "[notification-sweep] cleared stale notifications");
+        return cleared;
+    }
+    catch (err) {
+        deps.logger?.error({ err: err instanceof Error ? err.message : String(err) }, "[notification-sweep] sweep failed");
+        throw err;
+    }
+}
+/**
+ * Default `deleteStale` for production — issues the bounded DELETE
+ * against the shared application pool, which DBOS by default connects
+ * to. The CTE form keeps the join + the LIMIT in one statement.
+ */
+export function makeDefaultDeleteStale(pool) {
+    return async (limit) => {
+        const result = await pool.query({
+            text: `WITH stale AS (
+              SELECT destination_uuid, topic, message
+              FROM dbos.notifications n
+              WHERE n.consumed = false
+                AND EXISTS (
+                  SELECT 1 FROM dbos.workflow_status w
+                  WHERE w.workflow_uuid = n.destination_uuid
+                    AND w.status IN ('SUCCESS','ERROR','CANCELLED')
+                )
+              LIMIT $1
+            )
+            DELETE FROM dbos.notifications n
+            USING stale s
+            WHERE n.destination_uuid = s.destination_uuid
+              AND n.topic = s.topic
+              AND n.message = s.message`,
+            values: [limit],
+        });
+        return result.rowCount ?? 0;
+    };
+}
+export function registerNotificationSweep(deps) {
+    const deleteStale = deps.deleteStale ?? makeDefaultDeleteStale(deps.pool);
+    // The scheduled target must itself be a registered workflow — DBOS's
+    // scheduler loop skips (and errors on) any `@scheduled` function that lacks
+    // a workflow registration. Register, then schedule the same callable.
+    const sweep = DBOS.registerWorkflow(async () => {
+        await sweepStaleNotifications({ deleteStale, logger: deps.logger });
+    }, { name: "dbos-notifications-sweep" });
+    DBOS.registerScheduled(sweep, {
+        name: "dbos-notifications-sweep",
+        crontab: SWEEP_CRON,
+    });
+}

package/dist/sandbox/reaper.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Sandbox reaper — the sole cleanup for orphaned sandboxes and stale registry
+ * rows (see the harness-sandbox-exec spec, CONTEXT.md "Sandbox reaper").
+ *
+ * Distinct from the liveness watchdog (`sandbox/watchdog.ts`): the watchdog
+ * sweeps registry→cluster and unblocks stuck recvs; the reaper sweeps
+ * cluster→registry and garbage-collects. It lists every Cortex-managed sandbox
+ * machine in the configured namespace, reads each machine's owner-workflow-id
+ * label, and reaps any whose owning workflow is terminal/missing — tearing the
+ * machine down and reconciling its step row.
+ *
+ * Why a sweep and not teardown-on-cancel: DBOS forbids running a step in a
+ * cancelled workflow, so the body cannot tear down on the cancel path; and only
+ * a cluster-side sweep catches permanent scale-down orphans (see the harness-durable-runtime spec), whose
+ * workflows never recover to run any cleanup.
+ *
+ * The pure decision logic (`classifyManaged`) and the sweep (`reapOnce`) are
+ * hoisted for unit testing without a DBOS runtime; `registerSandboxReaper` is
+ * the thin production wiring.
+ */
+import type pino from "pino";
+import type { Pool } from "pg";
+import type { SandboxClient } from "./client.js";
+import type { ManagedSandbox } from "./types.js";
+export type ReapDecision = "leave" | "reap";
+/**
+ * Decide whether a managed machine is reapable. The owning workflow's status
+ * is authoritative — DBOS writes it at enqueue, *before* the machine exists, so
+ * a just-spawned machine is always observably in-flight and never mistaken for
+ * an orphan. Only the no-status case (no owner label, or the workflow is gone)
+ * falls back to a creation-time grace, the one window the status signal can't
+ * cover.
+ */
+export declare function classifyManaged(sb: ManagedSandbox, status: {
+    status: string;
+} | null, nowMs: number, graceMs: number): ReapDecision;
+/** Map the owning workflow's terminal status to the step row's terminal status. */
+export declare function terminalStepStatus(workflowStatus: string | null): "canceled" | "failed" | "completed";
+export interface ReaperDeps {
+    pool: Pool;
+    sandboxClient: Pick<SandboxClient, "listManagedSandboxes" | "teardownById">;
+    getStatus: (workflowId: string) => Promise<{
+        status: string;
+    } | null>;
+    /** Creation-time grace for label-less / orphaned machines. */
+    graceMs?: number;
+    /** Wall-clock ms; injected for tests. */
+    nowMs?: () => number;
+    logger?: Pick<pino.Logger, "info" | "warn" | "error">;
+}
+export interface ReapSummary {
+    managedCount: number;
+    reapedCount: number;
+    rowsReconciled: number;
+    leftCount: number;
+}
+/**
+ * One cluster→registry sweep. Lists managed machines, classifies each against
+ * its owner workflow's status, and reaps the terminal/orphaned ones. A reap
+ * failure on one machine is logged and skipped — the next sweep retries.
+ */
+export declare function reapOnce(deps: ReaperDeps): Promise<ReapSummary>;
+export interface RegisterReaperDeps {
+    pool: Pool;
+    sandboxClient: Pick<SandboxClient, "listManagedSandboxes" | "teardownById">;
+    graceMs?: number;
+    logger?: Pick<pino.Logger, "info" | "warn" | "error">;
+}
+/** Production registration: a single (unsharded) `@DBOS.scheduled` workflow. */
+export declare function registerSandboxReaper(deps: RegisterReaperDeps): void;

package/dist/sandbox/reaper.js

@@ -0,0 +1,124 @@
+/**
+ * Sandbox reaper — the sole cleanup for orphaned sandboxes and stale registry
+ * rows (see the harness-sandbox-exec spec, CONTEXT.md "Sandbox reaper").
+ *
+ * Distinct from the liveness watchdog (`sandbox/watchdog.ts`): the watchdog
+ * sweeps registry→cluster and unblocks stuck recvs; the reaper sweeps
+ * cluster→registry and garbage-collects. It lists every Cortex-managed sandbox
+ * machine in the configured namespace, reads each machine's owner-workflow-id
+ * label, and reaps any whose owning workflow is terminal/missing — tearing the
+ * machine down and reconciling its step row.
+ *
+ * Why a sweep and not teardown-on-cancel: DBOS forbids running a step in a
+ * cancelled workflow, so the body cannot tear down on the cancel path; and only
+ * a cluster-side sweep catches permanent scale-down orphans (see the harness-durable-runtime spec), whose
+ * workflows never recover to run any cleanup.
+ *
+ * The pure decision logic (`classifyManaged`) and the sweep (`reapOnce`) are
+ * hoisted for unit testing without a DBOS runtime; `registerSandboxReaper` is
+ * the thin production wiring.
+ */
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { unwrapOrThrow } from "../lib/result.js";
+import { reconcileReapedSandbox } from "../state/active-sandboxes.js";
+const REAPER_CRON = "0 */5 * * * *";
+const DEFAULT_GRACE_MS = 10 * 60_000;
+/**
+ * Workflow statuses that mean "still owns its sandbox — never reap". Must
+ * include every non-terminal DBOS state (`mapDbosToRunStatus` in
+ * `routes/run-status.ts` is the canonical active set): `RUNNING` especially —
+ * an actively-executing step's workflow is RUNNING, and omitting it would reap
+ * a live sandbox out from under a running step.
+ */
+const ACTIVE_WORKFLOW_STATUSES = new Set(["PENDING", "ENQUEUED", "RUNNING"]);
+/**
+ * Decide whether a managed machine is reapable. The owning workflow's status
+ * is authoritative — DBOS writes it at enqueue, *before* the machine exists, so
+ * a just-spawned machine is always observably in-flight and never mistaken for
+ * an orphan. Only the no-status case (no owner label, or the workflow is gone)
+ * falls back to a creation-time grace, the one window the status signal can't
+ * cover.
+ */
+export function classifyManaged(sb, status, nowMs, graceMs) {
+    if (status && ACTIVE_WORKFLOW_STATUSES.has(status.status))
+        return "leave";
+    if (status)
+        return "reap";
+    const ageMs = sb.createdAtMs == null ? Number.POSITIVE_INFINITY : nowMs - sb.createdAtMs;
+    return ageMs >= graceMs ? "reap" : "leave";
+}
+/** Map the owning workflow's terminal status to the step row's terminal status. */
+export function terminalStepStatus(workflowStatus) {
+    switch (workflowStatus) {
+        case "CANCELLED":
+            return "canceled";
+        case "SUCCESS":
+            return "completed";
+        default:
+            // ERROR, MAX_RECOVERY_ATTEMPTS_EXCEEDED, or no workflow at all.
+            return "failed";
+    }
+}
+/**
+ * One cluster→registry sweep. Lists managed machines, classifies each against
+ * its owner workflow's status, and reaps the terminal/orphaned ones. A reap
+ * failure on one machine is logged and skipped — the next sweep retries.
+ */
+export async function reapOnce(deps) {
+    const nowMs = (deps.nowMs ?? (() => Date.now()))();
+    const graceMs = deps.graceMs ?? DEFAULT_GRACE_MS;
+    const managed = await deps.sandboxClient.listManagedSandboxes();
+    let reapedCount = 0;
+    let rowsReconciled = 0;
+    let leftCount = 0;
+    for (const sb of managed) {
+        const status = sb.ownerWorkflowId ? await deps.getStatus(sb.ownerWorkflowId) : null;
+        if (classifyManaged(sb, status, nowMs, graceMs) === "leave") {
+            leftCount++;
+            continue;
+        }
+        try {
+            await deps.sandboxClient.teardownById(sb.sandboxId);
+            reapedCount++;
+            const reconciled = unwrapOrThrow(await reconcileReapedSandbox(deps.pool, sb.sandboxId, terminalStepStatus(status?.status ?? null)));
+            if (reconciled)
+                rowsReconciled++;
+        }
+        catch (err) {
+            deps.logger?.warn({
+                sandboxId: sb.sandboxId,
+                err: err instanceof Error ? err.message : String(err),
+            }, "[sandbox-reaper] reap failed — skipping this round");
+        }
+    }
+    const summary = {
+        managedCount: managed.length,
+        reapedCount,
+        rowsReconciled,
+        leftCount,
+    };
+    deps.logger?.info(summary, "[sandbox-reaper] sweep completed");
+    return summary;
+}
+/** Production registration: a single (unsharded) `@DBOS.scheduled` workflow. */
+export function registerSandboxReaper(deps) {
+    const reaper = DBOS.registerWorkflow(async () => {
+        // The whole sweep is ONE step: `listManagedSandboxes` is live cluster
+        // state (set membership + order vary across runs) and it gates how many
+        // per-machine `getWorkflowStatus` reads happen. In the workflow body
+        // `getWorkflowStatus` is a function-ID-counting op, so a differing list
+        // on replay would diverge the recorded sequence. Inside a step the sweep
+        // is checkpointed wholesale and `getWorkflowStatus` is a plain read.
+        await DBOS.runStep(() => reapOnce({
+            pool: deps.pool,
+            sandboxClient: deps.sandboxClient,
+            getStatus: async (workflowId) => (await DBOS.getWorkflowStatus(workflowId)),
+            graceMs: deps.graceMs,
+            logger: deps.logger,
+        }), { name: "sandbox-reaper-sweep" });
+    }, { name: "sandbox-reaper" });
+    DBOS.registerScheduled(reaper, {
+        name: "sandbox-reaper",
+        crontab: REAPER_CRON,
+    });
+}

package/dist/sandbox/sandbox-error.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Sandbox Result glue for cortex.
+ *
+ * The sandbox backend boundaries — the Docker SDK ops (`docker-client.ts`) and
+ * the K8s client ops (`k8s-client.ts`) — model failure as values: each external
+ * op returns `ResultAsync<T, SandboxError>`. The ONLY `try/catch` against the
+ * SDK/driver call lives in those files (via the `trySandbox` helper here);
+ * everything up to the composition seam flows `ResultAsync`. (`submitExec`
+ * (`submit-exec.ts`) is NOT on this channel — it wraps its own `/exec` POST in a
+ * `DBOS.runStep` and throws on a non-202 so the step boundary records the
+ * failure.)
+ *
+ * House rules realized here (see `lib/result.ts`):
+ *  - Absence / "already gone" is NOT an error. A 404 on inspect/isAlive is the
+ *    OK-channel fact `ok(false)`; a 404 on teardown is a successful idempotent
+ *    teardown `ok(undefined)`; a 409 name-conflict that the owner-guard adopts
+ *    is the OK-channel outcome of create. `SandboxError` is reserved for a
+ *    genuine failure — a create that cannot complete, a name-collision the
+ *    owner-guard REFUSES, a submit the server rejects, a teardown the driver
+ *    errors on, a liveness probe that errors.
+ *  - Control-flow exceptions are NOT failures and are never captured as a
+ *    `SandboxError`. The recv loop's `HardCancelError` / `ExecTimeoutError`
+ *    (`await-exec.ts`) and DBOS's `DBOSWorkflowCancelledError` / `AbortError`
+ *    are control-flow signalling — they live OUTSIDE this error channel and
+ *    propagate untouched. `await-exec.ts` is NOT converted; `trySandbox` only
+ *    ever wraps the single SDK/HTTP call handed to it.
+ *
+ * The public `SandboxClient` interface keeps `Promise<T>` method signatures.
+ * `create-sandbox.ts` is the composition seam: it builds the backend ops, then
+ * `unwrapOrThrow`s each op's `ResultAsync` so the seam re-throws a genuine
+ * `SandboxError`. (`submitExec` is the exception — it is already its own DBOS
+ * step and throws directly, so the seam forwards its `Promise<void>` as-is.)
+ * Workflow consumers (`sandbox-step.ts`) already wrap each seam call in
+ * `DBOS.runStep`, so the re-thrown error records the step as failed;
+ * non-workflow consumers (`reaper`, `watchdog`, `ephemeral-runner`,
+ * `data-profile`, `run-exec`) already handle a thrown failure. They are
+ * untouched by this wave.
+ */
+import { ResultAsync } from "neverthrow";
+/**
+ * A sandbox backend failure. Absence / idempotent "already gone" is NOT
+ * modelled — those ride the OK channel (`ok(false)` for a dead/missing machine,
+ * `ok(undefined)` for a teardown that found nothing to remove). `op` is a
+ * stable label (e.g. `"docker.createSandbox"`); `sandboxId` is carried when the
+ * op knows it. `status` carries the originating HTTP status when one exists.
+ */
+export type SandboxError = {
+    readonly type: "container_create_failed";
+    readonly op: string;
+    readonly sandboxId?: string;
+    readonly status?: number;
+    readonly cause: unknown;
+} | {
+    readonly type: "name_conflict";
+    readonly op: string;
+    readonly sandboxId: string;
+    readonly owner: string | null;
+    readonly cause?: unknown;
+} | {
+    readonly type: "not_found";
+    readonly op: string;
+    readonly sandboxId?: string;
+    readonly cause?: unknown;
+} | {
+    readonly type: "submit_failed";
+    readonly op: string;
+    readonly execId: string;
+    readonly status?: number;
+    readonly cause: unknown;
+} | {
+    readonly type: "teardown_failed";
+    readonly op: string;
+    readonly sandboxId?: string;
+    readonly status?: number;
+    readonly cause: unknown;
+} | {
+    readonly type: "liveness_failed";
+    readonly op: string;
+    readonly sandboxId?: string;
+    readonly status?: number;
+    readonly cause: unknown;
+};
+/**
+ * The HTTP/driver status a backend SDK error carries. dockerode exposes it on
+ * `.statusCode`; `@kubernetes/client-node` (v1.x) on `.code` (a number).
+ * Mirror the per-client `statusCodeOf` so 404/409 classification is identical.
+ * A Node system error (`code: "ECONNREFUSED"`) has a string `code`, so guard on
+ * `typeof === "number"` to avoid returning it as a status.
+ */
+export declare function sandboxStatusOf(cause: unknown): number | undefined;
+/** A one-line, user-facing description of a `SandboxError` for logs/error bodies. */
+export declare function describeSandboxError(e: SandboxError): string;
+/**
+ * Wrap a single sandbox backend call (Docker SDK op, K8s API op, or the
+ * `/exec` POST). A throw becomes `err(toError(status, cause))`; the caller
+ * supplies `toError` so each op classifies its own throw (create vs submit vs
+ * teardown vs liveness) and special-cases 404/409.
+ *
+ * `fn` runs the single SDK/HTTP call and returns the already-mapped value
+ * (`T`). Keep `fn` to that one call plus trivial mapping; do NOT embed
+ * control-flow that could throw a non-backend error (the recv loop's
+ * `HardCancelError`/`ExecTimeoutError` and DBOS cancellation must stay outside
+ * any `trySandbox` so they propagate as control-flow, never become an `err`).
+ */
+export declare function trySandbox<T>(fn: () => Promise<T>, toError: (status: number | undefined, cause: unknown) => SandboxError): ResultAsync<T, SandboxError>;

package/dist/sandbox/sandbox-error.js

@@ -0,0 +1,92 @@
+/**
+ * Sandbox Result glue for cortex.
+ *
+ * The sandbox backend boundaries — the Docker SDK ops (`docker-client.ts`) and
+ * the K8s client ops (`k8s-client.ts`) — model failure as values: each external
+ * op returns `ResultAsync<T, SandboxError>`. The ONLY `try/catch` against the
+ * SDK/driver call lives in those files (via the `trySandbox` helper here);
+ * everything up to the composition seam flows `ResultAsync`. (`submitExec`
+ * (`submit-exec.ts`) is NOT on this channel — it wraps its own `/exec` POST in a
+ * `DBOS.runStep` and throws on a non-202 so the step boundary records the
+ * failure.)
+ *
+ * House rules realized here (see `lib/result.ts`):
+ *  - Absence / "already gone" is NOT an error. A 404 on inspect/isAlive is the
+ *    OK-channel fact `ok(false)`; a 404 on teardown is a successful idempotent
+ *    teardown `ok(undefined)`; a 409 name-conflict that the owner-guard adopts
+ *    is the OK-channel outcome of create. `SandboxError` is reserved for a
+ *    genuine failure — a create that cannot complete, a name-collision the
+ *    owner-guard REFUSES, a submit the server rejects, a teardown the driver
+ *    errors on, a liveness probe that errors.
+ *  - Control-flow exceptions are NOT failures and are never captured as a
+ *    `SandboxError`. The recv loop's `HardCancelError` / `ExecTimeoutError`
+ *    (`await-exec.ts`) and DBOS's `DBOSWorkflowCancelledError` / `AbortError`
+ *    are control-flow signalling — they live OUTSIDE this error channel and
+ *    propagate untouched. `await-exec.ts` is NOT converted; `trySandbox` only
+ *    ever wraps the single SDK/HTTP call handed to it.
+ *
+ * The public `SandboxClient` interface keeps `Promise<T>` method signatures.
+ * `create-sandbox.ts` is the composition seam: it builds the backend ops, then
+ * `unwrapOrThrow`s each op's `ResultAsync` so the seam re-throws a genuine
+ * `SandboxError`. (`submitExec` is the exception — it is already its own DBOS
+ * step and throws directly, so the seam forwards its `Promise<void>` as-is.)
+ * Workflow consumers (`sandbox-step.ts`) already wrap each seam call in
+ * `DBOS.runStep`, so the re-thrown error records the step as failed;
+ * non-workflow consumers (`reaper`, `watchdog`, `ephemeral-runner`,
+ * `data-profile`, `run-exec`) already handle a thrown failure. They are
+ * untouched by this wave.
+ */
+import { ResultAsync, err, ok } from "neverthrow";
+const _assertDomainError = true;
+/**
+ * The HTTP/driver status a backend SDK error carries. dockerode exposes it on
+ * `.statusCode`; `@kubernetes/client-node` (v1.x) on `.code` (a number).
+ * Mirror the per-client `statusCodeOf` so 404/409 classification is identical.
+ * A Node system error (`code: "ECONNREFUSED"`) has a string `code`, so guard on
+ * `typeof === "number"` to avoid returning it as a status.
+ */
+export function sandboxStatusOf(cause) {
+    const e = cause;
+    if (typeof e.code === "number")
+        return e.code;
+    return e.statusCode ?? e.response?.statusCode;
+}
+/** A one-line, user-facing description of a `SandboxError` for logs/error bodies. */
+export function describeSandboxError(e) {
+    switch (e.type) {
+        case "container_create_failed":
+            return `sandbox create failed (${e.op}${e.sandboxId ? `: ${e.sandboxId}` : ""})`;
+        case "name_conflict":
+            return `sandbox name collision (${e.op}: ${e.sandboxId} owned by ${e.owner ?? "<unlabeled>"})`;
+        case "not_found":
+            return `sandbox not found (${e.op}${e.sandboxId ? `: ${e.sandboxId}` : ""})`;
+        case "submit_failed":
+            return `sandbox exec submit failed (${e.op}: execId=${e.execId})`;
+        case "teardown_failed":
+            return `sandbox teardown failed (${e.op}${e.sandboxId ? `: ${e.sandboxId}` : ""})`;
+        case "liveness_failed":
+            return `sandbox liveness probe failed (${e.op}${e.sandboxId ? `: ${e.sandboxId}` : ""})`;
+    }
+}
+/**
+ * Wrap a single sandbox backend call (Docker SDK op, K8s API op, or the
+ * `/exec` POST). A throw becomes `err(toError(status, cause))`; the caller
+ * supplies `toError` so each op classifies its own throw (create vs submit vs
+ * teardown vs liveness) and special-cases 404/409.
+ *
+ * `fn` runs the single SDK/HTTP call and returns the already-mapped value
+ * (`T`). Keep `fn` to that one call plus trivial mapping; do NOT embed
+ * control-flow that could throw a non-backend error (the recv loop's
+ * `HardCancelError`/`ExecTimeoutError` and DBOS cancellation must stay outside
+ * any `trySandbox` so they propagate as control-flow, never become an `err`).
+ */
+export function trySandbox(fn, toError) {
+    return new ResultAsync((async () => {
+        try {
+            return ok(await fn());
+        }
+        catch (cause) {
+            return err(toError(sandboxStatusOf(cause), cause));
+        }
+    })());
+}