@agenthippo-ai/agenthippo 0.2.3-dataplane → 1.110.11

package/out/sandbox/processDriver.js

@@ -0,0 +1,318 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) AgentHippo.ai. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProcessSandboxDriver = exports.PROVIDER_KEY_PASSTHROUGH_VARS = exports.NS_PROBE_ARGS = exports.UNSHARE_WRAP_ARGS = void 0;
+exports.namespaceIsolationFromEnv = namespaceIsolationFromEnv;
+exports.probeNamespaceIsolation = probeNamespaceIsolation;
+exports.providerKeyPassthroughFromEnv = providerKeyPassthroughFromEnv;
+exports.sandboxEnvCarriesProviderKey = sandboxEnvCarriesProviderKey;
+exports.baseSandboxEnv = baseSandboxEnv;
+/**
+ * The `process` driver (tier 0) — today's behavior, re-transported over stdio.
+ *
+ * This is the laptop and free-tier path and it must stay first-class forever: it is the only
+ * driver with no dependency on a container runtime. It provides **no isolation boundary** — the
+ * sandbox shares the host's uid, PID namespace and filesystem — which is why `tier` is 0 and why
+ * per-route revocation throws instead of pretending.
+ *
+ * On Linux hosts that allow unprivileged user namespaces, the spawn is wrapped in
+ * `unshare --user --pid --mount-proc` (see UNSHARE_WRAP_ARGS): each sandbox then sees only its own
+ * process tree in /proc, so a concurrent tenant cannot read another turn's env or argv
+ * (/proc/<pid>/environ is same-uid readable and /proc/<pid>/cmdline is WORLD-readable — the
+ * "residual P4" nonce-lift). This is hardening, not a tier bump: the filesystem is still the
+ * host's, and tier 1 — as the container driver's downgrade defines it — still implies containment
+ * this does not have. The tier stays 0 and the wrapper is disclosed in the boot log instead.
+ */
+const node_child_process_1 = require("node:child_process");
+const types_1 = require("./types");
+/** Resolve the namespace-isolation mode a deployment asked for (serve's default is 'auto'). */
+function namespaceIsolationFromEnv(env = process.env) {
+    const v = env.AGENTHIPPO_SANDBOX_NS_ISOLATION?.trim().toLowerCase();
+    if (v === 'off' || v === '0') {
+        return 'off';
+    }
+    if (v === 'require') {
+        return 'require';
+    }
+    return 'auto';
+}
+/**
+ * The wrapper: `unshare <these> <command> <args…>`.
+ *
+ * `--user` because that is what an unprivileged process is allowed to unshare, and creating it
+ * first is what authorizes `--pid`/`--mount-proc` without root. `--map-current-user` keeps
+ * getuid() stable inside (an unmapped ns reports the overflow uid and `os.userInfo()` in the
+ * runner host throws). `--pid --fork --mount-proc` is the actual fix: the sandbox's /proc shows
+ * only its own tree, so a sibling's environ/cmdline are unnameable, not merely unreadable.
+ * `--kill-child=SIGTERM` forwards our TTL/destroy SIGTERM to the inner tree — without it, killing
+ * the unshare parent orphans a live PID namespace.
+ *
+ * Known cost, accepted: the runner host becomes PID 1 of the namespace and Node does not reap
+ * orphans it did not spawn, so a tool subprocess that outlives its parent lingers as a zombie
+ * until the sandbox's TTL reaps everything. Bounded by sandbox lifetime, invisible outside it.
+ */
+exports.UNSHARE_WRAP_ARGS = [
+    '--user', '--map-current-user',
+    '--pid', '--fork', '--kill-child=SIGTERM',
+    '--mount-proc',
+];
+/**
+ * What the probe actually runs: the full wrapper with a NESTED `unshare --user` inside it, not a
+ * bare `true`. Engines that sandbox themselves (codex via bubblewrap) must create their own
+ * namespaces INSIDE ours, and a host can permit one level while its policy (Ubuntu 23.10+ gates
+ * unprivileged userns behind AppArmor profiles) refuses the second — a single-level probe would
+ * pass, and then every codex turn would die at bwrap. Probing the worst case means a host that
+ * would break a self-sandboxing engine gets the honest fallback for every engine, with no
+ * per-engine branch in the driver (which cannot know the engine at create time anyway: the
+ * sandbox serves whatever engine each turn requests). Verified live on Ubuntu 24.04: nested
+ * unshare and a codex-shaped bwrap (--unshare-all, --proc, --tmpfs /tmp) both work under the
+ * wrapper on an unrestricted kernel.
+ */
+exports.NS_PROBE_ARGS = [...exports.UNSHARE_WRAP_ARGS, 'unshare', '--user', 'true'];
+/**
+ * Can this host actually run the wrapper? Empirical, not inferential: distros differ in whether
+ * unprivileged user namespaces are allowed (and whether they can nest — see NS_PROBE_ARGS),
+ * container runtimes mask /proc so `--mount-proc` fails EPERM inside a default Docker/k8s
+ * container, and util-linux versions differ in which flags exist — running the real command
+ * answers all of it at once. Non-Linux hosts fail the spawn with ENOENT and land in the same
+ * `false`.
+ */
+async function probeNamespaceIsolation(spawnFn = node_child_process_1.spawn, timeoutMs = 5_000) {
+    return new Promise((resolve) => {
+        let timer;
+        let settled = false;
+        const done = (ok) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            if (timer) {
+                clearTimeout(timer);
+            }
+            resolve(ok);
+        };
+        let child;
+        try {
+            child = spawnFn('unshare', [...exports.NS_PROBE_ARGS], { stdio: 'ignore' });
+        }
+        catch {
+            done(false);
+            return;
+        }
+        timer = setTimeout(() => { child.kill('SIGKILL'); done(false); }, timeoutMs);
+        timer.unref?.();
+        child.once('error', () => done(false));
+        child.once('exit', (code) => done(code === 0));
+    });
+}
+/**
+ * Provider keys that some engine genuinely cannot do without, and which therefore break the
+ * "the sandbox holds no provider key" invariant when they are passed through.
+ */
+exports.PROVIDER_KEY_PASSTHROUGH_VARS = ['CURSOR_API_KEY'];
+/**
+ * The opt-in. `AGENTHIPPO_SANDBOX_ALLOW_PROVIDER_KEY=1` (or an explicit driver option) is required
+ * before a raw provider key crosses into a sandbox.
+ */
+function providerKeyPassthroughFromEnv(env = process.env) {
+    return env.AGENTHIPPO_SANDBOX_ALLOW_PROVIDER_KEY === '1';
+}
+/**
+ * Does this child env actually contain a raw provider key?
+ *
+ * Used to decide the tier the sandbox is TOLD it has. A sandbox holding a provider key can call the
+ * provider directly — around the proxy, around the meter, around the per-sandbox warrant — so it
+ * does not have the credential-containment property tier 0 already lacks a boundary for, and must
+ * not be reported as anything better.
+ */
+function sandboxEnvCarriesProviderKey(env) {
+    return exports.PROVIDER_KEY_PASSTHROUGH_VARS.some((k) => Boolean(env[k]));
+}
+class ProcessSandboxDriver {
+    opts;
+    kind = 'process';
+    tier = 0;
+    sandboxes = new Map();
+    seq = 0;
+    /** Probed once per driver, not per sandbox; a 'require' failure is cached and re-thrown. */
+    nsWrap;
+    constructor(opts) {
+        this.opts = opts;
+    }
+    namespaceWrap() {
+        if (!this.nsWrap) {
+            const mode = this.opts.namespaceIsolation ?? 'off';
+            if (mode === 'off') {
+                this.nsWrap = Promise.resolve(false);
+            }
+            else {
+                const probe = this.opts.probeFn ?? (() => probeNamespaceIsolation());
+                this.nsWrap = probe().then((ok) => {
+                    if (ok) {
+                        this.opts.logger?.info('[Sandbox] namespace isolation active: each sandbox gets its own PID namespace (/proc shows only its own tree)');
+                        return true;
+                    }
+                    if (mode === 'require') {
+                        throw new Error('AGENTHIPPO_SANDBOX_NS_ISOLATION=require, but this host cannot run ' +
+                            `\`unshare ${exports.UNSHARE_WRAP_ARGS.join(' ')}\` — no sandbox may exist here whose env is ` +
+                            'readable by a concurrent tenant. Enable unprivileged user namespaces (or use ' +
+                            '--sandbox container), or drop the requirement.');
+                    }
+                    this.opts.logger?.warn('namespace isolation unavailable on this host (unshare probe failed) — tier-0 sandboxes ' +
+                        'share one /proc, so a concurrent same-uid process can read a turn\'s env. Fine on a ' +
+                        'laptop; on a multi-tenant host set AGENTHIPPO_SANDBOX_NS_ISOLATION=require or use ' +
+                        '--sandbox container.');
+                    return false;
+                });
+            }
+        }
+        return this.nsWrap;
+    }
+    async create(spec) {
+        const id = `sandbox-${++this.seq}-${sanitize(spec.sandboxKey)}`;
+        const spawnFn = this.opts.spawnFn ?? node_child_process_1.spawn;
+        const wrap = await this.namespaceWrap();
+        const command = wrap ? 'unshare' : this.opts.command;
+        const args = wrap
+            ? [...exports.UNSHARE_WRAP_ARGS, this.opts.command, ...(this.opts.args ?? [])]
+            : this.opts.args ?? [];
+        const base = baseSandboxEnv({
+            ...(this.opts.allowProviderKeyPassthrough !== undefined
+                ? { allowProviderKeyPassthrough: this.opts.allowProviderKeyPassthrough }
+                : {}),
+            logger: { warn: (m) => console.warn(m) },
+        });
+        const childEnv = { ...base, ...this.opts.env, ...spec.env };
+        // A sandbox holding a raw provider key can reach the provider directly — around the proxy,
+        // the meter and the per-sandbox warrant. Tier 0 already claims no boundary, so there is no
+        // lower tier to drop to; report it explicitly instead of letting the disclosure live only
+        // in a comment.
+        const carriesProviderKey = sandboxEnvCarriesProviderKey(childEnv);
+        const child = spawnFn(command, args, {
+            cwd: this.opts.cwd,
+            // stderr passes through to the host log; stdin/stdout carry the frame protocol.
+            stdio: ['pipe', 'pipe', 'inherit'],
+            // A curated env, not an inherited one: the sandbox must not receive serve's credentials.
+            // But it DOES need the basics — an empty env leaves the sandbox with no PATH, so engine
+            // CLIs (claude, codex) cannot be discovered and every turn fails (observed live).
+            // The tier is stamped LAST so no caller-supplied env can overstate the boundary: a
+            // tier-0 child must never be told it is running behind a container, or engines that
+            // sandbox themselves (codex) would stand down with nothing underneath them.
+            env: {
+                ...childEnv,
+                [types_1.SANDBOX_TIER_ENV]: String(this.tier),
+                ...(carriesProviderKey ? { [types_1.SANDBOX_PROVIDER_KEY_ENV]: '1' } : {}),
+            },
+        });
+        if (carriesProviderKey) {
+            console.warn(`[Sandbox] ${id} runs WITH a raw provider key in its environment ` +
+                `(${exports.PROVIDER_KEY_PASSTHROUGH_VARS.filter((k) => childEnv[k]).join(', ')}). ` +
+                `This sandbox does not have the model-credential containment the other engines get; ` +
+                `disclose it in the deployment.`);
+        }
+        const closed = new Promise((resolve) => {
+            child.once('exit', () => resolve());
+            child.once('error', () => resolve());
+        });
+        const channel = {
+            input: child.stdout,
+            output: { write: (chunk) => child.stdin?.write(chunk) ?? false },
+            closed,
+        };
+        const sandbox = { child, channel };
+        // TTL is the eventual-kill guarantee: no sandbox outlives its lease revocation by more than
+        // one TTL, even if nothing else reaps it.
+        if (spec.limits.ttlSeconds > 0) {
+            sandbox.ttl = setTimeout(() => { void this.kill(id); }, spec.limits.ttlSeconds * 1000);
+            sandbox.ttl.unref?.();
+        }
+        this.sandboxes.set(id, sandbox);
+        return {
+            id,
+            tier: this.tier,
+            endpoint: 'stdio',
+            connect: () => channel,
+        };
+    }
+    /**
+     * Tier 0 has no network boundary, so a per-route revoke would be a lie. `all` is honest —
+     * it kills the sandbox, which does remove every route.
+     */
+    async revoke(handle, what) {
+        if (what !== 'all') {
+            throw new types_1.RevocationUnsupportedError('process', what);
+        }
+        await this.kill(handle.id);
+    }
+    async destroy(handle) {
+        await this.kill(handle.id);
+    }
+    async kill(id) {
+        const sandbox = this.sandboxes.get(id);
+        if (!sandbox) {
+            return;
+        }
+        this.sandboxes.delete(id);
+        if (sandbox.ttl) {
+            clearTimeout(sandbox.ttl);
+        }
+        if (sandbox.child.exitCode === null && !sandbox.child.killed) {
+            sandbox.child.kill('SIGTERM');
+        }
+    }
+}
+exports.ProcessSandboxDriver = ProcessSandboxDriver;
+/**
+ * The non-secret environment every sandbox needs to function. Deliberately an allowlist: anything
+ * not named here (provider keys, data-plane tokens, gateway secrets) does not cross.
+ */
+function baseSandboxEnv(opts = {}, sourceEnv = process.env) {
+    const allow = [
+        'PATH', 'HOME', 'LANG', 'LC_ALL', 'TMPDIR', 'TERM', 'NODE_ENV',
+        // Engine discovery — paths and directories, never credentials. These are OVERRIDES only:
+        // the sandbox resolves bundled engines itself via findClaudeCodePath()/findCodexPath(),
+        // which already know about .agenthippo-deps and the image's baked SDKs.
+        'CLAUDE_CODE_PATH', 'CODEX_PATH', 'AGENTHIPPO_CUSTOM_ENGINES_DIR', 'AGENTHIPPO_NO_AUTO_INSTALL',
+        // Boot-stable route URLs (not credentials). Sandboxed serve resolves ${env:AGENTHIPPO_DATA_MCP_URL}
+        // when loading pack MCP configs; without these the postgres gateway entry is dropped.
+        'AGENTHIPPO_DATA_MCP_URL', 'AGENTHIPPO_LITELLM_BASE_URL',
+    ];
+    const out = {};
+    for (const key of allow) {
+        const v = sourceEnv[key];
+        if (v) {
+            out[key] = v;
+        }
+    }
+    // DOCUMENTED EXCEPTION to invariant 1, now OPT-IN rather than automatic.
+    //
+    // The Cursor SDK accepts no base URL, so it cannot be routed through the inference proxy and
+    // its key has to live in the sandbox. That is a real constraint — but it was silently true for
+    // every sandbox, which made "the sandbox holds no provider key" false in the shipped
+    // customer-service sample without anything saying so. It now requires
+    // AGENTHIPPO_SANDBOX_ALLOW_PROVIDER_KEY=1, refuses by default, and when it IS on the sandbox is
+    // reported one tier lower (see `create`) because it genuinely does not have credential
+    // containment.
+    const allowPassthrough = opts.allowProviderKeyPassthrough ?? providerKeyPassthroughFromEnv(sourceEnv);
+    for (const key of exports.PROVIDER_KEY_PASSTHROUGH_VARS) {
+        const v = sourceEnv[key];
+        if (!v) {
+            continue;
+        }
+        if (!allowPassthrough) {
+            opts.logger?.warn(`[Sandbox] ${key} is set but was NOT passed into the sandbox: a raw provider key in a ` +
+                `sandbox can call the provider directly, around the inference proxy and its spend cap. ` +
+                `Cursor packs need it and cannot be proxied — set ` +
+                `AGENTHIPPO_SANDBOX_ALLOW_PROVIDER_KEY=1 to allow it and accept that this sandbox holds ` +
+                `a provider credential.`);
+            continue;
+        }
+        out[key] = v;
+    }
+    return out;
+}
+function sanitize(s) {
+    return s.replace(/[^a-zA-Z0-9_-]/g, '').slice(0, 32) || 'session';
+}

package/out/sandbox/sandboxRunner.js

@@ -0,0 +1,303 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) AgentHippo.ai. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSandboxRunner = createSandboxRunner;
+const remoteEngine_1 = require("../runner/remoteEngine");
+const paths_1 = require("../agentpack/paths");
+const credential_context_1 = require("../cli/credential-context");
+function createSandboxRunner(opts) {
+    const sandboxes = new Map();
+    /**
+     * Creations in flight, keyed by sandbox key.
+     *
+     * Two turns arriving together must not each build a sandbox. `sandboxes.get` → `driver.create`
+     * → `sandboxes.set` spans two awaits, so without this both callers miss, both create, and the
+     * loser is a LIVE container that no longer appears in the map: destroy, revoke and the TTL
+     * bookkeeping all miss it, so it keeps its model warrant and answers turns nothing can stop.
+     * That is a kill-switch hole, not a leak. Reachable today — the turn gate caps concurrency
+     * process-wide but never serializes one key.
+     */
+    const creating = new Map();
+    /**
+     * sessionKey → sandboxKey, so callers may keep naming a session while a sandbox holds several.
+     *
+     * Bounded for the reason governor.ts bounds `sessionUsers`: a long-lived serve sees unbounded
+     * distinct session keys (every anonymous gateway sender gets a fresh uuid), and this must never
+     * be the reason an instance falls over. Entries are never purged on teardown — a stale one
+     * resolves to a sandbox that is gone, which the lookups already treat as "nothing to do", and
+     * the next turn overwrites it.
+     */
+    const sessionToSandbox = new Map();
+    const MAX_TRACKED_SESSIONS = 10_000;
+    const rememberSession = (sessionKey, sandboxKey) => {
+        if (sessionToSandbox.size >= MAX_TRACKED_SESSIONS && !sessionToSandbox.has(sessionKey)) {
+            const oldest = sessionToSandbox.keys().next();
+            if (!oldest.done) {
+                sessionToSandbox.delete(oldest.value);
+            }
+        }
+        sessionToSandbox.set(sessionKey, sandboxKey);
+    };
+    /** Accepts a session key or a sandbox key; unknown keys pass through as sandbox keys. */
+    const resolveSandboxKey = (key) => sessionToSandbox.get(key) ?? key;
+    const createSandbox = async (sandboxKey, sessionKey) => {
+        // Admission first: no lease ⇒ no sandbox. This is the kill switch and the commercial gate,
+        // deliberately the same mechanism.
+        await opts.admit?.({ sandboxKey, sessionKey });
+        const handle = await opts.driver.create({
+            sandboxKey,
+            image: opts.image,
+            tier: opts.tier,
+            routes: opts.routes,
+            fs: (await opts.fsFor?.(sandboxKey)) ?? opts.fs ?? { readonly: [], writable: [] },
+            ...(opts.stateDirFor ? { stateDir: await opts.stateDirFor(sandboxKey) } : {}),
+            limits: opts.limits ?? { ttlSeconds: 30 * 60 },
+            env: { ...(opts.env ?? {}), ...(opts.provenanceEnv?.() ?? {}) },
+        });
+        const modelToken = await opts.mintModelToken?.(handle.id, sandboxKey);
+        const channel = handle.connect();
+        const engine = new remoteEngine_1.RemoteEngine({
+            id: 'sandbox',
+            displayName: `Sandbox ${handle.id}`,
+            channel,
+            model: {
+                baseUrl: opts.inferenceBaseUrl,
+                modelToken,
+                // Re-minted per turn: a warrant is capped at turn length, but a sandbox lives for
+                // its TTL. Binding the sandbox's identity here (handle.id) keeps the warrant
+                // non-transferable while letting its lifetime stay honest.
+                ...(opts.mintModelToken
+                    ? { mintToken: async () => opts.mintModelToken(handle.id, sandboxKey) }
+                    : {}),
+            },
+            logger: opts.logger,
+        });
+        const sandbox = {
+            handle, engine, dead: false, sessionKeys: new Set([sessionKey]), queue: Promise.resolve(),
+        };
+        // A sandbox dies on TTL reap, OOM, or node loss. Mark it so the next turn recreates rather
+        // than dialing a socket that is gone — otherwise the first message after a reap fails.
+        const onClosed = (err) => {
+            sandbox.dead = true;
+            // Still registered ⇒ the channel died on its own. destroySession, dispose and
+            // revoke('all') remove their entry first and have already torn the sandbox down.
+            const unexpected = sandboxes.get(sandboxKey) === sandbox;
+            if (unexpected) {
+                sandboxes.delete(sandboxKey);
+            }
+            const why = err instanceof Error ? `: ${err.message}` : '';
+            opts.logger?.info(`[Sandbox] ${handle.id} closed${why}; ${sandboxKey} will rehydrate on next message`);
+            // Forgetting a sandbox is not the same as destroying it. A channel that dies while the
+            // CONTAINER is healthy (a failed dial, a dropped socket) used to leave that container
+            // running forever — observed live: four orphans still answering `ready` eight hours
+            // after the run that abandoned them.
+            if (unexpected) {
+                void opts.driver.destroy(handle).catch(() => undefined);
+            }
+        };
+        // Both arms: the channel REJECTS with the socket-level reason (see unixSocketChannel), and
+        // a one-armed `.then` would turn that into an unhandled rejection.
+        void channel.closed?.then(() => onClosed(), onClosed);
+        sandboxes.set(sandboxKey, sandbox);
+        opts.logger?.info(`[Sandbox] created ${handle.id} for ${sandboxKey} (tier ${handle.tier})`);
+        return sandbox;
+    };
+    /**
+     * The one place a sandbox is looked up or built. NOT async, deliberately: `creating.set` must
+     * run in the same synchronous step as the misses above it, and an `async` wrapper invites
+     * someone to add an await between them, which silently reopens the create race.
+     */
+    const sandboxFor = (sandboxKey, sessionKey) => {
+        const existing = sandboxes.get(sandboxKey);
+        if (existing && !existing.dead) {
+            return Promise.resolve(existing);
+        }
+        if (existing?.dead) {
+            sandboxes.delete(sandboxKey);
+        }
+        const inFlight = creating.get(sandboxKey);
+        if (inFlight) {
+            return inFlight;
+        }
+        // `finally`, not `then`: a failed admission or a driver error must evict the entry, or the
+        // rejected promise is cached forever and every later turn replays a stale refusal.
+        const started = createSandbox(sandboxKey, sessionKey)
+            .finally(() => { creating.delete(sandboxKey); });
+        creating.set(sandboxKey, started);
+        return started;
+    };
+    /**
+     * Runs one turn, serialized when the peer cannot tag streamed events with their run id.
+     *
+     * An untagged peer's events are broadcast to every in-flight turn (see `RemoteEngine.dispatch`),
+     * so overlapping turns would write one session's tokens into another's transcript. The sandbox
+     * image is pinned by digest and upgrades independently of serve, so an older peer is a normal
+     * state, not an error — it just runs one turn at a time.
+     */
+    const exclusive = async (sandbox, fn) => {
+        if (sandbox.engine.supportsDemux()) {
+            return fn(sandbox);
+        }
+        const prior = sandbox.queue;
+        let release;
+        sandbox.queue = new Promise((resolve) => { release = resolve; });
+        await prior;
+        // The sandbox can be reaped or killed while a turn waits its place in line; running now
+        // would dial a socket that is gone. Phrased so the caller's retry recognises it.
+        if (sandbox.dead) {
+            release();
+            throw new Error('sandbox channel closed');
+        }
+        try {
+            return await fn(sandbox);
+        }
+        finally {
+            // In `finally` so a thrown turn cannot wedge every later turn on this sandbox.
+            release();
+        }
+    };
+    const run = async (options) => {
+        const sessionKey = options.sessionId ?? 'default';
+        // Which sandbox owns this turn. Absent ⇒ the session IS the sandbox, which is the historical
+        // behaviour and what any caller that does not thread the key keeps getting.
+        const sandboxKey = options.sandboxKey ?? sessionKey;
+        const join = (sandbox) => {
+            sandbox.sessionKeys.add(sessionKey);
+            rememberSession(sessionKey, sandboxKey);
+            return sandbox;
+        };
+        const collected = { text: '', thinking: '', toolCalls: [] };
+        // Collect for the JSON response while forwarding to serve's SSE emitter when present —
+        // both paths get the same events.
+        const emitter = teeEmitter(options.emitter, collected);
+        const nothingEmitted = () => collected.text === '' && collected.thinking === '' && collected.toolCalls.length === 0;
+        const attempt = async (sandbox) => {
+            await sandbox.engine.stream(options.message, {
+                engineId: options.engineId,
+                modelId: options.modelId,
+                workspaceRoot: options.workspaceRoot,
+                additionalDirectories: options.additionalDirectories,
+                sessionId: options.sessionId,
+                permissions: options.permissions,
+                // Everything serve hands its in-process runner must also cross into a sandbox,
+                // or the sandbox silently behaves differently from the single-user path:
+                // attribution (evidence), the per-turn data-plane nonce (OBO), and the
+                // bound gateway session.
+                analyticsUserId: options.analyticsUserId,
+                // Computed HERE, not in the child: this runs inside serve's own
+                // runWithCredential frame, and the child has none. Without it every user's
+                // engine home inside a sandbox resolved to the same unscoped directory, so
+                // tier 0 was strictly worse for transcript separation than no sandbox at all.
+                engineHomeUserKey: (0, paths_1.engineHomeUserKey)((0, credential_context_1.currentCredential)()?.userId),
+                turnEnv: options.turnEnv,
+                gatewaySessionKey: options.gatewaySessionKey,
+                // No nativeSessionId on purpose: the in-sandbox runner resumes from its own
+                // persisted session store under the session's state dir, exactly as the
+                // single-user path does. A thread id on the wire would name engine state the
+                // sandbox may not have — and it was a permanently-undefined field before this,
+                // so the resume path looked wired and never was.
+            }, emitter, opts.runtime, options.signal);
+        };
+        try {
+            const sandbox = join(await sandboxFor(sandboxKey, sessionKey));
+            try {
+                await exclusive(sandbox, attempt);
+            }
+            catch (err) {
+                // The sandbox died mid-turn (TTL reap, OOM, kill) or before this turn reached the
+                // front of its queue. Recreate and retry ONCE — but only if nothing has been emitted
+                // yet, so a streaming client never sees duplicates.
+                const message = errText(err);
+                if (!/channel closed/i.test(message) || !nothingEmitted()) {
+                    throw err;
+                }
+                // Only if it is still the dead one: a concurrent turn may already have replaced it,
+                // and deleting that would strand a live container the same way the create race did.
+                if (sandboxes.get(sandboxKey) === sandbox) {
+                    sandboxes.delete(sandboxKey);
+                }
+                opts.logger?.info(`[Sandbox] session ${sessionKey} lost its sandbox mid-turn; recreating and retrying once`);
+                await exclusive(join(await sandboxFor(sandboxKey, sessionKey)), attempt);
+            }
+            return { success: true, text: collected.text, thinking: collected.thinking, toolCalls: collected.toolCalls };
+        }
+        catch (err) {
+            return { success: false, text: collected.text, thinking: collected.thinking, toolCalls: collected.toolCalls, error: errText(err) };
+        }
+    };
+    return {
+        run,
+        dispose() {
+            for (const [key, sandbox] of sandboxes) {
+                void opts.driver.destroy(sandbox.handle).catch(() => undefined);
+                sandboxes.delete(key);
+            }
+        },
+        sandboxes: () => [...sandboxes.entries()].map(([sandboxKey, c]) => ({
+            sandboxKey, sandboxId: c.handle.id, tier: c.handle.tier, sessionKeys: [...c.sessionKeys],
+        })),
+        async destroySession(key) {
+            const sandboxKey = resolveSandboxKey(key);
+            const sandbox = sandboxes.get(sandboxKey);
+            if (!sandbox) {
+                return;
+            }
+            sandboxes.delete(sandboxKey);
+            await opts.driver.destroy(sandbox.handle);
+            opts.logger?.info(`[Sandbox] destroyed ${sandbox.handle.id} (${sandboxKey}${describeSessions(sandbox, key)})`);
+        },
+        async revokeRoute(key, what) {
+            const sandboxKey = resolveSandboxKey(key);
+            const sandbox = sandboxes.get(sandboxKey);
+            if (!sandbox) {
+                return;
+            }
+            await opts.driver.revoke(sandbox.handle, what);
+            if (what === 'all') {
+                sandboxes.delete(sandboxKey);
+            }
+            opts.logger?.info(`[Sandbox] revoked ${what} on ${sandbox.handle.id} (${sandboxKey}${describeSessions(sandbox, key)})`);
+        },
+    };
+}
+/** Forwards to serve's emitter (when streaming) while collecting for the JSON response. */
+function teeEmitter(downstream, into) {
+    return {
+        async thinking(delta) { into.thinking += delta; await downstream?.thinking(delta); },
+        async text(delta) { into.text += delta; await downstream?.text(delta); },
+        async toolStart(name, input, toolCallId, metadata) {
+            into.toolCalls.push({ name, input });
+            await downstream?.toolStart(name, input, toolCallId, metadata);
+        },
+        async toolEnd(name, toolCallId, result, isError, metadata) {
+            const entry = [...into.toolCalls].reverse().find((t) => t.name === name && t.result === undefined);
+            if (entry) {
+                entry.result = result;
+                entry.isError = isError;
+            }
+            await downstream?.toolEnd(name, toolCallId, result, isError, metadata);
+        },
+        async progress(message) { await downstream?.progress(message); },
+        async question(payload) {
+            await downstream?.question(payload);
+            // One-way: the runner already answered `declined` in-sandbox.
+            return { selected: [], freeText: null, declined: true };
+        },
+        async error(message) { await downstream?.error(message); },
+        async done() { await downstream?.done(); },
+    };
+}
+function errText(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+/**
+ * Names the OTHER sessions a teardown ends. Under user scope an order aimed at one session stops
+ * that user's other conversations too, and a stop wider than the order that caused it must not be
+ * silent in the log.
+ */
+function describeSessions(sandbox, requested) {
+    const others = [...sandbox.sessionKeys].filter((s) => s !== requested);
+    return others.length === 0 ? '' : `, also ending ${others.length}: ${others.join(', ')}`;
+}