@agenthippo-ai/agenthippo 0.2.3-dataplane → 1.110.11

package/out/test/sandbox-runner.test.js

@@ -0,0 +1,343 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  createSandboxRunner (PLAN/serve-v4-clean.md §3) + per-sandbox model token (blocker C).
+ *
+ *  Behaviors pinned here are the ones a live test caught the hard way:
+ *   - one sandbox per session, reused across turns;
+ *   - a dead sandbox is never handed out again, and the next turn recovers transparently;
+ *   - a retry never duplicates output a streaming client already saw;
+ *   - admission (lease) is checked BEFORE a sandbox exists — the kill switch;
+ *   - the model token is sandbox-bound, so lifting it from one sandbox fails in another.
+ *--------------------------------------------------------------------------------------------*/
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+require("mocha");
+const assert = __importStar(require("assert"));
+const node_stream_1 = require("node:stream");
+const sandboxRunner_1 = require("../sandbox/sandboxRunner");
+const support_1 = require("../sandbox/support");
+const credential_context_1 = require("../cli/credential-context");
+const warrant_1 = require("../authority/warrant");
+const host_1 = require("../runner/host");
+const sandbox_1 = require("../cli/commands/sandbox");
+const RUNTIME = { logger: { info() { }, warn() { }, error() { }, debug() { } } };
+/** A driver whose sandboxes are real runner hosts over in-memory pipes. */
+function fakeDriver(engineFor) {
+    const created = [];
+    const specs = [];
+    const destroyed = [];
+    const revoked = [];
+    const closers = new Map();
+    let seq = 0;
+    const driver = {
+        kind: 'process', tier: 0,
+        async create(spec) {
+            const id = `sandbox-${++seq}`;
+            created.push(spec.sandboxKey);
+            specs.push(spec);
+            const toRunner = new node_stream_1.PassThrough();
+            const toServe = new node_stream_1.PassThrough();
+            (0, host_1.startRunnerHost)({
+                engine: engineFor(), runtime: RUNTIME, input: toRunner,
+                output: { write: (c) => toServe.write(c) },
+                maintenanceIntervalMs: 60_000,
+            });
+            let closeChannel = () => undefined;
+            const closed = new Promise((res) => { closeChannel = res; });
+            closers.set(id, closeChannel);
+            return {
+                id, tier: 0, endpoint: 'stdio',
+                connect: () => ({ input: toServe, output: { write: (c) => toRunner.write(c) }, closed }),
+            };
+        },
+        async revoke(handle, what) { revoked.push({ id: handle.id, what }); },
+        async destroy(handle) { destroyed.push(handle.id); closers.get(handle.id)?.(); },
+    };
+    return { driver, created, specs, destroyed, revoked, kill: (id) => closers.get(id)?.() };
+}
+const echoEngine = () => ({
+    id: 'e', displayName: 'E',
+    async stream(message, _o, emitter) { await emitter.text(`ECHO:${message}`); },
+    onMaintenance() { }, dispose() { },
+});
+function baseOpts(driver) {
+    return {
+        driver, image: 'img@sha256:' + 'a'.repeat(64), tier: 0,
+        routes: [], runtime: RUNTIME,
+    };
+}
+suite('sandbox runner — session affinity and recovery', () => {
+    test('one sandbox per session, reused across turns', async () => {
+        const { driver, created } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)(baseOpts(driver));
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'b', workspaceRoot: '/w', sessionId: 's1' });
+        assert.deepStrictEqual(created, ['s1'], 'second turn on the same session reuses its sandbox');
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'c', workspaceRoot: '/w', sessionId: 's2' });
+        assert.deepStrictEqual(created, ['s1', 's2'], 'a different session gets its own sandbox — never cohabitation');
+        assert.strictEqual(runner.sandboxes().length, 2);
+    });
+    test('turn output flows back through the boundary', async () => {
+        const { driver } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)(baseOpts(driver));
+        const res = await runner.run({ engineId: 'e', modelId: 'm', message: 'hi', workspaceRoot: '/w', sessionId: 's1' });
+        assert.strictEqual(res.success, true);
+        assert.strictEqual(res.text, 'ECHO:hi');
+    });
+    test('a dead sandbox is dropped and the next turn recovers transparently', async () => {
+        const { driver, created, kill } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)(baseOpts(driver));
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        // TTL reap / OOM / node loss.
+        kill(runner.sandboxes()[0].sandboxId);
+        await new Promise((r) => setTimeout(r, 10));
+        const res = await runner.run({ engineId: 'e', modelId: 'm', message: 'b', workspaceRoot: '/w', sessionId: 's1' });
+        assert.strictEqual(res.success, true, 'the turn after a sandbox death must succeed, not fail once first');
+        assert.strictEqual(res.text, 'ECHO:b');
+        assert.deepStrictEqual(created, ['s1', 's1'], 'a fresh sandbox was created for the same session');
+    });
+    test('a sandbox whose channel dies is torn down, not left running', async () => {
+        // The container survives a dead channel — a refused dial or a dropped socket kills only
+        // serve's end. Dropping the map entry without destroying the sandbox leaked one container
+        // per failed turn (observed live: four orphans, healthy and unreachable, hours later).
+        const { driver, destroyed, kill } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)(baseOpts(driver));
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        const sandboxId = runner.sandboxes()[0].sandboxId;
+        kill(sandboxId);
+        await new Promise((r) => setTimeout(r, 10));
+        assert.deepStrictEqual(destroyed, [sandboxId], 'the abandoned sandbox is reaped');
+    });
+    test('a mid-turn retry never duplicates output a streaming client already saw', async () => {
+        // An engine that emits, then hangs — the channel dies after output exists, so retrying
+        // would double the client's stream. The runner must surface the failure instead.
+        const { driver, kill } = fakeDriver(() => ({
+            id: 'e', displayName: 'E',
+            async stream(_m, _o, emitter) {
+                await emitter.text('partial');
+                await new Promise(() => undefined); // never settles
+            },
+            onMaintenance() { }, dispose() { },
+        }));
+        const runner = (0, sandboxRunner_1.createSandboxRunner)(baseOpts(driver));
+        const seen = [];
+        const turn = runner.run({
+            engineId: 'e', modelId: 'm', message: 'x', workspaceRoot: '/w', sessionId: 's1',
+            emitter: {
+                async thinking() { }, async text(d) { seen.push(d); }, async toolStart() { },
+                async toolEnd() { }, async progress() { },
+                async question() { return { selected: [], freeText: null, declined: true }; },
+                async error() { }, async done() { },
+            },
+        });
+        await new Promise((r) => setTimeout(r, 20));
+        kill(runner.sandboxes()[0].sandboxId);
+        const res = await turn;
+        assert.strictEqual(res.success, false, 'output already streamed ⇒ fail rather than silently retry');
+        assert.deepStrictEqual(seen, ['partial'], 'the client saw the text exactly once');
+    });
+});
+suite('sandbox runner — admission is the kill switch', () => {
+    test('a refused lease prevents the sandbox from ever existing', async () => {
+        const { driver, created } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)({
+            ...baseOpts(driver),
+            admit: () => { throw new Error('lease revoked for pack analyst@1.4.2'); },
+        });
+        const res = await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        assert.strictEqual(res.success, false);
+        assert.match(res.error ?? '', /lease revoked/);
+        assert.strictEqual(created.length, 0, 'no sandbox is created when admission fails — new work stops immediately');
+    });
+    test('destroySession and revokeRoute reach the driver', async () => {
+        const { driver, destroyed, revoked } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)(baseOpts(driver));
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        const sandboxId = runner.sandboxes()[0].sandboxId;
+        await runner.revokeRoute('s1', 'inference');
+        assert.deepStrictEqual(revoked, [{ id: sandboxId, what: 'inference' }], 'cap breach revokes the inference route');
+        assert.strictEqual(runner.sandboxes().length, 1, 'revoking a route does not kill the sandbox');
+        await runner.destroySession('s1');
+        assert.deepStrictEqual(destroyed, [sandboxId]);
+        assert.strictEqual(runner.sandboxes().length, 0);
+    });
+});
+suite('sandbox filesystem is per session, not per instance', () => {
+    /**
+     * The container boundary does not extend to a host directory two containers both mount. With a
+     * single static `fs`, every session's sandbox mounted the SAME writable workspace — so one
+     * user's file output was the next user's readable input, one turn later.
+     */
+    test('fsFor gives each session its own writable mount', async () => {
+        const { driver, specs } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)({
+            ...baseOpts(driver),
+            fs: { readonly: ['/packs/a:/app/pack'], writable: ['/shared:/state/workspace'] },
+            fsFor: async (sessionKey) => ({
+                readonly: ['/packs/a:/app/pack'],
+                writable: [`/state/ws/${sessionKey}:/state/workspace`],
+            }),
+        });
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'b', workspaceRoot: '/w', sessionId: 's2' });
+        assert.deepStrictEqual(specs[0].fs.writable, ['/state/ws/s1:/state/workspace']);
+        assert.deepStrictEqual(specs[1].fs.writable, ['/state/ws/s2:/state/workspace']);
+        assert.notStrictEqual(specs[0].fs.writable[0], specs[1].fs.writable[0], 'two sessions never share a writable host directory');
+        // The pack is the same for everyone — read-only, and the whole point of pinning it.
+        assert.deepStrictEqual(specs[0].fs.readonly, specs[1].fs.readonly);
+    });
+    test('without fsFor the static fs still applies — --shared-workspace keeps working', async () => {
+        const { driver, specs } = fakeDriver(echoEngine);
+        const runner = (0, sandboxRunner_1.createSandboxRunner)({
+            ...baseOpts(driver),
+            fs: { readonly: [], writable: ['/shared:/state/workspace'] },
+        });
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'b', workspaceRoot: '/w', sessionId: 's2' });
+        assert.deepStrictEqual(specs[0].fs.writable, specs[1].fs.writable);
+    });
+});
+suite('per-sandbox model token (blocker C)', () => {
+    /**
+     * A warrant is hard-capped at turn length (300s) because nothing revokes one; a sandbox lives
+     * for its TTL (30 min by default). Minting once at create handed the sandbox a credential that
+     * spent most of its life expired — invisible only because the proxy does not check `exp` yet.
+     */
+    test('the model warrant is re-minted per turn, not once per sandbox', async () => {
+        const { driver } = fakeDriver(echoEngine);
+        const minted = [];
+        let n = 0;
+        const runner = (0, sandboxRunner_1.createSandboxRunner)({
+            ...baseOpts(driver),
+            mintModelToken: (sandboxId) => {
+                const token = `warrant-${sandboxId}-${++n}`;
+                minted.push(token);
+                return token;
+            },
+        });
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' });
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'b', workspaceRoot: '/w', sessionId: 's1' });
+        // One sandbox, three mints: one at create plus one per turn.
+        assert.strictEqual(runner.sandboxes().length, 1, 'the sandbox is reused across turns');
+        assert.ok(minted.length >= 3, `expected a mint per turn, got ${minted.length}`);
+        assert.strictEqual(new Set(minted).size, minted.length, 'each turn gets its own warrant');
+        // Still bound to the one sandbox — a fresh lifetime must not mean a fresh identity.
+        assert.ok(minted.every((t) => t.startsWith('warrant-sandbox-1-')), 'every warrant names its sandbox');
+    });
+    /**
+     * The warrant's `user` claim is the ONLY attribution that survives the gateway (it strips
+     * caller-supplied user metadata and re-stamps from the verified claim). serve reads the acting
+     * user via the ambient credential context at mint time, so what this pins is the plumbing that
+     * makes that correct: AsyncLocalStorage must reach mintModelToken through sandbox creation AND
+     * through the per-turn re-mint inside RemoteEngine.stream. serve minted `user: packId` for a
+     * long time precisely because nothing asserted what the mint callback could see.
+     */
+    test('mintModelToken sees each turn\'s ambient credential and the session key', async () => {
+        const { driver } = fakeDriver(echoEngine);
+        const seen = [];
+        const runner = (0, sandboxRunner_1.createSandboxRunner)({
+            ...baseOpts(driver),
+            mintModelToken: (sandboxId, sessionKey) => {
+                seen.push({ user: (0, credential_context_1.currentCredential)()?.userId, sessionKey });
+                return `warrant-${sandboxId}`;
+            },
+        });
+        await (0, credential_context_1.runWithCredential)({ userId: 'alice@corp.com' }, () => runner.run({ engineId: 'e', modelId: 'm', message: 'a', workspaceRoot: '/w', sessionId: 's1' }));
+        await (0, credential_context_1.runWithCredential)({ userId: 'bob@corp.com' }, () => runner.run({ engineId: 'e', modelId: 'm', message: 'b', workspaceRoot: '/w', sessionId: 's1' }));
+        await runner.run({ engineId: 'e', modelId: 'm', message: 'c', workspaceRoot: '/w', sessionId: 's1' });
+        // Turn 1: create-time mint + per-turn re-mint, both inside alice's credential scope.
+        assert.ok(seen.length >= 4, `expected create + per-turn mints, got ${seen.length}`);
+        assert.ok(seen.slice(0, 2).every((m) => m.user === 'alice@corp.com'), 'create-time and first-turn mints carry the first turn\'s user');
+        // Turn 2 reuses the sandbox but re-mints under bob — attribution is per turn, not per sandbox.
+        assert.strictEqual(seen[2].user, 'bob@corp.com', 're-mint carries the current turn\'s user');
+        // Outside any credential scope the mint sees no user — the caller's anon fallback, never a
+        // stale identity from an earlier turn.
+        assert.strictEqual(seen[3].user, undefined, 'no ambient credential leaks across turns');
+        assert.ok(seen.every((m) => m.sessionKey === 's1'), 'every mint is handed the session key');
+    });
+    test('the per-sandbox model credential is a signed warrant, not the provider key', () => {
+        // The HMAC-of-the-provider-key token this suite used to cover was removed: nothing shipped
+        // used it, and its own docstring conceded the cap it advertised was unenforced. The credential
+        // serve actually mints is an ES256 warrant, and these are the properties that matter.
+        const provider = 'sk-live-provider-key-must-never-leave';
+        const key = (0, warrant_1.generateWarrantKeypair)('test');
+        const pubs = { [key.kid]: key.publicKeyPem };
+        const token = (0, support_1.mintSandboxInferenceWarrant)(key, {
+            user: 'u@example.com', pack: 'p', packVersion: '1.0.0', sandboxId: 'sandbox-1', tier: 2,
+        });
+        assert.ok(!token.includes(provider), 'the token must not contain the provider key');
+        assert.ok((0, warrant_1.verifyWarrant)(token, pubs, { audience: 'inference', channelSandboxId: 'sandbox-1' }).ok, 'verifies for its own sandbox');
+        assert.ok(!(0, warrant_1.verifyWarrant)(token, pubs, { audience: 'inference', channelSandboxId: 'sandbox-2' }).ok, 'a token lifted into another sandbox FAILS — this is what makes the cap non-evadable');
+        assert.ok(!(0, warrant_1.verifyWarrant)(token, pubs, { audience: 'data', channelSandboxId: 'sandbox-1' }).ok, 'an inference warrant is not a data-plane warrant');
+    });
+    test('a tampered warrant is refused', () => {
+        const key = (0, warrant_1.generateWarrantKeypair)('test');
+        const pubs = { [key.kid]: key.publicKeyPem };
+        const token = (0, support_1.mintSandboxInferenceWarrant)(key, {
+            user: 'u@example.com', pack: 'p', packVersion: '1.0.0', sandboxId: 'sandbox-1', tier: 2,
+        });
+        // Flip a character in the MIDDLE of the signature segment. The final base64url
+        // character encodes only padding bits, so flipping it decodes to the same bytes
+        // roughly a quarter of the time and the token still verifies.
+        const dot = token.lastIndexOf('.');
+        const sigMid = dot + 1 + Math.floor((token.length - dot - 1) / 2);
+        const tampered = token.slice(0, sigMid) + (token[sigMid] === 'A' ? 'B' : 'A') + token.slice(sigMid + 1);
+        assert.ok(!(0, warrant_1.verifyWarrant)(tampered, pubs, { audience: 'inference', channelSandboxId: 'sandbox-1' }).ok, 'a bad signature is refused');
+        assert.ok(!(0, warrant_1.verifyWarrant)('sk-provider', pubs, { audience: 'inference', channelSandboxId: 'sandbox-1' }).ok, 'a raw provider key is not a valid sandbox credential');
+    });
+    test('two sandboxes get different warrants from the same key', () => {
+        const key = (0, warrant_1.generateWarrantKeypair)('test');
+        const base = { user: 'u@example.com', pack: 'p', packVersion: '1.0.0', tier: 2 };
+        const a = (0, support_1.mintSandboxInferenceWarrant)(key, { ...base, sandboxId: 'sandbox-1' });
+        const b = (0, support_1.mintSandboxInferenceWarrant)(key, { ...base, sandboxId: 'sandbox-2' });
+        assert.notStrictEqual(a, b);
+    });
+});
+suite('sandbox workspace authority', () => {
+    test('the wire never redirects a sandbox out of its own workspace', () => {
+        // The "codex discards the write" bug: serve sends its HOST workspace path on every run
+        // frame, and the sandbox used to mkdir + chdir into it. Inside a container that path is
+        // container-private, so the engine's files were visible via ls/cat in the sandbox and absent
+        // on the host — and the engine ran with none of the synced pack content around it.
+        assert.strictEqual((0, sandbox_1.resolveSandboxWorkspace)('/Users/someone/host/ws', '/state/workspace'), '/state/workspace', 'a host path from the wire must never win');
+        assert.strictEqual((0, sandbox_1.resolveSandboxWorkspace)(undefined, '/state/workspace'), '/state/workspace', 'no wire path — the sandbox workspace still governs');
+    });
+    test('tier 0 sees no mismatch, because serve passes the same path', () => {
+        assert.ok(!(0, sandbox_1.isForeignWorkspace)('/tmp/ws', '/tmp/ws'));
+        assert.ok(!(0, sandbox_1.isForeignWorkspace)('/tmp/ws/', '/tmp/ws'), 'trailing slash is the same directory');
+        assert.ok(!(0, sandbox_1.isForeignWorkspace)(undefined, '/tmp/ws'), 'nothing asked for — nothing to warn about');
+        assert.ok((0, sandbox_1.isForeignWorkspace)('/tmp/other', '/tmp/ws'), 'a genuinely different root is worth a warning');
+    });
+});