@agenthippo-ai/agenthippo 0.2.3-dataplane → 1.110.9

package/out/cli/commands/serve.js

@@ -42,18 +42,32 @@ exports.runGatewayTurn = runGatewayTurn;
 /**
  * Serve Command - HTTP Server
  */
+const crypto = __importStar(require("crypto"));
 const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
 const http = __importStar(require("http"));
 const path = __importStar(require("path"));
 const node_crypto_1 = require("node:crypto");
+const sandboxRunner_1 = require("../../sandbox/sandboxRunner");
+const deps_1 = require("../deps");
+const signature_1 = require("../../pack/signature");
+const reconcile_1 = require("../../pack/reconcile");
+const index_1 = require("../../store/index");
+const lease_1 = require("../../pack/lease");
+const caps_1 = require("../../pack/caps");
+const governor_1 = require("../../pack/governor");
+const containerDriver_1 = require("../../sandbox/containerDriver");
+const processDriver_1 = require("../../sandbox/processDriver");
+const support_1 = require("../../sandbox/support");
 const headless_1 = require("../../headless");
 const pack_loader_1 = require("../pack-loader");
 const utils_1 = require("../utils");
-const runtime_1 = require("../runtime");
+const pack_runtime_1 = require("../pack-runtime");
+const sessionShip_1 = require("../../evidence/sessionShip");
 const serve_hardening_1 = require("../serve-hardening");
 const credential_context_1 = require("../credential-context");
 const data_plane_1 = require("../data-plane");
-const runtime_2 = require("../runtime");
+const runtime_1 = require("../runtime");
 const outboundSend_1 = require("../../gateway/outboundSend");
 const sessionEvents_1 = require("../../analytics/sessionEvents");
 const base_1 = require("../../engines/base");
@@ -64,6 +78,11 @@ const customRegistry_1 = require("../../engines/customRegistry");
 const BANNER_LINE = '-'.repeat(62);
 async function serveCommand(options) {
     const apiConfig = (0, pack_loader_1.getApiConfig)();
+    if (apiConfig.useLiteLLM && !apiConfig.litellmApiKey) {
+        // Loud, because the failure mode it replaces was silent: traffic reached the gateway
+        // authenticated with whatever sat in the provider-key slot, with analytics disabled.
+        console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} AGENTHIPPO_LITELLM_BASE_URL is set without AGENTHIPPO_LITELLM_API_KEY — gateway calls will authenticate as 'sk-dummy' (a strict gateway will 401). Provider keys are never sent to the gateway.`);
+    }
     // Load agent pack (required)
     let pack;
     const syncLogger = options.verbose ? new headless_1.ConsoleLogger('[Sync]') : new headless_1.NoOpLogger();
@@ -97,7 +116,98 @@ async function serveCommand(options) {
         }
         process.exit(1);
     }
-    // Sync pack content to workspace
+    const logger = options.verbose ? new headless_1.ConsoleLogger('[Server]') : new headless_1.NoOpLogger();
+    // ---------------------------------------------------------------------------------------
+    // Pack admission — BEFORE anything from the pack reaches the workspace.
+    //
+    // `syncPackToWorkspace` used to run here, hundreds of lines before the signature was checked,
+    // so a pack that was about to be refused had already materialised its skills, AGENTS.md and
+    // MCP server definitions into the workspace and into the derived .claude/.codex directories.
+    // The refusal then stopped the turn while the content it was refusing sat on disk, reachable
+    // by anything else pointed at that workspace. Verification moves ahead of the write.
+    // ---------------------------------------------------------------------------------------
+    let activePackDir = pack.packPath;
+    let activeVersion = pack.manifest.metadata.version ?? '0.0.0';
+    let packDigestValue;
+    /**
+     * Trusted pack-signing keys. NOT frozen at boot.
+     *
+     * A trust root that is CONFIGURED BUT UNREACHABLE (a 503 during a rolling restart, a DNS blip)
+     * is now distinguishable from one that was never configured, and it is retried on the lease
+     * poll — one transient fetch at the wrong second must fail the turn, not brick the process.
+     */
+    let packTrust = await (0, signature_1.loadPackTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
+    const reportTrustRoot = (t, label = 'pack') => {
+        for (const w of t.warnings) {
+            console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${w}`);
+        }
+        if (t.status === 'unavailable') {
+            console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${label} trust root is configured but unusable: ${t.reason}`);
+        }
+    };
+    reportTrustRoot(packTrust);
+    /**
+     * Verified per turn, with a cheap change probe — see `createPackVerifier`. The RESULT is
+     * cached only as long as the pack tree looks untouched and never longer than the full-reverify
+     * interval; the REQUIREMENT is re-read from the lease on every poll, so an operator can turn
+     * enforcement on for a fleet already running.
+     */
+    const packVerifier = (0, signature_1.createPackVerifier)();
+    let packSignature = { ok: false, kind: 'no-pack', reason: 'not verified' };
+    const verifyPackAt = (packDir, version) => {
+        packSignature = packVerifier.refresh({ packDir, pack: packId, packVersion: version, keys: packTrust.keys });
+        packDigestValue = packSignature.ok ? packSignature.claims.digest : undefined;
+        logger.info(packSignature.ok
+            ? `[Pack] signature verified for ${packId}`
+            : `[Pack] unverified — ${packSignature.reason}`);
+        return packSignature;
+    };
+    /** The refusal an operator sees, with the remedy for THIS failure rather than a generic one. */
+    const packRefusalText = (result) => {
+        if (result.ok) {
+            return '';
+        }
+        // "Configured but unreachable" outranks whatever the verifier concluded: with no usable
+        // keys every signature fails as `no-trust-root`, and telling an operator to configure a
+        // trust root they already configured is the least useful thing we could say.
+        const kind = packTrust.status === 'unavailable' && result.kind === 'no-trust-root' ? 'trust-root-unavailable' : result.kind;
+        return (0, signature_1.describePackRefusal)({
+            kind,
+            detail: kind === 'trust-root-unavailable' ? (packTrust.reason ?? result.reason) : result.reason,
+            pack: packId,
+            packVersion: activeVersion,
+            packDir: activePackDir,
+            trustRootHint: packTrust.url ? `Trust root: ${packTrust.url} (${packTrust.reason ?? 'unavailable'})` : packTrust.reason,
+        });
+    };
+    verifyPackAt(activePackDir, activeVersion);
+    /**
+     * Signing REQUIRED by default.
+     *
+     * This is a deliberate break with the previous shipped behaviour, which logged
+     * `[Pack] unsigned or unverifiable — …` and then ran the pack anyway. Warn-and-run is not a
+     * gate; it is a log line next to an ungated code path, and it meant the default deployment
+     * shape executed whatever was in the pack directory.
+     *
+     * Tighten-only survives intact: the escape hatch below relaxes only this INSTANCE's default,
+     * and `resolveRequireSignedPack` checks the lease's demand first, so a control plane that sets
+     * `requireSignedPack` cannot be opted out of locally.
+     */
+    const allowUnsignedPack = Boolean(options.allowUnsignedPack) || (0, signature_1.allowUnsignedPackFromEnv)();
+    if (!packSignature.ok) {
+        if (!allowUnsignedPack) {
+            console.error(`${utils_1.COLORS.red}Error: ${packRefusalText(packSignature)}${utils_1.COLORS.reset}`);
+            process.exit(1);
+        }
+        // Unmissable, every boot, at warn level regardless of --verbose.
+        console.warn(BANNER_LINE);
+        console.warn(`${utils_1.COLORS.yellow}!! RUNNING AN UNVERIFIED PACK — signature checking is DISABLED on this instance.${utils_1.COLORS.reset}`);
+        console.warn(`${utils_1.COLORS.yellow}!! ${packSignature.reason}${utils_1.COLORS.reset}`);
+        console.warn(`${utils_1.COLORS.yellow}!! Whatever is in ${activePackDir} will be executed, verified by nobody.${utils_1.COLORS.reset}`);
+        console.warn(`${utils_1.COLORS.yellow}!! This is a LOCAL DEVELOPMENT setting (AGENTHIPPO_ALLOW_UNSIGNED_PACK). Do not ship it.${utils_1.COLORS.reset}`);
+        console.warn(BANNER_LINE);
+    }
+    // Sync pack content to workspace — only now that the pack has been admitted.
     const bundledPromptsDir = (0, utils_1.findBundledBasePromptsDir)();
     await (0, headless_1.syncPackToWorkspace)(options.workspace, pack.packPath, packId, defaultEngine, syncLogger, bundledPromptsDir);
     console.log(`OK: Loaded agent: ${pack.manifest.metadata.name} v${pack.manifest.metadata.version}`);
@@ -106,7 +216,12 @@ async function serveCommand(options) {
     // Resolve persistent session state (serve-v3 §A1: replaces the tmpdir default).
     const { dir: storagePath, ephemeral } = (0, serve_hardening_1.resolveStateDir)(options.stateDir);
     const bindHost = (0, serve_hardening_1.resolveBindHost)(options.host);
-    // Passed to createHeadlessRunner below so the runner's own SessionEventWriter uses the same
+    // Who serve believes a caller to be decides whose session state and whose data credentials they
+    // reach, so an unpinned identity header is not a detail to leave in a docstring.
+    for (const line of (0, serve_hardening_1.identityHeaderAdvice)(process.env, bindHost)) {
+        console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${line}`);
+    }
+    // Passed to createPackRunner below so the runner's own SessionEventWriter uses the same
     // root — a fail-closed warrant denial lands in the same per-session JSONL as the tool events
     // it precedes. The denial happens before runner.run() is ever called, so it needs its own
     // writer here too. See resolveAnalyticsSessionsDir's docstring for the precedence.
@@ -117,12 +232,42 @@ async function serveCommand(options) {
     // serve runs the connector broker-side and engines get a per-turn nonce instead of the
     // credential. The raw data credential never crosses into the agent's trust domain.
     const dataPlaneCfg = (0, data_plane_1.resolveDataPlaneConfig)(pack.manifest.spec, pack.packPath);
+    // EDGE-registration mode (P1 fix, obo-inprogress §2): the trusted edge (IdP Caddy)
+    // registers each turn with the sidecar and injects the nonce as a request header; serve
+    // merely adopts it. serve holds NO registration credential at all — nothing in this
+    // container's /proc can mint a nonce for an arbitrary userId. Takes precedence over the
+    // token-based sidecar mode below.
+    const edgeCfg = (0, data_plane_1.resolveEdgeDataPlane)();
+    const edgeDataPlane = edgeCfg ? new data_plane_1.EdgeDataPlane(edgeCfg) : null;
+    // Data-plane SIDECAR mode (secure-obo-design §7.6): when the connector runs in its OWN
+    // container, AGENTHIPPO_DATA_PLANE_CONTROL_URL/TOKEN + AGENTHIPPO_DATA_MCP_URL are set and
+    // serve holds NO backend base credential — it registers turns with the sidecar and hands the
+    // engine only the nonce. Takes precedence over an in-process connector.
+    const remoteCfg = edgeDataPlane ? null : (0, data_plane_1.resolveRemoteDataPlane)();
+    const remoteDataPlane = remoteCfg ? new data_plane_1.RemoteDataPlane(remoteCfg) : null;
     // One shared registry: a single per-turn nonce grants BOTH the data-plane and egress
-    // capabilities (secure-obo-design §7.6 step 2). Created when either broker will run.
-    const turnRegistry = (dataPlaneCfg || options.gatewayBridge) ? new data_plane_1.TurnRegistry() : null;
+    // capabilities (secure-obo-design §7.6 step 2). Created when either LOCAL broker will run.
+    const turnRegistry = ((dataPlaneCfg && !remoteDataPlane && !edgeDataPlane) || options.gatewayBridge) ? new data_plane_1.TurnRegistry() : null;
     const auditEvent = (sessionKey, event, fields) => sessionEventWriter.append(sessionKey, (0, base_1.generateRequestId)(), event, fields);
     let dataPlane = null;
-    if (dataPlaneCfg) {
+    if (edgeDataPlane) {
+        process.env[data_plane_1.DATA_MCP_URL_ENV] = edgeDataPlane.url;
+        // Belt-and-braces: edge mode must never see a register token, but if one leaked into the
+        // deploy env anyway, strip it before any engine child inherits it.
+        delete process.env.AGENTHIPPO_DATA_PLANE_TOKEN;
+        delete process.env.AGENTHIPPO_DATA_PLANE_CONTROL_URL;
+        console.log(`OK: Data plane EDGE-registered — gateway ${edgeDataPlane.url}; turn nonces are minted at the edge, serve holds no registration credential`);
+    }
+    else if (remoteDataPlane) {
+        process.env[data_plane_1.DATA_MCP_URL_ENV] = remoteDataPlane.url;
+        // The control-API token is captured in the RemoteDataPlane (serve memory). Strip it from
+        // the process env so no engine child inherits it — otherwise the agent could register a
+        // turn as ANY user and read their rows (forge-identity escalation). The engine keeps the
+        // gateway URL + its own per-turn nonce, which is all it needs (secure-obo-design §7.6).
+        delete process.env.AGENTHIPPO_DATA_PLANE_TOKEN;
+        console.log(`OK: Data plane SIDECAR — gateway ${remoteDataPlane.url}; base credentials stay in the connector container, not here`);
+    }
+    else if (dataPlaneCfg) {
         try {
             dataPlane = await data_plane_1.DataPlane.start(dataPlaneCfg, {
                 logger: options.verbose ? new headless_1.ConsoleLogger('[DataPlane]') : new headless_1.NoOpLogger(),
@@ -143,12 +288,42 @@ async function serveCommand(options) {
     // Egress broker (secure-obo-design §7.6): capture the gateway token broker-side BEFORE any
     // engine spawns, so it can be stripped from the process env the children inherit. The broker
     // starts alongside the gateway bridge below; the token is held only in this closure.
-    const brokerGatewayToken = options.gatewayBridge ? (0, runtime_2.readOpenClawGatewayToken)() : undefined;
+    // Captured UNCONDITIONALLY, and the env strip is immediate (serve-v4 §10 hole 2): the strip
+    // used to live inside the broker branch, so a serve without --gateway-bridge kept the token
+    // in process.env and every engine child (they all spread it) inherited a credential nothing
+    // in this deployment would ever legitimately use.
+    const brokerGatewayToken = (0, runtime_1.readOpenClawGatewayToken)();
+    delete process.env.OPENCLAW_GATEWAY_TOKEN;
     let egressBroker = null;
     // Mint one nonce per turn and assemble the per-spawn child env. Returns null outside any
     // broker (no capability to grant). The URLs ride process.env (boot-stable) for config
     // invariance; only the nonce is per-spawn.
-    const registerServeTurn = (reg) => {
+    const registerServeTurn = async (reg) => {
+        // Edge mode: the trusted edge already registered this turn and injected the nonce as a
+        // request header — adopt it. No edge nonce (misrouted request, or a channel turn that
+        // never traversed the edge) ⇒ no data capability, fail closed at the gateway.
+        if (edgeDataPlane) {
+            const edgeNonce = reg.edgeNonce?.trim();
+            if (!edgeNonce) {
+                console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} edge data plane active but no ${data_plane_1.TURN_NONCE_HEADER} header on this turn — data tools will fail closed (check the edge route's forward_auth)`);
+                return null;
+            }
+            const r = edgeDataPlane.adoptTurn(edgeNonce);
+            const env = { ...r.env };
+            if (egressBroker) {
+                env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
+            }
+            return { nonce: r.nonce, env, release: r.release };
+        }
+        // Sidecar: register with the connector container; serve never sees a base credential.
+        if (remoteDataPlane) {
+            const r = await remoteDataPlane.registerTurn(reg);
+            const env = { ...r.env };
+            if (egressBroker) {
+                env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
+            }
+            return { nonce: r.nonce, env, release: r.release };
+        }
         if (!turnRegistry) {
             return null;
         }
@@ -162,6 +337,16 @@ async function serveCommand(options) {
         }
         return { nonce, env, release };
     };
+    // Edge-registered turn nonce, injected by the trusted edge per chat request (edge mode only —
+    // otherwise ignored). Trusted like the pinned identity header: only the in-network edge can
+    // reach serve, and a forged value fails closed at the gateway (unknown nonce ⇒ 401).
+    const edgeTurnNonce = (headers) => {
+        if (!edgeDataPlane) {
+            return undefined;
+        }
+        const v = headers[data_plane_1.TURN_NONCE_HEADER];
+        return Array.isArray(v) ? v[0] : v;
+    };
     const turnLimiter = new serve_hardening_1.TurnLimiter(options.maxTurnsPerSession ?? 0);
     // Per-identity request-rate cap (req/min). Keyed on the trusted user id (falling back to
     // client IP) so rotating session_id can't bypass it. AGENTHIPPO_SERVE_RATE_LIMIT=0 disables.
@@ -212,18 +397,22 @@ async function serveCommand(options) {
             await (0, state_sync_1.ensureSessionLocal)(s3State, storagePath, sessionKey, stateLog);
         }
     };
-    const logger = options.verbose ? new headless_1.ConsoleLogger('[Server]') : new headless_1.NoOpLogger();
     // Find SDK CLI paths
     const claudeCodePath = (0, utils_1.findClaudeCodePath)();
     const codexPath = (0, utils_1.findCodexPath)();
-    // Validate required CLI is available
-    if (defaultEngine === 'claude' && !claudeCodePath) {
+    // Validate required CLI is available — but ONLY when this process will run the engine.
+    // A container sandbox runs the engine inside its own image, so requiring the SDK on the host
+    // is wrong and refuses to boot on exactly the minimal host a tier-2 deployment wants
+    // (observed live: a Linux host with no npm/codex could not start `--sandbox container`).
+    // The process driver shares this host, so it still needs the CLI present.
+    const engineRunsInSandbox = options.sandbox === 'container';
+    if (!engineRunsInSandbox && defaultEngine === 'claude' && !claudeCodePath) {
         console.error(`${utils_1.COLORS.red}Error: Claude Code CLI not found.${utils_1.COLORS.reset}`);
         console.error('   Install: npm install -g @anthropic-ai/claude-code');
         console.error('   Or set CLAUDE_CODE_PATH environment variable');
         process.exit(1);
     }
-    if (defaultEngine === 'codex' && !codexPath) {
+    if (!engineRunsInSandbox && defaultEngine === 'codex' && !codexPath) {
         console.error(`${utils_1.COLORS.red}Error: Codex CLI not found.${utils_1.COLORS.reset}`);
         console.error('   Install: npm install -g @openai/codex');
         console.error('   Or set CODEX_PATH environment variable');
@@ -234,38 +423,589 @@ async function serveCommand(options) {
     const permissions = {
         fileAccess: (0, pack_loader_1.getFileAccess)(pack),
     };
-    const runner = (0, headless_1.createHeadlessRunner)({
+    // R-3 policy provenance (PLAN/compliance.md §3.1): remember WHICH policy is in force and
+    // announce changes. `policy_applied` marks the boundary; every turn between two boundaries
+    // carries the version via the runner's provenance stamp below, so "was the cap enforced?"
+    // stays answerable after anyone edits a cap.
+    let lastPolicyVersion;
+    let provenanceAnnounced = false;
+    const notePolicy = (lease) => {
+        const v = (0, lease_1.policyVersion)(lease);
+        // Announce once at boot even with no lease (unmanaged deployments still have a digest,
+        // an engine build and a tier), then again on every policy change. This event is the
+        // PROVENANCE DIMENSION: turns don't repeat these facts — the evidence index joins a
+        // turn to the latest announcement for its instance at or before the turn's timestamp.
+        if (provenanceAnnounced && v === lastPolicyVersion) {
+            return;
+        }
+        provenanceAnnounced = true;
+        lastPolicyVersion = v;
+        void sessionEventWriter.append('serve', (0, base_1.generateRequestId)(), 'policy_applied', {
+            instance_id: instanceId,
+            policy_version: v ?? null,
+            pack_digest: packDigestValue ?? null,
+            engine_version: (0, deps_1.installedEngineVersion)(defaultEngine) ?? null,
+            tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : null,
+            enabled: lease?.enabled ?? null,
+            pinned_version: lease?.version ?? null,
+            caps: lease?.caps ?? null,
+            deny_count: lease?.deny?.length ?? 0,
+        });
+    };
+    // Same factory a sandbox uses (cli/pack-runtime.ts) — built after activeVersion is known.
+    let inProcessRunner;
+    /**
+     * Where sandboxes read the pack from.
+     *
+     * With --pull-packs this is a SYMLINK (`<state>/packs/current`), never the resolved directory:
+     * the sandbox drivers capture this path once at construction, so a stable name whose target is
+     * swapped is what lets a later version reach new sandboxes without rebuilding the runner. A
+     * sandbox already starting keeps the directory it opened, which is exactly the drain semantics
+     * version convergence already has.
+     */
+    const packsRoot = path.join(storagePath, 'packs');
+    const storeDownloader = async (slug, version) => {
+        const files = await (0, index_1.downloadFromStore)({ type: 'pack', slug, version });
+        return files.map((f) => ({ path: f.path, content: f.content }));
+    };
+    /** Converge to the pinned version if we can; never take the fleet down trying. */
+    const reconcileToPin = async (pinned) => {
+        if (!options.pullPacks || !pinned || pinned === activeVersion) {
+            return;
+        }
+        const res = await (0, reconcile_1.reconcilePinnedPack)({
+            pack: packId,
+            currentVersion: activeVersion,
+            pinnedVersion: pinned,
+            packsRoot,
+            download: storeDownloader,
+            publicKeysByKid: packTrust.keys,
+            logger: { info: (m) => logger.info(m), warn: (m) => logger.info(m) },
+        });
+        if (res.ok) {
+            activeVersion = res.version;
+            activePackDir = res.packDir;
+            verifyPackAt(activePackDir, activeVersion);
+            logger.info(`[Pack] now serving ${packId}@${activeVersion} from ${res.packDir}`);
+            return;
+        }
+        // Deliberately not fatal: the lease fails OPEN on transport, and a bad tag or a store
+        // outage must not become a fleet-wide stop. The instance keeps reporting the version it is
+        // really running, so the console shows "pinned to X, running Y" rather than an outage.
+        console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} cannot converge to ${packId}@${pinned}: ${res.reason}. Still serving @${activeVersion}.`);
+    };
+    const warrantKey = (0, support_1.loadWarrantKeypair)(logger);
+    // Lease: kill switch + version pin. No URL configured ⇒ unmanaged deployment, always admits.
+    /** Set once the sandbox runner exists; the lease-poll identity reads it lazily. */
+    let sandboxRunnerRef;
+    // Stable across restarts, unique per instance: an operator watching the console must see one
+    // row per instance that survives a redeploy, not a new row each boot (and not one row per NAT
+    // address for a whole fleet). Persisted beside the instance's own state.
+    const instanceId = (() => {
+        try {
+            const file = path.join((0, serve_hardening_1.resolveStateDir)(options.stateDir).dir, 'instance-id');
+            if (fs.existsSync(file)) {
+                return fs.readFileSync(file, 'utf-8').trim();
+            }
+            const id = `${os.hostname()}-${crypto.randomBytes(4).toString('hex')}`;
+            fs.mkdirSync(path.dirname(file), { recursive: true });
+            fs.writeFileSync(file, id);
+            return id;
+        }
+        catch {
+            // Never let bookkeeping stop a boot; a per-process id still beats collapsing by address.
+            return `${os.hostname()}-${crypto.randomBytes(4).toString('hex')}`;
+        }
+    })();
+    /** Lazily read by the lease-poll identity, which is built before the pack path is resolved. */
+    const activeVersionRef = { get: () => pack?.manifest.metadata.version };
+    // ONE constant for the poll interval and everything derived from it. The cache used to default
+    // to 60s while every poll defaulted to 30s and the banner printed 30 — so the real kill latency
+    // was up to 3x what an operator was told.
+    const leasePollSeconds = (0, lease_1.resolveLeasePollSeconds)(options.leasePollSeconds);
+    /**
+     * Trusted LEASE-signing keys — a separate root from the pack one.
+     *
+     * These used to be the same keyring (`signingKeys: () => packTrust.keys`), which made signed
+     * leases unreachable end to end: the control plane publishes the lease public key under purpose
+     * `lease-signing` and refuses to register one kid under two purposes, so a correctly-configured
+     * plane and a correctly-configured serve could never name the same key. With `requireSignedLease`
+     * sticky and no plane-side undo, that disagreement latches an instance into refusing turns.
+     *
+     * A DERIVED root that comes back `unavailable` is the normal state of every plane that publishes
+     * no lease-signing key, so it is only reported at boot when an operator named one outright.
+     * Otherwise `leaseSigningKeys` says everything there is to say, and only once a signed lease has
+     * actually arrived.
+     */
+    let leaseTrust = await (0, signature_1.loadLeaseTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
+    if (leaseTrust.explicitlyConfigured) {
+        reportTrustRoot(leaseTrust, 'lease');
+    }
+    let leaseTrustFallbackWarned = false;
+    /**
+     * BACK-COMPAT, deliberately not a cutover.
+     *
+     * A deployment that verifies signed leases against the pack keyring today keeps working: with no
+     * lease root named outright, the pack keys stay in play and the (possibly empty) derived lease
+     * keys are added to them. The warning fires ONCE, and only when a signed lease is actually being
+     * verified, so a deployment that never uses signed leases never sees it.
+     *
+     * Not a hard switch because `requireSignedLease` latches per instance and the plane has no undo:
+     * an instance that stopped recognising its lease key would refuse turns and could not be talked
+     * out of it from the console.
+     */
+    const leaseSigningKeys = () => {
+        const selected = (0, signature_1.selectLeaseSigningKeys)(leaseTrust, packTrust);
+        if (selected.packFallback && !leaseTrustFallbackWarned) {
+            leaseTrustFallbackWarned = true;
+            console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} verifying signed leases against the PACK trust root — ` +
+                'no lease trust root is configured on this instance. The control plane publishes lease keys under ' +
+                'purpose `lease-signing` and refuses to register one kid under two purposes, so this stops working ' +
+                'the moment the plane rotates to a dedicated lease key — and requireSignedLease is sticky with no ' +
+                'plane-side undo, so that failure cannot be fixed from the console. Set ' +
+                `${signature_1.LEASE_JWKS_URL_ENV}=https://<plane>/.well-known/jwks.json?purpose=lease-signing ` +
+                `(or ${signature_1.LEASE_SIGNING_KEYS_ENV} with inline {"<kid>":"<public PEM>"} for an air-gapped install).` +
+                (leaseTrust.reason ? ` The derived lease trust root is unusable: ${leaseTrust.reason}` : ''));
+        }
+        return selected.keys;
+    };
+    const leaseSource = (0, lease_1.createLeaseSource)({
+        url: process.env.AGENTHIPPO_LEASE_URL,
+        // The poll IS the registration — see LeaseSourceOptions.identity.
+        identity: () => ({
+            instance: instanceId,
+            pack: packId,
+            // The version actually being served: after a pull this is the pinned one, and the
+            // console's whole convergence view depends on it being the truth rather than boot state.
+            version: activeVersionRef.get(),
+            tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
+            sandboxes: sandboxRunnerRef?.sandboxes().length,
+            // Which engine, and which BUILD of it. Pack version alone cannot explain a fleet-wide
+            // behaviour change that came from an engine upgrade, and the CLI is not engine-version
+            // agnostic, so the console needs both to answer "what is actually running?".
+            engine: defaultEngine,
+            engineversion: (0, deps_1.installedEngineVersion)(defaultEngine),
+        }),
+        // Derived from the poll interval by ONE function, so the two can no longer drift. Worst-case
+        // kill latency is poll + cache, and that is the number the boot banner prints.
+        cacheSeconds: (0, lease_1.leaseCacheSecondsFor)(leasePollSeconds),
+        // Signed-lease verification uses the LEASE trust root, falling back to the pack one while no
+        // lease root is configured. Read through a closure so a keyring that arrives on a later poll
+        // is picked up without rebuilding this.
+        signingKeys: () => leaseSigningKeys(),
+        requireSignedLease: options.requireSignedLease,
+        instanceId,
+        ...(options.leaseMaxStalenessSeconds !== undefined ? { maxStalenessSeconds: options.leaseMaxStalenessSeconds } : {}),
+        logger: {
+            // Lease warnings are security-relevant (unsigned lease, staleness ceiling, insecure URL)
+            // and must not be hidden behind --verbose.
+            info: (m) => logger.info(m),
+            warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
+        },
+    });
+    /**
+     * Retry the trust root while it is missing, and re-verify once it arrives.
+     *
+     * Only while MISSING: a loaded trust root is not re-fetched, so key rotation still needs the
+     * documented restart and a compromised plane cannot swap the trust root under a running
+     * instance mid-flight. This exists solely so a transient failure at boot is transient — the
+     * `unavailable` status is exactly what makes "transient" distinguishable from "not configured".
+     */
+    const retrySigningKeys = async () => {
+        if (packTrust.status !== 'configured') {
+            const next = await (0, signature_1.loadPackTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
+            if (next.status === 'configured') {
+                packTrust = next;
+                reportTrustRoot(next);
+                logger.info(`[Pack] signing keys now available (${Object.keys(next.keys).length}); re-verifying`);
+                verifyPackAt(activePackDir, activeVersion);
+            }
+        }
+        // The lease root gets the SAME rule, not a better one: retried only while missing, never
+        // re-fetched once configured. Improving one and not the other would leave two roots with two
+        // rotation stories, which is how the rotation runbook silently stops being true for half of it.
+        if (leaseTrust.status !== 'configured') {
+            const next = await (0, signature_1.loadLeaseTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
+            if (next.status === 'configured') {
+                leaseTrust = next;
+                if (next.explicitlyConfigured) {
+                    reportTrustRoot(next, 'lease');
+                }
+                logger.info(`[Lease] lease-signing keys now available (${Object.keys(next.keys).length})`);
+            }
+        }
+    };
+    // Record the boot policy and watch for changes on the lease-poll cadence. The lease source
+    // caches, so this adds no extra fetches — it re-reads whatever the poll already trusts.
+    void leaseSource.get().then(notePolicy).catch(() => undefined);
+    {
+        const everyMs = leasePollSeconds * 1000;
+        const policyTimer = setInterval(() => {
+            void leaseSource.get().then(notePolicy).catch(() => undefined);
+            void retrySigningKeys().catch(() => undefined);
+        }, everyMs);
+        policyTimer.unref?.();
+    }
+    if (options.pullPacks) {
+        fs.mkdirSync(packsRoot, { recursive: true });
+        // Materialise whatever we boot with, so `current` is valid before any pin is considered and
+        // the drivers can capture one stable path.
+        const bootDir = path.join(packsRoot, activeVersion);
+        if (!fs.existsSync(bootDir)) {
+            fs.cpSync(pack.packPath, bootDir, { recursive: true });
+        }
+        (0, reconcile_1.promoteCurrent)(packsRoot, bootDir);
+        activePackDir = path.join(packsRoot, reconcile_1.CURRENT_LINK);
+        await reconcileToPin((await leaseSource.get())?.version);
+        // The pack is now read through `current`, which is a different path than the one verified
+        // above. Re-verify so `packSignature` describes the tree that will actually run.
+        verifyPackAt(activePackDir, activeVersion);
+    }
+    activeVersionRef.get = () => activeVersion;
+    inProcessRunner = (0, pack_runtime_1.createPackRunner)({
         storagePath,
         analyticsSessionsDir,
         packId,
-        packVersion: pack?.manifest.metadata.version,
-        bundledPromptsDir,
-        apiKey: apiConfig.apiKey,
-        baseUrl: apiConfig.baseUrl,
-        useLiteLLM: apiConfig.useLiteLLM,
-        // Per-destination keys so each engine authenticates to the endpoint it actually dials
-        // (codex→OpenAI, claude→Anthropic, or the LiteLLM proxy) — no provider-blind collapse.
-        litellmApiKey: apiConfig.litellmApiKey,
-        openaiApiKey: apiConfig.openaiApiKey,
-        anthropicApiKey: apiConfig.anthropicApiKey,
+        packVersion: activeVersion,
+        provenance: () => ({
+            policyVersion: lastPolicyVersion,
+            packDigest: packDigestValue,
+            engineVersion: (0, deps_1.installedEngineVersion)(defaultEngine),
+            tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
+            instanceId,
+        }),
+        workspaceRoot: options.workspace,
+        credentials: {
+            apiKey: apiConfig.apiKey,
+            baseUrl: apiConfig.baseUrl,
+            useLiteLLM: apiConfig.useLiteLLM,
+            litellmApiKey: apiConfig.litellmApiKey,
+            openaiApiKey: apiConfig.openaiApiKey,
+            anthropicApiKey: apiConfig.anthropicApiKey,
+        },
         logger,
-        freshState: false,
-        claudeCodeExecutablePath: claudeCodePath,
-        codexExecutablePath: codexPath,
         permissionHandler,
-        cliEnvPatch: (0, runtime_1.buildCliEnvPatch)({
-            cliPath: process.execPath,
-            workspaceRoot: options.workspace,
-            agentId: packId,
-            baseEnv: process.env,
-        }),
-        ...(claudeCheapModel ? { engineSettings: { claudeCheapModel } } : {}),
+        claudeCheapModel,
+        omitGatewayToken: true,
     });
+    // Converge on the same cadence the fleet already polls, so pinning a version in the console
+    // reaches instances without a redeploy. Errors are swallowed by reconcileToPin itself.
+    if (options.pullPacks) {
+        const every = leasePollSeconds * 1000;
+        const timer = setInterval(() => {
+            void leaseSource.get().then((l) => reconcileToPin(l?.version)).catch(() => undefined);
+        }, every);
+        timer.unref?.();
+    }
+    // Container sandboxes got NO filesystem mounts at all — the driver honours `fs`, but serve never
+    // passed it, so a sandbox's `/state/workspace` was container-private storage. Everything the
+    // agent wrote was invisible on the host and lost when the sandbox died. The process driver needs
+    // no mounts: tier 0 shares this host's filesystem already.
+    //
+    // The workspace is PER SESSION unless --shared-workspace says otherwise. One host directory
+    // mounted writable into every session's sandbox is a shared disk between users of the same
+    // agent: whatever session A writes (a query export, a downloaded file), session B reads on its
+    // next turn. That silently contradicts "one disposable sandbox per session" for everything that
+    // matters, since the container boundary does not extend to a directory both containers mount.
+    const sharedWorkspace = Boolean(options.sharedWorkspace);
+    const sandboxMounts = options.sandbox === 'container'
+        ? {
+            // serve's resolved pack, not the image's baked copy — the two cannot then disagree
+            // about which pack version is running.
+            readonly: [`${activePackDir}:${containerDriver_1.SANDBOX_PACK_DIR}`],
+            writable: [`${options.workspace}:${containerDriver_1.SANDBOX_WORKSPACE_DIR}`],
+        }
+        : undefined;
+    if (sandboxMounts && sharedWorkspace) {
+        console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} --shared-workspace: every session's sandbox mounts ` +
+            `${options.workspace} read-write, so sessions can read and overwrite each other's files. ` +
+            `Only safe when all sessions belong to one trust domain.`);
+    }
+    // serve wrote that workspace as root; the sandbox runs non-root and must be able to write it too.
+    // Boot-time and once, before the first turn can arrive.
+    let sandboxImageUser;
+    if (sandboxMounts && options.sandboxImage) {
+        await (0, support_1.alignSandboxWorkspaceOwnership)({
+            exec: support_1.dockerExec,
+            image: options.sandboxImage,
+            writable: sandboxMounts.writable,
+            // A failure here is what an operator must see even without --verbose; success is noise.
+            logger: { info: (m) => logger.info(m), warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`) },
+        });
+        sandboxImageUser = await (0, support_1.resolveSandboxImageUser)(support_1.dockerExec, options.sandboxImage);
+    }
+    // A session's engine state must outlive its sandboxes or the conversation restarts every time a
+    // sandbox is reaped (serve-v4-sandbox §8.4). Per session, under serve's own state dir, so the
+    // existing per-session sync/prune machinery already covers it.
+    const sandboxStateDirFor = (sessionKey) => (0, support_1.ensureSandboxStateDir)(path.join(storagePath, 'sandbox-state', (0, state_sync_1.safeSessionId)(sessionKey)), sandboxImageUser);
+    /**
+     * Mounts for ONE session's sandbox: the pack read-only, and a workspace only this session sees.
+     *
+     * The per-session workspace lives beside that session's engine state, so the same reap/prune
+     * machinery already covers it and the agent's output stays visible on the host — the property
+     * the shared mount was added for — without also being visible to every other session.
+     */
+    const sandboxFsFor = async (sessionKey) => {
+        if (!sandboxMounts) {
+            return { readonly: [], writable: [] };
+        }
+        if (sharedWorkspace) {
+            return sandboxMounts;
+        }
+        const dir = (0, support_1.ensureSandboxStateDir)(path.join(storagePath, 'sandbox-workspace', (0, state_sync_1.safeSessionId)(sessionKey)), sandboxImageUser);
+        return { readonly: sandboxMounts.readonly, writable: [`${dir}:${containerDriver_1.SANDBOX_WORKSPACE_DIR}`] };
+    };
+    /**
+     * "May this pack run, at this version, for this session?" — signature requirement, version pin,
+     * image digest and scoped deny rules, in ONE place.
+     *
+     * Both gates call this: sandbox creation (where it decides whether a sandbox may be born) and,
+     * on deployments with no sandbox, the per-turn governor gate. Two copies of these four checks
+     * is exactly the drift this codebase keeps paying for, and the no-sandbox path having no copy
+     * at all is why `requireSignedPack` and a version pin could be switched on in the console and
+     * enforce nothing on the default deployment shape.
+     */
+    const assertPackAdmitted = (currentLease, sessionKey, userId) => {
+        const required = (0, signature_1.resolveRequireSignedPack)({
+            // The lease's demand is read FIRST and cannot be waived by any local setting — that is
+            // what keeps the requirement tighten-only now that the local default is "required".
+            lease: currentLease?.requireSignedPack,
+            localRequire: options.requireSignedPack,
+            localAllowUnsigned: allowUnsignedPack,
+        });
+        if (required) {
+            // RE-VERIFIED, not read off a boot-time boolean. In the default no-sandbox shape the
+            // engine runs in THIS process against a pack scaffolded with `fileAccess: full`, so the
+            // agent can edit its own pack directory between turns; a cached "verified once at boot"
+            // would pass every one of those later turns. `createPackVerifier` keeps this cheap — a
+            // stat-only probe in the common case, a full digest when the tree looks touched or the
+            // re-verify interval has elapsed.
+            const result = packVerifier.verify({
+                packDir: activePackDir, pack: packId, packVersion: activeVersion, keys: packTrust.keys,
+            });
+            if (result !== packSignature) {
+                packSignature = result;
+                packDigestValue = result.ok ? result.claims.digest : undefined;
+            }
+            if (!result.ok) {
+                throw new lease_1.LeaseDeniedError(packRefusalText(result));
+            }
+        }
+        // sessionKey makes admission SCOPED: a deny rule naming this session or its user refuses
+        // at the same gate that enforces the fleet-wide switch.
+        (0, lease_1.assertAdmitted)(currentLease, {
+            pack: packId,
+            version: activeVersion,
+            image: options.sandboxImage,
+            sessionKey,
+            userHash: serve_hardening_1.userNamespaceHash,
+            userId,
+        });
+    };
+    // Assigned just below; the identity closure above reads it lazily so a poll can report live
+    // sandbox count without forcing the runner to be constructed before the lease source.
+    // eslint-disable-next-line prefer-const
+    // Sandbox mode: the same HeadlessRunner shape, body relocated into a per-session sandbox.
+    // serve's turn loop below is untouched — see sandbox/sandboxRunner.ts.
+    const sandboxRunner = options.sandbox
+        ? (0, sandboxRunner_1.createSandboxRunner)({
+            driver: options.sandbox === 'container'
+                // A non-internal route network is a real weakening of the tier-2 claim, so its warning
+                // must reach an operator who did not pass --verbose.
+                ? new containerDriver_1.ContainerSandboxDriver({
+                    exec: support_1.dockerExec,
+                    logger: {
+                        info: (m) => logger.info(m),
+                        warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
+                    },
+                })
+                : new processDriver_1.ProcessSandboxDriver({
+                    command: process.execPath,
+                    // --workspace is REQUIRED, not optional: a tier-0 sandbox shares this host's
+                    // filesystem, and serve sends this same path as `workspaceRoot` on every run
+                    // frame. Omitting it left the sandbox on the container-image default
+                    // (/state/workspace), so a serve-launched process sandbox died at boot with
+                    // ENOENT: mkdir '/state/workspace'.
+                    // pack.packPath, not options.agent: serve has ALREADY resolved the pack (a
+                    // name resolves against ~/.agent-hippo/agents and the workspace), and the
+                    // sandbox must run that exact directory rather than re-resolve a name from a
+                    // different cwd and possibly land on a different pack.
+                    args: [
+                        // Only forward the script path when the process actually HAS one. A
+                        // single-file build reports `process.argv[1]` as a path inside its own
+                        // virtual filesystem (`/$bunfs/root/index.js` for `bun build --compile`,
+                        // `/snapshot/...` for pkg / node SEA). Passing that made the child parse
+                        // it as the subcommand and die with `unknown command '/$bunfs/...'`, so
+                        // every tier-0 sandbox failed with a blank "sandbox channel closed".
+                        // An existence check is NOT enough to detect this: those runtimes mount
+                        // the virtual path, so `fs.existsSync` cheerfully returns true for it.
+                        ...(0, support_1.sandboxScriptArg)(),
+                        'sandbox', '--agent', activePackDir, '--stdio',
+                        '--state-dir', storagePath, '--workspace', options.workspace,
+                    ],
+                    // PID/user-namespace wrap where the host allows it (residual P4: on a
+                    // single-uid host, a concurrent tenant reads a turn's nonce from
+                    // /proc/<pid>/environ — and the claude CLI's argv from /proc/<pid>/cmdline,
+                    // which is world-readable). Default 'auto': hardened where possible, honest
+                    // warning where not; 'require' makes multi-tenant hosts fail closed.
+                    namespaceIsolation: (0, processDriver_1.namespaceIsolationFromEnv)(),
+                    logger: {
+                        info: (m) => logger.info(m),
+                        warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
+                    },
+                }),
+            image: options.sandboxImage ?? '',
+            tier: options.sandbox === 'container' ? 2 : 0,
+            ...(sandboxMounts ? { fs: sandboxMounts, fsFor: sandboxFsFor, stateDirFor: sandboxStateDirFor } : {}),
+            // The only routes a sandbox may reach. Everything else is unroutable, not merely denied.
+            routes: [
+                ...(process.env.AGENTHIPPO_DATA_MCP_URL ? [{ kind: 'data', target: process.env.AGENTHIPPO_DATA_MCP_URL }] : []),
+                ...(apiConfig.baseUrl ? [{ kind: 'inference', target: apiConfig.baseUrl }] : []),
+            ],
+            limits: { ttlSeconds: options.sandboxTtlSeconds ?? 30 * 60 },
+            provenanceEnv: () => (0, runtime_1.buildProvenanceEnv)({
+                policyVersion: lastPolicyVersion,
+                packDigest: packDigestValue,
+                engineVersion: (0, deps_1.installedEngineVersion)(defaultEngine),
+                tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
+                instanceId,
+            }),
+            // Kill switch + rollback, checked before a sandbox can exist. Revoking the lease stops
+            // new work fleet-wide immediately; TTL bounds the sandboxes already running.
+            admit: async (sessionKey) => {
+                const currentLease = await leaseSource.get();
+                assertPackAdmitted(currentLease, sessionKey);
+            },
+            inferenceBaseUrl: apiConfig.baseUrl,
+            // Per-sandbox inference warrant — signed, audience-bound, and naming the sandbox it was
+            // minted for. The provider key never enters a sandbox; the proxy verifies with a public
+            // key and can never mint, so a sandbox cannot re-issue itself more budget.
+            //
+            // `user` is the verified end user from the turn's credential scope, not the pack: the
+            // gateway strips caller-supplied x-litellm-metadata-user-id and re-stamps it from this
+            // claim, so the warrant is the only attribution channel that survives it — stamping
+            // packId here made every downstream user_id (LiteLLM, spend rows, evidence provenance)
+            // the pack. Every mint runs inside runWithCredential (create-time on first-turn
+            // acquire, then per-turn re-mints in RemoteEngine.stream), so the ambient credential is
+            // the turn's own. An unidentified turn gets a session-scoped marker rather than
+            // silently borrowing the pack's name — sessionKey is already user-namespaced, so the
+            // marker cannot collide across identified users.
+            mintModelToken: (sandboxId, sessionKey) => (0, support_1.mintSandboxInferenceWarrant)(warrantKey, {
+                user: (0, credential_context_1.currentCredential)()?.userId?.trim() || `anon:${sessionKey}`,
+                pack: packId,
+                packVersion: activeVersion,
+                sandboxId,
+                tier: options.sandbox === 'container' ? 2 : 0,
+                // A sandbox outlives one turn, so its model warrant gets the full cap rather than a
+                // turn-length TTL; the spend cap, not the clock, is what bounds it.
+                ttlSeconds: 300,
+            }),
+            logger,
+            runtime: { logger, editTracker: undefined, configProvider: undefined, permissionHandler },
+        })
+        : undefined;
+    sandboxRunnerRef = sandboxRunner ?? undefined;
+    // Caps + kill switch. Enforcement is non-cooperative — it revokes a route or destroys the
+    // sandbox — so it exists only where there ARE sandboxes. Without --sandbox we say so rather than
+    // pretending a cap is in force.
+    // 0 / NaN mean "unlimited" for every one of these — `--max-turns-per-session` has documented
+    // that meaning since before caps existed, and passing its default 0 through as a real cap
+    // refused the very first turn (`turn cap reached (1/0)`, observed live).
+    const positive = (n) => typeof n === 'number' && Number.isFinite(n) && n > 0 ? n : undefined;
+    const localCaps = {
+        ...(0, caps_1.capsFromEnv)(),
+        ...(positive(options.maxTurnsPerSession) !== undefined ? { maxTurnsPerSession: options.maxTurnsPerSession } : {}),
+        ...(positive(options.maxSessionSeconds) !== undefined ? { maxSessionSeconds: options.maxSessionSeconds } : {}),
+        ...(positive(options.maxConcurrentSandboxes) !== undefined ? { maxConcurrentSandboxes: options.maxConcurrentSandboxes } : {}),
+        ...(options.capOnBreach ? { onBreach: options.capOnBreach } : {}),
+    };
+    const capsRequested = localCaps.maxTurnsPerSession !== undefined
+        || localCaps.maxSessionSeconds !== undefined
+        || localCaps.maxConcurrentSandboxes !== undefined;
+    // A deployment must be governable whether or not it runs sandboxes. Gating the governor on
+    // `sandboxRunner` meant a plain `serve` never polled its lease at all: invisible to the control
+    // plane AND immune to the kill switch, which is the opposite of what an operator configuring
+    // a lease URL is asking for. Without sandboxes the enforcement that needs one degrades (no route
+    // revocation, no concurrency bound) but the substance survives — the turn is still refused.
+    const nullSandboxes = {
+        sandboxes: () => [],
+        destroySession: async () => undefined,
+        revokeRoute: async () => undefined,
+        // There is nothing here that can stop a turn already executing in this process, so a
+        // `kill` deny rule degrades to `drain`. The governor now says so loudly instead of
+        // sweeping an empty list and reporting the rule as applied.
+        supportsInFlightTermination: false,
+    };
+    const governed = Boolean(sandboxRunner || process.env.AGENTHIPPO_LEASE_URL || capsRequested);
+    const governor = governed
+        ? (0, governor_1.createGovernor)({
+            // The lease's caps bind; local flags may tighten them and never loosen them.
+            caps: (0, caps_1.mergeCaps)((await leaseSource.get())?.caps, localCaps),
+            sandboxes: sandboxRunner ?? nullSandboxes,
+            getLease: () => leaseSource.get(),
+            pollSeconds: leasePollSeconds,
+            // Scope for deny rules: the pack this instance runs, and the same hash that namespaces
+            // session keys — so the console can name a user by email without serve holding a roster.
+            pack: packId,
+            userHash: serve_hardening_1.userNamespaceHash,
+            // Only when there is no sandbox: with one, the same checks gate a sandbox's BIRTH and
+            // running sandboxes are meant to drain on TTL. Without one, this is the only gate that
+            // exists, so an unsigned pack or an unpinned version would otherwise keep serving.
+            ...(sandboxRunner ? {} : {
+                admitPack: (sessionKey, lease) => {
+                    try {
+                        // No userId here on purpose: the governor has ALREADY evaluated the deny rules for
+                        // this turn (with its followed-user map, which this call site cannot see), so the
+                        // only thing left for this gate is the signature/pin/digest half.
+                        assertPackAdmitted(lease, sessionKey);
+                        return { allowed: true };
+                    }
+                    catch (err) {
+                        return { allowed: false, reason: err instanceof Error ? err.message : String(err) };
+                    }
+                },
+            }),
+            logger,
+            audit: (event, fields) => auditEvent(String(fields.sessionKey ?? 'instance'), event, fields),
+        })
+        : undefined;
+    if (capsRequested && !sandboxRunner) {
+        console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} caps are configured but --sandbox is not set. Turn and duration caps still refuse the turn; the concurrency cap and route revocation need sandboxes, so those are IGNORED.`);
+    }
+    const baseRunner = sandboxRunner ?? inProcessRunner;
+    const runner = governor ? (0, governor_1.withGovernor)(baseRunner, governor) : baseRunner;
+    if (governor) {
+        governor.start();
+        const p = governor.policy();
+        // The banner prints the number an operator will be held to: worst-case kill latency is
+        // poll + cache, not the poll alone. It used to print "lease poll 30s" while the cache
+        // defaulted to 60, so a kill could take three times as long as advertised.
+        const killLatency = (0, lease_1.effectiveKillLatencySeconds)(leasePollSeconds);
+        logger.info(`[Serve] control plane: caps turns=${p.maxTurnsPerSession ?? '∞'} session=${p.maxSessionSeconds ?? '∞'}s sandboxes=${p.maxConcurrentSandboxes ?? '∞'} onBreach=${p.onBreach ?? 'revoke-inference'}; lease poll ${leasePollSeconds}s (worst-case kill latency ${killLatency}s; an admin refresh is immediate)`);
+        if (!sandboxRunner) {
+            logger.info('[Serve] no sandbox driver: a deny rule with mode=kill takes effect as DRAIN — the in-flight turn finishes and only the next one is refused.');
+        }
+        const leaseStatus = leaseSource.status();
+        if (process.env.AGENTHIPPO_LEASE_URL && !leaseStatus.requireSignedLease) {
+            logger.info('[Serve] lease signature is NOT required — set --require-signed-lease (or lease.requireSignedLease) once the control plane signs leases.');
+        }
+    }
+    if (sandboxRunner) {
+        logger.info(`[Serve] sandbox mode: ${options.sandbox} (tier ${options.sandbox === 'container' ? 2 : 0}) — engines run in per-session sandboxes`);
+        // A sandbox's ONLY model credential is the per-sandbox warrant, which is meaningful only to an
+        // inference proxy. With no proxy configured there is nothing to send it to, so the engine
+        // dials the provider bare and the first turn dies with an opaque 401/reconnect loop. Say so
+        // at boot — same reason the engine credential check above is a boot check, not a turn check.
+        if (!apiConfig.baseUrl) {
+            console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} --sandbox without an inference proxy: a sandbox holds no provider key, so model calls will fail 401. Set AGENTHIPPO_LITELLM_BASE_URL.`);
+        }
+    }
     /** Set when `--gateway-bridge` is enabled; cleared on shutdown. */
     let gatewayBridge = null;
     /** True once the pack is synced and (if bridged) the gateway is connected. Drives GET /ready. */
     let ready = false;
     // Create HTTP server
+    let evidenceShipHandle;
     const server = http.createServer(async (req, res) => {
         const cors = (0, serve_hardening_1.corsHeaders)();
         for (const [k, v] of Object.entries(cors)) {
@@ -279,8 +1019,63 @@ async function serveCommand(options) {
         try {
             // Liveness — always open.
             if (req.method === 'GET' && req.url === '/health') {
+                const health = { status: 'ok' };
+                // Reported unconditionally: with only `last_ok_at`, "shipping is off" and "shipping
+                // is on but has never once succeeded" look identical to an operator and to alerting.
+                health.evidence_ship_enabled = Boolean(evidenceShipHandle);
+                const lastOk = evidenceShipHandle?.lastOkAt();
+                if (lastOk !== undefined) {
+                    health.evidence_ship_last_ok_at = new Date(lastOk).toISOString();
+                }
+                // R-13 gap detection: evidence write failures and permanently-skipped files are
+                // COUNTED now rather than swallowed, but a counter nobody can read is the same as no
+                // counter. `disabledReason` matters most — "shipping is off because
+                // AGENTHIPPO_EVIDENCE_SHIP=0" and "shipping is on and failing" produce the same
+                // `enabled: false` shaped alert otherwise.
+                const ship = (0, sessionShip_1.getEvidenceShipStatus)();
+                health.evidence_ship_disabled_reason = ship.disabledReason ?? null;
+                health.evidence_ship_consecutive_failures = ship.consecutiveFailures;
+                health.evidence_records_staged = ship.recordsStaged;
+                health.evidence_records_shipped = ship.recordsShipped;
+                // Non-zero means the record is known to be INCOMPLETE, not merely late.
+                health.evidence_records_skipped_too_old = ship.recordsSkippedTooOld;
+                health.evidence_files_skipped_too_old = ship.filesSkippedTooOld;
+                health.evidence_content_capture = ship.contentCapture;
+                if (ship.lastHeartbeatAt !== undefined) {
+                    health.evidence_ship_last_heartbeat_at = new Date(ship.lastHeartbeatAt).toISOString();
+                }
+                const writes = (0, sessionEvents_1.getSessionEventWriterStats)();
+                health.evidence_writes_ok = writes.written;
+                health.evidence_writes_failed = writes.failed;
+                if (writes.lastError) {
+                    health.evidence_write_last_error = {
+                        at: new Date(writes.lastError.at).toISOString(),
+                        event_type: writes.lastError.eventType,
+                        message: writes.lastError.message,
+                    };
+                }
+                // Governance surface: an operator asking "is this instance actually governed?" gets
+                // the answer here rather than from boot logs that have long since scrolled away.
+                const leaseState = leaseSource.status();
+                health.lease_configured = Boolean(process.env.AGENTHIPPO_LEASE_URL);
+                health.lease_signed = leaseState.signed;
+                health.lease_signature_required = leaseState.requireSignedLease;
+                health.lease_staleness_seconds = leaseState.stalenessSeconds ?? null;
+                health.lease_fail_closed = leaseState.failClosed;
+                health.pack_signature_ok = packSignature.ok;
+                health.pack_signature_required = (0, signature_1.resolveRequireSignedPack)({
+                    localRequire: options.requireSignedPack, localAllowUnsigned: allowUnsignedPack,
+                });
+                health.pack_trust_root = packTrust.status;
+                // Reported separately from the pack root: "signed leases are required and the lease
+                // keyring is unavailable" is an outage in the making, and it is invisible in
+                // `pack_trust_root`. `lease_trust_root_source` distinguishes an operator-named root
+                // from the pack-keyring fallback, which is the state that stops working on rotation.
+                health.lease_trust_root = leaseTrust.status;
+                health.lease_trust_root_source = leaseTrust.explicitlyConfigured ? 'lease' : 'pack-fallback';
+                health.kill_in_flight_supported = Boolean(sandboxRunner);
                 res.writeHead(200, { 'Content-Type': 'application/json' });
-                res.end(JSON.stringify({ status: 'ok' }));
+                res.end(JSON.stringify(health));
                 return;
             }
             // Readiness — pack synced (+ gateway connected when bridged).
@@ -346,7 +1141,7 @@ async function serveCommand(options) {
                     }
                     await prepareSession(sessionKey);
                     if (userId?.trim()) {
-                        turnReg = registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken });
+                        turnReg = await registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken, edgeNonce: edgeTurnNonce(req.headers) });
                     }
                     const result = await (0, credential_context_1.runWithCredential)({ userId, dataToken }, () => handleChat(runner, body, sessionKey, defaultEngine, defaultModel, options.workspace, permissions, userId, turnReg?.env), { onDenied: logWarrantDenied(sessionKey) });
                     res.writeHead(200, { 'Content-Type': 'application/json' });
@@ -354,7 +1149,7 @@ async function serveCommand(options) {
                     statePusher?.schedule(sessionKey); // per-session delta push, off the response path
                 }
                 finally {
-                    turnReg?.release();
+                    await turnReg?.release();
                     release();
                 }
                 return;
@@ -401,7 +1196,7 @@ async function serveCommand(options) {
                     }
                     await prepareSession(sessionKey);
                     if (userId?.trim()) {
-                        turnReg = registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken });
+                        turnReg = await registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken, edgeNonce: edgeTurnNonce(req.headers) });
                     }
                     // Fail-closed credential check MUST run before SSE writeHead(200).
                     // Otherwise a warrant denial lands after headers are committed, the outer
@@ -418,7 +1213,7 @@ async function serveCommand(options) {
                     statePusher?.schedule(sessionKey); // per-session delta push, off the response path
                 }
                 finally {
-                    turnReg?.release();
+                    await turnReg?.release();
                     release();
                 }
                 return;
@@ -472,11 +1267,19 @@ async function serveCommand(options) {
             console.log('');
             console.log('Press Ctrl+C to stop.');
             console.log('');
+            const shipConfig = (0, sessionShip_1.resolveSessionShipConfig)(instanceId, analyticsSessionsDir, storagePath);
+            if (shipConfig) {
+                evidenceShipHandle = (0, sessionShip_1.startSessionShipper)({
+                    ...shipConfig,
+                    logger: { info: (m) => logger.info(m), warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`) },
+                });
+            }
             // Bind channels before bridge connect so config.patch reload does not race registration.
             const bindSpec = options.bind?.trim();
             if (bindSpec) {
                 try {
-                    await bindToGatewayChannels(gatewayAgentId, bindSpec);
+                    // The captured token — the env var was stripped at boot (§10 hole 2).
+                    await bindToGatewayChannels(gatewayAgentId, bindSpec, brokerGatewayToken);
                 }
                 catch (err) {
                     console.error(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} Failed to bind channels: ${err instanceof Error ? err.message : String(err)}`);
@@ -500,13 +1303,15 @@ async function serveCommand(options) {
                             audit: auditEvent,
                         });
                         process.env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
-                        // Now that the broker holds the token, strip it from the process env so no
-                        // engine child (all spread process.env) inherits it (§7.4 gap 1 close).
-                        delete process.env.OPENCLAW_GATEWAY_TOKEN;
+                        // The token itself was stripped from process.env at boot, broker or not.
                         console.log('OK: Egress broker on loopback — gateway token stays broker-side; agent sends via turn nonce');
                     }
                     catch (err) {
-                        console.error(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} Egress broker failed to start: ${err instanceof Error ? err.message : String(err)}`);
+                        // Fail CLOSED: without the broker, AGENTHIPPO_EGRESS_URL stays unset and the
+                        // runtime would bake the gateway token back into engine children (§7.6 undone
+                        // by a port-bind hiccup). Refuse to serve rather than degrade silently.
+                        console.error(`${utils_1.COLORS.red}Error:${utils_1.COLORS.reset} Egress broker failed to start: ${err instanceof Error ? err.message : String(err)}`);
+                        process.exit(1);
                     }
                 }
                 try {
@@ -580,6 +1385,17 @@ async function serveCommand(options) {
                     // best-effort
                 }
             }
+            // After the planes are closed, so the tail includes whatever they just wrote. SIGTERM is
+            // how every redeploy ends, and without this each one drops up to one interval of
+            // session evidence — permanently, when the state dir (and its cursor) is ephemeral.
+            if (evidenceShipHandle) {
+                try {
+                    await evidenceShipHandle.flush();
+                }
+                catch {
+                    // best-effort — never block shutdown on the evidence server being reachable
+                }
+            }
             server.close(() => {
                 runner.dispose();
                 process.exit(0);
@@ -711,9 +1527,9 @@ async function startServeGatewayBridge(params) {
     const { GatewayBridge: Bridge } = await import('../../gateway/bridge.js');
     const DEFAULT_GATEWAY_PORT = 18789;
     const gatewayUrl = process.env.OPENCLAW_GATEWAY_URL ?? `ws://127.0.0.1:${DEFAULT_GATEWAY_PORT}`;
-    // Use the captured token — serve strips OPENCLAW_GATEWAY_TOKEN from process.env once the
-    // egress broker holds it, so the bridge must not depend on the env var (§7.6).
-    const token = params.brokerGatewayToken ?? process.env.OPENCLAW_GATEWAY_TOKEN;
+    // The captured token — OPENCLAW_GATEWAY_TOKEN is stripped from process.env at boot (§7.6,
+    // §10 hole 2), so the bridge must never depend on the env var.
+    const token = params.brokerGatewayToken;
     const bridge = new Bridge({ gatewayUrl, token });
     await bridge.connect();
     await bridge.registerAgent({
@@ -759,7 +1575,7 @@ async function startServeGatewayBridge(params) {
                     // destinations server-side (§7.6). userId gates BOTH capabilities: an
                     // unidentified peer gets neither a nonce nor an egress route.
                     const turnReg = userId?.trim()
-                        ? params.registerTurn({ userId: userId.trim(), sessionKey, egressStoreKey: outboundKey })
+                        ? await params.registerTurn({ userId: userId.trim(), sessionKey, egressStoreKey: outboundKey })
                         : null;
                     try {
                         const result = await (0, credential_context_1.runWithCredential)({ userId }, () => params.runner.run({
@@ -785,7 +1601,7 @@ async function startServeGatewayBridge(params) {
                         return { success: result.success, error: result.error };
                     }
                     finally {
-                        turnReg?.release();
+                        await turnReg?.release();
                     }
                 },
             });
@@ -800,11 +1616,10 @@ async function startServeGatewayBridge(params) {
  * Connect to the gateway and set channel bindings for this agent.
  * Called when --bind is specified (e.g., --bind "telegram,whatsapp" or --bind "*").
  */
-async function bindToGatewayChannels(agentId, bindSpec) {
+async function bindToGatewayChannels(agentId, bindSpec, token) {
     const { createAgentIDEGatewayClient } = await import('../../gateway/client.js');
     const DEFAULT_GATEWAY_PORT = 18789;
     const gatewayUrl = process.env.OPENCLAW_GATEWAY_URL ?? `ws://127.0.0.1:${DEFAULT_GATEWAY_PORT}`;
-    const token = process.env.OPENCLAW_GATEWAY_TOKEN;
     const { client, clientInfo } = createAgentIDEGatewayClient(gatewayUrl, token);
     try {
         await client.connect(clientInfo);
@@ -901,7 +1716,7 @@ async function handleStreamingChat(runner, body, sessionKey, defaultEngine, defa
         },
     };
     try {
-        await runner.run({
+        const result = await runner.run({
             engineId: resolveRequestedEngineId(body.engine, defaultEngine),
             modelId: body.model || defaultModel,
             message: body.message,
@@ -913,6 +1728,14 @@ async function handleStreamingChat(runner, body, sessionKey, defaultEngine, defa
             permissions,
             turnEnv,
         });
+        // A policy refusal is a RESULT, not an exception: `withGovernor` returns
+        // `{success:false, error}` without ever touching the emitter. Ignoring it sent the client
+        // HTTP 200, zero events, then `done` — a kill switch, a cap breach or a deny rule looked
+        // exactly like the agent choosing to say nothing, and the operator's reason reached nobody.
+        // /v1/chat already returns the reason in its body; the two endpoints must not disagree.
+        if (!result.success && !streamEnded) {
+            sendEvent('error', { message: result.error ?? 'refused by policy' });
+        }
         if (!streamEnded) {
             sendEvent('done', {});
             endStream();