@hotcell/daemon 0.1.22 → 0.1.24

package/dist/api/server.d.ts

@@ -3,6 +3,7 @@ import { BackupRegistry } from "../backups.js";
 import { type Config } from "../config.js";
 import type { Driver } from "../driver/types.js";
 import type { Capacity } from "../capacity.js";
+import type { BootScheduler } from "../boot-scheduler.js";
 import type { MetricsHistory } from "../metrics.js";
 import { SandboxStore } from "../store.js";
 interface Deps {
@@ -12,6 +13,7 @@ interface Deps {
     backups: BackupRegistry;
     history?: MetricsHistory;
     capacity?: Capacity;
+    bootScheduler?: BootScheduler;
     /** Re-read provider keys (file/keychain/env) and hot-swap the egress gateway. */
     reloadKeys?: () => {
         providers: number;

package/dist/api/server.js

@@ -1,4 +1,5 @@
 import { createServer } from "node:http";
+import { audit, redactCommandForAudit } from "../audit.js";
 import { previewUrl } from "../config.js";
 import { kernelFor } from "../kernels.js";
 import { pauseSandbox, resumeSandbox } from "../lifecycle.js";
@@ -133,6 +134,7 @@ async function handleTerminalUpgrade(req, socket, head, deps) {
         return;
     }
     if (!isAllowedHost(req, config)) {
+        auditAccessDenied("auth.host_rejected", "GET", url.pathname, "host_not_allowed", driver, store);
         socket.write("HTTP/1.1 403 Forbidden\r\n\r\n");
         socket.destroy();
         return;
@@ -140,6 +142,7 @@ async function handleTerminalUpgrade(req, socket, head, deps) {
     if (config.apiKey) {
         const provided = url.searchParams.get("key") ?? bearerFromHeader(req);
         if (!provided || !safeEqual(provided, config.apiKey)) {
+            auditAccessDenied("auth.rejected", "GET", url.pathname, "invalid_api_key", driver, store);
             socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
             socket.destroy();
             return;
@@ -184,11 +187,13 @@ async function handleTerminalUpgrade(req, socket, head, deps) {
         term = await driver.openTerminal(id, { cols, rows, env: record.env });
     }
     catch (err) {
+        audit({ event: "sandbox.terminal.open.failed", sandboxId: id, driver: record.driver ?? driver.name });
         ws.send(`\r\n[sbx] failed to open terminal: ${errorMessage(err)}\r\n`);
         ws.close();
         span.setStatus("error").end();
         return;
     }
+    audit({ event: "sandbox.terminal.open", sandboxId: id, driver: record.driver ?? driver.name });
     log.info("terminal opened", { sandbox: id, traceId: span.traceId });
     term.stream.on("data", (chunk) => ws.send(chunk));
     term.stream.on("end", () => ws.close());
@@ -211,6 +216,7 @@ async function handleTerminalUpgrade(req, socket, head, deps) {
     });
     ws.onClose(() => {
         term.close();
+        audit({ event: "sandbox.terminal.close", sandboxId: id, driver: record.driver ?? driver.name });
         span.end();
         log.info("terminal closed", { sandbox: id });
     });
@@ -401,7 +407,18 @@ function resolveLimits(body, config) {
 function normalizeRoute(path) {
     return path
         .replace(/^\/sandboxes\/[^/]+/, "/sandboxes/:id")
-        .replace(/\/(processes|sessions|code-contexts|expose|backups)\/[^/]+/, "/$1/:sub");
+        .replace(/\/(processes|sessions|code-contexts|expose|backups|egress-tokens)\/[^/]+/g, "/$1/:sub");
+}
+/** Audit an access rejection without recording credentials or raw path identifiers. */
+function auditAccessDenied(event, method, path, reason, driver, store) {
+    const sandboxId = path.match(/^\/sandboxes\/([^/]+)/)?.[1];
+    const record = sandboxId ? store.get(sandboxId) : undefined;
+    audit({
+        event,
+        sandboxId,
+        driver: record?.driver ?? driver.name,
+        attributes: { method, route: normalizeRoute(path), reason },
+    });
 }
 /**
  * Enforce the API key when one is configured. `/healthz` and the dashboard HTML
@@ -459,9 +476,11 @@ async function handle(req, res, deps) {
     const path = url.pathname;
     const method = req.method ?? "GET";
     if (!isAllowedHost(req, config)) {
+        auditAccessDenied("auth.host_rejected", method, path, "host_not_allowed", driver, store);
         return sendJson(res, 403, { error: "host not allowed" });
     }
     if (!isAuthorized(req, config, path)) {
+        auditAccessDenied("auth.rejected", method, path, "invalid_api_key", driver, store);
         res.setHeader("WWW-Authenticate", "Bearer");
         return sendJson(res, 401, { error: "missing or invalid API key" });
     }
@@ -500,6 +519,8 @@ async function handle(req, res, deps) {
         if (!store.get(egressTokenIdMatch[1]))
             return sendJson(res, 404, { error: "not found" });
         const removed = store.removeEgressToken(egressTokenIdMatch[2]);
+        if (removed)
+            audit({ event: "gateway.token.revoked", sandboxId: egressTokenIdMatch[1], driver: store.get(egressTokenIdMatch[1])?.driver ?? driver.name, attributes: { "token.raw": egressTokenIdMatch[2] } });
         if (!removed)
             return sendJson(res, 404, { error: "token not found" });
         return sendJson(res, 200, { token: egressTokenIdMatch[2], revoked: true });
@@ -516,6 +537,7 @@ async function handle(req, res, deps) {
                 return sendJson(res, 400, { error: parsed.error });
             const token = SandboxStore.newEgressToken();
             store.addEgressToken(token, id, parsed.policy);
+            audit({ event: "gateway.token.minted", sandboxId: id, driver: store.get(id)?.driver ?? driver.name, attributes: { "token.raw": token, providers: (parsed.policy.providers ?? []).join(",") } });
             return sendJson(res, 201, {
                 token,
                 policy: parsed.policy,
@@ -538,13 +560,38 @@ async function handle(req, res, deps) {
     if (method === "GET" && path === "/capacity") {
         if (!deps.capacity)
             return sendJson(res, 200, { enforced: false });
-        return sendJson(res, 200, deps.capacity.snapshot());
+        return sendJson(res, 200, {
+            ...deps.capacity.snapshot(),
+            boot: deps.bootScheduler?.snapshot(),
+        });
     }
     // Hot-reload provider keys after `hotcell keys add/rm` — no daemon restart.
     if (method === "POST" && path === "/reload-keys") {
         const r = deps.reloadKeys?.() ?? { providers: 0 };
         return sendJson(res, 200, { ok: true, ...r });
     }
+    if (method === "POST" && path === "/audit-events") {
+        const body = await readJson(req, config.maxBodyBytes);
+        if (body.event !== "keys.import" || !Array.isArray(body.choices) || body.choices.length > 500) {
+            return sendJson(res, 400, { error: "invalid audit event payload" });
+        }
+        const choices = body.choices;
+        if (!choices.every((choice) => {
+            if (!choice || typeof choice !== "object")
+                return false;
+            const item = choice;
+            return typeof item.name === "string" && /^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(item.name)
+                && (item.disposition === "gateway" || item.disposition === "inject" || item.disposition === "skip");
+        }))
+            return sendJson(res, 400, { error: "invalid key import choice" });
+        const safeChoices = choices;
+        audit({ event: "keys.import", driver: driver.name, attributes: {
+                "key.count": safeChoices.length,
+                "key.names": safeChoices.map((item) => item.name).join(","),
+                "key.dispositions": safeChoices.map((item) => `${item.name}=${item.disposition}`).join(","),
+            } });
+        return sendJson(res, 202, { accepted: true });
+    }
     // Recent finished spans, for debugging and the trace view. Newest first.
     if (method === "GET" && path === "/traces") {
         const limit = Number(url.searchParams.get("limit") ?? "100") || 100;
@@ -557,7 +604,13 @@ async function handle(req, res, deps) {
     }
     if (method === "POST" && path === "/sandboxes") {
         const body = await readJson(req, config.maxBodyBytes);
-        return createSandbox(res, { config, driver, store, capacity: deps.capacity }, body);
+        return createSandbox(res, {
+            config,
+            driver,
+            store,
+            capacity: deps.capacity,
+            bootScheduler: deps.bootScheduler,
+        }, body);
     }
     if (method === "GET" && path === "/sandboxes") {
         return sendJson(res, 200, { sandboxes: store.list() });
@@ -665,9 +718,16 @@ async function handle(req, res, deps) {
     }
     const backupIdMatch = path.match(/^\/backups\/([^/]+)$/);
     if (method === "DELETE" && backupIdMatch) {
+        const backupInfo = await backups.get(backupIdMatch[1]);
         const removed = await backups.remove(backupIdMatch[1]);
         if (!removed)
             return sendJson(res, 404, { error: "backup not found" });
+        audit({
+            event: "sandbox.backup.delete",
+            sandboxId: backupInfo?.sandboxId,
+            driver: backupInfo ? store.get(backupInfo.sandboxId)?.driver ?? driver.name : driver.name,
+            attributes: { backupId: backupIdMatch[1] },
+        });
         return sendJson(res, 200, { backupId: backupIdMatch[1], deleted: true });
     }
     const backupsMatch = path.match(/^\/sandboxes\/([^/]+)\/backups$/);
@@ -750,6 +810,7 @@ async function handle(req, res, deps) {
             if (!record)
                 return sendJson(res, 404, { error: "not found" });
             await driver.destroy(id);
+            audit({ event: "sandbox.destroy", sandboxId: id, driver: record.driver ?? driver.name, attributes: { image: record.image } });
             store.remove(id);
             store.clearSandbox(id);
             deps.history?.clear(id);
@@ -758,7 +819,10 @@ async function handle(req, res, deps) {
     }
     sendJson(res, 404, { error: `no route for ${method} ${path}` });
 }
-async function createSandbox(res, { config, driver, store, capacity }, body) {
+function safeRepoForAudit(repo) {
+    return repo.replace(/:\/\/[^/@]+@/g, "://").split(/[?#]/, 1)[0]?.slice(0, 512) ?? "";
+}
+async function createSandbox(res, { config, driver, store, capacity, bootScheduler }, body) {
     const id = SandboxStore.newId();
     const image = typeof body.image === "string" ? body.image : config.defaultImage;
     // Per-sandbox isolation selection: pick the runtime driver (default SBX_DRIVER).
@@ -801,8 +865,10 @@ async function createSandbox(res, { config, driver, store, capacity }, body) {
         ok: true,
         release: () => { },
     };
-    if (!admitted.ok)
+    if (!admitted.ok) {
+        audit({ event: "sandbox.admission_denied", sandboxId: id, driver: driverName, attributes: { image, memoryMb: limits.memoryMb ?? 0, cpus: limits.cpus ?? 0, status: 503, reason: admitted.reason.slice(0, 256) } });
         return sendCapacityRefusal(res, admitted.reason);
+    }
     // Egress wiring. `egress: true` (or a policy object) opts the sandbox's LLM SDKs
     // into the gateway (provider base-URL + key env). Independently, when egress
     // enforcement is ON every sandbox is locked to the gateway, so it also gets a
@@ -878,19 +944,38 @@ async function createSandbox(res, { config, driver, store, capacity }, body) {
         usage: emptyUsage(),
     };
     store.add(record);
+    audit({ event: "sandbox.create", sandboxId: id, driver: driverName, attributes: { image, repo: repo ? safeRepoForAudit(repo) : "", branch: (repoRef ?? branchRaw ?? "").slice(0, 256), memoryMb: limits.memoryMb ?? 0, cpus: limits.cpus ?? 0 } });
     if (egressToken)
         store.addEgressToken(egressToken, id, egressPolicy);
-    const provision = () => driver.create({
-        id, image, driver: driverName, env, labels, persist, setup: setupSteps,
-        repo, repoRef, limits, networked, writableRootfs, cpuset,
-        onProgress: (phase) => {
+    const provision = () => (driverName === "firecracker" || driverName === "applevz") && bootScheduler
+        ? bootScheduler.run(`create:${id}`, () => driver.create({
+            id, image, driver: driverName, env, labels, persist, setup: setupSteps,
+            repo, repoRef, limits, networked, writableRootfs, cpuset,
+            onProgress: (phase) => {
+                const r = store.get(id);
+                if (r?.status === "creating") {
+                    r.statusReason = phase;
+                    store.add(r);
+                }
+            },
+        }), () => {
             const r = store.get(id);
             if (r?.status === "creating") {
-                r.statusReason = phase;
+                r.statusReason = "queued for microVM boot";
                 store.add(r);
             }
-        },
-    });
+        })
+        : driver.create({
+            id, image, driver: driverName, env, labels, persist, setup: setupSteps,
+            repo, repoRef, limits, networked, writableRootfs, cpuset,
+            onProgress: (phase) => {
+                const r = store.get(id);
+                if (r?.status === "creating") {
+                    r.statusReason = phase;
+                    store.add(r);
+                }
+            },
+        });
     // Terminal transitions. Both re-check the store: a DELETE mid-create is the
     // cancel path, and must win — never resurrect a destroyed record.
     const finishOk = () => {
@@ -904,6 +989,7 @@ async function createSandbox(res, { config, driver, store, capacity }, body) {
         r.statusReason = undefined;
         r.lastActivityAt = new Date().toISOString();
         store.add(r);
+        audit({ event: "sandbox.start", sandboxId: id, driver: r.driver ?? driver.name });
         return r;
     };
     const finishFail = async (err, keepRecord) => {
@@ -958,7 +1044,8 @@ async function stopSandbox(res, { driver, store }, id) {
     // Processes and exposed ports die with the container; sessions persist.
     store.clearRuntimeState(id);
     record.status = "stopped";
-    store.add(record); // write-through the status change
+    store.add(record);
+    audit({ event: "sandbox.stop", sandboxId: id, driver: record.driver ?? driver.name }); // write-through the status change
     sendJson(res, 200, record);
 }
 async function startSandbox(res, { driver, store, capacity }, id) {
@@ -1097,6 +1184,7 @@ async function createBackup(res, { driver, store, backups, capacity }, id) {
         bytes,
     };
     await backups.save(info);
+    audit({ event: "sandbox.backup.create", sandboxId: id, driver: record.driver ?? driver.name, attributes: { backupId, bytes } });
     sendJson(res, 201, info);
 }
 async function restoreBackup(res, { driver, store, backups, capacity }, id, body) {
@@ -1112,6 +1200,7 @@ async function restoreBackup(res, { driver, store, backups, capacity }, id, body
     if (!(await ensureLive(res, driver, store, id, capacity)))
         return;
     await driver.restoreBackup(id, backups.tarPath(backupId));
+    audit({ event: "sandbox.backup.restore", sandboxId: id, driver: store.get(id)?.driver ?? driver.name, attributes: { backupId } });
     sendJson(res, 200, { id, restored: backupId });
 }
 // --- code interpreter ------------------------------------------------------
@@ -1290,8 +1379,19 @@ async function execInSandbox(res, { driver, store, capacity }, id, body) {
     const write = (event) => {
         res.write(`data: ${JSON.stringify(event)}\n\n`);
     };
+    // The stream is live work: the reaper must see this sandbox as active for
+    // the whole run, not just the instant the request arrived.
+    store.beginExec(id);
+    const execStartedAt = Date.now();
+    audit({ event: "sandbox.exec.start", sandboxId: id, driver: record.driver ?? driver.name, attributes: { command: redactCommandForAudit(command), cwd: cwd ?? "" } });
+    let auditExitCode = 1;
+    const stopKeepalive = startSseKeepalive(res);
     try {
-        await driver.exec(id, runCommand, { cwd, env }, write);
+        await driver.exec(id, runCommand, { cwd, env }, (event) => {
+            if (event.type === "exit")
+                auditExitCode = event.exitCode;
+            write(event);
+        });
         if (session && cwdFile) {
             try {
                 const next = (await driver.readFile(id, { path: cwdFile })).trim();
@@ -1310,9 +1410,30 @@ async function execInSandbox(res, { driver, store, capacity }, id, body) {
         write({ type: "exit", exitCode: 1 });
     }
     finally {
+        stopKeepalive();
+        audit({ event: "sandbox.exec", sandboxId: id, driver: record.driver ?? driver.name, attributes: { command: redactCommandForAudit(command), cwd: cwd ?? "", exitCode: auditExitCode, durationMs: Date.now() - execStartedAt } });
+        store.endExec(id);
         res.end();
     }
 }
+/**
+ * Emit an SSE comment frame on a quiet interval for the life of a stream.
+ *
+ * An agent inside a sandbox can legitimately produce NOTHING on an exec stream
+ * for many minutes (a model thinking between tool calls, a subagent doing the
+ * work while the parent waits). HTTP clients infer death from that silence —
+ * undici's default fetch kills a body after 5 quiet minutes ("Body Timeout
+ * Error"), and it cost a consumer a complete 75-minute run at its final verify
+ * step. Comment frames are ignored by every SSE parser but keep bytes flowing,
+ * so no client-side timeout policy ever sees silence on a healthy stream.
+ */
+function startSseKeepalive(res, intervalMs = 15_000) {
+    const timer = setInterval(() => {
+        res.write(": hb\n\n");
+    }, intervalMs);
+    timer.unref?.();
+    return () => clearInterval(timer);
+}
 async function watchFiles(req, res, { driver, store, capacity }, id, watchPath, intervalMs) {
     if (!(await ensureLive(res, driver, store, id, capacity)))
         return;
@@ -1327,6 +1448,7 @@ async function watchFiles(req, res, { driver, store, capacity }, id, watchPath,
     // Stop the watcher when the client disconnects.
     const controller = new AbortController();
     req.on("close", () => controller.abort());
+    const stopKeepalive = startSseKeepalive(res);
     try {
         await driver.watchFiles(id, watchPath, { intervalMs, signal: controller.signal }, (event) => res.write(`data: ${JSON.stringify(event)}\n\n`));
     }
@@ -1334,6 +1456,7 @@ async function watchFiles(req, res, { driver, store, capacity }, id, watchPath,
         res.write(`data: ${JSON.stringify({ type: "error", error: String(err) })}\n\n`);
     }
     finally {
+        stopKeepalive();
         res.end();
     }
 }
@@ -1354,14 +1477,17 @@ async function handleFileAction(res, { driver, store, capacity }, id, action, bo
                     content: body.content,
                     mode: typeof body.mode === "string" ? body.mode : undefined,
                 });
+                audit({ event: "sandbox.file.write", sandboxId: id, driver: store.get(id)?.driver ?? driver.name, attributes: { path: path.slice(0, 512), bytes: Buffer.byteLength(body.content, "utf8") } });
                 return sendJson(res, 200, { ok: true });
             }
             case "read": {
                 const content = await driver.readFile(id, { path });
+                audit({ event: "sandbox.file.read", sandboxId: id, driver: store.get(id)?.driver ?? driver.name, attributes: { path: path.slice(0, 512), bytes: Buffer.byteLength(content, "utf8") } });
                 return sendJson(res, 200, { content });
             }
             case "mkdir": {
                 await driver.mkdir(id, { path, parents: body.parents === true });
+                audit({ event: "sandbox.file.mkdir", sandboxId: id, driver: store.get(id)?.driver ?? driver.name, attributes: { path: path.slice(0, 512), parents: body.parents === true } });
                 return sendJson(res, 200, { ok: true });
             }
             case "list": {
@@ -1400,6 +1526,7 @@ async function startProcess(res, { driver, store, capacity }, id, body) {
             logPath,
         };
         store.addProcess(id, proc);
+        audit({ event: "sandbox.process.start", sandboxId: id, driver: record.driver ?? driver.name, attributes: { command: redactCommandForAudit(command), cwd: opts.cwd ?? "", procId } });
         return sendJson(res, 201, proc);
     }
     catch (err) {
@@ -1457,6 +1584,7 @@ async function streamLogs(req, res, { driver, store }, id, procId, follow) {
     });
     const controller = new AbortController();
     req.on("close", () => controller.abort());
+    const stopKeepalive = startSseKeepalive(res);
     try {
         await driver.streamProcessLogs(id, proc.logPath, { follow, signal: controller.signal }, (data) => res.write(`data: ${JSON.stringify({ type: "log", data })}\n\n`));
     }
@@ -1464,6 +1592,7 @@ async function streamLogs(req, res, { driver, store }, id, procId, follow) {
         res.write(`data: ${JSON.stringify({ type: "log", data: errorMessage(err) })}\n\n`);
     }
     finally {
+        stopKeepalive();
         res.write(`data: ${JSON.stringify({ type: "end" })}\n\n`);
         res.end();
     }
@@ -1502,6 +1631,7 @@ async function exposePort(res, { config, driver, store, capacity }, id, body) {
         url: previewUrl(config, id, port),
     };
     store.addExposed(id, exposed);
+    audit({ event: "sandbox.preview.expose", sandboxId: id, driver: store.get(id)?.driver ?? driver.name, attributes: { port, tokenConfigured: exposed.token !== null } });
     return sendJson(res, 201, exposed);
 }
 function unexposePort(res, { store }, id, port) {
@@ -1510,6 +1640,7 @@ function unexposePort(res, { store }, id, port) {
     const removed = store.removeExposed(id, port);
     if (!removed)
         return sendJson(res, 404, { error: "port not exposed" });
+    audit({ event: "sandbox.preview.unexpose", sandboxId: id, driver: store.get(id)?.driver, attributes: { port } });
     return sendJson(res, 200, { ok: true });
 }
 function createSession(res, { store }, id, body) {