@wardby/cli 0.5.3 → 0.7.0

package/dist/native-worker/docker-launcher.js

@@ -0,0 +1,297 @@
+/**
+ * Launches, watches, stops, and removes native sandbox workers on the local
+ * Docker engine (docs/native-sandbox.md), from the isolation plan in
+ * docker-isolation.ts. Every operation is idempotent and keyed by the run id,
+ * so the executor can retry, a second launch attaches instead of starting
+ * twice, and a restarted server re-finds a running worker by name.
+ */
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { logger } from "../core/logger.js";
+import { maskSpans, mergeSpans, secretSpans, withEscapedForms } from "../core/secret-spans.js";
+import { buildGatewayConnectArgs, buildGatewayDisconnectArgs, buildNativeNetworkCreateArgs, buildNativeWarmNetworkCreateArgs, buildNativeWarmWorkerRunArgs, buildNativeWorkerRunArgs, gatewayConnectArgs, gatewayDisconnectArgs, nativeIsolationNames, nativeWarmIsolationNames, NATIVE_WARM_LABEL_FILTER, NATIVE_WARM_TOKEN_LABEL, NATIVE_WORKER_LABEL_FILTER, } from "./docker-isolation.js";
+import { warmDeliveryCommand } from "./warm-delivery.js";
+import { NATIVE_SANDBOX_WARM_DELIVERY_FAILED } from "./warm-pool.js";
+const launcherLog = logger.child({ module: "native-docker-launcher" });
+/**
+ * Bounded, so a noisy worker cannot flood the log. The worker holds no long-lived credentials (only
+ * its run's gateway capability), and it redacts the secret values it received (every `secrets.get`
+ * result, and the capability) from everything it writes to stderr, its failure line and logger lines
+ * included (stderr-redaction.ts), with the console bridge's redactor and its minimum length. The
+ * worker is untrusted, so the launcher masks the tail again itself (workerTailFields).
+ */
+export const WORKER_STDERR_TAIL_CHARS = 2000;
+/** How long a failed worker's diagnostic (its log read and its secret values read) may take in all. */
+export const WORKER_TAIL_TIMEOUT_MS = 10_000;
+/** `promise`, or a rejection once `ms` pass first. */
+export function withTimeout(promise, ms) {
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error("worker_diagnostic_timeout")), Math.max(0, ms));
+    });
+    return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+/**
+ * A failed worker's output tail as log fields: `stderrTail`, masked by exact value through the
+ * shared redactor against every secret value delivered to that worker (its gateway capability,
+ * from its input, and each value its tools may read, from `secretValues`, with their JSON-escaped
+ * forms). When those values are not given, cannot be read, or are not read within `timeoutMs`, no
+ * tail: `stderrTailOmitted: "secrets_unavailable"` (fail closed). No fields when the output is empty.
+ *
+ * `cut`: `output` is a slice of a longer output (a log read bounded by lines or bytes), so it may
+ * begin (`start`) or end (`end`) inside a value, which no exact match finds; as many characters as
+ * the longest value less one are masked at each cut end too.
+ */
+export async function workerTailFields(output, input, secretValues, cut = {}, timeoutMs = WORKER_TAIL_TIMEOUT_MS) {
+    if (!output.trim())
+        return {};
+    let delivered;
+    try {
+        if (!secretValues)
+            return { stderrTailOmitted: "secrets_unavailable" };
+        delivered = await withTimeout(secretValues(), timeoutMs);
+    }
+    catch {
+        return { stderrTailOmitted: "secrets_unavailable" };
+    }
+    const values = withEscapedForms([...(input.gateway ? [input.gateway.capability] : []), ...delivered]);
+    const spans = secretSpans(output, values);
+    const longest = Math.max(0, ...values.map((value) => value.length));
+    if (longest > 1) {
+        if (cut.start)
+            spans.push([0, Math.min(output.length, longest - 1)]);
+        if (cut.end)
+            spans.push([Math.max(0, output.length - (longest - 1)), output.length]);
+    }
+    // Masked whole, then cut: a cut can then split only a marker or plain text, never a value.
+    const stderrTail = maskSpans(output, mergeSpans(spans)).trim().slice(-WORKER_STDERR_TAIL_CHARS);
+    return stderrTail ? { stderrTail } : {};
+}
+export function processDockerCli(binary = "docker") {
+    return {
+        run(args, options = {}) {
+            return new Promise((resolve) => {
+                const child = spawn(binary, args, { stdio: ["pipe", "pipe", "pipe"] });
+                let stdout = "";
+                let stderr = "";
+                const timer = options.timeoutMs ? setTimeout(() => child.kill("SIGKILL"), options.timeoutMs) : undefined;
+                // One decoder per stream, so a multi-byte character split across chunks still decodes (and a
+                // secret holding it still matches the redactor exactly).
+                child.stdout.setEncoding("utf8");
+                child.stderr.setEncoding("utf8");
+                child.stdout.on("data", (chunk) => (stdout += chunk));
+                child.stderr.on("data", (chunk) => (stderr += chunk));
+                child.on("error", (err) => resolve({ code: null, stdout, stderr: err.message }));
+                child.on("close", (code) => {
+                    if (timer)
+                        clearTimeout(timer);
+                    resolve({ code, stdout, stderr });
+                });
+                child.stdin.on("error", () => { });
+                child.stdin.end(options.input ?? "");
+            });
+        },
+    };
+}
+export class DockerNativeWorkerLauncher {
+    options;
+    docker;
+    constructor(options) {
+        this.options = options;
+        this.docker = options.docker ?? processDockerCli();
+    }
+    async must(args, what, tolerate) {
+        const result = await this.docker.run(args);
+        if (result.code !== 0 && !(tolerate && tolerate.test(result.stderr))) {
+            throw new Error(`native_sandbox_docker_failed: ${what}: ${result.stderr.trim().slice(0, 500)}`);
+        }
+        return result;
+    }
+    /** The image must be present and pinned; a digest is pulled once, a local id must already exist. */
+    async ensureImage() {
+        const present = await this.docker.run(["image", "inspect", "--format", "{{.Id}}", this.options.image]);
+        if (present.code === 0)
+            return;
+        await this.must(["image", "pull", "--quiet", this.options.image], "pull the worker image");
+    }
+    async launch(input, launchOptions = {}) {
+        const runId = input.runId;
+        const names = nativeIsolationNames(runId);
+        const existing = await this.inspect(runId);
+        if (existing.state === "missing") {
+            await this.ensureImage();
+            await this.must(buildNativeNetworkCreateArgs(runId), "create the run network", /already exists/);
+            await this.must(buildGatewayConnectArgs(runId, this.options.gatewayContainer), "join the gateway to the run network", /already exists|is already connected/);
+            const args = buildNativeWorkerRunArgs({ runId, image: this.options.image, limits: this.options.limits });
+            // Attached so the input (capability included) goes in on stdin; the exit is read with
+            // `docker wait`, which a restarted server can call again.
+            void this.docker.run(args, { input: `${JSON.stringify(input)}\n` }).then(async (result) => {
+                if (result.code !== 0 && result.code !== null && !/Conflict/.test(result.stderr)) {
+                    const tail = await workerTailFields(result.stderr, input, launchOptions.secretValues);
+                    launcherLog.warn({ container: names.worker, code: result.code, ...tail }, "native worker exited non-zero");
+                }
+            });
+        }
+        else {
+            launcherLog.info({ container: names.worker }, "native worker already exists: attaching, not relaunching");
+        }
+        return this.handle(runId);
+    }
+    /** Waits for (and reports) the worker's exit; usable after a server restart. */
+    handle(runId) {
+        return this.handleByName(nativeIsolationNames(runId).worker);
+    }
+    handleByName(name) {
+        const exited = (async () => {
+            // `docker run` may not have created the container yet: wait for it to exist (bounded).
+            for (let attempt = 0; attempt < 50; attempt += 1) {
+                const waited = await this.docker.run(["container", "wait", name]);
+                if (waited.code === 0) {
+                    const code = Number(waited.stdout.trim());
+                    return Number.isInteger(code) ? code : null;
+                }
+                if (!/No such container/i.test(waited.stderr))
+                    return null;
+                await new Promise((resolve) => setTimeout(resolve, 200));
+            }
+            return null;
+        })();
+        return { exited, kill: () => void this.docker.run(["container", "kill", name]) };
+    }
+    async inspect(runId) {
+        return this.inspectByName(nativeIsolationNames(runId).worker);
+    }
+    async inspectByName(name) {
+        const result = await this.docker.run([
+            "container",
+            "inspect",
+            "--format",
+            "{{.State.Status}} {{.State.ExitCode}}",
+            name,
+        ]);
+        if (result.code !== 0)
+            return { state: "missing" };
+        const [status, exitCode] = result.stdout.trim().split(" ");
+        if (status === "running" || status === "created" || status === "restarting")
+            return { state: "running" };
+        return { state: "exited", exitCode: Number(exitCode) };
+    }
+    async kill(runId) {
+        await this.docker.run(["container", "kill", nativeIsolationNames(runId).worker]);
+    }
+    /** Removes the worker, its network, and the gateway's membership of it. Idempotent. */
+    async remove(runId) {
+        const names = nativeIsolationNames(runId);
+        await this.docker.run(["container", "rm", "--force", names.worker]);
+        await this.docker.run(buildGatewayDisconnectArgs(runId, this.options.gatewayContainer));
+        await this.docker.run(["network", "rm", names.network]);
+    }
+    /** Removes a worker found by the janitor, by its container name (its network shares the name's token). */
+    async removeByWorkerName(workerName) {
+        const token = /^wardby-native-([0-9a-f]{20})$/.exec(workerName)?.[1];
+        if (!token)
+            return;
+        await this.docker.run(["container", "rm", "--force", workerName]);
+        await this.docker.run(["network", "disconnect", "--force", `wardby-nnet-${token}`, this.options.gatewayContainer]);
+        await this.docker.run(["network", "rm", `wardby-nnet-${token}`]);
+    }
+    /** Every native worker container this engine holds, with its run-hash label (for the janitor). */
+    async listWorkers() {
+        const result = await this.docker.run([
+            "container",
+            "ls",
+            "--all",
+            "--filter",
+            `label=${NATIVE_WORKER_LABEL_FILTER}`,
+            "--format",
+            '{{.Names}} {{.Label "io.wardby.run-sha256"}}',
+        ]);
+        if (result.code !== 0)
+            return [];
+        return result.stdout
+            .split("\n")
+            .map((line) => line.trim().split(" "))
+            .filter(([name, runHash]) => name && runHash)
+            .map(([name, runHash]) => ({ name, runHash }));
+    }
+    // --- Warm pool workers (native sandbox phase 6) ---
+    warmSpecHash(waitMs) {
+        const { image, limits, gatewayContainer } = this.options;
+        return createHash("sha256")
+            .update(JSON.stringify({ launcher: "docker", image, limits, gatewayContainer, waitMs }))
+            .digest("hex");
+    }
+    /** Its network is internal from creation, so a running container is already isolated: claimable. */
+    async startWarm(token, waitMs) {
+        const names = nativeWarmIsolationNames(token);
+        try {
+            await this.ensureImage();
+            await this.must(buildNativeWarmNetworkCreateArgs(token), "create the warm worker network", /already exists/);
+            await this.must(gatewayConnectArgs(names.network, this.options.gatewayContainer), "join the gateway to the warm worker network", /already exists|is already connected/);
+            await this.must(buildNativeWarmWorkerRunArgs({ token, image: this.options.image, limits: this.options.limits, waitMs }), "start the warm worker");
+        }
+        catch (err) {
+            await this.removeWarm(token).catch(() => { });
+            throw err;
+        }
+    }
+    /** Still running, and attached to its own network and nothing else. */
+    async reattestWarm(token) {
+        const names = nativeWarmIsolationNames(token);
+        const result = await this.docker.run([
+            "container",
+            "inspect",
+            "--format",
+            "{{.State.Status}} {{json .NetworkSettings.Networks}}",
+            names.worker,
+        ]);
+        if (result.code !== 0)
+            return false;
+        const [status, ...rest] = result.stdout.trim().split(" ");
+        try {
+            const networks = Object.keys(JSON.parse(rest.join(" ")));
+            return status === "running" && networks.length === 1 && networks[0] === names.network;
+        }
+        catch {
+            return false;
+        }
+    }
+    async deliver(token, input) {
+        const result = await this.docker.run(["container", "exec", "-i", nativeWarmIsolationNames(token).worker, ...warmDeliveryCommand()], { input: `${JSON.stringify(input)}\n`, timeoutMs: 20_000 });
+        if (result.code !== 0) {
+            throw new Error(`${NATIVE_SANDBOX_WARM_DELIVERY_FAILED}: delivering the run's input exited ${result.code ?? "?"}.`);
+        }
+    }
+    warmHandle(token) {
+        return this.handleByName(nativeWarmIsolationNames(token).worker);
+    }
+    inspectWarm(token) {
+        return this.inspectByName(nativeWarmIsolationNames(token).worker);
+    }
+    async killWarm(token) {
+        await this.docker.run(["container", "kill", nativeWarmIsolationNames(token).worker]);
+    }
+    async removeWarm(token) {
+        const names = nativeWarmIsolationNames(token);
+        await this.docker.run(["container", "rm", "--force", names.worker]);
+        await this.docker.run(gatewayDisconnectArgs(names.network, this.options.gatewayContainer));
+        await this.docker.run(["network", "rm", names.network]);
+    }
+    async listWarm() {
+        const result = await this.docker.run([
+            "container",
+            "ls",
+            "--all",
+            "--filter",
+            `label=${NATIVE_WARM_LABEL_FILTER}`,
+            "--format",
+            `{{.Label "${NATIVE_WARM_TOKEN_LABEL}"}}`,
+        ]);
+        if (result.code !== 0)
+            return [];
+        return result.stdout
+            .split("\n")
+            .map((line) => line.trim())
+            .filter((token) => /^[0-9a-f]{20}$/.test(token));
+    }
+}

package/dist/native-worker/gateway-server.d.ts

@@ -0,0 +1,77 @@
+/**
+ * The native sandbox gateway over HTTP (native sandbox phase 3): stateless per
+ * request. Each request is authenticated by the hash of its bearer capability,
+ * rebuilt from its run's gateway session (the snapshot pinned at launch), the
+ * run row, and the call ledger, and served by the same trusted code the
+ * in-process path uses — so any control-plane replica can serve any call, and a
+ * replica restart is only a retry for the worker.
+ *
+ * Authority stays here: a model call reserves its worst case against the
+ * session budget before the provider is called and settles at the provider's
+ * reported usage; the worker's own usage numbers are never read.
+ */
+import { createNativeRunTools, durableDelegate, type NativeRunFinishContext, type NativeRunProviders, type RunnerDb, type SandboxSessionSnapshot } from "../core/runner.js";
+import { ReviewReadLedger } from "../core/review-reads.js";
+import type { LlmProvider, LlmStreamEvent } from "../providers/llm/types.js";
+import { PrismaGatewayLedger, type GatewaySessionRecord } from "./ledger.js";
+import { type GatewayParams } from "./protocol.js";
+/** Output tokens reserved for a model call that names no maxTokens (as the coding proxy does). */
+export declare const GATEWAY_DEFAULT_RESERVE_OUTPUT_TOKENS = 4096;
+/** The most output tokens one call may reserve for. */
+export declare const GATEWAY_MAX_RESERVE_OUTPUT_TOKENS = 128000;
+/** A replayable result is kept only up to this size; a larger one cannot be replayed on retry. */
+export declare const GATEWAY_MAX_REPLAY_BYTES: number;
+export interface GatewayServerDeps {
+    db: RunnerDb & ConstructorParameters<typeof PrismaGatewayLedger>[0];
+    providers: NativeRunProviders;
+    /** Delegation long-poll window; tests shorten it. */
+    pollWindowMs?: number;
+}
+/** What a request gets back: one JSON value, or a stream of model events. */
+export type GatewayResponse = {
+    kind: "json";
+    status: number;
+    body: unknown;
+} | {
+    kind: "stream";
+    status: 200;
+    events: AsyncIterable<LlmStreamEvent>;
+};
+/** Serves one gateway request. Never throws: every failure is a response. */
+export declare function serveGatewayRequest(deps: GatewayServerDeps, authorization: string | undefined, body: unknown): Promise<GatewayResponse>;
+/**
+ * Every secret value the rebuilt run can read (each user tool's readable or brokered values), and
+ * the request's own capability: what a cause code is checked against before it leaves the gateway.
+ */
+export declare function runSecretValues(ctx: Pick<RebuiltRun, "snapshot" | "tools">, capability: string): () => Promise<string[]>;
+/**
+ * A failure as its response. Only a checked cause code leaves the gateway (workerGatewayError),
+ * never the rest of an unexpected error's message; the server's own log line is redacted by the
+ * same values.
+ */
+export declare function failureResponse(err: unknown, secretValues: (() => Promise<string[]>) | undefined): Promise<GatewayResponse>;
+/** A model stream whose failure reaches the worker as the same checked error a JSON response carries. */
+export declare function checkedEvents(events: AsyncIterable<LlmStreamEvent>, secretValues: (() => Promise<string[]>) | undefined): AsyncIterable<LlmStreamEvent>;
+interface RebuiltRun {
+    session: GatewaySessionRecord;
+    snapshot: SandboxSessionSnapshot;
+    llm: LlmProvider;
+    tools: ReturnType<typeof createNativeRunTools>;
+    /** The run's review reads, restored from the session; saved back after each read (see builtin). */
+    reviewReads: ReviewReadLedger;
+    finishContext: NativeRunFinishContext;
+    delegation: Omit<Parameters<typeof durableDelegate>[0], "ledger" | "sessionId">;
+}
+/** Exported for its unit test; the HTTP handler is its only production caller. */
+export declare function streamModel(ctx: RebuiltRun, ledger: PrismaGatewayLedger, callId: string, params: GatewayParams<"llm.stream">): AsyncIterable<LlmStreamEvent>;
+/**
+ * Logged (at warn) in place of a user tool's console line when that tool's redaction list cannot
+ * be read: without the complete list the line cannot be shown to be free of its secret values, so
+ * none of it is logged. The tool's call still succeeds, as a console write does in-process.
+ */
+export declare const CONSOLE_WITHHELD_MESSAGE = "sandbox tool console line withheld: the tool's secret values could not be read";
+/** Serves one privileged bridge call for a worker (exported for tests). */
+export declare function hostCall(ctx: RebuiltRun, ledger: PrismaGatewayLedger, callId: string, params: GatewayParams<"host.call">): Promise<unknown>;
+/** The worker's terminal call: the run's one terminal write (masked by finishNativeRun), then the session ends. */
+export declare function finish(ctx: RebuiltRun, ledger: PrismaGatewayLedger, callId: string, params: GatewayParams<"finish">): Promise<null>;
+export {};