@wardby/cli 0.5.2 → 0.6.0

package/dist/native-worker/docker-launcher.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Launches, watches, stops, and removes native sandbox workers on the local
+ * Docker engine (docs/native-sandbox.md), from the isolation plan in
+ * docker-isolation.ts. Every operation is idempotent and keyed by the run id,
+ * so the executor can retry, a second launch attaches instead of starting
+ * twice, and a restarted server re-finds a running worker by name.
+ */
+import { type NativeWorkerLimits } from "./docker-isolation.js";
+import type { DetachedWorkerLauncher, WorkerHandle } from "./launch.js";
+import type { WorkerInput } from "./protocol.js";
+import { type WarmWorkerLauncher } from "./warm-pool.js";
+export interface DockerResult {
+    code: number | null;
+    stdout: string;
+    stderr: string;
+}
+/** The Docker CLI as this launcher uses it; tests replace it. */
+export interface DockerCli {
+    run(args: string[], options?: {
+        input?: string;
+        timeoutMs?: number;
+    }): Promise<DockerResult>;
+}
+export declare function processDockerCli(binary?: string): DockerCli;
+export type NativeWorkerState = {
+    state: "running";
+} | {
+    state: "exited";
+    exitCode: number;
+} | {
+    state: "missing";
+};
+export interface DockerNativeWorkerLauncherOptions {
+    image: string;
+    /** The container running `wardby native-gateway`, joined to each run's network. */
+    gatewayContainer: string;
+    limits: NativeWorkerLimits;
+    docker?: DockerCli;
+}
+export declare class DockerNativeWorkerLauncher implements DetachedWorkerLauncher, WarmWorkerLauncher {
+    private readonly options;
+    private readonly docker;
+    constructor(options: DockerNativeWorkerLauncherOptions);
+    private must;
+    /** The image must be present and pinned; a digest is pulled once, a local id must already exist. */
+    private ensureImage;
+    launch(input: WorkerInput): Promise<WorkerHandle>;
+    /** Waits for (and reports) the worker's exit; usable after a server restart. */
+    handle(runId: string): WorkerHandle;
+    private handleByName;
+    inspect(runId: string): Promise<NativeWorkerState>;
+    private inspectByName;
+    kill(runId: string): Promise<void>;
+    /** Removes the worker, its network, and the gateway's membership of it. Idempotent. */
+    remove(runId: string): Promise<void>;
+    /** Removes a worker found by the janitor, by its container name (its network shares the name's token). */
+    removeByWorkerName(workerName: string): Promise<void>;
+    /** Every native worker container this engine holds, with its run-hash label (for the janitor). */
+    listWorkers(): Promise<{
+        name: string;
+        runHash: string;
+    }[]>;
+    warmSpecHash(waitMs: number): string;
+    /** Its network is internal from creation, so a running container is already isolated: claimable. */
+    startWarm(token: string, waitMs: number): Promise<void>;
+    /** Still running, and attached to its own network and nothing else. */
+    reattestWarm(token: string): Promise<boolean>;
+    deliver(token: string, input: WorkerInput): Promise<void>;
+    warmHandle(token: string): WorkerHandle;
+    inspectWarm(token: string): Promise<NativeWorkerState>;
+    killWarm(token: string): Promise<void>;
+    removeWarm(token: string): Promise<void>;
+    listWarm(): Promise<string[]>;
+}

package/dist/native-worker/docker-launcher.js

@@ -0,0 +1,237 @@
+/**
+ * Launches, watches, stops, and removes native sandbox workers on the local
+ * Docker engine (docs/native-sandbox.md), from the isolation plan in
+ * docker-isolation.ts. Every operation is idempotent and keyed by the run id,
+ * so the executor can retry, a second launch attaches instead of starting
+ * twice, and a restarted server re-finds a running worker by name.
+ */
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { logger } from "../core/logger.js";
+import { buildGatewayConnectArgs, buildGatewayDisconnectArgs, buildNativeNetworkCreateArgs, buildNativeWarmNetworkCreateArgs, buildNativeWarmWorkerRunArgs, buildNativeWorkerRunArgs, gatewayConnectArgs, gatewayDisconnectArgs, nativeIsolationNames, nativeWarmIsolationNames, NATIVE_WARM_LABEL_FILTER, NATIVE_WARM_TOKEN_LABEL, NATIVE_WORKER_LABEL_FILTER, } from "./docker-isolation.js";
+import { warmDeliveryCommand } from "./warm-delivery.js";
+import { NATIVE_SANDBOX_WARM_DELIVERY_FAILED } from "./warm-pool.js";
+const launcherLog = logger.child({ module: "native-docker-launcher" });
+export function processDockerCli(binary = "docker") {
+    return {
+        run(args, options = {}) {
+            return new Promise((resolve) => {
+                const child = spawn(binary, args, { stdio: ["pipe", "pipe", "pipe"] });
+                let stdout = "";
+                let stderr = "";
+                const timer = options.timeoutMs ? setTimeout(() => child.kill("SIGKILL"), options.timeoutMs) : undefined;
+                child.stdout.on("data", (chunk) => (stdout += chunk.toString()));
+                child.stderr.on("data", (chunk) => (stderr += chunk.toString()));
+                child.on("error", (err) => resolve({ code: null, stdout, stderr: err.message }));
+                child.on("close", (code) => {
+                    if (timer)
+                        clearTimeout(timer);
+                    resolve({ code, stdout, stderr });
+                });
+                child.stdin.on("error", () => { });
+                child.stdin.end(options.input ?? "");
+            });
+        },
+    };
+}
+export class DockerNativeWorkerLauncher {
+    options;
+    docker;
+    constructor(options) {
+        this.options = options;
+        this.docker = options.docker ?? processDockerCli();
+    }
+    async must(args, what, tolerate) {
+        const result = await this.docker.run(args);
+        if (result.code !== 0 && !(tolerate && tolerate.test(result.stderr))) {
+            throw new Error(`native_sandbox_docker_failed: ${what}: ${result.stderr.trim().slice(0, 500)}`);
+        }
+        return result;
+    }
+    /** The image must be present and pinned; a digest is pulled once, a local id must already exist. */
+    async ensureImage() {
+        const present = await this.docker.run(["image", "inspect", "--format", "{{.Id}}", this.options.image]);
+        if (present.code === 0)
+            return;
+        await this.must(["image", "pull", "--quiet", this.options.image], "pull the worker image");
+    }
+    async launch(input) {
+        const runId = input.runId;
+        const names = nativeIsolationNames(runId);
+        const existing = await this.inspect(runId);
+        if (existing.state === "missing") {
+            await this.ensureImage();
+            await this.must(buildNativeNetworkCreateArgs(runId), "create the run network", /already exists/);
+            await this.must(buildGatewayConnectArgs(runId, this.options.gatewayContainer), "join the gateway to the run network", /already exists|is already connected/);
+            const args = buildNativeWorkerRunArgs({ runId, image: this.options.image, limits: this.options.limits });
+            // Attached so the input (capability included) goes in on stdin; the exit is read with
+            // `docker wait`, which a restarted server can call again.
+            void this.docker.run(args, { input: `${JSON.stringify(input)}\n` }).then((result) => {
+                if (result.code !== 0 && result.code !== null && !/Conflict/.test(result.stderr)) {
+                    launcherLog.warn({ container: names.worker, code: result.code }, "native worker exited non-zero");
+                }
+            });
+        }
+        else {
+            launcherLog.info({ container: names.worker }, "native worker already exists: attaching, not relaunching");
+        }
+        return this.handle(runId);
+    }
+    /** Waits for (and reports) the worker's exit; usable after a server restart. */
+    handle(runId) {
+        return this.handleByName(nativeIsolationNames(runId).worker);
+    }
+    handleByName(name) {
+        const exited = (async () => {
+            // `docker run` may not have created the container yet: wait for it to exist (bounded).
+            for (let attempt = 0; attempt < 50; attempt += 1) {
+                const waited = await this.docker.run(["container", "wait", name]);
+                if (waited.code === 0) {
+                    const code = Number(waited.stdout.trim());
+                    return Number.isInteger(code) ? code : null;
+                }
+                if (!/No such container/i.test(waited.stderr))
+                    return null;
+                await new Promise((resolve) => setTimeout(resolve, 200));
+            }
+            return null;
+        })();
+        return { exited, kill: () => void this.docker.run(["container", "kill", name]) };
+    }
+    async inspect(runId) {
+        return this.inspectByName(nativeIsolationNames(runId).worker);
+    }
+    async inspectByName(name) {
+        const result = await this.docker.run([
+            "container",
+            "inspect",
+            "--format",
+            "{{.State.Status}} {{.State.ExitCode}}",
+            name,
+        ]);
+        if (result.code !== 0)
+            return { state: "missing" };
+        const [status, exitCode] = result.stdout.trim().split(" ");
+        if (status === "running" || status === "created" || status === "restarting")
+            return { state: "running" };
+        return { state: "exited", exitCode: Number(exitCode) };
+    }
+    async kill(runId) {
+        await this.docker.run(["container", "kill", nativeIsolationNames(runId).worker]);
+    }
+    /** Removes the worker, its network, and the gateway's membership of it. Idempotent. */
+    async remove(runId) {
+        const names = nativeIsolationNames(runId);
+        await this.docker.run(["container", "rm", "--force", names.worker]);
+        await this.docker.run(buildGatewayDisconnectArgs(runId, this.options.gatewayContainer));
+        await this.docker.run(["network", "rm", names.network]);
+    }
+    /** Removes a worker found by the janitor, by its container name (its network shares the name's token). */
+    async removeByWorkerName(workerName) {
+        const token = /^wardby-native-([0-9a-f]{20})$/.exec(workerName)?.[1];
+        if (!token)
+            return;
+        await this.docker.run(["container", "rm", "--force", workerName]);
+        await this.docker.run(["network", "disconnect", "--force", `wardby-nnet-${token}`, this.options.gatewayContainer]);
+        await this.docker.run(["network", "rm", `wardby-nnet-${token}`]);
+    }
+    /** Every native worker container this engine holds, with its run-hash label (for the janitor). */
+    async listWorkers() {
+        const result = await this.docker.run([
+            "container",
+            "ls",
+            "--all",
+            "--filter",
+            `label=${NATIVE_WORKER_LABEL_FILTER}`,
+            "--format",
+            '{{.Names}} {{.Label "io.wardby.run-sha256"}}',
+        ]);
+        if (result.code !== 0)
+            return [];
+        return result.stdout
+            .split("\n")
+            .map((line) => line.trim().split(" "))
+            .filter(([name, runHash]) => name && runHash)
+            .map(([name, runHash]) => ({ name, runHash }));
+    }
+    // --- Warm pool workers (native sandbox phase 6) ---
+    warmSpecHash(waitMs) {
+        const { image, limits, gatewayContainer } = this.options;
+        return createHash("sha256")
+            .update(JSON.stringify({ launcher: "docker", image, limits, gatewayContainer, waitMs }))
+            .digest("hex");
+    }
+    /** Its network is internal from creation, so a running container is already isolated: claimable. */
+    async startWarm(token, waitMs) {
+        const names = nativeWarmIsolationNames(token);
+        try {
+            await this.ensureImage();
+            await this.must(buildNativeWarmNetworkCreateArgs(token), "create the warm worker network", /already exists/);
+            await this.must(gatewayConnectArgs(names.network, this.options.gatewayContainer), "join the gateway to the warm worker network", /already exists|is already connected/);
+            await this.must(buildNativeWarmWorkerRunArgs({ token, image: this.options.image, limits: this.options.limits, waitMs }), "start the warm worker");
+        }
+        catch (err) {
+            await this.removeWarm(token).catch(() => { });
+            throw err;
+        }
+    }
+    /** Still running, and attached to its own network and nothing else. */
+    async reattestWarm(token) {
+        const names = nativeWarmIsolationNames(token);
+        const result = await this.docker.run([
+            "container",
+            "inspect",
+            "--format",
+            "{{.State.Status}} {{json .NetworkSettings.Networks}}",
+            names.worker,
+        ]);
+        if (result.code !== 0)
+            return false;
+        const [status, ...rest] = result.stdout.trim().split(" ");
+        try {
+            const networks = Object.keys(JSON.parse(rest.join(" ")));
+            return status === "running" && networks.length === 1 && networks[0] === names.network;
+        }
+        catch {
+            return false;
+        }
+    }
+    async deliver(token, input) {
+        const result = await this.docker.run(["container", "exec", "-i", nativeWarmIsolationNames(token).worker, ...warmDeliveryCommand()], { input: `${JSON.stringify(input)}\n`, timeoutMs: 20_000 });
+        if (result.code !== 0) {
+            throw new Error(`${NATIVE_SANDBOX_WARM_DELIVERY_FAILED}: delivering the run's input exited ${result.code ?? "?"}.`);
+        }
+    }
+    warmHandle(token) {
+        return this.handleByName(nativeWarmIsolationNames(token).worker);
+    }
+    inspectWarm(token) {
+        return this.inspectByName(nativeWarmIsolationNames(token).worker);
+    }
+    async killWarm(token) {
+        await this.docker.run(["container", "kill", nativeWarmIsolationNames(token).worker]);
+    }
+    async removeWarm(token) {
+        const names = nativeWarmIsolationNames(token);
+        await this.docker.run(["container", "rm", "--force", names.worker]);
+        await this.docker.run(gatewayDisconnectArgs(names.network, this.options.gatewayContainer));
+        await this.docker.run(["network", "rm", names.network]);
+    }
+    async listWarm() {
+        const result = await this.docker.run([
+            "container",
+            "ls",
+            "--all",
+            "--filter",
+            `label=${NATIVE_WARM_LABEL_FILTER}`,
+            "--format",
+            `{{.Label "${NATIVE_WARM_TOKEN_LABEL}"}}`,
+        ]);
+        if (result.code !== 0)
+            return [];
+        return result.stdout
+            .split("\n")
+            .map((line) => line.trim())
+            .filter((token) => /^[0-9a-f]{20}$/.test(token));
+    }
+}

package/dist/native-worker/gateway-server.d.ts

@@ -0,0 +1,39 @@
+/**
+ * The native sandbox gateway over HTTP (native sandbox phase 3): stateless per
+ * request. Each request is authenticated by the hash of its bearer capability,
+ * rebuilt from its run's gateway session (the snapshot pinned at launch), the
+ * run row, and the call ledger, and served by the same trusted code the
+ * in-process path uses — so any control-plane replica can serve any call, and a
+ * replica restart is only a retry for the worker.
+ *
+ * Authority stays here: a model call reserves its worst case against the
+ * session budget before the provider is called and settles at the provider's
+ * reported usage; the worker's own usage numbers are never read.
+ */
+import { type NativeRunProviders, type RunnerDb } from "../core/runner.js";
+import type { LlmStreamEvent } from "../providers/llm/types.js";
+import { PrismaGatewayLedger } from "./ledger.js";
+/** Output tokens reserved for a model call that names no maxTokens (as the coding proxy does). */
+export declare const GATEWAY_DEFAULT_RESERVE_OUTPUT_TOKENS = 4096;
+/** The most output tokens one call may reserve for. */
+export declare const GATEWAY_MAX_RESERVE_OUTPUT_TOKENS = 128000;
+/** A replayable result is kept only up to this size; a larger one cannot be replayed on retry. */
+export declare const GATEWAY_MAX_REPLAY_BYTES: number;
+export interface GatewayServerDeps {
+    db: RunnerDb & ConstructorParameters<typeof PrismaGatewayLedger>[0];
+    providers: NativeRunProviders;
+    /** Delegation long-poll window; tests shorten it. */
+    pollWindowMs?: number;
+}
+/** What a request gets back: one JSON value, or a stream of model events. */
+export type GatewayResponse = {
+    kind: "json";
+    status: number;
+    body: unknown;
+} | {
+    kind: "stream";
+    status: 200;
+    events: AsyncIterable<LlmStreamEvent>;
+};
+/** Serves one gateway request. Never throws: every failure is a response. */
+export declare function serveGatewayRequest(deps: GatewayServerDeps, authorization: string | undefined, body: unknown): Promise<GatewayResponse>;

package/dist/native-worker/gateway-server.js

@@ -0,0 +1,315 @@
+/**
+ * The native sandbox gateway over HTTP (native sandbox phase 3): stateless per
+ * request. Each request is authenticated by the hash of its bearer capability,
+ * rebuilt from its run's gateway session (the snapshot pinned at launch), the
+ * run row, and the call ledger, and served by the same trusted code the
+ * in-process path uses — so any control-plane replica can serve any call, and a
+ * replica restart is only a retry for the worker.
+ *
+ * Authority stays here: a model call reserves its worst case against the
+ * session budget before the provider is called and settles at the provider's
+ * reported usage; the worker's own usage numbers are never read.
+ */
+import { logger } from "../core/logger.js";
+import { createNativeRunTools, durableDelegate, failNativeRun, finishNativeRun, nativeRunIntegrations, recordNativeRunProgress, runDrivability, sandboxCapabilityHash, } from "../core/runner.js";
+import { DELEGATE_TOOL_PREFIX } from "../core/tool-names.js";
+import { RoutingLlmProvider } from "../providers/llm/routing.js";
+import { isPrivilegedBridgeName } from "../sandbox/host-functions.js";
+import { PrismaGatewayLedger } from "./ledger.js";
+import { GatewayError, GatewayRequestSchema, parseParams, } from "./protocol.js";
+const serverLog = logger.child({ module: "native-gateway-server" });
+/** Output tokens reserved for a model call that names no maxTokens (as the coding proxy does). */
+export const GATEWAY_DEFAULT_RESERVE_OUTPUT_TOKENS = 4_096;
+/** The most output tokens one call may reserve for. */
+export const GATEWAY_MAX_RESERVE_OUTPUT_TOKENS = 128_000;
+/** A replayable result is kept only up to this size; a larger one cannot be replayed on retry. */
+export const GATEWAY_MAX_REPLAY_BYTES = 256 * 1024;
+const ok = (result) => ({
+    kind: "json",
+    status: 200,
+    body: { ok: true, result: result ?? null },
+});
+const fail = (status, code, message) => ({
+    kind: "json",
+    status,
+    body: { ok: false, error: { code, message } },
+});
+const STATUS_FOR = {
+    invalid_request: 400,
+    not_allowed: 403,
+    run_not_drivable: 409,
+    duplicate_call: 409,
+    budget_exhausted: 402,
+    not_ready: 503,
+    bridge_error: 200,
+    internal: 500,
+};
+/** Serves one gateway request. Never throws: every failure is a response. */
+export async function serveGatewayRequest(deps, authorization, body) {
+    try {
+        return await handle(deps, authorization, body);
+    }
+    catch (err) {
+        if (err instanceof GatewayError)
+            return fail(STATUS_FOR[err.code] ?? 500, err.code, err.message);
+        serverLog.error({ err }, "native gateway request failed");
+        return fail(500, "internal", "The gateway could not serve this call.");
+    }
+}
+async function handle(deps, authorization, body) {
+    const { db } = deps;
+    const ledger = new PrismaGatewayLedger(db);
+    const token = /^Bearer (\S+)$/.exec(authorization ?? "")?.[1];
+    if (!token)
+        return fail(401, "unauthorized", "A gateway capability is required.");
+    const session = await ledger.findSessionByCapabilityHash(sandboxCapabilityHash(token));
+    if (!session)
+        return fail(401, "unauthorized", "Unknown gateway capability.");
+    const parsed = GatewayRequestSchema.safeParse(body);
+    if (!parsed.success)
+        throw new GatewayError("invalid_request", parsed.error.issues[0]?.message ?? "invalid request");
+    const request = parsed.data;
+    if (request.runId !== session.runId)
+        throw new GatewayError("not_allowed", "request for another run");
+    let params;
+    try {
+        params = parseParams(request.method, request.params);
+    }
+    catch (err) {
+        throw new GatewayError("invalid_request", err instanceof Error ? err.message : String(err));
+    }
+    if (session.status !== "active" || Date.now() >= session.deadlineAt.getTime()) {
+        throw new GatewayError("run_not_drivable", "This run's gateway session has ended.");
+    }
+    // Nothing is served before the worker's network isolation is proven: until then its egress may
+    // not yet be limited to this gateway, so it must not receive tool code results, secrets, or data.
+    if (!session.networkReadyAt) {
+        throw new GatewayError("not_ready", "This run's network isolation is not proven yet; retry shortly.");
+    }
+    if ((await runDrivability(db, session.runId)) !== "drivable") {
+        throw new GatewayError("run_not_drivable", "This run is no longer running.");
+    }
+    const ctx = await rebuild(deps, session);
+    switch (request.method) {
+        case "llm.stream":
+            return {
+                kind: "stream",
+                status: 200,
+                events: streamModel(ctx, ledger, request.callId, params),
+            };
+        case "llm.countTokens": {
+            const p = params;
+            return ok(await ctx.llm.countTokens(p.model, p.messages, p.tools));
+        }
+        case "builtin.call":
+            return ok(await builtin(ctx, ledger, request.callId, params, deps.pollWindowMs));
+        case "host.call":
+            return ok(await hostCall(ctx, ledger, request.callId, params));
+        case "progress": {
+            const p = params;
+            const totals = await ledger.totals(session.id);
+            await recordNativeRunProgress(db, session.runId, {
+                turns: p.turns,
+                usage: { tokensIn: totals.tokensIn, tokensOut: totals.tokensOut, costUsd: totals.costUsd },
+            });
+            return ok(null);
+        }
+        case "text":
+            // Live text is for an attached observer; a background sandbox run has none.
+            return ok(null);
+        case "finish":
+            return ok(await finish(ctx, ledger, request.callId, params));
+    }
+}
+async function rebuild(deps, session) {
+    const { db, providers } = deps;
+    const snapshot = session.snapshot;
+    const existingRun = await db.run.findUniqueOrThrow({ where: { id: session.runId } });
+    const { reviewHosts, repoAccess, issueTrackers } = nativeRunIntegrations(providers, db);
+    const loaded = snapshot.loaded;
+    const llm = loaded.pricing && providers.llm instanceof RoutingLlmProvider
+        ? providers.llm.forRun(loaded.pricing.entry)
+        : providers.llm;
+    return {
+        session,
+        snapshot,
+        llm,
+        tools: createNativeRunTools({
+            runId: session.runId,
+            existingRun,
+            loaded,
+            providers,
+            db,
+            reviewHosts,
+            repoAccess,
+            issueTrackers,
+        }),
+        finishContext: { runId: session.runId, db, providers, reviewHosts, repoAccess, issueTrackers },
+        delegation: {
+            runId: session.runId,
+            existingRun,
+            loaded,
+            providers,
+            db,
+            issueTrackers,
+            pollWindowMs: deps.pollWindowMs,
+        },
+    };
+}
+async function* streamModel(ctx, ledger, callId, params) {
+    const req = params.request;
+    if (req.model !== ctx.snapshot.input.agent.model)
+        throw new GatewayError("not_allowed", "A run may only call its own model.");
+    const inputTokens = await ctx.llm.countTokens(req.model, req.messages, req.tools);
+    const outputTokens = Math.min(req.maxTokens ?? GATEWAY_DEFAULT_RESERVE_OUTPUT_TOKENS, GATEWAY_MAX_RESERVE_OUTPUT_TOKENS);
+    const reservationUsd = ctx.llm.priceUsd(req.model, { inputTokens, outputTokens });
+    const reserved = await ledger.reserve({ sessionId: ctx.session.id, callId, reservationUsd, now: new Date() });
+    if (reserved.outcome === "duplicate")
+        throw new GatewayError("duplicate_call", `callId "${callId}" was already used.`);
+    if (reserved.outcome === "inactive")
+        throw new GatewayError("run_not_drivable", "This run's gateway session has ended.");
+    if (reserved.outcome === "budget_exhausted") {
+        throw new GatewayError("budget_exhausted", "This run's budget cannot cover another model call.");
+    }
+    let settled = false;
+    let sawEvent = false;
+    let streamedText = "";
+    try {
+        for await (const event of ctx.llm.stream(req)) {
+            sawEvent = true;
+            if (event.type === "text")
+                streamedText += event.delta;
+            if (event.type === "done") {
+                settled = true;
+                await ledger.complete(ctx.session.id, callId, {
+                    inputTokens: event.usage.inputTokens,
+                    outputTokens: event.usage.outputTokens,
+                    cachedInputTokens: event.usage.cachedInputTokens ?? 0,
+                    cacheWriteTokens: event.usage.cacheWriteTokens ?? 0,
+                    costUsd: event.usage.costUsd,
+                });
+            }
+            yield event;
+        }
+    }
+    catch (err) {
+        if (!settled) {
+            // Refused before anything streamed: the provider billed nothing. Mid-stream: unknown, keep the hold.
+            if (sawEvent)
+                await ledger.markUncertain(ctx.session.id, callId);
+            else
+                await ledger.release(ctx.session.id, callId);
+            settled = true;
+        }
+        throw err;
+    }
+    finally {
+        if (!settled) {
+            // The worker stopped reading (its own budget cutoff) before usage arrived: bill an estimate.
+            const output = streamedText
+                ? await ctx.llm.countTokens(req.model, [{ role: "assistant", content: streamedText }]).catch(() => 0)
+                : 0;
+            await ledger.complete(ctx.session.id, callId, {
+                inputTokens,
+                outputTokens: output,
+                cachedInputTokens: 0,
+                cacheWriteTokens: 0,
+                costUsd: ctx.llm.priceUsd(req.model, { inputTokens, outputTokens: output }),
+            });
+        }
+    }
+}
+async function builtin(ctx, ledger, callId, params, pollWindowMs) {
+    if (!ctx.snapshot.input.builtinTools.includes(params.name)) {
+        throw new GatewayError("not_allowed", `"${params.name}" is not a built-in tool of this run.`);
+    }
+    if (params.name.startsWith(DELEGATE_TOOL_PREFIX)) {
+        const outcome = await durableDelegate({ ...ctx.delegation, pollWindowMs, ledger, sessionId: ctx.session.id }, callId, params.name, params.argsJson);
+        return "pending" in outcome ? { pending: true } : outcome.result;
+    }
+    const handler = ctx.tools.builtinHandler(params.name);
+    if (!handler)
+        throw new GatewayError("not_allowed", `"${params.name}" is not a built-in tool of this run.`);
+    return replayable(ledger, ctx.session.id, callId, "builtin.call", () => handler(params.argsJson));
+}
+async function hostCall(ctx, ledger, callId, params) {
+    if (!isPrivilegedBridgeName(params.bridge))
+        throw new GatewayError("not_allowed", "unknown bridge");
+    if (!ctx.snapshot.input.userTools[params.tool]) {
+        throw new GatewayError("not_allowed", `"${params.tool}" is not a user tool of this run.`);
+    }
+    const redact = params.bridge === "__bridge_console" ? await ctx.tools.readableSecretValues(params.tool) : undefined;
+    const host = ctx.tools.privilegedHostFor(params.tool, AbortSignal.timeout(30_000), redact);
+    if (!host)
+        throw new GatewayError("not_allowed", `"${params.tool}" is not a user tool of this run.`);
+    const bridge = host[params.bridge];
+    const run = async () => {
+        try {
+            return await bridge(params.argsJson);
+        }
+        catch (err) {
+            // What the tool sees, exactly as in-process (bridge.ts truncates the same way).
+            throw new GatewayError("bridge_error", err instanceof Error ? err.message.slice(0, 1024) : "Host function failed.");
+        }
+    };
+    // Console lines are not worth a ledger row, and a secret value must never be stored in one
+    // (a secrets.get is a side-effect-free read, so a retried callId simply reads it again).
+    if (params.bridge === "__bridge_console" || params.bridge === "__bridge_secretsGet")
+        return run();
+    return replayable(ledger, ctx.session.id, callId, "host.call", run);
+}
+/** Runs a one-shot call once per callId; a retry replays its recorded result. */
+async function replayable(ledger, sessionId, callId, method, run) {
+    const claimed = await ledger.claim(sessionId, callId, method);
+    if (claimed.outcome === "done") {
+        const stored = claimed.result;
+        if (stored.tooLarge)
+            throw new GatewayError("internal", "This call's result was too large to replay.");
+        if (stored.error)
+            throw new GatewayError(stored.error.code, stored.error.message);
+        return stored.value;
+    }
+    if (claimed.outcome === "in_flight")
+        throw new GatewayError("duplicate_call", `callId "${callId}" is still in flight.`);
+    try {
+        const value = await run();
+        const record = { value: value ?? null };
+        await ledger.recordResult(sessionId, callId, JSON.stringify(record).length <= GATEWAY_MAX_REPLAY_BYTES ? record : { tooLarge: true });
+        return value;
+    }
+    catch (err) {
+        if (err instanceof GatewayError && err.code === "bridge_error") {
+            await ledger.recordResult(sessionId, callId, { error: { code: err.code, message: err.message } });
+        }
+        throw err;
+    }
+}
+async function finish(ctx, ledger, callId, params) {
+    const claimed = await ledger.claim(ctx.session.id, callId, "finish");
+    if (claimed.outcome !== "claimed")
+        return null;
+    const totals = await ledger.totals(ctx.session.id);
+    try {
+        await finishNativeRun(ctx.finishContext, ctx.snapshot.input.agent.model, {
+            status: params.status,
+            finalText: params.finalText,
+            turns: params.turns,
+            ...(params.error !== undefined ? { error: params.error } : {}),
+            usage: {
+                tokensIn: totals.tokensIn,
+                tokensOut: totals.tokensOut,
+                costUsd: totals.costUsd,
+                cachedInputTokens: totals.cachedInputTokens,
+                cacheWriteTokens: totals.cacheWriteTokens,
+            },
+        });
+    }
+    catch (err) {
+        await failNativeRun(ctx.finishContext, err);
+    }
+    finally {
+        await ledger.recordResult(ctx.session.id, callId, { value: null });
+        await ledger.endSession(ctx.session.id, "finished");
+    }
+    return null;
+}