@wardby/cli 0.6.0 → 0.8.0

package/dist/native-worker/kubernetes-launcher.js

@@ -15,15 +15,24 @@ import { DISABLED_POWER, wakeTimeoutMs } from "../core/power.js";
 import { KubernetesAlreadyExistsError } from "../providers/jobs/kubernetes-api.js";
 import { canonicalResources } from "../providers/jobs/kubernetes-isolation.js";
 import { NATIVE_GATEWAY_PORT } from "./docker-isolation.js";
+import { WORKER_TAIL_TIMEOUT_MS, withTimeout, workerTailFields } from "./docker-launcher.js";
 import { createHash } from "node:crypto";
 import { Readable } from "node:stream";
 import { buildNativeInputSecret, buildNativeRunNetworkPolicy, buildNativeRunPod, buildNativeWarmNetworkPolicy, buildNativeWarmPod, NATIVE_PROBE, NATIVE_RUN_SELECTOR, NATIVE_RUN_SHA_LABEL, NATIVE_WARM_SELECTOR, NATIVE_WARM_TOKEN_LABEL, NATIVE_WORKER_CONTAINER, nativeEnforcementProbe, nativeKubernetesNames, nativeWarmWorkerName, } from "./kubernetes-isolation.js";
+import { NATIVE_SANDBOX_LAUNCH_ABANDONED } from "./launch.js";
 import { warmDeliveryCommand } from "./warm-delivery.js";
 import { NATIVE_SANDBOX_WARM_DELIVERY_FAILED } from "./warm-pool.js";
 const k8sLog = logger.child({ module: "native-kubernetes-launcher" });
 export const NATIVE_SANDBOX_NETWORK_UNENFORCED = "native_sandbox_network_unenforced";
 export const NATIVE_SANDBOX_ISOLATION_MISMATCH = "native_sandbox_isolation_mismatch";
 export const NATIVE_SANDBOX_CAPACITY = "native_sandbox_capacity";
+/**
+ * A failed worker's log is read this far back before its tail is redacted and cut. The API server
+ * applies `limitBytes` from the start of the selected lines, so it would cut the end (the worker's
+ * final failure line): it is only a memory cap, far above what the line bound selects in practice.
+ */
+const WORKER_LOG_TAIL_LINES = 100;
+const WORKER_LOG_TAIL_BYTES = 1024 * 1024;
 /** The API server's refusal of a pod the namespace's ResourceQuota has no room for (403 "exceeded quota"). */
 export function isQuotaRejection(err) {
     if (typeof err !== "object" || err === null)
@@ -96,7 +105,7 @@ export class KubernetesNativeWorkerLauncher {
                 throw err;
         }
     }
-    async launch(input) {
+    async launch(input, launchOptions = {}) {
         const runId = input.runId;
         // Low-power mode: the gateway may be scaled to zero; wake it before anything is created. Guarded
         // so mode off adds no await.
@@ -106,6 +115,10 @@ export class KubernetesNativeWorkerLauncher {
                 runId,
             });
         }
+        // A wake can take minutes: a run stopped meanwhile gets no pod.
+        if (launchOptions.stillWanted && !(await launchOptions.stillWanted())) {
+            throw new Error(`${NATIVE_SANDBOX_LAUNCH_ABANDONED}: the run ended while its worker was launching; no worker was started.`);
+        }
         const names = nativeKubernetesNames(runId);
         if ((await this.inspect(runId)).state === "missing") {
             const pod = buildNativeRunPod({
@@ -124,12 +137,37 @@ export class KubernetesNativeWorkerLauncher {
                 beforePod: () => this.createIfMissing(() => this.api.createSecret(this.ns, buildNativeInputSecret(input, this.ns))),
             });
             await this.options.onNetworkProven(runId);
+            // Read before the exit is reported, so the pod (and its log) still exists.
+            return this.handleByName(names.pod, (code) => this.logFailedExit(names.pod, code, input, launchOptions));
         }
         else {
             k8sLog.info({ pod: names.pod }, "native worker pod already exists: attaching, not relaunching");
         }
         return this.handle(runId);
     }
+    /** A non-zero exit, with the tail of the worker's log (bounded and redacted: workerTailFields), as the Docker launcher logs it. */
+    async logFailedExit(podName, code, input, launchOptions) {
+        if (code === 0)
+            return;
+        // One bound for the whole diagnostic, so a hung read never holds back the exit (and the run's end).
+        const by = Date.now() + (this.options.diagnosticTimeoutMs ?? WORKER_TAIL_TIMEOUT_MS);
+        let log;
+        try {
+            log = await withTimeout(this.api.readLogTail(this.ns, podName, NATIVE_WORKER_CONTAINER, WORKER_LOG_TAIL_LINES, WORKER_LOG_TAIL_BYTES), by - Date.now());
+        }
+        catch {
+            // The pod may already be gone, the read may be refused (no pods/log), or it timed out.
+            k8sLog.warn({ pod: podName, code, stderrTailOmitted: "log_read_failed" }, "native worker exited non-zero");
+            return;
+        }
+        // The read may always begin mid-text, even with fewer lines than the bound: the kubelet splits
+        // a line over 16 KiB into partial records (each counted as a line), and log rotation drops the
+        // oldest text, so the start is always masked as cut. The byte cap (counted from the start of
+        // the selected lines) cuts the end only when it is met.
+        const end = Buffer.byteLength(log) >= WORKER_LOG_TAIL_BYTES - 3;
+        const tail = await workerTailFields(log, input, launchOptions.secretValues, { start: true, end }, by - Date.now());
+        k8sLog.warn({ pod: podName, code, ...tail }, "native worker exited non-zero");
+    }
     /**
      * Policy first, so the pod never exists without it; then the pod; then attest and prove it. A
      * quota rejection becomes native_sandbox_capacity. Any failure removes what was made.
@@ -225,12 +263,14 @@ export class KubernetesNativeWorkerLauncher {
     handle(runId) {
         return this.handleByName(nativeKubernetesNames(runId).pod);
     }
-    handleByName(podName) {
+    handleByName(podName, onExit) {
         const exited = (async () => {
             for (;;) {
                 const state = await this.inspectByName(podName);
-                if (state.state === "exited")
+                if (state.state === "exited") {
+                    await onExit?.(state.exitCode).catch(() => { });
                     return state.exitCode;
+                }
                 if (state.state === "missing")
                     return null;
                 await this.sleep(this.options.pollMs ?? 1_000);

package/dist/native-worker/launch.d.ts

@@ -16,9 +16,25 @@ export interface WorkerHandle {
     exited: Promise<number | null>;
     kill(): void;
 }
+export interface WorkerLaunchOptions {
+    /**
+     * Asked after any slow wait (a low-power wake) and before a worker is created: false means the
+     * run ended meanwhile, and the launch stops with NATIVE_SANDBOX_LAUNCH_ABANDONED, creating nothing.
+     */
+    stillWanted?: () => Promise<boolean>;
+    /**
+     * Every secret value the worker may be handed (each user tool's readable or brokered secrets),
+     * for masking a failed worker's output tail before the launcher logs it; the run's capability is
+     * masked from the input itself. Rejects when any read fails. Absent or rejected: no tail is
+     * logged (fail closed).
+     */
+    secretValues?: () => Promise<readonly string[]>;
+}
+/** The run ended while its worker was launching, so none was created. */
+export declare const NATIVE_SANDBOX_LAUNCH_ABANDONED = "native_sandbox_launch_abandoned";
 /** Starts a worker for an HTTP-gateway input and returns at once. */
 export interface DetachedWorkerLauncher {
-    launch(input: WorkerInput): Promise<WorkerHandle>;
+    launch(input: WorkerInput, options?: WorkerLaunchOptions): Promise<WorkerHandle>;
     /**
      * False when the worker's network isolation only takes effect after it starts (a Kubernetes
      * NetworkPolicy): its session is then created not ready, and the launcher marks it ready once
@@ -42,4 +58,5 @@ export declare function startSandboxRun(options: {
     db: RunnerDb & ConstructorParameters<typeof PrismaGatewayLedger>[0];
     gatewayUrl: string;
     launcher: DetachedWorkerLauncher;
+    stillWanted?: () => Promise<boolean>;
 }): Promise<StartSandboxRunOutcome>;

package/dist/native-worker/launch.js

@@ -9,6 +9,8 @@
 import { spawn } from "node:child_process";
 import { createSandboxSession } from "../core/runner.js";
 import { PrismaGatewayLedger } from "./ledger.js";
+/** The run ended while its worker was launching, so none was created. */
+export const NATIVE_SANDBOX_LAUNCH_ABANDONED = "native_sandbox_launch_abandoned";
 /** The worker as a local child process with an empty environment (by default), its input on stdin. */
 export function processWorkerLauncher(spec) {
     return {
@@ -39,6 +41,9 @@ export async function startSandboxRun(options) {
     });
     if (session.kind === "ended")
         return session;
-    const handle = await options.launcher.launch(session.input);
+    const handle = await options.launcher.launch(session.input, {
+        ...(options.stillWanted ? { stillWanted: options.stillWanted } : {}),
+        secretValues: session.secretValues,
+    });
     return { kind: "launched", sessionId: session.sessionId, handle };
 }

package/dist/native-worker/ledger.d.ts

@@ -7,6 +7,7 @@
  * replica can serve any call, and a restart loses nothing.
  */
 import type { Prisma, PrismaClient } from "#prisma";
+import { ReviewReadLedger } from "../core/review-reads.js";
 export type GatewayLedgerDb = Pick<PrismaClient, "$transaction" | "$queryRaw" | "nativeGatewaySession" | "nativeGatewayCall">;
 export interface GatewaySessionRecord {
     id: string;
@@ -16,6 +17,8 @@ export interface GatewaySessionRecord {
     budgetUsd: number;
     snapshot: unknown;
     networkReadyAt: Date | null;
+    /** The run's saved review reads (ReviewReadsState), or null before it reads a pull request. */
+    reviewReads?: unknown;
 }
 export interface CreateGatewaySessionInput {
     runId: string;
@@ -77,6 +80,11 @@ export declare class PrismaGatewayLedger {
     markNetworkReady(sessionId: string, at?: Date): Promise<void>;
     /** markNetworkReady by the session's run (a launcher knows the run, not the session). */
     markNetworkReadyForRun(runId: string, at?: Date): Promise<void>;
+    /**
+     * Adds one request's review reads to the session's saved ones, under the session's row lock:
+     * reads only add, so parallel tool calls of one run each keep what they read.
+     */
+    mergeReviewReads(sessionId: string, reads: ReviewReadLedger): Promise<void>;
     endSession(sessionId: string, status: "finished" | "cancelled"): Promise<void>;
     /** Reserves `reservationUsd` for a model call, or says why not. Serialized per session. */
     reserve(input: {
@@ -89,8 +97,16 @@ export declare class PrismaGatewayLedger {
     complete(sessionId: string, callId: string, usage: SettledUsage): Promise<void>;
     /** Frees a reservation the provider never billed (it refused the request). */
     release(sessionId: string, callId: string): Promise<void>;
-    /** The outcome is unknown (the connection broke mid-call): the reservation stays held. */
-    markUncertain(sessionId: string, callId: string): Promise<void>;
+    /**
+     * The outcome is unknown (the connection broke mid-call). Without an estimate the
+     * reservation stays held; with one, the call counts at the estimate instead.
+     */
+    markUncertain(sessionId: string, callId: string, estimate?: SettledUsage): Promise<void>;
+    /**
+     * Median output tokens of the session's completed model calls (all on the run's one model), or
+     * undefined before the first one: the output of a call cut with nothing streamed.
+     */
+    medianOutputTokens(sessionId: string): Promise<number | undefined>;
     totals(sessionId: string): Promise<GatewayTotals>;
     /** Claims a one-shot call by callId: the first claimant runs it, a repeat waits or replays. */
     claim(sessionId: string, callId: string, method: string): Promise<ClaimOutcome>;
@@ -99,11 +115,15 @@ export declare class PrismaGatewayLedger {
     setDelegation(sessionId: string, callId: string, state: {
         status: "pending" | "waiting_budget";
         childAgentId: string;
+        childRepository?: string;
         childRunId?: string;
     }): Promise<void>;
     delegation(sessionId: string, callId: string): Promise<DelegationState | null>;
-    /** The sub-agents other delegations of this run are waiting for budget to start. */
-    waitingChildAgentIds(sessionId: string, exceptCallId: string): Promise<string[]>;
+    /** The sub-agents (and repositories, null = the child's default) other delegations of this run wait on budget to start. */
+    waitingDelegations(sessionId: string, exceptCallId: string): Promise<{
+        childAgentId: string;
+        childRepository: string | null;
+    }[]>;
     /** Delegations of this session whose child run is started and not yet reported. */
     openDelegationChildren(sessionId: string): Promise<string[]>;
 }

package/dist/native-worker/ledger.js

@@ -6,6 +6,7 @@
  * called, and is settled at its actual cost after — so any control-plane
  * replica can serve any call, and a restart loses nothing.
  */
+import { ReviewReadLedger } from "../core/review-reads.js";
 const LEDGER_TX = { maxWait: 5_000, timeout: 20_000 };
 function toSession(row) {
     return { ...row, budgetUsd: Number(row.budgetUsd) };
@@ -37,6 +38,24 @@ export class PrismaGatewayLedger {
             data: { networkReadyAt: at },
         });
     }
+    /**
+     * Adds one request's review reads to the session's saved ones, under the session's row lock:
+     * reads only add, so parallel tool calls of one run each keep what they read.
+     */
+    async mergeReviewReads(sessionId, reads) {
+        await this.db.$transaction(async (tx) => {
+            const locked = await tx.$queryRaw `
+        SELECT "reviewReads" FROM "NativeGatewaySession" WHERE "id" = ${sessionId} FOR UPDATE`;
+            if (!locked[0])
+                return;
+            const saved = ReviewReadLedger.fromJSON(locked[0].reviewReads);
+            saved.merge(reads);
+            await tx.nativeGatewaySession.update({
+                where: { id: sessionId },
+                data: { reviewReads: saved.toJSON() },
+            });
+        }, LEDGER_TX);
+    }
     async endSession(sessionId, status) {
         await this.db.nativeGatewaySession.updateMany({ where: { id: sessionId, status: "active" }, data: { status } });
     }
@@ -86,6 +105,7 @@ export class PrismaGatewayLedger {
                 outputTokens: usage.outputTokens,
                 cachedInputTokens: usage.cachedInputTokens,
                 cacheWriteTokens: usage.cacheWriteTokens,
+                costEstimated: false,
                 completedAt: new Date(),
             },
         });
@@ -97,16 +117,48 @@ export class PrismaGatewayLedger {
             data: { status: "released", completedAt: new Date() },
         });
     }
-    /** The outcome is unknown (the connection broke mid-call): the reservation stays held. */
-    async markUncertain(sessionId, callId) {
+    /**
+     * The outcome is unknown (the connection broke mid-call). Without an estimate the
+     * reservation stays held; with one, the call counts at the estimate instead.
+     */
+    async markUncertain(sessionId, callId, estimate) {
         await this.db.nativeGatewayCall.updateMany({
             where: { sessionId, callId, status: "reserved" },
-            data: { status: "uncertain" },
+            data: {
+                status: "uncertain",
+                ...(estimate
+                    ? {
+                        actualCostUsd: estimate.costUsd,
+                        inputTokens: estimate.inputTokens,
+                        outputTokens: estimate.outputTokens,
+                        cachedInputTokens: estimate.cachedInputTokens,
+                        cacheWriteTokens: estimate.cacheWriteTokens,
+                        costEstimated: true,
+                    }
+                    : {}),
+            },
         });
     }
+    /**
+     * Median output tokens of the session's completed model calls (all on the run's one model), or
+     * undefined before the first one: the output of a call cut with nothing streamed.
+     */
+    async medianOutputTokens(sessionId) {
+        const rows = await this.db.$queryRaw `
+      SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY "outputTokens") AS "median"
+      FROM "NativeGatewayCall"
+      WHERE "sessionId" = ${sessionId} AND "method" = 'llm.stream' AND "status" = 'completed'
+        AND "outputTokens" IS NOT NULL`;
+        const median = rows[0]?.median ?? null;
+        return median === null ? undefined : Number(median);
+    }
     async totals(sessionId) {
         const completed = await this.db.nativeGatewayCall.aggregate({
-            where: { sessionId, status: "completed", method: "llm.stream" },
+            where: {
+                sessionId,
+                method: "llm.stream",
+                OR: [{ status: "completed" }, { status: "uncertain", costEstimated: true }],
+            },
             _sum: {
                 inputTokens: true,
                 outputTokens: true,
@@ -150,6 +202,7 @@ export class PrismaGatewayLedger {
             data: {
                 status: state.status,
                 childAgentId: state.childAgentId,
+                ...(state.childRepository ? { childRepository: state.childRepository } : {}),
                 ...(state.childRunId ? { childRunId: state.childRunId } : {}),
             },
         });
@@ -161,13 +214,13 @@ export class PrismaGatewayLedger {
         const status = row.status === "waiting_budget" ? "waiting_budget" : row.status === "completed" ? "completed" : "pending";
         return { status, childRunId: row.childRunId, childAgentId: row.childAgentId, createdAt: row.createdAt };
     }
-    /** The sub-agents other delegations of this run are waiting for budget to start. */
-    async waitingChildAgentIds(sessionId, exceptCallId) {
+    /** The sub-agents (and repositories, null = the child's default) other delegations of this run wait on budget to start. */
+    async waitingDelegations(sessionId, exceptCallId) {
         const rows = await this.db.nativeGatewayCall.findMany({
             where: { sessionId, status: "waiting_budget", callId: { not: exceptCallId }, childAgentId: { not: null } },
-            select: { childAgentId: true },
+            select: { childAgentId: true, childRepository: true },
         });
-        return rows.map((row) => row.childAgentId);
+        return rows.map((row) => ({ childAgentId: row.childAgentId, childRepository: row.childRepository }));
     }
     /** Delegations of this session whose child run is started and not yet reported. */
     async openDelegationChildren(sessionId) {
@@ -183,6 +236,7 @@ async function heldUsd(db, sessionId) {
     SELECT COALESCE(SUM(
       CASE
         WHEN "status" = 'completed' THEN COALESCE("actualCostUsd", 0)
+        WHEN "status" = 'uncertain' AND "costEstimated" THEN COALESCE("actualCostUsd", 0)
         WHEN "status" IN ('reserved', 'uncertain') THEN COALESCE("reservationUsd", 0)
         ELSE 0
       END

package/dist/native-worker/loopback.js

@@ -5,14 +5,9 @@
  * shape a real transport has. For tests and local development; never a
  * sandbox (the worker runs in the server's own process).
  */
-import { GatewayError, parseMessage, WorkerInputSchema } from "./protocol.js";
+import { parseMessage, toGatewayError, WorkerInputSchema } from "./protocol.js";
 import { runNativeWorker } from "./worker.js";
 const wire = (value) => (value === undefined ? value : JSON.parse(JSON.stringify(value)));
-function asGatewayError(err) {
-    if (err instanceof GatewayError)
-        return new GatewayError(err.code, err.message);
-    return new GatewayError("internal", err instanceof Error ? err.message : String(err));
-}
 export function createLoopbackTransport(gateway) {
     return {
         async call(request) {
@@ -20,7 +15,7 @@ export function createLoopbackTransport(gateway) {
                 return wire(await gateway.call(wire(request)));
             }
             catch (err) {
-                throw asGatewayError(err);
+                throw toGatewayError(err);
             }
         },
         async *stream(request) {
@@ -29,7 +24,7 @@ export function createLoopbackTransport(gateway) {
                     yield wire(event);
             }
             catch (err) {
-                throw asGatewayError(err);
+                throw toGatewayError(err);
             }
         },
     };

package/dist/native-worker/main.d.ts

@@ -2,6 +2,7 @@
  * Native sandbox worker process entry. Reads its WorkerInput as the first line
  * on stdin, runs the run's engine against the gateway on stdin/stdout
  * (stdio.ts), and exits 0 once it has sent its result. stdout carries only
- * protocol frames; diagnostics (and the shared logger) go to stderr.
+ * protocol frames; diagnostics (and the shared logger) go to stderr, without any secret value the
+ * worker received (stderr-redaction.ts).
  */
 export {};

package/dist/native-worker/main.js

@@ -2,12 +2,14 @@
  * Native sandbox worker process entry. Reads its WorkerInput as the first line
  * on stdin, runs the run's engine against the gateway on stdin/stdout
  * (stdio.ts), and exits 0 once it has sent its result. stdout carries only
- * protocol frames; diagnostics (and the shared logger) go to stderr.
+ * protocol frames; diagnostics (and the shared logger) go to stderr, without any secret value the
+ * worker received (stderr-redaction.ts).
  */
 import { readFile, stat } from "node:fs/promises";
 import { parseMessage, WorkerInputSchema } from "./protocol.js";
 import { createHttpTransport } from "./http-transport.js";
 import { createStdioTransport } from "./stdio.js";
+import { flushStderrRedaction, installStderrRedaction, rememberSecretValue } from "./stderr-redaction.js";
 import { WARM_WORKER_UNCLAIMED_EXIT } from "./warm-delivery.js";
 import { runNativeWorker } from "./worker.js";
 function firstLine() {
@@ -57,6 +59,8 @@ async function readInput() {
 }
 async function main() {
     const input = parseMessage(await readInput(), WorkerInputSchema);
+    if (input.gateway)
+        rememberSecretValue(input.gateway.capability);
     // The gateway answers only after the worker asks, so choosing the transport after the input
     // line misses nothing.
     const transport = input.gateway
@@ -64,11 +68,18 @@ async function main() {
         : createStdioTransport(process.stdin, process.stdout);
     await runNativeWorker(input, transport);
 }
-main().then(() => process.exit(0), (err) => {
+const fail = (err) => {
+    // Anything still buffered for its line's end goes out first, so the failure line stands alone.
+    flushStderrRedaction();
     if (err instanceof UnclaimedError) {
         process.stderr.write(`native sandbox worker: ${err.message}\n`);
         process.exit(WARM_WORKER_UNCLAIMED_EXIT);
     }
     process.stderr.write(`native sandbox worker failed: ${err instanceof Error ? err.message : String(err)}\n`);
     process.exit(1);
-});
+};
+installStderrRedaction();
+// Node's own report of an uncaught error would bypass the redaction: fail the same way instead.
+process.on("uncaughtException", fail);
+process.on("unhandledRejection", fail);
+main().then(() => process.exit(0), fail);