@flow-engineer/flow 0.0.1 → 0.2.1

package/dist/testing/src/client.js

@@ -0,0 +1,956 @@
+// The developer's side of the tunnel: connects a local agent to the test cloud. The agent keeps its
+// own model and key; its tools are replaced by calls the cloud simulates, its replies go to the
+// cloud's platform simulators, and the person's messages arrive over one stream.
+import { createHash, randomUUID } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { Agent as HttpAgent, request as httpRequest } from "node:http";
+import { Agent as HttpsAgent, request as httpsRequest } from "node:https";
+import { basename, relative, resolve } from "node:path";
+import { toSpec } from "../../src/index.js";
+import { PROFILE_TOOL, profileTool } from "../../src/conversation.js";
+import { STAND_IN } from "../../src/tools.js";
+import { mimeOf, parseScenario, parseToolFile, suiteHash } from "./suite.js";
+/** The session opened, then the cloud ended it with a `failed` command. `code` is the reason's leading
+ * code (`capacity: ...` gives capacity), when it starts with one. */
+export class CloudEnded extends Error {
+    code;
+    constructor(error) {
+        super(`the cloud stopped the session: ${error}`);
+        this.name = "CloudEnded";
+        this.code = /^([a-z_]+):/.exec(error)?.[1];
+    }
+}
+/** The tunnel, the stream, or the connection failed after the session opened. `runs` are the ones already received. The CLI prints them and exits 2. */
+export class SessionStopped extends Error {
+    runs;
+    dashboard;
+    constructor(reason, runs, dashboard) {
+        super(reason);
+        this.name = "SessionStopped";
+        this.runs = runs;
+        this.dashboard = dashboard;
+    }
+}
+/** The cloud said session_gone, and its open said it can re-open: runInCloud plays the unfinished runs once more. */
+class SessionGone extends SessionStopped {
+    sessionId;
+    constructor(reason, runs, dashboard, sessionId) {
+        super(reason, runs, dashboard);
+        this.sessionId = sessionId;
+    }
+}
+/** A POST that sat until its own deadline. It fails that one call. Only a cloud that stores keys sends it again. The session abort is not one of these. */
+export class TimeoutError extends Error {
+    constructor() {
+        super("timed out");
+        this.name = "TimeoutError";
+    }
+}
+/** The cloud answered a tool call `ok: false`. Its message is the cloud's code: a tool file's own error,
+ * or one of {@link CLOUD_TOOL_ERRORS} when the cloud could not play the call. */
+class ToolFailed extends Error {
+}
+/** A tool call's codes for the cloud's own failure, not the tool's: its simulator failed, or the run had already ended. */
+const CLOUD_TOOL_ERRORS = new Set(["simulator_failed", "run_finished"]);
+/** The stream stopped carrying bytes: the idle watch fired, or the cloud closed it before finished. */
+export class StreamLost extends Error {
+    constructor(message, options) {
+        super(message, options);
+        this.name = "StreamLost";
+    }
+}
+const POST_TIMEOUT_MS = 30_000;
+const STREAM_IDLE_MS = 45_000;
+const RESUME_WINDOW_MS = 120_000;
+const STREAM_REFRESH_MS = 50 * 60_000;
+/** How long one call keeps trying after its first failure. The first try's own wait does not count, so a timed-out try is still sent again. */
+export const RETRY_BUDGET_MS = 30_000;
+/** The cloud abandons one model attempt at 90s and tries a timeout once more, then this POST has its 30s. */
+export const CLOUD_MODEL_BUDGET_MS = 90_000 * 2 + POST_TIMEOUT_MS;
+/** How long a tool call may take: the longest latency its suite file declares, plus {@link CLOUD_MODEL_BUDGET_MS} (210s).
+ * A tool with no file gets the simulator's own cap (60s) plus that 210s.
+ *
+ * The cloud asks its model before the declared latency, and the model's own time counts toward it.
+ * 210s is two 90s attempts plus the 30s POST budget, so a stuck model can finish before this POST gives up.
+ * When it runs out, the agent sees an error while the cloud may still record the call in the world log once the model returns. */
+export function toolTimeoutMs(files, tool, budgetMs = CLOUD_MODEL_BUDGET_MS) {
+    let declared = 0;
+    let known = false;
+    for (const file of files) {
+        if (!file.path.startsWith("tools/") || !file.path.endsWith(".yaml"))
+            continue;
+        try {
+            const parsed = parseToolFile(file);
+            if (parsed.tool !== tool)
+                continue;
+            known = true;
+            for (const response of parsed.responses)
+                declared = Math.max(declared, response.latency_ms);
+        }
+        catch { /* the cloud refuses the file; the call still waits the simulator's cap */ }
+    }
+    return budgetMs + (known ? declared : 60_000);
+}
+/** The base budget plus 1s per 100KB, so a 5MB event batch or file upload is not cut off at 30s. */
+export function postBudgetMs(baseMs, bytes) {
+    return baseMs + Math.floor(Math.max(0, bytes) / 100_000) * 1_000;
+}
+/** The open's headers. A cloud off this machine needs the CI key and the synthetic attestation;
+ * a local one needs neither, and gets them when they are given. */
+export function cloudHeaders(access) {
+    const local = isLoopback(access.cloud);
+    if (!local && !access.ciKey)
+        throw new Error("A hosted cloud needs the agent's CI key: set FLOW_FRAMEWORK_CLOUD_KEY.");
+    if ((!local || access.ciKey) && !access.synthetic) {
+        throw new Error("A hosted cloud runs synthetic tests only: pass --synthetic to attest that your test files hold no real customer or member data.");
+    }
+    if (access.environment && access.environment !== "staging" && access.environment !== "production") {
+        throw new Error("--environment must be staging or production.");
+    }
+    const headers = {};
+    if (access.ciKey)
+        headers.authorization = `Bearer ${access.ciKey}`;
+    if (access.synthetic)
+        headers["x-chinwag-synthetic"] = "true";
+    const optional = [
+        ["x-chinwag-agent-version", "--agent-version", access.agentVersion],
+        ["x-chinwag-ref", "--ref", access.ref],
+        ["x-chinwag-environment", "--environment", access.environment],
+    ];
+    for (const [header, flag, value] of optional) {
+        if (!value)
+            continue;
+        if (!/^[\x20-\x7e]{1,100}$/.test(value))
+            throw new Error(`${flag} must be 1 to 100 printable ASCII characters.`);
+        headers[header] = value;
+    }
+    return headers;
+}
+/** Where the session is played: Opened.tunnel when the cloud answers one, else the open's own address. */
+export function tunnelBase(cloud, opened) {
+    if (!opened.tunnel)
+        return cloud.replace(/\/$/, "");
+    let url = null;
+    try {
+        url = new URL(opened.tunnel);
+    }
+    catch { }
+    if (!url || (url.protocol !== "https:" && url.protocol !== "http:") || url.username || url.password) {
+        throw new Error(`the cloud answered a tunnel that is not an http(s) URL: ${opened.tunnel}`);
+    }
+    return opened.tunnel.replace(/\/$/, "");
+}
+function isLoopback(cloud) {
+    const host = new URL(cloud).hostname;
+    return host === "localhost" || host === "[::1]" || /^127(\.\d{1,3}){3}$/.test(host);
+}
+/** Plays the suite in the cloud. If the cloud loses the session (session_gone, on a cloud whose open said
+ * reopen), one new session plays only the runs that have no result yet, and the two reports are merged. */
+export async function runInCloud(options) {
+    try {
+        return await playSession(options);
+    }
+    catch (error) {
+        if (!(error instanceof SessionGone))
+            throw error;
+        const log = options.log ?? ((line) => console.log(line));
+        const unfinished = unfinishedRuns(options.files, options.runs ?? 1, error.runs);
+        if (!unfinished.length)
+            throw new SessionStopped(error.message, error.runs, error.dashboard);
+        const which = unfinished.map((r) => `${r.scenario} on ${r.channel} #${r.run}`).join(", ");
+        log(`The cloud lost session ${error.sessionId} (${error.message}). Opening a new session to play the ${unfinished.length} run${unfinished.length === 1 ? "" : "s"} with no result, at the cloud's cost again: ${which}`);
+        let second;
+        try {
+            // With nothing finished, the whole suite plays again without `only`: the cloud takes `only` just
+            // for files it already stored, and the first session may have been lost before its uploads.
+            second = await playSession(options, error.runs.length ? unfinished : undefined);
+        }
+        catch (again) {
+            // Never a third session: a second loss stops with what both sessions finished.
+            if (again instanceof SessionStopped)
+                throw new SessionStopped(again.message, [...error.runs, ...again.runs], again.dashboard);
+            throw new SessionStopped(`the cloud lost the session, and a new one did not open: ${again instanceof Error ? again.message : String(again)}`, error.runs, error.dashboard);
+        }
+        return { ...second, runs: [...error.runs, ...second.runs], lost: { sessionId: error.sessionId, dashboard: error.dashboard } };
+    }
+}
+/** The runs the suite plays (each scenario on each of its channels, `runs` times) that have no summary in `done`. */
+function unfinishedRuns(files, runs, done) {
+    const have = new Set(done.map((r) => `${r.scenario}\n${r.channel}\n${r.run}`));
+    const out = [];
+    for (const file of files) {
+        if (!file.path.startsWith("scenarios/"))
+            continue;
+        let scenario;
+        try {
+            scenario = parseScenario(file);
+        }
+        catch {
+            continue; /* the cloud would have refused the session */
+        }
+        for (const channel of scenario.channels) {
+            for (let run = 1; run <= runs; run++)
+                if (!have.has(`${scenario.id}\n${channel}\n${run}`))
+                    out.push({ scenario: scenario.id, channel, run });
+        }
+    }
+    return out;
+}
+/** One session, start to finish. `only`: the runs a re-open plays. */
+async function playSession(options, only) {
+    const log = options.log ?? ((line) => console.log(line));
+    const postTimeoutMs = options.postTimeoutMs ?? POST_TIMEOUT_MS;
+    const streamIdleMs = options.streamIdleMs ?? STREAM_IDLE_MS;
+    const retryBudgetMs = options.retryBudgetMs ?? RETRY_BUDGET_MS;
+    const openRetryCapMs = options.openRetryCapMs ?? OPEN_RETRY_CAP_MS;
+    const resumeWindowMs = options.resumeWindowMs ?? RESUME_WINDOW_MS;
+    const now = options.now ?? Date.now;
+    const streamRefreshMs = options.streamRefreshMs ?? STREAM_REFRESH_MS;
+    let idempotent = false;
+    const base = options.cloud.replace(/\/$/, "");
+    // The session's address is known once it opens; tools and events only use it after that.
+    let session = "";
+    let auth = {};
+    const controller = new AbortController();
+    let stream;
+    let stopped = false;
+    let failure;
+    let dashboard = "";
+    let openedId = "";
+    let reopenable = false;
+    const runs = [];
+    let idle;
+    let refresh;
+    const fail = (context, error) => {
+        if (stopped || failure)
+            return;
+        failure = new Error(`${context}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+        stream?.destroy(failure);
+        controller.abort();
+    };
+    const pending = new Set();
+    const watch = (work, context, allowRefusal = false) => {
+        const handled = work.then(() => { }, (error) => {
+            if (allowRefusal && error instanceof PostError && error.status === 409 && error.code === "platform_refused")
+                return;
+            // Retried already. The send fails for this reply; the stream stays up for the other runs.
+            if (allowRefusal && unavailable(error))
+                return;
+            fail(context, error);
+        });
+        pending.add(handled);
+        void handled.then(() => pending.delete(handled));
+    };
+    // Tools: same name, description and input; running one asks the cloud, which holds the request
+    // until its simulator has answered and the chosen latency has passed.
+    // 210s already includes the default POST budget. A session that sets its own uses that instead, so a test can still watch a timeout.
+    const toolBudget = options.postTimeoutMs == null ? CLOUD_MODEL_BUDGET_MS : postTimeoutMs;
+    // The built-in get_user_profile is played like the agent's own tools, so a test never reads a real
+    // profile: its stand-in is given to every agent, and counts only for one with a `profile`.
+    const ownProfile = options.tools.some((t) => t.name === PROFILE_TOOL);
+    const standIn = ownProfile ? [] : [Object.assign(profileTool(() => null), { [STAND_IN]: true })];
+    const played = [...options.tools, ...standIn];
+    const toolWait = new Map(played.map((t) => [t.name, toolTimeoutMs(options.files, t.name, toolBudget)]));
+    const tools = played.map((t) => ({
+        ...t,
+        run: (input, context) => {
+            const call = (async () => {
+                const request = { conversationId: context.conversationId, tool: t.name, input };
+                const result = await postRetry(`${session}/tool-calls`, request, auth, controller.signal, log, toolWait.get(t.name) ?? postTimeoutMs, {
+                    mode: idempotent ? "idempotent" : "never", key: `tool:${randomUUID()}`, budgetMs: retryBudgetMs, what: "tool call",
+                });
+                if (!result.ok)
+                    throw new ToolFailed(result.error);
+                return result.output;
+            })();
+            // The agent's own code may start a tool and never wait for it (an intercept's hand-off). A call
+            // the cloud failed is then that run's error, never an unhandled rejection that ends the session;
+            // whoever awaits the call still gets the failure.
+            call.catch((error) => {
+                if (stopped || (error instanceof ToolFailed && !CLOUD_TOOL_ERRORS.has(error.message)))
+                    return;
+                const why = error instanceof PostError && error.code ? error.code : error instanceof Error ? error.message : String(error);
+                log(`${context.conversationId}: the cloud failed the tool call ${t.name} (${why}): an infrastructure error for this run, not the agent's; the session plays on.`);
+            });
+            return call;
+        },
+    }));
+    // Events go up in small batches, so the cloud can show the trace and time the agent.
+    const queued = [];
+    let flushing = null;
+    let timer;
+    let eventSeq = 0;
+    const flush = async () => {
+        while (queued.length) {
+            const seq = ++eventSeq;
+            let bytes = Buffer.byteLength(`{"seq":${seq},"events":[]}`);
+            let count = 0;
+            while (count < queued.length && count < 500) {
+                const next = queued[count].bytes + (count ? 1 : 0);
+                if (bytes + next > 5_000_000)
+                    break;
+                bytes += next;
+                count++;
+            }
+            if (!count)
+                throw new Error("one event exceeds the 5MB upload limit");
+            const batch = queued.slice(0, count);
+            const started = Date.now();
+            const wait = started - batch[0].at;
+            let failed = false;
+            try {
+                await postRetry(`${session}/events`, undefined, auth, controller.signal, log, postTimeoutMs, {
+                    mode: idempotent ? "idempotent" : "never",
+                    key: `events:${seq}`,
+                    serialized: `{"seq":${seq},"events":[${batch.map((e) => e.json).join(",")}]}`,
+                    budgetMs: retryBudgetMs,
+                    what: "event upload",
+                });
+                queued.splice(0, count);
+            }
+            catch (error) {
+                const closed = error instanceof PostError && error.status === 409 && error.code === "session_closed";
+                const aborted = stopped && error instanceof Error && error.name === "AbortError";
+                if (closed || aborted) {
+                    log(`session already finished, dropped ${count} events`);
+                    queued.splice(0, count);
+                    continue;
+                }
+                failed = true;
+                // An older cloud does not store the key, so a retry might apply the batch twice.
+                const again = idempotent ? "retried" : "not retried";
+                throw new Error(`event upload failed; acceptance unknown, ${again}`, { cause: error });
+            }
+            finally {
+                const roundtrip = Date.now() - started;
+                if (failed || (options.verbose && (roundtrip >= 1_000 || wait >= 1_000))) {
+                    log(`event upload ${failed ? "failed" : "slow"}: roundtrip_ms=${roundtrip} count=${count} bytes=${bytes} queue_wait_ms=${wait}`);
+                }
+            }
+        }
+    };
+    const startFlush = () => {
+        if (flushing || stopped || failure || !session)
+            return;
+        flushing = flush().catch((error) => fail("events", error)).finally(() => { flushing = null; });
+    };
+    const events = (event) => {
+        if (stopped || failure)
+            return;
+        try {
+            const json = JSON.stringify(event);
+            queued.push({ json, bytes: Buffer.byteLength(json), at: Date.now() });
+            if (!flushing && !timer)
+                timer = setTimeout(() => { timer = undefined; startFlush(); }, 100);
+        }
+        catch (error) {
+            fail("event serialization", error);
+        }
+    };
+    // The test cloud judges from this. Other sinks do not get it unless they ask.
+    events.context = true;
+    try {
+        const agent = options.build({ tools, events: [events] });
+        const open = {
+            agent: { name: options.name, title: options.title, tools: [...options.tools, ...(agent.hasProfile?.() ? standIn : [])].map(toSpec), channels: agent.channels() },
+            files: options.files.map(({ path, hash }) => ({ path, hash })),
+            runs: options.runs ?? 1,
+            deliveryReceipts: true,
+            retries: true,
+            resume: true,
+            reopen: true,
+            ...(only ? { only } : {}),
+        };
+        const opened = await openOnce(`${base}/api/sessions`, open, options.openHeaders, controller.signal, postTimeoutMs, openRetryCapMs, log);
+        idempotent = opened.idempotency === true;
+        openedId = opened.sessionId;
+        reopenable = opened.reopen === true;
+        const tunnel = tunnelBase(base, opened);
+        auth = { authorization: `Bearer ${opened.token}` };
+        const missing = new Set(opened.missing);
+        const uploads = options.files.filter((f) => missing.has(f.hash));
+        if (uploads.length) {
+            await postRetry(`${tunnel}/api/files`, { files: uploads }, auth, controller.signal, log, postTimeoutMs, {
+                mode: idempotent ? "idempotent" : "never",
+                key: `files:${createHash("sha256").update(uploads.map((file) => file.hash).join("\n")).digest("hex")}`,
+                budgetMs: retryBudgetMs,
+                what: "file upload",
+            });
+        }
+        log(`Session ${opened.sessionId}: suite ${suiteHash(options.files).slice(0, 12)}, ${options.files.length} files (${uploads.length} new to the cloud, the rest it already had).`);
+        dashboard = opened.dashboard;
+        options.onDashboard?.(dashboard, opened.sessionId, typeof opened.agent === "string" && opened.agent ? opened.agent : undefined);
+        log(`Watch it: ${opened.dashboard}`);
+        session = `${tunnel}/api/sessions/${opened.sessionId}`;
+        if (failure)
+            throw failure;
+        startFlush();
+        const streamUrl = `${session}/stream`;
+        stream = await openStream(streamUrl, auth, controller.signal);
+        if (failure)
+            throw failure;
+        // A cloud that resumes numbers its commands: a lost stream is opened again after the last one
+        // handled, and a command the cloud sends again is skipped.
+        const resume = opened.resume === true;
+        let lastId = 0;
+        let refreshing = false;
+        // The cloud writes ": ping" every 15s. Three missed pings and the stream is dead.
+        const arm = () => {
+            if (idle)
+                clearTimeout(idle);
+            idle = setTimeout(() => {
+                const lost = new StreamLost(`no bytes for ${streamIdleMs / 1000}s`);
+                if (resume)
+                    stream?.destroy(lost);
+                else
+                    fail("the stream is dead", lost);
+            }, streamIdleMs);
+        };
+        // The tunnel cuts a request at an hour, so a resumable stream is replaced before then.
+        const scheduleRefresh = () => {
+            if (refresh)
+                clearTimeout(refresh);
+            if (resume)
+                refresh = setTimeout(() => { refreshing = true; stream?.destroy(); }, streamRefreshMs);
+        };
+        arm();
+        scheduleRefresh();
+        let draining = false;
+        let drained = false;
+        let receiveFails = 0;
+        // Drain and the drained POST wait on deliveries that can outlast the idle window. Pings
+        // still arrive, and sit unread until that wait ends, so the timer must be down for it.
+        const pauseIdle = () => {
+            if (idle)
+                clearTimeout(idle);
+            idle = undefined;
+        };
+        const drainWork = async () => {
+            // Delivery completion may enqueue recovery events or another outbound, so keep
+            // draining until both lanes are empty. Event batches remain strictly serial.
+            do {
+                while (pending.size)
+                    await Promise.all(pending);
+                if (failure)
+                    throw failure;
+                if (timer) {
+                    clearTimeout(timer);
+                    timer = undefined;
+                }
+                startFlush();
+                await flushing;
+                if (failure)
+                    throw failure;
+            } while (pending.size || queued.length);
+        };
+        /** One command from the stream. Returns the summary once the session has finished. */
+        const handle = async (command) => {
+            if (draining && (command.type === "message" || command.type === "reaction"))
+                throw new Error("the cloud sent a command after drain");
+            if (command.type === "message") {
+                const deliveries = [];
+                // Not async: register the promise during receive(), before its fire-and-forget
+                // interceptor returns null. Each command owns its deliveries, even in a burst.
+                let outboundIndex = 0;
+                const deliver = (outbound) => {
+                    const index = outboundIndex++;
+                    const delivery = postRetry(`${session}/outbound`, { conversationId: command.conversationId, outbound }, auth, controller.signal, log, postTimeoutMs, {
+                        mode: idempotent ? "idempotent" : "outbound",
+                        key: `outbound:${command.conversationId}:${command.messageId}:${index}`,
+                        budgetMs: retryBudgetMs,
+                        what: "outbound delivery",
+                    }).then(() => { });
+                    deliveries.push(delivery);
+                    // Keep the rejection for the normal conversation's delivery recovery, but
+                    // attach a handler immediately for fire-and-forget interceptor sends too.
+                    // A tunnel blip (no JSON from the cloud) is not a reason to end the session.
+                    watch(delivery, "outbound delivery", true);
+                    return delivery;
+                };
+                let taken;
+                try {
+                    taken = agent.receive(command.conversationId, withFiles(command.incoming, options.testsDir, log), deliver, {
+                        channel: command.channel,
+                        channelName: command.channelName,
+                        messageId: command.messageId,
+                    });
+                }
+                catch (error) {
+                    const detail = error instanceof Error ? error.message : String(error);
+                    receiveFails++;
+                    // One broken message can time out. An agent that throws on every message would
+                    // wait out the whole suite, one run at a time.
+                    if (receiveFails >= 3)
+                        throw new Error(`the agent throws on every message: ${detail}`);
+                    log(`receive failed: ${detail}`);
+                    return;
+                }
+                receiveFails = 0;
+                const receipt = { conversationId: command.conversationId, messageId: command.messageId, status: taken === null ? "intercepted" : "taken" };
+                const sendReceipt = () => postRetry(`${session}/receipts`, receipt, auth, controller.signal, log, postTimeoutMs, {
+                    mode: idempotent ? "idempotent" : "never",
+                    key: `receipt:${receipt.conversationId}:${receipt.messageId}`,
+                    budgetMs: retryBudgetMs,
+                    what: "delivery receipt",
+                });
+                // An intercepted reply waits until its outbound bodies land. A tunnel blip on one of them
+                // is not a receipt failure: this run can time out, and the later runs still play. A real
+                // refusal still ends the session, the same as a receipt the tunnel rejected.
+                const receiptWork = taken === null
+                    ? Promise.all(deliveries).then(sendReceipt, (error) => { if (!unavailable(error))
+                        throw error; })
+                    : sendReceipt();
+                watch(receiptWork, "delivery receipt");
+            }
+            else if (command.type === "reaction") {
+                agent.reacted(command.conversationId, command.emoji, command.toText);
+            }
+            else if (command.type === "run") {
+                runs.push(command.run);
+                options.onRun?.(command.run);
+            }
+            else if (command.type === "drain") {
+                if (draining)
+                    throw new Error("the cloud sent drain twice");
+                draining = true;
+                pauseIdle();
+                await drainWork();
+                await postRetry(`${session}/drained`, {}, auth, controller.signal, log, postTimeoutMs, {
+                    mode: idempotent ? "idempotent" : "never",
+                    key: `drained:${opened.sessionId}`,
+                    budgetMs: retryBudgetMs,
+                    what: "drained",
+                });
+                drained = true;
+                arm();
+            }
+            else if (command.type === "finished") {
+                pauseIdle();
+                if (!drained)
+                    await drainWork();
+                return command.summary;
+            }
+            else if (command.type === "failed") {
+                throw new CloudEnded(command.error);
+            }
+        };
+        /** The stream opened again after the last command handled, with backoff, until the resume window closes. */
+        const reconnect = async (why) => {
+            log(`stream ${why}; resuming after command ${lastId}`);
+            const deadline = now() + resumeWindowMs;
+            let last;
+            for (let attempt = 0;; attempt++) {
+                const backoff = attempt === 0 ? 0 : Math.min(10_000, 250 * 2 ** attempt) * (0.8 + Math.random() * 0.4);
+                const refused = last instanceof Error && last.cause instanceof PostError ? last.cause : undefined;
+                const wait = Math.max(backoff, refused?.retryAfterMs ?? 0);
+                if (now() + wait >= deadline) {
+                    const detail = last instanceof Error ? `; last try: ${last.message}` : "";
+                    throw new StreamLost(`the stream did not resume within ${resumeWindowMs / 1000}s (${why})${detail}`);
+                }
+                await pause(wait, controller.signal);
+                if (failure)
+                    throw failure;
+                try {
+                    return await openStream(streamUrl, { ...auth, "last-event-id": String(lastId) }, controller.signal, Math.max(1, Math.min(postTimeoutMs, deadline - now())));
+                }
+                catch (error) {
+                    if (failure)
+                        throw failure;
+                    const cause = error instanceof Error && error.cause instanceof PostError ? error.cause : error;
+                    const code = cause instanceof PostError && cause.code ? ` ${cause.code}` : "";
+                    if (!retryable(cause, "idempotent"))
+                        throw new StreamLost(`the stream could not resume: ${error instanceof Error ? error.message : String(error)}${code}`, { cause: error });
+                    last = error;
+                }
+            }
+        };
+        for (;;) {
+            const decoder = new TextDecoder();
+            const chunks = stream[Symbol.asyncIterator]();
+            let buffer = "";
+            let lost;
+            for (;;) {
+                let next;
+                try {
+                    next = await chunks.next();
+                }
+                catch (error) {
+                    lost = error;
+                    break;
+                }
+                if (next.done)
+                    break;
+                arm();
+                buffer += decoder.decode(next.value, { stream: true });
+                let end;
+                while ((end = buffer.indexOf("\n\n")) >= 0) {
+                    const frame = buffer.slice(0, end).split("\n");
+                    buffer = buffer.slice(end + 2);
+                    const data = frame.filter((l) => l.startsWith("data: ")).map((l) => l.slice(6)).join("\n");
+                    if (!data)
+                        continue;
+                    const idLine = frame.find((l) => l.startsWith("id:"));
+                    const id = idLine === undefined ? NaN : Number(idLine.slice(3).trim());
+                    if (Number.isSafeInteger(id) && id > 0) {
+                        // Sent again after a resume: it was handled on the stream before.
+                        if (id <= lastId)
+                            continue;
+                        lastId = id;
+                    }
+                    let command;
+                    try {
+                        command = JSON.parse(data);
+                    }
+                    catch {
+                        log("a stream frame was not valid JSON");
+                        continue;
+                    }
+                    const summary = await handle(command);
+                    if (summary)
+                        return summary;
+                }
+            }
+            if (failure)
+                throw failure;
+            const closed = new StreamLost("the cloud closed the stream before the session finished");
+            if (!resume)
+                throw lost ?? closed;
+            pauseIdle();
+            const why = refreshing ? "refreshed before the tunnel's hour limit" : `lost (${lost instanceof Error ? lost.message : "the cloud closed it"})`;
+            refreshing = false;
+            stream.destroy();
+            stream = await reconnect(why);
+            arm();
+            scheduleRefresh();
+        }
+    }
+    catch (error) {
+        if (error instanceof SessionStopped)
+            throw error;
+        const reason = failure ?? (error instanceof Error ? error : new Error(String(error)));
+        // Exit 2 is a transport failure after the session exists. A refusal, or anything
+        // before the open answers, is the error itself (the CLI prints it alone and exits by its kind, report.ts).
+        if (dashboard && isTransportFailure(reason)) {
+            if (reopenable && saysGone(reason))
+                throw new SessionGone(reason.message, runs, dashboard, openedId);
+            throw new SessionStopped(reason.message, runs, dashboard);
+        }
+        throw reason;
+    }
+    finally {
+        stopped = true;
+        if (timer)
+            clearTimeout(timer);
+        if (idle)
+            clearTimeout(idle);
+        if (refresh)
+            clearTimeout(refresh);
+        stream?.destroy();
+        controller.abort();
+        await Promise.all(pending);
+        await flushing;
+    }
+}
+/** The stream on a connection of its own. fetch shares one HTTP/2 connection per origin, and over
+ * HTTPS a request behind another open one on it waits until that one finishes: a tool call the
+ * cloud has not answered held every later event and reply, and the run ended agent_quiet.
+ * A refusal's status and JSON code are its cause, a PostError. A resume waits at most `timeoutMs` for the answer. */
+function openStream(url, headers, signal, timeoutMs) {
+    const request = url.startsWith("https:") ? httpsRequest : httpRequest;
+    return new Promise((resolve, reject) => {
+        const req = request(url, { headers, agent: false, signal }, (response) => {
+            if (timer)
+                clearTimeout(timer);
+            if (response.statusCode === 200)
+                return resolve(response);
+            const status = response.statusCode ?? 0;
+            let text = "";
+            response.setEncoding("utf8");
+            response.on("data", (chunk) => { if (text.length < 4_096)
+                text += chunk; });
+            response.on("error", () => { });
+            response.on("end", () => {
+                const cause = new PostError(url, status, text, retryAfterOf(flatHeaders(response.headers)));
+                reject(new Error(`could not open the stream (${status})`, { cause }));
+            });
+        });
+        const timer = timeoutMs === undefined ? undefined : setTimeout(() => req.destroy(new TimeoutError()), timeoutMs);
+        req.on("error", (error) => { if (timer)
+            clearTimeout(timer); reject(error); }).end();
+    });
+}
+/** The cloud answered a call with a status that is not 2xx. `code` is a JSON `error` string (the tunnel's
+ * refusals, which retries go by); `type` is a JSON `error` object's type (a hosted cloud's front door). */
+export class PostError extends Error {
+    status;
+    code;
+    type;
+    retryAfterMs;
+    constructor(url, status, text, retryAfterMs) {
+        super(`${url}: ${status} ${shown(text)}`);
+        this.name = "PostError";
+        this.status = status;
+        let error;
+        try {
+            error = JSON.parse(text).error;
+        }
+        catch { /* the body was not JSON */ }
+        if (typeof error === "string")
+            this.code = error;
+        else if (error && typeof error === "object" && typeof error.type === "string")
+            this.type = error.type;
+        this.retryAfterMs = retryAfterMs;
+    }
+}
+const OPEN_RETRY_CAP_MS = 60_000;
+/** The open, sent once more when the cloud is busy (a 429 that is not the spend cap, or a 503 capacity)
+ * and its Retry-After is at most `capMs`. */
+async function openOnce(url, body, headers, signal, timeoutMs, capMs, log) {
+    try {
+        return await post(url, body, headers, signal, undefined, timeoutMs);
+    }
+    catch (error) {
+        if (!(error instanceof PostError) || error.retryAfterMs === undefined || error.retryAfterMs > capMs)
+            throw error;
+        const why = error.type ?? error.code;
+        const busy = (error.status === 429 && why !== "spend_cap") || (error.status === 503 && why === "capacity");
+        if (!busy)
+            throw error;
+        log(`The cloud is busy (${error.status}${why ? ` ${why}` : ""}); trying once more in ${Math.ceil(error.retryAfterMs / 1000)} s.`);
+        await pause(error.retryAfterMs, signal);
+        return await post(url, body, headers, signal, undefined, timeoutMs);
+    }
+}
+/** This call can be dropped and the session continues: a frontend page with no JSON, a
+ * timeout, or a connection that died. */
+function unavailable(error) {
+    if (error instanceof TimeoutError)
+        return true;
+    if (error instanceof PostError)
+        return !error.code && (error.status === 502 || error.status === 503 || error.status === 504);
+    if (!(error instanceof Error) || error.name === "AbortError")
+        return false;
+    const code = error.code;
+    return code === "ECONNRESET" || code === "ECONNREFUSED" || code === "EPIPE" || code === "ETIMEDOUT" || code === "EAI_AGAIN" || error.message === "socket hang up";
+}
+/** The tunnel, the stream, or the connection, as opposed to a refusal the cloud wrote as JSON. */
+export function isTransportFailure(error) {
+    let current = error;
+    for (let guard = 0; current instanceof Error && guard < 6; guard++) {
+        if (current instanceof TimeoutError || current instanceof StreamLost)
+            return true;
+        if (current instanceof PostError)
+            return current.code === "session_gone" || (!current.code && (current.status === 502 || current.status === 503 || current.status === 504));
+        if (/^could not open the stream \((502|503|504)\)$/.test(current.message))
+            return true;
+        const code = current.code;
+        if (code === "ECONNRESET" || code === "ECONNREFUSED" || code === "EPIPE" || code === "ETIMEDOUT" || code === "EAI_AGAIN" || code === "ENOTFOUND" || current.message === "socket hang up")
+            return true;
+        current = current.cause;
+    }
+    return false;
+}
+/** The cloud no longer has the session: a tunnel call or the stream answered session_gone. */
+function saysGone(error) {
+    let current = error;
+    for (let guard = 0; current instanceof Error && guard < 6; guard++) {
+        if (current instanceof PostError && current.code === "session_gone")
+            return true;
+        current = current.cause;
+    }
+    return false;
+}
+/** Safe to send again, until the cloud stores an idempotency key. A 502/503/504 with no JSON
+ * never reached the container. A failure before the connection was made did not either.
+ * A timeout or a reset may mean the reply was already delivered. */
+function retryableOutbound(error) {
+    if (error instanceof PostError)
+        return !error.code && (error.status === 502 || error.status === 503 || error.status === 504);
+    if (!(error instanceof Error) || error.name === "AbortError")
+        return false;
+    const code = error.code;
+    return code === "ECONNREFUSED" || code === "EAI_AGAIN" || code === "ENOTFOUND";
+}
+function shown(text) {
+    const plain = text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
+    return plain.slice(0, 180);
+}
+function pause(ms, signal) {
+    if (signal?.aborted)
+        return Promise.reject(signal.reason);
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => { signal?.removeEventListener("abort", stop); resolve(); }, ms);
+        const stop = () => { clearTimeout(timer); reject(signal?.reason); };
+        signal?.addEventListener("abort", stop, { once: true });
+    });
+}
+/** Whether this failure may be sent again. A JSON refusal never is. An older cloud retries an
+ * outbound only when the body cannot have arrived. A cloud that stores keys also retries a
+ * timeout, a reset, a 429, and its own 503 unavailable. */
+function retryable(error, mode) {
+    if (mode === "never")
+        return false;
+    if (mode === "outbound")
+        return retryableOutbound(error);
+    if (error instanceof TimeoutError)
+        return true;
+    if (error instanceof PostError) {
+        if (error.status === 429)
+            return true;
+        if (error.status === 503 && error.code === "unavailable")
+            return true;
+        return !error.code && (error.status === 502 || error.status === 503 || error.status === 504);
+    }
+    if (!(error instanceof Error) || error.name === "AbortError")
+        return false;
+    const code = error.code;
+    return code === "ECONNRESET" || code === "ECONNREFUSED" || code === "EPIPE" || code === "ETIMEDOUT" || code === "EAI_AGAIN" || code === "ENOTFOUND" || error.message === "socket hang up";
+}
+function retryStatus(error) {
+    if (error instanceof PostError)
+        return String(error.status);
+    if (error instanceof TimeoutError)
+        return "timed out";
+    return "no response";
+}
+/** Wait between tries: Retry-After when the cloud set one, otherwise exponential with jitter. */
+function retryWaitMs(attempt, error) {
+    if (error instanceof PostError && error.retryAfterMs !== undefined)
+        return error.retryAfterMs;
+    const base = 200 * 2 ** attempt;
+    return base + Math.random() * base * 0.2;
+}
+/** One call, sent again when that is safe, until the budget has passed since its first failure.
+ * The same Idempotency-Key goes on every try. */
+async function postRetry(url, body, headers, signal, log, timeoutMs, options) {
+    let firstFailure;
+    let attempt = 0;
+    const headersWithKey = { ...headers, "idempotency-key": options.key };
+    for (;;) {
+        try {
+            return await post(url, body, headersWithKey, signal, options.serialized, timeoutMs);
+        }
+        catch (error) {
+            const again = retryable(error, options.mode);
+            firstFailure ??= Date.now();
+            const elapsed = Date.now() - firstFailure;
+            const wait = again ? retryWaitMs(attempt, error) : 0;
+            if (!again || elapsed + wait > options.budgetMs) {
+                if (again)
+                    log(`${options.what} failed after ${attempt + 1} tries`);
+                else if (options.mode !== "never" && unavailable(error))
+                    log(`${options.what} failed (${retryStatus(error)})`);
+                throw error;
+            }
+            log(`${options.what} unavailable (${retryStatus(error)}), trying again`);
+            await pause(wait, signal);
+            attempt++;
+        }
+    }
+}
+/** POSTs share a keep-alive agent and speak HTTP/1.1, so each request still in flight has its own
+ * socket. A tool call the cloud has not answered cannot hold the later ones, and a finished
+ * connection is reused. The stream stays on a connection of its own (openStream). */
+const httpPosts = new HttpAgent({ keepAlive: true });
+const httpsPosts = new HttpsAgent({ keepAlive: true });
+function requestOnce(url, payload, headers, signal, timeoutMs) {
+    const secure = url.startsWith("https:");
+    const request = secure ? httpsRequest : httpRequest;
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        let timer;
+        let onAbort;
+        const finish = (error, result) => {
+            if (settled)
+                return;
+            settled = true;
+            if (timer)
+                clearTimeout(timer);
+            if (onAbort)
+                signal?.removeEventListener("abort", onAbort);
+            if (error)
+                reject(error);
+            else
+                resolve(result);
+        };
+        const req = request(url, {
+            method: "POST",
+            headers: { "content-type": "application/json", ...headers, "content-length": Buffer.byteLength(payload) },
+            agent: secure ? httpsPosts : httpPosts,
+        }, (response) => {
+            const chunks = [];
+            response.on("error", finish);
+            response.on("aborted", () => finish(new Error("The request was aborted")));
+            response.on("data", (chunk) => chunks.push(chunk));
+            response.on("end", () => finish(undefined, { status: response.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8"), headers: flatHeaders(response.headers) }));
+        });
+        timer = setTimeout(() => { finish(new TimeoutError()); req.destroy(); }, timeoutMs);
+        if (signal) {
+            onAbort = () => { finish(abortError(signal)); req.destroy(); };
+            signal.addEventListener("abort", onAbort, { once: true });
+        }
+        req.on("error", (error) => finish(signal?.aborted ? abortError(signal) : error));
+        req.end(payload);
+    });
+}
+function abortError(signal) {
+    const reason = signal.reason;
+    return reason instanceof Error ? reason : new DOMException("The operation was aborted", "AbortError");
+}
+function flatHeaders(headers) {
+    const out = {};
+    for (const [name, value] of Object.entries(headers)) {
+        if (typeof value === "string")
+            out[name] = value;
+        else if (Array.isArray(value))
+            out[name] = value.join(", ");
+    }
+    return out;
+}
+function retryAfterOf(headers) {
+    const value = headers["retry-after"];
+    if (!value)
+        return undefined;
+    const seconds = Number(value);
+    if (Number.isFinite(seconds) && seconds >= 0)
+        return Math.round(seconds * 1000);
+    const date = Date.parse(value);
+    if (Number.isNaN(date))
+        return undefined;
+    return Math.max(0, date - Date.now());
+}
+async function post(url, body, headers = {}, signal, serialized, timeoutMs = POST_TIMEOUT_MS) {
+    const payload = serialized ?? JSON.stringify(body ?? {});
+    const { status, text, headers: responseHeaders } = await requestOnce(url, payload, headers, signal, postBudgetMs(timeoutMs, Buffer.byteLength(payload)));
+    if (status < 200 || status >= 300) {
+        throw new PostError(url, status, text, retryAfterOf(responseHeaders));
+    }
+    return (text ? JSON.parse(text) : {});
+}
+/** The person's message with the files it names read from the tests folder, and only from there. */
+function withFiles(incoming, testsDir, log) {
+    const { attachments, ...rest } = incoming;
+    if (!attachments?.length)
+        return rest;
+    return {
+        ...rest,
+        attachments: attachments.map((a) => ({
+            kind: a.kind,
+            mime: mimeOf(a.file),
+            name: basename(a.file),
+            data: () => {
+                const reading = readAttachment(testsDir, a.file);
+                // A caller that never awaits still must not become an uncaught exception.
+                reading.catch((error) => log(`attachment could not be read: ${a.file}: ${error instanceof Error ? error.message : String(error)}`));
+                return reading;
+            },
+        })),
+    };
+}
+function readAttachment(testsDir, file) {
+    const path = testsDir ? resolve(testsDir, file) : "";
+    const inside = testsDir && !relative(resolve(testsDir), path).startsWith("..") && path !== resolve(testsDir);
+    if (!inside)
+        return Promise.reject(new Error(`${file} is not a file in the tests folder`));
+    return readFile(path).then((bytes) => new Uint8Array(bytes));
+}