@flow-engineer/flow 0.0.0-stage → 0.2.0

package/dist/testing/src/http-agent.d.ts

@@ -0,0 +1,47 @@
+import { type ChannelInfo, type ChannelSummary, type Deliver, type EventSink, type Incoming, type Outbound, type ReceiveOptions } from "../../src/index.ts";
+export interface HttpAgentOptions {
+    url: string;
+    /** Sent as `authorization: Bearer <token>` (FLOW_FRAMEWORK_AGENT_TOKEN). */
+    token?: string;
+    /** The channel every message names; default the platform the run plays (whatsapp, sms, imessage, telegram, web). */
+    channel?: string;
+    /** How long one message waits for its answer. Default 60 s. */
+    timeoutMs?: number;
+    events?: EventSink[];
+    log?: (line: string) => void;
+}
+/** The person's message as the agent's endpoint gets it. */
+export interface HttpAgentRequest {
+    conversationId: string;
+    /** The id the cloud gave this message (m1, m2, ...): a reply's replyTo, a reaction's or a read's messageId. */
+    messageId: string;
+    channel: string;
+    message: {
+        text: string;
+        replyTo?: Incoming["replyTo"];
+        choice?: Incoming["choice"];
+        attachments?: Array<{
+            kind: "image" | "audio" | "document";
+            mime: string;
+            name?: string;
+            data: string;
+        }>;
+    };
+}
+export declare class TurnError extends Error {
+}
+/** POSTs one message and returns its replies, unchecked; throws a TurnError saying why when the answer fails the turn. */
+export declare function postMessage(options: Pick<HttpAgentOptions, "url" | "token">, timeoutMs: number, body: HttpAgentRequest): Promise<unknown[]>;
+/** The reply as the platform takes it, or why it cannot go: the checks a Flow Framework agent's send gets. */
+export declare function checkReply(reply: unknown, info: ChannelInfo): {
+    outbound: Outbound;
+} | {
+    refused: string;
+};
+export declare function httpAgent(options: HttpAgentOptions): {
+    /** None of its own: each run plays its platform as the platform is. */
+    channels(): ChannelSummary[];
+    receive(conversationId: string, message: string | Incoming, deliver: Deliver, receive?: ReceiveOptions): string | null;
+    /** The person reacted to one of its messages: not sent to the endpoint in this version. */
+    reacted(_conversationId: string, _emoji: string, _toText: string): void;
+};

package/dist/testing/src/http-agent.js

@@ -0,0 +1,155 @@
+// An agent you already have, on any framework, tested through its HTTP endpoint: `flow test
+// --agent-url <url>`. This plays the part of a Flow Framework Agent on the developer's machine: each of the
+// person's messages is one POST to the URL from here (so the agent never has to be reachable from
+// the internet), and the replies it answers go to the test cloud as a Flow Framework agent's would. Its own
+// tools run on its side, for real.
+//
+//   POST <url>  { conversationId, messageId, channel, message: { text, attachments?: [{ kind, mime, name?, data }] } }
+//   200         { replies: Outbound[] }        (data is base64; [] is no reply)
+//
+// A conversation's messages go one at a time, in order: the next is sent once the last one's answer
+// is in. Each reply is checked against what the run's platform can show, as a Flow Framework agent's send
+// is. Agents that answer later (through a callback) are not supported yet.
+import { Buffer } from "node:buffer";
+import { emitter } from "../../src/events.js";
+import { fitMessage, z } from "../../src/index.js";
+import { sendInput, sendRefusal } from "../../src/talk.js";
+const answerSchema = z.object({ replies: z.array(z.unknown()) });
+const otherReply = z.discriminatedUnion("type", [
+    z.object({ type: z.literal("read"), messageIds: z.array(z.string()) }),
+    z.object({ type: z.literal("reaction"), emoji: z.string().min(1), messageId: z.string().min(1) }),
+    z.looseObject({ type: z.literal("template"), name: z.string().min(1) }),
+    z.object({ type: z.literal("status"), text: z.string().nullable() }),
+    z.object({ type: z.literal("typing"), on: z.boolean() }),
+]);
+export class TurnError extends Error {
+}
+const problems = (error) => error.issues.map((i) => (i.code === "unrecognized_keys" ? `${i.keys.join(", ")}: not something this platform shows` : (i.path.length ? `${i.path.join(".")}: ` : "") + i.message)).join("; ");
+/** POSTs one message and returns its replies, unchecked; throws a TurnError saying why when the answer fails the turn. */
+export async function postMessage(options, timeoutMs, body) {
+    let response;
+    let text;
+    try {
+        response = await fetch(options.url, {
+            method: "POST",
+            headers: { "content-type": "application/json", ...(options.token ? { authorization: `Bearer ${options.token}` } : {}) },
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(timeoutMs),
+        });
+        text = await response.text();
+    }
+    catch (error) {
+        const e = error;
+        if (e.name === "TimeoutError")
+            throw new TurnError(`no answer from ${options.url} within ${timeoutMs / 1000} s`);
+        throw new TurnError(`could not reach ${options.url}: ${e.cause?.code ?? e.cause?.message ?? e.message}`);
+    }
+    if (response.status !== 200)
+        throw new TurnError(`${options.url} answered ${response.status}${text.trim() ? `: ${text.trim().slice(0, 200)}` : ""}`);
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        throw new TurnError(`${options.url} answered 200 with a body that is not JSON: ${text.slice(0, 200)}`);
+    }
+    const answer = answerSchema.safeParse(parsed);
+    if (!answer.success)
+        throw new TurnError(`${options.url} answered 200 without { "replies": [...] }: ${text.slice(0, 200)}`);
+    return answer.data.replies;
+}
+/** The reply as the platform takes it, or why it cannot go: the checks a Flow Framework agent's send gets. */
+export function checkReply(reply, info) {
+    const type = reply?.type;
+    if (type !== "message") {
+        const other = otherReply.safeParse(reply);
+        return other.success ? { outbound: other.data } : { refused: problems(other.error) };
+    }
+    const { type: _, replyTo, fields, ...rest } = reply;
+    const input = { ...rest, ...(replyTo !== undefined ? { reply_to: replyTo } : {}) };
+    const sent = sendInput(info).strict().safeParse(input);
+    if (!sent.success)
+        return { refused: problems(sent.error).replaceAll("reply_to", "replyTo") };
+    if (fields !== undefined && (typeof fields !== "object" || fields === null || Array.isArray(fields)))
+        return { refused: "fields: must be an object" };
+    const refusal = sendRefusal(info, input);
+    if (refusal)
+        return { refused: refusal };
+    return { outbound: fitMessage(info, { type: "message", ...rest, ...(replyTo !== undefined ? { replyTo } : {}), ...(fields ? { fields } : {}) }) };
+}
+export function httpAgent(options) {
+    const emit = emitter(options.events ?? []);
+    const log = options.log ?? ((line) => console.log(line));
+    const timeoutMs = options.timeoutMs ?? 60_000;
+    /** Each conversation's last message, so the next one waits for its answer. */
+    const queues = new Map();
+    async function turn(conversationId, request, incoming, info, deliver) {
+        const started = Date.now();
+        emit(conversationId, "turn.start", {});
+        try {
+            if (incoming.attachments?.length) {
+                request.message.attachments = await Promise.all(incoming.attachments.map(async (a) => ({ kind: a.kind, mime: a.mime, ...(a.name ? { name: a.name } : {}), data: Buffer.from(await a.data()).toString("base64") })));
+            }
+            for (const reply of await postMessage(options, timeoutMs, request)) {
+                const checked = checkReply(reply, info);
+                const text = reply?.text;
+                const shown = typeof text === "string" ? text : `(${String(reply?.type ?? "reply")})`;
+                if ("refused" in checked) {
+                    log(`${conversationId}: a reply was not sent: ${checked.refused}`);
+                    emit(conversationId, "message.undelivered", { text: shown, reason: checked.refused });
+                    continue;
+                }
+                const outbound = checked.outbound;
+                try {
+                    await deliver(outbound);
+                }
+                catch (error) {
+                    emit(conversationId, "message.undelivered", { text: shown, reason: error.message });
+                    continue;
+                }
+                if (outbound.type === "message") {
+                    const { type: _, ...sent } = outbound;
+                    emit(conversationId, "message.out", { ...sent, channel: request.channel });
+                }
+                else if (outbound.type === "reaction") {
+                    emit(conversationId, "reaction", { emoji: outbound.emoji, messageId: outbound.messageId });
+                }
+            }
+        }
+        catch (error) {
+            const reason = error instanceof TurnError ? error.message : `the message could not be sent: ${error.message}`;
+            log(`${conversationId}: ${reason}`);
+            emit(conversationId, "model.error", { error: reason });
+            emit(conversationId, "message.undelivered", { text: "(the agent's answer)", reason });
+        }
+        finally {
+            emit(conversationId, "turn.end", { ms: Date.now() - started });
+        }
+    }
+    return {
+        /** None of its own: each run plays its platform as the platform is. */
+        channels() {
+            return [];
+        },
+        receive(conversationId, message, deliver, receive = {}) {
+            const incoming = typeof message === "string" ? { text: message } : message;
+            const messageId = receive.messageId ?? "";
+            // The cloud's conversations are "<platform>:<person>".
+            const channel = options.channel ?? /^([a-z]+):/.exec(conversationId)?.[1] ?? "chat";
+            const request = {
+                conversationId,
+                messageId,
+                channel,
+                message: { text: incoming.text, ...(incoming.replyTo ? { replyTo: incoming.replyTo } : {}), ...(incoming.choice ? { choice: incoming.choice } : {}) },
+            };
+            const attachments = incoming.attachments?.map((a) => ({ kind: a.kind, mime: a.mime, ...(a.name ? { name: a.name } : {}) }));
+            emit(conversationId, "message.in", { text: incoming.text, messageId, channel, ...(attachments ? { attachments } : {}) });
+            const info = receive.channel ?? { reactions: false, status: false };
+            const next = (queues.get(conversationId) ?? Promise.resolve()).then(() => turn(conversationId, request, incoming, info, deliver));
+            queues.set(conversationId, next);
+            return messageId || null;
+        },
+        /** The person reacted to one of its messages: not sent to the endpoint in this version. */
+        reacted(_conversationId, _emoji, _toText) { },
+    };
+}

package/dist/testing/src/index.d.ts

@@ -0,0 +1,3 @@
+export { runInCloud, type CloudSummary, type ConnectOptions } from "./client.ts";
+export { hashOf, loadSuite, parseScenario, parseToolFile, suiteHash, type Scenario, type SuiteFile, type ToolFile } from "./suite.ts";
+export type { Check, Command, OpenSession, Opened, RunKey, RunSummary, SessionSummary, ToolCallRequest, ToolCallResult, UploadFile, Usage, WireIncoming } from "./protocol.ts";

package/dist/testing/src/index.js

@@ -0,0 +1,2 @@
+export { runInCloud } from "./client.js";
+export { hashOf, loadSuite, parseScenario, parseToolFile, suiteHash } from "./suite.js";

package/dist/testing/src/new.d.ts

@@ -0,0 +1,9 @@
+/** Throws unless `name` can be an agent's folder and package name. */
+export declare function checkName(name: string): void;
+/** The @flow-engineer/flow package's folder: the root of a clone, or the installed package. */
+export declare function packageRoot(from?: string): string;
+/** Makes `<into>/<name>` (the current folder by default) and returns it. */
+export declare function newStandalone(name: string, options?: {
+    into?: string;
+    root?: string;
+}): string;

package/dist/testing/src/new.js

@@ -0,0 +1,122 @@
+// `flow new <name>`: makes ./<name>/, an agent of its own started from the café example (Pip),
+// that depends on the @flow-engineer/flow package. Its imports become "@flow-engineer/flow" and "@flow-engineer/flow/simulators", and
+// the coding-agent skills go in .claude/skills/, with an AGENTS.md (and a CLAUDE.md importing it)
+// pointing at them. Inside a clone, `npm run new -- <name>` makes agents/<name>/ instead.
+import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+const NAME = /^[a-z0-9]+(-[a-z0-9]+)*$/;
+/** Throws unless `name` can be an agent's folder and package name. */
+export function checkName(name) {
+    if (!NAME.test(name))
+        throw new Error(`"${name}" is not a usable name: use lowercase letters, digits and dashes, such as my-agent.`);
+}
+/** The @flow-engineer/flow package's folder: the root of a clone, or the installed package. */
+export function packageRoot(from = dirname(fileURLToPath(import.meta.url))) {
+    for (let dir = from;; dir = dirname(dir)) {
+        const file = join(dir, "package.json");
+        if (existsSync(file) && JSON.parse(readFileSync(file, "utf8")).name === "@flow-engineer/flow")
+            return dir;
+        if (dirname(dir) === dir)
+            throw new Error("flow: can't find its own package.json");
+    }
+}
+const TSCONFIG = {
+    compilerOptions: {
+        target: "es2023",
+        module: "nodenext",
+        moduleResolution: "nodenext",
+        strict: true,
+        noEmit: true,
+        allowImportingTsExtensions: true,
+        erasableSyntaxOnly: true,
+        verbatimModuleSyntax: true,
+        skipLibCheck: true,
+        types: ["node"],
+    },
+    include: ["src"],
+};
+const ENV_EXAMPLE = `# One model key. The first line with a value is the one used.
+# Make it at the provider, then paste it here. Never commit this file.
+#   OPENROUTER_API_KEY   https://openrouter.ai/keys
+#   OPENAI_API_KEY       https://platform.openai.com/api-keys
+#   ANTHROPIC_API_KEY    https://console.anthropic.com/settings/keys
+#   GEMINI_API_KEY       https://aistudio.google.com/apikey
+OPENROUTER_API_KEY=
+OPENAI_API_KEY=
+ANTHROPIC_API_KEY=
+GEMINI_API_KEY=
+
+# Optional. The model id for that provider; empty is that provider's default.
+# CAFE_MODEL=
+
+# The test cloud (npx flow test --synthetic) and traces. Sign in at https://flow.engineer/observe,
+# add the agent, and on its Developers page make a CI key (oci_..., shown once); the trace key is
+# from the same site.
+FLOW_FRAMEWORK_CLOUD=https://flow.engineer/observe
+FLOW_FRAMEWORK_CLOUD_KEY=
+FLOW_FRAMEWORK_TRACES_URL=https://flow.engineer/observe
+FLOW_FRAMEWORK_TRACES_KEY=
+`;
+const agentsMd = (name) => `# ${name}
+
+A chat agent on Flow Framework (https://github.com/flow-engineer/flow), started from its café example (Pip).
+
+- Load the skills in \`.claude/skills/\` before changing it: \`flow-framework\` to build the agent, \`flow-framework-tools\`
+  for its tools and their test files, \`flow-framework-tests\` for its test scenarios.
+- \`agent/AGENTS.md\` is the agent's own prompt, not instructions for coding agents.
+- \`npm start\` runs it in the playground. The owner puts one model key in \`.env\` (never read or
+  print it): \`OPENROUTER_API_KEY\`, \`OPENAI_API_KEY\`, \`ANTHROPIC_API_KEY\` or \`GEMINI_API_KEY\`.
+  \`npm run typecheck\` checks the types, \`npm test -- --synthetic\` runs the tests in the test
+  cloud at https://flow.engineer/observe with the owner's \`FLOW_FRAMEWORK_CLOUD_KEY\` (\`npx flow --help\`).
+`;
+/** Makes `<into>/<name>` (the current folder by default) and returns it. */
+export function newStandalone(name, options = {}) {
+    checkName(name);
+    const root = options.root ?? packageRoot();
+    const dir = join(options.into ?? process.cwd(), name);
+    if (existsSync(dir))
+        throw new Error(`${dir} already exists: pick another name or remove it.`);
+    const cafe = join(root, "examples", "cafe");
+    for (const part of ["agent", "src", "tests"])
+        cpSync(join(cafe, part), join(dir, part), { recursive: true });
+    for (const file of readdirSync(join(dir, "src"), { recursive: true, encoding: "utf8" }).filter((f) => f.endsWith(".ts"))) {
+        const path = join(dir, "src", file);
+        const source = readFileSync(path, "utf8")
+            .replace(/(["'])(\.\.\/)+src\/index\.ts\1/g, '"@flow-engineer/flow"')
+            .replace(/(["'])(\.\.\/)+simulators\/src\/index\.ts\1/g, '"@flow-engineer/flow/simulators"');
+        if (/["']\.\.\/\.\.\//.test(source))
+            throw new Error(`flow: ${file} imports a file outside the agent; only Flow Framework's public entries can be rewritten.`);
+        writeFileSync(path, source);
+    }
+    const own = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
+    const pkg = {
+        name,
+        version: "0.1.0",
+        private: true,
+        description: `${name}: a Flow Framework agent, started from the café example.`,
+        type: "module",
+        engines: own.engines,
+        scripts: { start: "node --env-file-if-exists=.env src/main.ts", test: "flow test", typecheck: "tsc --noEmit" },
+        dependencies: { "@flow-engineer/flow": `^${own.version}` },
+        devDependencies: { "@types/node": own.devDependencies["@types/node"], typescript: own.devDependencies.typescript },
+    };
+    writeFileSync(join(dir, "package.json"), `${JSON.stringify(pkg, null, 2)}\n`);
+    writeFileSync(join(dir, "tsconfig.json"), `${JSON.stringify(TSCONFIG, null, 2)}\n`);
+    writeFileSync(join(dir, ".gitignore"), "node_modules/\n.env\n.flow/\n");
+    // npm install then refuses a Node older than the engines field says, before .ts files fail to load.
+    writeFileSync(join(dir, ".npmrc"), "engine-strict=true\n");
+    writeFileSync(join(dir, ".env.example"), ENV_EXAMPLE);
+    const skills = join(root, "coding-agents");
+    for (const file of readdirSync(skills).filter((f) => f.endsWith(".md"))) {
+        const text = readFileSync(join(skills, file), "utf8");
+        const skill = /^name:\s*(\S+)\s*$/m.exec(text)?.[1];
+        if (!skill)
+            throw new Error(`flow: coding-agents/${file} has no name in its front matter.`);
+        mkdirSync(join(dir, ".claude", "skills", skill), { recursive: true });
+        writeFileSync(join(dir, ".claude", "skills", skill, "SKILL.md"), text);
+    }
+    writeFileSync(join(dir, "AGENTS.md"), agentsMd(name));
+    writeFileSync(join(dir, "CLAUDE.md"), "@AGENTS.md\n");
+    return dir;
+}

package/dist/testing/src/print-run.d.ts

@@ -0,0 +1,3 @@
+import type { RunSummary } from "./protocol.ts";
+/** Prints each run once. A run already shown as it finished is not printed again with the summary. */
+export declare function runPrinter(log?: (line: string) => void): (run: RunSummary) => void;

package/dist/testing/src/print-run.js

@@ -0,0 +1,22 @@
+import { runVerdict } from "./report.js";
+const usd = (n) => `$${n.toFixed(4)}`;
+/** Prints each run once. A run already shown as it finished is not printed again with the summary. */
+export function runPrinter(log = console.log) {
+    const printed = new Set();
+    return (r) => {
+        if (printed.has(r.runId))
+            return;
+        printed.add(r.runId);
+        const verdict = runVerdict(r);
+        const gaveUp = verdict === "passed" && r.ended === "gave_up" ? " (passed, the person gave up)" : "";
+        const retried = r.retried ? "; tried once more" : "";
+        // An errored run: the cloud could not play or judge it, so its checks say nothing about the agent.
+        const error = verdict === "errored" ? `; error: ${r.error ?? r.ended} (the cloud could not play or judge this run, not the agent's fault)` : "";
+        const label = verdict === "errored" ? "ERROR" : verdict === "passed" ? "PASS" : "FAIL";
+        log(`${label}  ${r.scenario} #${r.run} (${r.channel}) ${r.runId}: ${r.turns} turn${r.turns === 1 ? "" : "s"}, ended: ${r.ended}${gaveUp}${retried}${error}; slowest acknowledgement ${r.slowestAckMs === null ? "never" : `${r.slowestAckMs} ms`}; cost ${usd(r.agentUsage.costUsd)} agent + ${usd(r.testUsage.costUsd)} test`);
+        if (verdict !== "errored")
+            for (const c of r.checks.filter((c) => !c.pass))
+                log(`      ✗ ${c.kind}: ${c.text}\n        ${c.reason}`);
+        log(`      watch: ${r.watch}`);
+    };
+}

package/dist/testing/src/protocol.d.ts

@@ -0,0 +1,159 @@
+import type { Attachment, ChannelInfo, ChannelSummary, Incoming, ToolSpec } from "../../src/index.ts";
+/** One graded expect item of a run. */
+export interface Check {
+    kind: "outcome" | "judge";
+    text: string;
+    pass: boolean;
+    reason: string;
+}
+/** Model tokens and their cost, added up over a run. */
+export interface Usage {
+    input: number;
+    output: number;
+    costUsd: number;
+}
+export interface OpenSession {
+    /** name: a stable key (the package name) that keeps the agent's history together; title: for people. */
+    agent: {
+        name: string;
+        title?: string;
+        tools: ToolSpec[];
+        channels?: ChannelSummary[];
+    };
+    /** Every file in the local suite; only these tests run. */
+    files: Array<{
+        path: string;
+        hash: string;
+    }>;
+    /** How many times each scenario runs. */
+    runs: number;
+    /** Acknowledge each message independently of the serial event uploads. */
+    deliveryReceipts?: boolean;
+    /** The client will retry a call when this cloud stores the result under Idempotency-Key. */
+    retries?: boolean;
+    /** The client opens a lost stream again with Last-Event-ID when this cloud resumes. */
+    resume?: boolean;
+    /** The client opens one new session for the unfinished runs when this cloud says session_gone. */
+    reopen?: boolean;
+    /** Play only these runs, numbered as given: a re-open after session_gone. Sent only to a cloud whose Opened said reopen,
+     * and only when some run finished: the cloud checks it against the suite's files it already stores. */
+    only?: RunKey[];
+}
+/** One run of a suite: a scenario's id, one of its channels, and which of the `runs` times (from 1). */
+export interface RunKey {
+    scenario: string;
+    channel: string;
+    run: number;
+}
+export interface Receipt {
+    conversationId: string;
+    messageId: string;
+    status: "taken" | "intercepted";
+}
+export interface Opened {
+    sessionId: string;
+    token: string;
+    /** Hashes the cloud does not have yet; upload these. */
+    missing: string[];
+    /** Where to watch this session's runs. */
+    dashboard: string;
+    /** The tunnel's absolute URL, where every later call goes (a hosted cloud's front door serves
+     * only the open). Absent or empty: the open's own address (a local cloud). */
+    tunnel?: string;
+    /** This cloud stores each Idempotency-Key with its result. Absent on an older cloud. */
+    idempotency?: boolean;
+    /** This cloud numbers stream commands with an SSE id, sends the ones after Last-Event-ID to a
+     * stream opened again, and holds them 2 minutes while none is attached. Absent on an older cloud. */
+    resume?: boolean;
+    /** This cloud answers session_gone for a session it lost, and plays `only` on an open. Absent on an older cloud. */
+    reopen?: boolean;
+}
+export interface UploadFile {
+    hash: string;
+    path: string;
+    content: string;
+}
+export type Command = {
+    type: "message";
+    conversationId: string;
+    messageId: string;
+    incoming: WireIncoming;
+    channel: ChannelInfo;
+    channelName?: string;
+} | {
+    type: "reaction";
+    conversationId: string;
+    emoji: string;
+    toText: string;
+} | {
+    type: "drain";
+} | {
+    type: "run";
+    run: RunSummary;
+} | {
+    type: "finished";
+    summary: SessionSummary;
+}
+/** The session stopped before it finished; the CLI exits with this error. */
+ | {
+    type: "failed";
+    error: string;
+};
+/**
+ * A person's message as it travels: a scenario's file goes by its path in the tests folder, and the
+ * agent's side reads it there, so the file never leaves the developer's machine.
+ */
+export type WireIncoming = Omit<Incoming, "attachments"> & {
+    attachments?: Array<{
+        kind: Attachment["kind"];
+        file: string;
+    }>;
+};
+export interface ToolCallRequest {
+    conversationId: string;
+    tool: string;
+    input: unknown;
+}
+export type ToolCallResult = {
+    ok: true;
+    output: unknown;
+} | {
+    ok: false;
+    error: string;
+};
+export interface RunSummary {
+    runId: string;
+    scenario: string;
+    run: number;
+    channel: string;
+    /** Every check passed. */
+    passed: boolean;
+    checks: Check[];
+    /** Why the person stopped: goal met, gave up, max turns, or the agent went quiet. */
+    ended: string;
+    turns: number;
+    messagesIn: number;
+    messagesOut: number;
+    toolCalls: number;
+    /** Slowest acknowledgement of a person's message, in ms (null if one was never acknowledged). */
+    slowestAckMs: number | null;
+    /** The agent's model calls in this run, from its own events, and the models that answered them. */
+    agentUsage: Usage;
+    agentModels: string[];
+    /** The test's own model calls in this run: the person, the tool simulator and the judge. */
+    testUsage: Usage;
+    watch: string;
+    /** The first try never reached the agent, so this run is the second try. */
+    retried?: boolean;
+    /** The cloud's code when it could not play or judge the run (judge_failed, spend_cap, simulator_failed, ...).
+     * The run then says nothing about the agent, whatever `passed` and `ended` say. */
+    error?: string;
+}
+export interface SessionSummary {
+    sessionId: string;
+    suiteHash: string;
+    /** The model that played the person and the tools and judged. */
+    testModel: string;
+    runs: RunSummary[];
+    dashboard: string;
+}

package/dist/testing/src/protocol.js

@@ -0,0 +1,28 @@
+// What travels through the tunnel between an agent on a developer's machine and the test cloud.
+// Every connection is opened by the agent's side (plain HTTP), so it works behind any router.
+//
+//   POST /api/sessions                 OpenSession → Opened     start a session, learn which files to upload
+//   POST /api/files                    { files: UploadFile[] }   upload the files the cloud lacks
+//   GET  /api/sessions/:id/stream      server-sent Commands     the person's side, cloud → agent
+//        Last-Event-ID: N                                    a resume: the commands after N
+//   POST /api/sessions/:id/outbound    { conversationId, outbound } → { ok } | 409 { error }
+//   POST /api/sessions/:id/tool-calls  ToolCallRequest → ToolCallResult   held open while the tool is simulated
+//   POST /api/sessions/:id/events      { seq, events: FlowEvent[] }
+//
+// Every POST after the open sends Idempotency-Key. An older cloud ignores it. When Opened
+// says idempotency, a repeat of a key is the stored answer and the client may send it again.
+//   POST /api/sessions/:id/receipts    Receipt → { ok }
+//   POST /api/sessions/:id/drained     {} → { ok }   flush before the terminal command
+//
+// When Opened says resume, every command on the stream carries `id: N` (1, 2, 3, ... per session).
+// A stream that drops, or sits silent past the idle watch, is opened again with Last-Event-ID set
+// to the last id the client handled (0 for none); the cloud sends the commands after it, and holds
+// them for 2 minutes while no stream is attached. The client skips an id it has handled.
+//
+// When Opened says reopen, a tunnel call or the stream may answer 410 { error: "session_gone" }: the
+// cloud lost the session (its tunnel restarted). The client opens one new session with `only`, the
+// runs that have no result yet, and merges the two reports. Never more than once.
+//
+// The open goes to the cloud's front door; every later call goes to Opened.tunnel (else the same
+// address) and carries `authorization: Bearer <token>` from Opened.
+export {};

package/dist/testing/src/report.d.ts

@@ -0,0 +1,74 @@
+/** How a `flow test` ended, one per exit status (0 passed; 1 failed, invalid; 2 errored, transport; 3 refused; 130 interrupted). */
+export type Outcome = "passed" | "failed" | "errored" | "transport" | "refused" | "interrupted" | "invalid";
+export type Verdict = "passed" | "failed" | "errored" | "not_played";
+export interface ReportError {
+    message: string;
+    /** The cloud's HTTP status, when it answered one. */
+    status: number | null;
+    /** The cloud's code for it, such as spend_cap or rate_limited. */
+    type: string | null;
+    retryAfterSeconds: number | null;
+}
+export interface ReportRun {
+    runId: string | null;
+    /** The scenario's file name without .yaml, as --only takes it. */
+    scenario: string;
+    channel: string;
+    run: number;
+    verdict: Verdict;
+    /** The cloud's code when it could not play or judge the run (judge_failed, spend_cap, ...); else null. */
+    error: string | null;
+    ended: string | null;
+    turns: number | null;
+    watch: string | null;
+    failedChecks: Array<{
+        kind: string;
+        text: string;
+        reason: string;
+    }>;
+}
+/** `flow test --json`: one document on stdout, whatever the exit. */
+export interface JsonReport {
+    version: 1;
+    outcome: Outcome;
+    exitCode: number;
+    sessionId: string | null;
+    /** The cloud's id for the agent, which `flow show` reads runs under. */
+    agent: string | null;
+    dashboard: string | null;
+    suiteHash: string | null;
+    error: ReportError | null;
+    counts: {
+        runs: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        notPlayed: number;
+    };
+    costUsd: {
+        agent: number;
+        test: number;
+    };
+    runs: ReportRun[];
+}
+/** A run's verdict: errored when the cloud names an error for it or ended it run_failed, since the cloud
+ * could not play or judge it and that says nothing about the agent (a judge_failed run can end goal_met). */
+export declare function runVerdict(run: {
+    passed: boolean;
+    ended: string;
+    error?: string;
+}): Exclude<Verdict, "not_played">;
+export declare const EXIT: Record<Outcome, number>;
+/** A session that ran to its summary: failed when a run failed, errored when the cloud could not play or judge a run (or never played it). */
+export declare function finishedOutcome(runs: Array<{
+    verdict: Verdict;
+}>): Outcome;
+/** What stopped a session, or the open, as an outcome. `opened`: the session had opened. */
+export declare function stoppedOutcome(error: unknown, opened: boolean): Outcome;
+/** The error as the JSON report says it: the cloud's status, code and Retry-After when it answered. */
+export declare function reportError(error: unknown, message: string): ReportError;
+/** The connection's error code, from the error or what caused it. */
+export declare function networkCode(error: unknown): string | undefined;
+/** The agent folder's .env, or the --dotenv given; variables already set win, and no value is ever printed.
+ * Returns what is wrong when a named file cannot be read. */
+export declare function loadEnv(file: string | undefined): string | undefined;