@flow-engineer/flow 0.0.1 → 0.2.1

package/dist/testing/src/show.js

@@ -0,0 +1,280 @@
+// `flow show <run-id|last>`: one run's full record from the test cloud (the transcript, the tool
+// calls the simulator played, every check with the judge's reason), so a coding agent can read why a
+// run failed without a browser. The agent's id comes from .flow/last-session.json, and the read
+// carries the agent's CI key: GET {cloud}/v1/agents/{agent}/tests/runs/{run}.
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import { OBSERVE_URL, setting } from "../../src/traces.js";
+import { RESULTS_DIR, resultsFile } from "../../src/names.js";
+import { loadEnv } from "./report.js";
+/** Where every session's results go, in the agent's folder; `flow new` git-ignores .flow/. */
+export const LAST_SESSION = join(RESULTS_DIR, "last-session.json");
+/** The last session's file in dir: LAST_SESSION, else, before a run has written it, the one from before the rename (.chinwag/). */
+export function lastSessionIn(dir) {
+    return resultsFile(dir, "last-session.json");
+}
+export const SHOW_LINE = `  flow show <run-id|last> [--json]
+                            Print one run's record from the test cloud: its checks with the
+                            judge's reasons, the transcript and the tool calls (flow show --help)`;
+const SHOW_USAGE = `Usage:
+  flow show <run-id|last> [options]
+
+Prints one run of the last \`flow test\` (or any run of this agent, by the id on its PASS or
+FAIL line): verdict, every check with the judge's reason, then the conversation in time order with
+each tool call the simulator played and what it changed. \`last\` is the first run that did not
+pass in .flow/last-session.json (its first run when all passed).
+
+Options:
+  --json                   The record exactly as the cloud answered it, on stdout
+  --cloud <url>            The test cloud (default: FLOW_FRAMEWORK_CLOUD, else https://flow.engineer/observe)
+  --dotenv <path>          Read the environment from this file instead of the folder's .env
+  -h, --help               Show this help
+
+Needs FLOW_FRAMEWORK_CLOUD_KEY (the agent's CI key, also read from .env), and the agent's id that
+\`flow test\` keeps in .flow/last-session.json.
+
+Exit status:
+  0    Printed
+  1    A wrong flag, no key or last session, no such run (404), or past the 30 days a run is kept (410)
+  2    The cloud did not answer, answered 5xx, or answered something that is not a run's record
+  3    The cloud refused the key (401, 403) or is busy (429)`;
+/** `flow show`: returns the exit status. */
+export async function showCli(args) {
+    let parsed;
+    try {
+        parsed = parseArgs({ args, allowPositionals: true, strict: true, options: {
+                json: { type: "boolean", default: false },
+                cloud: { type: "string" },
+                dotenv: { type: "string" },
+                help: { type: "boolean", short: "h", default: false },
+            } });
+    }
+    catch (error) {
+        const { code, message } = error;
+        const unknown = code === "ERR_PARSE_ARGS_UNKNOWN_OPTION" ? /'([^']+)'/.exec(message)?.[1] : undefined;
+        console.error(`flow: ${unknown ? `unknown option ${unknown}` : message.split(". ")[0]}\n\n${SHOW_USAGE}`);
+        return 1;
+    }
+    const { values, positionals } = parsed;
+    if (values.help) {
+        console.log(SHOW_USAGE);
+        return 0;
+    }
+    if (positionals.length !== 1) {
+        console.error(`flow show takes one run id, or last.\n\n${SHOW_USAGE}`);
+        return 1;
+    }
+    const envProblem = loadEnv(values.dotenv);
+    if (envProblem) {
+        console.error(envProblem);
+        return 1;
+    }
+    const cloud = (values.cloud || setting("CLOUD") || OBSERVE_URL).replace(/\/+$/, "");
+    const key = setting("CLOUD_KEY") || undefined;
+    if (!key) {
+        console.error(`flow show reads the run with the agent's CI key: set FLOW_FRAMEWORK_CLOUD_KEY.${cloud === OBSERVE_URL ? ` Make a key at ${OBSERVE_URL}: your agent, Developers.` : ""}`);
+        return 1;
+    }
+    const last = readLast();
+    if (typeof last === "string") {
+        console.error(last);
+        return 1;
+    }
+    let runId = positionals[0];
+    if (runId === "last") {
+        const pickedRun = pickLast(last);
+        if (!pickedRun) {
+            console.error(`${LAST_SESSION} has no run ids: run flow test again, then flow show last.`);
+            return 1;
+        }
+        runId = pickedRun;
+    }
+    const id = /^[A-Za-z0-9_-]{1,100}$/;
+    if (!id.test(runId)) {
+        console.error(`"${runId}" is not a run id: use the id on a run's PASS or FAIL line (trn_...), or last.`);
+        return 1;
+    }
+    if (!id.test(last.agent)) {
+        console.error(`${LAST_SESSION} names an agent id that is not one: run flow test again.`);
+        return 1;
+    }
+    const url = `${cloud}/v1/agents/${last.agent}/tests/runs/${runId}`;
+    let status;
+    let text;
+    try {
+        const response = await fetch(url, { headers: { authorization: `Bearer ${key}`, accept: "application/json" }, signal: AbortSignal.timeout(30_000) });
+        status = response.status;
+        text = await response.text();
+    }
+    catch (error) {
+        const cause = error.cause;
+        const why = error.name === "TimeoutError" ? "no answer in 30 s" : cause?.code ?? error.message;
+        console.error(`Could not reach ${cloud}: ${why}.`);
+        return 2;
+    }
+    if (status < 200 || status >= 300)
+        return refused(status, text, runId);
+    let record;
+    try {
+        record = JSON.parse(text);
+        if (!record || typeof record !== "object" || Array.isArray(record))
+            throw new Error("not an object");
+    }
+    catch {
+        console.error(`The cloud's answer for ${runId} was not a run's record (${status}).`);
+        return 2;
+    }
+    if (values.json)
+        process.stdout.write(text);
+    else
+        console.log(render(record, runId).join("\n"));
+    return 0;
+}
+/** The last session's results, or what to do when there are none or they name no agent. */
+function readLast() {
+    const path = lastSessionIn(process.cwd());
+    if (!existsSync(path))
+        return `No ${LAST_SESSION} in ${process.cwd()}: run flow test first; flow show reads the agent's id from it.`;
+    let last;
+    try {
+        last = JSON.parse(readFileSync(path, "utf8"));
+        if (!Array.isArray(last.runs))
+            throw new Error("it has no runs");
+    }
+    catch (error) {
+        return `${LAST_SESSION} could not be read (${error.message}): run flow test again.`;
+    }
+    if (!last.agent)
+        return `${LAST_SESSION} has no agent id: run flow test once more against a current cloud, which names it, then flow show.`;
+    return last;
+}
+/** The first run that did not pass, else the first run; says which on stderr and names the others that did not pass. */
+function pickLast(last) {
+    const withId = last.runs.filter((r) => r.runId);
+    const notPassed = withId.filter((r) => r.verdict === "failed" || r.verdict === "errored");
+    const chosen = notPassed[0] ?? withId[0];
+    if (!chosen)
+        return undefined;
+    console.error(`Showing ${chosen.scenario} #${chosen.run} (${chosen.channel}), ${chosen.verdict}: ${chosen.runId}`);
+    for (const r of notPassed.slice(1))
+        console.error(`  also did not pass: flow show ${r.runId}   (${r.scenario} #${r.run}, ${r.channel}, ${r.verdict})`);
+    return chosen.runId;
+}
+/** The cloud's refusal on stderr, and its exit status. */
+function refused(status, text, runId) {
+    let error;
+    try {
+        const body = JSON.parse(text).error;
+        error = typeof body === "string" ? { type: body } : body;
+    }
+    catch { /* the body was not JSON */ }
+    const said = (error?.type ? `${error.type}${error.message ? `: ${error.message}` : ""}` : text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().slice(0, 180)).replace(/\.$/, "");
+    const meaning = status === 404 ? ` No run ${runId} for this agent.` : status === 410 ? ` Run ${runId} is past the 30 days the cloud keeps runs.` : "";
+    console.error(`The cloud answered ${status}${said ? ` ${said}` : ""}.${meaning}${error?.hint ? `\n${error.hint}` : ""}`);
+    if (status === 401 || status === 403 || status === 429)
+        return 3;
+    if (status >= 500 || !error)
+        return 2;
+    return 1;
+}
+/** At most `max` characters of a value as JSON. */
+function brief(value, max = 300) {
+    const json = value === undefined ? "" : typeof value === "string" ? JSON.stringify(value) : JSON.stringify(value) ?? String(value);
+    return json.length > max ? `${json.slice(0, max)}…` : json;
+}
+/** The labels a button row or a list shows, whatever shape the record keeps them in. */
+function labels(value) {
+    if (typeof value === "string")
+        return [value];
+    if (Array.isArray(value))
+        return value.flatMap(labels);
+    if (!value || typeof value !== "object")
+        return [];
+    const o = value;
+    for (const field of ["title", "label", "text"])
+        if (typeof o[field] === "string")
+            return [o[field]];
+    return [...labels(o.sections), ...labels(o.rows)];
+}
+/** The run as text: header, checks (failed first), the conversation and tool calls in time order, then detail and the watch link. */
+function render(record, runId) {
+    const out = [];
+    const s = record.summary ?? {};
+    const checks = record.checks ?? [];
+    const passed = s.passed ?? (checks.length > 0 && checks.every((c) => c.pass));
+    // An error (judge_failed, spend_cap, ...) or run_failed: the cloud could not play or judge the run.
+    const error = record.error ?? s.error;
+    const verdict = error || s.ended === "run_failed" ? "ERROR" : passed ? "PASS" : "FAIL";
+    const where = record.scenario?.path ?? record.scenario?.id ?? "?";
+    out.push(`${verdict}  ${where}${record.scenario?.title ? ` (${record.scenario.title})` : ""}`);
+    out.push(`  channel ${record.channel ?? "?"}, run ${record.run ?? "?"}, ${record.runId ?? runId}; ended: ${s.ended ?? "?"}${s.turns === undefined ? "" : `, ${s.turns} turn${s.turns === 1 ? "" : "s"}`}`);
+    if (error)
+        out.push(`  error: ${typeof error === "string" ? error : brief(error)} (the cloud could not play or judge this run; it says nothing about the agent)`);
+    if (checks.length) {
+        out.push("", "Checks:");
+        for (const c of [...checks.filter((c) => !c.pass), ...checks.filter((c) => c.pass)]) {
+            out.push(`  ${c.pass ? "✓" : "✗"} ${c.kind ?? "check"}: ${c.text ?? ""}`);
+            if (c.reason)
+                out.push(...indent(c.reason, "      "));
+        }
+    }
+    const start = record.startedAt ?? Math.min(...(record.transcript ?? []).map((l) => l.at ?? Infinity), ...(record.toolCalls ?? []).map((t) => t.at ?? Infinity));
+    const when = (at) => (at === undefined || !Number.isFinite(start) ? "        " : `+${((at - start) / 1000).toFixed(1)}s`.padStart(8));
+    const items = [];
+    for (const line of record.transcript ?? []) {
+        const t = when(line.at);
+        let lines;
+        if (line.kind === "in") {
+            const files = (line.attachments ?? []).map((a) => {
+                const f = a;
+                return `[${f.kind ?? "file"}: ${f.file ?? f.name ?? "attached"}]`;
+            });
+            lines = indent([line.text ?? "", ...files].filter(Boolean).join(" "), `${t} person: `, " ".repeat(t.length + 9));
+        }
+        else if (line.kind === "out") {
+            const extra = [];
+            const buttons = labels(line.buttons);
+            if (buttons.length)
+                extra.push(`[buttons: ${buttons.join(" | ")}]`);
+            const list = labels(line.list);
+            if (list.length)
+                extra.push(`[list: ${list.join(" | ")}]`);
+            if (line.link)
+                extra.push(`[link: ${typeof line.link === "string" ? line.link : line.link.url ?? brief(line.link, 120)}]`);
+            if (line.image)
+                extra.push("[image]");
+            if (line.template)
+                extra.push(`[template: ${typeof line.template === "string" ? line.template : line.template.name ?? brief(line.template, 120)}]`);
+            lines = indent([line.text ?? "", ...extra].filter(Boolean).join(" "), `${t} agent: `, " ".repeat(t.length + 8));
+        }
+        else if (line.kind === "reaction") {
+            lines = [`${t} ${line.by ?? "someone"} reacted ${line.emoji ?? "?"} to ${line.messageId ?? "a message"}`];
+        }
+        else
+            continue;
+        items.push({ at: line.at ?? Infinity, order: items.length, lines });
+    }
+    for (const call of record.toolCalls ?? []) {
+        const t = when(call.at);
+        const result = call.ok === false ? `error: ${typeof call.error === "string" ? call.error : brief(call.error)}` : brief(call.output);
+        const notes = [call.kind, "simulated", call.latencyMs === undefined ? undefined : `${call.latencyMs} ms`].filter(Boolean).join(", ");
+        const lines = [`${t} tool ${call.tool ?? "?"}(${brief(call.input)}) → ${result} [${notes}]`];
+        for (const change of call.changes ?? [])
+            lines.push(`${" ".repeat(t.length)}   changes: ${change}`);
+        items.push({ at: call.at ?? Infinity, order: items.length, lines });
+    }
+    items.sort((a, b) => a.at - b.at || a.order - b.order);
+    if (items.length)
+        out.push("", "Conversation:", ...items.flatMap((i) => i.lines));
+    if (record.detail !== undefined && record.detail !== null && record.detail !== "") {
+        out.push("", ...indent(typeof record.detail === "string" ? record.detail : JSON.stringify(record.detail, null, 2), "Detail: ", "  "));
+    }
+    if (s.watch)
+        out.push("", `Watch: ${s.watch}`);
+    return out;
+}
+/** Text over several lines: the first after `first`, the rest after `rest`. */
+function indent(text, first, rest = first) {
+    return text.split("\n").map((line, i) => `${i === 0 ? first : rest}${line}`);
+}

package/dist/testing/src/suite.d.ts

@@ -0,0 +1,87 @@
+import { z } from "../../src/index.ts";
+export interface SuiteFile {
+    /** Relative to the tests folder: scenarios/latte-correction.yaml, tools/place_order.yaml. */
+    path: string;
+    hash: string;
+    content: string;
+}
+export declare function hashOf(content: string): string;
+/** Every scenario and tool file under `testsDir`, in a stable order. */
+export declare function loadSuite(testsDir: string): SuiteFile[];
+/** One hash for the whole suite: runs are compared only when it matches. */
+export declare function suiteHash(files: Array<{
+    path: string;
+    hash: string;
+}>): string;
+/** What is wrong with the `{{ now ... }}` placeholders in `text`, or null. A placeholder is the test's clock
+ * (`{{now}}`, `{{now + 7d | date}}`, `{{now + 6d @ 10:00}}`), filled by the cloud per run; any other `{{...}}` is left alone. */
+export declare function placeholderProblem(text: string): string | null;
+/** A tool call the run must (`calls`) or must not (`never`) make: a tool name, or the tool with input fields it must have. */
+declare const expectedCall: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+    tool: z.ZodString;
+    args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>]>;
+declare const scenarioSchema: z.ZodObject<{
+    id: z.ZodString;
+    title: z.ZodOptional<z.ZodString>;
+    channels: z.ZodArray<z.ZodEnum<{
+        imessage: "imessage";
+        sms: "sms";
+        telegram: "telegram";
+        web: "web";
+        whatsapp: "whatsapp";
+    }>>;
+    timezone: z.ZodOptional<z.ZodString>;
+    now: z.ZodOptional<z.ZodPreprocess<z.ZodString, unknown>>;
+    person: z.ZodObject<{
+        persona: z.ZodString;
+        goal: z.ZodString;
+        knows: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>;
+    opening: z.ZodArray<z.ZodObject<{
+        say: z.ZodDefault<z.ZodString>;
+        attach: z.ZodOptional<z.ZodObject<{
+            kind: z.ZodEnum<{
+                audio: "audio";
+                document: "document";
+                image: "image";
+            }>;
+            file: z.ZodString;
+        }, z.core.$strip>>;
+        after_ms: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    world: z.ZodOptional<z.ZodString>;
+    max_turns: z.ZodDefault<z.ZodNumber>;
+    expect: z.ZodDefault<z.ZodObject<{
+        outcome: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        judge: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        calls: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+            tool: z.ZodString;
+            args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>]>>>;
+        never: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+            tool: z.ZodString;
+            args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>]>>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const toolFileSchema: z.ZodObject<{
+    tool: z.ZodString;
+    behaviour: z.ZodOptional<z.ZodString>;
+    responses: z.ZodArray<z.ZodObject<{
+        when: z.ZodString;
+        latency_ms: z.ZodNumber;
+        input: z.ZodOptional<z.ZodUnknown>;
+        output: z.ZodOptional<z.ZodUnknown>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+/** The type of a file a scenario attaches, from its extension. */
+export declare function mimeOf(path: string): string;
+export type Scenario = z.infer<typeof scenarioSchema>;
+export type ToolFile = z.infer<typeof toolFileSchema>;
+export type ExpectedCall = z.infer<typeof expectedCall>;
+/** Parse and check a scenario. The error has one line per problem: `file:line: field: what is wrong`. */
+export declare function parseScenario(file: Pick<SuiteFile, "path" | "content">): Scenario;
+export declare function parseToolFile(file: Pick<SuiteFile, "path" | "content">): ToolFile;
+export {};

package/dist/testing/src/suite.js

@@ -0,0 +1,198 @@
+// A test suite on disk: tests/scenarios/*.yaml and tests/tools/*.yaml, each file hashed by its
+// content so the cloud stores it once and a run records exactly which tests it ran.
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { extname, isAbsolute, join } from "node:path";
+import { isMap, isSeq, LineCounter, parseDocument } from "yaml";
+import { z } from "../../src/index.js";
+export function hashOf(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+/** Every scenario and tool file under `testsDir`, in a stable order. */
+export function loadSuite(testsDir) {
+    const files = [];
+    for (const folder of ["scenarios", "tools"]) {
+        const dir = join(testsDir, folder);
+        if (!existsSync(dir))
+            continue;
+        for (const name of readdirSync(dir).filter((n) => n.endsWith(".yaml")).sort()) {
+            const content = readFileSync(join(dir, name), "utf8");
+            files.push({ path: `${folder}/${name}`, hash: hashOf(content), content });
+        }
+    }
+    return files;
+}
+/** One hash for the whole suite: runs are compared only when it matches. */
+export function suiteHash(files) {
+    return hashOf(files.map((f) => `${f.path}:${f.hash}`).sort().join("\n"));
+}
+/** `{{ now [±<n>m|h|d|w]* [@ HH:MM] [| iso|date|time|weekday] }}`, spaces optional between the parts. */
+const CLOCK = /^\s*now(?:\s*[+-]\s*\d+[mhdw])*(?:\s*@\s*(?:[01]\d|2[0-3]):[0-5]\d)?(?:\s*\|\s*(?:iso|date|time|weekday))?\s*$/;
+/** What is wrong with the `{{ now ... }}` placeholders in `text`, or null. A placeholder is the test's clock
+ * (`{{now}}`, `{{now + 7d | date}}`, `{{now + 6d @ 10:00}}`), filled by the cloud per run; any other `{{...}}` is left alone. */
+export function placeholderProblem(text) {
+    for (const [whole, inner] of text.matchAll(/\{\{([\s\S]*?)\}\}/g)) {
+        if (/^\s*now\b/.test(inner) && !CLOCK.test(inner)) {
+            return `unknown placeholder ${whole}: write it like {{now + 3d}}, {{now - 2h | time}} or {{now + 1w @ 10:00 | date}}`;
+        }
+    }
+    return null;
+}
+/** Every string in `value` whose placeholders are wrong, with where it is under `value`. */
+function* placeholderProblems(value, path = []) {
+    if (typeof value === "string") {
+        const problem = placeholderProblem(value);
+        if (problem)
+            yield { path, problem };
+    }
+    else if (Array.isArray(value)) {
+        for (let i = 0; i < value.length; i++)
+            yield* placeholderProblems(value[i], [...path, i]);
+    }
+    else if (value && typeof value === "object") {
+        for (const [key, item] of Object.entries(value))
+            yield* placeholderProblems(item, [...path, key]);
+    }
+}
+/** A value whose strings may hold placeholders, each checked. */
+const withClock = (schema) => schema.superRefine((value, ctx) => {
+    for (const { path, problem } of placeholderProblems(value))
+        ctx.addIssue({ code: "custom", message: problem, path });
+});
+/** A time zone Intl knows. */
+function isTimeZone(zone) {
+    try {
+        new Intl.DateTimeFormat("en-US", { timeZone: zone });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** YYYY-MM-DDTHH:MM[:SS], optionally Z or ±HH:MM, naming a real moment. */
+function isDateTime(text) {
+    const m = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?(?:Z|[+-](\d{2}):(\d{2}))?$/.exec(text);
+    if (!m)
+        return false;
+    const [y, mo, d, h, mi, s = 0, oh = 0, om = 0] = m.slice(1).map((part) => (part === undefined ? undefined : Number(part)));
+    const day = new Date(Date.UTC(y, mo - 1, d));
+    return day.getUTCFullYear() === y && day.getUTCMonth() === mo - 1 && day.getUTCDate() === d && h < 24 && mi < 60 && s < 60 && oh < 24 && om < 60;
+}
+/** A tool call the run must (`calls`) or must not (`never`) make: a tool name, or the tool with input fields it must have. */
+const expectedCall = z.union([
+    z.string().min(1),
+    z.strictObject({
+        tool: z.string().min(1),
+        /** Each field must equal the call's input field: nested objects in part, arrays and strings exactly. */
+        args: withClock(z.record(z.string(), z.unknown())).optional(),
+    }),
+], { error: "a tool name, or { tool: <name>, args: { <field>: <value> } }" });
+const scenarioSchema = z.object({
+    id: z.string().min(1),
+    title: z.string().optional(),
+    /** Every platform it runs on; each is a run of its own, with its own result. */
+    channels: z.array(z.enum(["sms", "whatsapp", "imessage", "telegram", "web"])).min(1),
+    /** The IANA time zone of the test's clock; UTC when left out. */
+    timezone: z.string().refine(isTimeZone, "not a time zone: use an IANA name such as America/New_York or UTC").optional(),
+    /** The test's clock when each run starts, local to `timezone` unless it has an offset; the real time when left out.
+     * YAML's default schema reads it as a string; a Date from another schema is taken as its UTC time. */
+    now: z
+        .preprocess((v) => (v instanceof Date ? v.toISOString().replace(/\.\d+Z$/, "Z") : v), z.string().refine(isDateTime, "not a date and time: write YYYY-MM-DDTHH:MM, with :SS, Z or an offset such as -04:00 if needed"))
+        .optional(),
+    person: z.object({
+        persona: withClock(z.string()),
+        goal: withClock(z.string()),
+        knows: z.array(withClock(z.string())).optional(),
+    }),
+    opening: z
+        .array(z
+        .object({
+        say: withClock(z.string()).default(""),
+        /** A file the person sends with it (say is its caption): a path in the tests folder. */
+        attach: z.object({ kind: z.enum(["image", "audio", "document"]), file: z.string().refine(inTests, "a path inside the tests folder, such as fixtures/receipt.jpg") }).optional(),
+        after_ms: z.number().int().min(0).optional(),
+    })
+        .refine((step) => step.say || step.attach, "say something, attach a file, or both"))
+        .min(1),
+    world: withClock(z.string()).optional(),
+    max_turns: z.number().int().min(1).max(50).default(8),
+    expect: z
+        .object({
+        outcome: z.array(withClock(z.string())).optional(),
+        judge: z.array(withClock(z.string())).optional(),
+        /** Each must be called at least once in the run; checked by code from the recorded calls. */
+        calls: z.array(expectedCall).optional(),
+        /** None may be called. */
+        never: z.array(expectedCall).optional(),
+    })
+        .default({}),
+});
+const toolFileSchema = z.object({
+    tool: z.string().min(1),
+    behaviour: z.string().optional(),
+    responses: z
+        .array(z.object({
+        when: z.string(),
+        latency_ms: z.number().int().min(0),
+        input: withClock(z.unknown()).optional(),
+        output: withClock(z.unknown()).optional(),
+        error: withClock(z.string()).optional(),
+    }))
+        .min(1),
+});
+/** A relative path that stays inside the tests folder. */
+function inTests(path) {
+    return !!path && !isAbsolute(path) && !path.split(/[\\/]/).includes("..");
+}
+/** The type of a file a scenario attaches, from its extension. */
+export function mimeOf(path) {
+    const types = { jpg: "image/jpeg", jpeg: "image/jpeg", png: "image/png", webp: "image/webp", gif: "image/gif", pdf: "application/pdf", ogg: "audio/ogg", opus: "audio/ogg", mp3: "audio/mpeg", m4a: "audio/mp4", wav: "audio/wav" };
+    return types[extname(path).slice(1).toLowerCase()] ?? "application/octet-stream";
+}
+/** Parse and check a scenario. The error has one line per problem: `file:line: field: what is wrong`. */
+export function parseScenario(file) {
+    return parseFile(scenarioSchema, file);
+}
+export function parseToolFile(file) {
+    return parseFile(toolFileSchema, file);
+}
+function parseFile(schema, file) {
+    const lines = new LineCounter();
+    const doc = parseDocument(file.content, { lineCounter: lines });
+    const at = (offset) => `${file.path}:${offset === undefined ? 1 : lines.linePos(offset).line}`;
+    if (doc.errors.length) {
+        // The YAML parser's own message ends with where it is and a drawing of the line; the prefix says where.
+        throw new Error(doc.errors.map((e) => `${at(e.pos[0])}: ${e.message.split("\n")[0].replace(/ at line \d+, column \d+:?$/, "")}`).join("\n"));
+    }
+    const result = schema.safeParse(doc.toJS());
+    if (result.success)
+        return result.data;
+    throw new Error(result.error.issues.map((issue) => {
+        const field = issue.path.map((key, i) => (typeof key === "number" ? `[${key}]` : `${i ? "." : ""}${String(key)}`)).join("");
+        const what = /received undefined$/.test(issue.message) ? "missing" : issue.message;
+        return `${at(offsetOf(doc.contents, issue.path))}: ${field ? `${field}: ` : ""}${what}`;
+    }).join("\n"));
+}
+/** Where the deepest part of `path` that the file has starts: the value, or for a missing key the key that holds it. */
+function offsetOf(root, path) {
+    let node = root;
+    let offset = root?.range?.[0];
+    let holder;
+    for (const key of path) {
+        if (isMap(node)) {
+            const pair = node.items.find((p) => p.key?.value === key);
+            if (!pair)
+                return holder ?? offset;
+            holder = pair.key.range?.[0];
+            node = pair.value;
+        }
+        else if (isSeq(node) && typeof key === "number" && node.items[key]) {
+            holder = undefined;
+            node = node.items[key];
+        }
+        else
+            break;
+        offset = node?.range?.[0] ?? offset;
+    }
+    return offset;
+}

package/docs/agents.md

@@ -0,0 +1,85 @@
+# Flow Framework for coding agents
+
+You are building or changing a chat agent for a person (the owner) so they can text it from their
+phone while it runs on their laptop. No tunnel, no public URL, and no key the owner copies by hand
+except their model key. Every command here runs without a terminal and never reads stdin, except
+`chat`, which reads the messages you pipe to it.
+
+Always name the package: `npx @flow-engineer/flow ...`. `npx flow` on its own fetches another npm
+package.
+
+## The flow
+
+```sh
+npx @flow-engineer/flow init                        # the starter agent in this (empty) folder; flow new <name> makes a folder
+npm install
+# the owner puts one model key in .env (OPENROUTER_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY or GEMINI_API_KEY)
+printf 'hi\nA latte please\n' | npx @flow-engineer/flow chat   # talk to it locally: no account, one message per line
+npx @flow-engineer/flow link --json                 # signs in and sets up if needed, then prints the phone link
+npm start                                           # keep it running; the owner texts it from the phone
+```
+
+- `chat` builds the agent from `src/agent.ts` and writes as its first test person. Each reply is
+  printed (`agent: ...`) once the turn is over; `--json` gives one JSON line per thing it sends.
+- `link` does the rest when the folder is not set up (no `FLOW_FRAMEWORK_JOIN_CODE` or
+  `FLOW_MESSAGING_KEY` in `.env`, or no sender for the WhatsApp or iMessage link asked for): it
+  runs `login` (opens the owner's browser itself and prints the link and code at once), then
+  `setup` (writes the keys and each open channel's sender into `.env`), then prints the link.
+  `--channel telegram|whatsapp|imessage` picks the channel (Telegram by default). Run it in the
+  agent's folder: `setup` and `link` refuse a folder with no `package.json` (`not_an_agent_folder`).
+- On WhatsApp and iMessage the agent can only reply after the owner writes first (WhatsApp: within
+  24 hours of their last message); it cannot message them first. `link --json` says so
+  (`reply_only: true`, and in `say_to_person`): tell the owner.
+- The starter's `npm start` is on Telegram, plus WhatsApp and iMessage when `.env` has their senders
+  (`FLOW_MESSAGING_WHATSAPP_SENDER`, `FLOW_MESSAGING_IMESSAGE_SENDER`). If it was running when
+  `link` added one, restart it.
+- Give the owner `say_to_person` from `link --json`, and the sign-in link and code from the
+  `pending` line while you wait.
+- `npm start` connects to Flow Messaging by an outbound event stream. Messages sent while it was
+  stopped arrive when it starts again (for as long as Flow keeps events, 30 days by default).
+
+## Exit codes (login, setup, link)
+
+| Code | Meaning | What to do |
+|---|---|---|
+| 0 | Done | Go on |
+| 1 | Usage, not signed in, not an agent's folder, or a login URL that is not https | Fix the command, or run `next` |
+| 2 | flow.engineer could not be reached, or did not answer in time (`timeout`) | Try again |
+| 3 | Refused: denied, expired, a product failed, a channel not open yet, a server with no CLI sign-in (`server_unsupported`), or a free-trial limit (`sandbox_allowance_used`, the iMessage people cap; HTTP 403) | Run `next`; for a trial limit `next` is a sentence: give it to the owner as it is |
+| 4 | Sign-in still waiting for the owner's approval, or flow.engineer asked to wait (`slow_down`, HTTP 429) | Approval: ask the owner to approve, then run `next` (the same code is kept). `slow_down`: wait `retry_after` seconds (printed as `Wait <n> s`), then run `next` |
+
+`login` and `link` wait at most `--timeout` seconds (120 by default) before exit 4, and no single
+request waits longer than 30 s. `login --wait` (or `link` again) goes on with the same code: no new
+code and no new browser tab. flow.engineer decides when a code has expired (an approved one can be
+collected a while after its `expires_in`); a waiting code that has expired is replaced by one new
+sign-in, and so is a code from another login URL.
+
+## The `next` lines
+
+Every error is `{"error":{"code","message","next"}}` on stdout with `--json`, and
+`<message> Next: <next>` on stderr. A waiting sign-in is `{"status":"pending",...,"next":...}`.
+`next` is the exact command or URL to run; when it came from flow.engineer it is the server's.
+Run it, or give it to the owner when it needs them (a browser approval, a page to open). A
+`slow_down` error also has `retry_after` (seconds, or `null` when the server gave none). A
+trial limit's `next` is a sentence, not a command: it is printed on its own line as it came; pass
+it on to the owner unchanged.
+
+flow.engineer may add `warnings` (up to 20 lines) to any setup answer. Each is printed on its own
+line on stderr; with `--json` setup's line has `warnings` (an empty list when there are none), and
+an error line has it when there were any. Pass them on to the owner.
+
+`setup` also shows the free trial: messages used per channel, and on iMessage the people it may
+reach (`iMessage: 1 of 2 people`; `trial.imessage.contacts` `{limit, used}` in `--json`).
+
+## Keys
+
+- Never ask the owner to paste a key into the chat, and never print, read aloud or commit `.env`.
+  `setup` writes the keys itself and prints only their last 4 characters.
+- The model key is the owner's: they type it into `.env` themselves.
+- `setup` keeps every value already in `.env` (`--replace` takes the new ones), but a name there
+  with no value (`cp .env.example .env`) counts as missing and is filled on its own line. Keys
+  flow.engineer made are written even when the rest of setup fails, since they are shown only once.
+  `.env` is set to mode 0600.
+- The sign-in token goes only to the server that issued it, over https (http only to localhost):
+  with `FLOW_FRAMEWORK_LOGIN_URL` naming another server, commands stop with `signed_in_elsewhere`.
+- CI never signs in: `npx @flow-engineer/flow setup --ci` says how to give it the test cloud key.

package/examples/cafe/agent/AGENTS.md

@@ -0,0 +1,10 @@
+# Pip
+
+You are Pip, the ordering assistant for Bean & Leaf, a small café. People text you to order drinks and food for pickup, ask what's on the menu, and check on their order.
+
+- Use get_menu for anything about items, sizes, milks or prices; never guess a price.
+- Before placing an order, read it back once with the total and ask them to confirm. Place it with place_order only after they say yes.
+- When the order is placed, tell them the order number and when it will be ready.
+- Keep messages short and friendly, like a barista texting.
+- Anything about allergies or dietary needs: load the allergens skill first.
+- You can't take payment by text; they pay at pickup.