@flow-engineer/flow 0.0.1 → 0.2.0

package/dist/testing/src/report.js

@@ -0,0 +1,90 @@
+// What `flow test --json` prints, how an error that stopped a session is classified into an
+// outcome and an exit status, and the pieces `flow test` and `flow show` share.
+import { existsSync } from "node:fs";
+import { CloudEnded, isTransportFailure, PostError, SessionStopped } from "./client.js";
+/** A run's verdict: errored when the cloud names an error for it or ended it run_failed, since the cloud
+ * could not play or judge it and that says nothing about the agent (a judge_failed run can end goal_met). */
+export function runVerdict(run) {
+    if (run.error || run.ended === "run_failed")
+        return "errored";
+    return run.passed ? "passed" : "failed";
+}
+export const EXIT = { passed: 0, failed: 1, invalid: 1, errored: 2, transport: 2, refused: 3, interrupted: 130 };
+/** A session that ran to its summary: failed when a run failed, errored when the cloud could not play or judge a run (or never played it). */
+export function finishedOutcome(runs) {
+    if (runs.some((r) => r.verdict === "failed"))
+        return "failed";
+    if (runs.some((r) => r.verdict !== "passed"))
+        return "errored";
+    return "passed";
+}
+/** What stopped a session, or the open, as an outcome. `opened`: the session had opened. */
+export function stoppedOutcome(error, opened) {
+    if (error instanceof SessionStopped)
+        return "transport";
+    if (error instanceof CloudEnded)
+        return error.code === "spend_cap" || error.code === "capacity" ? "refused" : "errored";
+    const refusal = postErrorOf(error);
+    if (refusal) {
+        const why = refusal.type ?? refusal.code;
+        if (refusal.status === 503 && why === "capacity")
+            return "refused";
+        if (refusal.status >= 400 && refusal.status < 500 && why)
+            return "refused";
+        return "transport";
+    }
+    if (isTransportFailure(error) || networkCode(error))
+        return "transport";
+    // Something on this machine: the agent would not build before the open, or broke the session after it.
+    return opened ? "failed" : "invalid";
+}
+/** The error as the JSON report says it: the cloud's status, code and Retry-After when it answered. */
+export function reportError(error, message) {
+    const refusal = postErrorOf(error);
+    const type = refusal ? refusal.type ?? refusal.code ?? null : error instanceof CloudEnded ? error.code ?? null : null;
+    const retry = refusal?.retryAfterMs;
+    return { message, status: refusal?.status ?? null, type, retryAfterSeconds: retry === undefined ? null : Math.ceil(retry / 1000) };
+}
+function postErrorOf(error) {
+    let current = error;
+    for (let guard = 0; current instanceof Error && guard < 6; guard++) {
+        if (current instanceof PostError)
+            return current;
+        current = current.cause;
+    }
+    return undefined;
+}
+/** The connection's error code, from the error or what caused it. */
+export function networkCode(error) {
+    let current = error;
+    for (let guard = 0; current instanceof Error && guard < 6; guard++) {
+        const code = current.code;
+        if (code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "ETIMEDOUT" || code === "ECONNRESET")
+            return code;
+        for (const inner of current instanceof AggregateError ? current.errors : []) {
+            const innerCode = networkCode(inner);
+            if (innerCode)
+                return innerCode;
+        }
+        current = current.cause;
+    }
+    return undefined;
+}
+/** The agent folder's .env, or the --dotenv given; variables already set win, and no value is ever printed.
+ * Returns what is wrong when a named file cannot be read. */
+export function loadEnv(file) {
+    if (file === undefined) {
+        if (existsSync(".env"))
+            process.loadEnvFile(".env");
+        return undefined;
+    }
+    if (!existsSync(file))
+        return `--dotenv ${file}: no such file.`;
+    try {
+        process.loadEnvFile(file);
+    }
+    catch (error) {
+        return `--dotenv ${file} could not be read: ${error.code ?? "unreadable"}.`;
+    }
+    return undefined;
+}

package/dist/testing/src/show.d.ts

@@ -0,0 +1,36 @@
+/** Where every session's results go, in the agent's folder; `flow new` git-ignores .flow/. */
+export declare const LAST_SESSION: string;
+/** The last session's file in dir: LAST_SESSION, else, before a run has written it, the one from before the rename (.chinwag/). */
+export declare function lastSessionIn(dir: string): string;
+/** .flow/last-session.json: what each run ended as, for `--failed`, `flow show` and coding agents. */
+export interface LastSession {
+    sessionId: string;
+    /** The cloud's id for the agent, from the open (an older cloud names none). */
+    agent?: string;
+    dashboard: string;
+    suiteHash: string;
+    /** When the session ended, as an ISO time. */
+    endedAt: string;
+    ended: "finished" | "stopped" | "interrupted";
+    /** The scenarios this session was limited to (--only, --failed); null when it ran the whole suite. */
+    only: string[] | null;
+    runs: Array<{
+        /** The scenario's file name without .yaml, as --only takes it. */
+        scenario: string;
+        channel: string;
+        run: number;
+        /** The cloud's id for the run, for `flow show`; null when it was not played. */
+        runId?: string | null;
+        verdict: "passed" | "failed" | "errored" | "not_played";
+        /** The checks as the test file says them, with the judge's reason. The file is local and
+         * git-ignored, and the suite is synthetic by attestation; the messages themselves are never kept. */
+        failedChecks: Array<{
+            kind: string;
+            text: string;
+            reason?: string;
+        }>;
+    }>;
+}
+export declare const SHOW_LINE = "  flow show <run-id|last> [--json]\n                            Print one run's record from the test cloud: its checks with the\n                            judge's reasons, the transcript and the tool calls (flow show --help)";
+/** `flow show`: returns the exit status. */
+export declare function showCli(args: string[]): Promise<number>;

package/dist/testing/src/show.js

@@ -0,0 +1,280 @@
+// `flow show <run-id|last>`: one run's full record from the test cloud (the transcript, the tool
+// calls the simulator played, every check with the judge's reason), so a coding agent can read why a
+// run failed without a browser. The agent's id comes from .flow/last-session.json, and the read
+// carries the agent's CI key: GET {cloud}/v1/agents/{agent}/tests/runs/{run}.
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import { OBSERVE_URL, setting } from "../../src/traces.js";
+import { RESULTS_DIR, resultsFile } from "../../src/names.js";
+import { loadEnv } from "./report.js";
+/** Where every session's results go, in the agent's folder; `flow new` git-ignores .flow/. */
+export const LAST_SESSION = join(RESULTS_DIR, "last-session.json");
+/** The last session's file in dir: LAST_SESSION, else, before a run has written it, the one from before the rename (.chinwag/). */
+export function lastSessionIn(dir) {
+    return resultsFile(dir, "last-session.json");
+}
+export const SHOW_LINE = `  flow show <run-id|last> [--json]
+                            Print one run's record from the test cloud: its checks with the
+                            judge's reasons, the transcript and the tool calls (flow show --help)`;
+const SHOW_USAGE = `Usage:
+  flow show <run-id|last> [options]
+
+Prints one run of the last \`flow test\` (or any run of this agent, by the id on its PASS or
+FAIL line): verdict, every check with the judge's reason, then the conversation in time order with
+each tool call the simulator played and what it changed. \`last\` is the first run that did not
+pass in .flow/last-session.json (its first run when all passed).
+
+Options:
+  --json                   The record exactly as the cloud answered it, on stdout
+  --cloud <url>            The test cloud (default: FLOW_FRAMEWORK_CLOUD, else https://flow.engineer/observe)
+  --dotenv <path>          Read the environment from this file instead of the folder's .env
+  -h, --help               Show this help
+
+Needs FLOW_FRAMEWORK_CLOUD_KEY (the agent's CI key, also read from .env), and the agent's id that
+\`flow test\` keeps in .flow/last-session.json.
+
+Exit status:
+  0    Printed
+  1    A wrong flag, no key or last session, no such run (404), or past the 30 days a run is kept (410)
+  2    The cloud did not answer, answered 5xx, or answered something that is not a run's record
+  3    The cloud refused the key (401, 403) or is busy (429)`;
+/** `flow show`: returns the exit status. */
+export async function showCli(args) {
+    let parsed;
+    try {
+        parsed = parseArgs({ args, allowPositionals: true, strict: true, options: {
+                json: { type: "boolean", default: false },
+                cloud: { type: "string" },
+                dotenv: { type: "string" },
+                help: { type: "boolean", short: "h", default: false },
+            } });
+    }
+    catch (error) {
+        const { code, message } = error;
+        const unknown = code === "ERR_PARSE_ARGS_UNKNOWN_OPTION" ? /'([^']+)'/.exec(message)?.[1] : undefined;
+        console.error(`flow: ${unknown ? `unknown option ${unknown}` : message.split(". ")[0]}\n\n${SHOW_USAGE}`);
+        return 1;
+    }
+    const { values, positionals } = parsed;
+    if (values.help) {
+        console.log(SHOW_USAGE);
+        return 0;
+    }
+    if (positionals.length !== 1) {
+        console.error(`flow show takes one run id, or last.\n\n${SHOW_USAGE}`);
+        return 1;
+    }
+    const envProblem = loadEnv(values.dotenv);
+    if (envProblem) {
+        console.error(envProblem);
+        return 1;
+    }
+    const cloud = (values.cloud || setting("CLOUD") || OBSERVE_URL).replace(/\/+$/, "");
+    const key = setting("CLOUD_KEY") || undefined;
+    if (!key) {
+        console.error(`flow show reads the run with the agent's CI key: set FLOW_FRAMEWORK_CLOUD_KEY.${cloud === OBSERVE_URL ? ` Make a key at ${OBSERVE_URL}: your agent, Developers.` : ""}`);
+        return 1;
+    }
+    const last = readLast();
+    if (typeof last === "string") {
+        console.error(last);
+        return 1;
+    }
+    let runId = positionals[0];
+    if (runId === "last") {
+        const pickedRun = pickLast(last);
+        if (!pickedRun) {
+            console.error(`${LAST_SESSION} has no run ids: run flow test again, then flow show last.`);
+            return 1;
+        }
+        runId = pickedRun;
+    }
+    const id = /^[A-Za-z0-9_-]{1,100}$/;
+    if (!id.test(runId)) {
+        console.error(`"${runId}" is not a run id: use the id on a run's PASS or FAIL line (trn_...), or last.`);
+        return 1;
+    }
+    if (!id.test(last.agent)) {
+        console.error(`${LAST_SESSION} names an agent id that is not one: run flow test again.`);
+        return 1;
+    }
+    const url = `${cloud}/v1/agents/${last.agent}/tests/runs/${runId}`;
+    let status;
+    let text;
+    try {
+        const response = await fetch(url, { headers: { authorization: `Bearer ${key}`, accept: "application/json" }, signal: AbortSignal.timeout(30_000) });
+        status = response.status;
+        text = await response.text();
+    }
+    catch (error) {
+        const cause = error.cause;
+        const why = error.name === "TimeoutError" ? "no answer in 30 s" : cause?.code ?? error.message;
+        console.error(`Could not reach ${cloud}: ${why}.`);
+        return 2;
+    }
+    if (status < 200 || status >= 300)
+        return refused(status, text, runId);
+    let record;
+    try {
+        record = JSON.parse(text);
+        if (!record || typeof record !== "object" || Array.isArray(record))
+            throw new Error("not an object");
+    }
+    catch {
+        console.error(`The cloud's answer for ${runId} was not a run's record (${status}).`);
+        return 2;
+    }
+    if (values.json)
+        process.stdout.write(text);
+    else
+        console.log(render(record, runId).join("\n"));
+    return 0;
+}
+/** The last session's results, or what to do when there are none or they name no agent. */
+function readLast() {
+    const path = lastSessionIn(process.cwd());
+    if (!existsSync(path))
+        return `No ${LAST_SESSION} in ${process.cwd()}: run flow test first; flow show reads the agent's id from it.`;
+    let last;
+    try {
+        last = JSON.parse(readFileSync(path, "utf8"));
+        if (!Array.isArray(last.runs))
+            throw new Error("it has no runs");
+    }
+    catch (error) {
+        return `${LAST_SESSION} could not be read (${error.message}): run flow test again.`;
+    }
+    if (!last.agent)
+        return `${LAST_SESSION} has no agent id: run flow test once more against a current cloud, which names it, then flow show.`;
+    return last;
+}
+/** The first run that did not pass, else the first run; says which on stderr and names the others that did not pass. */
+function pickLast(last) {
+    const withId = last.runs.filter((r) => r.runId);
+    const notPassed = withId.filter((r) => r.verdict === "failed" || r.verdict === "errored");
+    const chosen = notPassed[0] ?? withId[0];
+    if (!chosen)
+        return undefined;
+    console.error(`Showing ${chosen.scenario} #${chosen.run} (${chosen.channel}), ${chosen.verdict}: ${chosen.runId}`);
+    for (const r of notPassed.slice(1))
+        console.error(`  also did not pass: flow show ${r.runId}   (${r.scenario} #${r.run}, ${r.channel}, ${r.verdict})`);
+    return chosen.runId;
+}
+/** The cloud's refusal on stderr, and its exit status. */
+function refused(status, text, runId) {
+    let error;
+    try {
+        const body = JSON.parse(text).error;
+        error = typeof body === "string" ? { type: body } : body;
+    }
+    catch { /* the body was not JSON */ }
+    const said = (error?.type ? `${error.type}${error.message ? `: ${error.message}` : ""}` : text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().slice(0, 180)).replace(/\.$/, "");
+    const meaning = status === 404 ? ` No run ${runId} for this agent.` : status === 410 ? ` Run ${runId} is past the 30 days the cloud keeps runs.` : "";
+    console.error(`The cloud answered ${status}${said ? ` ${said}` : ""}.${meaning}${error?.hint ? `\n${error.hint}` : ""}`);
+    if (status === 401 || status === 403 || status === 429)
+        return 3;
+    if (status >= 500 || !error)
+        return 2;
+    return 1;
+}
+/** At most `max` characters of a value as JSON. */
+function brief(value, max = 300) {
+    const json = value === undefined ? "" : typeof value === "string" ? JSON.stringify(value) : JSON.stringify(value) ?? String(value);
+    return json.length > max ? `${json.slice(0, max)}…` : json;
+}
+/** The labels a button row or a list shows, whatever shape the record keeps them in. */
+function labels(value) {
+    if (typeof value === "string")
+        return [value];
+    if (Array.isArray(value))
+        return value.flatMap(labels);
+    if (!value || typeof value !== "object")
+        return [];
+    const o = value;
+    for (const field of ["title", "label", "text"])
+        if (typeof o[field] === "string")
+            return [o[field]];
+    return [...labels(o.sections), ...labels(o.rows)];
+}
+/** The run as text: header, checks (failed first), the conversation and tool calls in time order, then detail and the watch link. */
+function render(record, runId) {
+    const out = [];
+    const s = record.summary ?? {};
+    const checks = record.checks ?? [];
+    const passed = s.passed ?? (checks.length > 0 && checks.every((c) => c.pass));
+    // An error (judge_failed, spend_cap, ...) or run_failed: the cloud could not play or judge the run.
+    const error = record.error ?? s.error;
+    const verdict = error || s.ended === "run_failed" ? "ERROR" : passed ? "PASS" : "FAIL";
+    const where = record.scenario?.path ?? record.scenario?.id ?? "?";
+    out.push(`${verdict}  ${where}${record.scenario?.title ? ` (${record.scenario.title})` : ""}`);
+    out.push(`  channel ${record.channel ?? "?"}, run ${record.run ?? "?"}, ${record.runId ?? runId}; ended: ${s.ended ?? "?"}${s.turns === undefined ? "" : `, ${s.turns} turn${s.turns === 1 ? "" : "s"}`}`);
+    if (error)
+        out.push(`  error: ${typeof error === "string" ? error : brief(error)} (the cloud could not play or judge this run; it says nothing about the agent)`);
+    if (checks.length) {
+        out.push("", "Checks:");
+        for (const c of [...checks.filter((c) => !c.pass), ...checks.filter((c) => c.pass)]) {
+            out.push(`  ${c.pass ? "✓" : "✗"} ${c.kind ?? "check"}: ${c.text ?? ""}`);
+            if (c.reason)
+                out.push(...indent(c.reason, "      "));
+        }
+    }
+    const start = record.startedAt ?? Math.min(...(record.transcript ?? []).map((l) => l.at ?? Infinity), ...(record.toolCalls ?? []).map((t) => t.at ?? Infinity));
+    const when = (at) => (at === undefined || !Number.isFinite(start) ? "        " : `+${((at - start) / 1000).toFixed(1)}s`.padStart(8));
+    const items = [];
+    for (const line of record.transcript ?? []) {
+        const t = when(line.at);
+        let lines;
+        if (line.kind === "in") {
+            const files = (line.attachments ?? []).map((a) => {
+                const f = a;
+                return `[${f.kind ?? "file"}: ${f.file ?? f.name ?? "attached"}]`;
+            });
+            lines = indent([line.text ?? "", ...files].filter(Boolean).join(" "), `${t} person: `, " ".repeat(t.length + 9));
+        }
+        else if (line.kind === "out") {
+            const extra = [];
+            const buttons = labels(line.buttons);
+            if (buttons.length)
+                extra.push(`[buttons: ${buttons.join(" | ")}]`);
+            const list = labels(line.list);
+            if (list.length)
+                extra.push(`[list: ${list.join(" | ")}]`);
+            if (line.link)
+                extra.push(`[link: ${typeof line.link === "string" ? line.link : line.link.url ?? brief(line.link, 120)}]`);
+            if (line.image)
+                extra.push("[image]");
+            if (line.template)
+                extra.push(`[template: ${typeof line.template === "string" ? line.template : line.template.name ?? brief(line.template, 120)}]`);
+            lines = indent([line.text ?? "", ...extra].filter(Boolean).join(" "), `${t} agent: `, " ".repeat(t.length + 8));
+        }
+        else if (line.kind === "reaction") {
+            lines = [`${t} ${line.by ?? "someone"} reacted ${line.emoji ?? "?"} to ${line.messageId ?? "a message"}`];
+        }
+        else
+            continue;
+        items.push({ at: line.at ?? Infinity, order: items.length, lines });
+    }
+    for (const call of record.toolCalls ?? []) {
+        const t = when(call.at);
+        const result = call.ok === false ? `error: ${typeof call.error === "string" ? call.error : brief(call.error)}` : brief(call.output);
+        const notes = [call.kind, "simulated", call.latencyMs === undefined ? undefined : `${call.latencyMs} ms`].filter(Boolean).join(", ");
+        const lines = [`${t} tool ${call.tool ?? "?"}(${brief(call.input)}) → ${result} [${notes}]`];
+        for (const change of call.changes ?? [])
+            lines.push(`${" ".repeat(t.length)}   changes: ${change}`);
+        items.push({ at: call.at ?? Infinity, order: items.length, lines });
+    }
+    items.sort((a, b) => a.at - b.at || a.order - b.order);
+    if (items.length)
+        out.push("", "Conversation:", ...items.flatMap((i) => i.lines));
+    if (record.detail !== undefined && record.detail !== null && record.detail !== "") {
+        out.push("", ...indent(typeof record.detail === "string" ? record.detail : JSON.stringify(record.detail, null, 2), "Detail: ", "  "));
+    }
+    if (s.watch)
+        out.push("", `Watch: ${s.watch}`);
+    return out;
+}
+/** Text over several lines: the first after `first`, the rest after `rest`. */
+function indent(text, first, rest = first) {
+    return text.split("\n").map((line, i) => `${i === 0 ? first : rest}${line}`);
+}

package/dist/testing/src/suite.d.ts

@@ -0,0 +1,87 @@
+import { z } from "../../src/index.ts";
+export interface SuiteFile {
+    /** Relative to the tests folder: scenarios/latte-correction.yaml, tools/place_order.yaml. */
+    path: string;
+    hash: string;
+    content: string;
+}
+export declare function hashOf(content: string): string;
+/** Every scenario and tool file under `testsDir`, in a stable order. */
+export declare function loadSuite(testsDir: string): SuiteFile[];
+/** One hash for the whole suite: runs are compared only when it matches. */
+export declare function suiteHash(files: Array<{
+    path: string;
+    hash: string;
+}>): string;
+/** What is wrong with the `{{ now ... }}` placeholders in `text`, or null. A placeholder is the test's clock
+ * (`{{now}}`, `{{now + 7d | date}}`, `{{now + 6d @ 10:00}}`), filled by the cloud per run; any other `{{...}}` is left alone. */
+export declare function placeholderProblem(text: string): string | null;
+/** A tool call the run must (`calls`) or must not (`never`) make: a tool name, or the tool with input fields it must have. */
+declare const expectedCall: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+    tool: z.ZodString;
+    args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>]>;
+declare const scenarioSchema: z.ZodObject<{
+    id: z.ZodString;
+    title: z.ZodOptional<z.ZodString>;
+    channels: z.ZodArray<z.ZodEnum<{
+        imessage: "imessage";
+        sms: "sms";
+        telegram: "telegram";
+        web: "web";
+        whatsapp: "whatsapp";
+    }>>;
+    timezone: z.ZodOptional<z.ZodString>;
+    now: z.ZodOptional<z.ZodPreprocess<z.ZodString, unknown>>;
+    person: z.ZodObject<{
+        persona: z.ZodString;
+        goal: z.ZodString;
+        knows: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>;
+    opening: z.ZodArray<z.ZodObject<{
+        say: z.ZodDefault<z.ZodString>;
+        attach: z.ZodOptional<z.ZodObject<{
+            kind: z.ZodEnum<{
+                audio: "audio";
+                document: "document";
+                image: "image";
+            }>;
+            file: z.ZodString;
+        }, z.core.$strip>>;
+        after_ms: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    world: z.ZodOptional<z.ZodString>;
+    max_turns: z.ZodDefault<z.ZodNumber>;
+    expect: z.ZodDefault<z.ZodObject<{
+        outcome: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        judge: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        calls: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+            tool: z.ZodString;
+            args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>]>>>;
+        never: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+            tool: z.ZodString;
+            args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>]>>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const toolFileSchema: z.ZodObject<{
+    tool: z.ZodString;
+    behaviour: z.ZodOptional<z.ZodString>;
+    responses: z.ZodArray<z.ZodObject<{
+        when: z.ZodString;
+        latency_ms: z.ZodNumber;
+        input: z.ZodOptional<z.ZodUnknown>;
+        output: z.ZodOptional<z.ZodUnknown>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+/** The type of a file a scenario attaches, from its extension. */
+export declare function mimeOf(path: string): string;
+export type Scenario = z.infer<typeof scenarioSchema>;
+export type ToolFile = z.infer<typeof toolFileSchema>;
+export type ExpectedCall = z.infer<typeof expectedCall>;
+/** Parse and check a scenario. The error has one line per problem: `file:line: field: what is wrong`. */
+export declare function parseScenario(file: Pick<SuiteFile, "path" | "content">): Scenario;
+export declare function parseToolFile(file: Pick<SuiteFile, "path" | "content">): ToolFile;
+export {};