@itmetsam/samurai-framework 0.22.2 → 0.22.4

package/dist/cli/generate.js

@@ -0,0 +1,139 @@
+import { parseRunnerFlags, parseRunOverrides, } from "./args.js";
+const USAGE_LINE = 'Usage: samurai generate "<goal>" --file <spec> [--new <title>] [--test <name|index>] [--url <url>] [--model <provider:name>] [--max-steps <n>] [--yes-origin <origin>] [--secret <NAME>] [--json]';
+async function loadAgent() {
+    try {
+        return await import("../agent/index.js");
+    }
+    catch (error) {
+        if (error.code === "ERR_MODULE_NOT_FOUND")
+            throw new Error("samurai generate needs the packages ai and zod, and a provider package for your model; run: npm install ai zod @ai-sdk/anthropic (or @ai-sdk/openai, @ai-sdk/openai-compatible)");
+        throw error;
+    }
+}
+function origin(raw) {
+    try {
+        return new URL(raw).origin;
+    }
+    catch {
+        throw new Error(`--yes-origin: "${raw}" is not a URL such as https://shop.example`);
+    }
+}
+/** The model the run uses: `--model`, else `SAMURAI_AI_MODEL`, else the config's `ai.model` */
+async function chosenModel(agent, flag, env) {
+    const spec = flag ?? (await agent.defaultModelSpec(env));
+    if (!spec)
+        throw new Error("No model: pass --model provider:name (anthropic, openai or openai-compatible), set SAMURAI_AI_MODEL, or set ai.model in samurai.config.ts");
+    return {
+        spec,
+        model: await agent.modelFromSpec(spec, { projectDir: process.cwd(), env }),
+    };
+}
+/** `samurai generate "<goal>"`: lets the AI agent write a test and prints what it does. Returns the exit code */
+export async function generateCommand({ values, positionals }, io, deps = {}) {
+    const goal = positionals[0];
+    if (!goal)
+        throw new Error(`Which goal? ${USAGE_LINE}`);
+    const files = values.file ?? [];
+    if (files.length !== 1)
+        throw new Error(`Give exactly one --file <spec>. ${USAGE_LINE}`);
+    const maxSteps = values["max-steps"] === undefined ? undefined : Number(values["max-steps"]);
+    if (maxSteps !== undefined && (!Number.isInteger(maxSteps) || maxSteps < 1))
+        throw new Error(`--max-steps must be a whole number from 1, got "${values["max-steps"]}"`);
+    const flags = parseRunnerFlags(values);
+    if (flags.browsers && flags.browsers.length > 1)
+        throw new Error("--browser: the agent uses one browser");
+    const test = values.test === undefined
+        ? undefined
+        : /^\d+$/.test(values.test)
+            ? Number(values.test)
+            : values.test;
+    const allowedOrigins = (values["yes-origin"] ?? []).map(origin);
+    const agent = await (deps.loadAgent ?? loadAgent)();
+    const { spec, model } = await chosenModel(agent, values.model, io.env);
+    const json = values.json === true;
+    if (!json)
+        io.out(`Model ${spec.slice(0, spec.indexOf(":"))}: the pages the agent reads are sent to that provider.\n`);
+    const session = agent.runAgent({
+        ...parseRunOverrides(values, io.env),
+        model,
+        goal,
+        file: files[0],
+        ...(test !== undefined && { test }),
+        ...(values.new !== undefined && { create: values.new }),
+        ...(values.url !== undefined && { url: values.url }),
+        ...(flags.browsers && { browser: flags.browsers[0] }),
+        ...(flags.port !== undefined && { port: flags.port }),
+        ...(flags.headless !== undefined && { headless: flags.headless }),
+        ...(allowedOrigins.length > 0 && { allowedOrigins }),
+        ...(values.secret && { secrets: values.secret }),
+        ...(values["secret-field"] && { secretFields: values["secret-field"] }),
+        ...(maxSteps !== undefined && { limits: { maxSteps } }),
+        ...(io.signal && { signal: io.signal }),
+    });
+    let unanswerable;
+    for await (const event of session.events) {
+        if (json)
+            io.out(`${JSON.stringify(event)}\n`);
+        switch (event.type) {
+            case "plan":
+                if (!json)
+                    io.out(`${event.text}\n`);
+                break;
+            case "tool-call":
+                if (!json)
+                    io.out(`  · ${event.tool} ${summaryOf(event.args)}\n`);
+                break;
+            case "step":
+                if (!json && event.code !== undefined)
+                    io.out(`  ${event.event.op === "insert" ? "+" : "~"} ${event.code}\n`);
+                break;
+            case "question": {
+                const originQuestion = event.options?.join("/") === "allow/deny";
+                if (io.prompt && !json) {
+                    io.out(`? ${event.question}${event.options ? ` (${event.options.join("/")})` : ""}\n`);
+                    void io
+                        .prompt("> ")
+                        .then((answer) => session.answer(event.id, answer))
+                        .catch(() => undefined); // the run ended while the question was open
+                }
+                else if (originQuestion) {
+                    session.answer(event.id, "deny");
+                }
+                else {
+                    unanswerable = event.question;
+                    session.stop();
+                }
+                break;
+            }
+            case "ended":
+                if (!json)
+                    summarise(io, event.result);
+                break;
+            default:
+                break;
+        }
+    }
+    if (unanswerable !== undefined)
+        io.err(`The agent asked "${unanswerable}" and nobody can answer: stdin is not a terminal${json ? " (or --json is set)" : ""}. Run it in a terminal, or put the answer in the goal.\n`);
+    return (await session.result).reason === "finished" ? 0 : 1;
+}
+function summaryOf(args) {
+    const text = JSON.stringify(args ?? {});
+    return text.length > 100 ? `${text.slice(0, 99)}…` : text;
+}
+const ENDINGS = {
+    stopped: "Stopped.",
+    "step-limit": "Stopped at the step limit (--max-steps).",
+    "token-limit": "Stopped at the token limit.",
+    "time-limit": "Stopped at the time limit.",
+};
+function summarise(io, result) {
+    const base = `"${result.test}" has ${result.steps} ${result.steps === 1 ? "step" : "steps"} in ${result.file}.`;
+    if (result.reason === "error")
+        io.err(`\nThe run failed: ${result.error ?? "unknown error"}\n${base}\n`);
+    else
+        io.out(`\n${result.reason === "finished" ? "Done" : (ENDINGS[result.reason] ?? result.reason)} ${base}\n${result.summary ? `${result.summary}\n` : ""}`);
+    if (result.secrets.length > 0)
+        io.out(`Set ${result.secrets.map((name) => `SAMURAI_SECRET_${name}`).join(", ")} (environment variable or .env.<environment>) before running it.\n`);
+    io.out(`${result.tokens} tokens used. Review the steps before you run the test.\n`);
+}

package/dist/cli/io.d.ts

@@ -7,4 +7,6 @@ export interface Io {
     env: Partial<NodeJS.ProcessEnv>;
     /** Aborts a recording (Ctrl+C) */
     signal?: AbortSignal;
+    /** Asks the person at the terminal; set when stdin is a terminal, so a command may ask questions */
+    prompt?: (question: string) => Promise<string>;
 }

package/dist/cli/main.js

@@ -3,6 +3,7 @@ import { listTests, runTests } from "../runner/run.js";
 import { ConsoleReporter } from "./console-reporter.js";
 import { browsersCommand } from "./browsers.js";
 import { flowCommand } from "./flow.js";
+import { generateCommand } from "./generate.js";
 import { initProject } from "./init.js";
 import { recordCommand } from "./record.js";
 import { USAGE } from "./usage.js";
@@ -36,6 +37,8 @@ export async function main(argv, io) {
             io.out("\nNext: install the dependencies, then `samurai browsers install firefox`, then `samurai run`.\n");
             return 0;
         }
+        if (command === "generate")
+            return await generateCommand({ values, positionals }, io);
         if (command === "record")
             return await recordCommand({ values, positionals }, io);
         if (command !== "run" && command !== "list") {

package/dist/cli/prompt.d.ts

@@ -0,0 +1,2 @@
+/** Asks on the terminal when stdin is one; undefined otherwise (a pipe, CI), so a command knows it cannot ask */
+export declare function terminalPrompt(): ((question: string) => Promise<string>) | undefined;

package/dist/cli/prompt.js

@@ -0,0 +1,16 @@
+import { createInterface } from "node:readline";
+/** Asks on the terminal when stdin is one; undefined otherwise (a pipe, CI), so a command knows it cannot ask */
+export function terminalPrompt() {
+    if (!process.stdin.isTTY)
+        return undefined;
+    return (question) => new Promise((resolve) => {
+        const lines = createInterface({
+            input: process.stdin,
+            output: process.stdout,
+        });
+        lines.question(question, (answer) => {
+            lines.close();
+            resolve(answer);
+        });
+    });
+}

package/dist/cli/record.js

@@ -66,6 +66,8 @@ export async function recordCommand({ values, positionals }, io) {
     if (at !== undefined && (!Number.isInteger(at) || at < 0))
         throw new Error(`--at must be a step number, got "${values.at}"`);
     const flags = parseRunnerFlags(values);
+    if (flags.headless === true)
+        throw new Error("--headless: a recording needs a window; the recorder never runs without one");
     if (flags.browsers && flags.browsers.length > 1)
         throw new Error("--browser: a recording uses one browser");
     const test = values.test === undefined
@@ -81,7 +83,7 @@ export async function recordCommand({ values, positionals }, io) {
         ...(values.new !== undefined && { create: values.new }),
         ...(at !== undefined && { at }),
         ...(values.url !== undefined && { url: values.url }),
-        ...(flags.headless !== undefined && { headless: flags.headless }),
+        headless: false,
         ...(flags.port !== undefined && { port: flags.port }),
         ...(flags.browsers && { browser: flags.browsers[0] }),
         ...(io.signal && { signal: io.signal }),

package/dist/cli/usage.d.ts

@@ -1 +1 @@
-export declare const USAGE = "samurai: browser tests over WebDriver BiDi\n\nUsage:\n  samurai run [options]       Run the tests (the default command)\n  samurai list [options]      List the tests without running them\n  samurai record <spec>       Open a browser and record what you do into a spec file\n  samurai init [folder]       Scaffold a project: config, an example test, .env.example\n  samurai flow <command>      List, check and run the flows in ./flows (see below)\n  samurai browsers [command]  List, install and remove the browsers SAMURAI launches\n\nOptions for run and list:\n  --env <name>                Environment to run against (or SAMURAI_ENV)\n  --file <spec>               Only this spec file; repeatable\n  --grep <text>               Only tests whose full name contains the text\n  --group <name>              Only the tests of this group of the config's groups; repeat it for several\n  --browser <name>            Run in this browser (firefox or chrome) instead of the config's\n                              browsers; repeat it for more than one\n  --json                      Machine-readable output: JSON lines for run, one array for list\n\nOptions for run:\n  --timeout <ms>              Time one test may take\n  --expect-timeout <ms>       Time actions and assertions retry\n  --headless, --no-headless   Run the browser without a window, or with one.\n                              Default: the config's use.headless, else headless when CI is set\n  --port <n>                  Browser debugging port (default 9223)\n  --workers <n|n%>            Tests that run at the same time, each in a worker process: a number\n                              from 1, or a share of the CPUs like 50%. Default: the config's\n                              workers (or SAMURAI_WORKERS), else half the CPUs\n  --retries <n>               Run a failing test again up to n times, each with a fresh browser\n                              (or SAMURAI_RETRIES; default 0)\n  --shard <current/total>     Run one slice of the selected tests, to split a run over machines:\n                              --shard=2/4 runs the second of four. Whole files (and serial\n                              blocks) go to one shard; an empty shard succeeds\n\nOptions for record (also --env, --port, --json, --headless, --no-headless):\n  --browser <name>            Record in this browser (default: the first of the config's browsers)\n  --new <title>               Add a test with this title (creates the spec file if needed)\n  --test <name|index>         Record into this test; not needed when the file has one\n  --at <n>                    Insert from step n (default: after the last step)\n  --url <url>                 Page to start on (default: the environment's baseURL)\n  Stop with Ctrl+C or by closing the browser window.\n\n  -h, --help                  Show this help\n  -v, --version               Show the version\n\nExit code: 0 when every test passed, 1 when a test failed, 2 for a usage or configuration error.\n\nsamurai flow:\n  list                        The flows in ./flows\n  check [<flow>...]           Check flows without running them\n  run <flow>... | --all       Run flows; each writes result/flows/<id>.json\n  Options for run: --env, --headless, --timeout, --expect-timeout, --port, --json, --allow-unsupported\n  Run `samurai flow` alone for the details.\n\nsamurai browsers:\n  list                        The pinned browsers and every build in the cache (the default)\n  install [firefox|chrome|all]\n                              Download the pinned builds (default: all)\n  uninstall <name> [--build <id>] | --unused\n\nRun it in the project folder, the one holding samurai.config.ts.\n";
+export declare const USAGE = "samurai: browser tests over WebDriver BiDi\n\nUsage:\n  samurai run [options]       Run the tests (the default command)\n  samurai list [options]      List the tests without running them\n  samurai record <spec>       Open a browser and record what you do into a spec file\n  samurai generate \"<goal>\"   Let an AI agent write a test into a spec file (needs ai, zod and a provider)\n  samurai init [folder]       Scaffold a project: config, an example test, .env.example\n  samurai flow <command>      List, check and run the flows in ./flows (see below)\n  samurai browsers [command]  List, install and remove the browsers SAMURAI launches\n\nOptions for run and list:\n  --env <name>                Environment to run against (or SAMURAI_ENV)\n  --file <spec>               Only this spec file; repeatable\n  --grep <text>               Only tests whose full name contains the text\n  --group <name>              Only the tests of this group of the config's groups; repeat it for several\n  --browser <name>            Run in this browser (firefox or chrome) instead of the config's\n                              browsers; repeat it for more than one\n  --json                      Machine-readable output: JSON lines for run, one array for list\n\nOptions for run:\n  --timeout <ms>              Time one test may take\n  --expect-timeout <ms>       Time actions and assertions retry\n  --headless, --no-headless   Run the browser without a window, or with one.\n                              Default: the config's use.headless, else headless when CI is set\n  --port <n>                  Browser debugging port (default 9223)\n  --workers <n|n%>            Tests that run at the same time, each in a worker process: a number\n                              from 1, or a share of the CPUs like 50%. Default: the config's\n                              workers (or SAMURAI_WORKERS), else half the CPUs\n  --retries <n>               Run a failing test again up to n times, each with a fresh browser\n                              (or SAMURAI_RETRIES; default 0)\n  --shard <current/total>     Run one slice of the selected tests, to split a run over machines:\n                              --shard=2/4 runs the second of four. Whole files (and serial\n                              blocks) go to one shard; an empty shard succeeds\n\nOptions for record (also --env, --port, --json; always with a window):\n  --browser <name>            Record in this browser (default: the first of the config's browsers)\n  --new <title>               Add a test with this title (creates the spec file if needed)\n  --test <name|index>         Record into this test; not needed when the file has one\n  --at <n>                    Insert from step n (default: after the last step)\n  --url <url>                 Page to start on (default: the environment's baseURL)\n  Stop with Ctrl+C or by closing the browser window.\n\nOptions for generate (also --env, --browser, --port, --headless, --json):\n  --file <spec>               The spec file to write into (required, once)\n  --new <title>               Add a test with this title (creates the file if needed)\n  --test <name|index>         Write into this test; not needed when the file has one\n  --url <url>                 Page to start on (default: the environment's baseURL)\n  --model <provider:name>     anthropic, openai or openai-compatible, for example\n                              anthropic:claude-sonnet-4-5. Default: SAMURAI_AI_MODEL, then the\n                              config's ai.model. Keys: ANTHROPIC_API_KEY, OPENAI_API_KEY,\n                              SAMURAI_AI_API_KEY (+ SAMURAI_AI_BASE_URL for openai-compatible)\n  --max-steps <n>             Most model calls in the run (default 40)\n  --yes-origin <origin>       A site the agent may open without asking; repeatable\n  --secret <NAME>             A secret the agent may type (it never sees the value); repeatable\n  --secret-field <css>        A field whose value is never shown to the model; repeatable\n  Questions are asked in the terminal; without one (or with --json) an unknown site is refused\n  and any other question ends the run.\n\n  -h, --help                  Show this help\n  -v, --version               Show the version\n\nExit code: 0 when every test passed (generate: the agent finished), 1 when a test failed (generate: it ended at a limit, was stopped or failed), 2 for a usage or configuration error.\n\nsamurai flow:\n  list                        The flows in ./flows\n  check [<flow>...]           Check flows without running them\n  run <flow>... | --all       Run flows; each writes result/flows/<id>.json\n  Options for run: --env, --headless, --timeout, --expect-timeout, --port, --json, --allow-unsupported\n  Run `samurai flow` alone for the details.\n\nsamurai browsers:\n  list                        The pinned browsers and every build in the cache (the default)\n  install [firefox|chrome|all]\n                              Download the pinned builds (default: all)\n  uninstall <name> [--build <id>] | --unused\n\nRun it in the project folder, the one holding samurai.config.ts.\n";

package/dist/cli/usage.js

@@ -5,6 +5,7 @@ Usage:
   samurai run [options]       Run the tests (the default command)
   samurai list [options]      List the tests without running them
   samurai record <spec>       Open a browser and record what you do into a spec file
+  samurai generate "<goal>"   Let an AI agent write a test into a spec file (needs ai, zod and a provider)
   samurai init [folder]       Scaffold a project: config, an example test, .env.example
   samurai flow <command>      List, check and run the flows in ./flows (see below)
   samurai browsers [command]  List, install and remove the browsers SAMURAI launches
@@ -33,7 +34,7 @@ Options for run:
                               --shard=2/4 runs the second of four. Whole files (and serial
                               blocks) go to one shard; an empty shard succeeds
 
-Options for record (also --env, --port, --json, --headless, --no-headless):
+Options for record (also --env, --port, --json; always with a window):
   --browser <name>            Record in this browser (default: the first of the config's browsers)
   --new <title>               Add a test with this title (creates the spec file if needed)
   --test <name|index>         Record into this test; not needed when the file has one
@@ -41,10 +42,26 @@ Options for record (also --env, --port, --json, --headless, --no-headless):
   --url <url>                 Page to start on (default: the environment's baseURL)
   Stop with Ctrl+C or by closing the browser window.
 
+Options for generate (also --env, --browser, --port, --headless, --json):
+  --file <spec>               The spec file to write into (required, once)
+  --new <title>               Add a test with this title (creates the file if needed)
+  --test <name|index>         Write into this test; not needed when the file has one
+  --url <url>                 Page to start on (default: the environment's baseURL)
+  --model <provider:name>     anthropic, openai or openai-compatible, for example
+                              anthropic:claude-sonnet-4-5. Default: SAMURAI_AI_MODEL, then the
+                              config's ai.model. Keys: ANTHROPIC_API_KEY, OPENAI_API_KEY,
+                              SAMURAI_AI_API_KEY (+ SAMURAI_AI_BASE_URL for openai-compatible)
+  --max-steps <n>             Most model calls in the run (default 40)
+  --yes-origin <origin>       A site the agent may open without asking; repeatable
+  --secret <NAME>             A secret the agent may type (it never sees the value); repeatable
+  --secret-field <css>        A field whose value is never shown to the model; repeatable
+  Questions are asked in the terminal; without one (or with --json) an unknown site is refused
+  and any other question ends the run.
+
   -h, --help                  Show this help
   -v, --version               Show the version
 
-Exit code: 0 when every test passed, 1 when a test failed, 2 for a usage or configuration error.
+Exit code: 0 when every test passed (generate: the agent finished), 1 when a test failed (generate: it ended at a limit, was stopped or failed), 2 for a usage or configuration error.
 
 samurai flow:
   list                        The flows in ./flows

package/dist/index.js

@@ -1,4 +1,5 @@
 import { main } from "./cli/main.js";
+import { terminalPrompt } from "./cli/prompt.js";
 // Development entry (`npm run dev`, which runs `tsx src/index.ts`): the same as `samurai run`, straight from the sources
 const interrupt = new AbortController();
 process.on("SIGINT", () => {
@@ -12,4 +13,5 @@ process.exit(await main(process.argv.slice(2), {
     color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR,
     env: process.env,
     signal: interrupt.signal,
+    ...(terminalPrompt() && { prompt: terminalPrompt() }),
 }));

package/dist/recorder/normalise.d.ts

@@ -64,6 +64,12 @@ export type Observation = {
         flags: string;
     };
 }
+/** An action with no argument that a page event does not tell apart: the AI agent's hover, double click and clear */
+ | {
+    type: "action";
+    kind: "hover" | "dblclick" | "clear";
+    locator: LocatorSpec;
+}
 /** The toolbar's Page panel: the page's URL or title */
  | {
     type: "expectPage";

package/dist/recorder/normalise.js

@@ -71,6 +71,8 @@ export class Normaliser {
                     locator: observation.locator,
                     value: observation.value,
                 };
+            case "action":
+                return { kind: observation.kind, locator: observation.locator };
             case "wait-idle":
                 return { kind: "waitForNetworkIdle" };
             case "wait-time":

package/dist/recorder/session.d.ts

@@ -24,7 +24,7 @@ export interface RecordSpecOptions extends ProjectOptions {
     browser?: SupportedBrowser;
     /** The browser's debugging port. @default DEFAULT_PORT (9223) */
     port?: number;
-    /** Run the browser without a window. A person can't use it then; for tests and machines without a display. @default the config's `use.headless`, else headless on CI and a window otherwise */
+    /** Run the browser without a window. A person can't use it then, so this is only for tests of the recorder itself. Never read from the config, the environment or `CI`: a recording always opens a window. @default false */
     headless?: boolean;
     /** Aborting ends the recording, like closing the browser window */
     signal?: AbortSignal;

package/dist/recorder/session.js

@@ -28,7 +28,7 @@ export async function recordSpec(options) {
         const writer = new SpecWriter(spec.source, file, testIndex);
         const { browser, page } = await launchRecordingBrowser(browserName, {
             ...options,
-            headless: options.headless ?? settings.headless,
+            headless: options.headless ?? false,
         });
         const loaded = await loadRawConfig();
         const folder = filesFolder(projectDir(), loaded.filesDir);

package/dist/types/config.d.ts

@@ -204,6 +204,11 @@ export interface SamuraiTestConfig {
    * @default "files"
    */
   filesDir?: string;
+  /**
+   * The AI agent (`samurai generate`). `model` is a `provider:name` pair, such as `anthropic:claude-sonnet-4-5`;
+   * never a key (keys come from the environment). The `--model` flag and `SAMURAI_AI_MODEL` win over it
+   */
+  ai?: { model?: string };
   /** How pages are opened, and what the browser pretends: the options of `test.use()` apply to every test */
   use?: UseOptions &
     FixtureUse & {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itmetsam/samurai-framework",
-  "version": "0.22.2",
+  "version": "0.22.4",
   "description": "Browser test framework over WebDriver BiDi: auto-waiting locators, environments and secrets, a recorder, and a command line.",
   "main": "dist/api.js",
   "exports": {
@@ -52,6 +52,10 @@
       "types": "./dist/capabilities.d.ts",
       "default": "./dist/capabilities.js"
     },
+    "./agent": {
+      "types": "./dist/agent/index.d.ts",
+      "default": "./dist/agent/index.js"
+    },
     "./package.json": "./package.json"
   },
   "scripts": {
@@ -80,10 +84,12 @@
   "type": "module",
   "devDependencies": {
     "@types/node": "^25.6.0",
+    "ai": "^7.0.0",
     "esbuild": "^0.28.2",
     "knip": "^6.42.1",
     "madge": "^8.0.0",
-    "tsx": "^4.23.15"
+    "tsx": "^4.23.15",
+    "zod": "^4.1.8"
   },
   "dependencies": {
     "@puppeteer/browsers": "^3.2.4",
@@ -93,17 +99,26 @@
   "types": "dist/api.d.ts",
   "files": [
     "dist",
-    "bin"
+    "bin",
+    "AGENT-GUIDE.md"
   ],
   "bin": {
     "samurai": "bin/samurai.mjs"
   },
   "peerDependencies": {
-    "tsx": "^4.21.0"
+    "ai": "^7.0.0",
+    "tsx": "^4.21.0",
+    "zod": "^3.25.76 || ^4.1.8"
   },
   "peerDependenciesMeta": {
+    "ai": {
+      "optional": true
+    },
     "tsx": {
       "optional": true
+    },
+    "zod": {
+      "optional": true
     }
   },
   "repository": {