@itmetsam/samurai-framework 0.22.3 → 0.22.5

package/dist/agent/recording.js

@@ -0,0 +1,78 @@
+import path from "node:path";
+import { projectDir } from "../config/config.js";
+import { Normaliser, } from "../recorder/normalise.js";
+import { prepareSpec, readSpec, SpecWriter } from "../recorder/spec-source.js";
+import { stepToSource } from "../steps/emit.js";
+/** The statement an event wrote, `undefined` for a removal */
+function codeOf(event) {
+    return event.op === "remove" ? undefined : stepToSource(event.step);
+}
+/** What a tool tells the model about the steps it wrote */
+export function writtenText(events) {
+    const codes = events.flatMap((event) => codeOf(event) ?? []);
+    return codes.length > 0 ? ` Step written: ${codes.join(" ")}` : "";
+}
+/**
+ * Writes the agent's steps the way a recording does: an observation goes through the recorder's normaliser and
+ * `SpecWriter`, so the file is a valid spec after every call and holds only typed steps.
+ */
+export class StepRecorder {
+    constructor(options) {
+        this.options = options;
+        this.file = path.resolve(projectDir(), options.file);
+        const spec = prepareSpec(this.file, readSpec(this.file, options), options);
+        this.testName = spec.testName;
+        this.writer = new SpecWriter(spec.source, this.file, spec.testIndex);
+        this.steps = spec.at;
+        this.first = spec.at;
+        this.normaliser = this.fresh();
+    }
+    fresh() {
+        return new Normaliser({
+            at: this.steps,
+            ...(this.options.baseURL !== undefined && {
+                baseURL: this.options.baseURL,
+            }),
+        });
+    }
+    /** Steps in the test now */
+    get count() {
+        return this.steps;
+    }
+    /** The events written so far */
+    get events() {
+        return this.writer.events;
+    }
+    write(event) {
+        const source = this.writer.write(event);
+        if (event.op === "insert")
+            this.steps++;
+        else if (event.op === "remove")
+            this.steps--;
+        this.options.onStep?.(event, codeOf(event), source);
+    }
+    /** Writes what `observation` makes of the spec and returns the events, which hold the statements */
+    observe(observation) {
+        const events = this.normaliser.observe(observation, Date.now());
+        for (const event of events)
+            this.write(event);
+        return events;
+    }
+    /** Takes out the last `count` steps (at most the ones written in this run) and returns how many went */
+    undo(count) {
+        const removed = Math.max(0, Math.min(count, this.steps - this.first));
+        for (let i = 0; i < removed; i++)
+            this.write({ op: "remove", index: this.steps - 1 });
+        if (removed > 0)
+            this.normaliser = this.fresh();
+        return removed;
+    }
+    /** Writes an event a person's recording produced while the agent was not acting (see `rebase`) */
+    external(event) {
+        this.write(event);
+    }
+    /** Starts the agent's own normaliser again after the person's steps: it carries on after the last one */
+    rebase() {
+        this.normaliser = this.fresh();
+    }
+}

package/dist/agent/run-kind.d.ts

@@ -0,0 +1,48 @@
+import type Page from "../browser/page.js";
+import type { PromptInput } from "./prompt.js";
+import type { PersonRecording } from "./take-over.js";
+import type { Snapshotter } from "./snapshot.js";
+import type { AgentTool, ToolContext, ToolSink } from "./tools/context.js";
+import type { AgentEvent, BaseAgentOptions } from "./types.js";
+/** What a run keeps of its work: the spec's steps, or the plan's. `AgentResult` is made from it */
+export interface RunSink extends ToolSink {
+    /** Steps written so far */
+    readonly count: number;
+    /** The file the run writes */
+    readonly file: string;
+    /** Name of the test, or title of the plan */
+    readonly testName: string;
+}
+/** What the sink is made from */
+interface SinkInput {
+    baseURL: string | undefined;
+    emit(event: AgentEvent): void;
+}
+/**
+ * What a run swaps for another kind of run: what it records, which tools it has, what the prompt says and what
+ * guards the page. The browser, the limits, the origin policy, the controller and the event stream stay the same.
+ */
+export interface RunKind<O extends BaseAgentOptions = BaseAgentOptions, S extends RunSink = RunSink> {
+    /** The file the result names when no sink was made (the run failed before) */
+    file(options: O): string;
+    /** Makes the sink; throws a sentence when the file cannot be used. Runs before the browser opens */
+    sink(options: O, input: SinkInput): S;
+    /** The page is open on `url`: a first step may be recorded */
+    opened?(sink: S, url: string): void;
+    tools(context: ToolContext, sink: S): AgentTool[];
+    prompt(input: PromptInput, options: O): string;
+    /** The first message after the goal: the page as it is now, or that there is none */
+    opening(options: O, first: string): string;
+    /** Names the secrets the written steps read */
+    secrets(sink: S): string[];
+    /** Runs once the page is open, before the model's first call */
+    guard?(page: Page, sink: S): Promise<void>;
+    /** What the person's use of the browser does to the record */
+    person(input: {
+        page: Page;
+        sink: S;
+        snapshots: Snapshotter;
+        baseURL?: string;
+    }): PersonRecording;
+}
+export {};

package/dist/agent/run-kind.js

@@ -0,0 +1 @@
+export {};

package/dist/agent/run.d.ts

@@ -0,0 +1,15 @@
+import type Page from "../browser/page.js";
+import type { RunKind, RunSink } from "./run-kind.js";
+import { type AiModule } from "./sdk-tools.js";
+import type { AgentOptions, AgentSession, BaseAgentOptions } from "./types.js";
+/** What a run needs from outside, replaced in tests */
+export interface RunDeps {
+    importAi?: () => Promise<AiModule>;
+    /** Called with the page once the browser is open (a test plays the person with it) */
+    onPage?: (page: Page) => void;
+}
+/**
+ * Starts a run; its work goes on in the background and `result` settles when it ends. The kind says what the run
+ * records, which tools it has and what guards the page; the default writes a spec.
+ */
+export declare function startAgent<O extends BaseAgentOptions = AgentOptions, S extends RunSink = RunSink>(options: O, deps?: RunDeps, kind?: RunKind<O, S>): AgentSession;

package/dist/agent/run.js

@@ -0,0 +1,201 @@
+import { loadRawConfig, projectDir } from "../config/config.js";
+import { maskDeep, maskText } from "../config/mask.js";
+import { launchRecordingBrowser, prepareRecordingBrowser, } from "../recorder/session-browser.js";
+import { fixtureCatalog } from "../runner/fixture-registry.js";
+import { withProject } from "../runner/project.js";
+import { loadInstructions } from "./instructions.js";
+import { resolveLimits } from "./limits.js";
+import { runLoop } from "./loop.js";
+import { assertModelObject } from "./model.js";
+import { generateKind } from "./generate-kind.js";
+import { toSdkTools } from "./sdk-tools.js";
+import { AgentController } from "./session.js";
+import { Snapshotter } from "./snapshot.js";
+import { OriginPolicy } from "./tools/origins.js";
+const ACTION_TIMEOUT = 5000;
+async function loadAi(deps) {
+    try {
+        return await (deps.importAi ?? (() => import("ai")))();
+    }
+    catch (error) {
+        const missing = error.code;
+        throw new Error(missing === "ERR_MODULE_NOT_FOUND" || deps.importAi
+            ? 'agent(): the "ai" package is not installed. Install it, and a provider package for your model, with: npm install ai zod @ai-sdk/anthropic'
+            : `agent(): could not load the "ai" package: ${error.message}`);
+    }
+}
+/** The fixtures of the project's config, for the prompt; the agent works without them */
+async function fixturesOf(options) {
+    try {
+        const config = options.config ?? (await loadRawConfig());
+        return fixtureCatalog(config.fixtures ?? []);
+    }
+    catch {
+        return [];
+    }
+}
+function endReason(controller, tally, limit, failed) {
+    const stopped = controller.stopReason;
+    if (stopped)
+        return stopped;
+    if (failed)
+        return "error";
+    if (tally.finished)
+        return "finished";
+    return limit ?? "finished";
+}
+/**
+ * Starts a run; its work goes on in the background and `result` settles when it ends. The kind says what the run
+ * records, which tools it has and what guards the page; the default writes a spec.
+ */
+export function startAgent(options, deps = {}, kind = generateKind) {
+    assertModelObject(options.model);
+    const limits = resolveLimits(options.limits);
+    const person = {};
+    const controller = new AgentController({
+        begin: () => person.hooks?.begin() ?? Promise.resolve(),
+        end: () => person.hooks?.end() ?? Promise.resolve(""),
+    }, options.onEvent);
+    options.signal?.addEventListener("abort", () => controller.stop(), {
+        once: true,
+    });
+    if (options.signal?.aborted)
+        controller.stop();
+    const tally = { modelCalls: 0, tokens: 0, finished: false };
+    const done = { text: "" };
+    const work = async () => {
+        const ai = await loadAi(deps);
+        await withProject(options, async (run) => {
+            const { settings } = run;
+            const instructions = loadInstructions(projectDir());
+            const startUrl = options.url ?? settings.baseURL;
+            const prepared = await prepareRecordingBrowser({
+                ...(options.browser && { browser: options.browser }),
+                ...(options.port !== undefined && { port: options.port }),
+                signal: controller.signal,
+            });
+            const steps = kind.sink(options, {
+                baseURL: settings.baseURL,
+                emit: (event) => controller.emit(event),
+            });
+            done.steps = steps;
+            const { browser, page } = await launchRecordingBrowser(prepared, {
+                ...(options.port !== undefined && { port: options.port }),
+                signal: controller.signal,
+                headless: options.headless ?? false,
+            });
+            try {
+                deps.onPage?.(page);
+                if (options.url !== undefined)
+                    await page.goto(options.url);
+                else if (settings.baseURL !== undefined)
+                    await page.navigateTo(settings.baseURL);
+                const current = await page.url();
+                const onWeb = /^https?:/.test(current);
+                const snapshots = new Snapshotter(page, {
+                    secretFields: options.secretFields ?? [],
+                });
+                if (onWeb)
+                    kind.opened?.(steps, current);
+                const origins = new OriginPolicy([
+                    ...(options.allowedOrigins ?? []),
+                    ...(onWeb ? [current] : startUrl ? [startUrl] : []),
+                ]);
+                person.hooks = kind.person({
+                    page,
+                    sink: steps,
+                    snapshots,
+                    ...(settings.baseURL !== undefined && { baseURL: settings.baseURL }),
+                });
+                const context = {
+                    page,
+                    snapshots,
+                    steps,
+                    origins,
+                    secrets: {
+                        allowed: options.secrets ?? [],
+                        get: (name) => run.secrets.get(name),
+                    },
+                    vision: options.vision === true,
+                    actionTimeout: ACTION_TIMEOUT,
+                    ask: (question, choices) => controller.ask(question, choices),
+                    finish: (summary) => {
+                        done.summary = summary;
+                        tally.finished = true;
+                    },
+                    emit: (event) => controller.emit(event),
+                    secretOnScreen: false,
+                };
+                await kind.guard?.(page, steps);
+                const system = kind.prompt({
+                    instructions: instructions.text,
+                    fixtures: await fixturesOf(options),
+                    secrets: options.secrets ?? [],
+                    origins: origins.list(),
+                    vision: options.vision === true,
+                }, options);
+                const first = onWeb
+                    ? `The browser is open on ${current}. This is the page now:\n\n${maskText((await snapshots.take()).text)}`
+                    : "No page is open yet: start with navigate.";
+                const tools = toSdkTools(ai, kind.tools(context, steps), {
+                    wait: () => controller.gate(),
+                    note: async () => controller.takeNote(),
+                });
+                const outcome = await runLoop({
+                    ai,
+                    model: options.model,
+                    system,
+                    prompt: kind.opening(options, first),
+                    tools,
+                    controller,
+                    limits,
+                    tally,
+                });
+                done.text = outcome.text;
+                if (outcome.limit)
+                    done.limit = outcome.limit;
+            }
+            finally {
+                await person.hooks?.dispose().catch(() => undefined);
+                await browser.close().catch(() => undefined);
+            }
+        });
+    };
+    const stopTimer = controller.watchTime(limits.timeoutMs);
+    const result = work()
+        .then(() => false)
+        .catch((error) => {
+        done.error = maskText(error instanceof Error ? error.message : String(error));
+        return true;
+    })
+        .then(async (failed) => {
+        stopTimer();
+        await controller.settle();
+        const steps = done.steps;
+        const reason = endReason(controller, tally, done.limit, failed);
+        const summary = done.summary ?? (done.text.trim() || undefined);
+        const ended = maskDeep({
+            reason,
+            ...(reason === "finished" && summary !== undefined && { summary }),
+            ...(reason === "error" &&
+                done.error !== undefined && { error: done.error }),
+            file: steps?.file ?? kind.file(options),
+            test: steps?.testName ?? "",
+            steps: steps?.count ?? 0,
+            secrets: steps ? kind.secrets(steps) : [],
+            tokens: tally.tokens,
+        });
+        controller.end(ended);
+        return ended;
+    });
+    return {
+        events: controller.events,
+        result,
+        answer: (id, text) => controller.answer(id, text),
+        pause: () => controller.pause(),
+        resume: () => controller.resume(),
+        takeOver: () => controller.takeOver(),
+        handBack: () => controller.handBack(),
+        stop: () => controller.stop(),
+    };
+}

package/dist/agent/sdk-tools.d.ts

@@ -0,0 +1,15 @@
+import type { AgentTool } from "./tools/context.js";
+/** The AI SDK, loaded when a run starts: it is an optional peer dependency */
+export type AiModule = typeof import("ai");
+/** What stands between the model's call and the tool: waiting while paused, and the note after a hand back */
+export interface ToolGate {
+    /** Resolves when the agent may act; rejects when the run ended */
+    wait(): Promise<void>;
+    /** Text to give the model instead of running its call, once, after the person used the browser */
+    note(): Promise<string | undefined>;
+}
+/**
+ * The tools in the AI SDK's shape. The SDK is given the JSON schema only: it would answer a wrong call with a
+ * dump of the validation library's issues, so the tools validate themselves and answer in a sentence.
+ */
+export declare function toSdkTools(ai: AiModule, tools: readonly AgentTool[], gate: ToolGate): Record<string, unknown>;

package/dist/agent/sdk-tools.js

@@ -0,0 +1,52 @@
+function modelOutput(result) {
+    if (!result.ok)
+        return { type: "error-text", value: result.error };
+    if (!result.image)
+        return { type: "text", value: result.message };
+    return {
+        type: "content",
+        value: [
+            { type: "text", text: result.message },
+            {
+                type: "file",
+                data: { type: "data", data: result.image.data },
+                mediaType: result.image.mediaType,
+            },
+        ],
+    };
+}
+/**
+ * The tools in the AI SDK's shape. The SDK is given the JSON schema only: it would answer a wrong call with a
+ * dump of the validation library's issues, so the tools validate themselves and answer in a sentence.
+ */
+export function toSdkTools(ai, tools, gate) {
+    // A model may call several tools in one step and the SDK runs them at once. They share one page and one spec,
+    // so they run one after the other, in the order the model made them
+    let queue = Promise.resolve();
+    const inOrder = (work) => {
+        const turn = queue.then(work, work);
+        queue = turn.catch(() => undefined);
+        return turn;
+    };
+    const entries = tools.map((agentTool) => {
+        const schema = ai.jsonSchema(() => ai.zodSchema(agentTool.inputSchema).jsonSchema, {
+            validate: (value) => ({ success: true, value }),
+        });
+        return [
+            agentTool.name,
+            ai.tool({
+                description: agentTool.description,
+                inputSchema: schema,
+                execute: (input) => inOrder(async () => {
+                    await gate.wait();
+                    const note = await gate.note();
+                    if (note !== undefined)
+                        return { ok: false, error: note };
+                    return agentTool.run(input);
+                }),
+                toModelOutput: ({ output }) => modelOutput(output),
+            }),
+        ];
+    });
+    return Object.fromEntries(entries);
+}

package/dist/agent/session.d.ts

@@ -0,0 +1,62 @@
+import type { AgentEvent, AgentResult } from "./types.js";
+export type AbortReason = "stopped" | "time-limit" | "token-limit";
+/** What taking the browser over starts and ends: the recording of what the person does */
+export interface TakeOverHooks {
+    begin(): Promise<void>;
+    /** Ends the recording; the text tells the agent what the person did */
+    end(): Promise<string>;
+}
+/**
+ * The state of one run that the person controls: the event stream, pause and take over, the open questions, stop,
+ * and the active time. Time spent paused, taken over or waiting for an answer does not count against the time
+ * limit: it is the person's time, not the agent's.
+ */
+export declare class AgentController {
+    private readonly hooks;
+    private readonly onEvent?;
+    private readonly log;
+    private readonly readers;
+    private closed;
+    private mode;
+    private transition;
+    private readonly gateWaiters;
+    private readonly aborter;
+    private abortReason;
+    private readonly questions;
+    private asked;
+    private note;
+    private readonly startedAt;
+    private idleSince;
+    private idleTotal;
+    constructor(hooks: TakeOverHooks, onEvent?: ((event: AgentEvent) => void) | undefined);
+    /** Aborts when the run is stopped or a limit is reached */
+    get signal(): AbortSignal;
+    get stopReason(): AbortReason | undefined;
+    get events(): AsyncIterable<AgentEvent>;
+    private read;
+    emit(event: AgentEvent): void;
+    /** Emits `ended` once; later calls change nothing */
+    end(result: AgentResult): void;
+    /** Time the agent has been working, in ms */
+    activeMs(now?: number): number;
+    private touch;
+    private wake;
+    /** Resolves when the agent may act; rejects when the run was stopped or hit a limit */
+    gate(): Promise<void>;
+    pause(): void;
+    resume(): void;
+    takeOver(): void;
+    handBack(): void;
+    /** The note for the agent about what the person did, once, after a hand back */
+    takeNote(): string | undefined;
+    /** Waits for a hand back that is still ending the person's recording (the run is ending anyway) */
+    settle(): Promise<void>;
+    /** Asks the person; waits without counting against the time limit. Rejects when the run stops */
+    ask(question: string, options?: string[]): Promise<string>;
+    answer(id: string, text: string): void;
+    /** Ends the run early: `stop()` from the person, or a limit */
+    abort(reason: AbortReason): void;
+    stop(): void;
+    /** Aborts with `time-limit` once the active time passes `timeoutMs`. Returns the function that stops the check */
+    watchTime(timeoutMs: number): () => void;
+}

package/dist/agent/session.js

@@ -0,0 +1,196 @@
+import { maskDeep } from "../config/mask.js";
+/**
+ * The state of one run that the person controls: the event stream, pause and take over, the open questions, stop,
+ * and the active time. Time spent paused, taken over or waiting for an answer does not count against the time
+ * limit: it is the person's time, not the agent's.
+ */
+export class AgentController {
+    constructor(hooks, onEvent) {
+        this.hooks = hooks;
+        this.onEvent = onEvent;
+        this.log = [];
+        this.readers = new Set();
+        this.closed = false;
+        this.mode = "running";
+        this.transition = Promise.resolve();
+        this.gateWaiters = new Set();
+        this.aborter = new AbortController();
+        this.questions = new Map();
+        this.asked = 0;
+        this.startedAt = Date.now();
+        this.idleTotal = 0;
+    }
+    /** Aborts when the run is stopped or a limit is reached */
+    get signal() {
+        return this.aborter.signal;
+    }
+    get stopReason() {
+        return this.abortReason;
+    }
+    get events() {
+        return { [Symbol.asyncIterator]: () => this.read() };
+    }
+    async *read() {
+        for (let index = 0;;) {
+            while (index < this.log.length)
+                yield this.log[index++];
+            if (this.closed)
+                return;
+            await new Promise((resolve) => {
+                const wake = () => {
+                    this.readers.delete(wake);
+                    resolve();
+                };
+                this.readers.add(wake);
+            });
+        }
+    }
+    emit(event) {
+        if (this.closed)
+            return;
+        const safe = maskDeep(event);
+        this.log.push(safe);
+        try {
+            this.onEvent?.(safe);
+        }
+        catch {
+            // a failing listener must not stop the run
+        }
+        for (const wake of [...this.readers])
+            wake();
+    }
+    /** Emits `ended` once; later calls change nothing */
+    end(result) {
+        if (this.closed)
+            return;
+        this.emit({ type: "ended", result });
+        this.closed = true;
+        for (const wake of [...this.readers])
+            wake();
+    }
+    /** Time the agent has been working, in ms */
+    activeMs(now = Date.now()) {
+        const idle = this.idleSince === undefined ? 0 : now - this.idleSince;
+        return now - this.startedAt - this.idleTotal - idle;
+    }
+    touch() {
+        const idle = this.mode !== "running" || this.questions.size > 0;
+        const now = Date.now();
+        if (idle && this.idleSince === undefined)
+            this.idleSince = now;
+        if (!idle && this.idleSince !== undefined) {
+            this.idleTotal += now - this.idleSince;
+            this.idleSince = undefined;
+        }
+    }
+    wake() {
+        for (const resume of [...this.gateWaiters])
+            resume();
+        this.gateWaiters.clear();
+    }
+    /** Resolves when the agent may act; rejects when the run was stopped or hit a limit */
+    async gate() {
+        for (;;) {
+            this.signal.throwIfAborted();
+            await this.transition;
+            this.signal.throwIfAborted();
+            if (this.mode === "running")
+                return;
+            await new Promise((resolve) => this.gateWaiters.add(resolve));
+        }
+    }
+    pause() {
+        if (this.mode !== "running")
+            return;
+        this.mode = "paused";
+        this.touch();
+    }
+    resume() {
+        if (this.mode !== "paused")
+            return;
+        this.mode = "running";
+        this.touch();
+        this.wake();
+    }
+    takeOver() {
+        if (this.mode === "taken-over")
+            return;
+        this.mode = "taken-over";
+        this.touch();
+        this.transition = this.transition
+            .then(() => this.hooks.begin())
+            .catch(() => undefined);
+    }
+    handBack() {
+        if (this.mode !== "taken-over")
+            return;
+        this.transition = this.transition
+            .then(async () => {
+            this.note = await this.hooks.end();
+        })
+            .catch(() => undefined)
+            .then(() => {
+            this.mode = "running";
+            this.touch();
+            this.wake();
+        });
+    }
+    /** The note for the agent about what the person did, once, after a hand back */
+    takeNote() {
+        const note = this.note;
+        this.note = undefined;
+        return note;
+    }
+    /** Waits for a hand back that is still ending the person's recording (the run is ending anyway) */
+    settle() {
+        return this.transition;
+    }
+    /** Asks the person; waits without counting against the time limit. Rejects when the run stops */
+    ask(question, options) {
+        const id = `q${++this.asked}`;
+        return new Promise((resolve, reject) => {
+            this.questions.set(id, {
+                answer: (text) => resolve(text),
+                fail: reject,
+            });
+            this.touch();
+            this.emit({
+                type: "question",
+                id,
+                question,
+                ...(options && options.length > 0 && { options }),
+            });
+        }).finally(() => {
+            this.questions.delete(id);
+            this.touch();
+        });
+    }
+    answer(id, text) {
+        const open = this.questions.get(id);
+        if (!open)
+            throw new Error(`agent(): there is no open question ${id}`);
+        open.answer(text);
+    }
+    /** Ends the run early: `stop()` from the person, or a limit */
+    abort(reason) {
+        if (this.abortReason)
+            return;
+        this.abortReason = reason;
+        this.aborter.abort(new Error(`agent run ended: ${reason}`));
+        for (const open of this.questions.values())
+            open.fail(new Error("agent run ended"));
+        this.wake();
+    }
+    stop() {
+        this.abort("stopped");
+    }
+    /** Aborts with `time-limit` once the active time passes `timeoutMs`. Returns the function that stops the check */
+    watchTime(timeoutMs) {
+        const timer = setInterval(() => {
+            if (this.activeMs() >= timeoutMs)
+                this.abort("time-limit");
+        }, Math.min(250, Math.max(10, timeoutMs / 4)));
+        timer.unref();
+        return () => clearInterval(timer);
+    }
+}