@itmetsam/samurai-framework 0.22.3 → 0.22.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT-GUIDE.md +50 -0
- package/bin/samurai.mjs +2 -0
- package/dist/agent/generate-kind.d.ts +5 -0
- package/dist/agent/generate-kind.js +40 -0
- package/dist/agent/index.d.ts +19 -0
- package/dist/agent/index.js +23 -0
- package/dist/agent/instructions.d.ts +13 -0
- package/dist/agent/instructions.js +44 -0
- package/dist/agent/limits.d.ts +12 -0
- package/dist/agent/limits.js +18 -0
- package/dist/agent/loop.d.ts +32 -0
- package/dist/agent/loop.js +46 -0
- package/dist/agent/model.d.ts +9 -0
- package/dist/agent/model.js +14 -0
- package/dist/agent/plan/guard.d.ts +7 -0
- package/dist/agent/plan/guard.js +32 -0
- package/dist/agent/plan/plan-file.d.ts +59 -0
- package/dist/agent/plan/plan-file.js +201 -0
- package/dist/agent/plan/plan-kind.d.ts +5 -0
- package/dist/agent/plan/plan-kind.js +28 -0
- package/dist/agent/plan/prompt.d.ts +3 -0
- package/dist/agent/plan/prompt.js +37 -0
- package/dist/agent/plan/read-plan.d.ts +5 -0
- package/dist/agent/plan/read-plan.js +28 -0
- package/dist/agent/plan/recorder.d.ts +30 -0
- package/dist/agent/plan/recorder.js +60 -0
- package/dist/agent/plan/run.d.ts +8 -0
- package/dist/agent/plan/run.js +10 -0
- package/dist/agent/plan/tools.d.ts +4 -0
- package/dist/agent/plan/tools.js +95 -0
- package/dist/agent/prompt.d.ts +21 -0
- package/dist/agent/prompt.js +60 -0
- package/dist/agent/providers.d.ts +19 -0
- package/dist/agent/providers.js +95 -0
- package/dist/agent/recording.d.ts +45 -0
- package/dist/agent/recording.js +78 -0
- package/dist/agent/run-kind.d.ts +48 -0
- package/dist/agent/run-kind.js +1 -0
- package/dist/agent/run.d.ts +15 -0
- package/dist/agent/run.js +201 -0
- package/dist/agent/sdk-tools.d.ts +15 -0
- package/dist/agent/sdk-tools.js +52 -0
- package/dist/agent/session.d.ts +62 -0
- package/dist/agent/session.js +196 -0
- package/dist/agent/snapshot-script.d.ts +18 -0
- package/dist/agent/snapshot-script.js +163 -0
- package/dist/agent/snapshot.d.ts +54 -0
- package/dist/agent/snapshot.js +95 -0
- package/dist/agent/take-over.d.ts +19 -0
- package/dist/agent/take-over.js +49 -0
- package/dist/agent/tools/act.d.ts +2 -0
- package/dist/agent/tools/act.js +124 -0
- package/dist/agent/tools/ask.d.ts +1 -0
- package/dist/agent/tools/ask.js +19 -0
- package/dist/agent/tools/context.d.ts +70 -0
- package/dist/agent/tools/context.js +42 -0
- package/dist/agent/tools/expect.d.ts +1 -0
- package/dist/agent/tools/expect.js +152 -0
- package/dist/agent/tools/finish.d.ts +1 -0
- package/dist/agent/tools/finish.js +12 -0
- package/dist/agent/tools/index.d.ts +3 -0
- package/dist/agent/tools/index.js +23 -0
- package/dist/agent/tools/navigate.d.ts +1 -0
- package/dist/agent/tools/navigate.js +53 -0
- package/dist/agent/tools/origin-guard.d.ts +6 -0
- package/dist/agent/tools/origin-guard.js +19 -0
- package/dist/agent/tools/origins.d.ts +11 -0
- package/dist/agent/tools/origins.js +22 -0
- package/dist/agent/tools/screenshot.d.ts +1 -0
- package/dist/agent/tools/screenshot.js +39 -0
- package/dist/agent/tools/snapshot.d.ts +1 -0
- package/dist/agent/tools/snapshot.js +12 -0
- package/dist/agent/tools/undo.d.ts +1 -0
- package/dist/agent/tools/undo.js +23 -0
- package/dist/agent/tools/use-secret.d.ts +1 -0
- package/dist/agent/tools/use-secret.js +41 -0
- package/dist/agent/types.d.ts +128 -0
- package/dist/agent/types.js +1 -0
- package/dist/capabilities.d.ts +2 -0
- package/dist/capabilities.js +2 -0
- package/dist/cli/agent-run.d.ts +43 -0
- package/dist/cli/agent-run.js +132 -0
- package/dist/cli/args.d.ts +8 -0
- package/dist/cli/args.js +8 -0
- package/dist/cli/generate.d.ts +6 -0
- package/dist/cli/generate.js +48 -0
- package/dist/cli/io.d.ts +2 -0
- package/dist/cli/main.js +6 -0
- package/dist/cli/plan.d.ts +6 -0
- package/dist/cli/plan.js +28 -0
- package/dist/cli/prompt.d.ts +2 -0
- package/dist/cli/prompt.js +16 -0
- package/dist/cli/usage.d.ts +1 -1
- package/dist/cli/usage.js +26 -1
- package/dist/index.js +2 -0
- package/dist/recorder/normalise.d.ts +6 -0
- package/dist/recorder/normalise.js +2 -0
- package/dist/types/config.d.ts +5 -0
- package/package.json +19 -4
package/AGENT-GUIDE.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# SAMURAI for coding agents
|
|
2
|
+
|
|
3
|
+
A short guide for an AI coding agent that writes or runs tests with `@itmetsam/samurai-framework`. It describes what the package does today; the full guides are in the [wiki](https://github.com/degrootsam/samurai-framework/wiki).
|
|
4
|
+
|
|
5
|
+
## The project
|
|
6
|
+
|
|
7
|
+
- A project has `samurai.config.ts` (environments, base URL, groups), specs under its `srcDir` (`*.spec.ts`), and `.env.<environment>` files for secrets. Run every command in the project folder.
|
|
8
|
+
- A spec is a normal TypeScript file:
|
|
9
|
+
|
|
10
|
+
```ts
|
|
11
|
+
import { expect, test } from "@itmetsam/samurai-framework";
|
|
12
|
+
|
|
13
|
+
test("signs in", async ({ page, secrets }) => {
|
|
14
|
+
await page.goto("/login");
|
|
15
|
+
await page.getByLabel("Email").fill("ann@example.com");
|
|
16
|
+
await page.getByLabel("Password").fill(secrets.PASSWORD);
|
|
17
|
+
await page.getByRole("button", { name: "Sign in" }).click();
|
|
18
|
+
await expect(page.getByRole("heading", { name: "Dashboard" })).toBeVisible();
|
|
19
|
+
});
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
- Fixtures are `page`, `browser`, `browserName`, `env` (variables of the environment) and `secrets` (names are `UPPER_SNAKE_CASE`; set as `SAMURAI_SECRET_<NAME>` or in `.env.<environment>`). Never write a secret value into a spec.
|
|
23
|
+
|
|
24
|
+
## Writing steps the editor can read
|
|
25
|
+
|
|
26
|
+
A spec made of these statements can be read and edited by SAMURAI's step codec (anything else is kept as a `custom` step): `page.goto`, `click`, `fill`, `press`, `check`, `uncheck`, `selectOption`, `hover`, `dblclick`, `clear`, `setInputFiles`, `dropFiles`, `expect(...)` with the common matchers, `expect(page).toHaveURL` / `toHaveTitle`, `outputs.save`, `waitForNetworkIdle`, `waitForTimeout`, `page.screenshot({ name })`. Locators are chains of `getByRole`, `getByLabel`, `getByText`, `getByPlaceholder`, `getByAltText`, `getByTitle`, `getByTestId`, `getByCss` and `locator(xpath)`, with `nth`, `first`, `last` and `filter`. Prefer, in this order: test id, role and name, label, text, then CSS. Wiki: Steps and the recorder, Locators, Assertions.
|
|
27
|
+
|
|
28
|
+
## Commands
|
|
29
|
+
|
|
30
|
+
| Command | Does |
|
|
31
|
+
| ----------------------------------------------------------- | ----------------------------------------------------------------------------------- |
|
|
32
|
+
| `samurai list --json` | An array of `{ "name", "file" }` (plus `"modifier"`, `"reason"`); no browser starts |
|
|
33
|
+
| `samurai run --json` | JSON lines, one event per line, while the tests run |
|
|
34
|
+
| `samurai run --file tests/login.spec.ts --env staging` | Run one spec in one environment (`--grep`, `--group`, `--browser` also select) |
|
|
35
|
+
| `samurai record <spec> --new "title" --json` | Open a browser and record into a spec; one JSON event per step |
|
|
36
|
+
| `samurai flow list` / `check` / `run <flow>` (all `--json`) | Flows: graphs of tests |
|
|
37
|
+
| `samurai browsers install all` | Download the pinned Firefox and Chrome |
|
|
38
|
+
|
|
39
|
+
Exit codes: `0` everything passed, `1` a test failed (or a run ended early), `2` a usage or configuration error. A failing run prints what failed; fix the cause, then run that one spec again.
|
|
40
|
+
|
|
41
|
+
## Reading results
|
|
42
|
+
|
|
43
|
+
`samurai run` writes `result/report.json` and prints the same events with `--json` (`run-start`, `test-start`, `test-end` with a `result`, `run-end` with a `summary`). A test result has a `status` (`success`, `failed` or `skipped`), a `duration` and, for a failure, a flat `type` (`error`, `assertion` or `timeout`), `message` and `stack`; a test that passed on a retry is marked `flaky`. Secrets are masked in everything written.
|
|
44
|
+
|
|
45
|
+
## Rules of thumb
|
|
46
|
+
|
|
47
|
+
- Open pages with a path (`page.goto("/login")`); the base URL comes from the environment.
|
|
48
|
+
- Assertions retry on their own; do not add fixed waits. Use `waitForNetworkIdle` only when nothing on the page can be asserted.
|
|
49
|
+
- Keep tests short and independent; share setup with fixtures, not with the order of tests.
|
|
50
|
+
- Run `samurai list` after writing a spec to see that it was read, then run it.
|
package/bin/samurai.mjs
CHANGED
|
@@ -40,6 +40,7 @@ try {
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
const { main } = await import("../dist/cli/main.js");
|
|
43
|
+
const { terminalPrompt } = await import("../dist/cli/prompt.js");
|
|
43
44
|
// Ctrl+C (or a termination signal) ends a recording or a flow run cleanly (the browser is closed, the file is complete); a second one stops at once
|
|
44
45
|
const interrupt = new AbortController();
|
|
45
46
|
for (const signal of ["SIGINT", "SIGTERM"]) {
|
|
@@ -54,5 +55,6 @@ const code = await main(process.argv.slice(2), {
|
|
|
54
55
|
color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR,
|
|
55
56
|
env: process.env,
|
|
56
57
|
signal: interrupt.signal,
|
|
58
|
+
...(terminalPrompt() && { prompt: terminalPrompt() }),
|
|
57
59
|
});
|
|
58
60
|
process.exit(code);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { StepRecorder } from "./recording.js";
|
|
2
|
+
import type { RunKind } from "./run-kind.js";
|
|
3
|
+
import type { AgentOptions } from "./types.js";
|
|
4
|
+
/** The run of `runAgent`: typed steps go into a spec file */
|
|
5
|
+
export declare const generateKind: RunKind<AgentOptions, StepRecorder>;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { resultSecrets } from "../recorder/session-secrets.js";
|
|
2
|
+
import { buildSystemPrompt } from "./prompt.js";
|
|
3
|
+
import { StepRecorder } from "./recording.js";
|
|
4
|
+
import { personRecording } from "./take-over.js";
|
|
5
|
+
import { createTools } from "./tools/index.js";
|
|
6
|
+
/** The run of `runAgent`: typed steps go into a spec file */
|
|
7
|
+
export const generateKind = {
|
|
8
|
+
file: (options) => options.file,
|
|
9
|
+
sink: (options, { baseURL, emit }) => new StepRecorder({
|
|
10
|
+
file: options.file,
|
|
11
|
+
create: options.create,
|
|
12
|
+
test: options.test,
|
|
13
|
+
baseURL,
|
|
14
|
+
onStep: (event, code, source) => emit({
|
|
15
|
+
type: "step",
|
|
16
|
+
event,
|
|
17
|
+
...(code !== undefined && { code }),
|
|
18
|
+
source,
|
|
19
|
+
}),
|
|
20
|
+
}),
|
|
21
|
+
opened: (steps, url) => {
|
|
22
|
+
if (steps.count === 0)
|
|
23
|
+
steps.observe({ type: "goto", url });
|
|
24
|
+
},
|
|
25
|
+
tools: (context) => createTools(context),
|
|
26
|
+
prompt: (input, options) => buildSystemPrompt({
|
|
27
|
+
...input,
|
|
28
|
+
...(options.plan !== undefined && { hasPlan: true }),
|
|
29
|
+
}),
|
|
30
|
+
opening: (options, first) => options.plan !== undefined
|
|
31
|
+
? `Goal: ${options.goal}\n\nPlan (written earlier by another agent; follow it, but check each step on the live page):\n${options.plan.trim()}\n\n${first}`
|
|
32
|
+
: `Goal: ${options.goal}\n\n${first}`,
|
|
33
|
+
secrets: (steps) => resultSecrets(steps.events, []),
|
|
34
|
+
person: ({ page, sink, snapshots, baseURL }) => personRecording({
|
|
35
|
+
page,
|
|
36
|
+
steps: sink,
|
|
37
|
+
snapshots,
|
|
38
|
+
...(baseURL !== undefined && { baseURL }),
|
|
39
|
+
}),
|
|
40
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { AgentOptions, AgentSession, PlanOptions, PlanSession } from "./types.js";
|
|
2
|
+
export { DEFAULT_LIMITS, resolveLimits, type AgentLimits } from "./limits.js";
|
|
3
|
+
export { assertModelObject, type AgentModel } from "./model.js";
|
|
4
|
+
export { readPlan } from "./plan/read-plan.js";
|
|
5
|
+
export { defaultModelSpec, ModelSetupError, modelFromSpec, } from "./providers.js";
|
|
6
|
+
export type { AgentEndReason, AgentEvent, AgentOptions, AgentResult, AgentSession, PlanOptions, PlanResult, PlanSession, } from "./types.js";
|
|
7
|
+
/**
|
|
8
|
+
* Starts the agent on a goal. The model is an AI SDK model object; the run opens a browser, writes typed steps
|
|
9
|
+
* into the spec file as they succeed, and ends with an `ended` event whatever happens. Throws at once for a model
|
|
10
|
+
* that is a string. Needs the optional packages `ai` and `zod`. With `plan` (the text of a plan from `runPlan`)
|
|
11
|
+
* the agent follows that plan, checking each step on the live page.
|
|
12
|
+
*/
|
|
13
|
+
export declare function runAgent(options: AgentOptions): AgentSession;
|
|
14
|
+
/**
|
|
15
|
+
* Starts the planning agent on a goal: it explores the page and writes a Markdown plan into `out`, a step at a
|
|
16
|
+
* time, for `runAgent`'s `plan` option. The page is read-only: every request that is not a GET is stopped. Same
|
|
17
|
+
* events and `ended` event as `runAgent`, plus `plan-step`. Needs the optional packages `ai` and `zod`.
|
|
18
|
+
*/
|
|
19
|
+
export declare function runPlan(options: PlanOptions): PlanSession;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { startPlan } from "./plan/run.js";
|
|
2
|
+
import { startAgent } from "./run.js";
|
|
3
|
+
export { DEFAULT_LIMITS, resolveLimits } from "./limits.js";
|
|
4
|
+
export { assertModelObject } from "./model.js";
|
|
5
|
+
export { readPlan } from "./plan/read-plan.js";
|
|
6
|
+
export { defaultModelSpec, ModelSetupError, modelFromSpec, } from "./providers.js";
|
|
7
|
+
/**
|
|
8
|
+
* Starts the agent on a goal. The model is an AI SDK model object; the run opens a browser, writes typed steps
|
|
9
|
+
* into the spec file as they succeed, and ends with an `ended` event whatever happens. Throws at once for a model
|
|
10
|
+
* that is a string. Needs the optional packages `ai` and `zod`. With `plan` (the text of a plan from `runPlan`)
|
|
11
|
+
* the agent follows that plan, checking each step on the live page.
|
|
12
|
+
*/
|
|
13
|
+
export function runAgent(options) {
|
|
14
|
+
return startAgent(options);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Starts the planning agent on a goal: it explores the page and writes a Markdown plan into `out`, a step at a
|
|
18
|
+
* time, for `runAgent`'s `plan` option. The page is read-only: every request that is not a GET is stopped. Same
|
|
19
|
+
* events and `ended` event as `runAgent`, plus `plan-step`. Needs the optional packages `ai` and `zod`.
|
|
20
|
+
*/
|
|
21
|
+
export function runPlan(options) {
|
|
22
|
+
return startPlan(options);
|
|
23
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** The two instruction files together may not be larger than this (bytes) */
|
|
2
|
+
export declare const MAX_INSTRUCTIONS_BYTES: number;
|
|
3
|
+
export interface Instructions {
|
|
4
|
+
/** The files that were read, relative to the project folder */
|
|
5
|
+
files: string[];
|
|
6
|
+
/** Their text as it goes into the prompt; empty without files */
|
|
7
|
+
text: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Reads `AGENTS.md` and `.samurai/agent.md` from the project folder (both are optional). A file that is a link
|
|
11
|
+
* to somewhere outside the folder is an error, never read. Together they may hold at most 32 KB.
|
|
12
|
+
*/
|
|
13
|
+
export declare function loadInstructions(projectDir: string): Instructions;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { lstatSync, readFileSync, realpathSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/** The two instruction files together may not be larger than this (bytes) */
|
|
4
|
+
export const MAX_INSTRUCTIONS_BYTES = 32 * 1024;
|
|
5
|
+
/** The shared agent file first, the more specific SAMURAI one last: it wins where they disagree */
|
|
6
|
+
const FILES = ["AGENTS.md", path.join(".samurai", "agent.md")];
|
|
7
|
+
function readInside(root, name) {
|
|
8
|
+
const file = path.join(root, name);
|
|
9
|
+
try {
|
|
10
|
+
lstatSync(file);
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
const real = realpathSync(file);
|
|
16
|
+
const relative = path.relative(root, real);
|
|
17
|
+
if (relative.startsWith("..") || path.isAbsolute(relative))
|
|
18
|
+
throw new Error(`agent(): ${name} points outside the project folder, so it is not read`);
|
|
19
|
+
return readFileSync(real, "utf8");
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Reads `AGENTS.md` and `.samurai/agent.md` from the project folder (both are optional). A file that is a link
|
|
23
|
+
* to somewhere outside the folder is an error, never read. Together they may hold at most 32 KB.
|
|
24
|
+
*/
|
|
25
|
+
export function loadInstructions(projectDir) {
|
|
26
|
+
const root = realpathSync(projectDir);
|
|
27
|
+
const parts = [];
|
|
28
|
+
for (const name of FILES) {
|
|
29
|
+
const text = readInside(root, name);
|
|
30
|
+
if (text !== undefined)
|
|
31
|
+
parts.push({ name, text });
|
|
32
|
+
}
|
|
33
|
+
const bytes = parts.reduce((sum, p) => sum + Buffer.byteLength(p.text), 0);
|
|
34
|
+
if (bytes > MAX_INSTRUCTIONS_BYTES)
|
|
35
|
+
throw new Error(`agent(): the instruction files hold ${Math.ceil(bytes / 1024)} KB, over the ${MAX_INSTRUCTIONS_BYTES / 1024} KB limit (${parts
|
|
36
|
+
.map((p) => `${p.name} ${Math.ceil(Buffer.byteLength(p.text) / 1024)} KB`)
|
|
37
|
+
.join(", ")}); shorten them`);
|
|
38
|
+
return {
|
|
39
|
+
files: parts.map((p) => p.name),
|
|
40
|
+
text: parts
|
|
41
|
+
.map((p) => `### From ${p.name.replace(/\\/g, "/")}\n\n${p.text.trim()}`)
|
|
42
|
+
.join("\n\n"),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Hard limits of one agent run; reaching one ends the run cleanly with the steps written so far */
|
|
2
|
+
export interface AgentLimits {
|
|
3
|
+
/** Model calls (loop turns). A tool call is not a turn */
|
|
4
|
+
maxSteps: number;
|
|
5
|
+
/** Input plus output tokens, summed over the run */
|
|
6
|
+
maxTokens: number;
|
|
7
|
+
/** Wall-clock time of the run in ms */
|
|
8
|
+
timeoutMs: number;
|
|
9
|
+
}
|
|
10
|
+
export declare const DEFAULT_LIMITS: Readonly<AgentLimits>;
|
|
11
|
+
/** The defaults with `overrides` on top; a limit that is not a positive finite number throws */
|
|
12
|
+
export declare function resolveLimits(overrides?: Partial<AgentLimits>): AgentLimits;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export const DEFAULT_LIMITS = {
|
|
2
|
+
maxSteps: 40,
|
|
3
|
+
maxTokens: 400_000,
|
|
4
|
+
timeoutMs: 10 * 60_000,
|
|
5
|
+
};
|
|
6
|
+
/** The defaults with `overrides` on top; a limit that is not a positive finite number throws */
|
|
7
|
+
export function resolveLimits(overrides = {}) {
|
|
8
|
+
const limits = { ...DEFAULT_LIMITS };
|
|
9
|
+
for (const key of Object.keys(DEFAULT_LIMITS)) {
|
|
10
|
+
const value = overrides[key];
|
|
11
|
+
if (value === undefined)
|
|
12
|
+
continue;
|
|
13
|
+
if (!Number.isFinite(value) || value <= 0)
|
|
14
|
+
throw new RangeError(`agent(): limits.${key} must be a positive number, got ${String(value)}`);
|
|
15
|
+
limits[key] = value;
|
|
16
|
+
}
|
|
17
|
+
return limits;
|
|
18
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { AgentLimits } from "./limits.js";
|
|
2
|
+
import type { AgentModel } from "./model.js";
|
|
3
|
+
import type { AiModule } from "./sdk-tools.js";
|
|
4
|
+
import type { AgentController } from "./session.js";
|
|
5
|
+
/** What the loop counts, shared with the run that reports it */
|
|
6
|
+
export interface RunTally {
|
|
7
|
+
modelCalls: number;
|
|
8
|
+
tokens: number;
|
|
9
|
+
finished: boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface LoopInput {
|
|
12
|
+
ai: AiModule;
|
|
13
|
+
model: AgentModel;
|
|
14
|
+
system: string;
|
|
15
|
+
prompt: string;
|
|
16
|
+
tools: Record<string, unknown>;
|
|
17
|
+
controller: AgentController;
|
|
18
|
+
limits: AgentLimits;
|
|
19
|
+
tally: RunTally;
|
|
20
|
+
}
|
|
21
|
+
export interface LoopOutcome {
|
|
22
|
+
/** The model's last words when it ended without calling finish */
|
|
23
|
+
text: string;
|
|
24
|
+
/** The loop stopped because of a limit while the model still had work to do */
|
|
25
|
+
limit?: "step-limit" | "token-limit";
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The AI SDK's tool loop with SAMURAI's guards: model calls and tokens are counted after every step (and a
|
|
29
|
+
* `usage` event sent), a paused or taken-over run makes no new model call, and `finish` ends the loop. Time and
|
|
30
|
+
* `stop` abort through the controller's signal.
|
|
31
|
+
*/
|
|
32
|
+
export declare function runLoop(input: LoopInput): Promise<LoopOutcome>;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The AI SDK's tool loop with SAMURAI's guards: model calls and tokens are counted after every step (and a
|
|
3
|
+
* `usage` event sent), a paused or taken-over run makes no new model call, and `finish` ends the loop. Time and
|
|
4
|
+
* `stop` abort through the controller's signal.
|
|
5
|
+
*/
|
|
6
|
+
export async function runLoop(input) {
|
|
7
|
+
const { ai, controller, limits, tally } = input;
|
|
8
|
+
const result = await ai.generateText({
|
|
9
|
+
model: input.model,
|
|
10
|
+
system: input.system,
|
|
11
|
+
prompt: input.prompt,
|
|
12
|
+
tools: input.tools,
|
|
13
|
+
abortSignal: controller.signal,
|
|
14
|
+
stopWhen: [
|
|
15
|
+
ai.isStepCount(limits.maxSteps),
|
|
16
|
+
ai.hasToolCall("finish"),
|
|
17
|
+
() => tally.tokens >= limits.maxTokens,
|
|
18
|
+
],
|
|
19
|
+
prepareStep: async () => {
|
|
20
|
+
await controller.gate();
|
|
21
|
+
return undefined;
|
|
22
|
+
},
|
|
23
|
+
onStepFinish: (step) => {
|
|
24
|
+
tally.modelCalls++;
|
|
25
|
+
const inputTokens = step.usage.inputTokens ?? 0;
|
|
26
|
+
const outputTokens = step.usage.outputTokens ?? 0;
|
|
27
|
+
tally.tokens += step.usage.totalTokens ?? inputTokens + outputTokens;
|
|
28
|
+
if (step.text.trim() !== "")
|
|
29
|
+
controller.emit({ type: "plan", text: step.text.trim() });
|
|
30
|
+
controller.emit({
|
|
31
|
+
type: "usage",
|
|
32
|
+
inputTokens,
|
|
33
|
+
outputTokens,
|
|
34
|
+
totalTokens: step.usage.totalTokens ?? inputTokens + outputTokens,
|
|
35
|
+
});
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
const last = result.steps[result.steps.length - 1];
|
|
39
|
+
const unfinished = last !== undefined && last.toolCalls.length > 0;
|
|
40
|
+
if (tally.finished || !unfinished)
|
|
41
|
+
return { text: result.text };
|
|
42
|
+
return {
|
|
43
|
+
text: result.text,
|
|
44
|
+
limit: tally.tokens >= limits.maxTokens ? "token-limit" : "step-limit",
|
|
45
|
+
};
|
|
46
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { LanguageModel } from "ai";
|
|
2
|
+
/**
|
|
3
|
+
* A model object from an AI SDK provider package (`anthropic("...")`, `openai("...")`, an OpenAI-compatible
|
|
4
|
+
* endpoint). Never a string: the SDK resolves a bare string through the Vercel AI Gateway, a third party the
|
|
5
|
+
* user did not choose, which would receive the page content.
|
|
6
|
+
*/
|
|
7
|
+
export type AgentModel = Exclude<LanguageModel, string>;
|
|
8
|
+
/** `model` as an `AgentModel`; throws for a string id or anything that is not a model object */
|
|
9
|
+
export declare function assertModelObject(model: unknown): AgentModel;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
const VERSIONS = ["v2", "v3", "v4"];
|
|
2
|
+
/** `model` as an `AgentModel`; throws for a string id or anything that is not a model object */
|
|
3
|
+
export function assertModelObject(model) {
|
|
4
|
+
if (typeof model === "string")
|
|
5
|
+
throw new Error(`agent(): model must be a model object from a provider package, not the string "${model}" ` +
|
|
6
|
+
"(a string id would send the page through the Vercel AI Gateway). " +
|
|
7
|
+
'Install the provider (for example @ai-sdk/anthropic) and pass anthropic("model-name")');
|
|
8
|
+
const version = typeof model === "object" && model !== null
|
|
9
|
+
? model.specificationVersion
|
|
10
|
+
: undefined;
|
|
11
|
+
if (typeof version !== "string" || !VERSIONS.includes(version))
|
|
12
|
+
throw new Error("agent(): model must be an AI SDK language model object (it has no specificationVersion v2, v3 or v4)");
|
|
13
|
+
return model;
|
|
14
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type Page from "../../browser/page.js";
|
|
2
|
+
/**
|
|
3
|
+
* Makes the page read-only for the planner: every request that is not a `GET`, `HEAD` or `OPTIONS` is aborted
|
|
4
|
+
* before it leaves the browser (form submits, `fetch`, XHR, beacons), and `onBlocked` hears what it was. A `GET`
|
|
5
|
+
* that changes data and a WebSocket message are not covered. Other origins are the origin policy's business.
|
|
6
|
+
*/
|
|
7
|
+
export declare function installPlanGuard(page: Pick<Page, "route">, onBlocked: (what: string) => void): Promise<void>;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** Methods that read: they pass. `OPTIONS` is the CORS preflight, which carries no data of the page */
|
|
2
|
+
const READS = new Set(["GET", "HEAD", "OPTIONS"]);
|
|
3
|
+
/** "POST /api/feedback": the method and the path, without the query (it can hold values) */
|
|
4
|
+
function describeRequest(method, url) {
|
|
5
|
+
try {
|
|
6
|
+
return `${method} ${new URL(url).pathname}`;
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return method;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Makes the page read-only for the planner: every request that is not a `GET`, `HEAD` or `OPTIONS` is aborted
|
|
14
|
+
* before it leaves the browser (form submits, `fetch`, XHR, beacons), and `onBlocked` hears what it was. A `GET`
|
|
15
|
+
* that changes data and a WebSocket message are not covered. Other origins are the origin policy's business.
|
|
16
|
+
*/
|
|
17
|
+
export async function installPlanGuard(page, onBlocked) {
|
|
18
|
+
await page.route("**/*", async (route) => {
|
|
19
|
+
const request = route.request();
|
|
20
|
+
const { method, url } = request;
|
|
21
|
+
if (READS.has(method.toUpperCase())) {
|
|
22
|
+
await route.continue();
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
onBlocked(describeRequest(method.toUpperCase(), url));
|
|
26
|
+
// An aborted form submit leaves Chrome on its error page; an empty 204 answer to a navigation keeps the page
|
|
27
|
+
if (request.resourceType === "document" || request.navigation !== null)
|
|
28
|
+
await route.fulfill({ status: 204 });
|
|
29
|
+
else
|
|
30
|
+
await route.abort();
|
|
31
|
+
});
|
|
32
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/** A plan file may not be larger than this (bytes), for the planner and for `generate --plan` */
|
|
2
|
+
export declare const MAX_PLAN_BYTES: number;
|
|
3
|
+
interface PlanStep {
|
|
4
|
+
text: string;
|
|
5
|
+
expect?: string;
|
|
6
|
+
}
|
|
7
|
+
/** A plan as `parsePlan` reads it; every part but the steps may be missing in a hand-written file */
|
|
8
|
+
interface ParsedPlan {
|
|
9
|
+
title: string;
|
|
10
|
+
goal: string;
|
|
11
|
+
start: string;
|
|
12
|
+
steps: PlanStep[];
|
|
13
|
+
seen: string[];
|
|
14
|
+
notDone: string[];
|
|
15
|
+
questions: string[];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A plan file being written. The whole file is written again after every call, so it is valid Markdown with its
|
|
19
|
+
* four headings whenever the run stops. Texts are masked and made one line long.
|
|
20
|
+
*/
|
|
21
|
+
declare class PlanFile {
|
|
22
|
+
readonly file: string;
|
|
23
|
+
private readonly goal;
|
|
24
|
+
private start;
|
|
25
|
+
private readonly steps;
|
|
26
|
+
private readonly seenItems;
|
|
27
|
+
private readonly notDoneItems;
|
|
28
|
+
private readonly questionItems;
|
|
29
|
+
constructor(file: string, goal: string, start: string);
|
|
30
|
+
/** The title, which is also what the run reports as the name of the test */
|
|
31
|
+
get title(): string;
|
|
32
|
+
get count(): number;
|
|
33
|
+
private render;
|
|
34
|
+
/** Writes the file; refuses, and keeps the file as it was, when it would pass the size limit */
|
|
35
|
+
private save;
|
|
36
|
+
private add;
|
|
37
|
+
/** Appends a step and returns its number */
|
|
38
|
+
addStep(text: string, expect?: string): number;
|
|
39
|
+
/** Names the page the plan starts on, once the browser is there */
|
|
40
|
+
setStart(url: string): void;
|
|
41
|
+
seen(text: string): void;
|
|
42
|
+
notDone(text: string): void;
|
|
43
|
+
question(text: string): void;
|
|
44
|
+
/** Writes the first version of the file */
|
|
45
|
+
create(): void;
|
|
46
|
+
}
|
|
47
|
+
export type { PlanFile };
|
|
48
|
+
/** Starts a plan file: the headings are written at once. An existing file is an error unless `force` is set */
|
|
49
|
+
export declare function createPlan(input: {
|
|
50
|
+
goal: string;
|
|
51
|
+
start: string;
|
|
52
|
+
out: string;
|
|
53
|
+
force?: boolean;
|
|
54
|
+
}): PlanFile;
|
|
55
|
+
/**
|
|
56
|
+
* Reads a plan, the one `createPlan` wrote or one a person edited: blank lines, a missing title, missing sections
|
|
57
|
+
* and `None.` placeholders are fine. Only the `## Steps` heading is needed.
|
|
58
|
+
*/
|
|
59
|
+
export declare function parsePlan(text: string): ParsedPlan;
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { projectDir } from "../../config/config.js";
|
|
4
|
+
import { maskText } from "../../config/mask.js";
|
|
5
|
+
/** A plan file may not be larger than this (bytes), for the planner and for `generate --plan` */
|
|
6
|
+
export const MAX_PLAN_BYTES = 32 * 1024;
|
|
7
|
+
const NONE = "None.";
|
|
8
|
+
const TITLE_LENGTH = 80;
|
|
9
|
+
/** One line, masked: a new line in a text must not be able to start a heading or a step */
|
|
10
|
+
function oneLine(text) {
|
|
11
|
+
return maskText(text).replace(/\s+/g, " ").trim();
|
|
12
|
+
}
|
|
13
|
+
function bullets(items) {
|
|
14
|
+
return items.length > 0
|
|
15
|
+
? items.map((item) => `- ${item}`).join("\n")
|
|
16
|
+
: `- ${NONE}`;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* A plan file being written. The whole file is written again after every call, so it is valid Markdown with its
|
|
20
|
+
* four headings whenever the run stops. Texts are masked and made one line long.
|
|
21
|
+
*/
|
|
22
|
+
class PlanFile {
|
|
23
|
+
constructor(file, goal, start) {
|
|
24
|
+
this.file = file;
|
|
25
|
+
this.goal = goal;
|
|
26
|
+
this.start = start;
|
|
27
|
+
this.steps = [];
|
|
28
|
+
this.seenItems = [];
|
|
29
|
+
this.notDoneItems = [];
|
|
30
|
+
this.questionItems = [];
|
|
31
|
+
}
|
|
32
|
+
/** The title, which is also what the run reports as the name of the test */
|
|
33
|
+
get title() {
|
|
34
|
+
const first = this.goal.slice(0, TITLE_LENGTH).trim();
|
|
35
|
+
return first.length < this.goal.length ? `${first}…` : first;
|
|
36
|
+
}
|
|
37
|
+
get count() {
|
|
38
|
+
return this.steps.length;
|
|
39
|
+
}
|
|
40
|
+
render() {
|
|
41
|
+
const steps = this.steps.map((step, i) => `${i + 1}. ${step.text}${step.expect !== undefined ? `\n Expect: ${step.expect}` : ""}`);
|
|
42
|
+
return [
|
|
43
|
+
`# Plan: ${this.title}`,
|
|
44
|
+
"",
|
|
45
|
+
`Goal: ${this.goal}`,
|
|
46
|
+
`Start: ${this.start}`,
|
|
47
|
+
"",
|
|
48
|
+
"## Steps",
|
|
49
|
+
"",
|
|
50
|
+
steps.length > 0 ? steps.join("\n") : `- ${NONE}`,
|
|
51
|
+
"",
|
|
52
|
+
"## Seen on the page",
|
|
53
|
+
"",
|
|
54
|
+
bullets(this.seenItems),
|
|
55
|
+
"",
|
|
56
|
+
"## Not done",
|
|
57
|
+
"",
|
|
58
|
+
bullets(this.notDoneItems),
|
|
59
|
+
"",
|
|
60
|
+
"## Open questions",
|
|
61
|
+
"",
|
|
62
|
+
bullets(this.questionItems),
|
|
63
|
+
"",
|
|
64
|
+
].join("\n");
|
|
65
|
+
}
|
|
66
|
+
/** Writes the file; refuses, and keeps the file as it was, when it would pass the size limit */
|
|
67
|
+
save() {
|
|
68
|
+
const text = this.render();
|
|
69
|
+
if (Buffer.byteLength(text) > MAX_PLAN_BYTES)
|
|
70
|
+
throw new Error(`the plan file is full (${MAX_PLAN_BYTES / 1024} KB); finish with what it holds, or shorten the next entry`);
|
|
71
|
+
mkdirSync(path.dirname(this.file), { recursive: true });
|
|
72
|
+
writeFileSync(this.file, text);
|
|
73
|
+
}
|
|
74
|
+
add(list, text) {
|
|
75
|
+
list.push(oneLine(text));
|
|
76
|
+
try {
|
|
77
|
+
this.save();
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
list.pop();
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/** Appends a step and returns its number */
|
|
85
|
+
addStep(text, expect) {
|
|
86
|
+
const step = { text: oneLine(text) };
|
|
87
|
+
const wanted = expect === undefined ? "" : oneLine(expect);
|
|
88
|
+
if (wanted !== "")
|
|
89
|
+
step.expect = wanted;
|
|
90
|
+
this.steps.push(step);
|
|
91
|
+
try {
|
|
92
|
+
this.save();
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
this.steps.pop();
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
return this.steps.length;
|
|
99
|
+
}
|
|
100
|
+
/** Names the page the plan starts on, once the browser is there */
|
|
101
|
+
setStart(url) {
|
|
102
|
+
this.start = oneLine(url);
|
|
103
|
+
this.save();
|
|
104
|
+
}
|
|
105
|
+
seen(text) {
|
|
106
|
+
this.add(this.seenItems, text);
|
|
107
|
+
}
|
|
108
|
+
notDone(text) {
|
|
109
|
+
this.add(this.notDoneItems, text);
|
|
110
|
+
}
|
|
111
|
+
question(text) {
|
|
112
|
+
this.add(this.questionItems, text);
|
|
113
|
+
}
|
|
114
|
+
/** Writes the first version of the file */
|
|
115
|
+
create() {
|
|
116
|
+
this.save();
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/** Starts a plan file: the headings are written at once. An existing file is an error unless `force` is set */
|
|
120
|
+
export function createPlan(input) {
|
|
121
|
+
const file = path.resolve(projectDir(), input.out);
|
|
122
|
+
if (existsSync(file) && input.force !== true)
|
|
123
|
+
throw new Error(`createPlan(): ${file} already exists; pass force (--force) to replace it`);
|
|
124
|
+
const plan = new PlanFile(file, oneLine(input.goal), oneLine(input.start));
|
|
125
|
+
plan.create();
|
|
126
|
+
return plan;
|
|
127
|
+
}
|
|
128
|
+
const HEADING = /^##\s+(.*?)\s*$/;
|
|
129
|
+
const STEP = /^\s*\d+\.\s+(.*)$/;
|
|
130
|
+
const BULLET = /^\s*[-*]\s+(.*)$/;
|
|
131
|
+
const EXPECT = /^\s+Expect:\s*(.*)$/i;
|
|
132
|
+
const FIELD = /^(Goal|Start):\s*(.*)$/i;
|
|
133
|
+
function sectionOf(heading) {
|
|
134
|
+
const name = heading.toLowerCase();
|
|
135
|
+
if (name === "steps")
|
|
136
|
+
return "steps";
|
|
137
|
+
if (name.startsWith("seen"))
|
|
138
|
+
return "seen";
|
|
139
|
+
if (name === "not done")
|
|
140
|
+
return "notDone";
|
|
141
|
+
if (name.startsWith("open question"))
|
|
142
|
+
return "questions";
|
|
143
|
+
return "other";
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Reads a plan, the one `createPlan` wrote or one a person edited: blank lines, a missing title, missing sections
|
|
147
|
+
* and `None.` placeholders are fine. Only the `## Steps` heading is needed.
|
|
148
|
+
*/
|
|
149
|
+
export function parsePlan(text) {
|
|
150
|
+
const plan = {
|
|
151
|
+
title: "",
|
|
152
|
+
goal: "",
|
|
153
|
+
start: "",
|
|
154
|
+
steps: [],
|
|
155
|
+
seen: [],
|
|
156
|
+
notDone: [],
|
|
157
|
+
questions: [],
|
|
158
|
+
};
|
|
159
|
+
let section;
|
|
160
|
+
let hasSteps = false;
|
|
161
|
+
for (const line of text.split(/\r?\n/)) {
|
|
162
|
+
const heading = HEADING.exec(line);
|
|
163
|
+
if (heading) {
|
|
164
|
+
section = sectionOf(heading[1]);
|
|
165
|
+
if (section === "steps")
|
|
166
|
+
hasSteps = true;
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (line.trim() === "")
|
|
170
|
+
continue;
|
|
171
|
+
if (section === undefined) {
|
|
172
|
+
const title = /^#\s+(?:Plan:\s*)?(.*)$/i.exec(line);
|
|
173
|
+
const field = FIELD.exec(line);
|
|
174
|
+
if (title && plan.title === "")
|
|
175
|
+
plan.title = title[1].trim();
|
|
176
|
+
else if (field)
|
|
177
|
+
plan[field[1].toLowerCase()] = field[2].trim();
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (section === "other")
|
|
181
|
+
continue;
|
|
182
|
+
if (section === "steps") {
|
|
183
|
+
const last = plan.steps[plan.steps.length - 1];
|
|
184
|
+
const step = STEP.exec(line);
|
|
185
|
+
const expect = EXPECT.exec(line);
|
|
186
|
+
if (step)
|
|
187
|
+
plan.steps.push({ text: step[1].trim() });
|
|
188
|
+
else if (last && expect)
|
|
189
|
+
last.expect = expect[1].trim();
|
|
190
|
+
else if (last && !BULLET.test(line) && /^\s/.test(line))
|
|
191
|
+
last.text = `${last.text} ${line.trim()}`;
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const item = BULLET.exec(line)?.[1]?.trim();
|
|
195
|
+
if (item !== undefined && item !== NONE)
|
|
196
|
+
plan[section].push(item);
|
|
197
|
+
}
|
|
198
|
+
if (!hasSteps)
|
|
199
|
+
throw new Error('parsePlan(): the plan has no "## Steps" heading');
|
|
200
|
+
return plan;
|
|
201
|
+
}
|