@itmetsam/samurai-framework 0.22.3 → 0.22.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/AGENT-GUIDE.md +50 -0
  2. package/bin/samurai.mjs +2 -0
  3. package/dist/agent/generate-kind.d.ts +5 -0
  4. package/dist/agent/generate-kind.js +40 -0
  5. package/dist/agent/index.d.ts +19 -0
  6. package/dist/agent/index.js +23 -0
  7. package/dist/agent/instructions.d.ts +13 -0
  8. package/dist/agent/instructions.js +44 -0
  9. package/dist/agent/limits.d.ts +12 -0
  10. package/dist/agent/limits.js +18 -0
  11. package/dist/agent/loop.d.ts +32 -0
  12. package/dist/agent/loop.js +46 -0
  13. package/dist/agent/model.d.ts +9 -0
  14. package/dist/agent/model.js +14 -0
  15. package/dist/agent/plan/guard.d.ts +7 -0
  16. package/dist/agent/plan/guard.js +32 -0
  17. package/dist/agent/plan/plan-file.d.ts +59 -0
  18. package/dist/agent/plan/plan-file.js +201 -0
  19. package/dist/agent/plan/plan-kind.d.ts +5 -0
  20. package/dist/agent/plan/plan-kind.js +28 -0
  21. package/dist/agent/plan/prompt.d.ts +3 -0
  22. package/dist/agent/plan/prompt.js +37 -0
  23. package/dist/agent/plan/read-plan.d.ts +5 -0
  24. package/dist/agent/plan/read-plan.js +28 -0
  25. package/dist/agent/plan/recorder.d.ts +30 -0
  26. package/dist/agent/plan/recorder.js +60 -0
  27. package/dist/agent/plan/run.d.ts +8 -0
  28. package/dist/agent/plan/run.js +10 -0
  29. package/dist/agent/plan/tools.d.ts +4 -0
  30. package/dist/agent/plan/tools.js +95 -0
  31. package/dist/agent/prompt.d.ts +21 -0
  32. package/dist/agent/prompt.js +60 -0
  33. package/dist/agent/providers.d.ts +19 -0
  34. package/dist/agent/providers.js +95 -0
  35. package/dist/agent/recording.d.ts +45 -0
  36. package/dist/agent/recording.js +78 -0
  37. package/dist/agent/run-kind.d.ts +48 -0
  38. package/dist/agent/run-kind.js +1 -0
  39. package/dist/agent/run.d.ts +15 -0
  40. package/dist/agent/run.js +201 -0
  41. package/dist/agent/sdk-tools.d.ts +15 -0
  42. package/dist/agent/sdk-tools.js +52 -0
  43. package/dist/agent/session.d.ts +62 -0
  44. package/dist/agent/session.js +196 -0
  45. package/dist/agent/snapshot-script.d.ts +18 -0
  46. package/dist/agent/snapshot-script.js +163 -0
  47. package/dist/agent/snapshot.d.ts +54 -0
  48. package/dist/agent/snapshot.js +95 -0
  49. package/dist/agent/take-over.d.ts +19 -0
  50. package/dist/agent/take-over.js +49 -0
  51. package/dist/agent/tools/act.d.ts +2 -0
  52. package/dist/agent/tools/act.js +124 -0
  53. package/dist/agent/tools/ask.d.ts +1 -0
  54. package/dist/agent/tools/ask.js +19 -0
  55. package/dist/agent/tools/context.d.ts +70 -0
  56. package/dist/agent/tools/context.js +42 -0
  57. package/dist/agent/tools/expect.d.ts +1 -0
  58. package/dist/agent/tools/expect.js +152 -0
  59. package/dist/agent/tools/finish.d.ts +1 -0
  60. package/dist/agent/tools/finish.js +12 -0
  61. package/dist/agent/tools/index.d.ts +3 -0
  62. package/dist/agent/tools/index.js +23 -0
  63. package/dist/agent/tools/navigate.d.ts +1 -0
  64. package/dist/agent/tools/navigate.js +53 -0
  65. package/dist/agent/tools/origin-guard.d.ts +6 -0
  66. package/dist/agent/tools/origin-guard.js +19 -0
  67. package/dist/agent/tools/origins.d.ts +11 -0
  68. package/dist/agent/tools/origins.js +22 -0
  69. package/dist/agent/tools/screenshot.d.ts +1 -0
  70. package/dist/agent/tools/screenshot.js +39 -0
  71. package/dist/agent/tools/snapshot.d.ts +1 -0
  72. package/dist/agent/tools/snapshot.js +12 -0
  73. package/dist/agent/tools/undo.d.ts +1 -0
  74. package/dist/agent/tools/undo.js +23 -0
  75. package/dist/agent/tools/use-secret.d.ts +1 -0
  76. package/dist/agent/tools/use-secret.js +41 -0
  77. package/dist/agent/types.d.ts +128 -0
  78. package/dist/agent/types.js +1 -0
  79. package/dist/capabilities.d.ts +2 -0
  80. package/dist/capabilities.js +2 -0
  81. package/dist/cli/agent-run.d.ts +43 -0
  82. package/dist/cli/agent-run.js +132 -0
  83. package/dist/cli/args.d.ts +8 -0
  84. package/dist/cli/args.js +8 -0
  85. package/dist/cli/generate.d.ts +6 -0
  86. package/dist/cli/generate.js +48 -0
  87. package/dist/cli/io.d.ts +2 -0
  88. package/dist/cli/main.js +6 -0
  89. package/dist/cli/plan.d.ts +6 -0
  90. package/dist/cli/plan.js +28 -0
  91. package/dist/cli/prompt.d.ts +2 -0
  92. package/dist/cli/prompt.js +16 -0
  93. package/dist/cli/usage.d.ts +1 -1
  94. package/dist/cli/usage.js +26 -1
  95. package/dist/index.js +2 -0
  96. package/dist/recorder/normalise.d.ts +6 -0
  97. package/dist/recorder/normalise.js +2 -0
  98. package/dist/types/config.d.ts +5 -0
  99. package/package.json +19 -4
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Function declaration, called as `(options)` with `{ maxChars, secretFields }`. Writes the page as text (see
3
+ * `Snapshotter.take`) and remembers the elements it numbered, so `RESOLVE_SCRIPT` can find them again. Secret
4
+ * values are replaced here, in the page: they never reach the framework.
5
+ */
6
+ export declare const SNAPSHOT_SCRIPT: string;
7
+ /**
8
+ * Function declaration, called as `({ ref, focused, secretFields })` with the number of a reference, or `focused`
9
+ * for the element that has focus. Answers `{ status: "ok", target, candidates, secret, dotted, role, name }`, or a status
10
+ * that says why the reference does not resolve: `none` (no snapshot in this document), `unknown`, `older` (it was
11
+ * in an earlier snapshot), `gone` (removed since), `nofocus`.
12
+ */
13
+ export declare const RESOLVE_SCRIPT: string;
14
+ /**
15
+ * Function declaration, called as `({ secretFields })`. Answers the first of the caller's selectors that matches a
16
+ * visible field showing a value in plain text (a password input draws dots and does not count), else "".
17
+ */
18
+ export declare const SECRET_ON_SCREEN_SCRIPT: string;
@@ -0,0 +1,163 @@
1
+ import { ACCESSIBLE_NAME_JS } from "../locator/accessible-name.js";
2
+ import { CANDIDATES_JS } from "../recorder/candidates.js";
3
+ /** Name of the page-global that holds the references of the latest snapshot */
4
+ const STORE_KEY = "samurai.agent.refs";
5
+ /**
6
+ * Page-side: `isSecret(el, selectors)` is true for a field whose value must never leave the page: a password
7
+ * input, a one-time code or card field (by `autocomplete`), or an element matching one of the caller's selectors.
8
+ */
9
+ const SECRET_JS = String.raw `
10
+ const isSecret = (el, selectors) => {
11
+ if (el.localName === "input" && (el.getAttribute("type") || "").toLowerCase() === "password") return true;
12
+ if (/password|one-time-code|cc-/i.test(el.getAttribute("autocomplete") || "")) return true;
13
+ for (const selector of selectors) {
14
+ try { if (el.matches(selector)) return true; } catch (error) { /* a bad selector hides nothing */ }
15
+ }
16
+ return false;
17
+ };`;
18
+ /**
19
+ * Function declaration, called as `(options)` with `{ maxChars, secretFields }`. Writes the page as text (see
20
+ * `Snapshotter.take`) and remembers the elements it numbered, so `RESOLVE_SCRIPT` can find them again. Secret
21
+ * values are replaced here, in the page: they never reach the framework.
22
+ */
23
+ export const SNAPSHOT_SCRIPT = String.raw `(options) => {
24
+ ${ACCESSIBLE_NAME_JS}
25
+ ${SECRET_JS}
26
+ const KEY = Symbol.for("${STORE_KEY}");
27
+ const store = window[KEY] || (window[KEY] = { next: 1, ids: new WeakMap(), map: new Map() });
28
+ store.map = new Map();
29
+ const SHOWN = new Set(["button", "link", "textbox", "searchbox", "combobox", "listbox", "checkbox", "radio", "switch",
30
+ "slider", "spinbutton", "tab", "menuitem", "menuitemcheckbox", "menuitemradio", "option", "treeitem",
31
+ "heading", "img", "navigation", "main", "banner", "contentinfo", "region", "form", "dialog", "alertdialog", "alert",
32
+ "status", "table", "list", "listitem", "row", "cell", "columnheader", "rowheader", "gridcell", "article", "tablist",
33
+ "menu", "toolbar", "progressbar"]);
34
+ // Their text is their name: nothing inside is listed again
35
+ const LEAF = new Set(["button", "link", "heading", "checkbox", "radio", "option", "switch", "tab", "menuitem",
36
+ "menuitemcheckbox", "menuitemradio", "treeitem"]);
37
+ const SKIPPED = new Set(["script", "style", "noscript", "template", "head", "svg"]);
38
+ const cut = (text, max) => (text.length > max ? text.slice(0, max - 1) + "…" : text);
39
+ const quote = (text) => '"' + text.replace(/"/g, '\\"') + '"';
40
+ const lines = [];
41
+ let size = 0;
42
+ let omitted = 0;
43
+ const add = (line) => {
44
+ if (size + line.length + 1 > options.maxChars) { omitted++; return false; }
45
+ lines.push(line);
46
+ size += line.length + 1;
47
+ return true;
48
+ };
49
+ const refOf = (el) => {
50
+ let id = store.ids.get(el);
51
+ if (id === undefined) { id = store.next++; store.ids.set(el, id); }
52
+ return id;
53
+ };
54
+ const checkedOf = (el, role) => {
55
+ if (!["checkbox", "radio", "switch", "menuitemcheckbox"].includes(role)) return "";
56
+ const aria = el.getAttribute("aria-checked");
57
+ if (aria === "mixed" || el.indeterminate) return " mixed";
58
+ return (el.checked === true || aria === "true") ? " checked" : " unchecked";
59
+ };
60
+ const valueOf = (el, role) => {
61
+ if (!["textbox", "searchbox", "spinbutton", "combobox", "slider", "listbox"].includes(role)) return "";
62
+ let value = "";
63
+ if (el.localName === "select") value = Array.from(el.selectedOptions).map((o) => o.label || o.textContent).join(", ");
64
+ else if (typeof el.value === "string") value = el.value;
65
+ if (value === "") return "";
66
+ return " value=" + quote(isSecret(el, options.secretFields) ? "••••" : cut(squash(value), 80));
67
+ };
68
+ const statesOf = (el, role) => {
69
+ let out = checkedOf(el, role);
70
+ if (el.disabled === true || el.getAttribute("aria-disabled") === "true") out += " disabled";
71
+ if (el.getAttribute("aria-expanded") === "true") out += " expanded";
72
+ if (el.getAttribute("aria-expanded") === "false") out += " collapsed";
73
+ if (el.selected === true || el.getAttribute("aria-selected") === "true") out += " selected";
74
+ if (el.getAttribute("aria-pressed") === "true") out += " pressed";
75
+ if (el.required === true || el.getAttribute("aria-required") === "true") out += " required";
76
+ if (el.readOnly === true) out += " readonly";
77
+ if (el === document.activeElement) out += " focused";
78
+ if (role === "heading") out += " level=" + (Number(el.getAttribute("aria-level")) || Number(el.localName.slice(1)) || 1);
79
+ return out;
80
+ };
81
+ const ownText = (el) => squash(Array.from(el.childNodes).filter((n) => n.nodeType === 3).map((n) => n.nodeValue).join(" "));
82
+ const visit = (el, depth) => {
83
+ if (el.nodeType !== 1 || SKIPPED.has(el.localName)) return;
84
+ if (el.getAttribute("aria-hidden") === "true" || el.hidden) return;
85
+ const style = getComputedStyle(el);
86
+ if (style.display === "none") return;
87
+ const visible = style.visibility !== "hidden" && style.visibility !== "collapse";
88
+ const role = roleOf(el);
89
+ const clickable = el.hasAttribute("onclick") || (el.hasAttribute("tabindex") && Number(el.getAttribute("tabindex")) >= 0);
90
+ let indent = depth;
91
+ if (visible) {
92
+ let line = "";
93
+ if (SHOWN.has(role)) {
94
+ const name = cut(accessibleName(el) || ownText(el), 100);
95
+ line = role + (name ? " " + quote(name) : "") + valueOf(el, role) + statesOf(el, role);
96
+ } else if (clickable) {
97
+ line = "clickable " + quote(cut(accessibleName(el) || squash(el.textContent || ""), 100)) + statesOf(el, role);
98
+ } else if (!["input", "select", "textarea"].includes(el.localName)) {
99
+ const text = ownText(el);
100
+ if (text) line = "text " + quote(cut(text, 160));
101
+ }
102
+ if (line) {
103
+ const id = refOf(el);
104
+ if (add(" ".repeat(depth) + "[e" + id + "] " + line)) store.map.set(id, el);
105
+ indent = depth + 1;
106
+ }
107
+ }
108
+ if (LEAF.has(role)) return;
109
+ for (const child of el.children) visit(child, indent);
110
+ };
111
+ if (document.body) visit(document.body, 0);
112
+ const text = lines.join("\n");
113
+ return { text, omitted, count: store.map.size, url: location.href, title: document.title };
114
+ }`;
115
+ /**
116
+ * Function declaration, called as `({ ref, focused, secretFields })` with the number of a reference, or `focused`
117
+ * for the element that has focus. Answers `{ status: "ok", target, candidates, secret, dotted, role, name }`, or a status
118
+ * that says why the reference does not resolve: `none` (no snapshot in this document), `unknown`, `older` (it was
119
+ * in an earlier snapshot), `gone` (removed since), `nofocus`.
120
+ */
121
+ export const RESOLVE_SCRIPT = String.raw `(options) => {
122
+ const KEY = Symbol.for("${STORE_KEY}");
123
+ const store = window[KEY];
124
+ const accessible = (() => { ${ACCESSIBLE_NAME_JS}; return { accessibleName, roleOf }; })();
125
+ const candidates = (() => { ${CANDIDATES_JS}; return { candidatesOf }; })();
126
+ ${SECRET_JS}
127
+ let el;
128
+ if (options.focused) {
129
+ el = document.activeElement;
130
+ if (!el || el === document.body || el === document.documentElement) return { status: "nofocus" };
131
+ } else {
132
+ if (!store) return { status: "none" };
133
+ el = store.map.get(options.ref);
134
+ if (!el) return { status: options.ref >= store.next ? "unknown" : "older" };
135
+ if (!el.isConnected) return { status: "gone" };
136
+ }
137
+ return {
138
+ status: "ok",
139
+ target: el,
140
+ candidates: candidates.candidatesOf(el),
141
+ secret: isSecret(el, options.secretFields),
142
+ dotted: el.localName === "input" && (el.getAttribute("type") || "").toLowerCase() === "password",
143
+ role: accessible.roleOf(el),
144
+ name: accessible.accessibleName(el),
145
+ };
146
+ }`;
147
+ /**
148
+ * Function declaration, called as `({ secretFields })`. Answers the first of the caller's selectors that matches a
149
+ * visible field showing a value in plain text (a password input draws dots and does not count), else "".
150
+ */
151
+ export const SECRET_ON_SCREEN_SCRIPT = String.raw `(options) => {
152
+ for (const selector of options.secretFields) {
153
+ let found = [];
154
+ try { found = Array.from(document.querySelectorAll(selector)); } catch (error) { continue; }
155
+ for (const el of found) {
156
+ if (el.getClientRects().length === 0 || getComputedStyle(el).visibility === "hidden") continue;
157
+ if (el.localName === "input" && (el.getAttribute("type") || "").toLowerCase() === "password") continue;
158
+ const shown = typeof el.value === "string" ? el.value : (el.textContent || "");
159
+ if (shown.trim() !== "") return selector;
160
+ }
161
+ }
162
+ return "";
163
+ }`;
@@ -0,0 +1,54 @@
1
+ import type Page from "../browser/page.js";
2
+ import type Locator from "../locator/locator.js";
3
+ import type { LocatorSpec } from "../steps/model.js";
4
+ /** A reference does not point at an element of the latest snapshot; the text says why and what to do */
5
+ export declare class SnapshotRefError extends Error {
6
+ constructor(message: string);
7
+ }
8
+ /** The page as text for the model */
9
+ export interface Snapshot {
10
+ /** Header, then one line per element: `[e12] button "Save" disabled` */
11
+ text: string;
12
+ /** Elements that got a reference */
13
+ elements: number;
14
+ /** Lines that did not fit the size cap */
15
+ omitted: number;
16
+ }
17
+ /** An element of the latest snapshot, ready to act on */
18
+ export interface Located {
19
+ ref: string;
20
+ /** The locators the typed step is written with */
21
+ spec: LocatorSpec;
22
+ locator: Locator;
23
+ /** `textbox "Email"`, for messages */
24
+ description: string;
25
+ /** A password or a field the caller marked secret: its value is never shown or typed by the model */
26
+ secret: boolean;
27
+ /** The page itself draws the value as dots (a password input), so even a screenshot does not show it */
28
+ dotted: boolean;
29
+ }
30
+ export interface SnapshotOptions {
31
+ /** CSS selectors of fields to treat as secret, besides password inputs. @default none */
32
+ secretFields?: readonly string[];
33
+ /** The size cap of the text in characters. @default 16000 */
34
+ maxChars?: number;
35
+ }
36
+ /**
37
+ * Takes snapshots of a page and resolves their references. A reference (`e12`) stays the same for an element
38
+ * while it lives and is valid until the next snapshot; after that, only the elements the new snapshot lists
39
+ * resolve. A navigation to a new document starts over.
40
+ */
41
+ export declare class Snapshotter {
42
+ private readonly page;
43
+ private readonly secretFields;
44
+ private readonly maxChars;
45
+ constructor(page: Page, options?: SnapshotOptions);
46
+ take(): Promise<Snapshot>;
47
+ /** The first caller-marked selector that matches a visible field showing its value in plain text, else undefined */
48
+ secretOnScreen(): Promise<string | undefined>;
49
+ /** The element behind `ref`, with the best locator that matches only it. Throws `SnapshotRefError` */
50
+ locate(ref: string): Promise<Located>;
51
+ /** The element that has focus now (what `useSecret` fills). Throws `SnapshotRefError` when nothing is focused */
52
+ locateFocused(): Promise<Located>;
53
+ private locateWith;
54
+ }
@@ -0,0 +1,95 @@
1
+ import { rankLocator } from "../recorder/ranking.js";
2
+ import { locatorFromSpec } from "../steps/build.js";
3
+ import { RESOLVE_SCRIPT, SECRET_ON_SCREEN_SCRIPT, SNAPSHOT_SCRIPT, } from "./snapshot-script.js";
4
+ /** A reference does not point at an element of the latest snapshot; the text says why and what to do */
5
+ export class SnapshotRefError extends Error {
6
+ constructor(message) {
7
+ super(message);
8
+ this.name = "SnapshotRefError";
9
+ }
10
+ }
11
+ /** The size cap when none is given */
12
+ const DEFAULT_MAX_CHARS = 16_000;
13
+ /**
14
+ * Takes snapshots of a page and resolves their references. A reference (`e12`) stays the same for an element
15
+ * while it lives and is valid until the next snapshot; after that, only the elements the new snapshot lists
16
+ * resolve. A navigation to a new document starts over.
17
+ */
18
+ export class Snapshotter {
19
+ constructor(page, options = {}) {
20
+ this.page = page;
21
+ this.secretFields = options.secretFields ?? [];
22
+ this.maxChars = options.maxChars ?? DEFAULT_MAX_CHARS;
23
+ }
24
+ async take() {
25
+ const found = await this.page.evaluate(SNAPSHOT_SCRIPT, {
26
+ maxChars: this.maxChars,
27
+ secretFields: this.secretFields,
28
+ });
29
+ const header = `url: ${found.url}\ntitle: ${found.title}\n`;
30
+ const marker = found.omitted > 0
31
+ ? `\n… [snapshot truncated: ${found.omitted} more elements not shown]`
32
+ : "";
33
+ return {
34
+ text: header + found.text + marker,
35
+ elements: found.count,
36
+ omitted: found.omitted,
37
+ };
38
+ }
39
+ /** The first caller-marked selector that matches a visible field showing its value in plain text, else undefined */
40
+ async secretOnScreen() {
41
+ if (this.secretFields.length === 0)
42
+ return undefined;
43
+ const found = await this.page.evaluate(SECRET_ON_SCREEN_SCRIPT, { secretFields: this.secretFields });
44
+ return found === "" ? undefined : found;
45
+ }
46
+ /** The element behind `ref`, with the best locator that matches only it. Throws `SnapshotRefError` */
47
+ async locate(ref) {
48
+ const match = /^e(\d+)$/.exec(ref.trim());
49
+ if (!match)
50
+ throw new SnapshotRefError(`"${ref}" is not a snapshot reference; references look like e12 and come from a snapshot`);
51
+ const label = `e${Number(match[1])}`;
52
+ return this.locateWith(label, { ref: Number(match[1]) });
53
+ }
54
+ /** The element that has focus now (what `useSecret` fills). Throws `SnapshotRefError` when nothing is focused */
55
+ async locateFocused() {
56
+ return this.locateWith("the focused field", { focused: true });
57
+ }
58
+ async locateWith(label, which) {
59
+ const found = await this.page.evaluate(RESOLVE_SCRIPT, {
60
+ ...which,
61
+ secretFields: this.secretFields,
62
+ });
63
+ if (found.status !== "ok")
64
+ throw new SnapshotRefError(refProblem(label, found.status));
65
+ const spec = await rankLocator(this.page, found.target, found.candidates);
66
+ if (!spec)
67
+ throw new SnapshotRefError(`${label} has no locator that matches only that element; take a new snapshot`);
68
+ return {
69
+ ref: label,
70
+ spec,
71
+ locator: locatorFromSpec(this.page, spec),
72
+ description: describe(found.role, found.name),
73
+ secret: found.secret,
74
+ dotted: found.dotted,
75
+ };
76
+ }
77
+ }
78
+ function describe(role, name) {
79
+ const kind = role === "" ? "element" : role;
80
+ return name === "" ? kind : `${kind} "${name}"`;
81
+ }
82
+ function refProblem(label, status) {
83
+ switch (status) {
84
+ case "none":
85
+ return `${label} cannot be used: no snapshot has been taken of this page yet (the page is new or has reloaded); take a snapshot first`;
86
+ case "unknown":
87
+ return `${label} does not exist; use a reference from the latest snapshot`;
88
+ case "older":
89
+ return `${label} was in an earlier snapshot but is not in the latest one: the page changed; take a new snapshot`;
90
+ case "gone":
91
+ return `${label} is no longer on the page (it was removed after the snapshot); take a new snapshot`;
92
+ case "nofocus":
93
+ return "no field has focus; click the field first";
94
+ }
95
+ }
@@ -0,0 +1,19 @@
1
+ import type Page from "../browser/page.js";
2
+ import type { StepRecorder } from "./recording.js";
3
+ import type { TakeOverHooks } from "./session.js";
4
+ import type { Snapshotter } from "./snapshot.js";
5
+ export interface PersonRecording extends TakeOverHooks {
6
+ /** Ends a recording that is still open (the run ended while the person had the browser) */
7
+ dispose(): Promise<void>;
8
+ }
9
+ /**
10
+ * While the person drives, the framework's own recorder watches the same page and writes what they do into the
11
+ * spec after the agent's steps, so nothing they record is lost. On hand back the recorder stops, the agent's
12
+ * recording carries on after the last step, and the agent is told what happened and shown the page as it is.
13
+ */
14
+ export declare function personRecording(input: {
15
+ page: Page;
16
+ steps: StepRecorder;
17
+ snapshots: Snapshotter;
18
+ baseURL?: string;
19
+ }): PersonRecording;
@@ -0,0 +1,49 @@
1
+ import { maskText } from "../config/mask.js";
2
+ import { stepToSource } from "../steps/emit.js";
3
+ /**
4
+ * While the person drives, the framework's own recorder watches the same page and writes what they do into the
5
+ * spec after the agent's steps, so nothing they record is lost. On hand back the recorder stops, the agent's
6
+ * recording carries on after the last step, and the agent is told what happened and shown the page as it is.
7
+ */
8
+ export function personRecording(input) {
9
+ const { page, steps, snapshots } = input;
10
+ let recorder;
11
+ const written = new Map();
12
+ const close = async () => {
13
+ const open = recorder;
14
+ recorder = undefined;
15
+ if (!open)
16
+ return;
17
+ await open.stop().catch(() => undefined);
18
+ steps.rebase();
19
+ };
20
+ return {
21
+ async begin() {
22
+ written.clear();
23
+ recorder = await page.record({
24
+ at: steps.count,
25
+ initialGoto: false,
26
+ ...(input.baseURL !== undefined && { baseURL: input.baseURL }),
27
+ onEvent: (event) => {
28
+ steps.external(event);
29
+ if (event.op === "remove")
30
+ written.delete(event.index);
31
+ else
32
+ written.set(event.index, stepToSource(event.step));
33
+ },
34
+ });
35
+ },
36
+ async end() {
37
+ await close();
38
+ const mine = [...written.entries()]
39
+ .sort(([a], [b]) => a - b)
40
+ .map(([, code]) => code);
41
+ const did = mine.length > 0
42
+ ? `They added ${mine.length} ${mine.length === 1 ? "step" : "steps"} to the test: ${mine.join(" ")}`
43
+ : "They added no steps";
44
+ const page_ = maskText((await snapshots.take()).text);
45
+ return `The person used the browser while you were stopped. ${did}. The page may have changed; this is the page now:\n\n${page_}\n\nYour last call was not run. Call it again only if it is still needed.`;
46
+ },
47
+ dispose: close,
48
+ };
49
+ }
@@ -0,0 +1,2 @@
1
+ import { type ToolContext } from "./context.js";
2
+ export declare const act: (context: ToolContext) => import("./context.js").AgentTool;
@@ -0,0 +1,124 @@
1
+ import { z } from "zod";
2
+ import { writtenText } from "../recording.js";
3
+ import { leftAllowedOrigins } from "./origin-guard.js";
4
+ import { defineTool } from "./context.js";
5
+ const KINDS = [
6
+ "click",
7
+ "fill",
8
+ "select",
9
+ "check",
10
+ "uncheck",
11
+ "hover",
12
+ "dblclick",
13
+ "clear",
14
+ "press",
15
+ ];
16
+ /** The kinds that need a `value`, and what it is */
17
+ const VALUE_OF = {
18
+ fill: "the text to type",
19
+ select: "the option's value or visible text",
20
+ press: 'the key, for example "Enter" or "Control+A"',
21
+ };
22
+ const PAST = {
23
+ click: "Clicked",
24
+ fill: "Filled",
25
+ select: "Selected an option of",
26
+ check: "Checked",
27
+ uncheck: "Unchecked",
28
+ hover: "Hovered",
29
+ dblclick: "Double-clicked",
30
+ clear: "Cleared",
31
+ press: "Pressed a key on",
32
+ };
33
+ async function perform(kind, value, { locator }, timeout) {
34
+ const options = { timeout };
35
+ switch (kind) {
36
+ case "click":
37
+ return locator.click(options);
38
+ case "fill":
39
+ return locator.fill(value, options);
40
+ case "select":
41
+ await locator.selectOption(value, options);
42
+ return;
43
+ case "check":
44
+ return locator.check(options);
45
+ case "uncheck":
46
+ return locator.uncheck(options);
47
+ case "hover":
48
+ return locator.hover(options);
49
+ case "dblclick":
50
+ return locator.dblclick(options);
51
+ case "clear":
52
+ return locator.clear(options);
53
+ case "press":
54
+ return locator.press(value, options);
55
+ }
56
+ }
57
+ /** The observation the recorder's normaliser turns into the step for this action */
58
+ function observationFor(kind, value, located) {
59
+ const { spec: locator, ref: key } = located;
60
+ switch (kind) {
61
+ case "click":
62
+ return { type: "click", key, locator, textEntry: false };
63
+ case "check":
64
+ case "uncheck":
65
+ return {
66
+ type: "click",
67
+ key,
68
+ locator,
69
+ textEntry: false,
70
+ checked: kind === "check",
71
+ };
72
+ case "fill":
73
+ return { type: "input", key, locator, value };
74
+ case "select":
75
+ return { type: "select", key, locator, option: value };
76
+ case "press":
77
+ return { type: "press", key, locator, pressed: value };
78
+ case "hover":
79
+ case "dblclick":
80
+ case "clear":
81
+ return { type: "action", kind, locator };
82
+ }
83
+ }
84
+ export const act = defineTool({
85
+ name: "act",
86
+ description: "Do one thing to an element of the latest snapshot: click, fill (type text), select (an option of a dropdown), " +
87
+ "check, uncheck, hover, dblclick, clear or press (a key). Each success is written to the test as a step; " +
88
+ "a failure writes nothing. Take a new snapshot after the page changes.",
89
+ inputSchema: z.object({
90
+ kind: z.enum(KINDS),
91
+ target: z
92
+ .string()
93
+ .describe("A reference from the latest snapshot, for example e12"),
94
+ value: z
95
+ .string()
96
+ .optional()
97
+ .describe("fill: the text; select: the option; press: the key"),
98
+ }),
99
+ async execute({ kind, target, value }, context) {
100
+ const needs = VALUE_OF[kind];
101
+ if (needs !== undefined &&
102
+ (value === undefined || (kind !== "fill" && value === "")))
103
+ return { ok: false, error: `act(): ${kind} needs a value: ${needs}` };
104
+ if (needs === undefined && value !== undefined)
105
+ return { ok: false, error: `act(): ${kind} takes no value` };
106
+ const located = await context.snapshots.locate(target);
107
+ if (kind === "fill" && located.secret)
108
+ return {
109
+ ok: false,
110
+ error: `act(): ${located.ref} (${located.description}) holds a secret; focus it with a click and use useSecret, or ask the person`,
111
+ };
112
+ const text = value ?? "";
113
+ const before = await context.page.url();
114
+ await perform(kind, text, located, context.actionTimeout);
115
+ const problem = await leftAllowedOrigins(context, before);
116
+ if (problem)
117
+ return { ok: false, error: `act(): ${problem}` };
118
+ const events = context.steps.observe(observationFor(kind, text, located));
119
+ return {
120
+ ok: true,
121
+ message: `${PAST[kind]} ${located.ref} (${located.description}).${writtenText(events)}`,
122
+ };
123
+ },
124
+ });
@@ -0,0 +1 @@
1
+ export declare const ask: (context: import("./context.js").ToolContext) => import("./context.js").AgentTool;
@@ -0,0 +1,19 @@
1
+ import { z } from "zod";
2
+ import { defineTool } from "./context.js";
3
+ export const ask = defineTool({
4
+ name: "ask",
5
+ description: "Ask the person a question and wait for the answer. This is the only way to get what you do not know " +
6
+ "(a username, which of two options they want, permission). Do not guess; ask.",
7
+ inputSchema: z.object({
8
+ question: z.string().min(1),
9
+ options: z
10
+ .array(z.string().min(1))
11
+ .max(8)
12
+ .optional()
13
+ .describe("Answers to pick from, when there are a few"),
14
+ }),
15
+ async execute({ question, options }, context) {
16
+ const answer = await context.ask(question, options);
17
+ return { ok: true, message: `The person answered: ${answer}` };
18
+ },
19
+ });
@@ -0,0 +1,70 @@
1
+ import type { z } from "zod";
2
+ import type Page from "../../browser/page.js";
3
+ import type { Snapshotter } from "../snapshot.js";
4
+ import type { AgentEvent } from "../types.js";
5
+ import type { OriginPolicy } from "./origins.js";
6
+ import type { Observation, RecorderEvent } from "../../recorder/normalise.js";
7
+ /** Secrets the agent may use: it knows the names; values are read here and go no further than the page */
8
+ interface SecretAccess {
9
+ readonly allowed: readonly string[];
10
+ /** The value of a secret in the run's environment; undefined when it has none */
11
+ get(name: string): string | undefined;
12
+ }
13
+ /** What the shared tools need from the run's record of what was done: a spec writer, or the planner's tally */
14
+ export interface ToolSink {
15
+ /** Takes note of one thing done on the page and returns the events it wrote (none when nothing is written) */
16
+ observe(observation: Observation): RecorderEvent[];
17
+ /** Takes the last `count` steps back and returns how many went */
18
+ undo(count: number): number;
19
+ }
20
+ /** What the tools of one run share */
21
+ export interface ToolContext {
22
+ page: Page;
23
+ snapshots: Snapshotter;
24
+ steps: ToolSink;
25
+ origins: OriginPolicy;
26
+ secrets: SecretAccess;
27
+ /** The model takes images, so `screenshot` works */
28
+ vision: boolean;
29
+ /** Time (ms) an action or an assertion waits for the element. @default 5000 */
30
+ actionTimeout: number;
31
+ /** Asks the person and waits for the answer */
32
+ ask(question: string, options?: string[]): Promise<string>;
33
+ /** Records the agent's closing summary; the loop ends after the call */
34
+ finish(summary: string): void;
35
+ emit(event: AgentEvent): void;
36
+ /** A secret was typed into a field the page shows in plain text: screenshots would show it */
37
+ secretOnScreen: boolean;
38
+ }
39
+ /** What a tool answers the model with. A failure is a result too: the model reads it and tries something else */
40
+ export type ToolResult = {
41
+ ok: true;
42
+ message: string;
43
+ /** For a vision model */
44
+ image?: {
45
+ mediaType: "image/png";
46
+ data: string;
47
+ };
48
+ } | {
49
+ ok: false;
50
+ error: string;
51
+ };
52
+ /** A tool as the loop uses it: the model-facing description and schema, and `run` for one call */
53
+ export interface AgentTool {
54
+ readonly name: string;
55
+ readonly description: string;
56
+ readonly inputSchema: z.ZodType;
57
+ /** Validates `args`, runs the tool and answers; never throws. Secret values are removed from the answer */
58
+ run(args: unknown): Promise<ToolResult>;
59
+ }
60
+ interface ToolSpec<S extends z.ZodType, C extends ToolContext> {
61
+ name: string;
62
+ description: string;
63
+ inputSchema: S;
64
+ execute(input: z.infer<S>, context: C): Promise<ToolResult>;
65
+ }
66
+ /** The first lines of an error message: enough for the model, without a long call log */
67
+ export declare function shortMessage(error: unknown, max?: number): string;
68
+ /** A tool bound to a run. Wraps `execute` so every tool validates, reports and fails the same way */
69
+ export declare function defineTool<S extends z.ZodType, C extends ToolContext = ToolContext>(spec: ToolSpec<S, C>): (context: C) => AgentTool;
70
+ export {};
@@ -0,0 +1,42 @@
1
+ import { maskDeep } from "../../config/mask.js";
2
+ /** The first lines of an error message: enough for the model, without a long call log */
3
+ export function shortMessage(error, max = 500) {
4
+ const text = (error instanceof Error ? error.message : String(error)).trim();
5
+ const lines = text.split("\n").filter((line) => line.trim() !== "");
6
+ const joined = lines.slice(0, 4).join(" ");
7
+ return joined.length > max ? `${joined.slice(0, max)}…` : joined;
8
+ }
9
+ /** A tool bound to a run. Wraps `execute` so every tool validates, reports and fails the same way */
10
+ export function defineTool(spec) {
11
+ return (context) => ({
12
+ name: spec.name,
13
+ description: spec.description,
14
+ inputSchema: spec.inputSchema,
15
+ async run(args) {
16
+ context.emit({
17
+ type: "tool-call",
18
+ tool: spec.name,
19
+ args: maskDeep(args),
20
+ });
21
+ const parsed = spec.inputSchema.safeParse(args);
22
+ if (!parsed.success) {
23
+ const problems = parsed.error.issues
24
+ .map((issue) => `${issue.path.join(".") || "arguments"}: ${issue.message}`)
25
+ .join("; ");
26
+ return maskDeep({
27
+ ok: false,
28
+ error: `${spec.name}(): invalid arguments (${problems})`,
29
+ });
30
+ }
31
+ try {
32
+ return maskDeep(await spec.execute(parsed.data, context));
33
+ }
34
+ catch (error) {
35
+ return maskDeep({
36
+ ok: false,
37
+ error: `${spec.name}(): ${shortMessage(error)}`,
38
+ });
39
+ }
40
+ },
41
+ });
42
+ }