libretto 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/dist/cli/cli.js +20 -19
  2. package/dist/cli/commands/ai.js +1 -1
  3. package/dist/cli/commands/browser.js +3 -3
  4. package/dist/cli/commands/execution.js +3 -3
  5. package/dist/cli/commands/logs.js +1 -1
  6. package/dist/cli/core/browser.js +11 -6
  7. package/dist/cli/core/context.js +4 -18
  8. package/dist/cli/core/session.js +2 -2
  9. package/dist/cli/core/snapshot-analyzer.js +2 -2
  10. package/dist/cli/router.js +1 -1
  11. package/dist/cli/workers/run-integration-runtime.js +2 -2
  12. package/dist/shared/paths/paths.js +2 -1
  13. package/dist/shared/paths/repo-root.d.ts +3 -0
  14. package/dist/shared/paths/repo-root.js +24 -0
  15. package/package.json +6 -7
  16. package/scripts/postinstall.mjs +12 -3
  17. package/skills/libretto/SKILL.md +93 -404
  18. package/skills/libretto/references/auth-profiles.md +30 -0
  19. package/skills/libretto/references/pages-and-page-targeting.md +29 -0
  20. package/skills/libretto/references/reverse-engineering-network-requests.md +39 -0
  21. package/skills/libretto/references/user-action-log.md +31 -0
  22. package/src/cli/cli.ts +173 -0
  23. package/src/cli/commands/ai.ts +35 -0
  24. package/src/cli/commands/browser.ts +165 -0
  25. package/src/cli/commands/execution.ts +691 -0
  26. package/src/cli/commands/init.ts +327 -0
  27. package/src/cli/commands/logs.ts +128 -0
  28. package/src/cli/commands/shared.ts +70 -0
  29. package/src/cli/commands/snapshot.ts +327 -0
  30. package/src/cli/core/ai-config.ts +255 -0
  31. package/src/cli/core/api-snapshot-analyzer.ts +97 -0
  32. package/src/cli/core/browser.ts +839 -0
  33. package/src/cli/core/context.ts +122 -0
  34. package/src/cli/core/pause-signals.ts +35 -0
  35. package/src/cli/core/session-telemetry.ts +553 -0
  36. package/src/cli/core/session.ts +209 -0
  37. package/src/cli/core/snapshot-analyzer.ts +875 -0
  38. package/src/cli/core/snapshot-api-config.ts +236 -0
  39. package/src/cli/core/telemetry.ts +446 -0
  40. package/src/cli/framework/simple-cli.ts +1273 -0
  41. package/src/cli/index.ts +13 -0
  42. package/src/cli/router.ts +28 -0
  43. package/src/cli/workers/run-integration-runtime.ts +311 -0
  44. package/src/cli/workers/run-integration-worker-protocol.ts +14 -0
  45. package/src/cli/workers/run-integration-worker.ts +75 -0
  46. package/src/index.ts +120 -0
  47. package/src/runtime/download/download.ts +100 -0
  48. package/src/runtime/download/index.ts +7 -0
  49. package/src/runtime/extract/extract.ts +92 -0
  50. package/src/runtime/extract/index.ts +1 -0
  51. package/src/runtime/network/index.ts +5 -0
  52. package/src/runtime/network/network.ts +113 -0
  53. package/src/runtime/recovery/agent.ts +256 -0
  54. package/src/runtime/recovery/errors.ts +152 -0
  55. package/src/runtime/recovery/index.ts +7 -0
  56. package/src/runtime/recovery/recovery.ts +50 -0
  57. package/{dist/shared/condense-dom/condense-dom.cjs → src/shared/condense-dom/condense-dom.ts} +243 -115
  58. package/src/shared/config/config.ts +22 -0
  59. package/src/shared/config/index.ts +5 -0
  60. package/src/shared/debug/index.ts +1 -0
  61. package/src/shared/debug/pause.ts +85 -0
  62. package/src/shared/instrumentation/errors.ts +82 -0
  63. package/src/shared/instrumentation/index.ts +9 -0
  64. package/src/shared/instrumentation/instrument.ts +276 -0
  65. package/src/shared/llm/ai-sdk-adapter.ts +78 -0
  66. package/src/shared/llm/client.ts +217 -0
  67. package/src/shared/llm/index.ts +3 -0
  68. package/src/shared/llm/types.ts +63 -0
  69. package/src/shared/logger/index.ts +6 -0
  70. package/src/shared/logger/logger.ts +352 -0
  71. package/src/shared/logger/sinks.ts +144 -0
  72. package/src/shared/paths/paths.ts +109 -0
  73. package/src/shared/paths/repo-root.ts +27 -0
  74. package/src/shared/run/api.ts +2 -0
  75. package/src/shared/run/browser.ts +98 -0
  76. package/src/shared/state/index.ts +11 -0
  77. package/src/shared/state/session-state.ts +74 -0
  78. package/src/shared/visualization/ghost-cursor.ts +200 -0
  79. package/src/shared/visualization/highlight.ts +146 -0
  80. package/src/shared/visualization/index.ts +18 -0
  81. package/src/shared/workflow/workflow.ts +42 -0
  82. package/dist/index.cjs +0 -144
  83. package/dist/index.d.cts +0 -21
  84. package/dist/runtime/download/download.cjs +0 -70
  85. package/dist/runtime/download/download.d.cts +0 -35
  86. package/dist/runtime/download/index.cjs +0 -30
  87. package/dist/runtime/download/index.d.cts +0 -3
  88. package/dist/runtime/extract/extract.cjs +0 -88
  89. package/dist/runtime/extract/extract.d.cts +0 -23
  90. package/dist/runtime/extract/index.cjs +0 -28
  91. package/dist/runtime/extract/index.d.cts +0 -5
  92. package/dist/runtime/network/index.cjs +0 -28
  93. package/dist/runtime/network/index.d.cts +0 -4
  94. package/dist/runtime/network/network.cjs +0 -91
  95. package/dist/runtime/network/network.d.cts +0 -28
  96. package/dist/runtime/recovery/agent.cjs +0 -223
  97. package/dist/runtime/recovery/agent.d.cts +0 -13
  98. package/dist/runtime/recovery/errors.cjs +0 -124
  99. package/dist/runtime/recovery/errors.d.cts +0 -31
  100. package/dist/runtime/recovery/index.cjs +0 -34
  101. package/dist/runtime/recovery/index.d.cts +0 -7
  102. package/dist/runtime/recovery/recovery.cjs +0 -55
  103. package/dist/runtime/recovery/recovery.d.cts +0 -12
  104. package/dist/shared/condense-dom/condense-dom.d.cts +0 -34
  105. package/dist/shared/config/config.cjs +0 -44
  106. package/dist/shared/config/config.d.cts +0 -10
  107. package/dist/shared/config/index.cjs +0 -32
  108. package/dist/shared/config/index.d.cts +0 -1
  109. package/dist/shared/debug/index.cjs +0 -28
  110. package/dist/shared/debug/index.d.cts +0 -1
  111. package/dist/shared/debug/pause.cjs +0 -86
  112. package/dist/shared/debug/pause.d.cts +0 -12
  113. package/dist/shared/instrumentation/errors.cjs +0 -81
  114. package/dist/shared/instrumentation/errors.d.cts +0 -12
  115. package/dist/shared/instrumentation/index.cjs +0 -35
  116. package/dist/shared/instrumentation/index.d.cts +0 -6
  117. package/dist/shared/instrumentation/instrument.cjs +0 -206
  118. package/dist/shared/instrumentation/instrument.d.cts +0 -32
  119. package/dist/shared/llm/ai-sdk-adapter.cjs +0 -71
  120. package/dist/shared/llm/ai-sdk-adapter.d.cts +0 -22
  121. package/dist/shared/llm/client.cjs +0 -218
  122. package/dist/shared/llm/client.d.cts +0 -13
  123. package/dist/shared/llm/index.cjs +0 -31
  124. package/dist/shared/llm/index.d.cts +0 -5
  125. package/dist/shared/llm/types.cjs +0 -16
  126. package/dist/shared/llm/types.d.cts +0 -67
  127. package/dist/shared/logger/index.cjs +0 -37
  128. package/dist/shared/logger/index.d.cts +0 -2
  129. package/dist/shared/logger/logger.cjs +0 -232
  130. package/dist/shared/logger/logger.d.cts +0 -86
  131. package/dist/shared/logger/sinks.cjs +0 -160
  132. package/dist/shared/logger/sinks.d.cts +0 -9
  133. package/dist/shared/paths/paths.cjs +0 -104
  134. package/dist/shared/paths/paths.d.cts +0 -10
  135. package/dist/shared/run/api.cjs +0 -28
  136. package/dist/shared/run/api.d.cts +0 -2
  137. package/dist/shared/run/browser.cjs +0 -98
  138. package/dist/shared/run/browser.d.cts +0 -22
  139. package/dist/shared/state/index.cjs +0 -38
  140. package/dist/shared/state/index.d.cts +0 -2
  141. package/dist/shared/state/session-state.cjs +0 -92
  142. package/dist/shared/state/session-state.d.cts +0 -40
  143. package/dist/shared/visualization/ghost-cursor.cjs +0 -174
  144. package/dist/shared/visualization/ghost-cursor.d.cts +0 -37
  145. package/dist/shared/visualization/highlight.cjs +0 -134
  146. package/dist/shared/visualization/highlight.d.cts +0 -22
  147. package/dist/shared/visualization/index.cjs +0 -45
  148. package/dist/shared/visualization/index.d.cts +0 -3
  149. package/dist/shared/workflow/workflow.cjs +0 -47
  150. package/dist/shared/workflow/workflow.d.cts +0 -21
  151. package/skills/libretto/code-generation-rules.md +0 -223
  152. package/skills/libretto/integration-approach-selection.md +0 -174
@@ -1,28 +0,0 @@
1
- "use strict";
2
- var __defProp = Object.defineProperty;
3
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
- var __getOwnPropNames = Object.getOwnPropertyNames;
5
- var __hasOwnProp = Object.prototype.hasOwnProperty;
6
- var __export = (target, all) => {
7
- for (var name in all)
8
- __defProp(target, name, { get: all[name], enumerable: true });
9
- };
10
- var __copyProps = (to, from, except, desc) => {
11
- if (from && typeof from === "object" || typeof from === "function") {
12
- for (let key of __getOwnPropNames(from))
13
- if (!__hasOwnProp.call(to, key) && key !== except)
14
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
- }
16
- return to;
17
- };
18
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
- var network_exports = {};
20
- __export(network_exports, {
21
- pageRequest: () => import_network.pageRequest
22
- });
23
- module.exports = __toCommonJS(network_exports);
24
- var import_network = require("./network.js");
25
- // Annotate the CommonJS export names for ESM import in node:
26
- 0 && (module.exports = {
27
- pageRequest
28
- });
@@ -1,4 +0,0 @@
1
- export { PageRequestOptions, RequestConfig, pageRequest } from './network.cjs';
2
- import 'playwright';
3
- import 'zod';
4
- import '../../shared/logger/logger.cjs';
@@ -1,91 +0,0 @@
1
- "use strict";
2
- var __defProp = Object.defineProperty;
3
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
- var __getOwnPropNames = Object.getOwnPropertyNames;
5
- var __hasOwnProp = Object.prototype.hasOwnProperty;
6
- var __export = (target, all) => {
7
- for (var name in all)
8
- __defProp(target, name, { get: all[name], enumerable: true });
9
- };
10
- var __copyProps = (to, from, except, desc) => {
11
- if (from && typeof from === "object" || typeof from === "function") {
12
- for (let key of __getOwnPropNames(from))
13
- if (!__hasOwnProp.call(to, key) && key !== except)
14
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
- }
16
- return to;
17
- };
18
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
- var network_exports = {};
20
- __export(network_exports, {
21
- pageRequest: () => pageRequest
22
- });
23
- module.exports = __toCommonJS(network_exports);
24
- async function pageRequest(page, config, options) {
25
- const { url, method = "GET", headers = {}, body, bodyType = "json", responseType = "json" } = config;
26
- const { logger, schema } = options ?? {};
27
- const startTime = Date.now();
28
- const fetchHeaders = { ...headers };
29
- let fetchBody;
30
- if (body !== void 0) {
31
- if (bodyType === "form") {
32
- fetchHeaders["Content-Type"] = "application/x-www-form-urlencoded";
33
- if (typeof body === "string") {
34
- fetchBody = body;
35
- } else {
36
- fetchBody = new URLSearchParams(
37
- Object.entries(body).map(([k, v]) => [k, String(v)])
38
- ).toString();
39
- }
40
- } else {
41
- fetchHeaders["Content-Type"] = "application/json";
42
- fetchBody = typeof body === "string" ? body : JSON.stringify(body);
43
- }
44
- }
45
- const result = await page.evaluate(
46
- async ({ url: url2, method: method2, headers: headers2, body: body2, responseType: responseType2 }) => {
47
- const res = await fetch(url2, {
48
- method: method2,
49
- headers: headers2,
50
- body: body2 ?? void 0
51
- });
52
- const status = res.status;
53
- const ok = res.ok;
54
- let data;
55
- if (responseType2 === "json") {
56
- data = await res.json();
57
- } else {
58
- data = await res.text();
59
- }
60
- return { status, ok, data };
61
- },
62
- { url, method, headers: fetchHeaders, body: fetchBody, responseType }
63
- );
64
- const duration = Date.now() - startTime;
65
- if (!result.ok) {
66
- logger?.warn("network:request:error", {
67
- method,
68
- url,
69
- status: result.status,
70
- duration,
71
- body: typeof result.data === "string" ? result.data.slice(0, 500) : void 0
72
- });
73
- throw new Error(
74
- `pageRequest failed: ${method} ${url} returned ${result.status}`
75
- );
76
- }
77
- logger?.info("network:request", {
78
- method,
79
- url,
80
- status: result.status,
81
- duration
82
- });
83
- if (schema) {
84
- return schema.parse(result.data);
85
- }
86
- return result.data;
87
- }
88
- // Annotate the CommonJS export names for ESM import in node:
89
- 0 && (module.exports = {
90
- pageRequest
91
- });
@@ -1,28 +0,0 @@
1
- import { Page } from 'playwright';
2
- import z from 'zod';
3
- import { MinimalLogger } from '../../shared/logger/logger.cjs';
4
-
5
- type RequestConfig = {
6
- url: string;
7
- method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
8
- headers?: Record<string, string>;
9
- body?: Record<string, any> | string;
10
- /** How to serialize the body. Defaults to "json". */
11
- bodyType?: "json" | "form";
12
- /** How to parse the response. Defaults to "json". */
13
- responseType?: "json" | "text" | "xml";
14
- };
15
- type PageRequestOptions<T extends z.ZodType | undefined = undefined> = {
16
- logger?: MinimalLogger;
17
- /** Optional Zod schema to validate the response body. */
18
- schema?: T;
19
- };
20
- type PageRequestResult<T extends z.ZodType | undefined> = T extends z.ZodType ? z.infer<T> : any;
21
- /**
22
- * Executes a fetch() call inside the browser context via page.evaluate().
23
- * Provides typed request config, automatic response parsing, optional Zod
24
- * validation, and logging.
25
- */
26
- declare function pageRequest<T extends z.ZodType | undefined = undefined>(page: Page, config: RequestConfig, options?: PageRequestOptions<T>): Promise<PageRequestResult<T>>;
27
-
28
- export { type PageRequestOptions, type RequestConfig, pageRequest };
@@ -1,223 +0,0 @@
1
- "use strict";
2
- var __defProp = Object.defineProperty;
3
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
- var __getOwnPropNames = Object.getOwnPropertyNames;
5
- var __hasOwnProp = Object.prototype.hasOwnProperty;
6
- var __export = (target, all) => {
7
- for (var name in all)
8
- __defProp(target, name, { get: all[name], enumerable: true });
9
- };
10
- var __copyProps = (to, from, except, desc) => {
11
- if (from && typeof from === "object" || typeof from === "function") {
12
- for (let key of __getOwnPropNames(from))
13
- if (!__hasOwnProp.call(to, key) && key !== except)
14
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
- }
16
- return to;
17
- };
18
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
- var agent_exports = {};
20
- __export(agent_exports, {
21
- executeRecoveryAgent: () => executeRecoveryAgent
22
- });
23
- module.exports = __toCommonJS(agent_exports);
24
- var import_logger = require("../../shared/logger/logger.js");
25
- var import_zod = require("zod");
26
- function delay(ms) {
27
- return new Promise((resolve) => setTimeout(resolve, ms));
28
- }
29
- const KEY_MAPPINGS = {
30
- ENTER: "Enter",
31
- RETURN: "Enter",
32
- TAB: "Tab",
33
- SPACE: " ",
34
- BACKSPACE: "Backspace",
35
- DELETE: "Delete",
36
- ESCAPE: "Escape",
37
- ESC: "Escape",
38
- UP: "ArrowUp",
39
- DOWN: "ArrowDown",
40
- LEFT: "ArrowLeft",
41
- RIGHT: "ArrowRight",
42
- HOME: "Home",
43
- END: "End",
44
- PAGEUP: "PageUp",
45
- PAGEDOWN: "PageDown",
46
- CTRL: "Control",
47
- CONTROL: "Control",
48
- ALT: "Alt",
49
- SHIFT: "Shift",
50
- META: "Meta",
51
- CMD: "Meta",
52
- COMMAND: "Meta"
53
- };
54
- function mapKeyName(key) {
55
- return KEY_MAPPINGS[key.toUpperCase()] ?? key;
56
- }
57
- async function executeBrowserAction(page, action, logger = import_logger.defaultLogger) {
58
- switch (action.type) {
59
- case "click": {
60
- const { x, y, button = "left" } = action;
61
- const playwrightButton = button === "wheel" || button === "back" || button === "forward" ? "left" : button;
62
- await page.mouse.click(x, y, { button: playwrightButton });
63
- logger.info(`Clicked at (${x}, ${y}) with ${button} button`);
64
- break;
65
- }
66
- case "double_click": {
67
- const { x, y } = action;
68
- await page.mouse.dblclick(x, y);
69
- logger.info(`Double-clicked at (${x}, ${y})`);
70
- break;
71
- }
72
- case "scroll": {
73
- const { x, y, scroll_x, scroll_y } = action;
74
- await page.mouse.move(x, y);
75
- await page.evaluate(`window.scrollBy(${scroll_x}, ${scroll_y})`);
76
- logger.info(`Scrolled at (${x}, ${y}) by (${scroll_x}, ${scroll_y})`);
77
- break;
78
- }
79
- case "keypress": {
80
- for (const key of action.keys) {
81
- const mapped = mapKeyName(key);
82
- await page.keyboard.press(mapped);
83
- logger.info(`Pressed key: ${key} (mapped to ${mapped})`);
84
- }
85
- break;
86
- }
87
- case "type": {
88
- await page.keyboard.type(action.text);
89
- logger.info(`Typed text: ${action.text}`);
90
- break;
91
- }
92
- case "wait": {
93
- await delay(2e3);
94
- logger.info("Waited 2 seconds");
95
- break;
96
- }
97
- case "screenshot": {
98
- logger.info("Screenshot action (no-op, taken automatically)");
99
- break;
100
- }
101
- case "drag": {
102
- const { path } = action;
103
- const start = path[0];
104
- const end = path[path.length - 1];
105
- if (path.length >= 2 && start && end) {
106
- await page.mouse.move(start.x, start.y);
107
- await page.mouse.down();
108
- for (let i = 1; i < path.length; i++) {
109
- const point = path[i];
110
- if (point) await page.mouse.move(point.x, point.y);
111
- }
112
- await page.mouse.up();
113
- logger.info(`Dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
114
- }
115
- break;
116
- }
117
- case "move": {
118
- const { x, y } = action;
119
- await page.mouse.move(x, y);
120
- logger.info(`Moved mouse to (${x}, ${y})`);
121
- break;
122
- }
123
- case "done": {
124
- break;
125
- }
126
- }
127
- }
128
- const recoveryActionSchema = import_zod.z.object({
129
- reasoning: import_zod.z.string().describe("Your reasoning about what you see and what action to take"),
130
- action: import_zod.z.discriminatedUnion("type", [
131
- import_zod.z.object({
132
- type: import_zod.z.literal("click"),
133
- x: import_zod.z.number(),
134
- y: import_zod.z.number()
135
- }),
136
- import_zod.z.object({
137
- type: import_zod.z.literal("type"),
138
- text: import_zod.z.string()
139
- }),
140
- import_zod.z.object({
141
- type: import_zod.z.literal("keypress"),
142
- keys: import_zod.z.array(import_zod.z.string())
143
- }),
144
- import_zod.z.object({
145
- type: import_zod.z.literal("scroll"),
146
- x: import_zod.z.number(),
147
- y: import_zod.z.number(),
148
- scroll_x: import_zod.z.number(),
149
- scroll_y: import_zod.z.number()
150
- }),
151
- import_zod.z.object({
152
- type: import_zod.z.literal("wait")
153
- }),
154
- import_zod.z.object({
155
- type: import_zod.z.literal("done")
156
- })
157
- ])
158
- });
159
- async function executeRecoveryAgent(page, instruction, logger, llmClient) {
160
- if (!llmClient) {
161
- return;
162
- }
163
- const log = logger ?? import_logger.defaultLogger;
164
- log.info("Executing vision-based recovery agent", { instruction });
165
- const viewport = page.viewportSize();
166
- if (!viewport) {
167
- throw new Error("Viewport size not found");
168
- }
169
- let screenshot;
170
- try {
171
- screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
172
- } catch (screenshotError) {
173
- log.warn("Failed to take screenshot for recovery agent, skipping", {
174
- screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
175
- });
176
- throw new Error("Failed to take screenshot for recovery agent");
177
- }
178
- const maxSteps = 3;
179
- for (let step = 1; step <= maxSteps; step++) {
180
- const result = await llmClient.generateObjectFromMessages({
181
- schema: recoveryActionSchema,
182
- messages: [
183
- {
184
- role: "user",
185
- content: [
186
- {
187
- type: "text",
188
- text: `You are an expert browser support agent. Your job is to resolve issues when browser automation encounters unexpected website behavior (e.g., popups blocking interaction).
189
-
190
- Your task: ${instruction}
191
-
192
- Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
193
- Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`
194
- },
195
- {
196
- type: "image",
197
- image: `data:image/png;base64,${screenshot}`
198
- }
199
- ]
200
- }
201
- ],
202
- temperature: 0
203
- });
204
- log.info(`Recovery step ${step}/${maxSteps}`, {
205
- reasoning: result.reasoning,
206
- action: result.action
207
- });
208
- if (result.action.type === "done") {
209
- log.info("Recovery agent completed - no more actions needed");
210
- break;
211
- }
212
- await executeBrowserAction(page, result.action, log);
213
- await delay(2e3);
214
- screenshot = (await page.screenshot({ fullPage: false })).toString(
215
- "base64"
216
- );
217
- }
218
- log.info("Recovery agent execution completed");
219
- }
220
- // Annotate the CommonJS export names for ESM import in node:
221
- 0 && (module.exports = {
222
- executeRecoveryAgent
223
- });
@@ -1,13 +0,0 @@
1
- import { Page } from 'playwright';
2
- import { MinimalLogger } from '../../shared/logger/logger.cjs';
3
- import { LLMClient } from '../../shared/llm/types.cjs';
4
- import 'zod';
5
-
6
- /**
7
- * Executes a vision-based recovery agent to recover from browser automation failures.
8
- * Takes a screenshot, sends it to the LLM with the instruction, and executes
9
- * the LLM's suggested browser actions.
10
- */
11
- declare function executeRecoveryAgent(page: Page, instruction: string, logger?: MinimalLogger, llmClient?: LLMClient): Promise<void>;
12
-
13
- export { executeRecoveryAgent };
@@ -1,124 +0,0 @@
1
- "use strict";
2
- var __defProp = Object.defineProperty;
3
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
- var __getOwnPropNames = Object.getOwnPropertyNames;
5
- var __hasOwnProp = Object.prototype.hasOwnProperty;
6
- var __export = (target, all) => {
7
- for (var name in all)
8
- __defProp(target, name, { get: all[name], enumerable: true });
9
- };
10
- var __copyProps = (to, from, except, desc) => {
11
- if (from && typeof from === "object" || typeof from === "function") {
12
- for (let key of __getOwnPropNames(from))
13
- if (!__hasOwnProp.call(to, key) && key !== except)
14
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
- }
16
- return to;
17
- };
18
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
- var errors_exports = {};
20
- __export(errors_exports, {
21
- detectSubmissionError: () => detectSubmissionError
22
- });
23
- module.exports = __toCommonJS(errors_exports);
24
- var import_logger = require("../../shared/logger/logger.js");
25
- var import_zod = require("zod");
26
- const detectSubmissionErrorSchema = import_zod.z.object({
27
- hasError: import_zod.z.boolean().describe("Whether an error is visible on the page"),
28
- matchedKnownErrorId: import_zod.z.string().nullable().describe("The ID of the matched known error, or null if no match"),
29
- errorMessage: import_zod.z.string().nullable().describe("The error message visible on screen, or null if no error")
30
- });
31
- async function detectSubmissionError(page, error, logContext, llmClient, knownErrors = [], logger) {
32
- const log = logger ?? import_logger.defaultLogger;
33
- let screenshot;
34
- let domSnapshot;
35
- try {
36
- const cdpClient = await page.context().newCDPSession(page);
37
- await cdpClient.send("Page.enable");
38
- const { data } = await cdpClient.send("Page.captureScreenshot", {
39
- format: "png"
40
- });
41
- screenshot = data;
42
- } catch (screenshotError) {
43
- log.warn(
44
- "Failed to take screenshot via CDP for error detection, skipping LLM analysis",
45
- { screenshotError, originalError: error }
46
- );
47
- throw error;
48
- }
49
- try {
50
- const htmlContent = await page.content();
51
- domSnapshot = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
52
- } catch (domError) {
53
- log.warn("Failed to capture DOM snapshot", {
54
- domError: domError instanceof Error ? domError.message : String(domError)
55
- });
56
- }
57
- const knownErrorsDescription = knownErrors.length > 0 ? `
58
- Known error patterns to look for:
59
- ${knownErrors.map((e, i) => `${i + 1}. ID: "${e.id}" - Patterns: ${e.errorPatterns.join(", ")}`).join("\n")}
60
- ` : "";
61
- const prompt = `You are analyzing a screenshot and DOM of a web page to detect if an error occurred during a browser automation process.
62
-
63
- Context: ${logContext}
64
-
65
- ${knownErrorsDescription}
66
-
67
- Analyze the screenshot and DOM snapshot to determine:
68
- 1. Is there any error message, warning, or indication of failure visible on the page?
69
- 2. If yes, does it match any of the known error patterns listed above?
70
- 3. What is the exact error message or description of the problem?
71
-
72
- IMPORTANT:
73
- - Look carefully for error alerts, warning banners, error modals, red text, or any indication of failure
74
- - Check the DOM snapshot for error messages that may not be visible in the screenshot
75
- - If you see a known error pattern, use its exact ID in matchedKnownErrorId
76
- - If there's an error but it doesn't match any known pattern, set matchedKnownErrorId to null
77
- - If the page looks normal with no errors, set hasError to false
78
-
79
- ${domSnapshot ? `<dom_snapshot>
80
- ${domSnapshot}
81
- </dom_snapshot>` : ""}`;
82
- const result = await llmClient.generateObjectFromMessages({
83
- schema: detectSubmissionErrorSchema,
84
- messages: [
85
- {
86
- role: "user",
87
- content: [
88
- { type: "text", text: prompt },
89
- { type: "image", image: `data:image/png;base64,${screenshot}` }
90
- ]
91
- }
92
- ],
93
- temperature: 0
94
- });
95
- if (!result.hasError) {
96
- log.info("No error detected by LLM", { result });
97
- }
98
- if (result.matchedKnownErrorId) {
99
- const knownError = knownErrors.find(
100
- (e) => e.id === result.matchedKnownErrorId
101
- );
102
- if (knownError) {
103
- log.warn(logContext, {
104
- error,
105
- browserError: result.errorMessage,
106
- knownErrorId: result.matchedKnownErrorId
107
- });
108
- return {
109
- matched: true,
110
- errorId: knownError.id,
111
- message: knownError.userMessage
112
- };
113
- }
114
- }
115
- log.warn(logContext, {
116
- error,
117
- browserError: result.errorMessage
118
- });
119
- throw error;
120
- }
121
- // Annotate the CommonJS export names for ESM import in node:
122
- 0 && (module.exports = {
123
- detectSubmissionError
124
- });
@@ -1,31 +0,0 @@
1
- import { Page } from 'playwright';
2
- import { MinimalLogger } from '../../shared/logger/logger.cjs';
3
- import { LLMClient } from '../../shared/llm/types.cjs';
4
- import 'zod';
5
-
6
- /**
7
- * Known error type for classifying submission errors.
8
- * errorPatterns are what the LLM should look for on screen.
9
- * userMessage is the friendly message returned when matched.
10
- */
11
- type KnownSubmissionError = {
12
- id: string;
13
- errorPatterns: string[];
14
- userMessage: string;
15
- };
16
- type DetectedSubmissionError = {
17
- matched: true;
18
- errorId: string;
19
- message: string;
20
- };
21
- /**
22
- * Uses screenshot + LLM vision to detect if an error occurred during a submission process.
23
- * Captures a screenshot via CDP (handles unresponsive pages), sends it to the LLM,
24
- * and checks against the provided known error patterns.
25
- *
26
- * @returns DetectedSubmissionError if a known error is matched
27
- * @throws The original error if no known error matches
28
- */
29
- declare function detectSubmissionError(page: Page, error: unknown, logContext: string, llmClient: LLMClient, knownErrors?: KnownSubmissionError[], logger?: MinimalLogger): Promise<DetectedSubmissionError>;
30
-
31
- export { type DetectedSubmissionError, type KnownSubmissionError, detectSubmissionError };
@@ -1,34 +0,0 @@
1
- "use strict";
2
- var __defProp = Object.defineProperty;
3
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
- var __getOwnPropNames = Object.getOwnPropertyNames;
5
- var __hasOwnProp = Object.prototype.hasOwnProperty;
6
- var __export = (target, all) => {
7
- for (var name in all)
8
- __defProp(target, name, { get: all[name], enumerable: true });
9
- };
10
- var __copyProps = (to, from, except, desc) => {
11
- if (from && typeof from === "object" || typeof from === "function") {
12
- for (let key of __getOwnPropNames(from))
13
- if (!__hasOwnProp.call(to, key) && key !== except)
14
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
- }
16
- return to;
17
- };
18
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
- var recovery_exports = {};
20
- __export(recovery_exports, {
21
- attemptWithRecovery: () => import_recovery.attemptWithRecovery,
22
- detectSubmissionError: () => import_errors.detectSubmissionError,
23
- executeRecoveryAgent: () => import_agent.executeRecoveryAgent
24
- });
25
- module.exports = __toCommonJS(recovery_exports);
26
- var import_agent = require("./agent.js");
27
- var import_recovery = require("./recovery.js");
28
- var import_errors = require("./errors.js");
29
- // Annotate the CommonJS export names for ESM import in node:
30
- 0 && (module.exports = {
31
- attemptWithRecovery,
32
- detectSubmissionError,
33
- executeRecoveryAgent
34
- });
@@ -1,7 +0,0 @@
1
- export { executeRecoveryAgent } from './agent.cjs';
2
- export { attemptWithRecovery } from './recovery.cjs';
3
- export { DetectedSubmissionError, KnownSubmissionError, detectSubmissionError } from './errors.cjs';
4
- import 'playwright';
5
- import '../../shared/logger/logger.cjs';
6
- import '../../shared/llm/types.cjs';
7
- import 'zod';
@@ -1,55 +0,0 @@
1
- "use strict";
2
- var __defProp = Object.defineProperty;
3
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
- var __getOwnPropNames = Object.getOwnPropertyNames;
5
- var __hasOwnProp = Object.prototype.hasOwnProperty;
6
- var __export = (target, all) => {
7
- for (var name in all)
8
- __defProp(target, name, { get: all[name], enumerable: true });
9
- };
10
- var __copyProps = (to, from, except, desc) => {
11
- if (from && typeof from === "object" || typeof from === "function") {
12
- for (let key of __getOwnPropNames(from))
13
- if (!__hasOwnProp.call(to, key) && key !== except)
14
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
- }
16
- return to;
17
- };
18
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
- var recovery_exports = {};
20
- __export(recovery_exports, {
21
- attemptWithRecovery: () => attemptWithRecovery
22
- });
23
- module.exports = __toCommonJS(recovery_exports);
24
- var import_logger = require("../../shared/logger/logger.js");
25
- var import_agent = require("./agent.js");
26
- async function attemptWithRecovery(page, fn, logger, llmClient) {
27
- const log = logger ?? import_logger.defaultLogger;
28
- try {
29
- return await fn();
30
- } catch (error) {
31
- if (error instanceof Error && (error.message.includes("Target closed") || error.message.includes("browser has been closed") || error.message.includes("context or browser has been closed"))) {
32
- log.warn("Page/browser has been closed, cannot recover", {
33
- error: error.message
34
- });
35
- throw error;
36
- }
37
- if (!llmClient) {
38
- throw error;
39
- }
40
- log.info("Action failed, attempting popup recovery", {
41
- error: error instanceof Error ? error.message : String(error)
42
- });
43
- await (0, import_agent.executeRecoveryAgent)(
44
- page,
45
- "Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
46
- log,
47
- llmClient
48
- );
49
- return await fn();
50
- }
51
- }
52
- // Annotate the CommonJS export names for ESM import in node:
53
- 0 && (module.exports = {
54
- attemptWithRecovery
55
- });
@@ -1,12 +0,0 @@
1
- import { Page } from 'playwright';
2
- import { MinimalLogger } from '../../shared/logger/logger.cjs';
3
- import { LLMClient } from '../../shared/llm/types.cjs';
4
- import 'zod';
5
-
6
- /**
7
- * Attempts to execute a function, and if it fails, runs popup recovery
8
- * (if an LLM client is provided) and retries the function once.
9
- */
10
- declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger?: MinimalLogger, llmClient?: LLMClient): Promise<T>;
11
-
12
- export { attemptWithRecovery };