libretto 0.1.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/README.md +213 -17
  2. package/bin/libretto.mjs +18 -0
  3. package/dist/cli/cli.js +201 -0
  4. package/dist/cli/commands/ai.js +21 -0
  5. package/dist/cli/commands/browser.js +56 -0
  6. package/dist/cli/commands/execution.js +407 -0
  7. package/dist/cli/commands/logs.js +65 -0
  8. package/dist/cli/commands/snapshot.js +99 -0
  9. package/dist/cli/core/ai-config.js +149 -0
  10. package/dist/cli/core/browser.js +687 -0
  11. package/dist/cli/core/context.js +113 -0
  12. package/dist/cli/core/pause-signals.js +29 -0
  13. package/dist/cli/core/session.js +183 -0
  14. package/dist/cli/core/snapshot-analyzer.js +492 -0
  15. package/dist/cli/core/telemetry.js +350 -0
  16. package/dist/cli/index.js +13 -0
  17. package/dist/cli/workers/run-integration-runtime.js +204 -0
  18. package/dist/cli/workers/run-integration-worker-protocol.js +0 -0
  19. package/dist/cli/workers/run-integration-worker.js +83 -0
  20. package/dist/index.cjs +127 -0
  21. package/dist/index.d.cts +22 -0
  22. package/dist/index.d.ts +22 -0
  23. package/dist/index.js +110 -0
  24. package/dist/runtime/download/download.cjs +70 -0
  25. package/dist/runtime/download/download.d.cts +35 -0
  26. package/dist/runtime/download/download.d.ts +35 -0
  27. package/dist/runtime/download/download.js +45 -0
  28. package/dist/runtime/download/index.cjs +30 -0
  29. package/dist/runtime/download/index.d.cts +3 -0
  30. package/dist/runtime/download/index.d.ts +3 -0
  31. package/dist/runtime/download/index.js +8 -0
  32. package/dist/runtime/extract/extract.cjs +87 -0
  33. package/dist/runtime/extract/extract.d.cts +23 -0
  34. package/dist/runtime/extract/extract.d.ts +23 -0
  35. package/dist/runtime/extract/extract.js +63 -0
  36. package/dist/runtime/extract/index.cjs +28 -0
  37. package/dist/runtime/extract/index.d.cts +5 -0
  38. package/dist/runtime/extract/index.d.ts +5 -0
  39. package/dist/runtime/extract/index.js +4 -0
  40. package/dist/runtime/network/index.cjs +28 -0
  41. package/dist/runtime/network/index.d.cts +4 -0
  42. package/dist/runtime/network/index.d.ts +4 -0
  43. package/dist/runtime/network/index.js +6 -0
  44. package/dist/runtime/network/network.cjs +91 -0
  45. package/dist/runtime/network/network.d.cts +28 -0
  46. package/dist/runtime/network/network.d.ts +28 -0
  47. package/dist/runtime/network/network.js +67 -0
  48. package/dist/runtime/recovery/agent.cjs +218 -0
  49. package/dist/runtime/recovery/agent.d.cts +13 -0
  50. package/dist/runtime/recovery/agent.d.ts +13 -0
  51. package/dist/runtime/recovery/agent.js +194 -0
  52. package/dist/runtime/recovery/errors.cjs +122 -0
  53. package/dist/runtime/recovery/errors.d.cts +31 -0
  54. package/dist/runtime/recovery/errors.d.ts +31 -0
  55. package/dist/runtime/recovery/errors.js +98 -0
  56. package/dist/runtime/recovery/index.cjs +34 -0
  57. package/dist/runtime/recovery/index.d.cts +7 -0
  58. package/dist/runtime/recovery/index.d.ts +7 -0
  59. package/dist/runtime/recovery/index.js +10 -0
  60. package/dist/runtime/recovery/recovery.cjs +53 -0
  61. package/dist/runtime/recovery/recovery.d.cts +12 -0
  62. package/dist/runtime/recovery/recovery.d.ts +12 -0
  63. package/dist/runtime/recovery/recovery.js +29 -0
  64. package/dist/runtime/step/index.cjs +31 -0
  65. package/dist/runtime/step/index.d.cts +7 -0
  66. package/dist/runtime/step/index.d.ts +7 -0
  67. package/dist/runtime/step/index.js +6 -0
  68. package/dist/runtime/step/runner.cjs +208 -0
  69. package/dist/runtime/step/runner.d.cts +16 -0
  70. package/dist/runtime/step/runner.d.ts +16 -0
  71. package/dist/runtime/step/runner.js +187 -0
  72. package/dist/runtime/step/step.cjs +67 -0
  73. package/dist/runtime/step/step.d.cts +23 -0
  74. package/dist/runtime/step/step.d.ts +23 -0
  75. package/dist/runtime/step/step.js +43 -0
  76. package/dist/runtime/step/types.cjs +16 -0
  77. package/dist/runtime/step/types.d.cts +72 -0
  78. package/dist/runtime/step/types.d.ts +72 -0
  79. package/dist/runtime/step/types.js +0 -0
  80. package/dist/shared/config/config.cjs +44 -0
  81. package/dist/shared/config/config.d.cts +10 -0
  82. package/dist/shared/config/config.d.ts +10 -0
  83. package/dist/shared/config/config.js +18 -0
  84. package/dist/shared/config/index.cjs +32 -0
  85. package/dist/shared/config/index.d.cts +1 -0
  86. package/dist/shared/config/index.d.ts +1 -0
  87. package/dist/shared/config/index.js +10 -0
  88. package/dist/shared/debug/index.cjs +32 -0
  89. package/dist/shared/debug/index.d.cts +2 -0
  90. package/dist/shared/debug/index.d.ts +2 -0
  91. package/dist/shared/debug/index.js +10 -0
  92. package/dist/shared/debug/pause.cjs +56 -0
  93. package/dist/shared/debug/pause.d.cts +23 -0
  94. package/dist/shared/debug/pause.d.ts +23 -0
  95. package/dist/shared/debug/pause.js +30 -0
  96. package/dist/shared/instrumentation/errors.cjs +81 -0
  97. package/dist/shared/instrumentation/errors.d.cts +12 -0
  98. package/dist/shared/instrumentation/errors.d.ts +12 -0
  99. package/dist/shared/instrumentation/errors.js +57 -0
  100. package/dist/shared/instrumentation/index.cjs +35 -0
  101. package/dist/shared/instrumentation/index.d.cts +6 -0
  102. package/dist/shared/instrumentation/index.d.ts +6 -0
  103. package/dist/shared/instrumentation/index.js +12 -0
  104. package/dist/shared/instrumentation/instrument.cjs +206 -0
  105. package/dist/shared/instrumentation/instrument.d.cts +32 -0
  106. package/dist/shared/instrumentation/instrument.d.ts +32 -0
  107. package/dist/shared/instrumentation/instrument.js +190 -0
  108. package/dist/shared/llm/client.cjs +139 -0
  109. package/dist/shared/llm/client.d.cts +6 -0
  110. package/dist/shared/llm/client.d.ts +6 -0
  111. package/dist/shared/llm/client.js +115 -0
  112. package/dist/shared/llm/index.cjs +28 -0
  113. package/dist/shared/llm/index.d.cts +3 -0
  114. package/dist/shared/llm/index.d.ts +3 -0
  115. package/dist/shared/llm/index.js +4 -0
  116. package/dist/shared/llm/types.cjs +16 -0
  117. package/dist/shared/llm/types.d.cts +34 -0
  118. package/dist/shared/llm/types.d.ts +34 -0
  119. package/dist/shared/llm/types.js +0 -0
  120. package/dist/shared/logger/index.cjs +35 -0
  121. package/dist/shared/logger/index.d.cts +2 -0
  122. package/dist/shared/logger/index.d.ts +2 -0
  123. package/dist/shared/logger/index.js +12 -0
  124. package/dist/shared/logger/logger.cjs +200 -0
  125. package/dist/shared/logger/logger.d.cts +70 -0
  126. package/dist/shared/logger/logger.d.ts +70 -0
  127. package/dist/shared/logger/logger.js +176 -0
  128. package/dist/shared/logger/sinks.cjs +160 -0
  129. package/dist/shared/logger/sinks.d.cts +9 -0
  130. package/dist/shared/logger/sinks.d.ts +9 -0
  131. package/dist/shared/logger/sinks.js +124 -0
  132. package/dist/shared/paths/paths.cjs +104 -0
  133. package/dist/shared/paths/paths.d.cts +10 -0
  134. package/dist/shared/paths/paths.d.ts +10 -0
  135. package/dist/shared/paths/paths.js +73 -0
  136. package/dist/shared/run/api.cjs +35 -0
  137. package/dist/shared/run/api.d.cts +3 -0
  138. package/dist/shared/run/api.d.ts +3 -0
  139. package/dist/shared/run/api.js +12 -0
  140. package/dist/shared/run/browser.cjs +98 -0
  141. package/dist/shared/run/browser.d.cts +22 -0
  142. package/dist/shared/run/browser.d.ts +22 -0
  143. package/dist/shared/run/browser.js +74 -0
  144. package/dist/shared/state/index.cjs +38 -0
  145. package/dist/shared/state/index.d.cts +2 -0
  146. package/dist/shared/state/index.d.ts +2 -0
  147. package/dist/shared/state/index.js +16 -0
  148. package/dist/shared/state/session-state.cjs +85 -0
  149. package/dist/shared/state/session-state.d.cts +34 -0
  150. package/dist/shared/state/session-state.d.ts +34 -0
  151. package/dist/shared/state/session-state.js +56 -0
  152. package/dist/shared/visualization/ghost-cursor.cjs +174 -0
  153. package/dist/shared/visualization/ghost-cursor.d.cts +37 -0
  154. package/dist/shared/visualization/ghost-cursor.d.ts +37 -0
  155. package/dist/shared/visualization/ghost-cursor.js +145 -0
  156. package/dist/shared/visualization/highlight.cjs +134 -0
  157. package/dist/shared/visualization/highlight.d.cts +22 -0
  158. package/dist/shared/visualization/highlight.d.ts +22 -0
  159. package/dist/shared/visualization/highlight.js +108 -0
  160. package/dist/shared/visualization/index.cjs +45 -0
  161. package/dist/shared/visualization/index.d.cts +3 -0
  162. package/dist/shared/visualization/index.d.ts +3 -0
  163. package/dist/shared/visualization/index.js +24 -0
  164. package/dist/shared/workflow/workflow.cjs +47 -0
  165. package/dist/shared/workflow/workflow.d.cts +33 -0
  166. package/dist/shared/workflow/workflow.d.ts +33 -0
  167. package/dist/shared/workflow/workflow.js +21 -0
  168. package/package.json +123 -26
  169. package/.npmignore +0 -2
  170. package/bin/libretto +0 -31
  171. package/lib/connect.js +0 -34
  172. package/lib/export.js +0 -224
  173. package/lib/import.js +0 -168
  174. package/lib/index.js +0 -8
  175. package/lib/log.js +0 -9
  176. package/lib/validate.js +0 -20
  177. package/makefile +0 -8
  178. package/src/connect.coffee +0 -25
  179. package/src/export.coffee +0 -222
  180. package/src/import.coffee +0 -171
  181. package/src/index.coffee +0 -3
  182. package/src/log.coffee +0 -3
  183. package/src/validate.coffee +0 -10
@@ -0,0 +1,194 @@
1
+ function delay(ms) {
2
+ return new Promise((resolve) => setTimeout(resolve, ms));
3
+ }
4
+ const KEY_MAPPINGS = {
5
+ ENTER: "Enter",
6
+ RETURN: "Enter",
7
+ TAB: "Tab",
8
+ SPACE: " ",
9
+ BACKSPACE: "Backspace",
10
+ DELETE: "Delete",
11
+ ESCAPE: "Escape",
12
+ ESC: "Escape",
13
+ UP: "ArrowUp",
14
+ DOWN: "ArrowDown",
15
+ LEFT: "ArrowLeft",
16
+ RIGHT: "ArrowRight",
17
+ HOME: "Home",
18
+ END: "End",
19
+ PAGEUP: "PageUp",
20
+ PAGEDOWN: "PageDown",
21
+ CTRL: "Control",
22
+ CONTROL: "Control",
23
+ ALT: "Alt",
24
+ SHIFT: "Shift",
25
+ META: "Meta",
26
+ CMD: "Meta",
27
+ COMMAND: "Meta"
28
+ };
29
+ function mapKeyName(key) {
30
+ return KEY_MAPPINGS[key.toUpperCase()] ?? key;
31
+ }
32
+ async function executeBrowserAction(page, action, logger) {
33
+ switch (action.type) {
34
+ case "click": {
35
+ const { x, y, button = "left" } = action;
36
+ const playwrightButton = button === "wheel" || button === "back" || button === "forward" ? "left" : button;
37
+ await page.mouse.click(x, y, { button: playwrightButton });
38
+ logger.info(`Clicked at (${x}, ${y}) with ${button} button`);
39
+ break;
40
+ }
41
+ case "double_click": {
42
+ const { x, y } = action;
43
+ await page.mouse.dblclick(x, y);
44
+ logger.info(`Double-clicked at (${x}, ${y})`);
45
+ break;
46
+ }
47
+ case "scroll": {
48
+ const { x, y, scroll_x, scroll_y } = action;
49
+ await page.mouse.move(x, y);
50
+ await page.evaluate(`window.scrollBy(${scroll_x}, ${scroll_y})`);
51
+ logger.info(`Scrolled at (${x}, ${y}) by (${scroll_x}, ${scroll_y})`);
52
+ break;
53
+ }
54
+ case "keypress": {
55
+ for (const key of action.keys) {
56
+ const mapped = mapKeyName(key);
57
+ await page.keyboard.press(mapped);
58
+ logger.info(`Pressed key: ${key} (mapped to ${mapped})`);
59
+ }
60
+ break;
61
+ }
62
+ case "type": {
63
+ await page.keyboard.type(action.text);
64
+ logger.info(`Typed text: ${action.text}`);
65
+ break;
66
+ }
67
+ case "wait": {
68
+ await delay(2e3);
69
+ logger.info("Waited 2 seconds");
70
+ break;
71
+ }
72
+ case "screenshot": {
73
+ logger.info("Screenshot action (no-op, taken automatically)");
74
+ break;
75
+ }
76
+ case "drag": {
77
+ const { path } = action;
78
+ const start = path[0];
79
+ const end = path[path.length - 1];
80
+ if (path.length >= 2 && start && end) {
81
+ await page.mouse.move(start.x, start.y);
82
+ await page.mouse.down();
83
+ for (let i = 1; i < path.length; i++) {
84
+ const point = path[i];
85
+ if (point) await page.mouse.move(point.x, point.y);
86
+ }
87
+ await page.mouse.up();
88
+ logger.info(`Dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
89
+ }
90
+ break;
91
+ }
92
+ case "move": {
93
+ const { x, y } = action;
94
+ await page.mouse.move(x, y);
95
+ logger.info(`Moved mouse to (${x}, ${y})`);
96
+ break;
97
+ }
98
+ case "done": {
99
+ break;
100
+ }
101
+ }
102
+ }
103
+ import { z } from "zod";
104
+ const recoveryActionSchema = z.object({
105
+ reasoning: z.string().describe("Your reasoning about what you see and what action to take"),
106
+ action: z.discriminatedUnion("type", [
107
+ z.object({
108
+ type: z.literal("click"),
109
+ x: z.number(),
110
+ y: z.number()
111
+ }),
112
+ z.object({
113
+ type: z.literal("type"),
114
+ text: z.string()
115
+ }),
116
+ z.object({
117
+ type: z.literal("keypress"),
118
+ keys: z.array(z.string())
119
+ }),
120
+ z.object({
121
+ type: z.literal("scroll"),
122
+ x: z.number(),
123
+ y: z.number(),
124
+ scroll_x: z.number(),
125
+ scroll_y: z.number()
126
+ }),
127
+ z.object({
128
+ type: z.literal("wait")
129
+ }),
130
+ z.object({
131
+ type: z.literal("done")
132
+ })
133
+ ])
134
+ });
135
+ async function executeRecoveryAgent(page, instruction, logger, llmClient) {
136
+ logger.info("Executing vision-based recovery agent", { instruction });
137
+ const viewport = page.viewportSize();
138
+ if (!viewport) {
139
+ throw new Error("Viewport size not found");
140
+ }
141
+ let screenshot;
142
+ try {
143
+ screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
144
+ } catch (screenshotError) {
145
+ logger.warn("Failed to take screenshot for recovery agent, skipping", {
146
+ screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
147
+ });
148
+ throw new Error("Failed to take screenshot for recovery agent");
149
+ }
150
+ const maxSteps = 3;
151
+ for (let step = 1; step <= maxSteps; step++) {
152
+ const result = await llmClient.generateObjectFromMessages({
153
+ schema: recoveryActionSchema,
154
+ messages: [
155
+ {
156
+ role: "user",
157
+ content: [
158
+ {
159
+ type: "text",
160
+ text: `You are an expert browser support agent. Your job is to resolve issues when browser automation encounters unexpected website behavior (e.g., popups blocking interaction).
161
+
162
+ Your task: ${instruction}
163
+
164
+ Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
165
+ Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`
166
+ },
167
+ {
168
+ type: "image",
169
+ image: `data:image/png;base64,${screenshot}`
170
+ }
171
+ ]
172
+ }
173
+ ],
174
+ temperature: 0
175
+ });
176
+ logger.info(`Recovery step ${step}/${maxSteps}`, {
177
+ reasoning: result.reasoning,
178
+ action: result.action
179
+ });
180
+ if (result.action.type === "done") {
181
+ logger.info("Recovery agent completed - no more actions needed");
182
+ break;
183
+ }
184
+ await executeBrowserAction(page, result.action, logger);
185
+ await delay(2e3);
186
+ screenshot = (await page.screenshot({ fullPage: false })).toString(
187
+ "base64"
188
+ );
189
+ }
190
+ logger.info("Recovery agent execution completed");
191
+ }
192
+ export {
193
+ executeRecoveryAgent
194
+ };
@@ -0,0 +1,122 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var errors_exports = {};
20
+ __export(errors_exports, {
21
+ detectSubmissionError: () => detectSubmissionError
22
+ });
23
+ module.exports = __toCommonJS(errors_exports);
24
+ var import_zod = require("zod");
25
+ const detectSubmissionErrorSchema = import_zod.z.object({
26
+ hasError: import_zod.z.boolean().describe("Whether an error is visible on the page"),
27
+ matchedKnownErrorId: import_zod.z.string().nullable().describe("The ID of the matched known error, or null if no match"),
28
+ errorMessage: import_zod.z.string().nullable().describe("The error message visible on screen, or null if no error")
29
+ });
30
+ async function detectSubmissionError(page, logger, error, logContext, llmClient, knownErrors = []) {
31
+ let screenshot;
32
+ let domSnapshot;
33
+ try {
34
+ const cdpClient = await page.context().newCDPSession(page);
35
+ await cdpClient.send("Page.enable");
36
+ const { data } = await cdpClient.send("Page.captureScreenshot", {
37
+ format: "png"
38
+ });
39
+ screenshot = data;
40
+ } catch (screenshotError) {
41
+ logger.warn(
42
+ "Failed to take screenshot via CDP for error detection, skipping LLM analysis",
43
+ { screenshotError, originalError: error }
44
+ );
45
+ throw error;
46
+ }
47
+ try {
48
+ const htmlContent = await page.content();
49
+ domSnapshot = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
50
+ } catch (domError) {
51
+ logger.warn("Failed to capture DOM snapshot", {
52
+ domError: domError instanceof Error ? domError.message : String(domError)
53
+ });
54
+ }
55
+ const knownErrorsDescription = knownErrors.length > 0 ? `
56
+ Known error patterns to look for:
57
+ ${knownErrors.map((e, i) => `${i + 1}. ID: "${e.id}" - Patterns: ${e.errorPatterns.join(", ")}`).join("\n")}
58
+ ` : "";
59
+ const prompt = `You are analyzing a screenshot and DOM of a web page to detect if an error occurred during a browser automation process.
60
+
61
+ Context: ${logContext}
62
+
63
+ ${knownErrorsDescription}
64
+
65
+ Analyze the screenshot and DOM snapshot to determine:
66
+ 1. Is there any error message, warning, or indication of failure visible on the page?
67
+ 2. If yes, does it match any of the known error patterns listed above?
68
+ 3. What is the exact error message or description of the problem?
69
+
70
+ IMPORTANT:
71
+ - Look carefully for error alerts, warning banners, error modals, red text, or any indication of failure
72
+ - Check the DOM snapshot for error messages that may not be visible in the screenshot
73
+ - If you see a known error pattern, use its exact ID in matchedKnownErrorId
74
+ - If there's an error but it doesn't match any known pattern, set matchedKnownErrorId to null
75
+ - If the page looks normal with no errors, set hasError to false
76
+
77
+ ${domSnapshot ? `<dom_snapshot>
78
+ ${domSnapshot}
79
+ </dom_snapshot>` : ""}`;
80
+ const result = await llmClient.generateObjectFromMessages({
81
+ schema: detectSubmissionErrorSchema,
82
+ messages: [
83
+ {
84
+ role: "user",
85
+ content: [
86
+ { type: "text", text: prompt },
87
+ { type: "image", image: `data:image/png;base64,${screenshot}` }
88
+ ]
89
+ }
90
+ ],
91
+ temperature: 0
92
+ });
93
+ if (!result.hasError) {
94
+ logger.info("No error detected by LLM", { result });
95
+ }
96
+ if (result.matchedKnownErrorId) {
97
+ const knownError = knownErrors.find(
98
+ (e) => e.id === result.matchedKnownErrorId
99
+ );
100
+ if (knownError) {
101
+ logger.warn(logContext, {
102
+ error,
103
+ browserError: result.errorMessage,
104
+ knownErrorId: result.matchedKnownErrorId
105
+ });
106
+ return {
107
+ matched: true,
108
+ errorId: knownError.id,
109
+ message: knownError.userMessage
110
+ };
111
+ }
112
+ }
113
+ logger.warn(logContext, {
114
+ error,
115
+ browserError: result.errorMessage
116
+ });
117
+ throw error;
118
+ }
119
+ // Annotate the CommonJS export names for ESM import in node:
120
+ 0 && (module.exports = {
121
+ detectSubmissionError
122
+ });
@@ -0,0 +1,31 @@
1
+ import { Page } from 'playwright';
2
+ import { LoggerApi } from '../../shared/logger/logger.cjs';
3
+ import { LLMClient } from '../../shared/llm/types.cjs';
4
+ import 'zod';
5
+
6
+ /**
7
+ * Known error type for classifying submission errors.
8
+ * errorPatterns are what the LLM should look for on screen.
9
+ * userMessage is the friendly message returned when matched.
10
+ */
11
+ type KnownSubmissionError = {
12
+ id: string;
13
+ errorPatterns: string[];
14
+ userMessage: string;
15
+ };
16
+ type DetectedSubmissionError = {
17
+ matched: true;
18
+ errorId: string;
19
+ message: string;
20
+ };
21
+ /**
22
+ * Uses screenshot + LLM vision to detect if an error occurred during a submission process.
23
+ * Captures a screenshot via CDP (handles unresponsive pages), sends it to the LLM,
24
+ * and checks against the provided known error patterns.
25
+ *
26
+ * @returns DetectedSubmissionError if a known error is matched
27
+ * @throws The original error if no known error matches
28
+ */
29
+ declare function detectSubmissionError(page: Page, logger: LoggerApi, error: unknown, logContext: string, llmClient: LLMClient, knownErrors?: KnownSubmissionError[]): Promise<DetectedSubmissionError>;
30
+
31
+ export { type DetectedSubmissionError, type KnownSubmissionError, detectSubmissionError };
@@ -0,0 +1,31 @@
1
+ import { Page } from 'playwright';
2
+ import { LoggerApi } from '../../shared/logger/logger.js';
3
+ import { LLMClient } from '../../shared/llm/types.js';
4
+ import 'zod';
5
+
6
+ /**
7
+ * Known error type for classifying submission errors.
8
+ * errorPatterns are what the LLM should look for on screen.
9
+ * userMessage is the friendly message returned when matched.
10
+ */
11
+ type KnownSubmissionError = {
12
+ id: string;
13
+ errorPatterns: string[];
14
+ userMessage: string;
15
+ };
16
+ type DetectedSubmissionError = {
17
+ matched: true;
18
+ errorId: string;
19
+ message: string;
20
+ };
21
+ /**
22
+ * Uses screenshot + LLM vision to detect if an error occurred during a submission process.
23
+ * Captures a screenshot via CDP (handles unresponsive pages), sends it to the LLM,
24
+ * and checks against the provided known error patterns.
25
+ *
26
+ * @returns DetectedSubmissionError if a known error is matched
27
+ * @throws The original error if no known error matches
28
+ */
29
+ declare function detectSubmissionError(page: Page, logger: LoggerApi, error: unknown, logContext: string, llmClient: LLMClient, knownErrors?: KnownSubmissionError[]): Promise<DetectedSubmissionError>;
30
+
31
+ export { type DetectedSubmissionError, type KnownSubmissionError, detectSubmissionError };
@@ -0,0 +1,98 @@
1
+ import { z } from "zod";
2
+ const detectSubmissionErrorSchema = z.object({
3
+ hasError: z.boolean().describe("Whether an error is visible on the page"),
4
+ matchedKnownErrorId: z.string().nullable().describe("The ID of the matched known error, or null if no match"),
5
+ errorMessage: z.string().nullable().describe("The error message visible on screen, or null if no error")
6
+ });
7
+ async function detectSubmissionError(page, logger, error, logContext, llmClient, knownErrors = []) {
8
+ let screenshot;
9
+ let domSnapshot;
10
+ try {
11
+ const cdpClient = await page.context().newCDPSession(page);
12
+ await cdpClient.send("Page.enable");
13
+ const { data } = await cdpClient.send("Page.captureScreenshot", {
14
+ format: "png"
15
+ });
16
+ screenshot = data;
17
+ } catch (screenshotError) {
18
+ logger.warn(
19
+ "Failed to take screenshot via CDP for error detection, skipping LLM analysis",
20
+ { screenshotError, originalError: error }
21
+ );
22
+ throw error;
23
+ }
24
+ try {
25
+ const htmlContent = await page.content();
26
+ domSnapshot = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
27
+ } catch (domError) {
28
+ logger.warn("Failed to capture DOM snapshot", {
29
+ domError: domError instanceof Error ? domError.message : String(domError)
30
+ });
31
+ }
32
+ const knownErrorsDescription = knownErrors.length > 0 ? `
33
+ Known error patterns to look for:
34
+ ${knownErrors.map((e, i) => `${i + 1}. ID: "${e.id}" - Patterns: ${e.errorPatterns.join(", ")}`).join("\n")}
35
+ ` : "";
36
+ const prompt = `You are analyzing a screenshot and DOM of a web page to detect if an error occurred during a browser automation process.
37
+
38
+ Context: ${logContext}
39
+
40
+ ${knownErrorsDescription}
41
+
42
+ Analyze the screenshot and DOM snapshot to determine:
43
+ 1. Is there any error message, warning, or indication of failure visible on the page?
44
+ 2. If yes, does it match any of the known error patterns listed above?
45
+ 3. What is the exact error message or description of the problem?
46
+
47
+ IMPORTANT:
48
+ - Look carefully for error alerts, warning banners, error modals, red text, or any indication of failure
49
+ - Check the DOM snapshot for error messages that may not be visible in the screenshot
50
+ - If you see a known error pattern, use its exact ID in matchedKnownErrorId
51
+ - If there's an error but it doesn't match any known pattern, set matchedKnownErrorId to null
52
+ - If the page looks normal with no errors, set hasError to false
53
+
54
+ ${domSnapshot ? `<dom_snapshot>
55
+ ${domSnapshot}
56
+ </dom_snapshot>` : ""}`;
57
+ const result = await llmClient.generateObjectFromMessages({
58
+ schema: detectSubmissionErrorSchema,
59
+ messages: [
60
+ {
61
+ role: "user",
62
+ content: [
63
+ { type: "text", text: prompt },
64
+ { type: "image", image: `data:image/png;base64,${screenshot}` }
65
+ ]
66
+ }
67
+ ],
68
+ temperature: 0
69
+ });
70
+ if (!result.hasError) {
71
+ logger.info("No error detected by LLM", { result });
72
+ }
73
+ if (result.matchedKnownErrorId) {
74
+ const knownError = knownErrors.find(
75
+ (e) => e.id === result.matchedKnownErrorId
76
+ );
77
+ if (knownError) {
78
+ logger.warn(logContext, {
79
+ error,
80
+ browserError: result.errorMessage,
81
+ knownErrorId: result.matchedKnownErrorId
82
+ });
83
+ return {
84
+ matched: true,
85
+ errorId: knownError.id,
86
+ message: knownError.userMessage
87
+ };
88
+ }
89
+ }
90
+ logger.warn(logContext, {
91
+ error,
92
+ browserError: result.errorMessage
93
+ });
94
+ throw error;
95
+ }
96
+ export {
97
+ detectSubmissionError
98
+ };
@@ -0,0 +1,34 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var recovery_exports = {};
20
+ __export(recovery_exports, {
21
+ attemptWithRecovery: () => import_recovery.attemptWithRecovery,
22
+ detectSubmissionError: () => import_errors.detectSubmissionError,
23
+ executeRecoveryAgent: () => import_agent.executeRecoveryAgent
24
+ });
25
+ module.exports = __toCommonJS(recovery_exports);
26
+ var import_agent = require("./agent.js");
27
+ var import_recovery = require("./recovery.js");
28
+ var import_errors = require("./errors.js");
29
+ // Annotate the CommonJS export names for ESM import in node:
30
+ 0 && (module.exports = {
31
+ attemptWithRecovery,
32
+ detectSubmissionError,
33
+ executeRecoveryAgent
34
+ });
@@ -0,0 +1,7 @@
1
+ export { executeRecoveryAgent } from './agent.cjs';
2
+ export { attemptWithRecovery } from './recovery.cjs';
3
+ export { DetectedSubmissionError, KnownSubmissionError, detectSubmissionError } from './errors.cjs';
4
+ import 'playwright';
5
+ import '../../shared/logger/logger.cjs';
6
+ import '../../shared/llm/types.cjs';
7
+ import 'zod';
@@ -0,0 +1,7 @@
1
+ export { executeRecoveryAgent } from './agent.js';
2
+ export { attemptWithRecovery } from './recovery.js';
3
+ export { DetectedSubmissionError, KnownSubmissionError, detectSubmissionError } from './errors.js';
4
+ import 'playwright';
5
+ import '../../shared/logger/logger.js';
6
+ import '../../shared/llm/types.js';
7
+ import 'zod';
@@ -0,0 +1,10 @@
1
+ import { executeRecoveryAgent } from "./agent.js";
2
+ import { attemptWithRecovery } from "./recovery.js";
3
+ import {
4
+ detectSubmissionError
5
+ } from "./errors.js";
6
+ export {
7
+ attemptWithRecovery,
8
+ detectSubmissionError,
9
+ executeRecoveryAgent
10
+ };
@@ -0,0 +1,53 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var recovery_exports = {};
20
+ __export(recovery_exports, {
21
+ attemptWithRecovery: () => attemptWithRecovery
22
+ });
23
+ module.exports = __toCommonJS(recovery_exports);
24
+ var import_agent = require("./agent.js");
25
+ async function attemptWithRecovery(page, fn, logger, llmClient) {
26
+ try {
27
+ return await fn();
28
+ } catch (error) {
29
+ if (error instanceof Error && (error.message.includes("Target closed") || error.message.includes("browser has been closed") || error.message.includes("context or browser has been closed"))) {
30
+ logger.warn("Page/browser has been closed, cannot recover", {
31
+ error: error.message
32
+ });
33
+ throw error;
34
+ }
35
+ if (!llmClient) {
36
+ throw error;
37
+ }
38
+ logger.info("Action failed, attempting popup recovery", {
39
+ error: error instanceof Error ? error.message : String(error)
40
+ });
41
+ await (0, import_agent.executeRecoveryAgent)(
42
+ page,
43
+ "Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
44
+ logger,
45
+ llmClient
46
+ );
47
+ return await fn();
48
+ }
49
+ }
50
+ // Annotate the CommonJS export names for ESM import in node:
51
+ 0 && (module.exports = {
52
+ attemptWithRecovery
53
+ });
@@ -0,0 +1,12 @@
1
+ import { Page } from 'playwright';
2
+ import { LoggerApi } from '../../shared/logger/logger.cjs';
3
+ import { LLMClient } from '../../shared/llm/types.cjs';
4
+ import 'zod';
5
+
6
+ /**
7
+ * Attempts to execute a function, and if it fails, runs popup recovery
8
+ * (if an LLM client is provided) and retries the function once.
9
+ */
10
+ declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger: LoggerApi, llmClient?: LLMClient): Promise<T>;
11
+
12
+ export { attemptWithRecovery };
@@ -0,0 +1,12 @@
1
+ import { Page } from 'playwright';
2
+ import { LoggerApi } from '../../shared/logger/logger.js';
3
+ import { LLMClient } from '../../shared/llm/types.js';
4
+ import 'zod';
5
+
6
+ /**
7
+ * Attempts to execute a function, and if it fails, runs popup recovery
8
+ * (if an LLM client is provided) and retries the function once.
9
+ */
10
+ declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger: LoggerApi, llmClient?: LLMClient): Promise<T>;
11
+
12
+ export { attemptWithRecovery };
@@ -0,0 +1,29 @@
1
+ import { executeRecoveryAgent } from "./agent.js";
2
+ async function attemptWithRecovery(page, fn, logger, llmClient) {
3
+ try {
4
+ return await fn();
5
+ } catch (error) {
6
+ if (error instanceof Error && (error.message.includes("Target closed") || error.message.includes("browser has been closed") || error.message.includes("context or browser has been closed"))) {
7
+ logger.warn("Page/browser has been closed, cannot recover", {
8
+ error: error.message
9
+ });
10
+ throw error;
11
+ }
12
+ if (!llmClient) {
13
+ throw error;
14
+ }
15
+ logger.info("Action failed, attempting popup recovery", {
16
+ error: error instanceof Error ? error.message : String(error)
17
+ });
18
+ await executeRecoveryAgent(
19
+ page,
20
+ "Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
21
+ logger,
22
+ llmClient
23
+ );
24
+ return await fn();
25
+ }
26
+ }
27
+ export {
28
+ attemptWithRecovery
29
+ };
@@ -0,0 +1,31 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var step_exports = {};
20
+ __export(step_exports, {
21
+ createRunner: () => import_runner.createRunner,
22
+ step: () => import_step.step
23
+ });
24
+ module.exports = __toCommonJS(step_exports);
25
+ var import_step = require("./step.js");
26
+ var import_runner = require("./runner.js");
27
+ // Annotate the CommonJS export names for ESM import in node:
28
+ 0 && (module.exports = {
29
+ createRunner,
30
+ step
31
+ });