npm - libretto - Versions diffs - 0.4.4 → 0.5.0 - Mend

libretto 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (152) hide show

package/dist/cli/cli.js +20 -19
package/dist/cli/commands/ai.js +1 -1
package/dist/cli/commands/browser.js +3 -3
package/dist/cli/commands/execution.js +3 -3
package/dist/cli/commands/logs.js +1 -1
package/dist/cli/core/browser.js +11 -6
package/dist/cli/core/context.js +4 -18
package/dist/cli/core/session.js +2 -2
package/dist/cli/core/snapshot-analyzer.js +2 -2
package/dist/cli/router.js +1 -1
package/dist/cli/workers/run-integration-runtime.js +2 -2
package/dist/shared/paths/paths.js +2 -1
package/dist/shared/paths/repo-root.d.ts +3 -0
package/dist/shared/paths/repo-root.js +24 -0
package/package.json +6 -7
package/scripts/postinstall.mjs +12 -3
package/skills/libretto/SKILL.md +93 -404
package/skills/libretto/references/auth-profiles.md +30 -0
package/skills/libretto/references/pages-and-page-targeting.md +29 -0
package/skills/libretto/references/reverse-engineering-network-requests.md +39 -0
package/skills/libretto/references/user-action-log.md +31 -0
package/src/cli/cli.ts +173 -0
package/src/cli/commands/ai.ts +35 -0
package/src/cli/commands/browser.ts +165 -0
package/src/cli/commands/execution.ts +691 -0
package/src/cli/commands/init.ts +327 -0
package/src/cli/commands/logs.ts +128 -0
package/src/cli/commands/shared.ts +70 -0
package/src/cli/commands/snapshot.ts +327 -0
package/src/cli/core/ai-config.ts +255 -0
package/src/cli/core/api-snapshot-analyzer.ts +97 -0
package/src/cli/core/browser.ts +839 -0
package/src/cli/core/context.ts +122 -0
package/src/cli/core/pause-signals.ts +35 -0
package/src/cli/core/session-telemetry.ts +553 -0
package/src/cli/core/session.ts +209 -0
package/src/cli/core/snapshot-analyzer.ts +875 -0
package/src/cli/core/snapshot-api-config.ts +236 -0
package/src/cli/core/telemetry.ts +446 -0
package/src/cli/framework/simple-cli.ts +1273 -0
package/src/cli/index.ts +13 -0
package/src/cli/router.ts +28 -0
package/src/cli/workers/run-integration-runtime.ts +311 -0
package/src/cli/workers/run-integration-worker-protocol.ts +14 -0
package/src/cli/workers/run-integration-worker.ts +75 -0
package/src/index.ts +120 -0
package/src/runtime/download/download.ts +100 -0
package/src/runtime/download/index.ts +7 -0
package/src/runtime/extract/extract.ts +92 -0
package/src/runtime/extract/index.ts +1 -0
package/src/runtime/network/index.ts +5 -0
package/src/runtime/network/network.ts +113 -0
package/src/runtime/recovery/agent.ts +256 -0
package/src/runtime/recovery/errors.ts +152 -0
package/src/runtime/recovery/index.ts +7 -0
package/src/runtime/recovery/recovery.ts +50 -0
package/{dist/shared/condense-dom/condense-dom.cjs → src/shared/condense-dom/condense-dom.ts} +243 -115
package/src/shared/config/config.ts +22 -0
package/src/shared/config/index.ts +5 -0
package/src/shared/debug/index.ts +1 -0
package/src/shared/debug/pause.ts +85 -0
package/src/shared/instrumentation/errors.ts +82 -0
package/src/shared/instrumentation/index.ts +9 -0
package/src/shared/instrumentation/instrument.ts +276 -0
package/src/shared/llm/ai-sdk-adapter.ts +78 -0
package/src/shared/llm/client.ts +217 -0
package/src/shared/llm/index.ts +3 -0
package/src/shared/llm/types.ts +63 -0
package/src/shared/logger/index.ts +6 -0
package/src/shared/logger/logger.ts +352 -0
package/src/shared/logger/sinks.ts +144 -0
package/src/shared/paths/paths.ts +109 -0
package/src/shared/paths/repo-root.ts +27 -0
package/src/shared/run/api.ts +2 -0
package/src/shared/run/browser.ts +98 -0
package/src/shared/state/index.ts +11 -0
package/src/shared/state/session-state.ts +74 -0
package/src/shared/visualization/ghost-cursor.ts +200 -0
package/src/shared/visualization/highlight.ts +146 -0
package/src/shared/visualization/index.ts +18 -0
package/src/shared/workflow/workflow.ts +42 -0
package/dist/index.cjs +0 -144
package/dist/index.d.cts +0 -21
package/dist/runtime/download/download.cjs +0 -70
package/dist/runtime/download/download.d.cts +0 -35
package/dist/runtime/download/index.cjs +0 -30
package/dist/runtime/download/index.d.cts +0 -3
package/dist/runtime/extract/extract.cjs +0 -88
package/dist/runtime/extract/extract.d.cts +0 -23
package/dist/runtime/extract/index.cjs +0 -28
package/dist/runtime/extract/index.d.cts +0 -5
package/dist/runtime/network/index.cjs +0 -28
package/dist/runtime/network/index.d.cts +0 -4
package/dist/runtime/network/network.cjs +0 -91
package/dist/runtime/network/network.d.cts +0 -28
package/dist/runtime/recovery/agent.cjs +0 -223
package/dist/runtime/recovery/agent.d.cts +0 -13
package/dist/runtime/recovery/errors.cjs +0 -124
package/dist/runtime/recovery/errors.d.cts +0 -31
package/dist/runtime/recovery/index.cjs +0 -34
package/dist/runtime/recovery/index.d.cts +0 -7
package/dist/runtime/recovery/recovery.cjs +0 -55
package/dist/runtime/recovery/recovery.d.cts +0 -12
package/dist/shared/condense-dom/condense-dom.d.cts +0 -34
package/dist/shared/config/config.cjs +0 -44
package/dist/shared/config/config.d.cts +0 -10
package/dist/shared/config/index.cjs +0 -32
package/dist/shared/config/index.d.cts +0 -1
package/dist/shared/debug/index.cjs +0 -28
package/dist/shared/debug/index.d.cts +0 -1
package/dist/shared/debug/pause.cjs +0 -86
package/dist/shared/debug/pause.d.cts +0 -12
package/dist/shared/instrumentation/errors.cjs +0 -81
package/dist/shared/instrumentation/errors.d.cts +0 -12
package/dist/shared/instrumentation/index.cjs +0 -35
package/dist/shared/instrumentation/index.d.cts +0 -6
package/dist/shared/instrumentation/instrument.cjs +0 -206
package/dist/shared/instrumentation/instrument.d.cts +0 -32
package/dist/shared/llm/ai-sdk-adapter.cjs +0 -71
package/dist/shared/llm/ai-sdk-adapter.d.cts +0 -22
package/dist/shared/llm/client.cjs +0 -218
package/dist/shared/llm/client.d.cts +0 -13
package/dist/shared/llm/index.cjs +0 -31
package/dist/shared/llm/index.d.cts +0 -5
package/dist/shared/llm/types.cjs +0 -16
package/dist/shared/llm/types.d.cts +0 -67
package/dist/shared/logger/index.cjs +0 -37
package/dist/shared/logger/index.d.cts +0 -2
package/dist/shared/logger/logger.cjs +0 -232
package/dist/shared/logger/logger.d.cts +0 -86
package/dist/shared/logger/sinks.cjs +0 -160
package/dist/shared/logger/sinks.d.cts +0 -9
package/dist/shared/paths/paths.cjs +0 -104
package/dist/shared/paths/paths.d.cts +0 -10
package/dist/shared/run/api.cjs +0 -28
package/dist/shared/run/api.d.cts +0 -2
package/dist/shared/run/browser.cjs +0 -98
package/dist/shared/run/browser.d.cts +0 -22
package/dist/shared/state/index.cjs +0 -38
package/dist/shared/state/index.d.cts +0 -2
package/dist/shared/state/session-state.cjs +0 -92
package/dist/shared/state/session-state.d.cts +0 -40
package/dist/shared/visualization/ghost-cursor.cjs +0 -174
package/dist/shared/visualization/ghost-cursor.d.cts +0 -37
package/dist/shared/visualization/highlight.cjs +0 -134
package/dist/shared/visualization/highlight.d.cts +0 -22
package/dist/shared/visualization/index.cjs +0 -45
package/dist/shared/visualization/index.d.cts +0 -3
package/dist/shared/workflow/workflow.cjs +0 -47
package/dist/shared/workflow/workflow.d.cts +0 -21
package/skills/libretto/code-generation-rules.md +0 -223
package/skills/libretto/integration-approach-selection.md +0 -174

package/src/runtime/extract/extract.ts ADDED Viewed

@@ -0,0 +1,92 @@
+import type { Page } from "playwright";
+import type z from "zod";
+import { type MinimalLogger, defaultLogger } from "../../shared/logger/logger.js";
+import type { LLMClient } from "../../shared/llm/types.js";
+export type ExtractOptions<T extends z.ZodType> = {
+	page: Page;
+	instruction: string;
+	schema: T;
+	llmClient: LLMClient;
+	logger?: MinimalLogger;
+	/** Optional CSS selector to scope extraction to a specific element. */
+	selector?: string;
+};
+/**
+ * Generic AI-powered data extraction from page elements.
+ * Takes a screenshot (full-page via CDP or scoped to an element),
+ * captures DOM content, and uses an LLM to extract structured data
+ * matching the provided Zod schema.
+ */
+export async function extractFromPage<T extends z.ZodType>(
+	options: ExtractOptions<T>,
+): Promise<z.infer<T>> {
+	const { page, instruction, schema, selector, logger = defaultLogger, llmClient } = options;
+	let screenshot: string;
+	let domContent: string | undefined;
+	if (selector) {
+		const element = page.locator(selector);
+		await element.waitFor({ state: "visible", timeout: 10_000 });
+		const screenshotBuffer = await element.screenshot();
+		screenshot = screenshotBuffer.toString("base64");
+		try {
+			domContent = await element.innerHTML();
+			if (domContent.length > 30000) {
+				domContent = domContent.slice(0, 30000) + "\n... [truncated]";
+			}
+		} catch {
+			domContent = undefined;
+		}
+	} else {
+		const cdpClient = await page.context().newCDPSession(page);
+		await cdpClient.send("Page.enable");
+		const { data } = await cdpClient.send("Page.captureScreenshot", {
+			format: "png",
+		});
+		screenshot = data;
+		try {
+			const htmlContent = await page.content();
+			domContent =
+				htmlContent.length > 50000
+					? htmlContent.slice(0, 50000) + "\n... [truncated]"
+					: htmlContent;
+		} catch {
+			domContent = undefined;
+		}
+	}
+	const prompt = `You are analyzing a screenshot${selector ? " of a specific element" : ""} from a web page to extract structured data.
+Instruction: ${instruction}
+${domContent ? `Here is the HTML content for additional context:\n<html>\n${domContent}\n</html>` : ""}
+Extract the requested information from the screenshot and return it in the specified format. Be precise and only extract what is visible.`;
+	const result = await llmClient.generateObjectFromMessages({
+		schema,
+		messages: [
+			{
+				role: "user",
+				content: [
+					{ type: "text", text: prompt },
+					{ type: "image", image: `data:image/png;base64,${screenshot}` },
+				],
+			},
+		],
+		temperature: 0,
+	});
+	logger.info("extractFromPage completed", {
+		selector,
+		instruction: instruction.slice(0, 100),
+	});
+	return result;
+}

package/src/runtime/extract/index.ts ADDED Viewed

	@@ -0,0 +1 @@
1	+ export { extractFromPage, type ExtractOptions } from "./extract.js";

package/src/runtime/network/index.ts ADDED Viewed

@@ -0,0 +1,5 @@
+export {
+	pageRequest,
+	type RequestConfig,
+	type PageRequestOptions,
+} from "./network.js";

package/src/runtime/network/network.ts ADDED Viewed

@@ -0,0 +1,113 @@
+import type { Page } from "playwright";
+import type z from "zod";
+import type { MinimalLogger } from "../../shared/logger/logger.js";
+export type RequestConfig = {
+	url: string;
+	method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+	headers?: Record<string, string>;
+	body?: Record<string, any> | string;
+	/** How to serialize the body. Defaults to "json". */
+	bodyType?: "json" | "form";
+	/** How to parse the response. Defaults to "json". */
+	responseType?: "json" | "text" | "xml";
+};
+export type PageRequestOptions<T extends z.ZodType | undefined = undefined> = {
+	logger?: MinimalLogger;
+	/** Optional Zod schema to validate the response body. */
+	schema?: T;
+};
+type PageRequestResult<T extends z.ZodType | undefined> = T extends z.ZodType
+	? z.infer<T>
+	: any;
+/**
+ * Executes a fetch() call inside the browser context via page.evaluate().
+ * Provides typed request config, automatic response parsing, optional Zod
+ * validation, and logging.
+ */
+export async function pageRequest<T extends z.ZodType | undefined = undefined>(
+	page: Page,
+	config: RequestConfig,
+	options?: PageRequestOptions<T>,
+): Promise<PageRequestResult<T>> {
+	const { url, method = "GET", headers = {}, body, bodyType = "json", responseType = "json" } = config;
+	const { logger, schema } = options ?? {};
+	const startTime = Date.now();
+	// Build fetch options to pass into page.evaluate
+	const fetchHeaders: Record<string, string> = { ...headers };
+	let fetchBody: string | undefined;
+	if (body !== undefined) {
+		if (bodyType === "form") {
+			fetchHeaders["Content-Type"] = "application/x-www-form-urlencoded";
+			if (typeof body === "string") {
+				fetchBody = body;
+			} else {
+				fetchBody = new URLSearchParams(
+					Object.entries(body).map(([k, v]) => [k, String(v)]),
+				).toString();
+			}
+		} else {
+			fetchHeaders["Content-Type"] = "application/json";
+			fetchBody = typeof body === "string" ? body : JSON.stringify(body);
+		}
+	}
+	const result = await page.evaluate(
+		async ({ url, method, headers, body, responseType }) => {
+			const res = await fetch(url, {
+				method,
+				headers,
+				body: body ?? undefined,
+			});
+			const status = res.status;
+			const ok = res.ok;
+			let data: any;
+			if (responseType === "json") {
+				data = await res.json();
+			} else {
+				data = await res.text();
+			}
+			return { status, ok, data };
+		},
+		{ url, method, headers: fetchHeaders, body: fetchBody, responseType },
+	);
+	const duration = Date.now() - startTime;
+	if (!result.ok) {
+		logger?.warn("network:request:error", {
+			method,
+			url,
+			status: result.status,
+			duration,
+			body: typeof result.data === "string"
+				? result.data.slice(0, 500)
+				: undefined,
+		});
+		throw new Error(
+			`pageRequest failed: ${method} ${url} returned ${result.status}`,
+		);
+	}
+	logger?.info("network:request", {
+		method,
+		url,
+		status: result.status,
+		duration,
+	});
+	if (schema) {
+		return schema.parse(result.data) as PageRequestResult<T>;
+	}
+	return result.data as PageRequestResult<T>;
+}

package/src/runtime/recovery/agent.ts ADDED Viewed

@@ -0,0 +1,256 @@
+import type { Page } from "playwright";
+import { type MinimalLogger, defaultLogger } from "../../shared/logger/logger.js";
+import type { LLMClient } from "../../shared/llm/types.js";
+type BrowserAction =
+	| { type: "click"; x: number; y: number; button?: string }
+	| { type: "double_click"; x: number; y: number }
+	| {
+			type: "scroll";
+			x: number;
+			y: number;
+			scroll_x: number;
+			scroll_y: number;
+	  }
+	| { type: "keypress"; keys: string[] }
+	| { type: "type"; text: string }
+	| { type: "wait" }
+	| { type: "screenshot" }
+	| { type: "drag"; path: { x: number; y: number }[] }
+	| { type: "move"; x: number; y: number }
+	| { type: "done" };
+function delay(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+const KEY_MAPPINGS: Record<string, string> = {
+	ENTER: "Enter",
+	RETURN: "Enter",
+	TAB: "Tab",
+	SPACE: " ",
+	BACKSPACE: "Backspace",
+	DELETE: "Delete",
+	ESCAPE: "Escape",
+	ESC: "Escape",
+	UP: "ArrowUp",
+	DOWN: "ArrowDown",
+	LEFT: "ArrowLeft",
+	RIGHT: "ArrowRight",
+	HOME: "Home",
+	END: "End",
+	PAGEUP: "PageUp",
+	PAGEDOWN: "PageDown",
+	CTRL: "Control",
+	CONTROL: "Control",
+	ALT: "Alt",
+	SHIFT: "Shift",
+	META: "Meta",
+	CMD: "Meta",
+	COMMAND: "Meta",
+};
+function mapKeyName(key: string): string {
+	return KEY_MAPPINGS[key.toUpperCase()] ?? key;
+}
+async function executeBrowserAction(
+	page: Page,
+	action: BrowserAction,
+	logger: MinimalLogger = defaultLogger,
+): Promise<void> {
+	switch (action.type) {
+		case "click": {
+			const { x, y, button = "left" } = action;
+			const playwrightButton =
+				button === "wheel" || button === "back" || button === "forward"
+					? ("left" as const)
+					: (button as "left" | "right" | "middle");
+			await page.mouse.click(x, y, { button: playwrightButton });
+			logger.info(`Clicked at (${x}, ${y}) with ${button} button`);
+			break;
+		}
+		case "double_click": {
+			const { x, y } = action;
+			await page.mouse.dblclick(x, y);
+			logger.info(`Double-clicked at (${x}, ${y})`);
+			break;
+		}
+		case "scroll": {
+			const { x, y, scroll_x, scroll_y } = action;
+			await page.mouse.move(x, y);
+			await page.evaluate(`window.scrollBy(${scroll_x}, ${scroll_y})`);
+			logger.info(`Scrolled at (${x}, ${y}) by (${scroll_x}, ${scroll_y})`);
+			break;
+		}
+		case "keypress": {
+			for (const key of action.keys) {
+				const mapped = mapKeyName(key);
+				await page.keyboard.press(mapped);
+				logger.info(`Pressed key: ${key} (mapped to ${mapped})`);
+			}
+			break;
+		}
+		case "type": {
+			await page.keyboard.type(action.text);
+			logger.info(`Typed text: ${action.text}`);
+			break;
+		}
+		case "wait": {
+			await delay(2000);
+			logger.info("Waited 2 seconds");
+			break;
+		}
+		case "screenshot": {
+			logger.info("Screenshot action (no-op, taken automatically)");
+			break;
+		}
+		case "drag": {
+			const { path } = action;
+			const start = path[0];
+			const end = path[path.length - 1];
+			if (path.length >= 2 && start && end) {
+				await page.mouse.move(start.x, start.y);
+				await page.mouse.down();
+				for (let i = 1; i < path.length; i++) {
+					const point = path[i];
+					if (point) await page.mouse.move(point.x, point.y);
+				}
+				await page.mouse.up();
+				logger.info(`Dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
+			}
+			break;
+		}
+		case "move": {
+			const { x, y } = action;
+			await page.mouse.move(x, y);
+			logger.info(`Moved mouse to (${x}, ${y})`);
+			break;
+		}
+		case "done": {
+			break;
+		}
+	}
+}
+import { z } from "zod";
+const recoveryActionSchema = z.object({
+	reasoning: z
+		.string()
+		.describe("Your reasoning about what you see and what action to take"),
+	action: z.discriminatedUnion("type", [
+		z.object({
+			type: z.literal("click"),
+			x: z.number(),
+			y: z.number(),
+		}),
+		z.object({
+			type: z.literal("type"),
+			text: z.string(),
+		}),
+		z.object({
+			type: z.literal("keypress"),
+			keys: z.array(z.string()),
+		}),
+		z.object({
+			type: z.literal("scroll"),
+			x: z.number(),
+			y: z.number(),
+			scroll_x: z.number(),
+			scroll_y: z.number(),
+		}),
+		z.object({
+			type: z.literal("wait"),
+		}),
+		z.object({
+			type: z.literal("done"),
+		}),
+	]),
+});
+/**
+ * Executes a vision-based recovery agent to recover from browser automation failures.
+ * Takes a screenshot, sends it to the LLM with the instruction, and executes
+ * the LLM's suggested browser actions.
+ */
+export async function executeRecoveryAgent(
+	page: Page,
+	instruction: string,
+	logger?: MinimalLogger,
+	llmClient?: LLMClient,
+): Promise<void> {
+	if (!llmClient) {
+		return;
+	}
+	const log = logger ?? defaultLogger;
+	log.info("Executing vision-based recovery agent", { instruction });
+	const viewport = page.viewportSize();
+	if (!viewport) {
+		throw new Error("Viewport size not found");
+	}
+	let screenshot: string;
+	try {
+		screenshot = (
+			await page.screenshot({ fullPage: false, timeout: 10000 })
+		).toString("base64");
+	} catch (screenshotError) {
+		log.warn("Failed to take screenshot for recovery agent, skipping", {
+			screenshotError:
+				screenshotError instanceof Error
+					? screenshotError.message
+					: String(screenshotError),
+		});
+		throw new Error("Failed to take screenshot for recovery agent");
+	}
+	const maxSteps = 3;
+	for (let step = 1; step <= maxSteps; step++) {
+		const result = await llmClient.generateObjectFromMessages({
+			schema: recoveryActionSchema,
+			messages: [
+				{
+					role: "user",
+					content: [
+						{
+							type: "text",
+							text: `You are an expert browser support agent. Your job is to resolve issues when browser automation encounters unexpected website behavior (e.g., popups blocking interaction).
+Your task: ${instruction}
+Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
+Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`,
+						},
+						{
+							type: "image",
+							image: `data:image/png;base64,${screenshot}`,
+						},
+					],
+				},
+			],
+			temperature: 0,
+		});
+		log.info(`Recovery step ${step}/${maxSteps}`, {
+			reasoning: result.reasoning,
+			action: result.action,
+		});
+		if (result.action.type === "done") {
+			log.info("Recovery agent completed - no more actions needed");
+			break;
+		}
+		await executeBrowserAction(page, result.action, log);
+		await delay(2000);
+		// Take new screenshot for next iteration
+		screenshot = (await page.screenshot({ fullPage: false })).toString(
+			"base64",
+		);
+	}
+	log.info("Recovery agent execution completed");
+}

package/src/runtime/recovery/errors.ts ADDED Viewed

@@ -0,0 +1,152 @@
+import type { Page } from "playwright";
+import { type MinimalLogger, defaultLogger } from "../../shared/logger/logger.js";
+import type { LLMClient } from "../../shared/llm/types.js";
+import { z } from "zod";
+/**
+ * Known error type for classifying submission errors.
+ * errorPatterns are what the LLM should look for on screen.
+ * userMessage is the friendly message returned when matched.
+ */
+export type KnownSubmissionError = {
+	id: string;
+	errorPatterns: string[];
+	userMessage: string;
+};
+export type DetectedSubmissionError = {
+	matched: true;
+	errorId: string;
+	message: string;
+};
+const detectSubmissionErrorSchema = z.object({
+	hasError: z.boolean().describe("Whether an error is visible on the page"),
+	matchedKnownErrorId: z
+		.string()
+		.nullable()
+		.describe("The ID of the matched known error, or null if no match"),
+	errorMessage: z
+		.string()
+		.nullable()
+		.describe("The error message visible on screen, or null if no error"),
+});
+/**
+ * Uses screenshot + LLM vision to detect if an error occurred during a submission process.
+ * Captures a screenshot via CDP (handles unresponsive pages), sends it to the LLM,
+ * and checks against the provided known error patterns.
+ *
+ * @returns DetectedSubmissionError if a known error is matched
+ * @throws The original error if no known error matches
+ */
+export async function detectSubmissionError(
+	page: Page,
+	error: unknown,
+	logContext: string,
+	llmClient: LLMClient,
+	knownErrors: KnownSubmissionError[] = [],
+	logger?: MinimalLogger,
+): Promise<DetectedSubmissionError> {
+	const log = logger ?? defaultLogger;
+	// Capture screenshot using CDP to handle unresponsive pages
+	let screenshot: string;
+	let domSnapshot: string | undefined;
+	try {
+		const cdpClient = await page.context().newCDPSession(page);
+		await cdpClient.send("Page.enable");
+		const { data } = await cdpClient.send("Page.captureScreenshot", {
+			format: "png",
+		});
+		screenshot = data;
+	} catch (screenshotError) {
+		log.warn(
+			"Failed to take screenshot via CDP for error detection, skipping LLM analysis",
+			{ screenshotError, originalError: error },
+		);
+		throw error;
+	}
+	// Capture DOM snapshot for additional context
+	try {
+		const htmlContent = await page.content();
+		domSnapshot =
+			htmlContent.length > 50000
+				? htmlContent.slice(0, 50000) + "\n... [truncated]"
+				: htmlContent;
+	} catch (domError) {
+		log.warn("Failed to capture DOM snapshot", {
+			domError: domError instanceof Error ? domError.message : String(domError),
+		});
+	}
+	const knownErrorsDescription =
+		knownErrors.length > 0
+			? `\nKnown error patterns to look for:\n${knownErrors.map((e, i) => `${i + 1}. ID: "${e.id}" - Patterns: ${e.errorPatterns.join(", ")}`).join("\n")}\n`
+			: "";
+	const prompt = `You are analyzing a screenshot and DOM of a web page to detect if an error occurred during a browser automation process.
+Context: ${logContext}
+${knownErrorsDescription}
+Analyze the screenshot and DOM snapshot to determine:
+1. Is there any error message, warning, or indication of failure visible on the page?
+2. If yes, does it match any of the known error patterns listed above?
+3. What is the exact error message or description of the problem?
+IMPORTANT:
+- Look carefully for error alerts, warning banners, error modals, red text, or any indication of failure
+- Check the DOM snapshot for error messages that may not be visible in the screenshot
+- If you see a known error pattern, use its exact ID in matchedKnownErrorId
+- If there's an error but it doesn't match any known pattern, set matchedKnownErrorId to null
+- If the page looks normal with no errors, set hasError to false
+${domSnapshot ? `<dom_snapshot>\n${domSnapshot}\n</dom_snapshot>` : ""}`;
+	const result = await llmClient.generateObjectFromMessages({
+		schema: detectSubmissionErrorSchema,
+		messages: [
+			{
+				role: "user",
+				content: [
+					{ type: "text", text: prompt },
+					{ type: "image", image: `data:image/png;base64,${screenshot}` },
+				],
+			},
+		],
+		temperature: 0,
+	});
+	if (!result.hasError) {
+		log.info("No error detected by LLM", { result });
+	}
+	// Check if it matches a known error
+	if (result.matchedKnownErrorId) {
+		const knownError = knownErrors.find(
+			(e) => e.id === result.matchedKnownErrorId,
+		);
+		if (knownError) {
+			log.warn(logContext, {
+				error,
+				browserError: result.errorMessage,
+				knownErrorId: result.matchedKnownErrorId,
+			});
+			return {
+				matched: true,
+				errorId: knownError.id,
+				message: knownError.userMessage,
+			};
+		}
+	}
+	// Log and re-throw for unknown errors
+	log.warn(logContext, {
+		error,
+		browserError: result.errorMessage,
+	});
+	throw error;
+}

package/src/runtime/recovery/index.ts ADDED Viewed

@@ -0,0 +1,7 @@
+export { executeRecoveryAgent } from "./agent.js";
+export { attemptWithRecovery } from "./recovery.js";
+export {
+	detectSubmissionError,
+	type KnownSubmissionError,
+	type DetectedSubmissionError,
+} from "./errors.js";

package/src/runtime/recovery/recovery.ts ADDED Viewed

@@ -0,0 +1,50 @@
+import type { Page } from "playwright";
+import { type MinimalLogger, defaultLogger } from "../../shared/logger/logger.js";
+import type { LLMClient } from "../../shared/llm/types.js";
+import { executeRecoveryAgent } from "./agent.js";
+/**
+ * Attempts to execute a function, and if it fails, runs popup recovery
+ * (if an LLM client is provided) and retries the function once.
+ */
+export async function attemptWithRecovery<T>(
+	page: Page,
+	fn: () => Promise<T>,
+	logger?: MinimalLogger,
+	llmClient?: LLMClient,
+): Promise<T> {
+	const log = logger ?? defaultLogger;
+	try {
+		return await fn();
+	} catch (error) {
+		// Don't attempt recovery if the browser/page is closed
+		if (
+			error instanceof Error &&
+			(error.message.includes("Target closed") ||
+				error.message.includes("browser has been closed") ||
+				error.message.includes("context or browser has been closed"))
+		) {
+			log.warn("Page/browser has been closed, cannot recover", {
+				error: error.message,
+			});
+			throw error;
+		}
+		if (!llmClient) {
+			throw error;
+		}
+		log.info("Action failed, attempting popup recovery", {
+			error: error instanceof Error ? error.message : String(error),
+		});
+		await executeRecoveryAgent(
+			page,
+			"Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
+			log,
+			llmClient,
+		);
+		return await fn();
+	}
+}