npm - libretto - Versions diffs - 0.4.4 → 0.5.1 - Mend

libretto 0.4.4 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (194) hide show

package/README.md +106 -36
package/dist/cli/cli.js +39 -113
package/dist/cli/commands/ai.js +1 -1
package/dist/cli/commands/browser.js +87 -60
package/dist/cli/commands/execution.js +201 -88
package/dist/cli/commands/init.js +30 -8
package/dist/cli/commands/logs.js +5 -6
package/dist/cli/commands/shared.js +30 -29
package/dist/cli/commands/snapshot.js +26 -39
package/dist/cli/core/ai-config.js +9 -2
package/dist/cli/core/api-snapshot-analyzer.js +15 -5
package/dist/cli/core/browser.js +141 -33
package/dist/cli/core/context.js +7 -18
package/dist/cli/core/session-telemetry.js +5 -2
package/dist/cli/core/session.js +23 -10
package/dist/cli/core/snapshot-analyzer.js +16 -33
package/dist/cli/core/snapshot-api-config.js +2 -6
package/dist/cli/core/telemetry.js +10 -2
package/dist/cli/framework/simple-cli.js +45 -25
package/dist/cli/router.js +14 -21
package/dist/cli/workers/run-integration-runtime.js +26 -7
package/dist/cli/workers/run-integration-worker-protocol.js +3 -1
package/dist/cli/workers/run-integration-worker.js +1 -4
package/dist/index.d.ts +1 -2
package/dist/index.js +7 -10
package/dist/runtime/download/download.js +5 -1
package/dist/runtime/extract/extract.js +11 -2
package/dist/runtime/network/network.js +8 -1
package/dist/runtime/recovery/agent.js +6 -2
package/dist/runtime/recovery/errors.js +3 -1
package/dist/runtime/recovery/recovery.js +3 -1
package/dist/shared/condense-dom/condense-dom.js +6 -13
package/dist/shared/config/config.d.ts +1 -9
package/dist/shared/config/config.js +0 -18
package/dist/shared/config/index.d.ts +2 -1
package/dist/shared/config/index.js +0 -10
package/dist/shared/debug/pause.js +9 -3
package/dist/shared/instrumentation/instrument.js +101 -5
package/dist/shared/llm/ai-sdk-adapter.js +3 -1
package/dist/shared/llm/client.js +3 -1
package/dist/shared/logger/index.js +4 -1
package/dist/shared/paths/paths.js +2 -1
package/dist/shared/paths/repo-root.d.ts +3 -0
package/dist/shared/paths/repo-root.js +24 -0
package/dist/shared/run/api.js +3 -1
package/dist/shared/run/browser.js +7 -2
package/dist/shared/state/session-state.d.ts +2 -1
package/dist/shared/state/session-state.js +5 -2
package/dist/shared/visualization/ghost-cursor.js +19 -10
package/dist/shared/visualization/highlight.js +9 -6
package/dist/shared/workflow/workflow.d.ts +4 -5
package/dist/shared/workflow/workflow.js +3 -5
package/package.json +11 -8
package/scripts/check-skills-sync.mjs +25 -0
package/scripts/compare-eval-summary.mjs +47 -0
package/scripts/postinstall.mjs +26 -17
package/scripts/prepare-release.sh +97 -0
package/scripts/skills-libretto.mjs +103 -0
package/scripts/summarize-evals.mjs +135 -0
package/scripts/sync-skills.mjs +12 -0
package/skills/libretto/SKILL.md +130 -377
package/skills/libretto/references/auth-profiles.md +30 -0
package/skills/libretto/{code-generation-rules.md → references/code-generation-rules.md} +27 -42
package/skills/libretto/references/configuration-file-reference.md +53 -0
package/skills/libretto/references/pages-and-page-targeting.md +29 -0
package/skills/libretto/references/site-security-review.md +143 -0
package/src/cli/cli.ts +86 -0
package/src/cli/commands/ai.ts +35 -0
package/src/cli/commands/browser.ts +189 -0
package/src/cli/commands/execution.ts +822 -0
package/src/cli/commands/init.ts +350 -0
package/src/cli/commands/logs.ts +128 -0
package/src/cli/commands/shared.ts +69 -0
package/src/cli/commands/snapshot.ts +312 -0
package/src/cli/core/ai-config.ts +264 -0
package/src/cli/core/api-snapshot-analyzer.ts +108 -0
package/src/cli/core/browser.ts +976 -0
package/src/cli/core/context.ts +127 -0
package/src/cli/core/pause-signals.ts +35 -0
package/src/cli/core/session-telemetry.ts +564 -0
package/src/cli/core/session.ts +223 -0
package/src/cli/core/snapshot-analyzer.ts +855 -0
package/src/cli/core/snapshot-api-config.ts +231 -0
package/src/cli/core/telemetry.ts +459 -0
package/src/cli/framework/simple-cli.ts +1340 -0
package/src/cli/index.ts +13 -0
package/src/cli/router.ts +20 -0
package/src/cli/workers/run-integration-runtime.ts +338 -0
package/src/cli/workers/run-integration-worker-protocol.ts +16 -0
package/src/cli/workers/run-integration-worker.ts +72 -0
package/src/index.ts +127 -0
package/src/runtime/download/download.ts +104 -0
package/src/runtime/download/index.ts +7 -0
package/src/runtime/extract/extract.ts +102 -0
package/src/runtime/extract/index.ts +1 -0
package/src/runtime/network/index.ts +5 -0
package/src/runtime/network/network.ts +119 -0
package/{dist/runtime/recovery/agent.cjs → src/runtime/recovery/agent.ts} +114 -76
package/src/runtime/recovery/errors.ts +155 -0
package/src/runtime/recovery/index.ts +7 -0
package/src/runtime/recovery/recovery.ts +53 -0
package/{dist/shared/condense-dom/condense-dom.cjs → src/shared/condense-dom/condense-dom.ts} +249 -124
package/src/shared/config/config.ts +3 -0
package/src/shared/config/index.ts +0 -0
package/src/shared/debug/index.ts +1 -0
package/src/shared/debug/pause.ts +91 -0
package/src/shared/instrumentation/errors.ts +84 -0
package/src/shared/instrumentation/index.ts +9 -0
package/src/shared/instrumentation/instrument.ts +406 -0
package/src/shared/llm/ai-sdk-adapter.ts +81 -0
package/{dist/shared/llm/client.cjs → src/shared/llm/client.ts} +86 -80
package/src/shared/llm/index.ts +3 -0
package/src/shared/llm/types.ts +63 -0
package/src/shared/logger/index.ts +13 -0
package/src/shared/logger/logger.ts +358 -0
package/src/shared/logger/sinks.ts +148 -0
package/src/shared/paths/paths.ts +110 -0
package/src/shared/paths/repo-root.ts +27 -0
package/src/shared/run/api.ts +6 -0
package/src/shared/run/browser.ts +107 -0
package/src/shared/state/index.ts +11 -0
package/src/shared/state/session-state.ts +77 -0
package/src/shared/visualization/ghost-cursor.ts +213 -0
package/src/shared/visualization/highlight.ts +149 -0
package/src/shared/visualization/index.ts +18 -0
package/src/shared/workflow/workflow.ts +36 -0
package/dist/index.cjs +0 -144
package/dist/index.d.cts +0 -21
package/dist/runtime/download/download.cjs +0 -70
package/dist/runtime/download/download.d.cts +0 -35
package/dist/runtime/download/index.cjs +0 -30
package/dist/runtime/download/index.d.cts +0 -3
package/dist/runtime/extract/extract.cjs +0 -88
package/dist/runtime/extract/extract.d.cts +0 -23
package/dist/runtime/extract/index.cjs +0 -28
package/dist/runtime/extract/index.d.cts +0 -5
package/dist/runtime/network/index.cjs +0 -28
package/dist/runtime/network/index.d.cts +0 -4
package/dist/runtime/network/network.cjs +0 -91
package/dist/runtime/network/network.d.cts +0 -28
package/dist/runtime/recovery/agent.d.cts +0 -13
package/dist/runtime/recovery/errors.cjs +0 -124
package/dist/runtime/recovery/errors.d.cts +0 -31
package/dist/runtime/recovery/index.cjs +0 -34
package/dist/runtime/recovery/index.d.cts +0 -7
package/dist/runtime/recovery/recovery.cjs +0 -55
package/dist/runtime/recovery/recovery.d.cts +0 -12
package/dist/shared/condense-dom/condense-dom.d.cts +0 -34
package/dist/shared/config/config.cjs +0 -44
package/dist/shared/config/config.d.cts +0 -10
package/dist/shared/config/index.cjs +0 -32
package/dist/shared/config/index.d.cts +0 -1
package/dist/shared/debug/index.cjs +0 -28
package/dist/shared/debug/index.d.cts +0 -1
package/dist/shared/debug/pause.cjs +0 -86
package/dist/shared/debug/pause.d.cts +0 -12
package/dist/shared/instrumentation/errors.cjs +0 -81
package/dist/shared/instrumentation/errors.d.cts +0 -12
package/dist/shared/instrumentation/index.cjs +0 -35
package/dist/shared/instrumentation/index.d.cts +0 -6
package/dist/shared/instrumentation/instrument.cjs +0 -206
package/dist/shared/instrumentation/instrument.d.cts +0 -32
package/dist/shared/llm/ai-sdk-adapter.cjs +0 -71
package/dist/shared/llm/ai-sdk-adapter.d.cts +0 -22
package/dist/shared/llm/client.d.cts +0 -13
package/dist/shared/llm/index.cjs +0 -31
package/dist/shared/llm/index.d.cts +0 -5
package/dist/shared/llm/types.cjs +0 -16
package/dist/shared/llm/types.d.cts +0 -67
package/dist/shared/logger/index.cjs +0 -37
package/dist/shared/logger/index.d.cts +0 -2
package/dist/shared/logger/logger.cjs +0 -232
package/dist/shared/logger/logger.d.cts +0 -86
package/dist/shared/logger/sinks.cjs +0 -160
package/dist/shared/logger/sinks.d.cts +0 -9
package/dist/shared/paths/paths.cjs +0 -104
package/dist/shared/paths/paths.d.cts +0 -10
package/dist/shared/run/api.cjs +0 -28
package/dist/shared/run/api.d.cts +0 -2
package/dist/shared/run/browser.cjs +0 -98
package/dist/shared/run/browser.d.cts +0 -22
package/dist/shared/state/index.cjs +0 -38
package/dist/shared/state/index.d.cts +0 -2
package/dist/shared/state/session-state.cjs +0 -92
package/dist/shared/state/session-state.d.cts +0 -40
package/dist/shared/visualization/ghost-cursor.cjs +0 -174
package/dist/shared/visualization/ghost-cursor.d.cts +0 -37
package/dist/shared/visualization/highlight.cjs +0 -134
package/dist/shared/visualization/highlight.d.cts +0 -22
package/dist/shared/visualization/index.cjs +0 -45
package/dist/shared/visualization/index.d.cts +0 -3
package/dist/shared/workflow/workflow.cjs +0 -47
package/dist/shared/workflow/workflow.d.cts +0 -21
package/skills/libretto/integration-approach-selection.md +0 -174

package/src/runtime/extract/extract.ts ADDED Viewed

@@ -0,0 +1,102 @@
+import type { Page } from "playwright";
+import type z from "zod";
+import {
+  type MinimalLogger,
+  defaultLogger,
+} from "../../shared/logger/logger.js";
+import type { LLMClient } from "../../shared/llm/types.js";
+export type ExtractOptions<T extends z.ZodType> = {
+  page: Page;
+  instruction: string;
+  schema: T;
+  llmClient: LLMClient;
+  logger?: MinimalLogger;
+  /** Optional CSS selector to scope extraction to a specific element. */
+  selector?: string;
+};
+/**
+ * Generic AI-powered data extraction from page elements.
+ * Takes a screenshot (full-page via CDP or scoped to an element),
+ * captures DOM content, and uses an LLM to extract structured data
+ * matching the provided Zod schema.
+ */
+export async function extractFromPage<T extends z.ZodType>(
+  options: ExtractOptions<T>,
+): Promise<z.infer<T>> {
+  const {
+    page,
+    instruction,
+    schema,
+    selector,
+    logger = defaultLogger,
+    llmClient,
+  } = options;
+  let screenshot: string;
+  let domContent: string | undefined;
+  if (selector) {
+    const element = page.locator(selector);
+    await element.waitFor({ state: "visible", timeout: 10_000 });
+    const screenshotBuffer = await element.screenshot();
+    screenshot = screenshotBuffer.toString("base64");
+    try {
+      domContent = await element.innerHTML();
+      if (domContent.length > 30000) {
+        domContent = domContent.slice(0, 30000) + "\n... [truncated]";
+      }
+    } catch {
+      domContent = undefined;
+    }
+  } else {
+    const cdpClient = await page.context().newCDPSession(page);
+    await cdpClient.send("Page.enable");
+    const { data } = await cdpClient.send("Page.captureScreenshot", {
+      format: "png",
+    });
+    screenshot = data;
+    try {
+      const htmlContent = await page.content();
+      domContent =
+        htmlContent.length > 50000
+          ? htmlContent.slice(0, 50000) + "\n... [truncated]"
+          : htmlContent;
+    } catch {
+      domContent = undefined;
+    }
+  }
+  const prompt = `You are analyzing a screenshot${selector ? " of a specific element" : ""} from a web page to extract structured data.
+Instruction: ${instruction}
+${domContent ? `Here is the HTML content for additional context:\n<html>\n${domContent}\n</html>` : ""}
+Extract the requested information from the screenshot and return it in the specified format. Be precise and only extract what is visible.`;
+  const result = await llmClient.generateObjectFromMessages({
+    schema,
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: prompt },
+          { type: "image", image: `data:image/png;base64,${screenshot}` },
+        ],
+      },
+    ],
+    temperature: 0,
+  });
+  logger.info("extractFromPage completed", {
+    selector,
+    instruction: instruction.slice(0, 100),
+  });
+  return result;
+}

package/src/runtime/extract/index.ts ADDED Viewed

	@@ -0,0 +1 @@
1	+ export { extractFromPage, type ExtractOptions } from "./extract.js";

package/src/runtime/network/index.ts ADDED Viewed

@@ -0,0 +1,5 @@
+export {
+  pageRequest,
+  type RequestConfig,
+  type PageRequestOptions,
+} from "./network.js";

package/src/runtime/network/network.ts ADDED Viewed

@@ -0,0 +1,119 @@
+import type { Page } from "playwright";
+import type z from "zod";
+import type { MinimalLogger } from "../../shared/logger/logger.js";
+export type RequestConfig = {
+  url: string;
+  method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
+  headers?: Record<string, string>;
+  body?: Record<string, any> | string;
+  /** How to serialize the body. Defaults to "json". */
+  bodyType?: "json" | "form";
+  /** How to parse the response. Defaults to "json". */
+  responseType?: "json" | "text" | "xml";
+};
+export type PageRequestOptions<T extends z.ZodType | undefined = undefined> = {
+  logger?: MinimalLogger;
+  /** Optional Zod schema to validate the response body. */
+  schema?: T;
+};
+type PageRequestResult<T extends z.ZodType | undefined> = T extends z.ZodType
+  ? z.infer<T>
+  : any;
+/**
+ * Executes a fetch() call inside the browser context via page.evaluate().
+ * Provides typed request config, automatic response parsing, optional Zod
+ * validation, and logging.
+ */
+export async function pageRequest<T extends z.ZodType | undefined = undefined>(
+  page: Page,
+  config: RequestConfig,
+  options?: PageRequestOptions<T>,
+): Promise<PageRequestResult<T>> {
+  const {
+    url,
+    method = "GET",
+    headers = {},
+    body,
+    bodyType = "json",
+    responseType = "json",
+  } = config;
+  const { logger, schema } = options ?? {};
+  const startTime = Date.now();
+  // Build fetch options to pass into page.evaluate
+  const fetchHeaders: Record<string, string> = { ...headers };
+  let fetchBody: string | undefined;
+  if (body !== undefined) {
+    if (bodyType === "form") {
+      fetchHeaders["Content-Type"] = "application/x-www-form-urlencoded";
+      if (typeof body === "string") {
+        fetchBody = body;
+      } else {
+        fetchBody = new URLSearchParams(
+          Object.entries(body).map(([k, v]) => [k, String(v)]),
+        ).toString();
+      }
+    } else {
+      fetchHeaders["Content-Type"] = "application/json";
+      fetchBody = typeof body === "string" ? body : JSON.stringify(body);
+    }
+  }
+  const result = await page.evaluate(
+    async ({ url, method, headers, body, responseType }) => {
+      const res = await fetch(url, {
+        method,
+        headers,
+        body: body ?? undefined,
+      });
+      const status = res.status;
+      const ok = res.ok;
+      let data: any;
+      if (responseType === "json") {
+        data = await res.json();
+      } else {
+        data = await res.text();
+      }
+      return { status, ok, data };
+    },
+    { url, method, headers: fetchHeaders, body: fetchBody, responseType },
+  );
+  const duration = Date.now() - startTime;
+  if (!result.ok) {
+    logger?.warn("network:request:error", {
+      method,
+      url,
+      status: result.status,
+      duration,
+      body:
+        typeof result.data === "string" ? result.data.slice(0, 500) : undefined,
+    });
+    throw new Error(
+      `pageRequest failed: ${method} ${url} returned ${result.status}`,
+    );
+  }
+  logger?.info("network:request", {
+    method,
+    url,
+    status: result.status,
+    duration,
+  });
+  if (schema) {
+    return schema.parse(result.data) as PageRequestResult<T>;
+  }
+  return result.data as PageRequestResult<T>;
+}

package/{dist/runtime/recovery/agent.cjs → src/runtime/recovery/agent.ts} RENAMED Viewed

@@ -1,32 +1,33 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var agent_exports = {};
-__export(agent_exports, {
-  executeRecoveryAgent: () => executeRecoveryAgent
-});
-module.exports = __toCommonJS(agent_exports);
-var import_logger = require("../../shared/logger/logger.js");
-var import_zod = require("zod");
-function delay(ms) {
+import type { Page } from "playwright";
+import {
+  type MinimalLogger,
+  defaultLogger,
+} from "../../shared/logger/logger.js";
+import type { LLMClient } from "../../shared/llm/types.js";
+type BrowserAction =
+  | { type: "click"; x: number; y: number; button?: string }
+  | { type: "double_click"; x: number; y: number }
+  | {
+      type: "scroll";
+      x: number;
+      y: number;
+      scroll_x: number;
+      scroll_y: number;
+    }
+  | { type: "keypress"; keys: string[] }
+  | { type: "type"; text: string }
+  | { type: "wait" }
+  | { type: "screenshot" }
+  | { type: "drag"; path: { x: number; y: number }[] }
+  | { type: "move"; x: number; y: number }
+  | { type: "done" };
+function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
-const KEY_MAPPINGS = {
+const KEY_MAPPINGS: Record<string, string> = {
   ENTER: "Enter",
   RETURN: "Enter",
   TAB: "Tab",
@@ -49,16 +50,25 @@ const KEY_MAPPINGS = {
   SHIFT: "Shift",
   META: "Meta",
   CMD: "Meta",
-  COMMAND: "Meta"
+  COMMAND: "Meta",
 };
-function mapKeyName(key) {
+function mapKeyName(key: string): string {
   return KEY_MAPPINGS[key.toUpperCase()] ?? key;
 }
-async function executeBrowserAction(page, action, logger = import_logger.defaultLogger) {
+async function executeBrowserAction(
+  page: Page,
+  action: BrowserAction,
+  logger: MinimalLogger = defaultLogger,
+): Promise<void> {
   switch (action.type) {
     case "click": {
       const { x, y, button = "left" } = action;
-      const playwrightButton = button === "wheel" || button === "back" || button === "forward" ? "left" : button;
+      const playwrightButton =
+        button === "wheel" || button === "back" || button === "forward"
+          ? ("left" as const)
+          : (button as "left" | "right" | "middle");
       await page.mouse.click(x, y, { button: playwrightButton });
       logger.info(`Clicked at (${x}, ${y}) with ${button} button`);
       break;
@@ -90,7 +100,7 @@ async function executeBrowserAction(page, action, logger = import_logger.default
       break;
     }
     case "wait": {
-      await delay(2e3);
+      await delay(2000);
       logger.info("Waited 2 seconds");
       break;
     }
@@ -110,7 +120,9 @@ async function executeBrowserAction(page, action, logger = import_logger.default
           if (point) await page.mouse.move(point.x, point.y);
         }
         await page.mouse.up();
-        logger.info(`Dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
+        logger.info(
+          `Dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y})`,
+        );
       }
       break;
     }
@@ -125,56 +137,80 @@ async function executeBrowserAction(page, action, logger = import_logger.default
     }
   }
 }
-const recoveryActionSchema = import_zod.z.object({
-  reasoning: import_zod.z.string().describe("Your reasoning about what you see and what action to take"),
-  action: import_zod.z.discriminatedUnion("type", [
-    import_zod.z.object({
-      type: import_zod.z.literal("click"),
-      x: import_zod.z.number(),
-      y: import_zod.z.number()
+import { z } from "zod";
+const recoveryActionSchema = z.object({
+  reasoning: z
+    .string()
+    .describe("Your reasoning about what you see and what action to take"),
+  action: z.discriminatedUnion("type", [
+    z.object({
+      type: z.literal("click"),
+      x: z.number(),
+      y: z.number(),
+    }),
+    z.object({
+      type: z.literal("type"),
+      text: z.string(),
     }),
-    import_zod.z.object({
-      type: import_zod.z.literal("type"),
-      text: import_zod.z.string()
+    z.object({
+      type: z.literal("keypress"),
+      keys: z.array(z.string()),
     }),
-    import_zod.z.object({
-      type: import_zod.z.literal("keypress"),
-      keys: import_zod.z.array(import_zod.z.string())
+    z.object({
+      type: z.literal("scroll"),
+      x: z.number(),
+      y: z.number(),
+      scroll_x: z.number(),
+      scroll_y: z.number(),
     }),
-    import_zod.z.object({
-      type: import_zod.z.literal("scroll"),
-      x: import_zod.z.number(),
-      y: import_zod.z.number(),
-      scroll_x: import_zod.z.number(),
-      scroll_y: import_zod.z.number()
+    z.object({
+      type: z.literal("wait"),
     }),
-    import_zod.z.object({
-      type: import_zod.z.literal("wait")
+    z.object({
+      type: z.literal("done"),
     }),
-    import_zod.z.object({
-      type: import_zod.z.literal("done")
-    })
-  ])
+  ]),
 });
-async function executeRecoveryAgent(page, instruction, logger, llmClient) {
+/**
+ * Executes a vision-based recovery agent to recover from browser automation failures.
+ * Takes a screenshot, sends it to the LLM with the instruction, and executes
+ * the LLM's suggested browser actions.
+ */
+export async function executeRecoveryAgent(
+  page: Page,
+  instruction: string,
+  logger?: MinimalLogger,
+  llmClient?: LLMClient,
+): Promise<void> {
   if (!llmClient) {
     return;
   }
-  const log = logger ?? import_logger.defaultLogger;
+  const log = logger ?? defaultLogger;
   log.info("Executing vision-based recovery agent", { instruction });
   const viewport = page.viewportSize();
   if (!viewport) {
     throw new Error("Viewport size not found");
   }
-  let screenshot;
+  let screenshot: string;
   try {
-    screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
+    screenshot = (
+      await page.screenshot({ fullPage: false, timeout: 10000 })
+    ).toString("base64");
   } catch (screenshotError) {
     log.warn("Failed to take screenshot for recovery agent, skipping", {
-      screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
+      screenshotError:
+        screenshotError instanceof Error
+          ? screenshotError.message
+          : String(screenshotError),
     });
     throw new Error("Failed to take screenshot for recovery agent");
   }
   const maxSteps = 3;
   for (let step = 1; step <= maxSteps; step++) {
     const result = await llmClient.generateObjectFromMessages({
@@ -190,34 +226,36 @@ async function executeRecoveryAgent(page, instruction, logger, llmClient) {
 Your task: ${instruction}
 Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
-Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`
+Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`,
             },
             {
               type: "image",
-              image: `data:image/png;base64,${screenshot}`
-            }
-          ]
-        }
+              image: `data:image/png;base64,${screenshot}`,
+            },
+          ],
+        },
       ],
-      temperature: 0
+      temperature: 0,
     });
     log.info(`Recovery step ${step}/${maxSteps}`, {
       reasoning: result.reasoning,
-      action: result.action
+      action: result.action,
     });
     if (result.action.type === "done") {
       log.info("Recovery agent completed - no more actions needed");
       break;
     }
     await executeBrowserAction(page, result.action, log);
-    await delay(2e3);
+    await delay(2000);
+    // Take new screenshot for next iteration
     screenshot = (await page.screenshot({ fullPage: false })).toString(
-      "base64"
+      "base64",
     );
   }
   log.info("Recovery agent execution completed");
 }
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  executeRecoveryAgent
-});

package/src/runtime/recovery/errors.ts ADDED Viewed

@@ -0,0 +1,155 @@
+import type { Page } from "playwright";
+import {
+  type MinimalLogger,
+  defaultLogger,
+} from "../../shared/logger/logger.js";
+import type { LLMClient } from "../../shared/llm/types.js";
+import { z } from "zod";
+/**
+ * Known error type for classifying submission errors.
+ * errorPatterns are what the LLM should look for on screen.
+ * userMessage is the friendly message returned when matched.
+ */
+export type KnownSubmissionError = {
+  id: string;
+  errorPatterns: string[];
+  userMessage: string;
+};
+export type DetectedSubmissionError = {
+  matched: true;
+  errorId: string;
+  message: string;
+};
+const detectSubmissionErrorSchema = z.object({
+  hasError: z.boolean().describe("Whether an error is visible on the page"),
+  matchedKnownErrorId: z
+    .string()
+    .nullable()
+    .describe("The ID of the matched known error, or null if no match"),
+  errorMessage: z
+    .string()
+    .nullable()
+    .describe("The error message visible on screen, or null if no error"),
+});
+/**
+ * Uses screenshot + LLM vision to detect if an error occurred during a submission process.
+ * Captures a screenshot via CDP (handles unresponsive pages), sends it to the LLM,
+ * and checks against the provided known error patterns.
+ *
+ * @returns DetectedSubmissionError if a known error is matched
+ * @throws The original error if no known error matches
+ */
+export async function detectSubmissionError(
+  page: Page,
+  error: unknown,
+  logContext: string,
+  llmClient: LLMClient,
+  knownErrors: KnownSubmissionError[] = [],
+  logger?: MinimalLogger,
+): Promise<DetectedSubmissionError> {
+  const log = logger ?? defaultLogger;
+  // Capture screenshot using CDP to handle unresponsive pages
+  let screenshot: string;
+  let domSnapshot: string | undefined;
+  try {
+    const cdpClient = await page.context().newCDPSession(page);
+    await cdpClient.send("Page.enable");
+    const { data } = await cdpClient.send("Page.captureScreenshot", {
+      format: "png",
+    });
+    screenshot = data;
+  } catch (screenshotError) {
+    log.warn(
+      "Failed to take screenshot via CDP for error detection, skipping LLM analysis",
+      { screenshotError, originalError: error },
+    );
+    throw error;
+  }
+  // Capture DOM snapshot for additional context
+  try {
+    const htmlContent = await page.content();
+    domSnapshot =
+      htmlContent.length > 50000
+        ? htmlContent.slice(0, 50000) + "\n... [truncated]"
+        : htmlContent;
+  } catch (domError) {
+    log.warn("Failed to capture DOM snapshot", {
+      domError: domError instanceof Error ? domError.message : String(domError),
+    });
+  }
+  const knownErrorsDescription =
+    knownErrors.length > 0
+      ? `\nKnown error patterns to look for:\n${knownErrors.map((e, i) => `${i + 1}. ID: "${e.id}" - Patterns: ${e.errorPatterns.join(", ")}`).join("\n")}\n`
+      : "";
+  const prompt = `You are analyzing a screenshot and DOM of a web page to detect if an error occurred during a browser automation process.
+Context: ${logContext}
+${knownErrorsDescription}
+Analyze the screenshot and DOM snapshot to determine:
+1. Is there any error message, warning, or indication of failure visible on the page?
+2. If yes, does it match any of the known error patterns listed above?
+3. What is the exact error message or description of the problem?
+IMPORTANT:
+- Look carefully for error alerts, warning banners, error modals, red text, or any indication of failure
+- Check the DOM snapshot for error messages that may not be visible in the screenshot
+- If you see a known error pattern, use its exact ID in matchedKnownErrorId
+- If there's an error but it doesn't match any known pattern, set matchedKnownErrorId to null
+- If the page looks normal with no errors, set hasError to false
+${domSnapshot ? `<dom_snapshot>\n${domSnapshot}\n</dom_snapshot>` : ""}`;
+  const result = await llmClient.generateObjectFromMessages({
+    schema: detectSubmissionErrorSchema,
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: prompt },
+          { type: "image", image: `data:image/png;base64,${screenshot}` },
+        ],
+      },
+    ],
+    temperature: 0,
+  });
+  if (!result.hasError) {
+    log.info("No error detected by LLM", { result });
+  }
+  // Check if it matches a known error
+  if (result.matchedKnownErrorId) {
+    const knownError = knownErrors.find(
+      (e) => e.id === result.matchedKnownErrorId,
+    );
+    if (knownError) {
+      log.warn(logContext, {
+        error,
+        browserError: result.errorMessage,
+        knownErrorId: result.matchedKnownErrorId,
+      });
+      return {
+        matched: true,
+        errorId: knownError.id,
+        message: knownError.userMessage,
+      };
+    }
+  }
+  // Log and re-throw for unknown errors
+  log.warn(logContext, {
+    error,
+    browserError: result.errorMessage,
+  });
+  throw error;
+}

package/src/runtime/recovery/index.ts ADDED Viewed

@@ -0,0 +1,7 @@
+export { executeRecoveryAgent } from "./agent.js";
+export { attemptWithRecovery } from "./recovery.js";
+export {
+  detectSubmissionError,
+  type KnownSubmissionError,
+  type DetectedSubmissionError,
+} from "./errors.js";