libretto 0.2.6 → 0.3.0

package/dist/runtime/recovery/agent.js

@@ -1,199 +0,0 @@
-import { defaultLogger } from "../../shared/logger/logger.js";
-function delay(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-const KEY_MAPPINGS = {
-  ENTER: "Enter",
-  RETURN: "Enter",
-  TAB: "Tab",
-  SPACE: " ",
-  BACKSPACE: "Backspace",
-  DELETE: "Delete",
-  ESCAPE: "Escape",
-  ESC: "Escape",
-  UP: "ArrowUp",
-  DOWN: "ArrowDown",
-  LEFT: "ArrowLeft",
-  RIGHT: "ArrowRight",
-  HOME: "Home",
-  END: "End",
-  PAGEUP: "PageUp",
-  PAGEDOWN: "PageDown",
-  CTRL: "Control",
-  CONTROL: "Control",
-  ALT: "Alt",
-  SHIFT: "Shift",
-  META: "Meta",
-  CMD: "Meta",
-  COMMAND: "Meta"
-};
-function mapKeyName(key) {
-  return KEY_MAPPINGS[key.toUpperCase()] ?? key;
-}
-async function executeBrowserAction(page, action, logger = defaultLogger) {
-  switch (action.type) {
-    case "click": {
-      const { x, y, button = "left" } = action;
-      const playwrightButton = button === "wheel" || button === "back" || button === "forward" ? "left" : button;
-      await page.mouse.click(x, y, { button: playwrightButton });
-      logger.info(`Clicked at (${x}, ${y}) with ${button} button`);
-      break;
-    }
-    case "double_click": {
-      const { x, y } = action;
-      await page.mouse.dblclick(x, y);
-      logger.info(`Double-clicked at (${x}, ${y})`);
-      break;
-    }
-    case "scroll": {
-      const { x, y, scroll_x, scroll_y } = action;
-      await page.mouse.move(x, y);
-      await page.evaluate(`window.scrollBy(${scroll_x}, ${scroll_y})`);
-      logger.info(`Scrolled at (${x}, ${y}) by (${scroll_x}, ${scroll_y})`);
-      break;
-    }
-    case "keypress": {
-      for (const key of action.keys) {
-        const mapped = mapKeyName(key);
-        await page.keyboard.press(mapped);
-        logger.info(`Pressed key: ${key} (mapped to ${mapped})`);
-      }
-      break;
-    }
-    case "type": {
-      await page.keyboard.type(action.text);
-      logger.info(`Typed text: ${action.text}`);
-      break;
-    }
-    case "wait": {
-      await delay(2e3);
-      logger.info("Waited 2 seconds");
-      break;
-    }
-    case "screenshot": {
-      logger.info("Screenshot action (no-op, taken automatically)");
-      break;
-    }
-    case "drag": {
-      const { path } = action;
-      const start = path[0];
-      const end = path[path.length - 1];
-      if (path.length >= 2 && start && end) {
-        await page.mouse.move(start.x, start.y);
-        await page.mouse.down();
-        for (let i = 1; i < path.length; i++) {
-          const point = path[i];
-          if (point) await page.mouse.move(point.x, point.y);
-        }
-        await page.mouse.up();
-        logger.info(`Dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
-      }
-      break;
-    }
-    case "move": {
-      const { x, y } = action;
-      await page.mouse.move(x, y);
-      logger.info(`Moved mouse to (${x}, ${y})`);
-      break;
-    }
-    case "done": {
-      break;
-    }
-  }
-}
-import { z } from "zod";
-const recoveryActionSchema = z.object({
-  reasoning: z.string().describe("Your reasoning about what you see and what action to take"),
-  action: z.discriminatedUnion("type", [
-    z.object({
-      type: z.literal("click"),
-      x: z.number(),
-      y: z.number()
-    }),
-    z.object({
-      type: z.literal("type"),
-      text: z.string()
-    }),
-    z.object({
-      type: z.literal("keypress"),
-      keys: z.array(z.string())
-    }),
-    z.object({
-      type: z.literal("scroll"),
-      x: z.number(),
-      y: z.number(),
-      scroll_x: z.number(),
-      scroll_y: z.number()
-    }),
-    z.object({
-      type: z.literal("wait")
-    }),
-    z.object({
-      type: z.literal("done")
-    })
-  ])
-});
-async function executeRecoveryAgent(page, instruction, logger, llmClient) {
-  if (!llmClient) {
-    return;
-  }
-  const log = logger ?? defaultLogger;
-  log.info("Executing vision-based recovery agent", { instruction });
-  const viewport = page.viewportSize();
-  if (!viewport) {
-    throw new Error("Viewport size not found");
-  }
-  let screenshot;
-  try {
-    screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
-  } catch (screenshotError) {
-    log.warn("Failed to take screenshot for recovery agent, skipping", {
-      screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
-    });
-    throw new Error("Failed to take screenshot for recovery agent");
-  }
-  const maxSteps = 3;
-  for (let step = 1; step <= maxSteps; step++) {
-    const result = await llmClient.generateObjectFromMessages({
-      schema: recoveryActionSchema,
-      messages: [
-        {
-          role: "user",
-          content: [
-            {
-              type: "text",
-              text: `You are an expert browser support agent. Your job is to resolve issues when browser automation encounters unexpected website behavior (e.g., popups blocking interaction).
-
-Your task: ${instruction}
-
-Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
-Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`
-            },
-            {
-              type: "image",
-              image: `data:image/png;base64,${screenshot}`
-            }
-          ]
-        }
-      ],
-      temperature: 0
-    });
-    log.info(`Recovery step ${step}/${maxSteps}`, {
-      reasoning: result.reasoning,
-      action: result.action
-    });
-    if (result.action.type === "done") {
-      log.info("Recovery agent completed - no more actions needed");
-      break;
-    }
-    await executeBrowserAction(page, result.action, log);
-    await delay(2e3);
-    screenshot = (await page.screenshot({ fullPage: false })).toString(
-      "base64"
-    );
-  }
-  log.info("Recovery agent execution completed");
-}
-export {
-  executeRecoveryAgent
-};

package/dist/runtime/recovery/errors.cjs

@@ -1,124 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var errors_exports = {};
-__export(errors_exports, {
-  detectSubmissionError: () => detectSubmissionError
-});
-module.exports = __toCommonJS(errors_exports);
-var import_logger = require("../../shared/logger/logger.js");
-var import_zod = require("zod");
-const detectSubmissionErrorSchema = import_zod.z.object({
-  hasError: import_zod.z.boolean().describe("Whether an error is visible on the page"),
-  matchedKnownErrorId: import_zod.z.string().nullable().describe("The ID of the matched known error, or null if no match"),
-  errorMessage: import_zod.z.string().nullable().describe("The error message visible on screen, or null if no error")
-});
-async function detectSubmissionError(page, error, logContext, llmClient, knownErrors = [], logger) {
-  const log = logger ?? import_logger.defaultLogger;
-  let screenshot;
-  let domSnapshot;
-  try {
-    const cdpClient = await page.context().newCDPSession(page);
-    await cdpClient.send("Page.enable");
-    const { data } = await cdpClient.send("Page.captureScreenshot", {
-      format: "png"
-    });
-    screenshot = data;
-  } catch (screenshotError) {
-    log.warn(
-      "Failed to take screenshot via CDP for error detection, skipping LLM analysis",
-      { screenshotError, originalError: error }
-    );
-    throw error;
-  }
-  try {
-    const htmlContent = await page.content();
-    domSnapshot = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
-  } catch (domError) {
-    log.warn("Failed to capture DOM snapshot", {
-      domError: domError instanceof Error ? domError.message : String(domError)
-    });
-  }
-  const knownErrorsDescription = knownErrors.length > 0 ? `
-Known error patterns to look for:
-${knownErrors.map((e, i) => `${i + 1}. ID: "${e.id}" - Patterns: ${e.errorPatterns.join(", ")}`).join("\n")}
-` : "";
-  const prompt = `You are analyzing a screenshot and DOM of a web page to detect if an error occurred during a browser automation process.
-
-Context: ${logContext}
-
-${knownErrorsDescription}
-
-Analyze the screenshot and DOM snapshot to determine:
-1. Is there any error message, warning, or indication of failure visible on the page?
-2. If yes, does it match any of the known error patterns listed above?
-3. What is the exact error message or description of the problem?
-
-IMPORTANT:
-- Look carefully for error alerts, warning banners, error modals, red text, or any indication of failure
-- Check the DOM snapshot for error messages that may not be visible in the screenshot
-- If you see a known error pattern, use its exact ID in matchedKnownErrorId
-- If there's an error but it doesn't match any known pattern, set matchedKnownErrorId to null
-- If the page looks normal with no errors, set hasError to false
-
-${domSnapshot ? `<dom_snapshot>
-${domSnapshot}
-</dom_snapshot>` : ""}`;
-  const result = await llmClient.generateObjectFromMessages({
-    schema: detectSubmissionErrorSchema,
-    messages: [
-      {
-        role: "user",
-        content: [
-          { type: "text", text: prompt },
-          { type: "image", image: `data:image/png;base64,${screenshot}` }
-        ]
-      }
-    ],
-    temperature: 0
-  });
-  if (!result.hasError) {
-    log.info("No error detected by LLM", { result });
-  }
-  if (result.matchedKnownErrorId) {
-    const knownError = knownErrors.find(
-      (e) => e.id === result.matchedKnownErrorId
-    );
-    if (knownError) {
-      log.warn(logContext, {
-        error,
-        browserError: result.errorMessage,
-        knownErrorId: result.matchedKnownErrorId
-      });
-      return {
-        matched: true,
-        errorId: knownError.id,
-        message: knownError.userMessage
-      };
-    }
-  }
-  log.warn(logContext, {
-    error,
-    browserError: result.errorMessage
-  });
-  throw error;
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  detectSubmissionError
-});

package/dist/runtime/recovery/errors.d.cts

@@ -1,31 +0,0 @@
-import { Page } from 'playwright';
-import { MinimalLogger } from '../../shared/logger/logger.cjs';
-import { LLMClient } from '../../shared/llm/types.cjs';
-import 'zod';
-
-/**
- * Known error type for classifying submission errors.
- * errorPatterns are what the LLM should look for on screen.
- * userMessage is the friendly message returned when matched.
- */
-type KnownSubmissionError = {
-    id: string;
-    errorPatterns: string[];
-    userMessage: string;
-};
-type DetectedSubmissionError = {
-    matched: true;
-    errorId: string;
-    message: string;
-};
-/**
- * Uses screenshot + LLM vision to detect if an error occurred during a submission process.
- * Captures a screenshot via CDP (handles unresponsive pages), sends it to the LLM,
- * and checks against the provided known error patterns.
- *
- * @returns DetectedSubmissionError if a known error is matched
- * @throws The original error if no known error matches
- */
-declare function detectSubmissionError(page: Page, error: unknown, logContext: string, llmClient: LLMClient, knownErrors?: KnownSubmissionError[], logger?: MinimalLogger): Promise<DetectedSubmissionError>;
-
-export { type DetectedSubmissionError, type KnownSubmissionError, detectSubmissionError };

package/dist/runtime/recovery/errors.d.ts

@@ -1,31 +0,0 @@
-import { Page } from 'playwright';
-import { MinimalLogger } from '../../shared/logger/logger.js';
-import { LLMClient } from '../../shared/llm/types.js';
-import 'zod';
-
-/**
- * Known error type for classifying submission errors.
- * errorPatterns are what the LLM should look for on screen.
- * userMessage is the friendly message returned when matched.
- */
-type KnownSubmissionError = {
-    id: string;
-    errorPatterns: string[];
-    userMessage: string;
-};
-type DetectedSubmissionError = {
-    matched: true;
-    errorId: string;
-    message: string;
-};
-/**
- * Uses screenshot + LLM vision to detect if an error occurred during a submission process.
- * Captures a screenshot via CDP (handles unresponsive pages), sends it to the LLM,
- * and checks against the provided known error patterns.
- *
- * @returns DetectedSubmissionError if a known error is matched
- * @throws The original error if no known error matches
- */
-declare function detectSubmissionError(page: Page, error: unknown, logContext: string, llmClient: LLMClient, knownErrors?: KnownSubmissionError[], logger?: MinimalLogger): Promise<DetectedSubmissionError>;
-
-export { type DetectedSubmissionError, type KnownSubmissionError, detectSubmissionError };

package/dist/runtime/recovery/errors.js

@@ -1,100 +0,0 @@
-import { defaultLogger } from "../../shared/logger/logger.js";
-import { z } from "zod";
-const detectSubmissionErrorSchema = z.object({
-  hasError: z.boolean().describe("Whether an error is visible on the page"),
-  matchedKnownErrorId: z.string().nullable().describe("The ID of the matched known error, or null if no match"),
-  errorMessage: z.string().nullable().describe("The error message visible on screen, or null if no error")
-});
-async function detectSubmissionError(page, error, logContext, llmClient, knownErrors = [], logger) {
-  const log = logger ?? defaultLogger;
-  let screenshot;
-  let domSnapshot;
-  try {
-    const cdpClient = await page.context().newCDPSession(page);
-    await cdpClient.send("Page.enable");
-    const { data } = await cdpClient.send("Page.captureScreenshot", {
-      format: "png"
-    });
-    screenshot = data;
-  } catch (screenshotError) {
-    log.warn(
-      "Failed to take screenshot via CDP for error detection, skipping LLM analysis",
-      { screenshotError, originalError: error }
-    );
-    throw error;
-  }
-  try {
-    const htmlContent = await page.content();
-    domSnapshot = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
-  } catch (domError) {
-    log.warn("Failed to capture DOM snapshot", {
-      domError: domError instanceof Error ? domError.message : String(domError)
-    });
-  }
-  const knownErrorsDescription = knownErrors.length > 0 ? `
-Known error patterns to look for:
-${knownErrors.map((e, i) => `${i + 1}. ID: "${e.id}" - Patterns: ${e.errorPatterns.join(", ")}`).join("\n")}
-` : "";
-  const prompt = `You are analyzing a screenshot and DOM of a web page to detect if an error occurred during a browser automation process.
-
-Context: ${logContext}
-
-${knownErrorsDescription}
-
-Analyze the screenshot and DOM snapshot to determine:
-1. Is there any error message, warning, or indication of failure visible on the page?
-2. If yes, does it match any of the known error patterns listed above?
-3. What is the exact error message or description of the problem?
-
-IMPORTANT:
-- Look carefully for error alerts, warning banners, error modals, red text, or any indication of failure
-- Check the DOM snapshot for error messages that may not be visible in the screenshot
-- If you see a known error pattern, use its exact ID in matchedKnownErrorId
-- If there's an error but it doesn't match any known pattern, set matchedKnownErrorId to null
-- If the page looks normal with no errors, set hasError to false
-
-${domSnapshot ? `<dom_snapshot>
-${domSnapshot}
-</dom_snapshot>` : ""}`;
-  const result = await llmClient.generateObjectFromMessages({
-    schema: detectSubmissionErrorSchema,
-    messages: [
-      {
-        role: "user",
-        content: [
-          { type: "text", text: prompt },
-          { type: "image", image: `data:image/png;base64,${screenshot}` }
-        ]
-      }
-    ],
-    temperature: 0
-  });
-  if (!result.hasError) {
-    log.info("No error detected by LLM", { result });
-  }
-  if (result.matchedKnownErrorId) {
-    const knownError = knownErrors.find(
-      (e) => e.id === result.matchedKnownErrorId
-    );
-    if (knownError) {
-      log.warn(logContext, {
-        error,
-        browserError: result.errorMessage,
-        knownErrorId: result.matchedKnownErrorId
-      });
-      return {
-        matched: true,
-        errorId: knownError.id,
-        message: knownError.userMessage
-      };
-    }
-  }
-  log.warn(logContext, {
-    error,
-    browserError: result.errorMessage
-  });
-  throw error;
-}
-export {
-  detectSubmissionError
-};

package/dist/runtime/recovery/index.cjs

@@ -1,34 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var recovery_exports = {};
-__export(recovery_exports, {
-  attemptWithRecovery: () => import_recovery.attemptWithRecovery,
-  detectSubmissionError: () => import_errors.detectSubmissionError,
-  executeRecoveryAgent: () => import_agent.executeRecoveryAgent
-});
-module.exports = __toCommonJS(recovery_exports);
-var import_agent = require("./agent.js");
-var import_recovery = require("./recovery.js");
-var import_errors = require("./errors.js");
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  attemptWithRecovery,
-  detectSubmissionError,
-  executeRecoveryAgent
-});

package/dist/runtime/recovery/index.d.cts

@@ -1,7 +0,0 @@
-export { executeRecoveryAgent } from './agent.cjs';
-export { attemptWithRecovery } from './recovery.cjs';
-export { DetectedSubmissionError, KnownSubmissionError, detectSubmissionError } from './errors.cjs';
-import 'playwright';
-import '../../shared/logger/logger.cjs';
-import '../../shared/llm/types.cjs';
-import 'zod';

package/dist/runtime/recovery/index.d.ts

@@ -1,7 +0,0 @@
-export { executeRecoveryAgent } from './agent.js';
-export { attemptWithRecovery } from './recovery.js';
-export { DetectedSubmissionError, KnownSubmissionError, detectSubmissionError } from './errors.js';
-import 'playwright';
-import '../../shared/logger/logger.js';
-import '../../shared/llm/types.js';
-import 'zod';

package/dist/runtime/recovery/index.js

@@ -1,10 +0,0 @@
-import { executeRecoveryAgent } from "./agent.js";
-import { attemptWithRecovery } from "./recovery.js";
-import {
-  detectSubmissionError
-} from "./errors.js";
-export {
-  attemptWithRecovery,
-  detectSubmissionError,
-  executeRecoveryAgent
-};

package/dist/runtime/recovery/recovery.cjs

@@ -1,55 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var recovery_exports = {};
-__export(recovery_exports, {
-  attemptWithRecovery: () => attemptWithRecovery
-});
-module.exports = __toCommonJS(recovery_exports);
-var import_logger = require("../../shared/logger/logger.js");
-var import_agent = require("./agent.js");
-async function attemptWithRecovery(page, fn, logger, llmClient) {
-  const log = logger ?? import_logger.defaultLogger;
-  try {
-    return await fn();
-  } catch (error) {
-    if (error instanceof Error && (error.message.includes("Target closed") || error.message.includes("browser has been closed") || error.message.includes("context or browser has been closed"))) {
-      log.warn("Page/browser has been closed, cannot recover", {
-        error: error.message
-      });
-      throw error;
-    }
-    if (!llmClient) {
-      throw error;
-    }
-    log.info("Action failed, attempting popup recovery", {
-      error: error instanceof Error ? error.message : String(error)
-    });
-    await (0, import_agent.executeRecoveryAgent)(
-      page,
-      "Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
-      log,
-      llmClient
-    );
-    return await fn();
-  }
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  attemptWithRecovery
-});

package/dist/runtime/recovery/recovery.d.cts

@@ -1,12 +0,0 @@
-import { Page } from 'playwright';
-import { MinimalLogger } from '../../shared/logger/logger.cjs';
-import { LLMClient } from '../../shared/llm/types.cjs';
-import 'zod';
-
-/**
- * Attempts to execute a function, and if it fails, runs popup recovery
- * (if an LLM client is provided) and retries the function once.
- */
-declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger?: MinimalLogger, llmClient?: LLMClient): Promise<T>;
-
-export { attemptWithRecovery };

package/dist/runtime/recovery/recovery.d.ts

@@ -1,12 +0,0 @@
-import { Page } from 'playwright';
-import { MinimalLogger } from '../../shared/logger/logger.js';
-import { LLMClient } from '../../shared/llm/types.js';
-import 'zod';
-
-/**
- * Attempts to execute a function, and if it fails, runs popup recovery
- * (if an LLM client is provided) and retries the function once.
- */
-declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger?: MinimalLogger, llmClient?: LLMClient): Promise<T>;
-
-export { attemptWithRecovery };

package/dist/runtime/recovery/recovery.js

@@ -1,31 +0,0 @@
-import { defaultLogger } from "../../shared/logger/logger.js";
-import { executeRecoveryAgent } from "./agent.js";
-async function attemptWithRecovery(page, fn, logger, llmClient) {
-  const log = logger ?? defaultLogger;
-  try {
-    return await fn();
-  } catch (error) {
-    if (error instanceof Error && (error.message.includes("Target closed") || error.message.includes("browser has been closed") || error.message.includes("context or browser has been closed"))) {
-      log.warn("Page/browser has been closed, cannot recover", {
-        error: error.message
-      });
-      throw error;
-    }
-    if (!llmClient) {
-      throw error;
-    }
-    log.info("Action failed, attempting popup recovery", {
-      error: error instanceof Error ? error.message : String(error)
-    });
-    await executeRecoveryAgent(
-      page,
-      "Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
-      log,
-      llmClient
-    );
-    return await fn();
-  }
-}
-export {
-  attemptWithRecovery
-};