libretto 0.6.21 → 0.6.23

package/dist/runtime/recovery/agent.js

@@ -33,6 +33,83 @@ const KEY_MAPPINGS = {
 function mapKeyName(key) {
   return KEY_MAPPINGS[key.toUpperCase()] ?? key;
 }
+function clamp(value, min, max) {
+  return Math.min(Math.max(value, min), max);
+}
+function scalePoint(x, y, scale) {
+  return {
+    x: clamp(x * scale.scaleX, 0, Math.max(scale.viewportWidth - 1, 0)),
+    y: clamp(y * scale.scaleY, 0, Math.max(scale.viewportHeight - 1, 0))
+  };
+}
+function scaleBrowserAction(action, scale) {
+  switch (action.type) {
+    case "click": {
+      const point = scalePoint(action.x, action.y, scale);
+      return { ...action, ...point };
+    }
+    case "double_click": {
+      const point = scalePoint(action.x, action.y, scale);
+      return { ...action, ...point };
+    }
+    case "scroll": {
+      const point = scalePoint(action.x, action.y, scale);
+      return {
+        ...action,
+        ...point,
+        scroll_x: action.scroll_x * scale.scaleX,
+        scroll_y: action.scroll_y * scale.scaleY
+      };
+    }
+    case "drag":
+      return {
+        ...action,
+        path: action.path.map((point) => scalePoint(point.x, point.y, scale))
+      };
+    case "move": {
+      const point = scalePoint(action.x, action.y, scale);
+      return { ...action, ...point };
+    }
+    case "keypress":
+    case "type":
+    case "wait":
+    case "screenshot":
+    case "done":
+      return action;
+  }
+}
+function readPngDimensions(buffer) {
+  const pngSignature = "89504e470d0a1a0a";
+  if (buffer.subarray(0, 8).toString("hex") !== pngSignature) {
+    throw new Error("Recovery screenshot is not a PNG image.");
+  }
+  return {
+    width: buffer.readUInt32BE(16),
+    height: buffer.readUInt32BE(20)
+  };
+}
+async function takeViewportScreenshot(page) {
+  const viewport = page.viewportSize();
+  if (!viewport) {
+    throw new Error("Viewport size not found");
+  }
+  const screenshot = await page.screenshot({
+    fullPage: false,
+    scale: "css",
+    timeout: 1e4
+  });
+  const dimensions = readPngDimensions(screenshot);
+  return {
+    screenshot,
+    dimensions,
+    scale: {
+      scaleX: viewport.width / dimensions.width,
+      scaleY: viewport.height / dimensions.height,
+      viewportWidth: viewport.width,
+      viewportHeight: viewport.height
+    }
+  };
+}
 async function executeBrowserAction(page, action, logger = defaultLogger) {
   switch (action.type) {
     case "click": {
@@ -109,56 +186,79 @@ async function executeBrowserAction(page, action, logger = defaultLogger) {
 import { z } from "zod";
 const recoveryActionSchema = z.object({
   reasoning: z.string().describe("Your reasoning about what you see and what action to take"),
-  action: z.discriminatedUnion("type", [
-    z.object({
-      type: z.literal("click"),
-      x: z.number(),
-      y: z.number()
-    }),
-    z.object({
-      type: z.literal("type"),
-      text: z.string()
-    }),
-    z.object({
-      type: z.literal("keypress"),
-      keys: z.array(z.string())
-    }),
-    z.object({
-      type: z.literal("scroll"),
-      x: z.number(),
-      y: z.number(),
-      scroll_x: z.number(),
-      scroll_y: z.number()
-    }),
-    z.object({
-      type: z.literal("wait")
-    }),
-    z.object({
-      type: z.literal("done")
-    })
-  ])
+  action: z.object({
+    type: z.enum(["click", "type", "keypress", "scroll", "wait", "done"]).describe("The browser action to execute."),
+    x: z.number().nullable().describe("The screenshot pixel x coordinate for click/scroll."),
+    y: z.number().nullable().describe("The screenshot pixel y coordinate for click/scroll."),
+    text: z.string().nullable().describe("Text for type actions."),
+    keys: z.array(z.string()).nullable().describe("Keys for keypress actions."),
+    scroll_x: z.number().nullable().describe("Horizontal scroll delta."),
+    scroll_y: z.number().nullable().describe("Vertical scroll delta.")
+  })
 });
-async function executeRecoveryAgent(page, instruction, logger, model) {
+function numberOrThrow(value, field) {
+  if (typeof value === "number") return value;
+  throw new Error(`Recovery action is missing ${field}.`);
+}
+function normalizeRecoveryAction(action) {
+  switch (action.type) {
+    case "click":
+      return {
+        type: "click",
+        x: numberOrThrow(action.x, "x"),
+        y: numberOrThrow(action.y, "y")
+      };
+    case "type":
+      return { type: "type", text: action.text ?? "" };
+    case "keypress":
+      return { type: "keypress", keys: action.keys ?? [] };
+    case "scroll":
+      return {
+        type: "scroll",
+        x: numberOrThrow(action.x, "x"),
+        y: numberOrThrow(action.y, "y"),
+        scroll_x: numberOrThrow(action.scroll_x, "scroll_x"),
+        scroll_y: numberOrThrow(action.scroll_y, "scroll_y")
+      };
+    case "wait":
+      return { type: "wait" };
+    case "done":
+      return { type: "done" };
+  }
+}
+function getRecoveryStatus(steps) {
+  if (steps.length === 0) {
+    return "skipped";
+  }
+  const actionSteps = steps.filter((step) => step.action.type !== "done");
+  const completed = steps.at(-1)?.action.type === "done";
+  if (actionSteps.length === 0 && completed) {
+    return "no-action-needed";
+  }
+  if (completed) {
+    return "action-taken";
+  }
+  return "incomplete";
+}
+const DEFAULT_RECOVERY_MAX_STEPS = 3;
+async function executeRecoveryAgent(page, instruction, logger, model, maxSteps = DEFAULT_RECOVERY_MAX_STEPS) {
   if (!model) {
-    return;
+    return { status: "skipped", steps: [] };
   }
   const log = logger ?? defaultLogger;
   log.info("Executing vision-based recovery agent", { instruction });
-  const viewport = page.viewportSize();
-  if (!viewport) {
-    throw new Error("Viewport size not found");
-  }
-  let screenshot;
+  let screenshotState;
   try {
-    screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
+    screenshotState = await takeViewportScreenshot(page);
   } catch (screenshotError) {
     log.warn("Failed to take screenshot for recovery agent, skipping", {
       screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
     });
     throw new Error("Failed to take screenshot for recovery agent");
   }
-  const maxSteps = 3;
+  const steps = [];
   for (let step = 1; step <= maxSteps; step++) {
+    const { screenshot, dimensions, scale } = screenshotState;
     const { object: result } = await generateObject({
       model,
       schema: recoveryActionSchema,
@@ -172,33 +272,47 @@ async function executeRecoveryAgent(page, instruction, logger, model) {
 
 Your task: ${instruction}
 
-Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
+Screenshot: ${dimensions.width}x${dimensions.height}px. Coordinates must be screenshot pixel coordinates relative to the top-left corner of the screenshot. Complete this in as few steps as possible.
 Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`
             },
             {
               type: "image",
-              image: `data:image/png;base64,${screenshot}`
+              image: screenshot
             }
           ]
         }
       ],
       temperature: 0
     });
+    const imageAction = normalizeRecoveryAction(result.action);
+    const action = scaleBrowserAction(imageAction, scale);
     log.info(`Recovery step ${step}/${maxSteps}`, {
       reasoning: result.reasoning,
-      action: result.action
+      imageAction,
+      action,
+      screenshot: dimensions,
+      scale
+    });
+    steps.push({
+      step,
+      reasoning: result.reasoning,
+      action
     });
-    if (result.action.type === "done") {
+    if (action.type === "done") {
       log.info("Recovery agent completed - no more actions needed");
       break;
     }
-    await executeBrowserAction(page, result.action, log);
+    await executeBrowserAction(page, action, log);
     await delay(2e3);
-    screenshot = (await page.screenshot({ fullPage: false })).toString(
-      "base64"
-    );
+    if (step < maxSteps) {
+      screenshotState = await takeViewportScreenshot(page);
+    }
   }
   log.info("Recovery agent execution completed");
+  return {
+    status: getRecoveryStatus(steps),
+    steps
+  };
 }
 export {
   executeRecoveryAgent

package/dist/runtime/recovery/index.d.ts

@@ -1,6 +1,7 @@
-export { executeRecoveryAgent } from './agent.js';
+export { BrowserAction, RecoveryAgentResult, RecoveryAgentStep, executeRecoveryAgent } from './agent.js';
 export { attemptWithRecovery } from './recovery.js';
 export { DetectedSubmissionError, KnownSubmissionError, detectSubmissionError } from './errors.js';
+export { COMPUTER_USE_RECOVERY_MODELS, ComputerUseRecoveryActionOptions, POPUP_RECOVERY_INSTRUCTION, PopupRecoveryActionOptions, RecoveryAction, RecoveryActionContext, RecoveryActionHandler, RecoveryActionOptions, RecoveryActionResult, RecoveryActionTargetType, computerUseRecoveryAction, createRecoveryPage, popupRecoveryAction } from './page-fallbacks.js';
 import 'playwright';
 import '../../shared/logger/logger.js';
 import 'ai';

package/dist/runtime/recovery/index.js

@@ -1,10 +1,24 @@
-import { executeRecoveryAgent } from "./agent.js";
+import {
+  executeRecoveryAgent
+} from "./agent.js";
 import { attemptWithRecovery } from "./recovery.js";
 import {
   detectSubmissionError
 } from "./errors.js";
+import {
+  COMPUTER_USE_RECOVERY_MODELS,
+  POPUP_RECOVERY_INSTRUCTION,
+  computerUseRecoveryAction,
+  createRecoveryPage,
+  popupRecoveryAction
+} from "./page-fallbacks.js";
 export {
+  COMPUTER_USE_RECOVERY_MODELS,
+  POPUP_RECOVERY_INSTRUCTION,
   attemptWithRecovery,
+  computerUseRecoveryAction,
+  createRecoveryPage,
   detectSubmissionError,
-  executeRecoveryAgent
+  executeRecoveryAgent,
+  popupRecoveryAction
 };

package/dist/runtime/recovery/page-fallbacks.d.ts

@@ -0,0 +1,45 @@
+import { Page } from 'playwright';
+import { LanguageModel } from 'ai';
+
+type RecoveryActionTargetType = "page" | "locator";
+type RecoveryActionContext = {
+    page: Page;
+    targetType: RecoveryActionTargetType;
+    method: string;
+    args: readonly unknown[];
+    error: unknown;
+};
+type RecoveryActionResult = Record<string, unknown> | void;
+type RecoveryActionHandler = (context: RecoveryActionContext) => Promise<RecoveryActionResult>;
+type RecoveryAction = RecoveryActionHandler;
+type RecoveryActionOptions = {
+    recoveryAction: RecoveryAction;
+};
+type ComputerUseRecoveryModelOptions = {
+    languageModel: LanguageModel;
+} | {
+    provider: "openai";
+    apiKey: string;
+    model?: "gpt-5.5";
+} | {
+    provider: "anthropic";
+    apiKey: string;
+    model?: "claude-sonnet-4-6";
+};
+type ComputerUseRecoveryActionOptions = ComputerUseRecoveryModelOptions & {
+    instruction: string;
+    maxSteps?: number;
+};
+type PopupRecoveryActionOptions = ComputerUseRecoveryModelOptions & {
+    maxSteps?: number;
+};
+declare const POPUP_RECOVERY_INSTRUCTION: string;
+declare const COMPUTER_USE_RECOVERY_MODELS: {
+    readonly anthropic: "claude-sonnet-4-6";
+    readonly openai: "gpt-5.5";
+};
+declare function createRecoveryPage(page: Page, options: RecoveryActionOptions): Page;
+declare function computerUseRecoveryAction(options: ComputerUseRecoveryActionOptions): RecoveryAction;
+declare function popupRecoveryAction(options: PopupRecoveryActionOptions): RecoveryAction;
+
+export { COMPUTER_USE_RECOVERY_MODELS, type ComputerUseRecoveryActionOptions, POPUP_RECOVERY_INSTRUCTION, type PopupRecoveryActionOptions, type RecoveryAction, type RecoveryActionContext, type RecoveryActionHandler, type RecoveryActionOptions, type RecoveryActionResult, type RecoveryActionTargetType, computerUseRecoveryAction, createRecoveryPage, popupRecoveryAction };