libretto 0.6.20 → 0.6.22

package/dist/runtime/recovery/agent.js

@@ -33,6 +33,83 @@ const KEY_MAPPINGS = {
 function mapKeyName(key) {
   return KEY_MAPPINGS[key.toUpperCase()] ?? key;
 }
+function clamp(value, min, max) {
+  return Math.min(Math.max(value, min), max);
+}
+function scalePoint(x, y, scale) {
+  return {
+    x: clamp(x * scale.scaleX, 0, Math.max(scale.viewportWidth - 1, 0)),
+    y: clamp(y * scale.scaleY, 0, Math.max(scale.viewportHeight - 1, 0))
+  };
+}
+function scaleBrowserAction(action, scale) {
+  switch (action.type) {
+    case "click": {
+      const point = scalePoint(action.x, action.y, scale);
+      return { ...action, ...point };
+    }
+    case "double_click": {
+      const point = scalePoint(action.x, action.y, scale);
+      return { ...action, ...point };
+    }
+    case "scroll": {
+      const point = scalePoint(action.x, action.y, scale);
+      return {
+        ...action,
+        ...point,
+        scroll_x: action.scroll_x * scale.scaleX,
+        scroll_y: action.scroll_y * scale.scaleY
+      };
+    }
+    case "drag":
+      return {
+        ...action,
+        path: action.path.map((point) => scalePoint(point.x, point.y, scale))
+      };
+    case "move": {
+      const point = scalePoint(action.x, action.y, scale);
+      return { ...action, ...point };
+    }
+    case "keypress":
+    case "type":
+    case "wait":
+    case "screenshot":
+    case "done":
+      return action;
+  }
+}
+function readPngDimensions(buffer) {
+  const pngSignature = "89504e470d0a1a0a";
+  if (buffer.subarray(0, 8).toString("hex") !== pngSignature) {
+    throw new Error("Recovery screenshot is not a PNG image.");
+  }
+  return {
+    width: buffer.readUInt32BE(16),
+    height: buffer.readUInt32BE(20)
+  };
+}
+async function takeViewportScreenshot(page) {
+  const viewport = page.viewportSize();
+  if (!viewport) {
+    throw new Error("Viewport size not found");
+  }
+  const screenshot = await page.screenshot({
+    fullPage: false,
+    scale: "css",
+    timeout: 1e4
+  });
+  const dimensions = readPngDimensions(screenshot);
+  return {
+    screenshot,
+    dimensions,
+    scale: {
+      scaleX: viewport.width / dimensions.width,
+      scaleY: viewport.height / dimensions.height,
+      viewportWidth: viewport.width,
+      viewportHeight: viewport.height
+    }
+  };
+}
 async function executeBrowserAction(page, action, logger = defaultLogger) {
   switch (action.type) {
     case "click": {
@@ -109,56 +186,79 @@ async function executeBrowserAction(page, action, logger = defaultLogger) {
 import { z } from "zod";
 const recoveryActionSchema = z.object({
   reasoning: z.string().describe("Your reasoning about what you see and what action to take"),
-  action: z.discriminatedUnion("type", [
-    z.object({
-      type: z.literal("click"),
-      x: z.number(),
-      y: z.number()
-    }),
-    z.object({
-      type: z.literal("type"),
-      text: z.string()
-    }),
-    z.object({
-      type: z.literal("keypress"),
-      keys: z.array(z.string())
-    }),
-    z.object({
-      type: z.literal("scroll"),
-      x: z.number(),
-      y: z.number(),
-      scroll_x: z.number(),
-      scroll_y: z.number()
-    }),
-    z.object({
-      type: z.literal("wait")
-    }),
-    z.object({
-      type: z.literal("done")
-    })
-  ])
+  action: z.object({
+    type: z.enum(["click", "type", "keypress", "scroll", "wait", "done"]).describe("The browser action to execute."),
+    x: z.number().nullable().describe("The screenshot pixel x coordinate for click/scroll."),
+    y: z.number().nullable().describe("The screenshot pixel y coordinate for click/scroll."),
+    text: z.string().nullable().describe("Text for type actions."),
+    keys: z.array(z.string()).nullable().describe("Keys for keypress actions."),
+    scroll_x: z.number().nullable().describe("Horizontal scroll delta."),
+    scroll_y: z.number().nullable().describe("Vertical scroll delta.")
+  })
 });
-async function executeRecoveryAgent(page, instruction, logger, model) {
+function numberOrThrow(value, field) {
+  if (typeof value === "number") return value;
+  throw new Error(`Recovery action is missing ${field}.`);
+}
+function normalizeRecoveryAction(action) {
+  switch (action.type) {
+    case "click":
+      return {
+        type: "click",
+        x: numberOrThrow(action.x, "x"),
+        y: numberOrThrow(action.y, "y")
+      };
+    case "type":
+      return { type: "type", text: action.text ?? "" };
+    case "keypress":
+      return { type: "keypress", keys: action.keys ?? [] };
+    case "scroll":
+      return {
+        type: "scroll",
+        x: numberOrThrow(action.x, "x"),
+        y: numberOrThrow(action.y, "y"),
+        scroll_x: numberOrThrow(action.scroll_x, "scroll_x"),
+        scroll_y: numberOrThrow(action.scroll_y, "scroll_y")
+      };
+    case "wait":
+      return { type: "wait" };
+    case "done":
+      return { type: "done" };
+  }
+}
+function getRecoveryStatus(steps) {
+  if (steps.length === 0) {
+    return "skipped";
+  }
+  const actionSteps = steps.filter((step) => step.action.type !== "done");
+  const completed = steps.at(-1)?.action.type === "done";
+  if (actionSteps.length === 0 && completed) {
+    return "no-action-needed";
+  }
+  if (completed) {
+    return "action-taken";
+  }
+  return "incomplete";
+}
+const DEFAULT_RECOVERY_MAX_STEPS = 3;
+async function executeRecoveryAgent(page, instruction, logger, model, maxSteps = DEFAULT_RECOVERY_MAX_STEPS) {
   if (!model) {
-    return;
+    return { status: "skipped", steps: [] };
   }
   const log = logger ?? defaultLogger;
   log.info("Executing vision-based recovery agent", { instruction });
-  const viewport = page.viewportSize();
-  if (!viewport) {
-    throw new Error("Viewport size not found");
-  }
-  let screenshot;
+  let screenshotState;
   try {
-    screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
+    screenshotState = await takeViewportScreenshot(page);
   } catch (screenshotError) {
     log.warn("Failed to take screenshot for recovery agent, skipping", {
       screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
     });
     throw new Error("Failed to take screenshot for recovery agent");
   }
-  const maxSteps = 3;
+  const steps = [];
   for (let step = 1; step <= maxSteps; step++) {
+    const { screenshot, dimensions, scale } = screenshotState;
     const { object: result } = await generateObject({
       model,
       schema: recoveryActionSchema,
@@ -172,33 +272,47 @@ async function executeRecoveryAgent(page, instruction, logger, model) {
 
 Your task: ${instruction}
 
-Viewport: ${viewport.width}x${viewport.height}px. Complete this in as few steps as possible.
+Screenshot: ${dimensions.width}x${dimensions.height}px. Coordinates must be screenshot pixel coordinates relative to the top-left corner of the screenshot. Complete this in as few steps as possible.
 Analyze the screenshot and decide what action to take. If the task is complete or no action is needed, use the "done" action type.`
             },
             {
               type: "image",
-              image: `data:image/png;base64,${screenshot}`
+              image: screenshot
             }
           ]
         }
       ],
       temperature: 0
     });
+    const imageAction = normalizeRecoveryAction(result.action);
+    const action = scaleBrowserAction(imageAction, scale);
     log.info(`Recovery step ${step}/${maxSteps}`, {
       reasoning: result.reasoning,
-      action: result.action
+      imageAction,
+      action,
+      screenshot: dimensions,
+      scale
+    });
+    steps.push({
+      step,
+      reasoning: result.reasoning,
+      action
     });
-    if (result.action.type === "done") {
+    if (action.type === "done") {
       log.info("Recovery agent completed - no more actions needed");
       break;
     }
-    await executeBrowserAction(page, result.action, log);
+    await executeBrowserAction(page, action, log);
     await delay(2e3);
-    screenshot = (await page.screenshot({ fullPage: false })).toString(
-      "base64"
-    );
+    if (step < maxSteps) {
+      screenshotState = await takeViewportScreenshot(page);
+    }
   }
   log.info("Recovery agent execution completed");
+  return {
+    status: getRecoveryStatus(steps),
+    steps
+  };
 }
 export {
   executeRecoveryAgent

package/dist/runtime/recovery/index.d.ts

@@ -1,6 +1,7 @@
-export { executeRecoveryAgent } from './agent.js';
+export { BrowserAction, RecoveryAgentResult, RecoveryAgentStep, executeRecoveryAgent } from './agent.js';
 export { attemptWithRecovery } from './recovery.js';
 export { DetectedSubmissionError, KnownSubmissionError, detectSubmissionError } from './errors.js';
+export { COMPUTER_USE_RECOVERY_MODELS, ComputerUseRecoveryActionOptions, POPUP_RECOVERY_INSTRUCTION, PopupRecoveryActionOptions, RecoveryAction, RecoveryActionContext, RecoveryActionHandler, RecoveryActionOptions, RecoveryActionResult, RecoveryActionTargetType, computerUseRecoveryAction, createRecoveryPage, popupRecoveryAction } from './page-fallbacks.js';
 import 'playwright';
 import '../../shared/logger/logger.js';
 import 'ai';

package/dist/runtime/recovery/index.js

@@ -1,10 +1,24 @@
-import { executeRecoveryAgent } from "./agent.js";
+import {
+  executeRecoveryAgent
+} from "./agent.js";
 import { attemptWithRecovery } from "./recovery.js";
 import {
   detectSubmissionError
 } from "./errors.js";
+import {
+  COMPUTER_USE_RECOVERY_MODELS,
+  POPUP_RECOVERY_INSTRUCTION,
+  computerUseRecoveryAction,
+  createRecoveryPage,
+  popupRecoveryAction
+} from "./page-fallbacks.js";
 export {
+  COMPUTER_USE_RECOVERY_MODELS,
+  POPUP_RECOVERY_INSTRUCTION,
   attemptWithRecovery,
+  computerUseRecoveryAction,
+  createRecoveryPage,
   detectSubmissionError,
-  executeRecoveryAgent
+  executeRecoveryAgent,
+  popupRecoveryAction
 };

package/dist/runtime/recovery/page-fallbacks.d.ts

@@ -0,0 +1,45 @@
+import { Page } from 'playwright';
+import { LanguageModel } from 'ai';
+
+type RecoveryActionTargetType = "page" | "locator";
+type RecoveryActionContext = {
+    page: Page;
+    targetType: RecoveryActionTargetType;
+    method: string;
+    args: readonly unknown[];
+    error: unknown;
+};
+type RecoveryActionResult = Record<string, unknown> | void;
+type RecoveryActionHandler = (context: RecoveryActionContext) => Promise<RecoveryActionResult>;
+type RecoveryAction = RecoveryActionHandler;
+type RecoveryActionOptions = {
+    recoveryAction: RecoveryAction;
+};
+type ComputerUseRecoveryModelOptions = {
+    languageModel: LanguageModel;
+} | {
+    provider: "openai";
+    apiKey: string;
+    model?: "gpt-5.5";
+} | {
+    provider: "anthropic";
+    apiKey: string;
+    model?: "claude-sonnet-4-6";
+};
+type ComputerUseRecoveryActionOptions = ComputerUseRecoveryModelOptions & {
+    instruction: string;
+    maxSteps?: number;
+};
+type PopupRecoveryActionOptions = ComputerUseRecoveryModelOptions & {
+    maxSteps?: number;
+};
+declare const POPUP_RECOVERY_INSTRUCTION: string;
+declare const COMPUTER_USE_RECOVERY_MODELS: {
+    readonly anthropic: "claude-sonnet-4-6";
+    readonly openai: "gpt-5.5";
+};
+declare function createRecoveryPage(page: Page, options: RecoveryActionOptions): Page;
+declare function computerUseRecoveryAction(options: ComputerUseRecoveryActionOptions): RecoveryAction;
+declare function popupRecoveryAction(options: PopupRecoveryActionOptions): RecoveryAction;
+
+export { COMPUTER_USE_RECOVERY_MODELS, type ComputerUseRecoveryActionOptions, POPUP_RECOVERY_INSTRUCTION, type PopupRecoveryActionOptions, type RecoveryAction, type RecoveryActionContext, type RecoveryActionHandler, type RecoveryActionOptions, type RecoveryActionResult, type RecoveryActionTargetType, computerUseRecoveryAction, createRecoveryPage, popupRecoveryAction };

package/dist/runtime/recovery/page-fallbacks.js

@@ -0,0 +1,342 @@
+import { executeRecoveryAgent } from "./agent.js";
+const POPUP_RECOVERY_INSTRUCTION = [
+  "Look at the page for any popup, modal, cookie banner, overlay, dialog, or interstitial that blocks interaction.",
+  "If any blocking popup is visible, close it before returning done.",
+  "Prefer obvious close, dismiss, continue, accept, or X buttons.",
+  "Do not return done while a blocking overlay or dialog is still visible."
+].join(" ");
+const COMPUTER_USE_RECOVERY_MODELS = {
+  anthropic: "claude-sonnet-4-6",
+  openai: "gpt-5.5"
+};
+const PAGE_UI_METHODS = /* @__PURE__ */ new Set([
+  "click",
+  "dblclick",
+  "tap",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "pressSequentially",
+  "check",
+  "uncheck",
+  "setChecked",
+  "selectOption",
+  "setInputFiles",
+  "selectText",
+  "dispatchEvent",
+  "focus",
+  "blur",
+  "dragAndDrop"
+]);
+const PAGE_READ_METHODS = /* @__PURE__ */ new Set([
+  "title",
+  "content",
+  "screenshot",
+  "waitForLoadState",
+  "waitForRequest",
+  "waitForResponse",
+  "waitForURL"
+]);
+const LOCATOR_UI_METHODS = /* @__PURE__ */ new Set([
+  "click",
+  "dblclick",
+  "tap",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "pressSequentially",
+  "check",
+  "uncheck",
+  "setChecked",
+  "selectOption",
+  "setInputFiles",
+  "selectText",
+  "dispatchEvent",
+  "focus",
+  "blur",
+  "clear",
+  "dragTo",
+  "scrollIntoViewIfNeeded"
+]);
+const LOCATOR_READ_METHODS = /* @__PURE__ */ new Set([
+  "textContent",
+  "innerText",
+  "innerHTML",
+  "allTextContents",
+  "allInnerTexts",
+  "ariaSnapshot",
+  "boundingBox",
+  "count",
+  "getAttribute",
+  "inputValue",
+  "isChecked",
+  "isDisabled",
+  "isEditable",
+  "isEnabled",
+  "isVisible",
+  "isHidden",
+  "screenshot",
+  "waitFor"
+]);
+const PAGE_LOCATOR_FACTORY_METHODS = /* @__PURE__ */ new Set([
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId"
+]);
+const LOCATOR_FACTORY_METHODS = /* @__PURE__ */ new Set([
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+  "filter",
+  "and",
+  "or",
+  "first",
+  "last",
+  "nth"
+]);
+const FRAME_LOCATOR_FACTORY_METHODS = /* @__PURE__ */ new Set([
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+  "owner",
+  "first",
+  "last",
+  "nth",
+  "frameLocator"
+]);
+function isUiMethod(targetType, method) {
+  return targetType === "page" ? PAGE_UI_METHODS.has(method) : LOCATOR_UI_METHODS.has(method);
+}
+function isReadMethod(targetType, method) {
+  return targetType === "page" ? PAGE_READ_METHODS.has(method) : LOCATOR_READ_METHODS.has(method);
+}
+function isSupportedMethod(targetType, method) {
+  return isUiMethod(targetType, method) || isReadMethod(targetType, method);
+}
+async function runWithFallback(args) {
+  try {
+    return await args.invoke();
+  } catch (originalError) {
+    const baseContext = {
+      page: args.page,
+      targetType: args.targetType,
+      method: args.method,
+      args: args.methodArgs
+    };
+    if (!isSupportedMethod(baseContext.targetType, baseContext.method)) {
+      throw originalError;
+    }
+    try {
+      await args.options.recoveryAction({
+        ...baseContext,
+        error: originalError
+      });
+      return await args.invoke();
+    } catch {
+      throw originalError;
+    }
+  }
+}
+function bindOrWrapLocatorMethod(locator, rawPage, method, value, options, caches) {
+  if (typeof value !== "function") return value;
+  if (LOCATOR_FACTORY_METHODS.has(method)) {
+    return (...args) => {
+      const nextLocator = value.apply(locator, args);
+      return createFallbackLocator(nextLocator, rawPage, options, caches);
+    };
+  }
+  if (method === "all") {
+    return async (...args) => {
+      const locators = await value.apply(locator, args);
+      return locators.map(
+        (nextLocator) => createFallbackLocator(nextLocator, rawPage, options, caches)
+      );
+    };
+  }
+  if (method === "contentFrame") {
+    return (...args) => {
+      const frameLocator = value.apply(locator, args);
+      return createFallbackFrameLocator(frameLocator, rawPage, options, caches);
+    };
+  }
+  if (!isSupportedMethod("locator", method)) {
+    return value.bind(locator);
+  }
+  return (...args) => runWithFallback({
+    page: rawPage,
+    targetType: "locator",
+    method,
+    methodArgs: args,
+    invoke: () => value.apply(locator, args),
+    options
+  });
+}
+function createFallbackLocator(locator, rawPage, options, caches) {
+  const cached = caches.locators.get(locator);
+  if (cached) return cached;
+  const proxy = new Proxy(locator, {
+    get(target, prop, receiver) {
+      if (typeof prop !== "string") {
+        return Reflect.get(target, prop, receiver);
+      }
+      return bindOrWrapLocatorMethod(
+        target,
+        rawPage,
+        prop,
+        Reflect.get(target, prop, target),
+        options,
+        caches
+      );
+    }
+  });
+  caches.locators.set(locator, proxy);
+  return proxy;
+}
+function createFallbackFrameLocator(frameLocator, rawPage, options, caches) {
+  const cached = caches.frameLocators.get(frameLocator);
+  if (cached) return cached;
+  const proxy = new Proxy(frameLocator, {
+    get(target, prop, receiver) {
+      if (typeof prop !== "string") {
+        return Reflect.get(target, prop, receiver);
+      }
+      const value = Reflect.get(target, prop, target);
+      if (typeof value !== "function") return value;
+      if (FRAME_LOCATOR_FACTORY_METHODS.has(prop)) {
+        return (...args) => {
+          const result = value.apply(target, args);
+          if (prop === "first" || prop === "last" || prop === "nth") {
+            return createFallbackFrameLocator(
+              result,
+              rawPage,
+              options,
+              caches
+            );
+          }
+          if (prop === "frameLocator") {
+            return createFallbackFrameLocator(
+              result,
+              rawPage,
+              options,
+              caches
+            );
+          }
+          return createFallbackLocator(
+            result,
+            rawPage,
+            options,
+            caches
+          );
+        };
+      }
+      return value.bind(target);
+    }
+  });
+  caches.frameLocators.set(frameLocator, proxy);
+  return proxy;
+}
+function createRecoveryPage(page, options) {
+  const caches = {
+    locators: /* @__PURE__ */ new WeakMap(),
+    frameLocators: /* @__PURE__ */ new WeakMap()
+  };
+  return new Proxy(page, {
+    get(target, prop, receiver) {
+      if (typeof prop !== "string") {
+        return Reflect.get(target, prop, receiver);
+      }
+      const value = Reflect.get(target, prop, target);
+      if (typeof value !== "function") return value;
+      if (PAGE_LOCATOR_FACTORY_METHODS.has(prop)) {
+        return (...args) => {
+          const locator = value.apply(target, args);
+          return createFallbackLocator(locator, page, options, caches);
+        };
+      }
+      if (prop === "frameLocator") {
+        return (...args) => {
+          const frameLocator = value.apply(target, args);
+          return createFallbackFrameLocator(frameLocator, page, options, caches);
+        };
+      }
+      if (!isSupportedMethod("page", prop)) {
+        return value.bind(target);
+      }
+      return (...args) => runWithFallback({
+        page,
+        targetType: "page",
+        method: prop,
+        methodArgs: args,
+        invoke: () => value.apply(target, args),
+        options
+      });
+    }
+  });
+}
+async function resolveComputerUseRecoveryModel(options) {
+  if ("provider" in options) {
+    if (options.provider === "openai") {
+      const model2 = options.model ?? COMPUTER_USE_RECOVERY_MODELS.openai;
+      if (model2 !== COMPUTER_USE_RECOVERY_MODELS.openai) {
+        throw new Error(
+          `Unsupported OpenAI computer use recovery model "${model2}". Supported model: ${COMPUTER_USE_RECOVERY_MODELS.openai}.`
+        );
+      }
+      return import("@ai-sdk/openai").then(
+        ({ createOpenAI }) => createOpenAI({ apiKey: options.apiKey })(model2)
+      );
+    }
+    const model = options.model ?? COMPUTER_USE_RECOVERY_MODELS.anthropic;
+    if (model !== COMPUTER_USE_RECOVERY_MODELS.anthropic) {
+      throw new Error(
+        `Unsupported Anthropic computer use recovery model "${model}". Supported model: ${COMPUTER_USE_RECOVERY_MODELS.anthropic}.`
+      );
+    }
+    return import("@ai-sdk/anthropic").then(
+      ({ createAnthropic }) => createAnthropic({ apiKey: options.apiKey })(model)
+    );
+  }
+  return options.languageModel;
+}
+function computerUseRecoveryAction(options) {
+  return async ({ page }) => {
+    const model = await resolveComputerUseRecoveryModel(options);
+    return executeRecoveryAgent(
+      page,
+      options.instruction,
+      void 0,
+      model,
+      options.maxSteps
+    );
+  };
+}
+function popupRecoveryAction(options) {
+  return computerUseRecoveryAction({
+    ...options,
+    instruction: POPUP_RECOVERY_INSTRUCTION
+  });
+}
+export {
+  COMPUTER_USE_RECOVERY_MODELS,
+  POPUP_RECOVERY_INSTRUCTION,
+  computerUseRecoveryAction,
+  createRecoveryPage,
+  popupRecoveryAction
+};

package/dist/shared/state/index.d.ts

@@ -1,2 +1,2 @@
-export { SESSION_STATE_VERSION, SessionAccessMode, SessionAccessModeSchema, SessionState, SessionStateFile, SessionStateFileSchema, SessionStatus, SessionStatusSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState } from './session-state.js';
+export { ProviderState, SESSION_STATE_VERSION, SessionAccessMode, SessionAccessModeSchema, SessionState, SessionStateFile, SessionStateFileSchema, SessionStatus, SessionStatusSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState } from './session-state.js';
 import 'zod';