libretto 0.6.21 → 0.6.22

package/src/runtime/recovery/page-fallbacks.ts

@@ -0,0 +1,476 @@
+import type { FrameLocator, Locator, Page } from "playwright";
+import type { LanguageModel } from "ai";
+import { executeRecoveryAgent, type RecoveryAgentResult } from "./agent.js";
+
+export type RecoveryActionTargetType = "page" | "locator";
+
+export type RecoveryActionContext = {
+  page: Page;
+  targetType: RecoveryActionTargetType;
+  method: string;
+  args: readonly unknown[];
+  error: unknown;
+};
+
+export type RecoveryActionResult = Record<string, unknown> | void;
+
+export type RecoveryActionHandler = (
+  context: RecoveryActionContext,
+) => Promise<RecoveryActionResult>;
+
+export type RecoveryAction = RecoveryActionHandler;
+
+export type RecoveryActionOptions = {
+  recoveryAction: RecoveryAction;
+};
+
+type ComputerUseRecoveryModelOptions =
+  | {
+      languageModel: LanguageModel;
+    }
+  | {
+      provider: "openai";
+      apiKey: string;
+      model?: "gpt-5.5";
+    }
+  | {
+      provider: "anthropic";
+      apiKey: string;
+      model?: "claude-sonnet-4-6";
+    };
+
+export type ComputerUseRecoveryActionOptions = ComputerUseRecoveryModelOptions & {
+  instruction: string;
+  maxSteps?: number;
+};
+
+export type PopupRecoveryActionOptions = ComputerUseRecoveryModelOptions & {
+  maxSteps?: number;
+};
+
+export const POPUP_RECOVERY_INSTRUCTION = [
+  "Look at the page for any popup, modal, cookie banner, overlay, dialog, or interstitial that blocks interaction.",
+  "If any blocking popup is visible, close it before returning done.",
+  "Prefer obvious close, dismiss, continue, accept, or X buttons.",
+  "Do not return done while a blocking overlay or dialog is still visible.",
+].join(" ");
+
+export const COMPUTER_USE_RECOVERY_MODELS = {
+  anthropic: "claude-sonnet-4-6",
+  openai: "gpt-5.5",
+} as const;
+
+const PAGE_UI_METHODS = new Set([
+  "click",
+  "dblclick",
+  "tap",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "pressSequentially",
+  "check",
+  "uncheck",
+  "setChecked",
+  "selectOption",
+  "setInputFiles",
+  "selectText",
+  "dispatchEvent",
+  "focus",
+  "blur",
+  "dragAndDrop",
+]);
+
+const PAGE_READ_METHODS = new Set([
+  "title",
+  "content",
+  "screenshot",
+  "waitForLoadState",
+  "waitForRequest",
+  "waitForResponse",
+  "waitForURL",
+]);
+
+const LOCATOR_UI_METHODS = new Set([
+  "click",
+  "dblclick",
+  "tap",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "pressSequentially",
+  "check",
+  "uncheck",
+  "setChecked",
+  "selectOption",
+  "setInputFiles",
+  "selectText",
+  "dispatchEvent",
+  "focus",
+  "blur",
+  "clear",
+  "dragTo",
+  "scrollIntoViewIfNeeded",
+]);
+
+const LOCATOR_READ_METHODS = new Set([
+  "textContent",
+  "innerText",
+  "innerHTML",
+  "allTextContents",
+  "allInnerTexts",
+  "ariaSnapshot",
+  "boundingBox",
+  "count",
+  "getAttribute",
+  "inputValue",
+  "isChecked",
+  "isDisabled",
+  "isEditable",
+  "isEnabled",
+  "isVisible",
+  "isHidden",
+  "screenshot",
+  "waitFor",
+]);
+
+const PAGE_LOCATOR_FACTORY_METHODS = new Set([
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+]);
+
+const LOCATOR_FACTORY_METHODS = new Set([
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+  "filter",
+  "and",
+  "or",
+  "first",
+  "last",
+  "nth",
+]);
+
+const FRAME_LOCATOR_FACTORY_METHODS = new Set([
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+  "owner",
+  "first",
+  "last",
+  "nth",
+  "frameLocator",
+]);
+
+function isUiMethod(
+  targetType: RecoveryActionTargetType,
+  method: string,
+): boolean {
+  return targetType === "page"
+    ? PAGE_UI_METHODS.has(method)
+    : LOCATOR_UI_METHODS.has(method);
+}
+
+function isReadMethod(
+  targetType: RecoveryActionTargetType,
+  method: string,
+): boolean {
+  return targetType === "page"
+    ? PAGE_READ_METHODS.has(method)
+    : LOCATOR_READ_METHODS.has(method);
+}
+
+function isSupportedMethod(
+  targetType: RecoveryActionTargetType,
+  method: string,
+): boolean {
+  return isUiMethod(targetType, method) || isReadMethod(targetType, method);
+}
+
+async function runWithFallback<T>(args: {
+  page: Page;
+  targetType: RecoveryActionTargetType;
+  method: string;
+  methodArgs: readonly unknown[];
+  invoke: () => T | Promise<T>;
+  options: RecoveryActionOptions;
+}): Promise<T> {
+  try {
+    return await args.invoke();
+  } catch (originalError) {
+    const baseContext = {
+      page: args.page,
+      targetType: args.targetType,
+      method: args.method,
+      args: args.methodArgs,
+    } as const;
+    if (!isSupportedMethod(baseContext.targetType, baseContext.method)) {
+      throw originalError;
+    }
+
+    try {
+      await args.options.recoveryAction({
+        ...baseContext,
+        error: originalError,
+      });
+      return await args.invoke();
+    } catch {
+      throw originalError;
+    }
+  }
+}
+
+type ProxyCaches = {
+  locators: WeakMap<Locator, Locator>;
+  frameLocators: WeakMap<FrameLocator, FrameLocator>;
+};
+
+function bindOrWrapLocatorMethod(
+  locator: Locator,
+  rawPage: Page,
+  method: string,
+  value: unknown,
+  options: RecoveryActionOptions,
+  caches: ProxyCaches,
+): unknown {
+  if (typeof value !== "function") return value;
+
+  if (LOCATOR_FACTORY_METHODS.has(method)) {
+    return (...args: unknown[]) => {
+      const nextLocator = value.apply(locator, args) as Locator;
+      return createFallbackLocator(nextLocator, rawPage, options, caches);
+    };
+  }
+
+  if (method === "all") {
+    return async (...args: unknown[]) => {
+      const locators = (await value.apply(locator, args)) as Locator[];
+      return locators.map((nextLocator) =>
+        createFallbackLocator(nextLocator, rawPage, options, caches),
+      );
+    };
+  }
+
+  if (method === "contentFrame") {
+    return (...args: unknown[]) => {
+      const frameLocator = value.apply(locator, args) as FrameLocator;
+      return createFallbackFrameLocator(frameLocator, rawPage, options, caches);
+    };
+  }
+
+  if (!isSupportedMethod("locator", method)) {
+    return value.bind(locator);
+  }
+
+  return (...args: unknown[]) =>
+    runWithFallback({
+      page: rawPage,
+      targetType: "locator",
+      method,
+      methodArgs: args,
+      invoke: () => value.apply(locator, args),
+      options,
+    });
+}
+
+function createFallbackLocator(
+  locator: Locator,
+  rawPage: Page,
+  options: RecoveryActionOptions,
+  caches: ProxyCaches,
+): Locator {
+  const cached = caches.locators.get(locator);
+  if (cached) return cached;
+
+  const proxy = new Proxy(locator, {
+    get(target, prop, receiver) {
+      if (typeof prop !== "string") {
+        return Reflect.get(target, prop, receiver);
+      }
+      return bindOrWrapLocatorMethod(
+        target,
+        rawPage,
+        prop,
+        Reflect.get(target, prop, target),
+        options,
+        caches,
+      );
+    },
+  }) as Locator;
+
+  caches.locators.set(locator, proxy);
+  return proxy;
+}
+
+function createFallbackFrameLocator(
+  frameLocator: FrameLocator,
+  rawPage: Page,
+  options: RecoveryActionOptions,
+  caches: ProxyCaches,
+): FrameLocator {
+  const cached = caches.frameLocators.get(frameLocator);
+  if (cached) return cached;
+
+  const proxy = new Proxy(frameLocator, {
+    get(target, prop, receiver) {
+      if (typeof prop !== "string") {
+        return Reflect.get(target, prop, receiver);
+      }
+      const value = Reflect.get(target, prop, target);
+      if (typeof value !== "function") return value;
+
+      if (FRAME_LOCATOR_FACTORY_METHODS.has(prop)) {
+        return (...args: unknown[]) => {
+          const result = value.apply(target, args);
+          if (prop === "first" || prop === "last" || prop === "nth") {
+            return createFallbackFrameLocator(
+              result as FrameLocator,
+              rawPage,
+              options,
+              caches,
+            );
+          }
+          if (prop === "frameLocator") {
+            return createFallbackFrameLocator(
+              result as FrameLocator,
+              rawPage,
+              options,
+              caches,
+            );
+          }
+          return createFallbackLocator(
+            result as Locator,
+            rawPage,
+            options,
+            caches,
+          );
+        };
+      }
+
+      return value.bind(target);
+    },
+  }) as FrameLocator;
+
+  caches.frameLocators.set(frameLocator, proxy);
+  return proxy;
+}
+
+export function createRecoveryPage(
+  page: Page,
+  options: RecoveryActionOptions,
+): Page {
+  const caches: ProxyCaches = {
+    locators: new WeakMap(),
+    frameLocators: new WeakMap(),
+  };
+
+  return new Proxy(page, {
+    get(target, prop, receiver) {
+      if (typeof prop !== "string") {
+        return Reflect.get(target, prop, receiver);
+      }
+
+      const value = Reflect.get(target, prop, target);
+      if (typeof value !== "function") return value;
+
+      if (PAGE_LOCATOR_FACTORY_METHODS.has(prop)) {
+        return (...args: unknown[]) => {
+          const locator = value.apply(target, args) as Locator;
+          return createFallbackLocator(locator, page, options, caches);
+        };
+      }
+
+      if (prop === "frameLocator") {
+        return (...args: unknown[]) => {
+          const frameLocator = value.apply(target, args) as FrameLocator;
+          return createFallbackFrameLocator(frameLocator, page, options, caches);
+        };
+      }
+
+      if (!isSupportedMethod("page", prop)) {
+        return value.bind(target);
+      }
+
+      return (...args: unknown[]) =>
+        runWithFallback({
+          page,
+          targetType: "page",
+          method: prop,
+          methodArgs: args,
+          invoke: () => value.apply(target, args),
+          options,
+        });
+    },
+  }) as Page;
+}
+
+async function resolveComputerUseRecoveryModel(
+  options: ComputerUseRecoveryActionOptions,
+): Promise<LanguageModel> {
+  if ("provider" in options) {
+    if (options.provider === "openai") {
+      const model: string = options.model ?? COMPUTER_USE_RECOVERY_MODELS.openai;
+      if (model !== COMPUTER_USE_RECOVERY_MODELS.openai) {
+        throw new Error(
+          `Unsupported OpenAI computer use recovery model "${model}". Supported model: ${COMPUTER_USE_RECOVERY_MODELS.openai}.`,
+        );
+      }
+      return import("@ai-sdk/openai").then(({ createOpenAI }) =>
+        createOpenAI({ apiKey: options.apiKey })(model),
+      );
+    }
+    const model: string =
+      options.model ?? COMPUTER_USE_RECOVERY_MODELS.anthropic;
+    if (model !== COMPUTER_USE_RECOVERY_MODELS.anthropic) {
+      throw new Error(
+        `Unsupported Anthropic computer use recovery model "${model}". Supported model: ${COMPUTER_USE_RECOVERY_MODELS.anthropic}.`,
+      );
+    }
+    return import("@ai-sdk/anthropic").then(({ createAnthropic }) =>
+      createAnthropic({ apiKey: options.apiKey })(model),
+    );
+  }
+
+  return options.languageModel;
+}
+
+export function computerUseRecoveryAction(
+  options: ComputerUseRecoveryActionOptions,
+): RecoveryAction {
+  return async ({ page }): Promise<RecoveryAgentResult> => {
+    const model = await resolveComputerUseRecoveryModel(options);
+    return executeRecoveryAgent(
+      page,
+      options.instruction,
+      undefined,
+      model,
+      options.maxSteps,
+    );
+  };
+}
+
+export function popupRecoveryAction(
+  options: PopupRecoveryActionOptions,
+): RecoveryAction {
+  return computerUseRecoveryAction({
+    ...options,
+    instruction: POPUP_RECOVERY_INSTRUCTION,
+  });
+}

package/src/shared/state/index.ts

@@ -7,6 +7,7 @@ export {
   parseSessionStateContent,
   serializeSessionState,
   type SessionAccessMode,
+  type ProviderState,
   type SessionStatus,
   type SessionState,
   type SessionStateFile,

package/src/shared/state/session-state.ts

@@ -19,6 +19,7 @@ export const SessionViewportSchema = z.object({
 export const ProviderStateSchema = z.object({
   name: z.string(),
   sessionId: z.string(),
+  recordingUrl: z.string().url().optional(),
 });
 
 export const SessionStateFileSchema = z.object({
@@ -38,6 +39,7 @@ export const SessionStateFileSchema = z.object({
 
 export type SessionStatus = z.infer<typeof SessionStatusSchema>;
 export type SessionAccessMode = z.infer<typeof SessionAccessModeSchema>;
+export type ProviderState = z.infer<typeof ProviderStateSchema>;
 export type SessionStateFile = z.infer<typeof SessionStateFileSchema>;
 export type SessionState = Omit<SessionStateFile, "version">;
 

package/src/shared/workflow/workflow.ts

@@ -1,5 +1,9 @@
 import type { Page } from "playwright";
 import { z } from "zod";
+import {
+  createRecoveryPage,
+  type RecoveryAction,
+} from "../../runtime/recovery/page-fallbacks.js";
 
 export const LIBRETTO_WORKFLOW_BRAND = Symbol.for("libretto.workflow");
 
@@ -13,12 +17,23 @@ export type LibrettoWorkflowHandler<Input = unknown, Output = unknown> = (
   input: Input,
 ) => Promise<Output>;
 
-export type LibrettoWorkflowSchemas<
-  InputSchema extends z.ZodType,
-  OutputSchema extends z.ZodType,
+export type LibrettoWorkflowDefinition<
+  InputSchema extends z.ZodType = z.ZodType<unknown>,
+  OutputSchema extends z.ZodType = z.ZodType<unknown>,
 > = {
-  input: InputSchema;
-  output: OutputSchema;
+  input?: InputSchema;
+  output?: OutputSchema;
+  recoveryAction?: RecoveryAction;
+};
+
+export type LibrettoWorkflowOptions<
+  InputSchema extends z.ZodType = z.ZodType<unknown>,
+  OutputSchema extends z.ZodType = z.ZodType<unknown>,
+> = LibrettoWorkflowDefinition<InputSchema, OutputSchema> & {
+  handler: LibrettoWorkflowHandler<
+    z.infer<InputSchema>,
+    z.infer<OutputSchema>
+  >;
 };
 
 // Thrown when input fails Zod validation. The runner surfaces `.message`
@@ -89,6 +104,7 @@ export class LibrettoWorkflow<
   // this schema to JSON Schema at build time and exposes it via
   // /v1/workflows/get so API consumers know the workflow's output shape.
   public readonly outputSchema?: OutputSchema;
+  public readonly recoveryAction?: RecoveryAction;
   private readonly handler: LibrettoWorkflowHandler<
     z.infer<InputSchema>,
     z.infer<OutputSchema>
@@ -96,15 +112,22 @@ export class LibrettoWorkflow<
 
   constructor(
     name: string,
-    schemas: LibrettoWorkflowSchemas<InputSchema, OutputSchema> | undefined,
+    options:
+      | {
+          inputSchema?: InputSchema;
+          outputSchema?: OutputSchema;
+          recoveryAction?: RecoveryAction;
+        }
+      | undefined,
     handler: LibrettoWorkflowHandler<
       z.infer<InputSchema>,
       z.infer<OutputSchema>
     >,
   ) {
     this.name = name;
-    this.inputSchema = schemas?.input;
-    this.outputSchema = schemas?.output;
+    this.inputSchema = options?.inputSchema;
+    this.outputSchema = options?.outputSchema;
+    this.recoveryAction = options?.recoveryAction;
     this.handler = handler;
   }
 
@@ -113,7 +136,16 @@ export class LibrettoWorkflow<
     input: unknown,
   ): Promise<z.infer<OutputSchema>> {
     const parsed = parseWorkflowInput(this.name, this.inputSchema, input);
-    return this.handler(ctx, parsed);
+    const workflowContext =
+      !this.recoveryAction
+        ? ctx
+        : {
+            ...ctx,
+            page: createRecoveryPage(ctx.page, {
+              recoveryAction: this.recoveryAction,
+            }),
+          };
+    return this.handler(workflowContext, parsed);
   }
 }
 
@@ -122,6 +154,7 @@ export type ExportedLibrettoWorkflow = {
   readonly name: string;
   readonly inputSchema?: z.ZodType;
   readonly outputSchema?: z.ZodType;
+  readonly recoveryAction?: RecoveryAction;
   run: (ctx: LibrettoWorkflowContext, input: unknown) => Promise<unknown>;
 };
 
@@ -212,22 +245,47 @@ export function getWorkflowFromModuleExports(
   return null;
 }
 
-// Recommended 3-arg form: pass Zod schemas so input is validated at run time
-// and the hosted platform can expose typed I/O metadata via /v1/workflows/get.
-export function workflow<
+function getWorkflowConstructorOptions<
   InputSchema extends z.ZodType,
   OutputSchema extends z.ZodType,
+>(
+  options:
+    | LibrettoWorkflowDefinition<InputSchema, OutputSchema>
+    | LibrettoWorkflowOptions<InputSchema, OutputSchema>,
+): {
+  inputSchema?: InputSchema;
+  outputSchema?: OutputSchema;
+  recoveryAction?: RecoveryAction;
+} {
+  return {
+    inputSchema: options.input,
+    outputSchema: options.output,
+    recoveryAction: options.recoveryAction,
+  };
+}
+
+export function workflow<
+  InputSchema extends z.ZodType = z.ZodType<unknown>,
+  OutputSchema extends z.ZodType = z.ZodType<unknown>,
 >(
   name: string,
-  schemas: LibrettoWorkflowSchemas<InputSchema, OutputSchema>,
+  definition: LibrettoWorkflowDefinition<InputSchema, OutputSchema>,
   handler: LibrettoWorkflowHandler<
     z.infer<InputSchema>,
     z.infer<OutputSchema>
   >,
 ): LibrettoWorkflow<InputSchema, OutputSchema>;
 
+export function workflow<
+  InputSchema extends z.ZodType = z.ZodType<unknown>,
+  OutputSchema extends z.ZodType = z.ZodType<unknown>,
+>(
+  name: string,
+  options: LibrettoWorkflowOptions<InputSchema, OutputSchema>,
+): LibrettoWorkflow<InputSchema, OutputSchema>;
+
 // Legacy 2-arg form kept so deployments built before Zod schemas existed
-// continue to load. New code should always pass schemas.
+// continue to load. New code should pass input/output schemas when possible.
 export function workflow<Input = unknown, Output = unknown>(
   name: string,
   handler: LibrettoWorkflowHandler<Input, Output>,
@@ -235,18 +293,30 @@ export function workflow<Input = unknown, Output = unknown>(
 
 export function workflow(
   name: string,
-  schemasOrHandler:
-    | LibrettoWorkflowSchemas<z.ZodType, z.ZodType>
+  definitionOrHandler:
+    | LibrettoWorkflowDefinition<z.ZodType, z.ZodType>
+    | LibrettoWorkflowOptions
     | LibrettoWorkflowHandler,
   maybeHandler?: LibrettoWorkflowHandler,
 ): LibrettoWorkflow {
-  if (typeof schemasOrHandler === "function") {
-    return new LibrettoWorkflow(name, undefined, schemasOrHandler);
+  if (typeof definitionOrHandler === "function") {
+    return new LibrettoWorkflow(name, undefined, definitionOrHandler);
+  }
+  if ("handler" in definitionOrHandler) {
+    return new LibrettoWorkflow(
+      name,
+      getWorkflowConstructorOptions(definitionOrHandler),
+      definitionOrHandler.handler,
+    );
   }
   if (!maybeHandler) {
     throw new Error(
-      `workflow("${name}") called with schemas but no handler. Pass the handler as the third argument.`,
+      `workflow("${name}") called without a handler. Pass the handler as the third argument or in the options object.`,
     );
   }
-  return new LibrettoWorkflow(name, schemasOrHandler, maybeHandler);
+  return new LibrettoWorkflow(
+    name,
+    getWorkflowConstructorOptions(definitionOrHandler),
+    maybeHandler,
+  );
 }