libretto 0.4.4 → 0.5.1

package/src/shared/config/config.ts

@@ -0,0 +1,3 @@
+/**
+ * Runtime configuration for libretto.
+ */

package/src/shared/debug/index.ts

@@ -0,0 +1 @@
+export { pause } from "./pause.js";

package/src/shared/debug/pause.ts

@@ -0,0 +1,91 @@
+import { existsSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import { getSessionDir } from "../../cli/core/context.js";
+import {
+  getPauseSignalPaths,
+  removeSignalIfExists,
+} from "../../cli/core/pause-signals.js";
+import {
+  listSessionsWithStateFile,
+  readSessionState,
+} from "../../cli/core/session.js";
+
+function isPidRunning(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function getRunningSessions(): string[] {
+  return listSessionsWithStateFile().filter((candidate) => {
+    const state = readSessionState(candidate);
+    return state !== null && state.pid != null && isPidRunning(state.pid);
+  });
+}
+
+function throwMissingSessionError(): never {
+  const runningSessions = getRunningSessions();
+  const lines = ["pause(session) requires a non-empty session ID."];
+
+  if (runningSessions.length > 0) {
+    lines.push("", "Running sessions:");
+    for (const runningSession of runningSessions) {
+      lines.push(`  ${runningSession}`);
+    }
+  }
+
+  throw new Error(lines.join("\n"));
+}
+
+/**
+ * Standalone pause function.
+ *
+ * - In production (`NODE_ENV === "production"`), returns immediately (no-op).
+ * - Otherwise, writes a `.paused` signal file and polls for a `.resume` signal,
+ *   using the same file-based mechanism as the CLI runner.
+ *
+ * Import directly: `import { pause } from "libretto";`
+ */
+export async function pause(session: string): Promise<void> {
+  if (process.env.NODE_ENV === "production") {
+    return;
+  }
+
+  if (typeof session !== "string" || session.trim().length === 0) {
+    throwMissingSessionError();
+  }
+
+  const signalPaths = getPauseSignalPaths(session);
+  const { pausedSignalPath, resumeSignalPath } = signalPaths;
+
+  await mkdir(getSessionDir(session), { recursive: true });
+  await removeSignalIfExists(resumeSignalPath);
+
+  const details = {
+    sessionName: session,
+    pausedAt: new Date().toISOString(),
+    url: "unknown",
+  };
+
+  // Try to read the current page URL from the process (best-effort).
+  // The standalone pause doesn't have access to the page object,
+  // so we just record what we can.
+  await writeFile(pausedSignalPath, JSON.stringify(details, null, 2), "utf8");
+
+  console.log(`[pause] Paused (session: ${session})`);
+  console.log("[pause] Waiting for resume signal...");
+
+  const RESUME_POLL_INTERVAL_MS = 250;
+  while (!existsSync(resumeSignalPath)) {
+    await new Promise((resolve) =>
+      setTimeout(resolve, RESUME_POLL_INTERVAL_MS),
+    );
+  }
+
+  await removeSignalIfExists(resumeSignalPath);
+  await removeSignalIfExists(pausedSignalPath);
+  console.log("[pause] Resume signal received. Continuing workflow...");
+}

package/src/shared/instrumentation/errors.ts

@@ -0,0 +1,84 @@
+import type { Page, Locator } from "playwright";
+
+/**
+ * Enrich a timeout error from a pointer action (click/dblclick/hover) with
+ * diagnostic information about why the action may have failed.
+ *
+ * Mutates err.message in-place to append the enrichment.
+ * Best-effort: if any probe fails, we skip that check silently.
+ */
+export async function enrichTimeoutError(
+  err: any,
+  locator: Locator,
+  page: Page,
+): Promise<void> {
+  const reasons: string[] = [];
+
+  try {
+    const visible = await locator.isVisible().catch(() => null);
+    if (visible === false) {
+      reasons.push("Element is not visible");
+    }
+
+    // isInViewport is available in modern Playwright but may not exist in older versions
+    if (typeof (locator as any).isInViewport === "function") {
+      const inViewport = await (locator as any)
+        .isInViewport()
+        .catch(() => null);
+      if (inViewport === false) {
+        reasons.push("Element is outside of the viewport");
+      }
+    }
+
+    const enabled = await locator.isEnabled().catch(() => null);
+    if (enabled === false) {
+      reasons.push("Element is not enabled (disabled)");
+    }
+
+    // If the element appears visible and in viewport, check for intercepting elements
+    if (reasons.length === 0) {
+      const box = await locator.boundingBox().catch(() => null);
+      if (box) {
+        const centerX = box.x + box.width / 2;
+        const centerY = box.y + box.height / 2;
+
+        const interceptInfo = await page
+          .evaluate(
+            ({ x, y }) => {
+              const els = document.elementsFromPoint(x, y);
+              if (!els || els.length < 2) return null;
+              const topEl = els[0];
+              if (!topEl) return null;
+
+              // Build a brief preview of the intercepting element
+              const tag = topEl.tagName.toLowerCase();
+              const id = topEl.id ? `#${topEl.id}` : "";
+              const cls = topEl.className
+                ? `.${String(topEl.className).split(/\s+/).slice(0, 2).join(".")}`
+                : "";
+              const text = (topEl.textContent || "").trim().slice(0, 40);
+              return {
+                tag,
+                preview: `<${tag}${id}${cls}>${text ? ` "${text}"` : ""}`,
+              };
+            },
+            { x: centerX, y: centerY },
+          )
+          .catch(() => null);
+
+        if (interceptInfo) {
+          reasons.push(
+            `Element may be intercepted by ${interceptInfo.preview}`,
+          );
+        }
+      }
+    }
+  } catch {
+    // All enrichment is best-effort
+  }
+
+  if (reasons.length > 0) {
+    const enrichment = `\n[libretto diagnostics] ${reasons.join("; ")}`;
+    err.message = (err.message || "") + enrichment;
+  }
+}

package/src/shared/instrumentation/index.ts

@@ -0,0 +1,9 @@
+export {
+  instrumentPage,
+  installInstrumentation,
+  instrumentContext,
+  type InstrumentationOptions,
+  type InstrumentedPage,
+} from "./instrument.js";
+
+export { enrichTimeoutError } from "./errors.js";

package/src/shared/instrumentation/instrument.ts

@@ -0,0 +1,406 @@
+import type { Page, Locator, FrameLocator, BrowserContext } from "playwright";
+import type { MinimalLogger } from "../logger/logger.js";
+import type { GhostCursorOptions } from "../visualization/ghost-cursor.js";
+import type { HighlightOptions } from "../visualization/highlight.js";
+import {
+  ensureGhostCursor,
+  moveGhostCursor,
+  moveGhostCursorWithDistance,
+  ghostClick,
+  getGhostCursorPosition,
+} from "../visualization/ghost-cursor.js";
+import {
+  ensureHighlightLayer,
+  showHighlight,
+  clearHighlights,
+} from "../visualization/highlight.js";
+import { enrichTimeoutError } from "./errors.js";
+
+export type InstrumentationOptions = {
+  visualize?: boolean;
+  logger?: MinimalLogger;
+  highlightBeforeActionMs?: number;
+  ghostCursor?: GhostCursorOptions;
+  highlight?: HighlightOptions;
+};
+
+export type InstrumentedPage = Page & {
+  __librettoInstrumented: true;
+};
+
+const LOCATOR_ACTIONS = [
+  "click",
+  "dblclick",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "check",
+  "uncheck",
+  "selectOption",
+  "focus",
+] as const;
+
+const NAV_ACTIONS = ["goto", "reload", "goBack", "goForward"] as const;
+
+const POINTER_ACTIONS = new Set<string>(["click", "dblclick", "hover"]);
+
+const instrumentedTargets = new WeakSet<object>();
+
+// Per-page serialization queue so overlapping visualization actions don't glitch
+const pageQueues = new WeakMap<Page, Promise<void>>();
+
+function enqueue(page: Page, fn: () => Promise<void>): Promise<void> {
+  const prev = pageQueues.get(page) ?? Promise.resolve();
+  const next = prev.then(fn, fn);
+  pageQueues.set(page, next);
+  return next;
+}
+
+async function visualizeBeforeAction(
+  page: Page,
+  box: { x: number; y: number; width: number; height: number } | null,
+  actionName: string,
+  highlightMs: number,
+): Promise<void> {
+  if (!box) return;
+
+  // Re-ensure overlays in case DOM was replaced (e.g. page.setContent()).
+  await ensureGhostCursor(page);
+  await ensureHighlightLayer(page);
+
+  const centerX = box.x + box.width / 2;
+  const centerY = box.y + box.height / 2;
+
+  // Show highlight on the target element
+  await showHighlight(page, {
+    box,
+    durationMs: highlightMs + 200, // keep visible a bit past the cursor arrival
+  });
+
+  // Move ghost cursor to target
+  const cursorPos = await getGhostCursorPosition(page);
+  if (cursorPos) {
+    await moveGhostCursorWithDistance(page, cursorPos, {
+      x: centerX,
+      y: centerY,
+    });
+  } else {
+    await moveGhostCursor(page, { x: centerX, y: centerY, durationMs: 200 });
+  }
+
+  // For click actions, show click feedback
+  if (actionName === "click" || actionName === "dblclick") {
+    await ghostClick(page, { x: centerX, y: centerY });
+  }
+}
+
+async function visualizeAfterAction(page: Page): Promise<void> {
+  await clearHighlights(page);
+}
+
+function wrapLocatorActions(
+  locator: Locator,
+  page: Page,
+  opts: Required<
+    Pick<InstrumentationOptions, "visualize" | "highlightBeforeActionMs">
+  > &
+    InstrumentationOptions,
+): void {
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof (locator as any)[method] !== "function") continue;
+    const orig = (locator as any)[method].bind(locator);
+
+    (locator as any)[method] = async (...args: any[]) => {
+      if (opts.visualize) {
+        await enqueue(page, async () => {
+          try {
+            const box = await locator.boundingBox();
+            await visualizeBeforeAction(
+              page,
+              box,
+              method,
+              opts.highlightBeforeActionMs,
+            );
+          } catch {
+            // Best-effort visualization
+          }
+        });
+      }
+
+      try {
+        const result = await orig(...args);
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err: any) {
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        // Enrich timeout errors for pointer actions
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err)) {
+          await enrichTimeoutError(err, locator, page);
+        }
+        throw err;
+      }
+    };
+  }
+}
+
+const LOCATOR_FACTORY_METHODS = [
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+  "filter",
+  "and",
+  "or",
+  "first",
+  "last",
+  "nth",
+] as const;
+
+const FRAME_LOCATOR_FACTORY_METHODS = [
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+  "owner",
+  "first",
+  "last",
+  "nth",
+] as const;
+
+type InstrumentationRuntimeOptions = Required<
+  Pick<InstrumentationOptions, "visualize" | "highlightBeforeActionMs">
+> &
+  InstrumentationOptions;
+
+function instrumentLocator(
+  locator: Locator,
+  page: Page,
+  opts: InstrumentationRuntimeOptions,
+): Locator {
+  const target = locator as object;
+  if (instrumentedTargets.has(target)) {
+    return locator;
+  }
+  instrumentedTargets.add(target);
+
+  wrapLocatorActions(locator, page, opts);
+
+  for (const method of LOCATOR_FACTORY_METHODS) {
+    if (typeof (locator as any)[method] !== "function") continue;
+    const orig = (locator as any)[method].bind(locator);
+    (locator as any)[method] = (...args: any[]) => {
+      const nextLocator = orig(...args);
+      return instrumentLocator(nextLocator, page, opts);
+    };
+  }
+
+  if (typeof (locator as any).contentFrame === "function") {
+    const origContentFrame = (locator as any).contentFrame.bind(locator);
+    (locator as any).contentFrame = (...args: any[]) => {
+      const frameLocator = origContentFrame(...args);
+      return instrumentFrameLocator(frameLocator, page, opts);
+    };
+  }
+
+  return locator;
+}
+
+function instrumentFrameLocator(
+  frameLocator: FrameLocator,
+  page: Page,
+  opts: InstrumentationRuntimeOptions,
+): FrameLocator {
+  const target = frameLocator as object;
+  if (instrumentedTargets.has(target)) {
+    return frameLocator;
+  }
+  instrumentedTargets.add(target);
+
+  for (const method of FRAME_LOCATOR_FACTORY_METHODS) {
+    if (typeof (frameLocator as any)[method] !== "function") continue;
+    const orig = (frameLocator as any)[method].bind(frameLocator);
+    (frameLocator as any)[method] = (...args: any[]) => {
+      const result = orig(...args);
+      if (method === "first" || method === "last" || method === "nth") {
+        return instrumentFrameLocator(result, page, opts);
+      }
+      return instrumentLocator(result, page, opts);
+    };
+  }
+
+  if (typeof (frameLocator as any).frameLocator === "function") {
+    const origFrameLocator = (frameLocator as any).frameLocator.bind(
+      frameLocator,
+    );
+    (frameLocator as any).frameLocator = (...args: any[]) => {
+      const nestedFrameLocator = origFrameLocator(...args);
+      return instrumentFrameLocator(nestedFrameLocator, page, opts);
+    };
+  }
+
+  return frameLocator;
+}
+
+function isTimeoutError(err: any): boolean {
+  if (!err || typeof err.message !== "string") return false;
+  return (
+    err.message.includes("Timeout") ||
+    err.message.includes("timeout") ||
+    err.name === "TimeoutError"
+  );
+}
+
+const PAGE_LOCATOR_FACTORIES = [
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId",
+] as const;
+
+const PAGE_FRAME_LOCATOR_FACTORIES = ["frameLocator"] as const;
+
+/**
+ * In-place patching of a Page object to add visualization and error enrichment.
+ * Modifies the page directly (does not return a new object).
+ */
+export async function installInstrumentation(
+  page: Page,
+  options?: InstrumentationOptions,
+): Promise<void> {
+  if ((page as any).__librettoInstrumented) return;
+  (page as any).__librettoInstrumented = true;
+
+  const visualize = options?.visualize ?? false;
+  const highlightBeforeActionMs = options?.highlightBeforeActionMs ?? 350;
+  const mergedOpts = { ...options, visualize, highlightBeforeActionMs };
+
+  // Install overlay layers if visualization is on
+  if (visualize) {
+    await ensureGhostCursor(page, options?.ghostCursor);
+    await ensureHighlightLayer(page, options?.highlight);
+  }
+
+  // Wrap page-level locator actions (page.click, page.fill, etc.)
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof (page as any)[method] !== "function") continue;
+    const orig = (page as any)[method].bind(page);
+    (page as any)[method] = async (...args: any[]) => {
+      // For page-level actions, the first arg is typically the selector
+      if (visualize && typeof args[0] === "string") {
+        await enqueue(page, async () => {
+          try {
+            const loc = page.locator(args[0]);
+            const box = await loc.boundingBox();
+            await visualizeBeforeAction(
+              page,
+              box,
+              method,
+              highlightBeforeActionMs,
+            );
+          } catch {
+            // Best-effort
+          }
+        });
+      }
+
+      try {
+        const result = await orig(...args);
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err: any) {
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (
+          POINTER_ACTIONS.has(method) &&
+          isTimeoutError(err) &&
+          typeof args[0] === "string"
+        ) {
+          await enrichTimeoutError(err, page.locator(args[0]), page);
+        }
+        throw err;
+      }
+    };
+  }
+
+  // Wrap navigation actions (no visualization, just logging)
+  for (const method of NAV_ACTIONS) {
+    if (typeof (page as any)[method] !== "function") continue;
+    const orig = (page as any)[method].bind(page);
+    (page as any)[method] = async (...args: any[]) => {
+      options?.logger?.info(`instrumentation:${method}`, {
+        url: typeof args[0] === "string" ? args[0] : page.url(),
+      });
+      return orig(...args);
+    };
+  }
+
+  // Wrap locator factories to instrument returned locators
+  for (const factory of PAGE_LOCATOR_FACTORIES) {
+    if (typeof (page as any)[factory] !== "function") continue;
+    const origFactory = (page as any)[factory].bind(page);
+    (page as any)[factory] = (...factoryArgs: any[]) => {
+      const locator = origFactory(...factoryArgs);
+      return instrumentLocator(locator, page, mergedOpts);
+    };
+  }
+
+  for (const factory of PAGE_FRAME_LOCATOR_FACTORIES) {
+    if (typeof (page as any)[factory] !== "function") continue;
+    const origFactory = (page as any)[factory].bind(page);
+    (page as any)[factory] = (...factoryArgs: any[]) => {
+      const frameLocator = origFactory(...factoryArgs);
+      return instrumentFrameLocator(frameLocator, page, mergedOpts);
+    };
+  }
+}
+
+/**
+ * Returns a new object that proxies to the page with instrumentation applied.
+ * The original page is not modified.
+ */
+export async function instrumentPage(
+  page: Page,
+  options?: InstrumentationOptions,
+): Promise<InstrumentedPage> {
+  await installInstrumentation(page, options);
+  return page as InstrumentedPage;
+}
+
+/**
+ * Install overlays on a page and auto-install on all future pages in the context.
+ * Useful when connecting to an existing browser via CDP.
+ */
+export async function instrumentContext(
+  context: BrowserContext,
+  options?: InstrumentationOptions,
+): Promise<void> {
+  // Instrument all existing pages
+  for (const page of context.pages()) {
+    await installInstrumentation(page, options);
+  }
+
+  // Auto-instrument new pages
+  context.on("page", async (page) => {
+    await installInstrumentation(page, options);
+  });
+}

package/src/shared/llm/ai-sdk-adapter.ts

@@ -0,0 +1,81 @@
+import { generateObject, type LanguageModel } from "ai";
+import type { ZodType, output as ZodOutput } from "zod";
+import type { LLMClient, Message } from "./types.js";
+
+/**
+ * Creates a libretto LLMClient from a Vercel AI SDK LanguageModel.
+ *
+ * This eliminates the need for consumers to write their own adapter
+ * when using @ai-sdk/openai, @ai-sdk/anthropic, @ai-sdk/google-vertex,
+ * or any other Vercel AI SDK-compatible provider.
+ *
+ * @example
+ * ```typescript
+ * import { createLLMClientFromModel } from "libretto/llm";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * const llmClient = createLLMClientFromModel(openai("gpt-4o"));
+ * ```
+ */
+export function createLLMClientFromModel(model: LanguageModel): LLMClient {
+  return {
+    async generateObject<T extends ZodType>(opts: {
+      prompt: string;
+      schema: T;
+      temperature?: number;
+    }): Promise<ZodOutput<T>> {
+      const { object } = await generateObject({
+        model,
+        schema: opts.schema,
+        prompt: opts.prompt,
+        temperature: opts.temperature ?? 0,
+      });
+      return object as ZodOutput<T>;
+    },
+
+    async generateObjectFromMessages<T extends ZodType>(opts: {
+      messages: Message[];
+      schema: T;
+      temperature?: number;
+    }): Promise<ZodOutput<T>> {
+      // Convert libretto Message format to AI SDK message format
+      const messages = opts.messages.map((msg) => {
+        if (typeof msg.content === "string") {
+          return { role: msg.role, content: msg.content };
+        }
+        if (msg.role === "assistant") {
+          // AssistantContent only supports text parts (no images)
+          return {
+            role: "assistant" as const,
+            content: msg.content
+              .filter(
+                (part): part is typeof part & { type: "text" } =>
+                  part.type === "text",
+              )
+              .map((part) => ({ type: "text" as const, text: part.text })),
+          };
+        }
+        return {
+          role: "user" as const,
+          content: msg.content.map((part) =>
+            part.type === "text"
+              ? { type: "text" as const, text: part.text }
+              : {
+                  type: "image" as const,
+                  image: part.image,
+                  ...(part.mediaType ? { mediaType: part.mediaType } : {}),
+                },
+          ),
+        };
+      });
+
+      const { object } = await generateObject({
+        model,
+        schema: opts.schema,
+        messages,
+        temperature: opts.temperature ?? 0,
+      });
+      return object as ZodOutput<T>;
+    },
+  };
+}