libretto 0.3.0 → 0.3.1

package/dist/shared/instrumentation/instrument.cjs

@@ -0,0 +1,206 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var instrument_exports = {};
+__export(instrument_exports, {
+  installInstrumentation: () => installInstrumentation,
+  instrumentContext: () => instrumentContext,
+  instrumentPage: () => instrumentPage
+});
+module.exports = __toCommonJS(instrument_exports);
+var import_ghost_cursor = require("../visualization/ghost-cursor.js");
+var import_highlight = require("../visualization/highlight.js");
+var import_errors = require("./errors.js");
+const LOCATOR_ACTIONS = [
+  "click",
+  "dblclick",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "check",
+  "uncheck",
+  "selectOption",
+  "focus"
+];
+const NAV_ACTIONS = ["goto", "reload", "goBack", "goForward"];
+const POINTER_ACTIONS = /* @__PURE__ */ new Set(["click", "dblclick", "hover"]);
+const pageQueues = /* @__PURE__ */ new WeakMap();
+function enqueue(page, fn) {
+  const prev = pageQueues.get(page) ?? Promise.resolve();
+  const next = prev.then(fn, fn);
+  pageQueues.set(page, next);
+  return next;
+}
+async function visualizeBeforeAction(page, box, actionName, highlightMs) {
+  if (!box) return;
+  await (0, import_ghost_cursor.ensureGhostCursor)(page);
+  await (0, import_highlight.ensureHighlightLayer)(page);
+  const centerX = box.x + box.width / 2;
+  const centerY = box.y + box.height / 2;
+  await (0, import_highlight.showHighlight)(page, {
+    box,
+    durationMs: highlightMs + 200
+    // keep visible a bit past the cursor arrival
+  });
+  const cursorPos = await (0, import_ghost_cursor.getGhostCursorPosition)(page);
+  if (cursorPos) {
+    await (0, import_ghost_cursor.moveGhostCursorWithDistance)(page, cursorPos, {
+      x: centerX,
+      y: centerY
+    });
+  } else {
+    await (0, import_ghost_cursor.moveGhostCursor)(page, { x: centerX, y: centerY, durationMs: 200 });
+  }
+  if (actionName === "click" || actionName === "dblclick") {
+    await (0, import_ghost_cursor.ghostClick)(page, { x: centerX, y: centerY });
+  }
+}
+async function visualizeAfterAction(page) {
+  await (0, import_highlight.clearHighlights)(page);
+}
+function wrapLocatorActions(locator, page, opts) {
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof locator[method] !== "function") continue;
+    const orig = locator[method].bind(locator);
+    locator[method] = async (...args) => {
+      if (opts.visualize) {
+        await enqueue(page, async () => {
+          try {
+            const box = await locator.boundingBox();
+            await visualizeBeforeAction(
+              page,
+              box,
+              method,
+              opts.highlightBeforeActionMs
+            );
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err)) {
+          await (0, import_errors.enrichTimeoutError)(err, locator, page);
+        }
+        throw err;
+      }
+    };
+  }
+}
+function isTimeoutError(err) {
+  if (!err || typeof err.message !== "string") return false;
+  return err.message.includes("Timeout") || err.message.includes("timeout") || err.name === "TimeoutError";
+}
+const LOCATOR_FACTORIES = [
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId"
+];
+async function installInstrumentation(page, options) {
+  if (page.__librettoInstrumented) return;
+  page.__librettoInstrumented = true;
+  const visualize = options?.visualize ?? false;
+  const highlightBeforeActionMs = options?.highlightBeforeActionMs ?? 350;
+  const mergedOpts = { ...options, visualize, highlightBeforeActionMs };
+  if (visualize) {
+    await (0, import_ghost_cursor.ensureGhostCursor)(page, options?.ghostCursor);
+    await (0, import_highlight.ensureHighlightLayer)(page, options?.highlight);
+  }
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      if (visualize && typeof args[0] === "string") {
+        await enqueue(page, async () => {
+          try {
+            const loc = page.locator(args[0]);
+            const box = await loc.boundingBox();
+            await visualizeBeforeAction(page, box, method, highlightBeforeActionMs);
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err) && typeof args[0] === "string") {
+          await (0, import_errors.enrichTimeoutError)(err, page.locator(args[0]), page);
+        }
+        throw err;
+      }
+    };
+  }
+  for (const method of NAV_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      options?.logger?.info(`instrumentation:${method}`, {
+        url: typeof args[0] === "string" ? args[0] : page.url()
+      });
+      return orig(...args);
+    };
+  }
+  for (const factory of LOCATOR_FACTORIES) {
+    if (typeof page[factory] !== "function") continue;
+    const origFactory = page[factory].bind(page);
+    page[factory] = (...factoryArgs) => {
+      const locator = origFactory(...factoryArgs);
+      wrapLocatorActions(locator, page, mergedOpts);
+      return locator;
+    };
+  }
+}
+async function instrumentPage(page, options) {
+  await installInstrumentation(page, options);
+  return page;
+}
+async function instrumentContext(context, options) {
+  for (const page of context.pages()) {
+    await installInstrumentation(page, options);
+  }
+  context.on("page", async (page) => {
+    await installInstrumentation(page, options);
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  installInstrumentation,
+  instrumentContext,
+  instrumentPage
+});

package/dist/shared/instrumentation/instrument.d.cts

@@ -0,0 +1,32 @@
+import { Page, BrowserContext } from 'playwright';
+import { MinimalLogger } from '../logger/logger.cjs';
+import { GhostCursorOptions } from '../visualization/ghost-cursor.cjs';
+import { HighlightOptions } from '../visualization/highlight.cjs';
+
+type InstrumentationOptions = {
+    visualize?: boolean;
+    logger?: MinimalLogger;
+    highlightBeforeActionMs?: number;
+    ghostCursor?: GhostCursorOptions;
+    highlight?: HighlightOptions;
+};
+type InstrumentedPage = Page & {
+    __librettoInstrumented: true;
+};
+/**
+ * In-place patching of a Page object to add visualization and error enrichment.
+ * Modifies the page directly (does not return a new object).
+ */
+declare function installInstrumentation(page: Page, options?: InstrumentationOptions): Promise<void>;
+/**
+ * Returns a new object that proxies to the page with instrumentation applied.
+ * The original page is not modified.
+ */
+declare function instrumentPage(page: Page, options?: InstrumentationOptions): Promise<InstrumentedPage>;
+/**
+ * Install overlays on a page and auto-install on all future pages in the context.
+ * Useful when connecting to an existing browser via CDP.
+ */
+declare function instrumentContext(context: BrowserContext, options?: InstrumentationOptions): Promise<void>;
+
+export { type InstrumentationOptions, type InstrumentedPage, installInstrumentation, instrumentContext, instrumentPage };

package/dist/shared/instrumentation/instrument.d.ts

@@ -0,0 +1,32 @@
+import { Page, BrowserContext } from 'playwright';
+import { MinimalLogger } from '../logger/logger.js';
+import { GhostCursorOptions } from '../visualization/ghost-cursor.js';
+import { HighlightOptions } from '../visualization/highlight.js';
+
+type InstrumentationOptions = {
+    visualize?: boolean;
+    logger?: MinimalLogger;
+    highlightBeforeActionMs?: number;
+    ghostCursor?: GhostCursorOptions;
+    highlight?: HighlightOptions;
+};
+type InstrumentedPage = Page & {
+    __librettoInstrumented: true;
+};
+/**
+ * In-place patching of a Page object to add visualization and error enrichment.
+ * Modifies the page directly (does not return a new object).
+ */
+declare function installInstrumentation(page: Page, options?: InstrumentationOptions): Promise<void>;
+/**
+ * Returns a new object that proxies to the page with instrumentation applied.
+ * The original page is not modified.
+ */
+declare function instrumentPage(page: Page, options?: InstrumentationOptions): Promise<InstrumentedPage>;
+/**
+ * Install overlays on a page and auto-install on all future pages in the context.
+ * Useful when connecting to an existing browser via CDP.
+ */
+declare function instrumentContext(context: BrowserContext, options?: InstrumentationOptions): Promise<void>;
+
+export { type InstrumentationOptions, type InstrumentedPage, installInstrumentation, instrumentContext, instrumentPage };

package/dist/shared/instrumentation/instrument.js

@@ -0,0 +1,190 @@
+import {
+  ensureGhostCursor,
+  moveGhostCursor,
+  moveGhostCursorWithDistance,
+  ghostClick,
+  getGhostCursorPosition
+} from "../visualization/ghost-cursor.js";
+import {
+  ensureHighlightLayer,
+  showHighlight,
+  clearHighlights
+} from "../visualization/highlight.js";
+import { enrichTimeoutError } from "./errors.js";
+const LOCATOR_ACTIONS = [
+  "click",
+  "dblclick",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "check",
+  "uncheck",
+  "selectOption",
+  "focus"
+];
+const NAV_ACTIONS = ["goto", "reload", "goBack", "goForward"];
+const POINTER_ACTIONS = /* @__PURE__ */ new Set(["click", "dblclick", "hover"]);
+const pageQueues = /* @__PURE__ */ new WeakMap();
+function enqueue(page, fn) {
+  const prev = pageQueues.get(page) ?? Promise.resolve();
+  const next = prev.then(fn, fn);
+  pageQueues.set(page, next);
+  return next;
+}
+async function visualizeBeforeAction(page, box, actionName, highlightMs) {
+  if (!box) return;
+  await ensureGhostCursor(page);
+  await ensureHighlightLayer(page);
+  const centerX = box.x + box.width / 2;
+  const centerY = box.y + box.height / 2;
+  await showHighlight(page, {
+    box,
+    durationMs: highlightMs + 200
+    // keep visible a bit past the cursor arrival
+  });
+  const cursorPos = await getGhostCursorPosition(page);
+  if (cursorPos) {
+    await moveGhostCursorWithDistance(page, cursorPos, {
+      x: centerX,
+      y: centerY
+    });
+  } else {
+    await moveGhostCursor(page, { x: centerX, y: centerY, durationMs: 200 });
+  }
+  if (actionName === "click" || actionName === "dblclick") {
+    await ghostClick(page, { x: centerX, y: centerY });
+  }
+}
+async function visualizeAfterAction(page) {
+  await clearHighlights(page);
+}
+function wrapLocatorActions(locator, page, opts) {
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof locator[method] !== "function") continue;
+    const orig = locator[method].bind(locator);
+    locator[method] = async (...args) => {
+      if (opts.visualize) {
+        await enqueue(page, async () => {
+          try {
+            const box = await locator.boundingBox();
+            await visualizeBeforeAction(
+              page,
+              box,
+              method,
+              opts.highlightBeforeActionMs
+            );
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err)) {
+          await enrichTimeoutError(err, locator, page);
+        }
+        throw err;
+      }
+    };
+  }
+}
+function isTimeoutError(err) {
+  if (!err || typeof err.message !== "string") return false;
+  return err.message.includes("Timeout") || err.message.includes("timeout") || err.name === "TimeoutError";
+}
+const LOCATOR_FACTORIES = [
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId"
+];
+async function installInstrumentation(page, options) {
+  if (page.__librettoInstrumented) return;
+  page.__librettoInstrumented = true;
+  const visualize = options?.visualize ?? false;
+  const highlightBeforeActionMs = options?.highlightBeforeActionMs ?? 350;
+  const mergedOpts = { ...options, visualize, highlightBeforeActionMs };
+  if (visualize) {
+    await ensureGhostCursor(page, options?.ghostCursor);
+    await ensureHighlightLayer(page, options?.highlight);
+  }
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      if (visualize && typeof args[0] === "string") {
+        await enqueue(page, async () => {
+          try {
+            const loc = page.locator(args[0]);
+            const box = await loc.boundingBox();
+            await visualizeBeforeAction(page, box, method, highlightBeforeActionMs);
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err) && typeof args[0] === "string") {
+          await enrichTimeoutError(err, page.locator(args[0]), page);
+        }
+        throw err;
+      }
+    };
+  }
+  for (const method of NAV_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      options?.logger?.info(`instrumentation:${method}`, {
+        url: typeof args[0] === "string" ? args[0] : page.url()
+      });
+      return orig(...args);
+    };
+  }
+  for (const factory of LOCATOR_FACTORIES) {
+    if (typeof page[factory] !== "function") continue;
+    const origFactory = page[factory].bind(page);
+    page[factory] = (...factoryArgs) => {
+      const locator = origFactory(...factoryArgs);
+      wrapLocatorActions(locator, page, mergedOpts);
+      return locator;
+    };
+  }
+}
+async function instrumentPage(page, options) {
+  await installInstrumentation(page, options);
+  return page;
+}
+async function instrumentContext(context, options) {
+  for (const page of context.pages()) {
+    await installInstrumentation(page, options);
+  }
+  context.on("page", async (page) => {
+    await installInstrumentation(page, options);
+  });
+}
+export {
+  installInstrumentation,
+  instrumentContext,
+  instrumentPage
+};

package/dist/shared/llm/ai-sdk-adapter.cjs

@@ -0,0 +1,67 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ai_sdk_adapter_exports = {};
+__export(ai_sdk_adapter_exports, {
+  createLLMClientFromModel: () => createLLMClientFromModel
+});
+module.exports = __toCommonJS(ai_sdk_adapter_exports);
+var import_ai = require("ai");
+function createLLMClientFromModel(model) {
+  return {
+    async generateObject(opts) {
+      const { object } = await (0, import_ai.generateObject)({
+        model,
+        schema: opts.schema,
+        prompt: opts.prompt,
+        temperature: opts.temperature ?? 0
+      });
+      return object;
+    },
+    async generateObjectFromMessages(opts) {
+      const messages = opts.messages.map((msg) => {
+        if (typeof msg.content === "string") {
+          return { role: msg.role, content: msg.content };
+        }
+        if (msg.role === "assistant") {
+          return {
+            role: "assistant",
+            content: msg.content.filter((part) => part.type === "text").map((part) => ({ type: "text", text: part.text }))
+          };
+        }
+        return {
+          role: "user",
+          content: msg.content.map(
+            (part) => part.type === "text" ? { type: "text", text: part.text } : { type: "image", image: part.image }
+          )
+        };
+      });
+      const { object } = await (0, import_ai.generateObject)({
+        model,
+        schema: opts.schema,
+        messages,
+        temperature: opts.temperature ?? 0
+      });
+      return object;
+    }
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  createLLMClientFromModel
+});

package/dist/shared/llm/ai-sdk-adapter.d.cts

@@ -0,0 +1,22 @@
+import { LanguageModel } from 'ai';
+import { LLMClient } from './types.cjs';
+import 'zod';
+
+/**
+ * Creates a libretto LLMClient from a Vercel AI SDK LanguageModel.
+ *
+ * This eliminates the need for consumers to write their own adapter
+ * when using @ai-sdk/openai, @ai-sdk/anthropic, @ai-sdk/google-vertex,
+ * or any other Vercel AI SDK-compatible provider.
+ *
+ * @example
+ * ```typescript
+ * import { createLLMClientFromModel } from "libretto/llm";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * const llmClient = createLLMClientFromModel(openai("gpt-4o"));
+ * ```
+ */
+declare function createLLMClientFromModel(model: LanguageModel): LLMClient;
+
+export { createLLMClientFromModel };

package/dist/shared/llm/ai-sdk-adapter.d.ts

@@ -0,0 +1,22 @@
+import { LanguageModel } from 'ai';
+import { LLMClient } from './types.js';
+import 'zod';
+
+/**
+ * Creates a libretto LLMClient from a Vercel AI SDK LanguageModel.
+ *
+ * This eliminates the need for consumers to write their own adapter
+ * when using @ai-sdk/openai, @ai-sdk/anthropic, @ai-sdk/google-vertex,
+ * or any other Vercel AI SDK-compatible provider.
+ *
+ * @example
+ * ```typescript
+ * import { createLLMClientFromModel } from "libretto/llm";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * const llmClient = createLLMClientFromModel(openai("gpt-4o"));
+ * ```
+ */
+declare function createLLMClientFromModel(model: LanguageModel): LLMClient;
+
+export { createLLMClientFromModel };

package/dist/shared/llm/ai-sdk-adapter.js

@@ -0,0 +1,43 @@
+import { generateObject } from "ai";
+function createLLMClientFromModel(model) {
+  return {
+    async generateObject(opts) {
+      const { object } = await generateObject({
+        model,
+        schema: opts.schema,
+        prompt: opts.prompt,
+        temperature: opts.temperature ?? 0
+      });
+      return object;
+    },
+    async generateObjectFromMessages(opts) {
+      const messages = opts.messages.map((msg) => {
+        if (typeof msg.content === "string") {
+          return { role: msg.role, content: msg.content };
+        }
+        if (msg.role === "assistant") {
+          return {
+            role: "assistant",
+            content: msg.content.filter((part) => part.type === "text").map((part) => ({ type: "text", text: part.text }))
+          };
+        }
+        return {
+          role: "user",
+          content: msg.content.map(
+            (part) => part.type === "text" ? { type: "text", text: part.text } : { type: "image", image: part.image }
+          )
+        };
+      });
+      const { object } = await generateObject({
+        model,
+        schema: opts.schema,
+        messages,
+        temperature: opts.temperature ?? 0
+      });
+      return object;
+    }
+  };
+}
+export {
+  createLLMClientFromModel
+};