libretto 0.1.4 → 0.2.0

package/dist/shared/instrumentation/index.d.cts

@@ -0,0 +1,6 @@
+export { InstrumentationOptions, InstrumentedPage, installInstrumentation, instrumentContext, instrumentPage } from './instrument.cjs';
+export { enrichTimeoutError } from './errors.cjs';
+import 'playwright';
+import '../logger/logger.cjs';
+import '../visualization/ghost-cursor.cjs';
+import '../visualization/highlight.cjs';

package/dist/shared/instrumentation/index.d.ts

@@ -0,0 +1,6 @@
+export { InstrumentationOptions, InstrumentedPage, installInstrumentation, instrumentContext, instrumentPage } from './instrument.js';
+export { enrichTimeoutError } from './errors.js';
+import 'playwright';
+import '../logger/logger.js';
+import '../visualization/ghost-cursor.js';
+import '../visualization/highlight.js';

package/dist/shared/instrumentation/index.js

@@ -0,0 +1,12 @@
+import {
+  instrumentPage,
+  installInstrumentation,
+  instrumentContext
+} from "./instrument.js";
+import { enrichTimeoutError } from "./errors.js";
+export {
+  enrichTimeoutError,
+  installInstrumentation,
+  instrumentContext,
+  instrumentPage
+};

package/dist/shared/instrumentation/instrument.cjs

@@ -0,0 +1,206 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var instrument_exports = {};
+__export(instrument_exports, {
+  installInstrumentation: () => installInstrumentation,
+  instrumentContext: () => instrumentContext,
+  instrumentPage: () => instrumentPage
+});
+module.exports = __toCommonJS(instrument_exports);
+var import_ghost_cursor = require("../visualization/ghost-cursor.js");
+var import_highlight = require("../visualization/highlight.js");
+var import_errors = require("./errors.js");
+const LOCATOR_ACTIONS = [
+  "click",
+  "dblclick",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "check",
+  "uncheck",
+  "selectOption",
+  "focus"
+];
+const NAV_ACTIONS = ["goto", "reload", "goBack", "goForward"];
+const POINTER_ACTIONS = /* @__PURE__ */ new Set(["click", "dblclick", "hover"]);
+const pageQueues = /* @__PURE__ */ new WeakMap();
+function enqueue(page, fn) {
+  const prev = pageQueues.get(page) ?? Promise.resolve();
+  const next = prev.then(fn, fn);
+  pageQueues.set(page, next);
+  return next;
+}
+async function visualizeBeforeAction(page, box, actionName, highlightMs) {
+  if (!box) return;
+  await (0, import_ghost_cursor.ensureGhostCursor)(page);
+  await (0, import_highlight.ensureHighlightLayer)(page);
+  const centerX = box.x + box.width / 2;
+  const centerY = box.y + box.height / 2;
+  await (0, import_highlight.showHighlight)(page, {
+    box,
+    durationMs: highlightMs + 200
+    // keep visible a bit past the cursor arrival
+  });
+  const cursorPos = await (0, import_ghost_cursor.getGhostCursorPosition)(page);
+  if (cursorPos) {
+    await (0, import_ghost_cursor.moveGhostCursorWithDistance)(page, cursorPos, {
+      x: centerX,
+      y: centerY
+    });
+  } else {
+    await (0, import_ghost_cursor.moveGhostCursor)(page, { x: centerX, y: centerY, durationMs: 200 });
+  }
+  if (actionName === "click" || actionName === "dblclick") {
+    await (0, import_ghost_cursor.ghostClick)(page, { x: centerX, y: centerY });
+  }
+}
+async function visualizeAfterAction(page) {
+  await (0, import_highlight.clearHighlights)(page);
+}
+function wrapLocatorActions(locator, page, opts) {
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof locator[method] !== "function") continue;
+    const orig = locator[method].bind(locator);
+    locator[method] = async (...args) => {
+      if (opts.visualize) {
+        await enqueue(page, async () => {
+          try {
+            const box = await locator.boundingBox();
+            await visualizeBeforeAction(
+              page,
+              box,
+              method,
+              opts.highlightBeforeActionMs
+            );
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err)) {
+          await (0, import_errors.enrichTimeoutError)(err, locator, page);
+        }
+        throw err;
+      }
+    };
+  }
+}
+function isTimeoutError(err) {
+  if (!err || typeof err.message !== "string") return false;
+  return err.message.includes("Timeout") || err.message.includes("timeout") || err.name === "TimeoutError";
+}
+const LOCATOR_FACTORIES = [
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId"
+];
+async function installInstrumentation(page, options) {
+  if (page.__librettoInstrumented) return;
+  page.__librettoInstrumented = true;
+  const visualize = options?.visualize ?? false;
+  const highlightBeforeActionMs = options?.highlightBeforeActionMs ?? 350;
+  const mergedOpts = { ...options, visualize, highlightBeforeActionMs };
+  if (visualize) {
+    await (0, import_ghost_cursor.ensureGhostCursor)(page, options?.ghostCursor);
+    await (0, import_highlight.ensureHighlightLayer)(page, options?.highlight);
+  }
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      if (visualize && typeof args[0] === "string") {
+        await enqueue(page, async () => {
+          try {
+            const loc = page.locator(args[0]);
+            const box = await loc.boundingBox();
+            await visualizeBeforeAction(page, box, method, highlightBeforeActionMs);
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err) && typeof args[0] === "string") {
+          await (0, import_errors.enrichTimeoutError)(err, page.locator(args[0]), page);
+        }
+        throw err;
+      }
+    };
+  }
+  for (const method of NAV_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      options?.logger?.info(`instrumentation:${method}`, {
+        url: typeof args[0] === "string" ? args[0] : page.url()
+      });
+      return orig(...args);
+    };
+  }
+  for (const factory of LOCATOR_FACTORIES) {
+    if (typeof page[factory] !== "function") continue;
+    const origFactory = page[factory].bind(page);
+    page[factory] = (...factoryArgs) => {
+      const locator = origFactory(...factoryArgs);
+      wrapLocatorActions(locator, page, mergedOpts);
+      return locator;
+    };
+  }
+}
+async function instrumentPage(page, options) {
+  await installInstrumentation(page, options);
+  return page;
+}
+async function instrumentContext(context, options) {
+  for (const page of context.pages()) {
+    await installInstrumentation(page, options);
+  }
+  context.on("page", async (page) => {
+    await installInstrumentation(page, options);
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  installInstrumentation,
+  instrumentContext,
+  instrumentPage
+});

package/dist/shared/instrumentation/instrument.d.cts

@@ -0,0 +1,32 @@
+import { Page, BrowserContext } from 'playwright';
+import { LoggerApi } from '../logger/logger.cjs';
+import { GhostCursorOptions } from '../visualization/ghost-cursor.cjs';
+import { HighlightOptions } from '../visualization/highlight.cjs';
+
+type InstrumentationOptions = {
+    visualize?: boolean;
+    logger?: LoggerApi;
+    highlightBeforeActionMs?: number;
+    ghostCursor?: GhostCursorOptions;
+    highlight?: HighlightOptions;
+};
+type InstrumentedPage = Page & {
+    __librettoInstrumented: true;
+};
+/**
+ * In-place patching of a Page object to add visualization and error enrichment.
+ * Modifies the page directly (does not return a new object).
+ */
+declare function installInstrumentation(page: Page, options?: InstrumentationOptions): Promise<void>;
+/**
+ * Returns a new object that proxies to the page with instrumentation applied.
+ * The original page is not modified.
+ */
+declare function instrumentPage(page: Page, options?: InstrumentationOptions): Promise<InstrumentedPage>;
+/**
+ * Install overlays on a page and auto-install on all future pages in the context.
+ * Useful when connecting to an existing browser via CDP.
+ */
+declare function instrumentContext(context: BrowserContext, options?: InstrumentationOptions): Promise<void>;
+
+export { type InstrumentationOptions, type InstrumentedPage, installInstrumentation, instrumentContext, instrumentPage };

package/dist/shared/instrumentation/instrument.d.ts

@@ -0,0 +1,32 @@
+import { Page, BrowserContext } from 'playwright';
+import { LoggerApi } from '../logger/logger.js';
+import { GhostCursorOptions } from '../visualization/ghost-cursor.js';
+import { HighlightOptions } from '../visualization/highlight.js';
+
+type InstrumentationOptions = {
+    visualize?: boolean;
+    logger?: LoggerApi;
+    highlightBeforeActionMs?: number;
+    ghostCursor?: GhostCursorOptions;
+    highlight?: HighlightOptions;
+};
+type InstrumentedPage = Page & {
+    __librettoInstrumented: true;
+};
+/**
+ * In-place patching of a Page object to add visualization and error enrichment.
+ * Modifies the page directly (does not return a new object).
+ */
+declare function installInstrumentation(page: Page, options?: InstrumentationOptions): Promise<void>;
+/**
+ * Returns a new object that proxies to the page with instrumentation applied.
+ * The original page is not modified.
+ */
+declare function instrumentPage(page: Page, options?: InstrumentationOptions): Promise<InstrumentedPage>;
+/**
+ * Install overlays on a page and auto-install on all future pages in the context.
+ * Useful when connecting to an existing browser via CDP.
+ */
+declare function instrumentContext(context: BrowserContext, options?: InstrumentationOptions): Promise<void>;
+
+export { type InstrumentationOptions, type InstrumentedPage, installInstrumentation, instrumentContext, instrumentPage };

package/dist/shared/instrumentation/instrument.js

@@ -0,0 +1,190 @@
+import {
+  ensureGhostCursor,
+  moveGhostCursor,
+  moveGhostCursorWithDistance,
+  ghostClick,
+  getGhostCursorPosition
+} from "../visualization/ghost-cursor.js";
+import {
+  ensureHighlightLayer,
+  showHighlight,
+  clearHighlights
+} from "../visualization/highlight.js";
+import { enrichTimeoutError } from "./errors.js";
+const LOCATOR_ACTIONS = [
+  "click",
+  "dblclick",
+  "hover",
+  "fill",
+  "type",
+  "press",
+  "check",
+  "uncheck",
+  "selectOption",
+  "focus"
+];
+const NAV_ACTIONS = ["goto", "reload", "goBack", "goForward"];
+const POINTER_ACTIONS = /* @__PURE__ */ new Set(["click", "dblclick", "hover"]);
+const pageQueues = /* @__PURE__ */ new WeakMap();
+function enqueue(page, fn) {
+  const prev = pageQueues.get(page) ?? Promise.resolve();
+  const next = prev.then(fn, fn);
+  pageQueues.set(page, next);
+  return next;
+}
+async function visualizeBeforeAction(page, box, actionName, highlightMs) {
+  if (!box) return;
+  await ensureGhostCursor(page);
+  await ensureHighlightLayer(page);
+  const centerX = box.x + box.width / 2;
+  const centerY = box.y + box.height / 2;
+  await showHighlight(page, {
+    box,
+    durationMs: highlightMs + 200
+    // keep visible a bit past the cursor arrival
+  });
+  const cursorPos = await getGhostCursorPosition(page);
+  if (cursorPos) {
+    await moveGhostCursorWithDistance(page, cursorPos, {
+      x: centerX,
+      y: centerY
+    });
+  } else {
+    await moveGhostCursor(page, { x: centerX, y: centerY, durationMs: 200 });
+  }
+  if (actionName === "click" || actionName === "dblclick") {
+    await ghostClick(page, { x: centerX, y: centerY });
+  }
+}
+async function visualizeAfterAction(page) {
+  await clearHighlights(page);
+}
+function wrapLocatorActions(locator, page, opts) {
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof locator[method] !== "function") continue;
+    const orig = locator[method].bind(locator);
+    locator[method] = async (...args) => {
+      if (opts.visualize) {
+        await enqueue(page, async () => {
+          try {
+            const box = await locator.boundingBox();
+            await visualizeBeforeAction(
+              page,
+              box,
+              method,
+              opts.highlightBeforeActionMs
+            );
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (opts.visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err)) {
+          await enrichTimeoutError(err, locator, page);
+        }
+        throw err;
+      }
+    };
+  }
+}
+function isTimeoutError(err) {
+  if (!err || typeof err.message !== "string") return false;
+  return err.message.includes("Timeout") || err.message.includes("timeout") || err.name === "TimeoutError";
+}
+const LOCATOR_FACTORIES = [
+  "locator",
+  "getByRole",
+  "getByText",
+  "getByLabel",
+  "getByPlaceholder",
+  "getByAltText",
+  "getByTitle",
+  "getByTestId"
+];
+async function installInstrumentation(page, options) {
+  if (page.__librettoInstrumented) return;
+  page.__librettoInstrumented = true;
+  const visualize = options?.visualize ?? false;
+  const highlightBeforeActionMs = options?.highlightBeforeActionMs ?? 350;
+  const mergedOpts = { ...options, visualize, highlightBeforeActionMs };
+  if (visualize) {
+    await ensureGhostCursor(page, options?.ghostCursor);
+    await ensureHighlightLayer(page, options?.highlight);
+  }
+  for (const method of LOCATOR_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      if (visualize && typeof args[0] === "string") {
+        await enqueue(page, async () => {
+          try {
+            const loc = page.locator(args[0]);
+            const box = await loc.boundingBox();
+            await visualizeBeforeAction(page, box, method, highlightBeforeActionMs);
+          } catch {
+          }
+        });
+      }
+      try {
+        const result = await orig(...args);
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        return result;
+      } catch (err) {
+        if (visualize) {
+          enqueue(page, () => visualizeAfterAction(page));
+        }
+        if (POINTER_ACTIONS.has(method) && isTimeoutError(err) && typeof args[0] === "string") {
+          await enrichTimeoutError(err, page.locator(args[0]), page);
+        }
+        throw err;
+      }
+    };
+  }
+  for (const method of NAV_ACTIONS) {
+    if (typeof page[method] !== "function") continue;
+    const orig = page[method].bind(page);
+    page[method] = async (...args) => {
+      options?.logger?.info(`instrumentation:${method}`, {
+        url: typeof args[0] === "string" ? args[0] : page.url()
+      });
+      return orig(...args);
+    };
+  }
+  for (const factory of LOCATOR_FACTORIES) {
+    if (typeof page[factory] !== "function") continue;
+    const origFactory = page[factory].bind(page);
+    page[factory] = (...factoryArgs) => {
+      const locator = origFactory(...factoryArgs);
+      wrapLocatorActions(locator, page, mergedOpts);
+      return locator;
+    };
+  }
+}
+async function instrumentPage(page, options) {
+  await installInstrumentation(page, options);
+  return page;
+}
+async function instrumentContext(context, options) {
+  for (const page of context.pages()) {
+    await installInstrumentation(page, options);
+  }
+  context.on("page", async (page) => {
+    await installInstrumentation(page, options);
+  });
+}
+export {
+  installInstrumentation,
+  instrumentContext,
+  instrumentPage
+};

package/dist/shared/llm/client.cjs

@@ -0,0 +1,139 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var client_exports = {};
+__export(client_exports, {
+  createLLMClient: () => createLLMClient
+});
+module.exports = __toCommonJS(client_exports);
+var import_google_vertex = require("@ai-sdk/google-vertex");
+var import_anthropic = require("@ai-sdk/anthropic");
+var import_openai = require("@ai-sdk/openai");
+var import_ai = require("ai");
+function parseModel(model) {
+  const slashIndex = model.indexOf("/");
+  if (slashIndex === -1) {
+    throw new Error(
+      `Invalid model string "${model}". Expected format: "provider/model-id" (e.g. "google/gemini-3-flash-preview").`
+    );
+  }
+  const provider = model.slice(0, slashIndex);
+  const modelId = model.slice(slashIndex + 1);
+  if (!["google", "anthropic", "openai"].includes(provider)) {
+    throw new Error(
+      `Unsupported provider "${provider}". Supported providers: google, anthropic, openai.`
+    );
+  }
+  return { provider, modelId };
+}
+function getProviderModel(provider, modelId) {
+  switch (provider) {
+    case "google": {
+      const project = process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT;
+      if (!project) {
+        throw new Error(
+          "Missing GCP project for Vertex AI. Set GOOGLE_CLOUD_PROJECT environment variable and ensure application default credentials are configured (gcloud auth application-default login)."
+        );
+      }
+      const vertex = (0, import_google_vertex.createVertex)({
+        project,
+        location: process.env.GOOGLE_CLOUD_LOCATION || "global"
+      });
+      return vertex(modelId);
+    }
+    case "anthropic": {
+      const apiKey = process.env.ANTHROPIC_API_KEY;
+      if (!apiKey) {
+        throw new Error(
+          "Missing API key for Anthropic. Set ANTHROPIC_API_KEY environment variable."
+        );
+      }
+      const anthropic = (0, import_anthropic.createAnthropic)({ apiKey });
+      return anthropic(modelId);
+    }
+    case "openai": {
+      const apiKey = process.env.OPENAI_API_KEY;
+      if (!apiKey) {
+        throw new Error(
+          "Missing API key for OpenAI. Set OPENAI_API_KEY environment variable."
+        );
+      }
+      const openai = (0, import_openai.createOpenAI)({ apiKey });
+      return openai(modelId);
+    }
+  }
+}
+function convertUserContentParts(parts) {
+  return parts.map((part) => {
+    if (part.type === "text") {
+      return { type: "text", text: part.text };
+    }
+    return { type: "image", image: part.image };
+  });
+}
+function convertAssistantContentParts(parts) {
+  return parts.filter((part) => part.type === "text").map((part) => ({ type: "text", text: part.text }));
+}
+function convertMessages(messages) {
+  return messages.map((msg) => {
+    if (msg.role === "user") {
+      if (typeof msg.content === "string") {
+        return { role: "user", content: msg.content };
+      }
+      return {
+        role: "user",
+        content: convertUserContentParts(msg.content)
+      };
+    }
+    if (typeof msg.content === "string") {
+      return { role: "assistant", content: msg.content };
+    }
+    return {
+      role: "assistant",
+      content: convertAssistantContentParts(msg.content)
+    };
+  });
+}
+function createLLMClient(model) {
+  const { provider, modelId } = parseModel(model);
+  const aiModel = getProviderModel(provider, modelId);
+  return {
+    async generateObject(opts) {
+      const result = await (0, import_ai.generateObject)({
+        model: aiModel,
+        prompt: opts.prompt,
+        schema: opts.schema,
+        temperature: opts.temperature ?? 0
+      });
+      return result.object;
+    },
+    async generateObjectFromMessages(opts) {
+      const result = await (0, import_ai.generateObject)({
+        model: aiModel,
+        messages: convertMessages(opts.messages),
+        schema: opts.schema,
+        temperature: opts.temperature ?? 0
+      });
+      return result.object;
+    }
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  createLLMClient
+});

package/dist/shared/llm/client.d.cts

@@ -0,0 +1,6 @@
+import { LLMClient } from './types.cjs';
+import 'zod';
+
+declare function createLLMClient(model: string): LLMClient;
+
+export { createLLMClient };

package/dist/shared/llm/client.d.ts

@@ -0,0 +1,6 @@
+import { LLMClient } from './types.js';
+import 'zod';
+
+declare function createLLMClient(model: string): LLMClient;
+
+export { createLLMClient };