libretto 0.1.5 → 0.2.1

package/README.md

@@ -1,33 +1,231 @@
-Libretto loads fixture data into your mongodb database.
+# Libretto
 
-Exporting:
+A TypeScript library for browser automation with AI-powered recovery and data extraction. Built on Playwright.
+
+## Features
+
+- **AI-powered recovery** — Vision-based agent that automatically detects and dismisses popups or obstacles using an LLM
+- **Structured data extraction** — Extract typed data from web pages using AI vision + Zod schemas
+- **Error detection** — Classify form/submission errors against known patterns
+- **In-browser network requests** — Execute authenticated fetch calls inside the page context with optional Zod validation
+- **File downloads** — Trigger and intercept file downloads via click, with optional save-to-disk
+- **Dry-run mode** — Skip mutations in development without side effects
+- **Pluggable LLM** — Bring your own LLM provider (Claude, GPT, etc.) via a simple interface
+- **Pluggable logging** — All runtime functions accept an optional logger; defaults to console output
+
+## Installation
 
 ```bash
-libretto export /path/to/fixtures/dir --database=app-testing
+pnpm add libretto playwright zod
 ```
 
-Importing:
+`playwright` and `zod` are peer dependencies.
 
-```bash
-libretto import "/path/to/fixtures/dir/*" --database=app-testing
+## Quick Start
+
+```typescript
+import { chromium } from "playwright";
+import { extractFromPage, attemptWithRecovery } from "libretto";
+
+const browser = await chromium.launch();
+const page = await browser.newPage();
+
+await page.goto("https://example.com/login");
+await page.fill("#email", "user@example.com");
+await page.fill("#password", "secret");
+
+// Automatically retry with AI popup recovery on failure
+await attemptWithRecovery(page, () => page.click('button[type="submit"]'));
+
+await browser.close();
+```
+
+## Runtime Functions
+
+### Recovery
+
+#### `attemptWithRecovery(page, fn, logger?, llmClient?)`
+
+Executes a function and, if it fails, uses AI vision to detect and dismiss popups before retrying once.
+
+```typescript
+import { attemptWithRecovery } from "libretto";
+
+await attemptWithRecovery(page, async () => {
+  await page.click('button[type="submit"]');
+}, undefined, llmClient);
+```
+
+#### `executeRecoveryAgent(page, instruction, logger?, llmClient?)`
+
+Runs a multi-step vision-based recovery agent that takes screenshots and executes browser actions (click, type, scroll, etc.) to resolve obstacles.
+
+```typescript
+import { executeRecoveryAgent } from "libretto";
+
+await executeRecoveryAgent(
+  page,
+  "Close the cookie consent banner",
+  undefined,
+  llmClient,
+);
+```
+
+#### `detectSubmissionError(page, error, logContext, llmClient, knownErrors?, logger?)`
+
+Uses a screenshot + LLM vision to detect if an error occurred during a form submission. Matches against provided known error patterns.
+
+```typescript
+import { detectSubmissionError } from "libretto";
+
+try {
+  await page.click("#submit");
+} catch (error) {
+  const result = await detectSubmissionError(page, error, "checkout", llmClient, [
+    { id: "duplicate", errorPatterns: ["already exists"], userMessage: "Duplicate entry" },
+  ]);
+  console.log(result.errorId, result.message);
+}
+```
+
+### Data Extraction
+
+#### `extractFromPage(options)`
+
+Extract structured data from a page using AI vision + a Zod schema.
+
+```typescript
+import { extractFromPage } from "libretto";
+import { z } from "zod";
+
+const result = await extractFromPage({
+  page,
+  llmClient,
+  instruction: "Extract the product name and price",
+  schema: z.object({
+    name: z.string(),
+    price: z.number(),
+  }),
+  selector: ".product-card", // optional — scopes to a specific element
+});
+// result is typed as { name: string; price: number }
+```
+
+### Network
+
+#### `pageRequest(page, config, options?)`
+
+Executes a fetch call inside the browser context via `page.evaluate()`, inheriting the page's cookies and auth state. Supports optional Zod validation.
+
+```typescript
+import { pageRequest } from "libretto";
+import { z } from "zod";
+
+const data = await pageRequest(
+  page,
+  {
+    url: "https://example.com/api/profile",
+    method: "GET",
+    responseType: "json",
+  },
+  {
+    schema: z.object({ name: z.string(), email: z.string() }),
+  },
+);
+```
+
+### Downloads
+
+#### `downloadViaClick(page, selector, options?)`
+
+Triggers a file download by clicking a DOM element and intercepts the result.
+
+```typescript
+import { downloadViaClick } from "libretto";
+
+const { buffer, filename } = await downloadViaClick(page, "#download-btn");
 ```
 
-Using in tests:
+#### `downloadAndSave(page, selector, options?)`
+
+Same as `downloadViaClick` but also writes the file to disk.
 
+```typescript
+import { downloadAndSave } from "libretto";
 
-```javascript
-var exec = require("child_process").exec;
+const { savedTo } = await downloadAndSave(page, "#export-csv", {
+  savePath: "./exports/report.csv",
+});
+```
 
-describe("test#", function() {
+## LLM Client Interface
 
-  //load the fixtures
-  before(function(next) {
-    exec("./node_modules/.bin/libretto import " + __dirname + "/fixtures/scenario1/* --database=app-testing", next);
-  });
+Provide your own implementation backed by any LLM provider:
 
-  it("do some test", function(next) {
+```typescript
+import type { LLMClient } from "libretto";
 
-  });
+const myLLMClient: LLMClient = {
+  async generateObject({ prompt, schema, temperature }) {
+    // Call your LLM, return parsed + validated result
+  },
+  async generateObjectFromMessages({ messages, schema, temperature }) {
+    // Call your LLM with message history, return parsed + validated result
+  },
+};
+```
 
-})
+## Logging
+
+All runtime functions accept an optional `logger` parameter. When omitted, output goes to `console.log` with `[INFO]`, `[WARN]`, `[ERROR]` prefixes.
+
+For structured logging, use the built-in `Logger` class:
+
+```typescript
+import { Logger, createFileLogSink, prettyConsoleSink } from "libretto";
+
+const logger = new Logger()
+  .withSink(createFileLogSink({ filePath: "./app.log" }))
+  .withSink(prettyConsoleSink);
+
+const scoped = logger.withScope("auth");
+scoped.info("login attempt", { user: "alice" });
+scoped.error("login failed", { reason: "bad password" });
+```
+
+## Module Exports
+
+Libretto provides granular imports:
+
+| Import                   | Contents                                                  |
+| ------------------------ | --------------------------------------------------------- |
+| `libretto`               | Everything                                                |
+| `libretto/logger`        | `Logger`, `defaultLogger`, sinks                          |
+| `libretto/recovery`      | `attemptWithRecovery`, `executeRecoveryAgent`, `detectSubmissionError` |
+| `libretto/extract`       | `extractFromPage`                                         |
+| `libretto/network`       | `pageRequest`                                             |
+| `libretto/download`      | `downloadViaClick`, `downloadAndSave`                     |
+| `libretto/debug`         | `debugPause`                                              |
+| `libretto/config`        | `isDryRun`, `isDebugMode`, `shouldPauseBeforeMutation`    |
+| `libretto/instrumentation` | `instrumentPage`, `installInstrumentation`              |
+| `libretto/visualization` | Ghost cursor and highlight helpers                        |
+| `libretto/run`           | `launchBrowser`                                           |
+| `libretto/state`         | Session state serialization and parsing                   |
+| `libretto/llm`           | `LLMClient` type                                          |
+
+## Configuration
+
+Runtime flags via environment variables:
+
+| Env Variable          | Effect                                              |
+| --------------------- | --------------------------------------------------- |
+| `LIBRETTO_DEBUG`      | Enable debug mode                                   |
+| `LIBRETTO_DRY_RUN`   | Enable dry-run mode (defaults to `true` in development) |
+
+## Development
+
+```bash
+pnpm install
+pnpm build         # compile to dist/
+pnpm type-check    # typecheck without emitting
 ```

package/bin/libretto.mjs

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+import { existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { execSync } from "node:child_process";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const cliRoot = join(__dirname, "..");
+const distEntry = join(cliRoot, "dist", "cli", "index.js");
+
+if (!existsSync(distEntry)) {
+  // Build from the libretto monorepo root (builds core then CLI in order)
+  const monorepoRoot = join(cliRoot, "..", "..");
+  console.error("[libretto] dist not found, building...");
+  execSync("pnpm build", { cwd: monorepoRoot, stdio: "inherit" });
+}
+
+await import(distEntry);

package/dist/cli/cli.js

@@ -0,0 +1,203 @@
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { registerAICommands } from "./commands/ai.js";
+import { registerBrowserCommands } from "./commands/browser.js";
+import { registerExecutionCommands } from "./commands/execution.js";
+import { registerLogCommands } from "./commands/logs.js";
+import { registerSnapshotCommands } from "./commands/snapshot.js";
+import {
+  closeLogger,
+  createLoggerForSession,
+  ensureLibrettoSetup
+} from "./core/context.js";
+import {
+  SESSION_DEFAULT,
+  validateSessionName
+} from "./core/session.js";
+const CLI_COMMANDS = /* @__PURE__ */ new Set([
+  "open",
+  "run",
+  "ai",
+  "save",
+  "exec",
+  "snapshot",
+  "network",
+  "actions",
+  "pages",
+  "resume",
+  "close",
+  "--help",
+  "-h",
+  "help"
+]);
+function printUsage() {
+  console.log(`Usage: libretto-cli <command> [--session <name>]
+
+Commands:
+  open <url> [--headless] Launch browser and open URL (headed by default)
+                          Automatically loads saved profile if available
+  run <integrationFile> <integrationExport> [--params <json> | --params-file <path>] [--headed|--headless] [--debug]  Run an exported Libretto workflow from a file; pass --debug to enable pause-on-failure debugging (or --no-debug to disable)
+  ai configure [preset] [-- <command prefix...>]  Configure AI runtime for analysis commands
+  save <url|domain>       Save current browser session (cookies, localStorage, etc.)
+  exec <code> [--visualize]  Execute Playwright typescript code (--visualize enables ghost cursor + highlight)
+  snapshot [--objective <text> --context <text>]  Capture PNG + HTML; analyze when objective is provided (context optional)
+  network [--last N] [--filter regex] [--method M] [--clear]  View captured network requests
+  actions [--last N] [--filter regex] [--action TYPE] [--source SOURCE] [--clear]  View captured actions
+  pages                   List open pages in the active session
+  resume                  Resume a paused workflow in the active session
+  close                   Close the browser
+
+Options:
+  --session <name>        Use a named session (default: "default")
+                          Built-in sessions: default, dev-server, browser-agent
+
+Examples:
+  libretto-cli open https://linkedin.com
+
+  # ... manually log in ...
+  libretto-cli save linkedin.com
+  # Next time you open linkedin.com, you'll be logged in automatically
+
+  libretto-cli exec "await page.locator('button:has-text(\\"Sign in\\")').click()"
+  libretto-cli exec "await page.fill('input[name=\\"email\\"]', 'test@example.com')"
+  libretto-cli ai configure codex
+  libretto-cli ai configure claude
+  libretto-cli ai configure gemini
+  libretto-cli ai configure <codex|claude|gemini> -- <command prefix...>
+  libretto-cli snapshot
+  libretto-cli snapshot --objective "Find the submit button" --context "Submitting a referral form, already filled in patient details"
+  libretto-cli resume --session default
+  libretto-cli close
+
+  # Multiple sessions
+  libretto-cli open https://site1.com --session test1
+  libretto-cli open https://site2.com --session test2
+  libretto-cli exec "return await page.title()" --session test1
+
+Available in exec:
+  page, context, state, browser, networkLog, actionLog
+
+Profiles:
+  Profiles are saved to .libretto/profiles/<domain>.json (git-ignored)
+  They persist cookies, localStorage, and session data across browser launches.
+  Local profiles are machine-local and are not shared with other users/environments.
+  Sessions can expire; if run fails auth, log in again and re-save the profile.
+
+Sessions:
+  Session state is stored in .libretto/sessions/<session>/state.json
+  CLI logs are stored in .libretto/sessions/<session>/logs.jsonl
+  Each session runs an isolated browser instance on a dynamic port.
+`);
+}
+function filterSessionArgs(args) {
+  const result = [];
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === "--session") {
+      i++;
+    } else {
+      result.push(args[i]);
+    }
+  }
+  return result;
+}
+function parseSessionForLog(rawArgs) {
+  const idx = rawArgs.indexOf("--session");
+  if (idx < 0) return SESSION_DEFAULT;
+  const value = rawArgs[idx + 1];
+  if (!value || value.startsWith("--") || CLI_COMMANDS.has(value)) {
+    return SESSION_DEFAULT;
+  }
+  try {
+    validateSessionName(value);
+    return value;
+  } catch {
+    return SESSION_DEFAULT;
+  }
+}
+function validateLegacySessionArg(rawArgs) {
+  const idx = rawArgs.indexOf("--session");
+  if (idx < 0) return;
+  const value = rawArgs[idx + 1];
+  if (!value || value.startsWith("--") || CLI_COMMANDS.has(value)) {
+    throw new Error(
+      "Usage: libretto-cli <command> [--session <name>]\nMissing or invalid --session value."
+    );
+  }
+  validateSessionName(value);
+}
+function initializeLogger(rawArgs) {
+  const sessionForLog = parseSessionForLog(rawArgs);
+  const logger = createLoggerForSession(sessionForLog);
+  logger.info("cli-start", {
+    args: rawArgs,
+    cwd: process.cwd(),
+    session: sessionForLog
+  });
+  return logger;
+}
+async function withCliLogger(rawArgs, run) {
+  const logger = initializeLogger(rawArgs);
+  try {
+    return await run(logger);
+  } finally {
+    await closeLogger(logger);
+  }
+}
+function createParser(logger) {
+  let parser = yargs(hideBin(process.argv)).scriptName("libretto-cli").parserConfiguration({ "populate--": true }).option("session", {
+    type: "string",
+    default: SESSION_DEFAULT,
+    describe: "Use a named session",
+    global: true
+  }).middleware((argv) => {
+    validateSessionName(String(argv.session));
+  }).exitProcess(false).help(false).version(false).fail((msg, err) => {
+    if (err) throw err;
+    throw new Error(msg || "Command failed");
+  });
+  parser = registerBrowserCommands(parser, logger);
+  parser = registerExecutionCommands(parser, logger);
+  parser = registerLogCommands(parser);
+  parser = registerAICommands(parser);
+  parser = registerSnapshotCommands(parser, logger);
+  parser = parser.command("help", "Show usage", () => {
+  }, () => {
+    printUsage();
+  });
+  return parser;
+}
+async function runLibrettoCLI() {
+  const rawArgs = process.argv.slice(2);
+  let exitCode = 0;
+  ensureLibrettoSetup();
+  await withCliLogger(rawArgs, async (logger) => {
+    try {
+      validateLegacySessionArg(rawArgs);
+      const args = filterSessionArgs(rawArgs);
+      const command = args[0];
+      if (!command || command === "--help" || command === "-h" || command === "help") {
+        printUsage();
+        return;
+      }
+      if (!CLI_COMMANDS.has(command)) {
+        console.error(`Unknown command: ${command}
+`);
+        printUsage();
+        exitCode = 1;
+        return;
+      }
+      const parser = createParser(logger);
+      logger.info("cli-command", { command, args });
+      await parser.parseAsync();
+    } catch (err) {
+      logger.error("cli-error", { error: err, args: rawArgs });
+      const message = err instanceof Error ? err.message : String(err);
+      console.error(message);
+      exitCode = 1;
+    }
+  });
+  process.exit(exitCode);
+}
+export {
+  runLibrettoCLI
+};

package/dist/cli/commands/ai.js

@@ -0,0 +1,21 @@
+import { runAiConfigure } from "../core/ai-config.js";
+function registerAICommands(yargs) {
+  return yargs.command(
+    "ai configure [preset]",
+    "Configure AI runtime",
+    (cmd) => cmd.option("clear", { type: "boolean", default: false }),
+    (argv) => {
+      const customPrefix = Array.isArray(argv["--"]) ? argv["--"] : [];
+      runAiConfigure({
+        clear: Boolean(argv.clear),
+        preset: argv.preset,
+        customPrefix
+      }, {
+        configureCommandName: "libretto-cli ai configure"
+      });
+    }
+  );
+}
+export {
+  registerAICommands
+};

package/dist/cli/commands/browser.js

@@ -0,0 +1,59 @@
+import {
+  runClose as runCloseWithLogger,
+  runOpen,
+  runPages,
+  runSave
+} from "../core/browser.js";
+import { withSessionLogger } from "../core/context.js";
+function registerBrowserCommands(yargs, logger) {
+  return yargs.command(
+    "open [url]",
+    "Launch browser and open URL (headed by default)",
+    (cmd) => cmd.option("headed", {
+      type: "boolean",
+      default: false
+    }).option("headless", {
+      type: "boolean",
+      default: false
+    }),
+    async (argv) => {
+      const hasHeadedFlag = Boolean(argv.headed);
+      const hasHeadlessFlag = Boolean(argv.headless);
+      if (hasHeadedFlag && hasHeadlessFlag) {
+        throw new Error("Cannot pass both --headed and --headless.");
+      }
+      const headed = hasHeadedFlag || !hasHeadlessFlag;
+      const url = argv.url;
+      if (!url) {
+        throw new Error(
+          "Usage: libretto-cli open <url> [--headless] [--session <name>]"
+        );
+      }
+      await runOpen(url, headed, String(argv.session), logger);
+    }
+  ).command(
+    "save [urlOrDomain]",
+    "Save current browser session",
+    (cmd) => cmd,
+    async (argv) => {
+      const urlOrDomain = argv.urlOrDomain;
+      if (!urlOrDomain) {
+        throw new Error("Usage: libretto-cli save <url|domain> [--session <name>]");
+      }
+      await runSave(urlOrDomain, String(argv.session), logger);
+    }
+  ).command("pages", "List open pages in the session", (cmd) => cmd, async (argv) => {
+    await runPages(String(argv.session), logger);
+  }).command("close", "Close the browser", (cmd) => cmd, async (argv) => {
+    await runCloseWithLogger(String(argv.session), logger);
+  });
+}
+async function runClose(session) {
+  await withSessionLogger(session, async (logger) => {
+    await runCloseWithLogger(session, logger);
+  });
+}
+export {
+  registerBrowserCommands,
+  runClose
+};