libretto 0.4.4 → 0.5.0

package/dist/shared/visualization/index.cjs

@@ -1,45 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var visualization_exports = {};
-__export(visualization_exports, {
-  clearHighlights: () => import_highlight.clearHighlights,
-  ensureGhostCursor: () => import_ghost_cursor.ensureGhostCursor,
-  ensureHighlightLayer: () => import_highlight.ensureHighlightLayer,
-  getGhostCursorPosition: () => import_ghost_cursor.getGhostCursorPosition,
-  ghostClick: () => import_ghost_cursor.ghostClick,
-  hideGhostCursor: () => import_ghost_cursor.hideGhostCursor,
-  moveGhostCursor: () => import_ghost_cursor.moveGhostCursor,
-  moveGhostCursorWithDistance: () => import_ghost_cursor.moveGhostCursorWithDistance,
-  showHighlight: () => import_highlight.showHighlight
-});
-module.exports = __toCommonJS(visualization_exports);
-var import_ghost_cursor = require("./ghost-cursor.js");
-var import_highlight = require("./highlight.js");
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  clearHighlights,
-  ensureGhostCursor,
-  ensureHighlightLayer,
-  getGhostCursorPosition,
-  ghostClick,
-  hideGhostCursor,
-  moveGhostCursor,
-  moveGhostCursorWithDistance,
-  showHighlight
-});

package/dist/shared/visualization/index.d.cts

@@ -1,3 +0,0 @@
-export { GhostCursorOptions, GhostCursorStyle, ensureGhostCursor, getGhostCursorPosition, ghostClick, hideGhostCursor, moveGhostCursor, moveGhostCursorWithDistance } from './ghost-cursor.cjs';
-export { HighlightOptions, ShowHighlightParams, clearHighlights, ensureHighlightLayer, showHighlight } from './highlight.cjs';
-import 'playwright';

package/dist/shared/workflow/workflow.cjs

@@ -1,47 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var workflow_exports = {};
-__export(workflow_exports, {
-  LIBRETTO_WORKFLOW_BRAND: () => LIBRETTO_WORKFLOW_BRAND,
-  LibrettoWorkflow: () => LibrettoWorkflow,
-  workflow: () => workflow
-});
-module.exports = __toCommonJS(workflow_exports);
-const LIBRETTO_WORKFLOW_BRAND = /* @__PURE__ */ Symbol.for("libretto.workflow");
-class LibrettoWorkflow {
-  [LIBRETTO_WORKFLOW_BRAND] = true;
-  metadata;
-  handler;
-  constructor(metadata, handler) {
-    this.metadata = metadata;
-    this.handler = handler;
-  }
-  async run(ctx, input) {
-    return this.handler(ctx, input);
-  }
-}
-function workflow(metadata, handler) {
-  return new LibrettoWorkflow(metadata, handler);
-}
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  LIBRETTO_WORKFLOW_BRAND,
-  LibrettoWorkflow,
-  workflow
-});

package/dist/shared/workflow/workflow.d.cts

@@ -1,21 +0,0 @@
-import { Page } from 'playwright';
-import { MinimalLogger } from '../logger/logger.cjs';
-
-declare const LIBRETTO_WORKFLOW_BRAND: unique symbol;
-type LibrettoWorkflowMetadata = {};
-type LibrettoWorkflowContext<S = {}> = {
-    page: Page;
-    logger: MinimalLogger;
-    services: S;
-};
-type LibrettoWorkflowHandler<Input = unknown, Output = unknown, S = {}> = (ctx: LibrettoWorkflowContext<S>, input: Input) => Promise<Output>;
-declare class LibrettoWorkflow<Input = unknown, Output = unknown, S = {}> {
-    readonly [LIBRETTO_WORKFLOW_BRAND] = true;
-    readonly metadata: LibrettoWorkflowMetadata;
-    private readonly handler;
-    constructor(metadata: LibrettoWorkflowMetadata, handler: LibrettoWorkflowHandler<Input, Output, S>);
-    run(ctx: LibrettoWorkflowContext<S>, input: Input): Promise<Output>;
-}
-declare function workflow<Input = unknown, Output = unknown, S = {}>(metadata: LibrettoWorkflowMetadata, handler: LibrettoWorkflowHandler<Input, Output, S>): LibrettoWorkflow<Input, Output, S>;
-
-export { LIBRETTO_WORKFLOW_BRAND, LibrettoWorkflow, type LibrettoWorkflowContext, type LibrettoWorkflowHandler, type LibrettoWorkflowMetadata, workflow };

package/skills/libretto/code-generation-rules.md

@@ -1,223 +0,0 @@
-# Code Generation Rules
-
-These rules apply when generating production TypeScript files from interactive browser sessions. Read this file before writing any production code.
-
-## Workflow File Structure
-
-Generated files must export a `workflow()` instance so they can be run via `npx libretto run <file> <exportName>`. Import `workflow` and its types from `"libretto"`:
-
-```typescript
-import { workflow, pause, type LibrettoWorkflowContext } from "libretto";
-
-type Input = {
-  // Define the expected input shape — passed via --params JSON
-  query: string;
-  maxResults?: number;
-};
-
-type Output = {
-  // Define what the workflow returns
-  results: Array<{ name: string; value: string }>;
-};
-
-export const myWorkflow = workflow<Input, Output>(
-  {},
-  async (ctx, input): Promise<Output> => {
-    const { page } = ctx;
-
-    // workflow logic here — use ctx.page, ctx.logger, ctx.services
-    await page.goto("https://example.com");
-    // ...
-
-    return { results: [] };
-  },
-);
-```
-
-**Key points:**
-
-- The named export (e.g., `myWorkflow`) is what you pass as the second arg to `npx libretto run ./file.ts myWorkflow`
-- `ctx` provides `page`, `logger`, and `services` (generic, default `{}`)
-- `input` comes from `--params '{"query":"foo"}'` or `--params-file params.json` on the CLI
-- If the site requires a saved login session, pass `--auth-profile <domain>` to the CLI (created via `npx libretto save <domain>`)
-- Use `await pause()` (imported from `"libretto"`) to pause the workflow for debugging. It is a no-op in production.
-- The browser is launched and closed automatically by the CLI — do not launch or close it in the handler
-
-## Passing Application Dependencies via Services
-
-Use the third generic on `workflow<Input, Output, Services>` to inject
-dependencies that exist in your application but not in libretto's runtime
-(DB transactions, API clients, caches, etc.):
-
-```typescript
-import { type Transaction } from "./db";
-
-type MyServices = { tx?: Transaction };
-
-export const myWorkflow = workflow<Input, Output, MyServices>(
-  {},
-  async (ctx, input) => {
-    if (ctx.services.tx) {
-      await ctx.services.tx.insert(/* ... */);
-    } else {
-      ctx.logger.info("No DB transaction — skipping write");
-    }
-    // ... browser automation ...
-  },
-);
-```
-
-In production, the caller passes services when invoking `.run()`:
-
-```typescript
-await myWorkflow.run(
-  { page, logger, services: { tx } },
-  input,
-);
-```
-
-When running standalone via `npx libretto run`, services defaults to `{}`,
-so mark fields optional for anything unavailable in that context.
-
-## Playwright Locators for DOM Interaction
-
-Generated code must use Playwright locator APIs for all DOM interactions. Do not use `page.evaluate()` with `document.querySelector`, `querySelectorAll`, `textContent`, `click()`, or other DOM APIs when a Playwright locator can do the same thing.
-
-During the interactive `exec` phase, `page.evaluate` is fine for quick prototyping. In generated production code, translate those patterns into Playwright locators.
-
-### Translation Table
-
-| Operation        | Interactive (`exec`)                                        | Production file                                                        |
-| ---------------- | ----------------------------------------------------------- | ---------------------------------------------------------------------- |
-| Click            | `page.evaluate(() => document.getElementById('x').click())` | `page.locator('#x').click()`                                           |
-| Check state      | `page.evaluate(() => el.checked)`                           | `page.locator('#x').isChecked()`                                       |
-| Read text        | `page.evaluate(() => el.textContent)`                       | `page.locator('#x').textContent()`                                     |
-| Read all text    | `querySelectorAll(...).map(e => e.textContent)`             | `page.locator('.items').allTextContents()`                             |
-| Element position | `el.getBoundingClientRect()`                                | `page.locator('#x').boundingBox()`                                     |
-| Inline styles    | `el.style.top`                                              | `page.locator('#x').getAttribute('style')`                             |
-| Count elements   | `querySelectorAll(...).length`                              | `page.locator('.items').count()`                                       |
-| Select dropdown  | `selectEl.value = '...'`                                    | `page.locator('select').selectOption('...')`                           |
-| Iterate elements | `querySelectorAll(...).forEach(...)`                        | `const items = await locator.all(); for (const item of items) { ... }` |
-| Scoped query     | `parent.querySelector('.child')`                            | `parentLocator.locator('.child').textContent()`                        |
-| Batch extraction | `querySelectorAll('.item').forEach(e => { ... })`           | `for (const item of await locator.all()) { const text = await item.locator('.text').textContent(); ... }` |
-
-### Anti-Patterns
-
-These patterns come up frequently during interactive sessions and should not carry over into production code:
-
-```typescript
-// DON'T — batch-read via evaluate string
-const data = await page.evaluate(`(() => {
-  const posts = document.querySelectorAll('.post');
-  return Array.from(posts).map(p => ({
-    name: p.querySelector('.name')?.textContent,
-    content: p.querySelector('.content')?.textContent,
-  }));
-})()`);
-
-// DO — Playwright locators with a loop
-const posts = await page.locator('.post').all();
-for (const post of posts) {
-  const name = await post.locator('.name').textContent();
-  const content = await post.locator('.content').textContent();
-}
-```
-
-```typescript
-// DON'T — evaluate to count elements
-const count = await el.evaluate(`(el) => el.querySelectorAll('.item').length`);
-
-// DO
-const count = await el.locator('.item').count();
-```
-
-```typescript
-// DON'T — evaluate to read scoped text
-const text = await post.evaluate(
-  `(el) => el.querySelector('[data-view-name="foo"]')?.textContent`
-);
-
-// DO
-const text = await post.locator('[data-view-name="foo"]').textContent();
-```
-
-### When `page.evaluate()` Is Acceptable
-
-Use `page.evaluate()` only for operations that have no Playwright locator equivalent:
-
-1. **Browser-native APIs** — `getComputedStyle()`, `window.*` globals, `document.cookie`, scroll position
-2. **In-browser `fetch()` calls** — making HTTP requests from the browser context
-3. **Parsing operations** — using `DOMParser` to parse HTML/XML strings inside the browser
-
-A quick test: if the evaluate body contains `querySelector`, `querySelectorAll`, `textContent`, `click()`, `getAttribute()`, or iterates DOM elements, it should be rewritten with Playwright locators.
-
-When `page.evaluate()` is used for the acceptable cases above, keep the logic self-contained and return JSON-serializable values:
-
-```typescript
-const data = (await page.evaluate(`(() => {
-  const style = getComputedStyle(document.documentElement);
-  return style.getPropertyValue('--brand-color');
-})()`)) as string;
-```
-
-Do not rely on broad DOM querying inside `page.evaluate()` for production flows when Playwright locators can express the same interaction.
-
-## Network Request Methods
-
-When codifying network-based data extraction or form submissions, wrap `page.evaluate(() => fetch(...))` calls in typed methods on a shared API client class:
-
-```typescript
-class ApiClient {
-  constructor(private page: Page) {}
-
-  private async apiFetch(
-    url: string,
-    options?: { method?: string; body?: string },
-  ): Promise<string> {
-    return await this.page.evaluate(
-      async ({ url, method, body }) => {
-        const init: RequestInit = { method: method ?? "GET" };
-        if (body) {
-          init.headers = {
-            "Content-Type": "application/x-www-form-urlencoded",
-          };
-          init.body = body;
-        }
-        const response = await fetch(url, init);
-        if (!response.ok) throw new Error(`${response.status} for ${url}`);
-        return await response.text();
-      },
-      { url, method: options?.method, body: options?.body },
-    );
-  }
-
-  async fetchReferralList(status: string): Promise<Referral[]> {
-    const raw = await this.apiFetch(`/api/referrals?status=${status}`);
-    // parse and return typed data
-  }
-}
-```
-
-One method per endpoint. No try-catch in API methods — let errors propagate to the orchestrator. Parse XML/HTML inside `page.evaluate()` with `DOMParser`. Use string expressions for `page.evaluate()` to avoid DOM type errors.
-
-## Comments
-
-Add comments throughout generated code to explain what each logical block is doing. Comments should describe **intent**, not restate the code. Group related actions under a single comment rather than commenting every line.
-
-```typescript
-// Log in with credentials
-await page.locator('#username').fill(user);
-await page.locator('#password').fill(pass);
-await page.locator('#login').click();
-
-// Extract author and content from each feed post
-const posts = await page.locator('.post').all();
-for (const post of posts) {
-  const name = await post.locator('.name').textContent();
-  const content = await post.locator('.content').textContent();
-}
-```
-
-## Type Checking
-
-The generated file must pass `npx tsc --noEmit` before it's considered done. If there are type errors around DOM access, prefer locator APIs first, then use focused `page.evaluate()` only for browser-native APIs.

package/skills/libretto/integration-approach-selection.md

@@ -1,174 +0,0 @@
-# Integration Approach Selection Guide
-
-**Purpose:** You are connected to a live Chrome session on a target website. Your job is to probe the site for bot detection measures, assess its security posture, and determine the best integration strategy for data extraction. All strategies use Playwright for browser control — the question is what to **prioritize** for data capture: in-browser fetch calls, passive network interception, or DOM extraction.
-
-After completing the probes below, produce a **Site Assessment Summary** (see the output format at the end of this document).
-
----
-
-## Probing the Site
-
-Run these probes to build a picture of the site's detection posture. The examples below are starting points — use your judgment to investigate further based on what you find. Sites may use detection methods not listed here.
-
-### Probe 1: Bot Protection Services & Security Signals
-
-Look for signs that the site uses bot protection — either a third-party service or custom detection. There is no complete list of indicators; these are common examples.
-
-**Cookies to look for (examples, not exhaustive):**
-
-| Cookie Pattern | Associated Service |
-|---|---|
-| `_abck` | Akamai Bot Manager |
-| `_px*` | PerimeterX (HUMAN) |
-| `datadome` | DataDome |
-| `cf_clearance` | Cloudflare |
-| `_imp_apg_r_*` | Shape Security (F5) |
-| `x-kpsdk-*` | Kasada |
-
-But don't just check this list. Examine **all** cookies on the page — look for any cookies with obfuscated names, telemetry-related prefixes, or values that look like fingerprint hashes or encrypted tokens. Unknown security cookies are still security cookies.
-
-**Global variables to check (examples):**
-
-```js
-// Known telemetry globals — but probe broadly, not just these
-window._pxAppId   // PerimeterX
-window.bmak       // Akamai
-window.ddjskey    // DataDome
-```
-
-Also examine the page's scripts: look at the first `<script>` tags in the document source, check what external domains scripts load from (e.g., `*.akamaized.net`, `*.perimeterx.net`, `*.datadome.co`, `*.kasada.io`). Bot protection scripts are typically injected before any application code.
-
-**Challenge pages:**
-
-Check if the page is showing a challenge or interstitial instead of real content — "Checking your browser...", CAPTCHA iframes, blank pages with only a spinner. These indicate active bot protection that has already been triggered.
-
-**General guidance:** The goal is to determine whether the site has bot protection and roughly how aggressive it is. Don't limit yourself to known signatures — look at the overall page behavior, unusual scripts, and anything that seems like security telemetry.
-
-### Probe 2: Fetch / XHR Interception
-
-Check whether the site has monkey-patched `window.fetch` or `XMLHttpRequest`. If it has, making your own fetch calls from `page.evaluate()` is risky because the site can inspect call stacks and detect calls that don't originate from its own code.
-
-```js
-// Check if fetch has been wrapped
-window.fetch.toString()
-// Native: "function fetch() { [native code] }"
-// Patched: shows actual JavaScript source
-
-// Check XMLHttpRequest
-XMLHttpRequest.prototype.open.toString()
-
-// Check property descriptors for tampering
-Object.getOwnPropertyDescriptor(window, 'fetch')
-// Normal: { value: ƒ, writable: true, enumerable: true, configurable: true }
-
-// Proxy-based wrapping is harder to detect — native fetch has no prototype
-window.fetch.hasOwnProperty('prototype')  // true may indicate a Proxy wrapper
-```
-
-**Important:** Some sites use `Proxy` to wrap fetch, which makes `toString()` still return `"[native code]"`. The prototype check is a heuristic, not definitive. If you see any sign of fetch interception, treat it as patched.
-
-### Probe 3: Behavioral Monitoring
-
-Look for signs that the site collects behavioral telemetry (mouse movements, keystrokes, scroll patterns). Heavy monitoring means you should use natural, human-like interaction patterns when driving the UI.
-
-Things to check:
-- Unusually large numbers of event listeners on `document` or `body` for `mousemove`, `keydown`, `scroll`, `touchstart`, `click`
-- Known telemetry collection scripts
-- `MutationObserver` instances watching the DOM for injected elements
-- `requestAnimationFrame` loops that aren't tied to visible animations
-
-If you're in a DevTools context, `getEventListeners(document)` is the quickest way to assess this. Otherwise, use heuristics — heavy behavioral monitoring usually correlates with enterprise bot protection from Probe 1.
-
----
-
-## Choosing a Data Capture Strategy
-
-Every integration uses Playwright to control the browser. The question is what to **prioritize** for getting data out. In practice, most integrations use a mix — you'll always need some Playwright interaction for navigation, login flows, cookie consent, etc. The strategies below describe what to lean on for the core data extraction.
-
-### Strategy A: Prioritize `page.evaluate(fetch(...))`
-
-Make fetch calls directly from within the browser's JavaScript context. The requests share the browser's TLS fingerprint, cookies, and origin — they look identical to requests the site's own JS would make.
-
-**When to prioritize this:**
-- No enterprise bot protection detected
-- `fetch` is NOT monkey-patched
-- You've identified API endpoints that return the data you need
-- You need data that requires many API calls (deep pagination, bulk queries) where driving the UI would be slow
-
-**Why:** Maximum control and efficiency. You call exactly the endpoints you want with the parameters you want, skip UI rendering, and get structured JSON back. On sites without aggressive detection, this is the fastest and cleanest approach.
-
-**Risk:** If the site monitors fetch call stacks (Layer 4 detection), your calls will be flagged because they don't originate from the site's bundled code. This is uncommon but exists on high-security sites.
-
-**You'll still use Playwright for:** Initial navigation, login/auth flows, cookie consent, and any UI interactions needed to establish session state before making fetch calls.
-
-### Strategy B: Prioritize `page.on('response', ...)` (Passive Interception)
-
-Listen to network responses that the browser naturally makes as you navigate. You don't make any extra requests — you capture data flowing through the site's own API calls.
-
-**When to prioritize this:**
-- Enterprise bot protection is detected
-- `fetch` IS monkey-patched
-- The site's normal UI flow triggers API calls that return the data you need
-- You want to minimize detection risk as much as possible
-
-**Why:** Zero additional network risk. The requests that happen are the ones the site's own code triggers. You're just listening. No anomalous call stacks, no unexpected request patterns, no extra fetch calls for monitoring to flag.
-
-**Trade-off:** You only get data the page naturally loads. If you need page 50 of results, you have to click "next" 49 times via Playwright. You must set up listeners before the navigation that triggers the requests.
-
-**You'll still use Playwright for:** All navigation and interaction to trigger the data-bearing API calls, plus any data that isn't available via intercepted responses (DOM-only content).
-
-### Strategy C: Prioritize Playwright DOM Extraction
-
-Extract data directly from the rendered page using selectors and `page.evaluate()` to read DOM content.
-
-**When to prioritize this:**
-- Data is server-rendered (no JSON API calls observed)
-- The site doesn't expose the data you need via any API
-- You need visual/layout information that only exists in the DOM
-- As a fallback when Strategies A and B can't get specific pieces of data
-
-**Why:** Works regardless of the site's API architecture. If the data is visible on the page, you can extract it.
-
-**Trade-off:** Slower, fragile against DOM changes, and you only get data the UI renders (which may be less than what API responses contain).
-
----
-
-## Decision Summary
-
-| Site Profile | Primary Strategy | Supplement With |
-|---|---|---|
-| No bot protection, fetch not patched | **A** (`page.evaluate(fetch)`) | Playwright for navigation/auth |
-| No bot protection, fetch IS patched | **B** (`page.on('response', ...)`) | Playwright for navigation; DOM extraction as fallback |
-| Bot protection detected, fetch not patched | **B** (`page.on('response', ...)`) | Playwright for all navigation; cautious use of `page.evaluate(fetch)` only if needed |
-| Bot protection detected, fetch IS patched | **B** (`page.on('response', ...)`) | Playwright for all navigation; DOM extraction as fallback |
-| Server-rendered content (no API calls) | **C** (DOM extraction) | Playwright for all interaction |
-
----
-
-## Output: Site Assessment Summary
-
-After running the probes, produce a summary in this format. **Do NOT include a final strategy recommendation.** The security assessment determines what's *safe to use*, not what will *work*. Present this to the user for input, then use the safe approaches as you build the integration — adapting if specific endpoints don't work as expected (see "Handling Approach Mismatches" in SKILL.md).
-
-```
-## Site Assessment: [site URL]
-
-### Bot Detection Profile
-- **Enterprise bot protection:** [None detected / Detected — describe what you found (service name if identifiable, cookies, scripts, telemetry globals)]
-- **Fetch/XHR interception:** [Native (not patched) / Patched — describe what you found]
-- **Behavioral monitoring:** [None detected / Light / Heavy — describe indicators]
-- **Challenge pages:** [None / Present — describe type (CAPTCHA, interstitial, etc.)]
-- **Overall security posture:** [None / Low / Moderate / High / Very High]
-
-### API Surface
-- **API calls observed:** [List key endpoints discovered, or "None — content appears server-rendered"]
-- **Data format:** [JSON / GraphQL / HTML fragments / Other — note if any responses use proprietary/binary formats]
-- **Pagination:** [Describe how pagination works if applicable]
-
-### Safe Approaches
-- **`page.evaluate(fetch(...))`:** [Safe / Unsafe — brief rationale based on fetch patching, bot detection, etc.]
-- **`page.on('response', ...)`:** [Viable / Not viable — note if response formats are parseable or proprietary]
-- **DOM extraction:** [Always available as fallback]
-- **Interaction notes:** [any behavioral precautions — natural mouse movements, typing delays, etc.]
-```
-
-**Important:** This assessment tells you which tools are in your toolbox. Present it to the user, get their input, then start building the integration using the safe approaches.