libretto 0.5.0 → 0.5.1

package/scripts/skills-libretto.mjs

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+} from "node:fs";
+import { relative, resolve, join } from "node:path";
+
+export const SKILL_DIRS = [
+  "skills/libretto",
+  ".agents/skills/libretto",
+  ".claude/skills/libretto",
+];
+
+function walkFiles(dir, baseDir = dir) {
+  const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) =>
+    a.name.localeCompare(b.name),
+  );
+  const files = [];
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...walkFiles(fullPath, baseDir));
+      continue;
+    }
+    if (entry.isFile()) files.push(relative(baseDir, fullPath));
+  }
+
+  return files;
+}
+
+export function syncSkillDir(sourceDir, destDir) {
+  rmSync(destDir, { recursive: true, force: true });
+  mkdirSync(destDir, { recursive: true });
+  cpSync(sourceDir, destDir, { recursive: true });
+}
+
+export function syncRepoSkills(repoRoot) {
+  const sourceDir = resolve(repoRoot, "skills/libretto");
+  for (const dir of SKILL_DIRS.slice(1)) {
+    syncSkillDir(sourceDir, resolve(repoRoot, dir));
+  }
+}
+
+export function compareSkillDirs(repoRoot) {
+  const roots = SKILL_DIRS.map((dir) => ({
+    label: dir,
+    absPath: resolve(repoRoot, dir),
+  }));
+  const missing = roots.filter(({ absPath }) => !existsSync(absPath));
+  const mismatches = [];
+
+  if (missing.length > 0) {
+    return {
+      ok: false,
+      issues: missing.map(({ label }) => `missing directory: ${label}`),
+    };
+  }
+
+  const expectedFiles = walkFiles(roots[0].absPath);
+  const expectedFileSet = new Set(expectedFiles);
+
+  for (const root of roots.slice(1)) {
+    const actualFiles = walkFiles(root.absPath);
+    const actualFileSet = new Set(actualFiles);
+
+    for (const file of expectedFiles) {
+      if (!actualFileSet.has(file)) {
+        mismatches.push(`${root.label} is missing file: ${file}`);
+      }
+    }
+
+    for (const file of actualFiles) {
+      if (!expectedFileSet.has(file)) {
+        mismatches.push(`${root.label} has unexpected file: ${file}`);
+      }
+    }
+  }
+
+  for (const file of expectedFiles) {
+    const expectedContent = readFileSync(join(roots[0].absPath, file));
+    for (const root of roots.slice(1)) {
+      const targetPath = join(root.absPath, file);
+      if (!existsSync(targetPath)) continue;
+      const actualContent = readFileSync(targetPath);
+      if (!expectedContent.equals(actualContent)) {
+        mismatches.push(
+          `${root.label} differs from ${roots[0].label}: ${file}`,
+        );
+      }
+    }
+  }
+
+  return {
+    ok: mismatches.length === 0,
+    issues: mismatches,
+  };
+}

package/scripts/summarize-evals.mjs

@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+
+import { readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { basename, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+function usage() {
+  console.error(
+    "Usage: node scripts/summarize-evals.mjs <score-dir> <summary-json-path>",
+  );
+}
+
+function normalizeFailureRecord(failure) {
+  return {
+    criterion: String(failure?.criterion ?? "").trim(),
+    reason: String(failure?.reason ?? "").trim(),
+  };
+}
+
+function normalizeRecord(record) {
+  const failures = Array.isArray(record?.failures)
+    ? record.failures
+        .map(normalizeFailureRecord)
+        .filter(
+          (failure) =>
+            failure.criterion.length > 0 && failure.reason.length > 0,
+        )
+    : [];
+
+  return {
+    name: String(record?.name ?? "").trim(),
+    passed: Number(record?.passed ?? 0),
+    total: Number(record?.total ?? 0),
+    percent: Number(record?.percent ?? 0),
+    failures,
+  };
+}
+
+export function loadScoreRecords(scoreDirArg) {
+  const scoreDir = resolve(scoreDirArg);
+  return readdirSync(scoreDir, { withFileTypes: true })
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+    .map((entry) =>
+      JSON.parse(readFileSync(join(scoreDir, entry.name), "utf8")),
+    )
+    .map(normalizeRecord)
+    .sort((a, b) => String(a.name).localeCompare(String(b.name)));
+}
+
+export function buildSummary(records) {
+  const passed = records.reduce(
+    (sum, record) => sum + Number(record.passed || 0),
+    0,
+  );
+  const total = records.reduce(
+    (sum, record) => sum + Number(record.total || 0),
+    0,
+  );
+  const percent = total > 0 ? Number(((passed / total) * 100).toFixed(2)) : 0;
+  const failingRecords = records.filter((record) => record.failures.length > 0);
+
+  return {
+    generatedAt: new Date().toISOString(),
+    recordCount: records.length,
+    passed,
+    total,
+    percent,
+    failingRecordCount: failingRecords.length,
+    records,
+  };
+}
+
+export function buildMarkdown(summary, summaryPathArg) {
+  const lines = [
+    "# Eval Summary",
+    "",
+    `- Overall score: \`${summary.percent}%\``,
+    `- Passed criteria: \`${summary.passed}/${summary.total}\``,
+    `- Recorded score entries: \`${summary.recordCount}\``,
+    `- Failed evals: \`${summary.failingRecordCount}\``,
+    `- Summary file: \`${basename(summaryPathArg)}\``,
+  ];
+
+  if (summary.records.length > 0) {
+    lines.push("", "## Breakdown", "");
+    for (const record of summary.records) {
+      const status = record.failures.length > 0 ? "fail" : "pass";
+      lines.push(
+        `- ${status} \`${record.name}\`: \`${record.percent}%\` (${record.passed}/${record.total})`,
+      );
+    }
+  }
+
+  if (summary.failingRecordCount > 0) {
+    lines.push("", "## Failed Evals", "");
+    for (const record of summary.records.filter(
+      (candidate) => candidate.failures.length > 0,
+    )) {
+      lines.push(`### \`${record.name}\``);
+      lines.push("");
+      lines.push(
+        `- Score: \`${record.percent}%\` (${record.passed}/${record.total})`,
+      );
+      for (const failure of record.failures) {
+        lines.push(`- ${failure.criterion}: ${failure.reason}`);
+      }
+      lines.push("");
+    }
+  }
+
+  return `${lines.join("\n").trimEnd()}\n`;
+}
+
+function main(argv) {
+  const [, , scoreDirArg, summaryPathArg] = argv;
+
+  if (!scoreDirArg || !summaryPathArg) {
+    usage();
+    process.exit(1);
+  }
+
+  const summaryPath = resolve(summaryPathArg);
+  const records = loadScoreRecords(scoreDirArg);
+  const summary = buildSummary(records);
+
+  writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
+  process.stdout.write(buildMarkdown(summary, summaryPath));
+}
+
+if (
+  process.argv[1] &&
+  resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  main(process.argv);
+}

package/scripts/sync-skills.mjs

@@ -0,0 +1,12 @@
+#!/usr/bin/env node
+
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { SKILL_DIRS, syncRepoSkills } from "./skills-libretto.mjs";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const repoRoot = join(__dirname, "..");
+
+syncRepoSkills(repoRoot);
+console.log(`libretto: synced skill mirrors across ${SKILL_DIRS.join(", ")}`);

package/skills/libretto/SKILL.md

@@ -1,34 +1,36 @@
 ---
 name: libretto
-description: "Browser automation CLI for inspecting live pages, prototyping interactions, and running browser workflows."
+description: "Browser automation CLI for building, maintaining, and running browser automation workflows by inspecting live pages and prototyping interactions."
 license: MIT
 metadata:
   author: saffron-health
-  version: "0.4.0"
+  version: "0.4.2"
 ---
 
-# Libretto
+## How Libretto Works
 
-Use `npx libretto` to inspect live browser state, prototype interactions, and run existing browser workflows.
+- Libretto is a CLI for exploring live websites and building or debugging reusable browser automation scripts.
+- Use Libretto to inspect the real site first: open pages, observe state, inspect requests, and prototype interactions before writing code.
+- Libretto work must end in script changes. Create or edit the workflow file instead of stopping at interactive exploration.
 
-## Intro
+## Default Integration Approach
 
-- Use this skill when the truth is on the page.
-- Prefer Libretto when you need to see what the browser is doing, not when you only need to edit source files.
-- Treat Libretto as a session-based workflow: open a page, inspect it, try a focused action, then turn what you learned into code outside the CLI.
-- When building a new integration, prefer reverse-engineering network requests first. Fall back to browser automation when the request path is unclear, too fragile, or blocked by anti-bot systems.
+- Prefer network requests first for new integrations.
+- Read `references/site-security-review.md` before committing to a network-first approach on a new site.
+- Fall back to passive interception or Playwright-driven UI automation when the security review rules network requests out, the request path is not workable, or the user explicitly asks for Playwright.
 
 ## Setup
 
 - Ask the user to set up snapshot analysis before relying on `snapshot` for page understanding.
-- Use `npx libretto init` for first-time setup.
-- If they already have credentials, `npx libretto ai configure openai|anthropic|gemini|vertex` is enough.
+- Use `npx libretto init` for first-time workspace setup.
+- If credentials are already available, `npx libretto ai configure openai|anthropic|gemini|vertex` is usually enough.
 
-## Rules
+## Working Rules
 
 - Announce which session you are using and what page you are on.
 - Ask instead of guessing when it is unclear what to click, type, or submit.
-- Use `snapshot` to understand unknown page state before trying multiple selectors.
+- Read and follow guidelines in `references/code-generation-rules.md` before generating or editing production workflow code.
+- After interactive exploration and code generation, test key logic with `exec`, then verify the workflow file with `run --headless`.
 - Get explicit user confirmation before mutating actions or replaying network requests that may have side effects.
 - Never run multiple `exec` commands at the same time.
 - Keep the browser session open until the user says the session is done.
@@ -38,6 +40,7 @@ Use `npx libretto` to inspect live browser state, prototype interactions, and ru
 ### `open`
 
 - Open a page before using `exec` or `snapshot`.
+- Use `open` at the start of script authoring when you need live page state to decide how the workflow should work.
 - Use headed mode when the user needs to log in or watch the workflow.
 
 ```bash
@@ -45,37 +48,83 @@ npx libretto open https://example.com --headed
 npx libretto open https://example.com --headless --session debug-example
 ```
 
-### `exec`
+### `connect`
 
-- Use `exec` for focused inspection and short-lived interaction experiments.
-- Let failures throw. Do not hide `exec` failures with `try/catch`.
+- Use `connect` to attach to any existing Chrome DevTools Protocol (CDP) endpoint — a browser started with `--remote-debugging-port`, an Electron app, or any other CDP-compatible target.
+- After connecting, `exec`, `snapshot`, `pages`, and all other session commands work normally.
+- Libretto does not manage the connected process's lifecycle. `close` clears the session but does not terminate the remote process.
 
 ```bash
-npx libretto exec "return await page.url()"
-npx libretto exec "return await page.locator('button').count()"
-npx libretto exec --visualize "await page.locator('button:has-text(\"Continue\")').click()"
+npx libretto connect http://127.0.0.1:9222 --session my-session
+npx libretto connect http://127.0.0.1:9223 --session another-session
 ```
 
 ### `snapshot`
 
 - Use `snapshot` as the primary page observation tool.
-- When you want analysis, provide both `--objective` and `--context`.
-- If you only need the PNG and HTML files, omit `--objective`. That runs capture-only mode and skips AI analysis.
-- When using `--objective`, expect analysis to take time. Use a timeout of at least 2 minutes for shell-wrapped calls.
+- Always provide both `--objective` and `--context`.
+- Use it before guessing at selectors, after workflow failures, and whenever the visible page state is unclear.
+- When analysis is involved, expect it to take time. Use a timeout of at least 2 minutes for shell-wrapped calls.
 
 ```bash
-npx libretto snapshot
 npx libretto snapshot \
   --objective "Find the sign-in form and submit button" \
   --context "I just opened the login page and need the email field, password field, and submit button."
 npx libretto snapshot \
+  --session debug-example \
+  --page <page-id> \
   --objective "Explain why the table is empty" \
-  --context "I opened the referrals page and expected rows after applying filters."
+  --context "I opened the referrals page, applied filters, and expected rows to appear."
+```
+
+### `exec`
+
+- Use `exec` for focused inspection and short-lived interaction experiments.
+- Use `exec` to validate selectors, inspect data, or prototype a step before you encode it in the workflow file.
+- Let failures throw. Do not hide `exec` failures with `try/catch` or `.catch()`.
+- Do not run multiple `exec` commands in parallel.
+
+```bash
+npx libretto exec "return await page.url()"
+npx libretto exec "return await page.locator('button').count()"
+npx libretto exec --visualize "await page.locator('button:has-text(\"Continue\")').click()"
+```
+
+### `pages`
+
+- Use `pages` when a popup, new tab, or second page appears.
+- If `exec`, `snapshot`, `network`, or `actions` complains about multiple pages, list page ids first and then pass `--page`.
+
+```bash
+npx libretto pages --session debug-example
+npx libretto exec --session debug-example --page <page-id> "return await page.url()"
+```
+
+### `network`
+
+- Use `network` to inspect the requests the page already made.
+- Prefer this when discovering how a site loads data or when validating whether a network-first approach is workable.
+- Filter aggressively by method, URL pattern, and page when the log is noisy.
+
+```bash
+npx libretto network --session debug-example --last 20
+npx libretto network --session debug-example --method POST --filter 'referral|patient'
+npx libretto network --session debug-example --page <page-id>
+```
+
+### `actions`
+
+- Use `actions` when you need a quick record of recent user or agent interactions in the current session.
+- Keep it lightweight. It is a helper for orientation, not the main integration-building workflow.
+
+```bash
+npx libretto actions --session debug-example --last 20
+npx libretto actions --session debug-example --action click --source user
 ```
 
 ### `run`
 
-- Use `run` to execute an existing Libretto workflow.
+- Use `run` to verify a workflow file after creating it or editing it.
 - If the workflow fails, Libretto keeps the browser open. Inspect the failed state with `snapshot` and `exec` before editing code.
 - If the workflow pauses, resume it with `npx libretto resume --session <name>`.
 - Re-run the same workflow after each fix to verify the browser behavior end to end.
@@ -86,6 +135,34 @@ npx libretto run ./integration.ts main --params '{"status":"open"}'
 npx libretto run ./integration.ts main --auth-profile app.example.com --headed
 ```
 
+### `resume`
+
+- Workflows pause by calling `await pause()` in the workflow file.
+- Use `resume` when a workflow hit `await pause()`.
+- Keep resuming the same session until the workflow completes or pauses again.
+
+```bash
+npx libretto resume --session debug-example
+```
+
+### `save`
+
+- Use `save` when the user logs in manually and wants to reuse that authenticated browser state later.
+
+```bash
+npx libretto save app.example.com
+```
+
+### `close`
+
+- Use `close` only when the user is done with the session.
+- `close --all` is available for workspace cleanup.
+
+```bash
+npx libretto close --session debug-example
+npx libretto close --all
+```
+
 ## Examples
 
 ### Building new browser automation workflows
@@ -95,31 +172,16 @@ npx libretto run ./integration.ts main --auth-profile app.example.com --headed
 ```text
 <example>
 [Context: The user wants to build a new browser workflow and does not yet know the page structure]
-Assistant: Let me open the target page in headed mode so we can inspect the real workflow.
+Assistant: I'll inspect the real site first if needed, but before I finish I'll create `target-workflow.ts` so the task produces reusable automation code.
 Assistant: [Runs `npx libretto open https://target.example.com --headed`]
-Assistant: I'll first determine whether this flow can be built from captured network requests instead of UI-only automation.
-Assistant: [Runs `npx libretto snapshot --objective "Find the next required action" --context "We are starting the workflow from the landing page."`]
-Assistant: If the page exposes a clear and stable request path, I'll prioritize that. If the request path is unclear, fragile, or heavily defended, I'll fall back to browser automation.
-Assistant: [Uses `network`, `snapshot`, and `exec` as needed to prove the approach]
+Assistant: [Reads `references/site-security-review.md` before choosing between passive network inspection, direct browser fetch calls, and Playwright-first automation]
+Assistant: [Runs `npx libretto snapshot --objective "Find the next required action" --context "We are starting the workflow from the landing page and need the first meaningful step."`]
+Assistant: [Uses `network`, `snapshot`, and `exec` as needed to understand the site and decide the implementation path]
+Assistant: [Reads `references/code-generation-rules.md` before writing production workflow code]
 Assistant: I found the working path. I'll now update the workflow file outside Libretto and verify it with `npx libretto run ...`.
 </example>
 ```
 
-#### Network requests
-
-```text
-<example>
-[Context: The user wants to build an integration using network requests]
-Assistant: [Reads `references/reverse-engineering-network-requests.md`]
-Assistant: Let me open the page in headed mode. Perform the workflow and I'll use the network log to recreate it.
-Assistant: [Runs `npx libretto open https://target.example.com --headed`]
-[User performs workflow]
-User: I've completed the workflow
-Assistant: [Runs `npx libretto network --method POST --last 20`]
-Assistant: I found the relevant requests. I'll recreate the workflow from those requests, then test the resulting script with `npx libretto run ...`.
-</example>
-```
-
 ### Debugging existing workflows
 
 ```text
@@ -130,13 +192,15 @@ Assistant: [Runs `npx libretto run ./integration.ts main --session debug-flow --
 Assistant: The workflow failed and Libretto kept the browser open. I'll inspect the page state before changing code.
 Assistant: [Runs `npx libretto snapshot --session debug-flow --objective "Find the blocking error or broken selector target" --context "The workflow just failed after trying to continue from the review step, and I need to identify the visible blocker on the current page."`]
 Assistant: [Runs `npx libretto exec --session debug-flow "...focused inspection or prototype..."`]
+Assistant: [Reads `references/code-generation-rules.md` before patching the workflow file]
 Assistant: I found the issue. I'll patch the workflow code, then rerun `npx libretto run ...` to verify the fix.
 </example>
 ```
 
 ## References
 
-- For reverse-engineering captured requests, read `references/reverse-engineering-network-requests.md`.
-- For incorporating manual browser steps the user performed, read `references/user-action-log.md`.
-- For saving and reusing login state, read `references/auth-profiles.md`.
-- For multiple open pages and page targeting, read `references/pages-and-page-targeting.md`.
+- Read `references/configuration-file-reference.md` when you need to inspect or change `.libretto/config.json` for snapshot model selection or viewport defaults.
+- Read `references/site-security-review.md` before reviewing the site's security posture and deciding whether to lead with network requests, passive interception, or Playwright DOM automation on a new site.
+- Read `references/code-generation-rules.md` before writing or editing production workflow files.
+- Read `references/auth-profiles.md` when the site requires login and the simplest path is to save local browser state.
+- Read `references/pages-and-page-targeting.md` when a session has multiple open pages or you need `--page`.