libretto 0.5.1 → 0.5.2

package/README.md

@@ -13,15 +13,17 @@ Libretto is a toolkit for building robust web integrations. It gives your coding
 
 We at [Saffron Health](https://saffron.health) built Libretto to help us maintain our browser integrations to common healthcare software. We're open-sourcing it so other teams have an easier time doing the same thing.
 
+https://github.com/user-attachments/assets/9b9a0ab3-5133-4b20-b3be-459943349d18
+
 ## Installation
 
 ```bash
-npm install --save-dev libretto
+npm install libretto
 
 # Install skill, download Chromium if not already installed, configure snapshot analysis
 npx libretto init
 
-# Configure snapshot analysis model (see Configuration section below)
+# Configure or change the snapshot analysis model (see Configuration section below). `npx libretto init` sets this up the first time.
 npx libretto ai configure <openai | anthropic | gemini | vertex>
 ```
 
@@ -58,7 +60,7 @@ Agents can use Libretto to reproduce the failure, pause the workflow at any poin
 You can also use Libretto directly from the command line. All commands accept `--session <name>` to target a specific session.
 
 ```bash
-npx libretto init                          # initialize libretto in the current project
+npx libretto init                          # interactive; run yourself, not through an agent
 npx libretto open <url>                    # launch browser and open a URL (headed by default)
 npx libretto snapshot --objective "..." --context "..."  # capture PNG + HTML and analyze with an LLM
 npx libretto exec "<code>"                 # execute Playwright TypeScript against the open page
@@ -134,6 +136,8 @@ Maintained by the team at [Saffron Health](https://saffron.health).
 
 ## Development
 
+For local development in this repository:
+
 ```bash
 pnpm i
 pnpm build

package/dist/cli/commands/init.js

@@ -17,8 +17,8 @@ import {
   loadSnapshotEnv,
   resolveSnapshotApiModel
 } from "../core/snapshot-api-config.js";
-import { SimpleCLI } from "../framework/simple-cli.js";
 import { hasProviderCredentials } from "../../shared/llm/client.js";
+import { SimpleCLI } from "../framework/simple-cli.js";
 const PROVIDER_CHOICES = [
   {
     key: "1",
@@ -52,15 +52,6 @@ function promptUser(rl, question) {
     });
   });
 }
-function askYesNo(question) {
-  const rl = createInterface({ input: process.stdin, output: process.stdout });
-  return new Promise((resolve) => {
-    rl.question(`${question} (y/N) `, (answer) => {
-      rl.close();
-      resolve(answer.trim().toLowerCase() === "y");
-    });
-  });
-}
 function safeReadAiConfig() {
   try {
     return readAiConfig();
@@ -233,7 +224,7 @@ function detectAgentDirs(root) {
   if (existsSync(join(root, ".claude"))) dirs.push(join(root, ".claude"));
   return dirs;
 }
-async function copySkills() {
+function copySkills() {
   const agentDirs = detectAgentDirs(REPO_ROOT);
   if (agentDirs.length === 0) {
     console.log(
@@ -242,15 +233,6 @@ async function copySkills() {
     return;
   }
   const destinations = agentDirs.map((d) => join(d, "skills", "libretto"));
-  const dirNames = agentDirs.map((d) => basename(d)).join(" and ");
-  const existing = destinations.filter((d) => existsSync(d));
-  const verb = existing.length > 0 ? "Overwrite" : "Install";
-  const proceed = await askYesNo(`
-${verb} libretto skills in ${dirNames}?`);
-  if (!proceed) {
-    console.log("  Skipping skill copy.");
-    return;
-  }
   let sourceDir;
   try {
     sourceDir = getPackageSkillsDir();
@@ -289,10 +271,11 @@ const initCommand = SimpleCLI.command({
   } else {
     console.log("\nSkipping browser installation (--skip-browsers)");
   }
+  copySkills();
   if (process.stdin.isTTY) {
-    await copySkills();
     await runInteractiveApiSetup();
   } else {
+    loadSnapshotEnv();
     printSnapshotApiStatus();
   }
   console.log("\n\u2713 libretto init complete");

package/dist/cli/core/ai-config.js

@@ -11,10 +11,15 @@ const ViewportConfigSchema = z.object({
   width: z.number().int().min(1),
   height: z.number().int().min(1)
 });
+const WindowPositionConfigSchema = z.object({
+  x: z.number().int(),
+  y: z.number().int()
+});
 const LibrettoConfigSchema = z.object({
   version: z.literal(CURRENT_CONFIG_VERSION),
   ai: AiConfigSchema.optional(),
-  viewport: ViewportConfigSchema.optional()
+  viewport: ViewportConfigSchema.optional(),
+  windowPosition: WindowPositionConfigSchema.optional()
 }).passthrough();
 const DEFAULT_MODELS = {
   openai: "openai/gpt-5.4",
@@ -49,6 +54,10 @@ function formatExpectedConfigExample() {
       viewport: {
         width: 1280,
         height: 800
+      },
+      windowPosition: {
+        x: 1600,
+        y: 120
       }
     },
     null,
@@ -64,7 +73,7 @@ ${detail}` : null,
       "Expected config example:",
       formatExpectedConfigExample(),
       "Notes:",
-      '  - "ai" and "viewport" are optional.',
+      '  - "ai", "viewport", and "windowPosition" are optional.',
       '  - "ai.model" must be a provider/model string like "openai/gpt-5.4" or "anthropic/claude-sonnet-4-6".',
       "Fix the file to match this shape, or delete it and rerun:",
       `  npx libretto ai configure ${formatConfigureProviders()}`
@@ -189,6 +198,7 @@ export {
   CURRENT_CONFIG_VERSION,
   LibrettoConfigSchema,
   ViewportConfigSchema,
+  WindowPositionConfigSchema,
   clearAiConfig,
   readAiConfig,
   readLibrettoConfig,

package/dist/cli/core/browser.js

@@ -6,6 +6,14 @@ import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { createServer } from "node:net";
 import { spawn } from "node:child_process";
+import {
+  filterSemanticClasses,
+  INTERACTIVE_ROLE_NAMES,
+  INTERACTIVE_TAG_NAMES,
+  isObfuscatedClass,
+  TEST_ATTRIBUTE_NAMES,
+  TRUSTED_ATTRIBUTE_NAMES
+} from "../../shared/dom-semantics.js";
 import {
   getSessionActionsLogPath,
   getSessionNetworkLogPath,
@@ -245,10 +253,22 @@ function resolveViewport(cliViewport, logger) {
   });
   return DEFAULT_VIEWPORT;
 }
+function resolveWindowPosition(logger) {
+  const config = readLibrettoConfig();
+  if (config.windowPosition) {
+    logger.info("window-position-source", {
+      source: "config",
+      windowPosition: config.windowPosition
+    });
+    return config.windowPosition;
+  }
+  return void 0;
+}
 async function runOpen(rawUrl, headed, session, logger, options) {
   const url = normalizeUrl(rawUrl);
   const viewport = resolveViewport(options?.viewport, logger);
-  logger.info("open-start", { url, headed, session, viewport });
+  const windowPosition = headed ? resolveWindowPosition(logger) : void 0;
+  logger.info("open-start", { url, headed, session, viewport, windowPosition });
   assertSessionAvailableForStart(session, logger);
   const port = await pickFreePort();
   const runLogPath = logFileForSession(session);
@@ -277,6 +297,45 @@ async function runOpen(rawUrl, headed, session, logger, options) {
   const escapedLogPath = runLogPath.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
   const escapedNetworkLogPath = networkLogPath.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
   const escapedActionsLogPath = actionsLogPath.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+  const windowPositionArg = windowPosition ? `, '--window-position=${windowPosition.x},${windowPosition.y}'` : "";
+  const windowBoundsSetupCode = windowPosition ? `
+const requestedWindowBounds = { left: ${windowPosition.x}, top: ${windowPosition.y}, windowState: 'normal' };
+const pageCdp = await context.newCDPSession(page);
+let browserCdp;
+try {
+	const targetInfo = await pageCdp.send('Target.getTargetInfo');
+	const targetId = targetInfo?.targetInfo?.targetId;
+	browserCdp = await browser.newBrowserCDPSession();
+	const windowResult = await browserCdp.send(
+		'Browser.getWindowForTarget',
+		targetId ? { targetId } : {},
+	);
+	await browserCdp.send('Browser.setWindowBounds', {
+		windowId: windowResult.windowId,
+		bounds: requestedWindowBounds,
+	});
+	await new Promise((resolve) => setTimeout(resolve, 250));
+	const actualWindow = await browserCdp.send('Browser.getWindowBounds', {
+		windowId: windowResult.windowId,
+	});
+	childLog('info', 'window-bounds-set', {
+		windowId: windowResult.windowId,
+		requestedBounds: requestedWindowBounds,
+		actualBounds: actualWindow.bounds,
+	});
+} catch (error) {
+	childLog('warn', 'window-bounds-set-failed', {
+		requestedBounds: requestedWindowBounds,
+		message: error instanceof Error ? error.message : String(error),
+		stack: error instanceof Error ? error.stack : undefined,
+	});
+} finally {
+	await pageCdp.detach().catch(() => {});
+	if (browserCdp) {
+		await browserCdp.detach().catch(() => {});
+	}
+}
+` : "";
   const launcherCode = `
 import { chromium } from 'playwright';
 import { appendFileSync, mkdirSync } from 'node:fs';
@@ -288,14 +347,21 @@ const ACTIONS_LOG = '${escapedActionsLogPath}';
 mkdirSync(dirname(NETWORK_LOG), { recursive: true });
 
 // tsx/esbuild may emit __name() wrappers in Function#toString output.
-const __name = (target, value) =>
-	Object.defineProperty(target, 'name', { value, configurable: true });
+	const __name = (target, value) =>
+		Object.defineProperty(target, 'name', { value, configurable: true });
 
-${installSessionTelemetry.toString()}
+	const TEST_ATTRIBUTE_NAMES = ${JSON.stringify([...TEST_ATTRIBUTE_NAMES])};
+	const TRUSTED_ATTRIBUTE_NAMES = ${JSON.stringify([...TRUSTED_ATTRIBUTE_NAMES])};
+	const INTERACTIVE_TAG_NAMES = ${JSON.stringify([...INTERACTIVE_TAG_NAMES])};
+	const INTERACTIVE_ROLE_NAMES = ${JSON.stringify([...INTERACTIVE_ROLE_NAMES])};
+	const filterSemanticClasses = ${filterSemanticClasses.toString()};
+	const isObfuscatedClass = ${isObfuscatedClass.toString()};
 
-function logAction(entry) {
-	appendFileSync(ACTIONS_LOG, JSON.stringify(entry) + '\\n');
-}
+	${installSessionTelemetry.toString()}
+
+	function logAction(entry) {
+		appendFileSync(ACTIONS_LOG, JSON.stringify(entry) + '\\n');
+	}
 
 function logNetwork(entry) {
 	appendFileSync(NETWORK_LOG, JSON.stringify(entry) + '\\n');
@@ -317,7 +383,7 @@ function childLog(level, event, data = {}) {
 
 const browser = await chromium.launch({
 	headless: ${!headed},
-	args: ['--disable-blink-features=AutomationControlled', '--remote-debugging-port=${port}', '--remote-debugging-address=127.0.0.1', '--no-focus-on-check'],
+	args: ['--disable-blink-features=AutomationControlled', '--remote-debugging-port=${port}', '--remote-debugging-address=127.0.0.1', '--no-focus-on-check'${windowPositionArg}],
 });
 
 browser.on('disconnected', () => {
@@ -331,6 +397,7 @@ const context = await browser.newContext({
 });
 
 const page = await context.newPage();
+${windowBoundsSetupCode}
 page.setDefaultTimeout(30000);
 page.setDefaultNavigationTimeout(45000);