libretto 0.5.1 → 0.5.3-experimental.0

package/README.md

@@ -13,15 +13,17 @@ Libretto is a toolkit for building robust web integrations. It gives your coding
 
 We at [Saffron Health](https://saffron.health) built Libretto to help us maintain our browser integrations to common healthcare software. We're open-sourcing it so other teams have an easier time doing the same thing.
 
+https://github.com/user-attachments/assets/9b9a0ab3-5133-4b20-b3be-459943349d18
+
 ## Installation
 
 ```bash
-npm install --save-dev libretto
+npm install libretto
 
 # Install skill, download Chromium if not already installed, configure snapshot analysis
 npx libretto init
 
-# Configure snapshot analysis model (see Configuration section below)
+# Configure or change the snapshot analysis model (see Configuration section below). `npx libretto init` sets this up the first time.
 npx libretto ai configure <openai | anthropic | gemini | vertex>
 ```
 
@@ -58,11 +60,12 @@ Agents can use Libretto to reproduce the failure, pause the workflow at any poin
 You can also use Libretto directly from the command line. All commands accept `--session <name>` to target a specific session.
 
 ```bash
-npx libretto init                          # initialize libretto in the current project
+npx libretto init                          # interactive; run yourself, not through an agent
 npx libretto open <url>                    # launch browser and open a URL (headed by default)
 npx libretto snapshot --objective "..." --context "..."  # capture PNG + HTML and analyze with an LLM
-npx libretto exec "<code>"                 # execute Playwright TypeScript against the open page
-npx libretto run <file> <export>           # run an exported workflow from a file
+npx libretto exec "<code>"                 # execute Playwright TypeScript against the open page (single quoted argument)
+echo "<code>" | npx libretto exec -        # intentionally read Playwright TypeScript from stdin
+npx libretto run <file> <workflowName>     # run an exported workflow from a file
 npx libretto resume                        # resume a paused workflow
 npx libretto network                       # view captured network requests
 npx libretto actions                       # view captured user/agent actions
@@ -134,6 +137,8 @@ Maintained by the team at [Saffron Health](https://saffron.health).
 
 ## Development
 
+For local development in this repository:
+
 ```bash
 pnpm i
 pnpm build

package/dist/cli/commands/execution.js

@@ -445,9 +445,10 @@ async function runIntegrationFromFile(args, logger) {
   );
   const payload = JSON.stringify({
     integrationPath: args.integrationPath,
-    exportName: args.exportName,
+    workflowName: args.workflowName,
     session: args.session,
     params: args.params,
+    credentials: args.credentials,
     headless: args.headless,
     visualize: args.visualize,
     authProfileDomain: args.authProfileDomain,
@@ -496,11 +497,19 @@ Browser is still open. You can use \`exec\` to inspect it. Call \`run\` to re-ru
   setSessionStatus(args.session, "completed", logger);
   console.log("Integration completed.");
 }
+function readStdinSync() {
+  if (process.stdin.isTTY === true) return null;
+  try {
+    const content = readFileSync(0, "utf8");
+    return content.trim().length > 0 ? content : null;
+  } catch {
+    return null;
+  }
+}
 const execInput = SimpleCLI.input({
   positionals: [
-    SimpleCLI.positional("codeParts", z.array(z.string()).default([]), {
-      help: "Playwright TypeScript code to execute",
-      variadic: true
+    SimpleCLI.positional("code", z.string().optional(), {
+      help: "Playwright TypeScript code to execute"
     })
   ],
   named: {
@@ -511,28 +520,36 @@ const execInput = SimpleCLI.input({
     page: pageOption()
   }
 }).refine(
-  (input) => input.codeParts.length > 0,
-  `Usage: libretto exec <code> [--session <name>] [--visualize]`
+  (input) => input.code !== void 0,
+  `Usage: libretto exec <code|-> [--session <name>] [--visualize]
+       echo '<code>' | libretto exec - [--session <name>] [--visualize]`
 );
 const execCommand = SimpleCLI.command({
   description: "Execute Playwright TypeScript code"
 }).input(execInput).use(withRequiredSession()).handle(async ({ input, ctx }) => {
+  const code = input.code;
+  const codeFromArgsOrStdin = code === "-" ? readStdinSync() : code;
+  if (codeFromArgsOrStdin === null) {
+    throw new Error(
+      "Missing stdin input for `exec -`. Pipe Playwright code into stdin."
+    );
+  }
   await runExec(
-    input.codeParts.join(" "),
+    codeFromArgsOrStdin,
     ctx.session,
     ctx.logger,
     input.visualize,
     input.page
   );
 });
-const runUsage = `Usage: libretto run <integrationFile> <integrationExport> [--params <json> | --params-file <path>] [--tsconfig <path>] [--headed|--headless] [--no-visualize] [--viewport WxH]`;
+const runUsage = `Usage: libretto run <integrationFile> <workflowName> [--params <json> | --params-file <path>] [--credentials <json>] [--tsconfig <path>] [--headed|--headless] [--no-visualize] [--viewport WxH]`;
 const runInput = SimpleCLI.input({
   positionals: [
     SimpleCLI.positional("integrationFile", z.string().optional(), {
       help: "Path to the integration file"
     }),
-    SimpleCLI.positional("integrationExport", z.string().optional(), {
-      help: "Named workflow export to run"
+    SimpleCLI.positional("workflowName", z.string().optional(), {
+      help: "Workflow name to run (from workflow(name, handler))"
     })
   ],
   named: {
@@ -544,6 +561,9 @@ const runInput = SimpleCLI.input({
       name: "params-file",
       help: "Path to a JSON params file"
     }),
+    credentials: SimpleCLI.option(z.string().optional(), {
+      help: "Inline JSON credentials passed to ctx.credentials"
+    }),
     tsconfig: SimpleCLI.option(z.string().optional(), {
       help: "Path to a tsconfig used for workflow module resolution"
     }),
@@ -562,7 +582,7 @@ const runInput = SimpleCLI.input({
     })
   }
 }).refine(
-  (input) => Boolean(input.integrationFile && input.integrationExport),
+  (input) => Boolean(input.integrationFile && input.workflowName),
   runUsage
 ).refine(
   (input) => !(input.params && input.paramsFile),
@@ -594,6 +614,11 @@ const runCommand = SimpleCLI.command({
   await stopExistingFailedRunSession(ctx.session, ctx.logger);
   assertSessionAvailableForStart(ctx.session, ctx.logger);
   const params = resolveRunParams(input.params, input.paramsFile);
+  const rawCredentials = input.credentials ? parseJsonArg("--credentials", input.credentials) : void 0;
+  if (rawCredentials !== void 0 && (typeof rawCredentials !== "object" || rawCredentials === null || Array.isArray(rawCredentials))) {
+    throw new Error(`--credentials must be a JSON object (e.g., '{"key": "value"}').`);
+  }
+  const credentials = rawCredentials;
   const headlessMode = input.headed ? false : input.headless ? true : void 0;
   const visualize = !input.noVisualize;
   const viewport = resolveViewport(
@@ -603,9 +628,10 @@ const runCommand = SimpleCLI.command({
   await runIntegrationFromFile(
     {
       integrationPath: input.integrationFile,
-      exportName: input.integrationExport,
+      workflowName: input.workflowName,
       session: ctx.session,
       params,
+      credentials,
       tsconfigPath: input.tsconfig,
       headless: headlessMode ?? false,
       visualize,

package/dist/cli/commands/init.js

@@ -17,8 +17,8 @@ import {
   loadSnapshotEnv,
   resolveSnapshotApiModel
 } from "../core/snapshot-api-config.js";
-import { SimpleCLI } from "../framework/simple-cli.js";
 import { hasProviderCredentials } from "../../shared/llm/client.js";
+import { SimpleCLI } from "../framework/simple-cli.js";
 const PROVIDER_CHOICES = [
   {
     key: "1",
@@ -52,15 +52,6 @@ function promptUser(rl, question) {
     });
   });
 }
-function askYesNo(question) {
-  const rl = createInterface({ input: process.stdin, output: process.stdout });
-  return new Promise((resolve) => {
-    rl.question(`${question} (y/N) `, (answer) => {
-      rl.close();
-      resolve(answer.trim().toLowerCase() === "y");
-    });
-  });
-}
 function safeReadAiConfig() {
   try {
     return readAiConfig();
@@ -233,7 +224,7 @@ function detectAgentDirs(root) {
   if (existsSync(join(root, ".claude"))) dirs.push(join(root, ".claude"));
   return dirs;
 }
-async function copySkills() {
+function copySkills() {
   const agentDirs = detectAgentDirs(REPO_ROOT);
   if (agentDirs.length === 0) {
     console.log(
@@ -242,15 +233,6 @@ async function copySkills() {
     return;
   }
   const destinations = agentDirs.map((d) => join(d, "skills", "libretto"));
-  const dirNames = agentDirs.map((d) => basename(d)).join(" and ");
-  const existing = destinations.filter((d) => existsSync(d));
-  const verb = existing.length > 0 ? "Overwrite" : "Install";
-  const proceed = await askYesNo(`
-${verb} libretto skills in ${dirNames}?`);
-  if (!proceed) {
-    console.log("  Skipping skill copy.");
-    return;
-  }
   let sourceDir;
   try {
     sourceDir = getPackageSkillsDir();
@@ -289,10 +271,11 @@ const initCommand = SimpleCLI.command({
   } else {
     console.log("\nSkipping browser installation (--skip-browsers)");
   }
+  copySkills();
   if (process.stdin.isTTY) {
-    await copySkills();
     await runInteractiveApiSetup();
   } else {
+    loadSnapshotEnv();
     printSnapshotApiStatus();
   }
   console.log("\n\u2713 libretto init complete");

package/dist/cli/core/ai-config.js

@@ -11,10 +11,15 @@ const ViewportConfigSchema = z.object({
   width: z.number().int().min(1),
   height: z.number().int().min(1)
 });
+const WindowPositionConfigSchema = z.object({
+  x: z.number().int(),
+  y: z.number().int()
+});
 const LibrettoConfigSchema = z.object({
   version: z.literal(CURRENT_CONFIG_VERSION),
   ai: AiConfigSchema.optional(),
-  viewport: ViewportConfigSchema.optional()
+  viewport: ViewportConfigSchema.optional(),
+  windowPosition: WindowPositionConfigSchema.optional()
 }).passthrough();
 const DEFAULT_MODELS = {
   openai: "openai/gpt-5.4",
@@ -49,6 +54,10 @@ function formatExpectedConfigExample() {
       viewport: {
         width: 1280,
         height: 800
+      },
+      windowPosition: {
+        x: 1600,
+        y: 120
       }
     },
     null,
@@ -64,7 +73,7 @@ ${detail}` : null,
       "Expected config example:",
       formatExpectedConfigExample(),
       "Notes:",
-      '  - "ai" and "viewport" are optional.',
+      '  - "ai", "viewport", and "windowPosition" are optional.',
       '  - "ai.model" must be a provider/model string like "openai/gpt-5.4" or "anthropic/claude-sonnet-4-6".',
       "Fix the file to match this shape, or delete it and rerun:",
       `  npx libretto ai configure ${formatConfigureProviders()}`
@@ -189,6 +198,7 @@ export {
   CURRENT_CONFIG_VERSION,
   LibrettoConfigSchema,
   ViewportConfigSchema,
+  WindowPositionConfigSchema,
   clearAiConfig,
   readAiConfig,
   readLibrettoConfig,

package/dist/cli/core/browser.js

@@ -6,6 +6,14 @@ import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { createServer } from "node:net";
 import { spawn } from "node:child_process";
+import {
+  filterSemanticClasses,
+  INTERACTIVE_ROLE_NAMES,
+  INTERACTIVE_TAG_NAMES,
+  isObfuscatedClass,
+  TEST_ATTRIBUTE_NAMES,
+  TRUSTED_ATTRIBUTE_NAMES
+} from "../../shared/dom-semantics.js";
 import {
   getSessionActionsLogPath,
   getSessionNetworkLogPath,
@@ -245,10 +253,22 @@ function resolveViewport(cliViewport, logger) {
   });
   return DEFAULT_VIEWPORT;
 }
+function resolveWindowPosition(logger) {
+  const config = readLibrettoConfig();
+  if (config.windowPosition) {
+    logger.info("window-position-source", {
+      source: "config",
+      windowPosition: config.windowPosition
+    });
+    return config.windowPosition;
+  }
+  return void 0;
+}
 async function runOpen(rawUrl, headed, session, logger, options) {
   const url = normalizeUrl(rawUrl);
   const viewport = resolveViewport(options?.viewport, logger);
-  logger.info("open-start", { url, headed, session, viewport });
+  const windowPosition = headed ? resolveWindowPosition(logger) : void 0;
+  logger.info("open-start", { url, headed, session, viewport, windowPosition });
   assertSessionAvailableForStart(session, logger);
   const port = await pickFreePort();
   const runLogPath = logFileForSession(session);
@@ -277,6 +297,45 @@ async function runOpen(rawUrl, headed, session, logger, options) {
   const escapedLogPath = runLogPath.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
   const escapedNetworkLogPath = networkLogPath.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
   const escapedActionsLogPath = actionsLogPath.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+  const windowPositionArg = windowPosition ? `, '--window-position=${windowPosition.x},${windowPosition.y}'` : "";
+  const windowBoundsSetupCode = windowPosition ? `
+const requestedWindowBounds = { left: ${windowPosition.x}, top: ${windowPosition.y}, windowState: 'normal' };
+const pageCdp = await context.newCDPSession(page);
+let browserCdp;
+try {
+	const targetInfo = await pageCdp.send('Target.getTargetInfo');
+	const targetId = targetInfo?.targetInfo?.targetId;
+	browserCdp = await browser.newBrowserCDPSession();
+	const windowResult = await browserCdp.send(
+		'Browser.getWindowForTarget',
+		targetId ? { targetId } : {},
+	);
+	await browserCdp.send('Browser.setWindowBounds', {
+		windowId: windowResult.windowId,
+		bounds: requestedWindowBounds,
+	});
+	await new Promise((resolve) => setTimeout(resolve, 250));
+	const actualWindow = await browserCdp.send('Browser.getWindowBounds', {
+		windowId: windowResult.windowId,
+	});
+	childLog('info', 'window-bounds-set', {
+		windowId: windowResult.windowId,
+		requestedBounds: requestedWindowBounds,
+		actualBounds: actualWindow.bounds,
+	});
+} catch (error) {
+	childLog('warn', 'window-bounds-set-failed', {
+		requestedBounds: requestedWindowBounds,
+		message: error instanceof Error ? error.message : String(error),
+		stack: error instanceof Error ? error.stack : undefined,
+	});
+} finally {
+	await pageCdp.detach().catch(() => {});
+	if (browserCdp) {
+		await browserCdp.detach().catch(() => {});
+	}
+}
+` : "";
   const launcherCode = `
 import { chromium } from 'playwright';
 import { appendFileSync, mkdirSync } from 'node:fs';
@@ -288,14 +347,21 @@ const ACTIONS_LOG = '${escapedActionsLogPath}';
 mkdirSync(dirname(NETWORK_LOG), { recursive: true });
 
 // tsx/esbuild may emit __name() wrappers in Function#toString output.
-const __name = (target, value) =>
-	Object.defineProperty(target, 'name', { value, configurable: true });
+	const __name = (target, value) =>
+		Object.defineProperty(target, 'name', { value, configurable: true });
 
-${installSessionTelemetry.toString()}
+	const TEST_ATTRIBUTE_NAMES = ${JSON.stringify([...TEST_ATTRIBUTE_NAMES])};
+	const TRUSTED_ATTRIBUTE_NAMES = ${JSON.stringify([...TRUSTED_ATTRIBUTE_NAMES])};
+	const INTERACTIVE_TAG_NAMES = ${JSON.stringify([...INTERACTIVE_TAG_NAMES])};
+	const INTERACTIVE_ROLE_NAMES = ${JSON.stringify([...INTERACTIVE_ROLE_NAMES])};
+	const filterSemanticClasses = ${filterSemanticClasses.toString()};
+	const isObfuscatedClass = ${isObfuscatedClass.toString()};
 
-function logAction(entry) {
-	appendFileSync(ACTIONS_LOG, JSON.stringify(entry) + '\\n');
-}
+	${installSessionTelemetry.toString()}
+
+	function logAction(entry) {
+		appendFileSync(ACTIONS_LOG, JSON.stringify(entry) + '\\n');
+	}
 
 function logNetwork(entry) {
 	appendFileSync(NETWORK_LOG, JSON.stringify(entry) + '\\n');
@@ -317,7 +383,7 @@ function childLog(level, event, data = {}) {
 
 const browser = await chromium.launch({
 	headless: ${!headed},
-	args: ['--disable-blink-features=AutomationControlled', '--remote-debugging-port=${port}', '--remote-debugging-address=127.0.0.1', '--no-focus-on-check'],
+	args: ['--disable-blink-features=AutomationControlled', '--remote-debugging-port=${port}', '--remote-debugging-address=127.0.0.1', '--no-focus-on-check'${windowPositionArg}],
 });
 
 browser.on('disconnected', () => {
@@ -331,6 +397,7 @@ const context = await browser.newContext({
 });
 
 const page = await context.newPage();
+${windowBoundsSetupCode}
 page.setDefaultTimeout(30000);
 page.setDefaultNavigationTimeout(45000);