libretto 0.4.4 → 0.5.1

package/src/cli/index.ts

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import { runLibrettoCLI } from "./cli.js";
+import {
+  maybeConfigureLLMClientFactoryFromEnv,
+  setLLMClientFactory,
+} from "./core/context.js";
+
+export { setLLMClientFactory };
+export { runClose } from "./commands/browser.js";
+export { runLibrettoCLI };
+
+maybeConfigureLLMClientFactoryFromEnv();
+void runLibrettoCLI();

package/src/cli/router.ts

@@ -0,0 +1,20 @@
+import { aiCommands } from "./commands/ai.js";
+import { browserCommands } from "./commands/browser.js";
+import { executionCommands } from "./commands/execution.js";
+import { initCommand } from "./commands/init.js";
+import { logCommands } from "./commands/logs.js";
+import { snapshotCommand } from "./commands/snapshot.js";
+import { SimpleCLI } from "./framework/simple-cli.js";
+
+export const cliRoutes = {
+  ...browserCommands,
+  ...executionCommands,
+  ...logCommands,
+  ai: aiCommands,
+  init: initCommand,
+  snapshot: snapshotCommand,
+};
+
+export function createCLIApp() {
+  return SimpleCLI.define("libretto", cliRoutes);
+}

package/src/cli/workers/run-integration-runtime.ts

@@ -0,0 +1,338 @@
+import type { BrowserContext } from "playwright";
+import { appendFileSync, existsSync, readFileSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import { cwd } from "node:process";
+import { isAbsolute, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import {
+  instrumentContext,
+  launchBrowser,
+  type LibrettoWorkflowContext,
+} from "../../index.js";
+import type { LoggerApi } from "../../shared/logger/index.js";
+import { parseSessionStateContent } from "../../shared/state/index.js";
+import { getProfilePath, normalizeDomain } from "../core/browser.js";
+import {
+  getSessionActionsLogPath,
+  getSessionDir,
+  getSessionNetworkLogPath,
+  getSessionStatePath,
+} from "../core/context.js";
+import {
+  getPauseSignalPaths,
+  removeSignalIfExists,
+} from "../core/pause-signals.js";
+import { installSessionTelemetry } from "../core/session-telemetry.js";
+import type { RunIntegrationWorkerRequest } from "./run-integration-worker-protocol.js";
+
+const LIBRETTO_WORKFLOW_BRAND = Symbol.for("libretto.workflow");
+
+type LoadedLibrettoWorkflow = {
+  run: (ctx: LibrettoWorkflowContext, input: unknown) => Promise<unknown>;
+};
+
+type RunIntegrationOutcome =
+  | { status: "completed" }
+  | { status: "failed-held" };
+
+const FAILURE_HOLD_POLL_INTERVAL_MS = 250;
+const TSCONFIG_HINT =
+  "TypeScript compilation failed. Pass --tsconfig <path> to run against a specific tsconfig.";
+
+function isTsxCompileError(error: unknown): error is Error {
+  return (
+    error instanceof Error &&
+    (error.name === "TransformError" ||
+      error.message.startsWith("Cannot resolve tsconfig at path:"))
+  );
+}
+
+function mirrorStdoutToFile(filePath: string): () => void {
+  const stdout = process.stdout as NodeJS.WriteStream & {
+    write: (...args: any[]) => boolean;
+  };
+  const originalWrite = stdout.write.bind(stdout);
+
+  stdout.write = ((chunk: unknown, ...args: unknown[]) => {
+    try {
+      const buffer = Buffer.isBuffer(chunk)
+        ? chunk
+        : Buffer.from(String(chunk), "utf8");
+      appendFileSync(filePath, buffer);
+    } catch {
+      // Ignore log mirroring failures; primary stdout should still flow.
+    }
+    return originalWrite(chunk, ...args);
+  }) as typeof stdout.write;
+
+  return () => {
+    stdout.write = originalWrite as typeof stdout.write;
+  };
+}
+
+function readSessionStatePid(session: string): number | null {
+  const statePath = getSessionStatePath(session);
+  if (!existsSync(statePath)) return null;
+
+  try {
+    return (
+      parseSessionStateContent(readFileSync(statePath, "utf8"), statePath)
+        .pid ?? null
+    );
+  } catch {
+    return null;
+  }
+}
+
+async function waitForFailureSessionRelease(args: {
+  session: string;
+  expectedPid: number;
+  logger: LoggerApi;
+}): Promise<void> {
+  const { session, expectedPid, logger } = args;
+  logger.info("run-failure-session-hold", { session, expectedPid });
+
+  while (true) {
+    const currentPid = readSessionStatePid(session);
+    if (currentPid !== expectedPid) {
+      logger.info("run-failure-session-released", {
+        session,
+        expectedPid,
+        currentPid,
+      });
+      return;
+    }
+    await new Promise((resolveWait) =>
+      setTimeout(resolveWait, FAILURE_HOLD_POLL_INTERVAL_MS),
+    );
+  }
+}
+
+function isLoadedLibrettoWorkflow(
+  value: unknown,
+): value is LoadedLibrettoWorkflow {
+  if (!value || typeof value !== "object") return false;
+  const candidate = value as Record<PropertyKey, unknown>;
+  return (
+    candidate[LIBRETTO_WORKFLOW_BRAND] === true &&
+    typeof candidate.run === "function"
+  );
+}
+
+function resolveLocalAuthProfilePath(domain: string): string {
+  return getProfilePath(normalizeDomain(domain));
+}
+
+function getMissingLocalAuthProfileError(args: {
+  domain: string;
+  profilePath: string;
+  session: string;
+}): string {
+  const normalizedDomain = normalizeDomain(args.domain);
+  return [
+    `Local auth profile not found for domain "${normalizedDomain}".`,
+    `Expected profile file: ${args.profilePath}`,
+    "To create it:",
+    `  1. libretto open https://${normalizedDomain} --headed --session ${args.session}`,
+    "  2. Log in manually in the browser window.",
+    `  3. libretto save ${normalizedDomain} --session ${args.session}`,
+  ].join("\n");
+}
+
+function getAbsoluteIntegrationPath(integrationPath: string): string {
+  const absolutePath = isAbsolute(integrationPath)
+    ? integrationPath
+    : resolve(cwd(), integrationPath);
+  if (!existsSync(absolutePath)) {
+    throw new Error(`Integration file does not exist: ${absolutePath}`);
+  }
+  return absolutePath;
+}
+
+async function loadWorkflowExport(
+  absolutePath: string,
+  exportName: string,
+): Promise<LoadedLibrettoWorkflow> {
+  let loadedModule: Record<string, unknown>;
+  try {
+    loadedModule = (await import(pathToFileURL(absolutePath).href)) as Record<
+      string,
+      unknown
+    >;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const compileHint = isTsxCompileError(error) ? `\n${TSCONFIG_HINT}` : "";
+    throw new Error(
+      `Failed to import integration module at ${absolutePath}: ${message}${compileHint}`,
+    );
+  }
+
+  const targetExport = loadedModule[exportName];
+  if (!targetExport) {
+    const availableExports = Object.keys(loadedModule);
+    const detail =
+      availableExports.length > 0
+        ? ` Available exports: ${availableExports.join(", ")}`
+        : " The module has no exports.";
+    throw new Error(
+      `Export "${exportName}" was not found in ${absolutePath}.${detail}`,
+    );
+  }
+
+  if (!isLoadedLibrettoWorkflow(targetExport)) {
+    throw new Error(
+      [
+        `Export "${exportName}" in ${absolutePath} is not a valid Libretto workflow.`,
+        "",
+        'A workflow must be created using the workflow() function from "libretto":',
+        "",
+        '  import { workflow } from "libretto";',
+        "",
+        `  export const ${exportName} = workflow<InputType, OutputType>(`,
+        "    async (ctx, input) => {",
+        "      // ctx.session  — libretto session name",
+        "      // ctx.page     — Playwright Page instance",
+        "      // ctx.logger   — MinimalLogger",
+        "      // ctx.services — injected dependencies (generic, default {})",
+        "      // input        — JSON-serializable input matching InputType",
+        "      return output; // must match OutputType",
+        "    },",
+        "  );",
+      ].join("\n"),
+    );
+  }
+
+  return targetExport;
+}
+
+export async function installHeadedWorkflowVisualization(args: {
+  context: BrowserContext;
+  logger: LoggerApi;
+  instrument?: typeof instrumentContext;
+}): Promise<void> {
+  await (args.instrument ?? instrumentContext)(args.context, {
+    visualize: true,
+    logger: args.logger,
+  });
+}
+
+async function runIntegrationInternal(
+  args: RunIntegrationWorkerRequest,
+  options: {
+    logger: LoggerApi;
+  },
+): Promise<RunIntegrationOutcome> {
+  const { logger } = options;
+  const absolutePath = getAbsoluteIntegrationPath(args.integrationPath);
+  const workflow = await loadWorkflowExport(absolutePath, args.exportName);
+  const signalPaths = getPauseSignalPaths(args.session);
+  await removeSignalIfExists(signalPaths.pausedSignalPath);
+  await removeSignalIfExists(signalPaths.resumeSignalPath);
+  await removeSignalIfExists(signalPaths.completedSignalPath);
+  await removeSignalIfExists(signalPaths.failedSignalPath);
+  const restoreStdout = mirrorStdoutToFile(signalPaths.outputSignalPath);
+
+  console.log(
+    `Running integration "${args.exportName}" from ${absolutePath} (${args.headless ? "headless" : "headed"})...`,
+  );
+
+  const integrationLogger = logger.withScope("integration-run", {
+    integrationPath: absolutePath,
+    integrationExport: args.exportName,
+    session: args.session,
+  });
+
+  // Resolve auth profile from CLI flag (--auth-profile <domain>)
+  const authProfileDomain = args.authProfileDomain;
+  const storageStatePath = authProfileDomain
+    ? resolveLocalAuthProfilePath(authProfileDomain)
+    : undefined;
+  if (authProfileDomain && storageStatePath && !existsSync(storageStatePath)) {
+    throw new Error(
+      getMissingLocalAuthProfileError({
+        domain: authProfileDomain,
+        profilePath: storageStatePath,
+        session: args.session,
+      }),
+    );
+  }
+  const browserSession = await launchBrowser({
+    sessionName: args.session,
+    headless: args.headless,
+    storageStatePath,
+    viewport: args.viewport,
+  });
+  if (!args.headless && args.visualize !== false) {
+    await installHeadedWorkflowVisualization({
+      context: browserSession.context,
+      logger: integrationLogger,
+    });
+  }
+  const actionsLogPath = getSessionActionsLogPath(args.session);
+  const networkLogPath = getSessionNetworkLogPath(args.session);
+  await installSessionTelemetry({
+    context: browserSession.context,
+    initialPage: browserSession.page,
+    includeUserDomActions: true,
+    logAction: (entry) => {
+      appendFileSync(actionsLogPath, JSON.stringify(entry) + "\n");
+    },
+    logNetwork: (entry) => {
+      appendFileSync(networkLogPath, JSON.stringify(entry) + "\n");
+    },
+  });
+
+  const workflowContext: LibrettoWorkflowContext = {
+    session: args.session,
+    logger: integrationLogger,
+    page: browserSession.page,
+    services: {},
+  };
+
+  try {
+    try {
+      await workflow.run(workflowContext, args.params ?? {});
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      await writeFile(
+        signalPaths.failedSignalPath,
+        JSON.stringify(
+          {
+            failedAt: new Date().toISOString(),
+            message: errorMessage,
+            phase: "workflow",
+          },
+          null,
+          2,
+        ),
+        "utf8",
+      );
+      await waitForFailureSessionRelease({
+        session: args.session,
+        expectedPid: process.pid,
+        logger,
+      });
+      return { status: "failed-held" };
+    }
+    await writeFile(
+      signalPaths.completedSignalPath,
+      JSON.stringify({ completedAt: new Date().toISOString() }, null, 2),
+      "utf8",
+    );
+    console.log("Integration completed.");
+    return { status: "completed" };
+  } finally {
+    restoreStdout();
+    await browserSession.close();
+  }
+}
+
+export async function runIntegrationFromFileInWorker(
+  args: RunIntegrationWorkerRequest,
+  logger: LoggerApi,
+): Promise<RunIntegrationOutcome> {
+  return await runIntegrationInternal(args, {
+    logger,
+  });
+}

package/src/cli/workers/run-integration-worker-protocol.ts

@@ -0,0 +1,16 @@
+import { z } from "zod";
+
+export const RunIntegrationWorkerRequestSchema = z.object({
+  integrationPath: z.string().min(1),
+  exportName: z.string().min(1),
+  session: z.string().min(1),
+  params: z.unknown(),
+  headless: z.boolean(),
+  visualize: z.boolean().default(true),
+  authProfileDomain: z.string().optional(),
+  viewport: z.object({ width: z.number(), height: z.number() }).optional(),
+});
+
+export type RunIntegrationWorkerRequest = z.infer<
+  typeof RunIntegrationWorkerRequestSchema
+>;

package/src/cli/workers/run-integration-worker.ts

@@ -0,0 +1,72 @@
+import { writeFile } from "node:fs/promises";
+import { ZodError } from "zod";
+import {
+  RunIntegrationWorkerRequestSchema,
+  type RunIntegrationWorkerRequest,
+} from "./run-integration-worker-protocol.js";
+import { runIntegrationFromFileInWorker } from "./run-integration-runtime.js";
+import { ensureLibrettoSetup, withSessionLogger } from "../core/context.js";
+import { getPauseSignalPaths } from "../core/pause-signals.js";
+
+function parseWorkerRequest(argv: string[]): RunIntegrationWorkerRequest {
+  const rawPayload = argv[2];
+  if (!rawPayload) {
+    throw new Error("Missing worker payload argument.");
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(rawPayload);
+  } catch (error) {
+    throw new Error(
+      `Invalid worker payload JSON: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+
+  try {
+    return RunIntegrationWorkerRequestSchema.parse(parsed);
+  } catch (error) {
+    if (error instanceof ZodError) {
+      const details = error.issues
+        .map((issue) => `${issue.path.join(".") || "root"}: ${issue.message}`)
+        .join("; ");
+      throw new Error(`Worker payload is invalid: ${details}`);
+    }
+    throw error;
+  }
+}
+
+async function main(): Promise<void> {
+  let request: RunIntegrationWorkerRequest | null = null;
+  let exitCode = 0;
+  try {
+    request = parseWorkerRequest(process.argv);
+    const workerRequest = request;
+    ensureLibrettoSetup();
+    await withSessionLogger(workerRequest.session, async (logger) => {
+      await runIntegrationFromFileInWorker(workerRequest, logger);
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (request) {
+      const { failedSignalPath } = getPauseSignalPaths(request.session);
+      await writeFile(
+        failedSignalPath,
+        JSON.stringify(
+          {
+            failedAt: new Date().toISOString(),
+            message,
+            phase: "setup",
+          },
+          null,
+          2,
+        ),
+        "utf8",
+      );
+    }
+    exitCode = 1;
+  }
+  process.exit(exitCode);
+}
+
+void main();

package/src/index.ts

@@ -0,0 +1,127 @@
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+// Logger
+export {
+  Logger,
+  defaultLogger,
+  type LoggerApi,
+  type MinimalLogger,
+  type LoggerSink,
+  type LogOptions,
+} from "./shared/logger/logger.js";
+export {
+  createFileLogSink,
+  prettyConsoleSink,
+  jsonlConsoleSink,
+} from "./shared/logger/sinks.js";
+
+// LLM client interface
+export type {
+  LLMClient,
+  Message,
+  MessageContentPart,
+} from "./shared/llm/types.js";
+export { createLLMClientFromModel } from "./shared/llm/ai-sdk-adapter.js";
+export {
+  SESSION_STATE_VERSION,
+  SessionStatusSchema,
+  SessionStateFileSchema,
+  parseSessionStateData,
+  parseSessionStateContent,
+  serializeSessionState,
+  type SessionStatus,
+  type SessionState,
+  type SessionStateFile,
+} from "./shared/state/index.js";
+
+// Recovery
+export { executeRecoveryAgent } from "./runtime/recovery/agent.js";
+export { attemptWithRecovery } from "./runtime/recovery/recovery.js";
+export {
+  detectSubmissionError,
+  type KnownSubmissionError,
+  type DetectedSubmissionError,
+} from "./runtime/recovery/errors.js";
+
+// AI extraction
+export {
+  extractFromPage,
+  type ExtractOptions,
+} from "./runtime/extract/extract.js";
+
+// Network helpers
+export {
+  pageRequest,
+  type RequestConfig,
+  type PageRequestOptions,
+} from "./runtime/network/network.js";
+
+// Download helpers
+export {
+  downloadViaClick,
+  downloadAndSave,
+  type DownloadResult,
+  type DownloadViaClickOptions,
+  type SaveDownloadOptions,
+} from "./runtime/download/download.js";
+
+// Debug / Pause
+export { pause } from "./shared/debug/pause.js";
+
+// Instrumentation
+export {
+  instrumentPage,
+  installInstrumentation,
+  instrumentContext,
+  type InstrumentationOptions,
+  type InstrumentedPage,
+} from "./shared/instrumentation/instrument.js";
+
+// Visualization
+export {
+  ensureGhostCursor,
+  moveGhostCursor,
+  ghostClick,
+  hideGhostCursor,
+  type GhostCursorOptions,
+} from "./shared/visualization/ghost-cursor.js";
+export {
+  ensureHighlightLayer,
+  showHighlight,
+  clearHighlights,
+  type HighlightOptions,
+} from "./shared/visualization/highlight.js";
+
+// Run helpers
+export {
+  launchBrowser,
+  type LaunchBrowserArgs,
+  type BrowserSession,
+} from "./shared/run/api.js";
+
+// Workflow helpers
+export {
+  LibrettoWorkflow,
+  LIBRETTO_WORKFLOW_BRAND,
+  workflow,
+  type LibrettoWorkflowContext,
+  type LibrettoWorkflowHandler,
+} from "./shared/workflow/workflow.js";
+
+const isDirectExecution = (): boolean => {
+  const entryArg = process.argv[1];
+  if (!entryArg) {
+    return false;
+  }
+  return pathToFileURL(resolve(entryArg)).href === import.meta.url;
+};
+
+if (isDirectExecution()) {
+  void import("./cli/index.js").catch((error: unknown) => {
+    const message =
+      error instanceof Error ? (error.stack ?? error.message) : String(error);
+    process.stderr.write(`${message}\n`);
+    process.exitCode = 1;
+  });
+}

package/src/runtime/download/download.ts

@@ -0,0 +1,104 @@
+import { writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import type { Page, Download } from "playwright";
+import type { MinimalLogger } from "../../shared/logger/logger.js";
+
+export type DownloadResult = {
+  /** The raw file contents. */
+  buffer: Buffer;
+  /** The filename suggested by the server (Content-Disposition header or URL). */
+  filename: string;
+};
+
+export type DownloadViaClickOptions = {
+  logger?: MinimalLogger;
+  /** Timeout in milliseconds for waiting on the download event. Defaults to 30 000. */
+  timeout?: number;
+};
+
+/**
+ * Triggers a file download by clicking a DOM element and intercepts the
+ * resulting download using Playwright's download event.
+ *
+ * The download promise is registered **before** the click so the event is
+ * never missed.
+ */
+export async function downloadViaClick(
+  page: Page,
+  selector: string,
+  options?: DownloadViaClickOptions,
+): Promise<DownloadResult> {
+  const { logger, timeout = 30_000 } = options ?? {};
+
+  const startTime = Date.now();
+
+  // 1. Register the download listener BEFORE clicking
+  const downloadPromise = page.waitForEvent("download", { timeout });
+
+  // 2. Click the element that triggers the download
+  await page.locator(selector).click();
+
+  // 3. Await the download event
+  const download: Download = await downloadPromise;
+
+  // 4. Get the suggested filename
+  const filename = download.suggestedFilename();
+
+  // 5. Read the downloaded file into a buffer
+  const readStream = await download.createReadStream();
+  if (!readStream) {
+    throw new Error(
+      `Download stream unavailable for "${filename}". The browser may have been closed before the download completed.`,
+    );
+  }
+
+  const chunks: Buffer[] = [];
+  for await (const chunk of readStream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  const buffer = Buffer.concat(chunks);
+
+  const duration = Date.now() - startTime;
+
+  logger?.info("download:click", {
+    selector,
+    filename,
+    size: buffer.length,
+    duration,
+  });
+
+  return { buffer, filename };
+}
+
+export type SaveDownloadOptions = DownloadViaClickOptions & {
+  /** Absolute or relative path to save the file to. When omitted the suggested filename is used in the current working directory. */
+  savePath?: string;
+};
+
+/**
+ * Convenience wrapper around {@link downloadViaClick} that also writes the
+ * downloaded file to disk.
+ */
+export async function downloadAndSave(
+  page: Page,
+  selector: string,
+  options?: SaveDownloadOptions,
+): Promise<DownloadResult & { savedTo: string }> {
+  const { savePath, ...downloadOpts } = options ?? {};
+  const { buffer, filename } = await downloadViaClick(
+    page,
+    selector,
+    downloadOpts,
+  );
+
+  const dest = resolve(savePath ?? filename);
+  await writeFile(dest, buffer);
+
+  options?.logger?.info("download:saved", {
+    filename,
+    savedTo: dest,
+    size: buffer.length,
+  });
+
+  return { buffer, filename, savedTo: dest };
+}

package/src/runtime/download/index.ts

@@ -0,0 +1,7 @@
+export {
+  downloadViaClick,
+  downloadAndSave,
+  type DownloadResult,
+  type DownloadViaClickOptions,
+  type SaveDownloadOptions,
+} from "./download.js";