libretto 0.4.4 → 0.5.1

package/src/cli/commands/snapshot.ts

@@ -0,0 +1,312 @@
+import { mkdirSync } from "node:fs";
+import { z } from "zod";
+import type { LoggerApi } from "../../shared/logger/index.js";
+import { connect, disconnectBrowser } from "../core/browser.js";
+import { getSessionSnapshotRunDir } from "../core/context.js";
+import { condenseDom } from "../../shared/condense-dom/condense-dom.js";
+import { readSessionState } from "../core/session.js";
+import {
+  type InterpretArgs,
+  type ScreenshotPair,
+} from "../core/snapshot-analyzer.js";
+import { SimpleCLI } from "../framework/simple-cli.js";
+import { pageOption, sessionOption, withRequiredSession } from "./shared.js";
+import { runApiInterpret } from "../core/api-snapshot-analyzer.js";
+import { readAiConfig } from "../core/ai-config.js";
+import { resolveSnapshotApiModelOrThrow } from "../core/snapshot-api-config.js";
+
+const FALLBACK_SNAPSHOT_VIEWPORT = { width: 1280, height: 800 } as const;
+
+function generateSnapshotRunId(): string {
+  return `snapshot-${Date.now()}`;
+}
+
+type SnapshotViewportMetrics = {
+  configuredWidth: number | null;
+  configuredHeight: number | null;
+  innerWidth: number | null;
+  innerHeight: number | null;
+};
+
+function isZeroViewport(value: number | null): boolean {
+  return typeof value === "number" && value <= 0;
+}
+
+function shouldForceSnapshotViewport(
+  metrics: SnapshotViewportMetrics,
+): boolean {
+  return (
+    isZeroViewport(metrics.configuredWidth) ||
+    isZeroViewport(metrics.configuredHeight) ||
+    isZeroViewport(metrics.innerWidth) ||
+    isZeroViewport(metrics.innerHeight)
+  );
+}
+
+function isZeroWidthScreenshotError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    error.message.includes("Cannot take screenshot with 0 width")
+  );
+}
+
+async function readSnapshotViewportMetrics(page: {
+  viewportSize(): { width: number; height: number } | null;
+  evaluate<T>(pageFunction: () => T | Promise<T>): Promise<T>;
+}): Promise<SnapshotViewportMetrics> {
+  const configuredViewport = page.viewportSize();
+  let innerWidth: number | null = null;
+  let innerHeight: number | null = null;
+
+  try {
+    const innerViewport = await page.evaluate(() => ({
+      width: window.innerWidth,
+      height: window.innerHeight,
+    }));
+    innerWidth = innerViewport.width;
+    innerHeight = innerViewport.height;
+  } catch {}
+
+  return {
+    configuredWidth: configuredViewport?.width ?? null,
+    configuredHeight: configuredViewport?.height ?? null,
+    innerWidth,
+    innerHeight,
+  };
+}
+
+function resolveSnapshotViewport(
+  session: string,
+  logger: LoggerApi,
+): { width: number; height: number } {
+  const state = readSessionState(session, logger);
+  if (state?.viewport) {
+    logger.info("screenshot-viewport-from-session-state", {
+      session,
+      viewport: state.viewport,
+    });
+    return state.viewport;
+  }
+  logger.info("screenshot-viewport-fallback", {
+    session,
+    reason: "no viewport in session state",
+    viewport: FALLBACK_SNAPSHOT_VIEWPORT,
+  });
+  return FALLBACK_SNAPSHOT_VIEWPORT;
+}
+
+async function forceSnapshotViewport(
+  page: {
+    setViewportSize(size: { width: number; height: number }): Promise<void>;
+  },
+  viewport: { width: number; height: number },
+  logger: LoggerApi,
+  session: string,
+  pageId?: string,
+  reason?: string,
+): Promise<void> {
+  await page.setViewportSize(viewport);
+  logger.warn("screenshot-viewport-forced", {
+    session,
+    pageId,
+    reason,
+    viewport,
+  });
+}
+
+async function captureScreenshot(
+  session: string,
+  logger: LoggerApi,
+  pageId?: string,
+): Promise<ScreenshotPair> {
+  logger.info("screenshot-start", { session, pageId });
+  const snapshotRunId = generateSnapshotRunId();
+  const snapshotRunDir = getSessionSnapshotRunDir(session, snapshotRunId);
+  mkdirSync(snapshotRunDir, { recursive: true });
+  const { browser, page } = await connect(session, logger, 10000, {
+    pageId,
+    requireSinglePage: true,
+  });
+
+  try {
+    let title: string | null = null;
+    try {
+      title = await page.title();
+    } catch (error) {
+      logger.warn("screenshot-title-read-failed", {
+        session,
+        pageId,
+        error,
+      });
+    }
+
+    let pageUrl: string | null = null;
+    try {
+      pageUrl = page.url();
+    } catch (error) {
+      logger.warn("screenshot-url-read-failed", {
+        session,
+        pageId,
+        error,
+      });
+    }
+
+    const pngPath = `${snapshotRunDir}/page.png`;
+    const htmlPath = `${snapshotRunDir}/page.html`;
+    const condensedHtmlPath = `${snapshotRunDir}/page.condensed.html`;
+
+    const RENDER_SETTLE_TIMEOUT_MS = 10_000;
+    await Promise.race([
+      page.waitForLoadState("networkidle").catch(() => {}),
+      new Promise((resolve) => setTimeout(resolve, RENDER_SETTLE_TIMEOUT_MS)),
+    ]);
+
+    const restoreViewport = resolveSnapshotViewport(session, logger);
+    const viewportMetrics = await readSnapshotViewportMetrics(page);
+    logger.info("screenshot-viewport-metrics", {
+      session,
+      pageId,
+      restoreViewport,
+      ...viewportMetrics,
+    });
+    await forceSnapshotViewport(
+      page,
+      restoreViewport,
+      logger,
+      session,
+      pageId,
+      shouldForceSnapshotViewport(viewportMetrics)
+        ? "preflight-invalid-viewport"
+        : "preflight-normalize-viewport",
+    );
+
+    try {
+      await page.screenshot({ path: pngPath });
+    } catch (error) {
+      if (!isZeroWidthScreenshotError(error)) {
+        throw error;
+      }
+      await forceSnapshotViewport(
+        page,
+        restoreViewport,
+        logger,
+        session,
+        pageId,
+        "retry-after-zero-width-screenshot-error",
+      );
+      await page.screenshot({ path: pngPath });
+    }
+
+    const htmlContent = await page.content();
+    const fs = await import("node:fs/promises");
+    await fs.writeFile(htmlPath, htmlContent);
+
+    // Write condensed DOM
+    const condenseResult = condenseDom(htmlContent);
+    await fs.writeFile(condensedHtmlPath, condenseResult.html);
+
+    logger.info("screenshot-success", {
+      session,
+      pageUrl,
+      title,
+      pngPath,
+      htmlPath,
+      condensedHtmlPath,
+      snapshotRunId,
+      domCondenseStats: {
+        originalLength: condenseResult.originalLength,
+        condensedLength: condenseResult.condensedLength,
+        reductions: condenseResult.reductions,
+      },
+    });
+    return { pngPath, htmlPath, condensedHtmlPath, baseName: snapshotRunId };
+  } catch (err) {
+    let pageAlive = false;
+    let browserConnected = false;
+    try {
+      browserConnected = browser.isConnected();
+      pageAlive = !page.isClosed();
+    } catch {}
+    logger.error("screenshot-error", {
+      error: err,
+      session,
+      pageAlive,
+      browserConnected,
+      pageUrl: (() => {
+        try {
+          return page.url();
+        } catch {
+          return null;
+        }
+      })(),
+    });
+    throw err;
+  } finally {
+    disconnectBrowser(browser, logger, session);
+  }
+}
+
+async function runSnapshot(
+  session: string,
+  logger: LoggerApi,
+  pageId: string | undefined,
+  objective: string,
+  context: string,
+): Promise<void> {
+  const normalizedObjective = objective.trim();
+  const normalizedContext = context.trim();
+
+  const configuredAi = readAiConfig();
+  resolveSnapshotApiModelOrThrow(configuredAi);
+
+  const { pngPath, htmlPath, condensedHtmlPath } = await captureScreenshot(
+    session,
+    logger,
+    pageId,
+  );
+
+  console.log("Screenshot saved:");
+  console.log(`  PNG:             ${pngPath}`);
+  console.log(`  HTML:            ${htmlPath}`);
+  console.log(`  Condensed HTML:  ${condensedHtmlPath}`);
+
+  const interpretArgs: InterpretArgs = {
+    objective: normalizedObjective,
+    session,
+    context: normalizedContext,
+    pngPath,
+    htmlPath,
+    condensedHtmlPath,
+  };
+
+  // Analysis uses direct API calls via the Vercel AI SDK (see api-snapshot-analyzer.ts).
+  // The legacy CLI-agent path (spawning codex/claude/gemini as a subprocess) is preserved
+  // in snapshot-analyzer.ts — to switch back, replace this call with:
+  //   await runInterpret(interpretArgs, logger);
+  await runApiInterpret(interpretArgs, logger, configuredAi);
+}
+
+export const snapshotInput = SimpleCLI.input({
+  positionals: [],
+  named: {
+    session: sessionOption(),
+    page: pageOption(),
+    objective: SimpleCLI.option(z.string()),
+    context: SimpleCLI.option(z.string()),
+  },
+});
+
+export const snapshotCommand = SimpleCLI.command({
+  description: "Capture PNG + HTML and analyze with --objective and --context",
+})
+  .input(snapshotInput)
+  .use(withRequiredSession())
+  .handle(async ({ input, ctx }) => {
+    await runSnapshot(
+      ctx.session,
+      ctx.logger,
+      input.page,
+      input.objective,
+      input.context,
+    );
+  });

package/src/cli/core/ai-config.ts

@@ -0,0 +1,264 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { z } from "zod";
+import { LIBRETTO_CONFIG_PATH } from "./context.js";
+
+export const CURRENT_CONFIG_VERSION = 1;
+
+/**
+ * AI configuration schema.
+ *
+ * The `model` field is a provider/model-id string (e.g. "openai/gpt-5.4",
+ * "anthropic/claude-sonnet-4-6", "google/gemini-3-flash-preview", "vertex/gemini-2.5-pro").
+ *
+ * Legacy note: earlier versions stored a `preset` (codex|claude|gemini) and
+ * `commandPrefix` (CLI args to spawn a sub-agent process). That approach has
+ * been replaced by direct API calls via the Vercel AI SDK. The legacy CLI-agent
+ * code is preserved in snapshot-analyzer.ts but is not wired into the snapshot
+ * command.
+ */
+export const AiConfigSchema = z.object({
+  model: z.string().min(1),
+  updatedAt: z.string(),
+});
+export type AiConfig = z.infer<typeof AiConfigSchema>;
+
+export const ViewportConfigSchema = z.object({
+  width: z.number().int().min(1),
+  height: z.number().int().min(1),
+});
+export type ViewportConfig = z.infer<typeof ViewportConfigSchema>;
+
+export const LibrettoConfigSchema = z
+  .object({
+    version: z.literal(CURRENT_CONFIG_VERSION),
+    ai: AiConfigSchema.optional(),
+    viewport: ViewportConfigSchema.optional(),
+  })
+  .passthrough();
+export type LibrettoConfig = z.infer<typeof LibrettoConfigSchema>;
+
+/** Default models for each provider shorthand accepted by `ai configure`. */
+const DEFAULT_MODELS: Record<string, string> = {
+  openai: "openai/gpt-5.4",
+  anthropic: "anthropic/claude-sonnet-4-6",
+  gemini: "google/gemini-3-flash-preview",
+  vertex: "vertex/gemini-2.5-pro",
+};
+
+const PROVIDER_ALIASES: Record<string, string> = {
+  claude: DEFAULT_MODELS.anthropic,
+  google: DEFAULT_MODELS.gemini,
+};
+
+const CONFIGURE_PROVIDERS = [
+  "openai",
+  "anthropic",
+  "gemini",
+  "vertex",
+] as const;
+
+function formatConfigureProviders(separator = " | "): string {
+  return CONFIGURE_PROVIDERS.join(separator);
+}
+
+function formatConfigIssues(error: z.ZodError): string {
+  return error.issues
+    .map((issue) => `  - ${issue.path.join(".") || "root"}: ${issue.message}`)
+    .join("\n");
+}
+
+function formatExpectedConfigExample(): string {
+  return JSON.stringify(
+    {
+      version: CURRENT_CONFIG_VERSION,
+      ai: {
+        model: "openai/gpt-5.4",
+        updatedAt: "2026-01-01T00:00:00.000Z",
+      },
+      viewport: {
+        width: 1280,
+        height: 800,
+      },
+    },
+    null,
+    2,
+  );
+}
+
+function invalidConfigError(configPath: string, detail?: string): Error {
+  return new Error(
+    [
+      `AI config is invalid at ${configPath}.`,
+      detail ? `Problems:\n${detail}` : null,
+      "Expected config example:",
+      formatExpectedConfigExample(),
+      "Notes:",
+      '  - "ai" and "viewport" are optional.',
+      '  - "ai.model" must be a provider/model string like "openai/gpt-5.4" or "anthropic/claude-sonnet-4-6".',
+      "Fix the file to match this shape, or delete it and rerun:",
+      `  npx libretto ai configure ${formatConfigureProviders()}`,
+    ]
+      .filter(Boolean)
+      .join("\n"),
+  );
+}
+
+function parseConfig(raw: string, configPath: string): LibrettoConfig {
+  let parsedJson: unknown;
+  try {
+    parsedJson = JSON.parse(raw);
+  } catch (error) {
+    throw invalidConfigError(
+      configPath,
+      `  - root: Invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+
+  const parsed = LibrettoConfigSchema.safeParse(parsedJson);
+  if (!parsed.success) {
+    throw invalidConfigError(configPath, formatConfigIssues(parsed.error));
+  }
+  return parsed.data;
+}
+
+export function readLibrettoConfig(
+  configPath: string = LIBRETTO_CONFIG_PATH,
+): LibrettoConfig {
+  if (!existsSync(configPath)) {
+    return { version: CURRENT_CONFIG_VERSION };
+  }
+  return parseConfig(readFileSync(configPath, "utf-8"), configPath);
+}
+
+export function writeLibrettoConfig(
+  config: LibrettoConfig,
+  configPath: string = LIBRETTO_CONFIG_PATH,
+): LibrettoConfig {
+  const parsed = LibrettoConfigSchema.parse(config);
+  mkdirSync(dirname(configPath), { recursive: true });
+  writeFileSync(configPath, JSON.stringify(parsed, null, 2), "utf-8");
+  return parsed;
+}
+
+export function readAiConfig(
+  configPath: string = LIBRETTO_CONFIG_PATH,
+): AiConfig | null {
+  return readLibrettoConfig(configPath).ai ?? null;
+}
+
+export function writeAiConfig(
+  model: string,
+  configPath: string = LIBRETTO_CONFIG_PATH,
+): AiConfig {
+  const librettoConfig = readLibrettoConfig(configPath);
+  const ai = AiConfigSchema.parse({
+    model,
+    updatedAt: new Date().toISOString(),
+  });
+  writeLibrettoConfig(
+    {
+      ...librettoConfig,
+      version: CURRENT_CONFIG_VERSION,
+      ai,
+    },
+    configPath,
+  );
+  return ai;
+}
+
+export function clearAiConfig(
+  configPath: string = LIBRETTO_CONFIG_PATH,
+): boolean {
+  const librettoConfig = readLibrettoConfig(configPath);
+  if (!librettoConfig.ai) return false;
+  const { ai: _ai, ...rest } = librettoConfig;
+  writeLibrettoConfig(
+    {
+      ...rest,
+    },
+    configPath,
+  );
+  return true;
+}
+
+function printAiConfig(config: AiConfig, configPath: string): void {
+  console.log(`Model: ${config.model}`);
+  console.log(`Config file: ${configPath}`);
+  console.log(`Updated at: ${config.updatedAt}`);
+}
+
+/**
+ * Resolve the model string from a `ai configure` argument.
+ * Accepts a provider shorthand ("openai", "anthropic", "gemini", "vertex")
+ * or a full provider/model-id string ("openai/gpt-4o", "anthropic/claude-sonnet-4-6").
+ */
+function resolveModelFromInput(input: string): string | null {
+  const trimmed = input.trim();
+  if (!trimmed) return null;
+
+  // Full model string (contains a slash)
+  if (trimmed.includes("/")) return trimmed;
+
+  // Provider shorthand
+  const normalized = trimmed.toLowerCase();
+  return DEFAULT_MODELS[normalized] ?? PROVIDER_ALIASES[normalized] ?? null;
+}
+
+export function runAiConfigure(
+  input: {
+    preset?: string;
+    clear?: boolean;
+  },
+  options: {
+    configureCommandName?: string;
+    configPath?: string;
+  } = {},
+): void {
+  const configureCommandName =
+    options.configureCommandName ?? "npx libretto ai configure";
+  const configPath = options.configPath ?? LIBRETTO_CONFIG_PATH;
+
+  const presetArg = input.preset?.trim();
+
+  if (!presetArg && !input.clear) {
+    const config = readAiConfig(configPath);
+    if (!config) {
+      console.log(
+        `No AI config set. Choose a default model: ${configureCommandName} ${formatConfigureProviders()}`,
+      );
+      console.log(
+        "Provider credentials still come from your shell or .env file.",
+      );
+      return;
+    }
+    printAiConfig(config, configPath);
+    return;
+  }
+
+  if (input.clear) {
+    const removed = clearAiConfig(configPath);
+    if (removed) {
+      console.log(`Cleared AI config: ${configPath}`);
+    } else {
+      console.log("No AI config was set.");
+    }
+    return;
+  }
+
+  const model = resolveModelFromInput(presetArg!);
+  if (!model) {
+    console.log(
+      `Usage: ${configureCommandName} <${CONFIGURE_PROVIDERS.join("|")}|provider/model-id>\n` +
+        `       ${configureCommandName}\n` +
+        `       ${configureCommandName} --clear`,
+    );
+    throw new Error(
+      `Invalid provider or model. Use one of: ${formatConfigureProviders()}, or a full model string like "openai/gpt-4o".`,
+    );
+  }
+
+  const config = writeAiConfig(model, configPath);
+  console.log("AI config saved.");
+  printAiConfig(config, configPath);
+}

package/src/cli/core/api-snapshot-analyzer.ts

@@ -0,0 +1,108 @@
+/**
+ * API-based snapshot analyzer.
+ *
+ * Sends the DOM snapshot (condensed or full depending on sizing) and screenshot
+ * directly to a supported API provider via the Vercel AI SDK, without spawning
+ * a CLI process.
+ */
+
+import { readFileSync } from "node:fs";
+import type { LoggerApi } from "../../shared/logger/index.js";
+import { createLLMClient } from "../../shared/llm/client.js";
+import {
+  InterpretResultSchema,
+  buildInlinePromptSelection,
+  getMimeType,
+  readFileAsBase64,
+  type InterpretResult,
+  type InterpretArgs,
+} from "./snapshot-analyzer.js";
+import { readAiConfig, type AiConfig } from "./ai-config.js";
+import { resolveSnapshotApiModelOrThrow } from "./snapshot-api-config.js";
+
+export async function runApiInterpret(
+  args: InterpretArgs,
+  logger: LoggerApi,
+  configuredAi: AiConfig | null = readAiConfig(),
+): Promise<void> {
+  const selection = resolveSnapshotApiModelOrThrow(configuredAi);
+
+  logger.info("api-interpret-start", {
+    objective: args.objective,
+    pngPath: args.pngPath,
+    htmlPath: args.htmlPath,
+    condensedHtmlPath: args.condensedHtmlPath,
+    model: selection.model,
+    modelSource: selection.source,
+  });
+
+  const fullHtmlContent = readFileSync(args.htmlPath, "utf-8");
+  const condensedHtmlContent = readFileSync(args.condensedHtmlPath, "utf-8");
+
+  const promptSelection = buildInlinePromptSelection(
+    args,
+    fullHtmlContent,
+    condensedHtmlContent,
+    selection.model,
+  );
+
+  logger.info("api-interpret-dom-selection", {
+    configuredModel: promptSelection.stats.configuredModel,
+    fullDomEstimatedTokens: promptSelection.stats.fullDomEstimatedTokens,
+    condensedDomEstimatedTokens:
+      promptSelection.stats.condensedDomEstimatedTokens,
+    contextWindowTokens: promptSelection.budget.contextWindowTokens,
+    promptBudgetTokens: promptSelection.budget.promptBudgetTokens,
+    selectedDom: promptSelection.domSource,
+    selectedHtmlEstimatedTokens: promptSelection.htmlEstimatedTokens,
+    selectedPromptEstimatedTokens: promptSelection.promptEstimatedTokens,
+    selectionReason: promptSelection.selectionReason,
+    truncated: promptSelection.truncated,
+  });
+
+  const imageBase64 = readFileAsBase64(args.pngPath);
+  const imageMimeType = getMimeType(args.pngPath);
+  const imageBytes = Buffer.from(imageBase64, "base64");
+
+  const client = createLLMClient(selection.model);
+
+  const result = await client.generateObjectFromMessages({
+    schema: InterpretResultSchema,
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: promptSelection.prompt },
+          {
+            type: "image",
+            image: imageBytes,
+            mediaType: imageMimeType,
+          },
+        ],
+      },
+    ],
+    temperature: 0.1,
+  });
+
+  const parsed: InterpretResult = InterpretResultSchema.parse(result);
+
+  logger.info("api-interpret-success", {
+    selectorCount: parsed.selectors.length,
+    answer: parsed.answer.slice(0, 200),
+  });
+
+  console.log("");
+  console.log("Analysis:");
+  console.log(parsed.answer);
+  if (parsed.selectors.length > 0) {
+    console.log("");
+    console.log("Selectors:");
+    parsed.selectors.forEach((selector, index) => {
+      console.log(`  ${index + 1}. ${selector.label}: ${selector.selector}`);
+    });
+  }
+  if (parsed.notes?.trim()) {
+    console.log("");
+    console.log(`Notes: ${parsed.notes.trim()}`);
+  }
+}