libretto 0.1.5 → 0.2.0

package/dist/cli/core/snapshot-analyzer.js

@@ -0,0 +1,492 @@
+import {
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync
+} from "node:fs";
+import { extname, isAbsolute, join, resolve } from "node:path";
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { z } from "zod";
+import {
+  formatCommandPrefix,
+  readAiConfig
+} from "./ai-config.js";
+import {
+  getLLMClientFactory
+} from "./context.js";
+const InterpretResultSchema = z.object({
+  answer: z.string(),
+  selectors: z.array(
+    z.object({
+      label: z.string(),
+      selector: z.string(),
+      rationale: z.string()
+    })
+  ).default([]),
+  notes: z.string().optional().default("")
+});
+class UserCodingAgent {
+  constructor(config) {
+    this.config = config;
+  }
+  static resolveFromConfig(config) {
+    switch (config.preset) {
+      case "codex":
+        return new CodexUserCodingAgent(config);
+      case "claude":
+        return new ClaudeUserCodingAgent(config);
+      case "gemini":
+        return new GeminiUserCodingAgent(config);
+    }
+  }
+  static readConfiguredConfig() {
+    return readAiConfig();
+  }
+  static getConfigured() {
+    const config = this.readConfiguredConfig();
+    return config ? this.resolveFromConfig(config) : null;
+  }
+  get snapshotAnalyzerConfig() {
+    return this.config;
+  }
+  get command() {
+    const command = this.config.commandPrefix[0];
+    if (!command) {
+      throw new Error("AI config is invalid: command prefix is empty.");
+    }
+    return command;
+  }
+  get baseArgs() {
+    return this.config.commandPrefix.slice(1);
+  }
+  screenshotHint(pngPath) {
+    return `
+
+Screenshot file path: ${pngPath}
+Use the screenshot alongside the HTML snapshot context above.`;
+  }
+  async runAnalyzer(args, stdinText) {
+    const result = await runExternalCommand(this.command, args, stdinText);
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `Analyzer command failed (${formatCommandPrefix([this.command, ...args])}).
+${stripAnsi(result.stderr).trim() || stripAnsi(result.stdout).trim() || "No error output."}`
+      );
+    }
+    return result;
+  }
+  async runAndParse(args, stdinText) {
+    const result = await this.runAnalyzer(args, stdinText);
+    return parseInterpretResultFromText(result.stdout);
+  }
+}
+class CodexUserCodingAgent extends UserCodingAgent {
+  async analyzeSnapshot(prompt, pngPath) {
+    const tempDir = mkdtempSync(join(tmpdir(), "libretto-cli-analyzer-"));
+    const outputPath = join(
+      tempDir,
+      `snapshot-analyzer-${Date.now()}-${Math.random().toString(36).slice(2)}.json`
+    );
+    const args = [
+      ...this.baseArgs,
+      "--output-last-message",
+      outputPath,
+      "-i",
+      pngPath,
+      "-"
+    ];
+    const result = await this.runAnalyzer(args, prompt);
+    let outputText = result.stdout;
+    try {
+      if (existsSync(outputPath)) {
+        outputText = readFileSync(outputPath, "utf-8");
+      }
+      return parseInterpretResultFromText(outputText);
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+  }
+}
+class ClaudeUserCodingAgent extends UserCodingAgent {
+  async analyzeSnapshot(prompt, pngPath) {
+    const args = [...this.baseArgs, `${prompt}${this.screenshotHint(pngPath)}`];
+    return await this.runAndParse(args);
+  }
+}
+class GeminiUserCodingAgent extends UserCodingAgent {
+  async analyzeSnapshot(prompt, pngPath) {
+    const args = [...this.baseArgs, `${prompt}${this.screenshotHint(pngPath)}`];
+    return await this.runAndParse(args);
+  }
+}
+async function runExternalCommand(command, args, stdinText) {
+  return await new Promise((resolve2, reject) => {
+    const child = spawn(command, args, {
+      stdio: ["pipe", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on("error", (err) => {
+      const error = err;
+      if (error.code === "ENOENT") {
+        reject(
+          new Error(
+            `Command not found: ${command}. Configure AI with 'libretto-cli ai configure'.`
+          )
+        );
+        return;
+      }
+      reject(err);
+    });
+    child.on("close", (code) => {
+      resolve2({
+        exitCode: code ?? 1,
+        stdout,
+        stderr
+      });
+    });
+    if (stdinText !== void 0) {
+      child.stdin.write(stdinText);
+    }
+    child.stdin.end();
+  });
+}
+function stripAnsi(value) {
+  return value.replace(
+    /\u001b\[[0-9;]*[A-Za-z]|\u001b\][^\u0007]*(?:\u0007|\u001b\\)/g,
+    ""
+  );
+}
+function extractJsonObjectCandidates(text) {
+  const candidates = [];
+  const seen = /* @__PURE__ */ new Set();
+  const add = (value) => {
+    const trimmed = value.trim();
+    if (!trimmed || seen.has(trimmed)) return;
+    seen.add(trimmed);
+    candidates.push(trimmed);
+  };
+  try {
+    const direct = text.trim();
+    if (direct.startsWith("{") && direct.endsWith("}")) {
+      add(direct);
+    }
+  } catch {
+  }
+  const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)```/gi;
+  let codeBlockMatch;
+  while ((codeBlockMatch = codeBlockRegex.exec(text)) !== null) {
+    const body = codeBlockMatch[1]?.trim();
+    if (body && body.startsWith("{") && body.endsWith("}")) {
+      add(body);
+    }
+  }
+  const lines = text.split("\n");
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
+      add(trimmed);
+    }
+  }
+  let depth = 0;
+  let start = -1;
+  let inString = false;
+  let escaped = false;
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+        continue;
+      }
+      if (char === "\\") {
+        escaped = true;
+        continue;
+      }
+      if (char === '"') {
+        inString = false;
+      }
+      continue;
+    }
+    if (char === '"') {
+      inString = true;
+      continue;
+    }
+    if (char === "{") {
+      if (depth === 0) start = i;
+      depth += 1;
+      continue;
+    }
+    if (char === "}") {
+      if (depth > 0) depth -= 1;
+      if (depth === 0 && start >= 0) {
+        add(text.slice(start, i + 1));
+        start = -1;
+      }
+    }
+  }
+  return candidates;
+}
+function collectStringLeaves(value, out, depth = 0) {
+  if (depth > 6 || value == null) return;
+  if (typeof value === "string") {
+    out.push(value);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      collectStringLeaves(item, out, depth + 1);
+    }
+    return;
+  }
+  if (typeof value === "object") {
+    for (const nested of Object.values(value)) {
+      collectStringLeaves(nested, out, depth + 1);
+    }
+  }
+}
+function parseInterpretResultFromText(text) {
+  const cleaned = stripAnsi(text).trim();
+  const candidates = extractJsonObjectCandidates(cleaned);
+  if (candidates.length === 0) {
+    throw new Error(
+      "Analyzer output did not include a JSON object matching the interpret schema."
+    );
+  }
+  for (const candidate of candidates) {
+    try {
+      const parsed = JSON.parse(candidate);
+      const valid = InterpretResultSchema.safeParse(parsed);
+      if (valid.success) {
+        return valid.data;
+      }
+      const nestedStrings = [];
+      collectStringLeaves(parsed, nestedStrings);
+      for (const nestedText of nestedStrings) {
+        const nestedCandidates = extractJsonObjectCandidates(nestedText);
+        for (const nestedCandidate of nestedCandidates) {
+          try {
+            const nestedParsed = JSON.parse(nestedCandidate);
+            const nestedValid = InterpretResultSchema.safeParse(nestedParsed);
+            if (nestedValid.success) {
+              return nestedValid.data;
+            }
+          } catch {
+          }
+        }
+      }
+    } catch {
+    }
+  }
+  throw new Error(
+    "Analyzer output could not be parsed as valid interpret JSON. Ensure the configured command returns only the requested JSON object."
+  );
+}
+function resolvePath(filePath) {
+  return isAbsolute(filePath) ? filePath : resolve(process.cwd(), filePath);
+}
+function getMimeType(filePath) {
+  const ext = extname(filePath).toLowerCase();
+  if (ext === ".png") return "image/png";
+  if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg";
+  if (ext === ".webp") return "image/webp";
+  if (ext === ".gif") return "image/gif";
+  return "application/octet-stream";
+}
+function readFileAsBase64(filePath) {
+  return readFileSync(filePath).toString("base64");
+}
+function truncateText(text, maxChars) {
+  if (text.length <= maxChars) {
+    return { text, truncated: false };
+  }
+  const head = text.slice(0, Math.floor(maxChars * 0.6));
+  const tail = text.slice(-Math.floor(maxChars * 0.4));
+  return {
+    text: `${head}
+
+... [truncated] ...
+
+${tail}`,
+    truncated: true
+  };
+}
+function collectSelectorHints(html, limit = 120) {
+  const candidates = [];
+  const seen = /* @__PURE__ */ new Set();
+  const add = (value) => {
+    if (candidates.length >= limit || seen.has(value)) return;
+    seen.add(value);
+    candidates.push(value);
+  };
+  const selectors = [
+    { attr: "data-testid", format: (value) => `[data-testid="${value}"]` },
+    { attr: "data-test", format: (value) => `[data-test="${value}"]` },
+    { attr: "data-qa", format: (value) => `[data-qa="${value}"]` },
+    { attr: "aria-label", format: (value) => `[aria-label="${value}"]` },
+    { attr: "role", format: (value) => `[role="${value}"]` },
+    { attr: "name", format: (value) => `[name="${value}"]` },
+    { attr: "placeholder", format: (value) => `[placeholder="${value}"]` },
+    { attr: "id", format: (value) => `#${value}` }
+  ];
+  for (const selector of selectors) {
+    const regex = new RegExp(`${selector.attr}\\s*=\\s*["']([^"']+)["']`, "gi");
+    let match;
+    while ((match = regex.exec(html)) !== null) {
+      const value = match[1]?.trim();
+      if (!value) continue;
+      add(selector.format(value));
+      if (candidates.length >= limit) break;
+    }
+    if (candidates.length >= limit) break;
+  }
+  return candidates;
+}
+async function runInterpret(args, logger) {
+  logger.info("interpret-start", {
+    objective: args.objective,
+    pngPath: args.pngPath,
+    htmlPath: args.htmlPath
+  });
+  process.env.NODE_ENV = "development";
+  const pngPath = resolvePath(args.pngPath);
+  const htmlPath = resolvePath(args.htmlPath);
+  if (!existsSync(pngPath)) {
+    throw new Error(`PNG file not found: ${pngPath}`);
+  }
+  if (!existsSync(htmlPath)) {
+    throw new Error(`HTML file not found: ${htmlPath}`);
+  }
+  const htmlContent = readFileSync(htmlPath, "utf-8");
+  const htmlCharLimit = 5e5;
+  const { text: trimmedHtml, truncated } = truncateText(
+    htmlContent,
+    htmlCharLimit
+  );
+  const selectorHints = collectSelectorHints(htmlContent, 120);
+  let prompt = `# Objective
+${args.objective}
+
+`;
+  prompt += `# Context
+${args.context}
+
+`;
+  prompt += `# Instructions
+`;
+  prompt += `You are analyzing a screenshot and HTML snapshot of the same web page on behalf of an automation agent.
+`;
+  prompt += `The agent needs to interact with this page programmatically using Playwright.
+
+`;
+  prompt += `Based on the objective and context above:
+`;
+  prompt += `1. Answer the objective concisely
+`;
+  prompt += `2. Identify ALL interactive elements relevant to the objective and provide Playwright-ready CSS selectors
+`;
+  prompt += `3. Note any relevant page state (loading indicators, error messages, disabled elements, modals/overlays)
+`;
+  prompt += `4. If elements are inside iframes, identify the iframe selector and the element selector within it
+
+`;
+  prompt += `Output JSON with this shape:
+`;
+  prompt += `{"answer": string, "selectors": [{"label": string, "selector": string, "rationale": string}], "notes": string}
+
+`;
+  prompt += `Selectors should prefer robust attributes: data-testid, data-test, aria-label, name, id, role. Avoid fragile class-based or positional selectors.
+`;
+  prompt += `Only include selectors that exist in the HTML snapshot.
+
+`;
+  if (selectorHints.length > 0) {
+    prompt += `Selector hints from HTML attributes (use if relevant):
+`;
+    prompt += selectorHints.map((hint) => `- ${hint}`).join("\n");
+    prompt += "\n\n";
+  }
+  if (truncated) {
+    prompt += `HTML content is truncated to fit token limits.
+
+`;
+  }
+  prompt += `HTML snapshot:
+
+${trimmedHtml}`;
+  prompt += "\n\nReturn only a JSON object. Do not include markdown code fences or extra commentary.";
+  let parsed;
+  const configuredAgent = UserCodingAgent.getConfigured();
+  if (configuredAgent) {
+    const configuredAnalyzer = configuredAgent.snapshotAnalyzerConfig;
+    logger.info("interpret-analyzer-config", {
+      preset: configuredAnalyzer.preset,
+      commandPrefix: configuredAnalyzer.commandPrefix
+    });
+    parsed = await configuredAgent.analyzeSnapshot(prompt, pngPath);
+  } else {
+    const llmClientFactory = getLLMClientFactory();
+    if (!llmClientFactory) {
+      throw new Error(
+        "No AI config set. Run 'libretto-cli ai configure codex' (or claude/gemini). Library integrations can still set a factory via setLLMClientFactory()."
+      );
+    }
+    logger.info("interpret-analyzer-factory-fallback", {});
+    const imageBase64 = readFileAsBase64(pngPath);
+    const client = await llmClientFactory(logger, "google/gemini-3-flash-preview");
+    const result = await client.generateObjectFromMessages({
+      schema: InterpretResultSchema,
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: prompt },
+            {
+              type: "image",
+              image: `data:${getMimeType(pngPath)};base64,${imageBase64}`
+            }
+          ]
+        }
+      ],
+      temperature: 0.1
+    });
+    parsed = InterpretResultSchema.parse(result);
+  }
+  logger.info("interpret-success", {
+    selectorCount: parsed.selectors.length,
+    answer: parsed.answer.slice(0, 200)
+  });
+  const outputLines = [];
+  outputLines.push("Interpretation:");
+  outputLines.push(`Answer: ${parsed.answer}`);
+  outputLines.push("");
+  if (parsed.selectors.length === 0) {
+    outputLines.push("Selectors: none found.");
+  } else {
+    outputLines.push("Selectors:");
+    parsed.selectors.forEach((selector, index) => {
+      outputLines.push(`  ${index + 1}. ${selector.label}`);
+      outputLines.push(`     selector: ${selector.selector}`);
+      outputLines.push(`     rationale: ${selector.rationale}`);
+    });
+  }
+  if (parsed.notes.trim()) {
+    outputLines.push("");
+    outputLines.push(`Notes: ${parsed.notes.trim()}`);
+  }
+  console.log(outputLines.join("\n"));
+}
+function canAnalyzeSnapshots() {
+  return UserCodingAgent.getConfigured() !== null || getLLMClientFactory() !== null;
+}
+export {
+  canAnalyzeSnapshots,
+  runInterpret
+};