libretto 0.6.12 → 0.6.14

package/scripts/summarize-evals.mjs

@@ -1,135 +0,0 @@
-#!/usr/bin/env node
-
-import { readdirSync, readFileSync, writeFileSync } from "node:fs";
-import { basename, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-
-function usage() {
-  console.error(
-    "Usage: node scripts/summarize-evals.mjs <score-dir> <summary-json-path>",
-  );
-}
-
-function normalizeFailureRecord(failure) {
-  return {
-    criterion: String(failure?.criterion ?? "").trim(),
-    reason: String(failure?.reason ?? "").trim(),
-  };
-}
-
-function normalizeRecord(record) {
-  const failures = Array.isArray(record?.failures)
-    ? record.failures
-        .map(normalizeFailureRecord)
-        .filter(
-          (failure) =>
-            failure.criterion.length > 0 && failure.reason.length > 0,
-        )
-    : [];
-
-  return {
-    name: String(record?.name ?? "").trim(),
-    passed: Number(record?.passed ?? 0),
-    total: Number(record?.total ?? 0),
-    percent: Number(record?.percent ?? 0),
-    failures,
-  };
-}
-
-export function loadScoreRecords(scoreDirArg) {
-  const scoreDir = resolve(scoreDirArg);
-  return readdirSync(scoreDir, { withFileTypes: true })
-    .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
-    .map((entry) =>
-      JSON.parse(readFileSync(join(scoreDir, entry.name), "utf8")),
-    )
-    .map(normalizeRecord)
-    .sort((a, b) => String(a.name).localeCompare(String(b.name)));
-}
-
-export function buildSummary(records) {
-  const passed = records.reduce(
-    (sum, record) => sum + Number(record.passed || 0),
-    0,
-  );
-  const total = records.reduce(
-    (sum, record) => sum + Number(record.total || 0),
-    0,
-  );
-  const percent = total > 0 ? Number(((passed / total) * 100).toFixed(2)) : 0;
-  const failingRecords = records.filter((record) => record.failures.length > 0);
-
-  return {
-    generatedAt: new Date().toISOString(),
-    recordCount: records.length,
-    passed,
-    total,
-    percent,
-    failingRecordCount: failingRecords.length,
-    records,
-  };
-}
-
-export function buildMarkdown(summary, summaryPathArg) {
-  const lines = [
-    "# Eval Summary",
-    "",
-    `- Overall score: \`${summary.percent}%\``,
-    `- Passed criteria: \`${summary.passed}/${summary.total}\``,
-    `- Recorded score entries: \`${summary.recordCount}\``,
-    `- Failed evals: \`${summary.failingRecordCount}\``,
-    `- Summary file: \`${basename(summaryPathArg)}\``,
-  ];
-
-  if (summary.records.length > 0) {
-    lines.push("", "## Breakdown", "");
-    for (const record of summary.records) {
-      const status = record.failures.length > 0 ? "fail" : "pass";
-      lines.push(
-        `- ${status} \`${record.name}\`: \`${record.percent}%\` (${record.passed}/${record.total})`,
-      );
-    }
-  }
-
-  if (summary.failingRecordCount > 0) {
-    lines.push("", "## Failed Evals", "");
-    for (const record of summary.records.filter(
-      (candidate) => candidate.failures.length > 0,
-    )) {
-      lines.push(`### \`${record.name}\``);
-      lines.push("");
-      lines.push(
-        `- Score: \`${record.percent}%\` (${record.passed}/${record.total})`,
-      );
-      for (const failure of record.failures) {
-        lines.push(`- ${failure.criterion}: ${failure.reason}`);
-      }
-      lines.push("");
-    }
-  }
-
-  return `${lines.join("\n").trimEnd()}\n`;
-}
-
-function main(argv) {
-  const [, , scoreDirArg, summaryPathArg] = argv;
-
-  if (!scoreDirArg || !summaryPathArg) {
-    usage();
-    process.exit(1);
-  }
-
-  const summaryPath = resolve(summaryPathArg);
-  const records = loadScoreRecords(scoreDirArg);
-  const summary = buildSummary(records);
-
-  writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
-  process.stdout.write(buildMarkdown(summary, summaryPath));
-}
-
-if (
-  process.argv[1] &&
-  resolve(process.argv[1]) === fileURLToPath(import.meta.url)
-) {
-  main(process.argv);
-}

package/src/cli/commands/ai.ts

@@ -1,144 +0,0 @@
-import { z } from "zod";
-import {
-  CURRENT_CONFIG_VERSION,
-  readSnapshotModel,
-  writeSnapshotModel,
-  clearSnapshotModel,
-} from "../core/config.js";
-import { LIBRETTO_CONFIG_PATH } from "../core/context.js";
-import { DEFAULT_SNAPSHOT_MODELS } from "../core/ai-model.js";
-import { librettoCommand } from "../../shared/package-manager.js";
-import { SimpleCLI } from "../framework/simple-cli.js";
-
-const PROVIDER_ALIASES: Record<string, string> = {
-  claude: DEFAULT_SNAPSHOT_MODELS.anthropic,
-  gemini: DEFAULT_SNAPSHOT_MODELS.google,
-  google: DEFAULT_SNAPSHOT_MODELS.google,
-};
-
-const CONFIGURE_PROVIDERS = [
-  "openai",
-  "anthropic",
-  "gemini",
-  "vertex",
-] as const;
-
-function formatConfigureProviders(separator = " | "): string {
-  return CONFIGURE_PROVIDERS.join(separator);
-}
-
-function printSnapshotModelConfig(model: string, configPath: string): void {
-  console.log(`Snapshot model: ${model}`);
-  console.log(`Config file: ${configPath}`);
-}
-
-/**
- * Resolve the model string from a `ai configure` argument.
- * Accepts a provider shorthand ("openai", "anthropic", "gemini", "vertex")
- * or a full provider/model-id string ("openai/gpt-4o", "anthropic/claude-sonnet-4-6").
- */
-function resolveModelFromInput(input: string): string | null {
-  const trimmed = input.trim();
-  if (!trimmed) return null;
-
-  // Full model string (contains a slash)
-  if (trimmed.includes("/")) return trimmed;
-
-  // Provider shorthand
-  const normalized = trimmed.toLowerCase();
-  return (
-    (DEFAULT_SNAPSHOT_MODELS as Record<string, string>)[normalized] ??
-    PROVIDER_ALIASES[normalized] ??
-    null
-  );
-}
-
-export function runAiConfigure(
-  input: {
-    preset?: string;
-    clear?: boolean;
-  },
-  options: {
-    configureCommandName?: string;
-    configPath?: string;
-  } = {},
-): void {
-  const configureCommandName =
-    options.configureCommandName ?? librettoCommand("ai configure");
-  const configPath = options.configPath ?? LIBRETTO_CONFIG_PATH;
-
-  const presetArg = input.preset?.trim();
-
-  if (!presetArg && !input.clear) {
-    const model = readSnapshotModel(configPath);
-    if (!model) {
-      console.log(
-        `No snapshot model set. Choose a default model: ${configureCommandName} ${formatConfigureProviders()}`,
-      );
-      console.log(
-        "Provider credentials still come from your shell or .env file.",
-      );
-      return;
-    }
-    printSnapshotModelConfig(model, configPath);
-    return;
-  }
-
-  if (input.clear) {
-    const removed = clearSnapshotModel(configPath);
-    if (removed) {
-      console.log(`Cleared snapshot model config: ${configPath}`);
-    } else {
-      console.log("No snapshot model was set.");
-    }
-    return;
-  }
-
-  const model = resolveModelFromInput(presetArg!);
-  if (!model) {
-    console.log(
-      `Usage: ${configureCommandName} <${CONFIGURE_PROVIDERS.join("|")}|provider/model-id>\n` +
-        `       ${configureCommandName}\n` +
-        `       ${configureCommandName} --clear`,
-    );
-    throw new Error(
-      `Invalid provider or model. Use one of: ${formatConfigureProviders()}, or a full model string like "openai/gpt-4o".`,
-    );
-  }
-
-  writeSnapshotModel(model, configPath);
-  console.log("Snapshot model saved.");
-  printSnapshotModelConfig(model, configPath);
-}
-
-export const aiConfigureInput = SimpleCLI.input({
-  positionals: [
-    SimpleCLI.positional("preset", z.string().optional(), {
-      help: "Provider shorthand or provider/model-id",
-    }),
-  ],
-  named: {
-    clear: SimpleCLI.flag({ help: "Clear existing AI config" }),
-  },
-});
-
-export const aiCommands = SimpleCLI.group({
-  description: "AI commands",
-  routes: {
-    configure: SimpleCLI.command({
-      description: "Configure AI runtime",
-    })
-      .input(aiConfigureInput)
-      .handle(async ({ input }) => {
-        runAiConfigure(
-          {
-            clear: input.clear,
-            preset: input.preset,
-          },
-          {
-            configureCommandName: librettoCommand("ai configure"),
-          },
-        );
-      }),
-  },
-});

package/src/cli/core/ai-model.ts

@@ -1,301 +0,0 @@
-import { readSnapshotModel } from "./config.js";
-import { LIBRETTO_CONFIG_PATH } from "./context.js";
-import { librettoCommand } from "../../shared/package-manager.js";
-import {
-  hasProviderCredentials,
-  parseModel,
-  type Provider,
-} from "./resolve-model.js";
-
-// Re-export so existing consumers (e.g. tests) don't break.
-export { parseDotEnvAssignment } from "../../shared/env/load-env.js";
-
-// ── Default models ──────────────────────────────────────────────────────────
-
-export const DEFAULT_SNAPSHOT_MODELS = {
-  openai: "openai/gpt-5.4",
-  anthropic: "anthropic/claude-sonnet-4-6",
-  google: "google/gemini-3-flash-preview",
-  vertex: "vertex/gemini-2.5-flash",
-  openrouter: "openrouter/free",
-} as const satisfies Record<Provider, string>;
-
-// ── Source detection ────────────────────────────────────────────────────────
-
-/**
- * Detect which specific env var provides credentials for a provider.
- * Returns the env var name (e.g. "OPENAI_API_KEY", "GEMINI_API_KEY"),
- * or null if no credential is found.
- */
-function detectProviderEnvVar(
-  provider: Provider,
-  env: NodeJS.ProcessEnv = process.env,
-): string | null {
-  switch (provider) {
-    case "openai":
-      return env.OPENAI_API_KEY?.trim() ? "OPENAI_API_KEY" : null;
-    case "anthropic":
-      return env.ANTHROPIC_API_KEY?.trim() ? "ANTHROPIC_API_KEY" : null;
-    case "google":
-      if (env.GEMINI_API_KEY?.trim()) return "GEMINI_API_KEY";
-      if (env.GOOGLE_GENERATIVE_AI_API_KEY?.trim())
-        return "GOOGLE_GENERATIVE_AI_API_KEY";
-      return null;
-    case "vertex":
-      if (env.GOOGLE_CLOUD_PROJECT?.trim()) return "GOOGLE_CLOUD_PROJECT";
-      if (env.GCLOUD_PROJECT?.trim()) return "GCLOUD_PROJECT";
-      return null;
-    case "openrouter":
-      return env.OPENROUTER_API_KEY?.trim() ? "OPENROUTER_API_KEY" : null;
-  }
-}
-
-// ── Snapshot model resolution ───────────────────────────────────────────────
-
-export type SnapshotApiModelSelection = {
-  model: string;
-  provider: Provider;
-  source: "config" | `env:${string}`;
-};
-
-export class SnapshotApiUnavailableError extends Error {
-  constructor(message: string) {
-    super(message);
-    this.name = "SnapshotApiUnavailableError";
-  }
-}
-
-function providerSetupSentence(provider: Provider): string {
-  switch (provider) {
-    case "openai":
-      return "Add OPENAI_API_KEY to .env or as a shell environment variable.";
-    case "anthropic":
-      return "Add ANTHROPIC_API_KEY to .env or as a shell environment variable.";
-    case "google":
-      return "Add GEMINI_API_KEY or GOOGLE_GENERATIVE_AI_API_KEY to .env or as a shell environment variable.";
-    case "vertex":
-      return "Add GOOGLE_CLOUD_PROJECT or GCLOUD_PROJECT to .env or as a shell environment variable, and make sure application default credentials are configured.";
-    case "openrouter":
-      return "Add OPENROUTER_API_KEY to .env or as a shell environment variable.";
-  }
-}
-
-function defaultModelCommandLine(): string {
-  return librettoCommand(
-    "ai configure openai | anthropic | gemini | vertex | openrouter",
-  );
-}
-
-function providerMissingCredentialSummary(provider: Provider): string {
-  switch (provider) {
-    case "openai":
-      return "OPENAI_API_KEY is missing";
-    case "anthropic":
-      return "ANTHROPIC_API_KEY is missing";
-    case "google":
-      return "GEMINI_API_KEY and GOOGLE_GENERATIVE_AI_API_KEY are missing";
-    case "vertex":
-      return "GOOGLE_CLOUD_PROJECT and GCLOUD_PROJECT are missing";
-    case "openrouter":
-      return "OPENROUTER_API_KEY is missing";
-  }
-}
-
-function noSnapshotApiConfiguredMessage(): string {
-  return [
-    "Failed to analyze snapshot because no snapshot analyzer is configured.",
-    `Add OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY or GOOGLE_GENERATIVE_AI_API_KEY, GOOGLE_CLOUD_PROJECT, or OPENROUTER_API_KEY to .env or as a shell environment variable, or choose a default model with \`${defaultModelCommandLine()}\`.`,
-    `For more info, run \`${librettoCommand("setup")}\`.`,
-  ].join(" ");
-}
-
-function missingProviderSnapshotMessage(
-  selection: SnapshotApiModelSelection,
-): string {
-  const configuredSource =
-    selection.source === "config"
-      ? ` in ${LIBRETTO_CONFIG_PATH}`
-      : " from process env or .env";
-  return [
-    `Failed to analyze snapshot because ${selection.provider} is configured${configuredSource}, but ${providerMissingCredentialSummary(selection.provider)}.`,
-    providerSetupSentence(selection.provider),
-    `For more info, run \`${librettoCommand("setup")}\`.`,
-  ].join(" ");
-}
-
-// ── Model resolution ────────────────────────────────────────────────────────
-
-function inferAutoSnapshotModel(): SnapshotApiModelSelection | null {
-  const providersInPriorityOrder: Provider[] = [
-    "openai",
-    "anthropic",
-    "google",
-    "vertex",
-    "openrouter",
-  ];
-
-  for (const provider of providersInPriorityOrder) {
-    const envVar = detectProviderEnvVar(provider);
-    if (!envVar) continue;
-    return {
-      model: DEFAULT_SNAPSHOT_MODELS[provider],
-      provider,
-      source: `env:${envVar}`,
-    };
-  }
-
-  return null;
-}
-
-/**
- * Resolve which API model to use for snapshot analysis.
- *
- * Environment variables are loaded by the CLI entrypoint (`runLibrettoCLI` in
- * `cli.ts`) before this resolver runs. Keep dotenv loading centralized there so
- * model resolution and browser provider resolution share the same env setup.
- *
- * Priority:
- * 1. snapshotModel from .libretto/config.json (set via `ai configure`)
- * 2. Auto-detect from available API credentials in env
- */
-export function resolveSnapshotApiModel(
-  snapshotModel: string | null = readSnapshotModel(),
-): SnapshotApiModelSelection | null {
-  if (snapshotModel) {
-    const { provider } = parseModel(snapshotModel);
-    return {
-      model: snapshotModel,
-      provider,
-      source: "config",
-    };
-  }
-
-  return inferAutoSnapshotModel();
-}
-
-export function resolveSnapshotApiModelOrThrow(
-  snapshotModel: string | null = readSnapshotModel(),
-): SnapshotApiModelSelection {
-  const selection = resolveSnapshotApiModel(snapshotModel);
-  if (!selection) {
-    throw new SnapshotApiUnavailableError(noSnapshotApiConfiguredMessage());
-  }
-
-  if (!hasProviderCredentials(selection.provider)) {
-    throw new SnapshotApiUnavailableError(
-      missingProviderSnapshotMessage(selection),
-    );
-  }
-
-  return selection;
-}
-
-export function isSnapshotApiUnavailableError(error: unknown): boolean {
-  return error instanceof SnapshotApiUnavailableError;
-}
-
-// ── AI setup status ─────────────────────────────────────────────────────────
-
-/**
- * Workspace AI setup health states.
- *
- * - `ready`: a usable model was resolved and the matching provider has credentials.
- * - `configured-missing-credentials`: config pins a provider whose credentials are absent.
- * - `invalid-config`: `.libretto/config.json` exists but fails schema validation.
- * - `unconfigured`: no config and no env credentials detected.
- */
-export type AiSetupStatus =
-  | {
-      kind: "ready";
-      model: string;
-      provider: Provider;
-      source: "config" | `env:${string}`;
-    }
-  | {
-      kind: "configured-missing-credentials";
-      model: string;
-      provider: Provider;
-    }
-  | { kind: "invalid-config"; message: string }
-  | { kind: "unconfigured" };
-
-/**
- * Read snapshot model without throwing on invalid files.
- * Returns the model string or an error message.
- */
-function readSnapshotModelSafely(
-  configPath: string,
-): { ok: true; model: string | null } | { ok: false; message: string } {
-  try {
-    return { ok: true, model: readSnapshotModel(configPath) };
-  } catch (err) {
-    return {
-      ok: false,
-      message: err instanceof Error ? err.message : String(err),
-    };
-  }
-}
-
-/**
- * Resolve the workspace's current AI setup health.
- *
- * Uses the existing config reader and snapshot model resolver, but wraps
- * them to distinguish broken states (invalid config, missing credentials)
- * that the throwing APIs collapse into errors.
- *
- * 1. If config read throws → `invalid-config`.
- * 2. If config has a `snapshotModel` → check credentials for that provider.
- * 3. If no `snapshotModel` → auto-detect from env via existing resolver.
- */
-export function resolveAiSetupStatus(
-  configPath: string = LIBRETTO_CONFIG_PATH,
-): AiSetupStatus {
-  const result = readSnapshotModelSafely(configPath);
-
-  if (!result.ok) {
-    return { kind: "invalid-config", message: result.message };
-  }
-
-  // Config has a snapshotModel — use it directly to check credentials
-  if (result.model) {
-    let selection: SnapshotApiModelSelection | null;
-    try {
-      selection = resolveSnapshotApiModel(result.model);
-    } catch (err) {
-      return {
-        kind: "invalid-config",
-        message: err instanceof Error ? err.message : String(err),
-      };
-    }
-    if (!selection) {
-      // Should not happen when config has a model, but handle gracefully
-      return { kind: "unconfigured" };
-    }
-    if (hasProviderCredentials(selection.provider)) {
-      return {
-        kind: "ready",
-        model: selection.model,
-        provider: selection.provider,
-        source: selection.source,
-      };
-    }
-    return {
-      kind: "configured-missing-credentials",
-      model: selection.model,
-      provider: selection.provider,
-    };
-  }
-
-  // No snapshotModel — fall back to env auto-detect via existing resolver
-  const envSelection = resolveSnapshotApiModel(null);
-  if (envSelection && hasProviderCredentials(envSelection.provider)) {
-    return {
-      kind: "ready",
-      model: envSelection.model,
-      provider: envSelection.provider,
-      source: envSelection.source,
-    };
-  }
-
-  return { kind: "unconfigured" };
-}

package/src/cli/core/api-snapshot-analyzer.ts

@@ -1,110 +0,0 @@
-/**
- * API-based snapshot analyzer.
- *
- * Sends the DOM snapshot (condensed or full depending on sizing) and screenshot
- * directly to a supported API provider via the Vercel AI SDK, without spawning
- * a CLI process.
- */
-
-import { readFileSync } from "node:fs";
-import type { LoggerApi } from "../../shared/logger/index.js";
-import { generateObject } from "ai";
-import { resolveModel } from "./resolve-model.js";
-import {
-  InterpretResultSchema,
-  buildInlinePromptSelection,
-  getMimeType,
-  readFileAsBase64,
-  type InterpretResult,
-  type InterpretArgs,
-} from "./snapshot-analyzer.js";
-import { readSnapshotModel } from "./config.js";
-import { resolveSnapshotApiModelOrThrow } from "./ai-model.js";
-
-export async function runApiInterpret(
-  args: InterpretArgs,
-  logger: LoggerApi,
-  snapshotModel: string | null = readSnapshotModel(),
-): Promise<void> {
-  const selection = resolveSnapshotApiModelOrThrow(snapshotModel);
-
-  logger.info("api-interpret-start", {
-    objective: args.objective,
-    pngPath: args.pngPath,
-    htmlPath: args.htmlPath,
-    condensedHtmlPath: args.condensedHtmlPath,
-    model: selection.model,
-    modelSource: selection.source,
-  });
-
-  const fullHtmlContent = readFileSync(args.htmlPath, "utf-8");
-  const condensedHtmlContent = readFileSync(args.condensedHtmlPath, "utf-8");
-
-  const promptSelection = buildInlinePromptSelection(
-    args,
-    fullHtmlContent,
-    condensedHtmlContent,
-    selection.model,
-  );
-
-  logger.info("api-interpret-dom-selection", {
-    configuredModel: promptSelection.stats.configuredModel,
-    fullDomEstimatedTokens: promptSelection.stats.fullDomEstimatedTokens,
-    condensedDomEstimatedTokens:
-      promptSelection.stats.condensedDomEstimatedTokens,
-    contextWindowTokens: promptSelection.budget.contextWindowTokens,
-    promptBudgetTokens: promptSelection.budget.promptBudgetTokens,
-    selectedDom: promptSelection.domSource,
-    selectedHtmlEstimatedTokens: promptSelection.htmlEstimatedTokens,
-    selectedPromptEstimatedTokens: promptSelection.promptEstimatedTokens,
-    selectionReason: promptSelection.selectionReason,
-    truncated: promptSelection.truncated,
-  });
-
-  const imageBase64 = readFileAsBase64(args.pngPath);
-  const imageMimeType = getMimeType(args.pngPath);
-  const imageBytes = Buffer.from(imageBase64, "base64");
-
-  const model = await resolveModel(selection.model);
-
-  const { object: result } = await generateObject({
-    model,
-    schema: InterpretResultSchema,
-    messages: [
-      {
-        role: "user",
-        content: [
-          { type: "text", text: promptSelection.prompt },
-          {
-            type: "image",
-            image: imageBytes,
-            mediaType: imageMimeType,
-          },
-        ],
-      },
-    ],
-    temperature: 0.1,
-  });
-
-  const parsed: InterpretResult = InterpretResultSchema.parse(result);
-
-  logger.info("api-interpret-success", {
-    selectorCount: parsed.selectors.length,
-    answer: parsed.answer.slice(0, 200),
-  });
-
-  console.log("");
-  console.log("Analysis:");
-  console.log(parsed.answer);
-  if (parsed.selectors.length > 0) {
-    console.log("");
-    console.log("Selectors:");
-    parsed.selectors.forEach((selector, index) => {
-      console.log(`  ${index + 1}. ${selector.label}: ${selector.selector}`);
-    });
-  }
-  if (parsed.notes?.trim()) {
-    console.log("");
-    console.log(`Notes: ${parsed.notes.trim()}`);
-  }
-}