@sanity/ailf 0.1.0

package/dist/job-store.js

@@ -0,0 +1,188 @@
+/**
+ * job-store.ts
+ *
+ * Job tracking store backed by the Sanity Content Lake.
+ * Provides CRUD operations for `ailf.job` documents that track
+ * API-triggered pipeline evaluations.
+ *
+ * Job documents are created by the API gateway and updated by the
+ * CLI as the pipeline progresses through steps.
+ *
+ * @see docs/design-docs/api-service-gateway.md
+ * @see packages/studio/src/schema/job.ts — Sanity document schema
+ */
+import { getSanityClient } from "./sanity/client.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const JOB_TYPE = "ailf.job";
+/** Default job deadline: 45 minutes from creation */
+const DEFAULT_DEADLINE_MS = 45 * 60 * 1000;
+// ---------------------------------------------------------------------------
+// Job Store
+// ---------------------------------------------------------------------------
+export class JobStore {
+    client;
+    constructor(options = {}) {
+        if (options.client) {
+            this.client = options.client;
+        }
+        else {
+            this.client = getSanityClient({
+                ...(options.dataset ? { dataset: options.dataset } : {}),
+                ...(options.projectId ? { projectId: options.projectId } : {}),
+                ...(options.token ? { token: options.token } : {}),
+            });
+        }
+    }
+    /**
+     * Create a new job document in the Content Lake.
+     *
+     * Generates a job ID (e.g. "job_a1b2c3d4e5f6") and writes the
+     * initial document with status "received". The job ID is used
+     * directly as the Sanity document `_id`.
+     *
+     * @returns The generated job ID, or null on failure
+     */
+    async createJob(options) {
+        const jobId = generateJobId();
+        const now = new Date().toISOString();
+        const deadlineMs = options.deadlineMs ?? DEFAULT_DEADLINE_MS;
+        const deadlineAt = new Date(Date.now() + deadlineMs).toISOString();
+        try {
+            await this.client.create({
+                _id: jobId,
+                _type: JOB_TYPE,
+                apiKeyPrefix: options.apiKeyPrefix,
+                callback: options.callback ?? null,
+                completedAt: null,
+                config: options.config,
+                createdAt: now,
+                deadlineAt,
+                error: null,
+                execution: null,
+                jobId,
+                progress: null,
+                reportId: null,
+                startedAt: null,
+                status: "received",
+            });
+            return jobId;
+        }
+        catch (error) {
+            console.warn(`  ⚠️  Failed to create job: ${error instanceof Error ? error.message : String(error)}`);
+            return null;
+        }
+    }
+    /**
+     * Count jobs created today for a given API key prefix.
+     * Used for daily eval budget enforcement.
+     */
+    async countTodayJobs(apiKeyPrefix) {
+        try {
+            const todayStart = new Date();
+            todayStart.setUTCHours(0, 0, 0, 0);
+            const count = await this.client.fetch(`count(*[_type == $type && apiKeyPrefix == $prefix && createdAt >= $start])`, {
+                prefix: apiKeyPrefix,
+                start: todayStart.toISOString(),
+                type: JOB_TYPE,
+            });
+            return count;
+        }
+        catch {
+            return 0;
+        }
+    }
+    /**
+     * Count active (non-terminal) jobs for a given API key prefix.
+     * Used for concurrent job limit enforcement.
+     */
+    async countActiveJobs(apiKeyPrefix) {
+        try {
+            const count = await this.client.fetch(`count(*[_type == $type && apiKeyPrefix == $prefix && status in ["received", "queued", "running"]])`, {
+                prefix: apiKeyPrefix,
+                type: JOB_TYPE,
+            });
+            return count;
+        }
+        catch {
+            return 0;
+        }
+    }
+    /**
+     * Find stale jobs that have exceeded their deadline.
+     * Used by the timeout cron job.
+     */
+    async findStaleJobs() {
+        try {
+            const now = new Date().toISOString();
+            const docs = await this.client.fetch(`*[_type == $type && status in ["received", "queued", "running"] && deadlineAt < $now]`, { now, type: JOB_TYPE });
+            return docs.map(toJobDocument);
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Read a job by its job ID.
+     */
+    async readJob(jobId) {
+        try {
+            const doc = await this.client.fetch(`*[_type == $type && jobId == $id][0]`, { id: jobId, type: JOB_TYPE });
+            return doc ? toJobDocument(doc) : null;
+        }
+        catch (error) {
+            console.warn(`  ⚠️  Failed to read job: ${error instanceof Error ? error.message : String(error)}`);
+            return null;
+        }
+    }
+    /**
+     * Update a job's status and optional associated data.
+     */
+    async updateJob(jobId, update) {
+        try {
+            await this.client.patch(jobId).set(update).commit();
+            return true;
+        }
+        catch (error) {
+            console.warn(`  ⚠️  Failed to update job ${jobId}: ${error instanceof Error ? error.message : String(error)}`);
+            return false;
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Generate a job ID with "job_" prefix and random suffix.
+ *
+ * Uses crypto.randomUUID() for uniqueness, shortened for readability.
+ * The returned value is used directly as the Sanity document `_id`
+ * (no additional prefix wrapping).
+ *
+ * Example: "job_a1b2c3d4e5f6"
+ */
+function generateJobId() {
+    const uuid = crypto.randomUUID().replace(/-/g, "");
+    return `job_${uuid.slice(0, 12)}`;
+}
+/**
+ * Convert a raw Sanity document to a typed JobDocument.
+ */
+function toJobDocument(doc) {
+    return {
+        apiKeyPrefix: doc.apiKeyPrefix,
+        callback: doc.callback,
+        completedAt: doc.completedAt,
+        config: doc.config,
+        createdAt: doc.createdAt,
+        deadlineAt: doc.deadlineAt,
+        error: doc.error,
+        execution: doc.execution,
+        jobId: doc.jobId,
+        progress: doc.progress,
+        reportId: doc.reportId,
+        startedAt: doc.startedAt,
+        status: doc.status,
+    };
+}

package/dist/lib/agent-behavior-report.d.ts

@@ -0,0 +1,8 @@
+/**
+ * lib/agent-behavior-report.ts — DEPRECATED re-export shim.
+ * @deprecated Import from ../pipeline/agent-behavior-report.js instead.
+ */
+import "dotenv/config";
+export { analyzeResults, CANONICAL_DOC_MAP, detectFeatureArea, } from "../pipeline/agent-behavior-report.js";
+export type { AnalysisResult, FeatureAnalysis, TaskBehavior, TestResult, } from "../pipeline/agent-behavior-report.js";
+export declare function main(resultsPathArg?: string): void;

package/dist/lib/agent-behavior-report.js

@@ -0,0 +1,185 @@
+/**
+ * lib/agent-behavior-report.ts — DEPRECATED re-export shim.
+ * @deprecated Import from ../pipeline/agent-behavior-report.js instead.
+ */
+// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
+import "dotenv/config";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { dirname, join } from "path";
+export { analyzeResults, CANONICAL_DOC_MAP, detectFeatureArea, } from "../pipeline/agent-behavior-report.js";
+import { analyzeResults, } from "../pipeline/agent-behavior-report.js";
+export function main(resultsPathArg) {
+    const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
+    const resultsPath = resultsPathArg ??
+        process.argv[2] ??
+        join(ROOT, "results", "latest", "eval-results.json");
+    if (!existsSync(resultsPath)) {
+        console.error(`Results file not found: ${resultsPath}`);
+        console.error("Run an evaluation first: pnpm eval:observed");
+        process.exit(1);
+    }
+    console.log(`Reading results from: ${resultsPath}`);
+    console.log();
+    const json = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    const rawResults = Array.isArray(json.results)
+        ? json.results
+        : json.results.results;
+    const analysis = analyzeResults(rawResults);
+    if (!analysis.hasData) {
+        console.log("No agent behavior data found in the results.");
+        console.log("Make sure you ran the evaluation with the observed config:");
+        console.log("  pnpm eval:observed");
+        process.exit(0);
+    }
+    printReport(analysis);
+    // Persist detailed report as JSON
+    const outDir = join(ROOT, "results", "latest");
+    mkdirSync(outDir, { recursive: true });
+    const reportData = {
+        features: analysis.features.map((f) => ({
+            avgDocPages: f.avgDocPages,
+            avgNetworkMs: f.avgNetworkMs,
+            avgSearches: f.avgSearches,
+            canonicalCoverage: f.canonicalCoverage,
+            canonicalSlugs: f.canonicalSlugs,
+            docSlugsVisited: f.allDocSlugs,
+            externalDomains: f.allExternalDomains,
+            feature: f.feature,
+            searchQueries: f.allSearchQueries,
+            taskCount: f.tasks.length,
+        })),
+        tasks: analysis.tasks.map((t) => ({
+            behavior: t.behavior,
+            description: t.description,
+            feature: t.feature,
+            hasDocs: t.hasDocs,
+        })),
+        timestamp: new Date().toISOString(),
+        totalTasks: analysis.tasks.length,
+    };
+    writeFileSync(join(outDir, "agent-behavior-report.json"), JSON.stringify(reportData, null, 2));
+    console.log("Agent behavior report written to results/latest/agent-behavior-report.json");
+}
+// ---------------------------------------------------------------------------
+// Report output (kept in shim for backward compat)
+// ---------------------------------------------------------------------------
+function printReport(analysis) {
+    console.log("=".repeat(80));
+    console.log("            AGENT BEHAVIOR OBSERVATION REPORT");
+    console.log("=".repeat(80));
+    console.log();
+    // ---- Overview table ----
+    console.log("OVERVIEW BY FEATURE AREA");
+    console.log("-".repeat(80));
+    const h = "| Feature Area        | Tasks | Avg Docs | Avg Search | Avg Net(ms) | Canon% |";
+    const sep = "|---------------------|-------|----------|------------|-------------|--------|";
+    console.log(h);
+    console.log(sep);
+    for (const f of analysis.features) {
+        console.log(`| ${f.feature.padEnd(19)} | ` +
+            `${f.tasks.length.toString().padStart(5)} | ` +
+            `${f.avgDocPages.toFixed(1).padStart(8)} | ` +
+            `${f.avgSearches.toFixed(1).padStart(10)} | ` +
+            `${Math.round(f.avgNetworkMs).toString().padStart(11)} | ` +
+            `${(f.canonicalCoverage * 100).toFixed(0).padStart(5)}% |`);
+    }
+    console.log();
+    // ---- Canonical coverage breakdown ----
+    console.log("CANONICAL DOCUMENTATION COVERAGE");
+    console.log("-".repeat(80));
+    console.log();
+    for (const f of analysis.features) {
+        console.log(`  ${f.feature} (${(f.canonicalCoverage * 100).toFixed(0)}% canonical coverage):`);
+        if (f.canonicalSlugs.length === 0) {
+            console.log("    (no canonical docs defined)");
+        }
+        else {
+            for (const slug of f.canonicalSlugs) {
+                const found = f.allDocSlugs.some((visited) => visited.includes(slug));
+                const marker = found ? "[x]" : "[ ]";
+                console.log(`    ${marker} ${slug}`);
+            }
+        }
+        if (f.allDocSlugs.length > 0) {
+            const nonCanonical = f.allDocSlugs.filter((slug) => !f.canonicalSlugs.some((c) => slug.includes(c)));
+            if (nonCanonical.length > 0) {
+                console.log("    Additional docs visited:");
+                for (const slug of nonCanonical) {
+                    console.log(`      + ${slug}`);
+                }
+            }
+        }
+        console.log();
+    }
+    // ---- Search strategy ----
+    const allSearches = analysis.features.flatMap((f) => f.allSearchQueries);
+    if (allSearches.length > 0) {
+        console.log("SEARCH STRATEGY");
+        console.log("-".repeat(80));
+        console.log();
+        for (const f of analysis.features) {
+            if (f.allSearchQueries.length === 0)
+                continue;
+            console.log(`  ${f.feature}:`);
+            for (const q of f.allSearchQueries) {
+                console.log(`    -> "${q}"`);
+            }
+        }
+        console.log();
+    }
+    // ---- Per-task detail ----
+    console.log("PER-TASK DETAIL");
+    console.log("-".repeat(80));
+    console.log();
+    for (const f of analysis.features) {
+        console.log(`  ## ${f.feature}`);
+        console.log();
+        for (const t of f.tasks) {
+            const variant = t.hasDocs ? "[gold]" : "[baseline]";
+            console.log(`  ${variant} ${t.description}`);
+            console.log(`    Requests: ${t.behavior.totalRequests} | ` +
+                `Doc pages: ${t.behavior.docPagesVisited} | ` +
+                `Searches: ${t.behavior.searchesPerformed} | ` +
+                `External: ${t.behavior.externalRequestCount}`);
+            if (t.behavior.docSlugsVisited.length > 0) {
+                console.log(`    Docs: ${t.behavior.docSlugsVisited.join(", ")}`);
+            }
+            if (t.behavior.uniqueSearchQueries.length > 0) {
+                console.log(`    Queries: ${t.behavior.uniqueSearchQueries.map((q) => `"${q}"`).join(", ")}`);
+            }
+            console.log();
+        }
+    }
+    // ---- External domains ----
+    const allDomains = [
+        ...new Set(analysis.features.flatMap((f) => f.allExternalDomains)),
+    ];
+    if (allDomains.length > 0) {
+        console.log("EXTERNAL DOMAINS");
+        console.log("-".repeat(80));
+        console.log();
+        for (const d of allDomains) {
+            console.log(`  - ${d}`);
+        }
+        console.log();
+    }
+    // ---- Summary stats ----
+    console.log("OVERALL STATISTICS");
+    console.log("-".repeat(80));
+    console.log();
+    const totalTasks = analysis.tasks.length;
+    const tasksUsingDocs = analysis.tasks.filter((t) => t.behavior.usedDocs).length;
+    const tasksUsingSearch = analysis.tasks.filter((t) => t.behavior.usedSearch).length;
+    const avgCanonical = analysis.features.reduce((s, f) => s + f.canonicalCoverage, 0) /
+        (analysis.features.length || 1);
+    console.log(`  Total tasks observed:    ${totalTasks}`);
+    console.log(`  Tasks that used docs:    ${tasksUsingDocs}/${totalTasks} (${((tasksUsingDocs / totalTasks) * 100).toFixed(0)}%)`);
+    console.log(`  Tasks that used search:  ${tasksUsingSearch}/${totalTasks} (${((tasksUsingSearch / totalTasks) * 100).toFixed(0)}%)`);
+    console.log(`  Avg canonical coverage:  ${(avgCanonical * 100).toFixed(1)}%`);
+    console.log();
+}
+// Only run when invoked directly (not when imported)
+if (process.argv[1]?.endsWith("agent-behavior-report.ts") ||
+    process.argv[1]?.endsWith("agent-behavior-report.js")) {
+    main();
+}

package/dist/lib/baseline.d.ts

@@ -0,0 +1,19 @@
+/**
+ * lib/baseline.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/baseline.ts.
+ * This shim preserves backward compatibility for:
+ *   - Direct CLI invocation: `tsx src/lib/baseline.ts`
+ *   - Test imports that haven't been updated yet
+ *
+ * TODO: Update all importers to use pipeline/baseline.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/baseline.js instead.
+ */
+export type { BaselineMetadata, CompareResult, ScoreComparison, } from "../pipeline/baseline.js";
+export declare function saveBaseline(tag?: string): {
+    success: boolean;
+    message: string;
+};
+export declare function compareBaseline(baselineFile?: string): import("./baseline.js").CompareResult;
+export declare function listBaselines(): import("./baseline.js").BaselineMetadata[];

package/dist/lib/baseline.js

@@ -0,0 +1,153 @@
+/**
+ * lib/baseline.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/baseline.ts.
+ * This shim preserves backward compatibility for:
+ *   - Direct CLI invocation: `tsx src/lib/baseline.ts`
+ *   - Test imports that haven't been updated yet
+ *
+ * TODO: Update all importers to use pipeline/baseline.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/baseline.js instead.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { saveBaseline as _saveBaseline, compareBaseline as _compareBaseline, listBaselines as _listBaselines, } from "../pipeline/baseline.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "../..");
+export function saveBaseline(tag) {
+    return _saveBaseline(ROOT, tag);
+}
+export function compareBaseline(baselineFile) {
+    return _compareBaseline(ROOT, baselineFile);
+}
+export function listBaselines() {
+    return _listBaselines(ROOT);
+}
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+if (process.argv[1]?.endsWith("baseline.ts") ||
+    process.argv[1]?.endsWith("baseline.js")) {
+    const args = process.argv.slice(2);
+    const command = args[0] || "save";
+    function getArg(name) {
+        const idx = args.indexOf(`--${name}`);
+        return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
+    }
+    switch (command) {
+        case "compare": {
+            const file = getArg("file");
+            console.log("=== Baseline Comparison ===\n");
+            const result = compareBaseline(file);
+            if (!result.success) {
+                console.error(`  ❌ ${result.message}`);
+                process.exit(1);
+            }
+            console.log(`  ${result.message}\n`);
+            console.log("  " +
+                "Feature Area".padEnd(18) +
+                "Current".padEnd(10) +
+                "Baseline".padEnd(10) +
+                "Delta");
+            console.log("  " + "-".repeat(50));
+            for (const c of result.comparisons) {
+                const deltaStr = c.delta > 0 ? `+${c.delta}` : c.delta === 0 ? "=" : String(c.delta);
+                const icon = c.delta > 0 ? "📈" : c.delta < 0 ? "📉" : "➡️";
+                console.log("  " +
+                    c.feature.padEnd(18) +
+                    String(c.current).padEnd(10) +
+                    String(c.baseline).padEnd(10) +
+                    `${icon} ${deltaStr}`);
+            }
+            // Cost comparison (only if cost data exists)
+            const hasCostData = result.comparisons.some((c) => c.costCurrent !== undefined || c.costBaseline !== undefined);
+            if (hasCostData) {
+                console.log();
+                console.log("  Cost Comparison:");
+                console.log("  " +
+                    "Feature Area".padEnd(18) +
+                    "Current".padEnd(10) +
+                    "Baseline".padEnd(10) +
+                    "Delta");
+                console.log("  " + "-".repeat(50));
+                for (const c of result.comparisons) {
+                    if (c.costCurrent === undefined && c.costBaseline === undefined) {
+                        continue;
+                    }
+                    const cur = `$${(c.costCurrent ?? 0).toFixed(4)}`;
+                    const base = `$${(c.costBaseline ?? 0).toFixed(4)}`;
+                    const delta = c.costDelta ?? 0;
+                    const deltaStr = delta > 0
+                        ? `+$${delta.toFixed(4)}`
+                        : delta < 0
+                            ? `-$${Math.abs(delta).toFixed(4)}`
+                            : "=";
+                    const icon = delta > 0 ? "📈" : delta < 0 ? "📉" : "➡️";
+                    console.log("  " +
+                        c.feature.padEnd(18) +
+                        cur.padEnd(10) +
+                        base.padEnd(10) +
+                        `${icon} ${deltaStr}`);
+                }
+            }
+            console.log();
+            const overallIcon = result.overallDelta > 0 ? "📈" : result.overallDelta < 0 ? "📉" : "➡️";
+            const overallStr = result.overallDelta > 0
+                ? `+${result.overallDelta}`
+                : result.overallDelta === 0
+                    ? "="
+                    : String(result.overallDelta);
+            console.log(`  Overall: ${overallIcon} ${overallStr} points`);
+            break;
+        }
+        case "history": {
+            console.log("=== Baseline History ===\n");
+            const baselines = listBaselines();
+            if (baselines.length === 0) {
+                console.log("  No baselines saved yet.");
+            }
+            else {
+                const hasCosts = baselines.some((b) => b.totalCost !== undefined || b.graderCost !== undefined);
+                const costHeader = hasCosts ? "Cost".padEnd(10) : "";
+                console.log("  " +
+                    "Date".padEnd(22) +
+                    "Avg".padEnd(6) +
+                    "Areas".padEnd(7) +
+                    costHeader +
+                    "Tag");
+                console.log("  " + "-".repeat(hasCosts ? 60 : 50));
+                for (const b of baselines) {
+                    const date = new Date(b.timestamp).toLocaleString();
+                    const combinedCost = (b.totalCost ?? 0) + (b.graderCost ?? 0);
+                    const costStr = hasCosts
+                        ? (combinedCost > 0 ? `$${combinedCost.toFixed(2)}` : "-").padEnd(10)
+                        : "";
+                    console.log("  " +
+                        date.padEnd(22) +
+                        String(b.avgScore).padEnd(6) +
+                        String(b.areaCount).padEnd(7) +
+                        costStr +
+                        (b.tag ?? ""));
+                }
+            }
+            break;
+        }
+        case "save": {
+            const tag = getArg("tag");
+            console.log("=== Saving baseline snapshot ===\n");
+            const result = saveBaseline(tag);
+            if (result.success) {
+                console.log(`  ✅ ${result.message}`);
+            }
+            else {
+                console.error(`  ❌ ${result.message}`);
+                process.exit(1);
+            }
+            break;
+        }
+        default:
+            console.error(`Unknown command: "${command}". Use: save, history, compare`);
+            process.exit(1);
+    }
+}

package/dist/lib/calculate-scores.d.ts

@@ -0,0 +1,23 @@
+/**
+ * lib/calculate-scores.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/calculate-scores.ts.
+ * This shim preserves backward compatibility for:
+ *   - Direct CLI invocation: `tsx src/lib/calculate-scores.ts`
+ *   - Test imports that haven't been updated yet
+ *
+ * TODO: Update all importers to use pipeline/calculate-scores.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/calculate-scores.js instead.
+ */
+export { calculateAndWriteScores, calculateScoresPerModel, extractGraderJudgments, scoreAgenticResults, type CalculateScoresOptions, type PromptfooResultsWrapper, type RawPromptfooFile, type RawTestResult, } from "../pipeline/calculate-scores.js";
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, type ActualScoreEntry, type ComponentResult, type TestResult, type UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
+import type { CalculateScoresOptions } from "../pipeline/calculate-scores.js";
+/**
+ * Legacy main() entry point — wraps calculateAndWriteScores() with env var fallbacks.
+ *
+ * @deprecated Use calculateAndWriteScores() from pipeline/calculate-scores.ts instead.
+ */
+export declare function main(options?: Omit<CalculateScoresOptions, "rootDir"> & {
+    rootDir?: string;
+}): void;

package/dist/lib/calculate-scores.js

@@ -0,0 +1,42 @@
+/**
+ * lib/calculate-scores.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/calculate-scores.ts.
+ * This shim preserves backward compatibility for:
+ *   - Direct CLI invocation: `tsx src/lib/calculate-scores.ts`
+ *   - Test imports that haven't been updated yet
+ *
+ * TODO: Update all importers to use pipeline/calculate-scores.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/calculate-scores.js instead.
+ */
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+// Re-export everything from the real implementation
+export { calculateAndWriteScores, calculateScoresPerModel, extractGraderJudgments, scoreAgenticResults, } from "../pipeline/calculate-scores.js";
+// Re-export core types/functions for backward compatibility
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
+import { calculateAndWriteScores } from "../pipeline/calculate-scores.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(__dirname, "..", "..");
+/**
+ * Legacy main() entry point — wraps calculateAndWriteScores() with env var fallbacks.
+ *
+ * @deprecated Use calculateAndWriteScores() from pipeline/calculate-scores.ts instead.
+ */
+export function main(options) {
+    calculateAndWriteScores({
+        rootDir: options?.rootDir ?? ROOT,
+        allowedOrigins: options?.allowedOrigins,
+        mode: options?.mode ?? process.env.EVAL_MODE ?? "baseline",
+        resolvedSource: options?.resolvedSource,
+        resultsPath: options?.resultsPath,
+        searchMode: options?.searchMode,
+        source: options?.source,
+    });
+}
+// Only run when invoked directly (not when imported for testing)
+if (process.argv[1]?.endsWith("calculate-scores.ts") ||
+    process.argv[1]?.endsWith("calculate-scores.js")) {
+    main();
+}

package/dist/lib/compare.d.ts

@@ -0,0 +1,18 @@
+/**
+ * compare.ts
+ *
+ * CLI for structured comparison between two evaluation runs.
+ *
+ * Usage:
+ *   pnpm compare                                # compare current vs latest baseline
+ *   pnpm compare --baseline <path>              # compare current vs specific file
+ *   pnpm compare --baseline <path> --experiment <path>  # compare two specific files
+ *   pnpm compare --threshold 5                  # custom noise threshold
+ *   pnpm compare --output /tmp/comparison.json  # write JSON report to file
+ *   pnpm compare --format json                  # output raw JSON (default: table)
+ *
+ * Reads: results/latest/score-summary.json (as experiment, unless --experiment)
+ * Reads: results/baselines/<latest>.json (as baseline, unless --baseline)
+ */
+export { formatComparisonMarkdown, formatComparisonTable, } from "../_vendor/ailf-core/index.d.ts";
+export declare function main(): void;