@sanity/ailf 0.1.0 → 0.1.2

package/dist/scripts/debug-eval.js

@@ -1,73 +0,0 @@
-/**
- * debug-eval.ts
- *
- * Thin wrapper around `promptfoo eval` that reads DEBUG_EVAL_* environment
- * variables and forwards them as native promptfoo filter flags.
- *
- * Environment variables:
- *   DEBUG_EVAL=1              — required to enable debug mode
- *   DEBUG_EVAL_PATTERN=<re>   — --filter-pattern (regex on test description)
- *   DEBUG_EVAL_N=<number>     — --filter-first-n (default: 2 when no other filters set)
- *   DEBUG_EVAL_SAMPLE=<number> — --filter-sample (random N tests)
- *
- * Usage:
- *   tsx src/scripts/debug-eval.ts --config promptfooconfig.yaml
- *   tsx src/scripts/debug-eval.ts --config promptfooconfig.agentic.yaml --no-cache
- *
- * All extra argv are forwarded to promptfoo eval unchanged.
- */
-import { execSync } from "child_process";
-// ---------------------------------------------------------------------------
-// Parse DEBUG_EVAL_* environment
-// ---------------------------------------------------------------------------
-const debugEnabled = process.env.DEBUG_EVAL === "1";
-if (!debugEnabled) {
-    console.error("⚠ debug-eval.ts called without DEBUG_EVAL=1 — running full eval instead.");
-}
-const pattern = process.env.DEBUG_EVAL_PATTERN;
-const firstN = process.env.DEBUG_EVAL_N;
-const sample = process.env.DEBUG_EVAL_SAMPLE;
-// ---------------------------------------------------------------------------
-// Build filter flags
-// ---------------------------------------------------------------------------
-const filterFlags = [];
-if (debugEnabled) {
-    if (pattern) {
-        filterFlags.push(`--filter-pattern`, `'${pattern}'`);
-    }
-    if (sample) {
-        filterFlags.push(`--filter-sample`, sample);
-    }
-    if (firstN) {
-        filterFlags.push(`--filter-first-n`, firstN);
-    }
-    // Default: if no filters specified, limit to first 2 tests for speed
-    if (filterFlags.length === 0) {
-        filterFlags.push(`--filter-first-n`, "2");
-    }
-}
-// ---------------------------------------------------------------------------
-// Forward to promptfoo eval
-// ---------------------------------------------------------------------------
-// argv[0] = node, argv[1] = this script, argv[2..] = user args
-const passthroughArgs = process.argv.slice(2);
-const allArgs = ["eval", ...passthroughArgs, ...filterFlags];
-const cmd = `promptfoo ${allArgs.join(" ")}`;
-if (debugEnabled) {
-    console.log(`\n🐛 Debug mode enabled`);
-    console.log(`   Filters: ${filterFlags.join(" ") || "(default: first 2)"}`);
-    console.log(`   Command: ${cmd}\n`);
-}
-try {
-    execSync(cmd, {
-        env: process.env,
-        stdio: "inherit",
-    });
-}
-catch (err) {
-    // promptfoo exits non-zero when assertions fail — that's expected
-    const code = err !== null && typeof err === "object" && "status" in err
-        ? err.status
-        : 1;
-    process.exit(code);
-}

package/dist/scripts/discovery-report.d.ts

@@ -1,58 +0,0 @@
-/**
- * discovery-report.ts
- *
- * Generates an agent discoverability report from agentic mode retrieval
- * metrics. Reads score-summary.json (which contains `retrievalMetrics`
- * from agentic evaluation) and produces a markdown report showing:
- *
- *   - Retrieval summary (recall, precision, F1)
- *   - Per-area retrieval breakdown
- *   - Invisible documents (never retrieved by any task)
- *   - Recommendations for improving discoverability
- *
- * Phase 5c of the Scenario Matrix implementation (Scenarios 4.1 and 4.2).
- *
- * Usage:
- *   tsx src/scripts/discovery-report.ts                # stdout
- *   tsx src/scripts/discovery-report.ts --area groq    # filter by area
- *   tsx src/scripts/discovery-report.ts --output report.md
- *
- * @see docs/design-docs/retrieval-metrics.md
- */
-import "dotenv/config";
-import type { AreaRetrievalMetrics, RetrievalMetrics, ScoreSummary } from "../pipeline/types.js";
-export interface DiscoveryReport {
-    /** All areas included in the report (after filtering) */
-    areas: AreaRetrievalMetrics[];
-    /** Base URL from the score summary source config */
-    baseUrl: string | undefined;
-    /** Document slugs that were never retrieved by any task */
-    invisibleDocs: InvisibleDoc[];
-    /** Overall retrieval metrics */
-    overall: RetrievalMetrics["overall"];
-    /** Actionable recommendations */
-    recommendations: string[];
-    /** ISO timestamp of the source evaluation */
-    timestamp: string;
-    /** Total canonical docs across included areas */
-    totalCanonicalDocs: number;
-    /** Total hits (canonical docs successfully retrieved) */
-    totalHits: number;
-}
-export interface InvisibleDoc {
-    /** Tasks that reference this document via canonical_docs */
-    referencedBy: string[];
-    /** The document slug */
-    slug: string;
-}
-/**
- * Format a discovery report as markdown.
- */
-export declare function formatDiscoveryMarkdown(report: DiscoveryReport): string;
-/**
- * Generate a structured discovery report from a score summary.
- *
- * @param summary - Parsed score-summary.json
- * @param areaFilter - Optional area names to include (all if empty)
- */
-export declare function generateDiscoveryReport(summary: ScoreSummary, areaFilter?: string[]): DiscoveryReport;

package/dist/scripts/discovery-report.js

@@ -1,250 +0,0 @@
-/**
- * discovery-report.ts
- *
- * Generates an agent discoverability report from agentic mode retrieval
- * metrics. Reads score-summary.json (which contains `retrievalMetrics`
- * from agentic evaluation) and produces a markdown report showing:
- *
- *   - Retrieval summary (recall, precision, F1)
- *   - Per-area retrieval breakdown
- *   - Invisible documents (never retrieved by any task)
- *   - Recommendations for improving discoverability
- *
- * Phase 5c of the Scenario Matrix implementation (Scenarios 4.1 and 4.2).
- *
- * Usage:
- *   tsx src/scripts/discovery-report.ts                # stdout
- *   tsx src/scripts/discovery-report.ts --area groq    # filter by area
- *   tsx src/scripts/discovery-report.ts --output report.md
- *
- * @see docs/design-docs/retrieval-metrics.md
- */
-// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
-import "dotenv/config";
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-// ---------------------------------------------------------------------------
-// Core logic (exported for testing)
-// ---------------------------------------------------------------------------
-/**
- * Format a discovery report as markdown.
- */
-export function formatDiscoveryMarkdown(report) {
-    const lines = [];
-    // Header
-    lines.push("## 🔍 Agent Discoverability Report");
-    lines.push("");
-    if (report.baseUrl) {
-        lines.push(`**Base URL:** ${report.baseUrl}`);
-    }
-    lines.push("**Mode:** Agentic");
-    lines.push("");
-    // Retrieval summary table
-    lines.push("### Retrieval Summary");
-    lines.push("");
-    lines.push("| Metric | Value |");
-    lines.push("|---|---|");
-    lines.push(`| Recall (canonical docs found) | ${pct(report.overall.avgRecall)} (${report.totalHits}/${report.totalCanonicalDocs}) |`);
-    lines.push(`| Precision (relevant docs fetched) | ${pct(report.overall.avgPrecision)} |`);
-    lines.push(`| F1 Score | ${report.overall.avgF1.toFixed(2)} |`);
-    lines.push(`| Invisible docs | ${report.invisibleDocs.length} |`);
-    lines.push("");
-    // Per-area breakdown
-    if (report.areas.length > 0) {
-        lines.push("### Per-Area Breakdown");
-        lines.push("");
-        lines.push("| Area | Recall | Precision | F1 | Tasks |");
-        lines.push("|---|---|---|---|---|");
-        for (const area of sortedAreas(report.areas)) {
-            lines.push(`| ${area.area} | ${pct(area.avgRecall)} | ${pct(area.avgPrecision)} | ${area.avgF1.toFixed(2)} | ${area.taskCount} |`);
-        }
-        lines.push("");
-    }
-    // Invisible documents
-    if (report.invisibleDocs.length > 0) {
-        lines.push("### Invisible Documents (never retrieved by any task)");
-        lines.push("");
-        for (const doc of report.invisibleDocs) {
-            const refs = doc.referencedBy.join(", ");
-            lines.push(`- \`${doc.slug}\` — referenced by ${refs}`);
-        }
-        lines.push("");
-    }
-    // Recommendations
-    if (report.recommendations.length > 0) {
-        lines.push("### Recommendations");
-        lines.push("");
-        for (let i = 0; i < report.recommendations.length; i++) {
-            lines.push(`${i + 1}. ${report.recommendations[i]}`);
-        }
-        lines.push("");
-    }
-    return lines.join("\n");
-}
-/**
- * Generate a structured discovery report from a score summary.
- *
- * @param summary - Parsed score-summary.json
- * @param areaFilter - Optional area names to include (all if empty)
- */
-export function generateDiscoveryReport(summary, areaFilter) {
-    const metrics = summary.retrievalMetrics;
-    if (!metrics) {
-        throw new Error("score-summary.json does not contain retrievalMetrics. " +
-            "Run an agentic evaluation first: pnpm pipeline -- --mode agentic");
-    }
-    // Apply area filter
-    const areas = areaFilter && areaFilter.length > 0
-        ? metrics.areas.filter((a) => areaFilter.includes(a.area))
-        : metrics.areas;
-    if (areaFilter && areaFilter.length > 0 && areas.length === 0) {
-        throw new Error(`No retrieval data found for area(s): ${areaFilter.join(", ")}. ` +
-            `Available areas: ${metrics.areas.map((a) => a.area).join(", ")}`);
-    }
-    // Recompute overall metrics for filtered areas
-    const overall = areas.length === metrics.areas.length
-        ? metrics.overall
-        : computeOverall(areas);
-    // Build invisible docs list with task references
-    const invisibleDocs = buildInvisibleDocs(areas);
-    // Compute totals for the summary table
-    const allTasks = areas.flatMap((a) => a.tasks);
-    const allExpected = new Set(allTasks.flatMap((t) => t.expected));
-    const allHits = new Set(allTasks.flatMap((t) => t.hits));
-    const totalCanonicalDocs = allExpected.size;
-    const totalHits = allHits.size;
-    // Generate recommendations
-    const recommendations = generateRecommendations(invisibleDocs, areas, overall);
-    return {
-        areas,
-        baseUrl: summary.source?.baseUrl,
-        invisibleDocs,
-        overall,
-        recommendations,
-        timestamp: summary.timestamp,
-        totalCanonicalDocs,
-        totalHits,
-    };
-}
-// ---------------------------------------------------------------------------
-// Helpers (alphabetical for perfectionist/sort-modules)
-// ---------------------------------------------------------------------------
-function buildInvisibleDocs(areas) {
-    // Collect all invisible slugs and map them to the tasks that reference them
-    const slugToTasks = new Map();
-    for (const area of areas) {
-        for (const task of area.tasks) {
-            for (const slug of task.missed) {
-                // Check if this slug is globally invisible (never retrieved by ANY task)
-                const isGloballyInvisible = areas.every((a) => a.tasks.every((t) => !t.retrieved.includes(slug)));
-                if (isGloballyInvisible) {
-                    if (!slugToTasks.has(slug)) {
-                        slugToTasks.set(slug, new Set());
-                    }
-                    slugToTasks.get(slug).add(task.taskId);
-                }
-            }
-        }
-    }
-    return [...slugToTasks.entries()]
-        .map(([slug, tasks]) => ({
-        referencedBy: [...tasks].sort(),
-        slug,
-    }))
-        .sort((a, b) => b.referencedBy.length - a.referencedBy.length);
-}
-function computeOverall(areas) {
-    if (areas.length === 0) {
-        return { avgF1: 0, avgPrecision: 0, avgRecall: 0 };
-    }
-    // Weight by task count for fair averaging
-    const totalTasks = areas.reduce((s, a) => s + a.taskCount, 0);
-    if (totalTasks === 0) {
-        return { avgF1: 0, avgPrecision: 0, avgRecall: 0 };
-    }
-    const avgRecall = areas.reduce((s, a) => s + a.avgRecall * a.taskCount, 0) / totalTasks;
-    const avgPrecision = areas.reduce((s, a) => s + a.avgPrecision * a.taskCount, 0) / totalTasks;
-    const avgF1 = areas.reduce((s, a) => s + a.avgF1 * a.taskCount, 0) / totalTasks;
-    return { avgF1, avgPrecision, avgRecall };
-}
-function generateRecommendations(invisibleDocs, areas, overall) {
-    const recs = [];
-    // Recommend adding invisible docs to llms.txt
-    const highImpactInvisible = invisibleDocs.filter((d) => d.referencedBy.length > 0);
-    for (const doc of highImpactInvisible.slice(0, 5)) {
-        const taskWord = doc.referencedBy.length === 1 ? "task" : "tasks";
-        recs.push(`Add \`${doc.slug}\` to llms.txt (referenced by ${doc.referencedBy.length} ${taskWord})`);
-    }
-    // Recommend cross-linking for invisible docs
-    if (invisibleDocs.length > 0) {
-        recs.push(`Improve cross-linking to ${invisibleDocs.length} invisible document${invisibleDocs.length === 1 ? "" : "s"}`);
-    }
-    // Flag low-recall areas
-    const lowRecallAreas = areas.filter((a) => a.avgRecall < 0.5);
-    for (const area of lowRecallAreas) {
-        recs.push(`Investigate low recall in \`${area.area}\` (${pct(area.avgRecall)}) — agents miss most canonical docs`);
-    }
-    // Flag low-precision areas
-    const lowPrecisionAreas = areas.filter((a) => a.avgPrecision < 0.5);
-    for (const area of lowPrecisionAreas) {
-        recs.push(`Review search relevance for \`${area.area}\` (precision ${pct(area.avgPrecision)}) — agents fetch many irrelevant docs`);
-    }
-    // Overall recommendation
-    if (overall.avgF1 < 0.6) {
-        recs.push("Overall F1 is below 0.60 — consider a documentation restructure for agent accessibility");
-    }
-    return recs;
-}
-function main() {
-    const { areaFilter, output, summaryPath } = parseArgs(process.argv);
-    if (!existsSync(summaryPath)) {
-        console.error(`❌ Score summary not found: ${summaryPath}`);
-        console.error("Run an agentic evaluation first: pnpm pipeline -- --mode agentic");
-        process.exit(1);
-    }
-    const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
-    const report = generateDiscoveryReport(summary, areaFilter.length > 0 ? areaFilter : undefined);
-    const markdown = formatDiscoveryMarkdown(report);
-    if (output) {
-        writeFileSync(output, markdown, "utf-8");
-        console.log(`✅ Discovery report written to ${output}`);
-    }
-    else {
-        console.log(markdown);
-    }
-}
-function parseArgs(argv) {
-    const args = argv.slice(2);
-    let output;
-    const areaFilter = [];
-    let summaryPath = join(ROOT, "results", "latest", "score-summary.json");
-    for (let i = 0; i < args.length; i++) {
-        if (args[i] === "--output" && args[i + 1]) {
-            output = args[++i];
-        }
-        else if (args[i] === "--area" && args[i + 1]) {
-            areaFilter.push(...args[++i].split(","));
-        }
-        else if (args[i] === "--input" && args[i + 1]) {
-            summaryPath = args[++i];
-        }
-        else if (!args[i].startsWith("-")) {
-            summaryPath = args[i];
-        }
-    }
-    return { areaFilter, output, summaryPath };
-}
-function pct(value) {
-    return `${Math.round(value * 100)}%`;
-}
-function sortedAreas(areas) {
-    return [...areas].sort((a, b) => a.area.localeCompare(b.area));
-}
-// Only run when invoked directly
-if (process.argv[1]?.endsWith("discovery-report.ts") ||
-    process.argv[1]?.endsWith("discovery-report.js")) {
-    main();
-}

package/dist/scripts/fetch-docs.d.ts

@@ -1,35 +0,0 @@
-/**
- * Fetch-docs.ts
- *
- * Pulls documentation from the Sanity CMS and generates markdown context
- * files for use in Promptfoo evaluations. Always produces canonical contexts;
- * other outputs are opt-in:
- *
- *   1. Canonical contexts     — one file per evaluation task, containing
- *      only the manually-annotated "gold" documents for that task (always)
- *   2. Feature-area contexts  — one file per GROQ feature area query
- *      (opt-in via --include-feature-areas)
- *   3. Full corpus            — all articles in one file
- *      (opt-in via --include-corpus)
- */
-import "dotenv/config";
-/**
- * Result of resolving --sanity-document IDs against canonical docs.
- *
- * Documents specified by ID either replace a canonical doc (if the fetched
- * document's slug matches one in the canonical set) or are appended as
- * additional context (if the slug is not in the canonical set).
- */
-export interface DocumentOverlay {
-    /** Extra formatted content for docs that don't match any canonical slug */
-    appendedContent: string[];
-    /** Map from canonical slug → formatted content (replaces the normal fetch) */
-    replacements: Map<string, string>;
-}
-/** Result of comparing canonical docs between published and perspective */
-export interface ReleaseImpact {
-    added: string[];
-    modified: string[];
-    removed: string[];
-    unchanged: string[];
-}