@sanity/ailf 2.0.2 → 2.1.0

package/dist/scripts/agent-behavior-report.js

@@ -1,315 +0,0 @@
-/**
- * agent-behavior-report.ts
- *
- * Standalone script that reads Promptfoo evaluation results containing
- * agent behavior observation data and generates a detailed report.
- *
- * This provides deeper analysis than the summary included in the main
- * calculate-scores report, including:
- *
- *   - Per-task behavior breakdown (which specific pages each task visited)
- *   - Canonical doc coverage (did the agent find the "right" docs?)
- *   - Request timeline and latency analysis
- *   - Search strategy analysis
- *   - Cross-task navigation pattern detection
- *
- * Usage:
- *   tsx src/scripts/agent-behavior-report.ts [results-path]
- */
-// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
-import "dotenv/config";
-import { readFileSync, writeFileSync, mkdirSync, existsSync } from "fs";
-import { join, dirname } from "path";
-// Canonical doc mapping: task description patterns -> expected doc slugs
-// This maps what docs a well-informed agent *should* visit for each task
-const CANONICAL_DOC_MAP = {
-    frameworks: [
-        "remix",
-        "nuxt",
-        "svelte",
-        "astro",
-        "gatsby",
-        "client-libraries",
-    ],
-    functions: [
-        "functions",
-        "webhooks",
-        "groq-powered-webhooks",
-        "event-driven",
-        "automations",
-    ],
-    "nextjs-live": [
-        "next-js",
-        "live-content-api",
-        "content-source-maps",
-        "app-router",
-        "groq",
-        "client-libraries",
-    ],
-    "studio-setup": [
-        "studio",
-        "schema-types",
-        "structure-builder",
-        "configuration",
-        "plugins",
-    ],
-    "visual-editing": [
-        "visual-editing",
-        "presentation",
-        "preview",
-        "overlays",
-        "loaders",
-    ],
-};
-function analyzeResults(resultsPath) {
-    const json = JSON.parse(readFileSync(resultsPath, "utf-8"));
-    // Support both the flat shape ({ results: TestResult[] }) and the full
-    // Promptfoo envelope shape ({ results: { results: TestResult[] } }).
-    const results = Array.isArray(json.results)
-        ? json.results
-        : json.results.results;
-    const tasks = [];
-    for (const result of results) {
-        const metadata = result.metadata;
-        if (!metadata?.agentBehaviorSummary)
-            continue;
-        const behavior = metadata.agentBehaviorSummary;
-        tasks.push({
-            behavior,
-            description: result.description,
-            feature: detectFeatureArea(result.description),
-            hasDocs: !!(result.vars.docs && result.vars.docs.trim().length > 0),
-        });
-    }
-    if (tasks.length === 0) {
-        return { features: [], hasData: false, tasks: [] };
-    }
-    // Group by feature
-    const byFeature = {};
-    for (const t of tasks) {
-        if (!byFeature[t.feature])
-            byFeature[t.feature] = [];
-        byFeature[t.feature].push(t);
-    }
-    const features = Object.entries(byFeature)
-        .map(([feature, featureTasks]) => {
-        const allDocSlugs = [
-            ...new Set(featureTasks.flatMap((t) => t.behavior.docSlugsVisited)),
-        ];
-        const allSearchQueries = [
-            ...new Set(featureTasks.flatMap((t) => t.behavior.uniqueSearchQueries)),
-        ];
-        const allExternalDomains = [
-            ...new Set(featureTasks.flatMap((t) => t.behavior.externalDomains)),
-        ];
-        const canonicalSlugs = CANONICAL_DOC_MAP[feature] || [];
-        const matchedCanonical = canonicalSlugs.filter((slug) => allDocSlugs.some((visited) => visited.includes(slug)));
-        const canonicalCoverage = canonicalSlugs.length > 0
-            ? matchedCanonical.length / canonicalSlugs.length
-            : 0;
-        const count = featureTasks.length || 1;
-        return {
-            allDocSlugs,
-            allExternalDomains,
-            allSearchQueries,
-            avgDocPages: featureTasks.reduce((s, t) => s + t.behavior.docPagesVisited, 0) /
-                count,
-            avgNetworkMs: featureTasks.reduce((s, t) => s + t.behavior.totalNetworkMs, 0) /
-                count,
-            avgSearches: featureTasks.reduce((s, t) => s + t.behavior.searchesPerformed, 0) /
-                count,
-            canonicalCoverage,
-            canonicalSlugs,
-            feature,
-            tasks: featureTasks,
-        };
-    })
-        .sort((a, b) => a.feature.localeCompare(b.feature));
-    return { features, hasData: true, tasks };
-}
-function detectFeatureArea(description) {
-    const desc = description.toLowerCase();
-    if (desc.includes("studio"))
-        return "studio-setup";
-    if (desc.includes("visual") ||
-        desc.includes("presentation") ||
-        desc.includes("live preview"))
-        return "visual-editing";
-    if (desc.includes("function") || desc.includes("webhook"))
-        return "functions";
-    if (desc.includes("next") || desc.includes("app router"))
-        return "nextjs-live";
-    if (desc.includes("remix") ||
-        desc.includes("nuxt") ||
-        desc.includes("svelte"))
-        return "frameworks";
-    return "other";
-}
-// ---------------------------------------------------------------------------
-// Report output
-// ---------------------------------------------------------------------------
-function main() {
-    const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
-    const resultsPath = process.argv[2] || join(ROOT, "results", "latest", "eval-results.json");
-    if (!existsSync(resultsPath)) {
-        console.error(`Results file not found: ${resultsPath}`);
-        console.error("Run an evaluation first: pnpm eval:observed");
-        process.exit(1);
-    }
-    console.log(`Reading results from: ${resultsPath}`);
-    console.log();
-    const analysis = analyzeResults(resultsPath);
-    if (!analysis.hasData) {
-        console.log("No agent behavior data found in the results.");
-        console.log("Make sure you ran the evaluation with the observed config:");
-        console.log("  pnpm eval:observed");
-        process.exit(0);
-    }
-    printReport(analysis);
-    // Persist detailed report as JSON
-    const outDir = join(ROOT, "results", "latest");
-    mkdirSync(outDir, { recursive: true });
-    const reportData = {
-        features: analysis.features.map((f) => ({
-            avgDocPages: f.avgDocPages,
-            avgNetworkMs: f.avgNetworkMs,
-            avgSearches: f.avgSearches,
-            canonicalCoverage: f.canonicalCoverage,
-            canonicalSlugs: f.canonicalSlugs,
-            docSlugsVisited: f.allDocSlugs,
-            externalDomains: f.allExternalDomains,
-            feature: f.feature,
-            searchQueries: f.allSearchQueries,
-            taskCount: f.tasks.length,
-        })),
-        tasks: analysis.tasks.map((t) => ({
-            behavior: t.behavior,
-            description: t.description,
-            feature: t.feature,
-            hasDocs: t.hasDocs,
-        })),
-        timestamp: new Date().toISOString(),
-        totalTasks: analysis.tasks.length,
-    };
-    writeFileSync(join(outDir, "agent-behavior-report.json"), JSON.stringify(reportData, null, 2));
-    console.log("Agent behavior report written to results/latest/agent-behavior-report.json");
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-function printReport(analysis) {
-    console.log("=".repeat(80));
-    console.log("            AGENT BEHAVIOR OBSERVATION REPORT");
-    console.log("=".repeat(80));
-    console.log();
-    // ---- Overview table ----
-    console.log("OVERVIEW BY FEATURE AREA");
-    console.log("-".repeat(80));
-    const h = "| Feature Area        | Tasks | Avg Docs | Avg Search | Avg Net(ms) | Canon% |";
-    const sep = "|---------------------|-------|----------|------------|-------------|--------|";
-    console.log(h);
-    console.log(sep);
-    for (const f of analysis.features) {
-        console.log(`| ${f.feature.padEnd(19)} | ` +
-            `${f.tasks.length.toString().padStart(5)} | ` +
-            `${f.avgDocPages.toFixed(1).padStart(8)} | ` +
-            `${f.avgSearches.toFixed(1).padStart(10)} | ` +
-            `${Math.round(f.avgNetworkMs).toString().padStart(11)} | ` +
-            `${(f.canonicalCoverage * 100).toFixed(0).padStart(5)}% |`);
-    }
-    console.log();
-    // ---- Canonical coverage breakdown ----
-    console.log("CANONICAL DOCUMENTATION COVERAGE");
-    console.log("-".repeat(80));
-    console.log();
-    for (const f of analysis.features) {
-        console.log(`  ${f.feature} (${(f.canonicalCoverage * 100).toFixed(0)}% canonical coverage):`);
-        if (f.canonicalSlugs.length === 0) {
-            console.log("    (no canonical docs defined)");
-        }
-        else {
-            for (const slug of f.canonicalSlugs) {
-                const found = f.allDocSlugs.some((visited) => visited.includes(slug));
-                const marker = found ? "[x]" : "[ ]";
-                console.log(`    ${marker} ${slug}`);
-            }
-        }
-        if (f.allDocSlugs.length > 0) {
-            const nonCanonical = f.allDocSlugs.filter((slug) => !f.canonicalSlugs.some((c) => slug.includes(c)));
-            if (nonCanonical.length > 0) {
-                console.log("    Additional docs visited:");
-                for (const slug of nonCanonical) {
-                    console.log(`      + ${slug}`);
-                }
-            }
-        }
-        console.log();
-    }
-    // ---- Search strategy ----
-    const allSearches = analysis.features.flatMap((f) => f.allSearchQueries);
-    if (allSearches.length > 0) {
-        console.log("SEARCH STRATEGY");
-        console.log("-".repeat(80));
-        console.log();
-        for (const f of analysis.features) {
-            if (f.allSearchQueries.length === 0)
-                continue;
-            console.log(`  ${f.feature}:`);
-            for (const q of f.allSearchQueries) {
-                console.log(`    -> "${q}"`);
-            }
-        }
-        console.log();
-    }
-    // ---- Per-task detail ----
-    console.log("PER-TASK DETAIL");
-    console.log("-".repeat(80));
-    console.log();
-    for (const f of analysis.features) {
-        console.log(`  ## ${f.feature}`);
-        console.log();
-        for (const t of f.tasks) {
-            const variant = t.hasDocs ? "[gold]" : "[baseline]";
-            console.log(`  ${variant} ${t.description}`);
-            console.log(`    Requests: ${t.behavior.totalRequests} | ` +
-                `Doc pages: ${t.behavior.docPagesVisited} | ` +
-                `Searches: ${t.behavior.searchesPerformed} | ` +
-                `External: ${t.behavior.externalRequestCount}`);
-            if (t.behavior.docSlugsVisited.length > 0) {
-                console.log(`    Docs: ${t.behavior.docSlugsVisited.join(", ")}`);
-            }
-            if (t.behavior.uniqueSearchQueries.length > 0) {
-                console.log(`    Queries: ${t.behavior.uniqueSearchQueries.map((q) => `"${q}"`).join(", ")}`);
-            }
-            console.log();
-        }
-    }
-    // ---- External domains ----
-    const allDomains = [
-        ...new Set(analysis.features.flatMap((f) => f.allExternalDomains)),
-    ];
-    if (allDomains.length > 0) {
-        console.log("EXTERNAL DOMAINS");
-        console.log("-".repeat(80));
-        console.log();
-        for (const d of allDomains) {
-            console.log(`  - ${d}`);
-        }
-        console.log();
-    }
-    // ---- Summary stats ----
-    console.log("OVERALL STATISTICS");
-    console.log("-".repeat(80));
-    console.log();
-    const totalTasks = analysis.tasks.length;
-    const tasksUsingDocs = analysis.tasks.filter((t) => t.behavior.usedDocs).length;
-    const tasksUsingSearch = analysis.tasks.filter((t) => t.behavior.usedSearch).length;
-    const avgCanonical = analysis.features.reduce((s, f) => s + f.canonicalCoverage, 0) /
-        (analysis.features.length || 1);
-    console.log(`  Total tasks observed:    ${totalTasks}`);
-    console.log(`  Tasks that used docs:    ${tasksUsingDocs}/${totalTasks} (${((tasksUsingDocs / totalTasks) * 100).toFixed(0)}%)`);
-    console.log(`  Tasks that used search:  ${tasksUsingSearch}/${totalTasks} (${((tasksUsingSearch / totalTasks) * 100).toFixed(0)}%)`);
-    console.log(`  Avg canonical coverage:  ${(avgCanonical * 100).toFixed(1)}%`);
-    console.log();
-}
-main();

package/dist/scripts/baseline.d.ts

@@ -1,43 +0,0 @@
-/**
- * Baseline.ts
- *
- * Manages historical baseline snapshots of evaluation scores.
- * Allows saving, comparing, and listing score baselines over time.
- *
- * Usage:
- *   pnpm baseline:save                    # save current scores as baseline
- *   pnpm baseline:save --tag "pre-groq"   # save with a descriptive tag
- *   pnpm baseline:compare                 # compare current vs latest baseline
- *   pnpm baseline:history                 # list all saved baselines
- */
-interface BaselineMetadata {
-    areaCount: number;
-    avgScore: number;
-    filename: string;
-    graderCost?: number;
-    tag?: string;
-    timestamp: string;
-    totalCost?: number;
-}
-interface CompareResult {
-    comparisons?: ScoreComparison[];
-    message: string;
-    overallDelta?: number;
-    success: boolean;
-}
-interface ScoreComparison {
-    baseline: number;
-    costBaseline?: number;
-    costCurrent?: number;
-    costDelta?: number;
-    current: number;
-    delta: number;
-    feature: string;
-}
-export declare function compareBaseline(baselineFile?: string): CompareResult;
-export declare function listBaselines(): BaselineMetadata[];
-export declare function saveBaseline(tag?: string): {
-    success: boolean;
-    message: string;
-};
-export {};

package/dist/scripts/baseline.js

@@ -1,267 +0,0 @@
-/**
- * Baseline.ts
- *
- * Manages historical baseline snapshots of evaluation scores.
- * Allows saving, comparing, and listing score baselines over time.
- *
- * Usage:
- *   pnpm baseline:save                    # save current scores as baseline
- *   pnpm baseline:save --tag "pre-groq"   # save with a descriptive tag
- *   pnpm baseline:compare                 # compare current vs latest baseline
- *   pnpm baseline:history                 # list all saved baselines
- */
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-const BASELINES_DIR = join(ROOT, "results", "baselines");
-const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
-// ---------------------------------------------------------------------------
-// Compare
-// ---------------------------------------------------------------------------
-export function compareBaseline(baselineFile) {
-    if (!existsSync(SCORE_SUMMARY_PATH)) {
-        return {
-            message: "No current score-summary.json found.",
-            success: false,
-        };
-    }
-    // Find baseline to compare against
-    const baselines = listBaselines();
-    if (baselines.length === 0) {
-        return {
-            message: "No baselines saved yet. Run 'pnpm baseline:save' first.",
-            success: false,
-        };
-    }
-    const targetFile = baselineFile ?? baselines[0].filename;
-    const baselinePath = join(BASELINES_DIR, targetFile);
-    if (!existsSync(baselinePath)) {
-        return {
-            message: `Baseline file not found: ${targetFile}`,
-            success: false,
-        };
-    }
-    const current = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
-    const baseline = JSON.parse(readFileSync(baselinePath, "utf-8"));
-    const baselineMap = new Map(baseline.scores.map((s) => [s.feature, s.totalScore]));
-    const baselineCostMap = new Map(baseline.scores.map((s) => [s.feature, s.totalCost ?? 0]));
-    const comparisons = current.scores.map((s) => {
-        const baseScore = baselineMap.get(s.feature) ?? 0;
-        const currentCost = s.totalCost ?? 0;
-        const baseCost = baselineCostMap.get(s.feature) ?? 0;
-        return {
-            baseline: baseScore,
-            costBaseline: baseCost > 0 ? baseCost : undefined,
-            costCurrent: currentCost > 0 ? currentCost : undefined,
-            costDelta: currentCost > 0 || baseCost > 0 ? currentCost - baseCost : undefined,
-            current: s.totalScore,
-            delta: s.totalScore - baseScore,
-            feature: s.feature,
-        };
-    });
-    // Check for areas in baseline but not in current
-    for (const [feature, score] of baselineMap) {
-        if (!comparisons.find((c) => c.feature === feature)) {
-            comparisons.push({
-                baseline: score,
-                current: 0,
-                delta: -score,
-                feature,
-            });
-        }
-    }
-    comparisons.sort((a, b) => b.delta - a.delta);
-    const overallDelta = Math.round(current.overall.avgScore) - Math.round(baseline.overall.avgScore);
-    return {
-        comparisons,
-        message: `Compared against ${targetFile}`,
-        overallDelta,
-        success: true,
-    };
-}
-export function listBaselines() {
-    if (!existsSync(BASELINES_DIR)) {
-        return [];
-    }
-    const files = readdirSync(BASELINES_DIR)
-        .filter((f) => f.endsWith(".json"))
-        .sort()
-        .reverse(); // Newest first
-    return files.map((filename) => {
-        const raw = readFileSync(join(BASELINES_DIR, filename), "utf-8");
-        const data = JSON.parse(raw);
-        return {
-            areaCount: data.scores.length,
-            avgScore: Math.round(data.overall.avgScore),
-            filename,
-            graderCost: data.overall.cost?.graderTotal,
-            tag: data.baselineMeta?.tag,
-            timestamp: data.timestamp,
-            totalCost: data.overall.cost?.total,
-        };
-    });
-}
-export function saveBaseline(tag) {
-    if (!existsSync(SCORE_SUMMARY_PATH)) {
-        return {
-            message: "No score-summary.json found. Run 'pnpm calculate-scores' first.",
-            success: false,
-        };
-    }
-    const raw = readFileSync(SCORE_SUMMARY_PATH, "utf-8");
-    const summary = JSON.parse(raw);
-    mkdirSync(BASELINES_DIR, { recursive: true });
-    // Generate filename: YYYY-MM-DD_HHmmss[_tag].json
-    const now = new Date();
-    const datePart = now
-        .toISOString()
-        .slice(0, 19)
-        .replace(/[T:]/g, "_")
-        .replace(/-/g, "");
-    const tagPart = tag
-        ? `_${tag.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`
-        : "";
-    const filename = `${datePart}${tagPart}.json`;
-    const baseline = {
-        ...summary,
-        baselineMeta: {
-            savedAt: now.toISOString(),
-            // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string tag should be treated as no tag
-            tag: tag || undefined,
-        },
-    };
-    writeFileSync(join(BASELINES_DIR, filename), JSON.stringify(baseline, null, 2));
-    return {
-        message: `Saved baseline to results/baselines/${filename} (avg: ${Math.round(summary.overall.avgScore)}, ${summary.scores.length} areas)`,
-        success: true,
-    };
-}
-// ---------------------------------------------------------------------------
-// CLI
-// ---------------------------------------------------------------------------
-if (process.argv[1]?.endsWith("baseline.ts") ||
-    process.argv[1]?.endsWith("baseline.js")) {
-    const args = process.argv.slice(2);
-    const command = args[0] || "save";
-    function getArg(name) {
-        const idx = args.indexOf(`--${name}`);
-        return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
-    }
-    switch (command) {
-        case "compare": {
-            const file = getArg("file");
-            console.log("=== Baseline Comparison ===\n");
-            const result = compareBaseline(file);
-            if (!result.success) {
-                console.error(`  ❌ ${result.message}`);
-                process.exit(1);
-            }
-            console.log(`  ${result.message}\n`);
-            console.log("  " +
-                "Feature Area".padEnd(18) +
-                "Current".padEnd(10) +
-                "Baseline".padEnd(10) +
-                "Delta");
-            console.log("  " + "-".repeat(50));
-            for (const c of result.comparisons) {
-                const deltaStr = c.delta > 0 ? `+${c.delta}` : c.delta === 0 ? "=" : String(c.delta);
-                const icon = c.delta > 0 ? "📈" : c.delta < 0 ? "📉" : "➡️";
-                console.log("  " +
-                    c.feature.padEnd(18) +
-                    String(c.current).padEnd(10) +
-                    String(c.baseline).padEnd(10) +
-                    `${icon} ${deltaStr}`);
-            }
-            // Cost comparison (only if cost data exists)
-            const hasCostData = result.comparisons.some((c) => c.costCurrent !== undefined || c.costBaseline !== undefined);
-            if (hasCostData) {
-                console.log();
-                console.log("  " + "Cost Comparison:");
-                console.log("  " +
-                    "Feature Area".padEnd(18) +
-                    "Current".padEnd(10) +
-                    "Baseline".padEnd(10) +
-                    "Delta");
-                console.log("  " + "-".repeat(50));
-                for (const c of result.comparisons) {
-                    if (c.costCurrent === undefined && c.costBaseline === undefined) {
-                        continue;
-                    }
-                    const cur = `$${(c.costCurrent ?? 0).toFixed(4)}`;
-                    const base = `$${(c.costBaseline ?? 0).toFixed(4)}`;
-                    const delta = c.costDelta ?? 0;
-                    const deltaStr = delta > 0
-                        ? `+$${delta.toFixed(4)}`
-                        : delta < 0
-                            ? `-$${Math.abs(delta).toFixed(4)}`
-                            : "=";
-                    const icon = delta > 0 ? "📈" : delta < 0 ? "📉" : "➡️";
-                    console.log("  " +
-                        c.feature.padEnd(18) +
-                        cur.padEnd(10) +
-                        base.padEnd(10) +
-                        `${icon} ${deltaStr}`);
-                }
-            }
-            console.log();
-            const overallIcon = result.overallDelta > 0 ? "📈" : result.overallDelta < 0 ? "📉" : "➡️";
-            const overallStr = result.overallDelta > 0
-                ? `+${result.overallDelta}`
-                : result.overallDelta === 0
-                    ? "="
-                    : String(result.overallDelta);
-            console.log(`  Overall: ${overallIcon} ${overallStr} points`);
-            break;
-        }
-        case "history": {
-            console.log("=== Baseline History ===\n");
-            const baselines = listBaselines();
-            if (baselines.length === 0) {
-                console.log("  No baselines saved yet.");
-            }
-            else {
-                const hasCosts = baselines.some((b) => b.totalCost !== undefined || b.graderCost !== undefined);
-                const costHeader = hasCosts ? "Cost".padEnd(10) : "";
-                console.log("  " +
-                    "Date".padEnd(22) +
-                    "Avg".padEnd(6) +
-                    "Areas".padEnd(7) +
-                    costHeader +
-                    "Tag");
-                console.log("  " + "-".repeat(hasCosts ? 60 : 50));
-                for (const b of baselines) {
-                    const date = new Date(b.timestamp).toLocaleString();
-                    const combinedCost = (b.totalCost ?? 0) + (b.graderCost ?? 0);
-                    const costStr = hasCosts
-                        ? (combinedCost > 0 ? `$${combinedCost.toFixed(2)}` : "-").padEnd(10)
-                        : "";
-                    console.log("  " +
-                        date.padEnd(22) +
-                        String(b.avgScore).padEnd(6) +
-                        String(b.areaCount).padEnd(7) +
-                        costStr +
-                        (b.tag ?? ""));
-                }
-            }
-            break;
-        }
-        case "save": {
-            const tag = getArg("tag");
-            console.log("=== Saving baseline snapshot ===\n");
-            const result = saveBaseline(tag);
-            if (result.success) {
-                console.log(`  ✅ ${result.message}`);
-            }
-            else {
-                console.error(`  ❌ ${result.message}`);
-                process.exit(1);
-            }
-            break;
-        }
-        default:
-            console.error(`Unknown command: "${command}". Use: save, history, compare`);
-            process.exit(1);
-    }
-}