@sanity/ailf 1.0.0 → 2.0.1

package/dist/scripts/agent-behavior-report.js

@@ -0,0 +1,315 @@
+/**
+ * agent-behavior-report.ts
+ *
+ * Standalone script that reads Promptfoo evaluation results containing
+ * agent behavior observation data and generates a detailed report.
+ *
+ * This provides deeper analysis than the summary included in the main
+ * calculate-scores report, including:
+ *
+ *   - Per-task behavior breakdown (which specific pages each task visited)
+ *   - Canonical doc coverage (did the agent find the "right" docs?)
+ *   - Request timeline and latency analysis
+ *   - Search strategy analysis
+ *   - Cross-task navigation pattern detection
+ *
+ * Usage:
+ *   tsx src/scripts/agent-behavior-report.ts [results-path]
+ */
+// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
+import "dotenv/config";
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from "fs";
+import { join, dirname } from "path";
+// Canonical doc mapping: task description patterns -> expected doc slugs
+// This maps what docs a well-informed agent *should* visit for each task
+const CANONICAL_DOC_MAP = {
+    frameworks: [
+        "remix",
+        "nuxt",
+        "svelte",
+        "astro",
+        "gatsby",
+        "client-libraries",
+    ],
+    functions: [
+        "functions",
+        "webhooks",
+        "groq-powered-webhooks",
+        "event-driven",
+        "automations",
+    ],
+    "nextjs-live": [
+        "next-js",
+        "live-content-api",
+        "content-source-maps",
+        "app-router",
+        "groq",
+        "client-libraries",
+    ],
+    "studio-setup": [
+        "studio",
+        "schema-types",
+        "structure-builder",
+        "configuration",
+        "plugins",
+    ],
+    "visual-editing": [
+        "visual-editing",
+        "presentation",
+        "preview",
+        "overlays",
+        "loaders",
+    ],
+};
+function analyzeResults(resultsPath) {
+    const json = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    // Support both the flat shape ({ results: TestResult[] }) and the full
+    // Promptfoo envelope shape ({ results: { results: TestResult[] } }).
+    const results = Array.isArray(json.results)
+        ? json.results
+        : json.results.results;
+    const tasks = [];
+    for (const result of results) {
+        const metadata = result.metadata;
+        if (!metadata?.agentBehaviorSummary)
+            continue;
+        const behavior = metadata.agentBehaviorSummary;
+        tasks.push({
+            behavior,
+            description: result.description,
+            feature: detectFeatureArea(result.description),
+            hasDocs: !!(result.vars.docs && result.vars.docs.trim().length > 0),
+        });
+    }
+    if (tasks.length === 0) {
+        return { features: [], hasData: false, tasks: [] };
+    }
+    // Group by feature
+    const byFeature = {};
+    for (const t of tasks) {
+        if (!byFeature[t.feature])
+            byFeature[t.feature] = [];
+        byFeature[t.feature].push(t);
+    }
+    const features = Object.entries(byFeature)
+        .map(([feature, featureTasks]) => {
+        const allDocSlugs = [
+            ...new Set(featureTasks.flatMap((t) => t.behavior.docSlugsVisited)),
+        ];
+        const allSearchQueries = [
+            ...new Set(featureTasks.flatMap((t) => t.behavior.uniqueSearchQueries)),
+        ];
+        const allExternalDomains = [
+            ...new Set(featureTasks.flatMap((t) => t.behavior.externalDomains)),
+        ];
+        const canonicalSlugs = CANONICAL_DOC_MAP[feature] || [];
+        const matchedCanonical = canonicalSlugs.filter((slug) => allDocSlugs.some((visited) => visited.includes(slug)));
+        const canonicalCoverage = canonicalSlugs.length > 0
+            ? matchedCanonical.length / canonicalSlugs.length
+            : 0;
+        const count = featureTasks.length || 1;
+        return {
+            allDocSlugs,
+            allExternalDomains,
+            allSearchQueries,
+            avgDocPages: featureTasks.reduce((s, t) => s + t.behavior.docPagesVisited, 0) /
+                count,
+            avgNetworkMs: featureTasks.reduce((s, t) => s + t.behavior.totalNetworkMs, 0) /
+                count,
+            avgSearches: featureTasks.reduce((s, t) => s + t.behavior.searchesPerformed, 0) /
+                count,
+            canonicalCoverage,
+            canonicalSlugs,
+            feature,
+            tasks: featureTasks,
+        };
+    })
+        .sort((a, b) => a.feature.localeCompare(b.feature));
+    return { features, hasData: true, tasks };
+}
+function detectFeatureArea(description) {
+    const desc = description.toLowerCase();
+    if (desc.includes("studio"))
+        return "studio-setup";
+    if (desc.includes("visual") ||
+        desc.includes("presentation") ||
+        desc.includes("live preview"))
+        return "visual-editing";
+    if (desc.includes("function") || desc.includes("webhook"))
+        return "functions";
+    if (desc.includes("next") || desc.includes("app router"))
+        return "nextjs-live";
+    if (desc.includes("remix") ||
+        desc.includes("nuxt") ||
+        desc.includes("svelte"))
+        return "frameworks";
+    return "other";
+}
+// ---------------------------------------------------------------------------
+// Report output
+// ---------------------------------------------------------------------------
+function main() {
+    const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
+    const resultsPath = process.argv[2] || join(ROOT, "results", "latest", "eval-results.json");
+    if (!existsSync(resultsPath)) {
+        console.error(`Results file not found: ${resultsPath}`);
+        console.error("Run an evaluation first: pnpm eval:observed");
+        process.exit(1);
+    }
+    console.log(`Reading results from: ${resultsPath}`);
+    console.log();
+    const analysis = analyzeResults(resultsPath);
+    if (!analysis.hasData) {
+        console.log("No agent behavior data found in the results.");
+        console.log("Make sure you ran the evaluation with the observed config:");
+        console.log("  pnpm eval:observed");
+        process.exit(0);
+    }
+    printReport(analysis);
+    // Persist detailed report as JSON
+    const outDir = join(ROOT, "results", "latest");
+    mkdirSync(outDir, { recursive: true });
+    const reportData = {
+        features: analysis.features.map((f) => ({
+            avgDocPages: f.avgDocPages,
+            avgNetworkMs: f.avgNetworkMs,
+            avgSearches: f.avgSearches,
+            canonicalCoverage: f.canonicalCoverage,
+            canonicalSlugs: f.canonicalSlugs,
+            docSlugsVisited: f.allDocSlugs,
+            externalDomains: f.allExternalDomains,
+            feature: f.feature,
+            searchQueries: f.allSearchQueries,
+            taskCount: f.tasks.length,
+        })),
+        tasks: analysis.tasks.map((t) => ({
+            behavior: t.behavior,
+            description: t.description,
+            feature: t.feature,
+            hasDocs: t.hasDocs,
+        })),
+        timestamp: new Date().toISOString(),
+        totalTasks: analysis.tasks.length,
+    };
+    writeFileSync(join(outDir, "agent-behavior-report.json"), JSON.stringify(reportData, null, 2));
+    console.log("Agent behavior report written to results/latest/agent-behavior-report.json");
+}
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+function printReport(analysis) {
+    console.log("=".repeat(80));
+    console.log("            AGENT BEHAVIOR OBSERVATION REPORT");
+    console.log("=".repeat(80));
+    console.log();
+    // ---- Overview table ----
+    console.log("OVERVIEW BY FEATURE AREA");
+    console.log("-".repeat(80));
+    const h = "| Feature Area        | Tasks | Avg Docs | Avg Search | Avg Net(ms) | Canon% |";
+    const sep = "|---------------------|-------|----------|------------|-------------|--------|";
+    console.log(h);
+    console.log(sep);
+    for (const f of analysis.features) {
+        console.log(`| ${f.feature.padEnd(19)} | ` +
+            `${f.tasks.length.toString().padStart(5)} | ` +
+            `${f.avgDocPages.toFixed(1).padStart(8)} | ` +
+            `${f.avgSearches.toFixed(1).padStart(10)} | ` +
+            `${Math.round(f.avgNetworkMs).toString().padStart(11)} | ` +
+            `${(f.canonicalCoverage * 100).toFixed(0).padStart(5)}% |`);
+    }
+    console.log();
+    // ---- Canonical coverage breakdown ----
+    console.log("CANONICAL DOCUMENTATION COVERAGE");
+    console.log("-".repeat(80));
+    console.log();
+    for (const f of analysis.features) {
+        console.log(`  ${f.feature} (${(f.canonicalCoverage * 100).toFixed(0)}% canonical coverage):`);
+        if (f.canonicalSlugs.length === 0) {
+            console.log("    (no canonical docs defined)");
+        }
+        else {
+            for (const slug of f.canonicalSlugs) {
+                const found = f.allDocSlugs.some((visited) => visited.includes(slug));
+                const marker = found ? "[x]" : "[ ]";
+                console.log(`    ${marker} ${slug}`);
+            }
+        }
+        if (f.allDocSlugs.length > 0) {
+            const nonCanonical = f.allDocSlugs.filter((slug) => !f.canonicalSlugs.some((c) => slug.includes(c)));
+            if (nonCanonical.length > 0) {
+                console.log("    Additional docs visited:");
+                for (const slug of nonCanonical) {
+                    console.log(`      + ${slug}`);
+                }
+            }
+        }
+        console.log();
+    }
+    // ---- Search strategy ----
+    const allSearches = analysis.features.flatMap((f) => f.allSearchQueries);
+    if (allSearches.length > 0) {
+        console.log("SEARCH STRATEGY");
+        console.log("-".repeat(80));
+        console.log();
+        for (const f of analysis.features) {
+            if (f.allSearchQueries.length === 0)
+                continue;
+            console.log(`  ${f.feature}:`);
+            for (const q of f.allSearchQueries) {
+                console.log(`    -> "${q}"`);
+            }
+        }
+        console.log();
+    }
+    // ---- Per-task detail ----
+    console.log("PER-TASK DETAIL");
+    console.log("-".repeat(80));
+    console.log();
+    for (const f of analysis.features) {
+        console.log(`  ## ${f.feature}`);
+        console.log();
+        for (const t of f.tasks) {
+            const variant = t.hasDocs ? "[gold]" : "[baseline]";
+            console.log(`  ${variant} ${t.description}`);
+            console.log(`    Requests: ${t.behavior.totalRequests} | ` +
+                `Doc pages: ${t.behavior.docPagesVisited} | ` +
+                `Searches: ${t.behavior.searchesPerformed} | ` +
+                `External: ${t.behavior.externalRequestCount}`);
+            if (t.behavior.docSlugsVisited.length > 0) {
+                console.log(`    Docs: ${t.behavior.docSlugsVisited.join(", ")}`);
+            }
+            if (t.behavior.uniqueSearchQueries.length > 0) {
+                console.log(`    Queries: ${t.behavior.uniqueSearchQueries.map((q) => `"${q}"`).join(", ")}`);
+            }
+            console.log();
+        }
+    }
+    // ---- External domains ----
+    const allDomains = [
+        ...new Set(analysis.features.flatMap((f) => f.allExternalDomains)),
+    ];
+    if (allDomains.length > 0) {
+        console.log("EXTERNAL DOMAINS");
+        console.log("-".repeat(80));
+        console.log();
+        for (const d of allDomains) {
+            console.log(`  - ${d}`);
+        }
+        console.log();
+    }
+    // ---- Summary stats ----
+    console.log("OVERALL STATISTICS");
+    console.log("-".repeat(80));
+    console.log();
+    const totalTasks = analysis.tasks.length;
+    const tasksUsingDocs = analysis.tasks.filter((t) => t.behavior.usedDocs).length;
+    const tasksUsingSearch = analysis.tasks.filter((t) => t.behavior.usedSearch).length;
+    const avgCanonical = analysis.features.reduce((s, f) => s + f.canonicalCoverage, 0) /
+        (analysis.features.length || 1);
+    console.log(`  Total tasks observed:    ${totalTasks}`);
+    console.log(`  Tasks that used docs:    ${tasksUsingDocs}/${totalTasks} (${((tasksUsingDocs / totalTasks) * 100).toFixed(0)}%)`);
+    console.log(`  Tasks that used search:  ${tasksUsingSearch}/${totalTasks} (${((tasksUsingSearch / totalTasks) * 100).toFixed(0)}%)`);
+    console.log(`  Avg canonical coverage:  ${(avgCanonical * 100).toFixed(1)}%`);
+    console.log();
+}
+main();

package/dist/scripts/baseline.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Baseline.ts
+ *
+ * Manages historical baseline snapshots of evaluation scores.
+ * Allows saving, comparing, and listing score baselines over time.
+ *
+ * Usage:
+ *   pnpm baseline:save                    # save current scores as baseline
+ *   pnpm baseline:save --tag "pre-groq"   # save with a descriptive tag
+ *   pnpm baseline:compare                 # compare current vs latest baseline
+ *   pnpm baseline:history                 # list all saved baselines
+ */
+interface BaselineMetadata {
+    areaCount: number;
+    avgScore: number;
+    filename: string;
+    graderCost?: number;
+    tag?: string;
+    timestamp: string;
+    totalCost?: number;
+}
+interface CompareResult {
+    comparisons?: ScoreComparison[];
+    message: string;
+    overallDelta?: number;
+    success: boolean;
+}
+interface ScoreComparison {
+    baseline: number;
+    costBaseline?: number;
+    costCurrent?: number;
+    costDelta?: number;
+    current: number;
+    delta: number;
+    feature: string;
+}
+export declare function compareBaseline(baselineFile?: string): CompareResult;
+export declare function listBaselines(): BaselineMetadata[];
+export declare function saveBaseline(tag?: string): {
+    success: boolean;
+    message: string;
+};
+export {};

package/dist/scripts/baseline.js

@@ -0,0 +1,267 @@
+/**
+ * Baseline.ts
+ *
+ * Manages historical baseline snapshots of evaluation scores.
+ * Allows saving, comparing, and listing score baselines over time.
+ *
+ * Usage:
+ *   pnpm baseline:save                    # save current scores as baseline
+ *   pnpm baseline:save --tag "pre-groq"   # save with a descriptive tag
+ *   pnpm baseline:compare                 # compare current vs latest baseline
+ *   pnpm baseline:history                 # list all saved baselines
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+const BASELINES_DIR = join(ROOT, "results", "baselines");
+const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
+// ---------------------------------------------------------------------------
+// Compare
+// ---------------------------------------------------------------------------
+export function compareBaseline(baselineFile) {
+    if (!existsSync(SCORE_SUMMARY_PATH)) {
+        return {
+            message: "No current score-summary.json found.",
+            success: false,
+        };
+    }
+    // Find baseline to compare against
+    const baselines = listBaselines();
+    if (baselines.length === 0) {
+        return {
+            message: "No baselines saved yet. Run 'pnpm baseline:save' first.",
+            success: false,
+        };
+    }
+    const targetFile = baselineFile ?? baselines[0].filename;
+    const baselinePath = join(BASELINES_DIR, targetFile);
+    if (!existsSync(baselinePath)) {
+        return {
+            message: `Baseline file not found: ${targetFile}`,
+            success: false,
+        };
+    }
+    const current = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
+    const baseline = JSON.parse(readFileSync(baselinePath, "utf-8"));
+    const baselineMap = new Map(baseline.scores.map((s) => [s.feature, s.totalScore]));
+    const baselineCostMap = new Map(baseline.scores.map((s) => [s.feature, s.totalCost ?? 0]));
+    const comparisons = current.scores.map((s) => {
+        const baseScore = baselineMap.get(s.feature) ?? 0;
+        const currentCost = s.totalCost ?? 0;
+        const baseCost = baselineCostMap.get(s.feature) ?? 0;
+        return {
+            baseline: baseScore,
+            costBaseline: baseCost > 0 ? baseCost : undefined,
+            costCurrent: currentCost > 0 ? currentCost : undefined,
+            costDelta: currentCost > 0 || baseCost > 0 ? currentCost - baseCost : undefined,
+            current: s.totalScore,
+            delta: s.totalScore - baseScore,
+            feature: s.feature,
+        };
+    });
+    // Check for areas in baseline but not in current
+    for (const [feature, score] of baselineMap) {
+        if (!comparisons.find((c) => c.feature === feature)) {
+            comparisons.push({
+                baseline: score,
+                current: 0,
+                delta: -score,
+                feature,
+            });
+        }
+    }
+    comparisons.sort((a, b) => b.delta - a.delta);
+    const overallDelta = Math.round(current.overall.avgScore) - Math.round(baseline.overall.avgScore);
+    return {
+        comparisons,
+        message: `Compared against ${targetFile}`,
+        overallDelta,
+        success: true,
+    };
+}
+export function listBaselines() {
+    if (!existsSync(BASELINES_DIR)) {
+        return [];
+    }
+    const files = readdirSync(BASELINES_DIR)
+        .filter((f) => f.endsWith(".json"))
+        .sort()
+        .reverse(); // Newest first
+    return files.map((filename) => {
+        const raw = readFileSync(join(BASELINES_DIR, filename), "utf-8");
+        const data = JSON.parse(raw);
+        return {
+            areaCount: data.scores.length,
+            avgScore: Math.round(data.overall.avgScore),
+            filename,
+            graderCost: data.overall.cost?.graderTotal,
+            tag: data.baselineMeta?.tag,
+            timestamp: data.timestamp,
+            totalCost: data.overall.cost?.total,
+        };
+    });
+}
+export function saveBaseline(tag) {
+    if (!existsSync(SCORE_SUMMARY_PATH)) {
+        return {
+            message: "No score-summary.json found. Run 'pnpm calculate-scores' first.",
+            success: false,
+        };
+    }
+    const raw = readFileSync(SCORE_SUMMARY_PATH, "utf-8");
+    const summary = JSON.parse(raw);
+    mkdirSync(BASELINES_DIR, { recursive: true });
+    // Generate filename: YYYY-MM-DD_HHmmss[_tag].json
+    const now = new Date();
+    const datePart = now
+        .toISOString()
+        .slice(0, 19)
+        .replace(/[T:]/g, "_")
+        .replace(/-/g, "");
+    const tagPart = tag
+        ? `_${tag.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`
+        : "";
+    const filename = `${datePart}${tagPart}.json`;
+    const baseline = {
+        ...summary,
+        baselineMeta: {
+            savedAt: now.toISOString(),
+            // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string tag should be treated as no tag
+            tag: tag || undefined,
+        },
+    };
+    writeFileSync(join(BASELINES_DIR, filename), JSON.stringify(baseline, null, 2));
+    return {
+        message: `Saved baseline to results/baselines/${filename} (avg: ${Math.round(summary.overall.avgScore)}, ${summary.scores.length} areas)`,
+        success: true,
+    };
+}
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+if (process.argv[1]?.endsWith("baseline.ts") ||
+    process.argv[1]?.endsWith("baseline.js")) {
+    const args = process.argv.slice(2);
+    const command = args[0] || "save";
+    function getArg(name) {
+        const idx = args.indexOf(`--${name}`);
+        return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
+    }
+    switch (command) {
+        case "compare": {
+            const file = getArg("file");
+            console.log("=== Baseline Comparison ===\n");
+            const result = compareBaseline(file);
+            if (!result.success) {
+                console.error(`  ❌ ${result.message}`);
+                process.exit(1);
+            }
+            console.log(`  ${result.message}\n`);
+            console.log("  " +
+                "Feature Area".padEnd(18) +
+                "Current".padEnd(10) +
+                "Baseline".padEnd(10) +
+                "Delta");
+            console.log("  " + "-".repeat(50));
+            for (const c of result.comparisons) {
+                const deltaStr = c.delta > 0 ? `+${c.delta}` : c.delta === 0 ? "=" : String(c.delta);
+                const icon = c.delta > 0 ? "📈" : c.delta < 0 ? "📉" : "➡️";
+                console.log("  " +
+                    c.feature.padEnd(18) +
+                    String(c.current).padEnd(10) +
+                    String(c.baseline).padEnd(10) +
+                    `${icon} ${deltaStr}`);
+            }
+            // Cost comparison (only if cost data exists)
+            const hasCostData = result.comparisons.some((c) => c.costCurrent !== undefined || c.costBaseline !== undefined);
+            if (hasCostData) {
+                console.log();
+                console.log("  " + "Cost Comparison:");
+                console.log("  " +
+                    "Feature Area".padEnd(18) +
+                    "Current".padEnd(10) +
+                    "Baseline".padEnd(10) +
+                    "Delta");
+                console.log("  " + "-".repeat(50));
+                for (const c of result.comparisons) {
+                    if (c.costCurrent === undefined && c.costBaseline === undefined) {
+                        continue;
+                    }
+                    const cur = `$${(c.costCurrent ?? 0).toFixed(4)}`;
+                    const base = `$${(c.costBaseline ?? 0).toFixed(4)}`;
+                    const delta = c.costDelta ?? 0;
+                    const deltaStr = delta > 0
+                        ? `+$${delta.toFixed(4)}`
+                        : delta < 0
+                            ? `-$${Math.abs(delta).toFixed(4)}`
+                            : "=";
+                    const icon = delta > 0 ? "📈" : delta < 0 ? "📉" : "➡️";
+                    console.log("  " +
+                        c.feature.padEnd(18) +
+                        cur.padEnd(10) +
+                        base.padEnd(10) +
+                        `${icon} ${deltaStr}`);
+                }
+            }
+            console.log();
+            const overallIcon = result.overallDelta > 0 ? "📈" : result.overallDelta < 0 ? "📉" : "➡️";
+            const overallStr = result.overallDelta > 0
+                ? `+${result.overallDelta}`
+                : result.overallDelta === 0
+                    ? "="
+                    : String(result.overallDelta);
+            console.log(`  Overall: ${overallIcon} ${overallStr} points`);
+            break;
+        }
+        case "history": {
+            console.log("=== Baseline History ===\n");
+            const baselines = listBaselines();
+            if (baselines.length === 0) {
+                console.log("  No baselines saved yet.");
+            }
+            else {
+                const hasCosts = baselines.some((b) => b.totalCost !== undefined || b.graderCost !== undefined);
+                const costHeader = hasCosts ? "Cost".padEnd(10) : "";
+                console.log("  " +
+                    "Date".padEnd(22) +
+                    "Avg".padEnd(6) +
+                    "Areas".padEnd(7) +
+                    costHeader +
+                    "Tag");
+                console.log("  " + "-".repeat(hasCosts ? 60 : 50));
+                for (const b of baselines) {
+                    const date = new Date(b.timestamp).toLocaleString();
+                    const combinedCost = (b.totalCost ?? 0) + (b.graderCost ?? 0);
+                    const costStr = hasCosts
+                        ? (combinedCost > 0 ? `$${combinedCost.toFixed(2)}` : "-").padEnd(10)
+                        : "";
+                    console.log("  " +
+                        date.padEnd(22) +
+                        String(b.avgScore).padEnd(6) +
+                        String(b.areaCount).padEnd(7) +
+                        costStr +
+                        (b.tag ?? ""));
+                }
+            }
+            break;
+        }
+        case "save": {
+            const tag = getArg("tag");
+            console.log("=== Saving baseline snapshot ===\n");
+            const result = saveBaseline(tag);
+            if (result.success) {
+                console.log(`  ✅ ${result.message}`);
+            }
+            else {
+                console.error(`  ❌ ${result.message}`);
+                process.exit(1);
+            }
+            break;
+        }
+        default:
+            console.error(`Unknown command: "${command}". Use: save, history, compare`);
+            process.exit(1);
+    }
+}