@sanity/ailf 0.1.0

package/dist/scripts/baseline.js

@@ -0,0 +1,267 @@
+/**
+ * Baseline.ts
+ *
+ * Manages historical baseline snapshots of evaluation scores.
+ * Allows saving, comparing, and listing score baselines over time.
+ *
+ * Usage:
+ *   pnpm baseline:save                    # save current scores as baseline
+ *   pnpm baseline:save --tag "pre-groq"   # save with a descriptive tag
+ *   pnpm baseline:compare                 # compare current vs latest baseline
+ *   pnpm baseline:history                 # list all saved baselines
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+const BASELINES_DIR = join(ROOT, "results", "baselines");
+const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
+// ---------------------------------------------------------------------------
+// Compare
+// ---------------------------------------------------------------------------
+export function compareBaseline(baselineFile) {
+    if (!existsSync(SCORE_SUMMARY_PATH)) {
+        return {
+            message: "No current score-summary.json found.",
+            success: false,
+        };
+    }
+    // Find baseline to compare against
+    const baselines = listBaselines();
+    if (baselines.length === 0) {
+        return {
+            message: "No baselines saved yet. Run 'pnpm baseline:save' first.",
+            success: false,
+        };
+    }
+    const targetFile = baselineFile ?? baselines[0].filename;
+    const baselinePath = join(BASELINES_DIR, targetFile);
+    if (!existsSync(baselinePath)) {
+        return {
+            message: `Baseline file not found: ${targetFile}`,
+            success: false,
+        };
+    }
+    const current = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
+    const baseline = JSON.parse(readFileSync(baselinePath, "utf-8"));
+    const baselineMap = new Map(baseline.scores.map((s) => [s.feature, s.totalScore]));
+    const baselineCostMap = new Map(baseline.scores.map((s) => [s.feature, s.totalCost ?? 0]));
+    const comparisons = current.scores.map((s) => {
+        const baseScore = baselineMap.get(s.feature) ?? 0;
+        const currentCost = s.totalCost ?? 0;
+        const baseCost = baselineCostMap.get(s.feature) ?? 0;
+        return {
+            baseline: baseScore,
+            costBaseline: baseCost > 0 ? baseCost : undefined,
+            costCurrent: currentCost > 0 ? currentCost : undefined,
+            costDelta: currentCost > 0 || baseCost > 0 ? currentCost - baseCost : undefined,
+            current: s.totalScore,
+            delta: s.totalScore - baseScore,
+            feature: s.feature,
+        };
+    });
+    // Check for areas in baseline but not in current
+    for (const [feature, score] of baselineMap) {
+        if (!comparisons.find((c) => c.feature === feature)) {
+            comparisons.push({
+                baseline: score,
+                current: 0,
+                delta: -score,
+                feature,
+            });
+        }
+    }
+    comparisons.sort((a, b) => b.delta - a.delta);
+    const overallDelta = Math.round(current.overall.avgScore) - Math.round(baseline.overall.avgScore);
+    return {
+        comparisons,
+        message: `Compared against ${targetFile}`,
+        overallDelta,
+        success: true,
+    };
+}
+export function listBaselines() {
+    if (!existsSync(BASELINES_DIR)) {
+        return [];
+    }
+    const files = readdirSync(BASELINES_DIR)
+        .filter((f) => f.endsWith(".json"))
+        .sort()
+        .reverse(); // Newest first
+    return files.map((filename) => {
+        const raw = readFileSync(join(BASELINES_DIR, filename), "utf-8");
+        const data = JSON.parse(raw);
+        return {
+            areaCount: data.scores.length,
+            avgScore: Math.round(data.overall.avgScore),
+            filename,
+            graderCost: data.overall.cost?.graderTotal,
+            tag: data.baselineMeta?.tag,
+            timestamp: data.timestamp,
+            totalCost: data.overall.cost?.total,
+        };
+    });
+}
+export function saveBaseline(tag) {
+    if (!existsSync(SCORE_SUMMARY_PATH)) {
+        return {
+            message: "No score-summary.json found. Run 'pnpm calculate-scores' first.",
+            success: false,
+        };
+    }
+    const raw = readFileSync(SCORE_SUMMARY_PATH, "utf-8");
+    const summary = JSON.parse(raw);
+    mkdirSync(BASELINES_DIR, { recursive: true });
+    // Generate filename: YYYY-MM-DD_HHmmss[_tag].json
+    const now = new Date();
+    const datePart = now
+        .toISOString()
+        .slice(0, 19)
+        .replace(/[T:]/g, "_")
+        .replace(/-/g, "");
+    const tagPart = tag
+        ? `_${tag.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`
+        : "";
+    const filename = `${datePart}${tagPart}.json`;
+    const baseline = {
+        ...summary,
+        baselineMeta: {
+            savedAt: now.toISOString(),
+            // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string tag should be treated as no tag
+            tag: tag || undefined,
+        },
+    };
+    writeFileSync(join(BASELINES_DIR, filename), JSON.stringify(baseline, null, 2));
+    return {
+        message: `Saved baseline to results/baselines/${filename} (avg: ${Math.round(summary.overall.avgScore)}, ${summary.scores.length} areas)`,
+        success: true,
+    };
+}
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+if (process.argv[1]?.endsWith("baseline.ts") ||
+    process.argv[1]?.endsWith("baseline.js")) {
+    const args = process.argv.slice(2);
+    const command = args[0] || "save";
+    function getArg(name) {
+        const idx = args.indexOf(`--${name}`);
+        return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
+    }
+    switch (command) {
+        case "compare": {
+            const file = getArg("file");
+            console.log("=== Baseline Comparison ===\n");
+            const result = compareBaseline(file);
+            if (!result.success) {
+                console.error(`  ❌ ${result.message}`);
+                process.exit(1);
+            }
+            console.log(`  ${result.message}\n`);
+            console.log("  " +
+                "Feature Area".padEnd(18) +
+                "Current".padEnd(10) +
+                "Baseline".padEnd(10) +
+                "Delta");
+            console.log("  " + "-".repeat(50));
+            for (const c of result.comparisons) {
+                const deltaStr = c.delta > 0 ? `+${c.delta}` : c.delta === 0 ? "=" : String(c.delta);
+                const icon = c.delta > 0 ? "📈" : c.delta < 0 ? "📉" : "➡️";
+                console.log("  " +
+                    c.feature.padEnd(18) +
+                    String(c.current).padEnd(10) +
+                    String(c.baseline).padEnd(10) +
+                    `${icon} ${deltaStr}`);
+            }
+            // Cost comparison (only if cost data exists)
+            const hasCostData = result.comparisons.some((c) => c.costCurrent !== undefined || c.costBaseline !== undefined);
+            if (hasCostData) {
+                console.log();
+                console.log("  " + "Cost Comparison:");
+                console.log("  " +
+                    "Feature Area".padEnd(18) +
+                    "Current".padEnd(10) +
+                    "Baseline".padEnd(10) +
+                    "Delta");
+                console.log("  " + "-".repeat(50));
+                for (const c of result.comparisons) {
+                    if (c.costCurrent === undefined && c.costBaseline === undefined) {
+                        continue;
+                    }
+                    const cur = `$${(c.costCurrent ?? 0).toFixed(4)}`;
+                    const base = `$${(c.costBaseline ?? 0).toFixed(4)}`;
+                    const delta = c.costDelta ?? 0;
+                    const deltaStr = delta > 0
+                        ? `+$${delta.toFixed(4)}`
+                        : delta < 0
+                            ? `-$${Math.abs(delta).toFixed(4)}`
+                            : "=";
+                    const icon = delta > 0 ? "📈" : delta < 0 ? "📉" : "➡️";
+                    console.log("  " +
+                        c.feature.padEnd(18) +
+                        cur.padEnd(10) +
+                        base.padEnd(10) +
+                        `${icon} ${deltaStr}`);
+                }
+            }
+            console.log();
+            const overallIcon = result.overallDelta > 0 ? "📈" : result.overallDelta < 0 ? "📉" : "➡️";
+            const overallStr = result.overallDelta > 0
+                ? `+${result.overallDelta}`
+                : result.overallDelta === 0
+                    ? "="
+                    : String(result.overallDelta);
+            console.log(`  Overall: ${overallIcon} ${overallStr} points`);
+            break;
+        }
+        case "history": {
+            console.log("=== Baseline History ===\n");
+            const baselines = listBaselines();
+            if (baselines.length === 0) {
+                console.log("  No baselines saved yet.");
+            }
+            else {
+                const hasCosts = baselines.some((b) => b.totalCost !== undefined || b.graderCost !== undefined);
+                const costHeader = hasCosts ? "Cost".padEnd(10) : "";
+                console.log("  " +
+                    "Date".padEnd(22) +
+                    "Avg".padEnd(6) +
+                    "Areas".padEnd(7) +
+                    costHeader +
+                    "Tag");
+                console.log("  " + "-".repeat(hasCosts ? 60 : 50));
+                for (const b of baselines) {
+                    const date = new Date(b.timestamp).toLocaleString();
+                    const combinedCost = (b.totalCost ?? 0) + (b.graderCost ?? 0);
+                    const costStr = hasCosts
+                        ? (combinedCost > 0 ? `$${combinedCost.toFixed(2)}` : "-").padEnd(10)
+                        : "";
+                    console.log("  " +
+                        date.padEnd(22) +
+                        String(b.avgScore).padEnd(6) +
+                        String(b.areaCount).padEnd(7) +
+                        costStr +
+                        (b.tag ?? ""));
+                }
+            }
+            break;
+        }
+        case "save": {
+            const tag = getArg("tag");
+            console.log("=== Saving baseline snapshot ===\n");
+            const result = saveBaseline(tag);
+            if (result.success) {
+                console.log(`  ✅ ${result.message}`);
+            }
+            else {
+                console.error(`  ❌ ${result.message}`);
+                process.exit(1);
+            }
+            break;
+        }
+        default:
+            console.error(`Unknown command: "${command}". Use: save, history, compare`);
+            process.exit(1);
+    }
+}

package/dist/scripts/calculate-scores.d.ts

@@ -0,0 +1,166 @@
+/**
+ * Calculate-scores.ts
+ *
+ * Reads Promptfoo evaluation output and computes the AI Literacy Score
+ * for each feature area. Each dimension is scored on a uniform 0–100 scale:
+ *
+ *   Task Completion    (0–100)  — Can the LLM implement the feature?
+ *   Code Correctness   (0–100)  — Is the code idiomatic and correct?
+ *   Doc Coverage       (0–100)  — Did docs provide the needed info?
+ *
+ * Dimensions are combined into a weighted composite (0–100) using weights
+ * from config/rubrics.yaml (default: Task×0.50 + Code×0.25 + Docs×0.25).
+ *
+ * Additionally compares with-docs vs without-docs scores to calculate
+ * the "Doc Lift" — how much documentation helps vs parametric knowledge.
+ *
+ * When tests are run with the InstrumentedProvider (agent-observer),
+ * this script also aggregates and reports agent behavior data: which
+ * documentation pages were visited, what searches were performed, and
+ * overall network activity patterns.
+ */
+import "dotenv/config";
+import type { FeatureScore, GraderJudgment, PerModelEntry } from "../pipeline/types.js";
+export interface ComponentResult {
+    assertion?: {
+        type: string;
+        value?: string;
+        /** Structured metadata propagated from rubric templates (Approach 5). */
+        metadata?: Record<string, unknown>;
+    };
+    pass: boolean;
+    reason?: string;
+    score?: number;
+}
+export interface PromptfooResultsWrapper {
+    results: RawTestResult[];
+    stats: {
+        successes: number;
+        failures: number;
+        tokenUsage?: {
+            assertions?: {
+                completion: number;
+                prompt: number;
+                total: number;
+            };
+            completion: number;
+            prompt: number;
+            total: number;
+        };
+    };
+}
+export interface RawPromptfooFile {
+    [key: string]: unknown;
+    config?: {
+        defaultTest?: {
+            options?: {
+                provider?: string;
+                rubricProvider?: string;
+            };
+        };
+    };
+    results: PromptfooResultsWrapper;
+}
+export interface RawTestResult {
+    cost?: number;
+    error?: string;
+    gradingResult: null | {
+        componentResults: ComponentResult[];
+        pass: boolean;
+    };
+    metadata?: Record<string, unknown>;
+    provider?: {
+        id?: string;
+        label?: string;
+    };
+    response: {
+        output: string;
+    };
+    testCase?: {
+        description?: string;
+        vars?: Record<string, string>;
+    };
+    vars: Record<string, string>;
+}
+export interface TestResult {
+    cost: number;
+    description: string;
+    gradingResult: {
+        componentResults: ComponentResult[];
+        pass: boolean;
+    };
+    metadata?: Record<string, unknown>;
+    /** Provider identifier (e.g., "openai:gpt-4o") */
+    providerId?: string;
+    /** Provider label (e.g., "GPT-4o") */
+    providerLabel?: string;
+    response: {
+        output: string;
+    };
+    vars: Record<string, string>;
+}
+export interface UrlMetadata {
+    otherUrls: string[];
+    sanityUrlCount: number;
+    sanityUrls: string[];
+    totalUrlCount: number;
+}
+/**
+ * Calculate scores grouped by model. Each model gets its own FeatureScore[]
+ * and model-level aggregates.
+ *
+ * Uses the provider.id from Promptfoo results to identify models.
+ * Falls back to provider.label, then "unknown" if neither is available.
+ *
+ * @returns Record keyed by model ID, or null if only one model was used
+ *          (per-model breakdown is redundant when there's only one model).
+ */
+export declare function calculateScoresPerModel(resultsPath: string, weights: Record<string, number>): null | PerModelEntry[];
+export declare function classifyRubric(component: ComponentResult): "codeCorrectness" | "docCoverage" | "taskCompletion" | null;
+export declare function detectFeatureArea(description: string): string;
+/**
+ * Extract grader judgments (reason text + scores) from evaluation results.
+ *
+ * This preserves the grader's natural language reasoning for downstream
+ * analysis (failure mode classification, gap analysis). Each llm-rubric
+ * assertion produces one GraderJudgment entry.
+ *
+ * Phase 3a prerequisite: structured judgment data for failure mode extraction.
+ */
+export declare function extractGraderJudgments(resultsPath: string): GraderJudgment[];
+/**
+ * Finds the URL-extraction assertion result in a test's componentResults
+ * and parses the structured JSON from its `reason` field.
+ */
+export declare function extractUrlMetadata(test: TestResult): null | UrlMetadata;
+export declare function parseRubricScore(component: ComponentResult): number;
+/**
+ * Score agentic evaluation results. In agentic mode, all test entries are
+ * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).
+ * The model retrieves docs via tools, so all results map to "actual" scores.
+ *
+ * Returns a record keyed by feature area with the composite actual score.
+ */
+interface ActualScoreEntry {
+    actualScore: number;
+    codeCorrectness: number;
+    docCoverage: number;
+    taskCompletion: number;
+    testCount: number;
+    totalCost: number;
+}
+export declare function scoreAgenticResults(resultsPath: string, weights: Record<string, number>): Record<string, ActualScoreEntry>;
+/**
+ * Merge baseline FeatureScore[] with agentic actual scores to produce
+ * the full three-layer decomposition.
+ *
+ * The merge is per feature area. For each area:
+ * - If baseline data exists: floor, ceiling, docLift, docQualityGap are populated
+ * - If agentic data exists: actualScore is populated
+ * - If both exist: retrievalGap and infrastructureEfficiency are computed
+ *
+ * @param baselineScores  Floor/ceiling scores from baseline evaluation (may be empty)
+ * @param agenticScores   Actual scores from agentic evaluation (may be empty)
+ */
+export declare function mergeScores(baselineScores: FeatureScore[], agenticScores: Record<string, ActualScoreEntry>): FeatureScore[];
+export {};