@sanity/ailf 1.0.0 → 2.0.1

package/dist/lib/grader-validate.js

@@ -0,0 +1,78 @@
+/**
+ * lib/grader-validate.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/grader-validate-runner.ts.
+ * The pure analysis functions live in pipeline/grader-validation.ts.
+ * This shim preserves backward compatibility for direct CLI invocation.
+ *
+ * TODO: Update all importers to use pipeline/ modules, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/grader-validate-runner.js instead.
+ */
+// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
+import "dotenv/config";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+// Re-export from pipeline modules
+export { formatValidationReport, runGraderValidate, } from "../pipeline/grader-validate-runner.js";
+export { classifyCorrelation, validateGrader, } from "../pipeline/grader-validation.js";
+import { runGraderValidate } from "../pipeline/grader-validate-runner.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+// ---------------------------------------------------------------------------
+// CLI argument parsing
+// ---------------------------------------------------------------------------
+function parseCliArgs() {
+    const args = process.argv.slice(2);
+    function getOption(name) {
+        const idx = args.indexOf(`--${name}`);
+        return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
+    }
+    function getFlag(name) {
+        return args.includes(`--${name}`);
+    }
+    const showHelp = getFlag("help") || getFlag("h");
+    if (showHelp) {
+        console.log(`
+Usage: pnpm grader-validate [options]
+
+Validate grader accuracy against human reference grades.
+
+Options:
+  --grader <model>     Grader model to validate (default: from config/models.yaml)
+  --threshold <n>      MAE threshold for pass/fail (default: 10)
+  --help, -h           Show this help
+`);
+        process.exit(0);
+    }
+    const thresholdStr = getOption("threshold");
+    return {
+        graderOverride: getOption("grader"),
+        maeThreshold: thresholdStr ? parseFloat(thresholdStr) : 10,
+    };
+}
+/**
+ * Legacy main() entry point.
+ * @deprecated Use runGraderValidate() from pipeline/grader-validate-runner.js instead.
+ */
+export async function main() {
+    const { graderOverride, maeThreshold } = parseCliArgs();
+    const result = await runGraderValidate({
+        graderModel: graderOverride,
+        maeThreshold,
+        rootDir: ROOT,
+    });
+    // Exit with error code if threshold not met
+    if (!result.passesThreshold) {
+        console.error(`\n  ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${maeThreshold}`);
+        process.exit(1);
+    }
+}
+// Only run when invoked directly
+if (process.argv[1]?.endsWith("grader-validate.ts") ||
+    process.argv[1]?.endsWith("grader-validate.js")) {
+    main().catch((err) => {
+        console.error("❌ Fatal error:", err);
+        process.exit(1);
+    });
+}

package/dist/lib/measure-retrieval.d.ts

@@ -0,0 +1,14 @@
+/**
+ * lib/measure-retrieval.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/measure-retrieval.ts.
+ * This shim preserves backward compatibility for direct CLI invocation.
+ *
+ * TODO: Update all importers to use pipeline/measure-retrieval.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/measure-retrieval.js instead.
+ */
+import "dotenv/config";
+export { calculateNDCG, calculateRecall, formatRetrievalTable, measureRetrieval, type MeasureRetrievalOptions, type RetrievalResult, type RetrievalSummary, type RetrieverFn, } from "../pipeline/measure-retrieval.js";
+/** @deprecated Use measureRetrieval() from pipeline/measure-retrieval.js instead. */
+export declare function main(): Promise<void>;

package/dist/lib/measure-retrieval.js

@@ -0,0 +1,71 @@
+/**
+ * lib/measure-retrieval.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/measure-retrieval.ts.
+ * This shim preserves backward compatibility for direct CLI invocation.
+ *
+ * TODO: Update all importers to use pipeline/measure-retrieval.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/measure-retrieval.js instead.
+ */
+// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
+import "dotenv/config";
+import { writeFileSync, mkdirSync } from "fs";
+import { join, dirname } from "path";
+import { getSanityClient } from "../sanity/client.js";
+import { formatRetrievalTable, measureRetrieval, } from "../pipeline/measure-retrieval.js";
+// Re-export pipeline types and functions
+export { calculateNDCG, calculateRecall, formatRetrievalTable, measureRetrieval, } from "../pipeline/measure-retrieval.js";
+// ---------------------------------------------------------------------------
+// Sanity text search retriever (side-effecting — uses Sanity client)
+// ---------------------------------------------------------------------------
+async function retrieveDocsForQuery(query, k = 10) {
+    const client = getSanityClient();
+    const results = await client.fetch(`
+    *[_type == "article" && !(_id in path("drafts.**"))]
+    | score(
+        boost(title match $query, 3),
+        boost(pt::text(content) match $query, 1)
+      )
+    | order(_score desc)
+    [0...$k] {
+      "slug": slug.current,
+      _score
+    }
+  `, { k, query });
+    return results.map((r) => r.slug);
+}
+// ---------------------------------------------------------------------------
+// Legacy main() entry point
+// ---------------------------------------------------------------------------
+/** @deprecated Use measureRetrieval() from pipeline/measure-retrieval.js instead. */
+export async function main() {
+    console.log("=== Sanity AI Literacy — Retrieval Quality Measurement ===\n");
+    const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
+    const summary = await measureRetrieval({
+        onProgress: (area, taskId, result) => {
+            console.log(`  ${taskId}:`);
+            console.log(`    Recall@5:  ${(result.recall_at_5 * 100).toFixed(1)}%`);
+            console.log(`    Recall@10: ${(result.recall_at_10 * 100).toFixed(1)}%`);
+            console.log(`    NDCG@10:   ${(result.ndcg_at_10 * 100).toFixed(1)}%`);
+        },
+        retriever: retrieveDocsForQuery,
+        rootDir: ROOT,
+    });
+    // Print summary
+    console.log();
+    console.log(formatRetrievalTable(summary));
+    // Persist results
+    const outDir = join(ROOT, "results", "latest");
+    mkdirSync(outDir, { recursive: true });
+    writeFileSync(join(outDir, "retrieval-results.json"), JSON.stringify(summary, null, 2));
+    console.log("\nResults written to results/latest/retrieval-results.json");
+}
+// Only run when invoked directly (not when imported)
+if (process.argv[1]?.endsWith("measure-retrieval.ts") ||
+    process.argv[1]?.endsWith("measure-retrieval.js")) {
+    main().catch((err) => {
+        console.error("Fatal error:", err);
+        process.exit(1);
+    });
+}

package/dist/lib/pr-comment.d.ts

@@ -0,0 +1,16 @@
+/**
+ * lib/pr-comment.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/pr-comment.ts.
+ *
+ * @deprecated Import from ../pipeline/pr-comment.js instead.
+ */
+export { generatePrComment, type PrCommentOptions, } from "../pipeline/pr-comment.js";
+import type { PrCommentOptions } from "../pipeline/pr-comment.js";
+/**
+ * Legacy main() entry point.
+ * @deprecated Use generatePrComment() from pipeline/pr-comment.ts instead.
+ */
+export declare function main(options?: Omit<PrCommentOptions, "rootDir"> & {
+    rootDir?: string;
+}): void;

package/dist/lib/pr-comment.js

@@ -0,0 +1,28 @@
+/**
+ * lib/pr-comment.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/pr-comment.ts.
+ *
+ * @deprecated Import from ../pipeline/pr-comment.js instead.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+export { generatePrComment, } from "../pipeline/pr-comment.js";
+import { generatePrComment } from "../pipeline/pr-comment.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+/**
+ * Legacy main() entry point.
+ * @deprecated Use generatePrComment() from pipeline/pr-comment.ts instead.
+ */
+export function main(options) {
+    generatePrComment({
+        rootDir: options?.rootDir ?? ROOT,
+        outputPath: options?.outputPath,
+        promptfooUrl: options?.promptfooUrl,
+    });
+}
+if (process.argv[1]?.endsWith("pr-comment.ts") ||
+    process.argv[1]?.endsWith("pr-comment.js")) {
+    main();
+}

package/dist/lib/readiness-report.d.ts

@@ -0,0 +1,13 @@
+/**
+ * lib/readiness-report.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/readiness-report.ts.
+ *
+ * @deprecated Import from ../pipeline/readiness-report.js instead.
+ */
+export { formatReadinessMarkdown, generateReadinessReport, type DimensionCheck, type HistoryEntry, type ReadinessReport, } from "../pipeline/readiness-report.js";
+/**
+ * Legacy main() entry point.
+ * @deprecated Use generateReadinessReport() + formatReadinessMarkdown() directly.
+ */
+export declare function main(): void;

package/dist/lib/readiness-report.js

@@ -0,0 +1,108 @@
+/**
+ * lib/readiness-report.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/readiness-report.ts.
+ *
+ * @deprecated Import from ../pipeline/readiness-report.js instead.
+ */
+import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { load } from "js-yaml";
+import { ThresholdConfigSchema, } from "../pipeline/schemas.js";
+export { formatReadinessMarkdown, generateReadinessReport, } from "../pipeline/readiness-report.js";
+import { generateReadinessReport, formatReadinessMarkdown, } from "../pipeline/readiness-report.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
+const GAP_ANALYSIS_PATH = join(ROOT, "results", "latest", "gap-analysis.json");
+const THRESHOLDS_PATH = join(ROOT, "config", "thresholds.yaml");
+const BASELINES_DIR = join(ROOT, "results", "baselines");
+/**
+ * Legacy main() entry point.
+ * @deprecated Use generateReadinessReport() + formatReadinessMarkdown() directly.
+ */
+export function main() {
+    const args = process.argv.slice(2);
+    let area;
+    let history = false;
+    let output;
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg === "--area" && i + 1 < args.length) {
+            area = args[++i];
+        }
+        else if (arg === "--history") {
+            history = true;
+        }
+        else if (arg === "--output" && i + 1 < args.length) {
+            output = args[++i];
+        }
+    }
+    if (!area) {
+        console.error("Usage: readiness-report --area <area> [--history] [--output <file>]");
+        process.exit(1);
+    }
+    // Load data
+    if (!existsSync(SCORE_SUMMARY_PATH)) {
+        throw new Error(`Score summary not found at ${SCORE_SUMMARY_PATH}. Run \`pnpm pipeline\` first.`);
+    }
+    const scoreSummary = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
+    if (!existsSync(THRESHOLDS_PATH)) {
+        throw new Error(`Threshold config not found at ${THRESHOLDS_PATH}.`);
+    }
+    const rawThresholds = readFileSync(THRESHOLDS_PATH, "utf-8");
+    const thresholdConfig = ThresholdConfigSchema.parse(load(rawThresholds));
+    let gapAnalysis;
+    if (existsSync(GAP_ANALYSIS_PATH)) {
+        gapAnalysis = JSON.parse(readFileSync(GAP_ANALYSIS_PATH, "utf-8"));
+    }
+    const historyEntries = [];
+    if (history && existsSync(BASELINES_DIR)) {
+        const files = readdirSync(BASELINES_DIR)
+            .filter((f) => f.endsWith(".json"))
+            .sort();
+        for (const file of files) {
+            try {
+                const raw = readFileSync(join(BASELINES_DIR, file), "utf-8");
+                const data = JSON.parse(raw);
+                const areaScore = data.scores?.find((s) => s.feature === area);
+                if (!areaScore)
+                    continue;
+                const nameWithoutExt = file.replace(/\.json$/, "");
+                const parts = nameWithoutExt.split("_");
+                const tag = parts.length > 4 ? parts.slice(4).join("_") : undefined;
+                historyEntries.push({
+                    score: areaScore.totalScore,
+                    tag,
+                    timestamp: data.timestamp ?? nameWithoutExt,
+                });
+            }
+            catch {
+                // Skip malformed baseline files
+            }
+        }
+    }
+    const report = generateReadinessReport({
+        area,
+        gapAnalysis,
+        history: historyEntries,
+        scoreSummary,
+        thresholdConfig,
+    });
+    const markdown = formatReadinessMarkdown(report);
+    if (output) {
+        writeFileSync(output, markdown, "utf-8");
+        console.error(`✅ Readiness report written to ${output}`);
+    }
+    else {
+        console.log(markdown);
+    }
+    if (!report.pass) {
+        process.exit(1);
+    }
+}
+if (process.argv[1]?.endsWith("readiness-report.ts") ||
+    process.argv[1]?.endsWith("readiness-report.js")) {
+    main();
+}

package/dist/lib/webhook-server.d.ts

@@ -0,0 +1,11 @@
+/**
+ * lib/webhook-server.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/webhook-server.ts.
+ * This shim preserves backward compatibility for direct CLI invocation.
+ *
+ * TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/webhook-server.js instead.
+ */
+export { startWebhookServer, type WebhookServerOptions, } from "../pipeline/webhook-server.js";

package/dist/lib/webhook-server.js

@@ -0,0 +1,24 @@
+/**
+ * lib/webhook-server.ts — DEPRECATED re-export shim.
+ *
+ * The real implementation has moved to pipeline/webhook-server.ts.
+ * This shim preserves backward compatibility for direct CLI invocation.
+ *
+ * TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
+ *
+ * @deprecated Import from ../pipeline/webhook-server.js instead.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { startWebhookServer } from "../pipeline/webhook-server.js";
+export { startWebhookServer, } from "../pipeline/webhook-server.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+// When imported directly (e.g., `tsx src/lib/webhook-server.ts`), start the server
+startWebhookServer({
+    dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
+    debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10),
+    githubToken: process.env.GITHUB_TOKEN ?? "",
+    port: parseInt(process.env.WEBHOOK_PORT ?? "3333", 10),
+    rootDir: ROOT,
+});

package/dist/lib/weekly-digest.d.ts

@@ -0,0 +1,24 @@
+/**
+ * weekly-digest.ts
+ *
+ * CLI script to generate and deliver a weekly evaluation digest.
+ *
+ * Queries the Sanity Content Lake for all reports within the configured
+ * lookback window, computes trend analysis, and delivers the digest
+ * via configured channels (Slack, stdout, or both).
+ *
+ * Usage:
+ *   pnpm weekly-digest                    # send to configured Slack webhook
+ *   pnpm weekly-digest --dry-run          # print to stdout only
+ *   pnpm weekly-digest --lookback 14      # 14-day lookback window
+ *   pnpm weekly-digest --json             # output raw JSON
+ *
+ * Environment variables:
+ *   SLACK_WEBHOOK_URL     — Slack incoming webhook URL
+ *   SANITY_API_TOKEN      — Sanity read token
+ *   AILF_TRIGGER_TYPE     — set to "scheduled" by the cron workflow
+ *   AILF_SCHEDULE         — the schedule name (e.g., "weekly-digest")
+ *
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
+ */
+export declare function main(): Promise<void>;

package/dist/lib/weekly-digest.js

@@ -0,0 +1,148 @@
+/**
+ * weekly-digest.ts
+ *
+ * CLI script to generate and deliver a weekly evaluation digest.
+ *
+ * Queries the Sanity Content Lake for all reports within the configured
+ * lookback window, computes trend analysis, and delivers the digest
+ * via configured channels (Slack, stdout, or both).
+ *
+ * Usage:
+ *   pnpm weekly-digest                    # send to configured Slack webhook
+ *   pnpm weekly-digest --dry-run          # print to stdout only
+ *   pnpm weekly-digest --lookback 14      # 14-day lookback window
+ *   pnpm weekly-digest --json             # output raw JSON
+ *
+ * Environment variables:
+ *   SLACK_WEBHOOK_URL     — Slack incoming webhook URL
+ *   SANITY_API_TOKEN      — Sanity read token
+ *   AILF_TRIGGER_TYPE     — set to "scheduled" by the cron workflow
+ *   AILF_SCHEDULE         — the schedule name (e.g., "weekly-digest")
+ *
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
+ */
+import { config as dotenvConfig } from "dotenv";
+import { existsSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { generateDigest } from "../schedules/digest.js";
+import { getDigestConfig } from "../schedules/loader.js";
+import { formatWeeklyDigest } from "../sinks/slack/format.js";
+// Load root .env (same override behavior as pipeline.ts)
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
+if (existsSync(envPath)) {
+    dotenvConfig({ override: true, path: envPath });
+}
+// ---------------------------------------------------------------------------
+// CLI argument parsing
+// ---------------------------------------------------------------------------
+const args = process.argv.slice(2);
+function getOption(name) {
+    const idx = args.indexOf(`--${name}`);
+    return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
+}
+function hasFlag(name) {
+    return args.includes(`--${name}`);
+}
+const DRY_RUN = hasFlag("dry-run");
+const JSON_OUTPUT = hasFlag("json");
+const lookbackOverride = getOption("lookback");
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+export async function main() {
+    console.log();
+    console.log("=== AI Literacy Weekly Digest ===");
+    console.log();
+    // Load digest config
+    const digestConfig = getDigestConfig();
+    const lookbackDays = lookbackOverride
+        ? parseInt(lookbackOverride, 10)
+        : (digestConfig?.lookbackDays ?? 7);
+    console.log(`  Lookback: ${lookbackDays} days`);
+    console.log(`  Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
+    console.log();
+    // Generate digest — uses AILF_REPORT_* env vars for report store access,
+    // independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
+    const digest = await generateDigest({
+        dataset: process.env.AILF_REPORT_DATASET,
+        lookbackDays,
+        projectId: process.env.AILF_REPORT_PROJECT_ID,
+        token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
+    });
+    if (!digest) {
+        console.log("  No reports found in the lookback window. Nothing to send.");
+        process.exit(0);
+    }
+    // Output
+    console.log(`  Reports found: ${digest.reportCount}`);
+    console.log(`  Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
+    console.log(`  Improved: ${digest.improved.join(", ") || "none"}`);
+    console.log(`  Regressed: ${digest.regressed.join(", ") || "none"}`);
+    console.log(`  Stable: ${digest.stable.join(", ") || "none"}`);
+    console.log();
+    if (JSON_OUTPUT) {
+        console.log(JSON.stringify(digest, null, 2));
+        return;
+    }
+    // Format for Slack
+    const message = formatWeeklyDigest(digest);
+    if (DRY_RUN) {
+        console.log("  --- Slack Message Preview ---");
+        console.log(`  Text: ${message.text}`);
+        console.log();
+        for (const block of message.blocks) {
+            if (block.text) {
+                console.log(`  [${block.type}] ${block.text.text}`);
+            }
+            if (block.fields) {
+                for (const field of block.fields) {
+                    console.log(`    [field] ${field.text}`);
+                }
+            }
+            if (block.elements) {
+                for (const el of block.elements) {
+                    console.log(`    [element] ${el.text}`);
+                }
+            }
+        }
+        console.log();
+        return;
+    }
+    // Deliver via Slack
+    const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
+    if (!webhookUrl) {
+        console.warn("  ⚠️  No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
+        console.log("  Printing digest to stdout instead:");
+        console.log();
+        console.log(`  ${message.text}`);
+        return;
+    }
+    console.log("  Sending to Slack...");
+    try {
+        const response = await fetch(webhookUrl, {
+            body: JSON.stringify(message),
+            headers: { "Content-Type": "application/json" },
+            method: "POST",
+        });
+        if (response.ok) {
+            console.log("  ✅ Digest sent successfully");
+        }
+        else {
+            const text = await response.text();
+            console.warn(`  ⚠️  Slack delivery failed: ${response.status} ${text}`);
+        }
+    }
+    catch (error) {
+        console.warn(`  ⚠️  Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+// Only run when invoked directly (not when imported)
+if (process.argv[1]?.endsWith("weekly-digest.ts") ||
+    process.argv[1]?.endsWith("weekly-digest.js")) {
+    main().catch((error) => {
+        console.error("Fatal error:", error);
+        process.exit(1);
+    });
+}

package/dist/orchestration/build-app-context.js

@@ -8,7 +8,9 @@
  * Once all commands construct ResolvedConfig directly (or use --config),
  * this bridge can be deleted.
  */
+import { join } from "node:path";
 import { createAppContext } from "../composition-root.js";
+import { tryLoadConfigFile } from "../pipeline/compiler/config-loader.js";
 /**
  * Map legacy ResolvedOptions to ResolvedConfig.
  *
@@ -50,6 +52,7 @@ export function mapToResolvedConfig(opts, rootDir) {
         noCache: opts.noCache,
         noRemoteCache: opts.noRemoteCache,
         graderReplications: opts.graderReplications,
+        outputDir: opts.outputDir,
         outputPath: opts.outputPath,
         urls: opts.urlArgs.length > 0 ? opts.urlArgs : undefined,
         headers: opts.headerArgs.length > 0
@@ -75,6 +78,10 @@ export function mapToResolvedConfig(opts, rootDir) {
         remote: opts.remote ?? false,
         apiUrl: opts.apiUrl ?? "https://ailf-api.sanity.build",
         apiKey: opts.apiKey,
+        captureEnabled: opts.captureEnabled ?? false,
+        captureDir: opts.captureDir ?? join(rootDir, "results", "captures"),
+        captureCompress: opts.captureCompress ?? true,
+        captureExtras: opts.captureExtras ?? true,
     };
 }
 /**
@@ -85,5 +92,11 @@ export function mapToResolvedConfig(opts, rootDir) {
  */
 export function buildAppContext(opts, rootDir) {
     const config = mapToResolvedConfig(opts, rootDir);
+    // Inject config-file-only values that don't come from CLI options.
+    // evalBudgetMs lives on ModelsConfig, not CLI flags.
+    const models = tryLoadConfigFile("models", rootDir);
+    if (models?.data?.evalBudgetMs) {
+        config.evalBudgetMs = models.data.evalBudgetMs;
+    }
     return createAppContext(config);
 }

package/dist/orchestration/build-step-sequence.js

@@ -35,8 +35,10 @@ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
     if (config.repoTasksPath) {
         steps.push(new MirrorRepoTasksStep());
     }
-    // Step 1: Fetch documentation (step handles --skip-fetch internally)
-    steps.push(new FetchDocsStep());
+    // Step 1: Fetch documentation (literacy mode only — other modes don't use canonical docs)
+    if (config.mode === "literacy") {
+        steps.push(new FetchDocsStep());
+    }
     // Step 2: Generate Promptfoo configs
     steps.push(new GenerateConfigsStep());
     // Step 3: Run evaluation (steps handle --skip-eval internally)

package/dist/orchestration/cache-context.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Build cache context strings from the resolved pipeline configuration.
+ *
+ * These non-file strings participate in cache key computation so that
+ * different CLI filters (mode, variant, area, task, tag) produce
+ * distinct cache entries. Without them, running `--mode knowledge-probe`
+ * after `--mode literacy` would return cached literacy results.
+ *
+ * @see packages/core/src/ports/pipeline-step.ts — cacheContext() method
+ * @see packages/eval/src/pipeline/cache.ts — hashFiles() context parameter
+ */
+import type { ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Derive deterministic context strings from the resolved pipeline config.
+ *
+ * Included in every cacheable step's key so that:
+ * - `--mode literacy` and `--mode knowledge-probe` never share cache
+ * - `--variant agentic` and `--variant baseline` never share cache
+ * - `--area studio` and `--area groq` never share cache
+ * - `--task T001` and `--task T002` never share cache
+ * - `--tag critical` and `--tag smoke` never share cache
+ */
+export declare function buildCacheContext(config: ResolvedConfig): string[];

package/dist/orchestration/cache-context.js

@@ -0,0 +1,43 @@
+/**
+ * Build cache context strings from the resolved pipeline configuration.
+ *
+ * These non-file strings participate in cache key computation so that
+ * different CLI filters (mode, variant, area, task, tag) produce
+ * distinct cache entries. Without them, running `--mode knowledge-probe`
+ * after `--mode literacy` would return cached literacy results.
+ *
+ * @see packages/core/src/ports/pipeline-step.ts — cacheContext() method
+ * @see packages/eval/src/pipeline/cache.ts — hashFiles() context parameter
+ */
+/**
+ * Derive deterministic context strings from the resolved pipeline config.
+ *
+ * Included in every cacheable step's key so that:
+ * - `--mode literacy` and `--mode knowledge-probe` never share cache
+ * - `--variant agentic` and `--variant baseline` never share cache
+ * - `--area studio` and `--area groq` never share cache
+ * - `--task T001` and `--task T002` never share cache
+ * - `--tag critical` and `--tag smoke` never share cache
+ */
+export function buildCacheContext(config) {
+    const context = [];
+    // Mode is always present — it's required in ResolvedConfig
+    context.push(`mode:${config.mode}`);
+    // Variant (literacy sub-mode: baseline, agentic, observed, full)
+    if (config.variant) {
+        context.push(`variant:${config.variant}`);
+    }
+    // Area filter — sorted for deterministic hashing
+    if (config.areas && config.areas.length > 0) {
+        context.push(`areas:${[...config.areas].sort().join(",")}`);
+    }
+    // Task filter — sorted for deterministic hashing
+    if (config.tasks && config.tasks.length > 0) {
+        context.push(`tasks:${[...config.tasks].sort().join(",")}`);
+    }
+    // Tag filter — sorted for deterministic hashing
+    if (config.tags && config.tags.length > 0) {
+        context.push(`tags:${[...config.tags].sort().join(",")}`);
+    }
+    return context;
+}