@sanity/ailf 3.8.0 → 3.9.0

package/dist/cli-program.js

@@ -0,0 +1,137 @@
+/**
+ * cli-program.ts — pure factory for the AILF Commander program.
+ *
+ * Splits the program construction out of cli.ts so the CLI is testable
+ * in-process. cli.ts owns bootstrap side effects (dotenv loading,
+ * retired-flag/env/cmd checks, AILF_LOG_LEVEL pre-scan, parseAsync); this
+ * module owns command wiring.
+ *
+ * The W0078 M4 black-box harness imports `buildCliProgram()` directly so
+ * tests can construct a fresh program, attach `exitOverride()`, capture
+ * stdout/stderr, and parse a synthetic argv — all without spawning a
+ * subprocess.
+ *
+ * @see packages/eval/src/__tests__/cli-harness/run-cli.ts
+ */
+import { Command } from "commander";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { CommandGroup, configureProgram } from "./commands/shared/help.js";
+import { createAgentReportCommand } from "./commands/agent-report.js";
+import { createBaselineCommand } from "./commands/baseline.js";
+import { createCacheCommand } from "./commands/cache.js";
+import { createCalculateScoresCommand } from "./commands/calculate-scores.js";
+import { createCheckStalenessCommand } from "./commands/check-staleness.js";
+import { createChronicFailuresCommand } from "./commands/chronic-failures.js";
+import { createCompareCommand } from "./commands/compare.js";
+import { createCompletionCommand } from "./commands/completion.js";
+import { createCoverageAuditCommand } from "./commands/coverage-audit.js";
+import { createDiscoveryReportCommand } from "./commands/discovery-report.js";
+import { createEvalCommand } from "./commands/eval.js";
+import { createFetchDocsCommand } from "./commands/fetch-docs.js";
+import { createGenerateConfigsCommand } from "./commands/generate-configs.js";
+import { createGraderCommand } from "./commands/grader/index.js";
+import { createInitCommand } from "./commands/init.js";
+import { createInteractiveCommand } from "./commands/interactive.js";
+import { createLookupDocCommand } from "./commands/lookup-doc.js";
+import { createMeasureRetrievalCommand } from "./commands/measure-retrieval.js";
+import { createPrCommentCommand } from "./commands/pr-comment.js";
+import { createPublishCommand } from "./commands/publish.js";
+import { createReadinessReportCommand } from "./commands/readiness-report.js";
+import { createRunCommand } from "./commands/run.js";
+import { createRunsCommand } from "./commands/runs.js";
+import { createValidateConfigCommand } from "./commands/validate.js";
+import { createValidateTasksCommand } from "./commands/validate-tasks.js";
+import { createWebhookServerCommand } from "./commands/webhook-server.js";
+import { createWeeklyDigestCommand } from "./commands/weekly-digest.js";
+/**
+ * Construct the Commander program with every subcommand registered.
+ *
+ * Pure factory — no I/O beyond reading package.json for the version, no
+ * `process.exit()`, no `process.argv` access. Tests can call this and
+ * attach `program.exitOverride()` before parsing to capture exit codes
+ * instead of terminating the process.
+ *
+ * Registration order determines group display order in `--help`. Commands
+ * within a group appear in the order they're added.
+ */
+export function buildCliProgram(opts) {
+    const { evalRoot } = opts;
+    const pkgPath = resolve(evalRoot, "package.json");
+    const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+    const program = new Command()
+        .name("ailf")
+        .description("AI Literacy Framework — evaluate how well docs enable AI coding tools\n\nMeasure whether AI coding agents can find the right documentation\nand produce correct implementations of your product features.")
+        .version(pkg.version)
+        .option("-v, --verbose", "Increase log output")
+        .option("-q, --quiet", "Suppress non-error output")
+        .option("--dotenv <path>", "Override default .env file path")
+        .option("--explain", "Show execution plan without running")
+        .option("--format <fmt>", "Output format for --explain (console, json)", "console")
+        .option("-y, --yes", "With --explain: show plan then prompt to confirm execution");
+    configureProgram(program);
+    // Global --explain hook — intercepts any command before execution
+    program.hook("preAction", async (thisCommand, actionCommand) => {
+        const globalOpts = thisCommand.opts();
+        if (!globalOpts.explain)
+            return;
+        const { handleExplain } = await import("./commands/explain-handler.js");
+        try {
+            await handleExplain(actionCommand, globalOpts.yes ?? false, evalRoot);
+            process.exit(0);
+        }
+        catch (err) {
+            // Sentinel from --yes confirmation: user wants to proceed
+            if (err !== null &&
+                typeof err === "object" &&
+                "__proceedArgv" in err) {
+                const filteredArgv = err.__proceedArgv;
+                console.log("\n  ▸ Proceeding with execution…\n");
+                await program.parseAsync(filteredArgv);
+                return;
+            }
+            throw err;
+        }
+    });
+    // ── Core Workflow ──────────────────────────────────────────────────────
+    program.addCommand(createRunCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createCompareCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createBaselineCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createPublishCommand().helpGroup(CommandGroup.CoreWorkflow));
+    program.addCommand(createRunsCommand().helpGroup(CommandGroup.CoreWorkflow));
+    // ── Analysis & Reports ────────────────────────────────────────────────
+    const reportCommand = new Command("report")
+        .description("Generate analysis and reporting outputs from evaluation runs")
+        .addCommand(createReadinessReportCommand())
+        .addCommand(createChronicFailuresCommand())
+        .addCommand(createCoverageAuditCommand())
+        .addCommand(createDiscoveryReportCommand())
+        .addCommand(createAgentReportCommand())
+        .addCommand(createWeeklyDigestCommand())
+        .addCommand(createCheckStalenessCommand());
+    program.addCommand(reportCommand.helpGroup(CommandGroup.AnalysisReports));
+    // ── Grader Reliability ────────────────────────────────────────────────
+    program.addCommand(createGraderCommand().helpGroup(CommandGroup.GraderReliability));
+    // ── Setup & Configuration ─────────────────────────────────────────────
+    program.addCommand(createInitCommand().helpGroup(CommandGroup.SetupConfig));
+    const validateCommand = new Command("validate")
+        .description("Validate AILF configuration and task files")
+        .addCommand(createValidateConfigCommand())
+        .addCommand(createValidateTasksCommand());
+    program.addCommand(validateCommand.helpGroup(CommandGroup.SetupConfig));
+    program.addCommand(createFetchDocsCommand().helpGroup(CommandGroup.SetupConfig));
+    program.addCommand(createCacheCommand().helpGroup(CommandGroup.SetupConfig));
+    // ── Pipeline Internals ────────────────────────────────────────────────
+    program.addCommand(createEvalCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createCalculateScoresCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createPrCommentCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createGenerateConfigsCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createMeasureRetrievalCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createLookupDocCommand().helpGroup(CommandGroup.PipelineInternals));
+    program.addCommand(createWebhookServerCommand().helpGroup(CommandGroup.PipelineInternals));
+    // ── Developer Tools ───────────────────────────────────────────────────
+    program.addCommand(createInteractiveCommand().helpGroup(CommandGroup.DeveloperTools));
+    // Shell completion — must be registered last (needs full program tree)
+    program.addCommand(createCompletionCommand(program).helpGroup(CommandGroup.DeveloperTools));
+    return program;
+}

package/dist/cli.d.ts

@@ -25,5 +25,11 @@
  *
  * Dev mode (without building):
  *   tsx src/cli.ts run --debug
+ *
+ * Module split: this file owns *bootstrap side effects* (dotenv,
+ * retired-flag/env/cmd checks, AILF_LOG_LEVEL pre-scan, parseAsync).
+ * The Commander wiring lives in ./cli-program.ts so the W0078 M4 black-box
+ * harness can build the program in-process without firing those side
+ * effects.
  */
 export {};

package/dist/cli.js

@@ -1,6 +1,4 @@
 #!/usr/bin/env node
-/* oxlint-disable import/first -- imports are intentionally interleaved with
-   command registration for readability and lazy loading */
 /**
  * cli.ts — AILF CLI entry point.
  *
@@ -27,15 +25,20 @@
  *
  * Dev mode (without building):
  *   tsx src/cli.ts run --debug
+ *
+ * Module split: this file owns *bootstrap side effects* (dotenv,
+ * retired-flag/env/cmd checks, AILF_LOG_LEVEL pre-scan, parseAsync).
+ * The Commander wiring lives in ./cli-program.ts so the W0078 M4 black-box
+ * harness can build the program in-process without firing those side
+ * effects.
  */
 import { config as dotenvConfig } from "dotenv";
-import { existsSync, readFileSync } from "fs";
+import { existsSync } from "fs";
 import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
+import { buildCliProgram } from "./cli-program.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, "..");
-/** Path to the eval package root (packages/eval). Used by --explain. */
-const EVAL_ROOT = ROOT;
 // ---------------------------------------------------------------------------
 // Load .env — must happen before Commander parses so that .env()
 // fallbacks resolve correctly.
@@ -220,127 +223,14 @@ if (retiredFlag || retiredCommand || retiredEnv) {
     process.exit(2);
 }
 // ---------------------------------------------------------------------------
-// Build CLI program
-// ---------------------------------------------------------------------------
-import { Command } from "commander";
-import { CommandGroup, configureProgram } from "./commands/shared/help.js";
-// Read version from package.json
-const pkgPath = resolve(ROOT, "package.json");
-const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
-const program = new Command()
-    .name("ailf")
-    .description("AI Literacy Framework — evaluate how well docs enable AI coding tools\n\nMeasure whether AI coding agents can find the right documentation\nand produce correct implementations of your product features.")
-    .version(pkg.version)
-    .option("-v, --verbose", "Increase log output")
-    .option("-q, --quiet", "Suppress non-error output")
-    .option("--dotenv <path>", "Override default .env file path")
-    .option("--explain", "Show execution plan without running")
-    .option("--format <fmt>", "Output format for --explain (console, json)", "console")
-    .option("-y, --yes", "With --explain: show plan then prompt to confirm execution");
-configureProgram(program);
-// ---------------------------------------------------------------------------
-// Global --explain hook — intercepts any command before execution
+// Build CLI program (delegates Commander wiring to ./cli-program.ts)
 // ---------------------------------------------------------------------------
-program.hook("preAction", async (thisCommand, actionCommand) => {
-    const globalOpts = thisCommand.opts();
-    if (!globalOpts.explain)
-        return;
-    const { handleExplain } = await import("./commands/explain-handler.js");
-    try {
-        await handleExplain(actionCommand, globalOpts.yes ?? false, EVAL_ROOT);
-        process.exit(0);
-    }
-    catch (err) {
-        // Sentinel from --yes confirmation: user wants to proceed
-        if (err !== null &&
-            typeof err === "object" &&
-            "__proceedArgv" in err) {
-            const filteredArgv = err.__proceedArgv;
-            console.log("\n  ▸ Proceeding with execution…\n");
-            await program.parseAsync(filteredArgv);
-            return;
-        }
-        throw err;
-    }
-});
+const program = buildCliProgram({ evalRoot: ROOT });
 // ---------------------------------------------------------------------------
-// Register commands
+// Parse and run — default to showing help when no arguments given.
 //
-// Registration order determines group display order in --help.
-// Within each group, commands appear in the order they are added.
-// ---------------------------------------------------------------------------
-// ── Core Workflow ──────────────────────────────────────────────────────
-import { createRunCommand } from "./commands/run.js";
-program.addCommand(createRunCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createCompareCommand } from "./commands/compare.js";
-program.addCommand(createCompareCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createBaselineCommand } from "./commands/baseline.js";
-program.addCommand(createBaselineCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createPublishCommand } from "./commands/publish.js";
-program.addCommand(createPublishCommand().helpGroup(CommandGroup.CoreWorkflow));
-import { createRunsCommand } from "./commands/runs.js";
-program.addCommand(createRunsCommand().helpGroup(CommandGroup.CoreWorkflow));
-// ── Analysis & Reports ────────────────────────────────────────────────
-import { createReadinessReportCommand } from "./commands/readiness-report.js";
-import { createChronicFailuresCommand } from "./commands/chronic-failures.js";
-import { createCoverageAuditCommand } from "./commands/coverage-audit.js";
-import { createDiscoveryReportCommand } from "./commands/discovery-report.js";
-import { createAgentReportCommand } from "./commands/agent-report.js";
-import { createWeeklyDigestCommand } from "./commands/weekly-digest.js";
-import { createCheckStalenessCommand } from "./commands/check-staleness.js";
-const reportCommand = new Command("report")
-    .description("Generate analysis and reporting outputs from evaluation runs")
-    .addCommand(createReadinessReportCommand())
-    .addCommand(createChronicFailuresCommand())
-    .addCommand(createCoverageAuditCommand())
-    .addCommand(createDiscoveryReportCommand())
-    .addCommand(createAgentReportCommand())
-    .addCommand(createWeeklyDigestCommand())
-    .addCommand(createCheckStalenessCommand());
-program.addCommand(reportCommand.helpGroup(CommandGroup.AnalysisReports));
-// ── Grader Reliability ────────────────────────────────────────────────
-import { createGraderCommand } from "./commands/grader/index.js";
-program.addCommand(createGraderCommand().helpGroup(CommandGroup.GraderReliability));
-// ── Setup & Configuration ─────────────────────────────────────────────
-import { createInitCommand } from "./commands/init.js";
-program.addCommand(createInitCommand().helpGroup(CommandGroup.SetupConfig));
-import { createValidateConfigCommand } from "./commands/validate.js";
-import { createValidateTasksCommand } from "./commands/validate-tasks.js";
-const validateCommand = new Command("validate")
-    .description("Validate AILF configuration and task files")
-    .addCommand(createValidateConfigCommand())
-    .addCommand(createValidateTasksCommand());
-program.addCommand(validateCommand.helpGroup(CommandGroup.SetupConfig));
-import { createFetchDocsCommand } from "./commands/fetch-docs.js";
-program.addCommand(createFetchDocsCommand().helpGroup(CommandGroup.SetupConfig));
-import { createCacheCommand } from "./commands/cache.js";
-program.addCommand(createCacheCommand().helpGroup(CommandGroup.SetupConfig));
-// ── Pipeline Internals ────────────────────────────────────────────────
-import { createEvalCommand } from "./commands/eval.js";
-program.addCommand(createEvalCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createCalculateScoresCommand } from "./commands/calculate-scores.js";
-program.addCommand(createCalculateScoresCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createPrCommentCommand } from "./commands/pr-comment.js";
-program.addCommand(createPrCommentCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createGenerateConfigsCommand } from "./commands/generate-configs.js";
-program.addCommand(createGenerateConfigsCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createMeasureRetrievalCommand } from "./commands/measure-retrieval.js";
-program.addCommand(createMeasureRetrievalCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createLookupDocCommand } from "./commands/lookup-doc.js";
-program.addCommand(createLookupDocCommand().helpGroup(CommandGroup.PipelineInternals));
-import { createWebhookServerCommand } from "./commands/webhook-server.js";
-program.addCommand(createWebhookServerCommand().helpGroup(CommandGroup.PipelineInternals));
-// ── Developer Tools ───────────────────────────────────────────────────
-import { createInteractiveCommand } from "./commands/interactive.js";
-program.addCommand(createInteractiveCommand().helpGroup(CommandGroup.DeveloperTools));
-// Shell completion — must be registered last (needs full program tree)
-import { createCompletionCommand } from "./commands/completion.js";
-program.addCommand(createCompletionCommand(program).helpGroup(CommandGroup.DeveloperTools));
-// ---------------------------------------------------------------------------
-// Parse and run — default to showing help when no arguments given
-// ---------------------------------------------------------------------------
-// If no command is specified (just `ailf`), show help.
 // The interactive wizard is still available via `ailf interactive`.
+// ---------------------------------------------------------------------------
 if (process.argv.length <= 2) {
     program.outputHelp();
 }

package/dist/config/canary-tasks.ts

@@ -0,0 +1,64 @@
+/**
+ * canary-tasks.ts — The Tier 3 canary set.
+ *
+ * Five tasks the Tier 3 nightly workflow runs against live LLMs every day.
+ * Composition follows the design doc's "weighted toward modes/areas with
+ * the most production usage and the highest historical regression rates"
+ * recommendation: GROQ and Content Lake (foundational consumer surfaces),
+ * Portable Text (historically drift-prone), Studio schema authoring (the
+ * second-most-used surface after queries), and a knowledge-probe pairing
+ * for cross-mode coverage.
+ *
+ * Each entry's `rationale` is the canary's load-bearing field — without it,
+ * future maintainers can't reason about whether a regression is meaningful
+ * or whether the slot has lost value. Update the rationale when you swap a
+ * canary entry; never silently replace one.
+ *
+ * Validated against the live task inventory by `scripts/check-canary-tasks.ts`
+ * (`pnpm check`). Dangling task IDs fail the build.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see .github/workflows/tier-3-nightly.yml — consumer
+ */
+
+import { defineCanaryTasks } from "../_vendor/ailf-core/index.js"
+
+export default defineCanaryTasks({
+  tasks: [
+    {
+      taskId: "groq-blog-queries",
+      mode: "literacy",
+      rationale:
+        "Canonical first-use path for Sanity's most-used API. GROQ is the largest doc surface and the highest-leverage canary slot — drift here means drift in the most-consumed documentation. Filtering and pagination together exercise the largest cross-section of GROQ syntax in a single task.",
+    },
+    {
+      taskId: "content-lake-mutations",
+      mode: "literacy",
+      rationale:
+        "Foundational client API. CRUD is structurally distinct from query reasoning, so this catches regressions in mutation/transaction documentation that GROQ canary slots cannot reach. Every Sanity consumer eventually writes to the Content Lake.",
+    },
+    {
+      taskId: "portable-text-rendering",
+      mode: "literacy",
+      rationale:
+        "Major doc surface flagged as historically drift-prone in the testing audit. React-rendering of Portable Text mixes documentation, type definitions, and worked examples — a regression on any axis surfaces here first.",
+    },
+    {
+      taskId: "studio-blog-schema",
+      mode: "literacy",
+      rationale:
+        "Schema authoring (`defineType` / `defineField`) is the second-most-used surface after queries. Tests structural Studio docs that change shape across versions; pairs naturally with the GROQ canary because consumers typically author schemas before querying them.",
+    },
+    {
+      taskId: "kp-groq-projections",
+      mode: "knowledge-probe",
+      rationale:
+        "Cross-mode coverage. Pairs with `groq-blog-queries` (literacy) so we catch GROQ drift in both implementation (write code) and recall (explain syntax) modes. Knowledge-probe is the only non-literacy mode in the canary today; expand once mcp-server tasks land in the repo.",
+    },
+    // mcp-server canary slot — add a third mode here when a committed
+    // mcp-server task lands under packages/eval/tasks/mcp-server/. Today
+    // there are no production mcp-server tasks (only fixtures); the trigger
+    // is upstream and adding a placeholder slot would dangle. Surfaced at
+    // Phase 5 close (2026-04-27) — see W0116 retrospective.
+  ],
+})

package/dist/config/test-budgets.ts

@@ -0,0 +1,24 @@
+/**
+ * test-budgets.ts — Per-provider daily USD spend caps for Tier 3 CI runs.
+ *
+ * Each cap is the maximum cost a single Tier 3 nightly run may incur for
+ * that provider. The Tier 3 workflow (`.github/workflows/tier-3-nightly.yml`)
+ * fails loudly if any provider's actual spend exceeds its cap.
+ *
+ * The design doc names a $30–60/day envelope across all providers. Caps
+ * here divide that envelope per-provider; tighten as baseline canary spend
+ * becomes measurable.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see scripts/tier-3-budget-check.mjs — enforcement
+ */
+
+import { defineTestBudgets } from "../_vendor/ailf-core/index.js"
+
+export default defineTestBudgets({
+  perProviderDaily: {
+    anthropic: 30,
+    openai: 30,
+  },
+  warnFraction: 0.8,
+})

package/dist/pipeline/calculate-scores.d.ts

@@ -1,6 +1,6 @@
-import { type ActualScoreEntry, type ComponentResult, type Logger, type StoredTestResult, type TestSummary } from "../_vendor/ailf-core/index.d.ts";
+import { type ActualScoreEntry, type ComponentResult, type Logger, type StoredTestResult, type TestResult, type TestSummary } from "../_vendor/ailf-core/index.d.ts";
 import { type ResolvedSourceConfig } from "../sources.js";
-import type { GraderJudgment, PerModelEntry } from "./types.js";
+import type { FeatureScore, GraderJudgment, PerModelEntry } from "./types.js";
 export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, type ActualScoreEntry, type ComponentResult, type StoredTestResult, type TestResult, type UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
 export interface PromptfooResultsWrapper {
     results: RawTestResult[];
@@ -91,6 +91,21 @@ export declare function extractGraderJudgments(resultsPath: string): GraderJudgm
  * See D0029 and docs/design-docs/score-drill-down.md (Phase 1).
  */
 export declare function extractStoredTestResults(resultsPath: string): StoredTestResult[];
+/**
+ * Score knowledge-probe evaluation results.
+ *
+ * Knowledge-probe mode evaluates parametric recall: the model has no `docs`
+ * var and answers from training-data knowledge alone. The compiler explicitly
+ * deletes `vars.docs`, so every result lands in the without-docs bucket of
+ * the literacy scoring path — collapsing testCount and ceilingScore to zero.
+ *
+ * This branch mirrors the shape of `scoreAgentHarnessResults` but groups by
+ * feature area (KP results carry `__featureArea` from the compiler), and
+ * uses the `knowledge-probe` profile (factual-correctness / completeness /
+ * currency). Literacy-specific fields (ceilingScore, floorScore, docLift,
+ * docQualityGap) are zero — KP has no with-docs/without-docs decomposition.
+ */
+export declare function scoreKnowledgeProbeResults(results: TestResult[], profile: Record<string, number>): FeatureScore[];
 /**
  * Score agentic evaluation results. In agentic mode, all test entries are
  * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).

package/dist/pipeline/calculate-scores.js

@@ -719,6 +719,55 @@ function extractTaskId(description) {
     return description.trim() || "unknown";
 }
 // ---------------------------------------------------------------------------
+// Knowledge-probe scoring — closed-book recall with no docs context
+// ---------------------------------------------------------------------------
+/**
+ * Score knowledge-probe evaluation results.
+ *
+ * Knowledge-probe mode evaluates parametric recall: the model has no `docs`
+ * var and answers from training-data knowledge alone. The compiler explicitly
+ * deletes `vars.docs`, so every result lands in the without-docs bucket of
+ * the literacy scoring path — collapsing testCount and ceilingScore to zero.
+ *
+ * This branch mirrors the shape of `scoreAgentHarnessResults` but groups by
+ * feature area (KP results carry `__featureArea` from the compiler), and
+ * uses the `knowledge-probe` profile (factual-correctness / completeness /
+ * currency). Literacy-specific fields (ceilingScore, floorScore, docLift,
+ * docQualityGap) are zero — KP has no with-docs/without-docs decomposition.
+ */
+export function scoreKnowledgeProbeResults(results, profile) {
+    const byFeature = {};
+    for (const result of results) {
+        const feature = result.vars.__featureArea || detectFeatureArea(result.description);
+        if (!byFeature[feature]) {
+            byFeature[feature] = [];
+        }
+        byFeature[feature].push(result);
+    }
+    const scores = [];
+    for (const [feature, featureResults] of Object.entries(byFeature)) {
+        const scored = scoreTestGroup(featureResults, profile, feature);
+        scores.push({
+            assertionPassRate: scored.dimensions.assertionPassRate,
+            ceilingScore: 0,
+            codeCorrectness: scored.dimensions.codeCorrectness ?? 0,
+            dimensions: scored.dimensions,
+            docCoverage: scored.dimensions.docCoverage ?? 0,
+            docLift: 0,
+            docQualityGap: 0,
+            feature,
+            floorScore: 0,
+            groupType: "feature",
+            negativeDocLift: false,
+            taskCompletion: scored.dimensions.taskCompletion ?? 0,
+            testCount: featureResults.length,
+            totalCost: scored.totalCost,
+            totalScore: scored.composite,
+        });
+    }
+    return scores.sort((a, b) => a.feature.localeCompare(b.feature));
+}
+// ---------------------------------------------------------------------------
 // Agentic scoring — all results are "actual" (agent retrieves docs via tools)
 // ---------------------------------------------------------------------------
 /**
@@ -893,6 +942,56 @@ export function calculateAndWriteScores(options) {
         const testSummary = computeTestSummary(baselineResultsPath);
         return { belowCritical: summary.belowCritical, testSummary };
     }
+    // ── Knowledge-probe scoring path ────────────────────────────
+    // Knowledge-probe mode evaluates parametric recall (no docs context).
+    // The KP compiler deletes `vars.docs`, so the literacy path would bucket
+    // every result into `withoutDocs` and collapse testCount + dimensions
+    // to zero. This branch groups by feature area only and uses the
+    // `knowledge-probe` profile (factual-correctness / completeness /
+    // currency). See docs/design-docs/mode-agnostic-scoring.md.
+    if (mode === "knowledge-probe") {
+        const probeProfile = resolveProfile("knowledge-probe", "gold", rubricConfig);
+        log.debug("Knowledge-probe scoring profile", probeProfile);
+        const results = readAndNormalizeResults(baselineResultsPath);
+        const scores = scoreKnowledgeProbeResults(results, probeProfile);
+        log.debug("Knowledge-probe scores calculated", {
+            featureCount: scores.length,
+            features: scores.map((s) => ({
+                feature: s.feature,
+                totalScore: s.totalScore,
+                testCount: s.testCount,
+                dimensions: s.dimensions,
+            })),
+        });
+        const urlRefs = aggregateUrlReferences(baselineResultsPath);
+        const sourceVerification = buildSourceVerification(ROOT, source, {
+            allowedOrigins: options.allowedOrigins,
+            mode,
+            searchMode: options.searchMode,
+        });
+        const graderCost = extractGraderCost(baselineResultsPath);
+        const summary = printReport(scores, urlRefs, source, null, // no agent behavior — KP is closed-book
+        graderCost, null, // no per-model breakdown for now
+        null, // no source isolation — KP doesn't fetch sources
+        sourceVerification, "knowledge-probe", log);
+        // Persist
+        const outDir = join(ROOT, "results", "latest");
+        mkdirSync(outDir, { recursive: true });
+        writeFileSync(join(outDir, "score-summary.json"), JSON.stringify(summary, null, 2));
+        log.info("Score summary written to results/latest/score-summary.json");
+        const judgments = extractGraderJudgments(baselineResultsPath);
+        if (judgments.length > 0) {
+            writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
+            log.info(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
+        }
+        const testResults = extractStoredTestResults(baselineResultsPath);
+        if (testResults.length > 0) {
+            writeFileSync(join(outDir, "test-results.json"), JSON.stringify(testResults, null, 2));
+            log.info(`Test results written to results/latest/test-results.json (${testResults.length} results)`);
+        }
+        const testSummary = computeTestSummary(baselineResultsPath);
+        return { belowCritical: summary.belowCritical, testSummary };
+    }
     // ── Literacy scoring path ───────────────────────────────────
     // Gold (with-docs) entries use the "default" profile (3 dimensions).
     // Baseline (without-docs) entries use "output-only" (2 dimensions,

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.d.ts

@@ -9,6 +9,11 @@ import type { KnowledgeProbeCompileOptions } from "./types.js";
  * Tool-use assertions are rejected (knowledge probes don't use tools).
  * LLM-graded assertions receive the configured grader provider.
  * All other assertions are passed through.
+ *
+ * Templated `llm-rubric` assertions (those with `template` + `criteria`) go
+ * through the shared rubric resolver so the compiled assertion carries
+ * `metadata.dimension` — without this, the scoring engine can't classify
+ * KP grader output and dimension scores collapse to zero (W0128 / DOC-2077).
  */
 export declare function mapKnowledgeProbeAssertion(assertion: {
     type: string;

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.js

@@ -1,12 +1,18 @@
 /**
  * Assertion mapping for knowledge probe evaluations.
  */
+import { resolveTemplatedAssertion } from "../../rubric-resolution.js";
 /**
  * Map a raw knowledge probe assertion to a Promptfoo assertion.
  *
  * Tool-use assertions are rejected (knowledge probes don't use tools).
  * LLM-graded assertions receive the configured grader provider.
  * All other assertions are passed through.
+ *
+ * Templated `llm-rubric` assertions (those with `template` + `criteria`) go
+ * through the shared rubric resolver so the compiled assertion carries
+ * `metadata.dimension` — without this, the scoring engine can't classify
+ * KP grader output and dimension scores collapse to zero (W0128 / DOC-2077).
  */
 export function mapKnowledgeProbeAssertion(assertion, options, warnings) {
     switch (assertion.type) {
@@ -27,9 +33,26 @@ export function mapKnowledgeProbeAssertion(assertion, options, warnings) {
                     ? { weight: assertion.weight }
                     : {}),
             };
-        // LLM-graded assertions — add grader provider
-        case "g-eval":
         case "llm-rubric":
+            // Templated form (template + criteria) → resolve to full rubric text
+            // with dimension metadata attached.
+            if ("template" in assertion && "criteria" in assertion) {
+                return resolveTemplatedAssertion(assertion, options?.rubricConfig, options?.graderProvider, warnings);
+            }
+            // Inline value form — pass through with grader provider, no metadata.
+            // Back-compat for tasks not yet migrated to the templated form.
+            return {
+                type: "llm-rubric",
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+                ...(options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            };
+        // Other LLM-graded assertions — add grader provider
+        case "g-eval":
         case "model-graded-closedqa":
         case "model-graded-factuality":
             return {

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.js

@@ -37,7 +37,11 @@ export const handler = {
         if (!("mode" in task) || task.mode !== "knowledge-probe") {
             throw new Error(`Knowledge probe handler received task with mode "${task.mode ?? "undefined"}" — expected "knowledge-probe"`);
         }
-        const result = compileKnowledgeProbeTask(task, { graderProvider: ctx.graderProvider, models: ctx.models });
+        const result = compileKnowledgeProbeTask(task, {
+            graderProvider: ctx.graderProvider,
+            models: ctx.models,
+            rubricConfig: ctx.rubricConfig,
+        });
         return {
             providers: result.providers,
             tests: result.tests,

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts

@@ -2,6 +2,7 @@
  * Public types for the knowledge-probe mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 /** Options for compiling a knowledge probe task */
 export interface KnowledgeProbeCompileOptions {
     /** Grader provider for LLM-graded assertions */
@@ -12,6 +13,9 @@ export interface KnowledgeProbeCompileOptions {
         label: string;
         config?: Record<string, unknown>;
     }[];
+    /** Rubric config (templates, weights, profiles) — needed to resolve
+     * templated `llm-rubric` assertions to dimension metadata. */
+    rubricConfig?: RubricConfig;
 }
 /** Result of compiling a single knowledge probe task */
 export interface KnowledgeProbeCompileResult {