@sanity/ailf 2.2.0 → 2.3.1

package/dist/pipeline/calculate-scores.js

@@ -546,11 +546,13 @@ function scoreResults(results, goldProfile, baselineProfile, modelId) {
         const featureScore = {
             ceilingScore,
             codeCorrectness: gold.dimensions.codeCorrectness ?? 0,
+            dimensions: gold.dimensions,
             docCoverage: gold.dimensions.docCoverage ?? 0,
             docLift,
             docQualityGap: 100 - ceilingScore,
             feature,
             floorScore,
+            groupType: "feature",
             ...(modelId && { modelId }),
             negativeDocLift: docLift < 0,
             taskCompletion: gold.dimensions.taskCompletion ?? 0,
@@ -563,6 +565,69 @@ function scoreResults(results, goldProfile, baselineProfile, modelId) {
     return scores.sort((a, b) => a.feature.localeCompare(b.feature));
 }
 // ---------------------------------------------------------------------------
+// Agent-harness scoring — groups by task ID, single variant
+// ---------------------------------------------------------------------------
+/**
+ * Score agent-harness evaluation results. Unlike literacy mode, agent-harness
+ * tasks don't have a with-docs/without-docs split. All results are scored
+ * as a single "actual" variant using the agent-harness profile.
+ *
+ * Groups results by task ID (extracted from the test description prefix)
+ * rather than by feature area. Each group produces a FeatureScore with
+ * groupType: "task".
+ *
+ * Literacy-specific fields (ceilingScore, floorScore, docLift, docQualityGap)
+ * are set to 0 for backward compatibility with downstream consumers.
+ */
+function scoreAgentHarnessResults(results, profile) {
+    // Group by task ID (extracted from description: "task-id — Title")
+    const byTask = {};
+    for (const result of results) {
+        const taskId = extractTaskId(result.description);
+        if (!byTask[taskId]) {
+            byTask[taskId] = [];
+        }
+        byTask[taskId].push(result);
+    }
+    const scores = [];
+    for (const [taskId, taskResults] of Object.entries(byTask)) {
+        const scored = scoreTestGroup(taskResults, profile, taskId);
+        const totalCost = scored.totalCost;
+        // Detect feature area for backward compat (used by report grouping)
+        const feature = taskResults[0]?.vars.__featureArea ??
+            detectFeatureArea(taskResults[0]?.description ?? taskId);
+        scores.push({
+            assertionPassRate: scored.dimensions.assertionPassRate,
+            ceilingScore: 0,
+            codeCorrectness: scored.dimensions.codeCorrectness ?? 0,
+            dimensions: scored.dimensions,
+            docCoverage: scored.dimensions.docCoverage ?? 0,
+            docLift: 0,
+            docQualityGap: 0,
+            feature,
+            floorScore: 0,
+            groupType: "task",
+            negativeDocLift: false,
+            taskCompletion: scored.dimensions.taskCompletion ?? 0,
+            testCount: taskResults.length,
+            totalCost,
+            totalScore: scored.composite,
+        });
+    }
+    return scores.sort((a, b) => a.feature.localeCompare(b.feature));
+}
+/**
+ * Extract task ID from a test description string.
+ * Descriptions follow the pattern: "task-id — Title"
+ */
+function extractTaskId(description) {
+    const dashIndex = description.indexOf(" — ");
+    if (dashIndex > 0) {
+        return description.slice(0, dashIndex).trim();
+    }
+    return description.trim() || "unknown";
+}
+// ---------------------------------------------------------------------------
 // Agentic scoring — all results are "actual" (agent retrieves docs via tools)
 // ---------------------------------------------------------------------------
 /**
@@ -684,11 +749,57 @@ export function calculateAndWriteScores(options) {
     if (source) {
         log.info(`Source: ${sourceName} (${source.baseUrl})`);
     }
-    // Load rubric config and resolve scoring profiles per variant.
+    // Load rubric config — shared across all modes
+    const rubricConfig = loadRubricTemplates(ROOT);
+    // ── Agent-harness scoring path ──────────────────────────────
+    // Agent-harness mode uses its own scoring path because:
+    // 1. No with-docs/without-docs split — all results are "actual"
+    // 2. Groups by task ID, not feature area
+    // 3. Uses the agent-harness profile (assertion-pass-rate, agent-output, tool-usage)
+    // See docs/design-docs/mode-agnostic-scoring.md
+    if (mode === "agent-harness") {
+        const agentProfile = resolveProfile("agent-harness", "gold", rubricConfig);
+        log.debug("Agent-harness scoring profile", agentProfile);
+        const results = readAndNormalizeResults(baselineResultsPath);
+        const scores = scoreAgentHarnessResults(results, agentProfile);
+        log.debug("Agent-harness scores calculated", {
+            taskCount: scores.length,
+            tasks: scores.map((s) => ({
+                feature: s.feature,
+                totalScore: s.totalScore,
+                testCount: s.testCount,
+                dimensions: s.dimensions,
+            })),
+        });
+        const urlRefs = aggregateUrlReferences(baselineResultsPath);
+        const sourceVerification = buildSourceVerification(ROOT, source, {
+            allowedOrigins: options.allowedOrigins,
+            mode,
+            searchMode: options.searchMode,
+        });
+        const graderCost = extractGraderCost(baselineResultsPath);
+        const summary = printReport(scores, urlRefs, source, null, // no agent behavior (that's for literacy agentic mode)
+        graderCost, null, // no per-model breakdown
+        null, // no source isolation
+        sourceVerification, "agent-harness", log);
+        // Persist
+        const outDir = join(ROOT, "results", "latest");
+        mkdirSync(outDir, { recursive: true });
+        writeFileSync(join(outDir, "score-summary.json"), JSON.stringify(summary, null, 2));
+        log.info("Score summary written to results/latest/score-summary.json");
+        // Extract and persist grader judgments
+        const judgments = extractGraderJudgments(baselineResultsPath);
+        if (judgments.length > 0) {
+            writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
+            log.info(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
+        }
+        const testSummary = computeTestSummary(baselineResultsPath);
+        return { belowCritical: summary.belowCritical, testSummary };
+    }
+    // ── Literacy scoring path ───────────────────────────────────
     // Gold (with-docs) entries use the "default" profile (3 dimensions).
     // Baseline (without-docs) entries use "output-only" (2 dimensions,
     // doc-coverage excluded). See docs/design-docs/named-scoring-profiles.md.
-    const rubricConfig = loadRubricTemplates(ROOT);
     const goldProfile = resolveProfile("literacy", "gold", rubricConfig, LiteracyVariant.STANDARD);
     const baselineProfileWeights = resolveProfile("literacy", LiteracyVariant.STANDARD, rubricConfig, LiteracyVariant.STANDARD);
     log.debug("Loaded scoring profiles", {

package/dist/pipeline/compare.js

@@ -146,12 +146,6 @@ function buildAreaDelta(area, baselineScore, experimentScore, threshold, isMisma
     const bTotal = b?.totalScore ?? 0;
     const eTotal = e?.totalScore ?? 0;
     const delta = eTotal - bTotal;
-    const bTask = b?.taskCompletion ?? 0;
-    const eTask = e?.taskCompletion ?? 0;
-    const bCode = b?.codeCorrectness ?? 0;
-    const eCode = e?.codeCorrectness ?? 0;
-    const bDoc = b?.docCoverage ?? 0;
-    const eDoc = e?.docCoverage ?? 0;
     // Support both new field names and legacy data (old baselines/Sanity docs)
     const bRaw = b;
     const eRaw = e;
@@ -183,19 +177,7 @@ function buildAreaDelta(area, baselineScore, experimentScore, threshold, isMisma
         ceilingDelta: eCeiling - bCeiling,
         change: isMismatched ? "not-evaluated" : classifyChange(delta, threshold),
         delta,
-        dimensions: {
-            codeCorrectness: {
-                baseline: bCode,
-                delta: eCode - bCode,
-                experiment: eCode,
-            },
-            docCoverage: { baseline: bDoc, delta: eDoc - bDoc, experiment: eDoc },
-            taskCompletion: {
-                baseline: bTask,
-                delta: eTask - bTask,
-                experiment: eTask,
-            },
-        },
+        dimensions: buildDimensionDeltas(b, e),
         docLiftDelta: eLift - bLift,
         experiment: eTotal,
         floorDelta: eFloor - bFloor,
@@ -206,6 +188,55 @@ function buildAreaDelta(area, baselineScore, experimentScore, threshold, isMisma
         ...(hasCost && { costDelta: eCost - bCost }),
     };
 }
+/**
+ * Build per-dimension deltas from the generic dimensions map when available,
+ * falling back to the three legacy literacy fields for backward compatibility.
+ *
+ * This ensures non-literacy modes (agent-harness, mcp-server, etc.) get their
+ * actual dimensions (e.g., agentOutput, toolUsage) in comparison reports
+ * instead of hardcoded zeros for codeCorrectness/docCoverage/taskCompletion.
+ */
+function buildDimensionDeltas(b, e) {
+    const bDims = b?.dimensions;
+    const eDims = e?.dimensions;
+    // When the generic dimensions map is populated, use it — this covers
+    // agent-harness (agentOutput, toolUsage), literacy (taskCompletion,
+    // codeCorrectness, docCoverage), and any future mode dimensions.
+    if (bDims || eDims) {
+        const allKeys = new Set([
+            ...Object.keys(bDims ?? {}),
+            ...Object.keys(eDims ?? {}),
+        ]);
+        const result = {};
+        for (const key of allKeys) {
+            const bVal = bDims?.[key] ?? 0;
+            const eVal = eDims?.[key] ?? 0;
+            result[key] = { baseline: bVal, delta: eVal - bVal, experiment: eVal };
+        }
+        return result;
+    }
+    // Legacy fallback: older reports may lack the dimensions map entirely.
+    // Read from the three named FeatureScore fields instead.
+    const bTask = b?.taskCompletion ?? 0;
+    const eTask = e?.taskCompletion ?? 0;
+    const bCode = b?.codeCorrectness ?? 0;
+    const eCode = e?.codeCorrectness ?? 0;
+    const bDoc = b?.docCoverage ?? 0;
+    const eDoc = e?.docCoverage ?? 0;
+    return {
+        codeCorrectness: {
+            baseline: bCode,
+            delta: eCode - bCode,
+            experiment: eCode,
+        },
+        docCoverage: { baseline: bDoc, delta: eDoc - bDoc, experiment: eDoc },
+        taskCompletion: {
+            baseline: bTask,
+            delta: eTask - bTask,
+            experiment: eTask,
+        },
+    };
+}
 // ---------------------------------------------------------------------------
 // Main compare function
 // ---------------------------------------------------------------------------

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js

@@ -232,6 +232,70 @@ describe("compileAgentHarnessTask — assertions", () => {
         }), { graderProvider: "openai:chat:gpt-5" });
         assert.equal(result.tests[0].assert?.[0]?.provider, "openai:chat:gpt-5");
     });
+    it("resolves templated llm-rubric with rubric text and dimension metadata", () => {
+        const rubricConfig = {
+            templates: {
+                "agent-output": {
+                    dimension: "agent-output",
+                    header: "Score the agent's final output from 0 to 100:",
+                    scale: ["0: Failed", "50: Partial", "100: Complete"],
+                    criteria_label: "Check for:",
+                },
+            },
+        };
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    template: "agent-output",
+                    criteria: ["File created", "Correct content"],
+                },
+            ],
+        }), { rubricConfig, graderProvider: "anthropic:messages:claude-opus-4-5" });
+        const assertion = result.tests[0].assert?.[0];
+        assert.ok(assertion, "should produce an assertion");
+        assert.equal(assertion.type, "llm-rubric");
+        // Rubric text should be fully rendered (not empty)
+        assert.ok(assertion.value.includes("Score the agent"), "should contain rendered rubric header");
+        assert.ok(assertion.value.includes("File created"), "should contain task-specific criteria");
+        // Dimension metadata should be attached
+        const metadata = assertion.metadata;
+        assert.ok(metadata, "should have metadata");
+        assert.equal(metadata.dimension, "agent-output");
+        assert.equal(metadata.maxScore, 100);
+        // Grader provider should be set
+        assert.equal(assertion.provider, "anthropic:messages:claude-opus-4-5");
+    });
+    it("warns when rubric template is unknown", () => {
+        const rubricConfig = { templates: {} };
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    template: "nonexistent-template",
+                    criteria: ["Something"],
+                },
+            ],
+        }), { rubricConfig });
+        // Unknown template produces a warning and no assertion
+        assert.ok(result.warnings.some((w) => w.includes("nonexistent-template")), "should warn about unknown template");
+        // The assertion should be null (filtered out)
+        assert.equal(result.tests[0].assert?.length ?? 0, 0, "should not produce an assertion for unknown template");
+    });
+    it("warns when rubricConfig is not provided for templated assertion", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    template: "agent-output",
+                    criteria: ["Something"],
+                },
+            ],
+        })
+        // No rubricConfig in options
+        );
+        assert.ok(result.warnings.some((w) => w.includes("No rubric config")), "should warn about missing rubric config");
+    });
 });
 // ---------------------------------------------------------------------------
 // compileAgentHarnessTask — lifecycle extensions

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts

@@ -5,6 +5,12 @@
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
  *
+ * Templated LLM-rubric assertions (those with `template` + `criteria`)
+ * are resolved via the shared rubric-resolution module, producing fully
+ * assembled rubric text and dimension metadata. This is critical for
+ * scoring — without it, the grader receives empty rubrics and the
+ * scoring pipeline has no dimension data to work with (DOC-2029).
+ *
  * Agent-specific assertions use file-based references to the assertions
  * runtime module (dist/agent-harness/assertions-runtime.js) because
  * promptfoo's inline `type: javascript` assertions run in a restricted

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js

@@ -5,6 +5,12 @@
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
  *
+ * Templated LLM-rubric assertions (those with `template` + `criteria`)
+ * are resolved via the shared rubric-resolution module, producing fully
+ * assembled rubric text and dimension metadata. This is critical for
+ * scoring — without it, the grader receives empty rubrics and the
+ * scoring pipeline has no dimension data to work with (DOC-2029).
+ *
  * Agent-specific assertions use file-based references to the assertions
  * runtime module (dist/agent-harness/assertions-runtime.js) because
  * promptfoo's inline `type: javascript` assertions run in a restricted
@@ -14,6 +20,7 @@
  * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
  * @see src/agent-harness/assertions-runtime.ts — runtime implementations
  */
+import { resolveTemplatedAssertion } from "../../rubric-resolution.js";
 /** Base path for the file-based assertion runtime module */
 const RUNTIME = "file://dist/agent-harness/assertions-runtime.js";
 // ---------------------------------------------------------------------------
@@ -44,6 +51,13 @@ export function mapAgentAssertion(assertion, options, warnings) {
                     : {}),
             };
         case "llm-rubric":
+            // Templated assertions (template + criteria) need full resolution
+            // to produce rubric text and dimension metadata for scoring.
+            if ("template" in assertion && "criteria" in assertion) {
+                const resolved = resolveTemplatedAssertion(assertion, options?.rubricConfig, options?.graderProvider, warnings);
+                return resolved;
+            }
+            // Non-templated llm-rubric (inline value) — pass through
             return {
                 type: "llm-rubric",
                 ...("value" in assertion ? { value: assertion.value } : {}),

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js

@@ -28,6 +28,7 @@ export const handler = {
         const result = compileAgentHarnessTask(task, {
             graderProvider: ctx.graderProvider,
             rootDir: ctx.rootDir,
+            rubricConfig: ctx.rubricConfig,
         });
         return {
             providers: result.providers,

package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts

@@ -2,6 +2,7 @@
  * Shared types for the agent harness mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 import type { SandboxType } from "../../sandbox/sandbox-strategy.js";
 /** Options for compiling an agent harness task */
 export interface AgentHarnessCompileOptions {
@@ -9,6 +10,8 @@ export interface AgentHarnessCompileOptions {
     graderProvider?: string;
     /** Root directory for fixture resolution */
     rootDir?: string;
+    /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
+    rubricConfig?: RubricConfig;
 }
 /** Result of compiling a single agent harness task */
 export interface AgentHarnessCompileResult {

package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js

@@ -4,6 +4,7 @@
  * Handles rubric template resolution, doc-coverage auto-generation,
  * and baseline assertion filtering.
  */
+import { resolveTemplatedAssertion } from "../../rubric-resolution.js";
 // ---------------------------------------------------------------------------
 // Assertion resolution
 // ---------------------------------------------------------------------------
@@ -37,33 +38,6 @@ export function resolveAssertions(task, options, warnings) {
     return assertions;
 }
 // ---------------------------------------------------------------------------
-// Rubric template resolution
-// ---------------------------------------------------------------------------
-function resolveTemplatedAssertion(a, rubricConfig, graderProvider, warnings) {
-    if (!rubricConfig) {
-        warnings.push(`No rubric config — template "${a.template}" cannot be resolved`);
-        return null;
-    }
-    const template = rubricConfig.templates[a.template];
-    if (!template) {
-        warnings.push(`Unknown rubric template: "${a.template}"`);
-        return null;
-    }
-    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
-    const criteriaText = a.criteria.map((c) => `- ${c}`).join("\n");
-    const rubricValue = `${template.header}\n${scaleText}\n\n` +
-        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
-        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
-    return {
-        type: "llm-rubric",
-        value: rubricValue,
-        ...(graderProvider ? { provider: graderProvider } : {}),
-        ...(template.dimension
-            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
-            : {}),
-    };
-}
-// ---------------------------------------------------------------------------
 // Doc-coverage assertion
 // ---------------------------------------------------------------------------
 function buildDocCoverageAssertion(rubricConfig, graderProvider) {

package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts

@@ -2,6 +2,8 @@
  * Shared types for the literacy mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+export type { RubricConfig } from "../../rubric-resolution.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 /** Options for compiling a literacy task */
 export interface LiteracyCompileOptions {
     /** Grader provider for LLM-graded assertions */
@@ -19,15 +21,6 @@ export interface LiteracyCompileOptions {
     /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
     rubricConfig?: RubricConfig;
 }
-/** Minimal rubric config needed by the handler */
-export interface RubricConfig {
-    templates: Record<string, {
-        dimension?: string;
-        header: string;
-        scale: string[];
-        criteria_label?: string;
-    }>;
-}
 /** Result of compiling a single literacy task */
 export interface LiteracyCompileResult {
     /** Promptfoo provider configs */

package/dist/pipeline/compiler/rubric-resolution.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Shared rubric template resolution for all evaluation modes.
+ *
+ * Resolves templated LLM-rubric assertions (those with `template` + `criteria`
+ * fields) into fully assembled Promptfoo assertions with rubric text and
+ * dimension metadata.
+ *
+ * Used by both literacy and agent-harness compilers. Extracted from
+ * literacy/assertions.ts to fix the compilation bug where agent-harness
+ * tasks with templated rubrics produced empty rubric text (DOC-2029).
+ *
+ * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see config/rubrics.ts — template definitions
+ */
+import type { PromptfooAssertion } from "./assertion-mapper.js";
+/** Minimal rubric config needed for template resolution */
+export interface RubricConfig {
+    templates: Record<string, {
+        criteria_label?: string;
+        dimension?: string;
+        header: string;
+        scale: string[];
+    }>;
+}
+/**
+ * Resolve a templated LLM-rubric assertion into a fully assembled
+ * Promptfoo assertion with rubric text and dimension metadata.
+ *
+ * A "templated" assertion has `template` (referencing a key in rubrics.ts)
+ * and `criteria` (task-specific bullet points). The template provides the
+ * scoring header, scale, and dimension metadata. The criteria are appended
+ * to create the final rubric prompt.
+ *
+ * Returns null (with a warning) if the template can't be resolved.
+ */
+export declare function resolveTemplatedAssertion(assertion: {
+    criteria: string[];
+    template: string;
+    type: string;
+}, rubricConfig: RubricConfig | undefined, graderProvider: string | undefined, warnings: string[]): PromptfooAssertion | null;

package/dist/pipeline/compiler/rubric-resolution.js

@@ -0,0 +1,52 @@
+/**
+ * Shared rubric template resolution for all evaluation modes.
+ *
+ * Resolves templated LLM-rubric assertions (those with `template` + `criteria`
+ * fields) into fully assembled Promptfoo assertions with rubric text and
+ * dimension metadata.
+ *
+ * Used by both literacy and agent-harness compilers. Extracted from
+ * literacy/assertions.ts to fix the compilation bug where agent-harness
+ * tasks with templated rubrics produced empty rubric text (DOC-2029).
+ *
+ * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see config/rubrics.ts — template definitions
+ */
+// ---------------------------------------------------------------------------
+// Template resolution
+// ---------------------------------------------------------------------------
+/**
+ * Resolve a templated LLM-rubric assertion into a fully assembled
+ * Promptfoo assertion with rubric text and dimension metadata.
+ *
+ * A "templated" assertion has `template` (referencing a key in rubrics.ts)
+ * and `criteria` (task-specific bullet points). The template provides the
+ * scoring header, scale, and dimension metadata. The criteria are appended
+ * to create the final rubric prompt.
+ *
+ * Returns null (with a warning) if the template can't be resolved.
+ */
+export function resolveTemplatedAssertion(assertion, rubricConfig, graderProvider, warnings) {
+    if (!rubricConfig) {
+        warnings.push(`No rubric config — template "${assertion.template}" cannot be resolved`);
+        return null;
+    }
+    const template = rubricConfig.templates[assertion.template];
+    if (!template) {
+        warnings.push(`Unknown rubric template: "${assertion.template}"`);
+        return null;
+    }
+    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
+    const criteriaText = assertion.criteria.map((c) => `- ${c}`).join("\n");
+    const rubricValue = `${template.header}\n${scaleText}\n\n` +
+        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
+        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
+    return {
+        type: "llm-rubric",
+        value: rubricValue,
+        ...(graderProvider ? { provider: graderProvider } : {}),
+        ...(template.dimension
+            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
+            : {}),
+    };
+}

package/dist/pipeline/compiler/scoring-bridge.js

@@ -41,19 +41,25 @@ import { classifyRubric, parseRubricScore } from "../../_vendor/ailf-core/index.
 export function scoreTestGroup(tests, profile, taskId) {
     let totalCost = 0;
     // Step 1: Convert all ComponentResults into AssertionScore[] (0–1 scale)
+    //
+    // Two assertion types contribute to scoring:
+    //   - llm-rubric: dimension from metadata, score from grader (0–100 → [0,1])
+    //   - javascript: mapped to "assertion-pass-rate" dimension (pass=1, fail=0)
+    //
+    // Other types (cost, trajectory, contains, etc.) are metadata or guards —
+    // they don't produce dimension scores.
     const assertionScores = [];
     for (const test of tests) {
         totalCost += test.cost;
         for (const comp of test.gradingResult.componentResults) {
-            if (comp.assertion?.type !== "llm-rubric")
-                continue;
-            const converted = componentToAssertionScore(comp);
+            const converted = componentToScore(comp);
             if (converted)
                 assertionScores.push(converted);
         }
     }
     // Step 2: Aggregate into DimensionScores (0–1 scale)
     const dimensionLabels = {
+        "assertion-pass-rate": "Assertion Pass Rate",
         "code-correctness": "Code Correctness",
         "doc-coverage": "Doc Coverage",
         "task-completion": "Task Completion",
@@ -86,12 +92,34 @@ export function scoreTestGroup(tests, profile, taskId) {
 // Conversion helpers
 // ---------------------------------------------------------------------------
 /**
- * Convert a single Promptfoo ComponentResult into the scoring engine's
- * AssertionScore format.
+ * Route a ComponentResult to the appropriate scoring conversion.
  *
- * Returns null if the component doesn't map to a known dimension.
+ * Dispatches by assertion type:
+ *   - llm-rubric  → dimension from metadata, grader score (0–100 → [0,1])
+ *   - javascript   → "assertion-pass-rate" dimension, binary (pass=1, fail=0)
+ *   - everything else → null (not a scoring-relevant assertion)
+ *
+ * This replaces the previous llm-rubric-only filter that caused agent-harness
+ * javascript assertions to be invisible to the scoring engine (DOC-2029).
+ */
+function componentToScore(comp) {
+    const type = comp.assertion?.type;
+    if (type === "llm-rubric") {
+        return llmRubricToScore(comp);
+    }
+    if (type === "javascript") {
+        return javascriptAssertionToScore(comp);
+    }
+    // Other types (cost, trajectory, contains, etc.) don't produce scores
+    return null;
+}
+/**
+ * Convert an LLM-rubric ComponentResult into an AssertionScore.
+ *
+ * The dimension comes from metadata (set during rubric template resolution).
+ * Returns null if the component doesn't map to any dimension.
  */
-function componentToAssertionScore(comp) {
+function llmRubricToScore(comp) {
     const dim = classifyRubric(comp);
     if (!dim)
         return null;
@@ -108,6 +136,30 @@ function componentToAssertionScore(comp) {
         weight: 1.0,
     };
 }
+/**
+ * Convert a javascript assertion ComponentResult into an AssertionScore.
+ *
+ * Javascript assertions (fileExists, fileContains, commandSucceeds, etc.)
+ * produce binary pass/fail results. They map to the "assertion-pass-rate"
+ * dimension — the fraction of structural assertions that passed.
+ *
+ * Zero-weight assertions (like URL extraction) are excluded from scoring.
+ */
+function javascriptAssertionToScore(comp) {
+    // Skip zero-weight assertions (diagnostic-only, e.g., URL extraction)
+    const weight = comp.assertion?.weight;
+    if (weight === 0)
+        return null;
+    return {
+        assertionType: "javascript",
+        dimension: "assertion-pass-rate",
+        latencyMs: 0,
+        pass: comp.pass,
+        reason: comp.reason ?? "",
+        score: comp.pass ? 1.0 : 0.0,
+        weight: 1.0,
+    };
+}
 /** Convert kebab-case dimension key to camelCase (e.g., "task-completion" → "taskCompletion") */
 function kebabToCamel(kebab) {
     return kebab.replace(/-([a-z])/g, (_, c) => c.toUpperCase());

package/dist/pipeline/provenance.js

@@ -59,6 +59,12 @@ export function buildProvenance(input) {
         evalFingerprint: input.evalFingerprint,
         hasLineage: Boolean(lineage),
     });
+    // Non-literacy modes (agent-harness, mcp-server, etc.) don't use the
+    // config/models.ts model matrix — listing those models would be misleading.
+    // Only include them for literacy mode where they're the actual eval targets.
+    const evaluatedModels = input.mode === "literacy"
+        ? models.models.map((m) => ({ id: m.id, label: m.label }))
+        : [];
     return {
         areas: input.areas,
         autoScope: input.autoScope,
@@ -68,7 +74,7 @@ export function buildProvenance(input) {
         graderModel: models.grader.id,
         lineage,
         mode: input.mode,
-        models: models.models.map((m) => ({ id: m.id, label: m.label })),
+        models: evaluatedModels,
         promptfooUrl: input.promptfooUrl,
         promptfooUrls: input.promptfooUrls,
         source: {