@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/calculate-scores.js

@@ -9,7 +9,7 @@
  *   Doc Coverage       (0–100)  — Did docs provide the needed info?
  *
  * Dimensions are combined into a weighted composite (0–100) using named
- * scoring profiles from config/rubrics.yaml. Gold (with-docs) entries use
+ * scoring profiles from config/rubrics. Gold (with-docs) entries use
  * the "default" profile; baseline (without-docs) entries use "output-only"
  * which excludes doc-coverage (undefined without docs).
  * See docs/design-docs/named-scoring-profiles.md.
@@ -29,15 +29,17 @@
  */
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
+import { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
 import { calculateCost } from "../agent-observer/pricing.js";
 import { ConsoleLogger } from "../adapters/loggers/index.js";
+import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
 import { checkResultsExist } from "./checks.js";
-import { loadRubricTemplates } from "./expand-tasks.js";
+import { loadRubricTemplates } from "./rubric-loader.js";
 import { resolveProfile } from "./profile-resolution.js";
 import { loadSource } from "../sources.js";
-import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
-import { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
-// Re-export pure functions from core for backward compatibility.
+import { LiteracyVariant } from "./normalize-mode.js";
+import { scoreTestGroup } from "./compiler/scoring-bridge.js";
+// Re-export from core for backward compatibility.
 // Existing imports from this file continue to work unchanged.
 export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
 /**
@@ -137,14 +139,8 @@ export function extractGraderJudgments(resultsPath) {
                     // Not JSON — use raw reason string
                 }
             }
-            // Map internal dimension names to hyphenated form
-            const dimensionMap = {
-                codeCorrectness: "code-correctness",
-                docCoverage: "doc-coverage",
-                taskCompletion: "task-completion",
-            };
             judgments.push({
-                dimension: dimensionMap[kind] ?? kind,
+                dimension: kind,
                 modelId,
                 reason,
                 score,
@@ -281,7 +277,7 @@ function aggregateUrlReferences(resultsPath) {
  * verification report.
  */
 function buildSourceVerification(root, source, verificationCtx) {
-    const mode = verificationCtx?.mode ?? "baseline";
+    const mode = verificationCtx?.mode ?? LiteracyVariant.STANDARD;
     const sourceUrl = source?.baseUrl ?? "default";
     const searchMode = verificationCtx?.searchMode;
     const allowedOrigins = verificationCtx?.allowedOrigins;
@@ -493,62 +489,6 @@ function readAndNormalizeResults(resultsPath, log) {
     }
     return valid;
 }
-/**
- * Accumulate raw dimension scores across an array of test results.
- * Dimension-agnostic: any dimension returned by classifyRubric() is tracked.
- */
-function accumulateDimensions(tests) {
-    const dimensions = {};
-    let totalCost = 0;
-    for (const test of tests) {
-        totalCost += test.cost;
-        for (const comp of test.gradingResult.componentResults) {
-            if (comp.assertion?.type !== "llm-rubric")
-                continue;
-            const score = parseRubricScore(comp);
-            const kind = classifyRubric(comp);
-            if (kind) {
-                dimensions[kind] = (dimensions[kind] ?? 0) + score;
-            }
-        }
-    }
-    return { dimensions, totalCost };
-}
-/**
- * Average accumulated dimension scores by a count.
- * Returns a dimension → average score map.
- */
-function averageDimensions(accumulated, count) {
-    const avg = {};
-    for (const [dim, total] of Object.entries(accumulated.dimensions)) {
-        avg[dim] = total / count;
-    }
-    return avg;
-}
-/**
- * Compute a weighted composite score from dimension averages and a profile.
- * Only dimensions present in the profile contribute to the composite.
- * Dimensions not in the profile are ignored (e.g., doc-coverage on baseline).
- *
- * The profile maps camelCase dimension names (as returned by classifyRubric)
- * to kebab-case keys (as used in rubrics.yaml). This function handles the
- * mapping internally.
- */
-function weightedComposite(dimensionAverages, profile) {
-    // Map profile keys (kebab-case: "task-completion") to classifyRubric
-    // output (camelCase: "taskCompletion")
-    const kebabToCamel = {
-        "code-correctness": "codeCorrectness",
-        "doc-coverage": "docCoverage",
-        "task-completion": "taskCompletion",
-    };
-    let total = 0;
-    for (const [profileKey, weight] of Object.entries(profile)) {
-        const dimKey = kebabToCamel[profileKey] ?? profileKey;
-        total += (dimensionAverages[dimKey] ?? 0) * weight;
-    }
-    return total;
-}
 /**
  * Core scoring logic: takes a pre-filtered array of TestResult and produces
  * FeatureScore[] grouped by feature area. This is the shared implementation
@@ -577,35 +517,28 @@ function scoreResults(results, goldProfile, baselineProfile, modelId) {
     }
     const scores = [];
     for (const [feature, data] of Object.entries(byFeature)) {
-        // --- With docs (gold / ceiling) ---
-        const goldDims = accumulateDimensions(data.withDocs);
-        let featureCost = goldDims.totalCost;
-        const countWithDocs = data.withDocs.length || 1;
-        const avgGold = averageDimensions(goldDims, countWithDocs);
-        const withDocsTotal = weightedComposite(avgGold, goldProfile);
+        // --- With docs (gold / ceiling) — scored via 4-tier engine ---
+        const gold = scoreTestGroup(data.withDocs, goldProfile, feature);
         // --- Without docs (baseline / floor) ---
         // Uses the baseline profile (e.g. "output-only") which may exclude
         // dimensions like doc-coverage that are undefined without docs.
         // See docs/design-docs/named-scoring-profiles.md.
-        const baselineDims = accumulateDimensions(data.withoutDocs);
-        featureCost += baselineDims.totalCost;
-        const countWithoutDocs = data.withoutDocs.length || 1;
-        const avgBaseline = averageDimensions(baselineDims, countWithoutDocs);
-        const withoutDocsScore = weightedComposite(avgBaseline, baselineProfile);
-        const ceilingScore = Math.round(withDocsTotal);
-        const floorScore = Math.round(withoutDocsScore);
+        const baseline = scoreTestGroup(data.withoutDocs, baselineProfile, feature);
+        const featureCost = gold.totalCost + baseline.totalCost;
+        const ceilingScore = gold.composite;
+        const floorScore = baseline.composite;
         const docLift = ceilingScore - floorScore;
         const featureScore = {
             ceilingScore,
-            codeCorrectness: Math.round(avgGold.codeCorrectness ?? 0),
-            docCoverage: Math.round(avgGold.docCoverage ?? 0),
+            codeCorrectness: gold.dimensions.codeCorrectness ?? 0,
+            docCoverage: gold.dimensions.docCoverage ?? 0,
             docLift,
             docQualityGap: 100 - ceilingScore,
             feature,
             floorScore,
             ...(modelId && { modelId }),
             negativeDocLift: docLift < 0,
-            taskCompletion: Math.round(avgGold.taskCompletion ?? 0),
+            taskCompletion: gold.dimensions.taskCompletion ?? 0,
             testCount: data.withDocs.length,
             totalCost: featureCost,
             totalScore: ceilingScore,
@@ -638,17 +571,14 @@ export function scoreAgenticResults(resultsPath, profile) {
     }
     const entries = {};
     for (const [feature, featureResults] of Object.entries(byFeature)) {
-        const count = featureResults.length || 1;
-        const accumulated = accumulateDimensions(featureResults);
-        const avg = averageDimensions(accumulated, count);
-        const actualScore = Math.round(weightedComposite(avg, profile));
+        const scored = scoreTestGroup(featureResults, profile, feature);
         entries[feature] = {
-            actualScore,
-            codeCorrectness: Math.round(avg.codeCorrectness ?? 0),
-            docCoverage: Math.round(avg.docCoverage ?? 0),
-            taskCompletion: Math.round(avg.taskCompletion ?? 0),
+            actualScore: scored.composite,
+            codeCorrectness: scored.dimensions.codeCorrectness ?? 0,
+            docCoverage: scored.dimensions.docCoverage ?? 0,
+            taskCompletion: scored.dimensions.taskCompletion ?? 0,
             testCount: featureResults.length,
-            totalCost: accumulated.totalCost,
+            totalCost: scored.totalCost,
         };
     }
     return entries;
@@ -677,17 +607,14 @@ export function scoreAgenticResultsPerModel(resultsPath, profile) {
     for (const [modelId, features] of Object.entries(byModel)) {
         perModel[modelId] = {};
         for (const [feature, featureResults] of Object.entries(features)) {
-            const count = featureResults.length || 1;
-            const accumulated = accumulateDimensions(featureResults);
-            const avg = averageDimensions(accumulated, count);
-            const actualScore = Math.round(weightedComposite(avg, profile));
+            const scored = scoreTestGroup(featureResults, profile, feature);
             perModel[modelId][feature] = {
-                actualScore,
-                codeCorrectness: Math.round(avg.codeCorrectness ?? 0),
-                docCoverage: Math.round(avg.docCoverage ?? 0),
-                taskCompletion: Math.round(avg.taskCompletion ?? 0),
+                actualScore: scored.composite,
+                codeCorrectness: scored.dimensions.codeCorrectness ?? 0,
+                docCoverage: scored.dimensions.docCoverage ?? 0,
+                taskCompletion: scored.dimensions.taskCompletion ?? 0,
                 testCount: featureResults.length,
-                totalCost: accumulated.totalCost,
+                totalCost: scored.totalCost,
             };
         }
     }
@@ -725,7 +652,7 @@ export function calculateAndWriteScores(options) {
         }
     }
     // Determine mode — controls which result files are read
-    const mode = options.mode ?? "baseline";
+    const mode = options.mode ?? LiteracyVariant.STANDARD;
     const baselineResultsPath = options.resultsPath ?? join(ROOT, "results", "latest", "eval-results.json");
     // Agentic results path (only used in full mode)
     const agenticResultsPath = join(ROOT, "results", "latest", "eval-results-agentic.json");
@@ -747,8 +674,8 @@ export function calculateAndWriteScores(options) {
     // Baseline (without-docs) entries use "output-only" (2 dimensions,
     // doc-coverage excluded). See docs/design-docs/named-scoring-profiles.md.
     const rubricConfig = loadRubricTemplates(ROOT);
-    const goldProfile = resolveProfile("baseline", "gold", rubricConfig);
-    const baselineProfileWeights = resolveProfile("baseline", "baseline", rubricConfig);
+    const goldProfile = resolveProfile("literacy", "gold", rubricConfig, LiteracyVariant.STANDARD);
+    const baselineProfileWeights = resolveProfile("literacy", LiteracyVariant.STANDARD, rubricConfig, LiteracyVariant.STANDARD);
     log.debug("Loaded scoring profiles", {
         gold: goldProfile,
         baseline: baselineProfileWeights,
@@ -776,9 +703,9 @@ export function calculateAndWriteScores(options) {
     let agentBehavior = null;
     let sourceIsolation = null;
     let evaluationMode;
-    if (mode === "full" && existsSync(agenticResultsPath)) {
+    if (mode === LiteracyVariant.FULL && existsSync(agenticResultsPath)) {
         log.info(`\nReading agentic results from: ${agenticResultsPath}`);
-        const agenticProfile = resolveProfile("agentic", "gold", rubricConfig);
+        const agenticProfile = resolveProfile("literacy", "gold", rubricConfig, LiteracyVariant.AGENTIC);
         const agenticScores = scoreAgenticResults(agenticResultsPath, agenticProfile);
         log.debug("Agentic scores calculated", {
             featureCount: Object.keys(agenticScores).length,
@@ -789,7 +716,7 @@ export function calculateAndWriteScores(options) {
             })),
         });
         scores = mergeScores(baselineScores, agenticScores);
-        evaluationMode = "full";
+        evaluationMode = LiteracyVariant.FULL;
         // Merge agentic actual scores into the per-model breakdown
         if (perModel) {
             const agenticPerModel = scoreAgenticResultsPerModel(agenticResultsPath, agenticProfile);
@@ -812,17 +739,20 @@ export function calculateAndWriteScores(options) {
             graderCost.completionTokens += agenticGraderCost.completionTokens;
         }
     }
-    else if (mode === "agentic") {
+    else if (mode === LiteracyVariant.AGENTIC) {
         scores = baselineScores;
         agentBehavior = aggregateAgentBehavior(baselineResultsPath);
         sourceIsolation = aggregateSourceIsolation(baselineResultsPath, options?.allowedOrigins);
-        evaluationMode = "agentic";
+        evaluationMode = LiteracyVariant.AGENTIC;
     }
     else {
         scores = baselineScores;
         agentBehavior = aggregateAgentBehavior(baselineResultsPath);
         sourceIsolation = aggregateSourceIsolation(baselineResultsPath, options?.allowedOrigins);
-        evaluationMode = mode === "observed" ? "observed" : "baseline";
+        evaluationMode =
+            mode === LiteracyVariant.OBSERVED
+                ? LiteracyVariant.OBSERVED
+                : LiteracyVariant.STANDARD;
     }
     const summary = printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode, log);
     // Persist
@@ -833,7 +763,7 @@ export function calculateAndWriteScores(options) {
     // Extract and persist grader judgments (Phase 3a: failure mode extraction)
     const judgments = extractGraderJudgments(baselineResultsPath);
     // In full mode, also extract judgments from agentic results
-    if (mode === "full" && existsSync(agenticResultsPath)) {
+    if (mode === LiteracyVariant.FULL && existsSync(agenticResultsPath)) {
         const agenticJudgments = extractGraderJudgments(agenticResultsPath);
         judgments.push(...agenticJudgments);
     }

package/dist/pipeline/checks.js

@@ -117,7 +117,7 @@ export function checkGeneratedConfigsExist(rootDir) {
     const baselinePath = resolve(rootDir, "promptfooconfig.yaml");
     if (!existsSync(baselinePath)) {
         issues.push({
-            message: "Baseline config 'promptfooconfig.yaml' not found. Run 'pnpm generate-configs'.",
+            message: "Baseline config 'promptfooconfig.yaml' not found. Run the pipeline to generate it.",
             path: baselinePath,
             severity: "error",
             source: "checkGeneratedConfigsExist",
@@ -131,7 +131,7 @@ export function checkGeneratedConfigsExist(rootDir) {
         const configPath = resolve(rootDir, name);
         if (!existsSync(configPath)) {
             issues.push({
-                message: `Optional config \`${name}\` not found. Run \`pnpm generate-configs\` to create it.`,
+                message: `Optional config \`${name}\` not found. Run the pipeline to generate it.`,
                 path: configPath,
                 severity: "warning",
                 source: "checkGeneratedConfigsExist",

package/dist/pipeline/compare.js

@@ -79,14 +79,14 @@ export function compare(baseline, experiment, options) {
     // Per-dimension average deltas (only for areas present in both summaries)
     const commonAreas = areas.filter((a) => baselineAreas.has(a.area) && experimentAreas.has(a.area));
     const commonCount = commonAreas.length || 1;
-    const perDimension = {
-        codeCorrectness: commonAreas.reduce((s, a) => s + a.dimensions.codeCorrectness.delta, 0) /
-            commonCount,
-        docCoverage: commonAreas.reduce((s, a) => s + a.dimensions.docCoverage.delta, 0) /
-            commonCount,
-        taskCompletion: commonAreas.reduce((s, a) => s + a.dimensions.taskCompletion.delta, 0) /
-            commonCount,
-    };
+    // Collect all dimension keys from area deltas and average each
+    const allDimKeys = new Set(commonAreas.flatMap((a) => Object.keys(a.dimensions)));
+    const perDimension = {};
+    for (const dim of allDimKeys) {
+        perDimension[dim] =
+            commonAreas.reduce((s, a) => s + (a.dimensions[dim]?.delta ?? 0), 0) /
+                commonCount;
+    }
     // Doc Lift average delta (common areas only)
     const docLift = commonAreas.reduce((s, a) => s + a.docLiftDelta, 0) / commonCount;
     // Cost delta (if both summaries have cost data)

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * agent-harness-handler.test.ts — Tests for agent harness mode compilation.
+ *
+ * Tests validation, provider assembly, tool permission resolution,
+ * assertion mapping, sandbox config, lifecycle extensions, and
+ * end-to-end compilation of example tasks.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/agent-harness-handler.test.ts
+ */
+export {};

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js

@@ -0,0 +1,288 @@
+/**
+ * agent-harness-handler.test.ts — Tests for agent harness mode compilation.
+ *
+ * Tests validation, provider assembly, tool permission resolution,
+ * assertion mapping, sandbox config, lifecycle extensions, and
+ * end-to-end compilation of example tasks.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/agent-harness-handler.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { LiteracyVariant } from "../../normalize-mode.js";
+import { compileAgentHarnessTask, AGENT_HARNESS_PROMPT_TEMPLATES, handler as agentHandler, validateAgentHarnessTask, } from "../mode-handlers/agent-harness/index.js";
+import { allAgentHarnessExampleTasks, scaffoldProjectTask, modifyCodeTask, multiFileRefactorTask, } from "../mode-handlers/__fixtures__/agent-harness-example-tasks.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function makeTask(overrides) {
+    return {
+        mode: "agent-harness",
+        id: "test-agent-task",
+        title: "Test Agent Task",
+        description: "A test agent harness task",
+        area: "studio",
+        ...overrides,
+    };
+}
+// ---------------------------------------------------------------------------
+// handler.getPrompts() — prompt template ownership
+// ---------------------------------------------------------------------------
+describe("AgentHarnessHandler.getPrompts", () => {
+    it("returns prompt templates", () => {
+        const prompts = agentHandler.getPrompts();
+        assert.ok(prompts, "getPrompts() should return a record");
+        assert.ok(Object.keys(prompts).length > 0, "should return at least one template");
+    });
+    it("returns templates keyed by agent-specific IDs (not literacy names)", () => {
+        const prompts = agentHandler.getPrompts();
+        const keys = Object.keys(prompts);
+        // Must not use literacy template names
+        assert.ok(!keys.includes("with-docs"), "should not use literacy key 'with-docs'");
+        assert.ok(!keys.includes("without-docs"), "should not use literacy key 'without-docs'");
+        assert.ok(!keys.includes(LiteracyVariant.AGENTIC), "should not use literacy key 'agentic'");
+        // Must have agent-appropriate key(s)
+        assert.ok(keys.includes("agent-harness"), "should include 'agent-harness' template");
+    });
+    it("agent-harness template describes task for agent execution", () => {
+        const prompts = agentHandler.getPrompts();
+        const template = prompts["agent-harness"];
+        assert.ok(template, "agent-harness template should exist");
+        assert.ok(template.template.includes("{{task}}"), "should include {{task}} placeholder");
+        // Should reference agent / sandbox / tool execution context
+        assert.ok(/sandbox|file|tool|implement|code/i.test(template.template), "template should reference agent execution concepts");
+    });
+    it("template has correct PromptTemplate shape", () => {
+        const prompts = agentHandler.getPrompts();
+        const template = prompts["agent-harness"];
+        assert.equal(template.id, "agent-harness");
+        assert.ok(template.label, "should have a human-readable label");
+        assert.ok(template.template, "should have a template string");
+        assert.ok(Array.isArray(template.variables), "should declare variables");
+        assert.ok(template.variables.includes("task"), "variables should include 'task'");
+    });
+    it("exported AGENT_HARNESS_PROMPT_TEMPLATES matches handler.getPrompts()", () => {
+        const fromHandler = agentHandler.getPrompts();
+        assert.deepEqual(fromHandler, AGENT_HARNESS_PROMPT_TEMPLATES);
+    });
+});
+// ---------------------------------------------------------------------------
+// validateAgentHarnessTask
+// ---------------------------------------------------------------------------
+describe("validateAgentHarnessTask", () => {
+    it("passes for a valid minimal task", () => {
+        const errors = validateAgentHarnessTask(makeTask());
+        assert.equal(errors.length, 0);
+    });
+    it("errors on missing ID", () => {
+        const errors = validateAgentHarnessTask(makeTask({ id: "" }));
+        assert.ok(errors.some((e) => e.field === "id"));
+    });
+    it("errors on missing title", () => {
+        const errors = validateAgentHarnessTask(makeTask({ title: "" }));
+        assert.ok(errors.some((e) => e.field === "title"));
+    });
+});
+// ---------------------------------------------------------------------------
+// compileAgentHarnessTask — provider assembly
+// ---------------------------------------------------------------------------
+describe("compileAgentHarnessTask — providers", () => {
+    it("produces a provider", () => {
+        const result = compileAgentHarnessTask(makeTask());
+        assert.ok(result.providers.length > 0);
+        assert.ok(result.providers[0].id.startsWith("agent:"));
+    });
+    it("resolves coding tool preset", () => {
+        const result = compileAgentHarnessTask(makeTask({ tools: ["coding"] }));
+        const config = result.providers[0].config;
+        const tools = config.allowedTools;
+        assert.ok(tools.includes("Bash"));
+        assert.ok(tools.includes("Read"));
+        assert.ok(tools.includes("Write"));
+        assert.ok(tools.includes("Edit"));
+    });
+    it("resolves read-only tool preset", () => {
+        const result = compileAgentHarnessTask(makeTask({ tools: ["read-only"] }));
+        const config = result.providers[0].config;
+        const tools = config.allowedTools;
+        assert.ok(tools.includes("Read"));
+        assert.ok(tools.includes("Grep"));
+        assert.ok(!tools.includes("Write"), "read-only should not include Write");
+    });
+    it("mixes preset and explicit tools", () => {
+        const result = compileAgentHarnessTask(makeTask({ tools: ["read-only", "WebFetch"] }));
+        const config = result.providers[0].config;
+        const tools = config.allowedTools;
+        assert.ok(tools.includes("Read"));
+        assert.ok(tools.includes("WebFetch"));
+    });
+    it("includes sandbox config in provider", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            sandbox: { type: "docker", image: "node:22-slim" },
+        }));
+        const config = result.providers[0].config;
+        const sandbox = config.sandbox;
+        assert.equal(sandbox.type, "docker");
+        assert.equal(sandbox.image, "node:22-slim");
+    });
+});
+// ---------------------------------------------------------------------------
+// compileAgentHarnessTask — test cases
+// ---------------------------------------------------------------------------
+describe("compileAgentHarnessTask — test cases", () => {
+    it("produces at least one test case", () => {
+        const result = compileAgentHarnessTask(makeTask());
+        assert.ok(result.tests.length > 0);
+    });
+    it("includes task description in vars", () => {
+        const result = compileAgentHarnessTask(makeTask({ description: "Do the thing" }));
+        assert.equal(result.tests[0].vars.task, "Do the thing");
+    });
+    it("prefers prompt.vars.task over description", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            description: "Description",
+            prompt: { vars: { task: "Custom prompt" } },
+        }));
+        assert.equal(result.tests[0].vars.task, "Custom prompt");
+    });
+    it("creates multi-turn test case", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            multiTurn: {
+                turns: [
+                    { role: "user", content: "Hello" },
+                    { role: "assistant", content: "Hi" },
+                ],
+            },
+        }));
+        assert.equal(result.tests.length, 2);
+        assert.ok(result.tests[1].description.includes("[multi-turn]"));
+    });
+    it("sets sandbox metadata in vars", () => {
+        const result = compileAgentHarnessTask(makeTask({ sandbox: { type: "docker" } }));
+        assert.equal(result.tests[0].vars.__sandboxType, "docker");
+    });
+});
+// ---------------------------------------------------------------------------
+// compileAgentHarnessTask — assertions
+// ---------------------------------------------------------------------------
+describe("compileAgentHarnessTask — assertions", () => {
+    it("maps file-exists to javascript assertion", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [{ type: "file-exists", value: "sanity.config.ts" }],
+        }));
+        const assertion = result.tests[0].assert?.[0];
+        assert.ok(assertion);
+        assert.equal(assertion.type, "javascript");
+        assert.ok(assertion.value.includes("sanity.config.ts"));
+    });
+    it("maps file-contains to javascript assertion", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [
+                {
+                    type: "file-contains",
+                    value: { path: "config.ts", content: "projectId" },
+                },
+            ],
+        }));
+        const assertion = result.tests[0].assert?.[0];
+        assert.ok(assertion);
+        assert.equal(assertion.type, "javascript");
+        assert.ok(assertion.value.includes("projectId"));
+    });
+    it("maps command-succeeds to javascript assertion", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [{ type: "command-succeeds", value: "npx tsc --noEmit" }],
+        }));
+        const assertion = result.tests[0].assert?.[0];
+        assert.ok(assertion);
+        assert.equal(assertion.type, "javascript");
+        assert.ok(assertion.value.includes("tsc"));
+    });
+    it("maps diff-matches to javascript assertion", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [{ type: "diff-matches", value: "createClient" }],
+        }));
+        const assertion = result.tests[0].assert?.[0];
+        assert.ok(assertion);
+        assert.equal(assertion.type, "javascript");
+        assert.ok(assertion.value.includes("git diff"));
+    });
+    it("passes through standard assertions", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [{ type: "contains", value: "sanity" }],
+        }));
+        assert.equal(result.tests[0].assert?.[0]?.type, "contains");
+    });
+    it("sets grader provider on llm-rubric", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [{ type: "llm-rubric", value: "Check quality" }],
+        }), { graderProvider: "openai:chat:gpt-5" });
+        assert.equal(result.tests[0].assert?.[0]?.provider, "openai:chat:gpt-5");
+    });
+});
+// ---------------------------------------------------------------------------
+// compileAgentHarnessTask — lifecycle extensions
+// ---------------------------------------------------------------------------
+describe("compileAgentHarnessTask — lifecycle", () => {
+    it("produces beforeEach and afterEach extensions", () => {
+        const result = compileAgentHarnessTask(makeTask());
+        assert.equal(result.extensions.length, 2);
+        assert.equal(result.extensions[0].type, "beforeEach");
+        assert.equal(result.extensions[1].type, "afterEach");
+    });
+    it("beforeEach hook creates working directory", () => {
+        const result = compileAgentHarnessTask(makeTask());
+        assert.ok(result.extensions[0].code.includes("mkdirSync"));
+        assert.ok(result.extensions[0].code.includes("__workingDir"));
+    });
+    it("afterEach hook cleans up", () => {
+        const result = compileAgentHarnessTask(makeTask());
+        assert.ok(result.extensions[1].code.includes("rmSync"));
+    });
+    it("sandbox config captures task settings", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            sandbox: {
+                type: "docker",
+                image: "node:22",
+                limits: { cpus: 2, networkAccess: false },
+            },
+            fixtures: ["file://schema.ts"],
+        }));
+        assert.equal(result.sandboxConfig.type, "docker");
+        assert.equal(result.sandboxConfig.image, "node:22");
+        assert.deepEqual(result.sandboxConfig.fixtures, ["file://schema.ts"]);
+        assert.equal(result.sandboxConfig.limits?.cpus, 2);
+        assert.equal(result.sandboxConfig.limits?.networkAccess, false);
+    });
+});
+// ---------------------------------------------------------------------------
+// Example task compilation (end-to-end)
+// ---------------------------------------------------------------------------
+describe("example agent harness tasks — end-to-end", () => {
+    it("compiles all example tasks without errors", () => {
+        for (const task of allAgentHarnessExampleTasks) {
+            const result = compileAgentHarnessTask(task);
+            assert.ok(result.providers.length > 0, `${task.id}: should produce providers`);
+            assert.ok(result.tests.length > 0, `${task.id}: should produce test cases`);
+            assert.ok(result.extensions.length > 0, `${task.id}: should produce lifecycle extensions`);
+        }
+    });
+    it("scaffold task has file-exists assertions", () => {
+        const result = compileAgentHarnessTask(scaffoldProjectTask);
+        assert.ok(result.tests[0].assert);
+        assert.ok(result.tests[0].assert.length >= 3);
+        // First two are file-exists (javascript), third is file-contains, fourth is command-succeeds
+        assert.equal(result.tests[0].assert[0].type, "javascript");
+    });
+    it("modify task has file-contains assertions", () => {
+        const result = compileAgentHarnessTask(modifyCodeTask);
+        assert.ok(result.tests[0].assert);
+        assert.ok(result.tests[0].assert.some((a) => a.type === "javascript" &&
+            a.value.includes("useDocumentOperation")));
+    });
+    it("refactor task has docker sandbox config", () => {
+        const result = compileAgentHarnessTask(multiFileRefactorTask);
+        assert.equal(result.sandboxConfig.type, "docker");
+        assert.equal(result.sandboxConfig.image, "node:22-slim");
+    });
+});

package/dist/pipeline/compiler/__tests__/assertion-mapper.test.d.ts

@@ -0,0 +1,9 @@
+/**
+ * assertion-mapper.test.ts — Unit tests for the assertion type mapper.
+ *
+ * Tests mapping of AILF assertion types to Promptfoo assertion types,
+ * mode compatibility checking, negation support, and templated assertions.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/assertion-mapper.test.ts
+ */
+export {};