@sanity/ailf 0.4.1 → 1.0.0

package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.js

@@ -0,0 +1,471 @@
+/**
+ * scoring-and-presets.test.ts — Tests for 4-tier scoring engine,
+ * storage schema, and plugin registry / presets.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/scoring-and-presets.test.ts
+ */
+import assert from "node:assert/strict";
+import { dirname, resolve } from "node:path";
+import { describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+import { aggregateAreas, aggregateDimensions, computeEnsembleScore, computeTaskScore, normalizeScore, } from "../../../_vendor/ailf-core/index.js";
+import { CURRENT_SCHEMA_VERSION, InMemoryPluginRegistry, isSchemaVersioned, migrateDocument, } from "../../../_vendor/ailf-core/index.js";
+import { createSanityLiteracyPreset, sanityLiteracyPreset, } from "../presets/sanity-literacy.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function makeAssertion(overrides) {
+    return {
+        pass: true,
+        score: 0.8,
+        reason: "Good",
+        assertionType: "llm-rubric",
+        dimension: "task-completion",
+        latencyMs: 100,
+        weight: 1.0,
+        ...overrides,
+    };
+}
+function makeDimension(overrides) {
+    return {
+        dimensionId: "task-completion",
+        label: "Task Completion",
+        score: 0.8,
+        assertionCount: 2,
+        passCount: 2,
+        aggregation: "weighted-mean",
+        assertions: [],
+        ...overrides,
+    };
+}
+// ---------------------------------------------------------------------------
+// Tier 1 → Tier 2: Assertion → Dimension aggregation
+// ---------------------------------------------------------------------------
+describe("aggregateDimensions", () => {
+    it("groups assertions by dimension", () => {
+        const assertions = [
+            makeAssertion({ dimension: "code-correctness", score: 0.9 }),
+            makeAssertion({ dimension: "code-correctness", score: 0.7 }),
+            makeAssertion({ dimension: "task-completion", score: 0.8 }),
+        ];
+        const dims = aggregateDimensions(assertions);
+        assert.equal(dims.length, 2);
+        const cc = dims.find((d) => d.dimensionId === "code-correctness");
+        assert.ok(cc);
+        assert.equal(cc.assertionCount, 2);
+    });
+    it("uses weighted-mean by default", () => {
+        const assertions = [
+            makeAssertion({ score: 0.6, weight: 1.0 }),
+            makeAssertion({ score: 0.8, weight: 3.0 }),
+        ];
+        const dims = aggregateDimensions(assertions);
+        // Weighted mean: (0.6*1 + 0.8*3) / (1+3) = 3.0/4 = 0.75
+        assert.ok(Math.abs(dims[0].score - 0.75) < 0.01);
+    });
+    it("falls back to pass rate when no numeric scores", () => {
+        const assertions = [
+            makeAssertion({ score: null, pass: true }),
+            makeAssertion({ score: null, pass: false }),
+        ];
+        const dims = aggregateDimensions(assertions);
+        assert.equal(dims[0].score, 0.5);
+    });
+    it("applies custom dimension labels", () => {
+        const assertions = [makeAssertion({ dimension: "tc" })];
+        const dims = aggregateDimensions(assertions, {
+            dimensionLabels: { tc: "Task Completion" },
+        });
+        assert.equal(dims[0].label, "Task Completion");
+    });
+});
+// ---------------------------------------------------------------------------
+// Tier 2 → Tier 3: Dimension → Task scoring
+// ---------------------------------------------------------------------------
+describe("computeTaskScore", () => {
+    it("computes weighted score from dimensions", () => {
+        const dims = [
+            makeDimension({ dimensionId: "tc", score: 0.8 }),
+            makeDimension({ dimensionId: "cc", score: 0.6 }),
+        ];
+        const task = computeTaskScore(dims, {
+            taskId: "test-task",
+            weights: { tc: 0.6, cc: 0.4 },
+        });
+        // 0.8*0.6 + 0.6*0.4 = 0.48 + 0.24 = 0.72
+        assert.ok(Math.abs(task.score - 0.72) < 0.01);
+    });
+    it("normalizes weights that don't sum to 1", () => {
+        const dims = [
+            makeDimension({ dimensionId: "tc", score: 1.0 }),
+            makeDimension({ dimensionId: "cc", score: 0.0 }),
+        ];
+        const task = computeTaskScore(dims, {
+            taskId: "test-task",
+            weights: { tc: 2, cc: 2 },
+        });
+        // (1.0*2 + 0.0*2) / (2+2) = 2/4 = 0.5
+        assert.ok(Math.abs(task.score - 0.5) < 0.01);
+    });
+    it("checks against threshold", () => {
+        const dims = [makeDimension({ dimensionId: "tc", score: 0.6 })];
+        const passing = computeTaskScore(dims, {
+            taskId: "t1",
+            weights: { tc: 1.0 },
+            threshold: 0.5,
+        });
+        assert.equal(passing.passesThreshold, true);
+        const failing = computeTaskScore(dims, {
+            taskId: "t2",
+            weights: { tc: 1.0 },
+            threshold: 0.7,
+        });
+        assert.equal(failing.passesThreshold, false);
+    });
+    it("records weight source", () => {
+        const task = computeTaskScore([makeDimension()], {
+            taskId: "t1",
+            weights: { "task-completion": 1.0 },
+            weightSource: "rubrics.yaml:default",
+        });
+        assert.equal(task.weightSource, "rubrics.yaml:default");
+    });
+});
+// ---------------------------------------------------------------------------
+// Tier 3 → Tier 4: Task → Area aggregation
+// ---------------------------------------------------------------------------
+describe("aggregateAreas", () => {
+    it("groups tasks by area prefix", () => {
+        const tasks = [
+            computeTaskScore([makeDimension({ score: 0.8 })], {
+                taskId: "groq-basic",
+                weights: { "task-completion": 1.0 },
+            }),
+            computeTaskScore([makeDimension({ score: 0.6 })], {
+                taskId: "groq-advanced",
+                weights: { "task-completion": 1.0 },
+            }),
+            computeTaskScore([makeDimension({ score: 0.9 })], {
+                taskId: "studio-schema",
+                weights: { "task-completion": 1.0 },
+            }),
+        ];
+        const areas = aggregateAreas(tasks);
+        assert.equal(areas.length, 2);
+        const groq = areas.find((a) => a.areaId === "groq");
+        assert.ok(groq);
+        assert.equal(groq.taskCount, 2);
+        assert.ok(Math.abs(groq.score - 0.7) < 0.01); // (0.8+0.6)/2
+        const studio = areas.find((a) => a.areaId === "studio");
+        assert.ok(studio);
+        assert.equal(studio.taskCount, 1);
+    });
+    it("computes delta from previous scores", () => {
+        const tasks = [
+            computeTaskScore([makeDimension({ score: 0.8 })], {
+                taskId: "groq-basic",
+                weights: { "task-completion": 1.0 },
+            }),
+        ];
+        const areas = aggregateAreas(tasks, { groq: 0.6 });
+        assert.ok(areas[0].delta !== null);
+        assert.ok(Math.abs(areas[0].delta - 0.2) < 0.01);
+    });
+});
+// ---------------------------------------------------------------------------
+// Score normalization
+// ---------------------------------------------------------------------------
+describe("normalizeScore", () => {
+    it("normalizes LLM rubric scores (0-100 → 0-1)", () => {
+        assert.ok(Math.abs(normalizeScore(75, "llm-rubric") - 0.75) < 0.01);
+    });
+    it("passes through already-normalized scores", () => {
+        assert.ok(Math.abs(normalizeScore(0.75, "llm-rubric") - 0.75) < 0.01);
+    });
+    it("normalizes boolean assertions to 0 or 1", () => {
+        assert.equal(normalizeScore(1, "contains"), 1);
+        assert.equal(normalizeScore(0, "contains"), 0);
+    });
+    it("clamps similarity scores to [0, 1]", () => {
+        assert.equal(normalizeScore(1.5, "similar"), 1);
+        assert.equal(normalizeScore(-0.1, "similar"), 0);
+    });
+});
+// ---------------------------------------------------------------------------
+// Ensemble grading
+// ---------------------------------------------------------------------------
+describe("computeEnsembleScore", () => {
+    it("computes mean ensemble score", () => {
+        const { score, agreement } = computeEnsembleScore([0.8, 0.6, 0.7], "mean");
+        assert.ok(Math.abs(score - 0.7) < 0.01);
+        assert.ok(agreement > 0);
+    });
+    it("computes median ensemble score", () => {
+        const { score } = computeEnsembleScore([0.9, 0.5, 0.7], "median");
+        assert.ok(Math.abs(score - 0.7) < 0.01);
+    });
+    it("computes max ensemble score", () => {
+        const { score } = computeEnsembleScore([0.9, 0.5, 0.7], "max");
+        assert.ok(Math.abs(score - 0.9) < 0.01);
+    });
+    it("agreement is 1 for identical scores", () => {
+        const { agreement } = computeEnsembleScore([0.8, 0.8, 0.8]);
+        assert.ok(Math.abs(agreement - 1.0) < 0.01);
+    });
+    it("agreement decreases with divergent scores", () => {
+        const { agreement } = computeEnsembleScore([0.0, 1.0]);
+        assert.ok(agreement < 0.6);
+    });
+});
+// ---------------------------------------------------------------------------
+// Storage schema
+// ---------------------------------------------------------------------------
+describe("storage schema", () => {
+    it("CURRENT_SCHEMA_VERSION is 1", () => {
+        assert.equal(CURRENT_SCHEMA_VERSION, 1);
+    });
+    it("isSchemaVersioned detects versioned docs", () => {
+        assert.equal(isSchemaVersioned({ schemaVersion: 1 }), true);
+        assert.equal(isSchemaVersioned({}), false);
+        assert.equal(isSchemaVersioned(null), false);
+    });
+    it("migrateDocument is no-op for current version", () => {
+        const doc = { schemaVersion: 1, _type: "ailf.run" };
+        const migrated = migrateDocument(doc);
+        assert.equal(migrated.schemaVersion, 1);
+    });
+});
+// ---------------------------------------------------------------------------
+// Plugin registry
+// ---------------------------------------------------------------------------
+describe("InMemoryPluginRegistry", () => {
+    it("registers and retrieves modes", () => {
+        const registry = new InMemoryPluginRegistry();
+        registry.registerMode({
+            id: "custom",
+            label: "Custom Mode",
+            validProviderPatterns: [".*"],
+            rubricTemplateIds: [],
+            handlerModule: "./custom.js",
+        });
+        assert.equal(registry.getModes().length, 1);
+        assert.equal(registry.getMode("custom")?.label, "Custom Mode");
+    });
+    it("registers and retrieves assertions", () => {
+        const registry = new InMemoryPluginRegistry();
+        registry.registerAssertion({
+            type: "api-match",
+            label: "API Match",
+            compatibleModes: ["custom"],
+            handlerModule: "./api-match.js",
+        });
+        assert.equal(registry.getAssertions().length, 1);
+    });
+    it("registers a complete preset", () => {
+        const registry = new InMemoryPluginRegistry();
+        registry.registerPreset(sanityLiteracyPreset);
+        // Preset should register its modes, assertions, rubric templates
+        assert.ok(registry.getMode("literacy"));
+        assert.ok(registry.getAssertions().length > 0);
+        assert.ok(registry.getRubricTemplates().length > 0);
+        assert.ok(registry.getPresets().length === 1);
+    });
+});
+// ---------------------------------------------------------------------------
+// sanity-literacy preset
+// ---------------------------------------------------------------------------
+describe("sanityLiteracyPreset", () => {
+    it("has correct manifest", () => {
+        assert.equal(sanityLiteracyPreset.name, "sanity-literacy");
+        assert.equal(sanityLiteracyPreset.manifest.pluginApiVersion, 1);
+    });
+    it("registers literacy mode", () => {
+        assert.equal(sanityLiteracyPreset.modes?.length, 1);
+        assert.equal(sanityLiteracyPreset.modes[0].id, "literacy");
+    });
+    it("includes core assertion types", () => {
+        const types = sanityLiteracyPreset.assertions.map((a) => a.type);
+        assert.ok(types.includes("contains"));
+        assert.ok(types.includes("llm-rubric"));
+        assert.ok(types.includes("javascript"));
+    });
+    it("includes 3 rubric templates", () => {
+        assert.equal(sanityLiteracyPreset.rubricTemplates?.length, 3);
+        const ids = sanityLiteracyPreset.rubricTemplates.map((t) => t.id);
+        assert.ok(ids.includes("task-completion"));
+        assert.ok(ids.includes("code-correctness"));
+        assert.ok(ids.includes("doc-coverage"));
+    });
+    it("rubric template scales match config/rubrics.ts authoritative source", () => {
+        const templates = sanityLiteracyPreset.rubricTemplates;
+        const tc = templates.find((t) => t.id === "task-completion");
+        assert.deepEqual(tc.scale, [
+            "0: Couldn't attempt — missing critical information",
+            "20: Attempted but fundamentally wrong approach",
+            "50: Partial implementation — major functional gaps",
+            "80: Mostly complete — minor issues or missing edge cases",
+            "100: Fully functional code — works as expected",
+        ]);
+        assert.equal(tc.criteriaLabel, "Must demonstrate:");
+        const cc = templates.find((t) => t.id === "code-correctness");
+        assert.deepEqual(cc.scale, [
+            "0: Broken code, syntax errors, or deprecated APIs",
+            "30: Works but uses anti-patterns or inefficient approaches",
+            "50: Works but not idiomatic",
+            "80: Follows most best practices",
+            "100: Follows all best practices, idiomatic implementation",
+        ]);
+        assert.equal(cc.criteriaLabel, "Check for:");
+        const dc = templates.find((t) => t.id === "doc-coverage");
+        assert.deepEqual(dc.scale, [
+            "0: Had to hallucinate/guess most implementation details",
+            "30: Significant gaps — filled with assumptions",
+            "50: Some gaps — inferred from partial information",
+            "80: Minor gaps — almost everything was documented",
+            "100: Complete coverage — all necessary info was in docs",
+        ]);
+    });
+    it("includes sanity:// fixture resolver", () => {
+        assert.ok(sanityLiteracyPreset.fixtureResolvers?.some((r) => r.scheme === "sanity://"));
+    });
+    it("includes 3 prompt templates", () => {
+        const templates = sanityLiteracyPreset.promptTemplates;
+        assert.ok(templates);
+        assert.ok(templates["with-docs"]);
+        assert.ok(templates["without-docs"]);
+        assert.ok(templates["agentic"]);
+        assert.equal(Object.keys(templates).length, 3);
+    });
+    it("prompt template content matches literacy handler", () => {
+        const templates = sanityLiteracyPreset.promptTemplates;
+        assert.ok(templates["with-docs"].template.includes("{{docs}}"));
+        assert.ok(templates["with-docs"].template.includes("{{task}}"));
+        assert.ok(templates["without-docs"].template.includes("{{task}}"));
+        assert.ok(templates["agentic"].template.includes("{{task}}"));
+    });
+    it("includes default and output-only scoring profiles", () => {
+        const profiles = sanityLiteracyPreset.scoringProfiles;
+        assert.ok(profiles);
+        assert.deepEqual(profiles["default"], {
+            "task-completion": 0.5,
+            "code-correctness": 0.25,
+            "doc-coverage": 0.25,
+        });
+        assert.deepEqual(profiles["output-only"], {
+            "task-completion": 0.6,
+            "code-correctness": 0.4,
+        });
+    });
+    it("includes 3 source definitions", () => {
+        const sources = sanityLiteracyPreset.sourceDefs;
+        assert.ok(sources);
+        assert.equal(sources.length, 3);
+        const names = sources.map((s) => s.name);
+        assert.ok(names.includes("production"));
+        assert.ok(names.includes("branch"));
+        assert.ok(names.includes("local"));
+    });
+    it("production source has correct baseUrl", () => {
+        const prod = sanityLiteracyPreset.sourceDefs.find((s) => s.name === "production");
+        assert.ok(prod);
+        assert.equal(prod.baseUrl, "https://www.sanity.io/docs");
+    });
+    it("includes feature registry with all features", () => {
+        const features = sanityLiteracyPreset.featureDefs;
+        assert.ok(features);
+        assert.equal(features.features.length, 14);
+        const ids = features.features.map((f) => f.id);
+        // Covered features
+        assert.ok(ids.includes("groq"));
+        assert.ok(ids.includes("visual-editing"));
+        assert.ok(ids.includes("nextjs-live"));
+        assert.ok(ids.includes("functions"));
+        assert.ok(ids.includes("studio-setup"));
+        assert.ok(ids.includes("frameworks"));
+        // Uncovered features
+        assert.ok(ids.includes("portable-text"));
+        assert.ok(ids.includes("image-assets"));
+        assert.ok(ids.includes("mutations"));
+        assert.ok(ids.includes("schemas"));
+        assert.ok(ids.includes("authentication"));
+        assert.ok(ids.includes("webhooks"));
+        assert.ok(ids.includes("realtime"));
+        assert.ok(ids.includes("ai-assist"));
+    });
+    it("includes a docFetcher factory", () => {
+        assert.equal(typeof sanityLiteracyPreset.docFetcher, "function");
+        // The factory should return a SanityDocFetcher instance
+        const fetcher = sanityLiteracyPreset.docFetcher();
+        assert.ok(fetcher);
+        assert.equal(typeof fetcher.fetch, "function");
+    });
+});
+// ---------------------------------------------------------------------------
+// createSanityLiteracyPreset factory
+// ---------------------------------------------------------------------------
+describe("createSanityLiteracyPreset", () => {
+    it("returns a preset with all extension points populated", () => {
+        const preset = createSanityLiteracyPreset({ rootDir: "/tmp/test" });
+        assert.equal(preset.name, "sanity-literacy");
+        assert.ok(preset.modes);
+        assert.ok(preset.assertions);
+        assert.ok(preset.rubricTemplates);
+        assert.ok(preset.fixtureResolvers);
+        assert.ok(preset.promptTemplates);
+        assert.ok(preset.scoringProfiles);
+        assert.ok(preset.docFetcher);
+        assert.ok(preset.sourceDefs);
+        assert.ok(preset.featureDefs);
+    });
+    it("registers all extension points into the registry", () => {
+        const registry = new InMemoryPluginRegistry();
+        const preset = createSanityLiteracyPreset({ rootDir: "/tmp/test" });
+        registry.registerPreset(preset);
+        assert.ok(registry.getMode("literacy"));
+        assert.ok(registry.getAssertions().length > 0);
+        assert.ok(registry.getRubricTemplates().length === 3);
+        assert.ok(Object.keys(registry.getPromptTemplates()).length === 3);
+        assert.ok(Object.keys(registry.getScoringProfiles()).length === 2);
+        assert.ok(registry.getDocFetcherFactory());
+        assert.equal(registry.getSourceDefs().length, 3);
+        assert.ok(registry.getFeatureDefs());
+        assert.equal(registry.getFeatureDefs().features.length, 14);
+    });
+});
+// ---------------------------------------------------------------------------
+// Preset is single source of truth for sources and features
+// ---------------------------------------------------------------------------
+describe("preset is single source of truth for Sanity config", () => {
+    it("config/sources.ts exports an empty array", async () => {
+        const { tryLoadConfigFile } = await import("../../compiler/config-loader.js");
+        const ROOT = resolve(__dirname, "..", "..", "..", "..");
+        const loaded = tryLoadConfigFile("sources", ROOT);
+        assert.ok(loaded, "config/sources.ts should exist");
+        const sources = loaded.data;
+        assert.ok(Array.isArray(sources), "should export an array");
+        assert.equal(sources.length, 0, "config/sources should be empty (preset provides sources)");
+    });
+    it("config/features.ts exports an empty features array", async () => {
+        const { tryLoadConfigFile } = await import("../../compiler/config-loader.js");
+        const ROOT = resolve(__dirname, "..", "..", "..", "..");
+        const loaded = tryLoadConfigFile("features", ROOT);
+        assert.ok(loaded, "config/features.ts should exist");
+        assert.ok(Array.isArray(loaded.data.features), "should have a features array");
+        assert.equal(loaded.data.features.length, 0, "config/features should be empty (preset provides features)");
+    });
+    it("preset contains all 3 source entries", () => {
+        const sources = sanityLiteracyPreset.sourceDefs;
+        assert.equal(sources.length, 3);
+        const names = sources.map((s) => s.name).sort();
+        assert.deepEqual(names, ["branch", "local", "production"]);
+    });
+    it("preset contains all 14 feature entries", () => {
+        const features = sanityLiteracyPreset.featureDefs.features;
+        assert.equal(features.length, 14);
+        const covered = features.filter((f) => f.status === "covered");
+        const uncovered = features.filter((f) => f.status === "uncovered");
+        assert.equal(covered.length, 6, "should have 6 covered features");
+        assert.equal(uncovered.length, 8, "should have 8 uncovered features");
+    });
+});

package/dist/pipeline/compiler/__tests__/scoring-bridge.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * scoring-bridge.test.ts — Tests for the 4-tier scoring engine bridge.
+ *
+ * Verifies that `scoreTestGroup` produces the same 0–100 output as the
+ * legacy `accumulateDimensions → averageDimensions → weightedComposite`
+ * chain when given identical inputs.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/scoring-bridge.test.ts
+ */
+export {};

package/dist/pipeline/compiler/__tests__/scoring-bridge.test.js

@@ -0,0 +1,184 @@
+/**
+ * scoring-bridge.test.ts — Tests for the 4-tier scoring engine bridge.
+ *
+ * Verifies that `scoreTestGroup` produces the same 0–100 output as the
+ * legacy `accumulateDimensions → averageDimensions → weightedComposite`
+ * chain when given identical inputs.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/scoring-bridge.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { scoreTestGroup } from "../scoring-bridge.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function makeTestResult(overrides) {
+    const dims = overrides?.dimensions ?? {};
+    const componentResults = [];
+    if (dims.taskCompletion !== undefined) {
+        componentResults.push({
+            assertion: {
+                type: "llm-rubric",
+                metadata: { dimension: "task-completion" },
+            },
+            pass: true,
+            reason: JSON.stringify({ score: dims.taskCompletion }),
+            score: dims.taskCompletion / 100,
+        });
+    }
+    if (dims.codeCorrectness !== undefined) {
+        componentResults.push({
+            assertion: {
+                type: "llm-rubric",
+                metadata: { dimension: "code-correctness" },
+            },
+            pass: true,
+            reason: JSON.stringify({ score: dims.codeCorrectness }),
+            score: dims.codeCorrectness / 100,
+        });
+    }
+    if (dims.docCoverage !== undefined) {
+        componentResults.push({
+            assertion: {
+                type: "llm-rubric",
+                metadata: { dimension: "doc-coverage" },
+            },
+            pass: true,
+            reason: JSON.stringify({ score: dims.docCoverage }),
+            score: dims.docCoverage / 100,
+        });
+    }
+    return {
+        cost: overrides?.cost ?? 0.01,
+        description: overrides?.description ?? "test",
+        gradingResult: {
+            componentResults,
+            pass: true,
+        },
+        response: { output: "mock output" },
+        vars: overrides?.vars ?? { task: "test", docs: "" },
+    };
+}
+const DEFAULT_PROFILE = {
+    "code-correctness": 0.35,
+    "doc-coverage": 0.25,
+    "task-completion": 0.4,
+};
+const OUTPUT_ONLY_PROFILE = {
+    "code-correctness": 0.55,
+    "task-completion": 0.45,
+};
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+describe("scoreTestGroup — basic scoring", () => {
+    it("returns zeroes for empty test array", () => {
+        const result = scoreTestGroup([], DEFAULT_PROFILE);
+        assert.equal(result.composite, 0);
+        assert.equal(result.totalCost, 0);
+        assert.deepEqual(result.dimensions, {});
+    });
+    it("scores a single test with all dimensions", () => {
+        const tests = [
+            makeTestResult({
+                dimensions: {
+                    taskCompletion: 80,
+                    codeCorrectness: 70,
+                    docCoverage: 60,
+                },
+            }),
+        ];
+        const result = scoreTestGroup(tests, DEFAULT_PROFILE);
+        // Expected: 80*0.4 + 70*0.35 + 60*0.25 = 32 + 24.5 + 15 = 71.5 → 72
+        assert.equal(result.dimensions.taskCompletion, 80);
+        assert.equal(result.dimensions.codeCorrectness, 70);
+        assert.equal(result.dimensions.docCoverage, 60);
+        assert.equal(result.composite, 72);
+    });
+    it("averages across multiple tests", () => {
+        const tests = [
+            makeTestResult({
+                dimensions: { taskCompletion: 80, codeCorrectness: 60 },
+            }),
+            makeTestResult({
+                dimensions: { taskCompletion: 60, codeCorrectness: 80 },
+            }),
+        ];
+        const result = scoreTestGroup(tests, OUTPUT_ONLY_PROFILE);
+        // taskCompletion avg = 70, codeCorrectness avg = 70
+        // Expected: 70*0.45 + 70*0.55 = 31.5 + 38.5 = 70
+        assert.equal(result.dimensions.taskCompletion, 70);
+        assert.equal(result.dimensions.codeCorrectness, 70);
+        assert.equal(result.composite, 70);
+    });
+    it("accumulates cost across tests", () => {
+        const tests = [
+            makeTestResult({ cost: 0.05, dimensions: { taskCompletion: 80 } }),
+            makeTestResult({ cost: 0.03, dimensions: { taskCompletion: 70 } }),
+        ];
+        const result = scoreTestGroup(tests, DEFAULT_PROFILE);
+        assert.ok(Math.abs(result.totalCost - 0.08) < 0.001);
+    });
+});
+describe("scoreTestGroup — profile handling", () => {
+    it("uses output-only profile (excludes doc-coverage)", () => {
+        const tests = [
+            makeTestResult({
+                dimensions: {
+                    taskCompletion: 80,
+                    codeCorrectness: 60,
+                    docCoverage: 100,
+                },
+            }),
+        ];
+        const result = scoreTestGroup(tests, OUTPUT_ONLY_PROFILE);
+        // doc-coverage should be present in dimensions but NOT affect composite
+        // Expected: 80*0.45 + 60*0.55 = 36 + 33 = 69
+        assert.equal(result.dimensions.docCoverage, 100);
+        assert.equal(result.composite, 69);
+    });
+    it("handles profile with only one dimension", () => {
+        const tests = [
+            makeTestResult({
+                dimensions: { taskCompletion: 90, codeCorrectness: 50 },
+            }),
+        ];
+        const result = scoreTestGroup(tests, { "task-completion": 1.0 });
+        // Only taskCompletion should count
+        assert.equal(result.composite, 90);
+    });
+});
+describe("scoreTestGroup — edge cases", () => {
+    it("handles tests with no rubric components", () => {
+        const test = {
+            cost: 0.01,
+            description: "no rubrics",
+            gradingResult: {
+                componentResults: [
+                    { assertion: { type: "javascript" }, pass: true, score: 1 },
+                ],
+                pass: true,
+            },
+            response: { output: "mock" },
+            vars: { task: "test", docs: "" },
+        };
+        const result = scoreTestGroup([test], DEFAULT_PROFILE);
+        // No llm-rubric components → 0 composite
+        assert.equal(result.composite, 0);
+        assert.equal(result.totalCost, 0.01);
+    });
+    it("provides raw DimensionScore objects for advanced consumers", () => {
+        const tests = [
+            makeTestResult({
+                dimensions: { taskCompletion: 80, codeCorrectness: 60 },
+            }),
+        ];
+        const result = scoreTestGroup(tests, DEFAULT_PROFILE);
+        assert.ok(result.rawDimensions.length >= 2);
+        const tcDim = result.rawDimensions.find((d) => d.dimensionId === "task-completion");
+        assert.ok(tcDim);
+        assert.ok(tcDim.score >= 0 && tcDim.score <= 1); // 0–1 scale
+        assert.equal(tcDim.assertionCount, 1);
+    });
+});

package/dist/pipeline/compiler/__tests__/task-graph-builder.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * task-graph-builder.test.ts — Unit tests for TaskGraphBuilder.
+ *
+ * Tests DAG construction, cycle detection, filtering, and priority assignment.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/task-graph-builder.test.ts
+ */
+export {};