@sanity/ailf 2.0.0 → 2.0.2

package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js

@@ -0,0 +1,185 @@
+/**
+ * comparison-formatters.test.ts
+ *
+ * Verifies that formatComparisonMarkdown() and formatComparisonTable()
+ * dynamically derive column headers from the dimension keys present
+ * in the report data, rather than hardcoding literacy-specific names.
+ *
+ * Run: npx tsx --test src/__tests__/comparison-formatters.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { formatComparisonMarkdown, formatComparisonTable, } from "../services/comparison-formatters.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Minimal ScoreSummary stub — only fields the formatters actually read */
+function stubSummary(avgScore) {
+    return {
+        belowCritical: [],
+        lowestArea: "area-a",
+        lowestScore: 40,
+        overall: {
+            avgCeilingScore: 80,
+            avgScore,
+            avgDocLift: 10,
+            avgDocQualityGap: 20,
+            avgFloorScore: 30,
+            negativeDocLiftCount: 0,
+        },
+        scores: [],
+        timestamp: "2026-04-05T00:00:00.000Z",
+    };
+}
+function makeReport(overrides) {
+    return {
+        areas: [
+            {
+                area: "area-a",
+                baseline: 60,
+                experiment: 65,
+                delta: 5,
+                change: "improved",
+                dimensions: overrides.areaDimensions,
+                ceilingDelta: 0,
+                docLiftDelta: 2,
+                floorDelta: 0,
+            },
+        ],
+        baseline: stubSummary(60),
+        experiment: stubSummary(65),
+        deltas: {
+            overall: 5,
+            perArea: { "area-a": 5 },
+            perDimension: overrides.perDimension,
+            docLift: 2,
+        },
+        generatedAt: "2026-04-05T00:00:00.000Z",
+        improved: ["area-a"],
+        regressed: [],
+        unchanged: [],
+        notEvaluated: [],
+        mismatched: { onlyInBaseline: [], onlyInExperiment: [] },
+        noiseThreshold: 2,
+        noiseThresholdEmpirical: false,
+    };
+}
+// ---------------------------------------------------------------------------
+// Tests — literacy dimensions (backward compatibility)
+// ---------------------------------------------------------------------------
+describe("formatComparisonMarkdown", () => {
+    it("renders literacy dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "task-completion": { baseline: 60, experiment: 65, delta: 5 },
+                "code-correctness": { baseline: 50, experiment: 55, delta: 5 },
+                "doc-coverage": { baseline: 40, experiment: 42, delta: 2 },
+            },
+            perDimension: {
+                "task-completion": 5,
+                "code-correctness": 5,
+                "doc-coverage": 2,
+            },
+        });
+        const md = formatComparisonMarkdown(report);
+        // Column headers should be title-cased from kebab-case
+        assert.ok(md.includes("Task Completion"), "should have Task Completion column header");
+        assert.ok(md.includes("Code Correctness"), "should have Code Correctness column header");
+        assert.ok(md.includes("Doc Coverage"), "should have Doc Coverage column header");
+        // Per-dimension averages section should also show dynamic labels
+        assert.ok(md.includes("| Task Completion |"), "dimension averages should include Task Completion");
+        assert.ok(md.includes("| Code Correctness |"), "dimension averages should include Code Correctness");
+        assert.ok(md.includes("| Doc Coverage |"), "dimension averages should include Doc Coverage");
+    });
+    it("renders MCP dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "input-validation": { baseline: 50, experiment: 60, delta: 10 },
+                "output-correctness": { baseline: 70, experiment: 75, delta: 5 },
+                "error-handling": { baseline: 40, experiment: 45, delta: 5 },
+                security: { baseline: 80, experiment: 82, delta: 2 },
+            },
+            perDimension: {
+                "input-validation": 10,
+                "output-correctness": 5,
+                "error-handling": 5,
+                security: 2,
+            },
+        });
+        const md = formatComparisonMarkdown(report);
+        // 4 MCP columns instead of 3 literacy columns
+        assert.ok(md.includes("Input Validation"), "should have Input Validation column");
+        assert.ok(md.includes("Output Correctness"), "should have Output Correctness column");
+        assert.ok(md.includes("Error Handling"), "should have Error Handling column");
+        assert.ok(md.includes("Security"), "should have Security column");
+        // Per-dimension averages
+        assert.ok(md.includes("| Input Validation |"), "dimension averages should include Input Validation");
+        assert.ok(md.includes("| Security |"), "dimension averages should include Security");
+    });
+});
+describe("formatComparisonTable", () => {
+    it("renders literacy dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "task-completion": { baseline: 60, experiment: 65, delta: 5 },
+                "code-correctness": { baseline: 50, experiment: 55, delta: 5 },
+                "doc-coverage": { baseline: 40, experiment: 42, delta: 2 },
+            },
+            perDimension: {
+                "task-completion": 5,
+                "code-correctness": 5,
+                "doc-coverage": 2,
+            },
+        });
+        const table = formatComparisonTable(report);
+        // Dimension averages section
+        assert.ok(table.includes("Task Completion:"), "should show Task Completion in dimension averages");
+        assert.ok(table.includes("Code Correctness:"), "should show Code Correctness in dimension averages");
+        assert.ok(table.includes("Doc Coverage:"), "should show Doc Coverage in dimension averages");
+        // Per-area table header
+        assert.ok(table.includes("Task Completion"), "per-area table should have Task Completion header");
+        assert.ok(table.includes("Code Correctness"), "per-area table should have Code Correctness header");
+        assert.ok(table.includes("Doc Coverage"), "per-area table should have Doc Coverage header");
+    });
+    it("renders MCP dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "input-validation": { baseline: 50, experiment: 60, delta: 10 },
+                "output-correctness": { baseline: 70, experiment: 75, delta: 5 },
+                "error-handling": { baseline: 40, experiment: 45, delta: 5 },
+                security: { baseline: 80, experiment: 82, delta: 2 },
+            },
+            perDimension: {
+                "input-validation": 10,
+                "output-correctness": 5,
+                "error-handling": 5,
+                security: 2,
+            },
+        });
+        const table = formatComparisonTable(report);
+        // 4 MCP columns in the per-area table
+        assert.ok(table.includes("Input Validation"), "should have Input Validation");
+        assert.ok(table.includes("Output Correctness"), "should have Output Correctness");
+        assert.ok(table.includes("Error Handling"), "should have Error Handling");
+        assert.ok(table.includes("Security"), "should have Security");
+        // Should NOT have literacy dimension headers
+        assert.ok(!table.includes("Task Completion"), "should not contain Task Completion");
+        assert.ok(!table.includes("Doc Coverage"), "should not contain Doc Coverage");
+    });
+    it("includes delta values for each dimension in the per-area rows", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "input-validation": { baseline: 50, experiment: 60, delta: 10 },
+                "output-correctness": { baseline: 70, experiment: 75, delta: 5 },
+            },
+            perDimension: {
+                "input-validation": 10,
+                "output-correctness": 5,
+            },
+        });
+        const table = formatComparisonTable(report);
+        // The per-area row should include the delta values (+10 and +5)
+        assert.ok(table.includes("+10"), "should show +10 delta for area-a");
+        assert.ok(table.includes("+5"), "should show +5 delta for area-a");
+    });
+});

package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * noop-collector.test.ts — verifies the NoOpArtifactCollector is truly zero-cost.
+ *
+ * Run: npx tsx --test src/artifact-capture/__tests__/noop-collector.test.ts
+ */
+export {};

package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js

@@ -0,0 +1,42 @@
+/**
+ * noop-collector.test.ts — verifies the NoOpArtifactCollector is truly zero-cost.
+ *
+ * Run: npx tsx --test src/artifact-capture/__tests__/noop-collector.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { NoOpArtifactCollector } from "../noop-collector.js";
+describe("NoOpArtifactCollector", () => {
+    it("enabled returns false", () => {
+        const collector = new NoOpArtifactCollector();
+        assert.equal(collector.enabled, false);
+    });
+    it("extrasEnabled returns false", () => {
+        const collector = new NoOpArtifactCollector();
+        assert.equal(collector.extrasEnabled, false);
+    });
+    it("capture() is callable and returns void", () => {
+        const collector = new NoOpArtifactCollector();
+        const result = collector.capture("step", "type", { data: true });
+        assert.equal(result, undefined);
+    });
+    it("captureFile() is callable and returns void", () => {
+        const collector = new NoOpArtifactCollector();
+        const result = collector.captureFile("step", "type", "/some/path");
+        assert.equal(result, undefined);
+    });
+    it("flush() returns zero-count result", async () => {
+        const collector = new NoOpArtifactCollector();
+        const result = await collector.flush();
+        assert.equal(result.artifactCount, 0);
+        assert.equal(result.destination, "");
+        assert.equal(result.totalBytes, 0);
+        assert.equal(result.compressed, false);
+    });
+    it("flush() returns the same frozen object every time", async () => {
+        const collector = new NoOpArtifactCollector();
+        const a = await collector.flush();
+        const b = await collector.flush();
+        assert.equal(a, b);
+    });
+});

package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts

@@ -0,0 +1,14 @@
+/**
+ * No-op artifact collector — used when --capture is not set.
+ *
+ * All methods are constant-time stubs. Zero overhead on the
+ * default pipeline path.
+ */
+import type { ArtifactCollector, CaptureFlushResult } from "../ports/artifact-collector.js";
+export declare class NoOpArtifactCollector implements ArtifactCollector {
+    readonly enabled = false;
+    readonly extrasEnabled = false;
+    capture(): void;
+    captureFile(): void;
+    flush(): Promise<CaptureFlushResult>;
+}

package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js

@@ -0,0 +1,25 @@
+/**
+ * No-op artifact collector — used when --capture is not set.
+ *
+ * All methods are constant-time stubs. Zero overhead on the
+ * default pipeline path.
+ */
+const EMPTY_RESULT = Object.freeze({
+    artifactCount: 0,
+    compressed: false,
+    destination: "",
+    totalBytes: 0,
+});
+export class NoOpArtifactCollector {
+    enabled = false;
+    extrasEnabled = false;
+    capture() {
+        // no-op
+    }
+    captureFile() {
+        // no-op
+    }
+    async flush() {
+        return EMPTY_RESULT;
+    }
+}

package/dist/_vendor/ailf-core/config-helpers.d.ts

@@ -51,6 +51,12 @@ export declare function defineTask(task: GeneralizedTaskDefinition): Generalized
  * Define the model registry (models to evaluate and grader model).
  *
  * Used in `config/models.ts` for typed model configuration.
+ *
+ * Validates:
+ * - Every `modes` entry is a canonical eval mode name
+ * - Every `variants` key is a mode the model is enrolled in
+ *
+ * @throws {Error} On invalid mode names or mismatched variant keys
  */
 export declare function defineModels(models: ModelsConfig): ModelsConfig;
 /**

package/dist/_vendor/ailf-core/config-helpers.js

@@ -25,6 +25,7 @@
  *
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md (canonical)
  */
+import { CANONICAL_EVAL_MODES } from "../ailf-shared/index.js";
 // ---------------------------------------------------------------------------
 // Config-level helpers
 // ---------------------------------------------------------------------------
@@ -57,8 +58,36 @@ export function defineTask(task) {
  * Define the model registry (models to evaluate and grader model).
  *
  * Used in `config/models.ts` for typed model configuration.
+ *
+ * Validates:
+ * - Every `modes` entry is a canonical eval mode name
+ * - Every `variants` key is a mode the model is enrolled in
+ *
+ * @throws {Error} On invalid mode names or mismatched variant keys
  */
 export function defineModels(models) {
+    const validModes = new Set(CANONICAL_EVAL_MODES);
+    for (const model of models.models) {
+        if (model.modes) {
+            for (const mode of model.modes) {
+                if (!validModes.has(mode)) {
+                    throw new Error(`Model "${model.label ?? model.id}": unknown mode "${mode}". ` +
+                        `Valid modes: ${CANONICAL_EVAL_MODES.join(", ")}`);
+                }
+            }
+        }
+        if (model.variants) {
+            const enrolledModes = new Set(model.modes ?? []);
+            for (const variantMode of Object.keys(model.variants)) {
+                if (!enrolledModes.has(variantMode)) {
+                    throw new Error(`Model "${model.label ?? model.id}": variants specified for ` +
+                        `mode "${variantMode}" but model does not include "${variantMode}" ` +
+                        `in its modes array. Add "${variantMode}" to modes or remove ` +
+                        `the variants entry.`);
+                }
+            }
+        }
+    }
     return models;
 }
 // ---------------------------------------------------------------------------