@sanity/ailf 3.8.0 → 3.8.1

package/dist/pipeline/compiler/__tests__/literacy-handler.test.js

@@ -1,486 +0,0 @@
-/**
- * literacy-handler.test.ts — Tests for literacy mode compilation.
- *
- * Tests validation, gold/baseline entry generation, rubric template
- * resolution, doc-coverage auto-generation, prompt assignment, baseline
- * filtering, and the literacy bridge for LiteracyTaskDefinition.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/literacy-handler.test.ts
- */
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import { LiteracyVariant } from "../../normalize-mode.js";
-import { compileLiteracyTask, validateLiteracyTask, } from "../mode-handlers/literacy/index.js";
-import { compileLiteracyTasks, compareCompilerOutputs, } from "../literacy-bridge.js";
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-const RUBRIC_CONFIG = {
-    templates: {
-        "task-completion": {
-            dimension: "task-completion",
-            header: "Score task completion from 0 to 100:",
-            scale: [
-                "0: Not attempted",
-                "50: Partially complete",
-                "100: Fully complete",
-            ],
-            criteria_label: "Check for:",
-        },
-        "code-correctness": {
-            dimension: "code-correctness",
-            header: "Score code correctness from 0 to 100:",
-            scale: ["0: Does not compile", "50: Compiles but bugs", "100: Correct"],
-            criteria_label: "Check for:",
-        },
-        "doc-coverage": {
-            dimension: "doc-coverage",
-            header: "Score documentation coverage from 0 to 100:",
-            scale: [
-                "0: No docs used",
-                "50: Some docs used",
-                "100: All relevant docs used",
-            ],
-        },
-    },
-};
-function makeTask(overrides) {
-    return {
-        mode: "literacy",
-        id: "test-literacy-task",
-        title: "Test literacy task",
-        area: "groq",
-        prompt: { text: "Write a GROQ query to fetch all blog posts" },
-        context: {
-            docs: [{ slug: "groq-overview", reason: "Main GROQ reference" }],
-        },
-        referenceSolution: "canonical/groq-blog.ts",
-        docCoverage: true,
-        assertions: [
-            {
-                type: "llm-rubric",
-                template: "task-completion",
-                criteria: ["Uses _type filter", "Has projection"],
-            },
-            {
-                type: "contains",
-                value: "_type",
-            },
-        ],
-        baseline: { enabled: true, rubric: "full" },
-        ...overrides,
-    };
-}
-// ---------------------------------------------------------------------------
-// validateLiteracyTask
-// ---------------------------------------------------------------------------
-describe("validateLiteracyTask", () => {
-    it("passes for a valid task", () => {
-        const errors = validateLiteracyTask(makeTask());
-        assert.equal(errors.length, 0);
-    });
-    it("errors on missing ID", () => {
-        const errors = validateLiteracyTask(makeTask({ id: "" }));
-        assert.ok(errors.some((e) => e.field === "id"));
-    });
-    it("errors on missing title", () => {
-        const errors = validateLiteracyTask(makeTask({ title: "" }));
-        assert.ok(errors.some((e) => e.field === "title"));
-    });
-    it("errors on missing prompt text", () => {
-        const errors = validateLiteracyTask(makeTask({ prompt: { text: "" } }));
-        assert.ok(errors.some((e) => e.field === "prompt"));
-    });
-});
-// ---------------------------------------------------------------------------
-// compileLiteracyTask — gold + baseline entries
-// ---------------------------------------------------------------------------
-describe("compileLiteracyTask — entries", () => {
-    it("produces gold + baseline entries in baseline mode", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.equal(result.tests.length, 2);
-        assert.ok(result.tests[0].description.includes("(gold)"));
-        assert.ok(result.tests[1].description.includes("(baseline)"));
-    });
-    it("produces only gold entry in agentic mode", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            evalMode: LiteracyVariant.AGENTIC,
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.equal(result.tests.length, 1);
-        assert.ok(result.tests[0].description.includes("(gold)"));
-    });
-    it("skips baseline when baseline.enabled is false", () => {
-        const result = compileLiteracyTask(makeTask({ baseline: { enabled: false } }), { rubricConfig: RUBRIC_CONFIG });
-        assert.equal(result.tests.length, 1);
-    });
-    it("gold entry has docs variable", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.ok(result.tests[0].vars.docs.includes("file://"));
-    });
-    it("baseline entry has empty docs", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.equal(result.tests[1].vars.docs, "");
-    });
-    it("gold entry has with-docs prompt filter", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.deepEqual(result.tests[0].prompts, ["with-docs"]);
-    });
-    it("baseline entry has without-docs prompt filter", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.deepEqual(result.tests[1].prompts, ["without-docs"]);
-    });
-    it("sets empty docs when task has no canonical docs", () => {
-        const result = compileLiteracyTask(makeTask({ context: { docs: [] } }), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.equal(result.tests[0].vars.docs, "");
-    });
-});
-// ---------------------------------------------------------------------------
-// compileLiteracyTask — prompts
-// ---------------------------------------------------------------------------
-describe("compileLiteracyTask — prompts", () => {
-    it("generates with-docs and without-docs prompts in baseline mode", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.equal(result.prompts.length, 2);
-        assert.equal(result.prompts[0].id, "with-docs");
-        assert.equal(result.prompts[1].id, "without-docs");
-    });
-    it("generates single agentic prompt in agentic mode", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            evalMode: LiteracyVariant.AGENTIC,
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        assert.equal(result.prompts.length, 1);
-        assert.equal(result.prompts[0].id, LiteracyVariant.AGENTIC);
-    });
-});
-// ---------------------------------------------------------------------------
-// compileLiteracyTask — assertion resolution
-// ---------------------------------------------------------------------------
-describe("compileLiteracyTask — assertions", () => {
-    it("resolves templated assertions from rubric config", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        const goldAsserts = result.tests[0].assert;
-        const rubricAssert = goldAsserts.find((a) => a.type === "llm-rubric");
-        assert.ok(rubricAssert);
-        assert.ok(rubricAssert.value.includes("Score task completion"));
-        assert.ok(rubricAssert.value.includes("Uses _type filter"));
-    });
-    it("includes dimension metadata on templated assertions", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        const goldAsserts = result.tests[0].assert;
-        const rubricAssert = goldAsserts.find((a) => a.type === "llm-rubric");
-        assert.ok(rubricAssert);
-        const meta = rubricAssert.metadata;
-        assert.equal(meta?.dimension, "task-completion");
-        assert.equal(meta?.maxScore, 100);
-    });
-    it("passes through value assertions", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        const goldAsserts = result.tests[0].assert;
-        const containsAssert = goldAsserts.find((a) => a.type === "contains");
-        assert.ok(containsAssert);
-        assert.equal(containsAssert.value, "_type");
-    });
-    it("auto-generates doc-coverage assertion when docCoverage is true", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        const goldAsserts = result.tests[0].assert;
-        const docCovAsserts = goldAsserts.filter((a) => a.type === "llm-rubric" &&
-            a.metadata &&
-            a.metadata.dimension === "doc-coverage");
-        assert.equal(docCovAsserts.length, 1);
-    });
-    it("does not auto-generate doc-coverage when docCoverage is false", () => {
-        const result = compileLiteracyTask(makeTask({ docCoverage: false }), {
-            rubricConfig: RUBRIC_CONFIG,
-        });
-        const goldAsserts = result.tests[0].assert;
-        const docCovAsserts = goldAsserts.filter((a) => a.type === "llm-rubric" &&
-            a.metadata &&
-            a.metadata.dimension === "doc-coverage");
-        assert.equal(docCovAsserts.length, 0);
-    });
-    it("sets grader provider on llm-rubric assertions", () => {
-        const result = compileLiteracyTask(makeTask(), {
-            rubricConfig: RUBRIC_CONFIG,
-            graderProvider: "openai:chat:gpt-5",
-        });
-        const rubrics = result.tests[0].assert.filter((a) => a.type === "llm-rubric");
-        assert.ok(rubrics.every((r) => r.provider === "openai:chat:gpt-5"));
-    });
-});
-// ---------------------------------------------------------------------------
-// compileLiteracyTask — baseline assertion filtering
-// ---------------------------------------------------------------------------
-describe("compileLiteracyTask — baseline assertions", () => {
-    it("carries all assertions when baseline rubric is full", () => {
-        const result = compileLiteracyTask(makeTask({ baseline: { enabled: true, rubric: "full" } }), { rubricConfig: RUBRIC_CONFIG });
-        const goldCount = result.tests[0].assert.length;
-        const baselineCount = result.tests[1].assert.length;
-        assert.equal(goldCount, baselineCount);
-    });
-    it("abbreviates baseline assertions when rubric is abbreviated", () => {
-        const result = compileLiteracyTask(makeTask({ baseline: { enabled: true, rubric: "abbreviated" } }), { rubricConfig: RUBRIC_CONFIG });
-        const baselineAsserts = result.tests[1].assert;
-        // Abbreviated: only one summary llm-rubric
-        assert.equal(baselineAsserts.length, 1);
-        assert.equal(baselineAsserts[0].type, "llm-rubric");
-        assert.ok(baselineAsserts[0].value.includes("Score task completion"));
-    });
-    it("has no assertions when baseline rubric is none", () => {
-        const result = compileLiteracyTask(makeTask({ baseline: { enabled: true, rubric: "none" } }), { rubricConfig: RUBRIC_CONFIG });
-        assert.equal(result.tests[1].assert, undefined);
-    });
-});
-// ---------------------------------------------------------------------------
-// compileLiteracyTasks (bridge)
-// ---------------------------------------------------------------------------
-describe("compileLiteracyTasks — bridge", () => {
-    it("compiles multiple tasks", () => {
-        const tasks = [
-            makeTask({ id: "task-1", title: "First task" }),
-            makeTask({ id: "task-2", title: "Second task" }),
-        ];
-        const result = compileLiteracyTasks(tasks, {
-            rootDir: "/tmp/fake-root",
-        });
-        assert.equal(result.tasks.length, 2);
-        assert.ok(result.totalTests >= 4); // 2 tasks × (gold + baseline)
-    });
-});
-// ---------------------------------------------------------------------------
-// compareCompilerOutputs (parallel comparison gate — task 7b)
-// ---------------------------------------------------------------------------
-describe("compareCompilerOutputs — comparison gate", () => {
-    it("passes when outputs match", () => {
-        const bridge = compileLiteracyTasks([makeTask({ id: "task-1", title: "Test" })], { rootDir: "/tmp/fake-root" });
-        // Simulate legacy entries that match
-        const legacyEntries = [
-            {
-                description: "task-1 Test (gold)",
-                vars: { task: "Write a GROQ query", docs: "file://..." },
-                assert: bridge.tasks[0].result.tests[0].assert?.map((a) => ({
-                    type: a.type,
-                    value: a.value,
-                })),
-            },
-            {
-                description: "task-1 Test (baseline)",
-                vars: { task: "Write a GROQ query", docs: "" },
-                assert: bridge.tasks[0].result.tests[1].assert?.map((a) => ({
-                    type: a.type,
-                    value: a.value,
-                })),
-            },
-        ];
-        const comparison = compareCompilerOutputs(legacyEntries, bridge);
-        assert.ok(typeof comparison.passed === "boolean");
-        assert.ok(comparison.summary.length > 0);
-    });
-    it("detects test count mismatch", () => {
-        const bridge = compileLiteracyTasks([makeTask({ id: "task-1", title: "Test" })], { rootDir: "/tmp/fake-root" });
-        // Provide only gold entry — missing baseline creates a count mismatch
-        const legacyEntries = [
-            {
-                description: "task-1 Test (gold)",
-                vars: { task: "Write a GROQ query", docs: "file://..." },
-                assert: bridge.tasks[0].result.tests[0].assert?.map((a) => ({
-                    type: a.type,
-                    value: a.value,
-                })),
-            },
-        ];
-        const comparison = compareCompilerOutputs(legacyEntries, bridge);
-        assert.equal(comparison.passed, false, "expected mismatch to be detected");
-        assert.ok(comparison.discrepancies.length > 0, "expected discrepancies");
-        assert.ok(comparison.discrepancies.some((d) => d.field === "testCount"), "expected testCount discrepancy");
-    });
-    it("detects assertion count mismatch on gold entries", () => {
-        const bridge = compileLiteracyTasks([makeTask({ id: "task-1", title: "Test" })], { rootDir: "/tmp/fake-root" });
-        // The bridge gold test has assertions from the task definition.
-        // Provide a legacy gold entry with a different number of assertions
-        // to trigger the assertion count check.
-        const bridgeGoldAssertCount = bridge.tasks[0].result.tests[0].assert?.length ?? 0;
-        assert.ok(bridgeGoldAssertCount > 0, "bridge should have at least one assertion");
-        const legacyEntries = [
-            {
-                description: "task-1 Test (gold)",
-                vars: { task: "Write a GROQ query", docs: "file://..." },
-                // Deliberately provide wrong number of assertions
-                assert: [],
-            },
-            {
-                description: "task-1 Test (baseline)",
-                vars: { task: "Write a GROQ query", docs: "" },
-            },
-        ];
-        const comparison = compareCompilerOutputs(legacyEntries, bridge);
-        assert.equal(comparison.passed, false, "expected mismatch to be detected");
-        assert.ok(comparison.discrepancies.some((d) => d.field === "assertionCount"), "expected assertionCount discrepancy");
-    });
-});
-// ---------------------------------------------------------------------------
-// Compilation output stability (AC: identical configs before and after)
-// ---------------------------------------------------------------------------
-describe("compileLiteracyTasks — output stability", () => {
-    it("produces identical output across two compilations of the same input", () => {
-        const tasks = [
-            makeTask({ id: "task-1", title: "First task" }),
-            makeTask({ id: "task-2", title: "Second task" }),
-        ];
-        const opts = { rootDir: "/tmp/fake-root" };
-        const run1 = compileLiteracyTasks(tasks, opts);
-        const run2 = compileLiteracyTasks(tasks, opts);
-        // Same number of tasks and total tests
-        assert.equal(run1.tasks.length, run2.tasks.length);
-        assert.equal(run1.totalTests, run2.totalTests);
-        // Per-task: same test count, same descriptions, same assertion structure
-        for (let i = 0; i < run1.tasks.length; i++) {
-            const r1 = run1.tasks[i].result;
-            const r2 = run2.tasks[i].result;
-            assert.equal(r1.tests.length, r2.tests.length, `test count mismatch for task ${i}`);
-            for (let j = 0; j < r1.tests.length; j++) {
-                assert.equal(r1.tests[j].description, r2.tests[j].description, `description mismatch at task ${i} test ${j}`);
-                assert.equal(r1.tests[j].assert?.length ?? 0, r2.tests[j].assert?.length ?? 0, `assertion count mismatch at task ${i} test ${j}`);
-                assert.deepEqual(r1.tests[j].vars, r2.tests[j].vars, `vars mismatch at task ${i} test ${j}`);
-                // Verify assertion types and values match exactly
-                if (r1.tests[j].assert) {
-                    for (let k = 0; k < r1.tests[j].assert.length; k++) {
-                        assert.equal(r1.tests[j].assert[k].type, r2.tests[j].assert[k].type, `assertion type mismatch at task ${i} test ${j} assert ${k}`);
-                        assert.equal(r1.tests[j].assert[k].value, r2.tests[j].assert[k].value, `assertion value mismatch at task ${i} test ${j} assert ${k}`);
-                    }
-                }
-            }
-        }
-    });
-    it("produces identical output in baseline and agentic variants", () => {
-        const task = makeTask({ id: "stable-task", title: "Stable task" });
-        const baselineResult = compileLiteracyTask(task, {
-            rubricConfig: RUBRIC_CONFIG,
-            evalMode: LiteracyVariant.STANDARD,
-        });
-        const baselineResult2 = compileLiteracyTask(task, {
-            rubricConfig: RUBRIC_CONFIG,
-            evalMode: LiteracyVariant.STANDARD,
-        });
-        // Baseline produces gold + baseline entries
-        assert.equal(baselineResult.tests.length, baselineResult2.tests.length);
-        assert.deepEqual(baselineResult.tests.map((t) => t.description), baselineResult2.tests.map((t) => t.description));
-        const agenticResult = compileLiteracyTask(task, {
-            rubricConfig: RUBRIC_CONFIG,
-            evalMode: LiteracyVariant.AGENTIC,
-        });
-        const agenticResult2 = compileLiteracyTask(task, {
-            rubricConfig: RUBRIC_CONFIG,
-            evalMode: LiteracyVariant.AGENTIC,
-        });
-        // Agentic produces only gold entry
-        assert.equal(agenticResult.tests.length, agenticResult2.tests.length);
-        assert.deepEqual(agenticResult.tests.map((t) => t.description), agenticResult2.tests.map((t) => t.description));
-    });
-});
-// ---------------------------------------------------------------------------
-// compileLiteracyTasks — TaskGraphBuilder integration (task 7c)
-// ---------------------------------------------------------------------------
-describe("compileLiteracyTasks — TaskGraphBuilder wiring", () => {
-    it("filters out archived tasks via graph builder", () => {
-        const tasks = [
-            makeTask({ id: "active-task", title: "Active" }),
-            makeTask({
-                id: "archived-task",
-                title: "Archived",
-                status: "archived",
-            }),
-        ];
-        const result = compileLiteracyTasks(tasks, {
-            rootDir: "/tmp/fake-root",
-        });
-        assert.equal(result.tasks.length, 1);
-        assert.equal(result.tasks[0].taskId, "active-task");
-        assert.ok(result.warnings.some((w) => w.includes("filtered out")));
-    });
-    it("warns about duplicate task IDs via graph builder", () => {
-        const tasks = [
-            makeTask({ id: "dup-task", title: "First" }),
-            makeTask({ id: "dup-task", title: "Second" }),
-        ];
-        const result = compileLiteracyTasks(tasks, {
-            rootDir: "/tmp/fake-root",
-        });
-        assert.ok(result.warnings.some((w) => w.includes("Duplicate")));
-    });
-    it("throws on cyclic dependencies", () => {
-        const tasks = [
-            makeTask({
-                id: "task-a",
-                title: "A",
-                prompt: {
-                    text: "Write a GROQ query to fetch all blog posts",
-                    vars: { dependsOn: ["task-b"] },
-                },
-            }),
-            makeTask({
-                id: "task-b",
-                title: "B",
-                prompt: {
-                    text: "Write a GROQ query to fetch all blog posts",
-                    vars: { dependsOn: ["task-a"] },
-                },
-            }),
-        ];
-        assert.throws(() => compileLiteracyTasks(tasks, { rootDir: "/tmp/fake-root" }), /cycle/i);
-    });
-    it("respects topological ordering from dependencies", () => {
-        const tasks = [
-            makeTask({
-                id: "step-2",
-                title: "Step 2",
-                prompt: {
-                    text: "Write a GROQ query to fetch all blog posts",
-                    vars: { dependsOn: ["step-1"] },
-                },
-            }),
-            makeTask({ id: "step-1", title: "Step 1" }),
-        ];
-        const result = compileLiteracyTasks(tasks, {
-            rootDir: "/tmp/fake-root",
-        });
-        // step-1 should come before step-2 despite being second in input
-        assert.equal(result.tasks.length, 2);
-        assert.equal(result.tasks[0].taskId, "step-1");
-        assert.equal(result.tasks[1].taskId, "step-2");
-    });
-    it("returns empty result when all tasks are filtered out", () => {
-        const tasks = [
-            makeTask({ id: "archived-1", status: "archived" }),
-            makeTask({ id: "archived-2", status: "archived" }),
-        ];
-        const result = compileLiteracyTasks(tasks, {
-            rootDir: "/tmp/fake-root",
-        });
-        assert.equal(result.tasks.length, 0);
-        assert.equal(result.totalTests, 0);
-    });
-});

package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.d.ts

@@ -1,10 +0,0 @@
-/**
- * mcp-server-handler.test.ts — Tests for MCP server mode compilation.
- *
- * Tests the full MCP compilation pipeline: task validation, provider
- * assembly, assertion mapping, test case generation, and end-to-end
- * compilation of example tasks.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/mcp-server-handler.test.ts
- */
-export {};