@sanity/ailf 3.7.0 → 3.8.1

package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js

@@ -1,314 +0,0 @@
-/**
- * knowledge-probe-handler.test.ts — Tests for knowledge probe mode compilation.
- *
- * Tests validation, provider assembly, prompt generation, assertion mapping
- * (including rejection of tool-use assertions), metadata generation, and
- * end-to-end compilation of example tasks.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/knowledge-probe-handler.test.ts
- */
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import { LiteracyVariant } from "../../normalize-mode.js";
-import { compileKnowledgeProbeTask, handler as probeHandler, KNOWLEDGE_PROBE_PROMPT_TEMPLATES, validateKnowledgeProbeTask, } from "../mode-handlers/knowledge-probe/index.js";
-import { allKnowledgeProbeExampleTasks, groqProjectionTask, defineTypeApiTask, ecosystemComparisonTask, } from "../mode-handlers/__fixtures__/knowledge-probe-example-tasks.js";
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function makeTask(overrides) {
-    return {
-        mode: "knowledge-probe",
-        id: "test-probe",
-        title: "Test Knowledge Probe",
-        description: "A test knowledge probe",
-        area: "groq",
-        ...overrides,
-    };
-}
-// ---------------------------------------------------------------------------
-// handler.getPrompts() — prompt template ownership
-// ---------------------------------------------------------------------------
-describe("KnowledgeProbeHandler.getPrompts", () => {
-    it("returns prompt templates", () => {
-        const prompts = probeHandler.getPrompts();
-        assert.ok(prompts, "getPrompts() should return a record");
-        assert.ok(Object.keys(prompts).length > 0, "should return at least one template");
-    });
-    it("returns templates keyed by probe-specific IDs (not literacy names)", () => {
-        const prompts = probeHandler.getPrompts();
-        const keys = Object.keys(prompts);
-        // Must not use literacy template names
-        assert.ok(!keys.includes("with-docs"), "should not use literacy key 'with-docs'");
-        assert.ok(!keys.includes("without-docs"), "should not use literacy key 'without-docs'");
-        assert.ok(!keys.includes(LiteracyVariant.AGENTIC), "should not use literacy key 'agentic'");
-        // Must have probe-appropriate key(s)
-        assert.ok(keys.includes("knowledge-probe"), "should include 'knowledge-probe' template");
-    });
-    it("knowledge-probe template asks factual questions without context", () => {
-        const prompts = probeHandler.getPrompts();
-        const template = prompts["knowledge-probe"];
-        assert.ok(template, "knowledge-probe template should exist");
-        assert.ok(template.template.includes("{{task}}"), "should include {{task}} placeholder");
-        // Should NOT reference documentation context
-        assert.ok(!template.template.includes("{{docs}}"), "should NOT include {{docs}} — probes test raw model knowledge");
-    });
-    it("template has correct PromptTemplate shape", () => {
-        const prompts = probeHandler.getPrompts();
-        const template = prompts["knowledge-probe"];
-        assert.equal(template.id, "knowledge-probe");
-        assert.ok(template.label, "should have a human-readable label");
-        assert.ok(template.template, "should have a template string");
-        assert.ok(Array.isArray(template.variables), "should declare variables");
-        assert.ok(template.variables.includes("task"), "variables should include 'task'");
-    });
-    it("exported KNOWLEDGE_PROBE_PROMPT_TEMPLATES matches handler.getPrompts()", () => {
-        const fromHandler = probeHandler.getPrompts();
-        assert.deepEqual(fromHandler, KNOWLEDGE_PROBE_PROMPT_TEMPLATES);
-    });
-});
-// ---------------------------------------------------------------------------
-// validateKnowledgeProbeTask
-// ---------------------------------------------------------------------------
-describe("validateKnowledgeProbeTask", () => {
-    it("passes for a valid minimal task", () => {
-        const errors = validateKnowledgeProbeTask(makeTask());
-        assert.equal(errors.length, 0);
-    });
-    it("errors on missing ID", () => {
-        const errors = validateKnowledgeProbeTask(makeTask({ id: "" }));
-        assert.ok(errors.some((e) => e.field === "id"));
-    });
-    it("errors on missing title", () => {
-        const errors = validateKnowledgeProbeTask(makeTask({ title: "" }));
-        assert.ok(errors.some((e) => e.field === "title"));
-    });
-    it("errors when no prompt or description is provided", () => {
-        const errors = validateKnowledgeProbeTask(makeTask({ description: undefined }));
-        assert.ok(errors.some((e) => e.field === "prompt"));
-    });
-    it("passes with prompt.text instead of description", () => {
-        const errors = validateKnowledgeProbeTask(makeTask({
-            description: undefined,
-            prompt: { text: "Explain GROQ" },
-        }));
-        assert.equal(errors.length, 0);
-    });
-    it("passes with prompt.vars.task instead of description", () => {
-        const errors = validateKnowledgeProbeTask(makeTask({
-            description: undefined,
-            prompt: { vars: { task: "Explain GROQ" } },
-        }));
-        assert.equal(errors.length, 0);
-    });
-});
-// ---------------------------------------------------------------------------
-// compileKnowledgeProbeTask — basic compilation
-// ---------------------------------------------------------------------------
-describe("compileKnowledgeProbeTask — basic", () => {
-    it("produces prompts and test cases", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.ok(result.prompts.length > 0, "Should produce prompts");
-        assert.ok(result.tests.length > 0, "Should produce test cases");
-    });
-    it("produces exactly one test case (no baseline variant)", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        // Knowledge probes have no baseline — there's no "without docs" variant
-        // because there are no docs in the first place
-        assert.equal(result.tests.length, 1);
-    });
-    it("does NOT include docs in test case vars", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.equal(result.tests[0].vars.docs, undefined);
-    });
-    it("includes task in vars from description", () => {
-        const result = compileKnowledgeProbeTask(makeTask({ description: "Explain GROQ projections" }));
-        assert.equal(result.tests[0].vars.task, "Explain GROQ projections");
-    });
-    it("prefers prompt.vars.task over description", () => {
-        const result = compileKnowledgeProbeTask(makeTask({
-            description: "Description",
-            prompt: { vars: { task: "Custom prompt text" } },
-        }));
-        assert.equal(result.tests[0].vars.task, "Custom prompt text");
-    });
-    it("includes mode metadata in vars", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.equal(result.tests[0].vars.__mode, "knowledge-probe");
-    });
-    it("includes probe strategy in vars", () => {
-        const result = compileKnowledgeProbeTask(makeTask({ probeStrategy: "depth-first" }));
-        assert.equal(result.tests[0].vars.__probeStrategy, "depth-first");
-    });
-    it("defaults probe strategy to breadth-first", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.equal(result.tests[0].vars.__probeStrategy, "breadth-first");
-    });
-});
-// ---------------------------------------------------------------------------
-// compileKnowledgeProbeTask — prompts
-// ---------------------------------------------------------------------------
-describe("compileKnowledgeProbeTask — prompts", () => {
-    it("uses a single no-docs prompt", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.equal(result.prompts.length, 1);
-        assert.equal(result.prompts[0].id, "knowledge-probe");
-    });
-    it("uses description as prompt text", () => {
-        const result = compileKnowledgeProbeTask(makeTask({ description: "Explain GROQ" }));
-        assert.equal(result.prompts[0].raw, "Explain GROQ");
-    });
-    it("prefers prompt.text over description", () => {
-        const result = compileKnowledgeProbeTask(makeTask({
-            description: "Desc",
-            prompt: { text: "Custom prompt" },
-        }));
-        assert.equal(result.prompts[0].raw, "Custom prompt");
-    });
-    it("includes system message when provided", () => {
-        const result = compileKnowledgeProbeTask(makeTask({
-            prompt: {
-                text: "Explain GROQ",
-                systemMessage: "You are a Sanity expert.",
-            },
-        }));
-        assert.ok(result.prompts[0].raw.includes("You are a Sanity expert."));
-        assert.ok(result.prompts[0].raw.includes("Explain GROQ"));
-    });
-});
-// ---------------------------------------------------------------------------
-// compileKnowledgeProbeTask — providers
-// ---------------------------------------------------------------------------
-describe("compileKnowledgeProbeTask — providers", () => {
-    it("builds providers from model list", () => {
-        const result = compileKnowledgeProbeTask(makeTask(), {
-            models: [
-                { id: "openai:chat:gpt-4o", label: "GPT-4o" },
-                { id: "anthropic:messages:claude-sonnet-4-6", label: "Claude" },
-            ],
-        });
-        assert.equal(result.providers.length, 2);
-        assert.equal(result.providers[0].id, "openai:chat:gpt-4o");
-        assert.equal(result.providers[1].id, "anthropic:messages:claude-sonnet-4-6");
-    });
-    it("returns empty providers when no models specified", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.equal(result.providers.length, 0);
-    });
-});
-// ---------------------------------------------------------------------------
-// compileKnowledgeProbeTask — assertions
-// ---------------------------------------------------------------------------
-describe("compileKnowledgeProbeTask — assertions", () => {
-    it("maps standard assertions", () => {
-        const result = compileKnowledgeProbeTask(makeTask({
-            assertions: [
-                { type: "contains", value: "GROQ" },
-                { type: "regex", value: "select\\(" },
-            ],
-        }));
-        assert.equal(result.tests[0].assert?.length, 2);
-        assert.equal(result.tests[0].assert[0].type, "contains");
-        assert.equal(result.tests[0].assert[1].type, "regex");
-    });
-    it("maps LLM-graded assertions with grader provider", () => {
-        const result = compileKnowledgeProbeTask(makeTask({
-            assertions: [{ type: "llm-rubric", value: "Check accuracy" }],
-        }), { graderProvider: "openai:chat:gpt-5" });
-        assert.equal(result.tests[0].assert[0].type, "llm-rubric");
-        assert.equal(result.tests[0].assert[0].provider, "openai:chat:gpt-5");
-    });
-    it("rejects tool-use assertions with warning", () => {
-        const result = compileKnowledgeProbeTask(makeTask({
-            assertions: [
-                { type: "tool-called", value: "getDocument" },
-                { type: "tool-input-matches", value: {} },
-                { type: "tool-output-matches", value: {} },
-                { type: "skill-used", value: "search" },
-                { type: "tool-call-f1", value: 0.8 },
-            ],
-        }));
-        // All tool-use assertions should be skipped
-        assert.equal(result.tests[0].assert?.length ?? 0, 0);
-        // Should have 5 warnings
-        assert.equal(result.warnings.length, 5);
-        assert.ok(result.warnings.every((w) => w.includes("not applicable")));
-    });
-    it("preserves assertion weights", () => {
-        const result = compileKnowledgeProbeTask(makeTask({
-            assertions: [{ type: "contains", value: "GROQ", weight: 0.3 }],
-        }));
-        assert.equal(result.tests[0].assert[0].weight, 0.3);
-    });
-});
-// ---------------------------------------------------------------------------
-// compileKnowledgeProbeTask — metadata
-// ---------------------------------------------------------------------------
-describe("compileKnowledgeProbeTask — metadata", () => {
-    it("includes correct mode metadata", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.equal(result.metadata.mode, "knowledge-probe");
-        assert.equal(result.metadata.noDocContext, true);
-        assert.equal(result.metadata.retrievalMetrics, false);
-    });
-    it("includes probe strategy in metadata", () => {
-        const result = compileKnowledgeProbeTask(makeTask({ probeStrategy: "coverage-guided" }));
-        assert.equal(result.metadata.probeStrategy, "coverage-guided");
-    });
-    it("defaults probe strategy to breadth-first", () => {
-        const result = compileKnowledgeProbeTask(makeTask());
-        assert.equal(result.metadata.probeStrategy, "breadth-first");
-    });
-});
-// ---------------------------------------------------------------------------
-// Example task compilation (end-to-end)
-// ---------------------------------------------------------------------------
-describe("example knowledge probe tasks — end-to-end", () => {
-    it("compiles all example tasks without errors", () => {
-        for (const task of allKnowledgeProbeExampleTasks) {
-            const result = compileKnowledgeProbeTask(task, {
-                models: [
-                    { id: "openai:chat:gpt-4o", label: "GPT-4o" },
-                    { id: "anthropic:messages:claude-sonnet-4-6", label: "Claude" },
-                ],
-            });
-            assert.ok(result.tests.length > 0, `${task.id}: should produce test cases`);
-            assert.ok(result.prompts.length > 0, `${task.id}: should produce prompts`);
-            assert.ok(result.providers.length > 0, `${task.id}: should have providers`);
-        }
-    });
-    it("GROQ probe has correct assertion types", () => {
-        const result = compileKnowledgeProbeTask(groqProjectionTask);
-        assert.ok(result.tests[0].assert);
-        // 2 contains + 2 llm-rubric
-        assert.equal(result.tests[0].assert.length, 4);
-        const types = result.tests[0].assert.map((a) => a.type);
-        assert.ok(types.includes("contains"));
-        assert.ok(types.includes("llm-rubric"));
-    });
-    it("defineType probe tests API currency", () => {
-        const result = compileKnowledgeProbeTask(defineTypeApiTask);
-        assert.ok(result.tests[0].assert);
-        // 2 contains + 2 llm-rubric
-        assert.equal(result.tests[0].assert.length, 4);
-    });
-    it("ecosystem comparison has contains-any assertions", () => {
-        const result = compileKnowledgeProbeTask(ecosystemComparisonTask);
-        assert.ok(result.tests[0].assert);
-        const containsAny = result.tests[0].assert.filter((a) => a.type === "contains-any");
-        assert.equal(containsAny.length, 2); // GROQ + GraphQL
-    });
-    it("no example task includes docs in vars", () => {
-        for (const task of allKnowledgeProbeExampleTasks) {
-            const result = compileKnowledgeProbeTask(task);
-            assert.equal(result.tests[0].vars.docs, undefined, `${task.id}: should not include docs`);
-        }
-    });
-    it("all example tasks have knowledge-probe metadata", () => {
-        for (const task of allKnowledgeProbeExampleTasks) {
-            const result = compileKnowledgeProbeTask(task);
-            assert.equal(result.metadata.mode, "knowledge-probe");
-            assert.equal(result.metadata.noDocContext, true);
-            assert.equal(result.metadata.retrievalMetrics, false);
-        }
-    });
-});

package/dist/pipeline/compiler/__tests__/literacy-handler.test.d.ts

@@ -1,10 +0,0 @@
-/**
- * literacy-handler.test.ts — Tests for literacy mode compilation.
- *
- * Tests validation, gold/baseline entry generation, rubric template
- * resolution, doc-coverage auto-generation, prompt assignment, baseline
- * filtering, and the literacy bridge for LiteracyTaskDefinition.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/literacy-handler.test.ts
- */
-export {};