@sanity/ailf 3.7.0 → 3.8.1

package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js

@@ -1,332 +0,0 @@
-/**
- * promptfoo-compiler.test.ts — Unit tests for the PromptfooCompiler.
- *
- * Tests compilation of a TaskGraph into Promptfoo configuration,
- * including provider assembly, prompt generation, and test case creation.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/promptfoo-compiler.test.ts
- */
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import { tmpdir } from "os";
-import { compileToPromptfoo } from "../promptfoo-compiler.js";
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function makeModels(models = []) {
-    return {
-        defaults: { temperature: 0.2 },
-        grader: { id: "openai:chat:gpt-5", label: "GPT-5" },
-        models: models.map((m) => ({
-            config: {},
-            id: m.id,
-            label: m.label,
-            modes: m.modes,
-        })),
-    };
-}
-function makeNode(id, prompt) {
-    return {
-        dependsOn: [],
-        priority: 0,
-        resolvedPrompt: prompt ?? `Implement ${id}`,
-        resolvedVariables: {
-            declarations: [],
-            provenance: {},
-            values: {
-                task: prompt ?? `Implement ${id}`,
-                docs: `Documentation for ${id}`,
-            },
-        },
-        taskId: id,
-    };
-}
-function makeGraph(nodes, edges = []) {
-    const nodeMap = new Map();
-    for (const n of nodes)
-        nodeMap.set(n.taskId, n);
-    return {
-        compilationTarget: "promptfoo",
-        edges,
-        fixtures: new Map(),
-        nodes: nodeMap,
-    };
-}
-// ---------------------------------------------------------------------------
-// compileToPromptfoo — basic compilation
-// ---------------------------------------------------------------------------
-describe("compileToPromptfoo", () => {
-    it("compiles a single-task graph into a Promptfoo config", () => {
-        const graph = makeGraph([makeNode("groq-basics")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels([{ id: "openai:chat:gpt-4o", label: "GPT-4o" }]),
-            rootDir: tmpdir(),
-        });
-        assert.ok(result.config);
-        assert.equal(result.taskCount, 1);
-        // Literacy mode produces gold + baseline test cases
-        assert.equal(result.testCaseCount, 2);
-        assert.equal(result.config.tests.length, 2);
-    });
-    it("generates correct test case descriptions", () => {
-        const graph = makeGraph([makeNode("groq-basics")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-        });
-        assert.equal(result.config.tests[0].description, "groq-basics");
-        assert.equal(result.config.tests[1].description, "groq-basics [baseline]");
-    });
-    it("includes task variables in test case vars", () => {
-        const graph = makeGraph([makeNode("task-1", "Do the thing")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-        });
-        const vars = result.config.tests[0].vars;
-        assert.equal(vars.task, "Do the thing");
-        assert.equal(vars.docs, "Documentation for task-1");
-    });
-    it("baseline test case has empty docs", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-        });
-        // Second test case is the baseline
-        assert.equal(result.config.tests[1].vars.docs, "");
-    });
-    it("builds providers from model registry", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels([
-                { id: "openai:chat:gpt-4o", label: "GPT-4o" },
-                {
-                    id: "anthropic:messages:claude-sonnet-4-6",
-                    label: "Claude Sonnet",
-                },
-            ]),
-            rootDir: tmpdir(),
-        });
-        assert.equal(result.config.providers.length, 2);
-        assert.equal(result.config.providers[0].id, "openai:chat:gpt-4o");
-        assert.equal(result.config.providers[1].id, "anthropic:messages:claude-sonnet-4-6");
-    });
-    it("filters models by mode", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels([
-                { id: "model-a", label: "A", modes: ["literacy"] },
-                { id: "model-b", label: "B", modes: ["mcp-server"] },
-            ]),
-            rootDir: tmpdir(),
-        });
-        assert.equal(result.config.providers.length, 1);
-        assert.equal(result.config.providers[0].id, "model-a");
-    });
-    it("generates default prompts for literacy mode", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-        });
-        assert.equal(result.config.prompts.length, 2);
-        assert.equal(result.config.prompts[0].id, "with-docs");
-        assert.equal(result.config.prompts[1].id, "without-docs");
-    });
-    it("generates default prompts for non-literacy modes", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "mcp-server",
-            models: makeModels(),
-            rootDir: tmpdir(),
-        });
-        assert.equal(result.config.prompts.length, 1);
-        assert.equal(result.config.prompts[0].id, "default");
-        // Non-literacy modes don't create baseline variants
-        assert.equal(result.config.tests.length, 1);
-    });
-    it("sets grader provider in defaultTest options", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-            graderProvider: "openai:chat:gpt-5",
-        });
-        assert.ok(result.config.defaultTest);
-        assert.equal(result.config.defaultTest.options?.provider, "openai:chat:gpt-5");
-    });
-    it("compiles multiple tasks in priority order", () => {
-        const nodeA = makeNode("task-a");
-        nodeA.priority = 0;
-        const nodeB = makeNode("task-b");
-        nodeB.priority = 1;
-        const graph = makeGraph([nodeB, nodeA]); // Intentionally reversed
-        const result = compileToPromptfoo(graph, {
-            mode: "mcp-server",
-            models: makeModels(),
-            rootDir: tmpdir(),
-        });
-        // Should be sorted by priority: task-a first, then task-b
-        assert.equal(result.config.tests[0].description, "task-a");
-        assert.equal(result.config.tests[1].description, "task-b");
-    });
-    it("sets output path when provided", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-            outputPath: "results/latest/eval-results.json",
-        });
-        assert.equal(result.config.outputPath, "results/latest/eval-results.json");
-    });
-    it("merges model defaults with per-model config", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: {
-                defaults: { temperature: 0.2, max_tokens: 4096 },
-                grader: { id: "gpt-5" },
-                models: [
-                    {
-                        id: "model-a",
-                        label: "A",
-                        config: { temperature: 0.5 },
-                    },
-                ],
-            },
-            rootDir: tmpdir(),
-        });
-        assert.equal(result.config.providers.length, 1);
-        const config = result.config.providers[0].config;
-        // Per-model config overrides defaults
-        assert.equal(config.temperature, 0.5);
-        assert.equal(config.max_tokens, 4096);
-    });
-});
-// ---------------------------------------------------------------------------
-// Handler-owned prompt resolution
-// ---------------------------------------------------------------------------
-describe("compileToPromptfoo — handler prompt resolution", () => {
-    /** Build a minimal handler that returns prompts from getPrompts() */
-    function makeHandlerWithPrompts(prompts) {
-        return {
-            compileTask() {
-                return { providers: [], tests: [], prompts: [], warnings: [] };
-            },
-            getPrompts() {
-                return prompts;
-            },
-        };
-    }
-    /** Build a handler that does NOT implement getPrompts */
-    function makeHandlerWithoutPrompts() {
-        return {
-            compileTask() {
-                return { providers: [], tests: [], prompts: [], warnings: [] };
-            },
-        };
-    }
-    it("uses handler prompts when handler.getPrompts() returns templates", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const handler = makeHandlerWithPrompts({
-            "custom-prompt": {
-                id: "custom-prompt",
-                label: "Custom Mode Prompt",
-                template: "Do the thing: {{task}}",
-                variables: ["task"],
-            },
-        });
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-            handler,
-        });
-        assert.equal(result.config.prompts.length, 1);
-        assert.equal(result.config.prompts[0].id, "custom-prompt");
-        assert.equal(result.config.prompts[0].label, "Custom Mode Prompt");
-        assert.equal(result.config.prompts[0].raw, "Do the thing: {{task}}");
-    });
-    it("falls back to default prompts when handler has no getPrompts()", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const handler = makeHandlerWithoutPrompts();
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-            handler,
-        });
-        // Should fall back to built-in literacy defaults
-        assert.equal(result.config.prompts.length, 2);
-        assert.equal(result.config.prompts[0].id, "with-docs");
-        assert.equal(result.config.prompts[1].id, "without-docs");
-    });
-    it("falls back to default prompts when no handler is provided", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const result = compileToPromptfoo(graph, {
-            mode: "mcp-server",
-            models: makeModels(),
-            rootDir: tmpdir(),
-            // No handler
-        });
-        // Non-literacy modes get generic default prompt
-        assert.equal(result.config.prompts.length, 1);
-        assert.equal(result.config.prompts[0].id, "default");
-    });
-    it("prefers handler prompts over explicit options.prompts", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        const handler = makeHandlerWithPrompts({
-            "handler-owned": {
-                id: "handler-owned",
-                label: "Handler Prompt",
-                template: "From handler: {{task}}",
-            },
-        });
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-            handler,
-            // Explicit prompts should be ignored when handler provides them
-            prompts: [
-                { id: "explicit", label: "Explicit", raw: "explicit {{task}}" },
-            ],
-        });
-        assert.equal(result.config.prompts.length, 1);
-        assert.equal(result.config.prompts[0].id, "handler-owned");
-    });
-    it("falls back to explicit prompts when handler.getPrompts() returns undefined", () => {
-        const graph = makeGraph([makeNode("task-1")]);
-        // Handler with getPrompts defined but returning undefined
-        const handler = {
-            compileTask() {
-                return { providers: [], tests: [], prompts: [], warnings: [] };
-            },
-            getPrompts() {
-                return undefined;
-            },
-        };
-        const explicitPrompts = [
-            { id: "explicit-p", label: "Explicit Prompt", raw: "explicit: {{task}}" },
-        ];
-        const result = compileToPromptfoo(graph, {
-            mode: "literacy",
-            models: makeModels(),
-            rootDir: tmpdir(),
-            handler,
-            prompts: explicitPrompts,
-        });
-        assert.equal(result.config.prompts.length, 1);
-        assert.equal(result.config.prompts[0].id, "explicit-p");
-    });
-});

package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts

@@ -1,12 +0,0 @@
-/**
- * sandbox-and-fixtures.test.ts — Tests for sandbox strategy + fixture provisioning.
- *
- * Tests TempDir sandbox (the universal fallback), fixture provisioning
- * pipeline stages, ignoreFields stripping, and sandbox selection logic.
- *
- * Docker and GitWorktree sandboxes are not tested here (they require
- * external dependencies). CI tests for Docker are in a separate workflow.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/sandbox-and-fixtures.test.ts
- */
-export {};

package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js

@@ -1,210 +0,0 @@
-/**
- * sandbox-and-fixtures.test.ts — Tests for sandbox strategy + fixture provisioning.
- *
- * Tests TempDir sandbox (the universal fallback), fixture provisioning
- * pipeline stages, ignoreFields stripping, and sandbox selection logic.
- *
- * Docker and GitWorktree sandboxes are not tested here (they require
- * external dependencies). CI tests for Docker are in a separate workflow.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/sandbox-and-fixtures.test.ts
- */
-import assert from "node:assert/strict";
-import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import { tmpdir } from "os";
-import { resolve } from "path";
-import { TempDirSandboxStrategy } from "../sandbox/tempdir-sandbox.js";
-import { selectSandboxStrategy } from "../sandbox/sandbox-selector.js";
-import { provisionFixtures } from "../sandbox/fixture-provisioner.js";
-import { stripFields, compareWithIgnoredFields, buildIgnoreFieldsWrapper, } from "../ignore-fields.js";
-// ---------------------------------------------------------------------------
-// TempDirSandboxStrategy
-// ---------------------------------------------------------------------------
-describe("TempDirSandboxStrategy", () => {
-    const strategy = new TempDirSandboxStrategy();
-    it("is always available", async () => {
-        assert.equal(await strategy.isAvailable(), true);
-    });
-    it("provisions a sandbox with a working directory", async () => {
-        const sandbox = await strategy.provision({
-            type: "tempdir",
-            taskId: "test-task",
-        });
-        assert.ok(sandbox.id.startsWith("ailf-sandbox-"));
-        assert.ok(existsSync(sandbox.workingDir));
-        assert.equal(sandbox.strategy, "tempdir");
-        // Clean up
-        await strategy.teardown(sandbox);
-        assert.ok(!existsSync(sandbox.workingDir));
-    });
-    it("collects artifacts from sandbox", async () => {
-        const sandbox = await strategy.provision({
-            type: "tempdir",
-            taskId: "artifact-test",
-        });
-        // Create some files
-        writeFileSync(resolve(sandbox.workingDir, "file1.ts"), "export const x = 1");
-        mkdirSync(resolve(sandbox.workingDir, "subdir"), { recursive: true });
-        writeFileSync(resolve(sandbox.workingDir, "subdir", "file2.ts"), "export const y = 2");
-        const artifacts = await strategy.collectArtifacts(sandbox);
-        assert.ok(artifacts.modifiedFiles.includes("file1.ts"));
-        assert.ok(artifacts.modifiedFiles.includes("subdir/file2.ts"));
-        assert.ok(artifacts.durationMs >= 0);
-        await strategy.teardown(sandbox);
-    });
-});
-// ---------------------------------------------------------------------------
-// selectSandboxStrategy
-// ---------------------------------------------------------------------------
-describe("selectSandboxStrategy", () => {
-    it("returns TempDir as universal fallback", async () => {
-        // Force fallback by requesting an unlikely-available strategy
-        const result = await selectSandboxStrategy("tempdir");
-        assert.equal(result.strategy.type, "tempdir");
-        assert.equal(result.isFallback, false);
-    });
-    it("reports fallback when preferred strategy is unavailable", async () => {
-        // git-worktree is available when git is installed, but docker may not be
-        // Use tempdir as a reliable test
-        const result = await selectSandboxStrategy("tempdir");
-        assert.equal(result.isFallback, false);
-    });
-});
-// ---------------------------------------------------------------------------
-// Fixture provisioning
-// ---------------------------------------------------------------------------
-const FIXTURE_DIR = resolve(tmpdir(), `ailf-fixture-test-${process.pid}`);
-beforeEach(() => {
-    mkdirSync(FIXTURE_DIR, { recursive: true });
-});
-afterEach(() => {
-    rmSync(FIXTURE_DIR, { recursive: true, force: true });
-});
-describe("provisionFixtures", () => {
-    it("resolves file:// fixtures", async () => {
-        writeFileSync(resolve(FIXTURE_DIR, "test.md"), "# Hello World");
-        const result = await provisionFixtures([{ uri: "file://test.md", inject: "vars", key: "docs" }], { rootDir: FIXTURE_DIR });
-        assert.equal(result.fixtures.length, 1);
-        assert.equal(result.fixtures[0].content, "# Hello World");
-        assert.equal(result.vars.docs, "# Hello World");
-        assert.ok(result.manifest["file://test.md"]);
-    });
-    it("injects fixtures into sandbox working directory", async () => {
-        writeFileSync(resolve(FIXTURE_DIR, "schema.ts"), "export default {}");
-        const sandbox = await new TempDirSandboxStrategy().provision({
-            type: "tempdir",
-            taskId: "inject-test",
-        });
-        const result = await provisionFixtures([
-            {
-                uri: "file://schema.ts",
-                inject: "working_dir",
-                key: "src/schema.ts",
-            },
-        ], { rootDir: FIXTURE_DIR, sandbox });
-        assert.equal(result.fixtures.length, 1);
-        const injectedPath = resolve(sandbox.workingDir, "src", "schema.ts");
-        assert.ok(existsSync(injectedPath));
-        assert.equal(readFileSync(injectedPath, "utf-8"), "export default {}");
-        rmSync(sandbox.workingDir, { recursive: true, force: true });
-    });
-    it("warns on missing file:// fixture", async () => {
-        const result = await provisionFixtures([{ uri: "file://nonexistent.md", inject: "vars", key: "docs" }], { rootDir: FIXTURE_DIR });
-        assert.equal(result.fixtures.length, 0);
-        assert.ok(result.warnings.some((w) => w.includes("not found")));
-    });
-    it("applies strip-html transform", async () => {
-        writeFileSync(resolve(FIXTURE_DIR, "page.html"), "<h1>Title</h1><p>Body text</p>");
-        const result = await provisionFixtures([
-            {
-                uri: "file://page.html",
-                inject: "vars",
-                key: "content",
-                transform: "strip-html",
-            },
-        ], { rootDir: FIXTURE_DIR });
-        assert.equal(result.vars.content, "TitleBody text");
-    });
-    it("applies truncate transform", async () => {
-        const longContent = "x".repeat(20_000);
-        writeFileSync(resolve(FIXTURE_DIR, "long.txt"), longContent);
-        const result = await provisionFixtures([
-            {
-                uri: "file://long.txt",
-                inject: "vars",
-                key: "docs",
-                transform: "truncate",
-            },
-        ], { rootDir: FIXTURE_DIR });
-        const content = result.vars.docs;
-        assert.ok(content.length < 20_000);
-        assert.ok(content.includes("truncated"));
-    });
-    it("produces content hash manifest", async () => {
-        writeFileSync(resolve(FIXTURE_DIR, "a.md"), "alpha");
-        writeFileSync(resolve(FIXTURE_DIR, "b.md"), "beta");
-        const result = await provisionFixtures([
-            { uri: "file://a.md", inject: "vars", key: "a" },
-            { uri: "file://b.md", inject: "vars", key: "b" },
-        ], { rootDir: FIXTURE_DIR });
-        assert.equal(Object.keys(result.manifest).length, 2);
-        assert.ok(result.manifest["file://a.md"]);
-        assert.ok(result.manifest["file://b.md"]);
-        // Different content → different hashes
-        assert.notEqual(result.manifest["file://a.md"], result.manifest["file://b.md"]);
-    });
-});
-// ---------------------------------------------------------------------------
-// stripFields / ignoreFields
-// ---------------------------------------------------------------------------
-describe("stripFields", () => {
-    it("removes a top-level field", () => {
-        const result = stripFields({ a: 1, b: 2, c: 3 }, ["b"]);
-        assert.deepEqual(result, { a: 1, c: 3 });
-    });
-    it("removes a nested field", () => {
-        const result = stripFields({ meta: { createdAt: "2024-01-01", author: "Alice" }, title: "Hello" }, ["meta.createdAt"]);
-        assert.deepEqual(result, { meta: { author: "Alice" }, title: "Hello" });
-    });
-    it("removes multiple fields", () => {
-        const result = stripFields({ a: 1, b: 2, c: { d: 3, e: 4 } }, ["b", "c.d"]);
-        assert.deepEqual(result, { a: 1, c: { e: 4 } });
-    });
-    it("handles non-existent fields gracefully", () => {
-        const result = stripFields({ a: 1 }, ["b", "c.d"]);
-        assert.deepEqual(result, { a: 1 });
-    });
-    it("handles null/undefined input", () => {
-        assert.equal(stripFields(null, ["a"]), null);
-        assert.equal(stripFields(undefined, ["a"]), undefined);
-    });
-    it("does not mutate the original object", () => {
-        const original = { a: 1, b: 2 };
-        stripFields(original, ["b"]);
-        assert.deepEqual(original, { a: 1, b: 2 });
-    });
-});
-describe("compareWithIgnoredFields", () => {
-    it("returns true when objects match after stripping", () => {
-        const result = compareWithIgnoredFields({ title: "Hello", _rev: "abc123", _updatedAt: "2024-01-01" }, { title: "Hello", _rev: "def456", _updatedAt: "2024-06-01" }, ["_rev", "_updatedAt"]);
-        assert.equal(result, true);
-    });
-    it("returns false when objects differ in non-ignored fields", () => {
-        const result = compareWithIgnoredFields({ title: "Hello", _rev: "abc123" }, { title: "World", _rev: "def456" }, ["_rev"]);
-        assert.equal(result, false);
-    });
-});
-describe("buildIgnoreFieldsWrapper", () => {
-    it("returns original code when no fields to ignore", () => {
-        const code = "return true";
-        assert.equal(buildIgnoreFieldsWrapper(code, []), code);
-    });
-    it("wraps code with stripFields function", () => {
-        const wrapped = buildIgnoreFieldsWrapper("// check", ["_rev", "meta.ts"]);
-        assert.ok(wrapped.includes("function stripFields"));
-        assert.ok(wrapped.includes("__ignoreFields"));
-        assert.ok(wrapped.includes('"_rev"'));
-        assert.ok(wrapped.includes('"meta.ts"'));
-    });
-});

package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts

@@ -1,7 +0,0 @@
-/**
- * scoring-and-presets.test.ts — Tests for 4-tier scoring engine,
- * storage schema, and plugin registry / presets.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/scoring-and-presets.test.ts
- */
-export {};