@sanity/ailf 0.4.1 → 1.0.0

package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js

@@ -0,0 +1,333 @@
+/**
+ * promptfoo-compiler.test.ts — Unit tests for the PromptfooCompiler.
+ *
+ * Tests compilation of a TaskGraph into Promptfoo configuration,
+ * including provider assembly, prompt generation, and test case creation.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/promptfoo-compiler.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { tmpdir } from "os";
+import { LiteracyVariant } from "../../normalize-mode.js";
+import { compileToPromptfoo } from "../promptfoo-compiler.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function makeModels(models = []) {
+    return {
+        defaults: { temperature: 0.2 },
+        grader: { id: "openai:chat:gpt-5", label: "GPT-5" },
+        models: models.map((m) => ({
+            config: {},
+            id: m.id,
+            label: m.label,
+            modes: m.modes,
+        })),
+    };
+}
+function makeNode(id, prompt) {
+    return {
+        dependsOn: [],
+        priority: 0,
+        resolvedPrompt: prompt ?? `Implement ${id}`,
+        resolvedVariables: {
+            declarations: [],
+            provenance: {},
+            values: {
+                task: prompt ?? `Implement ${id}`,
+                docs: `Documentation for ${id}`,
+            },
+        },
+        taskId: id,
+    };
+}
+function makeGraph(nodes, edges = []) {
+    const nodeMap = new Map();
+    for (const n of nodes)
+        nodeMap.set(n.taskId, n);
+    return {
+        compilationTarget: "promptfoo",
+        edges,
+        fixtures: new Map(),
+        nodes: nodeMap,
+    };
+}
+// ---------------------------------------------------------------------------
+// compileToPromptfoo — basic compilation
+// ---------------------------------------------------------------------------
+describe("compileToPromptfoo", () => {
+    it("compiles a single-task graph into a Promptfoo config", () => {
+        const graph = makeGraph([makeNode("groq-basics")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels([{ id: "openai:chat:gpt-4o", label: "GPT-4o" }]),
+            rootDir: tmpdir(),
+        });
+        assert.ok(result.config);
+        assert.equal(result.taskCount, 1);
+        // Literacy mode produces gold + baseline test cases
+        assert.equal(result.testCaseCount, 2);
+        assert.equal(result.config.tests.length, 2);
+    });
+    it("generates correct test case descriptions", () => {
+        const graph = makeGraph([makeNode("groq-basics")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+        });
+        assert.equal(result.config.tests[0].description, "groq-basics");
+        assert.equal(result.config.tests[1].description, "groq-basics [baseline]");
+    });
+    it("includes task variables in test case vars", () => {
+        const graph = makeGraph([makeNode("task-1", "Do the thing")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+        });
+        const vars = result.config.tests[0].vars;
+        assert.equal(vars.task, "Do the thing");
+        assert.equal(vars.docs, "Documentation for task-1");
+    });
+    it("baseline test case has empty docs", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+        });
+        // Second test case is the baseline
+        assert.equal(result.config.tests[1].vars.docs, "");
+    });
+    it("builds providers from model registry", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels([
+                { id: "openai:chat:gpt-4o", label: "GPT-4o" },
+                {
+                    id: "anthropic:messages:claude-sonnet-4-6",
+                    label: "Claude Sonnet",
+                },
+            ]),
+            rootDir: tmpdir(),
+        });
+        assert.equal(result.config.providers.length, 2);
+        assert.equal(result.config.providers[0].id, "openai:chat:gpt-4o");
+        assert.equal(result.config.providers[1].id, "anthropic:messages:claude-sonnet-4-6");
+    });
+    it("filters models by mode", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels([
+                { id: "model-a", label: "A", modes: [LiteracyVariant.STANDARD] },
+                { id: "model-b", label: "B", modes: ["agentic-naive"] },
+            ]),
+            rootDir: tmpdir(),
+        });
+        assert.equal(result.config.providers.length, 1);
+        assert.equal(result.config.providers[0].id, "model-a");
+    });
+    it("generates default prompts for literacy mode", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+        });
+        assert.equal(result.config.prompts.length, 2);
+        assert.equal(result.config.prompts[0].id, "with-docs");
+        assert.equal(result.config.prompts[1].id, "without-docs");
+    });
+    it("generates default prompts for non-literacy modes", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "mcp-server",
+            models: makeModels(),
+            rootDir: tmpdir(),
+        });
+        assert.equal(result.config.prompts.length, 1);
+        assert.equal(result.config.prompts[0].id, "default");
+        // Non-literacy modes don't create baseline variants
+        assert.equal(result.config.tests.length, 1);
+    });
+    it("sets grader provider in defaultTest options", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+            graderProvider: "openai:chat:gpt-5",
+        });
+        assert.ok(result.config.defaultTest);
+        assert.equal(result.config.defaultTest.options?.provider, "openai:chat:gpt-5");
+    });
+    it("compiles multiple tasks in priority order", () => {
+        const nodeA = makeNode("task-a");
+        nodeA.priority = 0;
+        const nodeB = makeNode("task-b");
+        nodeB.priority = 1;
+        const graph = makeGraph([nodeB, nodeA]); // Intentionally reversed
+        const result = compileToPromptfoo(graph, {
+            mode: "mcp-server",
+            models: makeModels(),
+            rootDir: tmpdir(),
+        });
+        // Should be sorted by priority: task-a first, then task-b
+        assert.equal(result.config.tests[0].description, "task-a");
+        assert.equal(result.config.tests[1].description, "task-b");
+    });
+    it("sets output path when provided", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+            outputPath: "results/latest/eval-results.json",
+        });
+        assert.equal(result.config.outputPath, "results/latest/eval-results.json");
+    });
+    it("merges model defaults with per-model config", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: {
+                defaults: { temperature: 0.2, max_tokens: 4096 },
+                grader: { id: "gpt-5" },
+                models: [
+                    {
+                        id: "model-a",
+                        label: "A",
+                        config: { temperature: 0.5 },
+                    },
+                ],
+            },
+            rootDir: tmpdir(),
+        });
+        assert.equal(result.config.providers.length, 1);
+        const config = result.config.providers[0].config;
+        // Per-model config overrides defaults
+        assert.equal(config.temperature, 0.5);
+        assert.equal(config.max_tokens, 4096);
+    });
+});
+// ---------------------------------------------------------------------------
+// Handler-owned prompt resolution
+// ---------------------------------------------------------------------------
+describe("compileToPromptfoo — handler prompt resolution", () => {
+    /** Build a minimal handler that returns prompts from getPrompts() */
+    function makeHandlerWithPrompts(prompts) {
+        return {
+            compileTask() {
+                return { providers: [], tests: [], prompts: [], warnings: [] };
+            },
+            getPrompts() {
+                return prompts;
+            },
+        };
+    }
+    /** Build a handler that does NOT implement getPrompts */
+    function makeHandlerWithoutPrompts() {
+        return {
+            compileTask() {
+                return { providers: [], tests: [], prompts: [], warnings: [] };
+            },
+        };
+    }
+    it("uses handler prompts when handler.getPrompts() returns templates", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const handler = makeHandlerWithPrompts({
+            "custom-prompt": {
+                id: "custom-prompt",
+                label: "Custom Mode Prompt",
+                template: "Do the thing: {{task}}",
+                variables: ["task"],
+            },
+        });
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+            handler,
+        });
+        assert.equal(result.config.prompts.length, 1);
+        assert.equal(result.config.prompts[0].id, "custom-prompt");
+        assert.equal(result.config.prompts[0].label, "Custom Mode Prompt");
+        assert.equal(result.config.prompts[0].raw, "Do the thing: {{task}}");
+    });
+    it("falls back to default prompts when handler has no getPrompts()", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const handler = makeHandlerWithoutPrompts();
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+            handler,
+        });
+        // Should fall back to built-in literacy defaults
+        assert.equal(result.config.prompts.length, 2);
+        assert.equal(result.config.prompts[0].id, "with-docs");
+        assert.equal(result.config.prompts[1].id, "without-docs");
+    });
+    it("falls back to default prompts when no handler is provided", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const result = compileToPromptfoo(graph, {
+            mode: "mcp-server",
+            models: makeModels(),
+            rootDir: tmpdir(),
+            // No handler
+        });
+        // Non-literacy modes get generic default prompt
+        assert.equal(result.config.prompts.length, 1);
+        assert.equal(result.config.prompts[0].id, "default");
+    });
+    it("prefers handler prompts over explicit options.prompts", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        const handler = makeHandlerWithPrompts({
+            "handler-owned": {
+                id: "handler-owned",
+                label: "Handler Prompt",
+                template: "From handler: {{task}}",
+            },
+        });
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+            handler,
+            // Explicit prompts should be ignored when handler provides them
+            prompts: [
+                { id: "explicit", label: "Explicit", raw: "explicit {{task}}" },
+            ],
+        });
+        assert.equal(result.config.prompts.length, 1);
+        assert.equal(result.config.prompts[0].id, "handler-owned");
+    });
+    it("falls back to explicit prompts when handler.getPrompts() returns undefined", () => {
+        const graph = makeGraph([makeNode("task-1")]);
+        // Handler with getPrompts defined but returning undefined
+        const handler = {
+            compileTask() {
+                return { providers: [], tests: [], prompts: [], warnings: [] };
+            },
+            getPrompts() {
+                return undefined;
+            },
+        };
+        const explicitPrompts = [
+            { id: "explicit-p", label: "Explicit Prompt", raw: "explicit: {{task}}" },
+        ];
+        const result = compileToPromptfoo(graph, {
+            mode: "literacy",
+            models: makeModels(),
+            rootDir: tmpdir(),
+            handler,
+            prompts: explicitPrompts,
+        });
+        assert.equal(result.config.prompts.length, 1);
+        assert.equal(result.config.prompts[0].id, "explicit-p");
+    });
+});

package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts

@@ -0,0 +1,12 @@
+/**
+ * sandbox-and-fixtures.test.ts — Tests for sandbox strategy + fixture provisioning.
+ *
+ * Tests TempDir sandbox (the universal fallback), fixture provisioning
+ * pipeline stages, ignoreFields stripping, and sandbox selection logic.
+ *
+ * Docker and GitWorktree sandboxes are not tested here (they require
+ * external dependencies). CI tests for Docker are in a separate workflow.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/sandbox-and-fixtures.test.ts
+ */
+export {};

package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js

@@ -0,0 +1,210 @@
+/**
+ * sandbox-and-fixtures.test.ts — Tests for sandbox strategy + fixture provisioning.
+ *
+ * Tests TempDir sandbox (the universal fallback), fixture provisioning
+ * pipeline stages, ignoreFields stripping, and sandbox selection logic.
+ *
+ * Docker and GitWorktree sandboxes are not tested here (they require
+ * external dependencies). CI tests for Docker are in a separate workflow.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/sandbox-and-fixtures.test.ts
+ */
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { tmpdir } from "os";
+import { resolve } from "path";
+import { TempDirSandboxStrategy } from "../sandbox/tempdir-sandbox.js";
+import { selectSandboxStrategy } from "../sandbox/sandbox-selector.js";
+import { provisionFixtures } from "../sandbox/fixture-provisioner.js";
+import { stripFields, compareWithIgnoredFields, buildIgnoreFieldsWrapper, } from "../ignore-fields.js";
+// ---------------------------------------------------------------------------
+// TempDirSandboxStrategy
+// ---------------------------------------------------------------------------
+describe("TempDirSandboxStrategy", () => {
+    const strategy = new TempDirSandboxStrategy();
+    it("is always available", async () => {
+        assert.equal(await strategy.isAvailable(), true);
+    });
+    it("provisions a sandbox with a working directory", async () => {
+        const sandbox = await strategy.provision({
+            type: "tempdir",
+            taskId: "test-task",
+        });
+        assert.ok(sandbox.id.startsWith("ailf-sandbox-"));
+        assert.ok(existsSync(sandbox.workingDir));
+        assert.equal(sandbox.strategy, "tempdir");
+        // Clean up
+        await strategy.teardown(sandbox);
+        assert.ok(!existsSync(sandbox.workingDir));
+    });
+    it("collects artifacts from sandbox", async () => {
+        const sandbox = await strategy.provision({
+            type: "tempdir",
+            taskId: "artifact-test",
+        });
+        // Create some files
+        writeFileSync(resolve(sandbox.workingDir, "file1.ts"), "export const x = 1");
+        mkdirSync(resolve(sandbox.workingDir, "subdir"), { recursive: true });
+        writeFileSync(resolve(sandbox.workingDir, "subdir", "file2.ts"), "export const y = 2");
+        const artifacts = await strategy.collectArtifacts(sandbox);
+        assert.ok(artifacts.modifiedFiles.includes("file1.ts"));
+        assert.ok(artifacts.modifiedFiles.includes("subdir/file2.ts"));
+        assert.ok(artifacts.durationMs >= 0);
+        await strategy.teardown(sandbox);
+    });
+});
+// ---------------------------------------------------------------------------
+// selectSandboxStrategy
+// ---------------------------------------------------------------------------
+describe("selectSandboxStrategy", () => {
+    it("returns TempDir as universal fallback", async () => {
+        // Force fallback by requesting an unlikely-available strategy
+        const result = await selectSandboxStrategy("tempdir");
+        assert.equal(result.strategy.type, "tempdir");
+        assert.equal(result.isFallback, false);
+    });
+    it("reports fallback when preferred strategy is unavailable", async () => {
+        // git-worktree is available when git is installed, but docker may not be
+        // Use tempdir as a reliable test
+        const result = await selectSandboxStrategy("tempdir");
+        assert.equal(result.isFallback, false);
+    });
+});
+// ---------------------------------------------------------------------------
+// Fixture provisioning
+// ---------------------------------------------------------------------------
+const FIXTURE_DIR = resolve(tmpdir(), `ailf-fixture-test-${process.pid}`);
+beforeEach(() => {
+    mkdirSync(FIXTURE_DIR, { recursive: true });
+});
+afterEach(() => {
+    rmSync(FIXTURE_DIR, { recursive: true, force: true });
+});
+describe("provisionFixtures", () => {
+    it("resolves file:// fixtures", async () => {
+        writeFileSync(resolve(FIXTURE_DIR, "test.md"), "# Hello World");
+        const result = await provisionFixtures([{ uri: "file://test.md", inject: "vars", key: "docs" }], { rootDir: FIXTURE_DIR });
+        assert.equal(result.fixtures.length, 1);
+        assert.equal(result.fixtures[0].content, "# Hello World");
+        assert.equal(result.vars.docs, "# Hello World");
+        assert.ok(result.manifest["file://test.md"]);
+    });
+    it("injects fixtures into sandbox working directory", async () => {
+        writeFileSync(resolve(FIXTURE_DIR, "schema.ts"), "export default {}");
+        const sandbox = await new TempDirSandboxStrategy().provision({
+            type: "tempdir",
+            taskId: "inject-test",
+        });
+        const result = await provisionFixtures([
+            {
+                uri: "file://schema.ts",
+                inject: "working_dir",
+                key: "src/schema.ts",
+            },
+        ], { rootDir: FIXTURE_DIR, sandbox });
+        assert.equal(result.fixtures.length, 1);
+        const injectedPath = resolve(sandbox.workingDir, "src", "schema.ts");
+        assert.ok(existsSync(injectedPath));
+        assert.equal(readFileSync(injectedPath, "utf-8"), "export default {}");
+        rmSync(sandbox.workingDir, { recursive: true, force: true });
+    });
+    it("warns on missing file:// fixture", async () => {
+        const result = await provisionFixtures([{ uri: "file://nonexistent.md", inject: "vars", key: "docs" }], { rootDir: FIXTURE_DIR });
+        assert.equal(result.fixtures.length, 0);
+        assert.ok(result.warnings.some((w) => w.includes("not found")));
+    });
+    it("applies strip-html transform", async () => {
+        writeFileSync(resolve(FIXTURE_DIR, "page.html"), "<h1>Title</h1><p>Body text</p>");
+        const result = await provisionFixtures([
+            {
+                uri: "file://page.html",
+                inject: "vars",
+                key: "content",
+                transform: "strip-html",
+            },
+        ], { rootDir: FIXTURE_DIR });
+        assert.equal(result.vars.content, "TitleBody text");
+    });
+    it("applies truncate transform", async () => {
+        const longContent = "x".repeat(20_000);
+        writeFileSync(resolve(FIXTURE_DIR, "long.txt"), longContent);
+        const result = await provisionFixtures([
+            {
+                uri: "file://long.txt",
+                inject: "vars",
+                key: "docs",
+                transform: "truncate",
+            },
+        ], { rootDir: FIXTURE_DIR });
+        const content = result.vars.docs;
+        assert.ok(content.length < 20_000);
+        assert.ok(content.includes("truncated"));
+    });
+    it("produces content hash manifest", async () => {
+        writeFileSync(resolve(FIXTURE_DIR, "a.md"), "alpha");
+        writeFileSync(resolve(FIXTURE_DIR, "b.md"), "beta");
+        const result = await provisionFixtures([
+            { uri: "file://a.md", inject: "vars", key: "a" },
+            { uri: "file://b.md", inject: "vars", key: "b" },
+        ], { rootDir: FIXTURE_DIR });
+        assert.equal(Object.keys(result.manifest).length, 2);
+        assert.ok(result.manifest["file://a.md"]);
+        assert.ok(result.manifest["file://b.md"]);
+        // Different content → different hashes
+        assert.notEqual(result.manifest["file://a.md"], result.manifest["file://b.md"]);
+    });
+});
+// ---------------------------------------------------------------------------
+// stripFields / ignoreFields
+// ---------------------------------------------------------------------------
+describe("stripFields", () => {
+    it("removes a top-level field", () => {
+        const result = stripFields({ a: 1, b: 2, c: 3 }, ["b"]);
+        assert.deepEqual(result, { a: 1, c: 3 });
+    });
+    it("removes a nested field", () => {
+        const result = stripFields({ meta: { createdAt: "2024-01-01", author: "Alice" }, title: "Hello" }, ["meta.createdAt"]);
+        assert.deepEqual(result, { meta: { author: "Alice" }, title: "Hello" });
+    });
+    it("removes multiple fields", () => {
+        const result = stripFields({ a: 1, b: 2, c: { d: 3, e: 4 } }, ["b", "c.d"]);
+        assert.deepEqual(result, { a: 1, c: { e: 4 } });
+    });
+    it("handles non-existent fields gracefully", () => {
+        const result = stripFields({ a: 1 }, ["b", "c.d"]);
+        assert.deepEqual(result, { a: 1 });
+    });
+    it("handles null/undefined input", () => {
+        assert.equal(stripFields(null, ["a"]), null);
+        assert.equal(stripFields(undefined, ["a"]), undefined);
+    });
+    it("does not mutate the original object", () => {
+        const original = { a: 1, b: 2 };
+        stripFields(original, ["b"]);
+        assert.deepEqual(original, { a: 1, b: 2 });
+    });
+});
+describe("compareWithIgnoredFields", () => {
+    it("returns true when objects match after stripping", () => {
+        const result = compareWithIgnoredFields({ title: "Hello", _rev: "abc123", _updatedAt: "2024-01-01" }, { title: "Hello", _rev: "def456", _updatedAt: "2024-06-01" }, ["_rev", "_updatedAt"]);
+        assert.equal(result, true);
+    });
+    it("returns false when objects differ in non-ignored fields", () => {
+        const result = compareWithIgnoredFields({ title: "Hello", _rev: "abc123" }, { title: "World", _rev: "def456" }, ["_rev"]);
+        assert.equal(result, false);
+    });
+});
+describe("buildIgnoreFieldsWrapper", () => {
+    it("returns original code when no fields to ignore", () => {
+        const code = "return true";
+        assert.equal(buildIgnoreFieldsWrapper(code, []), code);
+    });
+    it("wraps code with stripFields function", () => {
+        const wrapped = buildIgnoreFieldsWrapper("// check", ["_rev", "meta.ts"]);
+        assert.ok(wrapped.includes("function stripFields"));
+        assert.ok(wrapped.includes("__ignoreFields"));
+        assert.ok(wrapped.includes('"_rev"'));
+        assert.ok(wrapped.includes('"meta.ts"'));
+    });
+});

package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts

@@ -0,0 +1,7 @@
+/**
+ * scoring-and-presets.test.ts — Tests for 4-tier scoring engine,
+ * storage schema, and plugin registry / presets.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/scoring-and-presets.test.ts
+ */
+export {};