@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.js

@@ -0,0 +1,259 @@
+/**
+ * Example MCP server task definitions — Sanity MCP Server reference.
+ *
+ * These are reference implementations showing how to author MCP server
+ * evaluation tasks. They model the real Sanity MCP Server (mcp.sanity.io)
+ * and its actual tool names. They serve as:
+ *
+ * 1. Documentation — developers learn the API by reading examples
+ * 2. Test fixtures — the compiler tests compile these and verify output
+ * 3. Validation — proves the compiler handles real-world task shapes
+ * 4. Reference — shows all transport types, auth patterns, and assertion styles
+ *
+ * NOTE: These are NOT executable without a valid SANITY_MCP_AUTH_TOKEN.
+ * They exist purely for compilation testing and as authoring examples.
+ * For real evaluation, write tasks in an external repo's .ailf/tasks/
+ * directory or a standalone config file.
+ *
+ * @see https://www.sanity.io/docs/ai/mcp-server — Sanity MCP Server docs
+ * @see https://www.promptfoo.dev/docs/providers/mcp/ — Promptfoo MCP provider
+ */
+// ---------------------------------------------------------------------------
+// Shared server config — reusable across tasks targeting the same server
+// ---------------------------------------------------------------------------
+const SANITY_MCP_SERVER = {
+    transport: "streamable-http",
+    url: "https://mcp.sanity.io",
+    auth: {
+        type: "bearer",
+        token: "{{env.SANITY_MCP_AUTH_TOKEN}}",
+    },
+};
+// ---------------------------------------------------------------------------
+// Task 1: GROQ query via query_documents tool
+// ---------------------------------------------------------------------------
+/**
+ * Tests that a model can construct a valid GROQ query and use the
+ * query_documents tool to fetch results from a Sanity dataset.
+ */
+export const queryDocumentsTask = {
+    mode: "mcp-server",
+    id: "mcp-query-documents",
+    title: "MCP: Query documents with GROQ",
+    description: "Use the Sanity MCP server's query_documents tool to fetch " +
+        "documents from a dataset using GROQ. Verify the model constructs " +
+        "a valid GROQ query and interprets the response correctly.",
+    area: "mcp",
+    difficulty: "intermediate",
+    tags: ["mcp", "groq", "query"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: ["query_documents", "get_schema"],
+    prompt: {
+        text: "Using the Sanity MCP server, query for all documents of type " +
+            '"article" in the production dataset. Return their titles and ' +
+            "slugs, ordered by creation date descending. Limit to 5 results.",
+    },
+    assertions: [
+        { type: "tool-called", value: "query_documents" },
+        { type: "contains", value: "_type" },
+        { type: "contains", value: "article" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Constructs a valid GROQ query with _type filter",
+                "Includes a projection with title and slug fields",
+                "Orders by _createdAt descending",
+                "Limits results to 5 with slice syntax [0...5] or [0..4]",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 2: Schema inspection via get_schema tool
+// ---------------------------------------------------------------------------
+/**
+ * Tests that a model can retrieve and interpret schema information
+ * from the MCP server.
+ */
+export const inspectSchemaTask = {
+    mode: "mcp-server",
+    id: "mcp-get-schema",
+    title: "MCP: Inspect schema and describe document types",
+    description: "Use the Sanity MCP server's get_schema tool to inspect the " +
+        "content model and describe the available document types.",
+    area: "mcp",
+    difficulty: "basic",
+    tags: ["mcp", "schema"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: ["get_schema"],
+    prompt: {
+        text: "Using the Sanity MCP server, retrieve the schema for the current " +
+            "workspace. List all document types and describe the fields of the " +
+            'most complex one. Focus on reference fields and "block" (Portable Text) fields.',
+    },
+    assertions: [
+        { type: "tool-called", value: "get_schema" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Successfully calls get_schema to retrieve the workspace schema",
+                "Identifies and lists the document types",
+                "Describes field structures including types and references",
+                "Correctly identifies Portable Text (block) fields",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 3: Multi-step create → verify → publish (multi-turn)
+// ---------------------------------------------------------------------------
+/**
+ * Tests a multi-step document lifecycle workflow using multiple tools
+ * across a multi-turn conversation. This is the most complex example,
+ * demonstrating tool chaining and verification patterns.
+ */
+export const createAndPublishTask = {
+    mode: "mcp-server",
+    id: "mcp-create-publish",
+    title: "MCP: Create a draft document and publish it",
+    description: "Use the Sanity MCP server to create a new draft document, " +
+        "verify it was created, then publish it. Tests the multi-step " +
+        "workflow of create → verify → publish.",
+    area: "mcp",
+    difficulty: "advanced",
+    tags: ["mcp", "crud", "multi-step"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: [
+        "create_documents_from_json",
+        "get_document",
+        "publish_documents",
+        "query_documents",
+    ],
+    multiTurn: {
+        turns: [
+            {
+                role: "user",
+                content: "Create a new draft article document with the title " +
+                    '"MCP Integration Test" and a slug of "mcp-integration-test". ' +
+                    "Use the create_documents_from_json tool.",
+            },
+            {
+                role: "user",
+                content: "Now verify the draft was created by querying for it. " +
+                    "Use query_documents with a GROQ query that fetches the " +
+                    'document by its slug "mcp-integration-test".',
+            },
+            {
+                role: "user",
+                content: "Finally, publish the draft document using the publish_documents tool.",
+            },
+        ],
+    },
+    assertions: [
+        { type: "tool-called", value: "create_documents_from_json" },
+        { type: "tool-called", value: "query_documents" },
+        { type: "tool-called", value: "publish_documents" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Creates a draft document with the correct _type and fields",
+                "Verifies the draft exists via GROQ query",
+                "Publishes the draft using the correct document ID",
+                "Handles the multi-step workflow in the correct order",
+            ],
+        },
+        {
+            type: "llm-rubric",
+            template: "mcp-error-handling",
+            criteria: [
+                "Handles potential errors (missing type, invalid schema) gracefully",
+                "Verifies each step before proceeding to the next",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 4: Semantic search (embeddings)
+// ---------------------------------------------------------------------------
+/**
+ * Tests that a model can discover embeddings indices and perform
+ * semantic search against them.
+ */
+export const semanticSearchTask = {
+    mode: "mcp-server",
+    id: "mcp-semantic-search",
+    title: "MCP: Semantic search for related content",
+    description: "Use the Sanity MCP server's semantic_search tool to find " +
+        "content related to a natural language query.",
+    area: "mcp",
+    difficulty: "intermediate",
+    tags: ["mcp", "search", "embeddings"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: ["semantic_search", "list_embeddings_indices", "get_document"],
+    prompt: {
+        text: "Using the Sanity MCP server, first list the available embeddings " +
+            "indices, then perform a semantic search for content related to " +
+            '"how to set up real-time previews in Next.js". Return the top 3 ' +
+            "most relevant documents with their titles and relevance scores.",
+    },
+    assertions: [
+        { type: "tool-called", value: "list_embeddings_indices" },
+        { type: "tool-called", value: "semantic_search" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Lists available embeddings indices before searching",
+                "Selects an appropriate index for the search",
+                "Constructs a meaningful search query",
+                "Returns relevant results with titles and scores",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 5: Local stdio server (shows command-based transport)
+// ---------------------------------------------------------------------------
+/**
+ * Demonstrates a stdio-based MCP server (local process) for contrast
+ * with the remote streamable-http tasks above.
+ */
+export const stdioServerTask = {
+    mode: "mcp-server",
+    id: "mcp-stdio-local",
+    title: "MCP: Local stdio server — getDocument",
+    description: "Call a locally running MCP server via stdio transport to " +
+        "fetch a document. Demonstrates command-based server config.",
+    area: "mcp",
+    difficulty: "basic",
+    tags: ["mcp", "stdio", "local"],
+    serverConfig: {
+        transport: "stdio",
+        command: "node dist/sanity-mcp-server.js",
+        env: {
+            SANITY_PROJECT_ID: "test-project",
+            SANITY_DATASET: "production",
+        },
+        startupTimeoutMs: 5000,
+    },
+    capabilities: ["tools/list", "tools/call"],
+    prompt: {
+        text: "Fetch the document with ID 'article-123' from the Sanity Content Lake.",
+    },
+    assertions: [
+        { type: "tool-called", value: "getDocument" },
+        { type: "contains", value: "article-123" },
+    ],
+    options: { timeout: 30_000 },
+};
+/** All example MCP tasks — used by compiler tests */
+export const allMCPExampleTasks = [
+    queryDocumentsTask,
+    inspectSchemaTask,
+    createAndPublishTask,
+    semanticSearchTask,
+    stdioServerTask,
+];

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Assertion mapping and builders for agent harness tasks.
+ *
+ * Handles agent-specific assertion types (file-exists, file-contains,
+ * command-succeeds, diff-matches) as well as standard pass-through
+ * assertion types.
+ */
+import type { PromptfooAssertion } from "../../assertion-mapper.js";
+import type { AgentHarnessCompileOptions } from "./types.js";
+export declare function mapAgentAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}, options: AgentHarnessCompileOptions | undefined, warnings: string[]): PromptfooAssertion | null;
+export declare function buildFileExistsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+export declare function buildFileContainsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+/**
+ * SECURITY: Trusted-input boundary.
+ *
+ * The `command-succeeds` assertion executes an arbitrary shell command
+ * inside the sandbox's working directory. The command string comes from
+ * task definitions (YAML or TypeScript config files), which are authored
+ * by developers — not from user input or LLM output.
+ *
+ * This is intentional: the assertion is designed to verify agent output
+ * by running build/test commands (e.g., "npm test", "tsc --noEmit").
+ *
+ * Task definitions are the trust boundary. If you accept task definitions
+ * from untrusted sources, validate commands against an allowlist first.
+ */
+export declare function buildCommandSucceedsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+export declare function buildDiffMatchesAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js

@@ -0,0 +1,187 @@
+/**
+ * Assertion mapping and builders for agent harness tasks.
+ *
+ * Handles agent-specific assertion types (file-exists, file-contains,
+ * command-succeeds, diff-matches) as well as standard pass-through
+ * assertion types.
+ */
+// ---------------------------------------------------------------------------
+// Assertion mapping
+// ---------------------------------------------------------------------------
+export function mapAgentAssertion(assertion, options, warnings) {
+    switch (assertion.type) {
+        case "file-exists":
+            return buildFileExistsAssertion(assertion);
+        case "file-contains":
+            return buildFileContainsAssertion(assertion);
+        case "command-succeeds":
+            return buildCommandSucceedsAssertion(assertion);
+        case "diff-matches":
+            return buildDiffMatchesAssertion(assertion);
+        // Standard assertions pass through
+        case "contains":
+        case "equals":
+        case "regex":
+        case "is-json":
+        case "javascript":
+        case "python":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+            };
+        case "llm-rubric":
+            return {
+                type: "llm-rubric",
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+                ...(options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            };
+        default:
+            warnings.push(`Agent task: unknown assertion type "${assertion.type}" — passed through`);
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+            };
+    }
+}
+// ---------------------------------------------------------------------------
+// Agent-specific assertion builders
+// ---------------------------------------------------------------------------
+export function buildFileExistsAssertion(assertion) {
+    const filePath = String(assertion.value ?? "");
+    // Use JSON.stringify for all interpolated values in generated JS to
+    // prevent broken strings from filePaths containing quotes/backslashes
+    const safeFilePath = JSON.stringify(filePath);
+    return {
+        type: "javascript",
+        value: `// file-exists: ${filePath}\n` +
+            `(function() {\n` +
+            `  const fs = require('fs');\n` +
+            `  const path = require('path');\n` +
+            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
+            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
+            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
+            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
+            `  }\n` +
+            `  const exists = fs.existsSync(target);\n` +
+            `  return {\n` +
+            `    pass: exists,\n` +
+            `    score: exists ? 1 : 0,\n` +
+            `    reason: exists\n` +
+            `      ? 'File exists: ' + ${safeFilePath}\n` +
+            `      : 'Expected file not found: ' + ${safeFilePath},\n` +
+            `  };\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+export function buildFileContainsAssertion(assertion) {
+    const config = assertion.value;
+    const filePath = config?.path ?? "";
+    const expectedContent = config?.content ?? "";
+    const safeFilePath = JSON.stringify(filePath);
+    return {
+        type: "javascript",
+        value: `// file-contains: ${filePath}\n` +
+            `(function() {\n` +
+            `  const fs = require('fs');\n` +
+            `  const path = require('path');\n` +
+            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
+            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
+            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
+            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
+            `  }\n` +
+            `  if (!fs.existsSync(target)) {\n` +
+            `    return { pass: false, score: 0, reason: 'File not found: ' + ${safeFilePath} };\n` +
+            `  }\n` +
+            `  const content = fs.readFileSync(target, 'utf-8');\n` +
+            `  const contains = content.includes(${JSON.stringify(expectedContent)});\n` +
+            `  return {\n` +
+            `    pass: contains,\n` +
+            `    score: contains ? 1 : 0,\n` +
+            `    reason: contains\n` +
+            `      ? 'File contains expected content'\n` +
+            `      : 'File does not contain expected content',\n` +
+            `  };\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+/**
+ * SECURITY: Trusted-input boundary.
+ *
+ * The `command-succeeds` assertion executes an arbitrary shell command
+ * inside the sandbox's working directory. The command string comes from
+ * task definitions (YAML or TypeScript config files), which are authored
+ * by developers — not from user input or LLM output.
+ *
+ * This is intentional: the assertion is designed to verify agent output
+ * by running build/test commands (e.g., "npm test", "tsc --noEmit").
+ *
+ * Task definitions are the trust boundary. If you accept task definitions
+ * from untrusted sources, validate commands against an allowlist first.
+ */
+export function buildCommandSucceedsAssertion(assertion) {
+    const command = String(assertion.value ?? "");
+    return {
+        type: "javascript",
+        value: `// command-succeeds: ${command}\n` +
+            `(function() {\n` +
+            `  const { execSync } = require('child_process');\n` +
+            `  const workDir = context.vars.__workingDir || '.';\n` +
+            `  try {\n` +
+            `    execSync(${JSON.stringify(command)}, { cwd: workDir, timeout: 30000 });\n` +
+            `    return { pass: true, score: 1, reason: 'Command succeeded: ' + ${JSON.stringify(command)} };\n` +
+            `  } catch (err) {\n` +
+            `    return {\n` +
+            `      pass: false,\n` +
+            `      score: 0,\n` +
+            `      reason: 'Command failed: ' + (err.message || err),\n` +
+            `    };\n` +
+            `  }\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+export function buildDiffMatchesAssertion(assertion) {
+    const expected = assertion.value;
+    return {
+        type: "javascript",
+        value: `// diff-matches\n` +
+            `(function() {\n` +
+            `  const { execSync } = require('child_process');\n` +
+            `  const workDir = context.vars.__workingDir || '.';\n` +
+            `  try {\n` +
+            `    const diff = execSync('git diff', { cwd: workDir, encoding: 'utf-8' });\n` +
+            `    const expected = ${JSON.stringify(expected)};\n` +
+            `    if (typeof expected === 'string') {\n` +
+            `      const contains = diff.includes(expected);\n` +
+            `      return {\n` +
+            `        pass: contains,\n` +
+            `        score: contains ? 1 : 0,\n` +
+            `        reason: contains ? 'Diff matches expected pattern' : 'Diff does not match',\n` +
+            `      };\n` +
+            `    }\n` +
+            `    return { pass: diff.length > 0, score: diff.length > 0 ? 1 : 0, reason: 'Diff exists' };\n` +
+            `  } catch (err) {\n` +
+            `    return { pass: false, score: 0, reason: 'Failed to get diff: ' + err.message };\n` +
+            `  }\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Agent harness task compilation — core compilation logic.
+ *
+ * Maps agent harness task definitions to Promptfoo configuration with:
+ * - Claude Agent SDK / OpenAI Codex SDK providers
+ * - Tool permission configuration (preset/allowed/disallowed)
+ * - Sandbox setup/teardown via Promptfoo extensions
+ * - Fixture provisioning into sandbox working directory
+ */
+import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { AgentHarnessCompileOptions, AgentHarnessCompileResult } from "./types.js";
+/**
+ * Compile an agent harness task definition into Promptfoo configuration.
+ */
+export declare function compileAgentHarnessTask(task: AgentHarnessTaskDefinition, options?: AgentHarnessCompileOptions): AgentHarnessCompileResult;
+export declare function buildAgentProvider(task: AgentHarnessTaskDefinition, _warnings: string[]): PromptfooProvider[];
+export declare function buildAgentPrompts(task: AgentHarnessTaskDefinition): PromptfooPrompt[];
+export declare function buildAgentTestCases(task: AgentHarnessTaskDefinition, options: AgentHarnessCompileOptions | undefined, warnings: string[]): PromptfooTestCase[];

package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.js

@@ -0,0 +1,138 @@
+/**
+ * Agent harness task compilation — core compilation logic.
+ *
+ * Maps agent harness task definitions to Promptfoo configuration with:
+ * - Claude Agent SDK / OpenAI Codex SDK providers
+ * - Tool permission configuration (preset/allowed/disallowed)
+ * - Sandbox setup/teardown via Promptfoo extensions
+ * - Fixture provisioning into sandbox working directory
+ */
+import { mapAgentAssertion } from "./assertions.js";
+import { buildLifecycleExtensions, buildSandboxConfig } from "./sandbox.js";
+import { TOOL_PRESETS } from "./tool-presets.js";
+import { validateAgentHarnessTask } from "./validation.js";
+/**
+ * Compile an agent harness task definition into Promptfoo configuration.
+ */
+export function compileAgentHarnessTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateAgentHarnessTask(task);
+    for (const err of validationErrors) {
+        warnings.push(`Agent harness task "${task.id}": ${err.field} — ${err.message}`);
+    }
+    // Build provider
+    const providers = buildAgentProvider(task, warnings);
+    // Build prompts
+    const prompts = buildAgentPrompts(task);
+    // Build test cases
+    const tests = buildAgentTestCases(task, options, warnings);
+    // Build sandbox extensions
+    const sandboxConfig = buildSandboxConfig(task);
+    const extensions = buildLifecycleExtensions(task, sandboxConfig);
+    return { providers, tests, prompts, extensions, sandboxConfig, warnings };
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+export function buildAgentProvider(task, _warnings) {
+    // Resolve tool permissions
+    const tools = resolveToolPermissions(task.tools);
+    const config = {};
+    if (tools.length > 0) {
+        config.allowedTools = tools;
+    }
+    if (task.sandbox) {
+        config.sandbox = {
+            type: task.sandbox.type,
+            ...(task.sandbox.image ? { image: task.sandbox.image } : {}),
+        };
+    }
+    // Default to Claude Agent SDK provider
+    return [
+        {
+            id: `agent:${task.id}`,
+            label: `Agent Harness: ${task.title}`,
+            config,
+        },
+    ];
+}
+/**
+ * Resolve tool permissions from task config.
+ *
+ * Handles:
+ * - Preset names ("coding", "read-only", "full-access")
+ * - Explicit tool names ("Bash", "Read", "Write")
+ * - Mixed arrays ["coding", "WebSearch"] → preset expansion + extras
+ */
+function resolveToolPermissions(tools) {
+    if (!tools || tools.length === 0)
+        return [];
+    const resolved = new Set();
+    for (const tool of tools) {
+        const preset = TOOL_PRESETS[tool];
+        if (preset) {
+            for (const t of preset)
+                resolved.add(t);
+        }
+        else {
+            resolved.add(tool);
+        }
+    }
+    return [...resolved];
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+export function buildAgentPrompts(task) {
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Agent task: ${task.title}`;
+    return [
+        {
+            id: "agent-harness",
+            label: `Agent: ${task.title}`,
+            raw: String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+export function buildAgentTestCases(task, options, warnings) {
+    const assertions = [];
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            const mapped = mapAgentAssertion(assertion, options, warnings);
+            if (mapped)
+                assertions.push(mapped);
+        }
+    }
+    const vars = {
+        task: task.prompt?.vars?.task ?? task.description ?? `Complete: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+        // Internal metadata for sandbox lifecycle hooks
+        __sandboxType: task.sandbox?.type ?? "tempdir",
+        __fixtures: task.fixtures ?? [],
+    };
+    const tests = [
+        {
+            description: `${task.id} — ${task.title}`,
+            vars,
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        },
+    ];
+    // Multi-turn support
+    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
+        tests.push({
+            description: `${task.id} — ${task.title} [multi-turn]`,
+            vars: {
+                ...vars,
+                __multiTurn: task.multiTurn.turns,
+            },
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        });
+    }
+    return tests;
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Agent harness mode handler — compiles AgentHarnessTaskDefinition into Promptfoo config.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
+ */
+import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";
+export { AGENT_HARNESS_PROMPT_TEMPLATES } from "./prompts.js";
+export { TOOL_PRESETS } from "./tool-presets.js";
+export { validateAgentHarnessTask } from "./validation.js";
+export { mapAgentAssertion, buildFileExistsAssertion, buildFileContainsAssertion, buildCommandSucceedsAssertion, buildDiffMatchesAssertion, } from "./assertions.js";
+export { buildLifecycleExtensions, buildBeforeEachHook, buildAfterEachHook, buildSandboxConfig, } from "./sandbox.js";
+export { compileAgentHarnessTask, buildAgentProvider, buildAgentPrompts, buildAgentTestCases, } from "./compiler.js";
+export type { AgentHarnessCompileOptions, AgentHarnessCompileResult, AgentHarnessValidationError, PromptfooExtension, SandboxConfigMeta, } from "./types.js";
+/** ModeHandler-conformant export for the agent-harness evaluation mode. */
+export declare const handler: ModeHandler;