@sanity/ailf 1.0.0 → 2.0.0

package/dist/pipeline/compiler/mode-bases/literacy.js

@@ -0,0 +1,78 @@
+/**
+ * Literacy mode base — shared evaluation methodology for documentation literacy.
+ *
+ * Defines HOW literacy evaluations are scored (rubrics, weights, prompts),
+ * independently of WHAT documentation is being evaluated. Domain presets
+ * like `sanity-literacy` target this mode base and add their own sources,
+ * features, and doc fetcher.
+ *
+ * @see docs/MODES.md
+ */
+import { LITERACY_PROMPT_TEMPLATES } from "../mode-handlers/literacy/index.js";
+export function createLiteracyModeBase() {
+    return {
+        mode: {
+            id: "literacy",
+            label: "Documentation Literacy",
+            validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
+            rubricTemplateIds: [
+                "task-completion",
+                "code-correctness",
+                "doc-coverage",
+            ],
+            handlerModule: "./mode-handlers/literacy/index.js",
+        },
+        rubricTemplates: [
+            {
+                id: "task-completion",
+                dimension: "task-completion",
+                header: "Score task completion from 0 to 100:",
+                scale: [
+                    "0: Couldn't attempt — missing critical information",
+                    "20: Attempted but fundamentally wrong approach",
+                    "50: Partial implementation — major functional gaps",
+                    "80: Mostly complete — minor issues or missing edge cases",
+                    "100: Fully functional code — works as expected",
+                ],
+                criteriaLabel: "Must demonstrate:",
+            },
+            {
+                id: "code-correctness",
+                dimension: "code-correctness",
+                header: "Score code correctness from 0 to 100:",
+                scale: [
+                    "0: Broken code, syntax errors, or deprecated APIs",
+                    "30: Works but uses anti-patterns or inefficient approaches",
+                    "50: Works but not idiomatic",
+                    "80: Follows most best practices",
+                    "100: Follows all best practices, idiomatic implementation",
+                ],
+                criteriaLabel: "Check for:",
+            },
+            {
+                id: "doc-coverage",
+                dimension: "doc-coverage",
+                header: "Score documentation coverage from 0 to 100:",
+                scale: [
+                    "0: Had to hallucinate/guess most implementation details",
+                    "30: Significant gaps — filled with assumptions",
+                    "50: Some gaps — inferred from partial information",
+                    "80: Minor gaps — almost everything was documented",
+                    "100: Complete coverage — all necessary info was in docs",
+                ],
+            },
+        ],
+        scoringProfiles: {
+            default: {
+                "task-completion": 0.5,
+                "code-correctness": 0.25,
+                "doc-coverage": 0.25,
+            },
+            "output-only": {
+                "task-completion": 0.6,
+                "code-correctness": 0.4,
+            },
+        },
+        promptTemplates: LITERACY_PROMPT_TEMPLATES,
+    };
+}

package/dist/pipeline/compiler/mode-bases/mcp-server.d.ts

@@ -0,0 +1,10 @@
+/**
+ * MCP Server mode base — evaluation methodology for MCP server tool-use testing.
+ *
+ * Defines rubric templates and scoring for evaluating how well an LLM can
+ * discover and use MCP server tools correctly.
+ *
+ * @see docs/MODES.md
+ */
+import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
+export declare function createMcpServerModeBase(): ModeBase;

package/dist/pipeline/compiler/mode-bases/mcp-server.js

@@ -0,0 +1,70 @@
+/**
+ * MCP Server mode base — evaluation methodology for MCP server tool-use testing.
+ *
+ * Defines rubric templates and scoring for evaluating how well an LLM can
+ * discover and use MCP server tools correctly.
+ *
+ * @see docs/MODES.md
+ */
+export function createMcpServerModeBase() {
+    return {
+        mode: {
+            id: "mcp-server",
+            label: "MCP Server Testing",
+            validProviderPatterns: ["^mcp:", "^file://"],
+            rubricTemplateIds: [
+                "mcp-input-validation",
+                "mcp-output-correctness",
+                "mcp-error-handling",
+            ],
+            handlerModule: "./mode-handlers/mcp-server/index.js",
+        },
+        rubricTemplates: [
+            {
+                id: "mcp-input-validation",
+                dimension: "mcp-input-validation",
+                header: "Score MCP input validation from 0 to 100:",
+                scale: [
+                    "0: Tool called with completely wrong parameters",
+                    "30: Some parameters correct but critical ones missing or wrong",
+                    "50: Parameters mostly correct but types or formats are off",
+                    "80: All required parameters correct, minor optional parameter issues",
+                    "100: Perfect tool invocation with all parameters correctly formed",
+                ],
+                criteriaLabel: "Check for:",
+            },
+            {
+                id: "mcp-output-correctness",
+                dimension: "mcp-output-correctness",
+                header: "Score MCP output correctness from 0 to 100:",
+                scale: [
+                    "0: Output is completely wrong or tool returned an error",
+                    "30: Output partially correct but missing key data",
+                    "50: Output correct but incomplete or improperly formatted",
+                    "80: Output correct and well-formatted, minor issues",
+                    "100: Output is correct, complete, and properly formatted",
+                ],
+                criteriaLabel: "Check for:",
+            },
+            {
+                id: "mcp-error-handling",
+                dimension: "mcp-error-handling",
+                header: "Score MCP error handling from 0 to 100:",
+                scale: [
+                    "0: No error handling — crashes or returns garbage on bad input",
+                    "30: Basic error detection but poor recovery or messaging",
+                    "50: Errors detected and reported but not gracefully handled",
+                    "80: Good error handling with clear messages and appropriate fallbacks",
+                    "100: Excellent error handling — validates input, provides actionable errors, degrades gracefully",
+                ],
+            },
+        ],
+        scoringProfiles: {
+            default: {
+                "mcp-input-validation": 0.4,
+                "mcp-output-correctness": 0.4,
+                "mcp-error-handling": 0.2,
+            },
+        },
+    };
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Assertion mapping and builders for agent harness tasks.
+ *
+ * Handles agent-specific assertion types (file-exists, file-contains,
+ * command-succeeds, diff-matches) as well as standard pass-through
+ * assertion types.
+ */
+import type { PromptfooAssertion } from "../../assertion-mapper.js";
+import type { AgentHarnessCompileOptions } from "./types.js";
+export declare function mapAgentAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}, options: AgentHarnessCompileOptions | undefined, warnings: string[]): PromptfooAssertion | null;
+export declare function buildFileExistsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+export declare function buildFileContainsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+/**
+ * SECURITY: Trusted-input boundary.
+ *
+ * The `command-succeeds` assertion executes an arbitrary shell command
+ * inside the sandbox's working directory. The command string comes from
+ * task definitions (YAML or TypeScript config files), which are authored
+ * by developers — not from user input or LLM output.
+ *
+ * This is intentional: the assertion is designed to verify agent output
+ * by running build/test commands (e.g., "npm test", "tsc --noEmit").
+ *
+ * Task definitions are the trust boundary. If you accept task definitions
+ * from untrusted sources, validate commands against an allowlist first.
+ */
+export declare function buildCommandSucceedsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+export declare function buildDiffMatchesAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js

@@ -0,0 +1,187 @@
+/**
+ * Assertion mapping and builders for agent harness tasks.
+ *
+ * Handles agent-specific assertion types (file-exists, file-contains,
+ * command-succeeds, diff-matches) as well as standard pass-through
+ * assertion types.
+ */
+// ---------------------------------------------------------------------------
+// Assertion mapping
+// ---------------------------------------------------------------------------
+export function mapAgentAssertion(assertion, options, warnings) {
+    switch (assertion.type) {
+        case "file-exists":
+            return buildFileExistsAssertion(assertion);
+        case "file-contains":
+            return buildFileContainsAssertion(assertion);
+        case "command-succeeds":
+            return buildCommandSucceedsAssertion(assertion);
+        case "diff-matches":
+            return buildDiffMatchesAssertion(assertion);
+        // Standard assertions pass through
+        case "contains":
+        case "equals":
+        case "regex":
+        case "is-json":
+        case "javascript":
+        case "python":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+            };
+        case "llm-rubric":
+            return {
+                type: "llm-rubric",
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+                ...(options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            };
+        default:
+            warnings.push(`Agent task: unknown assertion type "${assertion.type}" — passed through`);
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+            };
+    }
+}
+// ---------------------------------------------------------------------------
+// Agent-specific assertion builders
+// ---------------------------------------------------------------------------
+export function buildFileExistsAssertion(assertion) {
+    const filePath = String(assertion.value ?? "");
+    // Use JSON.stringify for all interpolated values in generated JS to
+    // prevent broken strings from filePaths containing quotes/backslashes
+    const safeFilePath = JSON.stringify(filePath);
+    return {
+        type: "javascript",
+        value: `// file-exists: ${filePath}\n` +
+            `(function() {\n` +
+            `  const fs = require('fs');\n` +
+            `  const path = require('path');\n` +
+            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
+            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
+            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
+            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
+            `  }\n` +
+            `  const exists = fs.existsSync(target);\n` +
+            `  return {\n` +
+            `    pass: exists,\n` +
+            `    score: exists ? 1 : 0,\n` +
+            `    reason: exists\n` +
+            `      ? 'File exists: ' + ${safeFilePath}\n` +
+            `      : 'Expected file not found: ' + ${safeFilePath},\n` +
+            `  };\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+export function buildFileContainsAssertion(assertion) {
+    const config = assertion.value;
+    const filePath = config?.path ?? "";
+    const expectedContent = config?.content ?? "";
+    const safeFilePath = JSON.stringify(filePath);
+    return {
+        type: "javascript",
+        value: `// file-contains: ${filePath}\n` +
+            `(function() {\n` +
+            `  const fs = require('fs');\n` +
+            `  const path = require('path');\n` +
+            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
+            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
+            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
+            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
+            `  }\n` +
+            `  if (!fs.existsSync(target)) {\n` +
+            `    return { pass: false, score: 0, reason: 'File not found: ' + ${safeFilePath} };\n` +
+            `  }\n` +
+            `  const content = fs.readFileSync(target, 'utf-8');\n` +
+            `  const contains = content.includes(${JSON.stringify(expectedContent)});\n` +
+            `  return {\n` +
+            `    pass: contains,\n` +
+            `    score: contains ? 1 : 0,\n` +
+            `    reason: contains\n` +
+            `      ? 'File contains expected content'\n` +
+            `      : 'File does not contain expected content',\n` +
+            `  };\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+/**
+ * SECURITY: Trusted-input boundary.
+ *
+ * The `command-succeeds` assertion executes an arbitrary shell command
+ * inside the sandbox's working directory. The command string comes from
+ * task definitions (YAML or TypeScript config files), which are authored
+ * by developers — not from user input or LLM output.
+ *
+ * This is intentional: the assertion is designed to verify agent output
+ * by running build/test commands (e.g., "npm test", "tsc --noEmit").
+ *
+ * Task definitions are the trust boundary. If you accept task definitions
+ * from untrusted sources, validate commands against an allowlist first.
+ */
+export function buildCommandSucceedsAssertion(assertion) {
+    const command = String(assertion.value ?? "");
+    return {
+        type: "javascript",
+        value: `// command-succeeds: ${command}\n` +
+            `(function() {\n` +
+            `  const { execSync } = require('child_process');\n` +
+            `  const workDir = context.vars.__workingDir || '.';\n` +
+            `  try {\n` +
+            `    execSync(${JSON.stringify(command)}, { cwd: workDir, timeout: 30000 });\n` +
+            `    return { pass: true, score: 1, reason: 'Command succeeded: ' + ${JSON.stringify(command)} };\n` +
+            `  } catch (err) {\n` +
+            `    return {\n` +
+            `      pass: false,\n` +
+            `      score: 0,\n` +
+            `      reason: 'Command failed: ' + (err.message || err),\n` +
+            `    };\n` +
+            `  }\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+export function buildDiffMatchesAssertion(assertion) {
+    const expected = assertion.value;
+    return {
+        type: "javascript",
+        value: `// diff-matches\n` +
+            `(function() {\n` +
+            `  const { execSync } = require('child_process');\n` +
+            `  const workDir = context.vars.__workingDir || '.';\n` +
+            `  try {\n` +
+            `    const diff = execSync('git diff', { cwd: workDir, encoding: 'utf-8' });\n` +
+            `    const expected = ${JSON.stringify(expected)};\n` +
+            `    if (typeof expected === 'string') {\n` +
+            `      const contains = diff.includes(expected);\n` +
+            `      return {\n` +
+            `        pass: contains,\n` +
+            `        score: contains ? 1 : 0,\n` +
+            `        reason: contains ? 'Diff matches expected pattern' : 'Diff does not match',\n` +
+            `      };\n` +
+            `    }\n` +
+            `    return { pass: diff.length > 0, score: diff.length > 0 ? 1 : 0, reason: 'Diff exists' };\n` +
+            `  } catch (err) {\n` +
+            `    return { pass: false, score: 0, reason: 'Failed to get diff: ' + err.message };\n` +
+            `  }\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Agent harness task compilation — core compilation logic.
+ *
+ * Maps agent harness task definitions to Promptfoo configuration with:
+ * - Claude Agent SDK / OpenAI Codex SDK providers
+ * - Tool permission configuration (preset/allowed/disallowed)
+ * - Sandbox setup/teardown via Promptfoo extensions
+ * - Fixture provisioning into sandbox working directory
+ */
+import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { AgentHarnessCompileOptions, AgentHarnessCompileResult } from "./types.js";
+/**
+ * Compile an agent harness task definition into Promptfoo configuration.
+ */
+export declare function compileAgentHarnessTask(task: AgentHarnessTaskDefinition, options?: AgentHarnessCompileOptions): AgentHarnessCompileResult;
+export declare function buildAgentProvider(task: AgentHarnessTaskDefinition, _warnings: string[]): PromptfooProvider[];
+export declare function buildAgentPrompts(task: AgentHarnessTaskDefinition): PromptfooPrompt[];
+export declare function buildAgentTestCases(task: AgentHarnessTaskDefinition, options: AgentHarnessCompileOptions | undefined, warnings: string[]): PromptfooTestCase[];

package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.js

@@ -0,0 +1,138 @@
+/**
+ * Agent harness task compilation — core compilation logic.
+ *
+ * Maps agent harness task definitions to Promptfoo configuration with:
+ * - Claude Agent SDK / OpenAI Codex SDK providers
+ * - Tool permission configuration (preset/allowed/disallowed)
+ * - Sandbox setup/teardown via Promptfoo extensions
+ * - Fixture provisioning into sandbox working directory
+ */
+import { mapAgentAssertion } from "./assertions.js";
+import { buildLifecycleExtensions, buildSandboxConfig } from "./sandbox.js";
+import { TOOL_PRESETS } from "./tool-presets.js";
+import { validateAgentHarnessTask } from "./validation.js";
+/**
+ * Compile an agent harness task definition into Promptfoo configuration.
+ */
+export function compileAgentHarnessTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateAgentHarnessTask(task);
+    for (const err of validationErrors) {
+        warnings.push(`Agent harness task "${task.id}": ${err.field} — ${err.message}`);
+    }
+    // Build provider
+    const providers = buildAgentProvider(task, warnings);
+    // Build prompts
+    const prompts = buildAgentPrompts(task);
+    // Build test cases
+    const tests = buildAgentTestCases(task, options, warnings);
+    // Build sandbox extensions
+    const sandboxConfig = buildSandboxConfig(task);
+    const extensions = buildLifecycleExtensions(task, sandboxConfig);
+    return { providers, tests, prompts, extensions, sandboxConfig, warnings };
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+export function buildAgentProvider(task, _warnings) {
+    // Resolve tool permissions
+    const tools = resolveToolPermissions(task.tools);
+    const config = {};
+    if (tools.length > 0) {
+        config.allowedTools = tools;
+    }
+    if (task.sandbox) {
+        config.sandbox = {
+            type: task.sandbox.type,
+            ...(task.sandbox.image ? { image: task.sandbox.image } : {}),
+        };
+    }
+    // Default to Claude Agent SDK provider
+    return [
+        {
+            id: `agent:${task.id}`,
+            label: `Agent Harness: ${task.title}`,
+            config,
+        },
+    ];
+}
+/**
+ * Resolve tool permissions from task config.
+ *
+ * Handles:
+ * - Preset names ("coding", "read-only", "full-access")
+ * - Explicit tool names ("Bash", "Read", "Write")
+ * - Mixed arrays ["coding", "WebSearch"] → preset expansion + extras
+ */
+function resolveToolPermissions(tools) {
+    if (!tools || tools.length === 0)
+        return [];
+    const resolved = new Set();
+    for (const tool of tools) {
+        const preset = TOOL_PRESETS[tool];
+        if (preset) {
+            for (const t of preset)
+                resolved.add(t);
+        }
+        else {
+            resolved.add(tool);
+        }
+    }
+    return [...resolved];
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+export function buildAgentPrompts(task) {
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Agent task: ${task.title}`;
+    return [
+        {
+            id: "agent-harness",
+            label: `Agent: ${task.title}`,
+            raw: String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+export function buildAgentTestCases(task, options, warnings) {
+    const assertions = [];
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            const mapped = mapAgentAssertion(assertion, options, warnings);
+            if (mapped)
+                assertions.push(mapped);
+        }
+    }
+    const vars = {
+        task: task.prompt?.vars?.task ?? task.description ?? `Complete: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+        // Internal metadata for sandbox lifecycle hooks
+        __sandboxType: task.sandbox?.type ?? "tempdir",
+        __fixtures: task.fixtures ?? [],
+    };
+    const tests = [
+        {
+            description: `${task.id} — ${task.title}`,
+            vars,
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        },
+    ];
+    // Multi-turn support
+    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
+        tests.push({
+            description: `${task.id} — ${task.title} [multi-turn]`,
+            vars: {
+                ...vars,
+                __multiTurn: task.multiTurn.turns,
+            },
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        });
+    }
+    return tests;
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Agent harness mode handler — compiles AgentHarnessTaskDefinition into Promptfoo config.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
+ */
+import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";
+export { AGENT_HARNESS_PROMPT_TEMPLATES } from "./prompts.js";
+export { TOOL_PRESETS } from "./tool-presets.js";
+export { validateAgentHarnessTask } from "./validation.js";
+export { mapAgentAssertion, buildFileExistsAssertion, buildFileContainsAssertion, buildCommandSucceedsAssertion, buildDiffMatchesAssertion, } from "./assertions.js";
+export { buildLifecycleExtensions, buildBeforeEachHook, buildAfterEachHook, buildSandboxConfig, } from "./sandbox.js";
+export { compileAgentHarnessTask, buildAgentProvider, buildAgentPrompts, buildAgentTestCases, } from "./compiler.js";
+export type { AgentHarnessCompileOptions, AgentHarnessCompileResult, AgentHarnessValidationError, PromptfooExtension, SandboxConfigMeta, } from "./types.js";
+/** ModeHandler-conformant export for the agent-harness evaluation mode. */
+export declare const handler: ModeHandler;

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js

@@ -0,0 +1,43 @@
+/**
+ * Agent harness mode handler — compiles AgentHarnessTaskDefinition into Promptfoo config.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
+ */
+import { compileAgentHarnessTask } from "./compiler.js";
+import { AGENT_HARNESS_PROMPT_TEMPLATES } from "./prompts.js";
+// Re-export public API
+export { AGENT_HARNESS_PROMPT_TEMPLATES } from "./prompts.js";
+export { TOOL_PRESETS } from "./tool-presets.js";
+export { validateAgentHarnessTask } from "./validation.js";
+export { mapAgentAssertion, buildFileExistsAssertion, buildFileContainsAssertion, buildCommandSucceedsAssertion, buildDiffMatchesAssertion, } from "./assertions.js";
+export { buildLifecycleExtensions, buildBeforeEachHook, buildAfterEachHook, buildSandboxConfig, } from "./sandbox.js";
+export { compileAgentHarnessTask, buildAgentProvider, buildAgentPrompts, buildAgentTestCases, } from "./compiler.js";
+// ---------------------------------------------------------------------------
+// ModeHandler adapter — wraps compileAgentHarnessTask for registry dispatch
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the agent-harness evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return AGENT_HARNESS_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "agent-harness") {
+            throw new Error(`Agent harness handler received task with mode "${task.mode ?? "undefined"}" — expected "agent-harness"`);
+        }
+        const result = compileAgentHarnessTask(task, {
+            graderProvider: ctx.graderProvider,
+            rootDir: ctx.rootDir,
+        });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+            extras: {
+                extensions: result.extensions,
+                sandboxConfig: result.sandboxConfig,
+            },
+        };
+    },
+};

package/dist/pipeline/compiler/mode-handlers/agent-harness/prompts.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Canonical prompt templates for agent-harness-mode evaluations.
+ *
+ * Handler-owned prompts for agent harness evaluations. Describes the task
+ * for autonomous agent execution within a sandboxed environment with file
+ * system and tool access.
+ */
+import type { PromptTemplate } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const AGENT_HARNESS_PROMPT_TEMPLATES: Record<string, PromptTemplate>;

package/dist/pipeline/compiler/mode-handlers/agent-harness/prompts.js

@@ -0,0 +1,29 @@
+/**
+ * Canonical prompt templates for agent-harness-mode evaluations.
+ *
+ * Handler-owned prompts for agent harness evaluations. Describes the task
+ * for autonomous agent execution within a sandboxed environment with file
+ * system and tool access.
+ */
+export const AGENT_HARNESS_PROMPT_TEMPLATES = {
+    "agent-harness": {
+        id: "agent-harness",
+        label: "Agent Harness Task",
+        template: `You are a coding agent working in a sandboxed environment. You have access to file system tools (read, write, edit) and a shell to complete the following task.
+
+## Task
+{{task}}
+
+## Instructions
+
+1. Read existing files to understand the project structure before making changes
+2. Implement a complete, working solution — no placeholders or TODOs
+3. Ensure all necessary imports and dependencies are included
+4. Verify your implementation compiles and passes any provided test commands
+5. Keep changes minimal and focused on the task
+
+Complete the implementation:
+`,
+        variables: ["task"],
+    },
+};

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Sandbox configuration and lifecycle extensions for agent harness tasks.
+ *
+ * Builds Promptfoo beforeEach/afterEach hooks for provisioning and
+ * tearing down sandbox working directories.
+ */
+import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooExtension, SandboxConfigMeta } from "./types.js";
+export declare function buildSandboxConfig(task: AgentHarnessTaskDefinition): SandboxConfigMeta;
+export declare function buildLifecycleExtensions(task: AgentHarnessTaskDefinition, sandboxConfig: SandboxConfigMeta): PromptfooExtension[];
+export declare function buildBeforeEachHook(taskId: string, config: SandboxConfigMeta): string;
+export declare function buildAfterEachHook(taskId: string): string;