@sanity/ailf 1.0.0 → 2.0.1

package/dist/pipeline/compiler/ignore-fields.js

@@ -17,7 +17,7 @@
  * ]
  * ```
  *
- * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-4-agent-harness.md
  */
 // ---------------------------------------------------------------------------
 // Public API

package/dist/pipeline/compiler/index.d.ts

@@ -1,13 +1,10 @@
 /**
- * Config compiler — the heart of the new architecture.
+ * Config compiler — the heart of the compilation architecture.
  *
  * Converts task definitions from any source into a TaskGraph IR,
  * then compiles the graph into Promptfoo YAML configuration.
  *
- * This module coexists with the existing `generate-configs.ts` path.
- * Phase 7 will migrate callers to use the compiler exclusively.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 export { buildTaskGraph, detectCycle, type TaskGraphBuildOptions, type TaskGraphBuildResult, } from "./task-graph-builder.js";
 export { compileToPromptfoo, type CompilationResult, type CompiledPromptfooConfig, type PromptfooCompilerOptions, type PromptfooPrompt, type PromptfooProvider, type PromptfooTestCase, } from "./promptfoo-compiler.js";

package/dist/pipeline/compiler/index.js

@@ -1,13 +1,10 @@
 /**
- * Config compiler — the heart of the new architecture.
+ * Config compiler — the heart of the compilation architecture.
  *
  * Converts task definitions from any source into a TaskGraph IR,
  * then compiles the graph into Promptfoo YAML configuration.
  *
- * This module coexists with the existing `generate-configs.ts` path.
- * Phase 7 will migrate callers to use the compiler exclusively.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 // TaskGraph builder
 export { buildTaskGraph, detectCycle, } from "./task-graph-builder.js";

package/dist/pipeline/compiler/literacy-bridge.d.ts

@@ -16,10 +16,10 @@
  * - Prompts from config/prompts are integrated
  * - TaskGraphBuilder validates the DAG, deduplicates, and orders tasks
  *
- * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
  */
 import type { LiteracyTaskDefinition } from "../../_vendor/ailf-core/index.d.ts";
-import { type LiteracyCompileResult } from "./mode-handlers/literacy-handler.js";
+import { type LiteracyCompileResult } from "./mode-handlers/literacy/index.js";
 import { type LiteracyEvalSubMode } from "../normalize-mode.js";
 /** Options for compiling all literacy tasks via the new compiler */
 export interface LiteracyBridgeOptions {

package/dist/pipeline/compiler/literacy-bridge.js

@@ -16,9 +16,9 @@
  * - Prompts from config/prompts are integrated
  * - TaskGraphBuilder validates the DAG, deduplicates, and orders tasks
  *
- * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
  */
-import { compileLiteracyTask, } from "./mode-handlers/literacy-handler.js";
+import { compileLiteracyTask, } from "./mode-handlers/literacy/index.js";
 import { tryLoadConfigFile } from "./config-loader.js";
 import { buildTaskGraph } from "./task-graph-builder.js";
 // ---------------------------------------------------------------------------

package/dist/pipeline/compiler/mode-bases/agent-harness.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Agent Harness mode base — evaluation methodology for autonomous agent testing.
+ *
+ * Tests whether an autonomous agent can complete implementation tasks
+ * end-to-end, including tool use, file creation, and code generation.
+ *
+ * @see docs/modes.md
+ */
+import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
+export declare function createAgentHarnessBase(): ModeBase;

package/dist/pipeline/compiler/mode-bases/agent-harness.js

@@ -0,0 +1,21 @@
+/**
+ * Agent Harness mode base — evaluation methodology for autonomous agent testing.
+ *
+ * Tests whether an autonomous agent can complete implementation tasks
+ * end-to-end, including tool use, file creation, and code generation.
+ *
+ * @see docs/modes.md
+ */
+export function createAgentHarnessBase() {
+    return {
+        mode: {
+            id: "agent-harness",
+            label: "Agent Harness",
+            validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
+            rubricTemplateIds: [],
+            handlerModule: "./mode-handlers/agent-harness/index.js",
+        },
+        // Agent harness rubric templates and scoring profiles will be defined
+        // as the mode matures. The structural registration is in place.
+    };
+}

package/dist/pipeline/compiler/mode-bases/index.d.ts

@@ -0,0 +1,4 @@
+export { createAgentHarnessBase } from "./agent-harness.js";
+export { createKnowledgeProbeBase } from "./knowledge-probe.js";
+export { createLiteracyModeBase } from "./literacy.js";
+export { createMcpServerModeBase } from "./mcp-server.js";

package/dist/pipeline/compiler/mode-bases/index.js

@@ -0,0 +1,4 @@
+export { createAgentHarnessBase } from "./agent-harness.js";
+export { createKnowledgeProbeBase } from "./knowledge-probe.js";
+export { createLiteracyModeBase } from "./literacy.js";
+export { createMcpServerModeBase } from "./mcp-server.js";

package/dist/pipeline/compiler/mode-bases/knowledge-probe.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Knowledge Probe mode base — evaluation methodology for testing model knowledge.
+ *
+ * Tests what the model knows about a topic without providing documentation,
+ * establishing a baseline of model knowledge.
+ *
+ * @see docs/modes.md
+ */
+import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
+export declare function createKnowledgeProbeBase(): ModeBase;

package/dist/pipeline/compiler/mode-bases/knowledge-probe.js

@@ -0,0 +1,22 @@
+/**
+ * Knowledge Probe mode base — evaluation methodology for testing model knowledge.
+ *
+ * Tests what the model knows about a topic without providing documentation,
+ * establishing a baseline of model knowledge.
+ *
+ * @see docs/modes.md
+ */
+export function createKnowledgeProbeBase() {
+    return {
+        mode: {
+            id: "knowledge-probe",
+            label: "Knowledge Probe",
+            validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
+            rubricTemplateIds: [],
+            handlerModule: "./mode-handlers/knowledge-probe/index.js",
+        },
+        // Knowledge probe uses the same rubric dimensions as literacy
+        // but without doc-coverage (since no docs are provided).
+        // Rubric templates will be inherited or defined as the mode matures.
+    };
+}

package/dist/pipeline/compiler/mode-bases/literacy.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Literacy mode base — shared evaluation methodology for documentation literacy.
+ *
+ * Defines HOW literacy evaluations are scored (rubrics, weights, prompts),
+ * independently of WHAT documentation is being evaluated. Domain presets
+ * like `sanity-literacy` target this mode base and add their own sources,
+ * features, and doc fetcher.
+ *
+ * @see docs/modes.md
+ */
+import { type ModeBase, type ModelEntry } from "../../../_vendor/ailf-core/index.d.ts";
+export declare function createLiteracyModeBase(): ModeBase;
+/**
+ * Check whether a model participates in a specific literacy variant.
+ *
+ * A model matches if:
+ * 1. It's enrolled in the `literacy` eval mode (or has no `modes` field)
+ * 2. The variant is in its resolved variant set (defaults to all variants)
+ *
+ * This is the single source of truth for literacy variant matching —
+ * import this instead of reimplementing the pattern.
+ */
+export declare function modelMatchesLiteracyVariant(model: ModelEntry, variant: string): boolean;

package/dist/pipeline/compiler/mode-bases/literacy.js

@@ -0,0 +1,132 @@
+/**
+ * Literacy mode base — shared evaluation methodology for documentation literacy.
+ *
+ * Defines HOW literacy evaluations are scored (rubrics, weights, prompts),
+ * independently of WHAT documentation is being evaluated. Domain presets
+ * like `sanity-literacy` target this mode base and add their own sources,
+ * features, and doc fetcher.
+ *
+ * @see docs/modes.md
+ */
+import { modelMatchesMode, resolveModelVariants, } from "../../../_vendor/ailf-core/index.js";
+import { LITERACY_PROMPT_TEMPLATES } from "../mode-handlers/literacy/index.js";
+// ---------------------------------------------------------------------------
+// Mode base factory
+// ---------------------------------------------------------------------------
+export function createLiteracyModeBase() {
+    return {
+        mode: {
+            id: "literacy",
+            label: "Documentation Literacy",
+            validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
+            rubricTemplateIds: [
+                "task-completion",
+                "code-correctness",
+                "doc-coverage",
+            ],
+            handlerModule: "./mode-handlers/literacy/index.js",
+            variants: [
+                {
+                    id: "baseline",
+                    label: "Standard (baseline)",
+                    description: "Standard with-docs and without-docs evaluation prompts",
+                },
+                {
+                    id: "observed",
+                    label: "Observed (HTTP-instrumented)",
+                    description: "HTTP-instrumented evaluation that records model behavior",
+                },
+                {
+                    id: "agentic-naive",
+                    label: "Agentic (naive)",
+                    description: "Model uses tools to find docs with default system prompt",
+                },
+                {
+                    id: "agentic-optimized",
+                    label: "Agentic (optimized)",
+                    description: "Model uses tools to find docs with optimized system prompt",
+                },
+            ],
+        },
+        rubricTemplates: [
+            {
+                id: "task-completion",
+                dimension: "task-completion",
+                header: "Score task completion from 0 to 100:",
+                scale: [
+                    "0: Couldn't attempt — missing critical information",
+                    "20: Attempted but fundamentally wrong approach",
+                    "50: Partial implementation — major functional gaps",
+                    "80: Mostly complete — minor issues or missing edge cases",
+                    "100: Fully functional code — works as expected",
+                ],
+                criteriaLabel: "Must demonstrate:",
+            },
+            {
+                id: "code-correctness",
+                dimension: "code-correctness",
+                header: "Score code correctness from 0 to 100:",
+                scale: [
+                    "0: Broken code, syntax errors, or deprecated APIs",
+                    "30: Works but uses anti-patterns or inefficient approaches",
+                    "50: Works but not idiomatic",
+                    "80: Follows most best practices",
+                    "100: Follows all best practices, idiomatic implementation",
+                ],
+                criteriaLabel: "Check for:",
+            },
+            {
+                id: "doc-coverage",
+                dimension: "doc-coverage",
+                header: "Score documentation coverage from 0 to 100:",
+                scale: [
+                    "0: Had to hallucinate/guess most implementation details",
+                    "30: Significant gaps — filled with assumptions",
+                    "50: Some gaps — inferred from partial information",
+                    "80: Minor gaps — almost everything was documented",
+                    "100: Complete coverage — all necessary info was in docs",
+                ],
+            },
+        ],
+        scoringProfiles: {
+            default: {
+                "task-completion": 0.5,
+                "code-correctness": 0.25,
+                "doc-coverage": 0.25,
+            },
+            "output-only": {
+                "task-completion": 0.6,
+                "code-correctness": 0.4,
+            },
+        },
+        promptTemplates: LITERACY_PROMPT_TEMPLATES,
+    };
+}
+// ---------------------------------------------------------------------------
+// Shared variant matching helper
+// ---------------------------------------------------------------------------
+/** Lazily initialized mode base — avoids re-creating on every call */
+let _cachedBase;
+function getLiteracyModeBase() {
+    if (!_cachedBase)
+        _cachedBase = createLiteracyModeBase();
+    return _cachedBase;
+}
+/**
+ * Check whether a model participates in a specific literacy variant.
+ *
+ * A model matches if:
+ * 1. It's enrolled in the `literacy` eval mode (or has no `modes` field)
+ * 2. The variant is in its resolved variant set (defaults to all variants)
+ *
+ * This is the single source of truth for literacy variant matching —
+ * import this instead of reimplementing the pattern.
+ */
+export function modelMatchesLiteracyVariant(model, variant) {
+    if (!modelMatchesMode(model, "literacy"))
+        return false;
+    const variants = resolveModelVariants(model, getLiteracyModeBase());
+    if (!variants)
+        return true;
+    return variants.includes(variant);
+}

package/dist/pipeline/compiler/mode-bases/mcp-server.d.ts

@@ -0,0 +1,10 @@
+/**
+ * MCP Server mode base — evaluation methodology for MCP server tool-use testing.
+ *
+ * Defines rubric templates and scoring for evaluating how well an LLM can
+ * discover and use MCP server tools correctly.
+ *
+ * @see docs/modes.md
+ */
+import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
+export declare function createMcpServerModeBase(): ModeBase;

package/dist/pipeline/compiler/mode-bases/mcp-server.js

@@ -0,0 +1,70 @@
+/**
+ * MCP Server mode base — evaluation methodology for MCP server tool-use testing.
+ *
+ * Defines rubric templates and scoring for evaluating how well an LLM can
+ * discover and use MCP server tools correctly.
+ *
+ * @see docs/modes.md
+ */
+export function createMcpServerModeBase() {
+    return {
+        mode: {
+            id: "mcp-server",
+            label: "MCP Server Testing",
+            validProviderPatterns: ["^mcp:", "^file://"],
+            rubricTemplateIds: [
+                "mcp-input-validation",
+                "mcp-output-correctness",
+                "mcp-error-handling",
+            ],
+            handlerModule: "./mode-handlers/mcp-server/index.js",
+        },
+        rubricTemplates: [
+            {
+                id: "mcp-input-validation",
+                dimension: "mcp-input-validation",
+                header: "Score MCP input validation from 0 to 100:",
+                scale: [
+                    "0: Tool called with completely wrong parameters",
+                    "30: Some parameters correct but critical ones missing or wrong",
+                    "50: Parameters mostly correct but types or formats are off",
+                    "80: All required parameters correct, minor optional parameter issues",
+                    "100: Perfect tool invocation with all parameters correctly formed",
+                ],
+                criteriaLabel: "Check for:",
+            },
+            {
+                id: "mcp-output-correctness",
+                dimension: "mcp-output-correctness",
+                header: "Score MCP output correctness from 0 to 100:",
+                scale: [
+                    "0: Output is completely wrong or tool returned an error",
+                    "30: Output partially correct but missing key data",
+                    "50: Output correct but incomplete or improperly formatted",
+                    "80: Output correct and well-formatted, minor issues",
+                    "100: Output is correct, complete, and properly formatted",
+                ],
+                criteriaLabel: "Check for:",
+            },
+            {
+                id: "mcp-error-handling",
+                dimension: "mcp-error-handling",
+                header: "Score MCP error handling from 0 to 100:",
+                scale: [
+                    "0: No error handling — crashes or returns garbage on bad input",
+                    "30: Basic error detection but poor recovery or messaging",
+                    "50: Errors detected and reported but not gracefully handled",
+                    "80: Good error handling with clear messages and appropriate fallbacks",
+                    "100: Excellent error handling — validates input, provides actionable errors, degrades gracefully",
+                ],
+            },
+        ],
+        scoringProfiles: {
+            default: {
+                "mcp-input-validation": 0.4,
+                "mcp-output-correctness": 0.4,
+                "mcp-error-handling": 0.2,
+            },
+        },
+    };
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Assertion mapping and builders for agent harness tasks.
+ *
+ * Handles agent-specific assertion types (file-exists, file-contains,
+ * command-succeeds, diff-matches) as well as standard pass-through
+ * assertion types.
+ */
+import type { PromptfooAssertion } from "../../assertion-mapper.js";
+import type { AgentHarnessCompileOptions } from "./types.js";
+export declare function mapAgentAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}, options: AgentHarnessCompileOptions | undefined, warnings: string[]): PromptfooAssertion | null;
+export declare function buildFileExistsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+export declare function buildFileContainsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+/**
+ * SECURITY: Trusted-input boundary.
+ *
+ * The `command-succeeds` assertion executes an arbitrary shell command
+ * inside the sandbox's working directory. The command string comes from
+ * task definitions (YAML or TypeScript config files), which are authored
+ * by developers — not from user input or LLM output.
+ *
+ * This is intentional: the assertion is designed to verify agent output
+ * by running build/test commands (e.g., "npm test", "tsc --noEmit").
+ *
+ * Task definitions are the trust boundary. If you accept task definitions
+ * from untrusted sources, validate commands against an allowlist first.
+ */
+export declare function buildCommandSucceedsAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;
+export declare function buildDiffMatchesAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}): PromptfooAssertion;

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js

@@ -0,0 +1,187 @@
+/**
+ * Assertion mapping and builders for agent harness tasks.
+ *
+ * Handles agent-specific assertion types (file-exists, file-contains,
+ * command-succeeds, diff-matches) as well as standard pass-through
+ * assertion types.
+ */
+// ---------------------------------------------------------------------------
+// Assertion mapping
+// ---------------------------------------------------------------------------
+export function mapAgentAssertion(assertion, options, warnings) {
+    switch (assertion.type) {
+        case "file-exists":
+            return buildFileExistsAssertion(assertion);
+        case "file-contains":
+            return buildFileContainsAssertion(assertion);
+        case "command-succeeds":
+            return buildCommandSucceedsAssertion(assertion);
+        case "diff-matches":
+            return buildDiffMatchesAssertion(assertion);
+        // Standard assertions pass through
+        case "contains":
+        case "equals":
+        case "regex":
+        case "is-json":
+        case "javascript":
+        case "python":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+            };
+        case "llm-rubric":
+            return {
+                type: "llm-rubric",
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+                ...(options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            };
+        default:
+            warnings.push(`Agent task: unknown assertion type "${assertion.type}" — passed through`);
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+            };
+    }
+}
+// ---------------------------------------------------------------------------
+// Agent-specific assertion builders
+// ---------------------------------------------------------------------------
+export function buildFileExistsAssertion(assertion) {
+    const filePath = String(assertion.value ?? "");
+    // Use JSON.stringify for all interpolated values in generated JS to
+    // prevent broken strings from filePaths containing quotes/backslashes
+    const safeFilePath = JSON.stringify(filePath);
+    return {
+        type: "javascript",
+        value: `// file-exists: ${filePath}\n` +
+            `(function() {\n` +
+            `  const fs = require('fs');\n` +
+            `  const path = require('path');\n` +
+            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
+            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
+            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
+            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
+            `  }\n` +
+            `  const exists = fs.existsSync(target);\n` +
+            `  return {\n` +
+            `    pass: exists,\n` +
+            `    score: exists ? 1 : 0,\n` +
+            `    reason: exists\n` +
+            `      ? 'File exists: ' + ${safeFilePath}\n` +
+            `      : 'Expected file not found: ' + ${safeFilePath},\n` +
+            `  };\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+export function buildFileContainsAssertion(assertion) {
+    const config = assertion.value;
+    const filePath = config?.path ?? "";
+    const expectedContent = config?.content ?? "";
+    const safeFilePath = JSON.stringify(filePath);
+    return {
+        type: "javascript",
+        value: `// file-contains: ${filePath}\n` +
+            `(function() {\n` +
+            `  const fs = require('fs');\n` +
+            `  const path = require('path');\n` +
+            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
+            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
+            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
+            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
+            `  }\n` +
+            `  if (!fs.existsSync(target)) {\n` +
+            `    return { pass: false, score: 0, reason: 'File not found: ' + ${safeFilePath} };\n` +
+            `  }\n` +
+            `  const content = fs.readFileSync(target, 'utf-8');\n` +
+            `  const contains = content.includes(${JSON.stringify(expectedContent)});\n` +
+            `  return {\n` +
+            `    pass: contains,\n` +
+            `    score: contains ? 1 : 0,\n` +
+            `    reason: contains\n` +
+            `      ? 'File contains expected content'\n` +
+            `      : 'File does not contain expected content',\n` +
+            `  };\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+/**
+ * SECURITY: Trusted-input boundary.
+ *
+ * The `command-succeeds` assertion executes an arbitrary shell command
+ * inside the sandbox's working directory. The command string comes from
+ * task definitions (YAML or TypeScript config files), which are authored
+ * by developers — not from user input or LLM output.
+ *
+ * This is intentional: the assertion is designed to verify agent output
+ * by running build/test commands (e.g., "npm test", "tsc --noEmit").
+ *
+ * Task definitions are the trust boundary. If you accept task definitions
+ * from untrusted sources, validate commands against an allowlist first.
+ */
+export function buildCommandSucceedsAssertion(assertion) {
+    const command = String(assertion.value ?? "");
+    return {
+        type: "javascript",
+        value: `// command-succeeds: ${command}\n` +
+            `(function() {\n` +
+            `  const { execSync } = require('child_process');\n` +
+            `  const workDir = context.vars.__workingDir || '.';\n` +
+            `  try {\n` +
+            `    execSync(${JSON.stringify(command)}, { cwd: workDir, timeout: 30000 });\n` +
+            `    return { pass: true, score: 1, reason: 'Command succeeded: ' + ${JSON.stringify(command)} };\n` +
+            `  } catch (err) {\n` +
+            `    return {\n` +
+            `      pass: false,\n` +
+            `      score: 0,\n` +
+            `      reason: 'Command failed: ' + (err.message || err),\n` +
+            `    };\n` +
+            `  }\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}
+export function buildDiffMatchesAssertion(assertion) {
+    const expected = assertion.value;
+    return {
+        type: "javascript",
+        value: `// diff-matches\n` +
+            `(function() {\n` +
+            `  const { execSync } = require('child_process');\n` +
+            `  const workDir = context.vars.__workingDir || '.';\n` +
+            `  try {\n` +
+            `    const diff = execSync('git diff', { cwd: workDir, encoding: 'utf-8' });\n` +
+            `    const expected = ${JSON.stringify(expected)};\n` +
+            `    if (typeof expected === 'string') {\n` +
+            `      const contains = diff.includes(expected);\n` +
+            `      return {\n` +
+            `        pass: contains,\n` +
+            `        score: contains ? 1 : 0,\n` +
+            `        reason: contains ? 'Diff matches expected pattern' : 'Diff does not match',\n` +
+            `      };\n` +
+            `    }\n` +
+            `    return { pass: diff.length > 0, score: diff.length > 0 ? 1 : 0, reason: 'Diff exists' };\n` +
+            `  } catch (err) {\n` +
+            `    return { pass: false, score: 0, reason: 'Failed to get diff: ' + err.message };\n` +
+            `  }\n` +
+            `})()`,
+        ...(typeof assertion.weight === "number"
+            ? { weight: assertion.weight }
+            : {}),
+    };
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Agent harness task compilation — core compilation logic.
+ *
+ * Maps agent harness task definitions to Promptfoo configuration with:
+ * - Claude Agent SDK / OpenAI Codex SDK providers
+ * - Tool permission configuration (preset/allowed/disallowed)
+ * - Sandbox setup/teardown via Promptfoo extensions
+ * - Fixture provisioning into sandbox working directory
+ */
+import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { AgentHarnessCompileOptions, AgentHarnessCompileResult } from "./types.js";
+/**
+ * Compile an agent harness task definition into Promptfoo configuration.
+ */
+export declare function compileAgentHarnessTask(task: AgentHarnessTaskDefinition, options?: AgentHarnessCompileOptions): AgentHarnessCompileResult;
+export declare function buildAgentProvider(task: AgentHarnessTaskDefinition, _warnings: string[]): PromptfooProvider[];
+export declare function buildAgentPrompts(task: AgentHarnessTaskDefinition): PromptfooPrompt[];
+export declare function buildAgentTestCases(task: AgentHarnessTaskDefinition, options: AgentHarnessCompileOptions | undefined, warnings: string[]): PromptfooTestCase[];