@sanity/ailf 2.0.2 → 2.2.0

package/dist/orchestration/steps/generate-configs-step.js

@@ -20,6 +20,8 @@ import { loadSource } from "../../sources.js";
 import { configToSourceOverrides } from "../config-to-source-overrides.js";
 export class GenerateConfigsStep {
     name = "generate-configs";
+    /** Task IDs from the last loadTasks call (pre-filter), for error messages. */
+    lastLoadedTaskIds = [];
     check(ctx) {
         const issues = validateModelsYaml(ctx.config.rootDir);
         return issues.filter((i) => i.severity === "error");
@@ -54,10 +56,10 @@ export class GenerateConfigsStep {
             // Load tasks
             const tasks = await this.loadTasks(ctx, mode, state);
             if (tasks.length === 0) {
+                const error = this.buildNoTasksError(ctx, mode);
                 return {
                     durationMs: Date.now() - start,
-                    error: `No ${mode} tasks found. Create *.task.ts files in ` +
-                        `packages/eval/tasks/${mode}/`,
+                    error,
                     status: "failed",
                 };
             }
@@ -249,6 +251,10 @@ export class GenerateConfigsStep {
         return filtered;
     }
     applyFilters(ctx, tasks) {
+        // Capture pre-filter IDs for diagnostic messages
+        this.lastLoadedTaskIds = tasks
+            .map((t) => t.id)
+            .filter((id) => !!id);
         let result = tasks;
         if (ctx.config.areas?.length) {
             const allowed = new Set(ctx.config.areas.map((a) => a.toLowerCase()));
@@ -273,6 +279,33 @@ export class GenerateConfigsStep {
         }
         return result;
     }
+    /**
+     * Build a descriptive error message when no tasks match the current filters.
+     * Distinguishes between "no tasks exist" and "tasks exist but filters exclude them".
+     */
+    buildNoTasksError(ctx, mode) {
+        const filters = [];
+        if (ctx.config.tasks?.length) {
+            filters.push(`--task ${ctx.config.tasks.join(", ")}`);
+        }
+        if (ctx.config.areas?.length) {
+            filters.push(`--area ${ctx.config.areas.join(", ")}`);
+        }
+        if (ctx.config.tags?.length) {
+            filters.push(`--tag ${ctx.config.tags.join(", ")}`);
+        }
+        if (filters.length > 0) {
+            // Collect available task IDs for the hint
+            const availableIds = this.lastLoadedTaskIds ?? [];
+            const hint = availableIds.length > 0
+                ? `\n  Available ${mode} task IDs: ${availableIds.join(", ")}`
+                : "";
+            return (`No ${mode} tasks match the current filters (${filters.join("; ")}).` +
+                hint);
+        }
+        return (`No ${mode} tasks found. Create *.task.ts files in ` +
+            `packages/eval/tasks/${mode}/`);
+    }
     // ---------------------------------------------------------------------------
     // Compilation helpers
     // ---------------------------------------------------------------------------

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js

@@ -87,15 +87,23 @@ describe("validateAgentHarnessTask", () => {
 // compileAgentHarnessTask — provider assembly
 // ---------------------------------------------------------------------------
 describe("compileAgentHarnessTask — providers", () => {
-    it("produces a provider", () => {
+    it("produces a Claude Agent SDK provider", () => {
         const result = compileAgentHarnessTask(makeTask());
         assert.ok(result.providers.length > 0);
-        assert.ok(result.providers[0].id.startsWith("agent:"));
+        assert.equal(result.providers[0].id, "anthropic:claude-agent-sdk");
     });
-    it("resolves coding tool preset", () => {
+    it("sets default agent config", () => {
+        const result = compileAgentHarnessTask(makeTask());
+        const config = result.providers[0].config;
+        assert.ok(config.model, "should set a model");
+        assert.ok(config.max_turns, "should set max_turns");
+        assert.ok(config.max_budget_usd, "should set budget cap");
+        assert.equal(config.permission_mode, "bypassPermissions");
+    });
+    it("resolves coding tool preset into custom_allowed_tools", () => {
         const result = compileAgentHarnessTask(makeTask({ tools: ["coding"] }));
         const config = result.providers[0].config;
-        const tools = config.allowedTools;
+        const tools = config.custom_allowed_tools;
         assert.ok(tools.includes("Bash"));
         assert.ok(tools.includes("Read"));
         assert.ok(tools.includes("Write"));
@@ -104,7 +112,7 @@ describe("compileAgentHarnessTask — providers", () => {
     it("resolves read-only tool preset", () => {
         const result = compileAgentHarnessTask(makeTask({ tools: ["read-only"] }));
         const config = result.providers[0].config;
-        const tools = config.allowedTools;
+        const tools = config.custom_allowed_tools;
         assert.ok(tools.includes("Read"));
         assert.ok(tools.includes("Grep"));
         assert.ok(!tools.includes("Write"), "read-only should not include Write");
@@ -112,19 +120,10 @@ describe("compileAgentHarnessTask — providers", () => {
     it("mixes preset and explicit tools", () => {
         const result = compileAgentHarnessTask(makeTask({ tools: ["read-only", "WebFetch"] }));
         const config = result.providers[0].config;
-        const tools = config.allowedTools;
+        const tools = config.custom_allowed_tools;
         assert.ok(tools.includes("Read"));
         assert.ok(tools.includes("WebFetch"));
     });
-    it("includes sandbox config in provider", () => {
-        const result = compileAgentHarnessTask(makeTask({
-            sandbox: { type: "docker", image: "node:22-slim" },
-        }));
-        const config = result.providers[0].config;
-        const sandbox = config.sandbox;
-        assert.equal(sandbox.type, "docker");
-        assert.equal(sandbox.image, "node:22-slim");
-    });
 });
 // ---------------------------------------------------------------------------
 // compileAgentHarnessTask — test cases
@@ -166,16 +165,20 @@ describe("compileAgentHarnessTask — test cases", () => {
 // compileAgentHarnessTask — assertions
 // ---------------------------------------------------------------------------
 describe("compileAgentHarnessTask — assertions", () => {
-    it("maps file-exists to javascript assertion", () => {
+    const RUNTIME = "file://dist/agent-harness/assertions-runtime.js";
+    it("maps file-exists to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [{ type: "file-exists", value: "sanity.config.ts" }],
         }));
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("sanity.config.ts"));
+        assert.equal(assertion.value, `${RUNTIME}:fileExists`);
+        assert.deepEqual(assertion.config, {
+            filePath: "sanity.config.ts",
+        });
     });
-    it("maps file-contains to javascript assertion", () => {
+    it("maps file-contains to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [
                 {
@@ -187,25 +190,35 @@ describe("compileAgentHarnessTask — assertions", () => {
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("projectId"));
+        assert.equal(assertion.value, `${RUNTIME}:fileContains`);
+        assert.deepEqual(assertion.config, {
+            filePath: "config.ts",
+            content: "projectId",
+        });
     });
-    it("maps command-succeeds to javascript assertion", () => {
+    it("maps command-succeeds to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [{ type: "command-succeeds", value: "npx tsc --noEmit" }],
         }));
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("tsc"));
+        assert.equal(assertion.value, `${RUNTIME}:commandSucceeds`);
+        assert.deepEqual(assertion.config, {
+            command: "npx tsc --noEmit",
+        });
     });
-    it("maps diff-matches to javascript assertion", () => {
+    it("maps diff-matches to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [{ type: "diff-matches", value: "createClient" }],
         }));
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("git diff"));
+        assert.equal(assertion.value, `${RUNTIME}:diffMatches`);
+        assert.deepEqual(assertion.config, {
+            expected: "createClient",
+        });
     });
     it("passes through standard assertions", () => {
         const result = compileAgentHarnessTask(makeTask({
@@ -250,7 +263,7 @@ describe("compileAgentHarnessTask — lifecycle", () => {
         }));
         assert.equal(result.sandboxConfig.type, "docker");
         assert.equal(result.sandboxConfig.image, "node:22");
-        assert.deepEqual(result.sandboxConfig.fixtures, ["file://schema.ts"]);
+        assert.deepEqual(result.sandboxConfig.fixtures, ["schema.ts"]);
         assert.equal(result.sandboxConfig.limits?.cpus, 2);
         assert.equal(result.sandboxConfig.limits?.networkAccess, false);
     });
@@ -278,7 +291,8 @@ describe("example agent harness tasks — end-to-end", () => {
         const result = compileAgentHarnessTask(modifyCodeTask);
         assert.ok(result.tests[0].assert);
         assert.ok(result.tests[0].assert.some((a) => a.type === "javascript" &&
-            a.value.includes("useDocumentOperation")));
+            a.value.includes("fileContains") &&
+            a.config != null));
     });
     it("refactor task has docker sandbox config", () => {
         const result = compileAgentHarnessTask(multiFileRefactorTask);

package/dist/pipeline/compiler/compiler-to-yaml.js

@@ -65,12 +65,38 @@ export function writeCompiledModeConfig(result, mode, options) {
     if (options.graderProvider) {
         graderOpts.provider = options.graderProvider;
     }
-    // Build provider entries
+    // For agent-harness mode, create sandbox directories and inject working_dir
+    // into provider configs. The sandbox must exist before the provider initializes
+    // (the Claude Agent SDK reads working_dir at construction time).
+    // Both working_dir and __workingDir use absolute paths to avoid ambiguity.
+    // @see https://www.promptfoo.dev/docs/providers/claude-agent-sdk/
+    const sandboxAbsPath = result.extras?.sandboxConfig
+        ? resolve(options.rootDir, `results/latest/sandbox-${mode}`)
+        : undefined;
+    if (sandboxAbsPath) {
+        mkdirSync(sandboxAbsPath, { recursive: true });
+    }
+    // Build provider entries, injecting working_dir for agent-harness providers
     const providerEntries = result.providers.map((p) => {
-        if (p.config)
-            return { id: p.id, label: p.label, config: p.config };
-        return p.label ? { id: p.id, label: p.label } : p.id;
+        if (!p.config)
+            return p.label ? { id: p.id, label: p.label } : p.id;
+        const config = { ...p.config };
+        if (sandboxAbsPath && p.id === "anthropic:claude-agent-sdk") {
+            config.working_dir = sandboxAbsPath;
+        }
+        return { id: p.id, label: p.label, config };
     });
+    // Inject __workingDir into test vars so assertions can find the sandbox
+    if (sandboxAbsPath) {
+        for (const test of expandedTests) {
+            if (test.vars) {
+                ;
+                test.vars.__workingDir = sandboxAbsPath;
+            }
+        }
+        // Re-write the tests file with the injected paths
+        writeFileSync(testsPath, JSON.stringify(expandedTests, null, 2), "utf-8");
+    }
     // Build prompt entries
     const prompts = result.prompts.map((p) => ({
         id: p.id,
@@ -88,10 +114,11 @@ export function writeCompiledModeConfig(result, mode, options) {
         tests: [testsFilename],
     });
     // Include extensions if present (agent-harness mode)
+    // Promptfoo expects extensions as string[] (file paths to JS modules),
+    // so we materialize the { type, code } objects as a .cjs file on disk.
     if (result.extras?.extensions) {
-        ;
-        config.extensions =
-            result.extras.extensions;
+        const extPaths = writeExtensionFile(options.rootDir, mode, result.extras.extensions);
+        config.extensions = extPaths;
     }
     writeConfig(options.rootDir, filename, config, options.logger);
 }
@@ -215,3 +242,47 @@ function writeYaml(path, data, header) {
     });
     writeFileSync(path, `${header}\n${yamlStr}`, "utf-8");
 }
+/**
+ * Materialize Promptfoo lifecycle extensions as a .cjs file on disk.
+ *
+ * Promptfoo extensions use a single-function dispatch pattern:
+ *   module.exports = async function(hookName, context) { ... }
+ *
+ * Each extension entry in the YAML references:
+ *   file://path/to/file.cjs:exportedFunctionName
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
+ */
+function writeExtensionFile(rootDir, mode, extensions) {
+    // Build a dispatch map: hookName → handler code
+    const hookMap = {};
+    for (const ext of extensions) {
+        hookMap[ext.type] = ext.code;
+    }
+    // Generate the single dispatch function that promptfoo expects
+    const hookCases = Object.entries(hookMap)
+        .map(([hookName, code]) => `  if (hookName === '${hookName}') {\n` +
+        `    const handler = ${code};\n` +
+        `    return handler(context);\n` +
+        `  }`)
+        .join("\n");
+    const fileContent = [
+        "// AUTO-GENERATED by compiler pipeline — do not edit directly.",
+        "// Run: npx @sanity/ailf generate-configs",
+        "//",
+        "// Promptfoo extension dispatch function.",
+        `// @see https://www.promptfoo.dev/docs/configuration/reference/`,
+        "",
+        "async function extensionHook(hookName, context) {",
+        hookCases,
+        "}",
+        "",
+        "module.exports = extensionHook;",
+        "",
+    ].join("\n");
+    const filename = `results/latest/${mode}-extensions.cjs`;
+    const outPath = resolve(rootDir, filename);
+    writeFileSync(outPath, fileContent, "utf-8");
+    // Single entry pointing to the dispatch function
+    return [`file://${filename}:extensionHook`];
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts

@@ -4,6 +4,15 @@
  * Handles agent-specific assertion types (file-exists, file-contains,
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
+ *
+ * Agent-specific assertions use file-based references to the assertions
+ * runtime module (dist/agent-harness/assertions-runtime.js) because
+ * promptfoo's inline `type: javascript` assertions run in a restricted
+ * eval() sandbox where require() is unavailable. File-based assertions
+ * run in a full Node.js context.
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ * @see src/agent-harness/assertions-runtime.ts — runtime implementations
  */
 import type { PromptfooAssertion } from "../../assertion-mapper.js";
 import type { AgentHarnessCompileOptions } from "./types.js";

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js

@@ -4,7 +4,18 @@
  * Handles agent-specific assertion types (file-exists, file-contains,
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
+ *
+ * Agent-specific assertions use file-based references to the assertions
+ * runtime module (dist/agent-harness/assertions-runtime.js) because
+ * promptfoo's inline `type: javascript` assertions run in a restricted
+ * eval() sandbox where require() is unavailable. File-based assertions
+ * run in a full Node.js context.
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ * @see src/agent-harness/assertions-runtime.ts — runtime implementations
  */
+/** Base path for the file-based assertion runtime module */
+const RUNTIME = "file://dist/agent-harness/assertions-runtime.js";
 // ---------------------------------------------------------------------------
 // Assertion mapping
 // ---------------------------------------------------------------------------
@@ -53,66 +64,29 @@ export function mapAgentAssertion(assertion, options, warnings) {
 }
 // ---------------------------------------------------------------------------
 // Agent-specific assertion builders
+//
+// Each builder returns a file-based assertion referencing the runtime
+// module with parameters passed via the `config` field.
 // ---------------------------------------------------------------------------
 export function buildFileExistsAssertion(assertion) {
-    const filePath = String(assertion.value ?? "");
-    // Use JSON.stringify for all interpolated values in generated JS to
-    // prevent broken strings from filePaths containing quotes/backslashes
-    const safeFilePath = JSON.stringify(filePath);
     return {
         type: "javascript",
-        value: `// file-exists: ${filePath}\n` +
-            `(function() {\n` +
-            `  const fs = require('fs');\n` +
-            `  const path = require('path');\n` +
-            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
-            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
-            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
-            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
-            `  }\n` +
-            `  const exists = fs.existsSync(target);\n` +
-            `  return {\n` +
-            `    pass: exists,\n` +
-            `    score: exists ? 1 : 0,\n` +
-            `    reason: exists\n` +
-            `      ? 'File exists: ' + ${safeFilePath}\n` +
-            `      : 'Expected file not found: ' + ${safeFilePath},\n` +
-            `  };\n` +
-            `})()`,
+        value: `${RUNTIME}:fileExists`,
+        config: { filePath: String(assertion.value ?? "") },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),
     };
 }
 export function buildFileContainsAssertion(assertion) {
-    const config = assertion.value;
-    const filePath = config?.path ?? "";
-    const expectedContent = config?.content ?? "";
-    const safeFilePath = JSON.stringify(filePath);
+    const val = assertion.value;
     return {
         type: "javascript",
-        value: `// file-contains: ${filePath}\n` +
-            `(function() {\n` +
-            `  const fs = require('fs');\n` +
-            `  const path = require('path');\n` +
-            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
-            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
-            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
-            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
-            `  }\n` +
-            `  if (!fs.existsSync(target)) {\n` +
-            `    return { pass: false, score: 0, reason: 'File not found: ' + ${safeFilePath} };\n` +
-            `  }\n` +
-            `  const content = fs.readFileSync(target, 'utf-8');\n` +
-            `  const contains = content.includes(${JSON.stringify(expectedContent)});\n` +
-            `  return {\n` +
-            `    pass: contains,\n` +
-            `    score: contains ? 1 : 0,\n` +
-            `    reason: contains\n` +
-            `      ? 'File contains expected content'\n` +
-            `      : 'File does not contain expected content',\n` +
-            `  };\n` +
-            `})()`,
+        value: `${RUNTIME}:fileContains`,
+        config: {
+            filePath: val?.path ?? "",
+            content: val?.content ?? "",
+        },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),
@@ -133,53 +107,22 @@ export function buildFileContainsAssertion(assertion) {
  * from untrusted sources, validate commands against an allowlist first.
  */
 export function buildCommandSucceedsAssertion(assertion) {
-    const command = String(assertion.value ?? "");
     return {
         type: "javascript",
-        value: `// command-succeeds: ${command}\n` +
-            `(function() {\n` +
-            `  const { execSync } = require('child_process');\n` +
-            `  const workDir = context.vars.__workingDir || '.';\n` +
-            `  try {\n` +
-            `    execSync(${JSON.stringify(command)}, { cwd: workDir, timeout: 30000 });\n` +
-            `    return { pass: true, score: 1, reason: 'Command succeeded: ' + ${JSON.stringify(command)} };\n` +
-            `  } catch (err) {\n` +
-            `    return {\n` +
-            `      pass: false,\n` +
-            `      score: 0,\n` +
-            `      reason: 'Command failed: ' + (err.message || err),\n` +
-            `    };\n` +
-            `  }\n` +
-            `})()`,
+        value: `${RUNTIME}:commandSucceeds`,
+        config: { command: String(assertion.value ?? "") },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),
     };
 }
 export function buildDiffMatchesAssertion(assertion) {
-    const expected = assertion.value;
     return {
         type: "javascript",
-        value: `// diff-matches\n` +
-            `(function() {\n` +
-            `  const { execSync } = require('child_process');\n` +
-            `  const workDir = context.vars.__workingDir || '.';\n` +
-            `  try {\n` +
-            `    const diff = execSync('git diff', { cwd: workDir, encoding: 'utf-8' });\n` +
-            `    const expected = ${JSON.stringify(expected)};\n` +
-            `    if (typeof expected === 'string') {\n` +
-            `      const contains = diff.includes(expected);\n` +
-            `      return {\n` +
-            `        pass: contains,\n` +
-            `        score: contains ? 1 : 0,\n` +
-            `        reason: contains ? 'Diff matches expected pattern' : 'Diff does not match',\n` +
-            `      };\n` +
-            `    }\n` +
-            `    return { pass: diff.length > 0, score: diff.length > 0 ? 1 : 0, reason: 'Diff exists' };\n` +
-            `  } catch (err) {\n` +
-            `    return { pass: false, score: 0, reason: 'Failed to get diff: ' + err.message };\n` +
-            `  }\n` +
-            `})()`,
+        value: `${RUNTIME}:diffMatches`,
+        config: {
+            ...(assertion.value != null ? { expected: assertion.value } : {}),
+        },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),

package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.js

@@ -27,8 +27,10 @@ export function compileAgentHarnessTask(task, options) {
     const prompts = buildAgentPrompts(task);
     // Build test cases
     const tests = buildAgentTestCases(task, options, warnings);
-    // Build sandbox extensions
-    const sandboxConfig = buildSandboxConfig(task);
+    // Build sandbox extensions — resolve fixture paths at compile time using
+    // the caller's cwd (monorepo root), not the eval package rootDir.
+    const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
+    const sandboxConfig = buildSandboxConfig(task, callerCwd);
     const extensions = buildLifecycleExtensions(task, sandboxConfig);
     return { providers, tests, prompts, extensions, sandboxConfig, warnings };
 }
@@ -36,22 +38,25 @@ export function compileAgentHarnessTask(task, options) {
 // Provider assembly
 // ---------------------------------------------------------------------------
 export function buildAgentProvider(task, _warnings) {
-    // Resolve tool permissions
     const tools = resolveToolPermissions(task.tools);
-    const config = {};
+    // Claude Agent SDK config.
+    // working_dir is set by the YAML writer to the sandbox path it creates.
+    // @see https://www.promptfoo.dev/docs/providers/claude-agent-sdk/
+    const config = {
+        model: "claude-sonnet-4-20250514",
+        max_turns: 25,
+        max_budget_usd: 1.0,
+        permission_mode: "bypassPermissions",
+        allow_dangerously_skip_permissions: true,
+    };
+    // Map AILF tool names to Claude Agent SDK tool config.
+    // Claude SDK uses custom_allowed_tools to replace defaults.
     if (tools.length > 0) {
-        config.allowedTools = tools;
-    }
-    if (task.sandbox) {
-        config.sandbox = {
-            type: task.sandbox.type,
-            ...(task.sandbox.image ? { image: task.sandbox.image } : {}),
-        };
+        config.custom_allowed_tools = tools;
     }
-    // Default to Claude Agent SDK provider
     return [
         {
-            id: `agent:${task.id}`,
+            id: "anthropic:claude-agent-sdk",
             label: `Agent Harness: ${task.title}`,
             config,
         },
@@ -112,9 +117,11 @@ export function buildAgentTestCases(task, options, warnings) {
     const vars = {
         task: task.prompt?.vars?.task ?? task.description ?? `Complete: ${task.title}`,
         ...(task.prompt?.vars ?? {}),
-        // Internal metadata for sandbox lifecycle hooks
+        // Internal metadata for sandbox lifecycle hooks.
+        // Fixture paths are plain strings (no file:// prefix) because
+        // promptfoo auto-resolves file:// in vars by reading file content.
         __sandboxType: task.sandbox?.type ?? "tempdir",
-        __fixtures: task.fixtures ?? [],
+        __fixtures: (task.fixtures ?? []).map((f) => f.startsWith("file://") ? f.slice(7) : f),
     };
     const tests = [
         {

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.d.ts

@@ -6,7 +6,14 @@
  */
 import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
 import type { PromptfooExtension, SandboxConfigMeta } from "./types.js";
-export declare function buildSandboxConfig(task: AgentHarnessTaskDefinition): SandboxConfigMeta;
+/**
+ * Build sandbox configuration from a task definition.
+ *
+ * Fixture paths are resolved to absolute at compile time using callerCwd
+ * (the directory the pipeline was invoked from), because promptfoo runs
+ * with cwd set to packages/eval/ — not the monorepo root where apps/ lives.
+ */
+export declare function buildSandboxConfig(task: AgentHarnessTaskDefinition, callerCwd?: string): SandboxConfigMeta;
 export declare function buildLifecycleExtensions(task: AgentHarnessTaskDefinition, sandboxConfig: SandboxConfigMeta): PromptfooExtension[];
 export declare function buildBeforeEachHook(taskId: string, config: SandboxConfigMeta): string;
 export declare function buildAfterEachHook(taskId: string): string;

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.js

@@ -4,14 +4,26 @@
  * Builds Promptfoo beforeEach/afterEach hooks for provisioning and
  * tearing down sandbox working directories.
  */
+import { resolve } from "path";
 // ---------------------------------------------------------------------------
 // Sandbox configuration
 // ---------------------------------------------------------------------------
-export function buildSandboxConfig(task) {
+/**
+ * Build sandbox configuration from a task definition.
+ *
+ * Fixture paths are resolved to absolute at compile time using callerCwd
+ * (the directory the pipeline was invoked from), because promptfoo runs
+ * with cwd set to packages/eval/ — not the monorepo root where apps/ lives.
+ */
+export function buildSandboxConfig(task, callerCwd) {
+    const cwd = callerCwd ?? process.cwd();
     return {
         type: task.sandbox?.type ?? "tempdir",
         image: task.sandbox?.image,
-        fixtures: task.fixtures ?? [],
+        fixtures: (task.fixtures ?? []).map((f) => {
+            const stripped = f.startsWith("file://") ? f.slice(7) : f;
+            return resolve(cwd, stripped);
+        }),
         limits: task.sandbox?.limits
             ? {
                 cpus: task.sandbox.limits.cpus,
@@ -39,23 +51,41 @@ export function buildLifecycleExtensions(task, sandboxConfig) {
     return extensions;
 }
 export function buildBeforeEachHook(taskId, config) {
-    return (`// beforeEach: provision sandbox for ${taskId}\n` +
-        `async function({ vars }) {\n` +
-        `  const { mkdirSync, writeFileSync } = require('fs');\n` +
-        `  const { tmpdir } = require('os');\n` +
+    // Promptfoo extension hooks receive (hookName, context).
+    // beforeEach context is { test } — vars live at context.test.vars.
+    // Must return context for mutations to persist.
+    //
+    // The sandbox directory is created by the YAML writer at config-gen time
+    // (deterministic path in results/latest/sandbox-{taskId}/) so it exists
+    // before the provider is initialized. This hook copies fixtures into it.
+    //
+    // @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
+    return (`// beforeEach: copy fixtures into sandbox for ${taskId}\n` +
+        `async function(context) {\n` +
+        `  const { cpSync, existsSync, mkdirSync } = require('fs');\n` +
         `  const { resolve } = require('path');\n` +
-        `  const id = 'ailf-${taskId}-' + require('crypto').randomUUID().slice(0, 8);\n` +
-        `  const workDir = resolve(tmpdir(), id);\n` +
+        `  const workDir = context.test.vars?.__workingDir;\n` +
+        `  if (!workDir) return context;\n` +
         `  mkdirSync(workDir, { recursive: true });\n` +
-        `  vars.__workingDir = workDir;\n` +
-        `  vars.__sandboxId = id;\n` +
-        `  // Fixture list: ${JSON.stringify(config.fixtures)}\n` +
+        `  // Copy fixtures into sandbox\n` +
+        `  const fixtures = ${JSON.stringify(config.fixtures)};\n` +
+        `  for (const fixture of fixtures) {\n` +
+        `    const src = resolve(process.cwd(), fixture);\n` +
+        `    if (existsSync(src)) {\n` +
+        `      cpSync(src, workDir, { recursive: true });\n` +
+        `    }\n` +
+        `  }\n` +
+        `  return context;\n` +
         `}`);
 }
 export function buildAfterEachHook(taskId) {
+    // Promptfoo extension hooks receive (hookName, context).
+    // afterEach context is { test, result } — vars live at context.test.vars.
+    // @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
     return (`// afterEach: collect artifacts + teardown for ${taskId}\n` +
-        `async function({ vars }) {\n` +
+        `async function(context) {\n` +
         `  const { rmSync, readdirSync, existsSync } = require('fs');\n` +
+        `  const vars = context.test.vars || {};\n` +
         `  const workDir = vars.__workingDir;\n` +
         `  if (workDir && existsSync(workDir)) {\n` +
         `    try {\n` +