@sanity/ailf 2.1.0 → 2.2.0

package/dist/_vendor/ailf-core/examples/index.d.ts

@@ -112,6 +112,55 @@ export declare const ailfConfigData: {
 };
 /** Raw YAML string for ailf-config example (preserves comments) */
 export declare const ailfConfigYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# .ailf/config.yaml \u2014 AI Literacy Framework project configuration\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This file configures how the AILF evaluation pipeline runs in this\n# repository. Place it at .ailf/config.yaml in your project root.\n#\n# Evaluations are submitted to the AILF API (ailf-api.sanity.build).\n# The API handles LLM calls, doc fetching, grading, and report\n# publishing. Your repo only needs one secret: AILF_API_KEY.\n#\n# Docs: https://github.com/sanity-labs/ai-literacy-framework\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n# Documentation source \u2014 which docs are being evaluated.\n#\n# This tells the pipeline which Sanity project and dataset contain\n# the documentation under test. For most users, this is Sanity's own\n# docs project.\n#\n# projectId \u2014 Sanity project ID (find yours at sanity.io/manage)\n# dataset   \u2014 the dataset to query (e.g., \"production\", \"next\")\n# baseUrl   \u2014 the public URL of your documentation site\n#             (used by agentic mode to test agent discoverability)\nsource:\n  projectId: \"3do82whm\"\n  dataset: next\n  baseUrl: \"https://www.sanity.io/docs\"\n\n# Trigger configuration \u2014 when evaluations run automatically.\n#\n# Each key is a trigger context. The pipeline checks which trigger\n# matches the current execution context (PR, merge, schedule, etc.)\n# and applies its settings.\n#\n# mode options:\n#   validate-only \u2014 check that task YAML parses correctly (fast, no LLM calls)\n#   eval          \u2014 run the full evaluation pipeline\n#\n# paths \u2014 only trigger when files matching these globs change\n# blocking \u2014 if true, a failing eval blocks the PR merge\n# notify \u2014 if true, post results to configured notification channels\ntriggers:\n  # On pull requests: just validate task files parse correctly\n  pr:\n    mode: validate-only\n\n  # When .ailf/ files change in a PR: run a real evaluation\n  pr-task-change:\n    mode: eval\n    paths: [\".ailf/**\"]\n\n  # On merge to main: run evaluation (non-blocking)\n  main:\n    mode: eval\n    blocking: false\n    notify: true\n";
+/** Parsed task data for example-agent-add-schema (JSON-safe) */
+export declare const exampleAgentAddSchemaData: readonly [{
+    readonly mode: "agent-harness";
+    readonly id: "example-agent-add-schema";
+    readonly title: "Add a document schema to a Sanity Studio project";
+    readonly description: "Example — tests whether an agent can create a schema file and register it in an existing Sanity Studio project";
+    readonly area: "studio";
+    readonly sandbox: {
+        readonly type: "tempdir";
+    };
+    readonly tools: readonly ["coding"];
+    readonly fixtures: readonly ["file://apps/studio-basic"];
+    readonly prompt: {
+        readonly text: "You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema.";
+    };
+    readonly assertions: readonly [{
+        readonly type: "file-exists";
+        readonly value: "schemas/post.ts";
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "schemas/post.ts";
+            readonly content: "defineType";
+        };
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "schemas/post.ts";
+            readonly content: "defineField";
+        };
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "schemas/index.ts";
+            readonly content: "post";
+        };
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "sanity.config.ts";
+            readonly content: "defineConfig";
+        };
+    }];
+    readonly status: "draft";
+}];
+/** TypeScript task template for example-agent-add-schema */
+export declare const exampleAgentAddSchemaTs = "/**\n * Example Task: Agent harness \u2014 add a schema to a Sanity Studio project.\n *\n * This is a starter template for agent-harness evaluations. It tests whether\n * an autonomous coding agent can modify a real project in a sandboxed\n * environment using file system tools.\n *\n * Unlike literacy or knowledge-probe tasks that evaluate text responses,\n * agent-harness tasks evaluate *side-effects*: files created, code modified,\n * commands executed. The agent gets a working directory with real files and\n * tools (Bash, Read, Write, Edit, etc.) and is graded on what it produces.\n *\n * This task is a DRAFT \u2014 it won't run unless activated or explicitly targeted.\n * To activate: change status to \"active\" or remove the status field.\n *\n * @see https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/modes.md#agent-harness\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  // \u2500\u2500 Mode \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // \"agent-harness\" runs a real coding agent (Claude Agent SDK) with\n  // filesystem tools in an isolated sandbox. The agent can read, write,\n  // edit files and run shell commands \u2014 then assertions verify the results.\n  //\n  // Other modes: \"literacy\" (text Q&A), \"mcp-server\" (tool use),\n  //              \"knowledge-probe\" (baseline knowledge).\n  mode: \"agent-harness\",\n\n  id: \"example-agent-add-schema\",\n  title: \"Add a document schema to a Sanity Studio project\",\n  description:\n    \"Example \u2014 tests whether an agent can create a schema file and \" +\n    \"register it in an existing Sanity Studio project\",\n\n  // Area groups tasks for scoring. All agent-harness tasks in the same\n  // area are aggregated together in the score report.\n  area: \"studio\",\n\n  // \u2500\u2500 Sandbox \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // The sandbox isolates the agent's file operations from your real repo.\n  // Currently only \"tempdir\" is implemented (creates a temporary directory).\n  // The pipeline creates the sandbox dir at config-generation time, copies\n  // fixtures into it before the agent starts, and the agent's working_dir\n  // is set to the sandbox path.\n  //\n  // See: docs/modes.md#sandbox-lifecycle\n  sandbox: { type: \"tempdir\" },\n\n  // \u2500\u2500 Tools \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Tool presets control which Claude Agent SDK tools the agent can use:\n  //   \"coding\"      \u2192 Bash, Read, Write, Edit, Glob, Grep\n  //   \"full-access\" \u2192 coding + WebSearch, WebFetch, TodoRead, TodoWrite\n  //   \"read-only\"   \u2192 Read, Glob, Grep, WebSearch\n  //\n  // You can also mix presets with explicit tool names:\n  //   tools: [\"coding\", \"WebFetch\"]\n  tools: [\"coding\"],\n\n  // \u2500\u2500 Fixtures \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Fixtures are files or directories copied into the sandbox before the\n  // agent starts. Paths use the file:// prefix and resolve relative to\n  // the directory where you run the pipeline (typically your repo root).\n  //\n  // The agent sees these files in its working directory as if they were\n  // a real project. For example, \"file://apps/studio-basic\" copies the\n  // entire studio-basic app into the sandbox root.\n  //\n  // If you don't have fixture projects, you can omit this field and the\n  // agent starts with an empty directory.\n  fixtures: [\"file://apps/studio-basic\"],\n\n  // \u2500\u2500 Prompt \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // The task instructions sent to the agent. Be specific about expected\n  // file paths and patterns \u2014 the assertions check for exact paths.\n  prompt: {\n    text: `You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema.`,\n  },\n\n  // \u2500\u2500 Assertions \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Agent-harness assertions verify the sandbox state after the agent runs.\n  // They execute in a full Node.js context (not eval()) so they can use\n  // fs, child_process, etc. All file paths resolve relative to the sandbox.\n  //\n  // Available assertion types:\n  //   file-exists      \u2014 check a file was created\n  //   file-contains    \u2014 check a file contains a substring\n  //   command-succeeds \u2014 run a shell command (exit 0 = pass)\n  //   diff-matches     \u2014 check git diff contains a pattern\n  //   llm-rubric       \u2014 LLM grades the agent's text output\n  assertions: [\n    // Verify the agent created the new schema file\n    { type: \"file-exists\", value: \"schemas/post.ts\" },\n\n    // Verify it uses the modern Sanity schema API\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineType\" },\n    },\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineField\" },\n    },\n\n    // Verify the schema was registered in the barrel export\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/index.ts\", content: \"post\" },\n    },\n\n    // Verify the existing config is still intact\n    {\n      type: \"file-contains\",\n      value: { path: \"sanity.config.ts\", content: \"defineConfig\" },\n    },\n  ],\n\n  status: \"draft\",\n})\n";
+/** Generated YAML for example-agent-add-schema (from parsed TS data) */
+export declare const exampleAgentAddSchemaYaml = "- mode: agent-harness\n  id: example-agent-add-schema\n  title: Add a document schema to a Sanity Studio project\n  description: Example \u2014 tests whether an agent can create a schema file and register it in an existing Sanity Studio project\n  area: studio\n  sandbox:\n    type: tempdir\n  tools:\n    - coding\n  fixtures:\n    - file://apps/studio-basic\n  prompt:\n    text: |-\n      You have a Sanity Studio project with an existing article schema.\n      Add a new \"post\" document type with the following fields:\n      1. title (string, required)\n      2. slug (slug, sourced from title, required)\n      3. author (string)\n      4. publishedAt (datetime)\n      5. body (array of block content)\n\n      Create the schema file at schemas/post.ts using defineType() and defineField().\n      Register it in schemas/index.ts alongside the existing article schema.\n  assertions:\n    - type: file-exists\n      value: schemas/post.ts\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineType\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineField\n    - type: file-contains\n      value:\n        path: schemas/index.ts\n        content: post\n    - type: file-contains\n      value:\n        path: sanity.config.ts\n        content: defineConfig\n  status: draft\n";
 /** Parsed task data for example-groq-blog-listing (JSON-safe) */
 export declare const exampleGroqBlogListingData: readonly [{
     readonly mode: "literacy";
@@ -367,7 +416,7 @@ export declare const taskTsFiles: Record<string, string>;
 /** Map of task ID (filename stem) → generated YAML string */
 export declare const taskYamlFiles: Record<string, string>;
 /** List of task file stems, in alphabetical order */
-export declare const TASK_FILE_NAMES: readonly ["example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
+export declare const TASK_FILE_NAMES: readonly ["example-agent-add-schema", "example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
 /** Task metadata for mode-based filtering in init and other consumers */
 export interface TaskExampleMeta {
     stem: string;

package/dist/_vendor/ailf-core/examples/index.js

@@ -143,6 +143,67 @@ export const ailfConfigData = {
 };
 /** Raw YAML string for ailf-config example (preserves comments) */
 export const ailfConfigYaml = "# ──────────────────────────────────────────────────────────────────────\n# .ailf/config.yaml — AI Literacy Framework project configuration\n# ──────────────────────────────────────────────────────────────────────\n#\n# This file configures how the AILF evaluation pipeline runs in this\n# repository. Place it at .ailf/config.yaml in your project root.\n#\n# Evaluations are submitted to the AILF API (ailf-api.sanity.build).\n# The API handles LLM calls, doc fetching, grading, and report\n# publishing. Your repo only needs one secret: AILF_API_KEY.\n#\n# Docs: https://github.com/sanity-labs/ai-literacy-framework\n# ──────────────────────────────────────────────────────────────────────\n\n# Documentation source — which docs are being evaluated.\n#\n# This tells the pipeline which Sanity project and dataset contain\n# the documentation under test. For most users, this is Sanity's own\n# docs project.\n#\n# projectId — Sanity project ID (find yours at sanity.io/manage)\n# dataset   — the dataset to query (e.g., \"production\", \"next\")\n# baseUrl   — the public URL of your documentation site\n#             (used by agentic mode to test agent discoverability)\nsource:\n  projectId: \"3do82whm\"\n  dataset: next\n  baseUrl: \"https://www.sanity.io/docs\"\n\n# Trigger configuration — when evaluations run automatically.\n#\n# Each key is a trigger context. The pipeline checks which trigger\n# matches the current execution context (PR, merge, schedule, etc.)\n# and applies its settings.\n#\n# mode options:\n#   validate-only — check that task YAML parses correctly (fast, no LLM calls)\n#   eval          — run the full evaluation pipeline\n#\n# paths — only trigger when files matching these globs change\n# blocking — if true, a failing eval blocks the PR merge\n# notify — if true, post results to configured notification channels\ntriggers:\n  # On pull requests: just validate task files parse correctly\n  pr:\n    mode: validate-only\n\n  # When .ailf/ files change in a PR: run a real evaluation\n  pr-task-change:\n    mode: eval\n    paths: [\".ailf/**\"]\n\n  # On merge to main: run evaluation (non-blocking)\n  main:\n    mode: eval\n    blocking: false\n    notify: true\n";
+/** Parsed task data for example-agent-add-schema (JSON-safe) */
+export const exampleAgentAddSchemaData = [
+    {
+        "mode": "agent-harness",
+        "id": "example-agent-add-schema",
+        "title": "Add a document schema to a Sanity Studio project",
+        "description": "Example — tests whether an agent can create a schema file and register it in an existing Sanity Studio project",
+        "area": "studio",
+        "sandbox": {
+            "type": "tempdir"
+        },
+        "tools": [
+            "coding"
+        ],
+        "fixtures": [
+            "file://apps/studio-basic"
+        ],
+        "prompt": {
+            "text": "You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema."
+        },
+        "assertions": [
+            {
+                "type": "file-exists",
+                "value": "schemas/post.ts"
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "schemas/post.ts",
+                    "content": "defineType"
+                }
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "schemas/post.ts",
+                    "content": "defineField"
+                }
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "schemas/index.ts",
+                    "content": "post"
+                }
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "sanity.config.ts",
+                    "content": "defineConfig"
+                }
+            }
+        ],
+        "status": "draft"
+    }
+];
+/** TypeScript task template for example-agent-add-schema */
+export const exampleAgentAddSchemaTs = "/**\n * Example Task: Agent harness — add a schema to a Sanity Studio project.\n *\n * This is a starter template for agent-harness evaluations. It tests whether\n * an autonomous coding agent can modify a real project in a sandboxed\n * environment using file system tools.\n *\n * Unlike literacy or knowledge-probe tasks that evaluate text responses,\n * agent-harness tasks evaluate *side-effects*: files created, code modified,\n * commands executed. The agent gets a working directory with real files and\n * tools (Bash, Read, Write, Edit, etc.) and is graded on what it produces.\n *\n * This task is a DRAFT — it won't run unless activated or explicitly targeted.\n * To activate: change status to \"active\" or remove the status field.\n *\n * @see https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/modes.md#agent-harness\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  // ── Mode ──────────────────────────────────────────────────────\n  // \"agent-harness\" runs a real coding agent (Claude Agent SDK) with\n  // filesystem tools in an isolated sandbox. The agent can read, write,\n  // edit files and run shell commands — then assertions verify the results.\n  //\n  // Other modes: \"literacy\" (text Q&A), \"mcp-server\" (tool use),\n  //              \"knowledge-probe\" (baseline knowledge).\n  mode: \"agent-harness\",\n\n  id: \"example-agent-add-schema\",\n  title: \"Add a document schema to a Sanity Studio project\",\n  description:\n    \"Example — tests whether an agent can create a schema file and \" +\n    \"register it in an existing Sanity Studio project\",\n\n  // Area groups tasks for scoring. All agent-harness tasks in the same\n  // area are aggregated together in the score report.\n  area: \"studio\",\n\n  // ── Sandbox ───────────────────────────────────────────────────\n  // The sandbox isolates the agent's file operations from your real repo.\n  // Currently only \"tempdir\" is implemented (creates a temporary directory).\n  // The pipeline creates the sandbox dir at config-generation time, copies\n  // fixtures into it before the agent starts, and the agent's working_dir\n  // is set to the sandbox path.\n  //\n  // See: docs/modes.md#sandbox-lifecycle\n  sandbox: { type: \"tempdir\" },\n\n  // ── Tools ─────────────────────────────────────────────────────\n  // Tool presets control which Claude Agent SDK tools the agent can use:\n  //   \"coding\"      → Bash, Read, Write, Edit, Glob, Grep\n  //   \"full-access\" → coding + WebSearch, WebFetch, TodoRead, TodoWrite\n  //   \"read-only\"   → Read, Glob, Grep, WebSearch\n  //\n  // You can also mix presets with explicit tool names:\n  //   tools: [\"coding\", \"WebFetch\"]\n  tools: [\"coding\"],\n\n  // ── Fixtures ──────────────────────────────────────────────────\n  // Fixtures are files or directories copied into the sandbox before the\n  // agent starts. Paths use the file:// prefix and resolve relative to\n  // the directory where you run the pipeline (typically your repo root).\n  //\n  // The agent sees these files in its working directory as if they were\n  // a real project. For example, \"file://apps/studio-basic\" copies the\n  // entire studio-basic app into the sandbox root.\n  //\n  // If you don't have fixture projects, you can omit this field and the\n  // agent starts with an empty directory.\n  fixtures: [\"file://apps/studio-basic\"],\n\n  // ── Prompt ────────────────────────────────────────────────────\n  // The task instructions sent to the agent. Be specific about expected\n  // file paths and patterns — the assertions check for exact paths.\n  prompt: {\n    text: `You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema.`,\n  },\n\n  // ── Assertions ────────────────────────────────────────────────\n  // Agent-harness assertions verify the sandbox state after the agent runs.\n  // They execute in a full Node.js context (not eval()) so they can use\n  // fs, child_process, etc. All file paths resolve relative to the sandbox.\n  //\n  // Available assertion types:\n  //   file-exists      — check a file was created\n  //   file-contains    — check a file contains a substring\n  //   command-succeeds — run a shell command (exit 0 = pass)\n  //   diff-matches     — check git diff contains a pattern\n  //   llm-rubric       — LLM grades the agent's text output\n  assertions: [\n    // Verify the agent created the new schema file\n    { type: \"file-exists\", value: \"schemas/post.ts\" },\n\n    // Verify it uses the modern Sanity schema API\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineType\" },\n    },\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineField\" },\n    },\n\n    // Verify the schema was registered in the barrel export\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/index.ts\", content: \"post\" },\n    },\n\n    // Verify the existing config is still intact\n    {\n      type: \"file-contains\",\n      value: { path: \"sanity.config.ts\", content: \"defineConfig\" },\n    },\n  ],\n\n  status: \"draft\",\n})\n";
+/** Generated YAML for example-agent-add-schema (from parsed TS data) */
+export const exampleAgentAddSchemaYaml = "- mode: agent-harness\n  id: example-agent-add-schema\n  title: Add a document schema to a Sanity Studio project\n  description: Example — tests whether an agent can create a schema file and register it in an existing Sanity Studio project\n  area: studio\n  sandbox:\n    type: tempdir\n  tools:\n    - coding\n  fixtures:\n    - file://apps/studio-basic\n  prompt:\n    text: |-\n      You have a Sanity Studio project with an existing article schema.\n      Add a new \"post\" document type with the following fields:\n      1. title (string, required)\n      2. slug (slug, sourced from title, required)\n      3. author (string)\n      4. publishedAt (datetime)\n      5. body (array of block content)\n\n      Create the schema file at schemas/post.ts using defineType() and defineField().\n      Register it in schemas/index.ts alongside the existing article schema.\n  assertions:\n    - type: file-exists\n      value: schemas/post.ts\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineType\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineField\n    - type: file-contains\n      value:\n        path: schemas/index.ts\n        content: post\n    - type: file-contains\n      value:\n        path: sanity.config.ts\n        content: defineConfig\n  status: draft\n";
 /** Parsed task data for example-groq-blog-listing (JSON-safe) */
 export const exampleGroqBlogListingData = [
     {
@@ -489,6 +550,7 @@ export const exampleStudioCustomInputYaml = "- mode: literacy\n  id: example-stu
 // ---------------------------------------------------------------------------
 /** All task example data as a flat array (JSON-safe) */
 export const allTaskData = [
+    ...exampleAgentAddSchemaData,
     ...exampleGroqBlogListingData,
     ...exampleIdBasedRefData,
     ...exampleKnowledgeProbeData,
@@ -499,6 +561,7 @@ export const allTaskData = [
 ];
 /** Map of task ID (filename stem) → raw TypeScript source */
 export const taskTsFiles = {
+    "example-agent-add-schema": exampleAgentAddSchemaTs,
     "example-groq-blog-listing": exampleGroqBlogListingTs,
     "example-id-based-ref": exampleIdBasedRefTs,
     "example-knowledge-probe": exampleKnowledgeProbeTs,
@@ -509,6 +572,7 @@ export const taskTsFiles = {
 };
 /** Map of task ID (filename stem) → generated YAML string */
 export const taskYamlFiles = {
+    "example-agent-add-schema": exampleAgentAddSchemaYaml,
     "example-groq-blog-listing": exampleGroqBlogListingYaml,
     "example-id-based-ref": exampleIdBasedRefYaml,
     "example-knowledge-probe": exampleKnowledgeProbeYaml,
@@ -518,8 +582,9 @@ export const taskYamlFiles = {
     "example-studio-custom-input": exampleStudioCustomInputYaml,
 };
 /** List of task file stems, in alphabetical order */
-export const TASK_FILE_NAMES = ["example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
+export const TASK_FILE_NAMES = ["example-agent-add-schema", "example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
 export const TASK_EXAMPLES = [
+    { stem: "example-agent-add-schema", mode: "agent-harness", status: "draft" },
     { stem: "example-groq-blog-listing", mode: "literacy", status: "draft" },
     { stem: "example-id-based-ref", mode: "literacy", status: "draft" },
     { stem: "example-knowledge-probe", mode: "knowledge-probe", status: "draft" },

package/dist/agent-harness/assertions-runtime.d.ts

@@ -0,0 +1,49 @@
+/**
+ * assertions-runtime.ts — Runtime assertion functions for agent harness tasks.
+ *
+ * These run as file-based promptfoo JavaScript assertions, which execute
+ * in a full Node.js context (unlike inline `type: javascript` assertions
+ * which run in a restricted eval() sandbox without require()).
+ *
+ * Each export is a named function with the signature:
+ *   (output: string, context: { vars, config, ... }) => GradingResult
+ *
+ * Referenced in promptfoo config as:
+ *   value: file://dist/agent-harness/assertions-runtime.js:functionName
+ *   config: { filePath: "...", ... }
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ */
+interface GradingResult {
+    pass: boolean;
+    score: number;
+    reason: string;
+}
+interface AssertionContext {
+    vars: Record<string, unknown>;
+    config?: Record<string, unknown>;
+}
+/**
+ * Assert that a file exists in the sandbox working directory.
+ * Config: { filePath: string }
+ */
+export declare function fileExists(_output: string, context: AssertionContext): GradingResult;
+/**
+ * Assert that a file contains expected content.
+ * Config: { filePath: string, content: string }
+ */
+export declare function fileContains(_output: string, context: AssertionContext): GradingResult;
+/**
+ * Assert that a shell command succeeds (exit code 0) in the sandbox.
+ * Config: { command: string, timeoutMs?: number }
+ *
+ * SECURITY: Commands come from developer-authored task definitions,
+ * not from user input or LLM output.
+ */
+export declare function commandSucceeds(_output: string, context: AssertionContext): GradingResult;
+/**
+ * Assert that the git diff in the sandbox contains expected content.
+ * Config: { expected?: string }
+ */
+export declare function diffMatches(_output: string, context: AssertionContext): GradingResult;
+export {};

package/dist/agent-harness/assertions-runtime.js

@@ -0,0 +1,138 @@
+/**
+ * assertions-runtime.ts — Runtime assertion functions for agent harness tasks.
+ *
+ * These run as file-based promptfoo JavaScript assertions, which execute
+ * in a full Node.js context (unlike inline `type: javascript` assertions
+ * which run in a restricted eval() sandbox without require()).
+ *
+ * Each export is a named function with the signature:
+ *   (output: string, context: { vars, config, ... }) => GradingResult
+ *
+ * Referenced in promptfoo config as:
+ *   value: file://dist/agent-harness/assertions-runtime.js:functionName
+ *   config: { filePath: "...", ... }
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ */
+import { existsSync, readFileSync } from "fs";
+import { execSync } from "child_process";
+import { resolve, sep } from "path";
+function resolveWorkDir(context) {
+    return resolve(context.vars.__workingDir || ".");
+}
+function guardTraversal(workDir, target, label) {
+    if (!target.startsWith(workDir + sep) && target !== workDir) {
+        return {
+            pass: false,
+            score: 0,
+            reason: `Path traversal: ${label} escapes sandbox`,
+        };
+    }
+    return null;
+}
+/**
+ * Assert that a file exists in the sandbox working directory.
+ * Config: { filePath: string }
+ */
+export function fileExists(_output, context) {
+    const filePath = context.config?.filePath ?? "";
+    const workDir = resolveWorkDir(context);
+    const target = resolve(workDir, filePath);
+    const traversal = guardTraversal(workDir, target, filePath);
+    if (traversal)
+        return traversal;
+    const exists = existsSync(target);
+    return {
+        pass: exists,
+        score: exists ? 1 : 0,
+        reason: exists
+            ? `File exists: ${filePath}`
+            : `Expected file not found: ${filePath}`,
+    };
+}
+/**
+ * Assert that a file contains expected content.
+ * Config: { filePath: string, content: string }
+ */
+export function fileContains(_output, context) {
+    const filePath = context.config?.filePath ?? "";
+    const expected = context.config?.content ?? "";
+    const workDir = resolveWorkDir(context);
+    const target = resolve(workDir, filePath);
+    const traversal = guardTraversal(workDir, target, filePath);
+    if (traversal)
+        return traversal;
+    if (!existsSync(target)) {
+        return { pass: false, score: 0, reason: `File not found: ${filePath}` };
+    }
+    const content = readFileSync(target, "utf-8");
+    const contains = content.includes(expected);
+    return {
+        pass: contains,
+        score: contains ? 1 : 0,
+        reason: contains
+            ? "File contains expected content"
+            : "File does not contain expected content",
+    };
+}
+/**
+ * Assert that a shell command succeeds (exit code 0) in the sandbox.
+ * Config: { command: string, timeoutMs?: number }
+ *
+ * SECURITY: Commands come from developer-authored task definitions,
+ * not from user input or LLM output.
+ */
+export function commandSucceeds(_output, context) {
+    const command = context.config?.command ?? "";
+    const timeoutMs = context.config?.timeoutMs ?? 30000;
+    const workDir = context.vars.__workingDir || ".";
+    try {
+        execSync(command, { cwd: workDir, timeout: timeoutMs });
+        return { pass: true, score: 1, reason: `Command succeeded: ${command}` };
+    }
+    catch (err) {
+        const error = err;
+        return {
+            pass: false,
+            score: 0,
+            reason: `Command failed: ${error.message || String(err)}`,
+        };
+    }
+}
+/**
+ * Assert that the git diff in the sandbox contains expected content.
+ * Config: { expected?: string }
+ */
+export function diffMatches(_output, context) {
+    const expected = context.config?.expected;
+    const workDir = context.vars.__workingDir || ".";
+    try {
+        const diff = execSync("git diff", {
+            cwd: workDir,
+            encoding: "utf-8",
+        });
+        if (typeof expected === "string") {
+            const contains = diff.includes(expected);
+            return {
+                pass: contains,
+                score: contains ? 1 : 0,
+                reason: contains
+                    ? "Diff matches expected pattern"
+                    : "Diff does not match",
+            };
+        }
+        return {
+            pass: diff.length > 0,
+            score: diff.length > 0 ? 1 : 0,
+            reason: "Diff exists",
+        };
+    }
+    catch (err) {
+        const error = err;
+        return {
+            pass: false,
+            score: 0,
+            reason: `Failed to get diff: ${error.message}`,
+        };
+    }
+}

package/dist/agent-harness/provider.d.ts

@@ -0,0 +1,58 @@
+/**
+ * provider.ts — Agent harness Promptfoo custom provider.
+ *
+ * Handles agent-harness evaluation tasks by calling the Anthropic API
+ * with the task prompt and returning the response. Sandbox provisioning
+ * and teardown are handled by lifecycle extensions (beforeEach/afterEach),
+ * not by the provider itself.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/agent-harness/provider.js
+ *       label: "Agent Harness: My Task"
+ *       config:
+ *         allowedTools: ["Bash", "Read", "Write"]
+ *         sandbox: { type: "tempdir" }
+ *
+ * Promptfoo loads this file and instantiates the default export class.
+ */
+interface CallApiContextParams {
+    prompt?: {
+        raw: string;
+        label?: string;
+    };
+    vars?: Record<string, object | string>;
+}
+interface ProviderOptions {
+    config?: Record<string, unknown>;
+    id?: string;
+}
+interface ProviderResponse {
+    cached?: boolean;
+    cost?: number;
+    error?: string;
+    metadata?: Record<string, unknown>;
+    output?: object | string;
+    tokenUsage?: {
+        total?: number;
+        prompt?: number;
+        completion?: number;
+        cached?: number;
+    };
+}
+export default class AgentHarnessProvider {
+    config: Record<string, unknown>;
+    private providerId;
+    constructor(options: ProviderOptions);
+    id(): string;
+    /**
+     * Main Promptfoo provider entry point. Called for each test case.
+     *
+     * Sends the task prompt to the Anthropic API and returns the response.
+     * The sandbox working directory is available in context.vars.__workingDir
+     * (set by the beforeEach extension).
+     */
+    callApi(prompt: string, context?: CallApiContextParams): Promise<ProviderResponse>;
+}
+export {};

package/dist/agent-harness/provider.js

@@ -0,0 +1,104 @@
+/**
+ * provider.ts — Agent harness Promptfoo custom provider.
+ *
+ * Handles agent-harness evaluation tasks by calling the Anthropic API
+ * with the task prompt and returning the response. Sandbox provisioning
+ * and teardown are handled by lifecycle extensions (beforeEach/afterEach),
+ * not by the provider itself.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/agent-harness/provider.js
+ *       label: "Agent Harness: My Task"
+ *       config:
+ *         allowedTools: ["Bash", "Read", "Write"]
+ *         sandbox: { type: "tempdir" }
+ *
+ * Promptfoo loads this file and instantiates the default export class.
+ */
+import { config as loadDotenv } from "dotenv";
+loadDotenv({
+    override: true,
+    path: new URL("../../.env", import.meta.url).pathname,
+});
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+export default class AgentHarnessProvider {
+    config;
+    providerId;
+    constructor(options) {
+        this.providerId = options.id ?? "agent-harness";
+        this.config = options.config ?? {};
+    }
+    id() {
+        return this.providerId;
+    }
+    /**
+     * Main Promptfoo provider entry point. Called for each test case.
+     *
+     * Sends the task prompt to the Anthropic API and returns the response.
+     * The sandbox working directory is available in context.vars.__workingDir
+     * (set by the beforeEach extension).
+     */
+    async callApi(prompt, context) {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            return {
+                error: "ANTHROPIC_API_KEY not set. Required for agent-harness evaluation.",
+            };
+        }
+        const model = this.config.model ?? "claude-sonnet-4-20250514";
+        const maxTokens = this.config.maxTokens ?? 4096;
+        const workingDir = context?.vars?.__workingDir;
+        const systemPrompt = workingDir
+            ? `You are an AI coding agent. Your working directory is: ${workingDir}\nComplete the following task.`
+            : "You are an AI coding agent. Complete the following task.";
+        try {
+            const response = await fetch("https://api.anthropic.com/v1/messages", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-api-key": apiKey,
+                    "anthropic-version": "2023-06-01",
+                },
+                body: JSON.stringify({
+                    model,
+                    max_tokens: maxTokens,
+                    system: systemPrompt,
+                    messages: [{ role: "user", content: prompt }],
+                }),
+            });
+            if (!response.ok) {
+                const errorBody = await response.text();
+                return {
+                    error: `Anthropic API error (${response.status}): ${errorBody}`,
+                };
+            }
+            const data = (await response.json());
+            const output = data.content
+                .filter((block) => block.type === "text")
+                .map((block) => block.text)
+                .join("\n");
+            return {
+                output,
+                tokenUsage: {
+                    prompt: data.usage?.input_tokens,
+                    completion: data.usage?.output_tokens,
+                    total: (data.usage?.input_tokens ?? 0) + (data.usage?.output_tokens ?? 0),
+                },
+                metadata: {
+                    model,
+                    workingDir,
+                    allowedTools: this.config.allowedTools,
+                    sandboxType: this.config.sandbox?.type ?? "none",
+                },
+            };
+        }
+        catch (err) {
+            const error = err;
+            return { error: `Agent harness provider error: ${error.message}` };
+        }
+    }
+}

package/dist/commands/init.js

@@ -138,6 +138,9 @@ async function runInit(opts) {
     else if (modeFilter === "knowledge-probe") {
         stemsToWrite = taskStemsForMode("knowledge-probe");
     }
+    else if (modeFilter === "agent-harness") {
+        stemsToWrite = taskStemsForMode("agent-harness");
+    }
     else {
         // Default (no --mode): write all tasks
         stemsToWrite = [...TASK_FILE_NAMES];

package/dist/orchestration/steps/generate-configs-step.d.ts

@@ -11,12 +11,19 @@
 import type { AppContext, PipelineState, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class GenerateConfigsStep implements PipelineStep {
     readonly name = "generate-configs";
+    /** Task IDs from the last loadTasks call (pre-filter), for error messages. */
+    private lastLoadedTaskIds;
     check(ctx: AppContext): ValidationIssue[];
     execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
     private compileLiteracyVariants;
     private compileSingleMode;
     private loadTasks;
     private applyFilters;
+    /**
+     * Build a descriptive error message when no tasks match the current filters.
+     * Distinguishes between "no tasks exist" and "tasks exist but filters exclude them".
+     */
+    private buildNoTasksError;
     /**
      * Compile all tasks through a handler, merging results.
      * For literacy mode, ctx can carry evalMode as an extension.

package/dist/orchestration/steps/generate-configs-step.js

@@ -20,6 +20,8 @@ import { loadSource } from "../../sources.js";
 import { configToSourceOverrides } from "../config-to-source-overrides.js";
 export class GenerateConfigsStep {
     name = "generate-configs";
+    /** Task IDs from the last loadTasks call (pre-filter), for error messages. */
+    lastLoadedTaskIds = [];
     check(ctx) {
         const issues = validateModelsYaml(ctx.config.rootDir);
         return issues.filter((i) => i.severity === "error");
@@ -54,10 +56,10 @@ export class GenerateConfigsStep {
             // Load tasks
             const tasks = await this.loadTasks(ctx, mode, state);
             if (tasks.length === 0) {
+                const error = this.buildNoTasksError(ctx, mode);
                 return {
                     durationMs: Date.now() - start,
-                    error: `No ${mode} tasks found. Create *.task.ts files in ` +
-                        `packages/eval/tasks/${mode}/`,
+                    error,
                     status: "failed",
                 };
             }
@@ -249,6 +251,10 @@ export class GenerateConfigsStep {
         return filtered;
     }
     applyFilters(ctx, tasks) {
+        // Capture pre-filter IDs for diagnostic messages
+        this.lastLoadedTaskIds = tasks
+            .map((t) => t.id)
+            .filter((id) => !!id);
         let result = tasks;
         if (ctx.config.areas?.length) {
             const allowed = new Set(ctx.config.areas.map((a) => a.toLowerCase()));
@@ -273,6 +279,33 @@ export class GenerateConfigsStep {
         }
         return result;
     }
+    /**
+     * Build a descriptive error message when no tasks match the current filters.
+     * Distinguishes between "no tasks exist" and "tasks exist but filters exclude them".
+     */
+    buildNoTasksError(ctx, mode) {
+        const filters = [];
+        if (ctx.config.tasks?.length) {
+            filters.push(`--task ${ctx.config.tasks.join(", ")}`);
+        }
+        if (ctx.config.areas?.length) {
+            filters.push(`--area ${ctx.config.areas.join(", ")}`);
+        }
+        if (ctx.config.tags?.length) {
+            filters.push(`--tag ${ctx.config.tags.join(", ")}`);
+        }
+        if (filters.length > 0) {
+            // Collect available task IDs for the hint
+            const availableIds = this.lastLoadedTaskIds ?? [];
+            const hint = availableIds.length > 0
+                ? `\n  Available ${mode} task IDs: ${availableIds.join(", ")}`
+                : "";
+            return (`No ${mode} tasks match the current filters (${filters.join("; ")}).` +
+                hint);
+        }
+        return (`No ${mode} tasks found. Create *.task.ts files in ` +
+            `packages/eval/tasks/${mode}/`);
+    }
     // ---------------------------------------------------------------------------
     // Compilation helpers
     // ---------------------------------------------------------------------------

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js

@@ -87,15 +87,23 @@ describe("validateAgentHarnessTask", () => {
 // compileAgentHarnessTask — provider assembly
 // ---------------------------------------------------------------------------
 describe("compileAgentHarnessTask — providers", () => {
-    it("produces a provider", () => {
+    it("produces a Claude Agent SDK provider", () => {
         const result = compileAgentHarnessTask(makeTask());
         assert.ok(result.providers.length > 0);
-        assert.ok(result.providers[0].id.startsWith("agent:"));
+        assert.equal(result.providers[0].id, "anthropic:claude-agent-sdk");
     });
-    it("resolves coding tool preset", () => {
+    it("sets default agent config", () => {
+        const result = compileAgentHarnessTask(makeTask());
+        const config = result.providers[0].config;
+        assert.ok(config.model, "should set a model");
+        assert.ok(config.max_turns, "should set max_turns");
+        assert.ok(config.max_budget_usd, "should set budget cap");
+        assert.equal(config.permission_mode, "bypassPermissions");
+    });
+    it("resolves coding tool preset into custom_allowed_tools", () => {
         const result = compileAgentHarnessTask(makeTask({ tools: ["coding"] }));
         const config = result.providers[0].config;
-        const tools = config.allowedTools;
+        const tools = config.custom_allowed_tools;
         assert.ok(tools.includes("Bash"));
         assert.ok(tools.includes("Read"));
         assert.ok(tools.includes("Write"));
@@ -104,7 +112,7 @@ describe("compileAgentHarnessTask — providers", () => {
     it("resolves read-only tool preset", () => {
         const result = compileAgentHarnessTask(makeTask({ tools: ["read-only"] }));
         const config = result.providers[0].config;
-        const tools = config.allowedTools;
+        const tools = config.custom_allowed_tools;
         assert.ok(tools.includes("Read"));
         assert.ok(tools.includes("Grep"));
         assert.ok(!tools.includes("Write"), "read-only should not include Write");
@@ -112,19 +120,10 @@ describe("compileAgentHarnessTask — providers", () => {
     it("mixes preset and explicit tools", () => {
         const result = compileAgentHarnessTask(makeTask({ tools: ["read-only", "WebFetch"] }));
         const config = result.providers[0].config;
-        const tools = config.allowedTools;
+        const tools = config.custom_allowed_tools;
         assert.ok(tools.includes("Read"));
         assert.ok(tools.includes("WebFetch"));
     });
-    it("includes sandbox config in provider", () => {
-        const result = compileAgentHarnessTask(makeTask({
-            sandbox: { type: "docker", image: "node:22-slim" },
-        }));
-        const config = result.providers[0].config;
-        const sandbox = config.sandbox;
-        assert.equal(sandbox.type, "docker");
-        assert.equal(sandbox.image, "node:22-slim");
-    });
 });
 // ---------------------------------------------------------------------------
 // compileAgentHarnessTask — test cases
@@ -166,16 +165,20 @@ describe("compileAgentHarnessTask — test cases", () => {
 // compileAgentHarnessTask — assertions
 // ---------------------------------------------------------------------------
 describe("compileAgentHarnessTask — assertions", () => {
-    it("maps file-exists to javascript assertion", () => {
+    const RUNTIME = "file://dist/agent-harness/assertions-runtime.js";
+    it("maps file-exists to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [{ type: "file-exists", value: "sanity.config.ts" }],
         }));
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("sanity.config.ts"));
+        assert.equal(assertion.value, `${RUNTIME}:fileExists`);
+        assert.deepEqual(assertion.config, {
+            filePath: "sanity.config.ts",
+        });
     });
-    it("maps file-contains to javascript assertion", () => {
+    it("maps file-contains to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [
                 {
@@ -187,25 +190,35 @@ describe("compileAgentHarnessTask — assertions", () => {
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("projectId"));
+        assert.equal(assertion.value, `${RUNTIME}:fileContains`);
+        assert.deepEqual(assertion.config, {
+            filePath: "config.ts",
+            content: "projectId",
+        });
     });
-    it("maps command-succeeds to javascript assertion", () => {
+    it("maps command-succeeds to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [{ type: "command-succeeds", value: "npx tsc --noEmit" }],
         }));
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("tsc"));
+        assert.equal(assertion.value, `${RUNTIME}:commandSucceeds`);
+        assert.deepEqual(assertion.config, {
+            command: "npx tsc --noEmit",
+        });
     });
-    it("maps diff-matches to javascript assertion", () => {
+    it("maps diff-matches to file-based javascript assertion", () => {
         const result = compileAgentHarnessTask(makeTask({
             assertions: [{ type: "diff-matches", value: "createClient" }],
         }));
         const assertion = result.tests[0].assert?.[0];
         assert.ok(assertion);
         assert.equal(assertion.type, "javascript");
-        assert.ok(assertion.value.includes("git diff"));
+        assert.equal(assertion.value, `${RUNTIME}:diffMatches`);
+        assert.deepEqual(assertion.config, {
+            expected: "createClient",
+        });
     });
     it("passes through standard assertions", () => {
         const result = compileAgentHarnessTask(makeTask({
@@ -250,7 +263,7 @@ describe("compileAgentHarnessTask — lifecycle", () => {
         }));
         assert.equal(result.sandboxConfig.type, "docker");
         assert.equal(result.sandboxConfig.image, "node:22");
-        assert.deepEqual(result.sandboxConfig.fixtures, ["file://schema.ts"]);
+        assert.deepEqual(result.sandboxConfig.fixtures, ["schema.ts"]);
         assert.equal(result.sandboxConfig.limits?.cpus, 2);
         assert.equal(result.sandboxConfig.limits?.networkAccess, false);
     });
@@ -278,7 +291,8 @@ describe("example agent harness tasks — end-to-end", () => {
         const result = compileAgentHarnessTask(modifyCodeTask);
         assert.ok(result.tests[0].assert);
         assert.ok(result.tests[0].assert.some((a) => a.type === "javascript" &&
-            a.value.includes("useDocumentOperation")));
+            a.value.includes("fileContains") &&
+            a.config != null));
     });
     it("refactor task has docker sandbox config", () => {
         const result = compileAgentHarnessTask(multiFileRefactorTask);

package/dist/pipeline/compiler/compiler-to-yaml.js

@@ -65,12 +65,38 @@ export function writeCompiledModeConfig(result, mode, options) {
     if (options.graderProvider) {
         graderOpts.provider = options.graderProvider;
     }
-    // Build provider entries
+    // For agent-harness mode, create sandbox directories and inject working_dir
+    // into provider configs. The sandbox must exist before the provider initializes
+    // (the Claude Agent SDK reads working_dir at construction time).
+    // Both working_dir and __workingDir use absolute paths to avoid ambiguity.
+    // @see https://www.promptfoo.dev/docs/providers/claude-agent-sdk/
+    const sandboxAbsPath = result.extras?.sandboxConfig
+        ? resolve(options.rootDir, `results/latest/sandbox-${mode}`)
+        : undefined;
+    if (sandboxAbsPath) {
+        mkdirSync(sandboxAbsPath, { recursive: true });
+    }
+    // Build provider entries, injecting working_dir for agent-harness providers
     const providerEntries = result.providers.map((p) => {
-        if (p.config)
-            return { id: p.id, label: p.label, config: p.config };
-        return p.label ? { id: p.id, label: p.label } : p.id;
+        if (!p.config)
+            return p.label ? { id: p.id, label: p.label } : p.id;
+        const config = { ...p.config };
+        if (sandboxAbsPath && p.id === "anthropic:claude-agent-sdk") {
+            config.working_dir = sandboxAbsPath;
+        }
+        return { id: p.id, label: p.label, config };
     });
+    // Inject __workingDir into test vars so assertions can find the sandbox
+    if (sandboxAbsPath) {
+        for (const test of expandedTests) {
+            if (test.vars) {
+                ;
+                test.vars.__workingDir = sandboxAbsPath;
+            }
+        }
+        // Re-write the tests file with the injected paths
+        writeFileSync(testsPath, JSON.stringify(expandedTests, null, 2), "utf-8");
+    }
     // Build prompt entries
     const prompts = result.prompts.map((p) => ({
         id: p.id,
@@ -88,10 +114,11 @@ export function writeCompiledModeConfig(result, mode, options) {
         tests: [testsFilename],
     });
     // Include extensions if present (agent-harness mode)
+    // Promptfoo expects extensions as string[] (file paths to JS modules),
+    // so we materialize the { type, code } objects as a .cjs file on disk.
     if (result.extras?.extensions) {
-        ;
-        config.extensions =
-            result.extras.extensions;
+        const extPaths = writeExtensionFile(options.rootDir, mode, result.extras.extensions);
+        config.extensions = extPaths;
     }
     writeConfig(options.rootDir, filename, config, options.logger);
 }
@@ -215,3 +242,47 @@ function writeYaml(path, data, header) {
     });
     writeFileSync(path, `${header}\n${yamlStr}`, "utf-8");
 }
+/**
+ * Materialize Promptfoo lifecycle extensions as a .cjs file on disk.
+ *
+ * Promptfoo extensions use a single-function dispatch pattern:
+ *   module.exports = async function(hookName, context) { ... }
+ *
+ * Each extension entry in the YAML references:
+ *   file://path/to/file.cjs:exportedFunctionName
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
+ */
+function writeExtensionFile(rootDir, mode, extensions) {
+    // Build a dispatch map: hookName → handler code
+    const hookMap = {};
+    for (const ext of extensions) {
+        hookMap[ext.type] = ext.code;
+    }
+    // Generate the single dispatch function that promptfoo expects
+    const hookCases = Object.entries(hookMap)
+        .map(([hookName, code]) => `  if (hookName === '${hookName}') {\n` +
+        `    const handler = ${code};\n` +
+        `    return handler(context);\n` +
+        `  }`)
+        .join("\n");
+    const fileContent = [
+        "// AUTO-GENERATED by compiler pipeline — do not edit directly.",
+        "// Run: npx @sanity/ailf generate-configs",
+        "//",
+        "// Promptfoo extension dispatch function.",
+        `// @see https://www.promptfoo.dev/docs/configuration/reference/`,
+        "",
+        "async function extensionHook(hookName, context) {",
+        hookCases,
+        "}",
+        "",
+        "module.exports = extensionHook;",
+        "",
+    ].join("\n");
+    const filename = `results/latest/${mode}-extensions.cjs`;
+    const outPath = resolve(rootDir, filename);
+    writeFileSync(outPath, fileContent, "utf-8");
+    // Single entry pointing to the dispatch function
+    return [`file://${filename}:extensionHook`];
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts

@@ -4,6 +4,15 @@
  * Handles agent-specific assertion types (file-exists, file-contains,
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
+ *
+ * Agent-specific assertions use file-based references to the assertions
+ * runtime module (dist/agent-harness/assertions-runtime.js) because
+ * promptfoo's inline `type: javascript` assertions run in a restricted
+ * eval() sandbox where require() is unavailable. File-based assertions
+ * run in a full Node.js context.
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ * @see src/agent-harness/assertions-runtime.ts — runtime implementations
  */
 import type { PromptfooAssertion } from "../../assertion-mapper.js";
 import type { AgentHarnessCompileOptions } from "./types.js";

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js

@@ -4,7 +4,18 @@
  * Handles agent-specific assertion types (file-exists, file-contains,
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
+ *
+ * Agent-specific assertions use file-based references to the assertions
+ * runtime module (dist/agent-harness/assertions-runtime.js) because
+ * promptfoo's inline `type: javascript` assertions run in a restricted
+ * eval() sandbox where require() is unavailable. File-based assertions
+ * run in a full Node.js context.
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ * @see src/agent-harness/assertions-runtime.ts — runtime implementations
  */
+/** Base path for the file-based assertion runtime module */
+const RUNTIME = "file://dist/agent-harness/assertions-runtime.js";
 // ---------------------------------------------------------------------------
 // Assertion mapping
 // ---------------------------------------------------------------------------
@@ -53,66 +64,29 @@ export function mapAgentAssertion(assertion, options, warnings) {
 }
 // ---------------------------------------------------------------------------
 // Agent-specific assertion builders
+//
+// Each builder returns a file-based assertion referencing the runtime
+// module with parameters passed via the `config` field.
 // ---------------------------------------------------------------------------
 export function buildFileExistsAssertion(assertion) {
-    const filePath = String(assertion.value ?? "");
-    // Use JSON.stringify for all interpolated values in generated JS to
-    // prevent broken strings from filePaths containing quotes/backslashes
-    const safeFilePath = JSON.stringify(filePath);
     return {
         type: "javascript",
-        value: `// file-exists: ${filePath}\n` +
-            `(function() {\n` +
-            `  const fs = require('fs');\n` +
-            `  const path = require('path');\n` +
-            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
-            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
-            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
-            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
-            `  }\n` +
-            `  const exists = fs.existsSync(target);\n` +
-            `  return {\n` +
-            `    pass: exists,\n` +
-            `    score: exists ? 1 : 0,\n` +
-            `    reason: exists\n` +
-            `      ? 'File exists: ' + ${safeFilePath}\n` +
-            `      : 'Expected file not found: ' + ${safeFilePath},\n` +
-            `  };\n` +
-            `})()`,
+        value: `${RUNTIME}:fileExists`,
+        config: { filePath: String(assertion.value ?? "") },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),
     };
 }
 export function buildFileContainsAssertion(assertion) {
-    const config = assertion.value;
-    const filePath = config?.path ?? "";
-    const expectedContent = config?.content ?? "";
-    const safeFilePath = JSON.stringify(filePath);
+    const val = assertion.value;
     return {
         type: "javascript",
-        value: `// file-contains: ${filePath}\n` +
-            `(function() {\n` +
-            `  const fs = require('fs');\n` +
-            `  const path = require('path');\n` +
-            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
-            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
-            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
-            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
-            `  }\n` +
-            `  if (!fs.existsSync(target)) {\n` +
-            `    return { pass: false, score: 0, reason: 'File not found: ' + ${safeFilePath} };\n` +
-            `  }\n` +
-            `  const content = fs.readFileSync(target, 'utf-8');\n` +
-            `  const contains = content.includes(${JSON.stringify(expectedContent)});\n` +
-            `  return {\n` +
-            `    pass: contains,\n` +
-            `    score: contains ? 1 : 0,\n` +
-            `    reason: contains\n` +
-            `      ? 'File contains expected content'\n` +
-            `      : 'File does not contain expected content',\n` +
-            `  };\n` +
-            `})()`,
+        value: `${RUNTIME}:fileContains`,
+        config: {
+            filePath: val?.path ?? "",
+            content: val?.content ?? "",
+        },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),
@@ -133,53 +107,22 @@ export function buildFileContainsAssertion(assertion) {
  * from untrusted sources, validate commands against an allowlist first.
  */
 export function buildCommandSucceedsAssertion(assertion) {
-    const command = String(assertion.value ?? "");
     return {
         type: "javascript",
-        value: `// command-succeeds: ${command}\n` +
-            `(function() {\n` +
-            `  const { execSync } = require('child_process');\n` +
-            `  const workDir = context.vars.__workingDir || '.';\n` +
-            `  try {\n` +
-            `    execSync(${JSON.stringify(command)}, { cwd: workDir, timeout: 30000 });\n` +
-            `    return { pass: true, score: 1, reason: 'Command succeeded: ' + ${JSON.stringify(command)} };\n` +
-            `  } catch (err) {\n` +
-            `    return {\n` +
-            `      pass: false,\n` +
-            `      score: 0,\n` +
-            `      reason: 'Command failed: ' + (err.message || err),\n` +
-            `    };\n` +
-            `  }\n` +
-            `})()`,
+        value: `${RUNTIME}:commandSucceeds`,
+        config: { command: String(assertion.value ?? "") },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),
     };
 }
 export function buildDiffMatchesAssertion(assertion) {
-    const expected = assertion.value;
     return {
         type: "javascript",
-        value: `// diff-matches\n` +
-            `(function() {\n` +
-            `  const { execSync } = require('child_process');\n` +
-            `  const workDir = context.vars.__workingDir || '.';\n` +
-            `  try {\n` +
-            `    const diff = execSync('git diff', { cwd: workDir, encoding: 'utf-8' });\n` +
-            `    const expected = ${JSON.stringify(expected)};\n` +
-            `    if (typeof expected === 'string') {\n` +
-            `      const contains = diff.includes(expected);\n` +
-            `      return {\n` +
-            `        pass: contains,\n` +
-            `        score: contains ? 1 : 0,\n` +
-            `        reason: contains ? 'Diff matches expected pattern' : 'Diff does not match',\n` +
-            `      };\n` +
-            `    }\n` +
-            `    return { pass: diff.length > 0, score: diff.length > 0 ? 1 : 0, reason: 'Diff exists' };\n` +
-            `  } catch (err) {\n` +
-            `    return { pass: false, score: 0, reason: 'Failed to get diff: ' + err.message };\n` +
-            `  }\n` +
-            `})()`,
+        value: `${RUNTIME}:diffMatches`,
+        config: {
+            ...(assertion.value != null ? { expected: assertion.value } : {}),
+        },
         ...(typeof assertion.weight === "number"
             ? { weight: assertion.weight }
             : {}),

package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.js

@@ -27,8 +27,10 @@ export function compileAgentHarnessTask(task, options) {
     const prompts = buildAgentPrompts(task);
     // Build test cases
     const tests = buildAgentTestCases(task, options, warnings);
-    // Build sandbox extensions
-    const sandboxConfig = buildSandboxConfig(task);
+    // Build sandbox extensions — resolve fixture paths at compile time using
+    // the caller's cwd (monorepo root), not the eval package rootDir.
+    const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
+    const sandboxConfig = buildSandboxConfig(task, callerCwd);
     const extensions = buildLifecycleExtensions(task, sandboxConfig);
     return { providers, tests, prompts, extensions, sandboxConfig, warnings };
 }
@@ -36,22 +38,25 @@ export function compileAgentHarnessTask(task, options) {
 // Provider assembly
 // ---------------------------------------------------------------------------
 export function buildAgentProvider(task, _warnings) {
-    // Resolve tool permissions
     const tools = resolveToolPermissions(task.tools);
-    const config = {};
+    // Claude Agent SDK config.
+    // working_dir is set by the YAML writer to the sandbox path it creates.
+    // @see https://www.promptfoo.dev/docs/providers/claude-agent-sdk/
+    const config = {
+        model: "claude-sonnet-4-20250514",
+        max_turns: 25,
+        max_budget_usd: 1.0,
+        permission_mode: "bypassPermissions",
+        allow_dangerously_skip_permissions: true,
+    };
+    // Map AILF tool names to Claude Agent SDK tool config.
+    // Claude SDK uses custom_allowed_tools to replace defaults.
     if (tools.length > 0) {
-        config.allowedTools = tools;
-    }
-    if (task.sandbox) {
-        config.sandbox = {
-            type: task.sandbox.type,
-            ...(task.sandbox.image ? { image: task.sandbox.image } : {}),
-        };
+        config.custom_allowed_tools = tools;
     }
-    // Default to Claude Agent SDK provider
     return [
         {
-            id: `agent:${task.id}`,
+            id: "anthropic:claude-agent-sdk",
             label: `Agent Harness: ${task.title}`,
             config,
         },
@@ -112,9 +117,11 @@ export function buildAgentTestCases(task, options, warnings) {
     const vars = {
         task: task.prompt?.vars?.task ?? task.description ?? `Complete: ${task.title}`,
         ...(task.prompt?.vars ?? {}),
-        // Internal metadata for sandbox lifecycle hooks
+        // Internal metadata for sandbox lifecycle hooks.
+        // Fixture paths are plain strings (no file:// prefix) because
+        // promptfoo auto-resolves file:// in vars by reading file content.
         __sandboxType: task.sandbox?.type ?? "tempdir",
-        __fixtures: task.fixtures ?? [],
+        __fixtures: (task.fixtures ?? []).map((f) => f.startsWith("file://") ? f.slice(7) : f),
     };
     const tests = [
         {

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.d.ts

@@ -6,7 +6,14 @@
  */
 import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
 import type { PromptfooExtension, SandboxConfigMeta } from "./types.js";
-export declare function buildSandboxConfig(task: AgentHarnessTaskDefinition): SandboxConfigMeta;
+/**
+ * Build sandbox configuration from a task definition.
+ *
+ * Fixture paths are resolved to absolute at compile time using callerCwd
+ * (the directory the pipeline was invoked from), because promptfoo runs
+ * with cwd set to packages/eval/ — not the monorepo root where apps/ lives.
+ */
+export declare function buildSandboxConfig(task: AgentHarnessTaskDefinition, callerCwd?: string): SandboxConfigMeta;
 export declare function buildLifecycleExtensions(task: AgentHarnessTaskDefinition, sandboxConfig: SandboxConfigMeta): PromptfooExtension[];
 export declare function buildBeforeEachHook(taskId: string, config: SandboxConfigMeta): string;
 export declare function buildAfterEachHook(taskId: string): string;

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.js

@@ -4,14 +4,26 @@
  * Builds Promptfoo beforeEach/afterEach hooks for provisioning and
  * tearing down sandbox working directories.
  */
+import { resolve } from "path";
 // ---------------------------------------------------------------------------
 // Sandbox configuration
 // ---------------------------------------------------------------------------
-export function buildSandboxConfig(task) {
+/**
+ * Build sandbox configuration from a task definition.
+ *
+ * Fixture paths are resolved to absolute at compile time using callerCwd
+ * (the directory the pipeline was invoked from), because promptfoo runs
+ * with cwd set to packages/eval/ — not the monorepo root where apps/ lives.
+ */
+export function buildSandboxConfig(task, callerCwd) {
+    const cwd = callerCwd ?? process.cwd();
     return {
         type: task.sandbox?.type ?? "tempdir",
         image: task.sandbox?.image,
-        fixtures: task.fixtures ?? [],
+        fixtures: (task.fixtures ?? []).map((f) => {
+            const stripped = f.startsWith("file://") ? f.slice(7) : f;
+            return resolve(cwd, stripped);
+        }),
         limits: task.sandbox?.limits
             ? {
                 cpus: task.sandbox.limits.cpus,
@@ -39,23 +51,41 @@ export function buildLifecycleExtensions(task, sandboxConfig) {
     return extensions;
 }
 export function buildBeforeEachHook(taskId, config) {
-    return (`// beforeEach: provision sandbox for ${taskId}\n` +
-        `async function({ vars }) {\n` +
-        `  const { mkdirSync, writeFileSync } = require('fs');\n` +
-        `  const { tmpdir } = require('os');\n` +
+    // Promptfoo extension hooks receive (hookName, context).
+    // beforeEach context is { test } — vars live at context.test.vars.
+    // Must return context for mutations to persist.
+    //
+    // The sandbox directory is created by the YAML writer at config-gen time
+    // (deterministic path in results/latest/sandbox-{taskId}/) so it exists
+    // before the provider is initialized. This hook copies fixtures into it.
+    //
+    // @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
+    return (`// beforeEach: copy fixtures into sandbox for ${taskId}\n` +
+        `async function(context) {\n` +
+        `  const { cpSync, existsSync, mkdirSync } = require('fs');\n` +
         `  const { resolve } = require('path');\n` +
-        `  const id = 'ailf-${taskId}-' + require('crypto').randomUUID().slice(0, 8);\n` +
-        `  const workDir = resolve(tmpdir(), id);\n` +
+        `  const workDir = context.test.vars?.__workingDir;\n` +
+        `  if (!workDir) return context;\n` +
         `  mkdirSync(workDir, { recursive: true });\n` +
-        `  vars.__workingDir = workDir;\n` +
-        `  vars.__sandboxId = id;\n` +
-        `  // Fixture list: ${JSON.stringify(config.fixtures)}\n` +
+        `  // Copy fixtures into sandbox\n` +
+        `  const fixtures = ${JSON.stringify(config.fixtures)};\n` +
+        `  for (const fixture of fixtures) {\n` +
+        `    const src = resolve(process.cwd(), fixture);\n` +
+        `    if (existsSync(src)) {\n` +
+        `      cpSync(src, workDir, { recursive: true });\n` +
+        `    }\n` +
+        `  }\n` +
+        `  return context;\n` +
         `}`);
 }
 export function buildAfterEachHook(taskId) {
+    // Promptfoo extension hooks receive (hookName, context).
+    // afterEach context is { test, result } — vars live at context.test.vars.
+    // @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
     return (`// afterEach: collect artifacts + teardown for ${taskId}\n` +
-        `async function({ vars }) {\n` +
+        `async function(context) {\n` +
         `  const { rmSync, readdirSync, existsSync } = require('fs');\n` +
+        `  const vars = context.test.vars || {};\n` +
         `  const workDir = vars.__workingDir;\n` +
         `  if (workDir && existsSync(workDir)) {\n` +
         `    try {\n` +

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -46,6 +46,7 @@
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.105",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.13.1",
     "tsx": "^4.19.2",