@sanity/ailf 2.0.2 → 2.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Sanity.io
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/_vendor/ailf-core/examples/index.d.ts

@@ -112,6 +112,55 @@ export declare const ailfConfigData: {
 };
 /** Raw YAML string for ailf-config example (preserves comments) */
 export declare const ailfConfigYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# .ailf/config.yaml \u2014 AI Literacy Framework project configuration\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This file configures how the AILF evaluation pipeline runs in this\n# repository. Place it at .ailf/config.yaml in your project root.\n#\n# Evaluations are submitted to the AILF API (ailf-api.sanity.build).\n# The API handles LLM calls, doc fetching, grading, and report\n# publishing. Your repo only needs one secret: AILF_API_KEY.\n#\n# Docs: https://github.com/sanity-labs/ai-literacy-framework\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n# Documentation source \u2014 which docs are being evaluated.\n#\n# This tells the pipeline which Sanity project and dataset contain\n# the documentation under test. For most users, this is Sanity's own\n# docs project.\n#\n# projectId \u2014 Sanity project ID (find yours at sanity.io/manage)\n# dataset   \u2014 the dataset to query (e.g., \"production\", \"next\")\n# baseUrl   \u2014 the public URL of your documentation site\n#             (used by agentic mode to test agent discoverability)\nsource:\n  projectId: \"3do82whm\"\n  dataset: next\n  baseUrl: \"https://www.sanity.io/docs\"\n\n# Trigger configuration \u2014 when evaluations run automatically.\n#\n# Each key is a trigger context. The pipeline checks which trigger\n# matches the current execution context (PR, merge, schedule, etc.)\n# and applies its settings.\n#\n# mode options:\n#   validate-only \u2014 check that task YAML parses correctly (fast, no LLM calls)\n#   eval          \u2014 run the full evaluation pipeline\n#\n# paths \u2014 only trigger when files matching these globs change\n# blocking \u2014 if true, a failing eval blocks the PR merge\n# notify \u2014 if true, post results to configured notification channels\ntriggers:\n  # On pull requests: just validate task files parse correctly\n  pr:\n    mode: validate-only\n\n  # When .ailf/ files change in a PR: run a real evaluation\n  pr-task-change:\n    mode: eval\n    paths: [\".ailf/**\"]\n\n  # On merge to main: run evaluation (non-blocking)\n  main:\n    mode: eval\n    blocking: false\n    notify: true\n";
+/** Parsed task data for example-agent-add-schema (JSON-safe) */
+export declare const exampleAgentAddSchemaData: readonly [{
+    readonly mode: "agent-harness";
+    readonly id: "example-agent-add-schema";
+    readonly title: "Add a document schema to a Sanity Studio project";
+    readonly description: "Example — tests whether an agent can create a schema file and register it in an existing Sanity Studio project";
+    readonly area: "studio";
+    readonly sandbox: {
+        readonly type: "tempdir";
+    };
+    readonly tools: readonly ["coding"];
+    readonly fixtures: readonly ["file://apps/studio-basic"];
+    readonly prompt: {
+        readonly text: "You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema.";
+    };
+    readonly assertions: readonly [{
+        readonly type: "file-exists";
+        readonly value: "schemas/post.ts";
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "schemas/post.ts";
+            readonly content: "defineType";
+        };
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "schemas/post.ts";
+            readonly content: "defineField";
+        };
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "schemas/index.ts";
+            readonly content: "post";
+        };
+    }, {
+        readonly type: "file-contains";
+        readonly value: {
+            readonly path: "sanity.config.ts";
+            readonly content: "defineConfig";
+        };
+    }];
+    readonly status: "draft";
+}];
+/** TypeScript task template for example-agent-add-schema */
+export declare const exampleAgentAddSchemaTs = "/**\n * Example Task: Agent harness \u2014 add a schema to a Sanity Studio project.\n *\n * This is a starter template for agent-harness evaluations. It tests whether\n * an autonomous coding agent can modify a real project in a sandboxed\n * environment using file system tools.\n *\n * Unlike literacy or knowledge-probe tasks that evaluate text responses,\n * agent-harness tasks evaluate *side-effects*: files created, code modified,\n * commands executed. The agent gets a working directory with real files and\n * tools (Bash, Read, Write, Edit, etc.) and is graded on what it produces.\n *\n * This task is a DRAFT \u2014 it won't run unless activated or explicitly targeted.\n * To activate: change status to \"active\" or remove the status field.\n *\n * @see https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/modes.md#agent-harness\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  // \u2500\u2500 Mode \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // \"agent-harness\" runs a real coding agent (Claude Agent SDK) with\n  // filesystem tools in an isolated sandbox. The agent can read, write,\n  // edit files and run shell commands \u2014 then assertions verify the results.\n  //\n  // Other modes: \"literacy\" (text Q&A), \"mcp-server\" (tool use),\n  //              \"knowledge-probe\" (baseline knowledge).\n  mode: \"agent-harness\",\n\n  id: \"example-agent-add-schema\",\n  title: \"Add a document schema to a Sanity Studio project\",\n  description:\n    \"Example \u2014 tests whether an agent can create a schema file and \" +\n    \"register it in an existing Sanity Studio project\",\n\n  // Area groups tasks for scoring. All agent-harness tasks in the same\n  // area are aggregated together in the score report.\n  area: \"studio\",\n\n  // \u2500\u2500 Sandbox \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // The sandbox isolates the agent's file operations from your real repo.\n  // Currently only \"tempdir\" is implemented (creates a temporary directory).\n  // The pipeline creates the sandbox dir at config-generation time, copies\n  // fixtures into it before the agent starts, and the agent's working_dir\n  // is set to the sandbox path.\n  //\n  // See: docs/modes.md#sandbox-lifecycle\n  sandbox: { type: \"tempdir\" },\n\n  // \u2500\u2500 Tools \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Tool presets control which Claude Agent SDK tools the agent can use:\n  //   \"coding\"      \u2192 Bash, Read, Write, Edit, Glob, Grep\n  //   \"full-access\" \u2192 coding + WebSearch, WebFetch, TodoRead, TodoWrite\n  //   \"read-only\"   \u2192 Read, Glob, Grep, WebSearch\n  //\n  // You can also mix presets with explicit tool names:\n  //   tools: [\"coding\", \"WebFetch\"]\n  tools: [\"coding\"],\n\n  // \u2500\u2500 Fixtures \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Fixtures are files or directories copied into the sandbox before the\n  // agent starts. Paths use the file:// prefix and resolve relative to\n  // the directory where you run the pipeline (typically your repo root).\n  //\n  // The agent sees these files in its working directory as if they were\n  // a real project. For example, \"file://apps/studio-basic\" copies the\n  // entire studio-basic app into the sandbox root.\n  //\n  // If you don't have fixture projects, you can omit this field and the\n  // agent starts with an empty directory.\n  fixtures: [\"file://apps/studio-basic\"],\n\n  // \u2500\u2500 Prompt \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // The task instructions sent to the agent. Be specific about expected\n  // file paths and patterns \u2014 the assertions check for exact paths.\n  prompt: {\n    text: `You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema.`,\n  },\n\n  // \u2500\u2500 Assertions \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Agent-harness assertions verify the sandbox state after the agent runs.\n  // They execute in a full Node.js context (not eval()) so they can use\n  // fs, child_process, etc. All file paths resolve relative to the sandbox.\n  //\n  // Available assertion types:\n  //   file-exists      \u2014 check a file was created\n  //   file-contains    \u2014 check a file contains a substring\n  //   command-succeeds \u2014 run a shell command (exit 0 = pass)\n  //   diff-matches     \u2014 check git diff contains a pattern\n  //   llm-rubric       \u2014 LLM grades the agent's text output\n  assertions: [\n    // Verify the agent created the new schema file\n    { type: \"file-exists\", value: \"schemas/post.ts\" },\n\n    // Verify it uses the modern Sanity schema API\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineType\" },\n    },\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineField\" },\n    },\n\n    // Verify the schema was registered in the barrel export\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/index.ts\", content: \"post\" },\n    },\n\n    // Verify the existing config is still intact\n    {\n      type: \"file-contains\",\n      value: { path: \"sanity.config.ts\", content: \"defineConfig\" },\n    },\n  ],\n\n  status: \"draft\",\n})\n";
+/** Generated YAML for example-agent-add-schema (from parsed TS data) */
+export declare const exampleAgentAddSchemaYaml = "- mode: agent-harness\n  id: example-agent-add-schema\n  title: Add a document schema to a Sanity Studio project\n  description: Example \u2014 tests whether an agent can create a schema file and register it in an existing Sanity Studio project\n  area: studio\n  sandbox:\n    type: tempdir\n  tools:\n    - coding\n  fixtures:\n    - file://apps/studio-basic\n  prompt:\n    text: |-\n      You have a Sanity Studio project with an existing article schema.\n      Add a new \"post\" document type with the following fields:\n      1. title (string, required)\n      2. slug (slug, sourced from title, required)\n      3. author (string)\n      4. publishedAt (datetime)\n      5. body (array of block content)\n\n      Create the schema file at schemas/post.ts using defineType() and defineField().\n      Register it in schemas/index.ts alongside the existing article schema.\n  assertions:\n    - type: file-exists\n      value: schemas/post.ts\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineType\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineField\n    - type: file-contains\n      value:\n        path: schemas/index.ts\n        content: post\n    - type: file-contains\n      value:\n        path: sanity.config.ts\n        content: defineConfig\n  status: draft\n";
 /** Parsed task data for example-groq-blog-listing (JSON-safe) */
 export declare const exampleGroqBlogListingData: readonly [{
     readonly mode: "literacy";
@@ -367,7 +416,7 @@ export declare const taskTsFiles: Record<string, string>;
 /** Map of task ID (filename stem) → generated YAML string */
 export declare const taskYamlFiles: Record<string, string>;
 /** List of task file stems, in alphabetical order */
-export declare const TASK_FILE_NAMES: readonly ["example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
+export declare const TASK_FILE_NAMES: readonly ["example-agent-add-schema", "example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
 /** Task metadata for mode-based filtering in init and other consumers */
 export interface TaskExampleMeta {
     stem: string;

package/dist/_vendor/ailf-core/examples/index.js

@@ -143,6 +143,67 @@ export const ailfConfigData = {
 };
 /** Raw YAML string for ailf-config example (preserves comments) */
 export const ailfConfigYaml = "# ──────────────────────────────────────────────────────────────────────\n# .ailf/config.yaml — AI Literacy Framework project configuration\n# ──────────────────────────────────────────────────────────────────────\n#\n# This file configures how the AILF evaluation pipeline runs in this\n# repository. Place it at .ailf/config.yaml in your project root.\n#\n# Evaluations are submitted to the AILF API (ailf-api.sanity.build).\n# The API handles LLM calls, doc fetching, grading, and report\n# publishing. Your repo only needs one secret: AILF_API_KEY.\n#\n# Docs: https://github.com/sanity-labs/ai-literacy-framework\n# ──────────────────────────────────────────────────────────────────────\n\n# Documentation source — which docs are being evaluated.\n#\n# This tells the pipeline which Sanity project and dataset contain\n# the documentation under test. For most users, this is Sanity's own\n# docs project.\n#\n# projectId — Sanity project ID (find yours at sanity.io/manage)\n# dataset   — the dataset to query (e.g., \"production\", \"next\")\n# baseUrl   — the public URL of your documentation site\n#             (used by agentic mode to test agent discoverability)\nsource:\n  projectId: \"3do82whm\"\n  dataset: next\n  baseUrl: \"https://www.sanity.io/docs\"\n\n# Trigger configuration — when evaluations run automatically.\n#\n# Each key is a trigger context. The pipeline checks which trigger\n# matches the current execution context (PR, merge, schedule, etc.)\n# and applies its settings.\n#\n# mode options:\n#   validate-only — check that task YAML parses correctly (fast, no LLM calls)\n#   eval          — run the full evaluation pipeline\n#\n# paths — only trigger when files matching these globs change\n# blocking — if true, a failing eval blocks the PR merge\n# notify — if true, post results to configured notification channels\ntriggers:\n  # On pull requests: just validate task files parse correctly\n  pr:\n    mode: validate-only\n\n  # When .ailf/ files change in a PR: run a real evaluation\n  pr-task-change:\n    mode: eval\n    paths: [\".ailf/**\"]\n\n  # On merge to main: run evaluation (non-blocking)\n  main:\n    mode: eval\n    blocking: false\n    notify: true\n";
+/** Parsed task data for example-agent-add-schema (JSON-safe) */
+export const exampleAgentAddSchemaData = [
+    {
+        "mode": "agent-harness",
+        "id": "example-agent-add-schema",
+        "title": "Add a document schema to a Sanity Studio project",
+        "description": "Example — tests whether an agent can create a schema file and register it in an existing Sanity Studio project",
+        "area": "studio",
+        "sandbox": {
+            "type": "tempdir"
+        },
+        "tools": [
+            "coding"
+        ],
+        "fixtures": [
+            "file://apps/studio-basic"
+        ],
+        "prompt": {
+            "text": "You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema."
+        },
+        "assertions": [
+            {
+                "type": "file-exists",
+                "value": "schemas/post.ts"
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "schemas/post.ts",
+                    "content": "defineType"
+                }
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "schemas/post.ts",
+                    "content": "defineField"
+                }
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "schemas/index.ts",
+                    "content": "post"
+                }
+            },
+            {
+                "type": "file-contains",
+                "value": {
+                    "path": "sanity.config.ts",
+                    "content": "defineConfig"
+                }
+            }
+        ],
+        "status": "draft"
+    }
+];
+/** TypeScript task template for example-agent-add-schema */
+export const exampleAgentAddSchemaTs = "/**\n * Example Task: Agent harness — add a schema to a Sanity Studio project.\n *\n * This is a starter template for agent-harness evaluations. It tests whether\n * an autonomous coding agent can modify a real project in a sandboxed\n * environment using file system tools.\n *\n * Unlike literacy or knowledge-probe tasks that evaluate text responses,\n * agent-harness tasks evaluate *side-effects*: files created, code modified,\n * commands executed. The agent gets a working directory with real files and\n * tools (Bash, Read, Write, Edit, etc.) and is graded on what it produces.\n *\n * This task is a DRAFT — it won't run unless activated or explicitly targeted.\n * To activate: change status to \"active\" or remove the status field.\n *\n * @see https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/modes.md#agent-harness\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  // ── Mode ──────────────────────────────────────────────────────\n  // \"agent-harness\" runs a real coding agent (Claude Agent SDK) with\n  // filesystem tools in an isolated sandbox. The agent can read, write,\n  // edit files and run shell commands — then assertions verify the results.\n  //\n  // Other modes: \"literacy\" (text Q&A), \"mcp-server\" (tool use),\n  //              \"knowledge-probe\" (baseline knowledge).\n  mode: \"agent-harness\",\n\n  id: \"example-agent-add-schema\",\n  title: \"Add a document schema to a Sanity Studio project\",\n  description:\n    \"Example — tests whether an agent can create a schema file and \" +\n    \"register it in an existing Sanity Studio project\",\n\n  // Area groups tasks for scoring. All agent-harness tasks in the same\n  // area are aggregated together in the score report.\n  area: \"studio\",\n\n  // ── Sandbox ───────────────────────────────────────────────────\n  // The sandbox isolates the agent's file operations from your real repo.\n  // Currently only \"tempdir\" is implemented (creates a temporary directory).\n  // The pipeline creates the sandbox dir at config-generation time, copies\n  // fixtures into it before the agent starts, and the agent's working_dir\n  // is set to the sandbox path.\n  //\n  // See: docs/modes.md#sandbox-lifecycle\n  sandbox: { type: \"tempdir\" },\n\n  // ── Tools ─────────────────────────────────────────────────────\n  // Tool presets control which Claude Agent SDK tools the agent can use:\n  //   \"coding\"      → Bash, Read, Write, Edit, Glob, Grep\n  //   \"full-access\" → coding + WebSearch, WebFetch, TodoRead, TodoWrite\n  //   \"read-only\"   → Read, Glob, Grep, WebSearch\n  //\n  // You can also mix presets with explicit tool names:\n  //   tools: [\"coding\", \"WebFetch\"]\n  tools: [\"coding\"],\n\n  // ── Fixtures ──────────────────────────────────────────────────\n  // Fixtures are files or directories copied into the sandbox before the\n  // agent starts. Paths use the file:// prefix and resolve relative to\n  // the directory where you run the pipeline (typically your repo root).\n  //\n  // The agent sees these files in its working directory as if they were\n  // a real project. For example, \"file://apps/studio-basic\" copies the\n  // entire studio-basic app into the sandbox root.\n  //\n  // If you don't have fixture projects, you can omit this field and the\n  // agent starts with an empty directory.\n  fixtures: [\"file://apps/studio-basic\"],\n\n  // ── Prompt ────────────────────────────────────────────────────\n  // The task instructions sent to the agent. Be specific about expected\n  // file paths and patterns — the assertions check for exact paths.\n  prompt: {\n    text: `You have a Sanity Studio project with an existing article schema.\nAdd a new \"post\" document type with the following fields:\n1. title (string, required)\n2. slug (slug, sourced from title, required)\n3. author (string)\n4. publishedAt (datetime)\n5. body (array of block content)\n\nCreate the schema file at schemas/post.ts using defineType() and defineField().\nRegister it in schemas/index.ts alongside the existing article schema.`,\n  },\n\n  // ── Assertions ────────────────────────────────────────────────\n  // Agent-harness assertions verify the sandbox state after the agent runs.\n  // They execute in a full Node.js context (not eval()) so they can use\n  // fs, child_process, etc. All file paths resolve relative to the sandbox.\n  //\n  // Available assertion types:\n  //   file-exists      — check a file was created\n  //   file-contains    — check a file contains a substring\n  //   command-succeeds — run a shell command (exit 0 = pass)\n  //   diff-matches     — check git diff contains a pattern\n  //   llm-rubric       — LLM grades the agent's text output\n  assertions: [\n    // Verify the agent created the new schema file\n    { type: \"file-exists\", value: \"schemas/post.ts\" },\n\n    // Verify it uses the modern Sanity schema API\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineType\" },\n    },\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/post.ts\", content: \"defineField\" },\n    },\n\n    // Verify the schema was registered in the barrel export\n    {\n      type: \"file-contains\",\n      value: { path: \"schemas/index.ts\", content: \"post\" },\n    },\n\n    // Verify the existing config is still intact\n    {\n      type: \"file-contains\",\n      value: { path: \"sanity.config.ts\", content: \"defineConfig\" },\n    },\n  ],\n\n  status: \"draft\",\n})\n";
+/** Generated YAML for example-agent-add-schema (from parsed TS data) */
+export const exampleAgentAddSchemaYaml = "- mode: agent-harness\n  id: example-agent-add-schema\n  title: Add a document schema to a Sanity Studio project\n  description: Example — tests whether an agent can create a schema file and register it in an existing Sanity Studio project\n  area: studio\n  sandbox:\n    type: tempdir\n  tools:\n    - coding\n  fixtures:\n    - file://apps/studio-basic\n  prompt:\n    text: |-\n      You have a Sanity Studio project with an existing article schema.\n      Add a new \"post\" document type with the following fields:\n      1. title (string, required)\n      2. slug (slug, sourced from title, required)\n      3. author (string)\n      4. publishedAt (datetime)\n      5. body (array of block content)\n\n      Create the schema file at schemas/post.ts using defineType() and defineField().\n      Register it in schemas/index.ts alongside the existing article schema.\n  assertions:\n    - type: file-exists\n      value: schemas/post.ts\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineType\n    - type: file-contains\n      value:\n        path: schemas/post.ts\n        content: defineField\n    - type: file-contains\n      value:\n        path: schemas/index.ts\n        content: post\n    - type: file-contains\n      value:\n        path: sanity.config.ts\n        content: defineConfig\n  status: draft\n";
 /** Parsed task data for example-groq-blog-listing (JSON-safe) */
 export const exampleGroqBlogListingData = [
     {
@@ -489,6 +550,7 @@ export const exampleStudioCustomInputYaml = "- mode: literacy\n  id: example-stu
 // ---------------------------------------------------------------------------
 /** All task example data as a flat array (JSON-safe) */
 export const allTaskData = [
+    ...exampleAgentAddSchemaData,
     ...exampleGroqBlogListingData,
     ...exampleIdBasedRefData,
     ...exampleKnowledgeProbeData,
@@ -499,6 +561,7 @@ export const allTaskData = [
 ];
 /** Map of task ID (filename stem) → raw TypeScript source */
 export const taskTsFiles = {
+    "example-agent-add-schema": exampleAgentAddSchemaTs,
     "example-groq-blog-listing": exampleGroqBlogListingTs,
     "example-id-based-ref": exampleIdBasedRefTs,
     "example-knowledge-probe": exampleKnowledgeProbeTs,
@@ -509,6 +572,7 @@ export const taskTsFiles = {
 };
 /** Map of task ID (filename stem) → generated YAML string */
 export const taskYamlFiles = {
+    "example-agent-add-schema": exampleAgentAddSchemaYaml,
     "example-groq-blog-listing": exampleGroqBlogListingYaml,
     "example-id-based-ref": exampleIdBasedRefYaml,
     "example-knowledge-probe": exampleKnowledgeProbeYaml,
@@ -518,8 +582,9 @@ export const taskYamlFiles = {
     "example-studio-custom-input": exampleStudioCustomInputYaml,
 };
 /** List of task file stems, in alphabetical order */
-export const TASK_FILE_NAMES = ["example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
+export const TASK_FILE_NAMES = ["example-agent-add-schema", "example-groq-blog-listing", "example-id-based-ref", "example-knowledge-probe", "example-mcp-tool-usage", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];
 export const TASK_EXAMPLES = [
+    { stem: "example-agent-add-schema", mode: "agent-harness", status: "draft" },
     { stem: "example-groq-blog-listing", mode: "literacy", status: "draft" },
     { stem: "example-id-based-ref", mode: "literacy", status: "draft" },
     { stem: "example-knowledge-probe", mode: "knowledge-probe", status: "draft" },

package/dist/agent-harness/assertions-runtime.d.ts

@@ -0,0 +1,49 @@
+/**
+ * assertions-runtime.ts — Runtime assertion functions for agent harness tasks.
+ *
+ * These run as file-based promptfoo JavaScript assertions, which execute
+ * in a full Node.js context (unlike inline `type: javascript` assertions
+ * which run in a restricted eval() sandbox without require()).
+ *
+ * Each export is a named function with the signature:
+ *   (output: string, context: { vars, config, ... }) => GradingResult
+ *
+ * Referenced in promptfoo config as:
+ *   value: file://dist/agent-harness/assertions-runtime.js:functionName
+ *   config: { filePath: "...", ... }
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ */
+interface GradingResult {
+    pass: boolean;
+    score: number;
+    reason: string;
+}
+interface AssertionContext {
+    vars: Record<string, unknown>;
+    config?: Record<string, unknown>;
+}
+/**
+ * Assert that a file exists in the sandbox working directory.
+ * Config: { filePath: string }
+ */
+export declare function fileExists(_output: string, context: AssertionContext): GradingResult;
+/**
+ * Assert that a file contains expected content.
+ * Config: { filePath: string, content: string }
+ */
+export declare function fileContains(_output: string, context: AssertionContext): GradingResult;
+/**
+ * Assert that a shell command succeeds (exit code 0) in the sandbox.
+ * Config: { command: string, timeoutMs?: number }
+ *
+ * SECURITY: Commands come from developer-authored task definitions,
+ * not from user input or LLM output.
+ */
+export declare function commandSucceeds(_output: string, context: AssertionContext): GradingResult;
+/**
+ * Assert that the git diff in the sandbox contains expected content.
+ * Config: { expected?: string }
+ */
+export declare function diffMatches(_output: string, context: AssertionContext): GradingResult;
+export {};

package/dist/agent-harness/assertions-runtime.js

@@ -0,0 +1,138 @@
+/**
+ * assertions-runtime.ts — Runtime assertion functions for agent harness tasks.
+ *
+ * These run as file-based promptfoo JavaScript assertions, which execute
+ * in a full Node.js context (unlike inline `type: javascript` assertions
+ * which run in a restricted eval() sandbox without require()).
+ *
+ * Each export is a named function with the signature:
+ *   (output: string, context: { vars, config, ... }) => GradingResult
+ *
+ * Referenced in promptfoo config as:
+ *   value: file://dist/agent-harness/assertions-runtime.js:functionName
+ *   config: { filePath: "...", ... }
+ *
+ * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
+ */
+import { existsSync, readFileSync } from "fs";
+import { execSync } from "child_process";
+import { resolve, sep } from "path";
+function resolveWorkDir(context) {
+    return resolve(context.vars.__workingDir || ".");
+}
+function guardTraversal(workDir, target, label) {
+    if (!target.startsWith(workDir + sep) && target !== workDir) {
+        return {
+            pass: false,
+            score: 0,
+            reason: `Path traversal: ${label} escapes sandbox`,
+        };
+    }
+    return null;
+}
+/**
+ * Assert that a file exists in the sandbox working directory.
+ * Config: { filePath: string }
+ */
+export function fileExists(_output, context) {
+    const filePath = context.config?.filePath ?? "";
+    const workDir = resolveWorkDir(context);
+    const target = resolve(workDir, filePath);
+    const traversal = guardTraversal(workDir, target, filePath);
+    if (traversal)
+        return traversal;
+    const exists = existsSync(target);
+    return {
+        pass: exists,
+        score: exists ? 1 : 0,
+        reason: exists
+            ? `File exists: ${filePath}`
+            : `Expected file not found: ${filePath}`,
+    };
+}
+/**
+ * Assert that a file contains expected content.
+ * Config: { filePath: string, content: string }
+ */
+export function fileContains(_output, context) {
+    const filePath = context.config?.filePath ?? "";
+    const expected = context.config?.content ?? "";
+    const workDir = resolveWorkDir(context);
+    const target = resolve(workDir, filePath);
+    const traversal = guardTraversal(workDir, target, filePath);
+    if (traversal)
+        return traversal;
+    if (!existsSync(target)) {
+        return { pass: false, score: 0, reason: `File not found: ${filePath}` };
+    }
+    const content = readFileSync(target, "utf-8");
+    const contains = content.includes(expected);
+    return {
+        pass: contains,
+        score: contains ? 1 : 0,
+        reason: contains
+            ? "File contains expected content"
+            : "File does not contain expected content",
+    };
+}
+/**
+ * Assert that a shell command succeeds (exit code 0) in the sandbox.
+ * Config: { command: string, timeoutMs?: number }
+ *
+ * SECURITY: Commands come from developer-authored task definitions,
+ * not from user input or LLM output.
+ */
+export function commandSucceeds(_output, context) {
+    const command = context.config?.command ?? "";
+    const timeoutMs = context.config?.timeoutMs ?? 30000;
+    const workDir = context.vars.__workingDir || ".";
+    try {
+        execSync(command, { cwd: workDir, timeout: timeoutMs });
+        return { pass: true, score: 1, reason: `Command succeeded: ${command}` };
+    }
+    catch (err) {
+        const error = err;
+        return {
+            pass: false,
+            score: 0,
+            reason: `Command failed: ${error.message || String(err)}`,
+        };
+    }
+}
+/**
+ * Assert that the git diff in the sandbox contains expected content.
+ * Config: { expected?: string }
+ */
+export function diffMatches(_output, context) {
+    const expected = context.config?.expected;
+    const workDir = context.vars.__workingDir || ".";
+    try {
+        const diff = execSync("git diff", {
+            cwd: workDir,
+            encoding: "utf-8",
+        });
+        if (typeof expected === "string") {
+            const contains = diff.includes(expected);
+            return {
+                pass: contains,
+                score: contains ? 1 : 0,
+                reason: contains
+                    ? "Diff matches expected pattern"
+                    : "Diff does not match",
+            };
+        }
+        return {
+            pass: diff.length > 0,
+            score: diff.length > 0 ? 1 : 0,
+            reason: "Diff exists",
+        };
+    }
+    catch (err) {
+        const error = err;
+        return {
+            pass: false,
+            score: 0,
+            reason: `Failed to get diff: ${error.message}`,
+        };
+    }
+}

package/dist/agent-harness/provider.d.ts

@@ -0,0 +1,58 @@
+/**
+ * provider.ts — Agent harness Promptfoo custom provider.
+ *
+ * Handles agent-harness evaluation tasks by calling the Anthropic API
+ * with the task prompt and returning the response. Sandbox provisioning
+ * and teardown are handled by lifecycle extensions (beforeEach/afterEach),
+ * not by the provider itself.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/agent-harness/provider.js
+ *       label: "Agent Harness: My Task"
+ *       config:
+ *         allowedTools: ["Bash", "Read", "Write"]
+ *         sandbox: { type: "tempdir" }
+ *
+ * Promptfoo loads this file and instantiates the default export class.
+ */
+interface CallApiContextParams {
+    prompt?: {
+        raw: string;
+        label?: string;
+    };
+    vars?: Record<string, object | string>;
+}
+interface ProviderOptions {
+    config?: Record<string, unknown>;
+    id?: string;
+}
+interface ProviderResponse {
+    cached?: boolean;
+    cost?: number;
+    error?: string;
+    metadata?: Record<string, unknown>;
+    output?: object | string;
+    tokenUsage?: {
+        total?: number;
+        prompt?: number;
+        completion?: number;
+        cached?: number;
+    };
+}
+export default class AgentHarnessProvider {
+    config: Record<string, unknown>;
+    private providerId;
+    constructor(options: ProviderOptions);
+    id(): string;
+    /**
+     * Main Promptfoo provider entry point. Called for each test case.
+     *
+     * Sends the task prompt to the Anthropic API and returns the response.
+     * The sandbox working directory is available in context.vars.__workingDir
+     * (set by the beforeEach extension).
+     */
+    callApi(prompt: string, context?: CallApiContextParams): Promise<ProviderResponse>;
+}
+export {};

package/dist/agent-harness/provider.js

@@ -0,0 +1,104 @@
+/**
+ * provider.ts — Agent harness Promptfoo custom provider.
+ *
+ * Handles agent-harness evaluation tasks by calling the Anthropic API
+ * with the task prompt and returning the response. Sandbox provisioning
+ * and teardown are handled by lifecycle extensions (beforeEach/afterEach),
+ * not by the provider itself.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/agent-harness/provider.js
+ *       label: "Agent Harness: My Task"
+ *       config:
+ *         allowedTools: ["Bash", "Read", "Write"]
+ *         sandbox: { type: "tempdir" }
+ *
+ * Promptfoo loads this file and instantiates the default export class.
+ */
+import { config as loadDotenv } from "dotenv";
+loadDotenv({
+    override: true,
+    path: new URL("../../.env", import.meta.url).pathname,
+});
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+export default class AgentHarnessProvider {
+    config;
+    providerId;
+    constructor(options) {
+        this.providerId = options.id ?? "agent-harness";
+        this.config = options.config ?? {};
+    }
+    id() {
+        return this.providerId;
+    }
+    /**
+     * Main Promptfoo provider entry point. Called for each test case.
+     *
+     * Sends the task prompt to the Anthropic API and returns the response.
+     * The sandbox working directory is available in context.vars.__workingDir
+     * (set by the beforeEach extension).
+     */
+    async callApi(prompt, context) {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            return {
+                error: "ANTHROPIC_API_KEY not set. Required for agent-harness evaluation.",
+            };
+        }
+        const model = this.config.model ?? "claude-sonnet-4-20250514";
+        const maxTokens = this.config.maxTokens ?? 4096;
+        const workingDir = context?.vars?.__workingDir;
+        const systemPrompt = workingDir
+            ? `You are an AI coding agent. Your working directory is: ${workingDir}\nComplete the following task.`
+            : "You are an AI coding agent. Complete the following task.";
+        try {
+            const response = await fetch("https://api.anthropic.com/v1/messages", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-api-key": apiKey,
+                    "anthropic-version": "2023-06-01",
+                },
+                body: JSON.stringify({
+                    model,
+                    max_tokens: maxTokens,
+                    system: systemPrompt,
+                    messages: [{ role: "user", content: prompt }],
+                }),
+            });
+            if (!response.ok) {
+                const errorBody = await response.text();
+                return {
+                    error: `Anthropic API error (${response.status}): ${errorBody}`,
+                };
+            }
+            const data = (await response.json());
+            const output = data.content
+                .filter((block) => block.type === "text")
+                .map((block) => block.text)
+                .join("\n");
+            return {
+                output,
+                tokenUsage: {
+                    prompt: data.usage?.input_tokens,
+                    completion: data.usage?.output_tokens,
+                    total: (data.usage?.input_tokens ?? 0) + (data.usage?.output_tokens ?? 0),
+                },
+                metadata: {
+                    model,
+                    workingDir,
+                    allowedTools: this.config.allowedTools,
+                    sandboxType: this.config.sandbox?.type ?? "none",
+                },
+            };
+        }
+        catch (err) {
+            const error = err;
+            return { error: `Agent harness provider error: ${error.message}` };
+        }
+    }
+}

package/dist/commands/init.js

@@ -138,6 +138,9 @@ async function runInit(opts) {
     else if (modeFilter === "knowledge-probe") {
         stemsToWrite = taskStemsForMode("knowledge-probe");
     }
+    else if (modeFilter === "agent-harness") {
+        stemsToWrite = taskStemsForMode("agent-harness");
+    }
     else {
         // Default (no --mode): write all tasks
         stemsToWrite = [...TASK_FILE_NAMES];

package/dist/orchestration/steps/generate-configs-step.d.ts

@@ -11,12 +11,19 @@
 import type { AppContext, PipelineState, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class GenerateConfigsStep implements PipelineStep {
     readonly name = "generate-configs";
+    /** Task IDs from the last loadTasks call (pre-filter), for error messages. */
+    private lastLoadedTaskIds;
     check(ctx: AppContext): ValidationIssue[];
     execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
     private compileLiteracyVariants;
     private compileSingleMode;
     private loadTasks;
     private applyFilters;
+    /**
+     * Build a descriptive error message when no tasks match the current filters.
+     * Distinguishes between "no tasks exist" and "tasks exist but filters exclude them".
+     */
+    private buildNoTasksError;
     /**
      * Compile all tasks through a handler, merging results.
      * For literacy mode, ctx can carry evalMode as an extension.