@sanity/ailf 1.0.0 → 2.0.0

package/README.md

@@ -17,5 +17,4 @@ for installation, quick start, configuration, and usage guides.
 
 | Package                                                                    | Description                                        |
 | -------------------------------------------------------------------------- | -------------------------------------------------- |
-| [`@sanity/ailf-tasks`](https://www.npmjs.com/package/@sanity/ailf-tasks)   | Lightweight task validator — schemas + YAML parser |
 | [`@sanity/ailf-studio`](https://www.npmjs.com/package/@sanity/ailf-studio) | Sanity Studio dashboard plugin for viewing reports |

package/config/models.ts

@@ -16,7 +16,13 @@ export default defineModels({
       id: "anthropic:messages:claude-opus-4-6",
       label: "Claude Opus 4.6",
       config: { temperature: 0.2, max_tokens: 4096 },
-      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+      modes: [
+        "baseline",
+        "observed",
+        "agentic-naive",
+        "agentic-optimized",
+        "mcp-server",
+      ],
     },
 
     // ── Google ─────────────────────────────────────────────────
@@ -35,14 +41,20 @@ export default defineModels({
       modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
     },
     {
-      id: "openai:chat:gpt-5.4",
+      id: "openai:responses:gpt-5.4",
       label: "GPT 5.4",
       config: {
         reasoning_effort: "medium",
         max_output_tokens: 4096,
         maxRetries: 1,
       },
-      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+      modes: [
+        "baseline",
+        "observed",
+        "agentic-naive",
+        "agentic-optimized",
+        "mcp-server",
+      ],
     },
 
     // ── Disabled models (uncomment to enable) ──────────────────

package/dist/_vendor/ailf-core/config-helpers.d.ts

@@ -31,6 +31,7 @@ import type { SchedulesFile } from "./schemas/schedules.js";
 import type { SinksFile } from "./schemas/sinks.js";
 import type { ModelsConfig } from "./types/index.js";
 import type { GeneralizedTaskDefinition } from "./types/generalized-task.js";
+import type { ModeBase, PresetDefinition } from "./types/plugin-registry.js";
 /**
  * Define an AILF evaluation configuration.
  *
@@ -132,27 +133,23 @@ export interface PromptEntry {
  */
 export declare function definePrompts(prompts: PromptEntry[]): PromptEntry[];
 /**
- * A preset is a named bundle of configuration that can be referenced
- * by tasks or suites to apply shared settings.
+ * Define a domain preset — targets a mode base and adds domain-specific config.
+ *
+ * A domain preset declares which mode it targets (by ID), provides domain
+ * configuration (sources, features, doc fetcher), and can optionally override
+ * mode base defaults (rubrics, scoring profiles, prompt templates).
+ *
+ * Used in `.ailf/preset.ts` or npm packages for typed preset authoring.
  */
-export interface PresetConfig {
-    /** Preset name */
-    name: string;
-    /** Default providers for tasks using this preset */
-    providers?: string[];
-    /** Default rubric template */
-    rubric?: string;
-    /** Default assertion templates */
-    assertions?: unknown[];
-    /** Default options */
-    options?: Record<string, unknown>;
-}
+export declare function definePreset(preset: PresetDefinition): PresetDefinition;
 /**
- * Define reusable configuration presets.
+ * Define a mode base — shared evaluation methodology for a mode.
  *
- * Used in `config/presets.ts` for typed preset configuration.
+ * A mode base defines HOW you evaluate (rubrics, scoring, prompts)
+ * independently of WHAT you're evaluating. Domain presets target a mode
+ * base and inherit its defaults.
  */
-export declare function definePreset(preset: PresetConfig): PresetConfig;
+export declare function defineModeBase(base: ModeBase): ModeBase;
 /**
  * A pricing table entry for cost estimation.
  */

package/dist/_vendor/ailf-core/config-helpers.js

@@ -132,14 +132,34 @@ export function defineSources(sources) {
 export function definePrompts(prompts) {
     return prompts;
 }
+// ---------------------------------------------------------------------------
+// Preset helpers
+// ---------------------------------------------------------------------------
 /**
- * Define reusable configuration presets.
+ * Define a domain preset — targets a mode base and adds domain-specific config.
+ *
+ * A domain preset declares which mode it targets (by ID), provides domain
+ * configuration (sources, features, doc fetcher), and can optionally override
+ * mode base defaults (rubrics, scoring profiles, prompt templates).
  *
- * Used in `config/presets.ts` for typed preset configuration.
+ * Used in `.ailf/preset.ts` or npm packages for typed preset authoring.
  */
 export function definePreset(preset) {
     return preset;
 }
+// ---------------------------------------------------------------------------
+// Mode base helpers
+// ---------------------------------------------------------------------------
+/**
+ * Define a mode base — shared evaluation methodology for a mode.
+ *
+ * A mode base defines HOW you evaluate (rubrics, scoring, prompts)
+ * independently of WHAT you're evaluating. Domain presets target a mode
+ * base and inherit its defaults.
+ */
+export function defineModeBase(base) {
+    return base;
+}
 /**
  * Define a pricing table for cost estimation.
  *

package/dist/_vendor/ailf-core/examples/index.d.ts

@@ -301,3 +301,19 @@ export interface ExampleRecord {
 export declare const EXAMPLES: Record<ExampleType, ExampleRecord>;
 /** GitHub Actions workflow template for AI Literacy evaluation */
 export declare const workflowYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# AI Literacy Evaluation \u2014 GitHub Actions workflow\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Evaluates your documentation quality on every pull request.\n# The AILF CLI reads your .ailf/tasks/ definitions, submits them\n# to the AILF API for evaluation, and writes a score report.\n#\n# Prerequisites:\n#   Add one secret to your repository (Settings \u2192 Secrets \u2192 Actions):\n#     AILF_API_KEY \u2014 your API key (starts with ailf_live_sk_)\n#     NPM_TOKEN   \u2014 npm token with read access to @sanity scope\n#\n# Customization:\n#   - Narrow the trigger paths to reduce cost (see comment below)\n#   - Check debug_mode for faster iteration (fewer tests)\n#   - See: https://github.com/sanity-labs/ai-literacy-framework\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nname: AI Literacy Eval\n\non:\n  pull_request:\n    branches: [main]\n    # Runs on every PR to main by default. To reduce cost:\n    #   paths: [\".ailf/**\", \"docs/**\"]\n\n  workflow_dispatch:\n    inputs:\n      debug_mode:\n        description: \"Run in debug mode (fewer tests, faster iteration)\"\n        type: boolean\n        default: false\n\nconcurrency:\n  group: ailf-eval-${{ github.event.pull_request.number || github.ref }}\n  cancel-in-progress: true\n\njobs:\n  evaluate:\n    name: AI Literacy Evaluation\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read\n      pull-requests: write\n    steps:\n      - uses: actions/checkout@v4\n\n      - name: Configure npm for @sanity scope\n        run:\n          echo \"//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}\" >>\n          ~/.npmrc\n\n      - name: Run evaluation\n        id: eval\n        env:\n          AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n        run: |\n          npx @sanity/ailf@latest pipeline --remote \\\n            --output /tmp/ailf-report.md \\\n            ${{ inputs.debug_mode && '--debug' || '' }}\n\n      - name: Post PR comment\n        if: always() && github.event_name == 'pull_request'\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n\n            // --- Constants ---\n            const MARKER = '<!-- ailf-score-report -->';\n            const HISTORY_START = '<!-- ailf-score-history -->';\n            const HISTORY_END = '<!-- /ailf-score-history -->';\n            const MAX_HISTORY = 3; // keep at most 3 prior runs\n\n            // --- Read new report ---\n            let newReport;\n            try {\n              newReport = fs.readFileSync('/tmp/ailf-report.md', 'utf-8');\n            } catch {\n              newReport = `## \u26A0\uFE0F AI Literacy Evaluation\\n\\nNo report generated. Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).`;\n            }\n\n            const prNumber = context.issue?.number || context.payload?.pull_request?.number;\n            if (!prNumber) {\n              console.log('No PR number found, skipping comment');\n              return;\n            }\n\n            // --- Find existing comment ---\n            const { data: comments } = await github.rest.issues.listComments({\n              owner: context.repo.owner, repo: context.repo.repo,\n              issue_number: prNumber,\n            });\n            const existing = comments.find(c => c.body?.includes(MARKER));\n\n            // --- Build history from previous comment ---\n            let historyEntries = [];\n            if (existing) {\n              const oldBody = existing.body || '';\n\n              // Collect existing collapsed history entries\n              const histStart = oldBody.indexOf(HISTORY_START);\n              const histEnd = oldBody.indexOf(HISTORY_END);\n              if (histStart !== -1 && histEnd !== -1) {\n                const historyContent = oldBody.slice(histStart + HISTORY_START.length, histEnd).trim();\n                // Split on </details> boundaries to get individual entries\n                if (historyContent) {\n                  historyEntries = historyContent\n                    .split(/<\\/details>\\s*/)\n                    .map(s => s.trim())\n                    .filter(s => s.startsWith('<details>'))\n                    .map(s => s + '\\n</details>');\n                }\n              }\n\n              // Extract the current report (will become the newest history entry)\n              let previousReport = '';\n              if (histStart !== -1) {\n                // Report is between MARKER and the \"Previous runs\" heading (or history section)\n                const markerIdx = oldBody.indexOf(MARKER);\n                // Find the --- separator before history\n                const separatorIdx = oldBody.lastIndexOf('---', histStart);\n                const endIdx = separatorIdx > markerIdx ? separatorIdx : histStart;\n                previousReport = oldBody.slice(markerIdx + MARKER.length, endIdx).trim();\n              } else {\n                // No history yet \u2014 everything after MARKER is the report\n                const markerIdx = oldBody.indexOf(MARKER);\n                if (markerIdx !== -1) {\n                  previousReport = oldBody.slice(markerIdx + MARKER.length).trim();\n                }\n              }\n\n              // Collapse the previous report into a <details> entry\n              if (previousReport) {\n                const scoreMatch = previousReport.match(/Overall:\\s*(\\d+)\\/100/);\n                const score = scoreMatch ? scoreMatch[1] : '?';\n                const dateMatch = previousReport.match(/Generated by.*?\u00B7\\s*([^\u00B7<\\n*]+)/);\n                const date = dateMatch\n                  ? dateMatch[1].trim()\n                  : new Date().toISOString().slice(0, 16).replace('T', ' ') + ' UTC';\n                const entry = `<details>\\n<summary>\uD83D\uDCDC ${date} \u2014 ${score}/100</summary>\\n\\n${previousReport}\\n\\n</details>`;\n                historyEntries.unshift(entry); // newest first\n              }\n\n              // Enforce max history limit\n              historyEntries = historyEntries.slice(0, MAX_HISTORY);\n            }\n\n            // --- Assemble final comment ---\n            const historySection = historyEntries.length > 0\n              ? `\\n\\n---\\n\\n### \uD83D\uDCDC Previous runs\\n\\n${HISTORY_START}\\n${historyEntries.join('\\n\\n')}\\n${HISTORY_END}`\n              : '';\n            const finalBody = `${MARKER}\\n${newReport}${historySection}`;\n\n            if (existing) {\n              await github.rest.issues.updateComment({\n                owner: context.repo.owner, repo: context.repo.repo,\n                comment_id: existing.id, body: finalBody,\n              });\n              console.log(`Updated comment (${historyEntries.length} history entries)`);\n            } else {\n              await github.rest.issues.createComment({\n                owner: context.repo.owner, repo: context.repo.repo,\n                issue_number: prNumber, body: finalBody,\n              });\n              console.log('Created new PR comment');\n            }\n\n      - name: Summary\n        if: always()\n        run: |\n          if [ -f /tmp/ailf-report.md ]; then\n            cat /tmp/ailf-report.md >> \"$GITHUB_STEP_SUMMARY\"\n          else\n            echo \"## \u26A0\uFE0F AI Literacy Evaluation\" >> \"$GITHUB_STEP_SUMMARY\"\n            echo \"\" >> \"$GITHUB_STEP_SUMMARY\"\n            echo \"No report generated. Check the workflow logs.\" >> \"$GITHUB_STEP_SUMMARY\"\n          fi\n";
+/** TypeScript project configuration template (ailf.config.ts) */
+export declare const ailfConfigTs = "/**\n * .ailf/ailf.config.ts \u2014 AI Literacy Framework project configuration.\n *\n * This file configures how the AILF evaluation pipeline runs in this\n * repository. Place it at .ailf/ailf.config.ts in your project root.\n *\n * Evaluations are submitted to the AILF API (ailf-api.sanity.build).\n * The API handles LLM calls, doc fetching, grading, and report\n * publishing. Your repo only needs one secret: AILF_API_KEY.\n *\n * Docs: https://github.com/sanity-labs/ai-literacy-framework\n */\n\nexport default {\n  /**\n   * Documentation source \u2014 which docs are being evaluated.\n   *\n   * This tells the pipeline which Sanity project and dataset contain\n   * the documentation under test. For most users, this is Sanity's own\n   * docs project.\n   */\n  source: {\n    /** Sanity project ID (find yours at sanity.io/manage) */\n    projectId: \"3do82whm\",\n    /** The dataset to query (e.g., \"production\", \"next\") */\n    dataset: \"next\",\n    /**\n     * The public URL of your documentation site.\n     * Used by agentic mode to test agent discoverability.\n     */\n    baseUrl: \"https://www.sanity.io/docs\",\n  },\n\n  /**\n   * Trigger configuration \u2014 when evaluations run automatically.\n   *\n   * Each key is a trigger context. The pipeline checks which trigger\n   * matches the current execution context (PR, merge, schedule, etc.)\n   * and applies its settings.\n   *\n   * Mode options:\n   *   \"validate-only\" \u2014 check that task files parse correctly (fast, no LLM calls)\n   *   \"eval\"          \u2014 run the full evaluation pipeline\n   */\n  triggers: {\n    /** On pull requests: just validate task files parse correctly. */\n    pr: {\n      mode: \"validate-only\",\n    },\n\n    /** When .ailf/ files change in a PR: run a real evaluation. */\n    \"pr-task-change\": {\n      mode: \"eval\",\n      paths: [\".ailf/**\"],\n    },\n\n    /** On merge to main: run evaluation (non-blocking). */\n    main: {\n      mode: \"eval\",\n      blocking: false,\n      notify: true,\n    },\n  },\n}\n";
+/** TypeScript task template for example-groq-blog-listing */
+export declare const exampleGroqBlogListingTs = "/**\n * Example Task: Blog listing with GROQ queries.\n *\n * This is a starter template \u2014 edit it for your own documentation.\n * Each task evaluates whether an AI coding agent can implement a feature\n * using your docs as context. Delete this file or replace it entirely.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * Full field reference:\n *   https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/CONTRIBUTING_TASKS.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  // \u2500\u2500 Mode \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // \"literacy\" tests whether AI coding tools can implement features\n  // using your docs as context. Other modes: \"mcp-server\",\n  // \"knowledge-probe\", \"agent-harness\", \"custom\".\n  mode: \"literacy\",\n\n  // \u2500\u2500 Identity \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Unique identifier \u2014 lowercase alphanumeric with hyphens.\n  // Must be unique across all task files in .ailf/tasks/.\n  id: \"example-groq-blog-listing\",\n  title: \"Blog listing with GROQ queries\",\n  description: \"Example \u2014 tests GROQ blog listing implementation\",\n\n  // Feature area this task belongs to. Tasks with the same area are\n  // grouped together in score summaries.\n  area: \"groq\",\n\n  // \u2500\u2500 Documentation context \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Canonical doc references for this task. The pipeline fetches these\n  // from Sanity and injects them into the prompt for baseline evaluation.\n  //\n  // This example uses slug-based references \u2014 the simplest form.\n  // See the other example tasks for path, id, and perspective references.\n  context: {\n    docs: [\n      {\n        slug: \"groq-introduction\",\n        reason: \"Core GROQ syntax and query language reference\",\n      },\n      {\n        slug: \"how-queries-work\",\n        reason: \"Query execution model and best practices\",\n      },\n    ],\n  },\n\n  // When true, the pipeline auto-generates an additional rubric that\n  // checks whether the LLM's response actually used the provided docs.\n  docCoverage: true,\n\n  // Path to a gold-standard implementation, relative to canonical/.\n  // The grader uses this as a reference when scoring code correctness.\n  referenceSolution: \"canonical/example-groq-blog-listing.ts\",\n\n  // \u2500\u2500 Prompt \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // prompt.text \u2014 the implementation prompt given to the LLM.\n  // Write this as if you're asking a developer to build the feature.\n  // Be specific about requirements so the grader can evaluate clearly.\n  prompt: {\n    text: `Create a Next.js page component that lists blog posts from Sanity\nusing GROQ. The page should display the title, slug, and published\ndate for each post, sorted by most recent first. Use the Sanity\nclient to fetch data.`,\n  },\n\n  // \u2500\u2500 Assertions \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  // Grading assertions \u2014 how the LLM's response is scored.\n  //\n  // \"llm-rubric\" assertions use a grader LLM to score against criteria.\n  // The \"template\" references a rubric template (e.g. task-completion).\n  //\n  // Available templates:\n  //   task-completion   \u2014 did the LLM implement the feature? (weight: 0.50)\n  //   code-correctness  \u2014 is the code idiomatic and correct? (weight: 0.25)\n  //\n  // You can also use value-based assertions:\n  //   { type: \"contains\", value: \"client.fetch\" }\n  //   { type: \"contains-any\", value: [\"createClient\", \"sanityClient\"] }\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Uses the groq tagged template literal\",\n        \"Fetches blog posts with title, slug, and publishedAt fields\",\n        \"Orders results by publishedAt in descending order\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"Uses createClient from @sanity/client or next-sanity\",\n        \"Exports a valid Next.js page component\",\n      ],\n    },\n  ],\n\n  // \u2500\u2500 Baseline variant \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n  //   enabled \u2014 set to false to skip this task entirely\n  //   rubric  \u2014 \"full\" (default), \"abbreviated\" (faster), or \"none\"\n  baseline: {\n    enabled: true,\n    rubric: \"full\",\n  },\n\n  // Example tasks ship as drafts so they don't run in production evals.\n  // Change to \"active\" (or remove this field) to activate.\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-id-based-ref */
+export declare const exampleIdBasedRefTs = "/**\n * Example Task: Document ID-based canonical doc references.\n *\n * Demonstrates using `id` to reference canonical documentation by\n * Sanity document `_id`. This is useful for:\n *   - Draft documents that don't have a stable slug yet\n *   - Programmatic references from imports or migrations\n *   - Documents where you know the _id but not the slug\n *\n * The `id` ref type can also carry optional `slug` and `path` fields\n * as human-readable annotations \u2014 these are NOT used for resolution,\n * only for display in logs and reports.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-id-based-ref\",\n  title: \"GROQ feature support (ID-based doc references)\",\n  description: \"Example \u2014 demonstrates ID-based canonical doc references\",\n\n  area: \"groq\",\n\n  // ID-based canonical doc references.\n  //\n  // Use the Sanity document _id to reference articles directly.\n  // Optional slug/path annotations help humans reading the file\n  // but are NOT used for resolution \u2014 only the `id` field matters.\n  //\n  // These IDs reference real articles in the Sanity docs (next dataset):\n  //   0ba88f1b... = \"GROQ feature support across Sanity\"\n  //   5b9c2863... = \"Custom GROQ functions\"\n  context: {\n    docs: [\n      {\n        id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\",\n        slug: \"groq-feature-support-by-context\", // annotation only\n        reason: \"GROQ feature support across different Sanity contexts\",\n      },\n      {\n        id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\",\n        slug: \"custom-groq-functions\", // annotation only\n        reason: \"Custom GROQ functions and pipelines\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n\n  prompt: {\n    text: `Explain how GROQ is used across different Sanity contexts.\nCover the following:\n1. Which GROQ features are available in each context (API queries,\n   webhooks, custom functions, access control)\n2. How to create and use custom GROQ functions\n3. Any differences in GROQ support between contexts\nProvide examples demonstrating context-specific GROQ patterns.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Explains GROQ availability across different Sanity contexts\",\n        \"Describes custom GROQ function creation and usage\",\n        \"Notes differences in GROQ support between contexts\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"GROQ examples use valid syntax\",\n        \"Custom function examples follow the correct API pattern\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-path-based-ref */
+export declare const examplePathBasedRefTs = "/**\n * Example Task: Path-based canonical doc references.\n *\n * Demonstrates using `path` to reference canonical documentation.\n * Paths are the preferred reference type because they uniquely identify\n * an article across sections (unlike slugs, which can collide).\n *\n * Path format:\n *   - Simple:    \"webhooks\"                \u2192 resolves by slug lookup\n *   - Sectioned: \"content-lake/webhooks\"   \u2192 disambiguates by section + slug\n *\n * This example demonstrates why paths matter: the slug \"documents\"\n * exists in both the \"content-lake\" and \"cli-reference\" sections.\n * Using \"content-lake/documents\" ensures we get the right one.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-path-based-ref\",\n  title: \"GROQ mutations (path-based doc references)\",\n  description: \"Example \u2014 demonstrates path-based canonical doc references\",\n\n  area: \"groq\",\n\n  // Path-based canonical doc references.\n  //\n  // Use \"section/slug\" format to uniquely identify articles:\n  //   - \"content-lake/mutations-introduction\" \u2192 the mutations article\n  //   - \"content-lake/documents\" \u2192 the documents article in Content Lake\n  //     (not the CLI \"documents\" article in cli-reference section)\n  //\n  // The \"documents\" slug exists in two sections \u2014 this is exactly why\n  // path-based references are preferred over slug-based references.\n  context: {\n    docs: [\n      {\n        path: \"content-lake/mutations-introduction\",\n        reason: \"Introduction to document mutations in the Content Lake\",\n      },\n      {\n        path: \"content-lake/documents\",\n        reason:\n          \"Document structure and types (Content Lake, not CLI reference)\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n\n  prompt: {\n    text: `Explain how to create, update, and delete documents in Sanity's\nContent Lake using mutations. Cover:\n1. The different mutation types (create, createOrReplace, patch, delete)\n2. Document structure and required fields (_id, _type)\n3. How to use patch operations to update specific fields\n4. Best practices for mutation patterns\nProvide working code examples using @sanity/client.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Explains create, createOrReplace, patch, and delete mutations\",\n        \"Describes required document fields (_id, _type)\",\n        \"Shows patch operations for field-level updates\",\n        \"Includes practical code examples\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"Uses correct @sanity/client mutation API\",\n        \"Patch operations use valid set/unset/inc syntax\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-perspective-ref */
+export declare const examplePerspectiveRefTs = "/**\n * Example Task: Perspective / content release doc references.\n *\n * Demonstrates using `perspective` to reference all documentation\n * articles within a content release. This is the key capability for\n * evaluating NEW feature documentation before it's published.\n *\n * How it works:\n *   - A perspective ref is one-to-many: the doc fetcher queries the\n *     named release and expands it to ALL articles versioned within it.\n *   - Downstream consumers see the same flat DocContext[] regardless\n *     of how docs were resolved.\n *   - When the release is published, the perspective entry becomes a\n *     no-op (articles are now in published). Migrate to explicit path\n *     or slug refs at your convenience.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-perspective-ref\",\n  title:\n    \"GROQ features from content release (perspective-based doc references)\",\n  description:\n    \"Example \u2014 demonstrates perspective-based canonical doc references\",\n\n  area: \"groq\",\n\n  // Perspective-based canonical doc reference.\n  //\n  // The perspective ID references a content release in the Sanity\n  // Content Lake. At evaluation time, the doc fetcher auto-discovers\n  // all articles versioned in this release and includes them as\n  // canonical documentation context.\n  //\n  // Release rE9TSJvR4 contains:\n  //   - \"GROQ-powered webhooks\" (webhooks)\n  //   - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n  //   - \"GROQ joins\" (groq-joins)\n  //\n  // You can combine perspective refs with explicit slug/path/id refs\n  // to include foundational published docs alongside release content.\n  // Here we add groq-data-types as a complementary published reference.\n  context: {\n    docs: [\n      {\n        perspective: \"rE9TSJvR4\",\n        reason: \"All GROQ documentation updates in the test content release\",\n      },\n      {\n        slug: \"groq-data-types\",\n        reason: \"GROQ data type reference (published, stable)\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n\n  prompt: {\n    text: `Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\n4. Working with different GROQ data types in filters\nProvide working GROQ query examples for each pattern.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Demonstrates GROQ join syntax for cross-document queries\",\n        \"Shows GROQ filter patterns for webhook configuration\",\n        \"Includes practical query examples from cheat sheet patterns\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"All GROQ queries use valid syntax\",\n        \"Reference joins use correct dereference operator (->)\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-studio-custom-input */
+export declare const exampleStudioCustomInputTs = "/**\n * Example Task: Custom input component in Sanity Studio.\n *\n * This is a starter template \u2014 edit it for your own documentation.\n * Delete this file or replace it with your own tasks.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-studio-custom-input\",\n  title: \"Custom input component in Sanity Studio\",\n  description: \"Example \u2014 tests Studio custom input implementation\",\n\n  area: \"studio\",\n\n  context: {\n    docs: [\n      {\n        slug: \"custom-input-widgets\",\n        reason: \"Guide for building custom form inputs in Sanity Studio\",\n      },\n      {\n        slug: \"form-components\",\n        reason: \"Form component API and customization patterns\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n  referenceSolution: \"canonical/example-studio-custom-input.ts\",\n\n  prompt: {\n    text: `Build a custom string input component for Sanity Studio that shows\na character count below the input field. The component should accept\na maxLength option from the field schema and display a warning when\nthe text exceeds the limit.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Implements a React component that renders a text input\",\n        \"Displays a live character count\",\n        \"Reads maxLength from schema options\",\n        \"Shows a visual warning when limit is exceeded\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"Uses the Sanity UI library for styling\",\n        \"Calls onChange with patch operations\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** Map of task ID (filename stem) → raw TypeScript source */
+export declare const taskTsFiles: Record<string, string>;
+/** List of TS task file stems, in alphabetical order */
+export declare const TASK_TS_FILE_NAMES: readonly ["example-groq-blog-listing", "example-id-based-ref", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];

package/dist/_vendor/ailf-core/examples/index.js

@@ -452,3 +452,28 @@ export const EXAMPLES = {
 // ---------------------------------------------------------------------------
 /** GitHub Actions workflow template for AI Literacy evaluation */
 export const workflowYaml = "# ──────────────────────────────────────────────────────────────────────\n# AI Literacy Evaluation — GitHub Actions workflow\n# ──────────────────────────────────────────────────────────────────────\n#\n# Evaluates your documentation quality on every pull request.\n# The AILF CLI reads your .ailf/tasks/ definitions, submits them\n# to the AILF API for evaluation, and writes a score report.\n#\n# Prerequisites:\n#   Add one secret to your repository (Settings → Secrets → Actions):\n#     AILF_API_KEY — your API key (starts with ailf_live_sk_)\n#     NPM_TOKEN   — npm token with read access to @sanity scope\n#\n# Customization:\n#   - Narrow the trigger paths to reduce cost (see comment below)\n#   - Check debug_mode for faster iteration (fewer tests)\n#   - See: https://github.com/sanity-labs/ai-literacy-framework\n# ──────────────────────────────────────────────────────────────────────\n\nname: AI Literacy Eval\n\non:\n  pull_request:\n    branches: [main]\n    # Runs on every PR to main by default. To reduce cost:\n    #   paths: [\".ailf/**\", \"docs/**\"]\n\n  workflow_dispatch:\n    inputs:\n      debug_mode:\n        description: \"Run in debug mode (fewer tests, faster iteration)\"\n        type: boolean\n        default: false\n\nconcurrency:\n  group: ailf-eval-${{ github.event.pull_request.number || github.ref }}\n  cancel-in-progress: true\n\njobs:\n  evaluate:\n    name: AI Literacy Evaluation\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read\n      pull-requests: write\n    steps:\n      - uses: actions/checkout@v4\n\n      - name: Configure npm for @sanity scope\n        run:\n          echo \"//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}\" >>\n          ~/.npmrc\n\n      - name: Run evaluation\n        id: eval\n        env:\n          AILF_API_KEY: ${{ secrets.AILF_API_KEY }}\n        run: |\n          npx @sanity/ailf@latest pipeline --remote \\\n            --output /tmp/ailf-report.md \\\n            ${{ inputs.debug_mode && '--debug' || '' }}\n\n      - name: Post PR comment\n        if: always() && github.event_name == 'pull_request'\n        uses: actions/github-script@v7\n        with:\n          script: |\n            const fs = require('fs');\n\n            // --- Constants ---\n            const MARKER = '<!-- ailf-score-report -->';\n            const HISTORY_START = '<!-- ailf-score-history -->';\n            const HISTORY_END = '<!-- /ailf-score-history -->';\n            const MAX_HISTORY = 3; // keep at most 3 prior runs\n\n            // --- Read new report ---\n            let newReport;\n            try {\n              newReport = fs.readFileSync('/tmp/ailf-report.md', 'utf-8');\n            } catch {\n              newReport = `## ⚠️ AI Literacy Evaluation\\n\\nNo report generated. Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).`;\n            }\n\n            const prNumber = context.issue?.number || context.payload?.pull_request?.number;\n            if (!prNumber) {\n              console.log('No PR number found, skipping comment');\n              return;\n            }\n\n            // --- Find existing comment ---\n            const { data: comments } = await github.rest.issues.listComments({\n              owner: context.repo.owner, repo: context.repo.repo,\n              issue_number: prNumber,\n            });\n            const existing = comments.find(c => c.body?.includes(MARKER));\n\n            // --- Build history from previous comment ---\n            let historyEntries = [];\n            if (existing) {\n              const oldBody = existing.body || '';\n\n              // Collect existing collapsed history entries\n              const histStart = oldBody.indexOf(HISTORY_START);\n              const histEnd = oldBody.indexOf(HISTORY_END);\n              if (histStart !== -1 && histEnd !== -1) {\n                const historyContent = oldBody.slice(histStart + HISTORY_START.length, histEnd).trim();\n                // Split on </details> boundaries to get individual entries\n                if (historyContent) {\n                  historyEntries = historyContent\n                    .split(/<\\/details>\\s*/)\n                    .map(s => s.trim())\n                    .filter(s => s.startsWith('<details>'))\n                    .map(s => s + '\\n</details>');\n                }\n              }\n\n              // Extract the current report (will become the newest history entry)\n              let previousReport = '';\n              if (histStart !== -1) {\n                // Report is between MARKER and the \"Previous runs\" heading (or history section)\n                const markerIdx = oldBody.indexOf(MARKER);\n                // Find the --- separator before history\n                const separatorIdx = oldBody.lastIndexOf('---', histStart);\n                const endIdx = separatorIdx > markerIdx ? separatorIdx : histStart;\n                previousReport = oldBody.slice(markerIdx + MARKER.length, endIdx).trim();\n              } else {\n                // No history yet — everything after MARKER is the report\n                const markerIdx = oldBody.indexOf(MARKER);\n                if (markerIdx !== -1) {\n                  previousReport = oldBody.slice(markerIdx + MARKER.length).trim();\n                }\n              }\n\n              // Collapse the previous report into a <details> entry\n              if (previousReport) {\n                const scoreMatch = previousReport.match(/Overall:\\s*(\\d+)\\/100/);\n                const score = scoreMatch ? scoreMatch[1] : '?';\n                const dateMatch = previousReport.match(/Generated by.*?·\\s*([^·<\\n*]+)/);\n                const date = dateMatch\n                  ? dateMatch[1].trim()\n                  : new Date().toISOString().slice(0, 16).replace('T', ' ') + ' UTC';\n                const entry = `<details>\\n<summary>📜 ${date} — ${score}/100</summary>\\n\\n${previousReport}\\n\\n</details>`;\n                historyEntries.unshift(entry); // newest first\n              }\n\n              // Enforce max history limit\n              historyEntries = historyEntries.slice(0, MAX_HISTORY);\n            }\n\n            // --- Assemble final comment ---\n            const historySection = historyEntries.length > 0\n              ? `\\n\\n---\\n\\n### 📜 Previous runs\\n\\n${HISTORY_START}\\n${historyEntries.join('\\n\\n')}\\n${HISTORY_END}`\n              : '';\n            const finalBody = `${MARKER}\\n${newReport}${historySection}`;\n\n            if (existing) {\n              await github.rest.issues.updateComment({\n                owner: context.repo.owner, repo: context.repo.repo,\n                comment_id: existing.id, body: finalBody,\n              });\n              console.log(`Updated comment (${historyEntries.length} history entries)`);\n            } else {\n              await github.rest.issues.createComment({\n                owner: context.repo.owner, repo: context.repo.repo,\n                issue_number: prNumber, body: finalBody,\n              });\n              console.log('Created new PR comment');\n            }\n\n      - name: Summary\n        if: always()\n        run: |\n          if [ -f /tmp/ailf-report.md ]; then\n            cat /tmp/ailf-report.md >> \"$GITHUB_STEP_SUMMARY\"\n          else\n            echo \"## ⚠️ AI Literacy Evaluation\" >> \"$GITHUB_STEP_SUMMARY\"\n            echo \"\" >> \"$GITHUB_STEP_SUMMARY\"\n            echo \"No report generated. Check the workflow logs.\" >> \"$GITHUB_STEP_SUMMARY\"\n          fi\n";
+// ---------------------------------------------------------------------------
+// TypeScript template exports (for ailf init --output-format ts)
+// ---------------------------------------------------------------------------
+/** TypeScript project configuration template (ailf.config.ts) */
+export const ailfConfigTs = "/**\n * .ailf/ailf.config.ts — AI Literacy Framework project configuration.\n *\n * This file configures how the AILF evaluation pipeline runs in this\n * repository. Place it at .ailf/ailf.config.ts in your project root.\n *\n * Evaluations are submitted to the AILF API (ailf-api.sanity.build).\n * The API handles LLM calls, doc fetching, grading, and report\n * publishing. Your repo only needs one secret: AILF_API_KEY.\n *\n * Docs: https://github.com/sanity-labs/ai-literacy-framework\n */\n\nexport default {\n  /**\n   * Documentation source — which docs are being evaluated.\n   *\n   * This tells the pipeline which Sanity project and dataset contain\n   * the documentation under test. For most users, this is Sanity's own\n   * docs project.\n   */\n  source: {\n    /** Sanity project ID (find yours at sanity.io/manage) */\n    projectId: \"3do82whm\",\n    /** The dataset to query (e.g., \"production\", \"next\") */\n    dataset: \"next\",\n    /**\n     * The public URL of your documentation site.\n     * Used by agentic mode to test agent discoverability.\n     */\n    baseUrl: \"https://www.sanity.io/docs\",\n  },\n\n  /**\n   * Trigger configuration — when evaluations run automatically.\n   *\n   * Each key is a trigger context. The pipeline checks which trigger\n   * matches the current execution context (PR, merge, schedule, etc.)\n   * and applies its settings.\n   *\n   * Mode options:\n   *   \"validate-only\" — check that task files parse correctly (fast, no LLM calls)\n   *   \"eval\"          — run the full evaluation pipeline\n   */\n  triggers: {\n    /** On pull requests: just validate task files parse correctly. */\n    pr: {\n      mode: \"validate-only\",\n    },\n\n    /** When .ailf/ files change in a PR: run a real evaluation. */\n    \"pr-task-change\": {\n      mode: \"eval\",\n      paths: [\".ailf/**\"],\n    },\n\n    /** On merge to main: run evaluation (non-blocking). */\n    main: {\n      mode: \"eval\",\n      blocking: false,\n      notify: true,\n    },\n  },\n}\n";
+/** TypeScript task template for example-groq-blog-listing */
+export const exampleGroqBlogListingTs = "/**\n * Example Task: Blog listing with GROQ queries.\n *\n * This is a starter template — edit it for your own documentation.\n * Each task evaluates whether an AI coding agent can implement a feature\n * using your docs as context. Delete this file or replace it entirely.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * Full field reference:\n *   https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/CONTRIBUTING_TASKS.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  // ── Mode ────────────────────────────────────────────────────────────\n  // \"literacy\" tests whether AI coding tools can implement features\n  // using your docs as context. Other modes: \"mcp-server\",\n  // \"knowledge-probe\", \"agent-harness\", \"custom\".\n  mode: \"literacy\",\n\n  // ── Identity ────────────────────────────────────────────────────────\n  // Unique identifier — lowercase alphanumeric with hyphens.\n  // Must be unique across all task files in .ailf/tasks/.\n  id: \"example-groq-blog-listing\",\n  title: \"Blog listing with GROQ queries\",\n  description: \"Example — tests GROQ blog listing implementation\",\n\n  // Feature area this task belongs to. Tasks with the same area are\n  // grouped together in score summaries.\n  area: \"groq\",\n\n  // ── Documentation context ───────────────────────────────────────────\n  // Canonical doc references for this task. The pipeline fetches these\n  // from Sanity and injects them into the prompt for baseline evaluation.\n  //\n  // This example uses slug-based references — the simplest form.\n  // See the other example tasks for path, id, and perspective references.\n  context: {\n    docs: [\n      {\n        slug: \"groq-introduction\",\n        reason: \"Core GROQ syntax and query language reference\",\n      },\n      {\n        slug: \"how-queries-work\",\n        reason: \"Query execution model and best practices\",\n      },\n    ],\n  },\n\n  // When true, the pipeline auto-generates an additional rubric that\n  // checks whether the LLM's response actually used the provided docs.\n  docCoverage: true,\n\n  // Path to a gold-standard implementation, relative to canonical/.\n  // The grader uses this as a reference when scoring code correctness.\n  referenceSolution: \"canonical/example-groq-blog-listing.ts\",\n\n  // ── Prompt ──────────────────────────────────────────────────────────\n  // prompt.text — the implementation prompt given to the LLM.\n  // Write this as if you're asking a developer to build the feature.\n  // Be specific about requirements so the grader can evaluate clearly.\n  prompt: {\n    text: `Create a Next.js page component that lists blog posts from Sanity\nusing GROQ. The page should display the title, slug, and published\ndate for each post, sorted by most recent first. Use the Sanity\nclient to fetch data.`,\n  },\n\n  // ── Assertions ──────────────────────────────────────────────────────\n  // Grading assertions — how the LLM's response is scored.\n  //\n  // \"llm-rubric\" assertions use a grader LLM to score against criteria.\n  // The \"template\" references a rubric template (e.g. task-completion).\n  //\n  // Available templates:\n  //   task-completion   — did the LLM implement the feature? (weight: 0.50)\n  //   code-correctness  — is the code idiomatic and correct? (weight: 0.25)\n  //\n  // You can also use value-based assertions:\n  //   { type: \"contains\", value: \"client.fetch\" }\n  //   { type: \"contains-any\", value: [\"createClient\", \"sanityClient\"] }\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Uses the groq tagged template literal\",\n        \"Fetches blog posts with title, slug, and publishedAt fields\",\n        \"Orders results by publishedAt in descending order\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"Uses createClient from @sanity/client or next-sanity\",\n        \"Exports a valid Next.js page component\",\n      ],\n    },\n  ],\n\n  // ── Baseline variant ────────────────────────────────────────────────\n  //   enabled — set to false to skip this task entirely\n  //   rubric  — \"full\" (default), \"abbreviated\" (faster), or \"none\"\n  baseline: {\n    enabled: true,\n    rubric: \"full\",\n  },\n\n  // Example tasks ship as drafts so they don't run in production evals.\n  // Change to \"active\" (or remove this field) to activate.\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-id-based-ref */
+export const exampleIdBasedRefTs = "/**\n * Example Task: Document ID-based canonical doc references.\n *\n * Demonstrates using `id` to reference canonical documentation by\n * Sanity document `_id`. This is useful for:\n *   - Draft documents that don't have a stable slug yet\n *   - Programmatic references from imports or migrations\n *   - Documents where you know the _id but not the slug\n *\n * The `id` ref type can also carry optional `slug` and `path` fields\n * as human-readable annotations — these are NOT used for resolution,\n * only for display in logs and reports.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-id-based-ref\",\n  title: \"GROQ feature support (ID-based doc references)\",\n  description: \"Example — demonstrates ID-based canonical doc references\",\n\n  area: \"groq\",\n\n  // ID-based canonical doc references.\n  //\n  // Use the Sanity document _id to reference articles directly.\n  // Optional slug/path annotations help humans reading the file\n  // but are NOT used for resolution — only the `id` field matters.\n  //\n  // These IDs reference real articles in the Sanity docs (next dataset):\n  //   0ba88f1b... = \"GROQ feature support across Sanity\"\n  //   5b9c2863... = \"Custom GROQ functions\"\n  context: {\n    docs: [\n      {\n        id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\",\n        slug: \"groq-feature-support-by-context\", // annotation only\n        reason: \"GROQ feature support across different Sanity contexts\",\n      },\n      {\n        id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\",\n        slug: \"custom-groq-functions\", // annotation only\n        reason: \"Custom GROQ functions and pipelines\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n\n  prompt: {\n    text: `Explain how GROQ is used across different Sanity contexts.\nCover the following:\n1. Which GROQ features are available in each context (API queries,\n   webhooks, custom functions, access control)\n2. How to create and use custom GROQ functions\n3. Any differences in GROQ support between contexts\nProvide examples demonstrating context-specific GROQ patterns.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Explains GROQ availability across different Sanity contexts\",\n        \"Describes custom GROQ function creation and usage\",\n        \"Notes differences in GROQ support between contexts\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"GROQ examples use valid syntax\",\n        \"Custom function examples follow the correct API pattern\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-path-based-ref */
+export const examplePathBasedRefTs = "/**\n * Example Task: Path-based canonical doc references.\n *\n * Demonstrates using `path` to reference canonical documentation.\n * Paths are the preferred reference type because they uniquely identify\n * an article across sections (unlike slugs, which can collide).\n *\n * Path format:\n *   - Simple:    \"webhooks\"                → resolves by slug lookup\n *   - Sectioned: \"content-lake/webhooks\"   → disambiguates by section + slug\n *\n * This example demonstrates why paths matter: the slug \"documents\"\n * exists in both the \"content-lake\" and \"cli-reference\" sections.\n * Using \"content-lake/documents\" ensures we get the right one.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-path-based-ref\",\n  title: \"GROQ mutations (path-based doc references)\",\n  description: \"Example — demonstrates path-based canonical doc references\",\n\n  area: \"groq\",\n\n  // Path-based canonical doc references.\n  //\n  // Use \"section/slug\" format to uniquely identify articles:\n  //   - \"content-lake/mutations-introduction\" → the mutations article\n  //   - \"content-lake/documents\" → the documents article in Content Lake\n  //     (not the CLI \"documents\" article in cli-reference section)\n  //\n  // The \"documents\" slug exists in two sections — this is exactly why\n  // path-based references are preferred over slug-based references.\n  context: {\n    docs: [\n      {\n        path: \"content-lake/mutations-introduction\",\n        reason: \"Introduction to document mutations in the Content Lake\",\n      },\n      {\n        path: \"content-lake/documents\",\n        reason:\n          \"Document structure and types (Content Lake, not CLI reference)\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n\n  prompt: {\n    text: `Explain how to create, update, and delete documents in Sanity's\nContent Lake using mutations. Cover:\n1. The different mutation types (create, createOrReplace, patch, delete)\n2. Document structure and required fields (_id, _type)\n3. How to use patch operations to update specific fields\n4. Best practices for mutation patterns\nProvide working code examples using @sanity/client.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Explains create, createOrReplace, patch, and delete mutations\",\n        \"Describes required document fields (_id, _type)\",\n        \"Shows patch operations for field-level updates\",\n        \"Includes practical code examples\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"Uses correct @sanity/client mutation API\",\n        \"Patch operations use valid set/unset/inc syntax\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-perspective-ref */
+export const examplePerspectiveRefTs = "/**\n * Example Task: Perspective / content release doc references.\n *\n * Demonstrates using `perspective` to reference all documentation\n * articles within a content release. This is the key capability for\n * evaluating NEW feature documentation before it's published.\n *\n * How it works:\n *   - A perspective ref is one-to-many: the doc fetcher queries the\n *     named release and expands it to ALL articles versioned within it.\n *   - Downstream consumers see the same flat DocContext[] regardless\n *     of how docs were resolved.\n *   - When the release is published, the perspective entry becomes a\n *     no-op (articles are now in published). Migrate to explicit path\n *     or slug refs at your convenience.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n *\n * @see docs/design-docs/canonical-doc-resolution.md\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-perspective-ref\",\n  title:\n    \"GROQ features from content release (perspective-based doc references)\",\n  description:\n    \"Example — demonstrates perspective-based canonical doc references\",\n\n  area: \"groq\",\n\n  // Perspective-based canonical doc reference.\n  //\n  // The perspective ID references a content release in the Sanity\n  // Content Lake. At evaluation time, the doc fetcher auto-discovers\n  // all articles versioned in this release and includes them as\n  // canonical documentation context.\n  //\n  // Release rE9TSJvR4 contains:\n  //   - \"GROQ-powered webhooks\" (webhooks)\n  //   - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n  //   - \"GROQ joins\" (groq-joins)\n  //\n  // You can combine perspective refs with explicit slug/path/id refs\n  // to include foundational published docs alongside release content.\n  // Here we add groq-data-types as a complementary published reference.\n  context: {\n    docs: [\n      {\n        perspective: \"rE9TSJvR4\",\n        reason: \"All GROQ documentation updates in the test content release\",\n      },\n      {\n        slug: \"groq-data-types\",\n        reason: \"GROQ data type reference (published, stable)\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n\n  prompt: {\n    text: `Using GROQ, demonstrate advanced query patterns including:\n1. Joining data across document types using references\n2. Filtering webhook payloads with GROQ projections\n3. Using the query cheat sheet patterns for common operations\n4. Working with different GROQ data types in filters\nProvide working GROQ query examples for each pattern.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Demonstrates GROQ join syntax for cross-document queries\",\n        \"Shows GROQ filter patterns for webhook configuration\",\n        \"Includes practical query examples from cheat sheet patterns\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"All GROQ queries use valid syntax\",\n        \"Reference joins use correct dereference operator (->)\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** TypeScript task template for example-studio-custom-input */
+export const exampleStudioCustomInputTs = "/**\n * Example Task: Custom input component in Sanity Studio.\n *\n * This is a starter template — edit it for your own documentation.\n * Delete this file or replace it with your own tasks.\n *\n * This example task ships as a DRAFT so it does not run in production\n * evaluations automatically. To activate it, change status to \"active\"\n * or remove the status field entirely (defaults to active).\n */\n\nimport { defineTask } from \"@sanity/ailf-core\"\n\nexport default defineTask({\n  mode: \"literacy\",\n  id: \"example-studio-custom-input\",\n  title: \"Custom input component in Sanity Studio\",\n  description: \"Example — tests Studio custom input implementation\",\n\n  area: \"studio\",\n\n  context: {\n    docs: [\n      {\n        slug: \"custom-input-widgets\",\n        reason: \"Guide for building custom form inputs in Sanity Studio\",\n      },\n      {\n        slug: \"form-components\",\n        reason: \"Form component API and customization patterns\",\n      },\n    ],\n  },\n\n  docCoverage: true,\n  referenceSolution: \"canonical/example-studio-custom-input.ts\",\n\n  prompt: {\n    text: `Build a custom string input component for Sanity Studio that shows\na character count below the input field. The component should accept\na maxLength option from the field schema and display a warning when\nthe text exceeds the limit.`,\n  },\n\n  assertions: [\n    {\n      type: \"llm-rubric\",\n      template: \"task-completion\",\n      criteria: [\n        \"Implements a React component that renders a text input\",\n        \"Displays a live character count\",\n        \"Reads maxLength from schema options\",\n        \"Shows a visual warning when limit is exceeded\",\n      ],\n    },\n    {\n      type: \"llm-rubric\",\n      template: \"code-correctness\",\n      criteria: [\n        \"Uses the Sanity UI library for styling\",\n        \"Calls onChange with patch operations\",\n      ],\n    },\n  ],\n\n  baseline: { enabled: true, rubric: \"full\" },\n  status: \"draft\",\n})\n";
+/** Map of task ID (filename stem) → raw TypeScript source */
+export const taskTsFiles = {
+    "example-groq-blog-listing": exampleGroqBlogListingTs,
+    "example-id-based-ref": exampleIdBasedRefTs,
+    "example-path-based-ref": examplePathBasedRefTs,
+    "example-perspective-ref": examplePerspectiveRefTs,
+    "example-studio-custom-input": exampleStudioCustomInputTs,
+};
+/** List of TS task file stems, in alphabetical order */
+export const TASK_TS_FILE_NAMES = ["example-groq-blog-listing", "example-id-based-ref", "example-path-based-ref", "example-perspective-ref", "example-studio-custom-input"];

package/dist/_vendor/ailf-core/index.d.ts

@@ -15,6 +15,6 @@ export * from "./schemas/index.js";
 export * from "./ports/index.js";
 export * from "./services/index.js";
 export * from "./examples/index.js";
-export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
-export type { PresetConfig, PricingEntry, PromptEntry, SourceEntry, } from "./config-helpers.js";
+export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
+export type { PricingEntry, PromptEntry, SourceEntry, } from "./config-helpers.js";
 export { env } from "./env-helper.js";

package/dist/_vendor/ailf-core/index.js

@@ -18,5 +18,5 @@ export * from "./examples/index.js";
 // ---------------------------------------------------------------------------
 // Architecture overhaul — Phase 0 helpers
 // ---------------------------------------------------------------------------
-export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
+export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
 export { env } from "./env-helper.js";

package/dist/_vendor/ailf-core/ports/context.d.ts

@@ -140,6 +140,8 @@ export interface ResolvedConfig {
     apiUrl: string;
     /** AILF API key (from AILF_API_KEY env var) */
     apiKey?: string;
+    /** External preset file paths or npm package names to load */
+    presets?: string[];
 }
 /**
  * Application context — the complete dependency carrier.

package/dist/_vendor/ailf-core/schemas/eval-config.d.ts

@@ -57,5 +57,6 @@ export declare const EvalConfigSchema: z.ZodObject<{
     source: z.ZodOptional<z.ZodString>;
     tasks: z.ZodOptional<z.ZodArray<z.ZodString>>;
     urls: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    presets: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strict>;
 export type EvalConfig = z.infer<typeof EvalConfigSchema>;

package/dist/_vendor/ailf-core/schemas/eval-config.js

@@ -81,5 +81,15 @@ export const EvalConfigSchema = z
     tasks: z.array(z.string()).optional(),
     /** Doc source URL overrides */
     urls: z.array(z.string().url()).optional(),
+    /**
+     * External presets to load — file paths or npm package names.
+     *
+     * Each entry is resolved as:
+     *   - Relative path (./foo or ../foo): loaded from disk via jiti
+     *   - Package name: resolved via Node require
+     *
+     * Presets are registered in order after built-in presets.
+     */
+    presets: z.array(z.string()).optional(),
 })
     .strict();

package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts

@@ -81,6 +81,7 @@ export declare const PipelineRequestSchema: z.ZodObject<{
     }>>;
     tasks: z.ZodOptional<z.ZodArray<z.ZodString>>;
     urls: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    presets: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 /** Inferred TypeScript type for a pipeline request payload. */
 export type PipelineRequest = z.infer<typeof PipelineRequestSchema>;

package/dist/_vendor/ailf-core/schemas/pipeline-request.js

@@ -89,4 +89,6 @@ export const PipelineRequestSchema = z.object({
     taskMode: z.enum(["content-lake", "yaml", "inline"]).optional(),
     tasks: z.array(z.string()).optional(),
     urls: z.array(z.string().url()).optional(),
+    /** External preset file paths or npm package names to load */
+    presets: z.array(z.string()).optional(),
 });

package/dist/_vendor/ailf-core/schemas/pipeline.d.ts

@@ -79,7 +79,6 @@ export declare const FeatureSchema: z.ZodObject<{
         planned: "planned";
         "out-of-scope": "out-of-scope";
     }>;
-    taskCount: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 /** Inferred TypeScript type for a product feature. */
 export type Feature = z.infer<typeof FeatureSchema>;
@@ -104,7 +103,6 @@ export declare const FeatureRegistrySchema: z.ZodObject<{
             planned: "planned";
             "out-of-scope": "out-of-scope";
         }>;
-        taskCount: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>>;
 }, z.core.$strip>;
 /** Inferred TypeScript type for the feature registry. */

package/dist/_vendor/ailf-core/schemas/pipeline.js

@@ -108,7 +108,6 @@ export const FeatureSchema = z.object({
     priority: z.enum(["critical", "high", "medium", "low"]),
     sections: z.array(z.string().min(1)).min(1),
     status: z.enum(["covered", "uncovered", "planned", "out-of-scope"]),
-    taskCount: z.number().int().min(0).optional(),
 });
 /**
  * Schema for the full config/features.yaml config file.

package/dist/_vendor/ailf-core/types/generalized-task.d.ts

@@ -226,6 +226,19 @@ export interface MCPServerTaskDefinition extends TaskCommonFields {
             content: string;
         }[];
     };
+    /**
+     * Override model list (provider IDs). When set, only these models are
+     * used instead of models filtered from the registry by `mcp-server` mode.
+     *
+     * @example ["anthropic:messages:claude-opus-4-6"]
+     */
+    models?: string[];
+    /**
+     * Maximum number of tool call rounds before forcing a final text response.
+     * Each round may include multiple parallel tool calls.
+     * @default 5
+     */
+    maxToolRounds?: number;
 }
 /**
  * Agent harness mode — evaluates autonomous agents in a sandbox.

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -14,7 +14,7 @@ export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from
 export type { DocumentRef } from "../../ailf-shared/index.d.ts";
 export type { StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
 export { CURRENT_SCHEMA_VERSION, isSchemaVersioned, migrateDocument, } from "./storage-schema.js";
-export type { AssertionRegistration, FixtureResolverRegistration, ModeRegistration, PluginManifest, PluginRegistry, PresetDefinition, ReportSinkRegistration, RubricTemplateRegistration, } from "./plugin-registry.js";
+export type { AssertionRegistration, FixtureResolverRegistration, ModeBase, ModeRegistration, PluginManifest, PluginRegistry, PresetDefinition, ReportSinkRegistration, RubricTemplateRegistration, } from "./plugin-registry.js";
 export { InMemoryPluginRegistry } from "./plugin-registry.js";
 export type { AgentHarnessConfig, AgentHarnessModeConfig, CustomModeConfig, EvalModeConfig, EvalModeType, KnowledgeBaseRef, KnowledgeProbeModeConfig, LiteracyModeConfig, MCPServerConfig, MCPServerModeConfig, ProbeStrategy, SandboxConfig, ToolDef, } from "./eval-mode-config.js";
 export { evalModeType } from "./eval-mode-config.js";
@@ -575,8 +575,6 @@ export interface ProductFeature {
     sections: string[];
     /** Coverage status */
     status: "covered" | "out-of-scope" | "planned" | "uncovered";
-    /** Number of evaluation tasks (if covered) */
-    taskCount?: number;
 }
 /** Full classification of a content release for evaluation */
 export interface ReleaseClassification {

package/dist/_vendor/ailf-core/types/plugin-registry.d.ts

@@ -1,12 +1,12 @@
 /**
  * Plugin registry — typed extension points for AILF evaluation capabilities.
  *
- * Twelve extension points: evaluation modes, providers, assertions,
- * rubric templates, fixture resolvers, report sinks, dashboard renderers,
- * prompt templates, scoring profiles, doc fetcher factory, source defs,
- * and feature defs.
- *
- * Presets bundle multiple extensions into a single installable unit.
+ * Three-tier architecture:
+ * - **Mode bases** define evaluation methodology (rubrics, scoring, prompts)
+ * - **Domain presets** target a mode base and add domain config (sources,
+ *   features, doc fetcher)
+ * - **Framework assertions** are generic evaluation primitives available to
+ *   all modes
  *
  * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
  */
@@ -33,8 +33,12 @@ export interface AssertionRegistration {
     type: string;
     /** Human-readable label */
     label: string;
-    /** Which modes this assertion is compatible with */
-    compatibleModes: string[];
+    /**
+     * Which modes this assertion is compatible with.
+     * When omitted, the assertion is compatible with all modes.
+     * When specified, acts as a whitelist of mode IDs.
+     */
+    compatibleModes?: string[];
     /** Assertion handler module path */
     handlerModule: string;
 }
@@ -65,6 +69,30 @@ export interface ReportSinkRegistration {
     /** Sink module path */
     handlerModule: string;
 }
+/**
+ * ModeBase — shared evaluation methodology for a mode.
+ *
+ * Defines HOW you evaluate (rubrics, scoring, prompts) independently of
+ * WHAT you're evaluating (sources, features, docs). Multiple domain presets
+ * can target the same mode base and inherit its defaults.
+ *
+ * Example: the "literacy" mode base defines rubric templates for
+ * task-completion, code-correctness, and doc-coverage. Both a Sanity docs
+ * preset and an external docs preset can target "literacy" and inherit
+ * these rubrics without redefining them.
+ */
+export interface ModeBase {
+    /** The mode registration (handler, provider patterns, rubric template IDs) */
+    mode: ModeRegistration;
+    /** Default rubric templates for this mode */
+    rubricTemplates?: RubricTemplateRegistration[];
+    /** Default scoring profiles for this mode (profile name → dimension weights) */
+    scoringProfiles?: Record<string, Record<string, number>>;
+    /** Default prompt templates for this mode (template name → template) */
+    promptTemplates?: Record<string, PromptTemplate>;
+    /** Mode-specific assertion types (beyond framework builtins) */
+    assertions?: AssertionRegistration[];
+}
 /** Plugin manifest describing a single plugin */
 export interface PluginManifest {
     /** Plugin name (npm package style) */
@@ -80,32 +108,49 @@ export interface PluginManifest {
     /** Dependencies on other plugins */
     requires?: string[];
 }
-/** A preset bundles multiple extensions into an installable unit */
+/**
+ * A domain preset targets a mode base and adds domain-specific configuration.
+ *
+ * The preset inherits evaluation methodology (rubrics, scoring, prompts) from
+ * its mode base. It can optionally override any inherited values.
+ */
 export interface PresetDefinition {
-    /** Preset name */
+    /** Preset name (unique identifier) */
     name: string;
     /** Plugin manifest */
     manifest: PluginManifest;
-    /** Evaluation modes to register */
-    modes?: ModeRegistration[];
-    /** Assertion types to register */
-    assertions?: AssertionRegistration[];
-    /** Rubric templates to register */
-    rubricTemplates?: RubricTemplateRegistration[];
-    /** Fixture resolvers to register */
+    /**
+     * Lifecycle status — mirrors task status semantics.
+     *   active:   registered and used in evaluations (default)
+     *   draft:    registered but skipped unless explicitly targeted
+     *   paused:   registered but skipped (can be resumed)
+     *   archived: not registered
+     */
+    status?: "active" | "archived" | "draft" | "paused";
+    /**
+     * Which mode this preset targets (by mode ID).
+     * Links to a registered ModeBase. The preset inherits rubrics,
+     * scoring profiles, and prompt templates from the base.
+     */
+    mode: string;
+    /** Fixture resolvers */
     fixtureResolvers?: FixtureResolverRegistration[];
-    /** Report sinks to register */
+    /** Report sinks */
     reportSinks?: ReportSinkRegistration[];
-    /** Prompt templates keyed by template name (e.g. "with-docs", "agentic") */
-    promptTemplates?: Record<string, PromptTemplate>;
-    /** Scoring profiles mapping profile name to dimension-weight pairs */
-    scoringProfiles?: Record<string, Record<string, number>>;
     /** Factory function that creates a DocFetcher instance */
     docFetcher?: () => DocFetcher;
     /** Documentation source definitions (production, branch, local, etc.) */
     sourceDefs?: SourceEntry[];
     /** Product feature registry for coverage tracking */
     featureDefs?: FeatureRegistry;
+    /** Override rubric templates (merged by ID with mode base) */
+    rubricTemplates?: RubricTemplateRegistration[];
+    /** Override scoring profiles (merged by name with mode base) */
+    scoringProfiles?: Record<string, Record<string, number>>;
+    /** Override prompt templates (merged by name with mode base) */
+    promptTemplates?: Record<string, PromptTemplate>;
+    /** Additional mode-specific assertions */
+    assertions?: AssertionRegistration[];
 }
 /**
  * PluginRegistry — central registry for all AILF extensions.
@@ -154,10 +199,16 @@ export interface PluginRegistry {
     registerSourceDefs(sources: SourceEntry[]): void;
     /** Get all registered source definitions */
     getSourceDefs(): SourceEntry[];
-    /** Register a feature registry (last-write-wins) */
+    /** Register a feature registry (merged by feature ID with existing) */
     registerFeatureDefs(features: FeatureRegistry): void;
     /** Get the registered feature registry, if any */
     getFeatureDefs(): FeatureRegistry | undefined;
+    /** Register a mode base (evaluation methodology) */
+    registerModeBase(base: ModeBase): void;
+    /** Get a mode base by mode ID */
+    getModeBase(modeId: string): ModeBase | undefined;
+    /** Get all registered mode bases */
+    getModeBases(): ModeBase[];
     /** Get all registered presets */
     getPresets(): PresetDefinition[];
 }
@@ -170,6 +221,7 @@ export declare class InMemoryPluginRegistry implements PluginRegistry {
     private readonly rubricTemplates_;
     private readonly fixtureResolvers_;
     private readonly reportSinks_;
+    private readonly modeBases_;
     private readonly presets_;
     private promptTemplates_;
     private scoringProfiles_;
@@ -199,4 +251,7 @@ export declare class InMemoryPluginRegistry implements PluginRegistry {
     getSourceDefs(): SourceEntry[];
     registerFeatureDefs(features: FeatureRegistry): void;
     getFeatureDefs(): FeatureRegistry | undefined;
+    registerModeBase(base: ModeBase): void;
+    getModeBase(modeId: string): ModeBase | undefined;
+    getModeBases(): ModeBase[];
 }