@sanity/ailf 0.4.1 → 0.5.0

package/config/rubrics.yaml

@@ -6,7 +6,7 @@
 # rubric text at expansion time.
 #
 # Each dimension is scored on a uniform 0–100 scale. Dimensions are
-# combined into a composite score using the weights below.
+# combined into a composite score using named scoring profiles below.
 #
 # Each template carries a `dimension` field that tags the scoring
 # dimension it belongs to. This metadata propagates through the
@@ -51,12 +51,31 @@ templates:
       - "80: Minor gaps — almost everything was documented"
       - "100: Complete coverage — all necessary info was in docs"
 
-# Weights for combining dimension scores into a composite (must sum to 1.0).
-# Composite = task-completion × 0.50 + code-correctness × 0.25 + doc-coverage × 0.25
-weights:
-  task-completion: 0.50
-  code-correctness: 0.25
-  doc-coverage: 0.25
+# Named scoring profiles — each is a dimension → weight map (must sum to 1.0).
+#
+# 'default': Full three-dimension composite for gold/ceiling entries (with docs).
+# 'output-only': Output quality dimensions only — excludes doc-coverage, which
+#   is semantically undefined on without-docs entries.
+#
+# See docs/design-docs/named-scoring-profiles.md for the rationale.
+profiles:
+  default:
+    task-completion: 0.50
+    code-correctness: 0.25
+    doc-coverage: 0.25
+  output-only:
+    task-completion: 0.60
+    code-correctness: 0.40
+
+# Mode-to-profile bindings — which profile to use for each (mode, variant) pair.
+# The scoring engine resolves: mode-profiles.<mode>.<variant> → profile name.
+# Falls back to 'default' when no explicit binding exists.
+mode-profiles:
+  baseline:
+    gold: default
+    baseline: output-only
+  agentic:
+    gold: default
 
 footer:
   'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}'

package/dist/_vendor/ailf-core/examples/index.d.ts

@@ -140,12 +140,12 @@ export declare const exampleGroqBlogListingData: readonly [{
     }];
     readonly baseline: {
         readonly enabled: true;
-        readonly rubric: "abbreviated";
+        readonly rubric: "full";
     };
     readonly status: "draft";
 }];
 /** Raw YAML string for example-groq-blog-listing (preserves comments) */
-export declare const exampleGroqBlogListingYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Blog listing with GROQ queries\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This is a starter template \u2014 edit it for your own documentation.\n# Each task evaluates whether an AI coding agent can implement a feature\n# using your docs as context. Delete this file or replace it entirely.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# Full field reference:\n#   https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/CONTRIBUTING_TASKS.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n# Unique identifier \u2014 lowercase alphanumeric with hyphens.\n# Must be unique across all task files in .ailf/tasks/.\n- id: example-groq-blog-listing\n\n  # Short human-readable summary. Shown in score tables and reports.\n  description: \"Example \u2014 Blog listing with GROQ queries\"\n\n  # Feature area this task belongs to. Tasks with the same area are\n  # grouped together in score summaries. Use a short kebab-case name.\n  featureArea: groq\n\n  # Gold-standard documentation articles for this task. The pipeline\n  # fetches these from Sanity and injects them into the prompt for\n  # baseline evaluation. Each entry needs:\n  #   slug   \u2014 the article's URL slug in your docs site\n  #   reason \u2014 why this doc is relevant (helps with auditing)\n  #\n  # This example uses slug-based references \u2014 the simplest form.\n  # See the other example tasks for path, id, and perspective references.\n  canonicalDocs:\n    - slug: groq-introduction\n      reason: \"Core GROQ syntax and query language reference\"\n    - slug: how-queries-work\n      reason: \"Query execution model and best practices\"\n\n  # When true, the pipeline auto-generates an additional rubric that\n  # checks whether the LLM's response actually used the provided docs.\n  docCoverage: true\n\n  # Path to a gold-standard implementation, relative to canonical/.\n  # The grader uses this as a reference when scoring code correctness.\n  referenceSolution: canonical/example-groq-blog-listing.ts\n\n  # vars.task \u2014 the implementation prompt given to the LLM.\n  # Write this as if you're asking a developer to build the feature.\n  # Be specific about requirements so the grader can evaluate clearly.\n  #\n  # vars.docs \u2014 leave empty (\"\"). The pipeline fills this in:\n  #   \u2022 Gold variant: injected with canonical doc content\n  #   \u2022 Baseline variant: left empty (tests model knowledge alone)\n  vars:\n    task: |\n      Create a Next.js page component that lists blog posts from Sanity\n      using GROQ. The page should display the title, slug, and published\n      date for each post, sorted by most recent first. Use the Sanity\n      client to fetch data.\n    docs: \"\"\n\n  # Grading assertions \u2014 how the LLM's response is scored.\n  #\n  # \"llm-rubric\" assertions use a grader LLM to score against criteria.\n  # The \"template\" references a rubric from config/rubrics.yaml.\n  # The \"criteria\" are task-specific bullets injected into the template.\n  #\n  # Available templates:\n  #   task-completion   \u2014 did the LLM implement the feature? (weight: 0.50)\n  #   code-correctness  \u2014 is the code idiomatic and correct? (weight: 0.25)\n  #\n  # You can also use value-based assertions:\n  #   - type: contains\n  #     value: \"client.fetch\"\n  #   - type: contains-any\n  #     value: [\"createClient\", \"sanityClient\"]\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Uses the groq tagged template literal\"\n        - \"Fetches blog posts with title, slug, and publishedAt fields\"\n        - \"Orders results by publishedAt in descending order\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses createClient from @sanity/client or next-sanity\"\n        - \"Exports a valid Next.js page component\"\n\n  # Baseline variant configuration.\n  #   enabled \u2014 set to false to skip this task entirely\n  #   rubric  \u2014 \"abbreviated\" (faster, default), \"full\", or \"none\"\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export declare const exampleGroqBlogListingYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Blog listing with GROQ queries\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This is a starter template \u2014 edit it for your own documentation.\n# Each task evaluates whether an AI coding agent can implement a feature\n# using your docs as context. Delete this file or replace it entirely.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# Full field reference:\n#   https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/CONTRIBUTING_TASKS.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n# Unique identifier \u2014 lowercase alphanumeric with hyphens.\n# Must be unique across all task files in .ailf/tasks/.\n- id: example-groq-blog-listing\n\n  # Short human-readable summary. Shown in score tables and reports.\n  description: \"Example \u2014 Blog listing with GROQ queries\"\n\n  # Feature area this task belongs to. Tasks with the same area are\n  # grouped together in score summaries. Use a short kebab-case name.\n  featureArea: groq\n\n  # Gold-standard documentation articles for this task. The pipeline\n  # fetches these from Sanity and injects them into the prompt for\n  # baseline evaluation. Each entry needs:\n  #   slug   \u2014 the article's URL slug in your docs site\n  #   reason \u2014 why this doc is relevant (helps with auditing)\n  #\n  # This example uses slug-based references \u2014 the simplest form.\n  # See the other example tasks for path, id, and perspective references.\n  canonicalDocs:\n    - slug: groq-introduction\n      reason: \"Core GROQ syntax and query language reference\"\n    - slug: how-queries-work\n      reason: \"Query execution model and best practices\"\n\n  # When true, the pipeline auto-generates an additional rubric that\n  # checks whether the LLM's response actually used the provided docs.\n  docCoverage: true\n\n  # Path to a gold-standard implementation, relative to canonical/.\n  # The grader uses this as a reference when scoring code correctness.\n  referenceSolution: canonical/example-groq-blog-listing.ts\n\n  # vars.task \u2014 the implementation prompt given to the LLM.\n  # Write this as if you're asking a developer to build the feature.\n  # Be specific about requirements so the grader can evaluate clearly.\n  #\n  # vars.docs \u2014 leave empty (\"\"). The pipeline fills this in:\n  #   \u2022 Gold variant: injected with canonical doc content\n  #   \u2022 Baseline variant: left empty (tests model knowledge alone)\n  vars:\n    task: |\n      Create a Next.js page component that lists blog posts from Sanity\n      using GROQ. The page should display the title, slug, and published\n      date for each post, sorted by most recent first. Use the Sanity\n      client to fetch data.\n    docs: \"\"\n\n  # Grading assertions \u2014 how the LLM's response is scored.\n  #\n  # \"llm-rubric\" assertions use a grader LLM to score against criteria.\n  # The \"template\" references a rubric from config/rubrics.yaml.\n  # The \"criteria\" are task-specific bullets injected into the template.\n  #\n  # Available templates:\n  #   task-completion   \u2014 did the LLM implement the feature? (weight: 0.50)\n  #   code-correctness  \u2014 is the code idiomatic and correct? (weight: 0.25)\n  #\n  # You can also use value-based assertions:\n  #   - type: contains\n  #     value: \"client.fetch\"\n  #   - type: contains-any\n  #     value: [\"createClient\", \"sanityClient\"]\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Uses the groq tagged template literal\"\n        - \"Fetches blog posts with title, slug, and publishedAt fields\"\n        - \"Orders results by publishedAt in descending order\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses createClient from @sanity/client or next-sanity\"\n        - \"Exports a valid Next.js page component\"\n\n  # Baseline variant configuration.\n  #   enabled \u2014 set to false to skip this task entirely\n  #   rubric  \u2014 \"full\" (default), \"abbreviated\" (faster), or \"none\"\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-id-based-ref (JSON-safe) */
 export declare const exampleIdBasedRefData: readonly [{
     readonly id: "example-id-based-ref";
@@ -176,12 +176,12 @@ export declare const exampleIdBasedRefData: readonly [{
     }];
     readonly baseline: {
         readonly enabled: true;
-        readonly rubric: "abbreviated";
+        readonly rubric: "full";
     };
     readonly status: "draft";
 }];
 /** Raw YAML string for example-id-based-ref (preserves comments) */
-export declare const exampleIdBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Document ID-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n#   - Draft documents that don't have a stable slug yet\n#   - Programmatic references from imports or migrations\n#   - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations \u2014 these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-id-based-ref\n  description: \"Example \u2014 GROQ feature support (ID-based doc references)\"\n\n  featureArea: groq\n\n  # ID-based canonical doc references.\n  #\n  # Use the Sanity document _id to reference articles directly.\n  # Optional slug/path annotations help humans reading the YAML\n  # but are NOT used for resolution \u2014 only the `id` field matters.\n  #\n  # These IDs reference real articles in the Sanity docs (next dataset):\n  #   0ba88f1b... = \"GROQ feature support across Sanity\"\n  #   5b9c2863... = \"Custom GROQ functions\"\n  canonicalDocs:\n    - id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\"\n      slug: groq-feature-support-by-context # annotation only \u2014 not used for resolution\n      reason: \"GROQ feature support across different Sanity contexts\"\n    - id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\"\n      slug: custom-groq-functions # annotation only \u2014 not used for resolution\n      reason: \"Custom GROQ functions and pipelines\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how GROQ is used across different Sanity contexts.\n      Cover the following:\n      1. Which GROQ features are available in each context (API queries,\n         webhooks, custom functions, access control)\n      2. How to create and use custom GROQ functions\n      3. Any differences in GROQ support between contexts\n      Provide examples demonstrating context-specific GROQ patterns.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains GROQ availability across different Sanity contexts\"\n        - \"Describes custom GROQ function creation and usage\"\n        - \"Notes differences in GROQ support between contexts\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"GROQ examples use valid syntax\"\n        - \"Custom function examples follow the correct API pattern\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export declare const exampleIdBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Document ID-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n#   - Draft documents that don't have a stable slug yet\n#   - Programmatic references from imports or migrations\n#   - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations \u2014 these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-id-based-ref\n  description: \"Example \u2014 GROQ feature support (ID-based doc references)\"\n\n  featureArea: groq\n\n  # ID-based canonical doc references.\n  #\n  # Use the Sanity document _id to reference articles directly.\n  # Optional slug/path annotations help humans reading the YAML\n  # but are NOT used for resolution \u2014 only the `id` field matters.\n  #\n  # These IDs reference real articles in the Sanity docs (next dataset):\n  #   0ba88f1b... = \"GROQ feature support across Sanity\"\n  #   5b9c2863... = \"Custom GROQ functions\"\n  canonicalDocs:\n    - id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\"\n      slug: groq-feature-support-by-context # annotation only \u2014 not used for resolution\n      reason: \"GROQ feature support across different Sanity contexts\"\n    - id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\"\n      slug: custom-groq-functions # annotation only \u2014 not used for resolution\n      reason: \"Custom GROQ functions and pipelines\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how GROQ is used across different Sanity contexts.\n      Cover the following:\n      1. Which GROQ features are available in each context (API queries,\n         webhooks, custom functions, access control)\n      2. How to create and use custom GROQ functions\n      3. Any differences in GROQ support between contexts\n      Provide examples demonstrating context-specific GROQ patterns.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains GROQ availability across different Sanity contexts\"\n        - \"Describes custom GROQ function creation and usage\"\n        - \"Notes differences in GROQ support between contexts\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"GROQ examples use valid syntax\"\n        - \"Custom function examples follow the correct API pattern\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-path-based-ref (JSON-safe) */
 export declare const examplePathBasedRefData: readonly [{
     readonly id: "example-path-based-ref";
@@ -210,12 +210,12 @@ export declare const examplePathBasedRefData: readonly [{
     }];
     readonly baseline: {
         readonly enabled: true;
-        readonly rubric: "abbreviated";
+        readonly rubric: "full";
     };
     readonly status: "draft";
 }];
 /** Raw YAML string for example-path-based-ref (preserves comments) */
-export declare const examplePathBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Path-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n#   - Simple:   \"webhooks\"              \u2192 resolves by slug lookup\n#   - Sectioned: \"content-lake/webhooks\" \u2192 disambiguates by section + slug\n#\n# This example demonstrates why paths matter: the slug \"documents\"\n# exists in both the \"content-lake\" and \"cli-reference\" sections.\n# Using \"content-lake/documents\" ensures we get the right one.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-path-based-ref\n  description: \"Example \u2014 GROQ mutations (path-based doc references)\"\n\n  featureArea: groq\n\n  # Path-based canonical doc references.\n  #\n  # Use \"section/slug\" format to uniquely identify articles:\n  #   - \"content-lake/mutations-introduction\" \u2192 the mutations article\n  #   - \"content-lake/documents\" \u2192 the documents article in Content Lake\n  #     (not the CLI \"documents\" article in cli-reference section)\n  #\n  # The \"documents\" slug exists in two sections \u2014 this is exactly why\n  # path-based references are preferred over slug-based references.\n  canonicalDocs:\n    - path: content-lake/mutations-introduction\n      reason: \"Introduction to document mutations in the Content Lake\"\n    - path: content-lake/documents\n      reason: \"Document structure and types (Content Lake, not CLI reference)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how to create, update, and delete documents in Sanity's\n      Content Lake using mutations. Cover:\n      1. The different mutation types (create, createOrReplace, patch, delete)\n      2. Document structure and required fields (_id, _type)\n      3. How to use patch operations to update specific fields\n      4. Best practices for mutation patterns\n      Provide working code examples using @sanity/client.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains create, createOrReplace, patch, and delete mutations\"\n        - \"Describes required document fields (_id, _type)\"\n        - \"Shows patch operations for field-level updates\"\n        - \"Includes practical code examples\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses correct @sanity/client mutation API\"\n        - \"Patch operations use valid set/unset/inc syntax\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export declare const examplePathBasedRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Path-based canonical doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n#   - Simple:   \"webhooks\"              \u2192 resolves by slug lookup\n#   - Sectioned: \"content-lake/webhooks\" \u2192 disambiguates by section + slug\n#\n# This example demonstrates why paths matter: the slug \"documents\"\n# exists in both the \"content-lake\" and \"cli-reference\" sections.\n# Using \"content-lake/documents\" ensures we get the right one.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-path-based-ref\n  description: \"Example \u2014 GROQ mutations (path-based doc references)\"\n\n  featureArea: groq\n\n  # Path-based canonical doc references.\n  #\n  # Use \"section/slug\" format to uniquely identify articles:\n  #   - \"content-lake/mutations-introduction\" \u2192 the mutations article\n  #   - \"content-lake/documents\" \u2192 the documents article in Content Lake\n  #     (not the CLI \"documents\" article in cli-reference section)\n  #\n  # The \"documents\" slug exists in two sections \u2014 this is exactly why\n  # path-based references are preferred over slug-based references.\n  canonicalDocs:\n    - path: content-lake/mutations-introduction\n      reason: \"Introduction to document mutations in the Content Lake\"\n    - path: content-lake/documents\n      reason: \"Document structure and types (Content Lake, not CLI reference)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how to create, update, and delete documents in Sanity's\n      Content Lake using mutations. Cover:\n      1. The different mutation types (create, createOrReplace, patch, delete)\n      2. Document structure and required fields (_id, _type)\n      3. How to use patch operations to update specific fields\n      4. Best practices for mutation patterns\n      Provide working code examples using @sanity/client.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains create, createOrReplace, patch, and delete mutations\"\n        - \"Describes required document fields (_id, _type)\"\n        - \"Shows patch operations for field-level updates\"\n        - \"Includes practical code examples\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses correct @sanity/client mutation API\"\n        - \"Patch operations use valid set/unset/inc syntax\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-perspective-ref (JSON-safe) */
 export declare const examplePerspectiveRefData: readonly [{
     readonly id: "example-perspective-ref";
@@ -244,12 +244,12 @@ export declare const examplePerspectiveRefData: readonly [{
     }];
     readonly baseline: {
         readonly enabled: true;
-        readonly rubric: "abbreviated";
+        readonly rubric: "full";
     };
     readonly status: "draft";
 }];
 /** Raw YAML string for example-perspective-ref (preserves comments) */
-export declare const examplePerspectiveRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Perspective / content release doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n#   - A perspective ref is one-to-many: the doc fetcher queries the\n#     named release and expands it to ALL articles versioned within it.\n#   - Downstream consumers see the same flat DocContext[] regardless\n#     of how docs were resolved.\n#   - When the release is published, the perspective entry becomes a\n#     no-op (articles are now in published). Migrate to explicit path\n#     or slug refs at your convenience.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-perspective-ref\n  description:\n    \"Example \u2014 GROQ features from content release (perspective-based doc\n    references)\"\n\n  featureArea: groq\n\n  # Perspective-based canonical doc reference.\n  #\n  # The perspective ID references a content release in the Sanity\n  # Content Lake. At evaluation time, the doc fetcher auto-discovers\n  # all articles versioned in this release and includes them as\n  # canonical documentation context.\n  #\n  # Release rE9TSJvR4 contains:\n  #   - \"GROQ-powered webhooks\" (webhooks)\n  #   - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n  #   - \"GROQ joins\" (groq-joins)\n  #\n  # You can combine perspective refs with explicit slug/path/id refs\n  # to include foundational published docs alongside release content.\n  # Here we add groq-data-types as a complementary published reference.\n  canonicalDocs:\n    - perspective: rE9TSJvR4\n      reason: \"All GROQ documentation updates in the test content release\"\n    - slug: groq-data-types\n      reason: \"GROQ data type reference (published, stable)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Using GROQ, demonstrate advanced query patterns including:\n      1. Joining data across document types using references\n      2. Filtering webhook payloads with GROQ projections\n      3. Using the query cheat sheet patterns for common operations\n      4. Working with different GROQ data types in filters\n      Provide working GROQ query examples for each pattern.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Demonstrates GROQ join syntax for cross-document queries\"\n        - \"Shows GROQ filter patterns for webhook configuration\"\n        - \"Includes practical query examples from cheat sheet patterns\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"All GROQ queries use valid syntax\"\n        - \"Reference joins use correct dereference operator (->)\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export declare const examplePerspectiveRefYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Perspective / content release doc references\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n#   - A perspective ref is one-to-many: the doc fetcher queries the\n#     named release and expands it to ALL articles versioned within it.\n#   - Downstream consumers see the same flat DocContext[] regardless\n#     of how docs were resolved.\n#   - When the release is published, the perspective entry becomes a\n#     no-op (articles are now in published). Migrate to explicit path\n#     or slug refs at your convenience.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-perspective-ref\n  description:\n    \"Example \u2014 GROQ features from content release (perspective-based doc\n    references)\"\n\n  featureArea: groq\n\n  # Perspective-based canonical doc reference.\n  #\n  # The perspective ID references a content release in the Sanity\n  # Content Lake. At evaluation time, the doc fetcher auto-discovers\n  # all articles versioned in this release and includes them as\n  # canonical documentation context.\n  #\n  # Release rE9TSJvR4 contains:\n  #   - \"GROQ-powered webhooks\" (webhooks)\n  #   - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n  #   - \"GROQ joins\" (groq-joins)\n  #\n  # You can combine perspective refs with explicit slug/path/id refs\n  # to include foundational published docs alongside release content.\n  # Here we add groq-data-types as a complementary published reference.\n  canonicalDocs:\n    - perspective: rE9TSJvR4\n      reason: \"All GROQ documentation updates in the test content release\"\n    - slug: groq-data-types\n      reason: \"GROQ data type reference (published, stable)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Using GROQ, demonstrate advanced query patterns including:\n      1. Joining data across document types using references\n      2. Filtering webhook payloads with GROQ projections\n      3. Using the query cheat sheet patterns for common operations\n      4. Working with different GROQ data types in filters\n      Provide working GROQ query examples for each pattern.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Demonstrates GROQ join syntax for cross-document queries\"\n        - \"Shows GROQ filter patterns for webhook configuration\"\n        - \"Includes practical query examples from cheat sheet patterns\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"All GROQ queries use valid syntax\"\n        - \"Reference joins use correct dereference operator (->)\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-studio-custom-input (JSON-safe) */
 export declare const exampleStudioCustomInputData: readonly [{
     readonly id: "example-studio-custom-input";
@@ -279,12 +279,12 @@ export declare const exampleStudioCustomInputData: readonly [{
     }];
     readonly baseline: {
         readonly enabled: true;
-        readonly rubric: "abbreviated";
+        readonly rubric: "full";
     };
     readonly status: "draft";
 }];
 /** Raw YAML string for example-studio-custom-input (preserves comments) */
-export declare const exampleStudioCustomInputYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Custom input component in Sanity Studio\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This is a starter template \u2014 edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-studio-custom-input\n  description: \"Example \u2014 Custom input component in Sanity Studio\"\n\n  featureArea: studio\n\n  # Slug-based canonical doc references.\n  canonicalDocs:\n    - slug: custom-input-widgets\n      reason: \"Guide for building custom form inputs in Sanity Studio\"\n    - slug: form-components\n      reason: \"Form component API and customization patterns\"\n\n  docCoverage: true\n  referenceSolution: canonical/example-studio-custom-input.ts\n\n  vars:\n    task: |\n      Build a custom string input component for Sanity Studio that shows\n      a character count below the input field. The component should accept\n      a maxLength option from the field schema and display a warning when\n      the text exceeds the limit.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Implements a React component that renders a text input\"\n        - \"Displays a live character count\"\n        - \"Reads maxLength from schema options\"\n        - \"Shows a visual warning when limit is exceeded\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses the Sanity UI library for styling\"\n        - \"Calls onChange with patch operations\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export declare const exampleStudioCustomInputYaml = "# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n# Example Task: Custom input component in Sanity Studio\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n#\n# This is a starter template \u2014 edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n# \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n- id: example-studio-custom-input\n  description: \"Example \u2014 Custom input component in Sanity Studio\"\n\n  featureArea: studio\n\n  # Slug-based canonical doc references.\n  canonicalDocs:\n    - slug: custom-input-widgets\n      reason: \"Guide for building custom form inputs in Sanity Studio\"\n    - slug: form-components\n      reason: \"Form component API and customization patterns\"\n\n  docCoverage: true\n  referenceSolution: canonical/example-studio-custom-input.ts\n\n  vars:\n    task: |\n      Build a custom string input component for Sanity Studio that shows\n      a character count below the input field. The component should accept\n      a maxLength option from the field schema and display a warning when\n      the text exceeds the limit.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Implements a React component that renders a text input\"\n        - \"Displays a live character count\"\n        - \"Reads maxLength from schema options\"\n        - \"Shows a visual warning when limit is exceeded\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses the Sanity UI library for styling\"\n        - \"Calls onChange with patch operations\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** All task example data as a flat array (JSON-safe) */
 export declare const allTaskData: readonly unknown[];
 /** Map of task ID (filename stem) → raw YAML string (preserves comments) */

package/dist/_vendor/ailf-core/examples/index.js

@@ -185,13 +185,13 @@ export const exampleGroqBlogListingData = [
         ],
         "baseline": {
             "enabled": true,
-            "rubric": "abbreviated"
+            "rubric": "full"
         },
         "status": "draft"
     }
 ];
 /** Raw YAML string for example-groq-blog-listing (preserves comments) */
-export const exampleGroqBlogListingYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Blog listing with GROQ queries\n# ──────────────────────────────────────────────────────────────────────\n#\n# This is a starter template — edit it for your own documentation.\n# Each task evaluates whether an AI coding agent can implement a feature\n# using your docs as context. Delete this file or replace it entirely.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# Full field reference:\n#   https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/CONTRIBUTING_TASKS.md\n# ──────────────────────────────────────────────────────────────────────\n\n# Unique identifier — lowercase alphanumeric with hyphens.\n# Must be unique across all task files in .ailf/tasks/.\n- id: example-groq-blog-listing\n\n  # Short human-readable summary. Shown in score tables and reports.\n  description: \"Example — Blog listing with GROQ queries\"\n\n  # Feature area this task belongs to. Tasks with the same area are\n  # grouped together in score summaries. Use a short kebab-case name.\n  featureArea: groq\n\n  # Gold-standard documentation articles for this task. The pipeline\n  # fetches these from Sanity and injects them into the prompt for\n  # baseline evaluation. Each entry needs:\n  #   slug   — the article's URL slug in your docs site\n  #   reason — why this doc is relevant (helps with auditing)\n  #\n  # This example uses slug-based references — the simplest form.\n  # See the other example tasks for path, id, and perspective references.\n  canonicalDocs:\n    - slug: groq-introduction\n      reason: \"Core GROQ syntax and query language reference\"\n    - slug: how-queries-work\n      reason: \"Query execution model and best practices\"\n\n  # When true, the pipeline auto-generates an additional rubric that\n  # checks whether the LLM's response actually used the provided docs.\n  docCoverage: true\n\n  # Path to a gold-standard implementation, relative to canonical/.\n  # The grader uses this as a reference when scoring code correctness.\n  referenceSolution: canonical/example-groq-blog-listing.ts\n\n  # vars.task — the implementation prompt given to the LLM.\n  # Write this as if you're asking a developer to build the feature.\n  # Be specific about requirements so the grader can evaluate clearly.\n  #\n  # vars.docs — leave empty (\"\"). The pipeline fills this in:\n  #   • Gold variant: injected with canonical doc content\n  #   • Baseline variant: left empty (tests model knowledge alone)\n  vars:\n    task: |\n      Create a Next.js page component that lists blog posts from Sanity\n      using GROQ. The page should display the title, slug, and published\n      date for each post, sorted by most recent first. Use the Sanity\n      client to fetch data.\n    docs: \"\"\n\n  # Grading assertions — how the LLM's response is scored.\n  #\n  # \"llm-rubric\" assertions use a grader LLM to score against criteria.\n  # The \"template\" references a rubric from config/rubrics.yaml.\n  # The \"criteria\" are task-specific bullets injected into the template.\n  #\n  # Available templates:\n  #   task-completion   — did the LLM implement the feature? (weight: 0.50)\n  #   code-correctness  — is the code idiomatic and correct? (weight: 0.25)\n  #\n  # You can also use value-based assertions:\n  #   - type: contains\n  #     value: \"client.fetch\"\n  #   - type: contains-any\n  #     value: [\"createClient\", \"sanityClient\"]\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Uses the groq tagged template literal\"\n        - \"Fetches blog posts with title, slug, and publishedAt fields\"\n        - \"Orders results by publishedAt in descending order\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses createClient from @sanity/client or next-sanity\"\n        - \"Exports a valid Next.js page component\"\n\n  # Baseline variant configuration.\n  #   enabled — set to false to skip this task entirely\n  #   rubric  — \"abbreviated\" (faster, default), \"full\", or \"none\"\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export const exampleGroqBlogListingYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Blog listing with GROQ queries\n# ──────────────────────────────────────────────────────────────────────\n#\n# This is a starter template — edit it for your own documentation.\n# Each task evaluates whether an AI coding agent can implement a feature\n# using your docs as context. Delete this file or replace it entirely.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# Full field reference:\n#   https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/CONTRIBUTING_TASKS.md\n# ──────────────────────────────────────────────────────────────────────\n\n# Unique identifier — lowercase alphanumeric with hyphens.\n# Must be unique across all task files in .ailf/tasks/.\n- id: example-groq-blog-listing\n\n  # Short human-readable summary. Shown in score tables and reports.\n  description: \"Example — Blog listing with GROQ queries\"\n\n  # Feature area this task belongs to. Tasks with the same area are\n  # grouped together in score summaries. Use a short kebab-case name.\n  featureArea: groq\n\n  # Gold-standard documentation articles for this task. The pipeline\n  # fetches these from Sanity and injects them into the prompt for\n  # baseline evaluation. Each entry needs:\n  #   slug   — the article's URL slug in your docs site\n  #   reason — why this doc is relevant (helps with auditing)\n  #\n  # This example uses slug-based references — the simplest form.\n  # See the other example tasks for path, id, and perspective references.\n  canonicalDocs:\n    - slug: groq-introduction\n      reason: \"Core GROQ syntax and query language reference\"\n    - slug: how-queries-work\n      reason: \"Query execution model and best practices\"\n\n  # When true, the pipeline auto-generates an additional rubric that\n  # checks whether the LLM's response actually used the provided docs.\n  docCoverage: true\n\n  # Path to a gold-standard implementation, relative to canonical/.\n  # The grader uses this as a reference when scoring code correctness.\n  referenceSolution: canonical/example-groq-blog-listing.ts\n\n  # vars.task — the implementation prompt given to the LLM.\n  # Write this as if you're asking a developer to build the feature.\n  # Be specific about requirements so the grader can evaluate clearly.\n  #\n  # vars.docs — leave empty (\"\"). The pipeline fills this in:\n  #   • Gold variant: injected with canonical doc content\n  #   • Baseline variant: left empty (tests model knowledge alone)\n  vars:\n    task: |\n      Create a Next.js page component that lists blog posts from Sanity\n      using GROQ. The page should display the title, slug, and published\n      date for each post, sorted by most recent first. Use the Sanity\n      client to fetch data.\n    docs: \"\"\n\n  # Grading assertions — how the LLM's response is scored.\n  #\n  # \"llm-rubric\" assertions use a grader LLM to score against criteria.\n  # The \"template\" references a rubric from config/rubrics.yaml.\n  # The \"criteria\" are task-specific bullets injected into the template.\n  #\n  # Available templates:\n  #   task-completion   — did the LLM implement the feature? (weight: 0.50)\n  #   code-correctness  — is the code idiomatic and correct? (weight: 0.25)\n  #\n  # You can also use value-based assertions:\n  #   - type: contains\n  #     value: \"client.fetch\"\n  #   - type: contains-any\n  #     value: [\"createClient\", \"sanityClient\"]\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Uses the groq tagged template literal\"\n        - \"Fetches blog posts with title, slug, and publishedAt fields\"\n        - \"Orders results by publishedAt in descending order\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses createClient from @sanity/client or next-sanity\"\n        - \"Exports a valid Next.js page component\"\n\n  # Baseline variant configuration.\n  #   enabled — set to false to skip this task entirely\n  #   rubric  — \"full\" (default), \"abbreviated\" (faster), or \"none\"\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-id-based-ref (JSON-safe) */
 export const exampleIdBasedRefData = [
     {
@@ -236,13 +236,13 @@ export const exampleIdBasedRefData = [
         ],
         "baseline": {
             "enabled": true,
-            "rubric": "abbreviated"
+            "rubric": "full"
         },
         "status": "draft"
     }
 ];
 /** Raw YAML string for example-id-based-ref (preserves comments) */
-export const exampleIdBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Document ID-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n#   - Draft documents that don't have a stable slug yet\n#   - Programmatic references from imports or migrations\n#   - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations — these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-id-based-ref\n  description: \"Example — GROQ feature support (ID-based doc references)\"\n\n  featureArea: groq\n\n  # ID-based canonical doc references.\n  #\n  # Use the Sanity document _id to reference articles directly.\n  # Optional slug/path annotations help humans reading the YAML\n  # but are NOT used for resolution — only the `id` field matters.\n  #\n  # These IDs reference real articles in the Sanity docs (next dataset):\n  #   0ba88f1b... = \"GROQ feature support across Sanity\"\n  #   5b9c2863... = \"Custom GROQ functions\"\n  canonicalDocs:\n    - id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\"\n      slug: groq-feature-support-by-context # annotation only — not used for resolution\n      reason: \"GROQ feature support across different Sanity contexts\"\n    - id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\"\n      slug: custom-groq-functions # annotation only — not used for resolution\n      reason: \"Custom GROQ functions and pipelines\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how GROQ is used across different Sanity contexts.\n      Cover the following:\n      1. Which GROQ features are available in each context (API queries,\n         webhooks, custom functions, access control)\n      2. How to create and use custom GROQ functions\n      3. Any differences in GROQ support between contexts\n      Provide examples demonstrating context-specific GROQ patterns.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains GROQ availability across different Sanity contexts\"\n        - \"Describes custom GROQ function creation and usage\"\n        - \"Notes differences in GROQ support between contexts\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"GROQ examples use valid syntax\"\n        - \"Custom function examples follow the correct API pattern\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export const exampleIdBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Document ID-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `id` to reference canonical documentation by\n# Sanity document `_id`. This is useful for:\n#   - Draft documents that don't have a stable slug yet\n#   - Programmatic references from imports or migrations\n#   - Documents where you know the _id but not the slug\n#\n# The `id` ref type can also carry optional `slug` and `path` fields\n# as human-readable annotations — these are NOT used for resolution,\n# only for display in logs and reports.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-id-based-ref\n  description: \"Example — GROQ feature support (ID-based doc references)\"\n\n  featureArea: groq\n\n  # ID-based canonical doc references.\n  #\n  # Use the Sanity document _id to reference articles directly.\n  # Optional slug/path annotations help humans reading the YAML\n  # but are NOT used for resolution — only the `id` field matters.\n  #\n  # These IDs reference real articles in the Sanity docs (next dataset):\n  #   0ba88f1b... = \"GROQ feature support across Sanity\"\n  #   5b9c2863... = \"Custom GROQ functions\"\n  canonicalDocs:\n    - id: \"0ba88f1b-d1a7-418a-9267-2e343d01886a\"\n      slug: groq-feature-support-by-context # annotation only — not used for resolution\n      reason: \"GROQ feature support across different Sanity contexts\"\n    - id: \"5b9c2863-ef01-4565-af8e-ee54e081ee74\"\n      slug: custom-groq-functions # annotation only — not used for resolution\n      reason: \"Custom GROQ functions and pipelines\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how GROQ is used across different Sanity contexts.\n      Cover the following:\n      1. Which GROQ features are available in each context (API queries,\n         webhooks, custom functions, access control)\n      2. How to create and use custom GROQ functions\n      3. Any differences in GROQ support between contexts\n      Provide examples demonstrating context-specific GROQ patterns.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains GROQ availability across different Sanity contexts\"\n        - \"Describes custom GROQ function creation and usage\"\n        - \"Notes differences in GROQ support between contexts\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"GROQ examples use valid syntax\"\n        - \"Custom function examples follow the correct API pattern\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-path-based-ref (JSON-safe) */
 export const examplePathBasedRefData = [
     {
@@ -286,13 +286,13 @@ export const examplePathBasedRefData = [
         ],
         "baseline": {
             "enabled": true,
-            "rubric": "abbreviated"
+            "rubric": "full"
         },
         "status": "draft"
     }
 ];
 /** Raw YAML string for example-path-based-ref (preserves comments) */
-export const examplePathBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Path-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n#   - Simple:   \"webhooks\"              → resolves by slug lookup\n#   - Sectioned: \"content-lake/webhooks\" → disambiguates by section + slug\n#\n# This example demonstrates why paths matter: the slug \"documents\"\n# exists in both the \"content-lake\" and \"cli-reference\" sections.\n# Using \"content-lake/documents\" ensures we get the right one.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-path-based-ref\n  description: \"Example — GROQ mutations (path-based doc references)\"\n\n  featureArea: groq\n\n  # Path-based canonical doc references.\n  #\n  # Use \"section/slug\" format to uniquely identify articles:\n  #   - \"content-lake/mutations-introduction\" → the mutations article\n  #   - \"content-lake/documents\" → the documents article in Content Lake\n  #     (not the CLI \"documents\" article in cli-reference section)\n  #\n  # The \"documents\" slug exists in two sections — this is exactly why\n  # path-based references are preferred over slug-based references.\n  canonicalDocs:\n    - path: content-lake/mutations-introduction\n      reason: \"Introduction to document mutations in the Content Lake\"\n    - path: content-lake/documents\n      reason: \"Document structure and types (Content Lake, not CLI reference)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how to create, update, and delete documents in Sanity's\n      Content Lake using mutations. Cover:\n      1. The different mutation types (create, createOrReplace, patch, delete)\n      2. Document structure and required fields (_id, _type)\n      3. How to use patch operations to update specific fields\n      4. Best practices for mutation patterns\n      Provide working code examples using @sanity/client.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains create, createOrReplace, patch, and delete mutations\"\n        - \"Describes required document fields (_id, _type)\"\n        - \"Shows patch operations for field-level updates\"\n        - \"Includes practical code examples\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses correct @sanity/client mutation API\"\n        - \"Patch operations use valid set/unset/inc syntax\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export const examplePathBasedRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Path-based canonical doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `path` to reference canonical documentation.\n# Paths are the preferred reference type because they uniquely identify\n# an article across sections (unlike slugs, which can collide).\n#\n# Path format:\n#   - Simple:   \"webhooks\"              → resolves by slug lookup\n#   - Sectioned: \"content-lake/webhooks\" → disambiguates by section + slug\n#\n# This example demonstrates why paths matter: the slug \"documents\"\n# exists in both the \"content-lake\" and \"cli-reference\" sections.\n# Using \"content-lake/documents\" ensures we get the right one.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-path-based-ref\n  description: \"Example — GROQ mutations (path-based doc references)\"\n\n  featureArea: groq\n\n  # Path-based canonical doc references.\n  #\n  # Use \"section/slug\" format to uniquely identify articles:\n  #   - \"content-lake/mutations-introduction\" → the mutations article\n  #   - \"content-lake/documents\" → the documents article in Content Lake\n  #     (not the CLI \"documents\" article in cli-reference section)\n  #\n  # The \"documents\" slug exists in two sections — this is exactly why\n  # path-based references are preferred over slug-based references.\n  canonicalDocs:\n    - path: content-lake/mutations-introduction\n      reason: \"Introduction to document mutations in the Content Lake\"\n    - path: content-lake/documents\n      reason: \"Document structure and types (Content Lake, not CLI reference)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Explain how to create, update, and delete documents in Sanity's\n      Content Lake using mutations. Cover:\n      1. The different mutation types (create, createOrReplace, patch, delete)\n      2. Document structure and required fields (_id, _type)\n      3. How to use patch operations to update specific fields\n      4. Best practices for mutation patterns\n      Provide working code examples using @sanity/client.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Explains create, createOrReplace, patch, and delete mutations\"\n        - \"Describes required document fields (_id, _type)\"\n        - \"Shows patch operations for field-level updates\"\n        - \"Includes practical code examples\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses correct @sanity/client mutation API\"\n        - \"Patch operations use valid set/unset/inc syntax\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-perspective-ref (JSON-safe) */
 export const examplePerspectiveRefData = [
     {
@@ -335,13 +335,13 @@ export const examplePerspectiveRefData = [
         ],
         "baseline": {
             "enabled": true,
-            "rubric": "abbreviated"
+            "rubric": "full"
         },
         "status": "draft"
     }
 ];
 /** Raw YAML string for example-perspective-ref (preserves comments) */
-export const examplePerspectiveRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Perspective / content release doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n#   - A perspective ref is one-to-many: the doc fetcher queries the\n#     named release and expands it to ALL articles versioned within it.\n#   - Downstream consumers see the same flat DocContext[] regardless\n#     of how docs were resolved.\n#   - When the release is published, the perspective entry becomes a\n#     no-op (articles are now in published). Migrate to explicit path\n#     or slug refs at your convenience.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-perspective-ref\n  description:\n    \"Example — GROQ features from content release (perspective-based doc\n    references)\"\n\n  featureArea: groq\n\n  # Perspective-based canonical doc reference.\n  #\n  # The perspective ID references a content release in the Sanity\n  # Content Lake. At evaluation time, the doc fetcher auto-discovers\n  # all articles versioned in this release and includes them as\n  # canonical documentation context.\n  #\n  # Release rE9TSJvR4 contains:\n  #   - \"GROQ-powered webhooks\" (webhooks)\n  #   - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n  #   - \"GROQ joins\" (groq-joins)\n  #\n  # You can combine perspective refs with explicit slug/path/id refs\n  # to include foundational published docs alongside release content.\n  # Here we add groq-data-types as a complementary published reference.\n  canonicalDocs:\n    - perspective: rE9TSJvR4\n      reason: \"All GROQ documentation updates in the test content release\"\n    - slug: groq-data-types\n      reason: \"GROQ data type reference (published, stable)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Using GROQ, demonstrate advanced query patterns including:\n      1. Joining data across document types using references\n      2. Filtering webhook payloads with GROQ projections\n      3. Using the query cheat sheet patterns for common operations\n      4. Working with different GROQ data types in filters\n      Provide working GROQ query examples for each pattern.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Demonstrates GROQ join syntax for cross-document queries\"\n        - \"Shows GROQ filter patterns for webhook configuration\"\n        - \"Includes practical query examples from cheat sheet patterns\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"All GROQ queries use valid syntax\"\n        - \"Reference joins use correct dereference operator (->)\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export const examplePerspectiveRefYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Perspective / content release doc references\n# ──────────────────────────────────────────────────────────────────────\n#\n# Demonstrates using `perspective` to reference all documentation\n# articles within a content release. This is the key capability for\n# evaluating NEW feature documentation before it's published.\n#\n# How it works:\n#   - A perspective ref is one-to-many: the doc fetcher queries the\n#     named release and expands it to ALL articles versioned within it.\n#   - Downstream consumers see the same flat DocContext[] regardless\n#     of how docs were resolved.\n#   - When the release is published, the perspective entry becomes a\n#     no-op (articles are now in published). Migrate to explicit path\n#     or slug refs at your convenience.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n#\n# @see docs/design-docs/canonical-doc-resolution.md\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-perspective-ref\n  description:\n    \"Example — GROQ features from content release (perspective-based doc\n    references)\"\n\n  featureArea: groq\n\n  # Perspective-based canonical doc reference.\n  #\n  # The perspective ID references a content release in the Sanity\n  # Content Lake. At evaluation time, the doc fetcher auto-discovers\n  # all articles versioned in this release and includes them as\n  # canonical documentation context.\n  #\n  # Release rE9TSJvR4 contains:\n  #   - \"GROQ-powered webhooks\" (webhooks)\n  #   - \"Query Cheat Sheet - GROQ\" (query-cheat-sheet)\n  #   - \"GROQ joins\" (groq-joins)\n  #\n  # You can combine perspective refs with explicit slug/path/id refs\n  # to include foundational published docs alongside release content.\n  # Here we add groq-data-types as a complementary published reference.\n  canonicalDocs:\n    - perspective: rE9TSJvR4\n      reason: \"All GROQ documentation updates in the test content release\"\n    - slug: groq-data-types\n      reason: \"GROQ data type reference (published, stable)\"\n\n  docCoverage: true\n\n  vars:\n    task: |\n      Using GROQ, demonstrate advanced query patterns including:\n      1. Joining data across document types using references\n      2. Filtering webhook payloads with GROQ projections\n      3. Using the query cheat sheet patterns for common operations\n      4. Working with different GROQ data types in filters\n      Provide working GROQ query examples for each pattern.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Demonstrates GROQ join syntax for cross-document queries\"\n        - \"Shows GROQ filter patterns for webhook configuration\"\n        - \"Includes practical query examples from cheat sheet patterns\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"All GROQ queries use valid syntax\"\n        - \"Reference joins use correct dereference operator (->)\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 /** Parsed task data for example-studio-custom-input (JSON-safe) */
 export const exampleStudioCustomInputData = [
     {
@@ -386,13 +386,13 @@ export const exampleStudioCustomInputData = [
         ],
         "baseline": {
             "enabled": true,
-            "rubric": "abbreviated"
+            "rubric": "full"
         },
         "status": "draft"
     }
 ];
 /** Raw YAML string for example-studio-custom-input (preserves comments) */
-export const exampleStudioCustomInputYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Custom input component in Sanity Studio\n# ──────────────────────────────────────────────────────────────────────\n#\n# This is a starter template — edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-studio-custom-input\n  description: \"Example — Custom input component in Sanity Studio\"\n\n  featureArea: studio\n\n  # Slug-based canonical doc references.\n  canonicalDocs:\n    - slug: custom-input-widgets\n      reason: \"Guide for building custom form inputs in Sanity Studio\"\n    - slug: form-components\n      reason: \"Form component API and customization patterns\"\n\n  docCoverage: true\n  referenceSolution: canonical/example-studio-custom-input.ts\n\n  vars:\n    task: |\n      Build a custom string input component for Sanity Studio that shows\n      a character count below the input field. The component should accept\n      a maxLength option from the field schema and display a warning when\n      the text exceeds the limit.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Implements a React component that renders a text input\"\n        - \"Displays a live character count\"\n        - \"Reads maxLength from schema options\"\n        - \"Shows a visual warning when limit is exceeded\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses the Sanity UI library for styling\"\n        - \"Calls onChange with patch operations\"\n\n  baseline:\n    enabled: true\n    rubric: abbreviated\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
+export const exampleStudioCustomInputYaml = "# ──────────────────────────────────────────────────────────────────────\n# Example Task: Custom input component in Sanity Studio\n# ──────────────────────────────────────────────────────────────────────\n#\n# This is a starter template — edit it for your own documentation.\n# Delete this file or replace it with your own tasks.\n#\n# This example task ships as a DRAFT so it does not run in production\n#   evaluations automatically. To activate it, change status to \"active\"\n# or remove the status line entirely (defaults to active).\n# ──────────────────────────────────────────────────────────────────────\n\n- id: example-studio-custom-input\n  description: \"Example — Custom input component in Sanity Studio\"\n\n  featureArea: studio\n\n  # Slug-based canonical doc references.\n  canonicalDocs:\n    - slug: custom-input-widgets\n      reason: \"Guide for building custom form inputs in Sanity Studio\"\n    - slug: form-components\n      reason: \"Form component API and customization patterns\"\n\n  docCoverage: true\n  referenceSolution: canonical/example-studio-custom-input.ts\n\n  vars:\n    task: |\n      Build a custom string input component for Sanity Studio that shows\n      a character count below the input field. The component should accept\n      a maxLength option from the field schema and display a warning when\n      the text exceeds the limit.\n    docs: \"\"\n\n  assert:\n    - type: llm-rubric\n      template: task-completion\n      criteria:\n        - \"Implements a React component that renders a text input\"\n        - \"Displays a live character count\"\n        - \"Reads maxLength from schema options\"\n        - \"Shows a visual warning when limit is exceeded\"\n\n    - type: llm-rubric\n      template: code-correctness\n      criteria:\n        - \"Uses the Sanity UI library for styling\"\n        - \"Calls onChange with patch operations\"\n\n  baseline:\n    enabled: true\n    rubric: full\n\n  # Example tasks ship as drafts so they don't run in production evals.\n  # Change to \"active\" (or remove this line) to activate.\n  status: draft\n";
 // ---------------------------------------------------------------------------
 // Aggregate task exports
 // ---------------------------------------------------------------------------

package/dist/_vendor/ailf-core/ports/task-source.d.ts

@@ -30,7 +30,7 @@ export type AssertionDefinition = TemplatedAssertion | ValueAssertion;
 export interface BaselineConfig {
     /** Whether to generate a baseline variant. Default: true */
     enabled?: boolean;
-    /** Rubric mode for baseline. Default: "abbreviated" */
+    /** Rubric mode for baseline. Default: "full" */
     rubric?: "abbreviated" | "full" | "none";
 }
 /**

package/dist/_vendor/ailf-core/schemas/pipeline.d.ts

@@ -25,21 +25,37 @@ export declare const RubricTemplateSchema: z.ZodObject<{
 }, z.core.$strip>;
 /** Inferred TypeScript type for a rubric template. */
 export type RubricTemplate = z.infer<typeof RubricTemplateSchema>;
+/**
+ * A named weight profile — maps dimension names to weights (must sum to 1.0).
+ * Each profile is a self-contained scoring formula used for a specific
+ * (mode, variant) pair.
+ */
+declare const WeightProfileSchema: z.ZodRecord<z.ZodString, z.ZodNumber>;
+/** Inferred type for a single weight profile. */
+export type WeightProfile = z.infer<typeof WeightProfileSchema>;
 /**
  * Schema for the full config/rubrics.yaml config file.
  *
- * Each dimension is scored on a uniform 0–100 scale. The `weights` section
- * defines how dimensions are combined into a composite score (must sum to 1.0).
+ * Each dimension is scored on a uniform 0–100 scale. Named scoring profiles
+ * define how dimensions are combined into composite scores. Mode-profile
+ * bindings declare which profile to use for each (mode, variant) pair.
+ *
+ * Supports both the new `profiles` format and the legacy flat `weights`
+ * format for backward compatibility.
+ *
+ * @see docs/design-docs/named-scoring-profiles.md
  */
 export declare const RubricConfigSchema: z.ZodObject<{
     footer: z.ZodString;
+    "mode-profiles": z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodString>>>;
+    profiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodNumber>>>;
     templates: z.ZodRecord<z.ZodString, z.ZodObject<{
         criteria_label: z.ZodOptional<z.ZodNullable<z.ZodString>>;
         dimension: z.ZodOptional<z.ZodString>;
         header: z.ZodString;
         scale: z.ZodArray<z.ZodString>;
     }, z.core.$strip>>;
-    weights: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    weights: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
 }, z.core.$strip>;
 /** Inferred TypeScript type for the rubrics config. */
 export type RubricConfig = z.infer<typeof RubricConfigSchema>;

package/dist/_vendor/ailf-core/schemas/pipeline.js

@@ -31,23 +31,54 @@ export const RubricTemplateSchema = z.object({
         .array(z.string().min(1))
         .min(1, "scale must have at least one entry"),
 });
+/**
+ * A named weight profile — maps dimension names to weights (must sum to 1.0).
+ * Each profile is a self-contained scoring formula used for a specific
+ * (mode, variant) pair.
+ */
+const WeightProfileSchema = z
+    .record(z.string(), z.number().min(0).max(1))
+    .refine((w) => {
+    const sum = Object.values(w).reduce((s, v) => s + v, 0);
+    return Math.abs(sum - 1.0) < 0.001;
+}, { message: "profile weights must sum to 1.0" });
+/**
+ * Mode-to-profile bindings — maps (mode, variant) pairs to profile names.
+ * Example: { baseline: { gold: "default", baseline: "output-only" } }
+ */
+const ModeProfilesSchema = z.record(z.string(), z.record(z.string(), z.string()));
 /**
  * Schema for the full config/rubrics.yaml config file.
  *
- * Each dimension is scored on a uniform 0–100 scale. The `weights` section
- * defines how dimensions are combined into a composite score (must sum to 1.0).
+ * Each dimension is scored on a uniform 0–100 scale. Named scoring profiles
+ * define how dimensions are combined into composite scores. Mode-profile
+ * bindings declare which profile to use for each (mode, variant) pair.
+ *
+ * Supports both the new `profiles` format and the legacy flat `weights`
+ * format for backward compatibility.
+ *
+ * @see docs/design-docs/named-scoring-profiles.md
  */
-export const RubricConfigSchema = z.object({
+export const RubricConfigSchema = z
+    .object({
     footer: z.string().min(1, "footer must be a non-empty string"),
+    "mode-profiles": ModeProfilesSchema.optional(),
+    profiles: z
+        .record(z.string(), WeightProfileSchema)
+        .refine((p) => "default" in p, {
+        message: "profiles must include a 'default' profile",
+    })
+        .optional(),
     templates: z
         .record(z.string(), RubricTemplateSchema)
         .refine((t) => Object.keys(t).length > 0, {
         message: "templates must have at least one entry",
     }),
-    weights: z.record(z.string(), z.number().min(0).max(1)).refine((w) => {
-        const sum = Object.values(w).reduce((s, v) => s + v, 0);
-        return Math.abs(sum - 1.0) < 0.001;
-    }, { message: "weights must sum to 1.0" }),
+    // Legacy: flat weight map. Treated as a single profile named "default".
+    weights: WeightProfileSchema.optional(),
+})
+    .refine((c) => c.profiles !== undefined || c.weights !== undefined, {
+    message: "rubrics.yaml must have either 'profiles' or 'weights'",
 });
 // ---------------------------------------------------------------------------
 // Feature registry schema — validates config/features.yaml (Phase 3c)

package/dist/adapters/task-sources/content-lake-task-source.js

@@ -131,7 +131,7 @@ function mapToTaskDefinition(raw) {
     // assertion types that aren't in the curated list). These bypass template
     // resolution and flow directly into the expanded Promptfoo test case as
     // value-based assertions. In baseline mode, buildBaselineAsserts() with
-    // "abbreviated" (the default) drops non-rubric assertions, so rawAssert
+    // "full" (the default) copies all assertions as-is, so rawAssert
     // entries only run in the gold variant — consistent with how regular
     // value-based assertions like `contains` or `regex` behave.
     const rawAssertions = (raw.rawAssert ?? [])

package/dist/pipeline/calculate-scores.d.ts

@@ -64,7 +64,7 @@ export interface RawTestResult {
  * @returns Record keyed by model ID, or null if only one model was used
  *          (per-model breakdown is redundant when there's only one model).
  */
-export declare function calculateScoresPerModel(resultsPath: string, weights: Record<string, number>): null | PerModelEntry[];
+export declare function calculateScoresPerModel(resultsPath: string, goldProfile: Record<string, number>, baselineProfile: Record<string, number>): null | PerModelEntry[];
 /**
  * Extract grader judgments (reason text + scores) from evaluation results.
  *
@@ -82,7 +82,7 @@ export declare function extractGraderJudgments(resultsPath: string): GraderJudgm
  *
  * Returns a record keyed by feature area with the composite actual score.
  */
-export declare function scoreAgenticResults(resultsPath: string, weights: Record<string, number>): Record<string, ActualScoreEntry>;
+export declare function scoreAgenticResults(resultsPath: string, profile: Record<string, number>): Record<string, ActualScoreEntry>;
 /**
  * Score agentic results broken down by model.
  *
@@ -90,7 +90,7 @@ export declare function scoreAgenticResults(resultsPath: string, weights: Record
  * producing a map of model → feature → ActualScoreEntry.
  * Used to enrich the per-model breakdown with actual scores in full mode.
  */
-export declare function scoreAgenticResultsPerModel(resultsPath: string, weights: Record<string, number>): Record<string, Record<string, ActualScoreEntry>>;
+export declare function scoreAgenticResultsPerModel(resultsPath: string, profile: Record<string, number>): Record<string, Record<string, ActualScoreEntry>>;
 /** Options for the calculate-scores main() function. */
 export interface CalculateScoresOptions {
     /** Allowed origins for source isolation reporting */

package/dist/pipeline/calculate-scores.js

@@ -8,8 +8,11 @@
  *   Code Correctness   (0–100)  — Is the code idiomatic and correct?
  *   Doc Coverage       (0–100)  — Did docs provide the needed info?
  *
- * Dimensions are combined into a weighted composite (0–100) using weights
- * from config/rubrics.yaml (default: Task×0.50 + Code×0.25 + Docs×0.25).
+ * Dimensions are combined into a weighted composite (0–100) using named
+ * scoring profiles from config/rubrics.yaml. Gold (with-docs) entries use
+ * the "default" profile; baseline (without-docs) entries use "output-only"
+ * which excludes doc-coverage (undefined without docs).
+ * See docs/design-docs/named-scoring-profiles.md.
  *
  * Additionally compares with-docs vs without-docs scores to calculate
  * the "Doc Lift" — how much documentation helps vs parametric knowledge.
@@ -30,6 +33,7 @@ import { calculateCost } from "../agent-observer/pricing.js";
 import { ConsoleLogger } from "../adapters/loggers/index.js";
 import { checkResultsExist } from "./checks.js";
 import { loadRubricTemplates } from "./expand-tasks.js";
+import { resolveProfile } from "./profile-resolution.js";
 import { loadSource } from "../sources.js";
 import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
 import { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
@@ -46,7 +50,7 @@ export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, par
  * @returns Record keyed by model ID, or null if only one model was used
  *          (per-model breakdown is redundant when there's only one model).
  */
-export function calculateScoresPerModel(resultsPath, weights) {
+export function calculateScoresPerModel(resultsPath, goldProfile, baselineProfile) {
     const results = readAndNormalizeResults(resultsPath);
     // Group results by provider
     const byModel = {};
@@ -66,7 +70,7 @@ export function calculateScoresPerModel(resultsPath, weights) {
     }
     const perModel = [];
     for (const [modelId, { label, results: modelResults }] of Object.entries(byModel)) {
-        const scores = scoreResults(modelResults, weights, modelId);
+        const scores = scoreResults(modelResults, goldProfile, baselineProfile, modelId);
         const totalTests = scores.reduce((s, sc) => s + sc.testCount, 0);
         const totalCost = scores.reduce((s, sc) => s + sc.totalCost, 0);
         const avgScore = scores.length > 0
@@ -318,9 +322,9 @@ function buildSourceVerification(root, source, verificationCtx) {
  * Calculate overall scores (all models combined).
  * This is the original scoring path — backward compatible.
  */
-function calculateScores(resultsPath, weights) {
+function calculateScores(resultsPath, goldProfile, baselineProfile) {
     const results = readAndNormalizeResults(resultsPath);
-    return scoreResults(results, weights);
+    return scoreResults(results, goldProfile, baselineProfile);
 }
 /**
  * Extracts agent behavior summary from a test result's metadata.
@@ -489,19 +493,73 @@ function readAndNormalizeResults(resultsPath, log) {
     }
     return valid;
 }
+/**
+ * Accumulate raw dimension scores across an array of test results.
+ * Dimension-agnostic: any dimension returned by classifyRubric() is tracked.
+ */
+function accumulateDimensions(tests) {
+    const dimensions = {};
+    let totalCost = 0;
+    for (const test of tests) {
+        totalCost += test.cost;
+        for (const comp of test.gradingResult.componentResults) {
+            if (comp.assertion?.type !== "llm-rubric")
+                continue;
+            const score = parseRubricScore(comp);
+            const kind = classifyRubric(comp);
+            if (kind) {
+                dimensions[kind] = (dimensions[kind] ?? 0) + score;
+            }
+        }
+    }
+    return { dimensions, totalCost };
+}
+/**
+ * Average accumulated dimension scores by a count.
+ * Returns a dimension → average score map.
+ */
+function averageDimensions(accumulated, count) {
+    const avg = {};
+    for (const [dim, total] of Object.entries(accumulated.dimensions)) {
+        avg[dim] = total / count;
+    }
+    return avg;
+}
+/**
+ * Compute a weighted composite score from dimension averages and a profile.
+ * Only dimensions present in the profile contribute to the composite.
+ * Dimensions not in the profile are ignored (e.g., doc-coverage on baseline).
+ *
+ * The profile maps camelCase dimension names (as returned by classifyRubric)
+ * to kebab-case keys (as used in rubrics.yaml). This function handles the
+ * mapping internally.
+ */
+function weightedComposite(dimensionAverages, profile) {
+    // Map profile keys (kebab-case: "task-completion") to classifyRubric
+    // output (camelCase: "taskCompletion")
+    const kebabToCamel = {
+        "code-correctness": "codeCorrectness",
+        "doc-coverage": "docCoverage",
+        "task-completion": "taskCompletion",
+    };
+    let total = 0;
+    for (const [profileKey, weight] of Object.entries(profile)) {
+        const dimKey = kebabToCamel[profileKey] ?? profileKey;
+        total += (dimensionAverages[dimKey] ?? 0) * weight;
+    }
+    return total;
+}
 /**
  * Core scoring logic: takes a pre-filtered array of TestResult and produces
  * FeatureScore[] grouped by feature area. This is the shared implementation
  * used by both the overall scoring and per-model scoring paths.
  *
  * @param results  Pre-filtered (valid) test results
- * @param weights  Dimension weights from rubrics.yaml
- * @param modelId  Optional model identifier to tag each FeatureScore
+ * @param goldProfile     Weight profile for gold (with-docs) entries
+ * @param baselineProfile Weight profile for baseline (without-docs) entries
+ * @param modelId         Optional model identifier to tag each FeatureScore
  */
-function scoreResults(results, weights, modelId) {
-    const wTask = weights["task-completion"] ?? 0.5;
-    const wCode = weights["code-correctness"] ?? 0.25;
-    const wDoc = weights["doc-coverage"] ?? 0.25;
+function scoreResults(results, goldProfile, baselineProfile, modelId) {
     // Group by feature + docs/no-docs
     const byFeature = {};
     for (const result of results) {
@@ -519,65 +577,35 @@ function scoreResults(results, weights, modelId) {
     }
     const scores = [];
     for (const [feature, data] of Object.entries(byFeature)) {
-        // --- With docs ---
-        let totalTask = 0;
-        let totalCode = 0;
-        let totalDoc = 0;
-        let featureCost = 0;
+        // --- With docs (gold / ceiling) ---
+        const goldDims = accumulateDimensions(data.withDocs);
+        let featureCost = goldDims.totalCost;
         const countWithDocs = data.withDocs.length || 1;
-        for (const test of data.withDocs) {
-            featureCost += test.cost;
-            for (const comp of test.gradingResult.componentResults) {
-                if (comp.assertion?.type !== "llm-rubric") {
-                    continue;
-                }
-                const score = parseRubricScore(comp);
-                const kind = classifyRubric(comp);
-                if (kind === "taskCompletion") {
-                    totalTask += score;
-                }
-                else if (kind === "codeCorrectness") {
-                    totalCode += score;
-                }
-                else if (kind === "docCoverage") {
-                    totalDoc += score;
-                }
-            }
-        }
-        // Per-dimension averages (each 0–100)
-        const avgTask = totalTask / countWithDocs;
-        const avgCode = totalCode / countWithDocs;
-        const avgDoc = totalDoc / countWithDocs;
-        // Weighted composite (0–100)
-        const withDocsTotal = avgTask * wTask + avgCode * wCode + avgDoc * wDoc;
-        // --- Without docs (baseline) ---
-        let baselineTotal = 0;
-        let baselineCount = 0;
-        for (const test of data.withoutDocs) {
-            featureCost += test.cost;
-            for (const comp of test.gradingResult.componentResults) {
-                if (comp.assertion?.type !== "llm-rubric") {
-                    continue;
-                }
-                baselineTotal += parseRubricScore(comp);
-                baselineCount++;
-            }
-        }
-        const withoutDocsScore = baselineCount > 0 ? baselineTotal / baselineCount : 0;
+        const avgGold = averageDimensions(goldDims, countWithDocs);
+        const withDocsTotal = weightedComposite(avgGold, goldProfile);
+        // --- Without docs (baseline / floor) ---
+        // Uses the baseline profile (e.g. "output-only") which may exclude
+        // dimensions like doc-coverage that are undefined without docs.
+        // See docs/design-docs/named-scoring-profiles.md.
+        const baselineDims = accumulateDimensions(data.withoutDocs);
+        featureCost += baselineDims.totalCost;
+        const countWithoutDocs = data.withoutDocs.length || 1;
+        const avgBaseline = averageDimensions(baselineDims, countWithoutDocs);
+        const withoutDocsScore = weightedComposite(avgBaseline, baselineProfile);
         const ceilingScore = Math.round(withDocsTotal);
         const floorScore = Math.round(withoutDocsScore);
         const docLift = ceilingScore - floorScore;
         const featureScore = {
             ceilingScore,
-            codeCorrectness: Math.round(avgCode),
-            docCoverage: Math.round(avgDoc),
+            codeCorrectness: Math.round(avgGold.codeCorrectness ?? 0),
+            docCoverage: Math.round(avgGold.docCoverage ?? 0),
             docLift,
             docQualityGap: 100 - ceilingScore,
             feature,
             floorScore,
             ...(modelId && { modelId }),
             negativeDocLift: docLift < 0,
-            taskCompletion: Math.round(avgTask),
+            taskCompletion: Math.round(avgGold.taskCompletion ?? 0),
             testCount: data.withDocs.length,
             totalCost: featureCost,
             totalScore: ceilingScore,
@@ -597,11 +625,8 @@ function scoreResults(results, weights, modelId) {
  * Returns a record keyed by feature area with the composite actual score.
  */
 // ActualScoreEntry — imported from @sanity/ailf-core via pipeline/types.js
-export function scoreAgenticResults(resultsPath, weights) {
+export function scoreAgenticResults(resultsPath, profile) {
     const results = readAndNormalizeResults(resultsPath);
-    const wTask = weights["task-completion"] ?? 0.5;
-    const wCode = weights["code-correctness"] ?? 0.25;
-    const wDoc = weights["doc-coverage"] ?? 0.25;
     // Group by feature area
     const byFeature = {};
     for (const result of results) {
@@ -613,37 +638,17 @@ export function scoreAgenticResults(resultsPath, weights) {
     }
     const entries = {};
     for (const [feature, featureResults] of Object.entries(byFeature)) {
-        let totalTask = 0;
-        let totalCode = 0;
-        let totalDoc = 0;
-        let featureCost = 0;
         const count = featureResults.length || 1;
-        for (const test of featureResults) {
-            featureCost += test.cost;
-            for (const comp of test.gradingResult.componentResults) {
-                if (comp.assertion?.type !== "llm-rubric")
-                    continue;
-                const score = parseRubricScore(comp);
-                const kind = classifyRubric(comp);
-                if (kind === "taskCompletion")
-                    totalTask += score;
-                else if (kind === "codeCorrectness")
-                    totalCode += score;
-                else if (kind === "docCoverage")
-                    totalDoc += score;
-            }
-        }
-        const avgTask = totalTask / count;
-        const avgCode = totalCode / count;
-        const avgDoc = totalDoc / count;
-        const actualScore = Math.round(avgTask * wTask + avgCode * wCode + avgDoc * wDoc);
+        const accumulated = accumulateDimensions(featureResults);
+        const avg = averageDimensions(accumulated, count);
+        const actualScore = Math.round(weightedComposite(avg, profile));
         entries[feature] = {
             actualScore,
-            codeCorrectness: Math.round(avgCode),
-            docCoverage: Math.round(avgDoc),
-            taskCompletion: Math.round(avgTask),
+            codeCorrectness: Math.round(avg.codeCorrectness ?? 0),
+            docCoverage: Math.round(avg.docCoverage ?? 0),
+            taskCompletion: Math.round(avg.taskCompletion ?? 0),
             testCount: featureResults.length,
-            totalCost: featureCost,
+            totalCost: accumulated.totalCost,
         };
     }
     return entries;
@@ -655,11 +660,8 @@ export function scoreAgenticResults(resultsPath, weights) {
  * producing a map of model → feature → ActualScoreEntry.
  * Used to enrich the per-model breakdown with actual scores in full mode.
  */
-export function scoreAgenticResultsPerModel(resultsPath, weights) {
+export function scoreAgenticResultsPerModel(resultsPath, profile) {
     const results = readAndNormalizeResults(resultsPath);
-    const wTask = weights["task-completion"] ?? 0.5;
-    const wCode = weights["code-correctness"] ?? 0.25;
-    const wDoc = weights["doc-coverage"] ?? 0.25;
     // Group by model, then feature
     const byModel = {};
     for (const result of results) {
@@ -675,37 +677,17 @@ export function scoreAgenticResultsPerModel(resultsPath, weights) {
     for (const [modelId, features] of Object.entries(byModel)) {
         perModel[modelId] = {};
         for (const [feature, featureResults] of Object.entries(features)) {
-            let totalTask = 0;
-            let totalCode = 0;
-            let totalDoc = 0;
-            let featureCost = 0;
             const count = featureResults.length || 1;
-            for (const test of featureResults) {
-                featureCost += test.cost;
-                for (const comp of test.gradingResult.componentResults) {
-                    if (comp.assertion?.type !== "llm-rubric")
-                        continue;
-                    const score = parseRubricScore(comp);
-                    const kind = classifyRubric(comp);
-                    if (kind === "taskCompletion")
-                        totalTask += score;
-                    else if (kind === "codeCorrectness")
-                        totalCode += score;
-                    else if (kind === "docCoverage")
-                        totalDoc += score;
-                }
-            }
-            const avgTask = totalTask / count;
-            const avgCode = totalCode / count;
-            const avgDoc = totalDoc / count;
-            const actualScore = Math.round(avgTask * wTask + avgCode * wCode + avgDoc * wDoc);
+            const accumulated = accumulateDimensions(featureResults);
+            const avg = averageDimensions(accumulated, count);
+            const actualScore = Math.round(weightedComposite(avg, profile));
             perModel[modelId][feature] = {
                 actualScore,
-                codeCorrectness: Math.round(avgCode),
-                docCoverage: Math.round(avgDoc),
-                taskCompletion: Math.round(avgTask),
+                codeCorrectness: Math.round(avg.codeCorrectness ?? 0),
+                docCoverage: Math.round(avg.docCoverage ?? 0),
+                taskCompletion: Math.round(avg.taskCompletion ?? 0),
                 testCount: featureResults.length,
-                totalCost: featureCost,
+                totalCost: accumulated.totalCost,
             };
         }
     }
@@ -760,10 +742,18 @@ export function calculateAndWriteScores(options) {
     if (source) {
         log.info(`Source: ${sourceName} (${source.baseUrl})`);
     }
-    // Load dimension weights from rubrics.yaml
+    // Load rubric config and resolve scoring profiles per variant.
+    // Gold (with-docs) entries use the "default" profile (3 dimensions).
+    // Baseline (without-docs) entries use "output-only" (2 dimensions,
+    // doc-coverage excluded). See docs/design-docs/named-scoring-profiles.md.
     const rubricConfig = loadRubricTemplates(ROOT);
-    log.debug("Loaded rubric weights", { weights: rubricConfig.weights });
-    const baselineScores = calculateScores(baselineResultsPath, rubricConfig.weights);
+    const goldProfile = resolveProfile("baseline", "gold", rubricConfig);
+    const baselineProfileWeights = resolveProfile("baseline", "baseline", rubricConfig);
+    log.debug("Loaded scoring profiles", {
+        gold: goldProfile,
+        baseline: baselineProfileWeights,
+    });
+    const baselineScores = calculateScores(baselineResultsPath, goldProfile, baselineProfileWeights);
     log.debug("Baseline scores calculated", {
         featureCount: baselineScores.length,
         features: baselineScores.map((s) => ({
@@ -773,7 +763,7 @@ export function calculateAndWriteScores(options) {
             docLift: s.docLift,
         })),
     });
-    const perModel = calculateScoresPerModel(baselineResultsPath, rubricConfig.weights);
+    const perModel = calculateScoresPerModel(baselineResultsPath, goldProfile, baselineProfileWeights);
     const urlRefs = aggregateUrlReferences(baselineResultsPath);
     const sourceVerification = buildSourceVerification(ROOT, source, {
         allowedOrigins: options.allowedOrigins,
@@ -788,7 +778,8 @@ export function calculateAndWriteScores(options) {
     let evaluationMode;
     if (mode === "full" && existsSync(agenticResultsPath)) {
         log.info(`\nReading agentic results from: ${agenticResultsPath}`);
-        const agenticScores = scoreAgenticResults(agenticResultsPath, rubricConfig.weights);
+        const agenticProfile = resolveProfile("agentic", "gold", rubricConfig);
+        const agenticScores = scoreAgenticResults(agenticResultsPath, agenticProfile);
         log.debug("Agentic scores calculated", {
             featureCount: Object.keys(agenticScores).length,
             features: Object.entries(agenticScores).map(([f, s]) => ({
@@ -801,7 +792,7 @@ export function calculateAndWriteScores(options) {
         evaluationMode = "full";
         // Merge agentic actual scores into the per-model breakdown
         if (perModel) {
-            const agenticPerModel = scoreAgenticResultsPerModel(agenticResultsPath, rubricConfig.weights);
+            const agenticPerModel = scoreAgenticResultsPerModel(agenticResultsPath, agenticProfile);
             for (const entry of perModel) {
                 const modelAgentic = agenticPerModel[entry.modelId];
                 if (modelAgentic) {

package/dist/pipeline/expand-tasks.d.ts

@@ -34,11 +34,11 @@
  *         value: ["client.fetch", "createClient"]
  *     baseline:
  *       enabled: true
- *       rubric: abbreviated
+ *       rubric: full
  *
  * Expands to:
  *   1. Gold entry — uses vars.docs as-is, resolves templates, appends doc-coverage
- *   2. Baseline entry — sets docs: "", adds transform, uses abbreviated rubric
+ *   2. Baseline entry — sets docs: "", uses full rubric (same assertions as gold)
  */
 import type { TaskDefinition } from "../_vendor/ailf-core/index.d.ts";
 import type { Logger } from "../_vendor/ailf-core/index.d.ts";
@@ -72,14 +72,14 @@ export interface LegacyTaskEntry {
 }
 /** A single task definition in the new format (input). */
 export interface SingleTaskDefinition {
-    /** Grading assertions (applied to gold; optionally abbreviated for baseline). */
+    /** Grading assertions (applied to both gold and baseline by default). */
     assert: AssertEntry[];
     /** Baseline generation options. */
     baseline?: {
         /** Whether to generate a baseline variant. Default: true. */
         enabled?: boolean;
         /** Rubric mode: 'full' copies all asserts, 'abbreviated' generates a
-         *  summary rubric, 'none' omits rubric asserts. Default: 'abbreviated'. */
+         *  summary rubric, 'none' omits rubric asserts. Default: 'full'. */
         rubric?: "abbreviated" | "full" | "none";
     };
     /** Human-readable description of what this task tests. */

package/dist/pipeline/expand-tasks.js

@@ -34,11 +34,11 @@
  *         value: ["client.fetch", "createClient"]
  *     baseline:
  *       enabled: true
- *       rubric: abbreviated
+ *       rubric: full
  *
  * Expands to:
  *   1. Gold entry — uses vars.docs as-is, resolves templates, appends doc-coverage
- *   2. Baseline entry — sets docs: "", adds transform, uses abbreviated rubric
+ *   2. Baseline entry — sets docs: "", uses full rubric (same assertions as gold)
  */
 import { existsSync, readFileSync, readdirSync } from "fs";
 import { resolve } from "path";
@@ -181,7 +181,7 @@ export function expandTask(task, rubricConfig, mode = "baseline") {
     // Restricted to the 'without-docs' prompt. Unless explicitly disabled.
     const baselineEnabled = task.baseline?.enabled !== false;
     if (baselineEnabled) {
-        const rubricMode = task.baseline?.rubric ?? "abbreviated";
+        const rubricMode = task.baseline?.rubric ?? "full";
         const baselineAsserts = buildBaselineAsserts(resolvedAsserts, rubricMode);
         entries.push({
             description: `${task.description} (baseline)`,

package/dist/pipeline/grader-consistency-runner.js

@@ -81,7 +81,8 @@ export function extractGradingJudgments(file) {
             continue;
         const description = result.testCase?.description ?? "unknown";
         const hasDocs = result.vars?.docs && result.vars.docs.trim().length > 0;
-        // Only grade "gold" (with-docs) tests — baseline tests have abbreviated rubrics
+        // Only grade "gold" (with-docs) tests — baseline tests use the output-only
+        // scoring profile and doc-coverage is undefined without docs
         if (!hasDocs)
             continue;
         const area = detectFeatureArea(description);

package/dist/pipeline/profile-resolution.d.ts

@@ -0,0 +1,39 @@
+/**
+ * pipeline/profile-resolution.ts
+ *
+ * Resolves the correct weight profile for a given (mode, variant) pair.
+ * The scoring engine calls this to determine which dimensions and weights
+ * apply to each test entry's composite score.
+ *
+ * Resolution order:
+ *   1. Explicit binding: mode-profiles.<mode>.<variant> → profile name
+ *   2. Fallback: the "default" profile
+ *
+ * Supports both the new `profiles` format and the legacy flat `weights`
+ * format (treated as a single profile named "default").
+ *
+ * @see docs/design-docs/named-scoring-profiles.md
+ */
+import type { RubricConfig, WeightProfile } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Resolve all named profiles from a RubricConfig, normalizing the legacy
+ * flat `weights` format into a `profiles` map with a single "default" entry.
+ *
+ * @returns A map of profile names → weight maps. Always includes "default".
+ */
+export declare function resolveProfiles(config: RubricConfig): Record<string, WeightProfile>;
+/**
+ * Resolve the weight profile for a specific (mode, variant) pair.
+ *
+ * @param mode    - Evaluation mode (e.g., "baseline", "agentic", "agent-task")
+ * @param variant - Entry variant: "gold" (with docs) or "baseline" (without docs)
+ * @param config  - Parsed rubrics.yaml config
+ * @returns The resolved weight profile (dimension → weight map)
+ *
+ * @example
+ *   resolveProfile("baseline", "gold", config)     // → default profile
+ *   resolveProfile("baseline", "baseline", config)  // → output-only profile
+ *   resolveProfile("agentic", "gold", config)       // → default profile
+ *   resolveProfile("unknown-mode", "gold", config)  // → default (fallback)
+ */
+export declare function resolveProfile(mode: string, variant: string, config: RubricConfig): WeightProfile;

package/dist/pipeline/profile-resolution.js

@@ -0,0 +1,69 @@
+/**
+ * pipeline/profile-resolution.ts
+ *
+ * Resolves the correct weight profile for a given (mode, variant) pair.
+ * The scoring engine calls this to determine which dimensions and weights
+ * apply to each test entry's composite score.
+ *
+ * Resolution order:
+ *   1. Explicit binding: mode-profiles.<mode>.<variant> → profile name
+ *   2. Fallback: the "default" profile
+ *
+ * Supports both the new `profiles` format and the legacy flat `weights`
+ * format (treated as a single profile named "default").
+ *
+ * @see docs/design-docs/named-scoring-profiles.md
+ */
+/**
+ * Resolve all named profiles from a RubricConfig, normalizing the legacy
+ * flat `weights` format into a `profiles` map with a single "default" entry.
+ *
+ * @returns A map of profile names → weight maps. Always includes "default".
+ */
+export function resolveProfiles(config) {
+    if (config.profiles) {
+        return config.profiles;
+    }
+    // Legacy format: flat weights → single "default" profile
+    if (config.weights) {
+        return { default: config.weights };
+    }
+    // Schema validation should prevent this, but be defensive
+    throw new Error("rubrics.yaml has neither 'profiles' nor 'weights' — cannot resolve scoring profiles");
+}
+/**
+ * Resolve the weight profile for a specific (mode, variant) pair.
+ *
+ * @param mode    - Evaluation mode (e.g., "baseline", "agentic", "agent-task")
+ * @param variant - Entry variant: "gold" (with docs) or "baseline" (without docs)
+ * @param config  - Parsed rubrics.yaml config
+ * @returns The resolved weight profile (dimension → weight map)
+ *
+ * @example
+ *   resolveProfile("baseline", "gold", config)     // → default profile
+ *   resolveProfile("baseline", "baseline", config)  // → output-only profile
+ *   resolveProfile("agentic", "gold", config)       // → default profile
+ *   resolveProfile("unknown-mode", "gold", config)  // → default (fallback)
+ */
+export function resolveProfile(mode, variant, config) {
+    const profiles = resolveProfiles(config);
+    const modeProfiles = config["mode-profiles"];
+    // Look up explicit binding: mode-profiles.<mode>.<variant> → profile name
+    const profileName = modeProfiles?.[mode]?.[variant];
+    if (profileName) {
+        const profile = profiles[profileName];
+        if (!profile) {
+            throw new Error(`mode-profiles.${mode}.${variant} references profile "${profileName}" ` +
+                `which does not exist. Available profiles: ${Object.keys(profiles).join(", ")}`);
+        }
+        return profile;
+    }
+    // Fall back to "default" profile
+    const defaultProfile = profiles["default"];
+    if (!defaultProfile) {
+        throw new Error(`No scoring profile found for mode="${mode}" variant="${variant}" ` +
+            `and no "default" profile exists. ` +
+            `Available profiles: ${Object.keys(profiles).join(", ")}`);
+    }
+    return defaultProfile;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "private": false,
   "publishConfig": {
     "access": "restricted"
@@ -40,8 +40,8 @@
     "@types/node": "^22.13.1",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
-    "@sanity/ailf-tasks": "0.1.4",
     "@sanity/ailf-core": "0.1.0",
+    "@sanity/ailf-tasks": "0.1.4",
     "@sanity/ailf-shared": "0.1.0"
   },
   "scripts": {