@sanity/ailf 2.0.0 → 2.0.1

package/tasks/.expanded.yaml

@@ -0,0 +1,565 @@
+# .expanded.yaml
+#
+# AUTO-GENERATED by compiler pipeline — do not edit directly.
+# Run: npx @sanity/ailf generate-configs
+
+- description: GROQ - Blog queries with filtering and pagination (gold)
+  vars:
+    task: |-
+      Write GROQ queries for a Sanity blog application:
+
+      1. Fetch all published blog posts ordered by publishedAt descending,
+         with a projection that includes: _id, title, slug (from slug.current),
+         publishedAt, excerpt, and the author's name (resolved from a reference)
+      2. Add pagination to return only the first 10 results
+      3. Fetch a single post by its slug parameter, including the full body
+         content and resolved author and category references
+      4. Fetch posts published after a specific date
+      5. Fetch posts that belong to a specific category (where categories
+         is an array of references)
+
+      Use @sanity/client with client.fetch() for all queries. Include
+      TypeScript types for the query results.
+    docs: file://contexts/canonical/groq-blog-queries.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - GROQ filter with _type == "post"
+        - Projection with aliased slug field ("slug": slug.current)
+        - Reference resolution with -> for author
+        - Ordering with | order(publishedAt desc)
+        - Slice/pagination syntax [0...10] or [0..9]
+        - Parameterized query with $slug for single post fetch
+        - Date filtering with dateTime() or string comparison
+        - Category filtering using references or array contains
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Valid GROQ syntax (proper filter brackets, projection braces)
+        - Uses @sanity/client createClient + client.fetch()
+        - Correct parameter passing syntax ($param)
+        - Proper reference dereference with ->
+        - No deprecated patterns
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains-any
+      value:
+        - client.fetch
+        - createClient
+      weight: 1
+    - type: contains-any
+      value:
+        - order(publishedAt
+        - order(_createdAt
+        - '| order('
+      weight: 1
+    - type: contains-any
+      value:
+        - '[0...10]'
+        - '[0..9]'
+        - '[0...'
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  prompts:
+    - with-docs
+- description: GROQ - Blog queries with filtering and pagination (baseline)
+  vars:
+    task: |-
+      Write GROQ queries for a Sanity blog application:
+
+      1. Fetch all published blog posts ordered by publishedAt descending,
+         with a projection that includes: _id, title, slug (from slug.current),
+         publishedAt, excerpt, and the author's name (resolved from a reference)
+      2. Add pagination to return only the first 10 results
+      3. Fetch a single post by its slug parameter, including the full body
+         content and resolved author and category references
+      4. Fetch posts published after a specific date
+      5. Fetch posts that belong to a specific category (where categories
+         is an array of references)
+
+      Use @sanity/client with client.fetch() for all queries. Include
+      TypeScript types for the query results.
+    docs: file://contexts/canonical/groq-blog-queries.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - GROQ filter with _type == "post"
+        - Projection with aliased slug field ("slug": slug.current)
+        - Reference resolution with -> for author
+        - Ordering with | order(publishedAt desc)
+        - Slice/pagination syntax [0...10] or [0..9]
+        - Parameterized query with $slug for single post fetch
+        - Date filtering with dateTime() or string comparison
+        - Category filtering using references or array contains
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Valid GROQ syntax (proper filter brackets, projection braces)
+        - Uses @sanity/client createClient + client.fetch()
+        - Correct parameter passing syntax ($param)
+        - Proper reference dereference with ->
+        - No deprecated patterns
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains-any
+      value:
+        - client.fetch
+        - createClient
+      weight: 1
+    - type: contains-any
+      value:
+        - order(publishedAt
+        - order(_createdAt
+        - '| order('
+      weight: 1
+    - type: contains-any
+      value:
+        - '[0...10]'
+        - '[0..9]'
+        - '[0...'
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  prompts:
+    - without-docs
+- description: GROQ - Joins and reference resolution (gold)
+  vars:
+    task: |-
+      Write GROQ queries that demonstrate join patterns in Sanity:
+
+      1. Follow a single reference to resolve an author's full profile
+         from a post (post.author -> author document with name, bio, image)
+      2. Resolve an array of category references from a post
+         (post.categories[]-> with title and slug)
+      3. Write a reverse reference query: given an author's ID, find all
+         posts by that author using a subquery and the parent scope operator (^)
+      4. Create a nested join: for each author, include their 5 most recent
+         posts as a nested array
+      5. Use the references() function to find all documents that reference
+         a specific document ID
+
+      Use @sanity/client with client.fetch(). Include TypeScript types.
+    docs: file://contexts/canonical/groq-joins-references.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - Single reference follow with -> operator
+        - Array reference resolution with []->
+        - Reverse reference / subquery using *[references(^._id)]
+        - Nested join pattern with parent scope (^)
+        - The references() function
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Correct -> dereference syntax
+        - Valid []-> array dereference
+        - Proper use of ^ parent scope operator
+        - Valid references() function usage
+        - No made-up syntax
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains
+      value: '->'
+      weight: 1
+    - type: contains-any
+      value:
+        - references(
+        - references(^
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  prompts:
+    - with-docs
+- description: GROQ - Joins and reference resolution (baseline)
+  vars:
+    task: |-
+      Write GROQ queries that demonstrate join patterns in Sanity:
+
+      1. Follow a single reference to resolve an author's full profile
+         from a post (post.author -> author document with name, bio, image)
+      2. Resolve an array of category references from a post
+         (post.categories[]-> with title and slug)
+      3. Write a reverse reference query: given an author's ID, find all
+         posts by that author using a subquery and the parent scope operator (^)
+      4. Create a nested join: for each author, include their 5 most recent
+         posts as a nested array
+      5. Use the references() function to find all documents that reference
+         a specific document ID
+
+      Use @sanity/client with client.fetch(). Include TypeScript types.
+    docs: file://contexts/canonical/groq-joins-references.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - Single reference follow with -> operator
+        - Array reference resolution with []->
+        - Reverse reference / subquery using *[references(^._id)]
+        - Nested join pattern with parent scope (^)
+        - The references() function
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Correct -> dereference syntax
+        - Valid []-> array dereference
+        - Proper use of ^ parent scope operator
+        - Valid references() function usage
+        - No made-up syntax
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains
+      value: '->'
+      weight: 1
+    - type: contains-any
+      value:
+        - references(
+        - references(^
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  prompts:
+    - without-docs
+- description: GROQ - Advanced filtering and projections (gold)
+  vars:
+    task: |-
+      Write GROQ queries demonstrating advanced filtering and projection patterns:
+
+      1. Use select() for conditional projections — return different fields
+         based on the document's _type (e.g., posts get excerpt, events get
+         date and venue)
+      2. Use coalesce() for fallback values — e.g., use seoTitle if it
+         exists, otherwise fall back to title
+      3. Use the match operator for full-text search in titles
+      4. Use count() to count documents matching a filter and to count
+         items within an array field
+      5. Use defined() to filter for documents that have a specific field set
+      6. Filter items within an array using [condition] syntax
+      7. Order results by multiple fields (e.g., featured status first,
+         then by publishedAt)
+
+      Use @sanity/client with client.fetch(). Include TypeScript types.
+    docs: file://contexts/canonical/groq-advanced-filtering.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - select() for conditional projections
+        - coalesce() for fallback values
+        - match operator for text search
+        - count() function usage
+        - defined() function for existence checks
+        - Array filtering with [condition]
+        - Multi-field ordering
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Valid select() syntax with => arrow notation
+        - Correct coalesce() usage
+        - Proper match operator usage (on text fields)
+        - Valid count() and defined() function calls
+        - Correct array filter syntax
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains-any
+      value:
+        - select(
+        - coalesce(
+      weight: 1
+    - type: contains-any
+      value:
+        - count(
+        - defined(
+      weight: 1
+    - type: contains-any
+      value:
+        - match
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  prompts:
+    - with-docs
+- description: GROQ - Advanced filtering and projections (baseline)
+  vars:
+    task: |-
+      Write GROQ queries demonstrating advanced filtering and projection patterns:
+
+      1. Use select() for conditional projections — return different fields
+         based on the document's _type (e.g., posts get excerpt, events get
+         date and venue)
+      2. Use coalesce() for fallback values — e.g., use seoTitle if it
+         exists, otherwise fall back to title
+      3. Use the match operator for full-text search in titles
+      4. Use count() to count documents matching a filter and to count
+         items within an array field
+      5. Use defined() to filter for documents that have a specific field set
+      6. Filter items within an array using [condition] syntax
+      7. Order results by multiple fields (e.g., featured status first,
+         then by publishedAt)
+
+      Use @sanity/client with client.fetch(). Include TypeScript types.
+    docs: file://contexts/canonical/groq-advanced-filtering.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - select() for conditional projections
+        - coalesce() for fallback values
+        - match operator for text search
+        - count() function usage
+        - defined() function for existence checks
+        - Array filtering with [condition]
+        - Multi-field ordering
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Valid select() syntax with => arrow notation
+        - Correct coalesce() usage
+        - Proper match operator usage (on text fields)
+        - Valid count() and defined() function calls
+        - Correct array filter syntax
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains-any
+      value:
+        - select(
+        - coalesce(
+      weight: 1
+    - type: contains-any
+      value:
+        - count(
+        - defined(
+      weight: 1
+    - type: contains-any
+      value:
+        - match
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  prompts:
+    - without-docs

package/tasks/knowledge-probe/define-type-api.task.ts

@@ -3,21 +3,30 @@
  *
  * Tests whether models know the current typed schema API
  * vs the legacy untyped format.
+ *
+ * Knowledge probes test the model's built-in knowledge WITHOUT providing documentation
+ * context (no `context.docs`). Contrast with "literacy" tasks which inject docs.
  */
 import { defineTask } from "@sanity/ailf-core"
 
 export default defineTask({
+  // "knowledge-probe" mode: no docs injected — measures what the model already knows
   mode: "knowledge-probe",
   id: "kp-define-type-api",
   title: "What is Sanity's defineType API?",
   description:
     "Explain how to define document schemas in Sanity using the defineType, " +
     "defineField, and defineArrayMember helper functions.",
+  // Used for score aggregation in reports and --area CLI filtering
   area: "studio",
+  // Metadata for reporting; does not affect evaluation behavior
   difficulty: "basic",
+  // Freeform labels for --tag CLI filtering
   tags: ["knowledge-probe", "studio", "schema"],
+  // Controls how the probe explores knowledge: "breadth-first" covers many topics, "depth-first" drills deep
   probeStrategy: "breadth-first",
   prompt: {
+    // Direct prompt text sent to the model (knowledge probes use text, literacy tasks use vars.task with a template)
     text:
       "Explain Sanity's schema definition API:\n\n" +
       "1. What is `defineType` and how do you use it?\n" +
@@ -34,6 +43,7 @@ export default defineTask({
   assertions: [
     { type: "contains", value: "defineType" },
     { type: "contains", value: "defineField" },
+    // Inline llm-rubric (value is the rubric text itself, unlike template+criteria in literacy tasks)
     {
       type: "llm-rubric",
       value:
@@ -41,6 +51,7 @@ export default defineTask({
         "Check that the code examples use the current API, not the legacy " +
         "untyped format. Penalize if the response uses the old pattern " +
         "without mentioning defineType.",
+      // weight: relative weight in the overall score (these two rubrics split evenly at 0.5 each)
       weight: 0.5,
     },
     {

package/tasks/knowledge-probe/groq-projections.task.ts

@@ -3,6 +3,8 @@
  *
  * Tests deep knowledge of Sanity's query language without
  * providing any documentation context.
+ *
+ * See define-type-api.task.ts for detailed explanations of knowledge-probe properties.
  */
 import { defineTask } from "@sanity/ailf-core"
 
@@ -18,6 +20,7 @@ export default defineTask({
   area: "groq",
   difficulty: "intermediate",
   tags: ["knowledge-probe", "groq", "syntax"],
+  // "depth-first" drills deep into one topic (vs "breadth-first" which covers many shallowly)
   probeStrategy: "depth-first",
   prompt: {
     text: