@sanity/ailf 2.0.0 → 2.0.1

package/dist/tasks/literacy/studio-setup.task.ts

@@ -0,0 +1,134 @@
+import { defineTask } from "../../_vendor/ailf-core/index.js"
+
+// See content-lake.task.ts for detailed explanations of common task properties
+export default [
+  defineTask({
+    id: "studio-blog-schema",
+    mode: "literacy",
+    title: "Studio Setup - Blog schema with posts, authors, categories",
+    description: "Studio Setup - Blog schema with posts, authors, categories",
+    area: "studio-setup",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "schemas-and-forms",
+          reason: "High-level overview of schemas and the form builder",
+        },
+        {
+          slug: "introduction-to-schemas",
+          reason: "Introduces schema concepts, defineType/defineField",
+        },
+        {
+          slug: "config-api-reference",
+          reason:
+            "Configuration API — defineConfig, plugins, schema registration",
+        },
+        {
+          slug: "reference-type",
+          reason: "Reference field type for author/category relationships",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/studio-setup/blog-schema.ts",
+    prompt: {
+      vars: {
+        task: `Set up a new Sanity Studio with a custom schema for a blog:
+
+1. Create document types for: posts, authors, categories
+2. Posts should have: title, slug, body (portable text), author reference, categories array
+3. Authors should have: name, bio, image
+4. Categories should have: title, description
+
+Include the schema definitions and sanity.config.ts setup.`,
+        docs: "file://contexts/canonical/studio-blog-schema.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Three document types (post, author, category)",
+          "Post with all required fields including portable text body",
+          "Reference from post to author",
+          "Array of references from post to categories",
+          "sanity.config.ts with schema registration",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Uses defineConfig, defineType, defineField (v3 syntax)",
+          "Does NOT use createSchema (deprecated v2)",
+          "Proper reference syntax with 'to' array",
+          "Correct portable text array definition",
+        ],
+      },
+      { type: "contains", value: "defineConfig", weight: 1 },
+      { type: "contains", value: "defineType", weight: 1 },
+      { type: "contains", value: "defineField", weight: 1 },
+      // not-contains: FAILS the assertion if the string appears — penalizes deprecated v2 schema API
+      { type: "not-contains", value: "createSchema", weight: 1 },
+      { type: "not-contains", value: "import Schema from", weight: 1 },
+    ],
+  }),
+
+  defineTask({
+    id: "studio-custom-tool",
+    mode: "literacy",
+    title: "Studio Setup - Custom tool in sidebar",
+    description: "Studio Setup - Custom tool in sidebar",
+    area: "studio-setup",
+    context: {
+      docs: [
+        {
+          slug: "studio-tools",
+          reason: "Overview of Studio tools system",
+        },
+        {
+          slug: "tool-api-reference",
+          reason: "Tool API — name, title, icon, component properties",
+        },
+        {
+          slug: "custom-studio-tool",
+          reason: "Step-by-step guide for creating a custom tool",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/studio-setup/custom-tool.tsx",
+    prompt: {
+      vars: {
+        task: `Add a custom tool to the Sanity Studio sidebar that displays
+a dashboard. The tool should:
+
+1. Appear in the studio navigation with a custom icon
+2. Have a title and name
+3. Render a React component showing a "Dashboard" heading
+
+Provide the tool definition and sanity.config.ts registration.`,
+        docs: "file://contexts/canonical/studio-custom-tool.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Tool object with name, title, icon, component",
+          "React component for the tool UI",
+          "Registration in sanity.config.ts tools array",
+        ],
+      },
+      { type: "contains", value: "tools", weight: 1 },
+      // javascript: custom JS assertion — `output` is the model's response string; must return boolean
+      {
+        type: "javascript",
+        value: `return output.includes('name:') &&
+        output.includes('component') &&
+        (output.includes('icon:') || output.includes('Icon'))`,
+      },
+    ],
+  }),
+]

package/dist/tasks/literacy/visual-editing.task.ts

@@ -0,0 +1,147 @@
+import { defineTask } from "../../_vendor/ailf-core/index.js"
+
+// See content-lake.task.ts for detailed explanations of common task properties
+export default [
+  defineTask({
+    id: "visual-editing-presentation",
+    mode: "literacy",
+    title: "Visual Editing - Presentation tool with click-to-edit",
+    description: "Visual Editing - Presentation tool with click-to-edit",
+    area: "visual-editing",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "configuring-the-presentation-tool",
+          reason: "Core presentationTool configuration and setup",
+        },
+        {
+          slug: "introduction-to-visual-editing",
+          reason: "Visual Editing concepts — stega, overlays, data attributes",
+        },
+        {
+          slug: "visual-editing-with-next-js-app-router",
+          reason: "Next.js App Router-specific visual editing guide",
+        },
+        {
+          slug: "stega",
+          reason: "Stega encoding for click-to-edit data attributes",
+        },
+      ],
+    },
+    referenceSolution:
+      "reference-solutions/visual-editing/presentation-nextjs.tsx",
+    prompt: {
+      vars: {
+        task: `Set up the Presentation tool with a Next.js 14 (App Router) frontend
+and implement click-to-edit functionality:
+
+1. Configure the Presentation tool in sanity.config.ts
+2. Set up the Next.js app to work with Visual Editing
+3. Implement data attributes so clicking content in the preview
+   opens the corresponding field in Studio
+
+Provide all necessary code for both Studio and Next.js sides.`,
+        docs: "file://contexts/canonical/visual-editing-presentation.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "presentationTool configured in sanity.config.ts",
+          "previewUrl or equivalent configured",
+          "Data attributes for click-to-edit (createDataAttribute or stega)",
+          "Next.js App Router patterns used correctly",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Uses @sanity/presentation (not deprecated packages)",
+          "Uses createDataAttribute or stega encoding correctly",
+          "Proper Next.js App Router patterns (not Pages Router)",
+          "No mixing of deprecated and current APIs",
+        ],
+      },
+      { type: "contains", value: "presentationTool", weight: 1 },
+      {
+        type: "contains-any",
+        value: [
+          "createDataAttribute",
+          "data-sanity",
+          "encodeDataAttribute",
+          "stega",
+        ],
+        weight: 1,
+      },
+      { type: "not-contains", value: "@sanity/preview-kit", weight: 1 },
+    ],
+  }),
+
+  defineTask({
+    id: "visual-editing-live-preview",
+    mode: "literacy",
+    title: "Visual Editing - Live preview with draft content",
+    description: "Visual Editing - Live preview with draft content",
+    area: "visual-editing",
+    context: {
+      docs: [
+        {
+          slug: "live-content-api",
+          reason: "Live Content API — defineLive, real-time subscriptions",
+        },
+        {
+          slug: "perspectives",
+          reason: "Draft vs published perspectives",
+        },
+        {
+          slug: "fetching-content-for-visual-editing",
+          reason: "Data fetching patterns for visual editing contexts",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/visual-editing/live-preview.tsx",
+    prompt: {
+      vars: {
+        task: `Implement live preview in a Next.js app that shows draft content
+from Sanity in real-time as editors make changes in the Studio.
+
+Requirements:
+- Use the Live Content API approach
+- Handle draft vs published perspectives correctly
+- Show real-time updates without page refresh
+
+Provide a complete implementation.`,
+        docs: "file://contexts/canonical/visual-editing-live-preview.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Live Content API usage (defineLive, useLiveQuery, or sanityFetch with live option)",
+          "Draft perspective configuration",
+          "Real-time subscription/update mechanism",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Modern API usage (not deprecated preview packages)",
+          "Proper perspective handling",
+          "Correct subscription lifecycle management",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: ["useLiveQuery", "defineLive", "live:", "perspective"],
+        weight: 1,
+      },
+    ],
+  }),
+]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "private": false,
   "publishConfig": {
     "access": "restricted"
@@ -31,28 +31,6 @@
     "canonical",
     "tasks"
   ],
-  "dependencies": {
-    "@google-cloud/bigquery": "^8.1.1",
-    "@inquirer/prompts": "^8.3.0",
-    "@modelcontextprotocol/sdk": "^1.29.0",
-    "@portabletext/markdown": "^1.0.0",
-    "@sanity/client": "^7.3.0",
-    "commander": "^14.0.3",
-    "dotenv": "^16.4.7",
-    "dotenv-cli": "^11.0.0",
-    "jiti": "^2.6.1",
-    "js-yaml": "^4.1.0",
-    "promptfoo": "^0.120.24",
-    "zod": "^4.3.6"
-  },
-  "devDependencies": {
-    "@types/js-yaml": "^4.0.9",
-    "@types/node": "^22.13.1",
-    "tsx": "^4.19.2",
-    "typescript": "^5.7.3",
-    "@sanity/ailf-shared": "0.1.0",
-    "@sanity/ailf-core": "0.1.0"
-  },
   "scripts": {
     "build": "tsc && tsx scripts/bundle-workspace-deps.ts",
     "generate-configs": "tsx src/cli.ts generate-configs",
@@ -80,5 +58,27 @@
     "discovery-report": "tsx src/cli.ts discovery-report",
     "webhook-server": "tsx src/cli.ts webhook-server",
     "weekly-digest": "tsx src/cli.ts weekly-digest"
+  },
+  "dependencies": {
+    "@google-cloud/bigquery": "^8.1.1",
+    "@inquirer/prompts": "^8.3.0",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "@portabletext/markdown": "^1.0.0",
+    "@sanity/client": "^7.3.0",
+    "commander": "^14.0.3",
+    "dotenv": "^16.4.7",
+    "dotenv-cli": "^11.0.0",
+    "jiti": "^2.6.1",
+    "js-yaml": "^4.1.0",
+    "promptfoo": "^0.120.24",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@sanity/ailf-core": "workspace:*",
+    "@sanity/ailf-shared": "workspace:*",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.13.1",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.3"
   }
-}
+}

package/tasks/.expanded.agentic.yaml

@@ -0,0 +1,280 @@
+# .expanded.agentic.yaml
+#
+# AUTO-GENERATED by compiler pipeline — do not edit directly.
+# Gold entries only (no baseline) for agentic evaluation mode.
+# Run: npx @sanity/ailf generate-configs
+
+- description: GROQ - Blog queries with filtering and pagination (gold)
+  vars:
+    task: |-
+      Write GROQ queries for a Sanity blog application:
+
+      1. Fetch all published blog posts ordered by publishedAt descending,
+         with a projection that includes: _id, title, slug (from slug.current),
+         publishedAt, excerpt, and the author's name (resolved from a reference)
+      2. Add pagination to return only the first 10 results
+      3. Fetch a single post by its slug parameter, including the full body
+         content and resolved author and category references
+      4. Fetch posts published after a specific date
+      5. Fetch posts that belong to a specific category (where categories
+         is an array of references)
+
+      Use @sanity/client with client.fetch() for all queries. Include
+      TypeScript types for the query results.
+    docs: file://contexts/canonical/groq-blog-queries.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - GROQ filter with _type == "post"
+        - Projection with aliased slug field ("slug": slug.current)
+        - Reference resolution with -> for author
+        - Ordering with | order(publishedAt desc)
+        - Slice/pagination syntax [0...10] or [0..9]
+        - Parameterized query with $slug for single post fetch
+        - Date filtering with dateTime() or string comparison
+        - Category filtering using references or array contains
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Valid GROQ syntax (proper filter brackets, projection braces)
+        - Uses @sanity/client createClient + client.fetch()
+        - Correct parameter passing syntax ($param)
+        - Proper reference dereference with ->
+        - No deprecated patterns
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains-any
+      value:
+        - client.fetch
+        - createClient
+      weight: 1
+    - type: contains-any
+      value:
+        - order(publishedAt
+        - order(_createdAt
+        - '| order('
+      weight: 1
+    - type: contains-any
+      value:
+        - '[0...10]'
+        - '[0..9]'
+        - '[0...'
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+- description: GROQ - Joins and reference resolution (gold)
+  vars:
+    task: |-
+      Write GROQ queries that demonstrate join patterns in Sanity:
+
+      1. Follow a single reference to resolve an author's full profile
+         from a post (post.author -> author document with name, bio, image)
+      2. Resolve an array of category references from a post
+         (post.categories[]-> with title and slug)
+      3. Write a reverse reference query: given an author's ID, find all
+         posts by that author using a subquery and the parent scope operator (^)
+      4. Create a nested join: for each author, include their 5 most recent
+         posts as a nested array
+      5. Use the references() function to find all documents that reference
+         a specific document ID
+
+      Use @sanity/client with client.fetch(). Include TypeScript types.
+    docs: file://contexts/canonical/groq-joins-references.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - Single reference follow with -> operator
+        - Array reference resolution with []->
+        - Reverse reference / subquery using *[references(^._id)]
+        - Nested join pattern with parent scope (^)
+        - The references() function
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Correct -> dereference syntax
+        - Valid []-> array dereference
+        - Proper use of ^ parent scope operator
+        - Valid references() function usage
+        - No made-up syntax
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains
+      value: '->'
+      weight: 1
+    - type: contains-any
+      value:
+        - references(
+        - references(^
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+- description: GROQ - Advanced filtering and projections (gold)
+  vars:
+    task: |-
+      Write GROQ queries demonstrating advanced filtering and projection patterns:
+
+      1. Use select() for conditional projections — return different fields
+         based on the document's _type (e.g., posts get excerpt, events get
+         date and venue)
+      2. Use coalesce() for fallback values — e.g., use seoTitle if it
+         exists, otherwise fall back to title
+      3. Use the match operator for full-text search in titles
+      4. Use count() to count documents matching a filter and to count
+         items within an array field
+      5. Use defined() to filter for documents that have a specific field set
+      6. Filter items within an array using [condition] syntax
+      7. Order results by multiple fields (e.g., featured status first,
+         then by publishedAt)
+
+      Use @sanity/client with client.fetch(). Include TypeScript types.
+    docs: file://contexts/canonical/groq-advanced-filtering.md
+    __featureArea: groq
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - select() for conditional projections
+        - coalesce() for fallback values
+        - match operator for text search
+        - count() function usage
+        - defined() function for existence checks
+        - Array filtering with [condition]
+        - Multi-field ordering
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: llm-rubric
+      value: |-
+        Score code correctness from 0 to 100:
+        - 0: Broken code, syntax errors, or deprecated APIs
+        - 30: Works but uses anti-patterns or inefficient approaches
+        - 50: Works but not idiomatic
+        - 80: Follows most best practices
+        - 100: Follows all best practices, idiomatic implementation
+
+        Check for:
+        - Valid select() syntax with => arrow notation
+        - Correct coalesce() usage
+        - Proper match operator usage (on text fields)
+        - Valid count() and defined() function calls
+        - Correct array filter syntax
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: code-correctness
+        maxScore: 100
+    - type: contains-any
+      value:
+        - select(
+        - coalesce(
+      weight: 1
+    - type: contains-any
+      value:
+        - count(
+        - defined(
+      weight: 1
+    - type: contains-any
+      value:
+        - match
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      provider: anthropic:messages:claude-opus-4-5-20251101
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100