@sanity/ailf 2.0.0 → 2.0.1

package/tasks/literacy/content-lake.task.ts

@@ -0,0 +1,181 @@
+import { defineTask } from "@sanity/ailf-core"
+
+export default [
+  defineTask({
+    id: "content-lake-mutations",
+    mode: "literacy",
+    title: "Content Lake - CRUD operations with @sanity/client",
+    description: "Content Lake - CRUD operations with @sanity/client",
+    // Used for score aggregation in reports and --area CLI filtering
+    area: "content-lake",
+    // Auto-generates an additional rubric checking if the response uses the provided docs
+    docCoverage: true,
+    context: {
+      // Sanity doc pages fetched and injected into the prompt as context; slug resolves via content lake
+      docs: [
+        {
+          slug: "js-client-mutations",
+          reason: "Creating and updating documents with @sanity/client",
+        },
+        {
+          slug: "js-client-transactions",
+          reason: "Atomic transactions — multiple mutations in one request",
+        },
+        {
+          slug: "js-client-deleting",
+          reason: "Deleting documents with @sanity/client",
+        },
+        {
+          slug: "http-patches",
+          reason: "Patch operations — set, unset, inc, insert, diffMatchPatch",
+        },
+      ],
+    },
+    // Path (relative to eval package root) to a gold-standard implementation the grader compares against
+    referenceSolution: "reference-solutions/content-lake/mutations.ts",
+    prompt: {
+      vars: {
+        // The instruction the model under evaluation sees (interpolated into the prompt template)
+        task: `Implement a content management service using @sanity/client that
+performs CRUD operations on Sanity documents.
+
+Requirements:
+1. Create a new document with typed fields
+2. Patch an existing document (set fields, unset fields, increment a counter)
+3. Delete a document by ID
+4. Use a transaction to atomically create multiple documents
+5. Include proper TypeScript types
+
+Provide a complete, reusable implementation.`,
+        // file:// URI resolved at runtime to a generated canonical context file (from npx @sanity/ailf fetch-docs)
+        docs: "file://contexts/canonical/content-lake-mutations.md",
+      },
+    },
+    // Grading criteria applied to the model's response; each assertion produces a pass/fail contributing to the task score
+    assertions: [
+      // llm-rubric with template: the grader model evaluates the response against these criteria
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Document creation with client.create() or client.createOrReplace()",
+          "Patch operations with set/unset/inc",
+          "Document deletion with client.delete()",
+          "Transaction with multiple mutations",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Correct @sanity/client API usage",
+          "Proper patch chaining syntax",
+          "Transaction commit pattern",
+          "TypeScript types for document shapes",
+        ],
+      },
+      // Deterministic string-match assertions — fast, cheap, no grader model needed
+      { type: "contains", value: "createClient", weight: 1 },
+      // contains-any: passes if ANY string in the array appears in the response
+      {
+        type: "contains-any",
+        value: [".create(", ".createOrReplace(", ".createIfNotExists("],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: [".patch(", ".set("],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: [".transaction(", "transaction"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: [".delete(", "client.delete"],
+        weight: 1,
+      },
+    ],
+  }),
+
+  defineTask({
+    id: "content-lake-realtime",
+    mode: "literacy",
+    title: "Content Lake - Real-time listeners with @sanity/client",
+    description: "Content Lake - Real-time listeners with @sanity/client",
+    area: "content-lake",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "js-client-realtime",
+          reason: "Listening to content updates with @sanity/client",
+        },
+        {
+          slug: "realtime-updates",
+          reason: "Real-time updates — listener protocol, event types",
+        },
+        {
+          slug: "js-client-getting-started",
+          reason: "Client setup — configuration, tokens, API version",
+        },
+        {
+          slug: "js-client-querying",
+          reason:
+            "Querying content — needed for initial data fetch before listening",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/content-lake/realtime.ts",
+    prompt: {
+      vars: {
+        task: `Implement real-time content synchronization using Sanity's listener API.
+
+Requirements:
+1. Set up a listener that watches for changes to documents of a specific type
+2. Handle mutation events (create, update, delete)
+3. Apply changes to a local in-memory cache
+4. Handle reconnection and error scenarios
+5. Provide a way to unsubscribe/clean up the listener
+
+Provide a complete implementation with TypeScript types.`,
+        docs: "file://contexts/canonical/content-lake-realtime.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Listener setup with GROQ filter for document type",
+          "Mutation event handling (create, update, delete differentiation)",
+          "Local cache synchronization",
+          "Cleanup/unsubscribe mechanism",
+          "Error or reconnection handling",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Correct client.listen() API usage",
+          "Proper GROQ filter syntax in listener",
+          "Subscription cleanup pattern (unsubscribe)",
+          "No deprecated listener APIs",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: [".listen(", "client.listen"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: ["subscribe", ".subscribe("],
+        weight: 1,
+      },
+    ],
+  }),
+]

package/tasks/literacy/frameworks.task.ts

@@ -1,5 +1,6 @@
 import { defineTask } from "@sanity/ailf-core"
 
+// See content-lake.task.ts for detailed explanations of common task properties
 export default [
   defineTask({
     id: "remix-integration",

package/tasks/literacy/functions.task.ts

@@ -1,5 +1,6 @@
 import { defineTask } from "@sanity/ailf-core"
 
+// See content-lake.task.ts for detailed explanations of common task properties
 export default [
   defineTask({
     id: "functions-webhook",

package/tasks/literacy/groq.task.ts

@@ -1,5 +1,6 @@
 import { defineTask } from "@sanity/ailf-core"
 
+// See content-lake.task.ts for detailed explanations of common task properties
 export default [
   defineTask({
     id: "groq-blog-queries",

package/tasks/literacy/image-handling.task.ts

@@ -0,0 +1,95 @@
+import { defineTask } from "@sanity/ailf-core"
+
+// See content-lake.task.ts for detailed explanations of common task properties
+export default [
+  defineTask({
+    id: "image-asset-pipeline",
+    mode: "literacy",
+    title:
+      "Image Handling - URL builder with transforms, hotspot/crop, responsive images",
+    description:
+      "Image Handling - URL builder with transforms, hotspot/crop, responsive images",
+    area: "image-handling",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "image-urls",
+          reason: "Image transformations — URL parameters, fit modes, format",
+        },
+        {
+          slug: "presenting-images",
+          reason: "Presenting images — best practices, responsive patterns",
+        },
+        {
+          slug: "image-type",
+          reason: "Image schema type — hotspot, crop, metadata fields",
+        },
+        {
+          slug: "js-client-assets",
+          reason: "Working with assets — upload, reference, URL generation",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/image-handling/asset-pipeline.tsx",
+    prompt: {
+      vars: {
+        task: `Build an image component for a Next.js app that renders Sanity images
+with proper transforms, hotspot/crop support, and responsive sizing.
+
+Requirements:
+1. Use @sanity/image-url to generate CDN URLs from image references
+2. Support hotspot and crop data from the image field
+3. Generate responsive srcSet with multiple widths
+4. Support format conversion (WebP with fallback)
+5. Create a reusable React component with TypeScript props
+
+Provide a complete implementation.`,
+        docs: "file://contexts/canonical/image-asset-pipeline.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "@sanity/image-url builder setup with project config",
+          "Hotspot and crop handling via .fit('crop') or equivalent",
+          "Responsive srcSet generation with multiple widths",
+          "WebP or auto format support",
+          "Reusable React component",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Correct @sanity/image-url API usage (imageUrlBuilder, .image(), .width(), etc.)",
+          "Proper hotspot/crop application (not ignoring crop metadata)",
+          "Valid srcSet syntax",
+          "No hardcoded CDN URLs (uses builder)",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: ["@sanity/image-url", "imageUrlBuilder", "image-url"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: ["srcSet", "srcset", "sizes"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: ["hotspot", "crop", ".fit("],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: [".width(", ".height(", ".size("],
+        weight: 1,
+      },
+    ],
+  }),
+]

package/tasks/literacy/nextjs-live.task.ts

@@ -1,12 +1,13 @@
 import { defineTask } from "@sanity/ailf-core"
 
+// See content-lake.task.ts for detailed explanations of common task properties
 export default [
   defineTask({
     id: "nextjs-app-router-integration",
     mode: "literacy",
     title: "Next.js - App Router integration with TypeScript",
     description: "Next.js - App Router integration with TypeScript",
-    area: "nextjs",
+    area: "nextjs-live",
     docCoverage: true,
     context: {
       docs: [

package/tasks/literacy/portable-text.task.ts

@@ -0,0 +1,169 @@
+import { defineTask } from "@sanity/ailf-core"
+
+// See content-lake.task.ts for detailed explanations of common task properties
+export default [
+  defineTask({
+    id: "portable-text-rendering",
+    mode: "literacy",
+    title: "Portable Text - Render rich text in React with custom components",
+    description:
+      "Portable Text - Render rich text in React with custom components",
+    area: "portable-text",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "previewing-block-content",
+          reason: "Previewing Portable Text — rendering in frontends",
+        },
+        {
+          slug: "customizing-block-content",
+          reason:
+            "Common PT editor patterns — custom blocks, marks, annotations",
+        },
+        {
+          slug: "presenting-images",
+          reason: "Presenting images — needed for inline image blocks in PT",
+        },
+        {
+          slug: "image-urls",
+          reason: "Image URL builder for transforming image references in PT",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/portable-text/rendering.tsx",
+    prompt: {
+      vars: {
+        task: `Render Portable Text content from Sanity in a React application using
+@portabletext/react.
+
+Requirements:
+1. Set up the PortableText component with custom components
+2. Handle image blocks with proper URL generation using @sanity/image-url
+3. Handle link marks (both internal and external links)
+4. Handle code blocks with syntax highlighting markup
+5. Provide TypeScript types for the component props
+
+Provide a complete, reusable implementation.`,
+        docs: "file://contexts/canonical/portable-text-rendering.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "PortableText component from @portabletext/react",
+          "Custom component map with image, link, and code block handlers",
+          "Image URL generation from Sanity image references",
+          "TypeScript types for props and component definitions",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Uses @portabletext/react (not deprecated @sanity/block-content-to-react)",
+          "Correct component map shape (types, marks, block, list, listItem)",
+          "Proper image URL builder usage with hotspot/crop support",
+          "No deprecated APIs or patterns",
+        ],
+      },
+      { type: "contains", value: "PortableText", weight: 1 },
+      {
+        type: "contains-any",
+        value: ["@portabletext/react", "portabletext/react"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: ["components", "serializers"],
+        weight: 1,
+      },
+      // not-contains: FAILS the assertion if the string appears — penalizes deprecated APIs
+      {
+        type: "not-contains",
+        value: "@sanity/block-content-to-react",
+        weight: 1,
+      },
+      { type: "not-contains", value: "BlockContent", weight: 1 },
+    ],
+  }),
+
+  defineTask({
+    id: "portable-text-custom-blocks",
+    mode: "literacy",
+    title: "Portable Text - Define and render custom block types",
+    description: "Portable Text - Define and render custom block types",
+    area: "portable-text",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "customizing-the-portable-text-editor",
+          reason: "Customizing the PT editor — adding custom block types",
+        },
+        {
+          slug: "customizing-block-content",
+          reason: "Common PT patterns — custom blocks, decorators, annotations",
+        },
+        {
+          slug: "portable-text-editor-configuration",
+          reason: "PT editor configuration — styles, lists, marks, of array",
+        },
+        {
+          slug: "previewing-block-content",
+          reason: "Previewing PT — rendering custom blocks in frontends",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/portable-text/custom-blocks.ts",
+    prompt: {
+      vars: {
+        task: `Define custom block types for a Portable Text field and render them
+in a React frontend.
+
+Requirements:
+1. Add a "code" block type with language and code fields
+2. Add a "callout" block type with tone (info/warning/error) and text
+3. Define the schema using defineArrayMember in a Sanity schema file
+4. Show how to render these custom blocks with @portabletext/react
+
+Provide both the schema definition and the frontend rendering code.`,
+        docs: "file://contexts/canonical/portable-text-custom-blocks.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Code block type with language and code fields",
+          "Callout block type with tone and text",
+          "Schema defined with defineArrayMember or equivalent",
+          "Frontend rendering for both custom block types",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Correct defineArrayMember usage for custom blocks",
+          'Proper "of" array structure in the PT field',
+          "Custom component map handles both block types",
+          "No deprecated schema patterns",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: ["defineArrayMember", "defineField"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: ["components", "serializers"],
+        weight: 1,
+      },
+    ],
+  }),
+]

package/tasks/literacy/studio-setup.task.ts

@@ -1,12 +1,13 @@
 import { defineTask } from "@sanity/ailf-core"
 
+// See content-lake.task.ts for detailed explanations of common task properties
 export default [
   defineTask({
     id: "studio-blog-schema",
     mode: "literacy",
     title: "Studio Setup - Blog schema with posts, authors, categories",
     description: "Studio Setup - Blog schema with posts, authors, categories",
-    area: "studio",
+    area: "studio-setup",
     docCoverage: true,
     context: {
       docs: [
@@ -68,6 +69,7 @@ Include the schema definitions and sanity.config.ts setup.`,
       { type: "contains", value: "defineConfig", weight: 1 },
       { type: "contains", value: "defineType", weight: 1 },
       { type: "contains", value: "defineField", weight: 1 },
+      // not-contains: FAILS the assertion if the string appears — penalizes deprecated v2 schema API
       { type: "not-contains", value: "createSchema", weight: 1 },
       { type: "not-contains", value: "import Schema from", weight: 1 },
     ],
@@ -78,7 +80,7 @@ Include the schema definitions and sanity.config.ts setup.`,
     mode: "literacy",
     title: "Studio Setup - Custom tool in sidebar",
     description: "Studio Setup - Custom tool in sidebar",
-    area: "studio",
+    area: "studio-setup",
     context: {
       docs: [
         {
@@ -120,6 +122,7 @@ Provide the tool definition and sanity.config.ts registration.`,
         ],
       },
       { type: "contains", value: "tools", weight: 1 },
+      // javascript: custom JS assertion — `output` is the model's response string; must return boolean
       {
         type: "javascript",
         value: `return output.includes('name:') &&

package/tasks/literacy/visual-editing.task.ts

@@ -1,5 +1,6 @@
 import { defineTask } from "@sanity/ailf-core"
 
+// See content-lake.task.ts for detailed explanations of common task properties
 export default [
   defineTask({
     id: "visual-editing-presentation",

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 Sanity.io
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/tasks/frameworks.yaml

@@ -1,98 +0,0 @@
-# tasks/frameworks.yaml
-#
-# Other Framework Integrations — Remix, Nuxt, etc.
-#
-# Each task is defined once. The pipeline auto-generates gold (with docs)
-# and baseline (without docs) variants from each definition.
-
-# ============================================================
-# TASK: Remix Integration
-# ============================================================
-- id: remix-integration
-  description: "Frameworks - Remix integration with data fetching"
-  doc_coverage: true
-  canonical_docs:
-    - slug: displaying-content-in-a-react-router-front-end
-      reason: "React Router front-end content display guide"
-    - slug: visual-editing-with-react-router
-      reason: "React Router / Remix integration with visual editing"
-    - slug: functions-js-client
-      reason: "Configuring @sanity/client for data fetching"
-  reference_solution: reference-solutions/frameworks/remix.tsx
-  vars:
-    task: |
-      Integrate Sanity into a Remix application:
-
-      1. Set up the Sanity client
-      2. Create a loader that fetches blog posts using GROQ
-      3. Build a route component that renders the fetched posts
-      4. Handle loading and error states properly
-
-      Provide all necessary files for a working Remix + Sanity integration.
-    docs: file://contexts/canonical/remix-integration.md
-  assert:
-    - type: llm-rubric
-      template: task-completion
-      criteria:
-        - Sanity client configuration
-        - Remix loader function with GROQ query
-        - Route component using useLoaderData
-        - Proper typing
-
-    - type: llm-rubric
-      template: code-correctness
-      criteria:
-        - Modern Remix patterns (v2 conventions)
-        - Proper loader/component separation
-        - Valid GROQ queries
-        - No deprecated APIs
-
-    - type: contains-any
-      value:
-        - "useLoaderData"
-        - "loader"
-      weight: 1
-
-# ============================================================
-# TASK: Nuxt 3 Integration
-# ============================================================
-- id: nuxt-integration
-  description: "Frameworks - Nuxt 4 integration"
-  doc_coverage: true
-  canonical_docs:
-    - slug: displaying-content-in-nuxt-js
-      reason: "Nuxt.js front-end content display guide"
-    - slug: visual-editing-with-nuxt
-      reason: "Nuxt visual editing integration"
-  reference_solution: reference-solutions/frameworks/nuxt.ts
-  vars:
-    task: |
-      Integrate Sanity into a Nuxt 4 application:
-
-      1. Install and configure the @nuxtjs/sanity module
-      2. Create a page that fetches and displays blog posts
-      3. Use Nuxt composables for data fetching
-
-      Provide all necessary configuration and component code.
-    docs: file://contexts/canonical/nuxt-integration.md
-  assert:
-    - type: llm-rubric
-      template: task-completion
-      criteria:
-        - "@nuxtjs/sanity module setup in nuxt.config.ts"
-        - Page component using Nuxt data fetching composables
-        - Sanity GROQ query
-
-    - type: llm-rubric
-      template: code-correctness
-      criteria:
-        - Nuxt 3 module configuration syntax
-        - Uses useSanityQuery or equivalent composable
-        - Proper Nuxt 3 patterns (not Nuxt 2)
-
-    - type: contains-any
-      value:
-        - "@nuxtjs/sanity"
-        - "useSanityQuery"
-        - "sanity:"
-      weight: 1