@sanity/ailf 0.5.0 → 1.0.0

package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Example agent harness task definitions — demonstrates the agent-harness mode.
+ *
+ * Three tasks of increasing complexity:
+ * 1. Scaffold a project from template
+ * 2. Modify existing code following docs
+ * 3. Multi-file refactoring task
+ */
+import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const scaffoldProjectTask: AgentHarnessTaskDefinition;
+export declare const modifyCodeTask: AgentHarnessTaskDefinition;
+export declare const multiFileRefactorTask: AgentHarnessTaskDefinition;
+/** All example agent harness tasks */
+export declare const allAgentHarnessExampleTasks: AgentHarnessTaskDefinition[];

package/dist/pipeline/compiler/mode-handlers/__fixtures__/agent-harness-example-tasks.js

@@ -0,0 +1,152 @@
+/**
+ * Example agent harness task definitions — demonstrates the agent-harness mode.
+ *
+ * Three tasks of increasing complexity:
+ * 1. Scaffold a project from template
+ * 2. Modify existing code following docs
+ * 3. Multi-file refactoring task
+ */
+// ---------------------------------------------------------------------------
+// Task 1: Scaffold a project from template
+// ---------------------------------------------------------------------------
+export const scaffoldProjectTask = {
+    mode: "agent-harness",
+    id: "agent-scaffold-project",
+    title: "Scaffold a Sanity Studio Project",
+    description: "The agent should create a new Sanity Studio project following the " +
+        "official documentation. It should use the CLI to initialize the project, " +
+        "configure sanity.config.ts, and set up basic schema types.",
+    area: "studio",
+    difficulty: "basic",
+    tags: ["agent", "scaffold", "studio"],
+    sandbox: {
+        type: "tempdir",
+    },
+    tools: ["coding"],
+    fixtures: ["file://fixtures/package-scaffold.json"],
+    prompt: {
+        text: "Create a new Sanity Studio project with the following requirements:\n" +
+            "1. Initialize with `npm create sanity@latest`\n" +
+            "2. Configure sanity.config.ts with project ID 'test-project' and dataset 'production'\n" +
+            "3. Create a 'post' schema type with title, slug, body, and author fields\n" +
+            "4. Ensure the project builds without errors",
+        vars: {
+            task: "Scaffold a Sanity Studio project with a post schema type. " +
+                "The project should build cleanly.",
+        },
+    },
+    assertions: [
+        { type: "file-exists", value: "sanity.config.ts" },
+        { type: "file-exists", value: "schemaTypes/post.ts" },
+        {
+            type: "file-contains",
+            value: { path: "sanity.config.ts", content: "test-project" },
+        },
+        { type: "command-succeeds", value: "npx tsc --noEmit" },
+    ],
+    options: {
+        timeout: 120_000,
+    },
+};
+// ---------------------------------------------------------------------------
+// Task 2: Modify existing code following docs
+// ---------------------------------------------------------------------------
+export const modifyCodeTask = {
+    mode: "agent-harness",
+    id: "agent-modify-schema",
+    title: "Add Document Actions Following Docs",
+    description: "Given an existing Studio project, the agent should add custom document " +
+        "actions following the Sanity documentation. The starting codebase has a " +
+        "basic Studio setup; the agent needs to create a custom publish action.",
+    area: "studio",
+    difficulty: "intermediate",
+    tags: ["agent", "modify", "document-actions"],
+    sandbox: {
+        type: "tempdir",
+    },
+    tools: ["coding"],
+    fixtures: ["file://fixtures/existing-studio-project/"],
+    prompt: {
+        text: "In the existing Sanity Studio project, add a custom document action " +
+            "that logs a message before publishing. Follow the Sanity docs for " +
+            "custom document actions.",
+        vars: {
+            task: "Add a custom document action that wraps the default publish action " +
+                "and logs 'Publishing document: <title>' before executing.",
+        },
+    },
+    assertions: [
+        { type: "file-exists", value: "actions/logPublishAction.ts" },
+        {
+            type: "file-contains",
+            value: {
+                path: "actions/logPublishAction.ts",
+                content: "useDocumentOperation",
+            },
+        },
+        {
+            type: "file-contains",
+            value: {
+                path: "sanity.config.ts",
+                content: "document.actions",
+            },
+        },
+    ],
+    options: {
+        timeout: 90_000,
+    },
+};
+// ---------------------------------------------------------------------------
+// Task 3: Multi-file refactoring task
+// ---------------------------------------------------------------------------
+export const multiFileRefactorTask = {
+    mode: "agent-harness",
+    id: "agent-migrate-api",
+    title: "Migrate from Client v5 to v6 API",
+    description: "The agent should migrate a multi-file project from the Sanity client v5 " +
+        "API to the v6 API, updating import paths, method calls, and configuration " +
+        "patterns across all files.",
+    area: "mutations",
+    difficulty: "advanced",
+    tags: ["agent", "migration", "refactoring"],
+    sandbox: {
+        type: "docker",
+        image: "node:22-slim",
+        limits: {
+            cpus: 2,
+            memoryBytes: 2_147_483_648, // 2GB
+            networkAccess: false,
+        },
+    },
+    tools: ["coding", "WebSearch"],
+    fixtures: ["file://fixtures/v5-project/"],
+    prompt: {
+        text: "Migrate this project from @sanity/client v5 to v6. Update all:\n" +
+            "1. Import statements (createClient instead of SanityClient)\n" +
+            "2. Client configuration (new config shape)\n" +
+            "3. Query method calls (fetch → client.fetch with new signature)\n" +
+            "4. Mutation helpers (create/patch/delete API changes)\n" +
+            "Ensure the project compiles after migration.",
+        vars: {
+            task: "Migrate the codebase from @sanity/client v5 to v6, " +
+                "updating all files. Project must compile cleanly after migration.",
+        },
+    },
+    assertions: [
+        {
+            type: "file-contains",
+            value: { path: "lib/sanity.ts", content: "createClient" },
+        },
+        { type: "command-succeeds", value: "npx tsc --noEmit" },
+        { type: "diff-matches", value: "createClient" },
+    ],
+    options: {
+        timeout: 180_000,
+    },
+};
+/** All example agent harness tasks */
+export const allAgentHarnessExampleTasks = [
+    scaffoldProjectTask,
+    modifyCodeTask,
+    multiFileRefactorTask,
+];

package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Example knowledge probe task definitions — demonstrates the knowledge-probe mode.
+ *
+ * These tasks measure raw model knowledge without documentation context.
+ * They answer: "What does this model know about X without any help?"
+ *
+ * Three tasks of increasing breadth:
+ * 1. GROQ projection syntax — deep, specific query language knowledge
+ * 2. defineType API — recent API knowledge + currency dimension
+ * 3. Sanity vs Contentful — broad ecosystem understanding
+ */
+import type { KnowledgeProbeTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+/**
+ * Tests deep knowledge of Sanity's query language.
+ * Models should demonstrate understanding of projection syntax,
+ * coercion, dereferencing, and the slice operator.
+ */
+export declare const groqProjectionTask: KnowledgeProbeTaskDefinition;
+/**
+ * Tests knowledge of the schema definition API, including recent changes.
+ * The currency dimension is key here — models should know about defineType,
+ * defineField, and the typed schema helpers, not the legacy untyped API.
+ */
+export declare const defineTypeApiTask: KnowledgeProbeTaskDefinition;
+/**
+ * Tests broader ecosystem knowledge and the ability to articulate
+ * differences between Sanity and a major competitor. This probes
+ * conceptual understanding, not just API syntax.
+ */
+export declare const ecosystemComparisonTask: KnowledgeProbeTaskDefinition;
+/** All example knowledge probe tasks */
+export declare const allKnowledgeProbeExampleTasks: KnowledgeProbeTaskDefinition[];

package/dist/pipeline/compiler/mode-handlers/__fixtures__/knowledge-probe-example-tasks.js

@@ -0,0 +1,176 @@
+/**
+ * Example knowledge probe task definitions — demonstrates the knowledge-probe mode.
+ *
+ * These tasks measure raw model knowledge without documentation context.
+ * They answer: "What does this model know about X without any help?"
+ *
+ * Three tasks of increasing breadth:
+ * 1. GROQ projection syntax — deep, specific query language knowledge
+ * 2. defineType API — recent API knowledge + currency dimension
+ * 3. Sanity vs Contentful — broad ecosystem understanding
+ */
+// ---------------------------------------------------------------------------
+// Task 1: GROQ projection syntax
+// ---------------------------------------------------------------------------
+/**
+ * Tests deep knowledge of Sanity's query language.
+ * Models should demonstrate understanding of projection syntax,
+ * coercion, dereferencing, and the slice operator.
+ */
+export const groqProjectionTask = {
+    mode: "knowledge-probe",
+    id: "kp-groq-projections",
+    title: "Explain GROQ Projection Syntax",
+    description: "Explain GROQ's projection syntax in detail, including: " +
+        "object projections ({}), array projections, spread operator (...), " +
+        "computed field names, the dereference operator (->), and " +
+        "conditional projections using select().",
+    area: "groq",
+    difficulty: "intermediate",
+    tags: ["knowledge-probe", "groq", "syntax"],
+    probeStrategy: "depth-first",
+    prompt: {
+        text: "Explain GROQ's projection syntax in Sanity. Cover these topics:\n\n" +
+            "1. Basic object projections with `{}`\n" +
+            "2. Nested projections and the spread operator `...`\n" +
+            "3. Computed field names\n" +
+            "4. The dereference operator `->` for following references\n" +
+            "5. Array slicing with `[0..5]` and `[0...5]`\n" +
+            "6. Conditional projections using `select()`\n\n" +
+            "Provide working code examples for each.",
+        vars: {
+            task: "Explain GROQ projection syntax with working code examples " +
+                "covering projections, spread, dereference, slicing, and select().",
+        },
+    },
+    assertions: [
+        { type: "contains", value: "->" },
+        { type: "contains", value: "select(" },
+        {
+            type: "llm-rubric",
+            value: "The response should demonstrate accurate knowledge of GROQ " +
+                "projection syntax with working code examples. Check that the " +
+                "dereference operator, spread syntax, and select() are correctly " +
+                "explained with valid GROQ code.",
+            weight: 0.6,
+        },
+        {
+            type: "llm-rubric",
+            value: "Evaluate whether the response reflects current GROQ syntax " +
+                "(post-2023). Check for deprecated patterns or outdated " +
+                "recommendations.",
+            weight: 0.4,
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 2: defineType API
+// ---------------------------------------------------------------------------
+/**
+ * Tests knowledge of the schema definition API, including recent changes.
+ * The currency dimension is key here — models should know about defineType,
+ * defineField, and the typed schema helpers, not the legacy untyped API.
+ */
+export const defineTypeApiTask = {
+    mode: "knowledge-probe",
+    id: "kp-define-type-api",
+    title: "What is Sanity's defineType API?",
+    description: "Explain how to define document schemas in Sanity using the defineType, " +
+        "defineField, and defineArrayMember helper functions. Include the " +
+        "motivation for typed schemas and common patterns.",
+    area: "studio",
+    difficulty: "basic",
+    tags: ["knowledge-probe", "studio", "schema"],
+    probeStrategy: "breadth-first",
+    prompt: {
+        text: "Explain Sanity's schema definition API:\n\n" +
+            "1. What is `defineType` and how do you use it?\n" +
+            "2. What are `defineField` and `defineArrayMember`?\n" +
+            "3. Why were these typed helpers introduced? What did they replace?\n" +
+            "4. Show a complete example of a document schema with various field types\n" +
+            "5. How do you add validation rules using the typed API?",
+        vars: {
+            task: "Explain Sanity's defineType/defineField schema API with examples, " +
+                "motivation, and validation rules.",
+        },
+    },
+    assertions: [
+        { type: "contains", value: "defineType" },
+        { type: "contains", value: "defineField" },
+        {
+            type: "llm-rubric",
+            value: "The response should accurately explain the typed schema helpers " +
+                "(defineType, defineField, defineArrayMember). Check that the code " +
+                "examples use the current API, not the legacy untyped format. " +
+                "Penalize if the response uses the old `{ type: 'document', fields: [...] }` " +
+                "pattern without mentioning defineType.",
+            weight: 0.5,
+        },
+        {
+            type: "llm-rubric",
+            value: "Evaluate currency: does the response know about defineType " +
+                "(introduced in Sanity v3)? Does it mention TypeScript type " +
+                "inference benefits? Does it use current import paths?",
+            weight: 0.5,
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 3: Sanity vs Contentful comparison
+// ---------------------------------------------------------------------------
+/**
+ * Tests broader ecosystem knowledge and the ability to articulate
+ * differences between Sanity and a major competitor. This probes
+ * conceptual understanding, not just API syntax.
+ */
+export const ecosystemComparisonTask = {
+    mode: "knowledge-probe",
+    id: "kp-sanity-vs-contentful",
+    title: "Compare Sanity and Contentful",
+    description: "Compare Sanity and Contentful as headless CMS platforms. " +
+        "Cover architecture, content modeling, querying, pricing, " +
+        "and developer experience differences.",
+    area: "general",
+    difficulty: "advanced",
+    tags: ["knowledge-probe", "ecosystem", "comparison"],
+    probeStrategy: "breadth-first",
+    prompt: {
+        text: "Compare Sanity and Contentful as headless CMS platforms. Address:\n\n" +
+            "1. Architecture differences (real-time vs CDN-based)\n" +
+            "2. Content modeling approaches\n" +
+            "3. Query languages (GROQ vs GraphQL)\n" +
+            "4. Developer experience and customization\n" +
+            "5. Pricing models\n" +
+            "6. When would you choose one over the other?",
+        vars: {
+            task: "Compare Sanity and Contentful across architecture, content modeling, " +
+                "querying, DX, pricing, and use case fit.",
+        },
+    },
+    assertions: [
+        { type: "contains-any", value: ["GROQ", "groq"] },
+        { type: "contains-any", value: ["GraphQL", "graphql"] },
+        {
+            type: "llm-rubric",
+            value: "The comparison should be factually accurate about both platforms. " +
+                "Check that key architectural differences are correctly stated: " +
+                "Sanity uses a real-time document store with GROQ; Contentful " +
+                "uses a CDN-based content infrastructure with GraphQL/REST. " +
+                "Penalize incorrect claims about either platform.",
+            weight: 0.5,
+        },
+        {
+            type: "llm-rubric",
+            value: "Evaluate completeness: does the response cover all requested " +
+                "areas (architecture, modeling, querying, DX, pricing, use cases)? " +
+                "Are the trade-offs balanced and fair?",
+            weight: 0.5,
+        },
+    ],
+};
+/** All example knowledge probe tasks */
+export const allKnowledgeProbeExampleTasks = [
+    groqProjectionTask,
+    defineTypeApiTask,
+    ecosystemComparisonTask,
+];

package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Example MCP server task definitions — Sanity MCP Server reference.
+ *
+ * These are reference implementations showing how to author MCP server
+ * evaluation tasks. They model the real Sanity MCP Server (mcp.sanity.io)
+ * and its actual tool names. They serve as:
+ *
+ * 1. Documentation — developers learn the API by reading examples
+ * 2. Test fixtures — the compiler tests compile these and verify output
+ * 3. Validation — proves the compiler handles real-world task shapes
+ * 4. Reference — shows all transport types, auth patterns, and assertion styles
+ *
+ * NOTE: These are NOT executable without a valid SANITY_MCP_AUTH_TOKEN.
+ * They exist purely for compilation testing and as authoring examples.
+ * For real evaluation, write tasks in an external repo's .ailf/tasks/
+ * directory or a standalone config file.
+ *
+ * @see https://www.sanity.io/docs/ai/mcp-server — Sanity MCP Server docs
+ * @see https://www.promptfoo.dev/docs/providers/mcp/ — Promptfoo MCP provider
+ */
+import type { MCPServerTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+/**
+ * Tests that a model can construct a valid GROQ query and use the
+ * query_documents tool to fetch results from a Sanity dataset.
+ */
+export declare const queryDocumentsTask: MCPServerTaskDefinition;
+/**
+ * Tests that a model can retrieve and interpret schema information
+ * from the MCP server.
+ */
+export declare const inspectSchemaTask: MCPServerTaskDefinition;
+/**
+ * Tests a multi-step document lifecycle workflow using multiple tools
+ * across a multi-turn conversation. This is the most complex example,
+ * demonstrating tool chaining and verification patterns.
+ */
+export declare const createAndPublishTask: MCPServerTaskDefinition;
+/**
+ * Tests that a model can discover embeddings indices and perform
+ * semantic search against them.
+ */
+export declare const semanticSearchTask: MCPServerTaskDefinition;
+/**
+ * Demonstrates a stdio-based MCP server (local process) for contrast
+ * with the remote streamable-http tasks above.
+ */
+export declare const stdioServerTask: MCPServerTaskDefinition;
+/** All example MCP tasks — used by compiler tests */
+export declare const allMCPExampleTasks: MCPServerTaskDefinition[];