@sanity/ailf 0.5.0 → 1.0.0

package/dist/pipeline/compiler/mode-handlers/__fixtures__/mcp-example-tasks.js

@@ -0,0 +1,259 @@
+/**
+ * Example MCP server task definitions — Sanity MCP Server reference.
+ *
+ * These are reference implementations showing how to author MCP server
+ * evaluation tasks. They model the real Sanity MCP Server (mcp.sanity.io)
+ * and its actual tool names. They serve as:
+ *
+ * 1. Documentation — developers learn the API by reading examples
+ * 2. Test fixtures — the compiler tests compile these and verify output
+ * 3. Validation — proves the compiler handles real-world task shapes
+ * 4. Reference — shows all transport types, auth patterns, and assertion styles
+ *
+ * NOTE: These are NOT executable without a valid SANITY_MCP_AUTH_TOKEN.
+ * They exist purely for compilation testing and as authoring examples.
+ * For real evaluation, write tasks in an external repo's .ailf/tasks/
+ * directory or a standalone config file.
+ *
+ * @see https://www.sanity.io/docs/ai/mcp-server — Sanity MCP Server docs
+ * @see https://www.promptfoo.dev/docs/providers/mcp/ — Promptfoo MCP provider
+ */
+// ---------------------------------------------------------------------------
+// Shared server config — reusable across tasks targeting the same server
+// ---------------------------------------------------------------------------
+const SANITY_MCP_SERVER = {
+    transport: "streamable-http",
+    url: "https://mcp.sanity.io",
+    auth: {
+        type: "bearer",
+        token: "{{env.SANITY_MCP_AUTH_TOKEN}}",
+    },
+};
+// ---------------------------------------------------------------------------
+// Task 1: GROQ query via query_documents tool
+// ---------------------------------------------------------------------------
+/**
+ * Tests that a model can construct a valid GROQ query and use the
+ * query_documents tool to fetch results from a Sanity dataset.
+ */
+export const queryDocumentsTask = {
+    mode: "mcp-server",
+    id: "mcp-query-documents",
+    title: "MCP: Query documents with GROQ",
+    description: "Use the Sanity MCP server's query_documents tool to fetch " +
+        "documents from a dataset using GROQ. Verify the model constructs " +
+        "a valid GROQ query and interprets the response correctly.",
+    area: "mcp",
+    difficulty: "intermediate",
+    tags: ["mcp", "groq", "query"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: ["query_documents", "get_schema"],
+    prompt: {
+        text: "Using the Sanity MCP server, query for all documents of type " +
+            '"article" in the production dataset. Return their titles and ' +
+            "slugs, ordered by creation date descending. Limit to 5 results.",
+    },
+    assertions: [
+        { type: "tool-called", value: "query_documents" },
+        { type: "contains", value: "_type" },
+        { type: "contains", value: "article" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Constructs a valid GROQ query with _type filter",
+                "Includes a projection with title and slug fields",
+                "Orders by _createdAt descending",
+                "Limits results to 5 with slice syntax [0...5] or [0..4]",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 2: Schema inspection via get_schema tool
+// ---------------------------------------------------------------------------
+/**
+ * Tests that a model can retrieve and interpret schema information
+ * from the MCP server.
+ */
+export const inspectSchemaTask = {
+    mode: "mcp-server",
+    id: "mcp-get-schema",
+    title: "MCP: Inspect schema and describe document types",
+    description: "Use the Sanity MCP server's get_schema tool to inspect the " +
+        "content model and describe the available document types.",
+    area: "mcp",
+    difficulty: "basic",
+    tags: ["mcp", "schema"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: ["get_schema"],
+    prompt: {
+        text: "Using the Sanity MCP server, retrieve the schema for the current " +
+            "workspace. List all document types and describe the fields of the " +
+            'most complex one. Focus on reference fields and "block" (Portable Text) fields.',
+    },
+    assertions: [
+        { type: "tool-called", value: "get_schema" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Successfully calls get_schema to retrieve the workspace schema",
+                "Identifies and lists the document types",
+                "Describes field structures including types and references",
+                "Correctly identifies Portable Text (block) fields",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 3: Multi-step create → verify → publish (multi-turn)
+// ---------------------------------------------------------------------------
+/**
+ * Tests a multi-step document lifecycle workflow using multiple tools
+ * across a multi-turn conversation. This is the most complex example,
+ * demonstrating tool chaining and verification patterns.
+ */
+export const createAndPublishTask = {
+    mode: "mcp-server",
+    id: "mcp-create-publish",
+    title: "MCP: Create a draft document and publish it",
+    description: "Use the Sanity MCP server to create a new draft document, " +
+        "verify it was created, then publish it. Tests the multi-step " +
+        "workflow of create → verify → publish.",
+    area: "mcp",
+    difficulty: "advanced",
+    tags: ["mcp", "crud", "multi-step"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: [
+        "create_documents_from_json",
+        "get_document",
+        "publish_documents",
+        "query_documents",
+    ],
+    multiTurn: {
+        turns: [
+            {
+                role: "user",
+                content: "Create a new draft article document with the title " +
+                    '"MCP Integration Test" and a slug of "mcp-integration-test". ' +
+                    "Use the create_documents_from_json tool.",
+            },
+            {
+                role: "user",
+                content: "Now verify the draft was created by querying for it. " +
+                    "Use query_documents with a GROQ query that fetches the " +
+                    'document by its slug "mcp-integration-test".',
+            },
+            {
+                role: "user",
+                content: "Finally, publish the draft document using the publish_documents tool.",
+            },
+        ],
+    },
+    assertions: [
+        { type: "tool-called", value: "create_documents_from_json" },
+        { type: "tool-called", value: "query_documents" },
+        { type: "tool-called", value: "publish_documents" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Creates a draft document with the correct _type and fields",
+                "Verifies the draft exists via GROQ query",
+                "Publishes the draft using the correct document ID",
+                "Handles the multi-step workflow in the correct order",
+            ],
+        },
+        {
+            type: "llm-rubric",
+            template: "mcp-error-handling",
+            criteria: [
+                "Handles potential errors (missing type, invalid schema) gracefully",
+                "Verifies each step before proceeding to the next",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 4: Semantic search (embeddings)
+// ---------------------------------------------------------------------------
+/**
+ * Tests that a model can discover embeddings indices and perform
+ * semantic search against them.
+ */
+export const semanticSearchTask = {
+    mode: "mcp-server",
+    id: "mcp-semantic-search",
+    title: "MCP: Semantic search for related content",
+    description: "Use the Sanity MCP server's semantic_search tool to find " +
+        "content related to a natural language query.",
+    area: "mcp",
+    difficulty: "intermediate",
+    tags: ["mcp", "search", "embeddings"],
+    serverConfig: SANITY_MCP_SERVER,
+    capabilities: ["semantic_search", "list_embeddings_indices", "get_document"],
+    prompt: {
+        text: "Using the Sanity MCP server, first list the available embeddings " +
+            "indices, then perform a semantic search for content related to " +
+            '"how to set up real-time previews in Next.js". Return the top 3 ' +
+            "most relevant documents with their titles and relevance scores.",
+    },
+    assertions: [
+        { type: "tool-called", value: "list_embeddings_indices" },
+        { type: "tool-called", value: "semantic_search" },
+        {
+            type: "llm-rubric",
+            template: "mcp-output-correctness",
+            criteria: [
+                "Lists available embeddings indices before searching",
+                "Selects an appropriate index for the search",
+                "Constructs a meaningful search query",
+                "Returns relevant results with titles and scores",
+            ],
+        },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Task 5: Local stdio server (shows command-based transport)
+// ---------------------------------------------------------------------------
+/**
+ * Demonstrates a stdio-based MCP server (local process) for contrast
+ * with the remote streamable-http tasks above.
+ */
+export const stdioServerTask = {
+    mode: "mcp-server",
+    id: "mcp-stdio-local",
+    title: "MCP: Local stdio server — getDocument",
+    description: "Call a locally running MCP server via stdio transport to " +
+        "fetch a document. Demonstrates command-based server config.",
+    area: "mcp",
+    difficulty: "basic",
+    tags: ["mcp", "stdio", "local"],
+    serverConfig: {
+        transport: "stdio",
+        command: "node dist/sanity-mcp-server.js",
+        env: {
+            SANITY_PROJECT_ID: "test-project",
+            SANITY_DATASET: "production",
+        },
+        startupTimeoutMs: 5000,
+    },
+    capabilities: ["tools/list", "tools/call"],
+    prompt: {
+        text: "Fetch the document with ID 'article-123' from the Sanity Content Lake.",
+    },
+    assertions: [
+        { type: "tool-called", value: "getDocument" },
+        { type: "contains", value: "article-123" },
+    ],
+    options: { timeout: 30_000 },
+};
+/** All example MCP tasks — used by compiler tests */
+export const allMCPExampleTasks = [
+    queryDocumentsTask,
+    inspectSchemaTask,
+    createAndPublishTask,
+    semanticSearchTask,
+    stdioServerTask,
+];

package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts

@@ -0,0 +1,70 @@
+/**
+ * AgentHarnessModeHandler — compilation rules for `agent-harness` mode.
+ *
+ * Maps agent harness task definitions to Promptfoo configuration with:
+ * - Claude Agent SDK / OpenAI Codex SDK providers
+ * - Tool permission configuration (preset/allowed/disallowed)
+ * - Sandbox setup/teardown via Promptfoo extensions
+ * - Fixture provisioning into sandbox working directory
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
+ */
+import type { AgentHarnessTaskDefinition, ModeHandler, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
+import type { SandboxType } from "../sandbox/sandbox-strategy.js";
+export declare const AGENT_HARNESS_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
+/** Options for compiling an agent harness task */
+export interface AgentHarnessCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Root directory for fixture resolution */
+    rootDir?: string;
+}
+/** Result of compiling a single agent harness task */
+export interface AgentHarnessCompileResult {
+    /** Promptfoo provider config */
+    providers: PromptfooProvider[];
+    /** Compiled test cases */
+    tests: PromptfooTestCase[];
+    /** Prompts for evaluation */
+    prompts: PromptfooPrompt[];
+    /** Promptfoo extensions for sandbox lifecycle */
+    extensions: PromptfooExtension[];
+    /** Sandbox configuration metadata */
+    sandboxConfig: SandboxConfigMeta;
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+/** Promptfoo extension hook */
+export interface PromptfooExtension {
+    type: "afterEach" | "beforeEach";
+    /** JavaScript code or module path for the hook */
+    code: string;
+}
+/** Metadata about sandbox configuration for this task */
+export interface SandboxConfigMeta {
+    type: SandboxType;
+    image?: string;
+    fixtures: string[];
+    limits?: {
+        cpus?: number;
+        memoryBytes?: number;
+        networkAccess?: boolean;
+    };
+}
+/** Validation errors for agent harness task definitions */
+export interface AgentHarnessValidationError {
+    field: string;
+    message: string;
+}
+/**
+ * Validate that an agent harness task definition has all required fields.
+ */
+export declare function validateAgentHarnessTask(task: AgentHarnessTaskDefinition): AgentHarnessValidationError[];
+/**
+ * Compile an agent harness task definition into Promptfoo configuration.
+ */
+export declare function compileAgentHarnessTask(task: AgentHarnessTaskDefinition, options?: AgentHarnessCompileOptions): AgentHarnessCompileResult;
+/** ModeHandler-conformant export for the agent-harness evaluation mode. */
+export declare const handler: ModeHandler;