@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-anthropic.js

@@ -0,0 +1,172 @@
+/**
+ * Anthropic multi-turn tool execution loop.
+ *
+ * Sends a prompt to the Anthropic Messages API with MCP tools attached.
+ * When the model calls a tool, executes it via the MCP client, feeds
+ * the result back, and continues until the model produces a final text
+ * response or maxToolRounds is exhausted.
+ */
+// ---------------------------------------------------------------------------
+// Tool loop implementation
+// ---------------------------------------------------------------------------
+/**
+ * Run a multi-turn tool loop using the Anthropic Messages API.
+ *
+ * The loop:
+ * 1. Sends the prompt with available tools to Claude
+ * 2. If Claude calls tools → executes them via MCP, sends results back
+ * 3. Repeats until Claude produces a text-only response or maxToolRounds is hit
+ * 4. On the last round, omits tools to force a synthesis response
+ */
+export async function runAnthropicToolLoop(config) {
+    const { prompt, tools, callTool, maxToolRounds, model, temperature, maxTokens, apiKey, } = config;
+    const anthropicTools = tools.map((t) => ({
+        name: t.name,
+        description: t.description || `MCP tool: ${t.name}`,
+        input_schema: t.inputSchema || { type: "object", properties: {} },
+    }));
+    const systemPrompt = "You are an AI assistant with access to tools provided by an MCP server. " +
+        "Use the available tools to complete the task. Call tools with correct parameters, " +
+        "interpret responses, and provide a complete answer.";
+    const messages = [{ content: prompt, role: "user" }];
+    let inputTokens = 0;
+    let outputTokens = 0;
+    const startTime = Date.now();
+    const toolCallLog = [];
+    for (let round = 0; round <= maxToolRounds; round++) {
+        const isLastRound = round === maxToolRounds;
+        // On the last round, omit tools to force a text-only response.
+        // Anthropic doesn't support tool_choice: "none" — the way to disable
+        // tools is to simply not include them in the request.
+        if (isLastRound) {
+            const lastMsg = messages[messages.length - 1];
+            const synthesisText = "You've used the tools available. Based on the information gathered, " +
+                "provide your complete, final answer now.";
+            if (lastMsg?.role === "user" && Array.isArray(lastMsg.content)) {
+                ;
+                lastMsg.content.push({
+                    type: "text",
+                    text: synthesisText,
+                });
+            }
+            else {
+                messages.push({ content: synthesisText, role: "user" });
+            }
+        }
+        const body = {
+            max_tokens: maxTokens,
+            messages,
+            model,
+            system: systemPrompt,
+            temperature,
+        };
+        if (!isLastRound) {
+            body.tools = anthropicTools;
+        }
+        const response = await fetch("https://api.anthropic.com/v1/messages", {
+            body: JSON.stringify(body),
+            headers: {
+                "anthropic-version": "2023-06-01",
+                "Content-Type": "application/json",
+                "x-api-key": apiKey,
+            },
+            method: "POST",
+        });
+        const data = (await response.json());
+        if (data.error) {
+            throw new Error(data.error.message ??
+                `Anthropic API error: ${JSON.stringify(data.error)}`);
+        }
+        inputTokens += data.usage?.input_tokens ?? 0;
+        outputTokens += data.usage?.output_tokens ?? 0;
+        if (!data.content?.length) {
+            return {
+                output: "",
+                toolCallLog,
+                tokenUsage: {
+                    prompt: inputTokens,
+                    completion: outputTokens,
+                    total: inputTokens + outputTokens,
+                },
+                toolRounds: round,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+        // Add assistant response to history
+        messages.push({ content: data.content, role: "assistant" });
+        // Check if model wants to use tools
+        const toolUseBlocks = data.content.filter((b) => b.type === "tool_use");
+        if (data.stop_reason !== "tool_use" || toolUseBlocks.length === 0) {
+            // Model is done — extract text
+            const textBlocks = data.content.filter((b) => b.type === "text");
+            const output = textBlocks.map((b) => b.text || "").join("\n") || "";
+            return {
+                output,
+                toolCallLog,
+                tokenUsage: {
+                    prompt: inputTokens,
+                    completion: outputTokens,
+                    total: inputTokens + outputTokens,
+                },
+                toolRounds: round,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+        // Execute each tool call via MCP
+        const toolResults = [];
+        for (const toolUse of toolUseBlocks) {
+            const toolName = toolUse.name;
+            const toolInput = (toolUse.input || {});
+            try {
+                const result = await callTool(toolName, toolInput);
+                const content = result.error
+                    ? JSON.stringify({ error: result.error })
+                    : result.content;
+                toolCallLog.push({ name: toolName, input: toolInput, output: content });
+                toolResults.push({
+                    content,
+                    tool_use_id: toolUse.id,
+                    type: "tool_result",
+                });
+            }
+            catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                toolCallLog.push({
+                    name: toolName,
+                    input: toolInput,
+                    output: `Error: ${errMsg}`,
+                });
+                toolResults.push({
+                    content: JSON.stringify({ error: errMsg }),
+                    tool_use_id: toolUse.id,
+                    type: "tool_result",
+                });
+            }
+        }
+        // Add tool results to conversation
+        messages.push({ content: toolResults, role: "user" });
+    }
+    // Exhausted rounds — extract last text
+    const lastAssistant = [...messages]
+        .reverse()
+        .find((m) => m.role === "assistant");
+    let lastText = "";
+    if (lastAssistant && Array.isArray(lastAssistant.content)) {
+        lastText = lastAssistant.content
+            .filter((b) => b.type === "text")
+            .map((b) => b.text || "")
+            .join("\n");
+    }
+    return {
+        output: lastText || "[Exhausted tool rounds without final answer]",
+        toolCallLog,
+        tokenUsage: {
+            prompt: inputTokens,
+            completion: outputTokens,
+            total: inputTokens + outputTokens,
+        },
+        toolRounds: maxToolRounds,
+        exhaustedRounds: true,
+        latencyMs: Date.now() - startTime,
+    };
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.d.ts

@@ -0,0 +1,14 @@
+/**
+ * OpenAI multi-turn tool execution loop.
+ *
+ * Placeholder for future implementation. Will use the OpenAI Chat
+ * Completions or Responses API with function calling to execute MCP
+ * tools in a multi-turn loop.
+ */
+import type { ToolLoopConfig, ToolLoopResult } from "./types.js";
+/**
+ * Run a multi-turn tool loop using the OpenAI API.
+ *
+ * @throws {Error} Not yet implemented
+ */
+export declare function runOpenAIToolLoop(_config: ToolLoopConfig): Promise<ToolLoopResult>;

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.js

@@ -0,0 +1,16 @@
+/**
+ * OpenAI multi-turn tool execution loop.
+ *
+ * Placeholder for future implementation. Will use the OpenAI Chat
+ * Completions or Responses API with function calling to execute MCP
+ * tools in a multi-turn loop.
+ */
+/**
+ * Run a multi-turn tool loop using the OpenAI API.
+ *
+ * @throws {Error} Not yet implemented
+ */
+export async function runOpenAIToolLoop(_config) {
+    throw new Error("OpenAI tool loop is not yet implemented. " +
+        "Use an Anthropic model (anthropic:messages:*) for MCP evaluations.");
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Shared types for the MCP tool provider.
+ */
+export interface CallApiContextParams {
+    prompt?: {
+        raw: string;
+        label?: string;
+    };
+    vars?: Record<string, object | string>;
+}
+export interface ProviderOptions {
+    config?: Record<string, unknown>;
+    id?: string;
+}
+export interface ProviderResponse {
+    cached?: boolean;
+    cost?: number;
+    error?: string;
+    metadata?: Record<string, unknown>;
+    output?: object | string;
+    tokenUsage?: {
+        total?: number;
+        prompt?: number;
+        completion?: number;
+        cached?: number;
+    };
+}
+export interface MCPTool {
+    name: string;
+    description?: string;
+    inputSchema?: Record<string, unknown>;
+}
+export interface MCPClient {
+    getAllTools: () => MCPTool[];
+    callTool: (name: string, args: Record<string, unknown>) => Promise<{
+        content: string;
+        error?: string;
+    }>;
+    cleanup: () => Promise<void>;
+}
+export interface ToolCallEntry {
+    name: string;
+    input: unknown;
+    output: string;
+}
+/** Configuration passed from the provider to an LLM backend's tool loop */
+export interface ToolLoopConfig {
+    /** The user's prompt */
+    prompt: string;
+    /** MCP tools available (already filtered by capabilities) */
+    tools: MCPTool[];
+    /** Function to call an MCP tool by name */
+    callTool: (name: string, args: Record<string, unknown>) => Promise<{
+        content: string;
+        error?: string;
+    }>;
+    /** Max rounds of tool calls before forcing a final answer */
+    maxToolRounds: number;
+    /** LLM model name (provider-specific, e.g., "claude-opus-4-6") */
+    model: string;
+    /** Sampling temperature */
+    temperature: number;
+    /** Max output tokens */
+    maxTokens: number;
+    /** API key for the LLM provider */
+    apiKey: string;
+}
+/** Result returned from an LLM backend's tool loop */
+export interface ToolLoopResult {
+    /** The LLM's final text output */
+    output: string;
+    /** Log of all tool calls made during the loop */
+    toolCallLog: ToolCallEntry[];
+    /** Token usage stats */
+    tokenUsage: {
+        prompt: number;
+        completion: number;
+        total: number;
+    };
+    /** Number of tool rounds completed */
+    toolRounds: number;
+    /** Whether the loop exhausted maxToolRounds */
+    exhaustedRounds?: boolean;
+    /** Wall-clock duration in ms */
+    latencyMs: number;
+}
+/**
+ * An LLM backend that can run a multi-turn tool loop.
+ *
+ * Each backend (Anthropic, OpenAI) implements this function signature.
+ * The provider selects the appropriate backend based on the model ID prefix.
+ */
+export type RunToolLoop = (config: ToolLoopConfig) => Promise<ToolLoopResult>;

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.js

@@ -0,0 +1,4 @@
+/**
+ * Shared types for the MCP tool provider.
+ */
+export {};

package/dist/pipeline/compiler/preset-loader.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Preset loader — resolves and loads external PresetDefinition files.
+ *
+ * Supports two reference formats:
+ * - Relative path (./foo.ts, ../presets/bar.ts) → loaded from disk via jiti
+ * - npm package name (my-preset-package) → resolved via Node require
+ *
+ * Uses jiti for synchronous TypeScript/JavaScript loading, matching the
+ * sync constraint of createAppContext() and createRegistry().
+ *
+ * @see packages/eval/src/composition-root.ts — where this is called
+ */
+import type { PresetDefinition } from "../../_vendor/ailf-core/index.d.ts";
+/**
+ * Load external presets from file paths or npm package names.
+ *
+ * @param refs - Preset references (relative paths or package names)
+ * @param rootDir - Base directory for resolving relative paths
+ * @returns Loaded PresetDefinition array (preserves input order)
+ * @throws If a preset file is not found or doesn't export a valid PresetDefinition
+ */
+export declare function loadExternalPresets(refs: string[], rootDir: string): PresetDefinition[];

package/dist/pipeline/compiler/preset-loader.js

@@ -0,0 +1,99 @@
+/**
+ * Preset loader — resolves and loads external PresetDefinition files.
+ *
+ * Supports two reference formats:
+ * - Relative path (./foo.ts, ../presets/bar.ts) → loaded from disk via jiti
+ * - npm package name (my-preset-package) → resolved via Node require
+ *
+ * Uses jiti for synchronous TypeScript/JavaScript loading, matching the
+ * sync constraint of createAppContext() and createRegistry().
+ *
+ * @see packages/eval/src/composition-root.ts — where this is called
+ */
+import { existsSync } from "fs";
+import { resolve } from "path";
+import { pathToFileURL } from "url";
+import { createJiti } from "jiti";
+/** Thrown for preset-specific load errors (distinguishes from third-party errors) */
+class PresetLoadError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "PresetLoadError";
+    }
+}
+/**
+ * Load external presets from file paths or npm package names.
+ *
+ * @param refs - Preset references (relative paths or package names)
+ * @param rootDir - Base directory for resolving relative paths
+ * @returns Loaded PresetDefinition array (preserves input order)
+ * @throws If a preset file is not found or doesn't export a valid PresetDefinition
+ */
+export function loadExternalPresets(refs, rootDir) {
+    const presets = [];
+    for (const ref of refs) {
+        const preset = loadSinglePreset(ref, rootDir);
+        presets.push(preset);
+    }
+    return presets;
+}
+function loadSinglePreset(ref, rootDir) {
+    const isPath = ref.startsWith("./") || ref.startsWith("../") || ref.startsWith("/");
+    const filePath = isPath ? resolve(rootDir, ref) : ref;
+    // Containment check: resolved path must be within rootDir to prevent
+    // path traversal (e.g., "../../other-project/secret.ts")
+    if (isPath) {
+        const resolvedRoot = resolve(rootDir);
+        if (!resolve(filePath).startsWith(resolvedRoot)) {
+            throw new PresetLoadError(`Preset "${ref}" resolves outside the project root (${resolvedRoot}). ` +
+                `Preset paths must be within the project directory.`);
+        }
+        if (!existsSync(filePath)) {
+            throw new PresetLoadError(`Preset file not found: ${filePath} (resolved from "${ref}" relative to ${rootDir})`);
+        }
+    }
+    try {
+        const jiti = createJiti(pathToFileURL(rootDir).href, {
+            interopDefault: true,
+            requireCache: true,
+        });
+        // jiti() is the synchronous loader
+        const mod = jiti(filePath);
+        const preset = extractDefault(mod);
+        if (!preset || typeof preset !== "object") {
+            throw new PresetLoadError(`Preset "${ref}" does not export a default value. ` +
+                `Use \`export default definePreset({ ... })\`.`);
+        }
+        // Validate minimal shape
+        const p = preset;
+        if (!p.name || !p.mode || !p.manifest) {
+            throw new PresetLoadError(`Preset "${ref}" is missing required fields. ` +
+                `A preset must have at least \`name\`, \`mode\`, and \`manifest\`.`);
+        }
+        return p;
+    }
+    catch (cause) {
+        if (cause instanceof PresetLoadError) {
+            throw cause; // Re-throw our own errors
+        }
+        const message = cause instanceof Error ? cause.message : String(cause);
+        throw new Error(`Failed to load preset "${ref}": ${message}`, {
+            cause: cause,
+        });
+    }
+}
+/**
+ * Extract the default export from a module, handling both ESM and CJS patterns.
+ */
+function extractDefault(mod) {
+    if (mod === null || mod === undefined)
+        return undefined;
+    if (typeof mod !== "object")
+        return mod;
+    const obj = mod;
+    // ESM: { default: value }
+    if ("default" in obj)
+        return obj.default;
+    // CJS: the module itself is the value
+    return obj;
+}

package/dist/pipeline/compiler/presets/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Presets — bundled evaluation capabilities.
+ *
+ * Each preset packages modes, assertions, rubric templates, fixture
+ * resolvers, prompt templates, scoring profiles, doc fetcher, source
+ * definitions, and feature definitions into a single installable unit.
+ */
+export { createSanityLiteracyPreset, registerSanityLiteracyPreset, sanityLiteracyPreset, } from "./sanity-literacy.js";
+export type { SanityLiteracyPresetOptions } from "./sanity-literacy.js";

package/dist/pipeline/compiler/presets/index.js

@@ -0,0 +1,8 @@
+/**
+ * Presets — bundled evaluation capabilities.
+ *
+ * Each preset packages modes, assertions, rubric templates, fixture
+ * resolvers, prompt templates, scoring profiles, doc fetcher, source
+ * definitions, and feature definitions into a single installable unit.
+ */
+export { createSanityLiteracyPreset, registerSanityLiteracyPreset, sanityLiteracyPreset, } from "./sanity-literacy.js";

package/dist/pipeline/compiler/presets/sanity-literacy.d.ts

@@ -0,0 +1,42 @@
+/**
+ * sanity-literacy preset — Sanity-specific domain configuration for literacy evaluation.
+ *
+ * This is a domain preset that targets the `literacy` mode base. It provides
+ * Sanity-specific configuration:
+ * - Sanity doc source definitions (production, branch, local)
+ * - Product feature registry for coverage auditing
+ * - DocFetcher factory (SanityDocFetcher)
+ * - Sanity fixture resolver (sanity:// scheme)
+ *
+ * Evaluation methodology (rubrics, scoring, prompts) is inherited from the
+ * `literacy` mode base — see mode-bases/literacy.ts.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-8-scoring-storage-presets.md
+ */
+import { type PresetDefinition } from "../../../_vendor/ailf-core/index.d.ts";
+export interface SanityLiteracyPresetOptions {
+    /** Root directory for SanityDocFetcher file path resolution (defaults to cwd) */
+    rootDir?: string;
+}
+/**
+ * Create the sanity-literacy preset with all Sanity-specific configuration.
+ *
+ * The `rootDir` option controls where the SanityDocFetcher resolves file
+ * paths. When omitted it defaults to `process.cwd()`, which matches the
+ * composition root's existing behavior.
+ */
+export declare function createSanityLiteracyPreset(options?: SanityLiteracyPresetOptions): PresetDefinition;
+/**
+ * Pre-built preset instance using default options (cwd for rootDir).
+ *
+ * Prefer `createSanityLiteracyPreset()` when you need to control rootDir
+ * or other options. This constant exists for backward compatibility with
+ * code that imports `sanityLiteracyPreset` directly.
+ */
+export declare const sanityLiteracyPreset: PresetDefinition;
+/**
+ * Register the sanity-literacy preset in a plugin registry.
+ */
+export declare function registerSanityLiteracyPreset(registry: {
+    registerPreset(preset: PresetDefinition): void;
+}): void;