@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Public types for the knowledge-probe mode handler.
+ */
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+/** Options for compiling a knowledge probe task */
+export interface KnowledgeProbeCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Model registry — knowledge probes run across all configured models */
+    models?: {
+        id: string;
+        label: string;
+        config?: Record<string, unknown>;
+    }[];
+}
+/** Result of compiling a single knowledge probe task */
+export interface KnowledgeProbeCompileResult {
+    /** Promptfoo provider configs (one per model) */
+    providers: PromptfooProvider[];
+    /** Compiled test cases */
+    tests: PromptfooTestCase[];
+    /** Prompts for evaluation */
+    prompts: PromptfooPrompt[];
+    /** Mode metadata for cross-model comparison */
+    metadata: KnowledgeProbeMetadata;
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+/** Metadata attached to knowledge probe results for comparison */
+export interface KnowledgeProbeMetadata {
+    /** Evaluation mode identifier */
+    mode: "knowledge-probe";
+    /** Probe strategy used */
+    probeStrategy: string;
+    /** Whether doc context was intentionally excluded */
+    noDocContext: true;
+    /** Whether retrieval metrics are applicable */
+    retrievalMetrics: false;
+}
+/** Validation errors for knowledge probe task definitions */
+export interface KnowledgeProbeValidationError {
+    field: string;
+    message: string;
+}

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.js

@@ -0,0 +1,4 @@
+/**
+ * Public types for the knowledge-probe mode handler.
+ */
+export {};

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/validation.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Validation logic for knowledge probe task definitions.
+ */
+import type { KnowledgeProbeTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { KnowledgeProbeValidationError } from "./types.js";
+/**
+ * Validate that a knowledge probe task definition has all required fields.
+ */
+export declare function validateKnowledgeProbeTask(task: KnowledgeProbeTaskDefinition): KnowledgeProbeValidationError[];

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/validation.js

@@ -0,0 +1,24 @@
+/**
+ * Validation logic for knowledge probe task definitions.
+ */
+/**
+ * Validate that a knowledge probe task definition has all required fields.
+ */
+export function validateKnowledgeProbeTask(task) {
+    const errors = [];
+    if (!task.id) {
+        errors.push({ field: "id", message: "Task ID is required" });
+    }
+    if (!task.title) {
+        errors.push({ field: "title", message: "Task title is required" });
+    }
+    // Knowledge probes must have either a prompt or a description
+    if (!task.prompt?.text && !task.prompt?.vars?.task && !task.description) {
+        errors.push({
+            field: "prompt",
+            message: "Knowledge probe tasks require either prompt.text, prompt.vars.task, " +
+                "or description — the question to ask the model",
+        });
+    }
+    return errors;
+}

package/dist/pipeline/compiler/mode-handlers/literacy/assertions.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Assertion resolution for literacy tasks.
+ *
+ * Handles rubric template resolution, doc-coverage auto-generation,
+ * and baseline assertion filtering.
+ */
+import type { LiteracyTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooAssertion } from "../../assertion-mapper.js";
+import type { LiteracyCompileOptions } from "./types.js";
+export declare function resolveAssertions(task: LiteracyTaskDefinition, options: LiteracyCompileOptions | undefined, warnings: string[]): PromptfooAssertion[];
+/**
+ * Build baseline assertions matching the legacy expand-tasks behavior.
+ *
+ * - "full": all assertions carried over
+ * - "abbreviated": only first llm-rubric with shortened prompt
+ * - "none": no assertions
+ */
+export declare function buildBaselineAssertions(goldAssertions: PromptfooAssertion[], rubricMode?: "abbreviated" | "full" | "none"): PromptfooAssertion[];

package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js

@@ -0,0 +1,118 @@
+/**
+ * Assertion resolution for literacy tasks.
+ *
+ * Handles rubric template resolution, doc-coverage auto-generation,
+ * and baseline assertion filtering.
+ */
+// ---------------------------------------------------------------------------
+// Assertion resolution
+// ---------------------------------------------------------------------------
+export function resolveAssertions(task, options, warnings) {
+    const assertions = [];
+    for (const a of task.assertions ?? []) {
+        if (a.type === "llm-rubric" && "template" in a) {
+            const resolved = resolveTemplatedAssertion(a, options?.rubricConfig, options?.graderProvider, warnings);
+            if (resolved)
+                assertions.push(resolved);
+        }
+        else {
+            assertions.push({
+                type: a.type,
+                ...("value" in a ? { value: a.value } : {}),
+                ...(typeof a.weight === "number"
+                    ? { weight: a.weight }
+                    : {}),
+                ...(a.type === "llm-rubric" && options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            });
+        }
+    }
+    // Doc-coverage auto-generation
+    if (task.docCoverage) {
+        const docCoverageAssertion = buildDocCoverageAssertion(options?.rubricConfig, options?.graderProvider);
+        if (docCoverageAssertion)
+            assertions.push(docCoverageAssertion);
+    }
+    return assertions;
+}
+// ---------------------------------------------------------------------------
+// Rubric template resolution
+// ---------------------------------------------------------------------------
+function resolveTemplatedAssertion(a, rubricConfig, graderProvider, warnings) {
+    if (!rubricConfig) {
+        warnings.push(`No rubric config — template "${a.template}" cannot be resolved`);
+        return null;
+    }
+    const template = rubricConfig.templates[a.template];
+    if (!template) {
+        warnings.push(`Unknown rubric template: "${a.template}"`);
+        return null;
+    }
+    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
+    const criteriaText = a.criteria.map((c) => `- ${c}`).join("\n");
+    const rubricValue = `${template.header}\n${scaleText}\n\n` +
+        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
+        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
+    return {
+        type: "llm-rubric",
+        value: rubricValue,
+        ...(graderProvider ? { provider: graderProvider } : {}),
+        ...(template.dimension
+            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
+            : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Doc-coverage assertion
+// ---------------------------------------------------------------------------
+function buildDocCoverageAssertion(rubricConfig, graderProvider) {
+    if (!rubricConfig?.templates["doc-coverage"])
+        return null;
+    const template = rubricConfig.templates["doc-coverage"];
+    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
+    const rubricValue = `${template.header}\n${scaleText}\n\n` +
+        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
+    return {
+        type: "llm-rubric",
+        value: rubricValue,
+        ...(graderProvider ? { provider: graderProvider } : {}),
+        ...(template.dimension
+            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
+            : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Baseline assertion filtering
+// ---------------------------------------------------------------------------
+/**
+ * Build baseline assertions matching the legacy expand-tasks behavior.
+ *
+ * - "full": all assertions carried over
+ * - "abbreviated": only first llm-rubric with shortened prompt
+ * - "none": no assertions
+ */
+export function buildBaselineAssertions(goldAssertions, rubricMode) {
+    const mode = rubricMode ?? "full";
+    if (mode === "none")
+        return [];
+    if (mode === "full")
+        return [...goldAssertions];
+    // Abbreviated: keep first llm-rubric as summary, skip rest
+    const abbreviated = [];
+    let foundFirst = false;
+    for (const a of goldAssertions) {
+        if (a.type === "llm-rubric") {
+            if (!foundFirst) {
+                foundFirst = true;
+                abbreviated.push({
+                    type: "llm-rubric",
+                    value: "Score task completion from 0 to 100 (same criteria as above).\n" +
+                        'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
+                    ...(a.provider ? { provider: a.provider } : {}),
+                });
+            }
+        }
+    }
+    return abbreviated;
+}

package/dist/pipeline/compiler/mode-handlers/literacy/compiler.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Literacy task compilation — core compilation logic.
+ *
+ * Produces the same structure as the legacy expand-tasks.ts path:
+ * - Gold entry with with-docs prompt and canonical doc context
+ * - Baseline entry with without-docs prompt and empty docs
+ * - Rubric assertions with structured dimension metadata
+ */
+import type { LiteracyTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { LiteracyCompileOptions, LiteracyCompileResult } from "./types.js";
+/**
+ * Compile a literacy task into Promptfoo configuration.
+ */
+export declare function compileLiteracyTask(task: LiteracyTaskDefinition, options?: LiteracyCompileOptions): LiteracyCompileResult;

package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js

@@ -0,0 +1,105 @@
+/**
+ * Literacy task compilation — core compilation logic.
+ *
+ * Produces the same structure as the legacy expand-tasks.ts path:
+ * - Gold entry with with-docs prompt and canonical doc context
+ * - Baseline entry with without-docs prompt and empty docs
+ * - Rubric assertions with structured dimension metadata
+ */
+import { LiteracyVariant, } from "../../../normalize-mode.js";
+import { buildBaselineAssertions, resolveAssertions } from "./assertions.js";
+import { LITERACY_PROMPT_TEMPLATES } from "./prompts.js";
+import { validateLiteracyTask } from "./validation.js";
+/**
+ * Compile a literacy task into Promptfoo configuration.
+ */
+export function compileLiteracyTask(task, options) {
+    const warnings = [];
+    const evalMode = options?.evalMode ?? LiteracyVariant.STANDARD;
+    // Validation
+    for (const err of validateLiteracyTask(task)) {
+        warnings.push(`Literacy task "${task.id}": ${err.field} — ${err.message}`);
+    }
+    const providers = buildProviders(options);
+    const prompts = buildPrompts(evalMode);
+    const tests = buildTestCases(task, evalMode, options, warnings);
+    return { providers, tests, prompts, warnings };
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+function buildProviders(options) {
+    if (options?.models && options.models.length > 0) {
+        return options.models.map((m) => ({
+            id: m.id,
+            label: m.label,
+            config: m.config,
+        }));
+    }
+    return [];
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+function templateToPromptfoo(pt) {
+    return { id: pt.id, label: pt.label, raw: pt.template };
+}
+function buildPrompts(evalMode) {
+    if (evalMode === "agentic") {
+        return [templateToPromptfoo(LITERACY_PROMPT_TEMPLATES["agentic"])];
+    }
+    return [
+        templateToPromptfoo(LITERACY_PROMPT_TEMPLATES["with-docs"]),
+        templateToPromptfoo(LITERACY_PROMPT_TEMPLATES["without-docs"]),
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+function buildTestCases(task, evalMode, options, warnings) {
+    const tests = [];
+    const promptText = task.prompt?.text ?? task.prompt?.template ?? "";
+    const contextDocs = task.context?.docs ?? [];
+    const taskArea = task.area ?? "";
+    const taskTitle = task.title;
+    const promptVars = task.prompt?.vars ?? {};
+    const hasDocs = contextDocs.length > 0;
+    const docsVar = hasDocs ? `file://contexts/canonical/${task.id}.md` : "";
+    const assertions = resolveAssertions(task, options, warnings);
+    // Gold entry — canonical docs injected
+    const goldVars = {
+        task: promptText,
+        docs: docsVar,
+        __featureArea: taskArea,
+        ...promptVars,
+    };
+    tests.push({
+        description: `${taskTitle} (gold)`,
+        vars: goldVars,
+        ...(evalMode === LiteracyVariant.STANDARD
+            ? { prompts: ["with-docs"] }
+            : {}),
+        ...(assertions.length > 0 ? { assert: assertions } : {}),
+    });
+    // Baseline entry — no docs (floor measurement)
+    if (evalMode !== "agentic") {
+        const baselineEnabled = task.baseline?.enabled !== false;
+        if (baselineEnabled) {
+            const baselineAssertions = buildBaselineAssertions(assertions, task.baseline?.rubric);
+            tests.push({
+                description: `${taskTitle} (baseline)`,
+                vars: {
+                    task: promptText,
+                    docs: "",
+                    __featureArea: taskArea,
+                    ...promptVars,
+                },
+                prompts: ["without-docs"],
+                ...(baselineAssertions.length > 0
+                    ? { assert: baselineAssertions }
+                    : {}),
+            });
+        }
+    }
+    return tests;
+}

package/dist/pipeline/compiler/mode-handlers/literacy/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Literacy mode handler — compiles LiteracyTaskDefinition into Promptfoo config.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ */
+import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";
+export { LITERACY_PROMPT_TEMPLATES } from "./prompts.js";
+export { validateLiteracyTask, type LiteracyValidationError, } from "./validation.js";
+export { compileLiteracyTask } from "./compiler.js";
+export type { LiteracyCompileOptions, LiteracyCompileResult, RubricConfig, } from "./types.js";
+export declare const handler: ModeHandler;

package/dist/pipeline/compiler/mode-handlers/literacy/index.js

@@ -0,0 +1,38 @@
+/**
+ * Literacy mode handler — compiles LiteracyTaskDefinition into Promptfoo config.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ */
+import { compileLiteracyTask } from "./compiler.js";
+import { LITERACY_PROMPT_TEMPLATES } from "./prompts.js";
+// Re-export public API
+export { LITERACY_PROMPT_TEMPLATES } from "./prompts.js";
+export { validateLiteracyTask, } from "./validation.js";
+export { compileLiteracyTask } from "./compiler.js";
+// ---------------------------------------------------------------------------
+// ModeHandler adapter — wraps compileLiteracyTask for registry dispatch
+// ---------------------------------------------------------------------------
+export const handler = {
+    getPrompts() {
+        return LITERACY_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (task.mode !== "literacy") {
+            throw new Error(`Literacy handler received task with mode "${task.mode}" — expected "literacy"`);
+        }
+        const result = compileLiteracyTask(task, {
+            graderProvider: ctx.graderProvider,
+            rootDir: ctx.rootDir,
+            models: ctx.models,
+            rubricConfig: ctx.rubricConfig,
+            evalMode: ctx
+                .evalMode,
+        });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+        };
+    },
+};

package/dist/pipeline/compiler/mode-handlers/literacy/prompts.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Canonical prompt templates for literacy-mode evaluations.
+ *
+ * These are the source-of-truth templates. Previously lived in
+ * config/prompts.ts as global templates; now handler-owned so
+ * non-literacy modes can define their own prompts without collision.
+ */
+import type { PromptTemplate } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const LITERACY_PROMPT_TEMPLATES: Record<string, PromptTemplate>;

package/dist/pipeline/compiler/mode-handlers/literacy/prompts.js

@@ -0,0 +1,74 @@
+/**
+ * Canonical prompt templates for literacy-mode evaluations.
+ *
+ * These are the source-of-truth templates. Previously lived in
+ * config/prompts.ts as global templates; now handler-owned so
+ * non-literacy modes can define their own prompts without collision.
+ */
+export const LITERACY_PROMPT_TEMPLATES = {
+    "with-docs": {
+        id: "with-docs",
+        label: "With Documentation",
+        template: `You are an expert Sanity.io developer. Use the following documentation to help implement the task.
+
+## Sanity Documentation
+{{docs}}
+
+## Task
+{{task}}
+
+## Requirements
+
+1. Use ONLY the APIs and patterns shown in the documentation
+2. Provide a complete, working implementation
+3. Include all necessary imports
+4. Follow Sanity best practices as documented
+
+Provide your implementation:
+`,
+        variables: ["docs", "task"],
+    },
+    "without-docs": {
+        id: "without-docs",
+        label: "Baseline (No Docs)",
+        template: `You are an expert Sanity.io developer.
+
+## Task
+{{task}}
+
+## Requirements
+
+1. Provide a complete, working implementation
+2. Include all necessary imports
+3. Follow Sanity best practices
+
+Provide your implementation:
+`,
+        variables: ["task"],
+    },
+    agentic: {
+        id: "agentic",
+        label: "Agentic (self-retrieval)",
+        template: `You are an expert developer helping implement a Sanity.io feature.
+You have access to web search and page fetching tools.
+
+IMPORTANT: Before writing any code, search for and read the relevant
+Sanity.io documentation to ensure you are using the latest APIs and
+best practices. Do not rely on memory alone.
+
+## Task
+{{task}}
+
+## Requirements
+
+1. Search for relevant Sanity documentation before implementing
+2. Use ONLY the APIs and patterns from the current official docs
+3. Provide a complete, working implementation
+4. Include all necessary imports
+5. Follow Sanity best practices as documented
+
+Provide your implementation:
+`,
+        variables: ["task"],
+    },
+};

package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Shared types for the literacy mode handler.
+ */
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+/** Options for compiling a literacy task */
+export interface LiteracyCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Root directory (for resolving file:// doc paths) */
+    rootDir?: string;
+    /** Evaluation sub-mode — controls which entries are generated */
+    evalMode?: import("../../../normalize-mode.js").LiteracyEvalSubMode;
+    /** Model providers to include */
+    models?: {
+        id: string;
+        label: string;
+        config?: Record<string, unknown>;
+    }[];
+    /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
+    rubricConfig?: RubricConfig;
+}
+/** Minimal rubric config needed by the handler */
+export interface RubricConfig {
+    templates: Record<string, {
+        dimension?: string;
+        header: string;
+        scale: string[];
+        criteria_label?: string;
+    }>;
+}
+/** Result of compiling a single literacy task */
+export interface LiteracyCompileResult {
+    /** Promptfoo provider configs */
+    providers: PromptfooProvider[];
+    /** Compiled test cases (gold + optional baseline) */
+    tests: PromptfooTestCase[];
+    /** Prompts for evaluation */
+    prompts: PromptfooPrompt[];
+    /** Warnings generated during compilation */
+    warnings: string[];
+}

package/dist/pipeline/compiler/mode-handlers/literacy/types.js

@@ -0,0 +1,4 @@
+/**
+ * Shared types for the literacy mode handler.
+ */
+export {};

package/dist/pipeline/compiler/mode-handlers/literacy/validation.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Validation for literacy task definitions.
+ */
+import type { LiteracyTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+export interface LiteracyValidationError {
+    field: string;
+    message: string;
+}
+/**
+ * Validate a literacy task definition.
+ */
+export declare function validateLiteracyTask(task: LiteracyTaskDefinition): LiteracyValidationError[];

package/dist/pipeline/compiler/mode-handlers/literacy/validation.js

@@ -0,0 +1,28 @@
+/**
+ * Validation for literacy task definitions.
+ */
+/**
+ * Validate a literacy task definition.
+ */
+export function validateLiteracyTask(task) {
+    const errors = [];
+    if (!task.id)
+        errors.push({ field: "id", message: "Task ID is required" });
+    if (!task.title) {
+        errors.push({
+            field: "title",
+            message: "Task title is required",
+        });
+    }
+    const promptText = task.prompt?.text ??
+        task.prompt?.template ??
+        task.prompt?.vars?.task ??
+        "";
+    if (!promptText) {
+        errors.push({
+            field: "prompt",
+            message: "Task prompt text is required",
+        });
+    }
+    return errors;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/assertions.d.ts

@@ -0,0 +1,42 @@
+/**
+ * MCP-specific assertion types — ergonomic assertions for MCP server testing.
+ *
+ * Each assertion type compiles down to a Promptfoo `javascript` assertion
+ * with the appropriate validation logic. The developer writes:
+ *
+ * ```typescript
+ * assertions: [
+ *   { type: "tool-called", value: "getDocument" },
+ *   { type: "tool-input-matches", value: { documentId: "doc-123" } },
+ *   { type: "tool-output-matches", value: { title: "Hello" } },
+ *   { type: "error-returned", value: { code: -32602 } },
+ * ]
+ * ```
+ *
+ * The compiler transforms these into Promptfoo-compatible `javascript`
+ * assertions that inspect the tool call trace in the evaluation output.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ */
+import type { PromptfooAssertion } from "../../assertion-mapper.js";
+import type { MCPAssertionContext } from "./types.js";
+/** An AILF assertion definition — accepts both core and generalized types */
+interface AssertionInput {
+    type: string;
+    value?: unknown;
+    weight?: number;
+    /** Allow additional properties from generalized assertions */
+    [key: string]: unknown;
+}
+/**
+ * Build MCP-specific assertions from task assertion definitions.
+ *
+ * Handles both MCP-specific types (tool-called, tool-input-matches, etc.)
+ * and standard assertion types (contains, llm-rubric, etc.) which are
+ * passed through unchanged.
+ */
+export declare function buildMCPAssertions(assertions: AssertionInput[], context: MCPAssertionContext): {
+    assertions: PromptfooAssertion[];
+    warnings: string[];
+};
+export {};