@sanity/ailf 0.5.0 → 1.0.0

package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js

@@ -0,0 +1,245 @@
+/**
+ * KnowledgeProbeModeHandler — compilation rules for `knowledge-probe` mode.
+ *
+ * The simplest mode handler. Knowledge probes measure raw model knowledge
+ * without documentation context, tool calling, or sandboxed execution.
+ * They answer: "What does this model know about X without any help?"
+ *
+ * Key properties:
+ * - No doc vars injected (intentionally empty)
+ * - Uses the without-docs prompt template (or custom prompt)
+ * - Standard LLM providers only (no agent SDKs, no MCP)
+ * - No retrieval metrics (precision/recall/F1 not applicable)
+ * - Results feed into the standard cross-model comparison pipeline
+ *
+ * This handler is the reference implementation for the mode handler pattern.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-5-knowledge-probe.md
+ * @see packages/core/src/types/generalized-task.ts — KnowledgeProbeTaskDefinition
+ */
+// ---------------------------------------------------------------------------
+// Canonical knowledge probe prompt templates
+// ---------------------------------------------------------------------------
+// Handler-owned prompts for knowledge probe evaluations. These ask factual
+// questions without injecting documentation context — measuring raw model
+// knowledge about Sanity concepts.
+export const KNOWLEDGE_PROBE_PROMPT_TEMPLATES = {
+    "knowledge-probe": {
+        id: "knowledge-probe",
+        label: "Knowledge Probe (No Docs)",
+        template: `Answer the following question about Sanity.io based on your existing knowledge. Do not search for or reference external documentation.
+
+## Question
+{{task}}
+
+## Instructions
+
+1. Answer based solely on what you already know
+2. Be specific — include API names, function signatures, and code examples where relevant
+3. If you are unsure about a detail, say so rather than guessing
+4. Provide a complete, accurate answer
+
+Your answer:
+`,
+        variables: ["task"],
+    },
+};
+/**
+ * Validate that a knowledge probe task definition has all required fields.
+ */
+export function validateKnowledgeProbeTask(task) {
+    const errors = [];
+    if (!task.id) {
+        errors.push({ field: "id", message: "Task ID is required" });
+    }
+    if (!task.title) {
+        errors.push({ field: "title", message: "Task title is required" });
+    }
+    // Knowledge probes must have either a prompt or a description
+    if (!task.prompt?.text && !task.prompt?.vars?.task && !task.description) {
+        errors.push({
+            field: "prompt",
+            message: "Knowledge probe tasks require either prompt.text, prompt.vars.task, " +
+                "or description — the question to ask the model",
+        });
+    }
+    return errors;
+}
+// ---------------------------------------------------------------------------
+// Compilation
+// ---------------------------------------------------------------------------
+/**
+ * Compile a knowledge probe task definition into Promptfoo configuration.
+ *
+ * This is intentionally minimal — knowledge probes map almost 1:1 to
+ * basic Promptfoo test cases. The AILF value-add is type-safe authoring,
+ * cross-model comparison, and score normalization.
+ */
+export function compileKnowledgeProbeTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateKnowledgeProbeTask(task);
+    for (const err of validationErrors) {
+        warnings.push(`Knowledge probe "${task.id}": ${err.field} — ${err.message}`);
+    }
+    // Build providers from model list (or use a default placeholder)
+    const providers = buildProviders(options);
+    // Build prompts — knowledge probes use a single no-docs prompt
+    const prompts = buildPrompts(task);
+    // Build test cases
+    const tests = buildTestCases(task, options, warnings);
+    // Build metadata
+    const metadata = {
+        mode: "knowledge-probe",
+        probeStrategy: task.probeStrategy ?? "breadth-first",
+        noDocContext: true,
+        retrievalMetrics: false,
+    };
+    return { providers, tests, prompts, metadata, warnings };
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+function buildProviders(options) {
+    if (options?.models && options.models.length > 0) {
+        return options.models.map((model) => ({
+            id: model.id,
+            label: model.label,
+            config: model.config,
+        }));
+    }
+    // No models specified — return empty (caller should provide models)
+    return [];
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+function buildPrompts(task) {
+    // Knowledge probes use a single prompt — no with-docs/without-docs split.
+    // The prompt IS the probe question.
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Knowledge probe: ${task.title}`;
+    const systemMessage = task.prompt?.systemMessage;
+    return [
+        {
+            id: "knowledge-probe",
+            label: `Probe: ${task.title}`,
+            raw: systemMessage
+                ? `[system]\n${systemMessage}\n\n[user]\n${String(promptText)}`
+                : String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+function buildTestCases(task, options, warnings) {
+    // Build assertions
+    const assertions = [];
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            const raw = assertion;
+            const mapped = mapKnowledgeProbeAssertion(raw, options, warnings);
+            if (mapped)
+                assertions.push(mapped);
+        }
+    }
+    // Build vars — intentionally no docs
+    const vars = {
+        task: task.prompt?.vars?.task ??
+            task.description ??
+            `Knowledge probe: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+        // Metadata for scoring pipeline
+        __mode: "knowledge-probe",
+        __probeStrategy: task.probeStrategy ?? "breadth-first",
+    };
+    // Explicitly do NOT include docs
+    // This is the defining characteristic of knowledge-probe mode
+    delete vars.docs;
+    return [
+        {
+            description: `${task.id} — ${task.title}`,
+            vars,
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Assertion mapping
+// ---------------------------------------------------------------------------
+function mapKnowledgeProbeAssertion(assertion, options, warnings) {
+    switch (assertion.type) {
+        // Standard assertions — pass through
+        case "contains":
+        case "contains-all":
+        case "contains-any":
+        case "equals":
+        case "is-json":
+        case "javascript":
+        case "python":
+        case "regex":
+        case "similar":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+            };
+        // LLM-graded assertions — add grader provider
+        case "g-eval":
+        case "llm-rubric":
+        case "model-graded-closedqa":
+        case "model-graded-factuality":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+                ...(options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            };
+        // Tool-use assertions are NOT valid for knowledge probes
+        case "skill-used":
+        case "tool-call-f1":
+        case "tool-called":
+        case "tool-input-matches":
+        case "tool-output-matches":
+            warnings.push(`Knowledge probe "${assertion.type}" assertion is not applicable — ` +
+                "knowledge probes don't use tools. Assertion skipped.");
+            return null;
+        default:
+            warnings.push(`Knowledge probe: unknown assertion type "${assertion.type}" — passed through`);
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+            };
+    }
+}
+// ---------------------------------------------------------------------------
+// ModeHandler adapter
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the knowledge-probe evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return KNOWLEDGE_PROBE_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "knowledge-probe") {
+            throw new Error(`Knowledge probe handler received task with mode "${task.mode ?? "undefined"}" — expected "knowledge-probe"`);
+        }
+        const result = compileKnowledgeProbeTask(task, { graderProvider: ctx.graderProvider, models: ctx.models });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+            extras: { metadata: result.metadata },
+        };
+    },
+};

package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts

@@ -0,0 +1,89 @@
+/**
+ * LiteracyModeHandler — compilation rules for `literacy` mode.
+ *
+ * This handler replaces the existing `generate-configs.ts` + `expand-tasks.ts`
+ * code path for literacy (documentation) evaluation. It compiles
+ * LiteracyTaskDefinition objects into Promptfoo structure:
+ *
+ * - Gold entry (with-docs prompt, canonical docs injected)
+ * - Baseline entry (without-docs prompt, empty docs)
+ * - Rubric template resolution from config/rubrics
+ * - Doc-coverage auto-generation when opted in
+ * - Structured dimension metadata on rubric assertions
+ *
+ * The handler accepts GeneralizedTaskDefinition, narrows to
+ * LiteracyTaskDefinition, and produces Promptfoo output.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ * @see packages/eval/src/pipeline/expand-tasks.ts — the legacy code path
+ */
+import type { LiteracyTaskDefinition, ModeHandler, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
+import { type LiteracyEvalSubMode } from "../../normalize-mode.js";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
+export declare const LITERACY_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
+/** Options for compiling a literacy task */
+export interface LiteracyCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Root directory (for resolving file:// doc paths) */
+    rootDir?: string;
+    /** Evaluation sub-mode — controls which entries are generated */
+    evalMode?: LiteracyEvalSubMode;
+    /** Model providers to include */
+    models?: {
+        id: string;
+        label: string;
+        config?: Record<string, unknown>;
+    }[];
+    /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
+    rubricConfig?: RubricConfig;
+}
+/** Minimal rubric config needed by the handler */
+export interface RubricConfig {
+    templates: Record<string, {
+        dimension?: string;
+        header: string;
+        scale: string[];
+        criteria_label?: string;
+    }>;
+}
+/** Result of compiling a single literacy task */
+export interface LiteracyCompileResult {
+    /** Promptfoo provider configs */
+    providers: PromptfooProvider[];
+    /** Compiled test cases (gold + optional baseline) */
+    tests: PromptfooTestCase[];
+    /** Prompts for evaluation */
+    prompts: PromptfooPrompt[];
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+export interface LiteracyValidationError {
+    field: string;
+    message: string;
+}
+/**
+ * Validate a literacy task definition.
+ */
+export declare function validateLiteracyTask(task: LiteracyTaskDefinition): LiteracyValidationError[];
+/**
+ * Compile a literacy task into Promptfoo configuration.
+ *
+ * Produces the same structure as the legacy expand-tasks.ts path:
+ * - Gold entry with with-docs prompt and canonical doc context
+ * - Baseline entry with without-docs prompt and empty docs
+ * - Rubric assertions with structured dimension metadata
+ */
+export declare function compileLiteracyTask(task: LiteracyTaskDefinition, options?: LiteracyCompileOptions): LiteracyCompileResult;
+/**
+ * ModeHandler-conformant export for the literacy evaluation mode.
+ *
+ * The pipeline looks up this handler via `registry.getMode("literacy")`
+ * and calls `handler.compileTask()`. The handler narrows the union to
+ * LiteracyTaskDefinition and delegates to `compileLiteracyTask()`.
+ *
+ * Note: The literacy handler's `evalMode` variant ("baseline" vs "agentic")
+ * is passed via `ctx.evalMode` — a literacy-specific extension of
+ * CompilationContext. The pipeline sets this when compiling literacy tasks.
+ */
+export declare const handler: ModeHandler;