@sanity/ailf 2.0.2 → 2.1.0

package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js

@@ -1,245 +0,0 @@
-/**
- * KnowledgeProbeModeHandler — compilation rules for `knowledge-probe` mode.
- *
- * The simplest mode handler. Knowledge probes measure raw model knowledge
- * without documentation context, tool calling, or sandboxed execution.
- * They answer: "What does this model know about X without any help?"
- *
- * Key properties:
- * - No doc vars injected (intentionally empty)
- * - Uses the without-docs prompt template (or custom prompt)
- * - Standard LLM providers only (no agent SDKs, no MCP)
- * - No retrieval metrics (precision/recall/F1 not applicable)
- * - Results feed into the standard cross-model comparison pipeline
- *
- * This handler is the reference implementation for the mode handler pattern.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-5-knowledge-probe.md
- * @see packages/core/src/types/generalized-task.ts — KnowledgeProbeTaskDefinition
- */
-// ---------------------------------------------------------------------------
-// Canonical knowledge probe prompt templates
-// ---------------------------------------------------------------------------
-// Handler-owned prompts for knowledge probe evaluations. These ask factual
-// questions without injecting documentation context — measuring raw model
-// knowledge about Sanity concepts.
-export const KNOWLEDGE_PROBE_PROMPT_TEMPLATES = {
-    "knowledge-probe": {
-        id: "knowledge-probe",
-        label: "Knowledge Probe (No Docs)",
-        template: `Answer the following question about Sanity.io based on your existing knowledge. Do not search for or reference external documentation.
-
-## Question
-{{task}}
-
-## Instructions
-
-1. Answer based solely on what you already know
-2. Be specific — include API names, function signatures, and code examples where relevant
-3. If you are unsure about a detail, say so rather than guessing
-4. Provide a complete, accurate answer
-
-Your answer:
-`,
-        variables: ["task"],
-    },
-};
-/**
- * Validate that a knowledge probe task definition has all required fields.
- */
-export function validateKnowledgeProbeTask(task) {
-    const errors = [];
-    if (!task.id) {
-        errors.push({ field: "id", message: "Task ID is required" });
-    }
-    if (!task.title) {
-        errors.push({ field: "title", message: "Task title is required" });
-    }
-    // Knowledge probes must have either a prompt or a description
-    if (!task.prompt?.text && !task.prompt?.vars?.task && !task.description) {
-        errors.push({
-            field: "prompt",
-            message: "Knowledge probe tasks require either prompt.text, prompt.vars.task, " +
-                "or description — the question to ask the model",
-        });
-    }
-    return errors;
-}
-// ---------------------------------------------------------------------------
-// Compilation
-// ---------------------------------------------------------------------------
-/**
- * Compile a knowledge probe task definition into Promptfoo configuration.
- *
- * This is intentionally minimal — knowledge probes map almost 1:1 to
- * basic Promptfoo test cases. The AILF value-add is type-safe authoring,
- * cross-model comparison, and score normalization.
- */
-export function compileKnowledgeProbeTask(task, options) {
-    const warnings = [];
-    // Validate
-    const validationErrors = validateKnowledgeProbeTask(task);
-    for (const err of validationErrors) {
-        warnings.push(`Knowledge probe "${task.id}": ${err.field} — ${err.message}`);
-    }
-    // Build providers from model list (or use a default placeholder)
-    const providers = buildProviders(options);
-    // Build prompts — knowledge probes use a single no-docs prompt
-    const prompts = buildPrompts(task);
-    // Build test cases
-    const tests = buildTestCases(task, options, warnings);
-    // Build metadata
-    const metadata = {
-        mode: "knowledge-probe",
-        probeStrategy: task.probeStrategy ?? "breadth-first",
-        noDocContext: true,
-        retrievalMetrics: false,
-    };
-    return { providers, tests, prompts, metadata, warnings };
-}
-// ---------------------------------------------------------------------------
-// Provider assembly
-// ---------------------------------------------------------------------------
-function buildProviders(options) {
-    if (options?.models && options.models.length > 0) {
-        return options.models.map((model) => ({
-            id: model.id,
-            label: model.label,
-            config: model.config,
-        }));
-    }
-    // No models specified — return empty (caller should provide models)
-    return [];
-}
-// ---------------------------------------------------------------------------
-// Prompt assembly
-// ---------------------------------------------------------------------------
-function buildPrompts(task) {
-    // Knowledge probes use a single prompt — no with-docs/without-docs split.
-    // The prompt IS the probe question.
-    const promptText = task.prompt?.text ??
-        task.prompt?.vars?.task ??
-        task.description ??
-        `Knowledge probe: ${task.title}`;
-    const systemMessage = task.prompt?.systemMessage;
-    return [
-        {
-            id: "knowledge-probe",
-            label: `Probe: ${task.title}`,
-            raw: systemMessage
-                ? `[system]\n${systemMessage}\n\n[user]\n${String(promptText)}`
-                : String(promptText),
-        },
-    ];
-}
-// ---------------------------------------------------------------------------
-// Test case assembly
-// ---------------------------------------------------------------------------
-function buildTestCases(task, options, warnings) {
-    // Build assertions
-    const assertions = [];
-    if (task.assertions) {
-        for (const assertion of task.assertions) {
-            const raw = assertion;
-            const mapped = mapKnowledgeProbeAssertion(raw, options, warnings);
-            if (mapped)
-                assertions.push(mapped);
-        }
-    }
-    // Build vars — intentionally no docs
-    const vars = {
-        task: task.prompt?.vars?.task ??
-            task.description ??
-            `Knowledge probe: ${task.title}`,
-        ...(task.prompt?.vars ?? {}),
-        // Metadata for scoring pipeline
-        __mode: "knowledge-probe",
-        __probeStrategy: task.probeStrategy ?? "breadth-first",
-    };
-    // Explicitly do NOT include docs
-    // This is the defining characteristic of knowledge-probe mode
-    delete vars.docs;
-    return [
-        {
-            description: `${task.id} — ${task.title}`,
-            vars,
-            ...(assertions.length > 0 ? { assert: assertions } : {}),
-        },
-    ];
-}
-// ---------------------------------------------------------------------------
-// Assertion mapping
-// ---------------------------------------------------------------------------
-function mapKnowledgeProbeAssertion(assertion, options, warnings) {
-    switch (assertion.type) {
-        // Standard assertions — pass through
-        case "contains":
-        case "contains-all":
-        case "contains-any":
-        case "equals":
-        case "is-json":
-        case "javascript":
-        case "python":
-        case "regex":
-        case "similar":
-            return {
-                type: assertion.type,
-                ...("value" in assertion ? { value: assertion.value } : {}),
-                ...(typeof assertion.weight === "number"
-                    ? { weight: assertion.weight }
-                    : {}),
-            };
-        // LLM-graded assertions — add grader provider
-        case "g-eval":
-        case "llm-rubric":
-        case "model-graded-closedqa":
-        case "model-graded-factuality":
-            return {
-                type: assertion.type,
-                ...("value" in assertion ? { value: assertion.value } : {}),
-                ...(typeof assertion.weight === "number"
-                    ? { weight: assertion.weight }
-                    : {}),
-                ...(options?.graderProvider
-                    ? { provider: options.graderProvider }
-                    : {}),
-            };
-        // Tool-use assertions are NOT valid for knowledge probes
-        case "skill-used":
-        case "tool-call-f1":
-        case "tool-called":
-        case "tool-input-matches":
-        case "tool-output-matches":
-            warnings.push(`Knowledge probe "${assertion.type}" assertion is not applicable — ` +
-                "knowledge probes don't use tools. Assertion skipped.");
-            return null;
-        default:
-            warnings.push(`Knowledge probe: unknown assertion type "${assertion.type}" — passed through`);
-            return {
-                type: assertion.type,
-                ...("value" in assertion ? { value: assertion.value } : {}),
-            };
-    }
-}
-// ---------------------------------------------------------------------------
-// ModeHandler adapter
-// ---------------------------------------------------------------------------
-/** ModeHandler-conformant export for the knowledge-probe evaluation mode. */
-export const handler = {
-    getPrompts() {
-        return KNOWLEDGE_PROBE_PROMPT_TEMPLATES;
-    },
-    compileTask(task, ctx) {
-        if (!("mode" in task) || task.mode !== "knowledge-probe") {
-            throw new Error(`Knowledge probe handler received task with mode "${task.mode ?? "undefined"}" — expected "knowledge-probe"`);
-        }
-        const result = compileKnowledgeProbeTask(task, { graderProvider: ctx.graderProvider, models: ctx.models });
-        return {
-            providers: result.providers,
-            tests: result.tests,
-            prompts: result.prompts,
-            warnings: result.warnings,
-            extras: { metadata: result.metadata },
-        };
-    },
-};

package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts

@@ -1,89 +0,0 @@
-/**
- * LiteracyModeHandler — compilation rules for `literacy` mode.
- *
- * This handler replaces the existing `generate-configs.ts` + `expand-tasks.ts`
- * code path for literacy (documentation) evaluation. It compiles
- * LiteracyTaskDefinition objects into Promptfoo structure:
- *
- * - Gold entry (with-docs prompt, canonical docs injected)
- * - Baseline entry (without-docs prompt, empty docs)
- * - Rubric template resolution from config/rubrics
- * - Doc-coverage auto-generation when opted in
- * - Structured dimension metadata on rubric assertions
- *
- * The handler accepts GeneralizedTaskDefinition, narrows to
- * LiteracyTaskDefinition, and produces Promptfoo output.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
- * @see packages/eval/src/pipeline/expand-tasks.ts — the legacy code path
- */
-import type { LiteracyTaskDefinition, ModeHandler, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
-import { type LiteracyEvalSubMode } from "../../normalize-mode.js";
-import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
-export declare const LITERACY_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
-/** Options for compiling a literacy task */
-export interface LiteracyCompileOptions {
-    /** Grader provider for LLM-graded assertions */
-    graderProvider?: string;
-    /** Root directory (for resolving file:// doc paths) */
-    rootDir?: string;
-    /** Evaluation sub-mode — controls which entries are generated */
-    evalMode?: LiteracyEvalSubMode;
-    /** Model providers to include */
-    models?: {
-        id: string;
-        label: string;
-        config?: Record<string, unknown>;
-    }[];
-    /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
-    rubricConfig?: RubricConfig;
-}
-/** Minimal rubric config needed by the handler */
-export interface RubricConfig {
-    templates: Record<string, {
-        dimension?: string;
-        header: string;
-        scale: string[];
-        criteria_label?: string;
-    }>;
-}
-/** Result of compiling a single literacy task */
-export interface LiteracyCompileResult {
-    /** Promptfoo provider configs */
-    providers: PromptfooProvider[];
-    /** Compiled test cases (gold + optional baseline) */
-    tests: PromptfooTestCase[];
-    /** Prompts for evaluation */
-    prompts: PromptfooPrompt[];
-    /** Warnings generated during compilation */
-    warnings: string[];
-}
-export interface LiteracyValidationError {
-    field: string;
-    message: string;
-}
-/**
- * Validate a literacy task definition.
- */
-export declare function validateLiteracyTask(task: LiteracyTaskDefinition): LiteracyValidationError[];
-/**
- * Compile a literacy task into Promptfoo configuration.
- *
- * Produces the same structure as the legacy expand-tasks.ts path:
- * - Gold entry with with-docs prompt and canonical doc context
- * - Baseline entry with without-docs prompt and empty docs
- * - Rubric assertions with structured dimension metadata
- */
-export declare function compileLiteracyTask(task: LiteracyTaskDefinition, options?: LiteracyCompileOptions): LiteracyCompileResult;
-/**
- * ModeHandler-conformant export for the literacy evaluation mode.
- *
- * The pipeline looks up this handler via `registry.getMode("literacy")`
- * and calls `handler.compileTask()`. The handler narrows the union to
- * LiteracyTaskDefinition and delegates to `compileLiteracyTask()`.
- *
- * Note: The literacy handler's `evalMode` variant ("baseline" vs "agentic")
- * is passed via `ctx.evalMode` — a literacy-specific extension of
- * CompilationContext. The pipeline sets this when compiling literacy tasks.
- */
-export declare const handler: ModeHandler;