@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js

@@ -0,0 +1,43 @@
+/**
+ * Agent harness mode handler — compiles AgentHarnessTaskDefinition into Promptfoo config.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
+ */
+import { compileAgentHarnessTask } from "./compiler.js";
+import { AGENT_HARNESS_PROMPT_TEMPLATES } from "./prompts.js";
+// Re-export public API
+export { AGENT_HARNESS_PROMPT_TEMPLATES } from "./prompts.js";
+export { TOOL_PRESETS } from "./tool-presets.js";
+export { validateAgentHarnessTask } from "./validation.js";
+export { mapAgentAssertion, buildFileExistsAssertion, buildFileContainsAssertion, buildCommandSucceedsAssertion, buildDiffMatchesAssertion, } from "./assertions.js";
+export { buildLifecycleExtensions, buildBeforeEachHook, buildAfterEachHook, buildSandboxConfig, } from "./sandbox.js";
+export { compileAgentHarnessTask, buildAgentProvider, buildAgentPrompts, buildAgentTestCases, } from "./compiler.js";
+// ---------------------------------------------------------------------------
+// ModeHandler adapter — wraps compileAgentHarnessTask for registry dispatch
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the agent-harness evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return AGENT_HARNESS_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "agent-harness") {
+            throw new Error(`Agent harness handler received task with mode "${task.mode ?? "undefined"}" — expected "agent-harness"`);
+        }
+        const result = compileAgentHarnessTask(task, {
+            graderProvider: ctx.graderProvider,
+            rootDir: ctx.rootDir,
+        });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+            extras: {
+                extensions: result.extensions,
+                sandboxConfig: result.sandboxConfig,
+            },
+        };
+    },
+};

package/dist/pipeline/compiler/mode-handlers/agent-harness/prompts.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Canonical prompt templates for agent-harness-mode evaluations.
+ *
+ * Handler-owned prompts for agent harness evaluations. Describes the task
+ * for autonomous agent execution within a sandboxed environment with file
+ * system and tool access.
+ */
+import type { PromptTemplate } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const AGENT_HARNESS_PROMPT_TEMPLATES: Record<string, PromptTemplate>;

package/dist/pipeline/compiler/mode-handlers/agent-harness/prompts.js

@@ -0,0 +1,29 @@
+/**
+ * Canonical prompt templates for agent-harness-mode evaluations.
+ *
+ * Handler-owned prompts for agent harness evaluations. Describes the task
+ * for autonomous agent execution within a sandboxed environment with file
+ * system and tool access.
+ */
+export const AGENT_HARNESS_PROMPT_TEMPLATES = {
+    "agent-harness": {
+        id: "agent-harness",
+        label: "Agent Harness Task",
+        template: `You are a coding agent working in a sandboxed environment. You have access to file system tools (read, write, edit) and a shell to complete the following task.
+
+## Task
+{{task}}
+
+## Instructions
+
+1. Read existing files to understand the project structure before making changes
+2. Implement a complete, working solution — no placeholders or TODOs
+3. Ensure all necessary imports and dependencies are included
+4. Verify your implementation compiles and passes any provided test commands
+5. Keep changes minimal and focused on the task
+
+Complete the implementation:
+`,
+        variables: ["task"],
+    },
+};

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Sandbox configuration and lifecycle extensions for agent harness tasks.
+ *
+ * Builds Promptfoo beforeEach/afterEach hooks for provisioning and
+ * tearing down sandbox working directories.
+ */
+import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooExtension, SandboxConfigMeta } from "./types.js";
+export declare function buildSandboxConfig(task: AgentHarnessTaskDefinition): SandboxConfigMeta;
+export declare function buildLifecycleExtensions(task: AgentHarnessTaskDefinition, sandboxConfig: SandboxConfigMeta): PromptfooExtension[];
+export declare function buildBeforeEachHook(taskId: string, config: SandboxConfigMeta): string;
+export declare function buildAfterEachHook(taskId: string): string;

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.js

@@ -0,0 +1,82 @@
+/**
+ * Sandbox configuration and lifecycle extensions for agent harness tasks.
+ *
+ * Builds Promptfoo beforeEach/afterEach hooks for provisioning and
+ * tearing down sandbox working directories.
+ */
+// ---------------------------------------------------------------------------
+// Sandbox configuration
+// ---------------------------------------------------------------------------
+export function buildSandboxConfig(task) {
+    return {
+        type: task.sandbox?.type ?? "tempdir",
+        image: task.sandbox?.image,
+        fixtures: task.fixtures ?? [],
+        limits: task.sandbox?.limits
+            ? {
+                cpus: task.sandbox.limits.cpus,
+                memoryBytes: task.sandbox.limits.memoryBytes,
+                networkAccess: task.sandbox.limits.networkAccess,
+            }
+            : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Lifecycle extensions
+// ---------------------------------------------------------------------------
+export function buildLifecycleExtensions(task, sandboxConfig) {
+    const extensions = [];
+    // beforeEach: provision sandbox + inject fixtures
+    extensions.push({
+        type: "beforeEach",
+        code: buildBeforeEachHook(task.id, sandboxConfig),
+    });
+    // afterEach: collect artifacts + teardown
+    extensions.push({
+        type: "afterEach",
+        code: buildAfterEachHook(task.id),
+    });
+    return extensions;
+}
+export function buildBeforeEachHook(taskId, config) {
+    return (`// beforeEach: provision sandbox for ${taskId}\n` +
+        `async function({ vars }) {\n` +
+        `  const { mkdirSync, writeFileSync } = require('fs');\n` +
+        `  const { tmpdir } = require('os');\n` +
+        `  const { resolve } = require('path');\n` +
+        `  const id = 'ailf-${taskId}-' + require('crypto').randomUUID().slice(0, 8);\n` +
+        `  const workDir = resolve(tmpdir(), id);\n` +
+        `  mkdirSync(workDir, { recursive: true });\n` +
+        `  vars.__workingDir = workDir;\n` +
+        `  vars.__sandboxId = id;\n` +
+        `  // Fixture list: ${JSON.stringify(config.fixtures)}\n` +
+        `}`);
+}
+export function buildAfterEachHook(taskId) {
+    return (`// afterEach: collect artifacts + teardown for ${taskId}\n` +
+        `async function({ vars }) {\n` +
+        `  const { rmSync, readdirSync, existsSync } = require('fs');\n` +
+        `  const workDir = vars.__workingDir;\n` +
+        `  if (workDir && existsSync(workDir)) {\n` +
+        `    try {\n` +
+        `      // Collect modified files list\n` +
+        `      const files = [];\n` +
+        `      function collect(dir, prefix) {\n` +
+        `        for (const e of readdirSync(dir, { withFileTypes: true })) {\n` +
+        `          const rel = prefix ? prefix + '/' + e.name : e.name;\n` +
+        `          if (e.isDirectory()) collect(require('path').resolve(dir, e.name), rel);\n` +
+        `          else files.push(rel);\n` +
+        `        }\n` +
+        `      }\n` +
+        `      collect(workDir, '');\n` +
+        `      vars.__artifacts = { modifiedFiles: files };\n` +
+        `    } finally {\n` +
+        `      // Guard: only delete directories under os.tmpdir()\n` +
+        `      const tmp = require('os').tmpdir();\n` +
+        `      if (require('path').resolve(workDir).startsWith(require('path').resolve(tmp))) {\n` +
+        `        rmSync(workDir, { recursive: true, force: true });\n` +
+        `      }\n` +
+        `    }\n` +
+        `  }\n` +
+        `}`);
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/tool-presets.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Predefined tool permission sets for agent harness evaluations.
+ */
+export declare const TOOL_PRESETS: Record<string, string[]>;

package/dist/pipeline/compiler/mode-handlers/agent-harness/tool-presets.js

@@ -0,0 +1,19 @@
+/**
+ * Predefined tool permission sets for agent harness evaluations.
+ */
+export const TOOL_PRESETS = {
+    coding: ["Bash", "Read", "Write", "Edit", "Glob", "Grep"],
+    "full-access": [
+        "Bash",
+        "Read",
+        "Write",
+        "Edit",
+        "Glob",
+        "Grep",
+        "WebSearch",
+        "WebFetch",
+        "TodoRead",
+        "TodoWrite",
+    ],
+    "read-only": ["Read", "Glob", "Grep", "WebSearch"],
+};

package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Shared types for the agent harness mode handler.
+ */
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { SandboxType } from "../../sandbox/sandbox-strategy.js";
+/** Options for compiling an agent harness task */
+export interface AgentHarnessCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Root directory for fixture resolution */
+    rootDir?: string;
+}
+/** Result of compiling a single agent harness task */
+export interface AgentHarnessCompileResult {
+    /** Promptfoo provider config */
+    providers: PromptfooProvider[];
+    /** Compiled test cases */
+    tests: PromptfooTestCase[];
+    /** Prompts for evaluation */
+    prompts: PromptfooPrompt[];
+    /** Promptfoo extensions for sandbox lifecycle */
+    extensions: PromptfooExtension[];
+    /** Sandbox configuration metadata */
+    sandboxConfig: SandboxConfigMeta;
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+/** Promptfoo extension hook */
+export interface PromptfooExtension {
+    type: "afterEach" | "beforeEach";
+    /** JavaScript code or module path for the hook */
+    code: string;
+}
+/** Metadata about sandbox configuration for this task */
+export interface SandboxConfigMeta {
+    type: SandboxType;
+    image?: string;
+    fixtures: string[];
+    limits?: {
+        cpus?: number;
+        memoryBytes?: number;
+        networkAccess?: boolean;
+    };
+}
+/** Validation errors for agent harness task definitions */
+export interface AgentHarnessValidationError {
+    field: string;
+    message: string;
+}

package/dist/pipeline/compiler/mode-handlers/agent-harness/types.js

@@ -0,0 +1,4 @@
+/**
+ * Shared types for the agent harness mode handler.
+ */
+export {};

package/dist/pipeline/compiler/mode-handlers/agent-harness/validation.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Validation for agent harness task definitions.
+ */
+import type { AgentHarnessTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { AgentHarnessValidationError } from "./types.js";
+/**
+ * Validate that an agent harness task definition has all required fields.
+ */
+export declare function validateAgentHarnessTask(task: AgentHarnessTaskDefinition): AgentHarnessValidationError[];

package/dist/pipeline/compiler/mode-handlers/agent-harness/validation.js

@@ -0,0 +1,16 @@
+/**
+ * Validation for agent harness task definitions.
+ */
+/**
+ * Validate that an agent harness task definition has all required fields.
+ */
+export function validateAgentHarnessTask(task) {
+    const errors = [];
+    if (!task.id) {
+        errors.push({ field: "id", message: "Task ID is required" });
+    }
+    if (!task.title) {
+        errors.push({ field: "title", message: "Task title is required" });
+    }
+    return errors;
+}

package/dist/pipeline/compiler/mode-handlers/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Mode handlers — per-mode compilation rules.
+ *
+ * Each handler translates mode-specific task definitions into
+ * Promptfoo-compatible configuration. The mode handler pattern
+ * is the extensibility mechanism for the compiler: adding a new
+ * evaluation mode means adding a new handler file.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+export { buildMCPAssertions, compileMCPTask, handler as mcpServerHandler, validateMCPTask, type MCPAssertionContext, type MCPCompileOptions, type MCPCompileResult, type MCPValidationError, } from "./mcp-server/index.js";
+export { compileLiteracyTask, handler as literacyHandler, validateLiteracyTask, type LiteracyCompileOptions, type LiteracyCompileResult, type LiteracyValidationError, type RubricConfig, } from "./literacy/index.js";
+export { compileKnowledgeProbeTask, handler as knowledgeProbeHandler, validateKnowledgeProbeTask, type KnowledgeProbeCompileOptions, type KnowledgeProbeCompileResult, type KnowledgeProbeMetadata, type KnowledgeProbeValidationError, } from "./knowledge-probe/index.js";
+export { compileAgentHarnessTask, handler as agentHarnessHandler, validateAgentHarnessTask, type AgentHarnessCompileOptions, type AgentHarnessCompileResult, type AgentHarnessValidationError, type PromptfooExtension, type SandboxConfigMeta, } from "./agent-harness/index.js";

package/dist/pipeline/compiler/mode-handlers/index.js

@@ -0,0 +1,19 @@
+/**
+ * Mode handlers — per-mode compilation rules.
+ *
+ * Each handler translates mode-specific task definitions into
+ * Promptfoo-compatible configuration. The mode handler pattern
+ * is the extensibility mechanism for the compiler: adding a new
+ * evaluation mode means adding a new handler file.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+// MCP Server mode
+export { buildMCPAssertions, compileMCPTask, handler as mcpServerHandler, validateMCPTask, } from "./mcp-server/index.js";
+// Literacy mode
+export { compileLiteracyTask, handler as literacyHandler, validateLiteracyTask, } from "./literacy/index.js";
+// Knowledge Probe mode
+export { compileKnowledgeProbeTask, handler as knowledgeProbeHandler, validateKnowledgeProbeTask, } from "./knowledge-probe/index.js";
+// Agent Harness mode
+export { compileAgentHarnessTask, handler as agentHarnessHandler, validateAgentHarnessTask, } from "./agent-harness/index.js";

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Assertion mapping for knowledge probe evaluations.
+ */
+import type { PromptfooAssertion } from "../../assertion-mapper.js";
+import type { KnowledgeProbeCompileOptions } from "./types.js";
+/**
+ * Map a raw knowledge probe assertion to a Promptfoo assertion.
+ *
+ * Tool-use assertions are rejected (knowledge probes don't use tools).
+ * LLM-graded assertions receive the configured grader provider.
+ * All other assertions are passed through.
+ */
+export declare function mapKnowledgeProbeAssertion(assertion: {
+    type: string;
+    [k: string]: unknown;
+}, options: KnowledgeProbeCompileOptions | undefined, warnings: string[]): PromptfooAssertion | null;

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.js

@@ -0,0 +1,61 @@
+/**
+ * Assertion mapping for knowledge probe evaluations.
+ */
+/**
+ * Map a raw knowledge probe assertion to a Promptfoo assertion.
+ *
+ * Tool-use assertions are rejected (knowledge probes don't use tools).
+ * LLM-graded assertions receive the configured grader provider.
+ * All other assertions are passed through.
+ */
+export function mapKnowledgeProbeAssertion(assertion, options, warnings) {
+    switch (assertion.type) {
+        // Standard assertions — pass through
+        case "contains":
+        case "contains-all":
+        case "contains-any":
+        case "equals":
+        case "is-json":
+        case "javascript":
+        case "python":
+        case "regex":
+        case "similar":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+            };
+        // LLM-graded assertions — add grader provider
+        case "g-eval":
+        case "llm-rubric":
+        case "model-graded-closedqa":
+        case "model-graded-factuality":
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+                ...(options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            };
+        // Tool-use assertions are NOT valid for knowledge probes
+        case "skill-used":
+        case "tool-call-f1":
+        case "tool-called":
+        case "tool-input-matches":
+        case "tool-output-matches":
+            warnings.push(`Knowledge probe "${assertion.type}" assertion is not applicable — ` +
+                "knowledge probes don't use tools. Assertion skipped.");
+            return null;
+        default:
+            warnings.push(`Knowledge probe: unknown assertion type "${assertion.type}" — passed through`);
+            return {
+                type: assertion.type,
+                ...("value" in assertion ? { value: assertion.value } : {}),
+            };
+    }
+}

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/compiler.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Core compilation logic for knowledge probe tasks.
+ *
+ * Compiles a knowledge probe task definition into Promptfoo configuration.
+ * This is intentionally minimal — knowledge probes map almost 1:1 to
+ * basic Promptfoo test cases. The AILF value-add is type-safe authoring,
+ * cross-model comparison, and score normalization.
+ */
+import type { KnowledgeProbeTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { KnowledgeProbeCompileOptions, KnowledgeProbeCompileResult } from "./types.js";
+/**
+ * Compile a knowledge probe task definition into Promptfoo configuration.
+ *
+ * This is intentionally minimal — knowledge probes map almost 1:1 to
+ * basic Promptfoo test cases. The AILF value-add is type-safe authoring,
+ * cross-model comparison, and score normalization.
+ */
+export declare function compileKnowledgeProbeTask(task: KnowledgeProbeTaskDefinition, options?: KnowledgeProbeCompileOptions): KnowledgeProbeCompileResult;

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/compiler.js

@@ -0,0 +1,112 @@
+/**
+ * Core compilation logic for knowledge probe tasks.
+ *
+ * Compiles a knowledge probe task definition into Promptfoo configuration.
+ * This is intentionally minimal — knowledge probes map almost 1:1 to
+ * basic Promptfoo test cases. The AILF value-add is type-safe authoring,
+ * cross-model comparison, and score normalization.
+ */
+import { mapKnowledgeProbeAssertion } from "./assertions.js";
+import { validateKnowledgeProbeTask } from "./validation.js";
+// ---------------------------------------------------------------------------
+// Public compilation entry point
+// ---------------------------------------------------------------------------
+/**
+ * Compile a knowledge probe task definition into Promptfoo configuration.
+ *
+ * This is intentionally minimal — knowledge probes map almost 1:1 to
+ * basic Promptfoo test cases. The AILF value-add is type-safe authoring,
+ * cross-model comparison, and score normalization.
+ */
+export function compileKnowledgeProbeTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateKnowledgeProbeTask(task);
+    for (const err of validationErrors) {
+        warnings.push(`Knowledge probe "${task.id}": ${err.field} — ${err.message}`);
+    }
+    // Build providers from model list (or use a default placeholder)
+    const providers = buildProviders(options);
+    // Build prompts — knowledge probes use a single no-docs prompt
+    const prompts = buildPrompts(task);
+    // Build test cases
+    const tests = buildTestCases(task, options, warnings);
+    // Build metadata
+    const metadata = {
+        mode: "knowledge-probe",
+        probeStrategy: task.probeStrategy ?? "breadth-first",
+        noDocContext: true,
+        retrievalMetrics: false,
+    };
+    return { providers, tests, prompts, metadata, warnings };
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+function buildProviders(options) {
+    if (options?.models && options.models.length > 0) {
+        return options.models.map((model) => ({
+            id: model.id,
+            label: model.label,
+            config: model.config,
+        }));
+    }
+    // No models specified — return empty (caller should provide models)
+    return [];
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+function buildPrompts(task) {
+    // Knowledge probes use a single prompt — no with-docs/without-docs split.
+    // The prompt IS the probe question.
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Knowledge probe: ${task.title}`;
+    const systemMessage = task.prompt?.systemMessage;
+    return [
+        {
+            id: "knowledge-probe",
+            label: `Probe: ${task.title}`,
+            raw: systemMessage
+                ? `[system]\n${systemMessage}\n\n[user]\n${String(promptText)}`
+                : String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+function buildTestCases(task, options, warnings) {
+    // Build assertions
+    const assertions = [];
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            const raw = assertion;
+            const mapped = mapKnowledgeProbeAssertion(raw, options, warnings);
+            if (mapped)
+                assertions.push(mapped);
+        }
+    }
+    // Build vars — intentionally no docs
+    const vars = {
+        task: task.prompt?.vars?.task ??
+            task.description ??
+            `Knowledge probe: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+        // Metadata for scoring pipeline
+        __mode: "knowledge-probe",
+        __probeStrategy: task.probeStrategy ?? "breadth-first",
+    };
+    // Explicitly do NOT include docs
+    // This is the defining characteristic of knowledge-probe mode
+    delete vars.docs;
+    return [
+        {
+            description: `${task.id} — ${task.title}`,
+            vars,
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        },
+    ];
+}

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.d.ts

@@ -0,0 +1,26 @@
+/**
+ * KnowledgeProbeModeHandler — compilation rules for `knowledge-probe` mode.
+ *
+ * The simplest mode handler. Knowledge probes measure raw model knowledge
+ * without documentation context, tool calling, or sandboxed execution.
+ * They answer: "What does this model know about X without any help?"
+ *
+ * Key properties:
+ * - No doc vars injected (intentionally empty)
+ * - Uses the without-docs prompt template (or custom prompt)
+ * - Standard LLM providers only (no agent SDKs, no MCP)
+ * - No retrieval metrics (precision/recall/F1 not applicable)
+ * - Results feed into the standard cross-model comparison pipeline
+ *
+ * This handler is the reference implementation for the mode handler pattern.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-5-knowledge-probe.md
+ * @see packages/core/src/types/generalized-task.ts — KnowledgeProbeTaskDefinition
+ */
+import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";
+export { compileKnowledgeProbeTask } from "./compiler.js";
+export { KNOWLEDGE_PROBE_PROMPT_TEMPLATES } from "./prompts.js";
+export type { KnowledgeProbeCompileOptions, KnowledgeProbeCompileResult, KnowledgeProbeMetadata, KnowledgeProbeValidationError, } from "./types.js";
+export { validateKnowledgeProbeTask } from "./validation.js";
+/** ModeHandler-conformant export for the knowledge-probe evaluation mode. */
+export declare const handler: ModeHandler;

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.js

@@ -0,0 +1,49 @@
+/**
+ * KnowledgeProbeModeHandler — compilation rules for `knowledge-probe` mode.
+ *
+ * The simplest mode handler. Knowledge probes measure raw model knowledge
+ * without documentation context, tool calling, or sandboxed execution.
+ * They answer: "What does this model know about X without any help?"
+ *
+ * Key properties:
+ * - No doc vars injected (intentionally empty)
+ * - Uses the without-docs prompt template (or custom prompt)
+ * - Standard LLM providers only (no agent SDKs, no MCP)
+ * - No retrieval metrics (precision/recall/F1 not applicable)
+ * - Results feed into the standard cross-model comparison pipeline
+ *
+ * This handler is the reference implementation for the mode handler pattern.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-5-knowledge-probe.md
+ * @see packages/core/src/types/generalized-task.ts — KnowledgeProbeTaskDefinition
+ */
+import { compileKnowledgeProbeTask } from "./compiler.js";
+import { KNOWLEDGE_PROBE_PROMPT_TEMPLATES } from "./prompts.js";
+// ---------------------------------------------------------------------------
+// Re-exports
+// ---------------------------------------------------------------------------
+export { compileKnowledgeProbeTask } from "./compiler.js";
+export { KNOWLEDGE_PROBE_PROMPT_TEMPLATES } from "./prompts.js";
+export { validateKnowledgeProbeTask } from "./validation.js";
+// ---------------------------------------------------------------------------
+// ModeHandler adapter
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the knowledge-probe evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return KNOWLEDGE_PROBE_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "knowledge-probe") {
+            throw new Error(`Knowledge probe handler received task with mode "${task.mode ?? "undefined"}" — expected "knowledge-probe"`);
+        }
+        const result = compileKnowledgeProbeTask(task, { graderProvider: ctx.graderProvider, models: ctx.models });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+            extras: { metadata: result.metadata },
+        };
+    },
+};

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/prompts.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Canonical knowledge probe prompt templates.
+ *
+ * Handler-owned prompts for knowledge probe evaluations. These ask factual
+ * questions without injecting documentation context — measuring raw model
+ * knowledge about Sanity concepts.
+ */
+import type { PromptTemplate } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const KNOWLEDGE_PROBE_PROMPT_TEMPLATES: Record<string, PromptTemplate>;

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/prompts.js

@@ -0,0 +1,28 @@
+/**
+ * Canonical knowledge probe prompt templates.
+ *
+ * Handler-owned prompts for knowledge probe evaluations. These ask factual
+ * questions without injecting documentation context — measuring raw model
+ * knowledge about Sanity concepts.
+ */
+export const KNOWLEDGE_PROBE_PROMPT_TEMPLATES = {
+    "knowledge-probe": {
+        id: "knowledge-probe",
+        label: "Knowledge Probe (No Docs)",
+        template: `Answer the following question about Sanity.io based on your existing knowledge. Do not search for or reference external documentation.
+
+## Question
+{{task}}
+
+## Instructions
+
+1. Answer based solely on what you already know
+2. Be specific — include API names, function signatures, and code examples where relevant
+3. If you are unsure about a detail, say so rather than guessing
+4. Provide a complete, accurate answer
+
+Your answer:
+`,
+        variables: ["task"],
+    },
+};