@sanity/ailf 2.0.1 → 2.1.0

package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js

@@ -1,485 +0,0 @@
-/**
- * AgentHarnessModeHandler — compilation rules for `agent-harness` mode.
- *
- * Maps agent harness task definitions to Promptfoo configuration with:
- * - Claude Agent SDK / OpenAI Codex SDK providers
- * - Tool permission configuration (preset/allowed/disallowed)
- * - Sandbox setup/teardown via Promptfoo extensions
- * - Fixture provisioning into sandbox working directory
- *
- * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
- * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
- */
-// ---------------------------------------------------------------------------
-// Canonical agent harness prompt templates
-// ---------------------------------------------------------------------------
-// Handler-owned prompts for agent harness evaluations. Describes the task
-// for autonomous agent execution within a sandboxed environment with file
-// system and tool access.
-export const AGENT_HARNESS_PROMPT_TEMPLATES = {
-    "agent-harness": {
-        id: "agent-harness",
-        label: "Agent Harness Task",
-        template: `You are a coding agent working in a sandboxed environment. You have access to file system tools (read, write, edit) and a shell to complete the following task.
-
-## Task
-{{task}}
-
-## Instructions
-
-1. Read existing files to understand the project structure before making changes
-2. Implement a complete, working solution — no placeholders or TODOs
-3. Ensure all necessary imports and dependencies are included
-4. Verify your implementation compiles and passes any provided test commands
-5. Keep changes minimal and focused on the task
-
-Complete the implementation:
-`,
-        variables: ["task"],
-    },
-};
-// ---------------------------------------------------------------------------
-// Tool permission presets
-// ---------------------------------------------------------------------------
-/** Predefined tool permission sets */
-const TOOL_PRESETS = {
-    coding: ["Bash", "Read", "Write", "Edit", "Glob", "Grep"],
-    "full-access": [
-        "Bash",
-        "Read",
-        "Write",
-        "Edit",
-        "Glob",
-        "Grep",
-        "WebSearch",
-        "WebFetch",
-        "TodoRead",
-        "TodoWrite",
-    ],
-    "read-only": ["Read", "Glob", "Grep", "WebSearch"],
-};
-/**
- * Validate that an agent harness task definition has all required fields.
- */
-export function validateAgentHarnessTask(task) {
-    const errors = [];
-    if (!task.id) {
-        errors.push({ field: "id", message: "Task ID is required" });
-    }
-    if (!task.title) {
-        errors.push({ field: "title", message: "Task title is required" });
-    }
-    return errors;
-}
-// ---------------------------------------------------------------------------
-// Compilation
-// ---------------------------------------------------------------------------
-/**
- * Compile an agent harness task definition into Promptfoo configuration.
- */
-export function compileAgentHarnessTask(task, options) {
-    const warnings = [];
-    // Validate
-    const validationErrors = validateAgentHarnessTask(task);
-    for (const err of validationErrors) {
-        warnings.push(`Agent harness task "${task.id}": ${err.field} — ${err.message}`);
-    }
-    // Build provider
-    const providers = buildAgentProvider(task, warnings);
-    // Build prompts
-    const prompts = buildAgentPrompts(task);
-    // Build test cases
-    const tests = buildAgentTestCases(task, options, warnings);
-    // Build sandbox extensions
-    const sandboxConfig = buildSandboxConfig(task);
-    const extensions = buildLifecycleExtensions(task, sandboxConfig);
-    return { providers, tests, prompts, extensions, sandboxConfig, warnings };
-}
-// ---------------------------------------------------------------------------
-// Provider assembly
-// ---------------------------------------------------------------------------
-function buildAgentProvider(task, _warnings) {
-    // Resolve tool permissions
-    const tools = resolveToolPermissions(task.tools);
-    const config = {};
-    if (tools.length > 0) {
-        config.allowedTools = tools;
-    }
-    if (task.sandbox) {
-        config.sandbox = {
-            type: task.sandbox.type,
-            ...(task.sandbox.image ? { image: task.sandbox.image } : {}),
-        };
-    }
-    // Default to Claude Agent SDK provider
-    return [
-        {
-            id: `agent:${task.id}`,
-            label: `Agent Harness: ${task.title}`,
-            config,
-        },
-    ];
-}
-/**
- * Resolve tool permissions from task config.
- *
- * Handles:
- * - Preset names ("coding", "read-only", "full-access")
- * - Explicit tool names ("Bash", "Read", "Write")
- * - Mixed arrays ["coding", "WebSearch"] → preset expansion + extras
- */
-function resolveToolPermissions(tools) {
-    if (!tools || tools.length === 0)
-        return [];
-    const resolved = new Set();
-    for (const tool of tools) {
-        const preset = TOOL_PRESETS[tool];
-        if (preset) {
-            for (const t of preset)
-                resolved.add(t);
-        }
-        else {
-            resolved.add(tool);
-        }
-    }
-    return [...resolved];
-}
-// ---------------------------------------------------------------------------
-// Prompt assembly
-// ---------------------------------------------------------------------------
-function buildAgentPrompts(task) {
-    const promptText = task.prompt?.text ??
-        task.prompt?.vars?.task ??
-        task.description ??
-        `Agent task: ${task.title}`;
-    return [
-        {
-            id: "agent-harness",
-            label: `Agent: ${task.title}`,
-            raw: String(promptText),
-        },
-    ];
-}
-// ---------------------------------------------------------------------------
-// Test case assembly
-// ---------------------------------------------------------------------------
-function buildAgentTestCases(task, options, warnings) {
-    const assertions = [];
-    if (task.assertions) {
-        for (const assertion of task.assertions) {
-            const mapped = mapAgentAssertion(assertion, options, warnings);
-            if (mapped)
-                assertions.push(mapped);
-        }
-    }
-    const vars = {
-        task: task.prompt?.vars?.task ?? task.description ?? `Complete: ${task.title}`,
-        ...(task.prompt?.vars ?? {}),
-        // Internal metadata for sandbox lifecycle hooks
-        __sandboxType: task.sandbox?.type ?? "tempdir",
-        __fixtures: task.fixtures ?? [],
-    };
-    const tests = [
-        {
-            description: `${task.id} — ${task.title}`,
-            vars,
-            ...(assertions.length > 0 ? { assert: assertions } : {}),
-        },
-    ];
-    // Multi-turn support
-    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
-        tests.push({
-            description: `${task.id} — ${task.title} [multi-turn]`,
-            vars: {
-                ...vars,
-                __multiTurn: task.multiTurn.turns,
-            },
-            ...(assertions.length > 0 ? { assert: assertions } : {}),
-        });
-    }
-    return tests;
-}
-// ---------------------------------------------------------------------------
-// Assertion mapping
-// ---------------------------------------------------------------------------
-function mapAgentAssertion(assertion, options, warnings) {
-    switch (assertion.type) {
-        case "file-exists":
-            return buildFileExistsAssertion(assertion);
-        case "file-contains":
-            return buildFileContainsAssertion(assertion);
-        case "command-succeeds":
-            return buildCommandSucceedsAssertion(assertion);
-        case "diff-matches":
-            return buildDiffMatchesAssertion(assertion);
-        // Standard assertions pass through
-        case "contains":
-        case "equals":
-        case "regex":
-        case "is-json":
-        case "javascript":
-        case "python":
-            return {
-                type: assertion.type,
-                ...("value" in assertion ? { value: assertion.value } : {}),
-                ...(typeof assertion.weight === "number"
-                    ? { weight: assertion.weight }
-                    : {}),
-            };
-        case "llm-rubric":
-            return {
-                type: "llm-rubric",
-                ...("value" in assertion ? { value: assertion.value } : {}),
-                ...(typeof assertion.weight === "number"
-                    ? { weight: assertion.weight }
-                    : {}),
-                ...(options?.graderProvider
-                    ? { provider: options.graderProvider }
-                    : {}),
-            };
-        default:
-            warnings.push(`Agent task: unknown assertion type "${assertion.type}" — passed through`);
-            return {
-                type: assertion.type,
-                ...("value" in assertion ? { value: assertion.value } : {}),
-            };
-    }
-}
-// ---------------------------------------------------------------------------
-// Agent-specific assertion builders
-// ---------------------------------------------------------------------------
-function buildFileExistsAssertion(assertion) {
-    const filePath = String(assertion.value ?? "");
-    // Use JSON.stringify for all interpolated values in generated JS to
-    // prevent broken strings from filePaths containing quotes/backslashes
-    const safeFilePath = JSON.stringify(filePath);
-    return {
-        type: "javascript",
-        value: `// file-exists: ${filePath}\n` +
-            `(function() {\n` +
-            `  const fs = require('fs');\n` +
-            `  const path = require('path');\n` +
-            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
-            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
-            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
-            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
-            `  }\n` +
-            `  const exists = fs.existsSync(target);\n` +
-            `  return {\n` +
-            `    pass: exists,\n` +
-            `    score: exists ? 1 : 0,\n` +
-            `    reason: exists\n` +
-            `      ? 'File exists: ' + ${safeFilePath}\n` +
-            `      : 'Expected file not found: ' + ${safeFilePath},\n` +
-            `  };\n` +
-            `})()`,
-        ...(typeof assertion.weight === "number"
-            ? { weight: assertion.weight }
-            : {}),
-    };
-}
-function buildFileContainsAssertion(assertion) {
-    const config = assertion.value;
-    const filePath = config?.path ?? "";
-    const expectedContent = config?.content ?? "";
-    const safeFilePath = JSON.stringify(filePath);
-    return {
-        type: "javascript",
-        value: `// file-contains: ${filePath}\n` +
-            `(function() {\n` +
-            `  const fs = require('fs');\n` +
-            `  const path = require('path');\n` +
-            `  const workDir = path.resolve(context.vars.__workingDir || '.');\n` +
-            `  const target = path.resolve(workDir, ${safeFilePath});\n` +
-            `  if (!target.startsWith(workDir + path.sep) && target !== workDir) {\n` +
-            `    return { pass: false, score: 0, reason: 'Path traversal: ' + ${safeFilePath} + ' escapes sandbox' };\n` +
-            `  }\n` +
-            `  if (!fs.existsSync(target)) {\n` +
-            `    return { pass: false, score: 0, reason: 'File not found: ' + ${safeFilePath} };\n` +
-            `  }\n` +
-            `  const content = fs.readFileSync(target, 'utf-8');\n` +
-            `  const contains = content.includes(${JSON.stringify(expectedContent)});\n` +
-            `  return {\n` +
-            `    pass: contains,\n` +
-            `    score: contains ? 1 : 0,\n` +
-            `    reason: contains\n` +
-            `      ? 'File contains expected content'\n` +
-            `      : 'File does not contain expected content',\n` +
-            `  };\n` +
-            `})()`,
-        ...(typeof assertion.weight === "number"
-            ? { weight: assertion.weight }
-            : {}),
-    };
-}
-/**
- * SECURITY: Trusted-input boundary.
- *
- * The `command-succeeds` assertion executes an arbitrary shell command
- * inside the sandbox's working directory. The command string comes from
- * task definitions (YAML or TypeScript config files), which are authored
- * by developers — not from user input or LLM output.
- *
- * This is intentional: the assertion is designed to verify agent output
- * by running build/test commands (e.g., "npm test", "tsc --noEmit").
- *
- * Task definitions are the trust boundary. If you accept task definitions
- * from untrusted sources, validate commands against an allowlist first.
- */
-function buildCommandSucceedsAssertion(assertion) {
-    const command = String(assertion.value ?? "");
-    return {
-        type: "javascript",
-        value: `// command-succeeds: ${command}\n` +
-            `(function() {\n` +
-            `  const { execSync } = require('child_process');\n` +
-            `  const workDir = context.vars.__workingDir || '.';\n` +
-            `  try {\n` +
-            `    execSync(${JSON.stringify(command)}, { cwd: workDir, timeout: 30000 });\n` +
-            `    return { pass: true, score: 1, reason: 'Command succeeded: ' + ${JSON.stringify(command)} };\n` +
-            `  } catch (err) {\n` +
-            `    return {\n` +
-            `      pass: false,\n` +
-            `      score: 0,\n` +
-            `      reason: 'Command failed: ' + (err.message || err),\n` +
-            `    };\n` +
-            `  }\n` +
-            `})()`,
-        ...(typeof assertion.weight === "number"
-            ? { weight: assertion.weight }
-            : {}),
-    };
-}
-function buildDiffMatchesAssertion(assertion) {
-    const expected = assertion.value;
-    return {
-        type: "javascript",
-        value: `// diff-matches\n` +
-            `(function() {\n` +
-            `  const { execSync } = require('child_process');\n` +
-            `  const workDir = context.vars.__workingDir || '.';\n` +
-            `  try {\n` +
-            `    const diff = execSync('git diff', { cwd: workDir, encoding: 'utf-8' });\n` +
-            `    const expected = ${JSON.stringify(expected)};\n` +
-            `    if (typeof expected === 'string') {\n` +
-            `      const contains = diff.includes(expected);\n` +
-            `      return {\n` +
-            `        pass: contains,\n` +
-            `        score: contains ? 1 : 0,\n` +
-            `        reason: contains ? 'Diff matches expected pattern' : 'Diff does not match',\n` +
-            `      };\n` +
-            `    }\n` +
-            `    return { pass: diff.length > 0, score: diff.length > 0 ? 1 : 0, reason: 'Diff exists' };\n` +
-            `  } catch (err) {\n` +
-            `    return { pass: false, score: 0, reason: 'Failed to get diff: ' + err.message };\n` +
-            `  }\n` +
-            `})()`,
-        ...(typeof assertion.weight === "number"
-            ? { weight: assertion.weight }
-            : {}),
-    };
-}
-// ---------------------------------------------------------------------------
-// Sandbox configuration
-// ---------------------------------------------------------------------------
-function buildSandboxConfig(task) {
-    return {
-        type: task.sandbox?.type ?? "tempdir",
-        image: task.sandbox?.image,
-        fixtures: task.fixtures ?? [],
-        limits: task.sandbox?.limits
-            ? {
-                cpus: task.sandbox.limits.cpus,
-                memoryBytes: task.sandbox.limits.memoryBytes,
-                networkAccess: task.sandbox.limits.networkAccess,
-            }
-            : undefined,
-    };
-}
-// ---------------------------------------------------------------------------
-// Lifecycle extensions
-// ---------------------------------------------------------------------------
-function buildLifecycleExtensions(task, sandboxConfig) {
-    const extensions = [];
-    // beforeEach: provision sandbox + inject fixtures
-    extensions.push({
-        type: "beforeEach",
-        code: buildBeforeEachHook(task.id, sandboxConfig),
-    });
-    // afterEach: collect artifacts + teardown
-    extensions.push({
-        type: "afterEach",
-        code: buildAfterEachHook(task.id),
-    });
-    return extensions;
-}
-function buildBeforeEachHook(taskId, config) {
-    return (`// beforeEach: provision sandbox for ${taskId}\n` +
-        `async function({ vars }) {\n` +
-        `  const { mkdirSync, writeFileSync } = require('fs');\n` +
-        `  const { tmpdir } = require('os');\n` +
-        `  const { resolve } = require('path');\n` +
-        `  const id = 'ailf-${taskId}-' + require('crypto').randomUUID().slice(0, 8);\n` +
-        `  const workDir = resolve(tmpdir(), id);\n` +
-        `  mkdirSync(workDir, { recursive: true });\n` +
-        `  vars.__workingDir = workDir;\n` +
-        `  vars.__sandboxId = id;\n` +
-        `  // Fixture list: ${JSON.stringify(config.fixtures)}\n` +
-        `}`);
-}
-function buildAfterEachHook(taskId) {
-    return (`// afterEach: collect artifacts + teardown for ${taskId}\n` +
-        `async function({ vars }) {\n` +
-        `  const { rmSync, readdirSync, existsSync } = require('fs');\n` +
-        `  const workDir = vars.__workingDir;\n` +
-        `  if (workDir && existsSync(workDir)) {\n` +
-        `    try {\n` +
-        `      // Collect modified files list\n` +
-        `      const files = [];\n` +
-        `      function collect(dir, prefix) {\n` +
-        `        for (const e of readdirSync(dir, { withFileTypes: true })) {\n` +
-        `          const rel = prefix ? prefix + '/' + e.name : e.name;\n` +
-        `          if (e.isDirectory()) collect(require('path').resolve(dir, e.name), rel);\n` +
-        `          else files.push(rel);\n` +
-        `        }\n` +
-        `      }\n` +
-        `      collect(workDir, '');\n` +
-        `      vars.__artifacts = { modifiedFiles: files };\n` +
-        `    } finally {\n` +
-        `      // Guard: only delete directories under os.tmpdir()\n` +
-        `      const tmp = require('os').tmpdir();\n` +
-        `      if (require('path').resolve(workDir).startsWith(require('path').resolve(tmp))) {\n` +
-        `        rmSync(workDir, { recursive: true, force: true });\n` +
-        `      }\n` +
-        `    }\n` +
-        `  }\n` +
-        `}`);
-}
-// ---------------------------------------------------------------------------
-// ModeHandler adapter
-// ---------------------------------------------------------------------------
-/** ModeHandler-conformant export for the agent-harness evaluation mode. */
-export const handler = {
-    getPrompts() {
-        return AGENT_HARNESS_PROMPT_TEMPLATES;
-    },
-    compileTask(task, ctx) {
-        if (!("mode" in task) || task.mode !== "agent-harness") {
-            throw new Error(`Agent harness handler received task with mode "${task.mode ?? "undefined"}" — expected "agent-harness"`);
-        }
-        const result = compileAgentHarnessTask(task, {
-            graderProvider: ctx.graderProvider,
-            rootDir: ctx.rootDir,
-        });
-        return {
-            providers: result.providers,
-            tests: result.tests,
-            prompts: result.prompts,
-            warnings: result.warnings,
-            extras: {
-                extensions: result.extensions,
-                sandboxConfig: result.sandboxConfig,
-            },
-        };
-    },
-};

package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts

@@ -1,76 +0,0 @@
-/**
- * KnowledgeProbeModeHandler — compilation rules for `knowledge-probe` mode.
- *
- * The simplest mode handler. Knowledge probes measure raw model knowledge
- * without documentation context, tool calling, or sandboxed execution.
- * They answer: "What does this model know about X without any help?"
- *
- * Key properties:
- * - No doc vars injected (intentionally empty)
- * - Uses the without-docs prompt template (or custom prompt)
- * - Standard LLM providers only (no agent SDKs, no MCP)
- * - No retrieval metrics (precision/recall/F1 not applicable)
- * - Results feed into the standard cross-model comparison pipeline
- *
- * This handler is the reference implementation for the mode handler pattern.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-5-knowledge-probe.md
- * @see packages/core/src/types/generalized-task.ts — KnowledgeProbeTaskDefinition
- */
-import type { KnowledgeProbeTaskDefinition, ModeHandler, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
-import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
-export declare const KNOWLEDGE_PROBE_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
-/** Options for compiling a knowledge probe task */
-export interface KnowledgeProbeCompileOptions {
-    /** Grader provider for LLM-graded assertions */
-    graderProvider?: string;
-    /** Model registry — knowledge probes run across all configured models */
-    models?: {
-        id: string;
-        label: string;
-        config?: Record<string, unknown>;
-    }[];
-}
-/** Result of compiling a single knowledge probe task */
-export interface KnowledgeProbeCompileResult {
-    /** Promptfoo provider configs (one per model) */
-    providers: PromptfooProvider[];
-    /** Compiled test cases */
-    tests: PromptfooTestCase[];
-    /** Prompts for evaluation */
-    prompts: PromptfooPrompt[];
-    /** Mode metadata for cross-model comparison */
-    metadata: KnowledgeProbeMetadata;
-    /** Warnings generated during compilation */
-    warnings: string[];
-}
-/** Metadata attached to knowledge probe results for comparison */
-export interface KnowledgeProbeMetadata {
-    /** Evaluation mode identifier */
-    mode: "knowledge-probe";
-    /** Probe strategy used */
-    probeStrategy: string;
-    /** Whether doc context was intentionally excluded */
-    noDocContext: true;
-    /** Whether retrieval metrics are applicable */
-    retrievalMetrics: false;
-}
-/** Validation errors for knowledge probe task definitions */
-export interface KnowledgeProbeValidationError {
-    field: string;
-    message: string;
-}
-/**
- * Validate that a knowledge probe task definition has all required fields.
- */
-export declare function validateKnowledgeProbeTask(task: KnowledgeProbeTaskDefinition): KnowledgeProbeValidationError[];
-/**
- * Compile a knowledge probe task definition into Promptfoo configuration.
- *
- * This is intentionally minimal — knowledge probes map almost 1:1 to
- * basic Promptfoo test cases. The AILF value-add is type-safe authoring,
- * cross-model comparison, and score normalization.
- */
-export declare function compileKnowledgeProbeTask(task: KnowledgeProbeTaskDefinition, options?: KnowledgeProbeCompileOptions): KnowledgeProbeCompileResult;
-/** ModeHandler-conformant export for the knowledge-probe evaluation mode. */
-export declare const handler: ModeHandler;