@sanity/ailf 2.0.0 → 2.0.2

package/dist/pipeline/compiler/ignore-fields.js

@@ -17,7 +17,7 @@
  * ]
  * ```
  *
- * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-4-agent-harness.md
  */
 // ---------------------------------------------------------------------------
 // Public API

package/dist/pipeline/compiler/index.d.ts

@@ -1,13 +1,10 @@
 /**
- * Config compiler — the heart of the new architecture.
+ * Config compiler — the heart of the compilation architecture.
  *
  * Converts task definitions from any source into a TaskGraph IR,
  * then compiles the graph into Promptfoo YAML configuration.
  *
- * This module coexists with the existing `generate-configs.ts` path.
- * Phase 7 will migrate callers to use the compiler exclusively.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 export { buildTaskGraph, detectCycle, type TaskGraphBuildOptions, type TaskGraphBuildResult, } from "./task-graph-builder.js";
 export { compileToPromptfoo, type CompilationResult, type CompiledPromptfooConfig, type PromptfooCompilerOptions, type PromptfooPrompt, type PromptfooProvider, type PromptfooTestCase, } from "./promptfoo-compiler.js";

package/dist/pipeline/compiler/index.js

@@ -1,13 +1,10 @@
 /**
- * Config compiler — the heart of the new architecture.
+ * Config compiler — the heart of the compilation architecture.
  *
  * Converts task definitions from any source into a TaskGraph IR,
  * then compiles the graph into Promptfoo YAML configuration.
  *
- * This module coexists with the existing `generate-configs.ts` path.
- * Phase 7 will migrate callers to use the compiler exclusively.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
 // TaskGraph builder
 export { buildTaskGraph, detectCycle, } from "./task-graph-builder.js";

package/dist/pipeline/compiler/literacy-bridge.d.ts

@@ -16,7 +16,7 @@
  * - Prompts from config/prompts are integrated
  * - TaskGraphBuilder validates the DAG, deduplicates, and orders tasks
  *
- * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
  */
 import type { LiteracyTaskDefinition } from "../../_vendor/ailf-core/index.d.ts";
 import { type LiteracyCompileResult } from "./mode-handlers/literacy/index.js";

package/dist/pipeline/compiler/literacy-bridge.js

@@ -16,7 +16,7 @@
  * - Prompts from config/prompts are integrated
  * - TaskGraphBuilder validates the DAG, deduplicates, and orders tasks
  *
- * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
  */
 import { compileLiteracyTask, } from "./mode-handlers/literacy/index.js";
 import { tryLoadConfigFile } from "./config-loader.js";

package/dist/pipeline/compiler/mode-bases/agent-harness.d.ts

@@ -4,7 +4,7 @@
  * Tests whether an autonomous agent can complete implementation tasks
  * end-to-end, including tool use, file creation, and code generation.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
 import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
 export declare function createAgentHarnessBase(): ModeBase;

package/dist/pipeline/compiler/mode-bases/agent-harness.js

@@ -4,7 +4,7 @@
  * Tests whether an autonomous agent can complete implementation tasks
  * end-to-end, including tool use, file creation, and code generation.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
 export function createAgentHarnessBase() {
     return {

package/dist/pipeline/compiler/mode-bases/knowledge-probe.d.ts

@@ -4,7 +4,7 @@
  * Tests what the model knows about a topic without providing documentation,
  * establishing a baseline of model knowledge.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
 import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
 export declare function createKnowledgeProbeBase(): ModeBase;

package/dist/pipeline/compiler/mode-bases/knowledge-probe.js

@@ -4,7 +4,7 @@
  * Tests what the model knows about a topic without providing documentation,
  * establishing a baseline of model knowledge.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
 export function createKnowledgeProbeBase() {
     return {

package/dist/pipeline/compiler/mode-bases/literacy.d.ts

@@ -6,7 +6,18 @@
  * like `sanity-literacy` target this mode base and add their own sources,
  * features, and doc fetcher.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
-import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
+import { type ModeBase, type ModelEntry } from "../../../_vendor/ailf-core/index.d.ts";
 export declare function createLiteracyModeBase(): ModeBase;
+/**
+ * Check whether a model participates in a specific literacy variant.
+ *
+ * A model matches if:
+ * 1. It's enrolled in the `literacy` eval mode (or has no `modes` field)
+ * 2. The variant is in its resolved variant set (defaults to all variants)
+ *
+ * This is the single source of truth for literacy variant matching —
+ * import this instead of reimplementing the pattern.
+ */
+export declare function modelMatchesLiteracyVariant(model: ModelEntry, variant: string): boolean;

package/dist/pipeline/compiler/mode-bases/literacy.js

@@ -6,9 +6,13 @@
  * like `sanity-literacy` target this mode base and add their own sources,
  * features, and doc fetcher.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
+import { modelMatchesMode, resolveModelVariants, } from "../../../_vendor/ailf-core/index.js";
 import { LITERACY_PROMPT_TEMPLATES } from "../mode-handlers/literacy/index.js";
+// ---------------------------------------------------------------------------
+// Mode base factory
+// ---------------------------------------------------------------------------
 export function createLiteracyModeBase() {
     return {
         mode: {
@@ -21,6 +25,28 @@ export function createLiteracyModeBase() {
                 "doc-coverage",
             ],
             handlerModule: "./mode-handlers/literacy/index.js",
+            variants: [
+                {
+                    id: "baseline",
+                    label: "Standard (baseline)",
+                    description: "Standard with-docs and without-docs evaluation prompts",
+                },
+                {
+                    id: "observed",
+                    label: "Observed (HTTP-instrumented)",
+                    description: "HTTP-instrumented evaluation that records model behavior",
+                },
+                {
+                    id: "agentic-naive",
+                    label: "Agentic (naive)",
+                    description: "Model uses tools to find docs with default system prompt",
+                },
+                {
+                    id: "agentic-optimized",
+                    label: "Agentic (optimized)",
+                    description: "Model uses tools to find docs with optimized system prompt",
+                },
+            ],
         },
         rubricTemplates: [
             {
@@ -76,3 +102,31 @@ export function createLiteracyModeBase() {
         promptTemplates: LITERACY_PROMPT_TEMPLATES,
     };
 }
+// ---------------------------------------------------------------------------
+// Shared variant matching helper
+// ---------------------------------------------------------------------------
+/** Lazily initialized mode base — avoids re-creating on every call */
+let _cachedBase;
+function getLiteracyModeBase() {
+    if (!_cachedBase)
+        _cachedBase = createLiteracyModeBase();
+    return _cachedBase;
+}
+/**
+ * Check whether a model participates in a specific literacy variant.
+ *
+ * A model matches if:
+ * 1. It's enrolled in the `literacy` eval mode (or has no `modes` field)
+ * 2. The variant is in its resolved variant set (defaults to all variants)
+ *
+ * This is the single source of truth for literacy variant matching —
+ * import this instead of reimplementing the pattern.
+ */
+export function modelMatchesLiteracyVariant(model, variant) {
+    if (!modelMatchesMode(model, "literacy"))
+        return false;
+    const variants = resolveModelVariants(model, getLiteracyModeBase());
+    if (!variants)
+        return true;
+    return variants.includes(variant);
+}

package/dist/pipeline/compiler/mode-bases/mcp-server.d.ts

@@ -4,7 +4,7 @@
  * Defines rubric templates and scoring for evaluating how well an LLM can
  * discover and use MCP server tools correctly.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
 import type { ModeBase } from "../../../_vendor/ailf-core/index.d.ts";
 export declare function createMcpServerModeBase(): ModeBase;

package/dist/pipeline/compiler/mode-bases/mcp-server.js

@@ -4,7 +4,7 @@
  * Defines rubric templates and scoring for evaluating how well an LLM can
  * discover and use MCP server tools correctly.
  *
- * @see docs/MODES.md
+ * @see docs/modes.md
  */
 export function createMcpServerModeBase() {
     return {

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.d.ts

@@ -1,7 +1,7 @@
 /**
  * Agent harness mode handler — compiles AgentHarnessTaskDefinition into Promptfoo config.
  *
- * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-4-agent-harness.md
  * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
  */
 import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js

@@ -1,7 +1,7 @@
 /**
  * Agent harness mode handler — compiles AgentHarnessTaskDefinition into Promptfoo config.
  *
- * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-4-agent-harness.md
  * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
  */
 import { compileAgentHarnessTask } from "./compiler.js";

package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts

@@ -0,0 +1,70 @@
+/**
+ * AgentHarnessModeHandler — compilation rules for `agent-harness` mode.
+ *
+ * Maps agent harness task definitions to Promptfoo configuration with:
+ * - Claude Agent SDK / OpenAI Codex SDK providers
+ * - Tool permission configuration (preset/allowed/disallowed)
+ * - Sandbox setup/teardown via Promptfoo extensions
+ * - Fixture provisioning into sandbox working directory
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ * @see packages/core/src/types/generalized-task.ts — AgentHarnessTaskDefinition
+ */
+import type { AgentHarnessTaskDefinition, ModeHandler, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
+import type { SandboxType } from "../sandbox/sandbox-strategy.js";
+export declare const AGENT_HARNESS_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
+/** Options for compiling an agent harness task */
+export interface AgentHarnessCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Root directory for fixture resolution */
+    rootDir?: string;
+}
+/** Result of compiling a single agent harness task */
+export interface AgentHarnessCompileResult {
+    /** Promptfoo provider config */
+    providers: PromptfooProvider[];
+    /** Compiled test cases */
+    tests: PromptfooTestCase[];
+    /** Prompts for evaluation */
+    prompts: PromptfooPrompt[];
+    /** Promptfoo extensions for sandbox lifecycle */
+    extensions: PromptfooExtension[];
+    /** Sandbox configuration metadata */
+    sandboxConfig: SandboxConfigMeta;
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+/** Promptfoo extension hook */
+export interface PromptfooExtension {
+    type: "afterEach" | "beforeEach";
+    /** JavaScript code or module path for the hook */
+    code: string;
+}
+/** Metadata about sandbox configuration for this task */
+export interface SandboxConfigMeta {
+    type: SandboxType;
+    image?: string;
+    fixtures: string[];
+    limits?: {
+        cpus?: number;
+        memoryBytes?: number;
+        networkAccess?: boolean;
+    };
+}
+/** Validation errors for agent harness task definitions */
+export interface AgentHarnessValidationError {
+    field: string;
+    message: string;
+}
+/**
+ * Validate that an agent harness task definition has all required fields.
+ */
+export declare function validateAgentHarnessTask(task: AgentHarnessTaskDefinition): AgentHarnessValidationError[];
+/**
+ * Compile an agent harness task definition into Promptfoo configuration.
+ */
+export declare function compileAgentHarnessTask(task: AgentHarnessTaskDefinition, options?: AgentHarnessCompileOptions): AgentHarnessCompileResult;
+/** ModeHandler-conformant export for the agent-harness evaluation mode. */
+export declare const handler: ModeHandler;