@sanity/ailf 1.0.0 → 2.0.1

package/dist/commands/validate-tasks.js

@@ -1,7 +1,7 @@
 /**
- * validate-tasks command — standalone validation of repo-based task YAML files.
+ * validate-tasks command — standalone validation of task files.
  *
- * Validates .ailf/tasks/*.yaml files against the RepoTaskSchema without
+ * Validates .ailf/tasks/*.yaml files against the CanonicalTaskSchema without
  * running the full pipeline. Useful for pre-commit hooks and CI checks
  * in external repos.
  *
@@ -16,11 +16,11 @@ import { existsSync, readdirSync, readFileSync } from "fs";
 import { resolve, relative } from "path";
 import { Command } from "commander";
 import { load } from "js-yaml";
-import { parseRepoTaskFile } from "../adapters/task-sources/repo-schemas.js";
-import { validateRepoTasks, formatValidationResult, } from "../adapters/task-sources/repo-validation.js";
+import { detectLegacyFieldNames, parseCanonicalTaskFile, } from "../adapters/task-sources/repo-schemas.js";
+import { validateCanonicalTasks, formatValidationResult, } from "../adapters/task-sources/repo-validation.js";
 export function createValidateTasksCommand() {
     return new Command("validate-tasks")
-        .description("Validate repo-based task YAML files (.ailf/tasks/) against the schema")
+        .description("Validate task YAML files (.ailf/tasks/) against the canonical schema")
         .argument("[path]", "Path to tasks directory (default: .ailf/tasks/)", ".ailf/tasks")
         .option("--strict", "Treat warnings as errors", false)
         .action(async (tasksPath, opts) => {
@@ -29,12 +29,12 @@ export function createValidateTasksCommand() {
         const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
         const resolvedPath = resolve(callerCwd, tasksPath);
         if (!existsSync(resolvedPath)) {
-            console.error(`❌ Directory not found: ${resolvedPath}`);
+            console.error(`Directory not found: ${resolvedPath}`);
             process.exit(1);
         }
         const yamlFiles = readdirSync(resolvedPath).filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."));
         if (yamlFiles.length === 0) {
-            console.error(`❌ No YAML files found in ${resolvedPath}`);
+            console.error(`No YAML files found in ${resolvedPath}`);
             process.exit(1);
         }
         console.log(`\nValidating ${yamlFiles.length} task file(s) in ${relative(process.cwd(), resolvedPath)}/\n`);
@@ -50,25 +50,36 @@ export function createValidateTasksCommand() {
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                console.error(`  ❌ ${file}: YAML parse error`);
+                console.error(`  ${file}: YAML parse error`);
                 console.error(`     ${msg}\n`);
                 hasErrors = true;
                 continue;
             }
             if (!Array.isArray(parsed)) {
-                console.error(`  ❌ ${file}: Expected a YAML array of task definitions`);
+                console.error(`  ${file}: Expected a YAML array of task definitions`);
+                hasErrors = true;
+                continue;
+            }
+            // Detect legacy field names before Zod validation
+            const legacyWarnings = detectLegacyFieldNames(parsed, file);
+            if (legacyWarnings.length > 0) {
+                console.error(`  ${file}: Uses legacy field names`);
+                for (const w of legacyWarnings) {
+                    console.error(`     ${w}`);
+                }
+                console.error();
                 hasErrors = true;
                 continue;
             }
             try {
-                const tasks = parseRepoTaskFile(parsed, file);
-                console.log(`  ✅ ${file}: ${tasks.length} task${tasks.length === 1 ? "" : "s"} valid`);
+                const tasks = parseCanonicalTaskFile(parsed, file);
+                console.log(`  ${file}: ${tasks.length} task${tasks.length === 1 ? "" : "s"} valid`);
                 totalTasks += tasks.length;
                 allTasks.push(...tasks);
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                console.error(`  ❌ ${file}: Schema validation failed`);
+                console.error(`  ${file}: Schema validation failed`);
                 console.error(`${msg
                     .split("\n")
                     .map((l) => `     ${l}`)
@@ -79,7 +90,7 @@ export function createValidateTasksCommand() {
         // Run semantic validation on all parsed tasks
         if (allTasks.length > 0) {
             console.log(); // blank line
-            const semanticResult = validateRepoTasks(allTasks);
+            const semanticResult = validateCanonicalTasks(allTasks);
             const formatted = formatValidationResult(semanticResult);
             console.log(formatted);
             if (!semanticResult.valid) {
@@ -87,10 +98,10 @@ export function createValidateTasksCommand() {
             }
             if (opts.strict && semanticResult.warnings.length > 0) {
                 hasErrors = true;
-                console.log("\n  ⚠️  --strict mode: warnings treated as errors");
+                console.log("\n  --strict mode: warnings treated as errors");
             }
         }
-        console.log(`\n${hasErrors ? "❌" : "✅"} ${totalTasks} task${totalTasks === 1 ? "" : "s"} across ${yamlFiles.length} file${yamlFiles.length === 1 ? "" : "s"}\n`);
+        console.log(`\n${hasErrors ? "FAIL" : "OK"} ${totalTasks} task${totalTasks === 1 ? "" : "s"} across ${yamlFiles.length} file${yamlFiles.length === 1 ? "" : "s"}\n`);
         process.exit(hasErrors ? 1 : 0);
     });
 }

package/dist/composition-root.d.ts

@@ -13,14 +13,25 @@
  * - After: one factory, one place to change adapter wiring
  *
  * @see packages/core/src/ports/context.ts — AppContext interface
- * @see docs/exec-plans/ports-and-adapters/phase-7-composition-root.md
+ * @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import { type AppContext, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+import { type AppContext, type AssertionRegistration, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
 /**
  * Create a fully wired AppContext from resolved configuration.
  *
  * Every adapter is constructed here and nowhere else (outside of tests).
- * Swapping an adapter (e.g., YamlTaskSource → ContentLakeTaskSource)
- * is a one-line change in this function.
+ * Swapping an adapter is a one-line change in this function.
  */
 export declare function createAppContext(config: ResolvedConfig): AppContext;
+/**
+ * Generic Promptfoo assertion types available to all evaluation modes.
+ *
+ * These are evaluation primitives (text matching, JSON validation, LLM grading)
+ * that aren't specific to any mode or domain. They're registered before any
+ * preset so every mode has access to them.
+ *
+ * `compatibleModes` is omitted — when undefined, the assertion is compatible
+ * with all modes. Mode-specific assertions can be registered by presets with
+ * explicit mode whitelists.
+ */
+export declare const FRAMEWORK_ASSERTIONS: AssertionRegistration[];

package/dist/composition-root.js

@@ -13,14 +13,18 @@
  * - After: one factory, one place to change adapter wiring
  *
  * @see packages/core/src/ports/context.ts — AppContext interface
- * @see docs/exec-plans/ports-and-adapters/phase-7-composition-root.md
+ * @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import { InMemoryPluginRegistry, } from "./_vendor/ailf-core/index.js";
+import { join } from "node:path";
+import { InMemoryPluginRegistry, NoOpArtifactCollector, } from "./_vendor/ailf-core/index.js";
+import { FilesystemArtifactCollector } from "./artifact-capture/filesystem-collector.js";
 import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
+import { loadExternalPresets } from "./pipeline/compiler/preset-loader.js";
 import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
 import { PromptfooEvalAdapter } from "./adapters/eval-runners/promptfoo-eval-adapter.js";
 import { ConsoleLogger, JsonLogger, QuietLogger, } from "./adapters/loggers/index.js";
-import { CompositeTaskSource, ContentLakeTaskSource, RepoTaskSource, YamlTaskSource, } from "./adapters/task-sources/index.js";
+import { CompositeTaskSource, ContentLakeTaskSource, RepoTaskSource, } from "./adapters/task-sources/index.js";
+import { createAgentHarnessBase, createKnowledgeProbeBase, createLiteracyModeBase, createMcpServerModeBase, } from "./pipeline/compiler/mode-bases/index.js";
 import { createSanityLiteracyPreset } from "./pipeline/compiler/presets/index.js";
 import { getSanityClient } from "./sanity/client.js";
 import { ReportStore } from "./report-store.js";
@@ -29,8 +33,7 @@ import { loadSinks } from "./sinks/index.js";
  * Create a fully wired AppContext from resolved configuration.
  *
  * Every adapter is constructed here and nowhere else (outside of tests).
- * Swapping an adapter (e.g., YamlTaskSource → ContentLakeTaskSource)
- * is a one-line change in this function.
+ * Swapping an adapter is a one-line change in this function.
  */
 export function createAppContext(config) {
     // Logger — selected by env var preferences
@@ -39,10 +42,12 @@ export function createAppContext(config) {
     const cache = config.noCache ? undefined : createCache(config);
     // Task source — selected by config.taskSourceType
     const taskSource = createTaskSource(config);
-    // Plugin registry — mode handlers, assertions, rubric templates, doc fetcher.
-    // The Sanity preset is registered here with config.rootDir so its doc fetcher
-    // factory resolves paths relative to the eval package root (not cwd).
-    const registry = createRegistry(config.rootDir);
+    // Plugin registry — mode bases, assertions, presets, doc fetcher.
+    // External presets from config are loaded and registered after built-ins.
+    const externalPresets = config.presets && config.presets.length > 0
+        ? loadExternalPresets(config.presets, config.rootDir)
+        : undefined;
+    const registry = createRegistry(config.rootDir, externalPresets);
     // Doc fetcher — provided by the registered preset's factory
     const docFetcherFactory = registry.getDocFetcherFactory();
     const docFetcher = docFetcherFactory ? docFetcherFactory() : undefined;
@@ -52,8 +57,23 @@ export function createAppContext(config) {
     const reportStore = createReportStore(config);
     // Sinks — loaded from config/sinks
     const sinks = loadSinks();
+    // Artifact collector — no-op by default, filesystem when --capture is set
+    const collector = config.captureEnabled
+        ? new FilesystemArtifactCollector({
+            captureDir: config.captureDir ?? join(config.rootDir, "results", "captures"),
+            mode: config.mode,
+            compress: config.captureCompress ?? true,
+            extras: config.captureExtras ?? true,
+            pipeline: {
+                variant: config.variant,
+                source: config.source,
+                areas: config.areas,
+            },
+        })
+        : new NoOpArtifactCollector();
     return {
         cache,
+        collector,
         config,
         docFetcher,
         evalRunner,
@@ -101,15 +121,12 @@ function createTaskSource(config) {
         }
         return new RepoTaskSource(config.repoTasksPath);
     }
-    // Primary source — selected by config.taskSourceType
-    // Content Lake tasks may require the report token for access.
-    const primary = config.taskSourceType === "yaml"
-        ? new YamlTaskSource(config.rootDir)
-        : new ContentLakeTaskSource(getSanityClient({
-            token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
-                process.env.SANITY_API_TOKEN ??
-                undefined,
-        }));
+    // Primary source — Content Lake (the only non-repo source remaining)
+    const primary = new ContentLakeTaskSource(getSanityClient({
+        token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
+            process.env.SANITY_API_TOKEN ??
+            undefined,
+    }));
     // If repo tasks path is set, combine primary + repo sources.
     // This is the "augment" mode — repo tasks extend the primary source.
     if (config.repoTasksPath) {
@@ -121,58 +138,86 @@ function createTaskSource(config) {
     return primary;
 }
 // ---------------------------------------------------------------------------
-// Built-in mode registrations for non-literacy modes
+// Layer 0: Framework built-in assertions
 // ---------------------------------------------------------------------------
-const BUILT_IN_MODES = [
+/**
+ * Generic Promptfoo assertion types available to all evaluation modes.
+ *
+ * These are evaluation primitives (text matching, JSON validation, LLM grading)
+ * that aren't specific to any mode or domain. They're registered before any
+ * preset so every mode has access to them.
+ *
+ * `compatibleModes` is omitted — when undefined, the assertion is compatible
+ * with all modes. Mode-specific assertions can be registered by presets with
+ * explicit mode whitelists.
+ */
+export const FRAMEWORK_ASSERTIONS = [
+    {
+        type: "contains",
+        label: "Contains text",
+        handlerModule: "promptfoo:builtin",
+    },
+    {
+        type: "contains-all",
+        label: "Contains all texts",
+        handlerModule: "promptfoo:builtin",
+    },
+    {
+        type: "contains-any",
+        label: "Contains any text",
+        handlerModule: "promptfoo:builtin",
+    },
+    { type: "equals", label: "Exact match", handlerModule: "promptfoo:builtin" },
+    { type: "regex", label: "Regex match", handlerModule: "promptfoo:builtin" },
+    { type: "is-json", label: "Valid JSON", handlerModule: "promptfoo:builtin" },
     {
-        id: "knowledge-probe",
-        label: "Knowledge Probe",
-        validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
-        rubricTemplateIds: [],
-        handlerModule: "./mode-handlers/knowledge-probe-handler.js",
+        type: "javascript",
+        label: "JavaScript assertion",
+        handlerModule: "promptfoo:builtin",
     },
     {
-        id: "mcp-server",
-        label: "MCP Server Testing",
-        validProviderPatterns: ["^mcp:", "^file://"],
-        rubricTemplateIds: [
-            "mcp-input-validation",
-            "mcp-output-correctness",
-            "mcp-error-handling",
-        ],
-        handlerModule: "./mode-handlers/mcp-server-handler.js",
+        type: "llm-rubric",
+        label: "LLM-graded rubric",
+        handlerModule: "promptfoo:builtin",
     },
     {
-        id: "agent-harness",
-        label: "Agent Harness",
-        validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
-        rubricTemplateIds: [],
-        handlerModule: "./mode-handlers/agent-harness-handler.js",
+        type: "similar",
+        label: "Semantic similarity",
+        handlerModule: "promptfoo:builtin",
     },
 ];
 /**
  * Build and populate the plugin registry.
  *
- * Preset registration flow:
- * 1. A preset is a PresetDefinition — a bundle of modes, assertions, rubric
- *    templates, prompt templates, scoring profiles, a doc fetcher factory,
- *    source definitions, and feature definitions.
- * 2. registerPreset() iterates the preset's fields and delegates each one to
- *    the appropriate register method (registerMode, registerRubricTemplate, …).
- * 3. After registration the rest of createAppContext() can pull capabilities
- *    from the registry (e.g. getDocFetcherFactory()) without knowing which
- *    preset provided them.
+ * Registration follows the five-layer model:
+ *
+ * Layer 0:   Framework built-in assertions (generic Promptfoo builtins)
+ * Layer 0.5: Mode bases (shared evaluation methodology per mode)
+ * Layer 1:   Domain presets (domain-specific config targeting a mode base)
  *
- * To add a new preset: create a PresetDefinition, then call
- * registry.registerPreset() here before the built-in mode registrations.
+ * Mode bases define HOW you evaluate (rubrics, scoring, prompts).
+ * Domain presets define WHAT you evaluate (sources, features, doc fetcher)
+ * and target a mode base by ID. When a preset is registered, it inherits
+ * its mode base's defaults and can optionally override them.
  */
-function createRegistry(rootDir) {
+function createRegistry(rootDir, externalPresets) {
     const registry = new InMemoryPluginRegistry();
-    // Register the sanity-literacy preset — the Sanity-specific evaluation bundle.
+    // Layer 0: Framework built-in assertions (available to all modes)
+    for (const assertion of FRAMEWORK_ASSERTIONS) {
+        registry.registerAssertion(assertion);
+    }
+    // Layer 0.5: Mode bases (evaluation methodology)
+    registry.registerModeBase(createLiteracyModeBase());
+    registry.registerModeBase(createMcpServerModeBase());
+    registry.registerModeBase(createKnowledgeProbeBase());
+    registry.registerModeBase(createAgentHarnessBase());
+    // Layer 1: Built-in domain presets
     registry.registerPreset(createSanityLiteracyPreset({ rootDir }));
-    // Register other built-in modes (not part of any preset yet)
-    for (const mode of BUILT_IN_MODES) {
-        registry.registerMode(mode);
+    // Layer 1+: External domain presets (from config.presets)
+    if (externalPresets) {
+        for (const preset of externalPresets) {
+            registry.registerPreset(preset);
+        }
     }
     return registry;
 }

package/dist/config/features.ts

@@ -0,0 +1,23 @@
+/**
+ * features.ts — Product feature registry for documentation coverage auditing.
+ *
+ * Default features are provided by the sanity-literacy preset registered
+ * in the composition root. This file exists as an override point — any
+ * features defined here take precedence over preset-provided features
+ * during coverage auditing.
+ *
+ * To track custom features, define them here:
+ *
+ *   export default defineFeatures({
+ *     features: [
+ *       { id: "my-feature", name: "My Feature", sections: ["api"], ... },
+ *     ],
+ *   })
+ *
+ * @see packages/eval/src/pipeline/compiler/presets/sanity-literacy.ts
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+
+import { defineFeatures } from "../_vendor/ailf-core/index.js"
+
+export default defineFeatures({ features: [] })

package/dist/config/models.ts

@@ -0,0 +1,100 @@
+/**
+ * models.ts — Central model registry for AILF evaluations.
+ *
+ * Define all models to test here. Each model declares which evaluation
+ * modes it participates in (e.g., "literacy", "mcp-server") and
+ * optionally which variants within those modes.
+ *
+ * When a model enrolls in a mode without specifying variants, all
+ * variants defined by the mode base are included by default.
+ *
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-1-ts-config-loading.md
+ */
+
+import { defineModels } from "../_vendor/ailf-core/index.js"
+
+export default defineModels({
+  models: [
+    // ── Anthropic ──────────────────────────────────────────────
+    {
+      id: "anthropic:messages:claude-opus-4-6",
+      label: "Claude Opus 4.6",
+      config: { temperature: 0.2, max_tokens: 4096 },
+      modes: ["literacy", "mcp-server", "knowledge-probe"],
+      // All literacy variants included by default (baseline, observed,
+      // agentic-naive, agentic-optimized)
+    },
+
+    // ── Google ─────────────────────────────────────────────────
+    // {
+    //   id: "google:gemini-2.5-pro",
+    //   label: "Gemini 2.5 Pro",
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    // },
+
+    // ── OpenAI ─────────────────────────────────────────────────
+    {
+      id: "openai:chat:gpt-5.2",
+      label: "GPT 5.2",
+      config: { temperature: 0.2, max_tokens: 4096 },
+      modes: ["literacy", "knowledge-probe"],
+      // All literacy variants included by default
+    },
+    {
+      id: "openai:responses:gpt-5.4",
+      label: "GPT 5.4",
+      config: {
+        reasoning_effort: "medium",
+        max_output_tokens: 4096,
+        maxRetries: 1,
+      },
+      timeoutMs: 600_000, // 10 min — reasoning model needs more headroom
+      modes: ["literacy", "mcp-server", "knowledge-probe"],
+      // All literacy variants included by default
+    },
+
+    // ── Disabled models (uncomment to enable) ──────────────────
+    // { id: "anthropic:claude-sonnet-4-20250514", label: "Claude Sonnet 4",
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline"] } },
+    // { id: "anthropic:claude-3.5-sonnet-20241022", label: "Claude 3.5 Sonnet",
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline", "agentic-naive", "agentic-optimized"] } },
+    // { id: "google:gemini-2.0-flash", label: "Gemini 2.0 Flash",
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline"] } },
+    // { id: "openrouter:deepseek/deepseek-r1", label: "DeepSeek R1",
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline"] } },
+  ],
+
+  // ── Grading Model ──────────────────────────────────────────
+  // Which model scores the responses. Separate from the models being tested.
+  grader: {
+    id: "anthropic:messages:claude-opus-4-5-20251101",
+    label: "Claude Opus 4.5 (grader)",
+  },
+
+  // ── Evaluation Options ─────────────────────────────────────
+  evalBudgetMs: 1_200_000, // 20 min per eval mode — outer kill switch
+  maxConcurrency: 32, // max parallel API calls — benchmarked in DOC-1896
+
+  // ── Default Config ─────────────────────────────────────────
+  // Applied to all models unless overridden per-model.
+  defaults: {
+    temperature: 0.2,
+    max_tokens: 4096,
+    maxToolRounds: 5, // for agentic modes
+    observerOptions: {
+      maxPreviewBytes: 2048,
+      captureResponsePreview: true,
+      includePatterns: ["sanity.io", "sanity.dev", "cdn.sanity.io"],
+      sensitiveHeaders: ["authorization", "cookie", "x-api-key"],
+    },
+  },
+})

package/dist/config/prompts.ts

@@ -0,0 +1,16 @@
+/**
+ * prompts.ts — User-override prompt templates.
+ *
+ * Canonical literacy prompt templates now live in the literacy mode handler:
+ *   src/pipeline/compiler/mode-handlers/literacy-handler.ts
+ *
+ * Each mode handler owns its own prompts via getPrompts(). This file exists
+ * for user-level overrides only. Add entries here to replace handler-owned
+ * defaults for specific prompt IDs.
+ *
+ * @see packages/eval/src/pipeline/compiler/mode-handlers/literacy-handler.ts
+ */
+
+import { definePrompts } from "../_vendor/ailf-core/index.js"
+
+export default definePrompts([])