@sanity/ailf 0.5.0 → 2.0.0

package/dist/commands/validate-tasks.js

@@ -1,7 +1,7 @@
 /**
- * validate-tasks command — standalone validation of repo-based task YAML files.
+ * validate-tasks command — standalone validation of task files.
  *
- * Validates .ailf/tasks/*.yaml files against the RepoTaskSchema without
+ * Validates .ailf/tasks/*.yaml files against the CanonicalTaskSchema without
  * running the full pipeline. Useful for pre-commit hooks and CI checks
  * in external repos.
  *
@@ -16,11 +16,11 @@ import { existsSync, readdirSync, readFileSync } from "fs";
 import { resolve, relative } from "path";
 import { Command } from "commander";
 import { load } from "js-yaml";
-import { parseRepoTaskFile } from "../adapters/task-sources/repo-schemas.js";
-import { validateRepoTasks, formatValidationResult, } from "../adapters/task-sources/repo-validation.js";
+import { detectLegacyFieldNames, parseCanonicalTaskFile, } from "../adapters/task-sources/repo-schemas.js";
+import { validateCanonicalTasks, formatValidationResult, } from "../adapters/task-sources/repo-validation.js";
 export function createValidateTasksCommand() {
     return new Command("validate-tasks")
-        .description("Validate repo-based task YAML files (.ailf/tasks/) against the schema")
+        .description("Validate task YAML files (.ailf/tasks/) against the canonical schema")
         .argument("[path]", "Path to tasks directory (default: .ailf/tasks/)", ".ailf/tasks")
         .option("--strict", "Treat warnings as errors", false)
         .action(async (tasksPath, opts) => {
@@ -29,12 +29,12 @@ export function createValidateTasksCommand() {
         const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
         const resolvedPath = resolve(callerCwd, tasksPath);
         if (!existsSync(resolvedPath)) {
-            console.error(`❌ Directory not found: ${resolvedPath}`);
+            console.error(`Directory not found: ${resolvedPath}`);
             process.exit(1);
         }
         const yamlFiles = readdirSync(resolvedPath).filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."));
         if (yamlFiles.length === 0) {
-            console.error(`❌ No YAML files found in ${resolvedPath}`);
+            console.error(`No YAML files found in ${resolvedPath}`);
             process.exit(1);
         }
         console.log(`\nValidating ${yamlFiles.length} task file(s) in ${relative(process.cwd(), resolvedPath)}/\n`);
@@ -50,25 +50,36 @@ export function createValidateTasksCommand() {
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                console.error(`  ❌ ${file}: YAML parse error`);
+                console.error(`  ${file}: YAML parse error`);
                 console.error(`     ${msg}\n`);
                 hasErrors = true;
                 continue;
             }
             if (!Array.isArray(parsed)) {
-                console.error(`  ❌ ${file}: Expected a YAML array of task definitions`);
+                console.error(`  ${file}: Expected a YAML array of task definitions`);
+                hasErrors = true;
+                continue;
+            }
+            // Detect legacy field names before Zod validation
+            const legacyWarnings = detectLegacyFieldNames(parsed, file);
+            if (legacyWarnings.length > 0) {
+                console.error(`  ${file}: Uses legacy field names`);
+                for (const w of legacyWarnings) {
+                    console.error(`     ${w}`);
+                }
+                console.error();
                 hasErrors = true;
                 continue;
             }
             try {
-                const tasks = parseRepoTaskFile(parsed, file);
-                console.log(`  ✅ ${file}: ${tasks.length} task${tasks.length === 1 ? "" : "s"} valid`);
+                const tasks = parseCanonicalTaskFile(parsed, file);
+                console.log(`  ${file}: ${tasks.length} task${tasks.length === 1 ? "" : "s"} valid`);
                 totalTasks += tasks.length;
                 allTasks.push(...tasks);
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                console.error(`  ❌ ${file}: Schema validation failed`);
+                console.error(`  ${file}: Schema validation failed`);
                 console.error(`${msg
                     .split("\n")
                     .map((l) => `     ${l}`)
@@ -79,7 +90,7 @@ export function createValidateTasksCommand() {
         // Run semantic validation on all parsed tasks
         if (allTasks.length > 0) {
             console.log(); // blank line
-            const semanticResult = validateRepoTasks(allTasks);
+            const semanticResult = validateCanonicalTasks(allTasks);
             const formatted = formatValidationResult(semanticResult);
             console.log(formatted);
             if (!semanticResult.valid) {
@@ -87,10 +98,10 @@ export function createValidateTasksCommand() {
             }
             if (opts.strict && semanticResult.warnings.length > 0) {
                 hasErrors = true;
-                console.log("\n  ⚠️  --strict mode: warnings treated as errors");
+                console.log("\n  --strict mode: warnings treated as errors");
             }
         }
-        console.log(`\n${hasErrors ? "❌" : "✅"} ${totalTasks} task${totalTasks === 1 ? "" : "s"} across ${yamlFiles.length} file${yamlFiles.length === 1 ? "" : "s"}\n`);
+        console.log(`\n${hasErrors ? "FAIL" : "OK"} ${totalTasks} task${totalTasks === 1 ? "" : "s"} across ${yamlFiles.length} file${yamlFiles.length === 1 ? "" : "s"}\n`);
         process.exit(hasErrors ? 1 : 0);
     });
 }

package/dist/composition-root.d.ts

@@ -15,7 +15,7 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import type { AppContext, ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+import { type AppContext, type AssertionRegistration, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
 /**
  * Create a fully wired AppContext from resolved configuration.
  *
@@ -24,3 +24,15 @@ import type { AppContext, ResolvedConfig } from "./_vendor/ailf-core/index.d.ts"
  * is a one-line change in this function.
  */
 export declare function createAppContext(config: ResolvedConfig): AppContext;
+/**
+ * Generic Promptfoo assertion types available to all evaluation modes.
+ *
+ * These are evaluation primitives (text matching, JSON validation, LLM grading)
+ * that aren't specific to any mode or domain. They're registered before any
+ * preset so every mode has access to them.
+ *
+ * `compatibleModes` is omitted — when undefined, the assertion is compatible
+ * with all modes. Mode-specific assertions can be registered by presets with
+ * explicit mode whitelists.
+ */
+export declare const FRAMEWORK_ASSERTIONS: AssertionRegistration[];

package/dist/composition-root.js

@@ -15,12 +15,15 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
+import { InMemoryPluginRegistry, } from "./_vendor/ailf-core/index.js";
 import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
+import { loadExternalPresets } from "./pipeline/compiler/preset-loader.js";
 import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
-import { SanityDocFetcher } from "./adapters/doc-fetchers/index.js";
 import { PromptfooEvalAdapter } from "./adapters/eval-runners/promptfoo-eval-adapter.js";
 import { ConsoleLogger, JsonLogger, QuietLogger, } from "./adapters/loggers/index.js";
 import { CompositeTaskSource, ContentLakeTaskSource, RepoTaskSource, YamlTaskSource, } from "./adapters/task-sources/index.js";
+import { createAgentHarnessBase, createKnowledgeProbeBase, createLiteracyModeBase, createMcpServerModeBase, } from "./pipeline/compiler/mode-bases/index.js";
+import { createSanityLiteracyPreset } from "./pipeline/compiler/presets/index.js";
 import { getSanityClient } from "./sanity/client.js";
 import { ReportStore } from "./report-store.js";
 import { loadSinks } from "./sinks/index.js";
@@ -38,13 +41,20 @@ export function createAppContext(config) {
     const cache = config.noCache ? undefined : createCache(config);
     // Task source — selected by config.taskSourceType
     const taskSource = createTaskSource(config);
-    // Doc fetcher — Sanity Content Lake
-    const docFetcher = new SanityDocFetcher(config.rootDir);
+    // Plugin registry — mode bases, assertions, presets, doc fetcher.
+    // External presets from config are loaded and registered after built-ins.
+    const externalPresets = config.presets && config.presets.length > 0
+        ? loadExternalPresets(config.presets, config.rootDir)
+        : undefined;
+    const registry = createRegistry(config.rootDir, externalPresets);
+    // Doc fetcher — provided by the registered preset's factory
+    const docFetcherFactory = registry.getDocFetcherFactory();
+    const docFetcher = docFetcherFactory ? docFetcherFactory() : undefined;
     // Eval runner — Promptfoo subprocess
     const evalRunner = new PromptfooEvalAdapter(config.rootDir);
     // Report store — Sanity Content Lake (for publish + auto-compare)
     const reportStore = createReportStore(config);
-    // Sinks — loaded from config/sinks.yaml
+    // Sinks — loaded from config/sinks
     const sinks = loadSinks();
     return {
         cache,
@@ -52,6 +62,7 @@ export function createAppContext(config) {
         docFetcher,
         evalRunner,
         logger,
+        registry,
         reportStore,
         sinks,
         taskSource,
@@ -113,6 +124,90 @@ function createTaskSource(config) {
     }
     return primary;
 }
+// ---------------------------------------------------------------------------
+// Layer 0: Framework built-in assertions
+// ---------------------------------------------------------------------------
+/**
+ * Generic Promptfoo assertion types available to all evaluation modes.
+ *
+ * These are evaluation primitives (text matching, JSON validation, LLM grading)
+ * that aren't specific to any mode or domain. They're registered before any
+ * preset so every mode has access to them.
+ *
+ * `compatibleModes` is omitted — when undefined, the assertion is compatible
+ * with all modes. Mode-specific assertions can be registered by presets with
+ * explicit mode whitelists.
+ */
+export const FRAMEWORK_ASSERTIONS = [
+    {
+        type: "contains",
+        label: "Contains text",
+        handlerModule: "promptfoo:builtin",
+    },
+    {
+        type: "contains-all",
+        label: "Contains all texts",
+        handlerModule: "promptfoo:builtin",
+    },
+    {
+        type: "contains-any",
+        label: "Contains any text",
+        handlerModule: "promptfoo:builtin",
+    },
+    { type: "equals", label: "Exact match", handlerModule: "promptfoo:builtin" },
+    { type: "regex", label: "Regex match", handlerModule: "promptfoo:builtin" },
+    { type: "is-json", label: "Valid JSON", handlerModule: "promptfoo:builtin" },
+    {
+        type: "javascript",
+        label: "JavaScript assertion",
+        handlerModule: "promptfoo:builtin",
+    },
+    {
+        type: "llm-rubric",
+        label: "LLM-graded rubric",
+        handlerModule: "promptfoo:builtin",
+    },
+    {
+        type: "similar",
+        label: "Semantic similarity",
+        handlerModule: "promptfoo:builtin",
+    },
+];
+/**
+ * Build and populate the plugin registry.
+ *
+ * Registration follows the five-layer model:
+ *
+ * Layer 0:   Framework built-in assertions (generic Promptfoo builtins)
+ * Layer 0.5: Mode bases (shared evaluation methodology per mode)
+ * Layer 1:   Domain presets (domain-specific config targeting a mode base)
+ *
+ * Mode bases define HOW you evaluate (rubrics, scoring, prompts).
+ * Domain presets define WHAT you evaluate (sources, features, doc fetcher)
+ * and target a mode base by ID. When a preset is registered, it inherits
+ * its mode base's defaults and can optionally override them.
+ */
+function createRegistry(rootDir, externalPresets) {
+    const registry = new InMemoryPluginRegistry();
+    // Layer 0: Framework built-in assertions (available to all modes)
+    for (const assertion of FRAMEWORK_ASSERTIONS) {
+        registry.registerAssertion(assertion);
+    }
+    // Layer 0.5: Mode bases (evaluation methodology)
+    registry.registerModeBase(createLiteracyModeBase());
+    registry.registerModeBase(createMcpServerModeBase());
+    registry.registerModeBase(createKnowledgeProbeBase());
+    registry.registerModeBase(createAgentHarnessBase());
+    // Layer 1: Built-in domain presets
+    registry.registerPreset(createSanityLiteracyPreset({ rootDir }));
+    // Layer 1+: External domain presets (from config.presets)
+    if (externalPresets) {
+        for (const preset of externalPresets) {
+            registry.registerPreset(preset);
+        }
+    }
+    return registry;
+}
 function createReportStore(config) {
     return new ReportStore({
         dataset: process.env.AILF_REPORT_DATASET ??

package/dist/index.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @sanity/ailf — Public API for the AI Literacy Framework.
+ *
+ * This module is the entry point for external consumers who import from
+ * `@sanity/ailf`. It re-exports the authoring API needed to write task
+ * definitions, configuration files, and validate task YAML.
+ *
+ * ## Task authoring
+ *
+ * ```typescript
+ * import { defineTask } from "@sanity/ailf"
+ *
+ * export default defineTask({
+ *   id: "groq-projection-basics",
+ *   mode: "literacy",
+ *   title: "GROQ Projection Basics",
+ *   area: "groq",
+ *   prompt: { text: "Write GROQ queries..." },
+ *   assertions: [
+ *     { type: "llm-rubric", template: "task-completion", criteria: ["..."] },
+ *   ],
+ * })
+ * ```
+ *
+ * ## Configuration authoring
+ *
+ * ```typescript
+ * import { defineConfig, env } from "@sanity/ailf"
+ *
+ * export default defineConfig({
+ *   projectId: env("SANITY_PROJECT_ID"),
+ *   dataset: env("SANITY_DATASET"),
+ * })
+ * ```
+ */
+export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./_vendor/ailf-core/index.d.ts";
+export type { PricingEntry, PromptEntry, SourceEntry } from "./_vendor/ailf-core/index.d.ts";
+export { env } from "./_vendor/ailf-core/index.d.ts";
+export type { AgentHarnessTaskDefinition, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PathDocRef, PerspectiveDocRef, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./_vendor/ailf-core/index.d.ts";
+export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, RUBRIC_TEMPLATE_NAMES, type CanonicalTask, type CuratedAssertionType, type RubricTemplateName, } from "./adapters/task-sources/repo-schemas.js";
+export { formatValidationResult, validateCanonicalTasks, type ValidationMessage, type ValidationResult, } from "./adapters/task-sources/repo-validation.js";

package/dist/index.js

@@ -0,0 +1,48 @@
+/**
+ * @sanity/ailf — Public API for the AI Literacy Framework.
+ *
+ * This module is the entry point for external consumers who import from
+ * `@sanity/ailf`. It re-exports the authoring API needed to write task
+ * definitions, configuration files, and validate task YAML.
+ *
+ * ## Task authoring
+ *
+ * ```typescript
+ * import { defineTask } from "@sanity/ailf"
+ *
+ * export default defineTask({
+ *   id: "groq-projection-basics",
+ *   mode: "literacy",
+ *   title: "GROQ Projection Basics",
+ *   area: "groq",
+ *   prompt: { text: "Write GROQ queries..." },
+ *   assertions: [
+ *     { type: "llm-rubric", template: "task-completion", criteria: ["..."] },
+ *   ],
+ * })
+ * ```
+ *
+ * ## Configuration authoring
+ *
+ * ```typescript
+ * import { defineConfig, env } from "@sanity/ailf"
+ *
+ * export default defineConfig({
+ *   projectId: env("SANITY_PROJECT_ID"),
+ *   dataset: env("SANITY_DATASET"),
+ * })
+ * ```
+ */
+// ---------------------------------------------------------------------------
+// Configuration helpers (define* identity functions for typed authoring)
+// ---------------------------------------------------------------------------
+export { defineConfig, defineFeatures, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Environment helper
+// ---------------------------------------------------------------------------
+export { env } from "./_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Validation — for programmatic validation of task YAML
+// ---------------------------------------------------------------------------
+export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, RUBRIC_TEMPLATE_NAMES, } from "./adapters/task-sources/repo-schemas.js";
+export { formatValidationResult, validateCanonicalTasks, } from "./adapters/task-sources/repo-validation.js";

package/dist/orchestration/build-app-context.js

@@ -20,6 +20,7 @@ export function mapToResolvedConfig(opts, rootDir) {
     return {
         rootDir,
         mode: opts.mode,
+        variant: opts.variant,
         noAutoScope: opts.noAutoScope ?? false,
         debug: opts.debug,
         areas: opts.areaOption

package/dist/orchestration/build-step-sequence.js

@@ -5,7 +5,7 @@
  * PipelineStep objects determined by config flags like skipFetch,
  * skipEval, compareEnabled, etc.
  */
-import { FULL_MODE_SUBMODES } from "../_vendor/ailf-shared/index.js";
+import { LiteracyVariant } from "../pipeline/normalize-mode.js";
 import { CallbackStep } from "./steps/callback-step.js";
 import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
 import { CompareStep } from "./steps/compare-step.js";
@@ -35,16 +35,36 @@ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
     if (config.repoTasksPath) {
         steps.push(new MirrorRepoTasksStep());
     }
-    // Step 1: Fetch documentation (step handles --skip-fetch internally)
-    steps.push(new FetchDocsStep());
+    // Step 1: Fetch documentation (literacy mode only — other modes don't use canonical docs)
+    if (config.mode === "literacy") {
+        steps.push(new FetchDocsStep());
+    }
     // Step 2: Generate Promptfoo configs
     steps.push(new GenerateConfigsStep());
     // Step 3: Run evaluation (steps handle --skip-eval internally)
-    const modes = config.mode === "full"
-        ? [...FULL_MODE_SUBMODES]
-        : [config.mode];
-    for (const mode of modes) {
-        steps.push(new RunEvalStep(mode));
+    //
+    // For literacy mode, the variant determines how many eval steps run:
+    //   "full" → baseline + agentic (two steps)
+    //   "baseline" / "agentic" / "observed" → one step
+    //   undefined → defaults to baseline
+    //
+    // For all other modes, one eval step per mode.
+    if (config.mode === "literacy") {
+        const variant = config.variant ?? LiteracyVariant.STANDARD;
+        if (variant === LiteracyVariant.FULL) {
+            for (const submode of [
+                LiteracyVariant.STANDARD,
+                LiteracyVariant.AGENTIC,
+            ]) {
+                steps.push(new RunEvalStep(submode));
+            }
+        }
+        else {
+            steps.push(new RunEvalStep(variant));
+        }
+    }
+    else {
+        steps.push(new RunEvalStep(config.mode));
     }
     // Step 3c: Grader consistency (optional, conditional)
     if (config.graderReplications) {

package/dist/orchestration/steps/calculate-scores-step.js

@@ -5,11 +5,11 @@
  * typed options derived from AppContext. No env bridge needed.
  */
 import { join } from "path";
-import { FULL_MODE_SUBMODES } from "../../_vendor/ailf-shared/index.js";
+import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { calculateAndWriteScores } from "../../pipeline/calculate-scores.js";
 import { checkResultsExist, checkScoreSummaryValid, } from "../../pipeline/checks.js";
-import { RESULTS_FILES } from "../../pipeline/eval-constants.js";
+import { resultsFileForMode } from "../../pipeline/eval-constants.js";
 import { loadSource } from "../../sources.js";
 import { configToSourceOverrides } from "../config-to-source-overrides.js";
 export class CalculateScoresStep {
@@ -23,10 +23,14 @@ export class CalculateScoresStep {
         // score-summary.json was already restored from the cached report.
         // Skip re-calculation — the raw eval-results files don't exist.
         if (state.remoteCacheHits?.size) {
-            const requiredModes = ctx.config.mode === "full"
-                ? [...FULL_MODE_SUBMODES]
-                : [ctx.config.mode];
-            const allCached = requiredModes.every((m) => state.remoteCacheHits.has(m));
+            // For literacy mode, determine required eval runs from variant
+            const variant = ctx.config.variant ?? LiteracyVariant.STANDARD;
+            const requiredRuns = ctx.config.mode === "literacy" && variant === LiteracyVariant.FULL
+                ? [LiteracyVariant.STANDARD, LiteracyVariant.AGENTIC]
+                : ctx.config.mode === "literacy"
+                    ? [variant]
+                    : [ctx.config.mode];
+            const allCached = requiredRuns.every((m) => state.remoteCacheHits.has(m));
             if (allCached) {
                 // Verify the restored score-summary.json is valid
                 const summaryIssues = checkScoreSummaryValid(ctx.config.rootDir);
@@ -40,10 +44,15 @@ export class CalculateScoresStep {
                 // If the summary is invalid, fall through to normal calculation
             }
         }
-        const primaryMode = ctx.config.mode === "full"
-            ? "baseline"
+        // Primary results file to score.
+        // For literacy: "full" variant uses baseline as primary; others use variant directly.
+        // For other modes: use the mode name.
+        const primaryResultsRun = ctx.config.mode === "literacy"
+            ? ctx.config.variant === LiteracyVariant.FULL
+                ? LiteracyVariant.STANDARD
+                : (ctx.config.variant ?? LiteracyVariant.STANDARD)
             : ctx.config.mode;
-        const resultsFile = RESULTS_FILES[primaryMode];
+        const resultsFile = resultsFileForMode(primaryResultsRun);
         // Precondition: results file exists
         const resultsIssues = checkResultsExist(ctx.config.rootDir, resultsFile);
         const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
@@ -68,9 +77,13 @@ export class CalculateScoresStep {
             const result = calculateAndWriteScores({
                 allowedOrigins: ctx.config.allowedOrigins,
                 logger: ctx.logger,
-                mode: ctx.config.mode,
+                // Pass the variant for literacy (scoring uses it to decide
+                // whether to read agentic results), or mode for other modes
+                mode: ctx.config.mode === "literacy"
+                    ? (ctx.config.variant ?? LiteracyVariant.STANDARD)
+                    : ctx.config.mode,
                 resolvedSource,
-                resultsPath: primaryMode !== "baseline"
+                resultsPath: primaryResultsRun !== LiteracyVariant.STANDARD
                     ? join(ctx.config.rootDir, resultsFile)
                     : undefined,
                 rootDir: ctx.config.rootDir,

package/dist/orchestration/steps/fetch-docs-step.js

@@ -28,13 +28,14 @@ export class FetchDocsStep {
         }
         const start = Date.now();
         // Precondition: at least one task has canonical doc mappings
-        const tasks = await ctx.taskSource.loadTasks(buildFilter(ctx));
-        const tasksWithDocs = tasks.filter((t) => t.canonicalDocs.length > 0);
+        const allTasks = await ctx.taskSource.loadTasks(buildFilter(ctx));
+        // Bridge: narrow to literacy tasks for canonical doc access
+        const literacyTasks = allTasks.filter((t) => t.mode === "literacy");
+        const tasksWithDocs = literacyTasks.filter((t) => (t.context?.docs?.length ?? 0) > 0);
         if (tasksWithDocs.length === 0) {
             return {
-                durationMs: Date.now() - start,
-                error: "No tasks with canonical_docs found. Add canonical_docs to your task definitions.",
-                status: "failed",
+                status: "skipped",
+                reason: "No literacy tasks with canonical_docs — nothing to fetch",
             };
         }
         // Resolve source once with typed overrides
@@ -100,7 +101,7 @@ export class FetchDocsStep {
         if (resolvedSource.perspective &&
             releaseImpact &&
             !ctx.config.noAutoScope) {
-            const autoScope = computeAutoScope(tasks, releaseImpact, resolvedSource.perspective);
+            const autoScope = computeAutoScope(literacyTasks, releaseImpact, resolvedSource.perspective);
             if (autoScope) {
                 state.releaseAutoScope = autoScope;
                 logAutoScope(autoScope);
@@ -171,7 +172,7 @@ function writeMetadataFiles(rootDir, metadata) {
  */
 function extractSlugsFromTask(task) {
     const slugs = [];
-    for (const ref of task.canonicalDocs) {
+    for (const ref of task.context?.docs ?? []) {
         if (isSlugRef(ref)) {
             slugs.push(ref.slug);
         }

package/dist/orchestration/steps/gap-analysis-step.js

@@ -89,7 +89,7 @@ export class GapAnalysisStep {
             const areaToDocRefs = new Map();
             let tasks = [];
             try {
-                tasks = await ctx.taskSource.loadTasks();
+                tasks = (await ctx.taskSource.loadTasks()).filter((t) => t.mode === "literacy");
             }
             catch {
                 // TaskSource may not be available in all contexts (e.g., standalone
@@ -99,17 +99,18 @@ export class GapAnalysisStep {
                 // Group tasks by feature area and build slug maps
                 const byArea = new Map();
                 for (const task of tasks) {
-                    const slugs = extractSlugsFromRefs(task.canonicalDocs);
+                    const slugs = extractSlugsFromRefs(task.context?.docs ?? []);
                     const refs = resolveRefs(slugs);
-                    // Map by description (what judgments use as taskId)
-                    descToDocRefs.set(task.description, refs);
+                    // Map by title (what judgments use as taskId)
+                    descToDocRefs.set(task.title, refs);
                     // Also map by task ID for prefix-based matching
                     descToDocRefs.set(task.id, refs);
                     // Group slugs by feature area
-                    if (!byArea.has(task.featureArea))
-                        byArea.set(task.featureArea, new Set());
+                    const area = task.area ?? "";
+                    if (!byArea.has(area))
+                        byArea.set(area, new Set());
                     for (const s of slugs)
-                        byArea.get(task.featureArea).add(s);
+                        byArea.get(area).add(s);
                 }
                 for (const [area, slugs] of byArea) {
                     areaToDocRefs.set(area, resolveRefs([...slugs]));

package/dist/orchestration/steps/generate-configs-step.d.ts

@@ -1,14 +1,27 @@
 /**
  * Pipeline step: Generate Promptfoo configuration files.
  *
- * Calls generateConfigs() from pipeline/generate-configs.ts with typed options
- * derived from AppContext. No env bridge needed — source is resolved and
- * passed directly.
+ * ALL modes route through the Plugin Registry. The step looks up the mode
+ * handler via ctx.registry.getMode() and delegates compilation to it.
+ *
+ * Literacy mode has a variant strategy: baseline/agentic/observed/full.
+ * When the variant is "full", the handler is called twice (baseline + agentic)
+ * and three YAML files are written. Other modes produce one YAML file.
  */
 import type { AppContext, PipelineState, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class GenerateConfigsStep implements PipelineStep {
     readonly name = "generate-configs";
     check(ctx: AppContext): ValidationIssue[];
     execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
+    private compileLiteracyVariants;
+    private compileSingleMode;
+    private loadTasks;
+    private applyFilters;
+    /**
+     * Compile all tasks through a handler, merging results.
+     * For literacy mode, ctx can carry evalMode as an extension.
+     */
+    private compileAll;
+    private checkLiteracyPostconditions;
     cacheInputs(ctx: AppContext): string[];
 }