@sanity/ailf 0.5.0 → 1.0.0

package/dist/commands/explain-handler.js

@@ -23,6 +23,7 @@ import { TASK_FILE_NAMES } from "../_vendor/ailf-core/index.js";
 import { buildPipelinePlan, buildSimpleCommandPlan, } from "../pipeline/plan.js";
 import { formatPlanConsole, formatPlanJson } from "../pipeline/plan-format.js";
 import { computeResolvedOptions } from "./pipeline-action.js";
+import { LiteracyVariant } from "../pipeline/normalize-mode.js";
 // ---------------------------------------------------------------------------
 // Registry
 // ---------------------------------------------------------------------------
@@ -84,8 +85,8 @@ const EXPLAIN_REGISTRY = {
         filesCreated: ["results/latest/score-summary.json"],
         filesRead: [
             "results/latest/eval-results.json",
-            "config/rubrics.yaml",
-            "config/models.yaml",
+            "config/rubrics.ts",
+            "config/models.ts",
         ],
         steps: [
             {
@@ -138,12 +139,12 @@ const EXPLAIN_REGISTRY = {
     },
     "coverage-audit": {
         description: "Cross-reference feature registry against evaluation tasks for coverage gaps",
-        filesRead: ["config/features.yaml", "tasks/*.yaml"],
+        filesRead: ["config/features.ts", "tasks/*.{yaml,task.ts,task.js}"],
         steps: [
             {
                 cacheStatus: "miss",
                 name: "Load feature registry",
-                reason: "Parse config/features.yaml for product feature list",
+                reason: "Parse config/features.ts for product feature list",
                 willRun: true,
             },
             {
@@ -201,7 +202,7 @@ const EXPLAIN_REGISTRY = {
     "fetch-docs": {
         description: "Fetch documentation from Sanity CMS and generate canonical context files",
         filesCreated: ["contexts/canonical/*.md"],
-        filesRead: ["config/sources.yaml", "config/models.yaml"],
+        filesRead: ["config/sources.ts", "config/models.ts"],
         steps: [
             {
                 cacheStatus: "miss",
@@ -224,7 +225,7 @@ const EXPLAIN_REGISTRY = {
         ],
     },
     "generate-configs": {
-        description: "Generate Promptfoo config files from models.yaml and task definitions",
+        description: "Generate Promptfoo config files from models.ts and task definitions",
         filesCreated: [
             "promptfooconfig.yaml",
             "promptfooconfig.observed.yaml",
@@ -232,16 +233,16 @@ const EXPLAIN_REGISTRY = {
             "tasks/.expanded.yaml",
         ],
         filesRead: [
-            "config/models.yaml",
-            "config/prompts.yaml",
-            "config/rubrics.yaml",
-            "config/sources.yaml",
+            "config/models.ts",
+            "config/prompts.ts",
+            "config/rubrics.ts",
+            "config/sources.ts",
         ],
         steps: [
             {
                 cacheStatus: "miss",
                 name: "Load models",
-                reason: "Parse config/models.yaml for active model list",
+                reason: "Parse config/models.ts for active model list",
                 willRun: true,
             },
             {
@@ -262,7 +263,7 @@ const EXPLAIN_REGISTRY = {
         description: "Grader reliability tools (consistency, compare, sensitivity, validate)",
         filesRead: [
             "results/latest/eval-results.json",
-            "config/rubrics.yaml",
+            "config/rubrics.ts",
             "canonical/reference-solutions/",
         ],
         steps: [
@@ -369,7 +370,7 @@ const EXPLAIN_REGISTRY = {
         filesRead: [
             "results/latest/score-summary.json",
             "results/latest/gap-analysis.json",
-            "config/thresholds.yaml",
+            "config/thresholds.ts",
             "results/baselines/",
         ],
         filesCreated: ["results/latest/readiness-report.md"],
@@ -377,7 +378,7 @@ const EXPLAIN_REGISTRY = {
             {
                 cacheStatus: "miss",
                 name: "Load scores + thresholds",
-                reason: "Read score-summary.json and thresholds.yaml for gate evaluation",
+                reason: "Read score-summary.json and thresholds.ts for gate evaluation",
                 willRun: true,
             },
             {
@@ -395,18 +396,18 @@ const EXPLAIN_REGISTRY = {
         ],
     },
     validate: {
-        description: "Validate all YAML config files, task definitions, reference solutions, and environment",
+        description: "Validate all config files, task definitions, reference solutions, and environment",
         filesRead: [
-            "config/models.yaml",
-            "config/rubrics.yaml",
-            "config/features.yaml",
-            "config/thresholds.yaml",
+            "config/models.ts",
+            "config/rubrics.ts",
+            "config/features.ts",
+            "config/thresholds.ts",
         ],
         steps: [
             {
                 cacheStatus: "miss",
                 name: "Validate configuration",
-                reason: "Parse all YAML configs through Zod schemas, cross-reference mappings",
+                reason: "Parse all config files through Zod schemas, cross-reference mappings",
                 willRun: true,
             },
             {
@@ -454,12 +455,12 @@ const EXPLAIN_REGISTRY = {
     },
     "weekly-digest": {
         description: "Generate and deliver a weekly evaluation trend digest via Slack",
-        filesRead: ["config/schedules.yaml", "config/sinks.yaml"],
+        filesRead: ["config/schedules.ts", "config/sinks.ts"],
         steps: [
             {
                 cacheStatus: "miss",
                 name: "Load digest config",
-                reason: "Read schedules.yaml for lookback window and delivery targets",
+                reason: "Read schedules.ts for lookback window and delivery targets",
                 willRun: true,
             },
             {
@@ -670,7 +671,7 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
         graderReplications: raw.graderReplications,
         header: raw.header ?? [],
         headers: raw.headers ?? [],
-        mode: raw.mode ?? "full",
+        mode: raw.mode ?? LiteracyVariant.FULL,
         output: raw.output,
         promptfooUrl: raw.promptfooUrl,
         publish: raw.publish,
@@ -714,6 +715,7 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
         gapAnalysisEnabled: resolved.gapAnalysisEnabled,
         graderReplications: resolved.graderReplications,
         mode: resolved.mode,
+        variant: resolved.variant,
         noCache: resolved.noCache,
         publishEnabled: resolved.publishEnabled,
         readinessEnabled: resolved.readinessEnabled,

package/dist/commands/fetch-docs.js

@@ -41,7 +41,7 @@ async function executeFetchDocs(opts) {
     // Build a minimal ResolvedConfig for the composition root
     const ctx = createAppContext({
         rootDir: ROOT,
-        mode: "baseline",
+        mode: "literacy",
         noAutoScope: false,
         skipFetch: false,
         skipEval: true,
@@ -83,7 +83,8 @@ async function executeFetchDocs(opts) {
     }
     // Canonical contexts — same code path as the pipeline
     const tasks = await ctx.taskSource.loadTasks();
-    const tasksWithDocs = tasks.filter((t) => t.canonicalDocs.length > 0);
+    // Bridge: narrow to literacy tasks with docs (only literacy tasks have context.docs)
+    const tasksWithDocs = tasks.filter((t) => t.mode === "literacy" && (t.context?.docs?.length ?? 0) > 0);
     if (tasksWithDocs.length > 0) {
         console.log("\nGenerating canonical (gold-retrieval) contexts...\n");
         const result = await fetcher.fetch(tasksWithDocs, resolvedSource);

package/dist/commands/generate-configs.js

@@ -19,7 +19,7 @@ export function createGenerateConfigsCommand() {
         try {
             const ctx = createAppContext({
                 rootDir: ROOT,
-                mode: "baseline",
+                mode: "literacy",
                 noAutoScope: false,
                 skipFetch: true,
                 skipEval: true,

package/dist/commands/interactive.js

@@ -9,6 +9,10 @@
  * Uses @inquirer/prompts for a clean, modern terminal UI.
  */
 import { Command } from "commander";
+import { LiteracyVariant } from "../pipeline/normalize-mode.js";
+// CLI command name for the baseline snapshot management subcommand.
+// Defined as a constant to avoid scattering the literal string across routing code.
+const BASELINE_CMD = "baseline";
 export function createInteractiveCommand() {
     return new Command("interactive")
         .description("Guided wizard for common evaluation workflows")
@@ -65,7 +69,7 @@ async function runInteractiveWizard() {
             {
                 description: "Save, compare, or list historical score snapshots",
                 name: "Manage baselines",
-                value: "baseline",
+                value: BASELINE_CMD,
             },
             {
                 description: "Weekly evaluation trends and area summaries",
@@ -93,7 +97,7 @@ async function runInteractiveWizard() {
         });
         return { args: dryRun ? ["--dry-run"] : [], command: "weekly-digest" };
     }
-    if (workflow === "baseline") {
+    if (workflow === BASELINE_CMD) {
         const subcommand = await select({
             choices: [
                 { name: "Save current scores", value: "save" },
@@ -102,7 +106,7 @@ async function runInteractiveWizard() {
             ],
             message: "Baseline operation:",
         });
-        return { args: [subcommand], command: "baseline" };
+        return { args: [subcommand], command: BASELINE_CMD };
     }
     if (workflow === "grader") {
         const subcommand = await select({
@@ -140,22 +144,22 @@ async function runInteractiveWizard() {
             {
                 description: "Evaluate with pre-fetched documentation context",
                 name: "Baseline (with docs vs without docs)",
-                value: "baseline",
+                value: LiteracyVariant.STANDARD,
             },
             {
                 description: "Baseline + record HTTP request patterns",
                 name: "Observed (instrumented)",
-                value: "observed",
+                value: LiteracyVariant.OBSERVED,
             },
             {
                 description: "Agent searches for docs itself via web tools",
                 name: "Agentic (agent-driven retrieval)",
-                value: "agentic",
+                value: LiteracyVariant.AGENTIC,
             },
         ],
         message: "Evaluation mode:",
     });
-    if (mode !== "baseline") {
+    if (mode !== LiteracyVariant.STANDARD) {
         args.push("--mode", mode);
     }
     // Step 3: Area scoping

package/dist/commands/pipeline-action.d.ts

@@ -31,6 +31,8 @@ export interface ResolvedOptions {
     headerArgs: string[];
     impactSummary?: ImpactSummary;
     mode: EvalMode;
+    /** Literacy variant — set when the user passes a legacy mode name */
+    variant?: string;
     noAutoScope: boolean;
     noCache: boolean;
     noRemoteCache: boolean;

package/dist/commands/pipeline-action.js

@@ -14,6 +14,7 @@ import { existsSync, readFileSync, writeFileSync } from "fs";
 import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
 import { classifyUrls } from "../pipeline/classify-url.js";
+import { normalizeMode } from "../pipeline/normalize-mode.js";
 import { assessImpact, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
 import { buildAppContext } from "../orchestration/build-app-context.js";
 import { buildStepSequence } from "../orchestration/build-step-sequence.js";
@@ -23,9 +24,8 @@ import { parseRepoConfig, } from "../adapters/task-sources/repo-schemas.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, "..", "..");
 // ---------------------------------------------------------------------------
-// Valid modes & search modes
+// Valid search modes
 // ---------------------------------------------------------------------------
-const VALID_MODES = ["baseline", "observed", "agentic", "full"];
 const VALID_SEARCH_MODES = ["open", "origin-only", "off"];
 /**
  * Pure option resolution — computes ResolvedOptions from CLI flags without
@@ -36,10 +36,19 @@ const VALID_SEARCH_MODES = ["open", "origin-only", "off"];
 export function computeResolvedOptions(opts) {
     // Resolve paths relative to the caller's cwd, not the eval package root
     const callerCwd = process.env.AILF_CALLER_CWD ?? process.cwd();
-    // Validate mode
-    const mode = opts.mode;
-    if (!VALID_MODES.includes(mode)) {
-        console.error(`❌ Invalid mode "${opts.mode}". Must be one of: ${VALID_MODES.join(", ")}`);
+    // Validate + normalize mode via the single boundary function.
+    // normalizeMode() maps legacy variant names (baseline, agentic, etc.)
+    // to canonical mode "literacy" + variant, and throws on invalid input.
+    let mode;
+    let variant;
+    try {
+        const normalized = normalizeMode(opts.mode);
+        mode = normalized.mode;
+        // Explicit --variant flag takes precedence over what normalizeMode inferred
+        variant = opts.variant ?? normalized.variant;
+    }
+    catch (err) {
+        console.error(`❌ ${err instanceof Error ? err.message : String(err)}`);
         process.exit(1);
     }
     // Debug options — any sub-flag (--debug-n, --debug-pattern, --debug-sample)
@@ -220,6 +229,7 @@ export function computeResolvedOptions(opts) {
         headerArgs,
         impactSummary,
         mode,
+        variant,
         noAutoScope: opts.autoScope === false,
         noCache: !opts.cache,
         noRemoteCache: opts.remoteCache === false,

package/dist/commands/pipeline.d.ts

@@ -35,6 +35,7 @@ export interface PipelineCliOptions {
     header: string[];
     headers: string[];
     mode: string;
+    variant?: string;
     output?: string;
     promptfooUrl?: string;
     publish?: boolean;

package/dist/commands/pipeline.js

@@ -8,11 +8,13 @@
  * @see docs/CLI.md for the full flag reference.
  */
 import { Command } from "commander";
+import { LiteracyVariant } from "../pipeline/normalize-mode.js";
 import { addAgenticOptions, addDebugOptions, addSanitySourceOptions, } from "./shared/options.js";
 export function createPipelineCommand() {
     const cmd = new Command("pipeline")
         .description("Run the full evaluation pipeline")
-        .option("-m, --mode <mode>", "Evaluation mode: full (default — floor + ceiling + actual), baseline (floor + ceiling only), agentic (actual only), observed", "full")
+        .option("-m, --mode <mode>", "Evaluation mode: literacy (default), mcp-server, agent-harness, knowledge-probe, custom. Legacy aliases (baseline, agentic, observed, full) are accepted and normalized to literacy + variant.", LiteracyVariant.FULL)
+        .option("--variant <variant>", "Literacy variant: full (default — standard + agentic), baseline (standard only), agentic (agentic only), observed. Only applies to --mode literacy.")
         .option("-s, --source <name>", "Documentation source name (from sources.yaml)")
         .option("-n, --dry-run", "Validate configuration only, no execution", false)
         .option("--skip-fetch", "Reuse cached documentation contexts", false)
@@ -44,7 +46,7 @@ export function createPipelineCommand() {
         .option("--publish-tag <tag>", "Label for published report")
         .option("--report-dataset <name>", "Sanity dataset for report store")
         .option("--report-project <id>", "Sanity project ID for report store")
-        .option("--config <path>", "Load pipeline config from a JSON/YAML file (overrides most CLI flags)")
+        .option("--config <path>", "Load pipeline config from a TS/JS/YAML/JSON file (overrides most CLI flags)")
         .option("-o, --output <path>", "Write PR comment markdown to file")
         .option("--promptfoo-url <url>", "Promptfoo share URL for report")
         .option("--task-source <type>", "Task definition source: content-lake (default — Sanity Content Lake), repo (repo tasks only, no Content Lake merge), yaml (tasks/*.yaml files, legacy)", "content-lake")

package/dist/commands/pr-comment.js

@@ -20,7 +20,7 @@ export function createPrCommentCommand() {
         try {
             const ctx = createAppContext({
                 rootDir: ROOT,
-                mode: "baseline",
+                mode: "literacy",
                 noAutoScope: false,
                 skipFetch: true,
                 skipEval: true,

package/dist/commands/publish.js

@@ -52,7 +52,7 @@ export function createPublishCommand() {
  */
 function buildProvenanceFromSummary(summary) {
     const areas = summary.scores.map((s) => s.feature);
-    const mode = (process.env.EVAL_MODE ?? "baseline");
+    const mode = (process.env.EVAL_MODE ?? "literacy");
     const source = {
         baseUrl: summary.source?.baseUrl ?? "https://www.sanity.io/docs",
         dataset: summary.source?.dataset ?? process.env.SANITY_DATASET ?? "next",
@@ -83,7 +83,7 @@ async function runPublishCommand(summaryPath, opts) {
         compareEnabled: false,
         discoveryReportEnabled: false,
         gapAnalysisEnabled: false,
-        mode: "baseline",
+        mode: "literacy",
         noAutoScope: false,
         noCache: true,
         noRemoteCache: true,

package/dist/commands/readiness-report.js

@@ -10,14 +10,14 @@ import { Command } from "commander";
 import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
 import { dirname, join, resolve } from "path";
 import { fileURLToPath } from "url";
-import { load } from "js-yaml";
+import { ConfigNotFoundError, loadConfigFile, } from "../pipeline/compiler/config-loader.js";
 import { formatReadinessMarkdown, generateReadinessReport, } from "../pipeline/readiness-report.js";
 import { ThresholdConfigSchema, } from "../pipeline/schemas.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, "..", "..");
 const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
 const GAP_ANALYSIS_PATH = join(ROOT, "results", "latest", "gap-analysis.json");
-const THRESHOLDS_PATH = join(ROOT, "config", "thresholds.yaml");
+// thresholds loaded via loadConfigFile below
 const BASELINES_DIR = join(ROOT, "results", "baselines");
 export function createReadinessReportCommand() {
     return new Command("readiness-report")
@@ -33,12 +33,19 @@ export function createReadinessReportCommand() {
         }
         const scoreSummary = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
         // Load threshold config
-        if (!existsSync(THRESHOLDS_PATH)) {
-            console.error(`❌ Threshold config not found at ${THRESHOLDS_PATH}.`);
+        let parsedThresholds;
+        try {
+            parsedThresholds = loadConfigFile("thresholds", ROOT).data;
+        }
+        catch (err) {
+            if (err instanceof ConfigNotFoundError) {
+                console.error("❌ Threshold config not found in config/.");
+            }
+            else {
+                console.error(`❌ Failed to load threshold config: ${err instanceof Error ? err.message : err}`);
+            }
             process.exit(1);
         }
-        const rawThresholds = readFileSync(THRESHOLDS_PATH, "utf-8");
-        const parsedThresholds = load(rawThresholds);
         const thresholdResult = ThresholdConfigSchema.safeParse(parsedThresholds);
         if (!thresholdResult.success) {
             const messages = thresholdResult.error.issues

package/dist/composition-root.d.ts

@@ -15,7 +15,7 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import type { AppContext, ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+import { type AppContext, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
 /**
  * Create a fully wired AppContext from resolved configuration.
  *

package/dist/composition-root.js

@@ -15,12 +15,13 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
+import { InMemoryPluginRegistry, } from "./_vendor/ailf-core/index.js";
 import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
 import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
-import { SanityDocFetcher } from "./adapters/doc-fetchers/index.js";
 import { PromptfooEvalAdapter } from "./adapters/eval-runners/promptfoo-eval-adapter.js";
 import { ConsoleLogger, JsonLogger, QuietLogger, } from "./adapters/loggers/index.js";
 import { CompositeTaskSource, ContentLakeTaskSource, RepoTaskSource, YamlTaskSource, } from "./adapters/task-sources/index.js";
+import { createSanityLiteracyPreset } from "./pipeline/compiler/presets/index.js";
 import { getSanityClient } from "./sanity/client.js";
 import { ReportStore } from "./report-store.js";
 import { loadSinks } from "./sinks/index.js";
@@ -38,13 +39,18 @@ export function createAppContext(config) {
     const cache = config.noCache ? undefined : createCache(config);
     // Task source — selected by config.taskSourceType
     const taskSource = createTaskSource(config);
-    // Doc fetcher — Sanity Content Lake
-    const docFetcher = new SanityDocFetcher(config.rootDir);
+    // Plugin registry — mode handlers, assertions, rubric templates, doc fetcher.
+    // The Sanity preset is registered here with config.rootDir so its doc fetcher
+    // factory resolves paths relative to the eval package root (not cwd).
+    const registry = createRegistry(config.rootDir);
+    // Doc fetcher — provided by the registered preset's factory
+    const docFetcherFactory = registry.getDocFetcherFactory();
+    const docFetcher = docFetcherFactory ? docFetcherFactory() : undefined;
     // Eval runner — Promptfoo subprocess
     const evalRunner = new PromptfooEvalAdapter(config.rootDir);
     // Report store — Sanity Content Lake (for publish + auto-compare)
     const reportStore = createReportStore(config);
-    // Sinks — loaded from config/sinks.yaml
+    // Sinks — loaded from config/sinks
     const sinks = loadSinks();
     return {
         cache,
@@ -52,6 +58,7 @@ export function createAppContext(config) {
         docFetcher,
         evalRunner,
         logger,
+        registry,
         reportStore,
         sinks,
         taskSource,
@@ -113,6 +120,62 @@ function createTaskSource(config) {
     }
     return primary;
 }
+// ---------------------------------------------------------------------------
+// Built-in mode registrations for non-literacy modes
+// ---------------------------------------------------------------------------
+const BUILT_IN_MODES = [
+    {
+        id: "knowledge-probe",
+        label: "Knowledge Probe",
+        validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
+        rubricTemplateIds: [],
+        handlerModule: "./mode-handlers/knowledge-probe-handler.js",
+    },
+    {
+        id: "mcp-server",
+        label: "MCP Server Testing",
+        validProviderPatterns: ["^mcp:", "^file://"],
+        rubricTemplateIds: [
+            "mcp-input-validation",
+            "mcp-output-correctness",
+            "mcp-error-handling",
+        ],
+        handlerModule: "./mode-handlers/mcp-server-handler.js",
+    },
+    {
+        id: "agent-harness",
+        label: "Agent Harness",
+        validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
+        rubricTemplateIds: [],
+        handlerModule: "./mode-handlers/agent-harness-handler.js",
+    },
+];
+/**
+ * Build and populate the plugin registry.
+ *
+ * Preset registration flow:
+ * 1. A preset is a PresetDefinition — a bundle of modes, assertions, rubric
+ *    templates, prompt templates, scoring profiles, a doc fetcher factory,
+ *    source definitions, and feature definitions.
+ * 2. registerPreset() iterates the preset's fields and delegates each one to
+ *    the appropriate register method (registerMode, registerRubricTemplate, …).
+ * 3. After registration the rest of createAppContext() can pull capabilities
+ *    from the registry (e.g. getDocFetcherFactory()) without knowing which
+ *    preset provided them.
+ *
+ * To add a new preset: create a PresetDefinition, then call
+ * registry.registerPreset() here before the built-in mode registrations.
+ */
+function createRegistry(rootDir) {
+    const registry = new InMemoryPluginRegistry();
+    // Register the sanity-literacy preset — the Sanity-specific evaluation bundle.
+    registry.registerPreset(createSanityLiteracyPreset({ rootDir }));
+    // Register other built-in modes (not part of any preset yet)
+    for (const mode of BUILT_IN_MODES) {
+        registry.registerMode(mode);
+    }
+    return registry;
+}
 function createReportStore(config) {
     return new ReportStore({
         dataset: process.env.AILF_REPORT_DATASET ??

package/dist/orchestration/build-app-context.js

@@ -20,6 +20,7 @@ export function mapToResolvedConfig(opts, rootDir) {
     return {
         rootDir,
         mode: opts.mode,
+        variant: opts.variant,
         noAutoScope: opts.noAutoScope ?? false,
         debug: opts.debug,
         areas: opts.areaOption

package/dist/orchestration/build-step-sequence.js

@@ -5,7 +5,7 @@
  * PipelineStep objects determined by config flags like skipFetch,
  * skipEval, compareEnabled, etc.
  */
-import { FULL_MODE_SUBMODES } from "../_vendor/ailf-shared/index.js";
+import { LiteracyVariant } from "../pipeline/normalize-mode.js";
 import { CallbackStep } from "./steps/callback-step.js";
 import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
 import { CompareStep } from "./steps/compare-step.js";
@@ -40,11 +40,29 @@ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
     // Step 2: Generate Promptfoo configs
     steps.push(new GenerateConfigsStep());
     // Step 3: Run evaluation (steps handle --skip-eval internally)
-    const modes = config.mode === "full"
-        ? [...FULL_MODE_SUBMODES]
-        : [config.mode];
-    for (const mode of modes) {
-        steps.push(new RunEvalStep(mode));
+    //
+    // For literacy mode, the variant determines how many eval steps run:
+    //   "full" → baseline + agentic (two steps)
+    //   "baseline" / "agentic" / "observed" → one step
+    //   undefined → defaults to baseline
+    //
+    // For all other modes, one eval step per mode.
+    if (config.mode === "literacy") {
+        const variant = config.variant ?? LiteracyVariant.STANDARD;
+        if (variant === LiteracyVariant.FULL) {
+            for (const submode of [
+                LiteracyVariant.STANDARD,
+                LiteracyVariant.AGENTIC,
+            ]) {
+                steps.push(new RunEvalStep(submode));
+            }
+        }
+        else {
+            steps.push(new RunEvalStep(variant));
+        }
+    }
+    else {
+        steps.push(new RunEvalStep(config.mode));
     }
     // Step 3c: Grader consistency (optional, conditional)
     if (config.graderReplications) {