@sanity/ailf 0.5.0 → 2.0.0

package/dist/orchestration/steps/generate-configs-step.js

@@ -1,13 +1,16 @@
 /**
  * Pipeline step: Generate Promptfoo configuration files.
  *
- * Calls generateConfigs() from pipeline/generate-configs.ts with typed options
- * derived from AppContext. No env bridge needed — source is resolved and
- * passed directly.
+ * ALL modes route through the Plugin Registry. The step looks up the mode
+ * handler via ctx.registry.getMode() and delegates compilation to it.
+ *
+ * Literacy mode has a variant strategy: baseline/agentic/observed/full.
+ * When the variant is "full", the handler is called twice (baseline + agentic)
+ * and three YAML files are written. Other modes produce one YAML file.
  */
+import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { checkGeneratedConfigsExist } from "../../pipeline/checks.js";
-import { generateConfigs } from "../../pipeline/generate-configs.js";
 import { validateModelsYaml } from "../../pipeline/validate.js";
 import { loadSource } from "../../sources.js";
 import { configToSourceOverrides } from "../config-to-source-overrides.js";
@@ -19,68 +22,246 @@ export class GenerateConfigsStep {
     }
     async execute(ctx, state) {
         const start = Date.now();
-        // Resolve source once with typed overrides
-        const overrides = configToSourceOverrides(ctx.config);
-        const resolvedSource = ctx.config.source
-            ? loadSource(ctx.config.source, overrides)
-            : undefined;
-        // Load tasks via the TaskSource port — this picks up Content Lake,
-        // repo-based, and YAML tasks depending on which adapter is wired.
-        let tasks;
+        const mode = ctx.config.mode;
+        // Look up mode handler in the registry
+        const registration = ctx.registry.getMode(mode);
+        if (!registration) {
+            return {
+                durationMs: Date.now() - start,
+                error: `No handler registered for mode "${mode}". ` +
+                    `Available modes: ${ctx.registry
+                        .getModes()
+                        .map((m) => m.id)
+                        .join(", ")}`,
+                status: "failed",
+            };
+        }
         try {
-            const filter = ctx.config.areas || ctx.config.tasks || ctx.config.tags
-                ? {
-                    areas: ctx.config.areas,
-                    taskIds: ctx.config.tasks,
-                    tags: ctx.config.tags,
-                }
+            // Dynamically import the handler module
+            const handlerModule = await import(`../../pipeline/compiler/${registration.handlerModule}`);
+            const handler = handlerModule.handler;
+            if (!handler?.compileTask) {
+                return {
+                    durationMs: Date.now() - start,
+                    error: `Handler module for "${mode}" does not export a valid ModeHandler`,
+                    status: "failed",
+                };
+            }
+            // Load tasks
+            const tasks = await this.loadTasks(ctx, mode, state);
+            if (tasks.length === 0) {
+                return {
+                    durationMs: Date.now() - start,
+                    error: `No ${mode} tasks found. Create *.task.ts files in ` +
+                        `packages/eval/tasks/${mode}/`,
+                    status: "failed",
+                };
+            }
+            // Load models
+            const { loadModelsAndProviders } = await import("../../pipeline/compiler/provider-assembler.js");
+            const overrides = configToSourceOverrides(ctx.config);
+            const resolvedSource = ctx.config.source
+                ? loadSource(ctx.config.source, overrides)
                 : undefined;
-            tasks = await ctx.taskSource.loadTasks(filter);
+            const { models, providers } = loadModelsAndProviders(ctx.config.rootDir, resolvedSource, ctx.config.searchMode, ctx.config.allowedOrigins);
+            // Literacy mode: variant expansion (baseline + agentic → 3 YAML files)
+            if (mode === "literacy") {
+                return this.compileLiteracyVariants(ctx, handler, tasks, models, providers, start);
+            }
+            // All other modes: single compilation → single YAML file
+            return this.compileSingleMode(ctx, handler, tasks, mode, models, start);
         }
         catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
             return {
                 durationMs: Date.now() - start,
-                error: `TaskSource.loadTasks failed: ${err instanceof Error ? err.message : String(err)}`,
+                error: `${mode} compilation failed: ${msg}`,
                 status: "failed",
             };
         }
-        // Release auto-scope: narrow tasks to those affected by the release.
-        // When explicit area/task filters are also active, this produces the
-        // intersection (only tasks matching BOTH the explicit filter AND the
-        // release impact are included).
+    }
+    // ---------------------------------------------------------------------------
+    // Literacy variant compilation (baseline + agentic → 3 YAML files)
+    // ---------------------------------------------------------------------------
+    async compileLiteracyVariants(ctx, handler, tasks, models, providers, start) {
+        ctx.logger.info(`Compiling ${tasks.length} literacy task(s) via registry handler...`);
+        // Filter models per variant
+        const baselineModels = models.models
+            .filter((m) => !m.modes || m.modes.includes(LiteracyVariant.STANDARD))
+            .map((m) => ({
+            id: m.id,
+            label: m.label,
+        }));
+        const agenticModels = models.models
+            .filter((m) => !m.modes ||
+            m.modes.includes("agentic-naive") ||
+            m.modes.includes("agentic-optimized"))
+            .map((m) => ({
+            id: m.id,
+            label: m.label,
+        }));
+        // Load rubric config for template resolution
+        let rubricConfig;
+        try {
+            const { loadRubricTemplates } = await import("../../pipeline/rubric-loader.js");
+            rubricConfig = loadRubricTemplates(ctx.config.rootDir);
+        }
+        catch {
+            ctx.logger.warn("  ⚠ Could not load rubric config — templates will not resolve");
+        }
+        // Compile for each variant
+        const baselineResults = this.compileAll(handler, tasks, {
+            rootDir: ctx.config.rootDir,
+            graderProvider: models.grader.id,
+            models: baselineModels,
+            rubricConfig,
+            evalMode: LiteracyVariant.STANDARD,
+        });
+        const agenticResults = this.compileAll(handler, tasks, {
+            rootDir: ctx.config.rootDir,
+            graderProvider: models.grader.id,
+            models: agenticModels,
+            rubricConfig,
+            evalMode: LiteracyVariant.AGENTIC,
+        });
+        // Log warnings
+        for (const w of [...baselineResults.warnings, ...agenticResults.warnings]) {
+            ctx.logger.warn(`  ⚠ ${w}`);
+        }
+        ctx.logger.info(`  Compiled ${tasks.length} task(s) → ${baselineResults.tests.length} baseline + ${agenticResults.tests.length} agentic entries`);
+        // Write 3 YAML files via the literacy-specific writer
+        const { writeCompiledLiteracyConfigs } = await import("../../pipeline/compiler/compiler-to-yaml.js");
+        writeCompiledLiteracyConfigs(baselineResults, agenticResults, providers, {
+            rootDir: ctx.config.rootDir,
+            graderProvider: models.grader.id,
+            maxConcurrency: models.maxConcurrency,
+            logger: ctx.logger,
+        });
+        return this.checkLiteracyPostconditions(ctx, start);
+    }
+    // ---------------------------------------------------------------------------
+    // Single-mode compilation (all non-literacy modes)
+    // ---------------------------------------------------------------------------
+    async compileSingleMode(ctx, handler, tasks, mode, models, start) {
+        ctx.logger.info(`Compiling ${tasks.length} ${mode} task(s) via registry handler...`);
+        // Filter models to those that declare this mode in their modes array
+        const modeModels = models.models
+            .filter((m) => !m.modes || m.modes.includes(mode))
+            .map((m) => ({
+            id: m.id,
+            label: m.label,
+            config: m.config,
+        }));
+        const merged = this.compileAll(handler, tasks, {
+            rootDir: ctx.config.rootDir,
+            graderProvider: models.grader.id,
+            models: modeModels,
+        });
+        for (const w of merged.warnings) {
+            ctx.logger.warn(`  ⚠ ${w}`);
+        }
+        ctx.logger.info(`  Compiled ${tasks.length} task(s) → ${merged.tests.length} test entries`);
+        const { writeCompiledModeConfig } = await import("../../pipeline/compiler/compiler-to-yaml.js");
+        writeCompiledModeConfig(merged, mode, {
+            rootDir: ctx.config.rootDir,
+            graderProvider: models.grader.id,
+            maxConcurrency: models.maxConcurrency,
+            logger: ctx.logger,
+        });
+        return {
+            durationMs: Date.now() - start,
+            status: "success",
+            summary: `Generated promptfooconfig.${mode}.yaml`,
+        };
+    }
+    // ---------------------------------------------------------------------------
+    // Task loading — unified for all modes
+    // ---------------------------------------------------------------------------
+    async loadTasks(ctx, mode, state) {
+        const { resolve } = await import("path");
+        const { discoverTsTaskFiles, loadTsTaskFile } = await import("../../adapters/task-sources/task-file-loader.js");
+        // Discover task files from the mode-specific directory and --repo-tasks-path
+        const tasksDir = resolve(ctx.config.rootDir, "tasks", mode);
+        const dirs = [tasksDir];
+        // Also search --repo-tasks-path (e.g., .ailf/tasks/) for repo-based tasks
+        if (ctx.config.repoTasksPath) {
+            const repoDir = resolve(ctx.config.repoTasksPath);
+            if (!dirs.includes(repoDir)) {
+                dirs.push(repoDir);
+            }
+        }
+        const tasks = [];
+        for (const dir of dirs) {
+            const files = discoverTsTaskFiles(dir);
+            for (const file of files) {
+                const raw = await loadTsTaskFile(file);
+                for (const t of raw.tasks) {
+                    const task = t;
+                    // Filter to matching mode (skip tasks from other modes in same dir)
+                    if (!("mode" in task) || task.mode === mode) {
+                        tasks.push(task);
+                    }
+                }
+            }
+        }
+        // Apply area/task/tag filters
+        const filtered = this.applyFilters(ctx, tasks);
+        // Release auto-scope
         if (state.releaseAutoScope && !ctx.config.noAutoScope) {
             const scopedIds = new Set(state.releaseAutoScope.affectedTaskIds);
-            const beforeCount = tasks.length;
-            tasks = tasks.filter((t) => scopedIds.has(t.id));
-            console.log(`  🎯 Auto-scoped to ${tasks.length} of ${beforeCount} task(s) affected by release` +
-                ` (${beforeCount - tasks.length} skipped, --no-auto-scope to override)`);
+            const beforeCount = filtered.length;
+            const scoped = filtered.filter((t) => "id" in t && scopedIds.has(t.id));
+            ctx.logger.info(`  🎯 Auto-scoped to ${scoped.length} of ${beforeCount} task(s) affected by release`);
+            return scoped;
         }
-        try {
-            generateConfigs({
-                allowedOrigins: ctx.config.allowedOrigins,
-                filter: ctx.config.areas || ctx.config.tasks || ctx.config.tags
-                    ? {
-                        areas: ctx.config.areas,
-                        taskIds: ctx.config.tasks,
-                        tags: ctx.config.tags,
-                    }
-                    : undefined,
-                logger: ctx.logger,
-                resolvedSource,
-                rootDir: ctx.config.rootDir,
-                searchMode: ctx.config.searchMode,
-                source: ctx.config.source,
-                tasks,
+        return filtered;
+    }
+    applyFilters(ctx, tasks) {
+        let result = tasks;
+        if (ctx.config.areas?.length) {
+            const allowed = new Set(ctx.config.areas.map((a) => a.toLowerCase()));
+            result = result.filter((t) => {
+                const area = t.area?.toLowerCase();
+                return area && allowed.has(area);
             });
         }
-        catch (err) {
-            return {
-                durationMs: Date.now() - start,
-                error: `generate-configs failed: ${err instanceof Error ? err.message : String(err)}`,
-                status: "failed",
-            };
+        if (ctx.config.tasks?.length) {
+            const allowed = new Set(ctx.config.tasks);
+            result = result.filter((t) => {
+                const id = t.id;
+                return id && allowed.has(id);
+            });
         }
-        // Postcondition: config files exist
+        if (ctx.config.tags?.length) {
+            const allowed = new Set(ctx.config.tags);
+            result = result.filter((t) => {
+                const tags = t.tags;
+                return tags?.some((tag) => allowed.has(tag));
+            });
+        }
+        return result;
+    }
+    // ---------------------------------------------------------------------------
+    // Compilation helpers
+    // ---------------------------------------------------------------------------
+    /**
+     * Compile all tasks through a handler, merging results.
+     * For literacy mode, ctx can carry evalMode as an extension.
+     */
+    compileAll(handler, tasks, ctx) {
+        const results = [];
+        const warnings = [];
+        for (const task of tasks) {
+            const result = handler.compileTask(task, ctx);
+            results.push(result);
+            warnings.push(...result.warnings);
+        }
+        return mergeCompileResults(results);
+    }
+    // ---------------------------------------------------------------------------
+    // Postcondition checks
+    // ---------------------------------------------------------------------------
+    checkLiteracyPostconditions(ctx, start) {
         const configIssues = checkGeneratedConfigsExist(ctx.config.rootDir);
         const configErrors = configIssues.filter((i) => i.severity === "error");
         if (configErrors.length > 0) {
@@ -100,3 +281,32 @@ export class GenerateConfigsStep {
         return getStepInputPaths(ctx.config.rootDir, "generate-configs");
     }
 }
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Merge multiple compile results into one.
+ *
+ * Note: `providers` and `prompts` are taken from the first result only.
+ * This is correct for single-mode compilation where all tasks share the
+ * same provider set. Cross-mode merging with per-task provider overrides
+ * would need deduplication here.
+ */
+function mergeCompileResults(results) {
+    const tests = results.flatMap((r) => r.tests);
+    const warnings = results.flatMap((r) => r.warnings);
+    const providers = results[0]?.providers ?? [];
+    const prompts = results[0]?.prompts ?? [];
+    const extras = {};
+    for (const r of results) {
+        if (r.extras)
+            Object.assign(extras, r.extras);
+    }
+    return {
+        providers,
+        tests,
+        prompts,
+        warnings,
+        ...(Object.keys(extras).length > 0 ? { extras } : {}),
+    };
+}

package/dist/orchestration/steps/grader-consistency-step.js

@@ -6,8 +6,9 @@
  */
 import { existsSync } from "fs";
 import { resolve } from "path";
+import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { checkResultsExist } from "../../pipeline/checks.js";
-import { RESULTS_FILES } from "../../pipeline/eval-constants.js";
+import { resultsFileForMode } from "../../pipeline/eval-constants.js";
 import { runGraderConsistency } from "../../pipeline/grader-consistency-runner.js";
 export class GraderConsistencyStep {
     name = "grader-consistency";
@@ -18,10 +19,12 @@ export class GraderConsistencyStep {
     async execute(ctx) {
         const start = Date.now();
         const replications = ctx.config.graderReplications ?? 5;
-        const concreteMode = ctx.config.mode === "full"
-            ? "baseline"
+        const primaryResultsRun = ctx.config.mode === "literacy"
+            ? ctx.config.variant === LiteracyVariant.FULL
+                ? LiteracyVariant.STANDARD
+                : (ctx.config.variant ?? LiteracyVariant.STANDARD)
             : ctx.config.mode;
-        const resultsFile = RESULTS_FILES[concreteMode];
+        const resultsFile = resultsFileForMode(primaryResultsRun);
         // Precondition: results file exists
         const resultsIssues = checkResultsExist(ctx.config.rootDir, resultsFile);
         const resultsErrors = resultsIssues.filter((i) => i.severity === "error");

package/dist/orchestration/steps/mirror-repo-tasks-step.js

@@ -43,7 +43,7 @@ export class MirrorRepoTasksStep {
             // RepoTaskSource via a fresh instance.
             const { RepoTaskSource } = await import("../../adapters/task-sources/repo-task-source.js");
             const repoSource = new RepoTaskSource(ctx.config.repoTasksPath);
-            const repoTasks = await repoSource.loadTasks();
+            const repoTasks = (await repoSource.loadTasks()).filter((t) => t.mode === "literacy");
             if (repoTasks.length === 0) {
                 return {
                     durationMs: Date.now() - start,

package/dist/orchestration/steps/readiness-step.js

@@ -6,7 +6,7 @@
  */
 import { existsSync, readFileSync, writeFileSync } from "fs";
 import { resolve } from "path";
-import { load } from "js-yaml";
+import { tryLoadConfigFile } from "../../pipeline/compiler/config-loader.js";
 import { formatReadinessMarkdown, generateReadinessReport, } from "../../pipeline/readiness-report.js";
 import { ThresholdConfigSchema } from "../../pipeline/schemas.js";
 export class ReadinessStep {
@@ -20,7 +20,6 @@ export class ReadinessStep {
         const start = Date.now();
         try {
             const scoreSummaryPath = resolve(root, "results", "latest", "score-summary.json");
-            const thresholdsPath = resolve(root, "config", "thresholds.yaml");
             if (!existsSync(scoreSummaryPath)) {
                 return {
                     durationMs: Date.now() - start,
@@ -28,16 +27,16 @@ export class ReadinessStep {
                     status: "failed",
                 };
             }
-            if (!existsSync(thresholdsPath)) {
+            const thresholdsLoaded = tryLoadConfigFile("thresholds", root);
+            if (!thresholdsLoaded) {
                 return {
                     durationMs: Date.now() - start,
-                    error: "config/thresholds.yaml not found",
+                    error: "config/thresholds not found",
                     status: "failed",
                 };
             }
             const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-            const rawThresholds = load(readFileSync(thresholdsPath, "utf-8"));
-            const thresholdConfig = ThresholdConfigSchema.parse(rawThresholds);
+            const thresholdConfig = ThresholdConfigSchema.parse(thresholdsLoaded.data);
             const gapPath = resolve(root, "results", "latest", "gap-analysis.json");
             const gapAnalysis = existsSync(gapPath)
                 ? JSON.parse(readFileSync(gapPath, "utf-8"))

package/dist/orchestration/steps/run-eval-step.d.ts

@@ -5,12 +5,11 @@
  * invocation. Builds a clean env object for the subprocess instead of
  * polluting global process.env.
  */
-import type { ConcreteEvalMode } from "../../_vendor/ailf-shared/index.d.ts";
 import type { AppContext, PipelineState, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class RunEvalStep implements PipelineStep {
     private readonly mode;
     readonly name: string;
-    constructor(mode: ConcreteEvalMode);
+    constructor(mode: string);
     check(): ValidationIssue[];
     execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
     cacheInputs(ctx: AppContext): string[];

package/dist/orchestration/steps/run-eval-step.js

@@ -10,7 +10,7 @@ import { resolve } from "path";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
 import { computeEvalFingerprint } from "../../pipeline/eval-fingerprint.js";
-import { buildFilterFlags, CONFIG_FILES, RESULTS_FILES, scanResultsForErrors, } from "../../pipeline/eval-constants.js";
+import { buildFilterFlags, configFileForMode, resultsFileForMode, scanResultsForErrors, } from "../../pipeline/eval-constants.js";
 export class RunEvalStep {
     mode;
     name;
@@ -59,7 +59,8 @@ export class RunEvalStep {
         // The generated Promptfoo config still includes their "without-docs"
         // variant (testing model knowledge alone), which doesn't need a
         // context file.
-        const tasksWithDocs = tasks.filter((t) => t.canonicalDocs.length > 0);
+        // Bridge: narrow to literacy tasks with docs
+        const tasksWithDocs = tasks.filter((t) => t.mode === "literacy" && (t.context?.docs?.length ?? 0) > 0);
         const taskIds = tasksWithDocs.map((t) => t.id);
         const contextIssues = checkCanonicalContextsExist(rootDir, taskIds);
         const contextErrors = contextIssues.filter((i) => i.severity === "error");
@@ -123,7 +124,7 @@ export class RunEvalStep {
         };
         // Only set env vars that differ from defaults — the subprocess inherits
         // process.env via PromptfooEvalAdapter's { ...process.env, ...config.env }
-        if (ctx.config.mode !== "baseline") {
+        if (ctx.config.mode !== "literacy") {
             subprocessEnv.EVAL_MODE = ctx.config.mode;
         }
         if (ctx.config.searchMode !== "open") {
@@ -135,7 +136,7 @@ export class RunEvalStep {
         // -----------------------------------------------------------------
         // Execute — use the EvalRunner port
         // -----------------------------------------------------------------
-        const configFile = CONFIG_FILES[this.mode];
+        const configFile = configFileForMode(this.mode);
         const filterFlags = buildFilterFlags(debug);
         const result = await ctx.evalRunner.run({
             concurrency,
@@ -145,7 +146,7 @@ export class RunEvalStep {
         });
         // Check if results were written despite non-zero exit
         if (result.status === "failed") {
-            const resultsExist = checkResultsExist(rootDir, RESULTS_FILES[this.mode]);
+            const resultsExist = checkResultsExist(rootDir, resultsFileForMode(this.mode));
             const hasResults = resultsExist.filter((i) => i.severity === "error").length === 0;
             if (!hasResults) {
                 return {
@@ -156,7 +157,7 @@ export class RunEvalStep {
             }
         }
         // Postcondition: results file exists
-        const resultsIssues = checkResultsExist(rootDir, RESULTS_FILES[this.mode]);
+        const resultsIssues = checkResultsExist(rootDir, resultsFileForMode(this.mode));
         const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
         if (resultsErrors.length > 0) {
             return {
@@ -166,7 +167,7 @@ export class RunEvalStep {
             };
         }
         // Scan results for errors
-        const errorSummary = scanResultsForErrors(resolve(rootDir, RESULTS_FILES[this.mode]));
+        const errorSummary = scanResultsForErrors(resolve(rootDir, resultsFileForMode(this.mode)));
         if (errorSummary) {
             console.log();
             console.log(errorSummary);

package/dist/pipeline/cache.d.ts

@@ -10,7 +10,7 @@
  *
  * Cache invalidation triggers:
  *   - Content change: any input file's content changes → hash changes → miss
- *   - Config change: config/models.yaml, config/sources.yaml, tasks/*.yaml changes → miss
+ *   - Config change: config/models, config/sources, tasks/*.yaml changes → miss
  *   - Manual bypass: --no-cache flag skips all cache lookups
  *   - Cache clear: delete results/cache/ to start fresh
  */

package/dist/pipeline/cache.js

@@ -10,7 +10,7 @@
  *
  * Cache invalidation triggers:
  *   - Content change: any input file's content changes → hash changes → miss
- *   - Config change: config/models.yaml, config/sources.yaml, tasks/*.yaml changes → miss
+ *   - Config change: config/models, config/sources, tasks/*.yaml changes → miss
  *   - Manual bypass: --no-cache flag skips all cache lookups
  *   - Cache clear: delete results/cache/ to start fresh
  */
@@ -18,6 +18,19 @@ import { createHash } from "crypto";
 import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync, } from "fs";
 import { join, resolve } from "path";
 // ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Resolve first existing config file (matches loadConfigFile priority chain) */
+function resolveConfig(rootDir, name) {
+    const r = (f) => resolve(rootDir, f);
+    for (const ext of [".ts", ".js", ".yaml", ".yml", ".json"]) {
+        const p = r(`config/${name}${ext}`);
+        if (existsSync(p))
+            return p;
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
 // Constants
 // ---------------------------------------------------------------------------
 const CACHE_DIR_NAME = "cache";
@@ -79,7 +92,10 @@ export function getStepInputPaths(rootDir, step) {
             const isBaseline = step === "eval-baseline" || step === "eval";
             const isAgentic = step === "eval-agentic" || step === "eval";
             const isObserved = step === "eval-observed" || step === "eval";
-            const paths = [r("config/models.yaml")];
+            const paths = [];
+            const modelsPath = resolveConfig(rootDir, "models");
+            if (modelsPath)
+                paths.push(modelsPath);
             // Config files — only the relevant ones for this mode
             if (isBaseline) {
                 paths.push(r("promptfooconfig.yaml"));
@@ -130,25 +146,37 @@ export function getStepInputPaths(rootDir, step) {
             return paths;
         }
         case "fetch-docs": {
-            // Inputs: config/sources.yaml, config/models.yaml, task files (which contain inline mappings)
-            const paths = [r("config/sources.yaml"), r("config/models.yaml")];
+            // Inputs: config sources + models, task files
+            const paths = [];
+            const sourcesPath = resolveConfig(rootDir, "sources");
+            const modelsPath2 = resolveConfig(rootDir, "models");
+            if (sourcesPath)
+                paths.push(sourcesPath);
+            if (modelsPath2)
+                paths.push(modelsPath2);
             // Include all task files (they define feature areas)
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
                 const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
                     .map((f) => join(tasksDir, f));
                 paths.push(...taskFiles);
             }
             return paths;
         }
         case "generate-configs": {
-            // Inputs: config/models.yaml, config/sources.yaml, all task files
-            const paths = [r("config/models.yaml"), r("config/sources.yaml")];
+            // Inputs: config models + sources, all task files
+            const paths = [];
+            const modelsPath3 = resolveConfig(rootDir, "models");
+            const sourcesPath2 = resolveConfig(rootDir, "sources");
+            if (modelsPath3)
+                paths.push(modelsPath3);
+            if (sourcesPath2)
+                paths.push(sourcesPath2);
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
                 const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
                     .map((f) => join(tasksDir, f));
                 paths.push(...taskFiles);
             }

package/dist/pipeline/calculate-scores.d.ts

@@ -1,9 +1,7 @@
-import type { Logger, TestSummary } from "../_vendor/ailf-core/index.d.ts";
+import { type ActualScoreEntry, type ComponentResult, type Logger, type TestSummary } from "../_vendor/ailf-core/index.d.ts";
 import { type ResolvedSourceConfig } from "../sources.js";
-import { type ActualScoreEntry, type ComponentResult } from "../_vendor/ailf-core/index.d.ts";
 import type { GraderJudgment, PerModelEntry } from "./types.js";
-export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.d.ts";
-export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, type ActualScoreEntry, type ComponentResult, type TestResult, type UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
 export interface PromptfooResultsWrapper {
     results: RawTestResult[];
     stats: {