@sanity/ailf 1.0.0 → 2.0.1

package/dist/orchestration/steps/generate-configs-step.js

@@ -8,8 +8,12 @@
  * When the variant is "full", the handler is called twice (baseline + agentic)
  * and three YAML files are written. Other modes produce one YAML file.
  */
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
 import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
+import { modelMatchesLiteracyVariant } from "../../pipeline/compiler/mode-bases/literacy.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
+import { buildCacheContext } from "../cache-context.js";
 import { checkGeneratedConfigsExist } from "../../pipeline/checks.js";
 import { validateModelsYaml } from "../../pipeline/validate.js";
 import { loadSource } from "../../sources.js";
@@ -85,21 +89,14 @@ export class GenerateConfigsStep {
     // ---------------------------------------------------------------------------
     async compileLiteracyVariants(ctx, handler, tasks, models, providers, start) {
         ctx.logger.info(`Compiling ${tasks.length} literacy task(s) via registry handler...`);
-        // Filter models per variant
+        // Filter models per variant using shared literacy variant matcher
         const baselineModels = models.models
-            .filter((m) => !m.modes || m.modes.includes(LiteracyVariant.STANDARD))
-            .map((m) => ({
-            id: m.id,
-            label: m.label,
-        }));
+            .filter((m) => modelMatchesLiteracyVariant(m, "baseline"))
+            .map((m) => ({ id: m.id, label: m.label }));
         const agenticModels = models.models
-            .filter((m) => !m.modes ||
-            m.modes.includes("agentic-naive") ||
-            m.modes.includes("agentic-optimized"))
-            .map((m) => ({
-            id: m.id,
-            label: m.label,
-        }));
+            .filter((m) => modelMatchesLiteracyVariant(m, "agentic-naive") ||
+            modelMatchesLiteracyVariant(m, "agentic-optimized"))
+            .map((m) => ({ id: m.id, label: m.label }));
         // Load rubric config for template resolution
         let rubricConfig;
         try {
@@ -137,6 +134,14 @@ export class GenerateConfigsStep {
             maxConcurrency: models.maxConcurrency,
             logger: ctx.logger,
         });
+        // Capture generated config files (use configFileForMode for legacy naming)
+        const { configFileForMode } = await import("../../pipeline/eval-constants.js");
+        for (const variant of ["baseline", "agentic", "observed"]) {
+            const configPath = resolve(ctx.config.rootDir, configFileForMode(variant));
+            if (existsSync(configPath)) {
+                ctx.collector.captureFile("generate-configs", `promptfoo-config-${variant}`, configPath, { mode: "literacy", variant });
+            }
+        }
         return this.checkLiteracyPostconditions(ctx, start);
     }
     // ---------------------------------------------------------------------------
@@ -144,13 +149,18 @@ export class GenerateConfigsStep {
     // ---------------------------------------------------------------------------
     async compileSingleMode(ctx, handler, tasks, mode, models, start) {
         ctx.logger.info(`Compiling ${tasks.length} ${mode} task(s) via registry handler...`);
+        // Filter models to those that declare this mode in their modes array
+        const modeModels = models.models
+            .filter((m) => !m.modes || m.modes.includes(mode))
+            .map((m) => ({
+            id: m.id,
+            label: m.label,
+            config: m.config,
+        }));
         const merged = this.compileAll(handler, tasks, {
             rootDir: ctx.config.rootDir,
             graderProvider: models.grader.id,
-            models: models.models.map((m) => ({
-                id: m.id,
-                label: m.label,
-            })),
+            models: modeModels,
         });
         for (const w of merged.warnings) {
             ctx.logger.warn(`  ⚠ ${w}`);
@@ -163,6 +173,18 @@ export class GenerateConfigsStep {
             maxConcurrency: models.maxConcurrency,
             logger: ctx.logger,
         });
+        // Capture generated config file
+        const configPath = resolve(ctx.config.rootDir, `promptfooconfig.${mode}.yaml`);
+        if (existsSync(configPath)) {
+            ctx.collector.captureFile("generate-configs", "promptfoo-config", configPath, { mode });
+        }
+        // Capture mode-specific test artifacts (extras)
+        if (ctx.collector.extrasEnabled) {
+            const testsPath = resolve(ctx.config.rootDir, "results", "latest", `${mode}-tests.json`);
+            if (existsSync(testsPath)) {
+                ctx.collector.captureFile("generate-configs", `${mode}-tests`, testsPath, { mode });
+            }
+        }
         return {
             durationMs: Date.now() - start,
             status: "success",
@@ -175,19 +197,45 @@ export class GenerateConfigsStep {
     async loadTasks(ctx, mode, state) {
         const { resolve } = await import("path");
         const { discoverTsTaskFiles, loadTsTaskFile } = await import("../../adapters/task-sources/task-file-loader.js");
-        const tasksDir = resolve(ctx.config.rootDir, "tasks", mode);
-        const files = discoverTsTaskFiles(tasksDir);
+        const { resolveVendoredSubdir } = await import("../../pipeline/compiler/config-loader.js");
+        // Discover task files from the mode-specific directory and --repo-tasks-path.
+        // Use vendored copies in dist/ when @sanity/ailf-core isn't resolvable
+        // (i.e., running outside the monorepo via npx).
+        const tasksDir = resolveVendoredSubdir(ctx.config.rootDir, `tasks/${mode}`);
+        const dirs = [tasksDir];
+        // Also search --repo-tasks-path (e.g., .ailf/tasks/) for repo-based tasks
+        if (ctx.config.repoTasksPath) {
+            const repoDir = resolve(ctx.config.repoTasksPath);
+            if (!dirs.includes(repoDir)) {
+                dirs.push(repoDir);
+            }
+        }
         const tasks = [];
-        for (const file of files) {
-            const raw = await loadTsTaskFile(file);
-            for (const t of raw.tasks) {
-                const task = t;
-                // Filter to matching mode (skip tasks from other modes in same dir)
-                if (!("mode" in task) || task.mode === mode) {
-                    tasks.push(task);
+        const skippedByMode = new Map();
+        for (const dir of dirs) {
+            const files = discoverTsTaskFiles(dir);
+            for (const file of files) {
+                const raw = await loadTsTaskFile(file);
+                for (const t of raw.tasks) {
+                    const task = t;
+                    // Filter to matching mode (skip tasks from other modes in same dir)
+                    if (!("mode" in task) || task.mode === mode) {
+                        tasks.push(task);
+                    }
+                    else {
+                        const taskMode = task.mode ?? "unknown";
+                        skippedByMode.set(taskMode, (skippedByMode.get(taskMode) ?? 0) + 1);
+                    }
                 }
             }
         }
+        if (skippedByMode.size > 0) {
+            const total = [...skippedByMode.values()].reduce((a, b) => a + b, 0);
+            const summary = [...skippedByMode.entries()]
+                .map(([m, n]) => `${n} ${m}`)
+                .join(", ");
+            ctx.logger.warn(`  ⚠ Skipped ${total} task(s) with non-matching mode (${summary}). Current pipeline mode: ${mode}. Run with --mode <mode> to include them.`);
+        }
         // Apply area/task/tag filters
         const filtered = this.applyFilters(ctx, tasks);
         // Release auto-scope
@@ -264,6 +312,9 @@ export class GenerateConfigsStep {
     cacheInputs(ctx) {
         return getStepInputPaths(ctx.config.rootDir, "generate-configs");
     }
+    cacheContext(ctx) {
+        return buildCacheContext(ctx.config);
+    }
 }
 // ---------------------------------------------------------------------------
 // Helpers

package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts

@@ -10,7 +10,7 @@
  * exist before evaluation begins.
  *
  * @see packages/eval/src/pipeline/mirror-repo-tasks.ts
- * @see docs/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
  */
 import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class MirrorRepoTasksStep implements PipelineStep {

package/dist/orchestration/steps/mirror-repo-tasks-step.js

@@ -10,7 +10,7 @@
  * exist before evaluation begins.
  *
  * @see packages/eval/src/pipeline/mirror-repo-tasks.ts
- * @see docs/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
  */
 import { getSanityClient } from "../../sanity/client.js";
 import { detectGitContext, mirrorRepoTasks, } from "../../pipeline/mirror-repo-tasks.js";

package/dist/orchestration/steps/publish-report-step.js

@@ -115,12 +115,31 @@ export class PublishReportStep {
         };
         // Share reportId with downstream steps (CallbackStep + orchestrator job update)
         state.reportId = reportId;
+        // Capture report object (Tier 2)
+        ctx.collector.capture("publish-report", "report-object", report);
+        // Capture auto-comparison if present (Tier 2)
+        if (comparison) {
+            ctx.collector.capture("publish-report", "auto-comparison", comparison);
+        }
         // Write to store (system of record — best-effort, P5)
         const sanityResult = ctx.reportStore
             ? await ctx.reportStore.write(report)
             : null;
         // Run sinks (fire-and-forget, P6)
         const publishResult = await runSinks(report, ctx);
+        // Capture sink results (Tier 2)
+        if (publishResult.sinkResults.length > 0) {
+            ctx.collector.capture("publish-report", "sink-results", {
+                sinkCount: publishResult.sinkResults.length,
+                results: publishResult.sinkResults.map((r) => ({
+                    name: r.name,
+                    status: r.result.status,
+                    ...(r.result.status === "success" ? { detail: r.result.detail } : {}),
+                    ...(r.result.status === "failed" ? { error: r.result.error } : {}),
+                    ...(r.result.status === "skipped" ? { reason: r.result.reason } : {}),
+                })),
+            });
+        }
         // Build result summary
         const parts = [];
         if (sanityResult) {

package/dist/orchestration/steps/readiness-step.js

@@ -4,7 +4,7 @@
  * Calls pure functions from pipeline/readiness-report.ts directly.
  * Optional step — failure doesn't stop the pipeline.
  */
-import { existsSync, readFileSync, writeFileSync } from "fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { resolve } from "path";
 import { tryLoadConfigFile } from "../../pipeline/compiler/config-loader.js";
 import { formatReadinessMarkdown, generateReadinessReport, } from "../../pipeline/readiness-report.js";
@@ -37,7 +37,8 @@ export class ReadinessStep {
             }
             const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
             const thresholdConfig = ThresholdConfigSchema.parse(thresholdsLoaded.data);
-            const gapPath = resolve(root, "results", "latest", "gap-analysis.json");
+            // Read gap-analysis.json from outputDir (gap-analysis step writes there)
+            const gapPath = resolve(ctx.config.outputDir, "gap-analysis.json");
             const gapAnalysis = existsSync(gapPath)
                 ? JSON.parse(readFileSync(gapPath, "utf-8"))
                 : undefined;
@@ -60,7 +61,11 @@ export class ReadinessStep {
                 console.log(md);
             }
             if (readinessLines.length > 0) {
-                writeFileSync(resolve(root, "results", "latest", "readiness-report.md"), readinessLines.join("\n---\n\n"));
+                // Write to outputDir (respects --output-dir)
+                mkdirSync(ctx.config.outputDir, { recursive: true });
+                const readinessPath = resolve(ctx.config.outputDir, "readiness-report.md");
+                writeFileSync(readinessPath, readinessLines.join("\n---\n\n"));
+                ctx.collector.captureFile("readiness", "readiness-report", readinessPath);
             }
             const passCount = readinessAreas.filter((area) => {
                 const areaScore = scoreSummary.scores.find((s) => s.feature === area);

package/dist/orchestration/steps/report-step.js

@@ -4,10 +4,10 @@
  * Calls generatePrComment() from pipeline/pr-comment.ts with typed options.
  * No env bridge or process.argv manipulation needed.
  */
-import { resolve } from "path";
+import { existsSync, mkdirSync } from "node:fs";
+import { dirname, resolve } from "path";
 import { checkScoreSummaryValid } from "../../pipeline/checks.js";
 import { generatePrComment } from "../../pipeline/pr-comment.js";
-const DEFAULT_REPORT_PATH = "results/latest/pr-comment.md";
 export class ReportStep {
     name = "report";
     check() {
@@ -15,7 +15,7 @@ export class ReportStep {
     }
     async execute(ctx) {
         const start = Date.now();
-        // Precondition: score summary exists
+        // Precondition: score summary exists (intermediate files stay in rootDir)
         const summaryIssues = checkScoreSummaryValid(ctx.config.rootDir);
         const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
         if (summaryErrors.length > 0) {
@@ -25,7 +25,12 @@ export class ReportStep {
                 status: "failed",
             };
         }
-        const resolvedOutput = ctx.config.outputPath ?? resolve(ctx.config.rootDir, DEFAULT_REPORT_PATH);
+        // User-facing output: --output flag wins, else outputDir
+        const resolvedOutput = ctx.config.outputPath ?? resolve(ctx.config.outputDir, "pr-comment.md");
+        // Ensure outputDir exists before writing (it may be a custom --output-dir
+        // that hasn't been created yet — writePipelineResult runs after the
+        // orchestrator returns, so we can't rely on it).
+        mkdirSync(dirname(resolvedOutput), { recursive: true });
         try {
             generatePrComment({
                 outputPath: resolvedOutput,
@@ -40,6 +45,14 @@ export class ReportStep {
                 status: "failed",
             };
         }
+        // Capture report artifacts
+        if (existsSync(resolvedOutput)) {
+            ctx.collector.captureFile("report", "pr-comment", resolvedOutput);
+        }
+        const pipelineResultPath = resolve(ctx.config.outputDir, "pipeline-result.json");
+        if (existsSync(pipelineResultPath)) {
+            ctx.collector.captureFile("report", "pipeline-result", pipelineResultPath);
+        }
         return {
             durationMs: Date.now() - start,
             status: "success",

package/dist/orchestration/steps/run-eval-step.d.ts

@@ -13,4 +13,5 @@ export declare class RunEvalStep implements PipelineStep {
     check(): ValidationIssue[];
     execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
     cacheInputs(ctx: AppContext): string[];
+    cacheContext(ctx: AppContext): string[];
 }

package/dist/orchestration/steps/run-eval-step.js

@@ -8,6 +8,7 @@
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { resolve } from "path";
 import { getStepInputPaths } from "../../pipeline/cache.js";
+import { buildCacheContext } from "../cache-context.js";
 import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
 import { computeEvalFingerprint } from "../../pipeline/eval-fingerprint.js";
 import { buildFilterFlags, configFileForMode, resultsFileForMode, scanResultsForErrors, } from "../../pipeline/eval-constants.js";
@@ -38,38 +39,41 @@ export class RunEvalStep {
             };
         }
         // Precondition: canonical context files exist for filtered tasks.
-        // Must apply the same area/task filter as fetch-docs so we only
-        // check contexts that were actually fetched.
-        const filter = ctx.config.areas || ctx.config.tasks || ctx.config.tags
-            ? {
-                ...(ctx.config.areas ? { areas: ctx.config.areas } : {}),
-                ...(ctx.config.tasks ? { taskIds: ctx.config.tasks } : {}),
-                ...(ctx.config.tags ? { tags: ctx.config.tags } : {}),
+        // Only applies to literacy mode — other modes don't use canonical doc contexts.
+        if (this.mode === "literacy") {
+            // Must apply the same area/task filter as fetch-docs so we only
+            // check contexts that were actually fetched.
+            const filter = ctx.config.areas || ctx.config.tasks || ctx.config.tags
+                ? {
+                    ...(ctx.config.areas ? { areas: ctx.config.areas } : {}),
+                    ...(ctx.config.tasks ? { taskIds: ctx.config.tasks } : {}),
+                    ...(ctx.config.tags ? { tags: ctx.config.tags } : {}),
+                }
+                : undefined;
+            let tasks = await ctx.taskSource.loadTasks(filter);
+            // Release auto-scope: narrow to affected tasks (mirrors GenerateConfigsStep)
+            if (state.releaseAutoScope && !ctx.config.noAutoScope) {
+                const scopedIds = new Set(state.releaseAutoScope.affectedTaskIds);
+                tasks = tasks.filter((t) => scopedIds.has(t.id));
+            }
+            // Only check context files for tasks that have canonical docs.
+            // Tasks without canonical docs are skipped by FetchDocsStep (they
+            // have no docs to fetch), so no context file is written for them.
+            // The generated Promptfoo config still includes their "without-docs"
+            // variant (testing model knowledge alone), which doesn't need a
+            // context file.
+            // Bridge: narrow to literacy tasks with docs
+            const tasksWithDocs = tasks.filter((t) => t.mode === "literacy" && (t.context?.docs?.length ?? 0) > 0);
+            const taskIds = tasksWithDocs.map((t) => t.id);
+            const contextIssues = checkCanonicalContextsExist(rootDir, taskIds);
+            const contextErrors = contextIssues.filter((i) => i.severity === "error");
+            if (contextErrors.length > 0) {
+                return {
+                    durationMs: Date.now() - start,
+                    error: `Context files missing. Run fetch-docs first. ${contextErrors.map((e) => e.message).join("; ")}`,
+                    status: "failed",
+                };
             }
-            : undefined;
-        let tasks = await ctx.taskSource.loadTasks(filter);
-        // Release auto-scope: narrow to affected tasks (mirrors GenerateConfigsStep)
-        if (state.releaseAutoScope && !ctx.config.noAutoScope) {
-            const scopedIds = new Set(state.releaseAutoScope.affectedTaskIds);
-            tasks = tasks.filter((t) => scopedIds.has(t.id));
-        }
-        // Only check context files for tasks that have canonical docs.
-        // Tasks without canonical docs are skipped by FetchDocsStep (they
-        // have no docs to fetch), so no context file is written for them.
-        // The generated Promptfoo config still includes their "without-docs"
-        // variant (testing model knowledge alone), which doesn't need a
-        // context file.
-        // Bridge: narrow to literacy tasks with docs
-        const tasksWithDocs = tasks.filter((t) => t.mode === "literacy" && (t.context?.docs?.length ?? 0) > 0);
-        const taskIds = tasksWithDocs.map((t) => t.id);
-        const contextIssues = checkCanonicalContextsExist(rootDir, taskIds);
-        const contextErrors = contextIssues.filter((i) => i.severity === "error");
-        if (contextErrors.length > 0) {
-            return {
-                durationMs: Date.now() - start,
-                error: `Context files missing. Run fetch-docs first. ${contextErrors.map((e) => e.message).join("; ")}`,
-                status: "failed",
-            };
         }
         // -----------------------------------------------------------------
         // Compute eval fingerprint (for remote cache + provenance)
@@ -109,6 +113,11 @@ export class RunEvalStep {
                 // required eval modes were satisfied from the remote cache.
                 state.remoteCacheHits ??= new Set();
                 state.remoteCacheHits.add(this.mode);
+                // Capture the restored score-summary from remote cache
+                const cachedSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
+                if (existsSync(cachedSummaryPath)) {
+                    ctx.collector.captureFile("run-eval", "score-summary-cached", cachedSummaryPath, { source: "remote-cache", mode: this.mode });
+                }
                 return {
                     durationMs: Date.now() - start,
                     status: "success",
@@ -143,6 +152,7 @@ export class RunEvalStep {
             configPath: configFile,
             env: subprocessEnv,
             filterFlags: filterFlags.trim() || undefined,
+            maxDurationMs: ctx.config.evalBudgetMs,
         });
         // Check if results were written despite non-zero exit
         if (result.status === "failed") {
@@ -172,6 +182,13 @@ export class RunEvalStep {
             console.log();
             console.log(errorSummary);
         }
+        // Capture eval results
+        const resultsPath = resolve(rootDir, resultsFileForMode(this.mode));
+        if (existsSync(resultsPath)) {
+            ctx.collector.captureFile("run-eval", `eval-results-${this.mode}`, resultsPath, {
+                mode: this.mode,
+            });
+        }
         const durationMs = Date.now() - start;
         return {
             durationMs,
@@ -182,6 +199,9 @@ export class RunEvalStep {
     cacheInputs(ctx) {
         return getStepInputPaths(ctx.config.rootDir, `eval-${this.mode}`);
     }
+    cacheContext(ctx) {
+        return buildCacheContext(ctx.config);
+    }
 }
 // ---------------------------------------------------------------------------
 // Remote cache helpers

package/dist/pipeline/agent-behavior-report.js

@@ -47,6 +47,12 @@ export const CANONICAL_DOC_MAP = {
 // ---------------------------------------------------------------------------
 export function detectFeatureArea(description) {
     const desc = description.toLowerCase();
+    if (desc.includes("portable text"))
+        return "portable-text";
+    if (desc.includes("content lake"))
+        return "content-lake";
+    if (desc.includes("image handling") || desc.includes("image asset"))
+        return "image-handling";
     if (desc.includes("studio"))
         return "studio-setup";
     if (desc.includes("visual") ||

package/dist/pipeline/attribution.d.ts

@@ -13,7 +13,7 @@
  * layered on top for ambiguous cases when higher precision is needed.
  *
  * @see docs/design-docs/scenario-matrix/per-document-attribution.md
- * @see docs/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
  */
 import type { AttributionReport, ComparisonReport } from "./types.js";
 import type { ResolvedMappings } from "./resolve-mappings.js";

package/dist/pipeline/attribution.js

@@ -13,7 +13,7 @@
  * layered on top for ambiguous cases when higher precision is needed.
  *
  * @see docs/design-docs/scenario-matrix/per-document-attribution.md
- * @see docs/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
  */
 // ---------------------------------------------------------------------------
 // Public API

package/dist/pipeline/cache.js

@@ -125,15 +125,18 @@ export function getStepInputPaths(rootDir, step) {
                 }
             }
             // Task files (contain assertions and test definitions).
-            // Exclude generated .expanded*.yaml files — those are already listed
-            // explicitly above per mode.
+            // Task files live in tasks/{mode}/ subdirectories (e.g., tasks/literacy/)
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
-                const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) &&
-                    !f.startsWith(".expanded"))
-                    .map((f) => join(tasksDir, f));
-                paths.push(...taskFiles);
+                for (const entry of readdirSync(tasksDir)) {
+                    const subDir = join(tasksDir, entry);
+                    if (statSync(subDir).isDirectory()) {
+                        const taskFiles = readdirSync(subDir)
+                            .filter((f) => /\.(task\.ts|task\.js)$/.test(f))
+                            .map((f) => join(subDir, f));
+                        paths.push(...taskFiles);
+                    }
+                }
             }
             // Reference solutions (used by grader assertions)
             const refDir = r("canonical/reference-solutions");
@@ -155,12 +158,18 @@ export function getStepInputPaths(rootDir, step) {
             if (modelsPath2)
                 paths.push(modelsPath2);
             // Include all task files (they define feature areas)
+            // Task files live in tasks/{mode}/ subdirectories (e.g., tasks/literacy/)
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
-                const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
-                    .map((f) => join(tasksDir, f));
-                paths.push(...taskFiles);
+                for (const entry of readdirSync(tasksDir)) {
+                    const subDir = join(tasksDir, entry);
+                    if (statSync(subDir).isDirectory()) {
+                        const taskFiles = readdirSync(subDir)
+                            .filter((f) => /\.(task\.ts|task\.js)$/.test(f))
+                            .map((f) => join(subDir, f));
+                        paths.push(...taskFiles);
+                    }
+                }
             }
             return paths;
         }
@@ -175,10 +184,15 @@ export function getStepInputPaths(rootDir, step) {
                 paths.push(sourcesPath2);
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
-                const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
-                    .map((f) => join(tasksDir, f));
-                paths.push(...taskFiles);
+                for (const entry of readdirSync(tasksDir)) {
+                    const subDir = join(tasksDir, entry);
+                    if (statSync(subDir).isDirectory()) {
+                        const taskFiles = readdirSync(subDir)
+                            .filter((f) => /\.(task\.ts|task\.js)$/.test(f))
+                            .map((f) => join(subDir, f));
+                        paths.push(...taskFiles);
+                    }
+                }
             }
             return paths;
         }

package/dist/pipeline/calculate-scores.d.ts

@@ -38,6 +38,8 @@ export interface RawTestResult {
         componentResults: ComponentResult[];
         pass: boolean;
     };
+    /** Per-test latency in ms (populated by Promptfoo when available) */
+    latencyMs?: number;
     metadata?: Record<string, unknown>;
     provider?: {
         id?: string;