@sanity/ailf 2.0.0 → 2.0.2

package/dist/orchestration/steps/fetch-docs-step.js

@@ -10,11 +10,13 @@
  * and stores a `releaseAutoScope` entry in PipelineState. Downstream
  * steps (GenerateConfigsStep, RunEvalStep) use this to narrow scope.
  */
-import { mkdirSync, writeFileSync } from "fs";
+import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { join } from "path";
 import { isIdRef, isPathRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
+import { buildCacheContext } from "../cache-context.js";
 import { checkCanonicalContextsExist } from "../../pipeline/checks.js";
+import { loadPipelineTasks } from "../load-pipeline-tasks.js";
 import { loadSource } from "../../sources.js";
 import { configToSourceOverrides } from "../config-to-source-overrides.js";
 export class FetchDocsStep {
@@ -27,8 +29,15 @@ export class FetchDocsStep {
             return { status: "skipped", reason: "--skip-fetch" };
         }
         const start = Date.now();
-        // Precondition: at least one task has canonical doc mappings
-        const allTasks = await ctx.taskSource.loadTasks(buildFilter(ctx));
+        // Load tasks from the filesystem — the same source GenerateConfigsStep
+        // uses. This replaces ctx.taskSource (ContentLakeTaskSource) which may
+        // have no ailf.task documents, causing a mismatch where generated
+        // configs reference context files that were never fetched.
+        const allTasks = await loadPipelineTasks({
+            rootDir: ctx.config.rootDir,
+            mode: ctx.config.mode,
+            repoTasksPath: ctx.config.repoTasksPath,
+        });
         // Bridge: narrow to literacy tasks for canonical doc access
         const literacyTasks = allTasks.filter((t) => t.mode === "literacy");
         const tasksWithDocs = literacyTasks.filter((t) => (t.context?.docs?.length ?? 0) > 0);
@@ -71,6 +80,21 @@ export class FetchDocsStep {
             if (result.metadata) {
                 writeMetadataFiles(ctx.config.rootDir, result.metadata);
             }
+            // Capture metadata files (mode-specific extras)
+            if (ctx.collector.extrasEnabled) {
+                const contextsDir = join(ctx.config.rootDir, "contexts");
+                for (const [type, filename] of [
+                    ["document-manifest", "document-manifest.json"],
+                    ["release-impact", "release-impact.json"],
+                    ["document-overlay", "document-overlay.json"],
+                    ["url-fetch", "url-fetch.json"],
+                ]) {
+                    const filePath = join(contextsDir, filename);
+                    if (existsSync(filePath)) {
+                        ctx.collector.captureFile("fetch-docs", type, filePath);
+                    }
+                }
+            }
         }
         catch (err) {
             return {
@@ -117,19 +141,9 @@ export class FetchDocsStep {
     cacheInputs(ctx) {
         return getStepInputPaths(ctx.config.rootDir, "fetch-docs");
     }
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function buildFilter(ctx) {
-    const { areas, tasks, tags } = ctx.config;
-    if (!areas && !tasks && !tags)
-        return undefined;
-    return {
-        ...(areas ? { areas } : {}),
-        ...(tasks ? { taskIds: tasks } : {}),
-        ...(tags ? { tags } : {}),
-    };
+    cacheContext(ctx) {
+        return buildCacheContext(ctx.config);
+    }
 }
 /**
  * Write metadata files returned by DocFetcher to the contexts/ directory.

package/dist/orchestration/steps/gap-analysis-step.js

@@ -14,7 +14,7 @@
  *
  * This is an optional step — failure doesn't stop the pipeline.
  */
-import { existsSync, readFileSync, writeFileSync } from "fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join, resolve } from "path";
 import { isSlugRef } from "../../_vendor/ailf-core/index.js";
 export class GapAnalysisStep {
@@ -56,7 +56,9 @@ export class GapAnalysisStep {
             console.log(formatFailureModesConsole(failureModeReport));
             const gapReport = buildGapAnalysisReport(failureModeReport, scoreSummary.scores);
             console.log(formatGapAnalysisConsole(gapReport));
-            const outDir = resolve(root, "results", "latest");
+            // Write user-facing artifacts to outputDir (respects --output-dir)
+            const outDir = ctx.config.outputDir;
+            mkdirSync(outDir, { recursive: true });
             writeFileSync(join(outDir, "failure-modes.json"), JSON.stringify(failureModeReport, null, 2));
             writeFileSync(join(outDir, "gap-analysis.json"), JSON.stringify(gapReport, null, 2));
             const manifestPath = resolve(root, "contexts", "document-manifest.json");
@@ -166,6 +168,15 @@ export class GapAnalysisStep {
                 scores: enrichedScores,
             };
             writeFileSync(scoreSummaryPath, JSON.stringify(enrichedSummary, null, 2));
+            // Capture gap analysis artifacts
+            const failureModesPath = join(outDir, "failure-modes.json");
+            if (existsSync(failureModesPath)) {
+                ctx.collector.captureFile("gap-analysis", "failure-modes", failureModesPath);
+            }
+            const gapReportPath = join(outDir, "gap-analysis.json");
+            if (existsSync(gapReportPath)) {
+                ctx.collector.captureFile("gap-analysis", "gap-report", gapReportPath);
+            }
             const gapCount = gapReport.gaps.length;
             const classRate = failureModeReport.classificationRate.toFixed(0);
             return {

package/dist/orchestration/steps/generate-configs-step.d.ts

@@ -24,4 +24,5 @@ export declare class GenerateConfigsStep implements PipelineStep {
     private compileAll;
     private checkLiteracyPostconditions;
     cacheInputs(ctx: AppContext): string[];
+    cacheContext(ctx: AppContext): string[];
 }

package/dist/orchestration/steps/generate-configs-step.js

@@ -8,8 +8,12 @@
  * When the variant is "full", the handler is called twice (baseline + agentic)
  * and three YAML files are written. Other modes produce one YAML file.
  */
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
 import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
+import { modelMatchesLiteracyVariant } from "../../pipeline/compiler/mode-bases/literacy.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
+import { buildCacheContext } from "../cache-context.js";
 import { checkGeneratedConfigsExist } from "../../pipeline/checks.js";
 import { validateModelsYaml } from "../../pipeline/validate.js";
 import { loadSource } from "../../sources.js";
@@ -85,21 +89,14 @@ export class GenerateConfigsStep {
     // ---------------------------------------------------------------------------
     async compileLiteracyVariants(ctx, handler, tasks, models, providers, start) {
         ctx.logger.info(`Compiling ${tasks.length} literacy task(s) via registry handler...`);
-        // Filter models per variant
+        // Filter models per variant using shared literacy variant matcher
         const baselineModels = models.models
-            .filter((m) => !m.modes || m.modes.includes(LiteracyVariant.STANDARD))
-            .map((m) => ({
-            id: m.id,
-            label: m.label,
-        }));
+            .filter((m) => modelMatchesLiteracyVariant(m, "baseline"))
+            .map((m) => ({ id: m.id, label: m.label }));
         const agenticModels = models.models
-            .filter((m) => !m.modes ||
-            m.modes.includes("agentic-naive") ||
-            m.modes.includes("agentic-optimized"))
-            .map((m) => ({
-            id: m.id,
-            label: m.label,
-        }));
+            .filter((m) => modelMatchesLiteracyVariant(m, "agentic-naive") ||
+            modelMatchesLiteracyVariant(m, "agentic-optimized"))
+            .map((m) => ({ id: m.id, label: m.label }));
         // Load rubric config for template resolution
         let rubricConfig;
         try {
@@ -137,6 +134,14 @@ export class GenerateConfigsStep {
             maxConcurrency: models.maxConcurrency,
             logger: ctx.logger,
         });
+        // Capture generated config files (use configFileForMode for legacy naming)
+        const { configFileForMode } = await import("../../pipeline/eval-constants.js");
+        for (const variant of ["baseline", "agentic", "observed"]) {
+            const configPath = resolve(ctx.config.rootDir, configFileForMode(variant));
+            if (existsSync(configPath)) {
+                ctx.collector.captureFile("generate-configs", `promptfoo-config-${variant}`, configPath, { mode: "literacy", variant });
+            }
+        }
         return this.checkLiteracyPostconditions(ctx, start);
     }
     // ---------------------------------------------------------------------------
@@ -168,6 +173,18 @@ export class GenerateConfigsStep {
             maxConcurrency: models.maxConcurrency,
             logger: ctx.logger,
         });
+        // Capture generated config file
+        const configPath = resolve(ctx.config.rootDir, `promptfooconfig.${mode}.yaml`);
+        if (existsSync(configPath)) {
+            ctx.collector.captureFile("generate-configs", "promptfoo-config", configPath, { mode });
+        }
+        // Capture mode-specific test artifacts (extras)
+        if (ctx.collector.extrasEnabled) {
+            const testsPath = resolve(ctx.config.rootDir, "results", "latest", `${mode}-tests.json`);
+            if (existsSync(testsPath)) {
+                ctx.collector.captureFile("generate-configs", `${mode}-tests`, testsPath, { mode });
+            }
+        }
         return {
             durationMs: Date.now() - start,
             status: "success",
@@ -180,8 +197,11 @@ export class GenerateConfigsStep {
     async loadTasks(ctx, mode, state) {
         const { resolve } = await import("path");
         const { discoverTsTaskFiles, loadTsTaskFile } = await import("../../adapters/task-sources/task-file-loader.js");
-        // Discover task files from the mode-specific directory and --repo-tasks-path
-        const tasksDir = resolve(ctx.config.rootDir, "tasks", mode);
+        const { resolveVendoredSubdir } = await import("../../pipeline/compiler/config-loader.js");
+        // Discover task files from the mode-specific directory and --repo-tasks-path.
+        // Use vendored copies in dist/ when @sanity/ailf-core isn't resolvable
+        // (i.e., running outside the monorepo via npx).
+        const tasksDir = resolveVendoredSubdir(ctx.config.rootDir, `tasks/${mode}`);
         const dirs = [tasksDir];
         // Also search --repo-tasks-path (e.g., .ailf/tasks/) for repo-based tasks
         if (ctx.config.repoTasksPath) {
@@ -191,6 +211,7 @@ export class GenerateConfigsStep {
             }
         }
         const tasks = [];
+        const skippedByMode = new Map();
         for (const dir of dirs) {
             const files = discoverTsTaskFiles(dir);
             for (const file of files) {
@@ -201,9 +222,20 @@ export class GenerateConfigsStep {
                     if (!("mode" in task) || task.mode === mode) {
                         tasks.push(task);
                     }
+                    else {
+                        const taskMode = task.mode ?? "unknown";
+                        skippedByMode.set(taskMode, (skippedByMode.get(taskMode) ?? 0) + 1);
+                    }
                 }
             }
         }
+        if (skippedByMode.size > 0) {
+            const total = [...skippedByMode.values()].reduce((a, b) => a + b, 0);
+            const summary = [...skippedByMode.entries()]
+                .map(([m, n]) => `${n} ${m}`)
+                .join(", ");
+            ctx.logger.warn(`  ⚠ Skipped ${total} task(s) with non-matching mode (${summary}). Current pipeline mode: ${mode}. Run with --mode <mode> to include them.`);
+        }
         // Apply area/task/tag filters
         const filtered = this.applyFilters(ctx, tasks);
         // Release auto-scope
@@ -280,6 +312,9 @@ export class GenerateConfigsStep {
     cacheInputs(ctx) {
         return getStepInputPaths(ctx.config.rootDir, "generate-configs");
     }
+    cacheContext(ctx) {
+        return buildCacheContext(ctx.config);
+    }
 }
 // ---------------------------------------------------------------------------
 // Helpers

package/dist/orchestration/steps/mirror-repo-tasks-step.d.ts

@@ -10,7 +10,7 @@
  * exist before evaluation begins.
  *
  * @see packages/eval/src/pipeline/mirror-repo-tasks.ts
- * @see docs/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
  */
 import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class MirrorRepoTasksStep implements PipelineStep {

package/dist/orchestration/steps/mirror-repo-tasks-step.js

@@ -10,7 +10,7 @@
  * exist before evaluation begins.
  *
  * @see packages/eval/src/pipeline/mirror-repo-tasks.ts
- * @see docs/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
  */
 import { getSanityClient } from "../../sanity/client.js";
 import { detectGitContext, mirrorRepoTasks, } from "../../pipeline/mirror-repo-tasks.js";

package/dist/orchestration/steps/publish-report-step.js

@@ -115,12 +115,31 @@ export class PublishReportStep {
         };
         // Share reportId with downstream steps (CallbackStep + orchestrator job update)
         state.reportId = reportId;
+        // Capture report object (Tier 2)
+        ctx.collector.capture("publish-report", "report-object", report);
+        // Capture auto-comparison if present (Tier 2)
+        if (comparison) {
+            ctx.collector.capture("publish-report", "auto-comparison", comparison);
+        }
         // Write to store (system of record — best-effort, P5)
         const sanityResult = ctx.reportStore
             ? await ctx.reportStore.write(report)
             : null;
         // Run sinks (fire-and-forget, P6)
         const publishResult = await runSinks(report, ctx);
+        // Capture sink results (Tier 2)
+        if (publishResult.sinkResults.length > 0) {
+            ctx.collector.capture("publish-report", "sink-results", {
+                sinkCount: publishResult.sinkResults.length,
+                results: publishResult.sinkResults.map((r) => ({
+                    name: r.name,
+                    status: r.result.status,
+                    ...(r.result.status === "success" ? { detail: r.result.detail } : {}),
+                    ...(r.result.status === "failed" ? { error: r.result.error } : {}),
+                    ...(r.result.status === "skipped" ? { reason: r.result.reason } : {}),
+                })),
+            });
+        }
         // Build result summary
         const parts = [];
         if (sanityResult) {

package/dist/orchestration/steps/readiness-step.js

@@ -4,7 +4,7 @@
  * Calls pure functions from pipeline/readiness-report.ts directly.
  * Optional step — failure doesn't stop the pipeline.
  */
-import { existsSync, readFileSync, writeFileSync } from "fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { resolve } from "path";
 import { tryLoadConfigFile } from "../../pipeline/compiler/config-loader.js";
 import { formatReadinessMarkdown, generateReadinessReport, } from "../../pipeline/readiness-report.js";
@@ -37,7 +37,8 @@ export class ReadinessStep {
             }
             const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
             const thresholdConfig = ThresholdConfigSchema.parse(thresholdsLoaded.data);
-            const gapPath = resolve(root, "results", "latest", "gap-analysis.json");
+            // Read gap-analysis.json from outputDir (gap-analysis step writes there)
+            const gapPath = resolve(ctx.config.outputDir, "gap-analysis.json");
             const gapAnalysis = existsSync(gapPath)
                 ? JSON.parse(readFileSync(gapPath, "utf-8"))
                 : undefined;
@@ -60,7 +61,11 @@ export class ReadinessStep {
                 console.log(md);
             }
             if (readinessLines.length > 0) {
-                writeFileSync(resolve(root, "results", "latest", "readiness-report.md"), readinessLines.join("\n---\n\n"));
+                // Write to outputDir (respects --output-dir)
+                mkdirSync(ctx.config.outputDir, { recursive: true });
+                const readinessPath = resolve(ctx.config.outputDir, "readiness-report.md");
+                writeFileSync(readinessPath, readinessLines.join("\n---\n\n"));
+                ctx.collector.captureFile("readiness", "readiness-report", readinessPath);
             }
             const passCount = readinessAreas.filter((area) => {
                 const areaScore = scoreSummary.scores.find((s) => s.feature === area);

package/dist/orchestration/steps/report-step.js

@@ -4,10 +4,10 @@
  * Calls generatePrComment() from pipeline/pr-comment.ts with typed options.
  * No env bridge or process.argv manipulation needed.
  */
-import { resolve } from "path";
+import { existsSync, mkdirSync } from "node:fs";
+import { dirname, resolve } from "path";
 import { checkScoreSummaryValid } from "../../pipeline/checks.js";
 import { generatePrComment } from "../../pipeline/pr-comment.js";
-const DEFAULT_REPORT_PATH = "results/latest/pr-comment.md";
 export class ReportStep {
     name = "report";
     check() {
@@ -15,7 +15,7 @@ export class ReportStep {
     }
     async execute(ctx) {
         const start = Date.now();
-        // Precondition: score summary exists
+        // Precondition: score summary exists (intermediate files stay in rootDir)
         const summaryIssues = checkScoreSummaryValid(ctx.config.rootDir);
         const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
         if (summaryErrors.length > 0) {
@@ -25,7 +25,12 @@ export class ReportStep {
                 status: "failed",
             };
         }
-        const resolvedOutput = ctx.config.outputPath ?? resolve(ctx.config.rootDir, DEFAULT_REPORT_PATH);
+        // User-facing output: --output flag wins, else outputDir
+        const resolvedOutput = ctx.config.outputPath ?? resolve(ctx.config.outputDir, "pr-comment.md");
+        // Ensure outputDir exists before writing (it may be a custom --output-dir
+        // that hasn't been created yet — writePipelineResult runs after the
+        // orchestrator returns, so we can't rely on it).
+        mkdirSync(dirname(resolvedOutput), { recursive: true });
         try {
             generatePrComment({
                 outputPath: resolvedOutput,
@@ -40,6 +45,14 @@ export class ReportStep {
                 status: "failed",
             };
         }
+        // Capture report artifacts
+        if (existsSync(resolvedOutput)) {
+            ctx.collector.captureFile("report", "pr-comment", resolvedOutput);
+        }
+        const pipelineResultPath = resolve(ctx.config.outputDir, "pipeline-result.json");
+        if (existsSync(pipelineResultPath)) {
+            ctx.collector.captureFile("report", "pipeline-result", pipelineResultPath);
+        }
         return {
             durationMs: Date.now() - start,
             status: "success",

package/dist/orchestration/steps/run-eval-step.d.ts

@@ -13,4 +13,5 @@ export declare class RunEvalStep implements PipelineStep {
     check(): ValidationIssue[];
     execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
     cacheInputs(ctx: AppContext): string[];
+    cacheContext(ctx: AppContext): string[];
 }

package/dist/orchestration/steps/run-eval-step.js

@@ -8,6 +8,7 @@
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { resolve } from "path";
 import { getStepInputPaths } from "../../pipeline/cache.js";
+import { buildCacheContext } from "../cache-context.js";
 import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
 import { computeEvalFingerprint } from "../../pipeline/eval-fingerprint.js";
 import { buildFilterFlags, configFileForMode, resultsFileForMode, scanResultsForErrors, } from "../../pipeline/eval-constants.js";
@@ -28,7 +29,7 @@ export class RunEvalStep {
         const start = Date.now();
         const { rootDir, debug, concurrency, noCache } = ctx.config;
         // Precondition: config file exists
-        const configIssues = checkGeneratedConfigsExist(rootDir);
+        const configIssues = checkGeneratedConfigsExist(rootDir, this.mode);
         const configErrors = configIssues.filter((i) => i.severity === "error");
         if (configErrors.length > 0) {
             return {
@@ -38,38 +39,41 @@ export class RunEvalStep {
             };
         }
         // Precondition: canonical context files exist for filtered tasks.
-        // Must apply the same area/task filter as fetch-docs so we only
-        // check contexts that were actually fetched.
-        const filter = ctx.config.areas || ctx.config.tasks || ctx.config.tags
-            ? {
-                ...(ctx.config.areas ? { areas: ctx.config.areas } : {}),
-                ...(ctx.config.tasks ? { taskIds: ctx.config.tasks } : {}),
-                ...(ctx.config.tags ? { tags: ctx.config.tags } : {}),
+        // Only applies to literacy mode — other modes don't use canonical doc contexts.
+        if (this.mode === "literacy") {
+            // Must apply the same area/task filter as fetch-docs so we only
+            // check contexts that were actually fetched.
+            const filter = ctx.config.areas || ctx.config.tasks || ctx.config.tags
+                ? {
+                    ...(ctx.config.areas ? { areas: ctx.config.areas } : {}),
+                    ...(ctx.config.tasks ? { taskIds: ctx.config.tasks } : {}),
+                    ...(ctx.config.tags ? { tags: ctx.config.tags } : {}),
+                }
+                : undefined;
+            let tasks = await ctx.taskSource.loadTasks(filter);
+            // Release auto-scope: narrow to affected tasks (mirrors GenerateConfigsStep)
+            if (state.releaseAutoScope && !ctx.config.noAutoScope) {
+                const scopedIds = new Set(state.releaseAutoScope.affectedTaskIds);
+                tasks = tasks.filter((t) => scopedIds.has(t.id));
+            }
+            // Only check context files for tasks that have canonical docs.
+            // Tasks without canonical docs are skipped by FetchDocsStep (they
+            // have no docs to fetch), so no context file is written for them.
+            // The generated Promptfoo config still includes their "without-docs"
+            // variant (testing model knowledge alone), which doesn't need a
+            // context file.
+            // Bridge: narrow to literacy tasks with docs
+            const tasksWithDocs = tasks.filter((t) => t.mode === "literacy" && (t.context?.docs?.length ?? 0) > 0);
+            const taskIds = tasksWithDocs.map((t) => t.id);
+            const contextIssues = checkCanonicalContextsExist(rootDir, taskIds);
+            const contextErrors = contextIssues.filter((i) => i.severity === "error");
+            if (contextErrors.length > 0) {
+                return {
+                    durationMs: Date.now() - start,
+                    error: `Context files missing. Run fetch-docs first. ${contextErrors.map((e) => e.message).join("; ")}`,
+                    status: "failed",
+                };
             }
-            : undefined;
-        let tasks = await ctx.taskSource.loadTasks(filter);
-        // Release auto-scope: narrow to affected tasks (mirrors GenerateConfigsStep)
-        if (state.releaseAutoScope && !ctx.config.noAutoScope) {
-            const scopedIds = new Set(state.releaseAutoScope.affectedTaskIds);
-            tasks = tasks.filter((t) => scopedIds.has(t.id));
-        }
-        // Only check context files for tasks that have canonical docs.
-        // Tasks without canonical docs are skipped by FetchDocsStep (they
-        // have no docs to fetch), so no context file is written for them.
-        // The generated Promptfoo config still includes their "without-docs"
-        // variant (testing model knowledge alone), which doesn't need a
-        // context file.
-        // Bridge: narrow to literacy tasks with docs
-        const tasksWithDocs = tasks.filter((t) => t.mode === "literacy" && (t.context?.docs?.length ?? 0) > 0);
-        const taskIds = tasksWithDocs.map((t) => t.id);
-        const contextIssues = checkCanonicalContextsExist(rootDir, taskIds);
-        const contextErrors = contextIssues.filter((i) => i.severity === "error");
-        if (contextErrors.length > 0) {
-            return {
-                durationMs: Date.now() - start,
-                error: `Context files missing. Run fetch-docs first. ${contextErrors.map((e) => e.message).join("; ")}`,
-                status: "failed",
-            };
         }
         // -----------------------------------------------------------------
         // Compute eval fingerprint (for remote cache + provenance)
@@ -109,6 +113,11 @@ export class RunEvalStep {
                 // required eval modes were satisfied from the remote cache.
                 state.remoteCacheHits ??= new Set();
                 state.remoteCacheHits.add(this.mode);
+                // Capture the restored score-summary from remote cache
+                const cachedSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
+                if (existsSync(cachedSummaryPath)) {
+                    ctx.collector.captureFile("run-eval", "score-summary-cached", cachedSummaryPath, { source: "remote-cache", mode: this.mode });
+                }
                 return {
                     durationMs: Date.now() - start,
                     status: "success",
@@ -143,6 +152,7 @@ export class RunEvalStep {
             configPath: configFile,
             env: subprocessEnv,
             filterFlags: filterFlags.trim() || undefined,
+            maxDurationMs: ctx.config.evalBudgetMs,
         });
         // Check if results were written despite non-zero exit
         if (result.status === "failed") {
@@ -172,6 +182,13 @@ export class RunEvalStep {
             console.log();
             console.log(errorSummary);
         }
+        // Capture eval results
+        const resultsPath = resolve(rootDir, resultsFileForMode(this.mode));
+        if (existsSync(resultsPath)) {
+            ctx.collector.captureFile("run-eval", `eval-results-${this.mode}`, resultsPath, {
+                mode: this.mode,
+            });
+        }
         const durationMs = Date.now() - start;
         return {
             durationMs,
@@ -182,6 +199,9 @@ export class RunEvalStep {
     cacheInputs(ctx) {
         return getStepInputPaths(ctx.config.rootDir, `eval-${this.mode}`);
     }
+    cacheContext(ctx) {
+        return buildCacheContext(ctx.config);
+    }
 }
 // ---------------------------------------------------------------------------
 // Remote cache helpers

package/dist/pipeline/agent-behavior-report.js

@@ -47,6 +47,12 @@ export const CANONICAL_DOC_MAP = {
 // ---------------------------------------------------------------------------
 export function detectFeatureArea(description) {
     const desc = description.toLowerCase();
+    if (desc.includes("portable text"))
+        return "portable-text";
+    if (desc.includes("content lake"))
+        return "content-lake";
+    if (desc.includes("image handling") || desc.includes("image asset"))
+        return "image-handling";
     if (desc.includes("studio"))
         return "studio-setup";
     if (desc.includes("visual") ||

package/dist/pipeline/attribution.d.ts

@@ -13,7 +13,7 @@
  * layered on top for ambiguous cases when higher precision is needed.
  *
  * @see docs/design-docs/scenario-matrix/per-document-attribution.md
- * @see docs/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
  */
 import type { AttributionReport, ComparisonReport } from "./types.js";
 import type { ResolvedMappings } from "./resolve-mappings.js";

package/dist/pipeline/attribution.js

@@ -13,7 +13,7 @@
  * layered on top for ambiguous cases when higher precision is needed.
  *
  * @see docs/design-docs/scenario-matrix/per-document-attribution.md
- * @see docs/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
  */
 // ---------------------------------------------------------------------------
 // Public API

package/dist/pipeline/cache.js

@@ -125,15 +125,18 @@ export function getStepInputPaths(rootDir, step) {
                 }
             }
             // Task files (contain assertions and test definitions).
-            // Exclude generated .expanded*.yaml files — those are already listed
-            // explicitly above per mode.
+            // Task files live in tasks/{mode}/ subdirectories (e.g., tasks/literacy/)
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
-                const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) &&
-                    !f.startsWith(".expanded"))
-                    .map((f) => join(tasksDir, f));
-                paths.push(...taskFiles);
+                for (const entry of readdirSync(tasksDir)) {
+                    const subDir = join(tasksDir, entry);
+                    if (statSync(subDir).isDirectory()) {
+                        const taskFiles = readdirSync(subDir)
+                            .filter((f) => /\.(task\.ts|task\.js)$/.test(f))
+                            .map((f) => join(subDir, f));
+                        paths.push(...taskFiles);
+                    }
+                }
             }
             // Reference solutions (used by grader assertions)
             const refDir = r("canonical/reference-solutions");
@@ -155,12 +158,18 @@ export function getStepInputPaths(rootDir, step) {
             if (modelsPath2)
                 paths.push(modelsPath2);
             // Include all task files (they define feature areas)
+            // Task files live in tasks/{mode}/ subdirectories (e.g., tasks/literacy/)
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
-                const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
-                    .map((f) => join(tasksDir, f));
-                paths.push(...taskFiles);
+                for (const entry of readdirSync(tasksDir)) {
+                    const subDir = join(tasksDir, entry);
+                    if (statSync(subDir).isDirectory()) {
+                        const taskFiles = readdirSync(subDir)
+                            .filter((f) => /\.(task\.ts|task\.js)$/.test(f))
+                            .map((f) => join(subDir, f));
+                        paths.push(...taskFiles);
+                    }
+                }
             }
             return paths;
         }
@@ -175,10 +184,15 @@ export function getStepInputPaths(rootDir, step) {
                 paths.push(sourcesPath2);
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
-                const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
-                    .map((f) => join(tasksDir, f));
-                paths.push(...taskFiles);
+                for (const entry of readdirSync(tasksDir)) {
+                    const subDir = join(tasksDir, entry);
+                    if (statSync(subDir).isDirectory()) {
+                        const taskFiles = readdirSync(subDir)
+                            .filter((f) => /\.(task\.ts|task\.js)$/.test(f))
+                            .map((f) => join(subDir, f));
+                        paths.push(...taskFiles);
+                    }
+                }
             }
             return paths;
         }

package/dist/pipeline/calculate-scores.d.ts

@@ -38,6 +38,8 @@ export interface RawTestResult {
         componentResults: ComponentResult[];
         pass: boolean;
     };
+    /** Per-test latency in ms (populated by Promptfoo when available) */
+    latencyMs?: number;
     metadata?: Record<string, unknown>;
     provider?: {
         id?: string;