@sanity/ailf 2.8.0 → 2.9.0

package/dist/orchestration/steps/fetch-docs-step.js

@@ -12,7 +12,8 @@
  */
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { join } from "path";
-import { isIdRef, isPathRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
+import { assoc, isIdRef, isPathRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
+import { emitFileContents } from "../../artifact-capture/emit-file.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { buildCacheContext } from "../cache-context.js";
 import { checkCanonicalContextsExist } from "../../pipeline/checks.js";
@@ -94,20 +95,13 @@ export class FetchDocsStep {
             if (result.metadata) {
                 writeMetadataFiles(ctx.config.rootDir, result.metadata);
             }
-            // Capture metadata files (mode-specific extras)
-            if (ctx.collector.extrasEnabled) {
-                const contextsDir = join(ctx.config.rootDir, "contexts");
-                for (const [type, filename] of [
-                    ["document-manifest", "document-manifest.json"],
-                    ["release-impact", "release-impact.json"],
-                    ["document-overlay", "document-overlay.json"],
-                    ["url-fetch", "url-fetch.json"],
-                ]) {
-                    const filePath = join(contextsDir, filename);
-                    if (existsSync(filePath)) {
-                        ctx.collector.captureFile("fetch-docs", type, filePath);
-                    }
-                }
+            // W0050 — documentManifest is run-scoped bulk JSON. The
+            // release-impact/document-overlay/url-fetch captures had no
+            // registered descriptors (they were extras-only); dropped per Q3
+            // ("producers always call emit; registered types only").
+            const documentManifestPath = join(ctx.config.rootDir, "contexts", "document-manifest.json");
+            if (existsSync(documentManifestPath)) {
+                await emitFileContents(ctx.artifactWriter, "documentManifest", assoc(ctx), documentManifestPath);
             }
         }
         catch (err) {

package/dist/orchestration/steps/finalize-run-step.js

@@ -17,6 +17,7 @@
  */
 import { existsSync, readFileSync } from "node:fs";
 import { resolve } from "node:path";
+import { AccumulatingArtifactWriter } from "../../artifact-capture/accumulating-artifact-writer.js";
 import { buildRunContext } from "../../pipeline/run-context.js";
 import { loadSource } from "../../sources.js";
 import { configToSourceOverrides } from "../config-to-source-overrides.js";
@@ -34,12 +35,11 @@ export class FinalizeRunStep {
     }
     async execute(ctx, state) {
         const start = Date.now();
-        if (!ctx.artifactWriter) {
-            return {
-                status: "skipped",
-                reason: "No artifactWriter wired — manifest is only written when a writer is available",
-            };
-        }
+        // W0050 — `ctx.artifactWriter` is now required on AppContext
+        // (composition root always provides one; NoOpArtifactWriter when
+        // `--no-artifacts`). The pre-W0050 guard that returned "skipped" has
+        // been removed — a NoOp writer's writeManifest returns null and the
+        // code below already handles that as a non-blocking failure.
         // Resolve the source (same input buildProvenance uses).
         const overrides = configToSourceOverrides(ctx.config);
         const resolvedSource = loadSource(ctx.config.source, overrides);
@@ -57,6 +57,20 @@ export class FinalizeRunStep {
             source: resolvedSource,
             taskIds: ctx.config.tasks,
         });
+        // W0051 revisit: the composition-root wraps `ctx.artifactWriter` in
+        // `AccumulatingArtifactWriter`, which keeps a map of every ref any
+        // producer emitted this run. Merge that into `state.artifactRefs` so
+        // the manifest reflects the FULL set — not just the subset producers
+        // happened to register manually. When the writer is a NoOp / plain
+        // decorator without accumulation, `aggregated` stays empty and the
+        // manifest falls back to the producer-side registration.
+        const aggregated = ctx.artifactWriter instanceof AccumulatingArtifactWriter
+            ? ctx.artifactWriter.getAccumulatedArtifactRefs()
+            : {};
+        const artifacts = {
+            ...aggregated,
+            ...(state.artifactRefs ?? {}),
+        };
         const manifest = {
             version: 1,
             runId: ctx.runId,
@@ -68,7 +82,7 @@ export class FinalizeRunStep {
                 ? { testSummary: state.testSummary }
                 : undefined,
             promptfooUrls: state.promptfooUrls,
-            artifacts: state.artifactRefs ?? {},
+            artifacts,
         };
         const ref = await ctx.artifactWriter.writeManifest(ctx.runId, manifest);
         if (!ref) {

package/dist/orchestration/steps/gap-analysis-step.js

@@ -16,7 +16,8 @@
  */
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join, resolve } from "path";
-import { isSlugRef } from "../../_vendor/ailf-core/index.js";
+import { assoc, isSlugRef } from "../../_vendor/ailf-core/index.js";
+import { emitFileContents } from "../../artifact-capture/emit-file.js";
 export class GapAnalysisStep {
     name = "gap-analysis";
     optional = true;
@@ -194,14 +195,29 @@ export class GapAnalysisStep {
                 ...(testResults !== undefined && { testResults }),
             };
             writeFileSync(scoreSummaryPath, JSON.stringify(enrichedSummary, null, 2));
-            // Capture gap analysis artifacts
-            const failureModesPath = join(outDir, "failure-modes.json");
-            if (existsSync(failureModesPath)) {
-                ctx.collector.captureFile("gap-analysis", "failure-modes", failureModesPath);
+            // W0051 Slice 2 — failureModes is per-entry keyed by {mode, category};
+            // one entry per classified FailureModeType. Zero-count categories are
+            // skipped to keep the manifest honest about what the run surfaced.
+            const classifiedByCategory = new Map();
+            for (const cj of failureModeReport.classifiedJudgments) {
+                const cat = cj.classification.mode;
+                const bucket = classifiedByCategory.get(cat) ?? [];
+                bucket.push(cj);
+                classifiedByCategory.set(cat, bucket);
+            }
+            for (const [category, classified] of classifiedByCategory) {
+                if (classified.length === 0)
+                    continue;
+                await ctx.artifactWriter.emit("failureModes", assoc(ctx, { mode: ctx.config.mode, category }), {
+                    category,
+                    count: classified.length,
+                    title: toTitleCase(category),
+                    judgments: classified.map((c) => c.judgment),
+                });
             }
             const gapReportPath = join(outDir, "gap-analysis.json");
             if (existsSync(gapReportPath)) {
-                ctx.collector.captureFile("gap-analysis", "gap-report", gapReportPath);
+                await emitFileContents(ctx.artifactWriter, "gapReport", assoc(ctx), gapReportPath);
             }
             const gapCount = gapReport.gaps.length;
             const classRate = failureModeReport.classificationRate.toFixed(0);
@@ -223,6 +239,18 @@ export class GapAnalysisStep {
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
+/**
+ * Render a kebab-case FailureModeType id as Title Case for the manifest
+ * entry's display title (e.g. `"missing-docs"` → `"Missing Docs"`). Kept
+ * local to the producer so the registry descriptor stays decoupled from
+ * eval-side types.
+ */
+function toTitleCase(id) {
+    return id
+        .split("-")
+        .map((w) => (w.length === 0 ? w : w[0].toUpperCase() + w.slice(1)))
+        .join(" ");
+}
 /**
  * Extract slug strings from polymorphic canonical doc refs.
  *

package/dist/orchestration/steps/generate-configs-step.d.ts

@@ -8,7 +8,7 @@
  * When the variant is "full", the handler is called twice (baseline + agentic)
  * and three YAML files are written. Other modes produce one YAML file.
  */
-import type { AppContext, PipelineState, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+import { type AppContext, type PipelineState, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class GenerateConfigsStep implements PipelineStep {
     readonly name = "generate-configs";
     /** Task IDs from the last loadTasks call (pre-filter), for error messages. */

package/dist/orchestration/steps/generate-configs-step.js

@@ -10,6 +10,8 @@
  */
 import { existsSync } from "node:fs";
 import { resolve } from "node:path";
+import { assoc, } from "../../_vendor/ailf-core/index.js";
+import { emitFileContents } from "../../artifact-capture/emit-file.js";
 import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { modelMatchesLiteracyVariant } from "../../pipeline/compiler/mode-bases/literacy.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
@@ -136,12 +138,14 @@ export class GenerateConfigsStep {
             maxConcurrency: models.maxConcurrency,
             logger: ctx.logger,
         });
-        // Capture generated config files (use configFileForMode for legacy naming)
+        // W0050 — configSnapshot is per-entry keyed by mode. For literacy,
+        // each variant produces a distinct config, so the variant name is the
+        // mode-axis value here.
         const { configFileForMode } = await import("../../pipeline/eval-constants.js");
         for (const variant of ["baseline", "agentic", "observed"]) {
             const configPath = resolve(ctx.config.rootDir, configFileForMode(variant));
             if (existsSync(configPath)) {
-                ctx.collector.captureFile("generate-configs", `promptfoo-config-${variant}`, configPath, { mode: "literacy", variant });
+                await emitFileContents(ctx.artifactWriter, "configSnapshot", assoc(ctx, { mode: `literacy-${variant}` }), configPath);
             }
         }
         return this.checkLiteracyPostconditions(ctx, start);
@@ -187,18 +191,14 @@ export class GenerateConfigsStep {
             maxConcurrency: models.maxConcurrency,
             logger: ctx.logger,
         });
-        // Capture generated config file
+        // W0050 — configSnapshot for a single-mode compile.
         const configPath = resolve(ctx.config.rootDir, `promptfooconfig.${mode}.yaml`);
         if (existsSync(configPath)) {
-            ctx.collector.captureFile("generate-configs", "promptfoo-config", configPath, { mode });
-        }
-        // Capture mode-specific test artifacts (extras)
-        if (ctx.collector.extrasEnabled) {
-            const testsPath = resolve(ctx.config.rootDir, "results", "latest", `${mode}-tests.json`);
-            if (existsSync(testsPath)) {
-                ctx.collector.captureFile("generate-configs", `${mode}-tests`, testsPath, { mode });
-            }
+            await emitFileContents(ctx.artifactWriter, "configSnapshot", assoc(ctx, { mode }), configPath);
         }
+        // W0050 — the mode-specific `${mode}-tests.json` file was an
+        // extras-only capture with no registered descriptor. Dropped; the
+        // same information lives in the configSnapshot + rawResults chain.
         return {
             durationMs: Date.now() - start,
             status: "success",

package/dist/orchestration/steps/publish-report-step.d.ts

@@ -10,7 +10,7 @@
  * - P5: Local-first (pipeline never fails because of a store write)
  * - P6: Sinks are fire-and-forget (failures logged, not thrown)
  */
-import type { AppContext, PipelineState, PipelineStep, PromptfooUrlEntry, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+import { type AppContext, type PipelineState, type PipelineStep, type PromptfooUrlEntry, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class PublishReportStep implements PipelineStep {
     private readonly pipelineStart;
     private readonly options;

package/dist/orchestration/steps/publish-report-step.js

@@ -12,6 +12,7 @@
  */
 import { readFileSync } from "fs";
 import { resolve } from "path";
+import { assoc, buildSlimReportSummary, } from "../../_vendor/ailf-core/index.js";
 import { checkScoreSummaryValid } from "../../pipeline/checks.js";
 import { buildProvenance, } from "../../pipeline/provenance.js";
 import { generateReportTitle } from "../../pipeline/report-title.js";
@@ -103,13 +104,19 @@ export class PublishReportStep {
             };
         }
         const title = generateReportTitle({ provenance });
+        // W0051 Slice 3: transform the full pipeline-internal ScoreSummary into
+        // the slim ReportSummary that lives on the Content Lake document.
+        // Prose fields (grader reasons, failureModes full text, gap prose,
+        // agentBehavior arrays) point at their external artifacts via
+        // `id = manifestEntryKey`; Studio hydrates on drill-down.
+        const slimSummary = buildSlimReportSummary(summary, ctx.config.mode);
         const report = {
             comparison: comparison ?? undefined,
             completedAt: now,
             durationMs,
             id: reportId,
             provenance,
-            summary,
+            summary: slimSummary,
             tag: this.options.publishTag ?? ctx.config.publishTag,
             title,
         };
@@ -126,19 +133,20 @@ export class PublishReportStep {
         // full output lives in the GCS artifact. When no testOutputs artifact
         // exists, leave the inline shape intact so Studio's drill-down UI
         // falls back to it.
-        if (artifactManifest?.testOutputs && summary.testResults?.length) {
+        if (artifactManifest?.testOutputs && slimSummary.testResults?.length) {
             report.summary = {
-                ...summary,
-                testResults: summary.testResults.map(slimTestResult),
+                ...slimSummary,
+                testResults: slimSummary.testResults.map(slimTestResult),
             };
         }
         // Share reportId with downstream steps (CallbackStep + orchestrator job update)
         state.reportId = reportId;
-        // Capture report object (Tier 2)
-        ctx.collector.capture("publish-report", "report-object", report);
-        // Capture auto-comparison if present (Tier 2)
+        // W0050 — migrated from ctx.collector.capture to the unified writer.
+        // reportSnapshot: full Report JSON for replay (run-scoped, bulk).
+        await ctx.artifactWriter.emit("reportSnapshot", assoc(ctx), report);
+        // autoComparison: delta vs baseline (run-scoped, bulk, optional).
         if (comparison) {
-            ctx.collector.capture("publish-report", "auto-comparison", comparison);
+            await ctx.artifactWriter.emit("autoComparison", assoc(ctx), comparison);
         }
         // Write to store (system of record — best-effort, P5)
         const sanityResult = ctx.reportStore
@@ -146,17 +154,14 @@ export class PublishReportStep {
             : null;
         // Run sinks (fire-and-forget, P6)
         const publishResult = await runSinks(report, ctx);
-        // Capture sink results (Tier 2)
-        if (publishResult.sinkResults.length > 0) {
-            ctx.collector.capture("publish-report", "sink-results", {
-                sinkCount: publishResult.sinkResults.length,
-                results: publishResult.sinkResults.map((r) => ({
-                    name: r.name,
-                    status: r.result.status,
-                    ...(r.result.status === "success" ? { detail: r.result.detail } : {}),
-                    ...(r.result.status === "failed" ? { error: r.result.error } : {}),
-                    ...(r.result.status === "skipped" ? { reason: r.result.reason } : {}),
-                })),
+        // sinkResults: per-sink outcome (run-scoped, per-entry keyed by sink name).
+        for (const r of publishResult.sinkResults) {
+            await ctx.artifactWriter.emit("sinkResults", assoc(ctx, { name: r.name }), {
+                name: r.name,
+                status: r.result.status,
+                ...(r.result.status === "success" ? { detail: r.result.detail } : {}),
+                ...(r.result.status === "failed" ? { error: r.result.error } : {}),
+                ...(r.result.status === "skipped" ? { reason: r.result.reason } : {}),
             });
         }
         // Build result summary

package/dist/orchestration/steps/readiness-step.d.ts

@@ -4,7 +4,7 @@
  * Calls pure functions from pipeline/readiness-report.ts directly.
  * Optional step — failure doesn't stop the pipeline.
  */
-import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class ReadinessStep implements PipelineStep {
     readonly name = "readiness";
     readonly optional = true;

package/dist/orchestration/steps/readiness-step.js

@@ -7,6 +7,8 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { resolve } from "path";
 import { tryLoadConfigFile } from "../../pipeline/compiler/config-loader.js";
+import { assoc, } from "../../_vendor/ailf-core/index.js";
+import { emitFileContents } from "../../artifact-capture/emit-file.js";
 import { formatReadinessMarkdown, generateReadinessReport, } from "../../pipeline/readiness-report.js";
 import { ThresholdConfigSchema } from "../../pipeline/schemas.js";
 export class ReadinessStep {
@@ -65,7 +67,8 @@ export class ReadinessStep {
                 mkdirSync(ctx.config.outputDir, { recursive: true });
                 const readinessPath = resolve(ctx.config.outputDir, "readiness-report.md");
                 writeFileSync(readinessPath, readinessLines.join("\n---\n\n"));
-                ctx.collector.captureFile("readiness", "readiness-report", readinessPath);
+                // W0050 — readinessReport is run-scoped bulk markdown.
+                await emitFileContents(ctx.artifactWriter, "readinessReport", assoc(ctx), readinessPath);
             }
             const passCount = readinessAreas.filter((area) => {
                 const areaScore = scoreSummary.scores.find((s) => s.feature === area);

package/dist/orchestration/steps/report-step.d.ts

@@ -4,7 +4,7 @@
  * Calls generatePrComment() from pipeline/pr-comment.ts with typed options.
  * No env bridge or process.argv manipulation needed.
  */
-import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class ReportStep implements PipelineStep {
     readonly name = "report";
     check(): ValidationIssue[];

package/dist/orchestration/steps/report-step.js

@@ -6,6 +6,8 @@
  */
 import { existsSync, mkdirSync } from "node:fs";
 import { dirname, resolve } from "path";
+import { assoc, } from "../../_vendor/ailf-core/index.js";
+import { emitFileContents } from "../../artifact-capture/emit-file.js";
 import { checkScoreSummaryValid } from "../../pipeline/checks.js";
 import { generatePrComment } from "../../pipeline/pr-comment.js";
 export class ReportStep {
@@ -45,13 +47,14 @@ export class ReportStep {
                 status: "failed",
             };
         }
-        // Capture report artifacts
+        // W0050 — captureFile → emitFileContents. Both are run-scoped bulk
+        // artifacts; the writer handles redaction + excluded-types gating.
         if (existsSync(resolvedOutput)) {
-            ctx.collector.captureFile("report", "pr-comment", resolvedOutput);
+            await emitFileContents(ctx.artifactWriter, "prComment", assoc(ctx), resolvedOutput);
         }
         const pipelineResultPath = resolve(ctx.config.outputDir, "pipeline-result.json");
         if (existsSync(pipelineResultPath)) {
-            ctx.collector.captureFile("report", "pipeline-result", pipelineResultPath);
+            await emitFileContents(ctx.artifactWriter, "pipelineResult", assoc(ctx), pipelineResultPath);
         }
         return {
             durationMs: Date.now() - start,

package/dist/orchestration/steps/run-eval-step.js

@@ -7,6 +7,7 @@
  */
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { resolve } from "path";
+import { emitPerEntryEvalResults } from "../../pipeline/emit-eval-results.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { buildCacheContext } from "../cache-context.js";
 import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
@@ -118,11 +119,11 @@ export class RunEvalStep {
                     state.promptfooUrls ??= [];
                     state.promptfooUrls.push(...remoteCacheResult.promptfooUrls);
                 }
-                // Capture the restored score-summary from remote cache
-                const cachedSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
-                if (existsSync(cachedSummaryPath)) {
-                    ctx.collector.captureFile("run-eval", "score-summary-cached", cachedSummaryPath, { source: "remote-cache", mode: this.mode });
-                }
+                // W0050 — score-summary-cached was an unregistered capture;
+                // scoreSummary is already emitted by calculate-scores-step on the
+                // non-cached path, which also runs when we have a remote cache hit
+                // (populating state.remoteCacheHits → CalculateScoresStep still
+                // invokes for the score-summary emit). Dropped here.
                 return {
                     durationMs: Date.now() - start,
                     status: "success",
@@ -187,12 +188,16 @@ export class RunEvalStep {
             console.log();
             console.log(errorSummary);
         }
-        // Capture eval results
+        // W0050 — decompose the promptfoo aggregate into the per-entry
+        // descriptors the W0049 registry expects: rawResults / renderedPrompts
+        // per (run, mode, task, model); graderPrompts / graderJudgments per
+        // (run, mode, task, model, grader). See pipeline/emit-eval-results.ts.
+        // `testOutputs` still flows through uploadTestOutputs() in
+        // calculate-scores-step. `traces` ships via agent-observer (out of
+        // scope for the promptfoo shape parser — follow-up).
         const resultsPath = resolve(rootDir, resultsFileForMode(this.mode));
         if (existsSync(resultsPath)) {
-            ctx.collector.captureFile("run-eval", `eval-results-${this.mode}`, resultsPath, {
-                mode: this.mode,
-            });
+            await emitPerEntryEvalResults(ctx.artifactWriter, ctx, this.mode, resultsPath);
         }
         // Extract Promptfoo share URL from eval results (Step 3b)
         if (ctx.evalRunner.extractShareUrl) {

package/dist/pipeline/compare.d.ts

@@ -15,7 +15,7 @@
  * @see docs/ideas/evaluation-roadmap.md — BP5: Make comparison a primitive
  * @see docs/ideas/metrics-design.md — Tier 4: Comparison results
  */
-import { type ChangeClass, type CompareOptions, type ComparisonReport, type ScoreSummary } from "./types.js";
+import { type ChangeClass, type ComparableSummary, type CompareOptions, type ComparisonReport } from "./types.js";
 /** Classify a delta as improved, regressed, or unchanged given a threshold */
 export declare function classifyChange(delta: number, threshold: number): ChangeClass;
 /**
@@ -28,4 +28,4 @@ export declare function classifyChange(delta: number, threshold: number): Change
  * @param options  Optional configuration (noise threshold, etc.)
  * @returns A ComparisonReport with deltas, classifications, and breakdowns
  */
-export declare function compare(baseline: ScoreSummary, experiment: ScoreSummary, options?: CompareOptions): ComparisonReport;
+export declare function compare(baseline: ComparableSummary, experiment: ComparableSummary, options?: CompareOptions): ComparisonReport;

package/dist/pipeline/emit-eval-results.d.ts

@@ -0,0 +1,38 @@
+/**
+ * emit-eval-results.ts — decompose the promptfoo results file into the
+ * per-entry descriptors that W0049's registry expects.
+ *
+ * Replaces the Phase-B-stopgap "route the aggregated JSON through the
+ * deprecated `evalResults` bulk descriptor" path. For each test in the
+ * promptfoo output we emit:
+ *
+ *   - `rawResults`      per (run, mode, task, model)           — the full result
+ *   - `renderedPrompts` per (run, mode, task, model)           — prompt the model saw
+ *   - `graderPrompts`   per (run, mode, task, model, grader)   — rubric text
+ *   - `graderJudgments` per (run, mode, task, model, grader)   — {score, reason, pass}
+ *
+ * `testOutputs` is still emitted separately by `calculate-scores-step`
+ * via `uploadTestOutputs()` (carried forward from W0048 for byte-
+ * equivalence with the original rollout).
+ *
+ * `traces` is NOT produced here — agentic trace data flows through the
+ * agent-observer, not through the promptfoo result shape. Traces
+ * emission is out of scope for this helper and lands when the observer
+ * integration migrates (follow-up; not in W0050).
+ *
+ * The "grader" axis value is the rubric dimension string produced by
+ * `classifyRubric` (e.g. "task-completion", "code-correctness"). Non-
+ * LLM-rubric component assertions (javascript, contains, etc.) don't
+ * have a natural grader identifier and are skipped — their outcomes
+ * still live inside the full `rawResults` object.
+ */
+import { type ArtifactWriter, type RunId } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Parse a promptfoo results file and emit the per-entry artifacts.
+ *
+ * Non-blocking: any individual emit failure warns but does not halt.
+ * File read/parse errors are caught and logged; the caller keeps going.
+ */
+export declare function emitPerEntryEvalResults(writer: ArtifactWriter, ctx: {
+    runId: RunId;
+}, mode: string, resultsPath: string): Promise<void>;

package/dist/pipeline/emit-eval-results.js

@@ -0,0 +1,100 @@
+/**
+ * emit-eval-results.ts — decompose the promptfoo results file into the
+ * per-entry descriptors that W0049's registry expects.
+ *
+ * Replaces the Phase-B-stopgap "route the aggregated JSON through the
+ * deprecated `evalResults` bulk descriptor" path. For each test in the
+ * promptfoo output we emit:
+ *
+ *   - `rawResults`      per (run, mode, task, model)           — the full result
+ *   - `renderedPrompts` per (run, mode, task, model)           — prompt the model saw
+ *   - `graderPrompts`   per (run, mode, task, model, grader)   — rubric text
+ *   - `graderJudgments` per (run, mode, task, model, grader)   — {score, reason, pass}
+ *
+ * `testOutputs` is still emitted separately by `calculate-scores-step`
+ * via `uploadTestOutputs()` (carried forward from W0048 for byte-
+ * equivalence with the original rollout).
+ *
+ * `traces` is NOT produced here — agentic trace data flows through the
+ * agent-observer, not through the promptfoo result shape. Traces
+ * emission is out of scope for this helper and lands when the observer
+ * integration migrates (follow-up; not in W0050).
+ *
+ * The "grader" axis value is the rubric dimension string produced by
+ * `classifyRubric` (e.g. "task-completion", "code-correctness"). Non-
+ * LLM-rubric component assertions (javascript, contains, etc.) don't
+ * have a natural grader identifier and are skipped — their outcomes
+ * still live inside the full `rawResults` object.
+ */
+import { readFileSync } from "node:fs";
+import { classifyRubric, parseRubricScore, } from "../_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
+/**
+ * Parse a promptfoo results file and emit the per-entry artifacts.
+ *
+ * Non-blocking: any individual emit failure warns but does not halt.
+ * File read/parse errors are caught and logged; the caller keeps going.
+ */
+export async function emitPerEntryEvalResults(writer, ctx, mode, resultsPath) {
+    let raw;
+    try {
+        raw = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.warn(`  ⚠️  emitPerEntryEvalResults: failed to read ${resultsPath} — ${message}`);
+        return;
+    }
+    // Promptfoo wraps results in either `{ results: { results: [...] } }`
+    // (older shape) or directly as `{ results: [...] }` (some adapters).
+    const wrapper = raw.results && "results" in raw.results
+        ? raw.results
+        : raw;
+    const rows = wrapper?.results ?? [];
+    if (rows.length === 0) {
+        console.warn(`  ⚠️  emitPerEntryEvalResults: ${resultsPath} has no results[]`);
+        return;
+    }
+    for (const result of rows) {
+        const taskId = result.testCase?.description ?? "unknown-task";
+        const modelId = result.provider?.id ?? result.provider?.label ?? "unknown-model";
+        const baseAssoc = {
+            run: ctx.runId,
+            mode,
+            task: taskId,
+            model: modelId,
+        };
+        // rawResults — full raw entry (bounded by descriptor capBytes: 1 MB)
+        await writer.emit("rawResults", baseAssoc, result);
+        // renderedPrompts — what the model saw + which provider it went to
+        if (result.prompt !== undefined) {
+            await writer.emit("renderedPrompts", baseAssoc, {
+                prompt: result.prompt,
+                provider: result.provider,
+            });
+        }
+        // Per-grader decomposition — only LLM-rubric assertions have a
+        // natural grader identity. Code assertions (javascript/contains/…)
+        // show up in rawResults but not as standalone graderJudgments.
+        const components = result.gradingResult?.componentResults ?? [];
+        for (const comp of components) {
+            if (comp.assertion?.type !== "llm-rubric")
+                continue;
+            const dimension = classifyRubric(comp);
+            if (!dimension)
+                continue;
+            const graderAssoc = { ...baseAssoc, grader: dimension };
+            await writer.emit("graderPrompts", graderAssoc, {
+                dimension,
+                assertion: comp.assertion,
+            });
+            await writer.emit("graderJudgments", graderAssoc, {
+                score: parseRubricScore(comp) ?? 0,
+                reason: comp.reason ?? "",
+                pass: comp.pass,
+            });
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.8.0",
+  "version": "2.9.0",
   "private": false,
   "publishConfig": {
     "access": "public"