@sanity/ailf 2.8.0 → 3.0.0

package/dist/orchestration/steps/report-step.d.ts

@@ -4,7 +4,7 @@
  * Calls generatePrComment() from pipeline/pr-comment.ts with typed options.
  * No env bridge or process.argv manipulation needed.
  */
-import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
 export declare class ReportStep implements PipelineStep {
     readonly name = "report";
     check(): ValidationIssue[];

package/dist/orchestration/steps/report-step.js

@@ -6,6 +6,8 @@
  */
 import { existsSync, mkdirSync } from "node:fs";
 import { dirname, resolve } from "path";
+import { assoc, } from "../../_vendor/ailf-core/index.js";
+import { emitFileContents } from "../../artifact-capture/emit-file.js";
 import { checkScoreSummaryValid } from "../../pipeline/checks.js";
 import { generatePrComment } from "../../pipeline/pr-comment.js";
 export class ReportStep {
@@ -45,13 +47,14 @@ export class ReportStep {
                 status: "failed",
             };
         }
-        // Capture report artifacts
+        // W0050 — captureFile → emitFileContents. Both are run-scoped bulk
+        // artifacts; the writer handles redaction + excluded-types gating.
         if (existsSync(resolvedOutput)) {
-            ctx.collector.captureFile("report", "pr-comment", resolvedOutput);
+            await emitFileContents(ctx.artifactWriter, "prComment", assoc(ctx), resolvedOutput);
         }
         const pipelineResultPath = resolve(ctx.config.outputDir, "pipeline-result.json");
         if (existsSync(pipelineResultPath)) {
-            ctx.collector.captureFile("report", "pipeline-result", pipelineResultPath);
+            await emitFileContents(ctx.artifactWriter, "pipelineResult", assoc(ctx), pipelineResultPath);
         }
         return {
             durationMs: Date.now() - start,

package/dist/orchestration/steps/run-eval-step.js

@@ -7,6 +7,7 @@
  */
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { resolve } from "path";
+import { emitPerEntryEvalResults } from "../../pipeline/emit-eval-results.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { buildCacheContext } from "../cache-context.js";
 import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
@@ -118,11 +119,11 @@ export class RunEvalStep {
                     state.promptfooUrls ??= [];
                     state.promptfooUrls.push(...remoteCacheResult.promptfooUrls);
                 }
-                // Capture the restored score-summary from remote cache
-                const cachedSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
-                if (existsSync(cachedSummaryPath)) {
-                    ctx.collector.captureFile("run-eval", "score-summary-cached", cachedSummaryPath, { source: "remote-cache", mode: this.mode });
-                }
+                // W0050 — score-summary-cached was an unregistered capture;
+                // scoreSummary is already emitted by calculate-scores-step on the
+                // non-cached path, which also runs when we have a remote cache hit
+                // (populating state.remoteCacheHits → CalculateScoresStep still
+                // invokes for the score-summary emit). Dropped here.
                 return {
                     durationMs: Date.now() - start,
                     status: "success",
@@ -187,12 +188,16 @@ export class RunEvalStep {
             console.log();
             console.log(errorSummary);
         }
-        // Capture eval results
+        // W0050 — decompose the promptfoo aggregate into the per-entry
+        // descriptors the W0049 registry expects: rawResults / renderedPrompts
+        // per (run, mode, task, model); graderPrompts / graderJudgments per
+        // (run, mode, task, model, grader). See pipeline/emit-eval-results.ts.
+        // `testOutputs` still flows through uploadTestOutputs() in
+        // calculate-scores-step. `traces` ships via agent-observer (out of
+        // scope for the promptfoo shape parser — follow-up).
         const resultsPath = resolve(rootDir, resultsFileForMode(this.mode));
         if (existsSync(resultsPath)) {
-            ctx.collector.captureFile("run-eval", `eval-results-${this.mode}`, resultsPath, {
-                mode: this.mode,
-            });
+            await emitPerEntryEvalResults(ctx.artifactWriter, ctx, this.mode, resultsPath);
         }
         // Extract Promptfoo share URL from eval results (Step 3b)
         if (ctx.evalRunner.extractShareUrl) {

package/dist/pipeline/compare.d.ts

@@ -15,7 +15,7 @@
  * @see docs/ideas/evaluation-roadmap.md — BP5: Make comparison a primitive
  * @see docs/ideas/metrics-design.md — Tier 4: Comparison results
  */
-import { type ChangeClass, type CompareOptions, type ComparisonReport, type ScoreSummary } from "./types.js";
+import { type ChangeClass, type ComparableSummary, type CompareOptions, type ComparisonReport } from "./types.js";
 /** Classify a delta as improved, regressed, or unchanged given a threshold */
 export declare function classifyChange(delta: number, threshold: number): ChangeClass;
 /**
@@ -28,4 +28,4 @@ export declare function classifyChange(delta: number, threshold: number): Change
  * @param options  Optional configuration (noise threshold, etc.)
  * @returns A ComparisonReport with deltas, classifications, and breakdowns
  */
-export declare function compare(baseline: ScoreSummary, experiment: ScoreSummary, options?: CompareOptions): ComparisonReport;
+export declare function compare(baseline: ComparableSummary, experiment: ComparableSummary, options?: CompareOptions): ComparisonReport;

package/dist/pipeline/emit-eval-results.d.ts

@@ -0,0 +1,38 @@
+/**
+ * emit-eval-results.ts — decompose the promptfoo results file into the
+ * per-entry descriptors that W0049's registry expects.
+ *
+ * Replaces the Phase-B-stopgap "route the aggregated JSON through the
+ * deprecated `evalResults` bulk descriptor" path. For each test in the
+ * promptfoo output we emit:
+ *
+ *   - `rawResults`      per (run, mode, task, model)           — the full result
+ *   - `renderedPrompts` per (run, mode, task, model)           — prompt the model saw
+ *   - `graderPrompts`   per (run, mode, task, model, grader)   — rubric text
+ *   - `graderJudgments` per (run, mode, task, model, grader)   — {score, reason, pass}
+ *
+ * `testOutputs` is still emitted separately by `calculate-scores-step`
+ * via `uploadTestOutputs()` (carried forward from W0048 for byte-
+ * equivalence with the original rollout).
+ *
+ * `traces` is NOT produced here — agentic trace data flows through the
+ * agent-observer, not through the promptfoo result shape. Traces
+ * emission is out of scope for this helper and lands when the observer
+ * integration migrates (follow-up; not in W0050).
+ *
+ * The "grader" axis value is the rubric dimension string produced by
+ * `classifyRubric` (e.g. "task-completion", "code-correctness"). Non-
+ * LLM-rubric component assertions (javascript, contains, etc.) don't
+ * have a natural grader identifier and are skipped — their outcomes
+ * still live inside the full `rawResults` object.
+ */
+import { type ArtifactWriter, type RunId } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Parse a promptfoo results file and emit the per-entry artifacts.
+ *
+ * Non-blocking: any individual emit failure warns but does not halt.
+ * File read/parse errors are caught and logged; the caller keeps going.
+ */
+export declare function emitPerEntryEvalResults(writer: ArtifactWriter, ctx: {
+    runId: RunId;
+}, mode: string, resultsPath: string): Promise<void>;

package/dist/pipeline/emit-eval-results.js

@@ -0,0 +1,100 @@
+/**
+ * emit-eval-results.ts — decompose the promptfoo results file into the
+ * per-entry descriptors that W0049's registry expects.
+ *
+ * Replaces the Phase-B-stopgap "route the aggregated JSON through the
+ * deprecated `evalResults` bulk descriptor" path. For each test in the
+ * promptfoo output we emit:
+ *
+ *   - `rawResults`      per (run, mode, task, model)           — the full result
+ *   - `renderedPrompts` per (run, mode, task, model)           — prompt the model saw
+ *   - `graderPrompts`   per (run, mode, task, model, grader)   — rubric text
+ *   - `graderJudgments` per (run, mode, task, model, grader)   — {score, reason, pass}
+ *
+ * `testOutputs` is still emitted separately by `calculate-scores-step`
+ * via `uploadTestOutputs()` (carried forward from W0048 for byte-
+ * equivalence with the original rollout).
+ *
+ * `traces` is NOT produced here — agentic trace data flows through the
+ * agent-observer, not through the promptfoo result shape. Traces
+ * emission is out of scope for this helper and lands when the observer
+ * integration migrates (follow-up; not in W0050).
+ *
+ * The "grader" axis value is the rubric dimension string produced by
+ * `classifyRubric` (e.g. "task-completion", "code-correctness"). Non-
+ * LLM-rubric component assertions (javascript, contains, etc.) don't
+ * have a natural grader identifier and are skipped — their outcomes
+ * still live inside the full `rawResults` object.
+ */
+import { readFileSync } from "node:fs";
+import { classifyRubric, parseRubricScore, } from "../_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
+/**
+ * Parse a promptfoo results file and emit the per-entry artifacts.
+ *
+ * Non-blocking: any individual emit failure warns but does not halt.
+ * File read/parse errors are caught and logged; the caller keeps going.
+ */
+export async function emitPerEntryEvalResults(writer, ctx, mode, resultsPath) {
+    let raw;
+    try {
+        raw = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.warn(`  ⚠️  emitPerEntryEvalResults: failed to read ${resultsPath} — ${message}`);
+        return;
+    }
+    // Promptfoo wraps results in either `{ results: { results: [...] } }`
+    // (older shape) or directly as `{ results: [...] }` (some adapters).
+    const wrapper = raw.results && "results" in raw.results
+        ? raw.results
+        : raw;
+    const rows = wrapper?.results ?? [];
+    if (rows.length === 0) {
+        console.warn(`  ⚠️  emitPerEntryEvalResults: ${resultsPath} has no results[]`);
+        return;
+    }
+    for (const result of rows) {
+        const taskId = result.testCase?.description ?? "unknown-task";
+        const modelId = result.provider?.id ?? result.provider?.label ?? "unknown-model";
+        const baseAssoc = {
+            run: ctx.runId,
+            mode,
+            task: taskId,
+            model: modelId,
+        };
+        // rawResults — full raw entry (bounded by descriptor capBytes: 1 MB)
+        await writer.emit("rawResults", baseAssoc, result);
+        // renderedPrompts — what the model saw + which provider it went to
+        if (result.prompt !== undefined) {
+            await writer.emit("renderedPrompts", baseAssoc, {
+                prompt: result.prompt,
+                provider: result.provider,
+            });
+        }
+        // Per-grader decomposition — only LLM-rubric assertions have a
+        // natural grader identity. Code assertions (javascript/contains/…)
+        // show up in rawResults but not as standalone graderJudgments.
+        const components = result.gradingResult?.componentResults ?? [];
+        for (const comp of components) {
+            if (comp.assertion?.type !== "llm-rubric")
+                continue;
+            const dimension = classifyRubric(comp);
+            if (!dimension)
+                continue;
+            const graderAssoc = { ...baseAssoc, grader: dimension };
+            await writer.emit("graderPrompts", graderAssoc, {
+                dimension,
+                assertion: comp.assertion,
+            });
+            await writer.emit("graderJudgments", graderAssoc, {
+                score: parseRubricScore(comp) ?? 0,
+                reason: comp.reason ?? "",
+                pass: comp.pass,
+            });
+        }
+    }
+}

package/dist/pipeline/map-request-to-config.js

@@ -74,10 +74,6 @@ export function mapRequestToConfig(request, rootDir) {
         callerGit: request.callerGit,
         callback: request.callback,
         jobId: request.jobId,
-        captureEnabled: false,
-        captureDir: undefined,
-        captureCompress: true,
-        captureExtras: true,
         remote: false,
         apiUrl: "https://ailf-api.sanity.build",
         presets: request.presets,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.8.0",
+  "version": "3.0.0",
   "private": false,
   "publishConfig": {
     "access": "public"

package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts

@@ -1,14 +0,0 @@
-/**
- * No-op artifact collector — used when --capture is not set.
- *
- * All methods are constant-time stubs. Zero overhead on the
- * default pipeline path.
- */
-import type { ArtifactCollector, CaptureFlushResult } from "../ports/artifact-collector.js";
-export declare class NoOpArtifactCollector implements ArtifactCollector {
-    readonly enabled = false;
-    readonly extrasEnabled = false;
-    capture(): void;
-    captureFile(): void;
-    flush(): Promise<CaptureFlushResult>;
-}

package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js

@@ -1,25 +0,0 @@
-/**
- * No-op artifact collector — used when --capture is not set.
- *
- * All methods are constant-time stubs. Zero overhead on the
- * default pipeline path.
- */
-const EMPTY_RESULT = Object.freeze({
-    artifactCount: 0,
-    compressed: false,
-    destination: "",
-    totalBytes: 0,
-});
-export class NoOpArtifactCollector {
-    enabled = false;
-    extrasEnabled = false;
-    capture() {
-        // no-op
-    }
-    captureFile() {
-        // no-op
-    }
-    async flush() {
-        return EMPTY_RESULT;
-    }
-}

package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts

@@ -1,94 +0,0 @@
-/**
- * Port: ArtifactCollector — captures pipeline artifacts during execution.
- *
- * Injected into AppContext. When capture is disabled (default), the
- * composition root provides NoOpArtifactCollector. When --capture is
- * set, provides FilesystemArtifactCollector.
- *
- * Design principles:
- * - P1: Zero-cost when off (no-op stub)
- * - P2: Capture, don't intercept (steps call capture() explicitly)
- * - P5: Non-blocking (failures swallowed, never block the pipeline)
- */
-/**
- * The contract for artifact capture during pipeline execution.
- *
- * Steps call capture() for in-memory data and captureFile() for
- * artifacts already on disk. The orchestrator calls flush() once
- * at pipeline end to write everything to the configured destination.
- */
-export interface ArtifactCollector {
-    /**
-     * Record an in-memory artifact produced during pipeline execution.
-     *
-     * Callers need not check `enabled` before calling — the NoOp
-     * implementation is zero-cost, so unconditional calls are safe.
-     *
-     * @param step - Pipeline step name (e.g., "run-eval")
-     * @param type - Artifact type identifier (e.g., "eval-results")
-     * @param data - Content to serialize (JSON or text)
-     * @param meta - Optional metadata (variant, model, etc.)
-     */
-    capture(step: string, type: string, data: unknown, meta?: Record<string, unknown>): void;
-    /**
-     * Record a file reference for an artifact already on disk.
-     * The file is copied into the capture directory on flush().
-     *
-     * @param step - Pipeline step name
-     * @param type - Artifact type identifier
-     * @param filePath - Absolute path to the existing file
-     * @param meta - Optional metadata
-     */
-    captureFile(step: string, type: string, filePath: string, meta?: Record<string, unknown>): void;
-    /**
-     * Flush all captured artifacts to the configured destination.
-     * Called once at pipeline end by the orchestrator.
-     */
-    flush(): Promise<CaptureFlushResult>;
-    /** Whether capture is active */
-    readonly enabled: boolean;
-    /** Whether mode-specific extras are being captured */
-    readonly extrasEnabled: boolean;
-}
-/** Result of flushing captured artifacts to the destination. */
-export interface CaptureFlushResult {
-    /** Total number of artifacts captured */
-    artifactCount: number;
-    /** Output path (directory or .tar.gz) */
-    destination: string;
-    /** Total bytes written (uncompressed) */
-    totalBytes: number;
-    /** Whether output was compressed */
-    compressed: boolean;
-}
-/** A single entry in the capture manifest. */
-export interface CaptureManifestEntry {
-    /** Pipeline step that produced this artifact */
-    step: string;
-    /** Artifact type identifier */
-    type: string;
-    /** Relative path within the capture directory */
-    path: string;
-    /** ISO 8601 timestamp of when capture() was called */
-    capturedAt: string;
-    /** Byte size of the artifact */
-    bytes: number;
-    /** Content format */
-    format: "json" | "markdown" | "text";
-    /** Optional metadata */
-    meta?: Record<string, unknown>;
-}
-/** The manifest.json written to each capture directory. */
-export interface CaptureManifest {
-    version: 1;
-    captureId: string;
-    startedAt: string;
-    completedAt: string;
-    pipeline: {
-        mode: string;
-        variant?: string;
-        source?: string;
-        areas?: string[];
-    };
-    artifacts: CaptureManifestEntry[];
-}

package/dist/_vendor/ailf-core/ports/artifact-collector.js

@@ -1,13 +0,0 @@
-/**
- * Port: ArtifactCollector — captures pipeline artifacts during execution.
- *
- * Injected into AppContext. When capture is disabled (default), the
- * composition root provides NoOpArtifactCollector. When --capture is
- * set, provides FilesystemArtifactCollector.
- *
- * Design principles:
- * - P1: Zero-cost when off (no-op stub)
- * - P2: Capture, don't intercept (steps call capture() explicitly)
- * - P5: Non-blocking (failures swallowed, never block the pipeline)
- */
-export {};

package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts

@@ -1,138 +0,0 @@
-/**
- * Types for cross-run capture comparison.
- *
- * The CaptureComparator reads two capture directories (baseline + experiment)
- * and produces a CaptureDiffReport. Types are defined in core so external
- * tooling can consume diff reports without depending on the eval package.
- *
- * Implementation lives in packages/eval/src/artifact-capture/comparator.ts.
- */
-/** How deeply to compare artifacts. */
-export type ComparisonMode = "strict" | "structural" | "inventory";
-/** Configurable thresholds for comparison. */
-export interface ComparisonOptions {
-    /** Comparison depth: inventory (existence), structural (shape), strict (content) */
-    mode: ComparisonMode;
-    /** Score regression thresholds */
-    scoreThresholds?: {
-        /** Maximum allowed aggregate score delta (percentage points, default 5) */
-        aggregate: number;
-        /** Maximum allowed per-task score drop (points, default 10) */
-        perTask: number;
-    };
-    /** Timing regression thresholds */
-    timingThresholds?: {
-        /** Multiplier — flag steps exceeding this ratio (default 2.0) */
-        multiplier: number;
-        /** Per-step overrides (step name → custom multiplier) */
-        perStep?: Record<string, number>;
-    };
-    /** JSON structural diff depth (default 3) */
-    jsonDiffDepth?: number;
-    /** Additional ephemeral fields to ignore (merged with defaults) */
-    ephemeralFields?: string[];
-}
-/** Inventory diff — which artifacts exist in each capture. */
-export interface InventoryDiff {
-    /** Artifact types in experiment but not in baseline */
-    added: string[];
-    /** Artifact types in baseline but not in experiment */
-    removed: string[];
-    /** Artifact types present in both */
-    common: string[];
-}
-/** A single structural change in a JSON artifact. */
-export interface JsonDiffEntry {
-    /** JSON pointer path (e.g., "config.mode") */
-    path: string;
-    /** Value in baseline (undefined if key is added) */
-    baseline?: unknown;
-    /** Value in experiment (undefined if key is removed) */
-    experiment?: unknown;
-}
-/** Content diff for a single artifact. */
-export interface ArtifactContentDiff {
-    /** Artifact type identifier (step/type) */
-    artifactKey: string;
-    /** Content format */
-    format: "json" | "markdown" | "text";
-    /** Structural changes (JSON) or line diff summary (text/markdown) */
-    changes: JsonDiffEntry[] | {
-        addedLines: number;
-        removedLines: number;
-    };
-}
-/** Score comparison between two captures. */
-export interface ScoreComparison {
-    /** Baseline aggregate score */
-    baselineMean: number;
-    /** Experiment aggregate score */
-    currentMean: number;
-    /** Absolute delta (current - baseline) */
-    delta: number;
-    /** Per-task score deltas */
-    perTask: {
-        task: string;
-        baseline: number;
-        current: number;
-        delta: number;
-    }[];
-    /** Tasks that breached configured thresholds */
-    breaches: string[];
-}
-/** Timing comparison between two captures. */
-export interface TimingComparison {
-    /** Total pipeline duration delta in ms */
-    totalDeltaMs: number;
-    /** Per-step timing */
-    perStep: {
-        step: string;
-        baselineMs: number;
-        currentMs: number;
-        ratio: number;
-    }[];
-    /** Steps that breached the timing multiplier threshold */
-    breaches: string[];
-}
-/** Metadata comparison between two captures. */
-export interface MetadataComparison {
-    /** Whether pipeline modes match */
-    modeMatch: boolean;
-    /** Whether pipeline variants match */
-    variantMatch: boolean;
-    /** Config key differences */
-    configDiffs: JsonDiffEntry[];
-}
-/** Security scan results. */
-export interface SecurityScan {
-    /** Whether any potential secret leaks were found */
-    leaksFound: boolean;
-    /** Details of each violation */
-    violations: {
-        /** Relative artifact file path */
-        file: string;
-        /** Description of the finding */
-        detail: string;
-    }[];
-}
-/** The full diff report produced by CaptureComparator. */
-export interface CaptureDiffReport {
-    /** Are the two captures semantically equivalent? */
-    equivalent: boolean;
-    /** Human-readable summary (1-3 sentences) */
-    summary: string;
-    /** Comparison mode used */
-    mode: ComparisonMode;
-    /** Artifact inventory diff */
-    inventory: InventoryDiff;
-    /** Content diffs for common artifacts (structural/strict modes only) */
-    content?: ArtifactContentDiff[];
-    /** Score comparison (if score-summary exists in both captures) */
-    scores?: ScoreComparison;
-    /** Timing comparison (if pipeline-context exists in both captures) */
-    timing?: TimingComparison;
-    /** Metadata comparison */
-    metadata?: MetadataComparison;
-    /** Security scan results */
-    security: SecurityScan;
-}

package/dist/_vendor/ailf-core/ports/capture-comparator.js

@@ -1,10 +0,0 @@
-/**
- * Types for cross-run capture comparison.
- *
- * The CaptureComparator reads two capture directories (baseline + experiment)
- * and produces a CaptureDiffReport. Types are defined in core so external
- * tooling can consume diff reports without depending on the eval package.
- *
- * Implementation lives in packages/eval/src/artifact-capture/comparator.ts.
- */
-export {};

package/dist/artifact-capture/comparator.d.ts

@@ -1,22 +0,0 @@
-/**
- * CaptureComparator — compares two capture directories and produces a diff report.
- *
- * Reads manifest.json from both directories and computes:
- * - Inventory diff (added/removed/common artifacts)
- * - Content diff (structural or strict, for common artifacts)
- * - Score comparison (from score-summary.json)
- * - Timing comparison (from pipeline-context.json)
- * - Metadata comparison (mode, variant, config keys)
- * - Security scan (regex for leaked secrets)
- *
- * Implementation for the types defined in @sanity/ailf-core.
- */
-import type { CaptureDiffReport, ComparisonOptions } from "../_vendor/ailf-core/index.d.ts";
-/**
- * Compare two capture directories and produce a structured diff report.
- *
- * @param baselineDir - Path to the baseline capture directory (contains manifest.json)
- * @param experimentDir - Path to the experiment capture directory
- * @param opts - Comparison options (mode, thresholds, etc.)
- */
-export declare function compareCaptures(baselineDir: string, experimentDir: string, opts?: Partial<ComparisonOptions>): CaptureDiffReport;