@sanity/ailf 2.7.1 → 2.9.0

package/dist/artifact-capture/local-fs-artifact-writer.js

@@ -0,0 +1,273 @@
+/**
+ * LocalFilesystemArtifactWriter — writes AILF run artifacts to the local
+ * filesystem under `{rootDir}/runs/{runId}/…`.
+ *
+ * D0033 M4 inverts D0032's "GCS default, local fallback" stance: the local
+ * writer is **always** attached, and GCS layers on top via a fanout writer
+ * when credentials are present. The result — every run produces a usable
+ * artifact bundle on disk even on airplanes, laptops, and CI without GCS
+ * creds. Studio retrieval works uniformly; only `ArtifactRef.store`
+ * differentiates.
+ *
+ * ## Path layout
+ *
+ * Paths mirror the GCS tree exactly, so the same descriptor's `objectPath`
+ * is used verbatim:
+ *   - bulk: `{rootDir}/runs/{runId}/{slug}.{ext}`
+ *   - per-entry: `{rootDir}/runs/{runId}/{slug}/{sanitizedKey}.{ext}`
+ *   - manifest: `{rootDir}/runs/{runId}/manifest.json`
+ *
+ * This keeps the L6 cross-reader contract test simple — a byte-compare of
+ * every object at every path, modulo timestamped manifest fields.
+ *
+ * ## Design choices
+ *
+ * - **Redaction at emit boundary (AC10).** `redactArtifactData` runs per
+ *   write, not post-hoc on a tarball. Same rules as the legacy collector.
+ * - **Exclude list gating (Q3).** `--capture-exclude=LIST` passes through
+ *   to the constructor; `emit()` returns null for excluded types before
+ *   touching disk.
+ * - **NDJSON uses plain `fs.appendFile`.** No compose rollover needed —
+ *   unlike GCS, local fs appends are atomic and unbounded. A crash mid-
+ *   append leaves a partial row visible; acceptable for dev runs.
+ * - **Non-blocking per P5.** Any fs error returns null + warns, never
+ *   throws. The pipeline must not fail because local disk is full.
+ *
+ * @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md (§ M4)
+ * @see packages/eval/src/artifact-capture/gcs-artifact-writer.ts (mirror)
+ */
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { ARTIFACT_REGISTRY, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
+import { redactArtifactData } from "./redact-artifact.js";
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+export class LocalFilesystemArtifactWriter {
+    options;
+    excludeSet;
+    constructor(options) {
+        this.options = options;
+        this.excludeSet = new Set(options.exclude ?? []);
+    }
+    // ---- Canonical W0049 API ------------------------------------------------
+    async emit(type, association, payload) {
+        if (this.excludeSet.has(type))
+            return null;
+        const descriptor = ARTIFACT_REGISTRY[type];
+        const runId = association.run;
+        if (!runId) {
+            console.warn(`  ⚠️  emit("${type}"): association.run is required, skipping`);
+            return null;
+        }
+        const redacted = redactArtifactData(payload);
+        const body = serializeForMime(redacted, descriptor.mime);
+        const bytes = Buffer.byteLength(body, "utf-8");
+        // Preview is built from the pre-redaction payload so the extract sees
+        // the same shape the producer handed us. The full entry is still redacted
+        // on disk; the preview lives only on the manifest and is bounded by the
+        // descriptor's capBytes.
+        const preview = buildManifestPreview(descriptor, payload);
+        if (descriptor.layout === "bulk") {
+            const relPath = descriptor.objectPath(runId);
+            const absPath = this.resolve(relPath);
+            const wrote = await this.writeAtomic(absPath, body);
+            if (!wrote)
+                return null;
+            return {
+                store: "local",
+                bucket: this.options.rootDir,
+                path: relPath,
+                bytes,
+                entryCount: entryCountOf(redacted),
+                layout: "bulk",
+                ...(preview === undefined ? {} : { preview }),
+            };
+        }
+        // per-entry
+        const entryKey = descriptor.formatEntryKey(association);
+        const relPath = descriptor.objectPath(runId, entryKey);
+        const absPath = this.resolve(relPath);
+        const wrote = await this.writeAtomic(absPath, body);
+        if (!wrote)
+            return null;
+        return {
+            store: "local",
+            bucket: this.options.rootDir,
+            path: `runs/${runId}/${descriptor.slug}`,
+            bytes,
+            entryCount: 1,
+            layout: "per-entry",
+            entries: [
+                {
+                    key: entryKey,
+                    bytes,
+                    association,
+                    ...(preview === undefined ? {} : { preview }),
+                },
+            ],
+        };
+    }
+    async appendNdjson(type, association, rows) {
+        if (this.excludeSet.has(type))
+            return null;
+        const descriptor = ARTIFACT_REGISTRY[type];
+        if (descriptor.mime !== "application/x-ndjson") {
+            console.warn(`  ⚠️  appendNdjson("${type}"): descriptor mime is ${descriptor.mime}, not application/x-ndjson — skipping`);
+            return null;
+        }
+        const runId = association.run;
+        if (!runId) {
+            console.warn(`  ⚠️  appendNdjson("${type}"): association.run is required, skipping`);
+            return null;
+        }
+        if (rows.length === 0)
+            return null;
+        const entryKey = descriptor.formatEntryKey(association);
+        const relPath = descriptor.objectPath(runId, entryKey);
+        const absPath = this.resolve(relPath);
+        const redactedRows = rows.map((r) => redactArtifactData(r));
+        const body = redactedRows.map((r) => JSON.stringify(r)).join("\n") + "\n";
+        const bytes = Buffer.byteLength(body, "utf-8");
+        try {
+            await fs.mkdir(path.dirname(absPath), { recursive: true });
+            await fs.appendFile(absPath, body, "utf-8");
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`  ⚠️  NDJSON append failed (non-blocking): ${absPath} — ${message}`);
+            return null;
+        }
+        // Local fs appendFile keeps no part state, so the returned ref always
+        // reports the *cumulative* size on disk — same semantic as the GCS
+        // writer's `state.totalBytes`, just computed differently.
+        let cumulative = bytes;
+        try {
+            const stat = await fs.stat(absPath);
+            cumulative = stat.size;
+        }
+        catch {
+            // If stat fails we still have the current batch's bytes — acceptable.
+        }
+        return {
+            store: "local",
+            bucket: this.options.rootDir,
+            path: `runs/${runId}/${descriptor.slug}`,
+            bytes: cumulative,
+            entryCount: 1,
+            layout: "per-entry",
+            entries: [{ key: entryKey, bytes: cumulative, association }],
+        };
+    }
+    async writeManifest(runId, manifest) {
+        const relPath = `runs/${runId}/manifest.json`;
+        const absPath = this.resolve(relPath);
+        const body = JSON.stringify(manifest);
+        const wrote = await this.writeAtomic(absPath, body);
+        if (!wrote)
+            return null;
+        return {
+            store: "local",
+            bucket: this.options.rootDir,
+            path: relPath,
+            bytes: Buffer.byteLength(body, "utf-8"),
+            layout: "bulk",
+        };
+    }
+    // ---- Deprecated legacy surface (W0052) ----------------------------------
+    /** @deprecated Use `emit()` instead. */
+    async writeBulk(type, runId, data) {
+        const descriptor = ARTIFACT_REGISTRY[type];
+        if (descriptor.layout !== "bulk") {
+            console.warn(`  ⚠️  writeBulk("${type}"): descriptor layout is "${descriptor.layout}", not "bulk" — skipping`);
+            return null;
+        }
+        return this.emit(type, { run: runId }, data);
+    }
+    /** @deprecated Use `emit()` per entry instead. */
+    async writePerEntry(type, runId, entries) {
+        const descriptor = ARTIFACT_REGISTRY[type];
+        if (descriptor.layout !== "per-entry") {
+            console.warn(`  ⚠️  writePerEntry("${type}"): descriptor layout is "${descriptor.layout}", not "per-entry" — skipping`);
+            return null;
+        }
+        if (!descriptor.parseEntryKey) {
+            console.warn(`  ⚠️  writePerEntry("${type}"): descriptor has no parseEntryKey`);
+            return null;
+        }
+        // Legacy shim: write each entry directly by its raw key, bypassing
+        // formatEntryKey. Preserves byte-equivalence with pre-W0050 paths
+        // for producers still on the legacy call surface.
+        const uploaded = [];
+        let totalBytes = 0;
+        for (const entry of entries) {
+            const parsed = descriptor.parseEntryKey(entry.key);
+            if (!parsed.ok) {
+                console.warn(`  ⚠️  Skipping entry with invalid key "${entry.key}": ${parsed.reason}`);
+                continue;
+            }
+            const redacted = redactArtifactData(entry.data);
+            const body = serializeForMime(redacted, descriptor.mime);
+            const bytes = Buffer.byteLength(body, "utf-8");
+            const relPath = descriptor.objectPath(runId, entry.key);
+            const absPath = this.resolve(relPath);
+            const wrote = await this.writeAtomic(absPath, body);
+            if (!wrote)
+                continue;
+            uploaded.push({ key: entry.key, bytes });
+            totalBytes += bytes;
+        }
+        if (uploaded.length === 0)
+            return null;
+        return {
+            store: "local",
+            bucket: this.options.rootDir,
+            path: `runs/${runId}/${descriptor.slug}`,
+            bytes: totalBytes,
+            entryCount: uploaded.length,
+            layout: "per-entry",
+            entries: uploaded,
+        };
+    }
+    // ---- Internals ----------------------------------------------------------
+    resolve(relPath) {
+        return path.resolve(this.options.rootDir, relPath);
+    }
+    /**
+     * Write the body to `absPath`, creating parent dirs as needed. Returns
+     * false + warns on any fs error (P5 non-blocking).
+     */
+    async writeAtomic(absPath, body) {
+        try {
+            await fs.mkdir(path.dirname(absPath), { recursive: true });
+            await fs.writeFile(absPath, body, "utf-8");
+            return true;
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`  ⚠️  Artifact write failed (non-blocking): ${absPath} — ${message}`);
+            return false;
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers (shared shape with GcsArtifactWriter)
+// ---------------------------------------------------------------------------
+function serializeForMime(payload, mime) {
+    if (mime === "text/markdown" || mime === "application/yaml") {
+        if (typeof payload === "string")
+            return payload;
+        return String(payload ?? "");
+    }
+    return JSON.stringify(payload);
+}
+function entryCountOf(data) {
+    if (typeof data === "object" &&
+        data !== null &&
+        "entries" in data &&
+        typeof data.entries === "object") {
+        return Object.keys(data.entries)
+            .length;
+    }
+    return undefined;
+}

package/dist/commands/explain-handler.js

@@ -727,6 +727,10 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
         captureCompress: raw.captureCompress ?? true,
         captureExtras: raw.captureExtras ?? true,
         captureDir: raw.captureDir,
+        artifacts: raw.artifacts ?? true,
+        artifactsDir: raw.artifactsDir,
+        artifactsDryRun: raw.artifactsDryRun ?? false,
+        captureExclude: raw.captureExclude,
     };
     const resolved = computeResolvedOptions(withDefaults);
     const planOpts = {

package/dist/commands/pipeline-action.d.ts

@@ -67,6 +67,11 @@ export interface ResolvedOptions {
     captureDir?: string;
     captureCompress: boolean;
     captureExtras: boolean;
+    /** D0033 / W0049 — unified artifact surface (W0050 wires it into writer). */
+    artifactsDisabled: boolean;
+    artifactsDir?: string;
+    artifactsDryRun: boolean;
+    artifactsExclude?: readonly string[];
 }
 /**
  * Pure option resolution — computes ResolvedOptions from CLI flags without

package/dist/commands/pipeline-action.js

@@ -263,13 +263,40 @@ export function computeResolvedOptions(opts) {
         tagOption,
         taskSourceType: resolvedTaskSourceType,
         urlArgs,
-        captureEnabled: opts.capture || process.env.AILF_CAPTURE === "1",
-        captureDir: opts.captureDir ?? process.env.AILF_CAPTURE_DIR,
+        captureEnabled: opts.capture,
+        captureDir: resolveArtifactsDir(opts),
         captureCompress: opts.captureCompress !== false &&
             process.env.AILF_CAPTURE_COMPRESS !== "0",
         captureExtras: opts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0",
+        artifactsDisabled: opts.artifacts === false,
+        artifactsDir: resolveArtifactsDir(opts),
+        artifactsDryRun: opts.artifactsDryRun,
+        artifactsExclude: parseCaptureExcludeList(opts.captureExclude),
     };
 }
+/**
+ * Resolve the artifacts / capture output directory from CLI flags and env
+ * vars. Precedence (highest first):
+ *   1. `--artifacts-dir` flag
+ *   2. `--capture-dir` flag (deprecated alias; no warning — silent rewrite)
+ *   3. `AILF_ARTIFACTS_DIR` env var
+ *   4. `AILF_CAPTURE_DIR` env var (deprecated alias; silent)
+ */
+function resolveArtifactsDir(opts) {
+    return (opts.artifactsDir ??
+        opts.captureDir ??
+        process.env.AILF_ARTIFACTS_DIR ??
+        process.env.AILF_CAPTURE_DIR);
+}
+function parseCaptureExcludeList(raw) {
+    if (!raw)
+        return undefined;
+    const list = raw
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+    return list.length > 0 ? list : undefined;
+}
 /** Resolve and validate the --task-source flag value. */
 function resolveTaskSourceType(raw) {
     if (!raw || raw === "content-lake")
@@ -282,6 +309,20 @@ function resolveTaskSourceType(raw) {
 // ---------------------------------------------------------------------------
 // Pipeline entry point
 // ---------------------------------------------------------------------------
+/**
+ * Module-level flag so the `--capture` deprecation warning fires exactly
+ * once per process even when `executePipeline` is invoked multiple times
+ * (e.g. tests, long-lived dev loops).
+ */
+let warnedCaptureDeprecation = false;
+function warnCaptureDeprecationIfNeeded(cliOpts) {
+    if (!cliOpts.capture)
+        return;
+    if (warnedCaptureDeprecation)
+        return;
+    warnedCaptureDeprecation = true;
+    console.warn("--capture is deprecated and will be removed in a future release; use --artifacts-dir or --no-artifacts instead");
+}
 /**
  * Execute the evaluation pipeline.
  *
@@ -291,6 +332,7 @@ function resolveTaskSourceType(raw) {
  * 4. Delegate to the PipelineOrchestrator
  */
 export async function executePipeline(cliOpts) {
+    warnCaptureDeprecationIfNeeded(cliOpts);
     // When --config is provided, resolve config from file instead of CLI flags
     if (cliOpts.config) {
         const { existsSync } = await import("fs");
@@ -317,9 +359,11 @@ export async function executePipeline(cliOpts) {
         config.outputDir = resolveOutputDir(cliOpts.outputDir);
         // Capture options — CLI flags and env vars aren't in the config file,
         // so merge them here (same logic as resolveOptions).
-        config.captureEnabled = cliOpts.capture || process.env.AILF_CAPTURE === "1";
-        if (cliOpts.captureDir ?? process.env.AILF_CAPTURE_DIR) {
-            config.captureDir = cliOpts.captureDir ?? process.env.AILF_CAPTURE_DIR;
+        // AILF_CAPTURE is a no-op post-W0049; only the flag toggles captureEnabled.
+        config.captureEnabled = cliOpts.capture;
+        const resolvedArtifactsDir = resolveArtifactsDir(cliOpts);
+        if (resolvedArtifactsDir) {
+            config.captureDir = resolvedArtifactsDir;
         }
         config.captureCompress =
             cliOpts.captureCompress !== false &&
@@ -328,6 +372,13 @@ export async function executePipeline(cliOpts) {
             cliOpts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0";
         config.captureGcsBucket ??= process.env.AILF_CAPTURE_GCS_BUCKET;
         config.captureGcsPrefix ??= process.env.AILF_CAPTURE_GCS_PREFIX;
+        config.artifactsDisabled ??= cliOpts.artifacts === false;
+        config.artifactsDir ??= resolvedArtifactsDir;
+        config.artifactsDryRun ??= cliOpts.artifactsDryRun;
+        const excludeList = parseCaptureExcludeList(cliOpts.captureExclude);
+        if (excludeList) {
+            config.artifactsExclude = excludeList;
+        }
         config.artifactGcsBucket ??= process.env.AILF_GCS_ARTIFACT_BUCKET;
         config.artifactUpload ??= parseArtifactUploadEnv(process.env.AILF_ARTIFACT_UPLOAD);
         // Create AppContext directly from the merged config so adapters

package/dist/commands/pipeline.d.ts

@@ -68,5 +68,9 @@ export interface PipelineCliOptions {
     captureDir?: string;
     captureCompress: boolean;
     captureExtras: boolean;
+    artifacts: boolean;
+    artifactsDir?: string;
+    artifactsDryRun: boolean;
+    captureExclude?: string;
 }
 export declare function createPipelineCommand(): Command;

package/dist/commands/pipeline.js

@@ -54,10 +54,14 @@ export function createPipelineCommand() {
         .option("--repo-tasks-path <path>", "Path to repo-based task definitions (.ailf/tasks/ directory)")
         .option("--remote", "Submit evaluation to the AILF API instead of running locally", false)
         .option("--api-url <url>", "AILF API base URL (default: https://ailf-api.sanity.build)")
-        .option("--capture", "Enable artifact capture for this run", false)
-        .option("--capture-dir <path>", "Base directory for capture output (default: .ailf/results/captures/)")
+        .option("--capture", "[DEPRECATED] Enable legacy artifact capture. Use --artifacts-dir / --no-artifacts instead.", false)
+        .option("--capture-dir <path>", "[DEPRECATED] Alias for --artifacts-dir.")
         .option("--no-capture-compress", "Disable tar.gz compression of captures")
         .option("--no-capture-extras", "Exclude mode-specific artifacts from captures")
+        .option("--no-artifacts", "Disable all artifact writers (D0033). Overrides --artifacts-dir.")
+        .option("--artifacts-dir <path>", "Root directory for local artifact output (D0033; default: .ailf/results/captures/)")
+        .option("--artifacts-dry-run", "Run artifact writers in dry-run mode — log intended writes, touch no storage", false)
+        .option("--capture-exclude <types>", "Comma-separated artifact types to skip (e.g. traces,graderPrompts)")
         .action(async (opts) => {
         const { executePipeline } = await import("./pipeline-action.js");
         await executePipeline(opts);

package/dist/commands/publish.js

@@ -27,6 +27,7 @@ import { addOutputDirOption } from "./shared/options.js";
 import { getCallerCwd, resolveOutputDir } from "./shared/resolve-output-dir.js";
 import { buildProvenance, } from "../pipeline/provenance.js";
 import { generateReportTitle } from "../pipeline/report-title.js";
+import { buildSlimReportSummary } from "../_vendor/ailf-core/index.js";
 import { generateReportId, } from "../report-store.js";
 import { withRetry } from "../sinks/retry.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -55,7 +56,7 @@ export function createPublishCommand() {
  * the summary metadata and environment. Some fields (contextHash,
  * promptfooUrl) are not available for manual publishes.
  */
-function buildProvenanceFromSummary(summary) {
+function buildProvenanceFromSummary(summary, runId) {
     const areas = summary.scores.map((s) => s.feature);
     const mode = (process.env.EVAL_MODE ?? "literacy");
     const source = {
@@ -76,6 +77,7 @@ function buildProvenanceFromSummary(summary) {
         areas,
         mode,
         rootDir: ROOT,
+        runId,
         source,
     };
 }
@@ -145,7 +147,7 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
     // -----------------------------------------------------------------------
     // 2. Build provenance
     // -----------------------------------------------------------------------
-    const provenanceInput = buildProvenanceFromSummary(summary);
+    const provenanceInput = buildProvenanceFromSummary(summary, ctx.runId);
     const provenance = buildProvenance(provenanceInput);
     // -----------------------------------------------------------------------
     // 3. Create report
@@ -173,13 +175,15 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
     }
     const reportId = generateReportId();
     const title = generateReportTitle({ provenance });
+    // W0051 Slice 3: slim the summary at publish time.
+    const slimSummary = buildSlimReportSummary(summary, provenance.mode);
     const report = {
         comparison: comparison ?? undefined,
         completedAt: now,
         durationMs: 0, // manual publish — no pipeline duration
         id: reportId,
         provenance,
-        summary,
+        summary: slimSummary,
         tag: opts.tag,
         title,
     };

package/dist/composition-root.d.ts

@@ -15,7 +15,7 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import { type AppContext, type ArtifactUploader, type AssertionRegistration, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+import { type AppContext, type ArtifactWriter, type AssertionRegistration, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
 /**
  * Create a fully wired AppContext from resolved configuration.
  *
@@ -24,21 +24,24 @@ import { type AppContext, type ArtifactUploader, type AssertionRegistration, typ
  */
 export declare function createAppContext(config: ResolvedConfig): AppContext;
 /**
- * Selects an ArtifactUploader implementation based on available credentials.
+ * Selects the `ArtifactWriter` wiring per D0033 M4:
  *
- * Selection order:
- *   1. config.artifactUpload === false → always skip (explicit opt-out)
- *   2. GOOGLE_APPLICATION_CREDENTIALS or GCLOUD_PROJECT present → direct GCS
- *   3. apiKey + apiUrl present → gateway-signed PUT URL
- *   4. Neither → skip silently (P5)
+ *   1. `--no-artifacts` (`config.artifactsDisabled === true`, or legacy
+ *      `config.artifactUpload === false`) → `NoOpArtifactWriter`.
+ *   2. Otherwise: always attach `LocalFilesystemArtifactWriter` under
+ *      `--artifacts-dir` (default `.ailf/results/captures`).
+ *   3. When a remote backend is reachable (ADC, GCLOUD_PROJECT, or an
+ *      AILF API key + URL), layer it via `FanoutArtifactWriter([local, gcs])`.
+ *      Local is listed first so a local success + remote failure still
+ *      produces a non-null ref.
  *
- * The bucket defaults to DEFAULT_ARTIFACT_BUCKET when not explicitly set —
- * users only need to override for self-hosted deployments with a different
- * bucket (and matching gateway signing credentials).
+ * Always returns a writer — pipeline code can assume `ctx.artifactWriter`
+ * is present. Producers post-W0050 drop their `if (ctx.artifactWriter)`
+ * guards in Slice 6.
  *
  * Exported for unit-test access; not part of the public package API.
  */
-export declare function createArtifactUploader(config: ResolvedConfig, logger: Logger): ArtifactUploader | undefined;
+export declare function createArtifactWriter(config: ResolvedConfig, logger: Logger): ArtifactWriter;
 /**
  * Generic Promptfoo assertion types available to all evaluation modes.
  *