@sanity/ailf 2.4.0 → 2.6.0

package/dist/_vendor/ailf-core/ports/context.d.ts

@@ -164,8 +164,25 @@ export interface ResolvedConfig {
     captureGcsBucket?: string;
     /** GCS object prefix for capture uploads (default: "captures/") */
     captureGcsPrefix?: string;
-    /** GCS bucket for report artifact uploads — enables ArtifactUploader (D0030) */
+    /**
+     * GCS bucket for report artifact uploads. Defaults to "ailf-artifacts"
+     * at the composition root — only set this to override (e.g., self-hosted
+     * deployment with a different bucket). Read access is governed by the
+     * gateway's signing credentials, so alternate bucket names require
+     * reconfiguring the gateway as well (D0030).
+     */
     artifactGcsBucket?: string;
+    /**
+     * Controls whether the ArtifactUploader is constructed.
+     *
+     * - `undefined` (default): auto — construct when credentials are available
+     *   (ADC for direct GCS, or AILF_API_KEY for gateway-signed URLs).
+     * - `true`: force-enable — still a no-op if no credentials are present (P5).
+     * - `false`: force-disable — skip artifact upload even when credentials exist.
+     *
+     * Sourced from AILF_ARTIFACT_UPLOAD env var or `artifactUpload` in ailf.config.ts.
+     */
+    artifactUpload?: boolean;
 }
 /**
  * Application context — the complete dependency carrier.

package/dist/adapters/api-client/build-request.js

@@ -15,8 +15,18 @@
 import { existsSync } from "fs";
 import { resolve } from "path";
 import { PipelineRequestSchema, } from "../../_vendor/ailf-core/index.js";
+import { LEGACY_EVAL_MODE_ALIASES } from "../../_vendor/ailf-shared/index.js";
 import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { RepoTaskSource } from "../task-sources/repo-task-source.js";
+const LEGACY_LITERACY_VARIANT_SET = new Set(LEGACY_EVAL_MODE_ALIASES);
+/**
+ * Resolve a raw `config.mode` (which may be a CLI literacy variant such as
+ * `"baseline"` or `"full"`) to the canonical task-level mode that appears on
+ * `GeneralizedTaskDefinition.mode`. Literacy variants all map to `"literacy"`.
+ */
+function resolveCanonicalTaskMode(configMode) {
+    return LEGACY_LITERACY_VARIANT_SET.has(configMode) ? "literacy" : configMode;
+}
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -33,10 +43,18 @@ import { RepoTaskSource } from "../task-sources/repo-task-source.js";
  */
 export async function buildRemoteRequest(options) {
     const { tasksDir, config } = options;
-    // 1. Load and validate local tasks
+    // 1. Load and validate local tasks, filtered to the requested mode.
+    //    `config.mode` may be a literacy variant (baseline/agentic/full/observed)
+    //    — those all map to task mode "literacy". Other modes match 1:1.
     const taskSource = new RepoTaskSource(tasksDir);
     const filterOptions = buildFilterOptions(config);
-    const tasks = (await taskSource.loadTasks(filterOptions)).filter((t) => t.mode === "literacy");
+    const allTasks = await taskSource.loadTasks(filterOptions);
+    const taskModeFilter = config.mode
+        ? resolveCanonicalTaskMode(config.mode)
+        : undefined;
+    const tasks = taskModeFilter
+        ? allTasks.filter((t) => t.mode === taskModeFilter)
+        : allTasks;
     if (tasks.length === 0) {
         throw new Error("No tasks found after applying filters.\n" +
             `  Tasks directory: ${tasksDir}\n` +
@@ -145,12 +163,13 @@ export function resolveTasksDir(rootDir, explicitPath) {
 // Helpers
 // ---------------------------------------------------------------------------
 /**
- * Convert a LiteracyTaskDefinition to the camelCase inline format expected
+ * Convert a GeneralizedTaskDefinition to the camelCase inline format expected
  * by the API.
  */
 function taskToInlineFormat(task) {
     const inline = {
         id: task.id,
+        mode: task.mode,
         description: task.title,
         featureArea: task.area ?? "",
         assert: task.assertions ?? [],
@@ -166,14 +185,17 @@ function taskToInlineFormat(task) {
             ...(task.prompt?.vars ?? {}),
         };
     }
-    if (task.docCoverage) {
-        inline.docCoverage = true;
-    }
-    if (task.referenceSolution) {
-        inline.referenceSolution = task.referenceSolution;
-    }
-    if (task.baseline) {
-        inline.baseline = task.baseline;
+    // Literacy-specific fields
+    if (task.mode === "literacy") {
+        if (task.docCoverage) {
+            inline.docCoverage = true;
+        }
+        if (task.referenceSolution) {
+            inline.referenceSolution = task.referenceSolution;
+        }
+        if (task.baseline) {
+            inline.baseline = task.baseline;
+        }
     }
     if (task.tags?.length) {
         inline.tags = task.tags;

package/dist/artifact-capture/api-gateway-artifact-uploader.d.ts

@@ -0,0 +1,41 @@
+/**
+ * ApiGatewayArtifactUploader — uploads report artifacts via the API Gateway.
+ *
+ * Counterpart to GcsReportArtifactUploader. Used when the CLI runs locally
+ * without GCS credentials. Two-step flow:
+ *
+ *   1. GET {apiBaseUrl}/v1/artifacts/{reportId}/upload-url?type={artifactType}
+ *      with Authorization: Bearer {apiKey} — returns a signed PUT URL.
+ *   2. PUT the JSON to that URL with Content-Type: application/json and
+ *      x-goog-if-generation-match: 0 (overwrite-protection contract from
+ *      the gateway's signed URL).
+ *
+ * The gateway stays out of the data path — Vercel only signs the URL,
+ * the artifact bytes go directly to GCS.
+ *
+ * Design principles:
+ * - P5: Non-blocking — any failure returns null and warns, never throws.
+ * - Stateless — no client to keep around between calls.
+ *
+ * @see docs/design-docs/external-artifact-store.md
+ * @see docs/decisions/D0030-external-artifact-store.md
+ */
+import type { ArtifactRef, ArtifactUploader } from "../_vendor/ailf-core/index.d.ts";
+export interface ApiGatewayUploaderOptions {
+    /** Base URL of the API gateway (e.g., "https://api.ailf.sanity.io"). */
+    apiBaseUrl: string;
+    /** AILF API key with the `artifact:write` scope. */
+    apiKey: string;
+    /** GCS bucket name — included in the returned ArtifactRef. */
+    bucket: string;
+}
+export declare class ApiGatewayArtifactUploader implements ArtifactUploader {
+    private readonly options;
+    constructor(options: ApiGatewayUploaderOptions);
+    upload(reportId: string, fileName: string, data: unknown): Promise<ArtifactRef | null>;
+    /**
+     * Fetch a signed upload URL from the gateway. Returns null on any non-2xx
+     * response or malformed body so the caller can stay non-blocking.
+     */
+    private fetchSignedUrl;
+}

package/dist/artifact-capture/api-gateway-artifact-uploader.js

@@ -0,0 +1,123 @@
+/**
+ * ApiGatewayArtifactUploader — uploads report artifacts via the API Gateway.
+ *
+ * Counterpart to GcsReportArtifactUploader. Used when the CLI runs locally
+ * without GCS credentials. Two-step flow:
+ *
+ *   1. GET {apiBaseUrl}/v1/artifacts/{reportId}/upload-url?type={artifactType}
+ *      with Authorization: Bearer {apiKey} — returns a signed PUT URL.
+ *   2. PUT the JSON to that URL with Content-Type: application/json and
+ *      x-goog-if-generation-match: 0 (overwrite-protection contract from
+ *      the gateway's signed URL).
+ *
+ * The gateway stays out of the data path — Vercel only signs the URL,
+ * the artifact bytes go directly to GCS.
+ *
+ * Design principles:
+ * - P5: Non-blocking — any failure returns null and warns, never throws.
+ * - Stateless — no client to keep around between calls.
+ *
+ * @see docs/design-docs/external-artifact-store.md
+ * @see docs/decisions/D0030-external-artifact-store.md
+ */
+// ---------------------------------------------------------------------------
+// File-name → artifact-type mapping (mirrors packages/api ARTIFACT_FILES)
+// ---------------------------------------------------------------------------
+/**
+ * Reverse map of the API gateway's ARTIFACT_FILES. The uploader port speaks
+ * file names; the gateway endpoint speaks artifact types. Keep these in sync
+ * with packages/api/src/routes/artifacts.ts.
+ */
+const FILE_TO_TYPE = {
+    "eval-results.json": "evalResults",
+    "grader-prompts.json": "graderPrompts",
+    "rendered-prompts.json": "renderedPrompts",
+    "task-definitions.json": "taskDefinitions",
+    "test-outputs.json": "testOutputs",
+};
+export class ApiGatewayArtifactUploader {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    async upload(reportId, fileName, data) {
+        const artifactType = FILE_TO_TYPE[fileName];
+        if (!artifactType) {
+            console.warn(`  ⚠️  Artifact upload skipped (unknown fileName): ${fileName}`);
+            return null;
+        }
+        const objectPath = `reports/${reportId}/${fileName}`;
+        const json = JSON.stringify(data);
+        const bytes = Buffer.byteLength(json, "utf-8");
+        try {
+            const signed = await this.fetchSignedUrl(reportId, artifactType);
+            if (!signed)
+                return null;
+            const putRes = await fetch(signed.url, {
+                body: json,
+                headers: signed.requiredHeaders,
+                method: "PUT",
+            });
+            if (!putRes.ok) {
+                console.warn(`  ⚠️  Artifact upload failed (non-blocking): ${objectPath} — GCS PUT ${putRes.status} ${putRes.statusText}`);
+                return null;
+            }
+            return {
+                bucket: signed.bucket,
+                bytes,
+                entryCount: extractEntryCount(data),
+                path: signed.path,
+                store: "gcs",
+            };
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`  ⚠️  Artifact upload failed (non-blocking): ${objectPath} — ${message}`);
+            return null;
+        }
+    }
+    /**
+     * Fetch a signed upload URL from the gateway. Returns null on any non-2xx
+     * response or malformed body so the caller can stay non-blocking.
+     */
+    async fetchSignedUrl(reportId, artifactType) {
+        const url = `${this.options.apiBaseUrl.replace(/\/$/, "")}/v1/artifacts/${encodeURIComponent(reportId)}/upload-url?type=${encodeURIComponent(artifactType)}`;
+        const res = await fetch(url, {
+            headers: {
+                Authorization: `Bearer ${this.options.apiKey}`,
+            },
+            method: "GET",
+        });
+        if (!res.ok) {
+            console.warn(`  ⚠️  Signed-URL request failed: ${res.status} ${res.statusText}`);
+            return null;
+        }
+        const body = (await res.json());
+        if (body.object !== "signed_upload_url" ||
+            typeof body.url !== "string" ||
+            typeof body.path !== "string" ||
+            typeof body.bucket !== "string" ||
+            !body.requiredHeaders) {
+            console.warn(`  ⚠️  Signed-URL response was malformed`);
+            return null;
+        }
+        return {
+            bucket: body.bucket,
+            method: "PUT",
+            object: "signed_upload_url",
+            path: body.path,
+            requiredHeaders: body.requiredHeaders,
+            url: body.url,
+        };
+    }
+}
+function extractEntryCount(data) {
+    if (typeof data === "object" &&
+        data !== null &&
+        "entries" in data &&
+        typeof data.entries === "object") {
+        return Object.keys(data.entries)
+            .length;
+    }
+    return undefined;
+}

package/dist/commands/pipeline-action.js

@@ -16,7 +16,7 @@ import { fileURLToPath } from "url";
 import { classifyUrls } from "../pipeline/classify-url.js";
 import { normalizeMode } from "../pipeline/normalize-mode.js";
 import { assessImpact, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
-import { buildAppContext } from "../orchestration/build-app-context.js";
+import { buildAppContext, parseArtifactUploadEnv, } from "../orchestration/build-app-context.js";
 import { buildStepSequence } from "../orchestration/build-step-sequence.js";
 import { orchestratePipeline } from "../orchestration/pipeline-orchestrator.js";
 import { load } from "js-yaml";
@@ -329,6 +329,7 @@ export async function executePipeline(cliOpts) {
         config.captureGcsBucket ??= process.env.AILF_CAPTURE_GCS_BUCKET;
         config.captureGcsPrefix ??= process.env.AILF_CAPTURE_GCS_PREFIX;
         config.artifactGcsBucket ??= process.env.AILF_GCS_ARTIFACT_BUCKET;
+        config.artifactUpload ??= parseArtifactUploadEnv(process.env.AILF_ARTIFACT_UPLOAD);
         // Create AppContext directly from the merged config so adapters
         // (especially taskSource) are wired from the file config's
         // taskSourceType — not from CLI defaults.

package/dist/composition-root.d.ts

@@ -15,7 +15,7 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import { type AppContext, type AssertionRegistration, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+import { type AppContext, type ArtifactUploader, type AssertionRegistration, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
 /**
  * Create a fully wired AppContext from resolved configuration.
  *
@@ -23,6 +23,22 @@ import { type AppContext, type AssertionRegistration, type ResolvedConfig } from
  * Swapping an adapter is a one-line change in this function.
  */
 export declare function createAppContext(config: ResolvedConfig): AppContext;
+/**
+ * Selects an ArtifactUploader implementation based on available credentials.
+ *
+ * Selection order:
+ *   1. config.artifactUpload === false → always skip (explicit opt-out)
+ *   2. GOOGLE_APPLICATION_CREDENTIALS or GCLOUD_PROJECT present → direct GCS
+ *   3. apiKey + apiUrl present → gateway-signed PUT URL
+ *   4. Neither → skip silently (P5)
+ *
+ * The bucket defaults to DEFAULT_ARTIFACT_BUCKET when not explicitly set —
+ * users only need to override for self-hosted deployments with a different
+ * bucket (and matching gateway signing credentials).
+ *
+ * Exported for unit-test access; not part of the public package API.
+ */
+export declare function createArtifactUploader(config: ResolvedConfig, logger: Logger): ArtifactUploader | undefined;
 /**
  * Generic Promptfoo assertion types available to all evaluation modes.
  *

package/dist/composition-root.js

@@ -17,6 +17,7 @@
  */
 import { join } from "node:path";
 import { InMemoryPluginRegistry, NoOpArtifactCollector, } from "./_vendor/ailf-core/index.js";
+import { ApiGatewayArtifactUploader } from "./artifact-capture/api-gateway-artifact-uploader.js";
 import { FilesystemArtifactCollector } from "./artifact-capture/filesystem-collector.js";
 import { GcsArtifactCollector } from "./artifact-capture/gcs-collector.js";
 import { GcsReportArtifactUploader } from "./artifact-capture/gcs-report-artifact-uploader.js";
@@ -82,10 +83,10 @@ export function createAppContext(config) {
             : fsCollector;
     }
     // Report artifact uploader — uploads structured files to GCS at known
-    // paths for Studio to fetch via signed URLs (D0030)
-    const artifactUploader = config.artifactGcsBucket
-        ? new GcsReportArtifactUploader({ bucket: config.artifactGcsBucket })
-        : undefined;
+    // paths for Studio to fetch via signed URLs (D0030). Auto-detects the
+    // right adapter from available credentials; defaults bucket to
+    // "ailf-artifacts". Set artifactUpload: false to opt out entirely.
+    const artifactUploader = createArtifactUploader(config, logger);
     return {
         artifactUploader,
         cache,
@@ -115,6 +116,53 @@ function createLogger() {
             process.env.AILF_VERBOSE === "1",
     });
 }
+/**
+ * Shared GCS bucket for report artifacts. Matches the gateway default at
+ * packages/api/src/routes/artifacts.ts — both sides assume ailf-artifacts
+ * unless explicitly overridden. The gateway's signing credentials are scoped
+ * to this bucket, so alternate names require reconfiguring the gateway.
+ */
+const DEFAULT_ARTIFACT_BUCKET = "ailf-artifacts";
+/**
+ * Selects an ArtifactUploader implementation based on available credentials.
+ *
+ * Selection order:
+ *   1. config.artifactUpload === false → always skip (explicit opt-out)
+ *   2. GOOGLE_APPLICATION_CREDENTIALS or GCLOUD_PROJECT present → direct GCS
+ *   3. apiKey + apiUrl present → gateway-signed PUT URL
+ *   4. Neither → skip silently (P5)
+ *
+ * The bucket defaults to DEFAULT_ARTIFACT_BUCKET when not explicitly set —
+ * users only need to override for self-hosted deployments with a different
+ * bucket (and matching gateway signing credentials).
+ *
+ * Exported for unit-test access; not part of the public package API.
+ */
+export function createArtifactUploader(config, logger) {
+    if (config.artifactUpload === false) {
+        logger.debug("Artifact upload explicitly disabled via artifactUpload=false");
+        return undefined;
+    }
+    const bucket = config.artifactGcsBucket ?? DEFAULT_ARTIFACT_BUCKET;
+    // CI / GCP runtime — direct GCS upload (fastest, no extra hop).
+    // We treat the presence of either env var as the user opting in to ADC.
+    const hasGcsCredentials = Boolean(process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.GCLOUD_PROJECT);
+    if (hasGcsCredentials) {
+        logger.debug(`Artifact uploader: GcsReportArtifactUploader (direct GCS via ADC, bucket=${bucket})`);
+        return new GcsReportArtifactUploader({ bucket });
+    }
+    // Local dev — request signed PUT URLs from the API gateway, no GCS creds needed.
+    if (config.apiKey && config.apiUrl) {
+        logger.debug(`Artifact uploader: ApiGatewayArtifactUploader (signed URL via ${config.apiUrl}, bucket=${bucket})`);
+        return new ApiGatewayArtifactUploader({
+            apiBaseUrl: config.apiUrl,
+            apiKey: config.apiKey,
+            bucket,
+        });
+    }
+    logger.debug("Artifact upload skipped: no GCS credentials or AILF_API_KEY available");
+    return undefined;
+}
 function createCache(config) {
     const local = new FilesystemCache(config.rootDir);
     if (config.noRemoteCache)

package/dist/orchestration/build-app-context.d.ts

@@ -18,6 +18,14 @@ import type { ResolvedOptions } from "../commands/pipeline-action.js";
  * are derived (e.g., areas from areaOption).
  */
 export declare function mapToResolvedConfig(opts: ResolvedOptions, rootDir: string): ResolvedConfig;
+/**
+ * Parse the AILF_ARTIFACT_UPLOAD env var into a tri-state.
+ *
+ * - "0" | "false" → false (force-disable)
+ * - "1" | "true"  → true  (force-enable; still a no-op without credentials)
+ * - unset | other → undefined (auto-detect from credentials)
+ */
+export declare function parseArtifactUploadEnv(value: string | undefined): boolean | undefined;
 /**
  * Build an AppContext from legacy ResolvedOptions.
  *

package/dist/orchestration/build-app-context.js

@@ -85,8 +85,26 @@ export function mapToResolvedConfig(opts, rootDir) {
         captureGcsBucket: process.env.AILF_CAPTURE_GCS_BUCKET,
         captureGcsPrefix: process.env.AILF_CAPTURE_GCS_PREFIX,
         artifactGcsBucket: process.env.AILF_GCS_ARTIFACT_BUCKET,
+        artifactUpload: parseArtifactUploadEnv(process.env.AILF_ARTIFACT_UPLOAD),
     };
 }
+/**
+ * Parse the AILF_ARTIFACT_UPLOAD env var into a tri-state.
+ *
+ * - "0" | "false" → false (force-disable)
+ * - "1" | "true"  → true  (force-enable; still a no-op without credentials)
+ * - unset | other → undefined (auto-detect from credentials)
+ */
+export function parseArtifactUploadEnv(value) {
+    if (value === undefined)
+        return undefined;
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "0" || normalized === "false")
+        return false;
+    if (normalized === "1" || normalized === "true")
+        return true;
+    return undefined;
+}
 /**
  * Build an AppContext from legacy ResolvedOptions.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.4.0",
+  "version": "2.6.0",
   "private": false,
   "publishConfig": {
     "access": "public"