@sanity/ailf 2.4.0 → 2.5.0

package/dist/adapters/api-client/build-request.js

@@ -15,8 +15,18 @@
 import { existsSync } from "fs";
 import { resolve } from "path";
 import { PipelineRequestSchema, } from "../../_vendor/ailf-core/index.js";
+import { LEGACY_EVAL_MODE_ALIASES } from "../../_vendor/ailf-shared/index.js";
 import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { RepoTaskSource } from "../task-sources/repo-task-source.js";
+const LEGACY_LITERACY_VARIANT_SET = new Set(LEGACY_EVAL_MODE_ALIASES);
+/**
+ * Resolve a raw `config.mode` (which may be a CLI literacy variant such as
+ * `"baseline"` or `"full"`) to the canonical task-level mode that appears on
+ * `GeneralizedTaskDefinition.mode`. Literacy variants all map to `"literacy"`.
+ */
+function resolveCanonicalTaskMode(configMode) {
+    return LEGACY_LITERACY_VARIANT_SET.has(configMode) ? "literacy" : configMode;
+}
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -33,10 +43,18 @@ import { RepoTaskSource } from "../task-sources/repo-task-source.js";
  */
 export async function buildRemoteRequest(options) {
     const { tasksDir, config } = options;
-    // 1. Load and validate local tasks
+    // 1. Load and validate local tasks, filtered to the requested mode.
+    //    `config.mode` may be a literacy variant (baseline/agentic/full/observed)
+    //    — those all map to task mode "literacy". Other modes match 1:1.
     const taskSource = new RepoTaskSource(tasksDir);
     const filterOptions = buildFilterOptions(config);
-    const tasks = (await taskSource.loadTasks(filterOptions)).filter((t) => t.mode === "literacy");
+    const allTasks = await taskSource.loadTasks(filterOptions);
+    const taskModeFilter = config.mode
+        ? resolveCanonicalTaskMode(config.mode)
+        : undefined;
+    const tasks = taskModeFilter
+        ? allTasks.filter((t) => t.mode === taskModeFilter)
+        : allTasks;
     if (tasks.length === 0) {
         throw new Error("No tasks found after applying filters.\n" +
             `  Tasks directory: ${tasksDir}\n` +
@@ -145,12 +163,13 @@ export function resolveTasksDir(rootDir, explicitPath) {
 // Helpers
 // ---------------------------------------------------------------------------
 /**
- * Convert a LiteracyTaskDefinition to the camelCase inline format expected
+ * Convert a GeneralizedTaskDefinition to the camelCase inline format expected
  * by the API.
  */
 function taskToInlineFormat(task) {
     const inline = {
         id: task.id,
+        mode: task.mode,
         description: task.title,
         featureArea: task.area ?? "",
         assert: task.assertions ?? [],
@@ -166,14 +185,17 @@ function taskToInlineFormat(task) {
             ...(task.prompt?.vars ?? {}),
         };
     }
-    if (task.docCoverage) {
-        inline.docCoverage = true;
-    }
-    if (task.referenceSolution) {
-        inline.referenceSolution = task.referenceSolution;
-    }
-    if (task.baseline) {
-        inline.baseline = task.baseline;
+    // Literacy-specific fields
+    if (task.mode === "literacy") {
+        if (task.docCoverage) {
+            inline.docCoverage = true;
+        }
+        if (task.referenceSolution) {
+            inline.referenceSolution = task.referenceSolution;
+        }
+        if (task.baseline) {
+            inline.baseline = task.baseline;
+        }
     }
     if (task.tags?.length) {
         inline.tags = task.tags;

package/dist/artifact-capture/api-gateway-artifact-uploader.d.ts

@@ -0,0 +1,41 @@
+/**
+ * ApiGatewayArtifactUploader — uploads report artifacts via the API Gateway.
+ *
+ * Counterpart to GcsReportArtifactUploader. Used when the CLI runs locally
+ * without GCS credentials. Two-step flow:
+ *
+ *   1. GET {apiBaseUrl}/v1/artifacts/{reportId}/upload-url?type={artifactType}
+ *      with Authorization: Bearer {apiKey} — returns a signed PUT URL.
+ *   2. PUT the JSON to that URL with Content-Type: application/json and
+ *      x-goog-if-generation-match: 0 (overwrite-protection contract from
+ *      the gateway's signed URL).
+ *
+ * The gateway stays out of the data path — Vercel only signs the URL,
+ * the artifact bytes go directly to GCS.
+ *
+ * Design principles:
+ * - P5: Non-blocking — any failure returns null and warns, never throws.
+ * - Stateless — no client to keep around between calls.
+ *
+ * @see docs/design-docs/external-artifact-store.md
+ * @see docs/decisions/D0030-external-artifact-store.md
+ */
+import type { ArtifactRef, ArtifactUploader } from "../_vendor/ailf-core/index.d.ts";
+export interface ApiGatewayUploaderOptions {
+    /** Base URL of the API gateway (e.g., "https://api.ailf.sanity.io"). */
+    apiBaseUrl: string;
+    /** AILF API key with the `artifact:write` scope. */
+    apiKey: string;
+    /** GCS bucket name — included in the returned ArtifactRef. */
+    bucket: string;
+}
+export declare class ApiGatewayArtifactUploader implements ArtifactUploader {
+    private readonly options;
+    constructor(options: ApiGatewayUploaderOptions);
+    upload(reportId: string, fileName: string, data: unknown): Promise<ArtifactRef | null>;
+    /**
+     * Fetch a signed upload URL from the gateway. Returns null on any non-2xx
+     * response or malformed body so the caller can stay non-blocking.
+     */
+    private fetchSignedUrl;
+}

package/dist/artifact-capture/api-gateway-artifact-uploader.js

@@ -0,0 +1,123 @@
+/**
+ * ApiGatewayArtifactUploader — uploads report artifacts via the API Gateway.
+ *
+ * Counterpart to GcsReportArtifactUploader. Used when the CLI runs locally
+ * without GCS credentials. Two-step flow:
+ *
+ *   1. GET {apiBaseUrl}/v1/artifacts/{reportId}/upload-url?type={artifactType}
+ *      with Authorization: Bearer {apiKey} — returns a signed PUT URL.
+ *   2. PUT the JSON to that URL with Content-Type: application/json and
+ *      x-goog-if-generation-match: 0 (overwrite-protection contract from
+ *      the gateway's signed URL).
+ *
+ * The gateway stays out of the data path — Vercel only signs the URL,
+ * the artifact bytes go directly to GCS.
+ *
+ * Design principles:
+ * - P5: Non-blocking — any failure returns null and warns, never throws.
+ * - Stateless — no client to keep around between calls.
+ *
+ * @see docs/design-docs/external-artifact-store.md
+ * @see docs/decisions/D0030-external-artifact-store.md
+ */
+// ---------------------------------------------------------------------------
+// File-name → artifact-type mapping (mirrors packages/api ARTIFACT_FILES)
+// ---------------------------------------------------------------------------
+/**
+ * Reverse map of the API gateway's ARTIFACT_FILES. The uploader port speaks
+ * file names; the gateway endpoint speaks artifact types. Keep these in sync
+ * with packages/api/src/routes/artifacts.ts.
+ */
+const FILE_TO_TYPE = {
+    "eval-results.json": "evalResults",
+    "grader-prompts.json": "graderPrompts",
+    "rendered-prompts.json": "renderedPrompts",
+    "task-definitions.json": "taskDefinitions",
+    "test-outputs.json": "testOutputs",
+};
+export class ApiGatewayArtifactUploader {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    async upload(reportId, fileName, data) {
+        const artifactType = FILE_TO_TYPE[fileName];
+        if (!artifactType) {
+            console.warn(`  ⚠️  Artifact upload skipped (unknown fileName): ${fileName}`);
+            return null;
+        }
+        const objectPath = `reports/${reportId}/${fileName}`;
+        const json = JSON.stringify(data);
+        const bytes = Buffer.byteLength(json, "utf-8");
+        try {
+            const signed = await this.fetchSignedUrl(reportId, artifactType);
+            if (!signed)
+                return null;
+            const putRes = await fetch(signed.url, {
+                body: json,
+                headers: signed.requiredHeaders,
+                method: "PUT",
+            });
+            if (!putRes.ok) {
+                console.warn(`  ⚠️  Artifact upload failed (non-blocking): ${objectPath} — GCS PUT ${putRes.status} ${putRes.statusText}`);
+                return null;
+            }
+            return {
+                bucket: signed.bucket,
+                bytes,
+                entryCount: extractEntryCount(data),
+                path: signed.path,
+                store: "gcs",
+            };
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`  ⚠️  Artifact upload failed (non-blocking): ${objectPath} — ${message}`);
+            return null;
+        }
+    }
+    /**
+     * Fetch a signed upload URL from the gateway. Returns null on any non-2xx
+     * response or malformed body so the caller can stay non-blocking.
+     */
+    async fetchSignedUrl(reportId, artifactType) {
+        const url = `${this.options.apiBaseUrl.replace(/\/$/, "")}/v1/artifacts/${encodeURIComponent(reportId)}/upload-url?type=${encodeURIComponent(artifactType)}`;
+        const res = await fetch(url, {
+            headers: {
+                Authorization: `Bearer ${this.options.apiKey}`,
+            },
+            method: "GET",
+        });
+        if (!res.ok) {
+            console.warn(`  ⚠️  Signed-URL request failed: ${res.status} ${res.statusText}`);
+            return null;
+        }
+        const body = (await res.json());
+        if (body.object !== "signed_upload_url" ||
+            typeof body.url !== "string" ||
+            typeof body.path !== "string" ||
+            typeof body.bucket !== "string" ||
+            !body.requiredHeaders) {
+            console.warn(`  ⚠️  Signed-URL response was malformed`);
+            return null;
+        }
+        return {
+            bucket: body.bucket,
+            method: "PUT",
+            object: "signed_upload_url",
+            path: body.path,
+            requiredHeaders: body.requiredHeaders,
+            url: body.url,
+        };
+    }
+}
+function extractEntryCount(data) {
+    if (typeof data === "object" &&
+        data !== null &&
+        "entries" in data &&
+        typeof data.entries === "object") {
+        return Object.keys(data.entries)
+            .length;
+    }
+    return undefined;
+}

package/dist/composition-root.d.ts

@@ -15,7 +15,7 @@
  * @see packages/core/src/ports/context.ts — AppContext interface
  * @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
  */
-import { type AppContext, type AssertionRegistration, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+import { type AppContext, type ArtifactUploader, type AssertionRegistration, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
 /**
  * Create a fully wired AppContext from resolved configuration.
  *
@@ -23,6 +23,15 @@ import { type AppContext, type AssertionRegistration, type ResolvedConfig } from
  * Swapping an adapter is a one-line change in this function.
  */
 export declare function createAppContext(config: ResolvedConfig): AppContext;
+/**
+ * Selects an ArtifactUploader implementation based on available credentials.
+ *
+ * Returns undefined when artifact upload is not configured — the publish
+ * step skips silently in that case (P5).
+ *
+ * Exported for unit-test access; not part of the public package API.
+ */
+export declare function createArtifactUploader(config: ResolvedConfig, logger: Logger): ArtifactUploader | undefined;
 /**
  * Generic Promptfoo assertion types available to all evaluation modes.
  *

package/dist/composition-root.js

@@ -17,6 +17,7 @@
  */
 import { join } from "node:path";
 import { InMemoryPluginRegistry, NoOpArtifactCollector, } from "./_vendor/ailf-core/index.js";
+import { ApiGatewayArtifactUploader } from "./artifact-capture/api-gateway-artifact-uploader.js";
 import { FilesystemArtifactCollector } from "./artifact-capture/filesystem-collector.js";
 import { GcsArtifactCollector } from "./artifact-capture/gcs-collector.js";
 import { GcsReportArtifactUploader } from "./artifact-capture/gcs-report-artifact-uploader.js";
@@ -82,10 +83,14 @@ export function createAppContext(config) {
             : fsCollector;
     }
     // Report artifact uploader — uploads structured files to GCS at known
-    // paths for Studio to fetch via signed URLs (D0030)
-    const artifactUploader = config.artifactGcsBucket
-        ? new GcsReportArtifactUploader({ bucket: config.artifactGcsBucket })
-        : undefined;
+    // paths for Studio to fetch via signed URLs (D0030).
+    //
+    // Selection (W0042):
+    //   1. Direct GCS upload when ADC env vars are present (CI / GCP runtime)
+    //   2. API Gateway signed-URL upload when only an AILF API key is present
+    //      (local dev — no GCS credentials needed)
+    //   3. Skipped silently when neither is configured
+    const artifactUploader = createArtifactUploader(config, logger);
     return {
         artifactUploader,
         cache,
@@ -115,6 +120,36 @@ function createLogger() {
             process.env.AILF_VERBOSE === "1",
     });
 }
+/**
+ * Selects an ArtifactUploader implementation based on available credentials.
+ *
+ * Returns undefined when artifact upload is not configured — the publish
+ * step skips silently in that case (P5).
+ *
+ * Exported for unit-test access; not part of the public package API.
+ */
+export function createArtifactUploader(config, logger) {
+    if (!config.artifactGcsBucket)
+        return undefined;
+    // CI / GCP runtime — direct GCS upload (fastest, no extra hop).
+    // We treat the presence of either env var as the user opting in to ADC.
+    const hasGcsCredentials = Boolean(process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.GCLOUD_PROJECT);
+    if (hasGcsCredentials) {
+        logger.debug("Artifact uploader: GcsReportArtifactUploader (direct GCS via ADC)");
+        return new GcsReportArtifactUploader({ bucket: config.artifactGcsBucket });
+    }
+    // Local dev — request signed PUT URLs from the API gateway, no GCS creds needed.
+    if (config.apiKey && config.apiUrl) {
+        logger.debug(`Artifact uploader: ApiGatewayArtifactUploader (signed URL via ${config.apiUrl})`);
+        return new ApiGatewayArtifactUploader({
+            apiBaseUrl: config.apiUrl,
+            apiKey: config.apiKey,
+            bucket: config.artifactGcsBucket,
+        });
+    }
+    logger.debug("Artifact upload skipped: AILF_GCS_ARTIFACT_BUCKET set but no GCS credentials or AILF_API_KEY available");
+    return undefined;
+}
 function createCache(config) {
     const local = new FilesystemCache(config.rootDir);
     if (config.noRemoteCache)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "private": false,
   "publishConfig": {
     "access": "public"