@sanity/ailf 2.9.0 → 3.1.0

package/dist/artifact-capture/api-gateway-artifact-writer.js

@@ -8,7 +8,7 @@
  * Endpoints:
  *   - Bulk:      GET  {apiBaseUrl}/v1/runs/{runId}/artifacts/{type}/upload-url
  *   - Per-entry: GET  {apiBaseUrl}/v1/runs/{runId}/artifacts/{type}/{entryKey}/upload-url
- *   - Manifest:  GET  {apiBaseUrl}/v1/runs/{runId}/artifacts/manifest/upload-url
+ *   - Manifest:  GET  {apiBaseUrl}/v1/runs/{runId}/artifacts/upload-url
  *
  * ## W0049 API surface
  *
@@ -28,10 +28,13 @@
  * @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
  */
 import { ARTIFACT_REGISTRY, NotImplementedError, } from "../_vendor/ailf-core/index.js";
+import { NO_OP_UPLOAD_METRICS, } from "./upload-metrics.js";
 export class ApiGatewayArtifactWriter {
     options;
+    metrics;
     constructor(options) {
         this.options = options;
+        this.metrics = options.metrics ?? NO_OP_UPLOAD_METRICS;
     }
     // ---- Canonical W0049 API ------------------------------------------------
     async emit(type, association, payload) {
@@ -46,12 +49,13 @@ export class ApiGatewayArtifactWriter {
             return this.putJson(uploadUrlPath, payload, {
                 layout: "bulk",
                 entryCount: entryCountOf(payload),
+                type,
             });
         }
         // per-entry
         const entryKey = descriptor.formatEntryKey(association);
         const uploadUrlPath = `/v1/runs/${encodeURIComponent(runId)}/artifacts/${encodeURIComponent(type)}/${encodeURIComponent(entryKey)}/upload-url`;
-        const result = await this.putJsonRaw(uploadUrlPath, payload);
+        const result = await this.putJsonRaw(uploadUrlPath, payload, type);
         if (!result)
             return null;
         return {
@@ -72,8 +76,11 @@ export class ApiGatewayArtifactWriter {
             "the batch route.");
     }
     async writeManifest(runId, manifest) {
-        const uploadUrlPath = `/v1/runs/${encodeURIComponent(runId)}/artifacts/manifest/upload-url`;
-        return this.putJson(uploadUrlPath, manifest, { layout: "bulk" });
+        const uploadUrlPath = `/v1/runs/${encodeURIComponent(runId)}/artifacts/upload-url`;
+        return this.putJson(uploadUrlPath, manifest, {
+            layout: "bulk",
+            type: "manifest",
+        });
     }
     // ---- Deprecated legacy surface (W0052) ----------------------------------
     /** @deprecated Use `emit()` instead. */
@@ -82,6 +89,7 @@ export class ApiGatewayArtifactWriter {
         return this.putJson(uploadUrlPath, data, {
             layout: "bulk",
             entryCount: entryCountOf(data),
+            type,
         });
     }
     /** @deprecated Use `emit()` per entry instead. */
@@ -101,7 +109,7 @@ export class ApiGatewayArtifactWriter {
                 continue;
             }
             const uploadUrlPath = `/v1/runs/${encodeURIComponent(runId)}/artifacts/${encodeURIComponent(type)}/${encodeURIComponent(entry.key)}/upload-url`;
-            const result = await this.putJsonRaw(uploadUrlPath, entry.data);
+            const result = await this.putJsonRaw(uploadUrlPath, entry.data, type);
             if (!result)
                 continue;
             bucket = result.bucket;
@@ -122,7 +130,7 @@ export class ApiGatewayArtifactWriter {
     }
     // ---- Internals ----------------------------------------------------------
     async putJson(uploadUrlPath, data, meta) {
-        const result = await this.putJsonRaw(uploadUrlPath, data);
+        const result = await this.putJsonRaw(uploadUrlPath, data, meta.type);
         if (!result)
             return null;
         return {
@@ -134,23 +142,38 @@ export class ApiGatewayArtifactWriter {
             layout: meta.layout,
         };
     }
-    async putJsonRaw(uploadUrlPath, data) {
+    async putJsonRaw(uploadUrlPath, data, type) {
         const json = JSON.stringify(data);
         const bytes = Buffer.byteLength(json, "utf-8");
         try {
-            const signed = await this.fetchSignedUrl(uploadUrlPath);
+            const signed = await this.fetchSignedUrl(uploadUrlPath, type);
             if (!signed)
                 return null;
-            const putRes = await fetch(signed.url, {
-                body: json,
-                headers: signed.requiredHeaders,
-                method: "PUT",
-            });
-            if (!putRes.ok) {
-                console.warn(`  ⚠️  Artifact upload failed (non-blocking): ${signed.path} — GCS PUT ${putRes.status} ${putRes.statusText}`);
-                return null;
+            const putStart = Date.now();
+            let putSuccess = false;
+            try {
+                const putRes = await fetch(signed.url, {
+                    body: json,
+                    headers: signed.requiredHeaders,
+                    method: "PUT",
+                });
+                if (!putRes.ok) {
+                    console.warn(`  ⚠️  Artifact upload failed (non-blocking): ${signed.path} — GCS PUT ${putRes.status} ${putRes.statusText}`);
+                    return null;
+                }
+                putSuccess = true;
+                return { bucket: signed.bucket, path: signed.path, bytes };
+            }
+            finally {
+                this.metrics.record({
+                    phase: "put",
+                    writer: "ApiGatewayArtifactWriter",
+                    type,
+                    ms: Date.now() - putStart,
+                    bytes,
+                    success: putSuccess,
+                });
             }
-            return { bucket: signed.bucket, path: signed.path, bytes };
         }
         catch (err) {
             const message = err instanceof Error ? err.message : String(err);
@@ -158,33 +181,47 @@ export class ApiGatewayArtifactWriter {
             return null;
         }
     }
-    async fetchSignedUrl(uploadUrlPath) {
-        const url = `${this.options.apiBaseUrl.replace(/\/$/, "")}${uploadUrlPath}`;
-        const res = await fetch(url, {
-            headers: { Authorization: `Bearer ${this.options.apiKey}` },
-            method: "GET",
-        });
-        if (!res.ok) {
-            console.warn(`  ⚠️  Signed-URL request failed: ${res.status} ${res.statusText}`);
-            return null;
+    async fetchSignedUrl(uploadUrlPath, type) {
+        const start = Date.now();
+        let success = false;
+        try {
+            const url = `${this.options.apiBaseUrl.replace(/\/$/, "")}${uploadUrlPath}`;
+            const res = await fetch(url, {
+                headers: { Authorization: `Bearer ${this.options.apiKey}` },
+                method: "GET",
+            });
+            if (!res.ok) {
+                console.warn(`  ⚠️  Signed-URL request failed: ${res.status} ${res.statusText}`);
+                return null;
+            }
+            const body = (await res.json());
+            if (body.object !== "signed_upload_url" ||
+                typeof body.url !== "string" ||
+                typeof body.path !== "string" ||
+                typeof body.bucket !== "string" ||
+                !body.requiredHeaders) {
+                console.warn(`  ⚠️  Signed-URL response was malformed`);
+                return null;
+            }
+            success = true;
+            return {
+                bucket: body.bucket,
+                method: "PUT",
+                object: "signed_upload_url",
+                path: body.path,
+                requiredHeaders: body.requiredHeaders,
+                url: body.url,
+            };
         }
-        const body = (await res.json());
-        if (body.object !== "signed_upload_url" ||
-            typeof body.url !== "string" ||
-            typeof body.path !== "string" ||
-            typeof body.bucket !== "string" ||
-            !body.requiredHeaders) {
-            console.warn(`  ⚠️  Signed-URL response was malformed`);
-            return null;
+        finally {
+            this.metrics.record({
+                phase: "sign",
+                writer: "ApiGatewayArtifactWriter",
+                type,
+                ms: Date.now() - start,
+                success,
+            });
         }
-        return {
-            bucket: body.bucket,
-            method: "PUT",
-            object: "signed_upload_url",
-            path: body.path,
-            requiredHeaders: body.requiredHeaders,
-            url: body.url,
-        };
     }
 }
 function entryCountOf(data) {

package/dist/artifact-capture/batching-api-gateway-artifact-writer.d.ts

@@ -0,0 +1,108 @@
+/**
+ * BatchingApiGatewayArtifactWriter — W0056 prototype B.
+ *
+ * Drop-in replacement for `ApiGatewayArtifactWriter` that coalesces `emit()`
+ * calls into batches, signs many URLs per Vercel round trip via
+ * `POST /v1/runs/:runId/artifacts/batch/upload-urls` (W0052), and PUTs to GCS
+ * with bounded concurrency.
+ *
+ * Why: the baseline in `docs/design-docs/artifact-upload-throughput.md` showed
+ * that parallel single-URL signs trigger `429 Too Many Requests` on the Vercel
+ * signing function. Batch signing eliminates one Vercel round trip per
+ * artifact and makes client-side parallelism safe on the API Gateway path.
+ *
+ * Flushing:
+ * - An `emit()` call pushes a `PendingEmit` onto a queue and returns a
+ *   deferred Promise. The queue self-flushes via `queueMicrotask` when
+ *   the first entry lands, and again whenever the buffer reaches
+ *   `batchSize`.
+ * - `writeManifest(...)` awaits any in-flight flush and drains the queue
+ *   before writing the manifest through a single-URL sign (the manifest
+ *   isn't in the registry's per-entry scheme).
+ *
+ * NDJSON streaming for `traces` is still not implemented here — the
+ * upstream `ApiGatewayArtifactWriter` throws `NotImplementedError` and so
+ * does this writer. Traces flow through the GCS-direct writer when ADC
+ * credentials are present.
+ */
+import { type ArtifactEntry, type ArtifactRef, type ArtifactType, type ArtifactWriter, type AssociationValues, type RunId, type RunManifest } from "../_vendor/ailf-core/index.d.ts";
+import { type UploadMetricsSink } from "./upload-metrics.js";
+export interface BatchingApiGatewayArtifactWriterOptions {
+    /** Base URL of the API gateway (e.g., "https://ailf-api.sanity.build"). */
+    apiBaseUrl: string;
+    /** AILF API key with the `artifact:write` scope. */
+    apiKey: string;
+    /** GCS bucket name — included in the returned ArtifactRef. */
+    bucket: string;
+    /**
+     * Maximum entries per `/batch/upload-urls` request. The existing Vercel
+     * route signs URLs in parallel inside a single Function invocation, so
+     * the cap protects the Function's CPU budget. Defaults to 256 (see
+     * `DEFAULT_BATCH_SIZE` for the rationale and the W0058 measurement
+     * that motivated the bump from 32).
+     */
+    batchSize?: number;
+    /** Bounded concurrency for the PUT fan-out after a batch sign resolves. */
+    putConcurrency?: number;
+    /** Optional metrics sink; defaults to no-op. */
+    metrics?: UploadMetricsSink;
+}
+export declare class BatchingApiGatewayArtifactWriter implements ArtifactWriter {
+    private readonly options;
+    private readonly pending;
+    private flushing;
+    private microtaskScheduled;
+    constructor(options: BatchingApiGatewayArtifactWriterOptions);
+    emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
+    appendNdjson(): Promise<ArtifactRef | null>;
+    writeManifest(runId: RunId, manifest: RunManifest): Promise<ArtifactRef | null>;
+    /** @deprecated — routes through `emit()` for backward compat. */
+    writeBulk(type: ArtifactType, runId: RunId, data: unknown): Promise<ArtifactRef | null>;
+    /**
+     * @deprecated The legacy `writePerEntry` surface requires back-deriving an
+     * `AssociationValues` from a wire entryKey, which the registry no longer
+     * exposes. Producers on this writer must use `emit()`. All current producers
+     * have already migrated.
+     */
+    writePerEntry(type: ArtifactType, _runId: RunId, _entries: readonly ArtifactEntry[]): Promise<ArtifactRef | null>;
+    private scheduleFlush;
+    /**
+     * Drain the pending queue with overlapped sign + PUT.
+     *
+     * Pre-pipeline behaviour was `sign(N) → put(N) → sign(N+1) → put(N+1)`.
+     * After this change the order becomes
+     * `sign(N) → (put(N) || sign(N+1)) → put(N+1)`:
+     *
+     * - We wait for stage N's sign to resolve, then launch stage N+1's sign
+     *   immediately (before awaiting stage N's PUT phase). This is the moment
+     *   emits that arrived while stage N's sign was in flight first become
+     *   visible to `startNextStage`.
+     * - Stage N's PUT fan-out then runs concurrently with stage N+1's sign
+     *   request, removing the per-batch sign-RTT pre-amble that made the
+     *   W0056 prototype ~10 % slower than the (unsafe) single-URL parallel
+     *   variant.
+     *
+     * Invariants preserved from the pre-pipeline implementation:
+     * - Emits resolve in batch-commit order within a stage.
+     * - All entries in a `Stage` share one `runId` (enforced by `startNextStage`).
+     * - A sign failure fails only that stage's group; later stages proceed.
+     * - run() keeps looping until `pending` is empty AND no prefetched stage
+     *   remains — so emits that arrive during a PUT phase are picked up in the
+     *   next iteration.
+     */
+    private drain;
+    /**
+     * Pop the next same-runId batch (up to `batchSize` entries) off the queue
+     * and launch its batch-sign request immediately. Returns a `Stage` whose
+     * `signPromise` is already in flight, so the caller can start the sign for
+     * the *next* batch while this one's PUTs are still pending.
+     *
+     * Returning `null` means the queue is empty.
+     */
+    private startNextStage;
+    private buildSignBody;
+    private batchSign;
+    private putToSignedUrl;
+    /** Single-URL sign + PUT — used for the manifest. */
+    private writeSingle;
+}