@hue-run/sdk 0.18.2 → 0.19.0

package/dist/blobs.js

@@ -8,9 +8,11 @@ import { capLogBody, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys } from ".
 import { truncateUtf8 } from "./safety.js";
 /**
  * Large values and binary media travel apart from their span through reservation, presigned PUT,
- * and completion. Text keeps a 16 KiB prefix. Binary media keeps digest metadata on its part.
- * Successful uploads are listed in `hue.blobs`; failures keep inline fallback or file metadata,
- * are reported as warnings, and are counted.
+ * and completion. Export hashes each value, writes its reference (text keeps a 16 KiB prefix,
+ * binary media digest metadata on its part) and its `hue.blobs` entry, and sends the record; the
+ * upload runs afterwards in the background, as Langfuse uploads media. A value refused when its
+ * record is exported keeps its inline fallback or file metadata; an upload that fails after export
+ * leaves its entry, which Hue reads as missing. Both are reported as warnings and counted.
  */
 /** The span attribute listing the values the SDK uploaded apart from the span: one compact JSON
  * object per value, `{"key", "sha256", "size", "content_type"}`, with an optional `name`. */
@@ -23,30 +25,42 @@ export const BLOB_PREFIX_BYTES = 16 * 1024;
 export const MAX_BLOB_ENTRIES = 64;
 const MAX_BLOB_KEY_LENGTH = 256;
 /**
- * The values over the inline limit that queued spans may hold whole for upload at once, charged
- * apart from `maxQueueBytes` (a string as two bytes a code unit, bytes as themselves). A value
- * the budget cannot hold is cut when it is queued, as before uploads existed; a single value
- * larger than the whole budget is held while nothing else is.
+ * The values over the inline limit that queued records and unsettled uploads may hold whole at
+ * once, charged apart from `maxQueueBytes` (a string as two bytes a code unit, bytes as
+ * themselves): each is held until its record is exported and its upload settles. A value the
+ * budget cannot hold is cut when it is queued, as before uploads existed; a single value larger
+ * than the whole budget is held while nothing else is.
  */
 export const MAX_HELD_BLOB_BYTES = 128 * 1024 * 1024;
 /** Uploads in flight at once, across both signals' exports. */
 const UPLOAD_CONCURRENCY = 4;
-/** How long a receiver without the upload route is believed to stay without it. */
+/** How long a receiver without the upload route is believed to stay without it: an answer misread
+ * as one (a proxy's page during a deploy) turns uploads off for this long, never for good. */
 const UNSUPPORTED_PAUSE_MILLIS = 10 * 60_000;
-/** How long uploads pause after one failed for a transient reason (a 5xx, the network, a
- * timeout), so an unavailable store meets no retry storm: values fall back to the inline cut. */
-const FAILURE_PAUSE_MILLIS = 30_000;
+/** Values hashed at once for export, apart from the uploads' places: export never waits for an
+ * upload to hash. */
+const HASH_CONCURRENCY = 4;
+/** Uploads queued and not yet settled; past it a value keeps its fallback. */
+const MAX_QUEUED_UPLOADS = 4096;
+/** How many times a reservation or a PUT is tried. */
+const UPLOAD_ATTEMPTS = 3;
 const MAX_JSON_RESPONSE_BYTES = 64 * 1024;
 const MAX_ERROR_RESPONSE_BYTES = 4096;
 /** Text is encoded to UTF-8 a slice at a time to hash and upload it: no full copy is held. */
 const TEXT_CHUNK_UNITS = 1 << 20;
 const RETRY_DELAY_MILLIS = 1000;
-/** The longest `Retry-After` a transient refusal of a reservation is waited for. */
-const MAX_TRANSIENT_WAIT_MILLIS = 5000;
-/** How long one export may spend uploading its values, on top of its requests. */
+/** The longest wait before an upload's next attempt, a `Retry-After` included. */
+export const MAX_RETRY_AFTER_MILLIS = 60_000;
+/** How long one export may spend hashing its values: past it a value keeps its fallback. Also
+ * the import's least upload deadline, and part of an upload's lifetime. */
 export function uploadBudgetMillis(timeoutMillis) {
     return timeoutMillis * 6;
 }
+/** How long one upload may take from when its record's export queued it: six export timeouts for
+ * its requests and up to a minute of Retry-After of its own. */
+export function uploadLifetimeMillis(timeoutMillis) {
+    return uploadBudgetMillis(timeoutMillis) + MAX_RETRY_AFTER_MILLIS;
+}
 /** The fixed description of each fallback, as an export issue reports it. */
 export const FALLBACK_MESSAGES = {
     unsupported: "This Hue server does not accept uploaded values; large values were cut inline and media kept digest metadata",
@@ -54,6 +68,14 @@ export const FALLBACK_MESSAGES = {
     tooLarge: "Content over its upload limit kept inline fallback or file metadata",
     budget: "Content did not fit the SDK's upload budget; large values were cut inline and media kept digest metadata",
 };
+/** The fixed description of each failure of an upload that ran after its record was exported:
+ * the record lists the value under `hue.blobs`, and Hue reads it as missing. */
+export const LATE_FALLBACK_MESSAGES = {
+    unsupported: "This Hue server does not accept uploaded values; values exported before it answered list hue.blobs entries that it does not store",
+    failed: "Content could not be uploaded after its record was exported; its hue.blobs entry names a value Hue does not store",
+    tooLarge: "Content over its upload limit could not be uploaded after its record was exported; its hue.blobs entry names a value Hue does not store",
+    budget: "Content did not fit the SDK's upload budget after its record was exported; its hue.blobs entry names a value Hue does not store",
+};
 /** The `hue.blobs` key of a staged file: its part's for an attachment placed in a value, else
  * its `hue.file` event's. */
 export function mediaKey(file) {
@@ -68,6 +90,32 @@ export function countFallback(tally, reason, traceId) {
     entry.traceIds.add(traceId);
     tally.fallbacks.set(reason, entry);
 }
+/** Adds one tally's outcomes to another's. */
+export function mergeTally(into, from) {
+    into.uploaded += from.uploaded;
+    for (const [reason, { count, traceIds }] of from.fallbacks) {
+        const entry = into.fallbacks.get(reason) ?? { count: 0, traceIds: new Set() };
+        entry.count += count;
+        for (const traceId of traceIds)
+            entry.traceIds.add(traceId);
+        into.fallbacks.set(reason, entry);
+    }
+}
+/** Adds `from` to `into` once `settled` all settle. Built apart from the step that queued the
+ * uploads, so a pending upload keeps alive the two tallies alone, never that step's record. */
+export function mergeWhenSettled(into, from, settled) {
+    return Promise.all(settled).then(() => mergeTally(into, from));
+}
+/** Counts an upload's outcome in `tally` once it settles. Built apart from the placement that
+ * queued it, so the upload keeps alive its own bytes alone, never the record placed around it. */
+function countWhenSettled(done, tally, traceId) {
+    return done.then((outcome) => {
+        if (outcome.ok)
+            tally.uploaded++;
+        else
+            countFallback(tally, outcome.reason, traceId);
+    });
+}
 /** A media type accepted by the upload contract. */
 export function isBlobContentType(value) {
     return isMediaContentType(value);
@@ -109,6 +157,13 @@ export function sizeOf(value) {
 export function heldBytesOf(value) {
     return isByteSource(value) ? 0 : value.byteLength;
 }
+/** The memory a queued upload keeps alive until it settles: a string as two bytes a code unit,
+ * bytes as themselves, a source the strings it reads from, each whole. */
+function heldBy(value) {
+    if (isByteSource(value))
+        return value.held ?? [];
+    return [{ bytes: typeof value === "string" ? value.length * 2 : value.byteLength }];
+}
 /** A string's UTF-8 encoding a slice at a time, never splitting a surrogate pair. */
 function* utf8Chunks(text) {
     // One buffer, reused: a reader is done with each slice before it asks for the next.
@@ -180,26 +235,111 @@ function parseJson(body) {
 function isObject(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value);
 }
+/** Places for work of one kind: a finishing run hands its place to the next one waiting, so none
+ * is ever taken twice. */
+class Slots {
+    size;
+    running = 0;
+    waiting = [];
+    released = false;
+    constructor(size) {
+        this.size = size;
+    }
+    /** Runs `work` once fewer than `size` runs are in progress. */
+    async run(work) {
+        await this.take();
+        try {
+            return await work();
+        }
+        finally {
+            this.give();
+        }
+    }
+    /** Runs `work` (a wait) from within a run without its place, and takes a place again after:
+     * a run that waits never holds the place another could use. Once `deadline` passes it stops
+     * waiting for a place and goes on with one over the size, to end without more work. */
+    async away(work, deadline = Infinity) {
+        this.give();
+        try {
+            return await work();
+        }
+        finally {
+            await this.take(deadline);
+        }
+    }
+    async take(deadline = Infinity) {
+        if (this.running < this.size || this.released) {
+            this.running++;
+            return;
+        }
+        await new Promise((resolve) => {
+            const start = () => {
+                clearTimeout(timer);
+                resolve();
+            };
+            const timer = deadline < Infinity
+                ? setTimeout(() => {
+                    const at = this.waiting.indexOf(start);
+                    if (at >= 0)
+                        this.waiting.splice(at, 1);
+                    this.running++;
+                    resolve();
+                }, deadline - Date.now())
+                : undefined;
+            this.waiting.push(start);
+        });
+    }
+    give() {
+        // A place taken over the size, past a deadline, is given back rather than handed on.
+        const next = this.running > this.size ? undefined : this.waiting.shift();
+        if (next)
+            next();
+        else
+            this.running--;
+    }
+    /** Starts every waiting run now; none waits again. */
+    release() {
+        this.released = true;
+        for (const run of this.waiting.splice(0)) {
+            this.running++;
+            run();
+        }
+    }
+}
 /**
  * Uploads values to Hue: reserves each with `POST /api/v1/otlp/blobs`, PUTs its bytes to the
- * presigned URL Hue answers (skipped when Hue already stores the value), and completes it. At
- * most {@link UPLOAD_CONCURRENCY} uploads run at once; every request has a deadline and none
- * follows a redirect. A receiver without the route (a 404 without Hue's JSON error, a 405 or
- * 501, or a 200 that is no reservation) is remembered for {@link UNSUPPORTED_PAUSE_MILLIS}, and a
- * transient failure or a refusal that is not about the value alone pauses uploads for
- * {@link FAILURE_PAUSE_MILLIS}, so values fall back to the inline cut instead of retrying.
+ * presigned URL Hue answers (skipped when Hue already stores the value), and completes it. An
+ * export queues its values ({@link enqueue}) and sends its records without waiting for them; the
+ * import stores each value before it sends its span ({@link store}). At most
+ * {@link UPLOAD_CONCURRENCY} uploads run at once, and {@link HASH_CONCURRENCY} hashes apart from
+ * them. A reservation or PUT that gets no answer, a 408, a 429 or a 5xx is tried
+ * {@link UPLOAD_ATTEMPTS} times; every request has a deadline and none follows a redirect. A
+ * receiver without the route (a 404 without Hue's JSON error, a 405 or 501, or a 200 that is no
+ * reservation) is remembered for {@link UNSUPPORTED_PAUSE_MILLIS}: values exported meanwhile keep
+ * their fallback, never an entry the receiver cannot resolve.
  */
 export class BlobUploader {
     options;
+    /** Until when the receiver is taken to lack the upload route. */
     unsupportedUntil = 0;
-    pausedUntil = 0;
     closed = false;
-    running = 0;
-    waiting = [];
+    uploads = new Slots(UPLOAD_CONCURRENCY);
+    hashes = new Slots(HASH_CONCURRENCY);
+    queued = 0;
+    queuedBytes = 0;
+    /** What queued uploads keep alive, each charged once, from the first upload holding it until
+     * the last settles: two files read in place from one recorded string charge it once, whole. */
+    holding = new Map();
+    sleeps = new Set();
     requests = new Set();
     outcomes = new Map();
+    /** The first wait before another attempt, doubled each time; tests shorten it. */
+    retryDelayMillis = RETRY_DELAY_MILLIS;
+    /** How long a queued upload may take; tests shorten it. */
+    lifetimeMillis;
     constructor(options) {
         this.options = options;
+        this.lifetimeMillis = uploadLifetimeMillis(options.timeoutMillis);
     }
     /** Whether a value over the inline limit is worth holding whole for upload now. */
     available() {
@@ -209,35 +349,31 @@ export class BlobUploader {
     unsupported() {
         return Date.now() < this.unsupportedUntil;
     }
-    /** Ends uploads in flight and refuses new ones: their values fall back to the inline cut. */
+    /** Ends uploads in flight, queued uploads and their waits, and refuses new ones: values not
+     * yet exported fall back to the inline cut, and uploads queued by exported records fail. */
     close() {
         this.closed = true;
         for (const request of this.requests)
             request.destroy();
         this.requests.clear();
         this.outcomes.clear();
-        for (const run of this.waiting.splice(0)) {
-            this.running++;
-            run();
-        }
+        for (const cancel of [...this.sleeps])
+            cancel();
+        this.uploads.release();
+        this.hashes.release();
     }
     /** Runs `work` once fewer than {@link UPLOAD_CONCURRENCY} uploads are in flight. */
-    async limit(work) {
-        // A finishing upload hands its place to the next one waiting, so none is ever taken twice.
-        if (this.running >= UPLOAD_CONCURRENCY && !this.closed)
-            await new Promise((resolve) => this.waiting.push(resolve));
-        else
-            this.running++;
-        try {
-            return await work();
-        }
-        finally {
-            const next = this.waiting.shift();
-            if (next)
-                next();
-            else
-                this.running--;
-        }
+    limit(work) {
+        return this.uploads.run(work);
+    }
+    /** The SHA-256 and size of `value` (see {@link digestOf}), hashed in a place of its own: an
+     * export never waits for an upload to hash its values. */
+    digest(value, deadline) {
+        return this.hashes.run(() => digestOf(value, deadline));
+    }
+    /** The bytes queued uploads keep alive until they settle. */
+    pendingBytes() {
+        return this.queuedBytes;
     }
     /** Why a value cannot be uploaded right now without trying, if it cannot. */
     refusal() {
@@ -245,21 +381,10 @@ export class BlobUploader {
             return "failed";
         if (this.unsupported())
             return "unsupported";
-        if (Date.now() < this.pausedUntil)
-            return "failed";
         return undefined;
     }
-    pause() {
-        this.pausedUntil = Date.now() + FAILURE_PAUSE_MILLIS;
-    }
-    /**
-     * Stores `value` under `key` of the trace: reserved, uploaded unless Hue already holds it, and
-     * completed. Only a fresh upload takes a {@link limit} slot: another reference to bytes the
-     * trace is already storing shares that upload's outcome without one, so repeated files never
-     * hold the slots new files need. The caller must not hold a slot.
-     */
-    async store(traceId, key, value, contentType, digest, upload) {
-        // Every occurrence is validated even when a previous occurrence already uploaded the bytes.
+    /** Why the value cannot be uploaded at all, whatever Hue would answer. */
+    invalid(traceId, key, value, contentType, digest) {
         if (!isBlobKey(key) ||
             !isBlobContentType(contentType) ||
             !/^[0-9a-f]{32}$/.test(traceId) ||
@@ -268,9 +393,95 @@ export class BlobUploader {
             !Number.isSafeInteger(digest.size) ||
             digest.size <= 0 ||
             digest.size !== sizeOf(value))
-            return { ok: false, reason: "failed" };
+            return "failed";
         if (digest.size > MAX_BLOB_BYTES)
-            return { ok: false, reason: "tooLarge" };
+            return "tooLarge";
+        return undefined;
+    }
+    /**
+     * Queues `value`'s upload for an exported record, deciding now what the record lists: a value
+     * refused without trying (see {@link refusal}), invalid, or past the queue's
+     * {@link MAX_QUEUED_UPLOADS} uploads or {@link MAX_HELD_BLOB_BYTES} keeps its fallback. A queued
+     * upload runs within {@link uploadLifetimeMillis} of now, holding `value` until it settles;
+     * `done` never rejects. `bytes` is what it adds to {@link pendingBytes}: the strings it keeps
+     * alive that no other queued upload holds, charged until `released`, once the last upload
+     * holding them settles.
+     */
+    enqueue(traceId, key, value, contentType, digest) {
+        const refused = this.refusal() ?? this.invalid(traceId, key, value, contentType, digest);
+        if (refused)
+            return { ok: false, reason: refused };
+        const held = heldBy(value);
+        const fresh = held.filter((text) => !this.holding.has(text));
+        const bytes = fresh.reduce((total, text) => total + text.bytes, 0);
+        // An upload whose strings are all held already adds no memory, so only the job cap applies.
+        if (this.queued >= MAX_QUEUED_UPLOADS ||
+            (bytes > 0 && this.queuedBytes > 0 && this.queuedBytes + bytes > MAX_HELD_BLOB_BYTES))
+            return { ok: false, reason: "budget" };
+        this.queued++;
+        this.queuedBytes += bytes;
+        const released = fresh.map((text) => new Promise((release) => {
+            this.holding.set(text, { uploads: 0, release });
+        }));
+        for (const text of held)
+            this.holding.get(text).uploads++;
+        const upload = {
+            deadline: Date.now() + this.lifetimeMillis,
+            wait: (millis) => this.sleep(millis),
+        };
+        // store() never rejects; the job holds `value` only until it settles.
+        const done = this.store(traceId, key, value, contentType, digest, upload).finally(() => {
+            this.queued--;
+            for (const text of held) {
+                const holding = this.holding.get(text);
+                if (--holding.uploads > 0)
+                    continue;
+                this.holding.delete(text);
+                this.queuedBytes -= text.bytes;
+                holding.release();
+            }
+        });
+        return { ok: true, done, bytes, released: Promise.all(released).then(() => { }) };
+    }
+    /** Waits `millis`; false when {@link close} ended the wait first. The timer keeps the process
+     * running, as an export in progress does, within the upload's lifetime. */
+    sleep(millis) {
+        if (this.closed)
+            return Promise.resolve(false);
+        return new Promise((resolve) => {
+            const done = (ran) => {
+                clearTimeout(timer);
+                this.sleeps.delete(cancel);
+                resolve(ran && !this.closed);
+            };
+            const cancel = () => done(false);
+            const timer = setTimeout(() => done(true), millis);
+            this.sleeps.add(cancel);
+        });
+    }
+    /** Waits before attempt `attempt + 1`: `Retry-After` when the answer gave one, otherwise a
+     * second doubled each attempt with jitter, at most a minute. The upload's place is given up
+     * meanwhile, so uploads queued behind it run. False when the wait would pass the upload's
+     * deadline, was ended, or no place came free again before the deadline. */
+    async retryWait(attempt, retryAfter, upload) {
+        const millis = Math.min(retryAfter ?? this.retryDelayMillis * (2 ** attempt + Math.random()), MAX_RETRY_AFTER_MILLIS);
+        if (this.closed || Date.now() + millis >= upload.deadline)
+            return false;
+        return ((await this.uploads.away(() => upload.wait(millis), upload.deadline)) &&
+            !this.closed &&
+            Date.now() < upload.deadline);
+    }
+    /**
+     * Stores `value` under `key` of the trace: reserved, uploaded unless Hue already holds it, and
+     * completed. Only a fresh upload takes a {@link limit} slot: another reference to bytes the
+     * trace is already storing shares that upload's outcome without one, so repeated files never
+     * hold the slots new files need. The caller must not hold a slot. Never rejects.
+     */
+    async store(traceId, key, value, contentType, digest, upload) {
+        // Every occurrence is validated even when a previous occurrence already uploaded the bytes.
+        const invalid = this.invalid(traceId, key, value, contentType, digest);
+        if (invalid)
+            return { ok: false, reason: invalid };
         const identity = `${traceId}:${digest.sha256}`;
         for (let known = this.outcomes.get(identity); known; known = this.outcomes.get(identity)) {
             const shared = await known.promise;
@@ -303,6 +514,9 @@ export class BlobUploader {
         const refused = this.refusal();
         if (refused)
             return { ok: false, reason: refused };
+        // Queued past its lifetime: not tried. Failed, not budget: the import holds its span back.
+        if (Date.now() >= upload.deadline)
+            return { ok: false, reason: "failed" };
         if (digest.size > MAX_BLOB_BYTES)
             return { ok: false, reason: "tooLarge" };
         if (!isBlobKey(key) ||
@@ -319,8 +533,7 @@ export class BlobUploader {
         });
         let reservation = await this.reserve(body, digest.size, upload);
         let presigns = 1;
-        let attempts = 0;
-        while (!("fallback" in reservation) && !reservation.exists) {
+        for (let attempt = 0; !("fallback" in reservation) && !reservation.exists;) {
             const { grant } = reservation;
             const put = await this.put(grant, value, digest.size, upload.deadline);
             if (put === "stored")
@@ -329,19 +542,13 @@ export class BlobUploader {
                 return { ok: false, reason: "failed", changed: true };
             if (this.closed)
                 return { ok: false, reason: "failed" };
-            const expired = put === "expired" || Date.now() >= grant.expiresAt;
-            if (put === "transient" &&
-                ++attempts < 2 &&
-                Date.now() + RETRY_DELAY_MILLIS < upload.deadline) {
-                await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MILLIS));
+            if (put === "transient") {
+                if (++attempt >= UPLOAD_ATTEMPTS || !(await this.retryWait(attempt - 1, undefined, upload)))
+                    return { ok: false, reason: "failed" };
                 if (Date.now() < grant.expiresAt)
                     continue;
             }
-            else if (put === "transient") {
-                this.pause();
-                return { ok: false, reason: "failed" };
-            }
-            else if (!expired)
+            else if (put !== "expired" && Date.now() < grant.expiresAt)
                 return { ok: false, reason: "failed" };
             // The URL expired: reserved again once.
             if (presigns++ >= 2)
@@ -350,13 +557,10 @@ export class BlobUploader {
         }
         if ("fallback" in reservation)
             return { ok: false, reason: reservation.fallback };
-        if (!reservation.exists) {
-            // For Hue's accounting only: a value is read whether or not it was completed, so a failed
-            // completion is not retried.
-            const completion = await this.request("POST", new URL("/api/v1/otlp/blobs/complete", this.options.baseUrl), this.jsonHeaders(), JSON.stringify({ traceId, sha256: digest.sha256 }), Math.min(upload.deadline, Date.now() + this.options.timeoutMillis), MAX_JSON_RESPONSE_BYTES);
-            if (!completion || completion.status >= 500 || completion.status === 429)
-                this.pause();
-        }
+        if (!reservation.exists)
+            // For Hue's accounting only: a value is read whether or not it was completed, and Hue
+            // stamps a missed completion on the next reservation, so its outcome is not read.
+            await this.request("POST", new URL("/api/v1/otlp/blobs/complete", this.options.baseUrl), this.jsonHeaders(), JSON.stringify({ traceId, sha256: digest.sha256 }), Math.min(upload.deadline, Date.now() + this.options.timeoutMillis), MAX_JSON_RESPONSE_BYTES);
         return { ok: true, contentType: reservation.contentType };
     }
     jsonHeaders() {
@@ -366,71 +570,46 @@ export class BlobUploader {
             "User-Agent": this.options.userAgent,
         };
     }
-    /** Reserves the value: Hue answers that it holds it already, or a presigned PUT. A rate limit
-     * is waited out within the export's hold, a transient refusal retried once. */
+    /** Reserves the value: Hue answers that it holds it already, or a presigned PUT. No answer, a
+     * 408, a 429 or a 5xx other than 501 is tried again, after its `Retry-After` when it gave one,
+     * within the upload's deadline. */
     async reserve(body, size, upload) {
         const url = new URL("/api/v1/otlp/blobs", this.options.baseUrl);
-        for (let attempt = 0; attempt < 3; attempt++) {
+        for (let attempt = 0;; attempt++) {
             if (this.closed)
                 return { fallback: "failed" };
             const reply = await this.request("POST", url, this.jsonHeaders(), body, Math.min(upload.deadline, Date.now() + this.options.timeoutMillis), MAX_JSON_RESPONSE_BYTES);
-            if (!reply) {
-                if (attempt === 0 && (await this.delay(RETRY_DELAY_MILLIS, upload)))
-                    continue;
-                this.pause();
-                return { fallback: "failed" };
-            }
-            const { status } = reply;
-            const answer = status === 200 || status === 404 || status === 409 ? parseJson(reply.body) : undefined;
-            // A receiver without the route answers its framework's not-found (Hue's own 404, an
-            // archived project, is JSON with an `error`), refuses the method, or answers something
-            // that is no reservation at all: it is not asked again for a while.
-            if (status === 405 ||
-                status === 501 ||
-                (status === 404 && !(isObject(answer) && typeof answer.error === "string")) ||
-                (status === 200 &&
-                    !(isObject(answer) && (answer.status === "exists" || answer.status === "upload")))) {
-                this.unsupportedUntil = Date.now() + UNSUPPORTED_PAUSE_MILLIS;
-                return { fallback: "unsupported" };
-            }
-            if (status === 200) {
-                const reservation = this.reservation(answer, size);
-                // A reservation that cannot be used would fail every value alike.
-                if ("fallback" in reservation)
-                    this.pause();
-                return reservation;
-            }
-            if (status === 409 && isObject(answer) && answer.status === "exists")
-                return this.reservation(answer, size);
-            if (status === 413)
-                return { fallback: "tooLarge" };
-            if (status === 429) {
-                const wait = retryAfterMillis(reply.headers) ?? RETRY_DELAY_MILLIS;
-                if (Date.now() + wait < upload.deadline && (await upload.wait(wait)))
-                    continue;
-                this.pause();
-                return { fallback: "failed" };
+            let retryAfter;
+            if (reply) {
+                const { status } = reply;
+                const answer = status === 200 || status === 404 || status === 409 ? parseJson(reply.body) : undefined;
+                // A receiver without the route answers its framework's not-found (Hue's own 404, an
+                // archived project, is JSON with an `error`), refuses the method, or answers something
+                // that is no reservation at all: it is not asked again for a while, since the values
+                // exported while it answers are listed for bytes it cannot store.
+                if (status === 405 ||
+                    status === 501 ||
+                    (status === 404 && !(isObject(answer) && typeof answer.error === "string")) ||
+                    (status === 200 &&
+                        !(isObject(answer) && (answer.status === "exists" || answer.status === "upload")))) {
+                    this.unsupportedUntil = Date.now() + UNSUPPORTED_PAUSE_MILLIS;
+                    return { fallback: "unsupported" };
+                }
+                // A reservation that cannot be used fails its value.
+                if (status === 200)
+                    return this.reservation(answer, size);
+                if (status === 409 && isObject(answer) && answer.status === "exists")
+                    return this.reservation(answer, size);
+                if (status === 413)
+                    return { fallback: "tooLarge" };
+                // Refusals of the value or the key (400, 401, 403, Hue's own 404, a conflict) are final.
+                if (status !== 408 && status !== 429 && status < 500)
+                    return { fallback: "failed" };
+                retryAfter = retryAfterMillis(reply.headers);
             }
-            if (status >= 500) {
-                const wait = retryAfterMillis(reply.headers) ?? RETRY_DELAY_MILLIS;
-                if (attempt === 0 && wait <= MAX_TRANSIENT_WAIT_MILLIS && (await this.delay(wait, upload)))
-                    continue;
-                this.pause();
+            if (attempt + 1 >= UPLOAD_ATTEMPTS || !(await this.retryWait(attempt, retryAfter, upload)))
                 return { fallback: "failed" };
-            }
-            // Bad values and conflicts refuse only this value. Transient service failures pause uploads.
-            if (status !== 400 && status !== 409)
-                this.pause();
-            return { fallback: "failed" };
         }
-        this.pause();
-        return { fallback: "failed" };
-    }
-    async delay(millis, upload) {
-        if (Date.now() + millis >= upload.deadline)
-            return false;
-        await new Promise((resolve) => setTimeout(resolve, millis));
-        return !this.closed;
     }
     /** A 200 answer read: `exists`, or an upload whose URL and headers are usable as given. */
     reservation(answer, size) {
@@ -626,9 +805,9 @@ export class BlobUploader {
 }
 /**
  * A span's own attributes with each candidate placed: binary media becomes a digest, while
- * text media over 64 KiB retains a 16 KiB prefix after upload or a digest on failure.
- * A value still over the inline limit is uploaded and replaced by its first
- * 16 KiB (or cut to the limit), each upload listed under `hue.blobs`, each whole value cut or
+ * text media over 64 KiB retains a 16 KiB prefix once uploaded or queued, or a digest when
+ * refused. A value still over the inline limit is uploaded and replaced by its first 16 KiB (or
+ * cut to the limit when refused), each upload listed under `hue.blobs`, each whole value cut or
  * uploaded listed under `hue.truncated`. Every fallback is counted in the tally.
  */
 export async function offloadAttributes(attributes, collector, options) {
@@ -636,12 +815,12 @@ export async function offloadAttributes(attributes, collector, options) {
 }
 /**
  * A record's own attributes, and a log record's body, with each candidate placed (see
- * {@link offloadAttributes}). A file is uploaded and placed in its digest form (a text file in a
- * provider part keeps a 16 KiB prefix), its `hue.blobs` entry keyed `<attribute>#<pointer>` or, in
- * a log record's body, `body#<pointer>`. A file that is not uploaded keeps its bytes inline, except
- * in a span's own recorded messages, where it keeps its digest as before. A log record's values
- * over the inline limit are cut, never uploaded whole, and a body still over it is replaced by the
- * receiver's marker, as before.
+ * {@link offloadAttributes}). A file is uploaded (or, for an export, queued) and placed in its
+ * digest form (a text file in a provider part keeps a 16 KiB prefix), its `hue.blobs` entry keyed
+ * `<attribute>#<pointer>` or, in a log record's body, `body#<pointer>`. A file refused or not
+ * uploaded before it is placed keeps its bytes inline, except in a span's own recorded messages,
+ * where it keeps its digest as before. A log record's values over the inline limit are cut, never
+ * uploaded whole, and a body still over it is replaced by the receiver's marker, as before.
  */
 export async function offloadRecord(attributes, collector, options) {
     const { uploader, traceId, valueBytes, upload, tally } = options;
@@ -655,14 +834,33 @@ export async function offloadRecord(attributes, collector, options) {
     const entries = [];
     const listed = [];
     const fallback = (reason) => countFallback(tally, reason, traceId);
+    /** Stores a value (the import), or queues its upload (an export): what the record lists. A
+     * queued value's entry names the type its own reference declared. `shared` when the value is
+     * memory its record holds (not decoded or rewritten from it). */
+    const submit = async (key, value, contentType, digest, shared) => {
+        const late = options.background;
+        if (!late) {
+            const outcome = await uploader.store(traceId, key, value, contentType, digest, upload);
+            if (outcome.ok)
+                tally.uploaded++;
+            return outcome;
+        }
+        const queued = uploader.enqueue(traceId, key, value, contentType, digest);
+        if (!queued.ok)
+            return queued;
+        const settled = countWhenSettled(queued.done, late.tally, traceId);
+        late.settled.push(settled);
+        late.queued?.(settled, shared ? queued.bytes : 0, queued.released);
+        return { ok: true, contentType };
+    };
     // A value the queue cut when it was admitted was never held whole to upload.
     for (let index = 0; index < collector.cut.length; index++)
         fallback(uploader.unsupported() ? "unsupported" : "budget");
     let inspected = 0;
-    /** Uploads each file of a value and places it; whether any was placed. A file that is not
-     * uploaded keeps its bytes inline, except in a recorded message: a span's own message keeps its
-     * digest, and a log record's keeps the digest of a file over the inline limit, as nested
-     * messages kept them before log records uploaded. */
+    /** Uploads (or queues) each file of a value and places it; whether any was placed. A file
+     * refused or not uploaded keeps its bytes inline, except in a recorded message: a span's own
+     * message keeps its digest, and a log record's keeps the digest of a file over the inline limit,
+     * as nested messages kept them before log records uploaded. */
     const placeFiles = async (index, key, files, long) => {
         const message = isMessageKey(key);
         let changed = false;
@@ -683,12 +881,12 @@ export async function offloadRecord(attributes, collector, options) {
             const partKey = `${key}#${prefix ? file.pointer : file.partPointer}`;
             const slot = slots > 0 && isBlobKey(partKey) ? (slots--, true) : false;
             return (async () => {
-                // Hashing a long file takes an upload place, as hashing a large value does; one the
-                // export's upload budget cannot hash stays as recorded and is not uploaded.
+                // A long file is hashed in a hashing place, as a large value is; one the export's
+                // hashing deadline cannot hash stays as recorded and is not uploaded.
                 let digest;
                 try {
                     digest = isByteSource(bytes)
-                        ? await uploader.limit(() => digestOf(bytes, upload.deadline))
+                        ? await uploader.digest(bytes, upload.deadline)
                         : options.memo
                             ? options.memo.digest(file, decoded, traceId)
                             : await digestOf(bytes);
@@ -698,7 +896,7 @@ export async function offloadRecord(attributes, collector, options) {
                     return;
                 }
                 const outcome = slot
-                    ? await uploader.store(traceId, partKey, bytes, contentType, digest, upload)
+                    ? await submit(partKey, bytes, contentType, digest, isByteSource(bytes))
                     : { ok: false, reason: "budget" };
                 if (outcome.ok) {
                     if (prefix)
@@ -710,7 +908,6 @@ export async function offloadRecord(attributes, collector, options) {
                         order: [index, position],
                         entry: blobEntry(partKey, digest.sha256, digest.size, outcome.contentType, file.name),
                     });
-                    tally.uploaded++;
                 }
                 else {
                     if (message && (!collector.log || size > INLINE_FILE_LIMIT)) {
@@ -833,10 +1030,10 @@ export async function offloadRecord(attributes, collector, options) {
         const stored = await (async () => {
             if (size > MAX_BLOB_BYTES)
                 return { ok: false, reason: "tooLarge" };
-            // Hashing a large value takes a slot as its upload does; the upload takes its own.
+            // A large value is hashed in a hashing place; its upload takes an upload place.
             let digest;
             try {
-                digest = await uploader.limit(() => digestOf(placed, upload.deadline));
+                digest = await uploader.digest(placed, upload.deadline);
             }
             catch {
                 return { ok: false, reason: "failed" };
@@ -846,7 +1043,7 @@ export async function offloadRecord(attributes, collector, options) {
                 : typeof placed === "string"
                     ? textContentType(placed)
                     : "application/octet-stream";
-            const outcome = await uploader.store(traceId, key, placed, contentType, digest, upload);
+            const outcome = await submit(key, placed, contentType, digest, !!joined || read === value);
             return outcome.ok ? { ...outcome, digest } : outcome;
         })();
         if (stored.ok) {
@@ -859,7 +1056,6 @@ export async function offloadRecord(attributes, collector, options) {
                 order: [index, Infinity],
                 entry: blobEntry(key, stored.digest.sha256, stored.digest.size, stored.contentType),
             });
-            tally.uploaded++;
         }
         else {
             result[key] = cut();
@@ -875,14 +1071,13 @@ export async function offloadRecord(attributes, collector, options) {
         return (async () => {
             const key = mediaKey(file);
             const outcome = slot
-                ? await uploader.store(traceId, key, file.bytes, file.contentType, file.digest, upload)
+                ? await submit(key, file.bytes, file.contentType, file.digest, true)
                 : { ok: false, reason: "budget" };
             if (outcome.ok) {
                 entries.push({
                     order: [collector.candidates.length, position],
                     entry: blobEntry(key, file.digest.sha256, file.digest.size, outcome.contentType, file.name),
                 });
-                tally.uploaded++;
             }
             else
                 fallback(outcome.reason);