@hue-run/sdk 0.18.1 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/blobs.js CHANGED
@@ -3,13 +3,16 @@ import { context } from "@opentelemetry/api";
3
3
  import { suppressTracing } from "@opentelemetry/core";
4
4
  import { isLoopbackHost } from "./config.js";
5
5
  import { digestPart, hashInlineFiles, hashNestedInlineFiles, INLINE_FILE_LIMIT, INLINE_FILE_TEXT_PER_RECORD, isMessageKey, keepsPrefix, literalKeys, mediaBytes, mediaContentType, mediaIn, isTextMediaType, mayInlineFiles, isMediaContentType, MAX_INLINE_FILE_TEXT, placeable, placedMediaText, prefixPart, readMediaText, reencodeMedia, } from "./inline-files.js";
6
+ import { isByteSource, JoinedText, longFileBytes, longHead, placedLargeText, readLargeMediaText, } from "./large-media.js";
6
7
  import { capLogBody, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys } from "./privacy.js";
7
8
  import { truncateUtf8 } from "./safety.js";
8
9
  /**
9
10
  * Large values and binary media travel apart from their span through reservation, presigned PUT,
10
- * and completion. Text keeps a 16 KiB prefix. Binary media keeps digest metadata on its part.
11
- * Successful uploads are listed in `hue.blobs`; failures keep inline fallback or file metadata,
12
- * are reported as warnings, and are counted.
11
+ * and completion. Export hashes each value, writes its reference (text keeps a 16 KiB prefix,
12
+ * binary media digest metadata on its part) and its `hue.blobs` entry, and sends the record; the
13
+ * upload runs afterwards in the background, as Langfuse uploads media. A value refused when its
14
+ * record is exported keeps its inline fallback or file metadata; an upload that fails after export
15
+ * leaves its entry, which Hue reads as missing. Both are reported as warnings and counted.
13
16
  */
14
17
  /** The span attribute listing the values the SDK uploaded apart from the span: one compact JSON
15
18
  * object per value, `{"key", "sha256", "size", "content_type"}`, with an optional `name`. */
@@ -22,30 +25,42 @@ export const BLOB_PREFIX_BYTES = 16 * 1024;
22
25
  export const MAX_BLOB_ENTRIES = 64;
23
26
  const MAX_BLOB_KEY_LENGTH = 256;
24
27
  /**
25
- * The values over the inline limit that queued spans may hold whole for upload at once, charged
26
- * apart from `maxQueueBytes` (a string as two bytes a code unit, bytes as themselves). A value
27
- * the budget cannot hold is cut when it is queued, as before uploads existed; a single value
28
- * larger than the whole budget is held while nothing else is.
28
+ * The values over the inline limit that queued records and unsettled uploads may hold whole at
29
+ * once, charged apart from `maxQueueBytes` (a string as two bytes a code unit, bytes as
30
+ * themselves): each is held until its record is exported and its upload settles. A value the
31
+ * budget cannot hold is cut when it is queued, as before uploads existed; a single value larger
32
+ * than the whole budget is held while nothing else is.
29
33
  */
30
34
  export const MAX_HELD_BLOB_BYTES = 128 * 1024 * 1024;
31
35
  /** Uploads in flight at once, across both signals' exports. */
32
36
  const UPLOAD_CONCURRENCY = 4;
33
- /** How long a receiver without the upload route is believed to stay without it. */
37
+ /** How long a receiver without the upload route is believed to stay without it: an answer misread
38
+ * as one (a proxy's page during a deploy) turns uploads off for this long, never for good. */
34
39
  const UNSUPPORTED_PAUSE_MILLIS = 10 * 60_000;
35
- /** How long uploads pause after one failed for a transient reason (a 5xx, the network, a
36
- * timeout), so an unavailable store meets no retry storm: values fall back to the inline cut. */
37
- const FAILURE_PAUSE_MILLIS = 30_000;
40
+ /** Values hashed at once for export, apart from the uploads' places: export never waits for an
41
+ * upload to hash. */
42
+ const HASH_CONCURRENCY = 4;
43
+ /** Uploads queued and not yet settled; past it a value keeps its fallback. */
44
+ const MAX_QUEUED_UPLOADS = 4096;
45
+ /** How many times a reservation or a PUT is tried. */
46
+ const UPLOAD_ATTEMPTS = 3;
38
47
  const MAX_JSON_RESPONSE_BYTES = 64 * 1024;
39
48
  const MAX_ERROR_RESPONSE_BYTES = 4096;
40
49
  /** Text is encoded to UTF-8 a slice at a time to hash and upload it: no full copy is held. */
41
50
  const TEXT_CHUNK_UNITS = 1 << 20;
42
51
  const RETRY_DELAY_MILLIS = 1000;
43
- /** The longest `Retry-After` a transient refusal of a reservation is waited for. */
44
- const MAX_TRANSIENT_WAIT_MILLIS = 5000;
45
- /** How long one export may spend uploading its values, on top of its requests. */
52
+ /** The longest wait before an upload's next attempt, a `Retry-After` included. */
53
+ export const MAX_RETRY_AFTER_MILLIS = 60_000;
54
+ /** How long one export may spend hashing its values: past it a value keeps its fallback. Also
55
+ * the import's least upload deadline, and part of an upload's lifetime. */
46
56
  export function uploadBudgetMillis(timeoutMillis) {
47
57
  return timeoutMillis * 6;
48
58
  }
59
+ /** How long one upload may take from when its record's export queued it: six export timeouts for
60
+ * its requests and up to a minute of Retry-After of its own. */
61
+ export function uploadLifetimeMillis(timeoutMillis) {
62
+ return uploadBudgetMillis(timeoutMillis) + MAX_RETRY_AFTER_MILLIS;
63
+ }
49
64
  /** The fixed description of each fallback, as an export issue reports it. */
50
65
  export const FALLBACK_MESSAGES = {
51
66
  unsupported: "This Hue server does not accept uploaded values; large values were cut inline and media kept digest metadata",
@@ -53,6 +68,14 @@ export const FALLBACK_MESSAGES = {
53
68
  tooLarge: "Content over its upload limit kept inline fallback or file metadata",
54
69
  budget: "Content did not fit the SDK's upload budget; large values were cut inline and media kept digest metadata",
55
70
  };
71
+ /** The fixed description of each failure of an upload that ran after its record was exported:
72
+ * the record lists the value under `hue.blobs`, and Hue reads it as missing. */
73
+ export const LATE_FALLBACK_MESSAGES = {
74
+ unsupported: "This Hue server does not accept uploaded values; values exported before it answered list hue.blobs entries that it does not store",
75
+ failed: "Content could not be uploaded after its record was exported; its hue.blobs entry names a value Hue does not store",
76
+ tooLarge: "Content over its upload limit could not be uploaded after its record was exported; its hue.blobs entry names a value Hue does not store",
77
+ budget: "Content did not fit the SDK's upload budget after its record was exported; its hue.blobs entry names a value Hue does not store",
78
+ };
56
79
  /** The `hue.blobs` key of a staged file: its part's for an attachment placed in a value, else
57
80
  * its `hue.file` event's. */
58
81
  export function mediaKey(file) {
@@ -67,6 +90,32 @@ export function countFallback(tally, reason, traceId) {
67
90
  entry.traceIds.add(traceId);
68
91
  tally.fallbacks.set(reason, entry);
69
92
  }
93
+ /** Adds one tally's outcomes to another's. */
94
+ export function mergeTally(into, from) {
95
+ into.uploaded += from.uploaded;
96
+ for (const [reason, { count, traceIds }] of from.fallbacks) {
97
+ const entry = into.fallbacks.get(reason) ?? { count: 0, traceIds: new Set() };
98
+ entry.count += count;
99
+ for (const traceId of traceIds)
100
+ entry.traceIds.add(traceId);
101
+ into.fallbacks.set(reason, entry);
102
+ }
103
+ }
104
+ /** Adds `from` to `into` once `settled` all settle. Built apart from the step that queued the
105
+ * uploads, so a pending upload keeps alive the two tallies alone, never that step's record. */
106
+ export function mergeWhenSettled(into, from, settled) {
107
+ return Promise.all(settled).then(() => mergeTally(into, from));
108
+ }
109
+ /** Counts an upload's outcome in `tally` once it settles. Built apart from the placement that
110
+ * queued it, so the upload keeps alive its own bytes alone, never the record placed around it. */
111
+ function countWhenSettled(done, tally, traceId) {
112
+ return done.then((outcome) => {
113
+ if (outcome.ok)
114
+ tally.uploaded++;
115
+ else
116
+ countFallback(tally, outcome.reason, traceId);
117
+ });
118
+ }
70
119
  /** A media type accepted by the upload contract. */
71
120
  export function isBlobContentType(value) {
72
121
  return isMediaContentType(value);
@@ -99,40 +148,60 @@ export function blobPrefix(value) {
99
148
  return Buffer.from(truncateUtf8(value, BLOB_PREFIX_BYTES), "utf8").toString("utf8");
100
149
  }
101
150
  /** A value's size in bytes as it is uploaded: a string's UTF-8 encoding, counted, not made. */
102
- function sizeOf(value) {
103
- return typeof value === "string" ? Buffer.byteLength(value, "utf8") : value.byteLength;
151
+ export function sizeOf(value) {
152
+ if (typeof value === "string")
153
+ return Buffer.byteLength(value, "utf8");
154
+ return isByteSource(value) ? value.size : value.byteLength;
155
+ }
156
+ /** The bytes a value holds in memory: none for a source read as it is uploaded. */
157
+ export function heldBytesOf(value) {
158
+ return isByteSource(value) ? 0 : value.byteLength;
159
+ }
160
+ /** The memory a queued upload keeps alive until it settles: a string as two bytes a code unit,
161
+ * bytes as themselves, a source the strings it reads from, each whole. */
162
+ function heldBy(value) {
163
+ if (isByteSource(value))
164
+ return value.held ?? [];
165
+ return [{ bytes: typeof value === "string" ? value.length * 2 : value.byteLength }];
104
166
  }
105
167
  /** A string's UTF-8 encoding a slice at a time, never splitting a surrogate pair. */
106
168
  function* utf8Chunks(text) {
169
+ // One buffer, reused: a reader is done with each slice before it asks for the next.
170
+ const buffer = Buffer.allocUnsafe(TEXT_CHUNK_UNITS * 3);
107
171
  for (let start = 0; start < text.length;) {
108
172
  let end = Math.min(text.length, start + TEXT_CHUNK_UNITS);
109
173
  if (end < text.length && /[\uD800-\uDBFF]/.test(text[end - 1]))
110
174
  end--;
111
- yield Buffer.from(text.slice(start, end), "utf8");
175
+ yield buffer.subarray(0, buffer.write(text.slice(start, end), "utf8"));
112
176
  start = end;
113
177
  }
114
178
  }
115
179
  /** The bytes an upload carries, a slice at a time: no second copy of a large value is made. */
116
- function* uploadChunks(value) {
117
- if (typeof value === "string") {
118
- yield* utf8Chunks(value);
119
- return;
120
- }
121
- for (let start = 0; start < value.byteLength; start += TEXT_CHUNK_UNITS)
122
- yield value.subarray(start, Math.min(value.byteLength, start + TEXT_CHUNK_UNITS));
180
+ function uploadChunks(value) {
181
+ if (isByteSource(value))
182
+ return value.chunks();
183
+ return (function* () {
184
+ if (typeof value === "string") {
185
+ yield* utf8Chunks(value);
186
+ return;
187
+ }
188
+ for (let start = 0; start < value.byteLength; start += TEXT_CHUNK_UNITS)
189
+ yield value.subarray(start, Math.min(value.byteLength, start + TEXT_CHUNK_UNITS));
190
+ })();
123
191
  }
124
192
  const nextTurn = () => new Promise((resolve) => setImmediate(resolve));
125
193
  /** The SHA-256 (lowercase hex) and byte size of what an upload of `value` carries, yielding to
126
- * the event loop between slices so a large value does not stall the application. */
127
- export async function digestOf(value) {
194
+ * the event loop between slices so a large value does not stall the application. Past
195
+ * `deadline`, it throws rather than read on. */
196
+ export async function digestOf(value, deadline = Infinity) {
128
197
  const hash = createHash("sha256");
129
198
  let size = 0;
130
- let slices = 0;
131
- for (const chunk of uploadChunks(value)) {
199
+ for await (const chunk of uploadChunks(value)) {
132
200
  hash.update(chunk);
133
201
  size += chunk.byteLength;
134
- if (++slices % 4 === 0)
135
- await nextTurn();
202
+ await nextTurn();
203
+ if (Date.now() >= deadline)
204
+ throw new Error("The upload ran out of time");
136
205
  }
137
206
  return { sha256: hash.digest("hex"), size };
138
207
  }
@@ -166,26 +235,111 @@ function parseJson(body) {
166
235
  function isObject(value) {
167
236
  return value !== null && typeof value === "object" && !Array.isArray(value);
168
237
  }
238
+ /** Places for work of one kind: a finishing run hands its place to the next one waiting, so none
239
+ * is ever taken twice. */
240
+ class Slots {
241
+ size;
242
+ running = 0;
243
+ waiting = [];
244
+ released = false;
245
+ constructor(size) {
246
+ this.size = size;
247
+ }
248
+ /** Runs `work` once fewer than `size` runs are in progress. */
249
+ async run(work) {
250
+ await this.take();
251
+ try {
252
+ return await work();
253
+ }
254
+ finally {
255
+ this.give();
256
+ }
257
+ }
258
+ /** Runs `work` (a wait) from within a run without its place, and takes a place again after:
259
+ * a run that waits never holds the place another could use. Once `deadline` passes it stops
260
+ * waiting for a place and goes on with one over the size, to end without more work. */
261
+ async away(work, deadline = Infinity) {
262
+ this.give();
263
+ try {
264
+ return await work();
265
+ }
266
+ finally {
267
+ await this.take(deadline);
268
+ }
269
+ }
270
+ async take(deadline = Infinity) {
271
+ if (this.running < this.size || this.released) {
272
+ this.running++;
273
+ return;
274
+ }
275
+ await new Promise((resolve) => {
276
+ const start = () => {
277
+ clearTimeout(timer);
278
+ resolve();
279
+ };
280
+ const timer = deadline < Infinity
281
+ ? setTimeout(() => {
282
+ const at = this.waiting.indexOf(start);
283
+ if (at >= 0)
284
+ this.waiting.splice(at, 1);
285
+ this.running++;
286
+ resolve();
287
+ }, deadline - Date.now())
288
+ : undefined;
289
+ this.waiting.push(start);
290
+ });
291
+ }
292
+ give() {
293
+ // A place taken over the size, past a deadline, is given back rather than handed on.
294
+ const next = this.running > this.size ? undefined : this.waiting.shift();
295
+ if (next)
296
+ next();
297
+ else
298
+ this.running--;
299
+ }
300
+ /** Starts every waiting run now; none waits again. */
301
+ release() {
302
+ this.released = true;
303
+ for (const run of this.waiting.splice(0)) {
304
+ this.running++;
305
+ run();
306
+ }
307
+ }
308
+ }
169
309
  /**
170
310
  * Uploads values to Hue: reserves each with `POST /api/v1/otlp/blobs`, PUTs its bytes to the
171
- * presigned URL Hue answers (skipped when Hue already stores the value), and completes it. At
172
- * most {@link UPLOAD_CONCURRENCY} uploads run at once; every request has a deadline and none
173
- * follows a redirect. A receiver without the route (a 404 without Hue's JSON error, a 405 or
174
- * 501, or a 200 that is no reservation) is remembered for {@link UNSUPPORTED_PAUSE_MILLIS}, and a
175
- * transient failure or a refusal that is not about the value alone pauses uploads for
176
- * {@link FAILURE_PAUSE_MILLIS}, so values fall back to the inline cut instead of retrying.
311
+ * presigned URL Hue answers (skipped when Hue already stores the value), and completes it. An
312
+ * export queues its values ({@link enqueue}) and sends its records without waiting for them; the
313
+ * import stores each value before it sends its span ({@link store}). At most
314
+ * {@link UPLOAD_CONCURRENCY} uploads run at once, and {@link HASH_CONCURRENCY} hashes apart from
315
+ * them. A reservation or PUT that gets no answer, a 408, a 429 or a 5xx is tried
316
+ * {@link UPLOAD_ATTEMPTS} times; every request has a deadline and none follows a redirect. A
317
+ * receiver without the route (a 404 without Hue's JSON error, a 405 or 501, or a 200 that is no
318
+ * reservation) is remembered for {@link UNSUPPORTED_PAUSE_MILLIS}: values exported meanwhile keep
319
+ * their fallback, never an entry the receiver cannot resolve.
177
320
  */
178
321
  export class BlobUploader {
179
322
  options;
323
+ /** Until when the receiver is taken to lack the upload route. */
180
324
  unsupportedUntil = 0;
181
- pausedUntil = 0;
182
325
  closed = false;
183
- running = 0;
184
- waiting = [];
326
+ uploads = new Slots(UPLOAD_CONCURRENCY);
327
+ hashes = new Slots(HASH_CONCURRENCY);
328
+ queued = 0;
329
+ queuedBytes = 0;
330
+ /** What queued uploads keep alive, each charged once, from the first upload holding it until
331
+ * the last settles: two files read in place from one recorded string charge it once, whole. */
332
+ holding = new Map();
333
+ sleeps = new Set();
185
334
  requests = new Set();
186
335
  outcomes = new Map();
336
+ /** The first wait before another attempt, doubled each time; tests shorten it. */
337
+ retryDelayMillis = RETRY_DELAY_MILLIS;
338
+ /** How long a queued upload may take; tests shorten it. */
339
+ lifetimeMillis;
187
340
  constructor(options) {
188
341
  this.options = options;
342
+ this.lifetimeMillis = uploadLifetimeMillis(options.timeoutMillis);
189
343
  }
190
344
  /** Whether a value over the inline limit is worth holding whole for upload now. */
191
345
  available() {
@@ -195,35 +349,31 @@ export class BlobUploader {
195
349
  unsupported() {
196
350
  return Date.now() < this.unsupportedUntil;
197
351
  }
198
- /** Ends uploads in flight and refuses new ones: their values fall back to the inline cut. */
352
+ /** Ends uploads in flight, queued uploads and their waits, and refuses new ones: values not
353
+ * yet exported fall back to the inline cut, and uploads queued by exported records fail. */
199
354
  close() {
200
355
  this.closed = true;
201
356
  for (const request of this.requests)
202
357
  request.destroy();
203
358
  this.requests.clear();
204
359
  this.outcomes.clear();
205
- for (const run of this.waiting.splice(0)) {
206
- this.running++;
207
- run();
208
- }
360
+ for (const cancel of [...this.sleeps])
361
+ cancel();
362
+ this.uploads.release();
363
+ this.hashes.release();
209
364
  }
210
365
  /** Runs `work` once fewer than {@link UPLOAD_CONCURRENCY} uploads are in flight. */
211
- async limit(work) {
212
- // A finishing upload hands its place to the next one waiting, so none is ever taken twice.
213
- if (this.running >= UPLOAD_CONCURRENCY && !this.closed)
214
- await new Promise((resolve) => this.waiting.push(resolve));
215
- else
216
- this.running++;
217
- try {
218
- return await work();
219
- }
220
- finally {
221
- const next = this.waiting.shift();
222
- if (next)
223
- next();
224
- else
225
- this.running--;
226
- }
366
+ limit(work) {
367
+ return this.uploads.run(work);
368
+ }
369
+ /** The SHA-256 and size of `value` (see {@link digestOf}), hashed in a place of its own: an
370
+ * export never waits for an upload to hash its values. */
371
+ digest(value, deadline) {
372
+ return this.hashes.run(() => digestOf(value, deadline));
373
+ }
374
+ /** The bytes queued uploads keep alive until they settle. */
375
+ pendingBytes() {
376
+ return this.queuedBytes;
227
377
  }
228
378
  /** Why a value cannot be uploaded right now without trying, if it cannot. */
229
379
  refusal() {
@@ -231,21 +381,10 @@ export class BlobUploader {
231
381
  return "failed";
232
382
  if (this.unsupported())
233
383
  return "unsupported";
234
- if (Date.now() < this.pausedUntil)
235
- return "failed";
236
384
  return undefined;
237
385
  }
238
- pause() {
239
- this.pausedUntil = Date.now() + FAILURE_PAUSE_MILLIS;
240
- }
241
- /**
242
- * Stores `value` under `key` of the trace: reserved, uploaded unless Hue already holds it, and
243
- * completed. Only a fresh upload takes a {@link limit} slot: another reference to bytes the
244
- * trace is already storing shares that upload's outcome without one, so repeated files never
245
- * hold the slots new files need. The caller must not hold a slot.
246
- */
247
- async store(traceId, key, value, contentType, digest, upload) {
248
- // Every occurrence is validated even when a previous occurrence already uploaded the bytes.
386
+ /** Why the value cannot be uploaded at all, whatever Hue would answer. */
387
+ invalid(traceId, key, value, contentType, digest) {
249
388
  if (!isBlobKey(key) ||
250
389
  !isBlobContentType(contentType) ||
251
390
  !/^[0-9a-f]{32}$/.test(traceId) ||
@@ -254,13 +393,102 @@ export class BlobUploader {
254
393
  !Number.isSafeInteger(digest.size) ||
255
394
  digest.size <= 0 ||
256
395
  digest.size !== sizeOf(value))
257
- return { ok: false, reason: "failed" };
396
+ return "failed";
258
397
  if (digest.size > MAX_BLOB_BYTES)
259
- return { ok: false, reason: "tooLarge" };
398
+ return "tooLarge";
399
+ return undefined;
400
+ }
401
+ /**
402
+ * Queues `value`'s upload for an exported record, deciding now what the record lists: a value
403
+ * refused without trying (see {@link refusal}), invalid, or past the queue's
404
+ * {@link MAX_QUEUED_UPLOADS} uploads or {@link MAX_HELD_BLOB_BYTES} keeps its fallback. A queued
405
+ * upload runs within {@link uploadLifetimeMillis} of now, holding `value` until it settles;
406
+ * `done` never rejects. `bytes` is what it adds to {@link pendingBytes}: the strings it keeps
407
+ * alive that no other queued upload holds, charged until `released`, once the last upload
408
+ * holding them settles.
409
+ */
410
+ enqueue(traceId, key, value, contentType, digest) {
411
+ const refused = this.refusal() ?? this.invalid(traceId, key, value, contentType, digest);
412
+ if (refused)
413
+ return { ok: false, reason: refused };
414
+ const held = heldBy(value);
415
+ const fresh = held.filter((text) => !this.holding.has(text));
416
+ const bytes = fresh.reduce((total, text) => total + text.bytes, 0);
417
+ // An upload whose strings are all held already adds no memory, so only the job cap applies.
418
+ if (this.queued >= MAX_QUEUED_UPLOADS ||
419
+ (bytes > 0 && this.queuedBytes > 0 && this.queuedBytes + bytes > MAX_HELD_BLOB_BYTES))
420
+ return { ok: false, reason: "budget" };
421
+ this.queued++;
422
+ this.queuedBytes += bytes;
423
+ const released = fresh.map((text) => new Promise((release) => {
424
+ this.holding.set(text, { uploads: 0, release });
425
+ }));
426
+ for (const text of held)
427
+ this.holding.get(text).uploads++;
428
+ const upload = {
429
+ deadline: Date.now() + this.lifetimeMillis,
430
+ wait: (millis) => this.sleep(millis),
431
+ };
432
+ // store() never rejects; the job holds `value` only until it settles.
433
+ const done = this.store(traceId, key, value, contentType, digest, upload).finally(() => {
434
+ this.queued--;
435
+ for (const text of held) {
436
+ const holding = this.holding.get(text);
437
+ if (--holding.uploads > 0)
438
+ continue;
439
+ this.holding.delete(text);
440
+ this.queuedBytes -= text.bytes;
441
+ holding.release();
442
+ }
443
+ });
444
+ return { ok: true, done, bytes, released: Promise.all(released).then(() => { }) };
445
+ }
446
+ /** Waits `millis`; false when {@link close} ended the wait first. The timer keeps the process
447
+ * running, as an export in progress does, within the upload's lifetime. */
448
+ sleep(millis) {
449
+ if (this.closed)
450
+ return Promise.resolve(false);
451
+ return new Promise((resolve) => {
452
+ const done = (ran) => {
453
+ clearTimeout(timer);
454
+ this.sleeps.delete(cancel);
455
+ resolve(ran && !this.closed);
456
+ };
457
+ const cancel = () => done(false);
458
+ const timer = setTimeout(() => done(true), millis);
459
+ this.sleeps.add(cancel);
460
+ });
461
+ }
462
+ /** Waits before attempt `attempt + 1`: `Retry-After` when the answer gave one, otherwise a
463
+ * second doubled each attempt with jitter, at most a minute. The upload's place is given up
464
+ * meanwhile, so uploads queued behind it run. False when the wait would pass the upload's
465
+ * deadline, was ended, or no place came free again before the deadline. */
466
+ async retryWait(attempt, retryAfter, upload) {
467
+ const millis = Math.min(retryAfter ?? this.retryDelayMillis * (2 ** attempt + Math.random()), MAX_RETRY_AFTER_MILLIS);
468
+ if (this.closed || Date.now() + millis >= upload.deadline)
469
+ return false;
470
+ return ((await this.uploads.away(() => upload.wait(millis), upload.deadline)) &&
471
+ !this.closed &&
472
+ Date.now() < upload.deadline);
473
+ }
474
+ /**
475
+ * Stores `value` under `key` of the trace: reserved, uploaded unless Hue already holds it, and
476
+ * completed. Only a fresh upload takes a {@link limit} slot: another reference to bytes the
477
+ * trace is already storing shares that upload's outcome without one, so repeated files never
478
+ * hold the slots new files need. The caller must not hold a slot. Never rejects.
479
+ */
480
+ async store(traceId, key, value, contentType, digest, upload) {
481
+ // Every occurrence is validated even when a previous occurrence already uploaded the bytes.
482
+ const invalid = this.invalid(traceId, key, value, contentType, digest);
483
+ if (invalid)
484
+ return { ok: false, reason: invalid };
260
485
  const identity = `${traceId}:${digest.sha256}`;
261
- const known = this.outcomes.get(identity);
262
- if (known)
263
- return known.promise;
486
+ for (let known = this.outcomes.get(identity); known; known = this.outcomes.get(identity)) {
487
+ const shared = await known.promise;
488
+ // Another reference's file changed since it was recorded: this one tries its own bytes.
489
+ if (shared.ok || !shared.changed)
490
+ return shared;
491
+ }
264
492
  if (this.outcomes.size >= 4096) {
265
493
  const oldest = [...this.outcomes].find(([, item]) => item.settled)?.[0];
266
494
  if (oldest === undefined)
@@ -286,6 +514,9 @@ export class BlobUploader {
286
514
  const refused = this.refusal();
287
515
  if (refused)
288
516
  return { ok: false, reason: refused };
517
+ // Queued past its lifetime: not tried. Failed, not budget: the import holds its span back.
518
+ if (Date.now() >= upload.deadline)
519
+ return { ok: false, reason: "failed" };
289
520
  if (digest.size > MAX_BLOB_BYTES)
290
521
  return { ok: false, reason: "tooLarge" };
291
522
  if (!isBlobKey(key) ||
@@ -302,27 +533,22 @@ export class BlobUploader {
302
533
  });
303
534
  let reservation = await this.reserve(body, digest.size, upload);
304
535
  let presigns = 1;
305
- let attempts = 0;
306
- while (!("fallback" in reservation) && !reservation.exists) {
536
+ for (let attempt = 0; !("fallback" in reservation) && !reservation.exists;) {
307
537
  const { grant } = reservation;
308
538
  const put = await this.put(grant, value, digest.size, upload.deadline);
309
539
  if (put === "stored")
310
540
  break;
541
+ if (put === "changed")
542
+ return { ok: false, reason: "failed", changed: true };
311
543
  if (this.closed)
312
544
  return { ok: false, reason: "failed" };
313
- const expired = put === "expired" || Date.now() >= grant.expiresAt;
314
- if (put === "transient" &&
315
- ++attempts < 2 &&
316
- Date.now() + RETRY_DELAY_MILLIS < upload.deadline) {
317
- await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MILLIS));
545
+ if (put === "transient") {
546
+ if (++attempt >= UPLOAD_ATTEMPTS || !(await this.retryWait(attempt - 1, undefined, upload)))
547
+ return { ok: false, reason: "failed" };
318
548
  if (Date.now() < grant.expiresAt)
319
549
  continue;
320
550
  }
321
- else if (put === "transient") {
322
- this.pause();
323
- return { ok: false, reason: "failed" };
324
- }
325
- else if (!expired)
551
+ else if (put !== "expired" && Date.now() < grant.expiresAt)
326
552
  return { ok: false, reason: "failed" };
327
553
  // The URL expired: reserved again once.
328
554
  if (presigns++ >= 2)
@@ -331,13 +557,10 @@ export class BlobUploader {
331
557
  }
332
558
  if ("fallback" in reservation)
333
559
  return { ok: false, reason: reservation.fallback };
334
- if (!reservation.exists) {
335
- // For Hue's accounting only: a value is read whether or not it was completed, so a failed
336
- // completion is not retried.
337
- const completion = await this.request("POST", new URL("/api/v1/otlp/blobs/complete", this.options.baseUrl), this.jsonHeaders(), JSON.stringify({ traceId, sha256: digest.sha256 }), Math.min(upload.deadline, Date.now() + this.options.timeoutMillis), MAX_JSON_RESPONSE_BYTES);
338
- if (!completion || completion.status >= 500 || completion.status === 429)
339
- this.pause();
340
- }
560
+ if (!reservation.exists)
561
+ // For Hue's accounting only: a value is read whether or not it was completed, and Hue
562
+ // stamps a missed completion on the next reservation, so its outcome is not read.
563
+ await this.request("POST", new URL("/api/v1/otlp/blobs/complete", this.options.baseUrl), this.jsonHeaders(), JSON.stringify({ traceId, sha256: digest.sha256 }), Math.min(upload.deadline, Date.now() + this.options.timeoutMillis), MAX_JSON_RESPONSE_BYTES);
341
564
  return { ok: true, contentType: reservation.contentType };
342
565
  }
343
566
  jsonHeaders() {
@@ -347,71 +570,46 @@ export class BlobUploader {
347
570
  "User-Agent": this.options.userAgent,
348
571
  };
349
572
  }
350
- /** Reserves the value: Hue answers that it holds it already, or a presigned PUT. A rate limit
351
- * is waited out within the export's hold, a transient refusal retried once. */
573
+ /** Reserves the value: Hue answers that it holds it already, or a presigned PUT. No answer, a
574
+ * 408, a 429 or a 5xx other than 501 is tried again, after its `Retry-After` when it gave one,
575
+ * within the upload's deadline. */
352
576
  async reserve(body, size, upload) {
353
577
  const url = new URL("/api/v1/otlp/blobs", this.options.baseUrl);
354
- for (let attempt = 0; attempt < 3; attempt++) {
578
+ for (let attempt = 0;; attempt++) {
355
579
  if (this.closed)
356
580
  return { fallback: "failed" };
357
581
  const reply = await this.request("POST", url, this.jsonHeaders(), body, Math.min(upload.deadline, Date.now() + this.options.timeoutMillis), MAX_JSON_RESPONSE_BYTES);
358
- if (!reply) {
359
- if (attempt === 0 && (await this.delay(RETRY_DELAY_MILLIS, upload)))
360
- continue;
361
- this.pause();
362
- return { fallback: "failed" };
363
- }
364
- const { status } = reply;
365
- const answer = status === 200 || status === 404 || status === 409 ? parseJson(reply.body) : undefined;
366
- // A receiver without the route answers its framework's not-found (Hue's own 404, an
367
- // archived project, is JSON with an `error`), refuses the method, or answers something
368
- // that is no reservation at all: it is not asked again for a while.
369
- if (status === 405 ||
370
- status === 501 ||
371
- (status === 404 && !(isObject(answer) && typeof answer.error === "string")) ||
372
- (status === 200 &&
373
- !(isObject(answer) && (answer.status === "exists" || answer.status === "upload")))) {
374
- this.unsupportedUntil = Date.now() + UNSUPPORTED_PAUSE_MILLIS;
375
- return { fallback: "unsupported" };
376
- }
377
- if (status === 200) {
378
- const reservation = this.reservation(answer, size);
379
- // A reservation that cannot be used would fail every value alike.
380
- if ("fallback" in reservation)
381
- this.pause();
382
- return reservation;
383
- }
384
- if (status === 409 && isObject(answer) && answer.status === "exists")
385
- return this.reservation(answer, size);
386
- if (status === 413)
387
- return { fallback: "tooLarge" };
388
- if (status === 429) {
389
- const wait = retryAfterMillis(reply.headers) ?? RETRY_DELAY_MILLIS;
390
- if (Date.now() + wait < upload.deadline && (await upload.wait(wait)))
391
- continue;
392
- this.pause();
393
- return { fallback: "failed" };
582
+ let retryAfter;
583
+ if (reply) {
584
+ const { status } = reply;
585
+ const answer = status === 200 || status === 404 || status === 409 ? parseJson(reply.body) : undefined;
586
+ // A receiver without the route answers its framework's not-found (Hue's own 404, an
587
+ // archived project, is JSON with an `error`), refuses the method, or answers something
588
+ // that is no reservation at all: it is not asked again for a while, since the values
589
+ // exported while it answers are listed for bytes it cannot store.
590
+ if (status === 405 ||
591
+ status === 501 ||
592
+ (status === 404 && !(isObject(answer) && typeof answer.error === "string")) ||
593
+ (status === 200 &&
594
+ !(isObject(answer) && (answer.status === "exists" || answer.status === "upload")))) {
595
+ this.unsupportedUntil = Date.now() + UNSUPPORTED_PAUSE_MILLIS;
596
+ return { fallback: "unsupported" };
597
+ }
598
+ // A reservation that cannot be used fails its value.
599
+ if (status === 200)
600
+ return this.reservation(answer, size);
601
+ if (status === 409 && isObject(answer) && answer.status === "exists")
602
+ return this.reservation(answer, size);
603
+ if (status === 413)
604
+ return { fallback: "tooLarge" };
605
+ // Refusals of the value or the key (400, 401, 403, Hue's own 404, a conflict) are final.
606
+ if (status !== 408 && status !== 429 && status < 500)
607
+ return { fallback: "failed" };
608
+ retryAfter = retryAfterMillis(reply.headers);
394
609
  }
395
- if (status >= 500) {
396
- const wait = retryAfterMillis(reply.headers) ?? RETRY_DELAY_MILLIS;
397
- if (attempt === 0 && wait <= MAX_TRANSIENT_WAIT_MILLIS && (await this.delay(wait, upload)))
398
- continue;
399
- this.pause();
610
+ if (attempt + 1 >= UPLOAD_ATTEMPTS || !(await this.retryWait(attempt, retryAfter, upload)))
400
611
  return { fallback: "failed" };
401
- }
402
- // Bad values and conflicts refuse only this value. Transient service failures pause uploads.
403
- if (status !== 400 && status !== 409)
404
- this.pause();
405
- return { fallback: "failed" };
406
612
  }
407
- this.pause();
408
- return { fallback: "failed" };
409
- }
410
- async delay(millis, upload) {
411
- if (Date.now() + millis >= upload.deadline)
412
- return false;
413
- await new Promise((resolve) => setTimeout(resolve, millis));
414
- return !this.closed;
415
613
  }
416
614
  /** A 200 answer read: `exists`, or an upload whose URL and headers are usable as given. */
417
615
  reservation(answer, size) {
@@ -464,14 +662,21 @@ export class BlobUploader {
464
662
  grant: { url, headers, expiresAt: Number.isFinite(expiresAt) ? expiresAt : Infinity },
465
663
  };
466
664
  }
467
- /** The PUT of the value's bytes, with exactly the headers Hue signed. */
665
+ /** The PUT of the value's bytes, with exactly the headers Hue signed. A source that no longer
666
+ * holds its bytes (a file changed since it was recorded) ends the request before its body is
667
+ * complete, and fails the value without a retry. */
468
668
  async put(grant, value, size, deadline) {
469
669
  const reply = await this.request("PUT", grant.url, grant.headers, value, deadline, MAX_ERROR_RESPONSE_BYTES, size);
670
+ if (isByteSource(value) && value.changed)
671
+ return "changed";
470
672
  if (!reply)
471
673
  return "transient";
472
- // 412: the object exists already (another upload of the same value won).
473
- if ((reply.status >= 200 && reply.status < 300) || reply.status === 412)
674
+ // 412: the object exists already (another upload of the same value won). A success answered
675
+ // before the whole body was written stores nothing that was sent.
676
+ if (reply.status === 412)
474
677
  return "stored";
678
+ if (reply.status >= 200 && reply.status < 300)
679
+ return reply.complete ? "stored" : "failed";
475
680
  if (reply.status === 403)
476
681
  return reply.body.toString("utf8").includes("Request has expired") ? "expired" : "failed";
477
682
  if (reply.status >= 500 || reply.status === 408 || reply.status === 429)
@@ -501,6 +706,8 @@ export class BlobUploader {
501
706
  return undefined;
502
707
  return new Promise((resolve) => {
503
708
  let settled = false;
709
+ // Whether the whole body was handed to the request.
710
+ let ended = false;
504
711
  let request;
505
712
  const finish = (reply) => {
506
713
  if (settled)
@@ -509,16 +716,20 @@ export class BlobUploader {
509
716
  clearTimeout(timer);
510
717
  if (request) {
511
718
  this.requests.delete(request);
512
- if (!reply)
719
+ // An answer before the whole body was handed over ends the request: a store that
720
+ // neither reads nor closes would otherwise hold it, and its source, open.
721
+ if (!reply || !ended)
513
722
  request.destroy();
514
723
  }
515
724
  resolve(reply);
516
725
  };
517
726
  const timer = setTimeout(() => finish(), Math.max(1, deadline - Date.now()));
727
+ // The bytes handed to the request: its body is complete once they are its length.
728
+ let sentBytes = 0;
729
+ const written = () => length === undefined || sentBytes === length;
518
730
  const requestHeaders = { ...headers };
519
731
  if (length === undefined)
520
- requestHeaders["Content-Length"] =
521
- typeof body === "string" ? Buffer.byteLength(body, "utf8") : body.byteLength;
732
+ requestHeaders["Content-Length"] = sizeOf(body);
522
733
  try {
523
734
  // Uploads are the SDK's own export traffic: an http instrumentation must not trace them.
524
735
  request = context.with(suppressTracing(context.active()), () => http.request(url, { method, headers: requestHeaders, agent: false }, (response) => {
@@ -528,6 +739,7 @@ export class BlobUploader {
528
739
  status: response.statusCode ?? 0,
529
740
  headers: response.headers,
530
741
  body: Buffer.concat(chunks),
742
+ complete: written(),
531
743
  });
532
744
  response.on("data", (chunk) => {
533
745
  size += chunk.byteLength;
@@ -552,35 +764,50 @@ export class BlobUploader {
552
764
  sent.on("error", () => finish());
553
765
  sent.setTimeout(this.options.timeoutMillis, () => finish());
554
766
  void (async () => {
555
- if (length === undefined) {
767
+ if (length === undefined && !isByteSource(body)) {
768
+ ended = true;
556
769
  sent.end(body);
557
770
  return;
558
771
  }
559
- for (const chunk of uploadChunks(body)) {
772
+ // A source or a text may reuse a slice's memory for the next: each slice is written out
773
+ // before the next is read. Bytes are views of memory that does not change.
774
+ const reused = !(body instanceof Uint8Array);
775
+ for await (const chunk of uploadChunks(body)) {
560
776
  if (settled)
561
777
  return;
562
- if (!sent.write(chunk))
563
- await new Promise((drained) => {
564
- const done = () => {
565
- sent.off("drain", done);
566
- sent.off("close", done);
567
- drained();
568
- };
778
+ sentBytes += chunk.byteLength;
779
+ await new Promise((proceed) => {
780
+ const done = () => {
781
+ sent.off("drain", done);
782
+ sent.off("close", done);
783
+ proceed();
784
+ };
785
+ sent.on("close", done);
786
+ if (reused) {
787
+ sent.write(chunk, () => done());
788
+ return;
789
+ }
790
+ if (sent.write(chunk))
791
+ done();
792
+ else
569
793
  sent.on("drain", done);
570
- sent.on("close", done);
571
- });
794
+ });
795
+ if (settled)
796
+ return;
572
797
  }
573
- if (!settled)
574
- sent.end();
798
+ if (settled)
799
+ return;
800
+ ended = true;
801
+ sent.end();
575
802
  })().catch(() => finish());
576
803
  });
577
804
  }
578
805
  }
579
806
  /**
580
807
  * A span's own attributes with each candidate placed: binary media becomes a digest, while
581
- * text media over 64 KiB retains a 16 KiB prefix after upload or a digest on failure.
582
- * A value still over the inline limit is uploaded and replaced by its first
583
- * 16 KiB (or cut to the limit), each upload listed under `hue.blobs`, each whole value cut or
808
+ * text media over 64 KiB retains a 16 KiB prefix once uploaded or queued, or a digest when
809
+ * refused. A value still over the inline limit is uploaded and replaced by its first 16 KiB (or
810
+ * cut to the limit when refused), each upload listed under `hue.blobs`, each whole value cut or
584
811
  * uploaded listed under `hue.truncated`. Every fallback is counted in the tally.
585
812
  */
586
813
  export async function offloadAttributes(attributes, collector, options) {
@@ -588,12 +815,12 @@ export async function offloadAttributes(attributes, collector, options) {
588
815
  }
589
816
  /**
590
817
  * A record's own attributes, and a log record's body, with each candidate placed (see
591
- * {@link offloadAttributes}). A file is uploaded and placed in its digest form (a text file in a
592
- * provider part keeps a 16 KiB prefix), its `hue.blobs` entry keyed `<attribute>#<pointer>` or, in
593
- * a log record's body, `body#<pointer>`. A file that is not uploaded keeps its bytes inline, except
594
- * in a span's own recorded messages, where it keeps its digest as before. A log record's values
595
- * over the inline limit are cut, never uploaded whole, and a body still over it is replaced by the
596
- * receiver's marker, as before.
818
+ * {@link offloadAttributes}). A file is uploaded (or, for an export, queued) and placed in its
819
+ * digest form (a text file in a provider part keeps a 16 KiB prefix), its `hue.blobs` entry keyed
820
+ * `<attribute>#<pointer>` or, in a log record's body, `body#<pointer>`. A file refused or not
821
+ * uploaded before it is placed keeps its bytes inline, except in a span's own recorded messages,
822
+ * where it keeps its digest as before. A log record's values over the inline limit are cut, never
823
+ * uploaded whole, and a body still over it is replaced by the receiver's marker, as before.
597
824
  */
598
825
  export async function offloadRecord(attributes, collector, options) {
599
826
  const { uploader, traceId, valueBytes, upload, tally } = options;
@@ -607,38 +834,73 @@ export async function offloadRecord(attributes, collector, options) {
607
834
  const entries = [];
608
835
  const listed = [];
609
836
  const fallback = (reason) => countFallback(tally, reason, traceId);
837
+ /** Stores a value (the import), or queues its upload (an export): what the record lists. A
838
+ * queued value's entry names the type its own reference declared. `shared` when the value is
839
+ * memory its record holds (not decoded or rewritten from it). */
840
+ const submit = async (key, value, contentType, digest, shared) => {
841
+ const late = options.background;
842
+ if (!late) {
843
+ const outcome = await uploader.store(traceId, key, value, contentType, digest, upload);
844
+ if (outcome.ok)
845
+ tally.uploaded++;
846
+ return outcome;
847
+ }
848
+ const queued = uploader.enqueue(traceId, key, value, contentType, digest);
849
+ if (!queued.ok)
850
+ return queued;
851
+ const settled = countWhenSettled(queued.done, late.tally, traceId);
852
+ late.settled.push(settled);
853
+ late.queued?.(settled, shared ? queued.bytes : 0, queued.released);
854
+ return { ok: true, contentType };
855
+ };
610
856
  // A value the queue cut when it was admitted was never held whole to upload.
611
857
  for (let index = 0; index < collector.cut.length; index++)
612
858
  fallback(uploader.unsupported() ? "unsupported" : "budget");
613
859
  let inspected = 0;
614
- /** Uploads each file of a value and places it; whether any was placed. A file that is not
615
- * uploaded keeps its bytes inline, except in a recorded message: a span's own message keeps its
616
- * digest, and a log record's keeps the digest of a file over the inline limit, as nested
617
- * messages kept them before log records uploaded. */
618
- const placeFiles = async (index, key, files) => {
860
+ /** Uploads (or queues) each file of a value and places it; whether any was placed. A file
861
+ * refused or not uploaded keeps its bytes inline, except in a recorded message: a span's own
862
+ * message keeps its digest, and a log record's keeps the digest of a file over the inline limit,
863
+ * as nested messages kept them before log records uploaded. */
864
+ const placeFiles = async (index, key, files, long) => {
619
865
  const message = isMessageKey(key);
620
866
  let changed = false;
621
867
  await Promise.all(files.map((file, position) => {
622
- const decoded = mediaBytes(file);
623
- if (!placeable(file, decoded))
868
+ // A file whose content is a long string is read from its place in the recorded text, a
869
+ // slice at a time: never decoded or copied whole.
870
+ const inPlace = long?.get(file.content);
871
+ const decoded = inPlace ? longFileBytes(file, inPlace) : mediaBytes(file);
872
+ const bytes = "source" in decoded ? decoded.source : decoded.bytes;
873
+ const size = sizeOf(bytes);
874
+ if (!placeable(file, decoded, size))
624
875
  return Promise.resolve();
625
876
  const contentType = mediaContentType(file, decoded);
626
877
  const textual = isTextMediaType(contentType);
627
- if (textual && decoded.bytes.byteLength <= INLINE_FILE_LIMIT)
878
+ if (textual && size <= INLINE_FILE_LIMIT)
628
879
  return Promise.resolve();
629
880
  const prefix = keepsPrefix(file, textual, decoded);
630
881
  const partKey = `${key}#${prefix ? file.pointer : file.partPointer}`;
631
882
  const slot = slots > 0 && isBlobKey(partKey) ? (slots--, true) : false;
632
883
  return (async () => {
633
- const digest = options.memo
634
- ? options.memo.digest(file, decoded, traceId)
635
- : await digestOf(decoded.bytes);
884
+ // A long file is hashed in a hashing place, as a large value is; one the export's
885
+ // hashing deadline cannot hash stays as recorded and is not uploaded.
886
+ let digest;
887
+ try {
888
+ digest = isByteSource(bytes)
889
+ ? await uploader.digest(bytes, upload.deadline)
890
+ : options.memo
891
+ ? options.memo.digest(file, decoded, traceId)
892
+ : await digestOf(bytes);
893
+ }
894
+ catch {
895
+ fallback("failed");
896
+ return;
897
+ }
636
898
  const outcome = slot
637
- ? await uploader.store(traceId, partKey, decoded.bytes, contentType, digest, upload)
899
+ ? await submit(partKey, bytes, contentType, digest, isByteSource(bytes))
638
900
  : { ok: false, reason: "budget" };
639
901
  if (outcome.ok) {
640
902
  if (prefix)
641
- prefixPart(file, blobPrefix(file.content));
903
+ prefixPart(file, blobPrefix(inPlace ? longHead(inPlace, BLOB_PREFIX_BYTES + 1) : file.content));
642
904
  else
643
905
  digestPart(file, digest);
644
906
  changed = true;
@@ -646,10 +908,9 @@ export async function offloadRecord(attributes, collector, options) {
646
908
  order: [index, position],
647
909
  entry: blobEntry(partKey, digest.sha256, digest.size, outcome.contentType, file.name),
648
910
  });
649
- tally.uploaded++;
650
911
  }
651
912
  else {
652
- if (message && (!collector.log || decoded.bytes.byteLength > INLINE_FILE_LIMIT)) {
913
+ if (message && (!collector.log || size > INLINE_FILE_LIMIT)) {
653
914
  digestPart(file, digest);
654
915
  changed = true;
655
916
  }
@@ -661,30 +922,59 @@ export async function offloadRecord(attributes, collector, options) {
661
922
  };
662
923
  const literal = literalKeys(attributes);
663
924
  const parts = async (index, key, text) => {
664
- // Both bounds count UTF-16 code units, as admission does. Literal text holds no file.
665
- if (!mayInlineFiles(key, text) || text.length > MAX_INLINE_FILE_TEXT || literal.has(key))
925
+ // Literal text holds no file.
926
+ if (!mayInlineFiles(key, text) || literal.has(key))
666
927
  return text;
667
- if (collector.inspected) {
668
- // A value admission skipped stays as recorded.
669
- if (!collector.inspected.has(key))
670
- return text;
928
+ let media;
929
+ let large;
930
+ if (collector.large?.has(key)) {
931
+ // Held whole past the inspection bounds: its long strings were read in place, the JSON
932
+ // around them within the per-attribute bound.
933
+ media = large = largeReads.get(index);
671
934
  }
672
935
  else {
673
- if (inspected + text.length > INLINE_FILE_TEXT_PER_RECORD)
936
+ // Both bounds count UTF-16 code units, as admission does.
937
+ if (text.length > MAX_INLINE_FILE_TEXT)
674
938
  return text;
675
- inspected += text.length;
939
+ if (collector.inspected) {
940
+ // A value admission skipped stays as recorded.
941
+ if (!collector.inspected.has(key))
942
+ return text;
943
+ }
944
+ else {
945
+ if (inspected + text.length > INLINE_FILE_TEXT_PER_RECORD)
946
+ return text;
947
+ inspected += text.length;
948
+ }
949
+ try {
950
+ media = readMediaText(key, text);
951
+ }
952
+ catch {
953
+ return text;
954
+ }
676
955
  }
677
- let media;
956
+ if (!media?.files.length)
957
+ return text;
958
+ if (!(await placeFiles(index, key, media.files, large?.long)))
959
+ return text;
960
+ return large ? await placedLargeText(large) : placedMediaText(media);
961
+ };
962
+ // The values read in place are scanned first, in candidate order, between turns of the event
963
+ // loop: places in `hue.blobs` are then taken in candidate order, as for any value.
964
+ const largeReads = new Map();
965
+ for (const [index, { key, value }] of collector.candidates.entries()) {
966
+ if (typeof value !== "string" ||
967
+ !collector.large?.has(key) ||
968
+ !mayInlineFiles(key, value) ||
969
+ literal.has(key))
970
+ continue;
678
971
  try {
679
- media = readMediaText(key, text);
972
+ largeReads.set(index, await readLargeMediaText(key, value, options.longUnits));
680
973
  }
681
974
  catch {
682
- return text;
975
+ // JSON that does not parse holds no file: the value stays as recorded.
683
976
  }
684
- if (!media?.files.length)
685
- return text;
686
- return (await placeFiles(index, key, media.files)) ? placedMediaText(media) : text;
687
- };
977
+ }
688
978
  /** A log record's body with its files placed. Each file's content counts toward the record's
689
979
  * inspection bounds, as an attribute's text does. */
690
980
  const placeBody = async (body) => {
@@ -710,14 +1000,22 @@ export async function offloadRecord(attributes, collector, options) {
710
1000
  return media.box.value;
711
1001
  };
712
1002
  const place = async (index, { key, value }) => {
713
- const placed = typeof value === "string" ? await parts(index, key, value) : value;
714
- const size = sizeOf(placed);
1003
+ const read = typeof value === "string" ? await parts(index, key, value) : value;
1004
+ // Text joined from pieces of the recorded text is made one string only when it is kept
1005
+ // inline; otherwise it is measured, hashed, uploaded and cut from its pieces.
1006
+ const joined = read instanceof JoinedText ? read : undefined;
1007
+ const size = joined ? joined.utf8Size() : sizeOf(read);
715
1008
  if (size <= valueBytes) {
716
- result[key] = placed;
1009
+ result[key] = joined ? joined.text() : read;
717
1010
  return;
718
1011
  }
1012
+ const placed = joined ? joined.source(size) : read;
719
1013
  listed.push({ order: index, key });
720
- const cut = () => typeof placed === "string" ? truncateUtf8(placed, valueBytes) : truncatedMarker(size);
1014
+ const cut = () => joined
1015
+ ? truncateUtf8(joined.head(valueBytes + 1), valueBytes)
1016
+ : typeof placed === "string"
1017
+ ? truncateUtf8(placed, valueBytes)
1018
+ : truncatedMarker(size);
721
1019
  // A log record's value is cut, as no log reader pages an uploaded text value.
722
1020
  if (collector.log) {
723
1021
  result[key] = cut();
@@ -732,19 +1030,32 @@ export async function offloadRecord(attributes, collector, options) {
732
1030
  const stored = await (async () => {
733
1031
  if (size > MAX_BLOB_BYTES)
734
1032
  return { ok: false, reason: "tooLarge" };
735
- // Hashing a large value takes a slot as its upload does; the upload takes its own.
736
- const digest = await uploader.limit(() => digestOf(placed));
737
- const contentType = typeof placed === "string" ? textContentType(placed) : "application/octet-stream";
738
- const outcome = await uploader.store(traceId, key, placed, contentType, digest, upload);
1033
+ // A large value is hashed in a hashing place; its upload takes an upload place.
1034
+ let digest;
1035
+ try {
1036
+ digest = await uploader.digest(placed, upload.deadline);
1037
+ }
1038
+ catch {
1039
+ return { ok: false, reason: "failed" };
1040
+ }
1041
+ const contentType = joined
1042
+ ? textContentType(joined.head(256) + joined.tail(256))
1043
+ : typeof placed === "string"
1044
+ ? textContentType(placed)
1045
+ : "application/octet-stream";
1046
+ const outcome = await submit(key, placed, contentType, digest, !!joined || read === value);
739
1047
  return outcome.ok ? { ...outcome, digest } : outcome;
740
1048
  })();
741
1049
  if (stored.ok) {
742
- result[key] = typeof placed === "string" ? blobPrefix(placed) : blobPrefix(placed);
1050
+ result[key] = joined
1051
+ ? blobPrefix(joined.head(BLOB_PREFIX_BYTES + 1))
1052
+ : typeof placed === "string"
1053
+ ? blobPrefix(placed)
1054
+ : blobPrefix(placed);
743
1055
  entries.push({
744
1056
  order: [index, Infinity],
745
1057
  entry: blobEntry(key, stored.digest.sha256, stored.digest.size, stored.contentType),
746
1058
  });
747
- tally.uploaded++;
748
1059
  }
749
1060
  else {
750
1061
  result[key] = cut();
@@ -754,20 +1065,19 @@ export async function offloadRecord(attributes, collector, options) {
754
1065
  const body = collector.body ? placeBody(collector.body.value) : undefined;
755
1066
  await Promise.all(collector.candidates.map((candidate, index) => place(index, candidate)));
756
1067
  await Promise.all((collector.attachments ?? []).map((file, position) => {
757
- if (!file.bytes.byteLength)
1068
+ if (!sizeOf(file.bytes))
758
1069
  return Promise.resolve();
759
1070
  const slot = slots > 0 ? (slots--, true) : false;
760
1071
  return (async () => {
761
1072
  const key = mediaKey(file);
762
1073
  const outcome = slot
763
- ? await uploader.store(traceId, key, file.bytes, file.contentType, file.digest, upload)
1074
+ ? await submit(key, file.bytes, file.contentType, file.digest, true)
764
1075
  : { ok: false, reason: "budget" };
765
1076
  if (outcome.ok) {
766
1077
  entries.push({
767
1078
  order: [collector.candidates.length, position],
768
1079
  entry: blobEntry(key, file.digest.sha256, file.digest.size, outcome.contentType, file.name),
769
1080
  });
770
- tally.uploaded++;
771
1081
  }
772
1082
  else
773
1083
  fallback(outcome.reason);