@hue-run/sdk 0.18.2 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +16 -12
- package/EVALUATIONS.md +1 -1
- package/README.md +110 -68
- package/dist/blobs.d.ts +116 -36
- package/dist/blobs.js +359 -162
- package/dist/cli/langfuse.d.ts +21 -5
- package/dist/cli/langfuse.js +123 -16
- package/dist/client.d.ts +11 -7
- package/dist/client.js +35 -26
- package/dist/inline-files.d.ts +21 -4
- package/dist/inline-files.js +100 -14
- package/dist/large-media.d.ts +15 -2
- package/dist/large-media.js +27 -16
- package/dist/transport.d.ts +38 -7
- package/dist/transport.js +240 -63
- package/dist/types.d.ts +5 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/transport.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
1
2
|
import { gzip } from "node:zlib";
|
|
2
3
|
import { ExportResultCode } from "@opentelemetry/core";
|
|
3
4
|
import { createOtlpNetworkExportDelegate, OTLPExporterBase, OTLPExporterError, } from "@opentelemetry/otlp-exporter-base";
|
|
@@ -6,7 +7,7 @@ import { LogsExporterMetricsHelper, ProtobufLogsSerializer, ProtobufTraceSeriali
|
|
|
6
7
|
import { BatchSpanProcessor, } from "@opentelemetry/sdk-trace";
|
|
7
8
|
import { BatchLogRecordProcessor, } from "@opentelemetry/sdk-logs";
|
|
8
9
|
import { isInsecureOrigin, REDACTION_CONTEXT_UNITS, validateOptions } from "./config.js";
|
|
9
|
-
import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, countFallback, BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, fallbackRecord, heldBytesOf, MAX_HELD_BLOB_BYTES, newTally, mediaKey, offloadAttributes, offloadRecord, sizeOf, uploadBudgetMillis, } from "./blobs.js";
|
|
10
|
+
import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, countFallback, BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, fallbackRecord, heldBytesOf, LATE_FALLBACK_MESSAGES, MAX_HELD_BLOB_BYTES, mergeWhenSettled, newTally, mediaKey, offloadAttributes, offloadRecord, sizeOf, uploadBudgetMillis, } from "./blobs.js";
|
|
10
11
|
import { isByteSource } from "./large-media.js";
|
|
11
12
|
import { advertisedLimits, BATCH_TARGET_BYTES, DEFAULT_RECEIVER_LIMITS, fitsWithoutCompression, } from "./limits.js";
|
|
12
13
|
import { announcesLiveSpan, LIVE_SPAN_INTERVAL_MILLIS, MAX_LIVE_SPANS, pendingPlaceholder, PLACEHOLDERS_HEADER, withoutPlaceholderMarkers, } from "./live-spans.js";
|
|
@@ -109,9 +110,11 @@ function mediaEntry(file) {
|
|
|
109
110
|
* them, reported like any refused request.
|
|
110
111
|
*/
|
|
111
112
|
export const MAX_RATE_LIMIT_HOLD_MILLIS = 60_000;
|
|
112
|
-
/** How long one export may take with requests of `timeoutMillis` each: its
|
|
113
|
+
/** How long one export may take with requests of `timeoutMillis` each: hashing its values over
|
|
113
114
|
* the inline limit, its requests, and a hold for a rate-limited receiver on top. Processors and
|
|
114
|
-
* providers wait this long for a drain.
|
|
115
|
+
* providers wait this long for a drain. A drain also waits for the uploads its exports queued,
|
|
116
|
+
* which run beside the requests and end within six timeouts and a minute of being queued, so they
|
|
117
|
+
* fit within the same budget. */
|
|
115
118
|
export function exportBudgetMillis(timeoutMillis) {
|
|
116
119
|
return uploadBudgetMillis(timeoutMillis) + timeoutMillis * 16 + 1000 + MAX_RATE_LIMIT_HOLD_MILLIS;
|
|
117
120
|
}
|
|
@@ -195,7 +198,11 @@ export class HueTransport {
|
|
|
195
198
|
logExporter;
|
|
196
199
|
closed = false;
|
|
197
200
|
shutdownPromise;
|
|
198
|
-
|
|
201
|
+
/** The latest flush's send of its records; the next flush's send starts after it. */
|
|
202
|
+
sendPromise;
|
|
203
|
+
/** Set while a flush sends its records: the processors leave the uploads to that flush, which
|
|
204
|
+
* waits for them after its send, so no flush's send waits behind another's uploads. */
|
|
205
|
+
sendingRecords = new AsyncLocalStorage();
|
|
199
206
|
// Open spans not yet announced. Each is considered once, at the next tick after it starts.
|
|
200
207
|
live = new Set();
|
|
201
208
|
liveTimer;
|
|
@@ -226,6 +233,10 @@ export class HueTransport {
|
|
|
226
233
|
/** Bytes of values over the inline limit queued spans hold whole for upload, and each span's. */
|
|
227
234
|
heldBytes = 0;
|
|
228
235
|
held = new WeakMap();
|
|
236
|
+
/** Bytes of each record's charge (its own held values and its staged files) that its queued
|
|
237
|
+
* uploads took over before the record was finished: charged to the uploads alone until they
|
|
238
|
+
* settle or the record is finished. */
|
|
239
|
+
heldByUploads = new WeakMap();
|
|
229
240
|
mediaMemo = new MediaDigestMemo();
|
|
230
241
|
// Context identity also matches safe API wrappers to the provider's ended span. Abandoned
|
|
231
242
|
// spans retain no span objects, and share the 2048-span / 128 MiB holding bounds.
|
|
@@ -242,6 +253,11 @@ export class HueTransport {
|
|
|
242
253
|
});
|
|
243
254
|
uploaded = 0;
|
|
244
255
|
uploadFallbacks = 0;
|
|
256
|
+
/** Each export's queued uploads, until they all settle and are reported. */
|
|
257
|
+
uploadReports = new Set();
|
|
258
|
+
/** Settles once every upload report registered so far has: each drain awaits this one promise,
|
|
259
|
+
* so a drain costs the same however many uploads are pending. */
|
|
260
|
+
uploadsSettled = Promise.resolve();
|
|
245
261
|
constructor(options) {
|
|
246
262
|
this.options = validateOptions(options);
|
|
247
263
|
Object.defineProperty(this, "options", { enumerable: false });
|
|
@@ -333,10 +349,25 @@ export class HueTransport {
|
|
|
333
349
|
this.issue("traces", "invalid", 1, "Telemetry processor could not accept a record", undefined, traceIdsOf([span]));
|
|
334
350
|
}
|
|
335
351
|
},
|
|
336
|
-
|
|
337
|
-
|
|
352
|
+
// The spans are sent first, then the uploads queued so far are waited for, unless a
|
|
353
|
+
// flush sends them: that flush waits for the uploads after its send.
|
|
354
|
+
forceFlush: async () => {
|
|
355
|
+
try {
|
|
356
|
+
await spans.forceFlush();
|
|
357
|
+
}
|
|
358
|
+
finally {
|
|
359
|
+
if (!this.sendingRecords.getStore())
|
|
360
|
+
await this.drainUploads();
|
|
361
|
+
}
|
|
362
|
+
},
|
|
363
|
+
shutdown: async () => {
|
|
338
364
|
this.stopLiveSpans();
|
|
339
|
-
|
|
365
|
+
try {
|
|
366
|
+
await spans.shutdown();
|
|
367
|
+
}
|
|
368
|
+
finally {
|
|
369
|
+
await this.drainUploads();
|
|
370
|
+
}
|
|
340
371
|
},
|
|
341
372
|
};
|
|
342
373
|
this.logRecordProcessor = {
|
|
@@ -367,8 +398,23 @@ export class HueTransport {
|
|
|
367
398
|
this.issue("logs", "invalid", 1, "Telemetry processor could not accept a record", undefined, traceIdsOf([log]));
|
|
368
399
|
}
|
|
369
400
|
},
|
|
370
|
-
forceFlush: () =>
|
|
371
|
-
|
|
401
|
+
forceFlush: async () => {
|
|
402
|
+
try {
|
|
403
|
+
await logs.forceFlush();
|
|
404
|
+
}
|
|
405
|
+
finally {
|
|
406
|
+
if (!this.sendingRecords.getStore())
|
|
407
|
+
await this.drainUploads();
|
|
408
|
+
}
|
|
409
|
+
},
|
|
410
|
+
shutdown: async () => {
|
|
411
|
+
try {
|
|
412
|
+
await logs.shutdown();
|
|
413
|
+
}
|
|
414
|
+
finally {
|
|
415
|
+
await this.drainUploads();
|
|
416
|
+
}
|
|
417
|
+
},
|
|
372
418
|
};
|
|
373
419
|
}
|
|
374
420
|
/**
|
|
@@ -400,8 +446,8 @@ export class HueTransport {
|
|
|
400
446
|
const hold = !advisory && this.holdsValues()
|
|
401
447
|
? {
|
|
402
448
|
valueBytes: this.limits.valueBytes,
|
|
403
|
-
budget:
|
|
404
|
-
first: this.heldBytes === 0,
|
|
449
|
+
budget: this.heldRoom(),
|
|
450
|
+
first: this.heldBytes === 0 && !this.uploader?.pendingBytes(),
|
|
405
451
|
}
|
|
406
452
|
: undefined;
|
|
407
453
|
const snapshot = signal === "traces"
|
|
@@ -534,6 +580,12 @@ export class HueTransport {
|
|
|
534
580
|
this.heldBytes -= held;
|
|
535
581
|
this.held.delete(record);
|
|
536
582
|
}
|
|
583
|
+
// Released whole above: the part its uploads took over stays charged to them.
|
|
584
|
+
const taken = this.heldByUploads.get(record);
|
|
585
|
+
if (taken) {
|
|
586
|
+
this.heldBytes += taken;
|
|
587
|
+
this.heldByUploads.delete(record);
|
|
588
|
+
}
|
|
537
589
|
}
|
|
538
590
|
}
|
|
539
591
|
/** @internal The same bounded digest memo is used by explicit and inline media. */
|
|
@@ -560,7 +612,7 @@ export class HueTransport {
|
|
|
560
612
|
const files = this.media.get(identity) ?? [];
|
|
561
613
|
if (files.length >= 64 ||
|
|
562
614
|
(!this.media.has(identity) && this.media.size >= 2048) ||
|
|
563
|
-
|
|
615
|
+
heldBytesOf(file.bytes) > this.heldRoom()) {
|
|
564
616
|
this.uploadFallbacks++;
|
|
565
617
|
this.issue("traces", "warning", 1, "File bytes did not fit the SDK's upload budget; their hue.file metadata was recorded", undefined, [span.spanContext().traceId]);
|
|
566
618
|
return false;
|
|
@@ -610,7 +662,7 @@ export class HueTransport {
|
|
|
610
662
|
const staged = this.media.get(identity) ?? [];
|
|
611
663
|
if (staged.length >= MAX_BLOB_ENTRIES ||
|
|
612
664
|
(!this.media.has(identity) && this.media.size >= 2048) ||
|
|
613
|
-
|
|
665
|
+
heldBytesOf(file.bytes) > this.heldRoom()) {
|
|
614
666
|
countFallback(tally, "budget", traceId);
|
|
615
667
|
continue;
|
|
616
668
|
}
|
|
@@ -662,7 +714,7 @@ export class HueTransport {
|
|
|
662
714
|
return;
|
|
663
715
|
this.logParts.delete(body);
|
|
664
716
|
const size = pending.files.reduce((total, file) => total + heldBytesOf(file.bytes), 0);
|
|
665
|
-
if (
|
|
717
|
+
if (size > this.heldRoom()) {
|
|
666
718
|
const tally = newTally();
|
|
667
719
|
for (const _file of pending.files)
|
|
668
720
|
countFallback(tally, "budget", pending.traceId);
|
|
@@ -673,6 +725,44 @@ export class HueTransport {
|
|
|
673
725
|
this.heldBytes += size;
|
|
674
726
|
this.held.set(queued, (this.held.get(queued) ?? 0) + size);
|
|
675
727
|
}
|
|
728
|
+
/** What the held-value budget has room for: values queued records hold and bytes unsettled
|
|
729
|
+
* uploads hold share it. */
|
|
730
|
+
heldRoom() {
|
|
731
|
+
return Math.max(0, MAX_HELD_BLOB_BYTES - this.heldBytes - (this.uploader?.pendingBytes() ?? 0));
|
|
732
|
+
}
|
|
733
|
+
/** An upload `record`'s export queued, as it is queued: flushes and the processors wait for it
|
|
734
|
+
* from now on, and the `shared` bytes it holds of the record's own are charged to the upload
|
|
735
|
+
* alone, so the held-value budget never counts them twice while the export goes on. A charge
|
|
736
|
+
* `released` before the record is finished is handed back to the record, which still holds
|
|
737
|
+
* them. */
|
|
738
|
+
uploadQueued(record, settled, shared, released) {
|
|
739
|
+
this.awaitUploads(settled);
|
|
740
|
+
const identity = this.mediaIdentity(record);
|
|
741
|
+
const staged = identity
|
|
742
|
+
? (this.media.get(identity) ?? []).reduce((total, file) => total + heldBytesOf(file.bytes), 0)
|
|
743
|
+
: 0;
|
|
744
|
+
const taken = this.heldByUploads.get(record) ?? 0;
|
|
745
|
+
const moved = Math.min(shared, (this.held.get(record) ?? 0) + staged - taken);
|
|
746
|
+
if (moved <= 0)
|
|
747
|
+
return;
|
|
748
|
+
this.heldBytes -= moved;
|
|
749
|
+
this.heldByUploads.set(record, taken + moved);
|
|
750
|
+
// Held weakly: a pending upload never keeps its finished record alive.
|
|
751
|
+
const queued = new WeakRef(record);
|
|
752
|
+
const handBack = () => {
|
|
753
|
+
const owner = queued.deref();
|
|
754
|
+
const still = owner && this.heldByUploads.get(owner);
|
|
755
|
+
if (!owner || !still)
|
|
756
|
+
return;
|
|
757
|
+
const back = Math.min(moved, still);
|
|
758
|
+
this.heldBytes += back;
|
|
759
|
+
if (still > back)
|
|
760
|
+
this.heldByUploads.set(owner, still - back);
|
|
761
|
+
else
|
|
762
|
+
this.heldByUploads.delete(owner);
|
|
763
|
+
};
|
|
764
|
+
void released.then(handBack, handBack);
|
|
765
|
+
}
|
|
676
766
|
/** Registers a span whose staged files are released if it is never exported. */
|
|
677
767
|
ownMedia(span, identity) {
|
|
678
768
|
if (this.mediaOwners.has(identity))
|
|
@@ -763,11 +853,19 @@ export class HueTransport {
|
|
|
763
853
|
return this.offloadsValues() && this.uploader?.available() === true;
|
|
764
854
|
}
|
|
765
855
|
/** @internal Exporter callback: a redacted span with the values its redaction left to the
|
|
766
|
-
* upload step placed:
|
|
767
|
-
|
|
856
|
+
* upload step placed: listed under `hue.blobs` with their uploads queued (each added to `late`
|
|
857
|
+
* once it settles, and charged in place of the `queued` record it came from), or cut as before.
|
|
858
|
+
* Never rejects. */
|
|
859
|
+
async offloadSpan(span, collector, upload, tally, late, queued) {
|
|
768
860
|
// A record without a readable context cannot name its trace: its values are cut.
|
|
769
861
|
const traceId = recordTraceId(span);
|
|
770
862
|
const valueBytes = this.limits.valueBytes;
|
|
863
|
+
// A record placed as a fallback counts its values there: its queued uploads are not added.
|
|
864
|
+
const own = {
|
|
865
|
+
tally: newTally(),
|
|
866
|
+
settled: [],
|
|
867
|
+
queued: (settled, shared, released) => this.uploadQueued(queued, settled, shared, released),
|
|
868
|
+
};
|
|
771
869
|
try {
|
|
772
870
|
if (!this.uploader)
|
|
773
871
|
throw new Error("Uploads are unavailable");
|
|
@@ -778,7 +876,10 @@ export class HueTransport {
|
|
|
778
876
|
upload,
|
|
779
877
|
tally,
|
|
780
878
|
memo: this.mediaMemo,
|
|
879
|
+
background: own,
|
|
781
880
|
});
|
|
881
|
+
if (own.settled.length)
|
|
882
|
+
late.settled.push(mergeWhenSettled(late.tally, own.tally, own.settled));
|
|
782
883
|
return { ...span, attributes: attributes };
|
|
783
884
|
}
|
|
784
885
|
catch {
|
|
@@ -789,10 +890,11 @@ export class HueTransport {
|
|
|
789
890
|
}
|
|
790
891
|
}
|
|
791
892
|
/** @internal Exporter callback: a redacted log record with the media its redaction left to
|
|
792
|
-
* the upload step placed: its attributes' and its body's files
|
|
793
|
-
*
|
|
794
|
-
*
|
|
795
|
-
|
|
893
|
+
* the upload step placed: its attributes' and its body's files listed under its own `hue.blobs`
|
|
894
|
+
* (keys `<attribute>#<pointer>`, `body#<pointer>`) with their uploads queued under the log
|
|
895
|
+
* record's trace (charged in place of the `queued` record); a file refused keeps its bytes
|
|
896
|
+
* inline, and the record is cut as before. Never rejects. */
|
|
897
|
+
async offloadLog(log, collector, upload, tally, late, queued) {
|
|
796
898
|
const traceId = log.spanContext?.traceId ?? "";
|
|
797
899
|
const valueBytes = this.limits.valueBytes;
|
|
798
900
|
const placed = (result) => ({
|
|
@@ -800,29 +902,39 @@ export class HueTransport {
|
|
|
800
902
|
attributes: result.attributes,
|
|
801
903
|
...(collector.body ? { body: result.body } : {}),
|
|
802
904
|
});
|
|
905
|
+
const own = {
|
|
906
|
+
tally: newTally(),
|
|
907
|
+
settled: [],
|
|
908
|
+
queued: (settled, shared, released) => this.uploadQueued(queued, settled, shared, released),
|
|
909
|
+
};
|
|
803
910
|
try {
|
|
804
911
|
if (!this.uploader)
|
|
805
912
|
throw new Error("Uploads are unavailable");
|
|
806
|
-
|
|
913
|
+
const result = await offloadRecord(log.attributes, collector, {
|
|
807
914
|
uploader: this.uploader,
|
|
808
915
|
traceId,
|
|
809
916
|
valueBytes,
|
|
810
917
|
upload,
|
|
811
918
|
tally,
|
|
812
919
|
memo: this.mediaMemo,
|
|
813
|
-
|
|
920
|
+
background: own,
|
|
921
|
+
});
|
|
922
|
+
if (own.settled.length)
|
|
923
|
+
late.settled.push(mergeWhenSettled(late.tally, own.tally, own.settled));
|
|
924
|
+
return placed(result);
|
|
814
925
|
}
|
|
815
926
|
catch {
|
|
816
927
|
return placed(fallbackRecord(log.attributes, collector, valueBytes, tally, traceId, this.mediaMemo));
|
|
817
928
|
}
|
|
818
929
|
}
|
|
819
930
|
/** @internal Exporter callback: counts an export's uploads and reports each kind of fallback
|
|
820
|
-
* once, as a warning naming its traces: the values were exported
|
|
821
|
-
|
|
931
|
+
* once, as a warning naming its traces: the values were exported cut as before, or, `late`,
|
|
932
|
+
* their uploads failed after their records listed them. */
|
|
933
|
+
reportOffload(tally, signal = "traces", late = false) {
|
|
822
934
|
this.uploaded += tally.uploaded;
|
|
823
935
|
for (const [reason, { count, traceIds }] of tally.fallbacks) {
|
|
824
936
|
this.uploadFallbacks += count;
|
|
825
|
-
this.issue(signal, "warning", count, FALLBACK_MESSAGES[reason], undefined, traceIds.size && traceIds.size <= MAX_ISSUE_TRACE_IDS ? [...traceIds] : undefined);
|
|
937
|
+
this.issue(signal, "warning", count, (late ? LATE_FALLBACK_MESSAGES : FALLBACK_MESSAGES)[reason], undefined, traceIds.size && traceIds.size <= MAX_ISSUE_TRACE_IDS ? [...traceIds] : undefined);
|
|
826
938
|
}
|
|
827
939
|
}
|
|
828
940
|
/** @internal Exporter callback: counts records the collector acknowledged. */
|
|
@@ -890,8 +1002,9 @@ export class HueTransport {
|
|
|
890
1002
|
}
|
|
891
1003
|
/** @internal Ends every rate-limit hold now and refuses new ones, once a shutdown's caller has
|
|
892
1004
|
* stopped waiting: the held records are reported lost to the rate limit, and no timer of theirs
|
|
893
|
-
* keeps the process running for the receiver's `Retry-After`.
|
|
894
|
-
* other starts: their
|
|
1005
|
+
* keeps the process running for the receiver's `Retry-After`. Queued and in-flight uploads end
|
|
1006
|
+
* too, and no other starts: their records' entries stay and the uploads count as failed, and
|
|
1007
|
+
* values not yet exported are exported cut, as before uploads existed. */
|
|
895
1008
|
releaseRateLimitHolds() {
|
|
896
1009
|
this.holdsReleased = true;
|
|
897
1010
|
for (const release of [...this.holds])
|
|
@@ -1015,27 +1128,67 @@ export class HueTransport {
|
|
|
1015
1128
|
this.endedSpans.delete(this.endedSpans.keys().next().value);
|
|
1016
1129
|
}
|
|
1017
1130
|
/**
|
|
1018
|
-
* Waits for the processors' and exporters' in-flight work
|
|
1131
|
+
* Waits for the processors' and exporters' in-flight work, then for the uploads their exports
|
|
1132
|
+
* queued; drain the providers first. Only the sends are serialized: a flush sends its records
|
|
1133
|
+
* once an earlier flush sent its own, never after that flush's uploads.
|
|
1019
1134
|
*
|
|
1020
1135
|
* @throws HueExportError when a new non-warning issue was recorded since the previous observation.
|
|
1021
1136
|
*/
|
|
1022
1137
|
flush() {
|
|
1138
|
+
return this.flushRecords().report;
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* @internal A flush in two steps: its send settles `sent`, then it waits on its own for the
|
|
1142
|
+
* uploads queued so far and settles `report` as {@link flush} does. `providers` (the client's
|
|
1143
|
+
* provider flush) runs first, before the flush joins the transport's sends, so a hung borrowed
|
|
1144
|
+
* provider never holds up the transport's own flushes. `sent` waits for it, though, and the
|
|
1145
|
+
* client starts each flush once the previous one's `sent` settles: it holds up the client's
|
|
1146
|
+
* later flushes.
|
|
1147
|
+
*/
|
|
1148
|
+
flushRecords(providers) {
|
|
1023
1149
|
const from = this.observedSequence;
|
|
1024
|
-
const
|
|
1025
|
-
this.
|
|
1150
|
+
const join = () => this.sendRecords();
|
|
1151
|
+
const sent = providers ? this.sendingRecords.run(true, providers).then(join, join) : join();
|
|
1152
|
+
const report = sent.then(async (processors) => {
|
|
1153
|
+
// Exports the processors did not start (a timer's, still in flight) queued uploads too.
|
|
1154
|
+
await this.drainUploads();
|
|
1155
|
+
return this.flushReport(from, processors);
|
|
1156
|
+
});
|
|
1157
|
+
return { sent, report };
|
|
1158
|
+
}
|
|
1159
|
+
/** Sends what the processors and exporters hold, once the previous flush's send ended. */
|
|
1160
|
+
sendRecords() {
|
|
1161
|
+
const send = () => this.sendingRecords.run(true, async () => {
|
|
1162
|
+
const processors = await Promise.allSettled([
|
|
1163
|
+
this.spanProcessor.forceFlush(),
|
|
1164
|
+
this.logRecordProcessor.forceFlush(),
|
|
1165
|
+
]);
|
|
1166
|
+
await Promise.all([this.traceExporter.forceFlush(), this.logExporter.forceFlush()]);
|
|
1167
|
+
return processors;
|
|
1168
|
+
});
|
|
1169
|
+
const sent = (this.sendPromise ?? Promise.resolve()).then(send, send);
|
|
1170
|
+
this.sendPromise = sent;
|
|
1026
1171
|
const clear = () => {
|
|
1027
|
-
if (this.
|
|
1028
|
-
this.
|
|
1172
|
+
if (this.sendPromise === sent)
|
|
1173
|
+
this.sendPromise = undefined;
|
|
1029
1174
|
};
|
|
1030
|
-
void
|
|
1031
|
-
return
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1175
|
+
void sent.then(clear, clear);
|
|
1176
|
+
return sent;
|
|
1177
|
+
}
|
|
1178
|
+
/** @internal Exporter callback: an export's queued uploads, reported once they all settle. */
|
|
1179
|
+
awaitUploads(report) {
|
|
1180
|
+
// Never an unhandled rejection, whatever the report does.
|
|
1181
|
+
const tracked = report.catch(() => { });
|
|
1182
|
+
this.uploadReports.add(tracked);
|
|
1183
|
+
void tracked.then(() => this.uploadReports.delete(tracked));
|
|
1184
|
+
const earlier = this.uploadsSettled;
|
|
1185
|
+
this.uploadsSettled = tracked.then(() => earlier);
|
|
1186
|
+
}
|
|
1187
|
+
/** Waits for the uploads queued so far; each ends within its lifetime, so this is bounded. */
|
|
1188
|
+
drainUploads() {
|
|
1189
|
+
return this.uploadsSettled;
|
|
1190
|
+
}
|
|
1191
|
+
flushReport(from, processors) {
|
|
1039
1192
|
for (const [index, result] of processors.entries())
|
|
1040
1193
|
if (result.status === "rejected") {
|
|
1041
1194
|
const signal = index === 0 ? "traces" : "logs";
|
|
@@ -1050,8 +1203,9 @@ export class HueTransport {
|
|
|
1050
1203
|
return report;
|
|
1051
1204
|
}
|
|
1052
1205
|
/**
|
|
1053
|
-
* Flushes and stops the processors
|
|
1054
|
-
* mode call it after shutting down the
|
|
1206
|
+
* Flushes and stops the processors, waits for queued uploads, then ends the uploader; records
|
|
1207
|
+
* emitted afterwards are dropped and counted. In attach mode call it after shutting down the
|
|
1208
|
+
* application's providers.
|
|
1055
1209
|
*
|
|
1056
1210
|
* @throws HueExportError when the final flush observed new failures.
|
|
1057
1211
|
*/
|
|
@@ -1067,7 +1221,10 @@ export class HueTransport {
|
|
|
1067
1221
|
this.logRecordProcessor.shutdown(),
|
|
1068
1222
|
]);
|
|
1069
1223
|
await Promise.allSettled([this.traceExporter.forceFlush(), this.logExporter.forceFlush()]);
|
|
1224
|
+
await this.drainUploads();
|
|
1070
1225
|
this.uploader?.close();
|
|
1226
|
+
// The reports of uploads the close ended land before shutdown resolves.
|
|
1227
|
+
await this.drainUploads();
|
|
1071
1228
|
for (const identity of this.media.keys())
|
|
1072
1229
|
this.releaseMediaIdentity(identity);
|
|
1073
1230
|
this.logParts.clear();
|
|
@@ -1266,7 +1423,24 @@ class ReportingExporter {
|
|
|
1266
1423
|
real.push(placeholder);
|
|
1267
1424
|
return real;
|
|
1268
1425
|
}
|
|
1426
|
+
/** Places and sends the records; the uploads their placement queued are reported once they
|
|
1427
|
+
* settle, after the records were sent, never awaited here. */
|
|
1269
1428
|
async exportRecords(records) {
|
|
1429
|
+
const late = { tally: newTally(), settled: [] };
|
|
1430
|
+
try {
|
|
1431
|
+
await this.placeAndSend(records, late);
|
|
1432
|
+
}
|
|
1433
|
+
finally {
|
|
1434
|
+
if (late.settled.length)
|
|
1435
|
+
this.reportWhenSettled(late);
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
/** Reports an export's uploads once they all settle. Built apart from the export, so a pending
|
|
1439
|
+
* upload never keeps the export's records alive. */
|
|
1440
|
+
reportWhenSettled(late) {
|
|
1441
|
+
this.transport.awaitUploads(Promise.all(late.settled).then(() => this.transport.reportOffload(late.tally, this.signal, true)));
|
|
1442
|
+
}
|
|
1443
|
+
async placeAndSend(records, late) {
|
|
1270
1444
|
// Completed records and placeholders travel in separate requests, completed records first,
|
|
1271
1445
|
// so a rejection count is always one kind of record's. Placeholders are advisory: losing one
|
|
1272
1446
|
// is a warning, never an export failure.
|
|
@@ -1288,21 +1462,17 @@ class ReportingExporter {
|
|
|
1288
1462
|
}
|
|
1289
1463
|
};
|
|
1290
1464
|
const resourceDeadline = Date.now() + this.transport.options.timeoutMillis;
|
|
1291
|
-
// How long this export has held its records for a rate-limited receiver
|
|
1292
|
-
// waits included.
|
|
1465
|
+
// How long this export has held its records for a rate-limited receiver.
|
|
1293
1466
|
const hold = { millis: 0 };
|
|
1294
|
-
// A span's values over the inline limit are
|
|
1295
|
-
// export's
|
|
1467
|
+
// A span's values over the inline limit are hashed and placed before it is measured, within
|
|
1468
|
+
// the export's hashing deadline; their uploads are queued, never awaited, and a placeholder's
|
|
1469
|
+
// values never upload.
|
|
1296
1470
|
const upload = {
|
|
1297
1471
|
deadline: Date.now() + uploadBudgetMillis(this.transport.options.timeoutMillis),
|
|
1298
|
-
wait: async (
|
|
1299
|
-
if (hold.millis + millis > MAX_RATE_LIMIT_HOLD_MILLIS)
|
|
1300
|
-
return false;
|
|
1301
|
-
hold.millis += millis;
|
|
1302
|
-
return this.transport.holdForRateLimit(millis);
|
|
1303
|
-
},
|
|
1472
|
+
wait: async () => false,
|
|
1304
1473
|
};
|
|
1305
1474
|
const tally = newTally();
|
|
1475
|
+
// Each record's attachments to upload once it is fitted, with the queued record holding them.
|
|
1306
1476
|
const helpers = new Map();
|
|
1307
1477
|
const placed = [];
|
|
1308
1478
|
for (const { record, markers } of this.ordered(records)) {
|
|
@@ -1351,8 +1521,8 @@ class ReportingExporter {
|
|
|
1351
1521
|
offload.body ||
|
|
1352
1522
|
offload.attachments?.length)
|
|
1353
1523
|
? this.signal === "traces"
|
|
1354
|
-
? this.transport.offloadSpan(redacted, offload, upload, tally)
|
|
1355
|
-
: this.transport.offloadLog(redacted, offload, upload, tally)
|
|
1524
|
+
? this.transport.offloadSpan(redacted, offload, upload, tally, late, record)
|
|
1525
|
+
: this.transport.offloadLog(redacted, offload, upload, tally, late, record)
|
|
1356
1526
|
: undefined;
|
|
1357
1527
|
placed.push({
|
|
1358
1528
|
record,
|
|
@@ -1403,7 +1573,7 @@ class ReportingExporter {
|
|
|
1403
1573
|
[BLOBS_KEY]: [...baseline, ...retained.map(mediaEntry)],
|
|
1404
1574
|
},
|
|
1405
1575
|
};
|
|
1406
|
-
helpers.set(redacted, { files: retained, baseline });
|
|
1576
|
+
helpers.set(redacted, { files: retained, baseline, source: record });
|
|
1407
1577
|
}
|
|
1408
1578
|
}
|
|
1409
1579
|
const bytes = 512 +
|
|
@@ -1605,7 +1775,11 @@ class ReportingExporter {
|
|
|
1605
1775
|
else
|
|
1606
1776
|
delete attributes[BLOBS_KEY];
|
|
1607
1777
|
}
|
|
1608
|
-
attachments = {
|
|
1778
|
+
attachments = {
|
|
1779
|
+
record: { ...fittedRecord, attributes },
|
|
1780
|
+
files,
|
|
1781
|
+
source: helper.source,
|
|
1782
|
+
};
|
|
1609
1783
|
remaining = [];
|
|
1610
1784
|
}
|
|
1611
1785
|
}
|
|
@@ -1623,15 +1797,17 @@ class ReportingExporter {
|
|
|
1623
1797
|
}
|
|
1624
1798
|
admitted.push({ fitted, attachments });
|
|
1625
1799
|
}
|
|
1626
|
-
// Decide every event/reference admission before
|
|
1627
|
-
//
|
|
1800
|
+
// Decide every event/reference admission before queueing helper uploads: a file fitting
|
|
1801
|
+
// removed is never uploaded. Their entries are written again as the fitted record listed
|
|
1802
|
+
// them, and the files upload after the records are sent.
|
|
1628
1803
|
const placed = await Promise.all(admitted.map(async ({ fitted, attachments }) => {
|
|
1629
1804
|
if (!attachments)
|
|
1630
1805
|
return fitted;
|
|
1631
1806
|
try {
|
|
1632
|
-
const uploaded = await this.transport.offloadSpan(attachments.record, { candidates: [], cut: [], attachments: attachments.files }, upload, tally);
|
|
1807
|
+
const uploaded = await this.transport.offloadSpan(attachments.record, { candidates: [], cut: [], attachments: attachments.files }, upload, tally, late, attachments.source);
|
|
1633
1808
|
measured.delete(fitted.record);
|
|
1634
|
-
//
|
|
1809
|
+
// The entries name each file's own type, as the fitted record's did; a file refused
|
|
1810
|
+
// when queued leaves its entry out, so the record is measured again, cheaply.
|
|
1635
1811
|
const final = await probe(uploaded);
|
|
1636
1812
|
if (final?.request)
|
|
1637
1813
|
measured.set(final.record, final.request);
|
|
@@ -1644,8 +1820,9 @@ class ReportingExporter {
|
|
|
1644
1820
|
return undefined;
|
|
1645
1821
|
}
|
|
1646
1822
|
}));
|
|
1647
|
-
//
|
|
1648
|
-
//
|
|
1823
|
+
// Values refused when their records were placed are reported once, before delivery failures
|
|
1824
|
+
// can consume the diagnostic callback's rate-limit slot; queued uploads are reported once
|
|
1825
|
+
// they settle. Placeholders never upload.
|
|
1649
1826
|
if (!advisory)
|
|
1650
1827
|
this.transport.reportOffload(tally, this.signal);
|
|
1651
1828
|
let batch = [];
|
package/dist/types.d.ts
CHANGED
|
@@ -136,11 +136,13 @@ export interface ExportReport {
|
|
|
136
136
|
instrumentationFailures: number;
|
|
137
137
|
/** Whole span attributes, inline media, attachments and opted-in file helper uploads stored
|
|
138
138
|
* apart from their records (or already held by Hue), each listed under the record's
|
|
139
|
-
* `hue.blobs
|
|
139
|
+
* `hue.blobs`; counted when the upload completes, which can be after the export that listed
|
|
140
|
+
* it. */
|
|
140
141
|
uploadedValues: number;
|
|
141
142
|
/** Whole-attribute, inline-media, attachment and opted-in file helper uploads that could not be
|
|
142
|
-
* attached
|
|
143
|
-
*
|
|
143
|
+
* attached, each with a warning issue: values exported in their fallback form (truncation,
|
|
144
|
+
* media digests, or file metadata), and values whose upload failed after their record was
|
|
145
|
+
* exported (their `hue.blobs` entries stay). */
|
|
144
146
|
uploadFallbacks: number;
|
|
145
147
|
}
|
|
146
148
|
/** Caller budget for {@link HueClient.flushSafe} and {@link HueClient.shutdownSafe}. */
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.19.1";
|
package/dist/version.js
CHANGED