@hue-run/sdk 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +8 -8
- package/EVALUATIONS.md +53 -2
- package/README.md +262 -37
- package/dist/attachment.d.ts +82 -0
- package/dist/attachment.js +225 -0
- package/dist/blobs.d.ts +41 -3
- package/dist/blobs.js +126 -41
- package/dist/client.d.ts +18 -0
- package/dist/client.js +148 -13
- package/dist/evals/client.d.ts +14 -1
- package/dist/evals/client.js +73 -9
- package/dist/evals/files.d.ts +11 -1
- package/dist/evals/files.js +160 -20
- package/dist/evals/media.d.ts +217 -0
- package/dist/evals/media.js +907 -0
- package/dist/evals/runner.js +130 -106
- package/dist/evals/types.d.ts +48 -0
- package/dist/evals.d.ts +2 -0
- package/dist/evals.js +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/inline-files.d.ts +131 -8
- package/dist/inline-files.js +565 -47
- package/dist/json-edits.d.ts +24 -0
- package/dist/json-edits.js +187 -0
- package/dist/privacy.d.ts +13 -0
- package/dist/privacy.js +84 -17
- package/dist/safety.d.ts +6 -2
- package/dist/safety.js +15 -3
- package/dist/snapshot.d.ts +1 -1
- package/dist/snapshot.js +123 -29
- package/dist/transport.d.ts +7 -0
- package/dist/transport.js +285 -55
- package/dist/types.d.ts +25 -6
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/transport.js
CHANGED
|
@@ -6,20 +6,75 @@ import { LogsExporterMetricsHelper, ProtobufLogsSerializer, ProtobufTraceSeriali
|
|
|
6
6
|
import { BatchSpanProcessor, } from "@opentelemetry/sdk-trace";
|
|
7
7
|
import { BatchLogRecordProcessor, } from "@opentelemetry/sdk-logs";
|
|
8
8
|
import { isInsecureOrigin, REDACTION_CONTEXT_UNITS, validateOptions } from "./config.js";
|
|
9
|
-
import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, countFallback, BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, MAX_HELD_BLOB_BYTES, newTally, offloadAttributes, uploadBudgetMillis, } from "./blobs.js";
|
|
9
|
+
import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, countFallback, BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, fallbackRecord, MAX_HELD_BLOB_BYTES, newTally, mediaKey, offloadAttributes, offloadRecord, uploadBudgetMillis, } from "./blobs.js";
|
|
10
10
|
import { advertisedLimits, BATCH_TARGET_BYTES, DEFAULT_RECEIVER_LIMITS, fitsWithoutCompression, } from "./limits.js";
|
|
11
11
|
import { announcesLiveSpan, LIVE_SPAN_INTERVAL_MILLIS, MAX_LIVE_SPANS, pendingPlaceholder, PLACEHOLDERS_HEADER, withoutPlaceholderMarkers, } from "./live-spans.js";
|
|
12
12
|
import { estimateRecordBytes, spanSource } from "./safety.js";
|
|
13
13
|
import { MediaDigestMemo, isMediaName } from "./inline-files.js";
|
|
14
|
+
import { locateParts } from "./attachment.js";
|
|
14
15
|
import { snapshotLog, snapshotSpan } from "./snapshot.js";
|
|
15
16
|
import { DROPPED_RECORDS_KEY, isContentKey, isTruncatedMarker, redactLog, redactSpan, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys, } from "./privacy.js";
|
|
16
17
|
import { sdkVersion } from "./version.js";
|
|
17
|
-
/**
|
|
18
|
-
function
|
|
18
|
+
/** A record's trace: a span's context, a log record's span identifiers; "" when unreadable. */
|
|
19
|
+
function recordTraceId(record) {
|
|
20
|
+
try {
|
|
21
|
+
return "spanContext" in record && typeof record.spanContext === "function"
|
|
22
|
+
? record.spanContext().traceId
|
|
23
|
+
: (record.spanContext?.traceId ?? "");
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return "";
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** The value of a record's attribute, or a log record's body, that holds placed attachments. */
|
|
30
|
+
function placedValue(record, attribute) {
|
|
31
|
+
if (attribute === "body" &&
|
|
32
|
+
!("spanContext" in record && typeof record.spanContext === "function"))
|
|
33
|
+
return record.body;
|
|
34
|
+
return record.attributes[attribute];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The staged files a record still authorizes. Each surviving `hue.file` event authorizes one
|
|
38
|
+
* staged request, including repeated digests. An attachment placed in a value is authorized by
|
|
39
|
+
* its own part in the value as exported: after redaction (`before` absent) each part is found in
|
|
40
|
+
* its value, and keyed at its pointer there, a part the redactor removed taking its upload with
|
|
41
|
+
* it; after fitting (`before`, the record as it was fitted) a part survives while fitting left its
|
|
42
|
+
* value as it was.
|
|
43
|
+
*/
|
|
44
|
+
function survivingMedia(files, record, tally, before) {
|
|
19
45
|
const consumed = new Set();
|
|
20
|
-
const traceId =
|
|
46
|
+
const traceId = recordTraceId(record);
|
|
47
|
+
const events = "events" in record ? record.events : [];
|
|
48
|
+
const located = new Map();
|
|
49
|
+
const locate = (attribute) => {
|
|
50
|
+
let parts = located.get(attribute);
|
|
51
|
+
if (parts)
|
|
52
|
+
return [];
|
|
53
|
+
let value = placedValue(record, attribute);
|
|
54
|
+
try {
|
|
55
|
+
if (typeof value === "string")
|
|
56
|
+
value = JSON.parse(value);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
value = undefined;
|
|
60
|
+
}
|
|
61
|
+
parts = locateParts(value, files.filter((file) => file.attribute === attribute)).map(({ file, pointer, name }) => {
|
|
62
|
+
const { name: _staged, ...rest } = file;
|
|
63
|
+
return { ...rest, key: `${attribute}#${pointer}`, ...(name ? { name } : {}) };
|
|
64
|
+
});
|
|
65
|
+
located.set(attribute, parts);
|
|
66
|
+
return parts;
|
|
67
|
+
};
|
|
21
68
|
return files.flatMap((file) => {
|
|
22
|
-
|
|
69
|
+
if (file.attribute !== undefined) {
|
|
70
|
+
if (!before)
|
|
71
|
+
return locate(file.attribute);
|
|
72
|
+
if (placedValue(record, file.attribute) === placedValue(before, file.attribute))
|
|
73
|
+
return [file];
|
|
74
|
+
countFallback(tally, "budget", traceId);
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
const index = events.findIndex((event, index) => !consumed.has(index) &&
|
|
23
78
|
event.name === "hue.file" &&
|
|
24
79
|
(!file.eventTime ||
|
|
25
80
|
(event.time[0] === file.eventTime[0] && event.time[1] === file.eventTime[1])) &&
|
|
@@ -31,7 +86,7 @@ function survivingMedia(files, span, tally) {
|
|
|
31
86
|
return [];
|
|
32
87
|
}
|
|
33
88
|
consumed.add(index);
|
|
34
|
-
const name =
|
|
89
|
+
const name = events[index].attributes?.["hue.file.name"];
|
|
35
90
|
return [
|
|
36
91
|
{
|
|
37
92
|
bytes: file.bytes,
|
|
@@ -44,7 +99,7 @@ function survivingMedia(files, span, tally) {
|
|
|
44
99
|
});
|
|
45
100
|
}
|
|
46
101
|
function mediaEntry(file) {
|
|
47
|
-
return blobEntry(
|
|
102
|
+
return blobEntry(mediaKey(file), file.digest.sha256, file.digest.size, file.contentType, file.name);
|
|
48
103
|
}
|
|
49
104
|
/**
|
|
50
105
|
* How long one export holds its records for a receiver that refused them for their rate (HTTP 429)
|
|
@@ -169,6 +224,10 @@ export class HueTransport {
|
|
|
169
224
|
// Context identity also matches safe API wrappers to the provider's ended span. Abandoned
|
|
170
225
|
// spans retain no span objects, and share the 2048-span / 128 MiB holding bounds.
|
|
171
226
|
media = new Map();
|
|
227
|
+
// Attachments placed in a log record's body, from `emit` until the record is admitted, and
|
|
228
|
+
// the admitted records' own.
|
|
229
|
+
logParts = new Map();
|
|
230
|
+
logMedia = new WeakMap();
|
|
172
231
|
mediaOwners = new Map();
|
|
173
232
|
mediaTimer;
|
|
174
233
|
mediaFinalizer = new FinalizationRegistry(({ identity, token }) => {
|
|
@@ -193,9 +252,10 @@ export class HueTransport {
|
|
|
193
252
|
...(offload ? { offload } : {}),
|
|
194
253
|
});
|
|
195
254
|
});
|
|
196
|
-
this.logExporter = new ReportingExporter(this, "logs", ProtobufLogsSerializer, LogsExporterMetricsHelper, (log, cache) => redactLog(log, this.options, cache, {
|
|
255
|
+
this.logExporter = new ReportingExporter(this, "logs", ProtobufLogsSerializer, LogsExporterMetricsHelper, (log, cache, offload) => redactLog(log, this.options, cache, {
|
|
197
256
|
valueBytes: this.limits.valueBytes,
|
|
198
257
|
cut: this.admissionCuts.get(log),
|
|
258
|
+
...(offload ? { offload } : {}),
|
|
199
259
|
}));
|
|
200
260
|
if (this.options.enabled !== false)
|
|
201
261
|
this.uploader = new BlobUploader({
|
|
@@ -275,9 +335,21 @@ export class HueTransport {
|
|
|
275
335
|
let admitted;
|
|
276
336
|
try {
|
|
277
337
|
const queued = this.enqueue("logs", log);
|
|
278
|
-
if (!queued)
|
|
338
|
+
if (!queued) {
|
|
339
|
+
const pending = log.body !== null && typeof log.body === "object"
|
|
340
|
+
? this.logParts.get(log.body)
|
|
341
|
+
: undefined;
|
|
342
|
+
if (pending) {
|
|
343
|
+
const tally = newTally();
|
|
344
|
+
for (const _file of pending.files)
|
|
345
|
+
countFallback(tally, "budget", pending.traceId);
|
|
346
|
+
this.reportOffload(tally, "logs");
|
|
347
|
+
this.logParts.delete(log.body);
|
|
348
|
+
}
|
|
279
349
|
return;
|
|
350
|
+
}
|
|
280
351
|
admitted = queued;
|
|
352
|
+
this.admitLogParts(log, queued);
|
|
281
353
|
logs.onEmit(queued);
|
|
282
354
|
}
|
|
283
355
|
catch {
|
|
@@ -316,7 +388,7 @@ export class HueTransport {
|
|
|
316
388
|
// and its full size could drop the record before that cut ever ran. A span's own value
|
|
317
389
|
// that export may upload is held whole instead, within the held-value budget.
|
|
318
390
|
const valueUnits = this.limits.valueBytes + REDACTION_CONTEXT_UNITS;
|
|
319
|
-
const hold =
|
|
391
|
+
const hold = !advisory && this.holdsValues()
|
|
320
392
|
? {
|
|
321
393
|
valueBytes: this.limits.valueBytes,
|
|
322
394
|
budget: Math.max(0, MAX_HELD_BLOB_BYTES - this.heldBytes),
|
|
@@ -325,7 +397,7 @@ export class HueTransport {
|
|
|
325
397
|
: undefined;
|
|
326
398
|
const snapshot = signal === "traces"
|
|
327
399
|
? snapshotSpan(record, remaining, this.options.captureContent, valueUnits, hold, this.mediaMemo)
|
|
328
|
-
: snapshotLog(record, remaining, this.options.captureContent, valueUnits);
|
|
400
|
+
: snapshotLog(record, remaining, this.options.captureContent, valueUnits, hold, this.mediaMemo);
|
|
329
401
|
if (snapshot.cut.size)
|
|
330
402
|
this.admissionCuts.set(snapshot.record, snapshot.cut);
|
|
331
403
|
if (snapshot.digested.size)
|
|
@@ -442,6 +514,7 @@ export class HueTransport {
|
|
|
442
514
|
finish(signal, records) {
|
|
443
515
|
for (const record of records) {
|
|
444
516
|
this.releaseMedia(record);
|
|
517
|
+
this.logMedia.delete(record);
|
|
445
518
|
const pending = signal === "traces" ? this.spans : this.logs;
|
|
446
519
|
this.pendingBytes -= pending.get(record) ?? 0;
|
|
447
520
|
pending.delete(record);
|
|
@@ -484,19 +557,122 @@ export class HueTransport {
|
|
|
484
557
|
const bytes = Uint8Array.from(file.bytes);
|
|
485
558
|
files.push({ ...file, bytes });
|
|
486
559
|
this.media.set(identity, files);
|
|
487
|
-
|
|
488
|
-
const source = spanSource(span);
|
|
489
|
-
const token = {};
|
|
490
|
-
this.mediaOwners.set(identity, { source: new WeakRef(source), token });
|
|
491
|
-
this.mediaFinalizer.register(source, { identity, token }, token);
|
|
492
|
-
if (!this.mediaTimer) {
|
|
493
|
-
this.mediaTimer = setInterval(() => this.sweepMedia(), LIVE_SPAN_INTERVAL_MILLIS);
|
|
494
|
-
this.mediaTimer.unref();
|
|
495
|
-
}
|
|
496
|
-
}
|
|
560
|
+
this.ownMedia(span, identity);
|
|
497
561
|
this.heldBytes += bytes.byteLength;
|
|
498
562
|
return true;
|
|
499
563
|
}
|
|
564
|
+
/**
|
|
565
|
+
* @internal Stages the attachments placed in a span's `attribute`, replacing those an earlier
|
|
566
|
+
* value of the attribute placed: each is uploaded at export when its part survives there. A
|
|
567
|
+
* file that cannot be held is counted as not uploaded; it keeps its digest part.
|
|
568
|
+
*/
|
|
569
|
+
stageParts(span, attribute, files) {
|
|
570
|
+
if (this.closed || !this.offloadsValues())
|
|
571
|
+
return;
|
|
572
|
+
const traceId = span.spanContext().traceId;
|
|
573
|
+
const identity = this.mediaIdentity(span);
|
|
574
|
+
const previous = this.media.get(identity);
|
|
575
|
+
if (previous?.some((file) => file.attribute === attribute)) {
|
|
576
|
+
const kept = previous.filter((file) => {
|
|
577
|
+
if (file.attribute !== attribute)
|
|
578
|
+
return true;
|
|
579
|
+
this.heldBytes -= file.bytes.byteLength;
|
|
580
|
+
return false;
|
|
581
|
+
});
|
|
582
|
+
if (kept.length)
|
|
583
|
+
this.media.set(identity, kept);
|
|
584
|
+
else
|
|
585
|
+
this.media.delete(identity);
|
|
586
|
+
}
|
|
587
|
+
const tally = newTally();
|
|
588
|
+
for (const file of files) {
|
|
589
|
+
if (!file.bytes.byteLength)
|
|
590
|
+
continue;
|
|
591
|
+
const refused = this.uploader?.refusal();
|
|
592
|
+
if (!this.holdsValues() || refused) {
|
|
593
|
+
countFallback(tally, refused ?? "unsupported", traceId);
|
|
594
|
+
continue;
|
|
595
|
+
}
|
|
596
|
+
this.sweepMedia();
|
|
597
|
+
const staged = this.media.get(identity) ?? [];
|
|
598
|
+
if (staged.length >= MAX_BLOB_ENTRIES ||
|
|
599
|
+
(!this.media.has(identity) && this.media.size >= 2048) ||
|
|
600
|
+
this.heldBytes + file.bytes.byteLength > MAX_HELD_BLOB_BYTES) {
|
|
601
|
+
countFallback(tally, "budget", traceId);
|
|
602
|
+
continue;
|
|
603
|
+
}
|
|
604
|
+
// Attachment bytes are the attachment's own copy: held as they are.
|
|
605
|
+
staged.push({ ...file, attribute });
|
|
606
|
+
this.media.set(identity, staged);
|
|
607
|
+
this.heldBytes += file.bytes.byteLength;
|
|
608
|
+
this.ownMedia(span, identity);
|
|
609
|
+
}
|
|
610
|
+
if (tally.fallbacks.size)
|
|
611
|
+
this.reportOffload(tally);
|
|
612
|
+
}
|
|
613
|
+
/**
|
|
614
|
+
* @internal Stages the attachments placed in a log record's body until the record is emitted
|
|
615
|
+
* (`body` is the object passed to `emit`); {@link unstageLogParts} drops what no processor of
|
|
616
|
+
* this transport admitted.
|
|
617
|
+
*/
|
|
618
|
+
stageLogParts(body, traceId, files) {
|
|
619
|
+
if (this.closed || !this.offloadsValues())
|
|
620
|
+
return;
|
|
621
|
+
const tally = newTally();
|
|
622
|
+
const staged = [];
|
|
623
|
+
for (const file of files) {
|
|
624
|
+
if (!file.bytes.byteLength)
|
|
625
|
+
continue;
|
|
626
|
+
const refused = this.uploader?.refusal();
|
|
627
|
+
if (!this.holdsValues() || refused)
|
|
628
|
+
countFallback(tally, refused ?? "unsupported", traceId);
|
|
629
|
+
else if (staged.length >= MAX_BLOB_ENTRIES)
|
|
630
|
+
countFallback(tally, "budget", traceId);
|
|
631
|
+
else
|
|
632
|
+
staged.push({ ...file, attribute: "body" });
|
|
633
|
+
}
|
|
634
|
+
if (tally.fallbacks.size)
|
|
635
|
+
this.reportOffload(tally, "logs");
|
|
636
|
+
if (staged.length)
|
|
637
|
+
this.logParts.set(body, { traceId, files: staged });
|
|
638
|
+
}
|
|
639
|
+
/** @internal Drops a log record's staged attachments once it was emitted. */
|
|
640
|
+
unstageLogParts(body) {
|
|
641
|
+
this.logParts.delete(body);
|
|
642
|
+
}
|
|
643
|
+
/** Holds the staged attachments of a log record being admitted with the queued record, within
|
|
644
|
+
* the held-value budget; those the budget refuses are counted as not uploaded. */
|
|
645
|
+
admitLogParts(log, queued) {
|
|
646
|
+
const body = log.body;
|
|
647
|
+
const pending = body !== null && typeof body === "object" ? this.logParts.get(body) : undefined;
|
|
648
|
+
if (!pending)
|
|
649
|
+
return;
|
|
650
|
+
this.logParts.delete(body);
|
|
651
|
+
const size = pending.files.reduce((total, file) => total + file.bytes.byteLength, 0);
|
|
652
|
+
if (this.heldBytes + size > MAX_HELD_BLOB_BYTES) {
|
|
653
|
+
const tally = newTally();
|
|
654
|
+
for (const _file of pending.files)
|
|
655
|
+
countFallback(tally, "budget", pending.traceId);
|
|
656
|
+
this.reportOffload(tally, "logs");
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
this.logMedia.set(queued, pending.files);
|
|
660
|
+
this.heldBytes += size;
|
|
661
|
+
this.held.set(queued, (this.held.get(queued) ?? 0) + size);
|
|
662
|
+
}
|
|
663
|
+
/** Registers a span whose staged files are released if it is never exported. */
|
|
664
|
+
ownMedia(span, identity) {
|
|
665
|
+
if (this.mediaOwners.has(identity))
|
|
666
|
+
return;
|
|
667
|
+
const source = spanSource(span);
|
|
668
|
+
const token = {};
|
|
669
|
+
this.mediaOwners.set(identity, { source: new WeakRef(source), token });
|
|
670
|
+
this.mediaFinalizer.register(source, { identity, token }, token);
|
|
671
|
+
if (!this.mediaTimer) {
|
|
672
|
+
this.mediaTimer = setInterval(() => this.sweepMedia(), LIVE_SPAN_INTERVAL_MILLIS);
|
|
673
|
+
this.mediaTimer.unref();
|
|
674
|
+
}
|
|
675
|
+
}
|
|
500
676
|
mediaIdentity(record) {
|
|
501
677
|
if (!("spanContext" in record) || typeof record.spanContext !== "function")
|
|
502
678
|
return undefined;
|
|
@@ -552,8 +728,11 @@ export class HueTransport {
|
|
|
552
728
|
}
|
|
553
729
|
}
|
|
554
730
|
}
|
|
555
|
-
/** @internal Staged requests are matched against surviving events before upload. */
|
|
731
|
+
/** @internal Staged requests are matched against surviving events and parts before upload. */
|
|
556
732
|
mediaAttachments(record) {
|
|
733
|
+
const logged = this.logMedia.get(record);
|
|
734
|
+
if (logged)
|
|
735
|
+
return logged;
|
|
557
736
|
const identity = this.mediaIdentity(record);
|
|
558
737
|
return identity ? (this.media.get(identity) ?? []) : [];
|
|
559
738
|
}
|
|
@@ -573,13 +752,8 @@ export class HueTransport {
|
|
|
573
752
|
/** @internal Exporter callback: a redacted span with the values its redaction left to the
|
|
574
753
|
* upload step placed: uploaded and listed under `hue.blobs`, or cut as before. Never rejects. */
|
|
575
754
|
async offloadSpan(span, collector, upload, tally) {
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
traceId = span.spanContext().traceId;
|
|
579
|
-
}
|
|
580
|
-
catch {
|
|
581
|
-
// A span without a readable context cannot name its trace: its values are cut.
|
|
582
|
-
}
|
|
755
|
+
// A record without a readable context cannot name its trace: its values are cut.
|
|
756
|
+
const traceId = recordTraceId(span);
|
|
583
757
|
const valueBytes = this.limits.valueBytes;
|
|
584
758
|
try {
|
|
585
759
|
if (!this.uploader)
|
|
@@ -601,13 +775,41 @@ export class HueTransport {
|
|
|
601
775
|
};
|
|
602
776
|
}
|
|
603
777
|
}
|
|
778
|
+
/** @internal Exporter callback: a redacted log record with the media its redaction left to
|
|
779
|
+
* the upload step placed: its attributes' and its body's files uploaded and listed under its own
|
|
780
|
+
* `hue.blobs` (keys `<attribute>#<pointer>`, `body#<pointer>`), under the log record's trace;
|
|
781
|
+
* a file not uploaded keeps its bytes inline, and the record is cut as before. Never rejects. */
|
|
782
|
+
async offloadLog(log, collector, upload, tally) {
|
|
783
|
+
const traceId = log.spanContext?.traceId ?? "";
|
|
784
|
+
const valueBytes = this.limits.valueBytes;
|
|
785
|
+
const placed = (result) => ({
|
|
786
|
+
...log,
|
|
787
|
+
attributes: result.attributes,
|
|
788
|
+
...(collector.body ? { body: result.body } : {}),
|
|
789
|
+
});
|
|
790
|
+
try {
|
|
791
|
+
if (!this.uploader)
|
|
792
|
+
throw new Error("Uploads are unavailable");
|
|
793
|
+
return placed(await offloadRecord(log.attributes, collector, {
|
|
794
|
+
uploader: this.uploader,
|
|
795
|
+
traceId,
|
|
796
|
+
valueBytes,
|
|
797
|
+
upload,
|
|
798
|
+
tally,
|
|
799
|
+
memo: this.mediaMemo,
|
|
800
|
+
}));
|
|
801
|
+
}
|
|
802
|
+
catch {
|
|
803
|
+
return placed(fallbackRecord(log.attributes, collector, valueBytes, tally, traceId, this.mediaMemo));
|
|
804
|
+
}
|
|
805
|
+
}
|
|
604
806
|
/** @internal Exporter callback: counts an export's uploads and reports each kind of fallback
|
|
605
807
|
* once, as a warning naming its traces: the values were exported, cut as before. */
|
|
606
|
-
reportOffload(tally) {
|
|
808
|
+
reportOffload(tally, signal = "traces") {
|
|
607
809
|
this.uploaded += tally.uploaded;
|
|
608
810
|
for (const [reason, { count, traceIds }] of tally.fallbacks) {
|
|
609
811
|
this.uploadFallbacks += count;
|
|
610
|
-
this.issue(
|
|
812
|
+
this.issue(signal, "warning", count, FALLBACK_MESSAGES[reason], undefined, traceIds.size && traceIds.size <= MAX_ISSUE_TRACE_IDS ? [...traceIds] : undefined);
|
|
611
813
|
}
|
|
612
814
|
}
|
|
613
815
|
/** @internal Exporter callback: counts records the collector acknowledged. */
|
|
@@ -848,6 +1050,7 @@ export class HueTransport {
|
|
|
848
1050
|
this.uploader?.close();
|
|
849
1051
|
for (const identity of this.media.keys())
|
|
850
1052
|
this.releaseMediaIdentity(identity);
|
|
1053
|
+
this.logParts.clear();
|
|
851
1054
|
this.mediaMemo.clear();
|
|
852
1055
|
}
|
|
853
1056
|
})();
|
|
@@ -988,6 +1191,10 @@ function shedContent(record, signal, shed) {
|
|
|
988
1191
|
: span.links,
|
|
989
1192
|
};
|
|
990
1193
|
}
|
|
1194
|
+
/** Whether a trace id can name an upload's trace: 32 lowercase hex digits, not all zero. */
|
|
1195
|
+
function isUploadTraceId(traceId) {
|
|
1196
|
+
return typeof traceId === "string" && /^[0-9a-f]{32}$/.test(traceId) && !/^0+$/.test(traceId);
|
|
1197
|
+
}
|
|
991
1198
|
class ReportingExporter {
|
|
992
1199
|
transport;
|
|
993
1200
|
signal;
|
|
@@ -1095,27 +1302,50 @@ class ReportingExporter {
|
|
|
1095
1302
|
clearTimeout(timer);
|
|
1096
1303
|
}
|
|
1097
1304
|
}
|
|
1098
|
-
const
|
|
1099
|
-
|
|
1305
|
+
const uploads = !markers && this.transport.offloadsValues();
|
|
1306
|
+
// A log record's media upload under its own trace: one without a trace is not offered.
|
|
1307
|
+
const offload = uploads &&
|
|
1308
|
+
(this.signal === "traces" ||
|
|
1309
|
+
isUploadTraceId(record.spanContext?.traceId))
|
|
1310
|
+
? this.signal === "traces"
|
|
1311
|
+
? { candidates: [], cut: [] }
|
|
1312
|
+
: { candidates: [], cut: [], log: true }
|
|
1100
1313
|
: undefined;
|
|
1101
1314
|
const redacted = this.redact(record, cache, offload);
|
|
1102
|
-
|
|
1315
|
+
let files = uploads
|
|
1103
1316
|
? survivingMedia(this.transport.mediaAttachments(record), redacted, tally)
|
|
1104
1317
|
: [];
|
|
1318
|
+
if (this.signal === "logs" && files.length) {
|
|
1319
|
+
// A log record's attachments upload with the media its body holds, in the one step
|
|
1320
|
+
// that places a log record's files; a record without a trace uploads none.
|
|
1321
|
+
if (offload)
|
|
1322
|
+
offload.attachments = files;
|
|
1323
|
+
else
|
|
1324
|
+
for (let index = 0; index < files.length; index++)
|
|
1325
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1326
|
+
files = [];
|
|
1327
|
+
}
|
|
1328
|
+
const offloaded = offload &&
|
|
1329
|
+
(offload.candidates.length ||
|
|
1330
|
+
offload.cut.length ||
|
|
1331
|
+
offload.body ||
|
|
1332
|
+
offload.attachments?.length)
|
|
1333
|
+
? this.signal === "traces"
|
|
1334
|
+
? this.transport.offloadSpan(redacted, offload, upload, tally)
|
|
1335
|
+
: this.transport.offloadLog(redacted, offload, upload, tally)
|
|
1336
|
+
: undefined;
|
|
1105
1337
|
placed.push({
|
|
1106
1338
|
record,
|
|
1107
1339
|
files,
|
|
1108
1340
|
...(markers ? { markers } : {}),
|
|
1109
|
-
redacted:
|
|
1110
|
-
? this.transport.offloadSpan(redacted, offload, upload, tally)
|
|
1111
|
-
: redacted,
|
|
1341
|
+
redacted: offloaded ?? redacted,
|
|
1112
1342
|
});
|
|
1113
1343
|
}
|
|
1114
1344
|
catch {
|
|
1115
|
-
if (!markers && this.
|
|
1345
|
+
if (!markers && this.transport.offloadsValues()) {
|
|
1116
1346
|
const files = this.transport.mediaAttachments(record);
|
|
1117
1347
|
for (let index = 0; index < files.length; index++)
|
|
1118
|
-
countFallback(tally, "budget", record
|
|
1348
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1119
1349
|
}
|
|
1120
1350
|
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
|
|
1121
1351
|
}
|
|
@@ -1137,20 +1367,19 @@ class ReportingExporter {
|
|
|
1137
1367
|
redacted = { ...redacted, attributes: kept };
|
|
1138
1368
|
}
|
|
1139
1369
|
if (files.length) {
|
|
1140
|
-
const
|
|
1141
|
-
const listed = span.attributes[BLOBS_KEY];
|
|
1370
|
+
const listed = redacted.attributes[BLOBS_KEY];
|
|
1142
1371
|
const baseline = Array.isArray(listed)
|
|
1143
1372
|
? listed.filter((entry) => typeof entry === "string")
|
|
1144
1373
|
: [];
|
|
1145
1374
|
retained = files.slice(0, Math.max(0, MAX_BLOB_ENTRIES - baseline.length));
|
|
1146
1375
|
for (let index = retained.length; index < files.length; index++)
|
|
1147
|
-
countFallback(tally, "budget",
|
|
1376
|
+
countFallback(tally, "budget", recordTraceId(redacted));
|
|
1148
1377
|
if (retained.length) {
|
|
1149
|
-
// Fit the event and its reference together before sending customer bytes.
|
|
1378
|
+
// Fit the event or part and its reference together before sending customer bytes.
|
|
1150
1379
|
redacted = {
|
|
1151
1380
|
...redacted,
|
|
1152
1381
|
attributes: {
|
|
1153
|
-
...
|
|
1382
|
+
...redacted.attributes,
|
|
1154
1383
|
[BLOBS_KEY]: [...baseline, ...retained.map(mediaEntry)],
|
|
1155
1384
|
},
|
|
1156
1385
|
};
|
|
@@ -1166,7 +1395,7 @@ class ReportingExporter {
|
|
|
1166
1395
|
}
|
|
1167
1396
|
catch {
|
|
1168
1397
|
for (let index = 0; index < retained.length; index++)
|
|
1169
|
-
countFallback(tally, "budget", record
|
|
1398
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1170
1399
|
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
|
|
1171
1400
|
}
|
|
1172
1401
|
}
|
|
@@ -1331,15 +1560,16 @@ class ReportingExporter {
|
|
|
1331
1560
|
try {
|
|
1332
1561
|
fitted = await fit(record);
|
|
1333
1562
|
if (fitted && helper) {
|
|
1334
|
-
const
|
|
1335
|
-
const
|
|
1563
|
+
const fittedRecord = fitted.record;
|
|
1564
|
+
const traceId = recordTraceId(fittedRecord);
|
|
1565
|
+
const matched = survivingMedia(remaining, fittedRecord, tally, record);
|
|
1336
1566
|
remaining = matched;
|
|
1337
|
-
const listed =
|
|
1567
|
+
const listed = fittedRecord.attributes[BLOBS_KEY];
|
|
1338
1568
|
const entries = Array.isArray(listed) ? [...listed] : [];
|
|
1339
1569
|
const files = matched.filter((file) => {
|
|
1340
1570
|
const index = entries.indexOf(mediaEntry(file), helper.baseline.length);
|
|
1341
1571
|
if (index < 0) {
|
|
1342
|
-
countFallback(tally, "budget",
|
|
1572
|
+
countFallback(tally, "budget", traceId);
|
|
1343
1573
|
return false;
|
|
1344
1574
|
}
|
|
1345
1575
|
entries.splice(index, 1);
|
|
@@ -1347,7 +1577,7 @@ class ReportingExporter {
|
|
|
1347
1577
|
});
|
|
1348
1578
|
remaining = files;
|
|
1349
1579
|
// Drop provisional entries, retaining only the original references after fitting.
|
|
1350
|
-
const attributes = { ...
|
|
1580
|
+
const attributes = { ...fittedRecord.attributes };
|
|
1351
1581
|
if (Array.isArray(listed)) {
|
|
1352
1582
|
const baseline = listed.slice(0, helper.baseline.length);
|
|
1353
1583
|
if (baseline.length)
|
|
@@ -1355,19 +1585,19 @@ class ReportingExporter {
|
|
|
1355
1585
|
else
|
|
1356
1586
|
delete attributes[BLOBS_KEY];
|
|
1357
1587
|
}
|
|
1358
|
-
attachments = {
|
|
1588
|
+
attachments = { record: { ...fittedRecord, attributes }, files };
|
|
1359
1589
|
remaining = [];
|
|
1360
1590
|
}
|
|
1361
1591
|
}
|
|
1362
1592
|
catch {
|
|
1363
1593
|
for (let index = 0; index < remaining.length; index++)
|
|
1364
|
-
countFallback(tally, "budget", record
|
|
1594
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1365
1595
|
invalid(advisory, "Telemetry record could not be serialized", record);
|
|
1366
1596
|
continue;
|
|
1367
1597
|
}
|
|
1368
1598
|
if (!fitted) {
|
|
1369
1599
|
for (let index = 0; index < remaining.length; index++)
|
|
1370
|
-
countFallback(tally, "budget", record
|
|
1600
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1371
1601
|
invalid(advisory, OVER_REQUEST_LIMIT, record);
|
|
1372
1602
|
continue;
|
|
1373
1603
|
}
|
|
@@ -1379,7 +1609,7 @@ class ReportingExporter {
|
|
|
1379
1609
|
if (!attachments)
|
|
1380
1610
|
return fitted;
|
|
1381
1611
|
try {
|
|
1382
|
-
const uploaded = await this.transport.offloadSpan(attachments.
|
|
1612
|
+
const uploaded = await this.transport.offloadSpan(attachments.record, { candidates: [], cut: [], attachments: attachments.files }, upload, tally);
|
|
1383
1613
|
measured.delete(fitted.record);
|
|
1384
1614
|
// Actual references can differ when an existing object supplies its media type.
|
|
1385
1615
|
const final = await probe(uploaded);
|
|
@@ -1397,7 +1627,7 @@ class ReportingExporter {
|
|
|
1397
1627
|
// Preparation includes explicit file uploads. Report their outcome once, before delivery
|
|
1398
1628
|
// failures can consume the diagnostic callback's rate-limit slot. Placeholders never upload.
|
|
1399
1629
|
if (!advisory)
|
|
1400
|
-
this.transport.reportOffload(tally);
|
|
1630
|
+
this.transport.reportOffload(tally, this.signal);
|
|
1401
1631
|
let batch = [];
|
|
1402
1632
|
let batchBytes = 0;
|
|
1403
1633
|
for (const fitted of placed) {
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { Attributes, Context, Span, SpanKind, Tracer, TracerProvider } from "@opentelemetry/api";
|
|
2
|
+
import type { AttributeValue } from "@opentelemetry/api";
|
|
2
3
|
import type { LoggerProvider } from "@opentelemetry/api-logs";
|
|
4
|
+
import type { Attachment } from "./attachment.js";
|
|
3
5
|
/**
|
|
4
6
|
* JSON-compatible data: the shape content has on the wire after the helpers encode it. Helpers
|
|
5
7
|
* accept `unknown` and validate at runtime; values that are not plain JSON are omitted with an
|
|
@@ -8,6 +10,14 @@ import type { LoggerProvider } from "@opentelemetry/api-logs";
|
|
|
8
10
|
export type JsonValue = null | boolean | number | string | JsonValue[] | {
|
|
9
11
|
[key: string]: JsonValue;
|
|
10
12
|
};
|
|
13
|
+
/**
|
|
14
|
+
* JSON data that may hold an {@link Attachment} anywhere: what a span attribute passed to a helper
|
|
15
|
+
* may be besides an OpenTelemetry attribute value. Such a value is recorded as JSON text with each
|
|
16
|
+
* attachment as its digest part.
|
|
17
|
+
*/
|
|
18
|
+
export type AttachmentJson = Attachment | JsonValue | AttachmentJson[] | {
|
|
19
|
+
[key: string]: AttachmentJson;
|
|
20
|
+
};
|
|
11
21
|
/** Telemetry signal named by export reports, issues and processors. */
|
|
12
22
|
export type Signal = "traces" | "logs";
|
|
13
23
|
/** Options accepted by every Hue client and transport, whether enabled or disabled. */
|
|
@@ -124,11 +134,13 @@ export interface ExportReport {
|
|
|
124
134
|
pendingBytes: number;
|
|
125
135
|
/** Helper capture or instrumentation failures that omitted telemetry while preserving application results. */
|
|
126
136
|
instrumentationFailures: number;
|
|
127
|
-
/** Whole span attributes, inline media, and opted-in file helper uploads stored
|
|
128
|
-
* their
|
|
137
|
+
/** Whole span attributes, inline media, attachments and opted-in file helper uploads stored
|
|
138
|
+
* apart from their records (or already held by Hue), each listed under the record's
|
|
139
|
+
* `hue.blobs`. */
|
|
129
140
|
uploadedValues: number;
|
|
130
|
-
/** Whole-attribute, inline-media, and opted-in file helper uploads that could not be
|
|
131
|
-
* Content falls back to truncation, media digests, or file metadata, with a warning
|
|
141
|
+
/** Whole-attribute, inline-media, attachment and opted-in file helper uploads that could not be
|
|
142
|
+
* attached. Content falls back to truncation, media digests, or file metadata, with a warning
|
|
143
|
+
* issue. */
|
|
132
144
|
uploadFallbacks: number;
|
|
133
145
|
}
|
|
134
146
|
/** Caller budget for {@link HueClient.flushSafe} and {@link HueClient.shutdownSafe}. */
|
|
@@ -149,8 +161,15 @@ export interface SafeLifecycleResult {
|
|
|
149
161
|
export interface SpanOptions {
|
|
150
162
|
/** OpenTelemetry span kind; `INTERNAL` by default. */
|
|
151
163
|
kind?: SpanKind;
|
|
152
|
-
/**
|
|
153
|
-
|
|
164
|
+
/**
|
|
165
|
+
* Attributes set when the span starts. A value that is a JSON object, an array OpenTelemetry
|
|
166
|
+
* cannot hold (not of one primitive type) or an {@link Attachment} is recorded as JSON text, with
|
|
167
|
+
* each attachment as its digest part and its bytes uploaded; put files under `metadata` for Hue to
|
|
168
|
+
* list them as the span's metadata files.
|
|
169
|
+
*/
|
|
170
|
+
attributes?: {
|
|
171
|
+
[key: string]: AttributeValue | AttachmentJson | undefined;
|
|
172
|
+
};
|
|
154
173
|
/** Recorded as `gen_ai.conversation.id` on this span and inherited by nested helper spans. */
|
|
155
174
|
sessionId?: string;
|
|
156
175
|
/** Recorded as `user.id` on this span and inherited by nested helper spans. */
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.18.0";
|
package/dist/version.js
CHANGED