@hue-run/sdk 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +130 -8
- package/EVALUATIONS.md +56 -2
- package/README.md +299 -37
- package/dist/attachment.d.ts +82 -0
- package/dist/attachment.js +225 -0
- package/dist/blobs.d.ts +41 -3
- package/dist/blobs.js +126 -41
- package/dist/cli/import.d.ts +35 -0
- package/dist/cli/import.js +692 -0
- package/dist/cli/langfuse.d.ts +309 -0
- package/dist/cli/langfuse.js +1075 -0
- package/dist/client.d.ts +18 -0
- package/dist/client.js +148 -13
- package/dist/evals/client.d.ts +19 -1
- package/dist/evals/client.js +91 -11
- package/dist/evals/files.d.ts +11 -1
- package/dist/evals/files.js +248 -20
- package/dist/evals/media.d.ts +221 -0
- package/dist/evals/media.js +919 -0
- package/dist/evals/runner.js +130 -106
- package/dist/evals/types.d.ts +49 -0
- package/dist/evals.d.ts +2 -0
- package/dist/evals.js +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/inline-files.d.ts +140 -8
- package/dist/inline-files.js +572 -51
- package/dist/json-edits.d.ts +24 -0
- package/dist/json-edits.js +187 -0
- package/dist/privacy.d.ts +13 -0
- package/dist/privacy.js +84 -17
- package/dist/safety.d.ts +6 -2
- package/dist/safety.js +15 -3
- package/dist/setup/cli.js +7 -3
- package/dist/snapshot.d.ts +1 -1
- package/dist/snapshot.js +123 -29
- package/dist/transport.d.ts +10 -0
- package/dist/transport.js +296 -56
- package/dist/types.d.ts +25 -6
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/transport.js
CHANGED
|
@@ -6,20 +6,75 @@ import { LogsExporterMetricsHelper, ProtobufLogsSerializer, ProtobufTraceSeriali
|
|
|
6
6
|
import { BatchSpanProcessor, } from "@opentelemetry/sdk-trace";
|
|
7
7
|
import { BatchLogRecordProcessor, } from "@opentelemetry/sdk-logs";
|
|
8
8
|
import { isInsecureOrigin, REDACTION_CONTEXT_UNITS, validateOptions } from "./config.js";
|
|
9
|
-
import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, countFallback, BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, MAX_HELD_BLOB_BYTES, newTally, offloadAttributes, uploadBudgetMillis, } from "./blobs.js";
|
|
9
|
+
import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, countFallback, BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, fallbackRecord, MAX_HELD_BLOB_BYTES, newTally, mediaKey, offloadAttributes, offloadRecord, uploadBudgetMillis, } from "./blobs.js";
|
|
10
10
|
import { advertisedLimits, BATCH_TARGET_BYTES, DEFAULT_RECEIVER_LIMITS, fitsWithoutCompression, } from "./limits.js";
|
|
11
11
|
import { announcesLiveSpan, LIVE_SPAN_INTERVAL_MILLIS, MAX_LIVE_SPANS, pendingPlaceholder, PLACEHOLDERS_HEADER, withoutPlaceholderMarkers, } from "./live-spans.js";
|
|
12
12
|
import { estimateRecordBytes, spanSource } from "./safety.js";
|
|
13
13
|
import { MediaDigestMemo, isMediaName } from "./inline-files.js";
|
|
14
|
+
import { locateParts } from "./attachment.js";
|
|
14
15
|
import { snapshotLog, snapshotSpan } from "./snapshot.js";
|
|
15
16
|
import { DROPPED_RECORDS_KEY, isContentKey, isTruncatedMarker, redactLog, redactSpan, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys, } from "./privacy.js";
|
|
16
17
|
import { sdkVersion } from "./version.js";
|
|
17
|
-
/**
|
|
18
|
-
function
|
|
18
|
+
/** A record's trace: a span's context, a log record's span identifiers; "" when unreadable. */
|
|
19
|
+
function recordTraceId(record) {
|
|
20
|
+
try {
|
|
21
|
+
return "spanContext" in record && typeof record.spanContext === "function"
|
|
22
|
+
? record.spanContext().traceId
|
|
23
|
+
: (record.spanContext?.traceId ?? "");
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return "";
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** The value of a record's attribute, or a log record's body, that holds placed attachments. */
|
|
30
|
+
function placedValue(record, attribute) {
|
|
31
|
+
if (attribute === "body" &&
|
|
32
|
+
!("spanContext" in record && typeof record.spanContext === "function"))
|
|
33
|
+
return record.body;
|
|
34
|
+
return record.attributes[attribute];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The staged files a record still authorizes. Each surviving `hue.file` event authorizes one
|
|
38
|
+
* staged request, including repeated digests. An attachment placed in a value is authorized by
|
|
39
|
+
* its own part in the value as exported: after redaction (`before` absent) each part is found in
|
|
40
|
+
* its value, and keyed at its pointer there, a part the redactor removed taking its upload with
|
|
41
|
+
* it; after fitting (`before`, the record as it was fitted) a part survives while fitting left its
|
|
42
|
+
* value as it was.
|
|
43
|
+
*/
|
|
44
|
+
function survivingMedia(files, record, tally, before) {
|
|
19
45
|
const consumed = new Set();
|
|
20
|
-
const traceId =
|
|
46
|
+
const traceId = recordTraceId(record);
|
|
47
|
+
const events = "events" in record ? record.events : [];
|
|
48
|
+
const located = new Map();
|
|
49
|
+
const locate = (attribute) => {
|
|
50
|
+
let parts = located.get(attribute);
|
|
51
|
+
if (parts)
|
|
52
|
+
return [];
|
|
53
|
+
let value = placedValue(record, attribute);
|
|
54
|
+
try {
|
|
55
|
+
if (typeof value === "string")
|
|
56
|
+
value = JSON.parse(value);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
value = undefined;
|
|
60
|
+
}
|
|
61
|
+
parts = locateParts(value, files.filter((file) => file.attribute === attribute)).map(({ file, pointer, name }) => {
|
|
62
|
+
const { name: _staged, ...rest } = file;
|
|
63
|
+
return { ...rest, key: `${attribute}#${pointer}`, ...(name ? { name } : {}) };
|
|
64
|
+
});
|
|
65
|
+
located.set(attribute, parts);
|
|
66
|
+
return parts;
|
|
67
|
+
};
|
|
21
68
|
return files.flatMap((file) => {
|
|
22
|
-
|
|
69
|
+
if (file.attribute !== undefined) {
|
|
70
|
+
if (!before)
|
|
71
|
+
return locate(file.attribute);
|
|
72
|
+
if (placedValue(record, file.attribute) === placedValue(before, file.attribute))
|
|
73
|
+
return [file];
|
|
74
|
+
countFallback(tally, "budget", traceId);
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
const index = events.findIndex((event, index) => !consumed.has(index) &&
|
|
23
78
|
event.name === "hue.file" &&
|
|
24
79
|
(!file.eventTime ||
|
|
25
80
|
(event.time[0] === file.eventTime[0] && event.time[1] === file.eventTime[1])) &&
|
|
@@ -31,7 +86,7 @@ function survivingMedia(files, span, tally) {
|
|
|
31
86
|
return [];
|
|
32
87
|
}
|
|
33
88
|
consumed.add(index);
|
|
34
|
-
const name =
|
|
89
|
+
const name = events[index].attributes?.["hue.file.name"];
|
|
35
90
|
return [
|
|
36
91
|
{
|
|
37
92
|
bytes: file.bytes,
|
|
@@ -44,7 +99,7 @@ function survivingMedia(files, span, tally) {
|
|
|
44
99
|
});
|
|
45
100
|
}
|
|
46
101
|
function mediaEntry(file) {
|
|
47
|
-
return blobEntry(
|
|
102
|
+
return blobEntry(mediaKey(file), file.digest.sha256, file.digest.size, file.contentType, file.name);
|
|
48
103
|
}
|
|
49
104
|
/**
|
|
50
105
|
* How long one export holds its records for a receiver that refused them for their rate (HTTP 429)
|
|
@@ -129,6 +184,9 @@ export class HueTransport {
|
|
|
129
184
|
* over the request limit), reported on the trace's root span as `hue.sdk.dropped_records` when
|
|
130
185
|
* the root is exported, so Hue reads the trace as incomplete by that many records. */
|
|
131
186
|
droppedByTrace = new Map();
|
|
187
|
+
/** Whether a trace's root carries `hue.sdk.dropped_records`; off for a replay of recorded
|
|
188
|
+
* spans, which must send the same root on every run. */
|
|
189
|
+
marksDroppedRecords = true;
|
|
132
190
|
instrumentationFailures = 0;
|
|
133
191
|
diagnosticPending = false;
|
|
134
192
|
lastDiagnosticAt = -Infinity;
|
|
@@ -169,6 +227,10 @@ export class HueTransport {
|
|
|
169
227
|
// Context identity also matches safe API wrappers to the provider's ended span. Abandoned
|
|
170
228
|
// spans retain no span objects, and share the 2048-span / 128 MiB holding bounds.
|
|
171
229
|
media = new Map();
|
|
230
|
+
// Attachments placed in a log record's body, from `emit` until the record is admitted, and
|
|
231
|
+
// the admitted records' own.
|
|
232
|
+
logParts = new Map();
|
|
233
|
+
logMedia = new WeakMap();
|
|
172
234
|
mediaOwners = new Map();
|
|
173
235
|
mediaTimer;
|
|
174
236
|
mediaFinalizer = new FinalizationRegistry(({ identity, token }) => {
|
|
@@ -193,9 +255,10 @@ export class HueTransport {
|
|
|
193
255
|
...(offload ? { offload } : {}),
|
|
194
256
|
});
|
|
195
257
|
});
|
|
196
|
-
this.logExporter = new ReportingExporter(this, "logs", ProtobufLogsSerializer, LogsExporterMetricsHelper, (log, cache) => redactLog(log, this.options, cache, {
|
|
258
|
+
this.logExporter = new ReportingExporter(this, "logs", ProtobufLogsSerializer, LogsExporterMetricsHelper, (log, cache, offload) => redactLog(log, this.options, cache, {
|
|
197
259
|
valueBytes: this.limits.valueBytes,
|
|
198
260
|
cut: this.admissionCuts.get(log),
|
|
261
|
+
...(offload ? { offload } : {}),
|
|
199
262
|
}));
|
|
200
263
|
if (this.options.enabled !== false)
|
|
201
264
|
this.uploader = new BlobUploader({
|
|
@@ -275,9 +338,21 @@ export class HueTransport {
|
|
|
275
338
|
let admitted;
|
|
276
339
|
try {
|
|
277
340
|
const queued = this.enqueue("logs", log);
|
|
278
|
-
if (!queued)
|
|
341
|
+
if (!queued) {
|
|
342
|
+
const pending = log.body !== null && typeof log.body === "object"
|
|
343
|
+
? this.logParts.get(log.body)
|
|
344
|
+
: undefined;
|
|
345
|
+
if (pending) {
|
|
346
|
+
const tally = newTally();
|
|
347
|
+
for (const _file of pending.files)
|
|
348
|
+
countFallback(tally, "budget", pending.traceId);
|
|
349
|
+
this.reportOffload(tally, "logs");
|
|
350
|
+
this.logParts.delete(log.body);
|
|
351
|
+
}
|
|
279
352
|
return;
|
|
353
|
+
}
|
|
280
354
|
admitted = queued;
|
|
355
|
+
this.admitLogParts(log, queued);
|
|
281
356
|
logs.onEmit(queued);
|
|
282
357
|
}
|
|
283
358
|
catch {
|
|
@@ -316,7 +391,7 @@ export class HueTransport {
|
|
|
316
391
|
// and its full size could drop the record before that cut ever ran. A span's own value
|
|
317
392
|
// that export may upload is held whole instead, within the held-value budget.
|
|
318
393
|
const valueUnits = this.limits.valueBytes + REDACTION_CONTEXT_UNITS;
|
|
319
|
-
const hold =
|
|
394
|
+
const hold = !advisory && this.holdsValues()
|
|
320
395
|
? {
|
|
321
396
|
valueBytes: this.limits.valueBytes,
|
|
322
397
|
budget: Math.max(0, MAX_HELD_BLOB_BYTES - this.heldBytes),
|
|
@@ -325,7 +400,7 @@ export class HueTransport {
|
|
|
325
400
|
: undefined;
|
|
326
401
|
const snapshot = signal === "traces"
|
|
327
402
|
? snapshotSpan(record, remaining, this.options.captureContent, valueUnits, hold, this.mediaMemo)
|
|
328
|
-
: snapshotLog(record, remaining, this.options.captureContent, valueUnits);
|
|
403
|
+
: snapshotLog(record, remaining, this.options.captureContent, valueUnits, hold, this.mediaMemo);
|
|
329
404
|
if (snapshot.cut.size)
|
|
330
405
|
this.admissionCuts.set(snapshot.record, snapshot.cut);
|
|
331
406
|
if (snapshot.digested.size)
|
|
@@ -442,6 +517,7 @@ export class HueTransport {
|
|
|
442
517
|
finish(signal, records) {
|
|
443
518
|
for (const record of records) {
|
|
444
519
|
this.releaseMedia(record);
|
|
520
|
+
this.logMedia.delete(record);
|
|
445
521
|
const pending = signal === "traces" ? this.spans : this.logs;
|
|
446
522
|
this.pendingBytes -= pending.get(record) ?? 0;
|
|
447
523
|
pending.delete(record);
|
|
@@ -484,19 +560,122 @@ export class HueTransport {
|
|
|
484
560
|
const bytes = Uint8Array.from(file.bytes);
|
|
485
561
|
files.push({ ...file, bytes });
|
|
486
562
|
this.media.set(identity, files);
|
|
487
|
-
|
|
488
|
-
const source = spanSource(span);
|
|
489
|
-
const token = {};
|
|
490
|
-
this.mediaOwners.set(identity, { source: new WeakRef(source), token });
|
|
491
|
-
this.mediaFinalizer.register(source, { identity, token }, token);
|
|
492
|
-
if (!this.mediaTimer) {
|
|
493
|
-
this.mediaTimer = setInterval(() => this.sweepMedia(), LIVE_SPAN_INTERVAL_MILLIS);
|
|
494
|
-
this.mediaTimer.unref();
|
|
495
|
-
}
|
|
496
|
-
}
|
|
563
|
+
this.ownMedia(span, identity);
|
|
497
564
|
this.heldBytes += bytes.byteLength;
|
|
498
565
|
return true;
|
|
499
566
|
}
|
|
567
|
+
/**
|
|
568
|
+
* @internal Stages the attachments placed in a span's `attribute`, replacing those an earlier
|
|
569
|
+
* value of the attribute placed: each is uploaded at export when its part survives there. A
|
|
570
|
+
* file that cannot be held is counted as not uploaded; it keeps its digest part.
|
|
571
|
+
*/
|
|
572
|
+
stageParts(span, attribute, files) {
|
|
573
|
+
if (this.closed || !this.offloadsValues())
|
|
574
|
+
return;
|
|
575
|
+
const traceId = span.spanContext().traceId;
|
|
576
|
+
const identity = this.mediaIdentity(span);
|
|
577
|
+
const previous = this.media.get(identity);
|
|
578
|
+
if (previous?.some((file) => file.attribute === attribute)) {
|
|
579
|
+
const kept = previous.filter((file) => {
|
|
580
|
+
if (file.attribute !== attribute)
|
|
581
|
+
return true;
|
|
582
|
+
this.heldBytes -= file.bytes.byteLength;
|
|
583
|
+
return false;
|
|
584
|
+
});
|
|
585
|
+
if (kept.length)
|
|
586
|
+
this.media.set(identity, kept);
|
|
587
|
+
else
|
|
588
|
+
this.media.delete(identity);
|
|
589
|
+
}
|
|
590
|
+
const tally = newTally();
|
|
591
|
+
for (const file of files) {
|
|
592
|
+
if (!file.bytes.byteLength)
|
|
593
|
+
continue;
|
|
594
|
+
const refused = this.uploader?.refusal();
|
|
595
|
+
if (!this.holdsValues() || refused) {
|
|
596
|
+
countFallback(tally, refused ?? "unsupported", traceId);
|
|
597
|
+
continue;
|
|
598
|
+
}
|
|
599
|
+
this.sweepMedia();
|
|
600
|
+
const staged = this.media.get(identity) ?? [];
|
|
601
|
+
if (staged.length >= MAX_BLOB_ENTRIES ||
|
|
602
|
+
(!this.media.has(identity) && this.media.size >= 2048) ||
|
|
603
|
+
this.heldBytes + file.bytes.byteLength > MAX_HELD_BLOB_BYTES) {
|
|
604
|
+
countFallback(tally, "budget", traceId);
|
|
605
|
+
continue;
|
|
606
|
+
}
|
|
607
|
+
// Attachment bytes are the attachment's own copy: held as they are.
|
|
608
|
+
staged.push({ ...file, attribute });
|
|
609
|
+
this.media.set(identity, staged);
|
|
610
|
+
this.heldBytes += file.bytes.byteLength;
|
|
611
|
+
this.ownMedia(span, identity);
|
|
612
|
+
}
|
|
613
|
+
if (tally.fallbacks.size)
|
|
614
|
+
this.reportOffload(tally);
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* @internal Stages the attachments placed in a log record's body until the record is emitted
|
|
618
|
+
* (`body` is the object passed to `emit`); {@link unstageLogParts} drops what no processor of
|
|
619
|
+
* this transport admitted.
|
|
620
|
+
*/
|
|
621
|
+
stageLogParts(body, traceId, files) {
|
|
622
|
+
if (this.closed || !this.offloadsValues())
|
|
623
|
+
return;
|
|
624
|
+
const tally = newTally();
|
|
625
|
+
const staged = [];
|
|
626
|
+
for (const file of files) {
|
|
627
|
+
if (!file.bytes.byteLength)
|
|
628
|
+
continue;
|
|
629
|
+
const refused = this.uploader?.refusal();
|
|
630
|
+
if (!this.holdsValues() || refused)
|
|
631
|
+
countFallback(tally, refused ?? "unsupported", traceId);
|
|
632
|
+
else if (staged.length >= MAX_BLOB_ENTRIES)
|
|
633
|
+
countFallback(tally, "budget", traceId);
|
|
634
|
+
else
|
|
635
|
+
staged.push({ ...file, attribute: "body" });
|
|
636
|
+
}
|
|
637
|
+
if (tally.fallbacks.size)
|
|
638
|
+
this.reportOffload(tally, "logs");
|
|
639
|
+
if (staged.length)
|
|
640
|
+
this.logParts.set(body, { traceId, files: staged });
|
|
641
|
+
}
|
|
642
|
+
/** @internal Drops a log record's staged attachments once it was emitted. */
|
|
643
|
+
unstageLogParts(body) {
|
|
644
|
+
this.logParts.delete(body);
|
|
645
|
+
}
|
|
646
|
+
/** Holds the staged attachments of a log record being admitted with the queued record, within
|
|
647
|
+
* the held-value budget; those the budget refuses are counted as not uploaded. */
|
|
648
|
+
admitLogParts(log, queued) {
|
|
649
|
+
const body = log.body;
|
|
650
|
+
const pending = body !== null && typeof body === "object" ? this.logParts.get(body) : undefined;
|
|
651
|
+
if (!pending)
|
|
652
|
+
return;
|
|
653
|
+
this.logParts.delete(body);
|
|
654
|
+
const size = pending.files.reduce((total, file) => total + file.bytes.byteLength, 0);
|
|
655
|
+
if (this.heldBytes + size > MAX_HELD_BLOB_BYTES) {
|
|
656
|
+
const tally = newTally();
|
|
657
|
+
for (const _file of pending.files)
|
|
658
|
+
countFallback(tally, "budget", pending.traceId);
|
|
659
|
+
this.reportOffload(tally, "logs");
|
|
660
|
+
return;
|
|
661
|
+
}
|
|
662
|
+
this.logMedia.set(queued, pending.files);
|
|
663
|
+
this.heldBytes += size;
|
|
664
|
+
this.held.set(queued, (this.held.get(queued) ?? 0) + size);
|
|
665
|
+
}
|
|
666
|
+
/** Registers a span whose staged files are released if it is never exported. */
|
|
667
|
+
ownMedia(span, identity) {
|
|
668
|
+
if (this.mediaOwners.has(identity))
|
|
669
|
+
return;
|
|
670
|
+
const source = spanSource(span);
|
|
671
|
+
const token = {};
|
|
672
|
+
this.mediaOwners.set(identity, { source: new WeakRef(source), token });
|
|
673
|
+
this.mediaFinalizer.register(source, { identity, token }, token);
|
|
674
|
+
if (!this.mediaTimer) {
|
|
675
|
+
this.mediaTimer = setInterval(() => this.sweepMedia(), LIVE_SPAN_INTERVAL_MILLIS);
|
|
676
|
+
this.mediaTimer.unref();
|
|
677
|
+
}
|
|
678
|
+
}
|
|
500
679
|
mediaIdentity(record) {
|
|
501
680
|
if (!("spanContext" in record) || typeof record.spanContext !== "function")
|
|
502
681
|
return undefined;
|
|
@@ -552,8 +731,11 @@ export class HueTransport {
|
|
|
552
731
|
}
|
|
553
732
|
}
|
|
554
733
|
}
|
|
555
|
-
/** @internal Staged requests are matched against surviving events before upload. */
|
|
734
|
+
/** @internal Staged requests are matched against surviving events and parts before upload. */
|
|
556
735
|
mediaAttachments(record) {
|
|
736
|
+
const logged = this.logMedia.get(record);
|
|
737
|
+
if (logged)
|
|
738
|
+
return logged;
|
|
557
739
|
const identity = this.mediaIdentity(record);
|
|
558
740
|
return identity ? (this.media.get(identity) ?? []) : [];
|
|
559
741
|
}
|
|
@@ -573,13 +755,8 @@ export class HueTransport {
|
|
|
573
755
|
/** @internal Exporter callback: a redacted span with the values its redaction left to the
|
|
574
756
|
* upload step placed: uploaded and listed under `hue.blobs`, or cut as before. Never rejects. */
|
|
575
757
|
async offloadSpan(span, collector, upload, tally) {
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
traceId = span.spanContext().traceId;
|
|
579
|
-
}
|
|
580
|
-
catch {
|
|
581
|
-
// A span without a readable context cannot name its trace: its values are cut.
|
|
582
|
-
}
|
|
758
|
+
// A record without a readable context cannot name its trace: its values are cut.
|
|
759
|
+
const traceId = recordTraceId(span);
|
|
583
760
|
const valueBytes = this.limits.valueBytes;
|
|
584
761
|
try {
|
|
585
762
|
if (!this.uploader)
|
|
@@ -601,13 +778,41 @@ export class HueTransport {
|
|
|
601
778
|
};
|
|
602
779
|
}
|
|
603
780
|
}
|
|
781
|
+
/** @internal Exporter callback: a redacted log record with the media its redaction left to
|
|
782
|
+
* the upload step placed: its attributes' and its body's files uploaded and listed under its own
|
|
783
|
+
* `hue.blobs` (keys `<attribute>#<pointer>`, `body#<pointer>`), under the log record's trace;
|
|
784
|
+
* a file not uploaded keeps its bytes inline, and the record is cut as before. Never rejects. */
|
|
785
|
+
async offloadLog(log, collector, upload, tally) {
|
|
786
|
+
const traceId = log.spanContext?.traceId ?? "";
|
|
787
|
+
const valueBytes = this.limits.valueBytes;
|
|
788
|
+
const placed = (result) => ({
|
|
789
|
+
...log,
|
|
790
|
+
attributes: result.attributes,
|
|
791
|
+
...(collector.body ? { body: result.body } : {}),
|
|
792
|
+
});
|
|
793
|
+
try {
|
|
794
|
+
if (!this.uploader)
|
|
795
|
+
throw new Error("Uploads are unavailable");
|
|
796
|
+
return placed(await offloadRecord(log.attributes, collector, {
|
|
797
|
+
uploader: this.uploader,
|
|
798
|
+
traceId,
|
|
799
|
+
valueBytes,
|
|
800
|
+
upload,
|
|
801
|
+
tally,
|
|
802
|
+
memo: this.mediaMemo,
|
|
803
|
+
}));
|
|
804
|
+
}
|
|
805
|
+
catch {
|
|
806
|
+
return placed(fallbackRecord(log.attributes, collector, valueBytes, tally, traceId, this.mediaMemo));
|
|
807
|
+
}
|
|
808
|
+
}
|
|
604
809
|
/** @internal Exporter callback: counts an export's uploads and reports each kind of fallback
|
|
605
810
|
* once, as a warning naming its traces: the values were exported, cut as before. */
|
|
606
|
-
reportOffload(tally) {
|
|
811
|
+
reportOffload(tally, signal = "traces") {
|
|
607
812
|
this.uploaded += tally.uploaded;
|
|
608
813
|
for (const [reason, { count, traceIds }] of tally.fallbacks) {
|
|
609
814
|
this.uploadFallbacks += count;
|
|
610
|
-
this.issue(
|
|
815
|
+
this.issue(signal, "warning", count, FALLBACK_MESSAGES[reason], undefined, traceIds.size && traceIds.size <= MAX_ISSUE_TRACE_IDS ? [...traceIds] : undefined);
|
|
611
816
|
}
|
|
612
817
|
}
|
|
613
818
|
/** @internal Exporter callback: counts records the collector acknowledged. */
|
|
@@ -620,13 +825,20 @@ export class HueTransport {
|
|
|
620
825
|
* (`consumeDroppedRecords`): a root written again under retry says the same, and a root whose
|
|
621
826
|
* request failed has not told Hue, so its count is not forgotten with it. */
|
|
622
827
|
withDroppedRecords(span) {
|
|
623
|
-
if (span.parentSpanContext)
|
|
828
|
+
if (span.parentSpanContext || !this.marksDroppedRecords)
|
|
624
829
|
return span;
|
|
625
830
|
const count = this.droppedByTrace.get(span.spanContext().traceId);
|
|
626
831
|
if (!count)
|
|
627
832
|
return span;
|
|
628
833
|
return { ...span, attributes: { ...span.attributes, [DROPPED_RECORDS_KEY]: count } };
|
|
629
834
|
}
|
|
835
|
+
/** @internal For an import that replays recorded spans: roots never carry the count of
|
|
836
|
+
* records this run did not deliver, so a later run that delivers them sends each root as it
|
|
837
|
+
* was first sent, which Hue keeps as an exact replay. The losses are still counted and
|
|
838
|
+
* reported. */
|
|
839
|
+
omitDroppedRecords() {
|
|
840
|
+
this.marksDroppedRecords = false;
|
|
841
|
+
}
|
|
630
842
|
/** @internal Exporter callback: these counts reached Hue on their traces' roots. Only what a
|
|
631
843
|
* root carried is forgotten: a record lost while its request was in flight is still counted. */
|
|
632
844
|
consumeDroppedRecords(counts) {
|
|
@@ -848,6 +1060,7 @@ export class HueTransport {
|
|
|
848
1060
|
this.uploader?.close();
|
|
849
1061
|
for (const identity of this.media.keys())
|
|
850
1062
|
this.releaseMediaIdentity(identity);
|
|
1063
|
+
this.logParts.clear();
|
|
851
1064
|
this.mediaMemo.clear();
|
|
852
1065
|
}
|
|
853
1066
|
})();
|
|
@@ -988,6 +1201,10 @@ function shedContent(record, signal, shed) {
|
|
|
988
1201
|
: span.links,
|
|
989
1202
|
};
|
|
990
1203
|
}
|
|
1204
|
+
/** Whether a trace id can name an upload's trace: 32 lowercase hex digits, not all zero. */
|
|
1205
|
+
function isUploadTraceId(traceId) {
|
|
1206
|
+
return typeof traceId === "string" && /^[0-9a-f]{32}$/.test(traceId) && !/^0+$/.test(traceId);
|
|
1207
|
+
}
|
|
991
1208
|
class ReportingExporter {
|
|
992
1209
|
transport;
|
|
993
1210
|
signal;
|
|
@@ -1095,27 +1312,50 @@ class ReportingExporter {
|
|
|
1095
1312
|
clearTimeout(timer);
|
|
1096
1313
|
}
|
|
1097
1314
|
}
|
|
1098
|
-
const
|
|
1099
|
-
|
|
1315
|
+
const uploads = !markers && this.transport.offloadsValues();
|
|
1316
|
+
// A log record's media upload under its own trace: one without a trace is not offered.
|
|
1317
|
+
const offload = uploads &&
|
|
1318
|
+
(this.signal === "traces" ||
|
|
1319
|
+
isUploadTraceId(record.spanContext?.traceId))
|
|
1320
|
+
? this.signal === "traces"
|
|
1321
|
+
? { candidates: [], cut: [] }
|
|
1322
|
+
: { candidates: [], cut: [], log: true }
|
|
1100
1323
|
: undefined;
|
|
1101
1324
|
const redacted = this.redact(record, cache, offload);
|
|
1102
|
-
|
|
1325
|
+
let files = uploads
|
|
1103
1326
|
? survivingMedia(this.transport.mediaAttachments(record), redacted, tally)
|
|
1104
1327
|
: [];
|
|
1328
|
+
if (this.signal === "logs" && files.length) {
|
|
1329
|
+
// A log record's attachments upload with the media its body holds, in the one step
|
|
1330
|
+
// that places a log record's files; a record without a trace uploads none.
|
|
1331
|
+
if (offload)
|
|
1332
|
+
offload.attachments = files;
|
|
1333
|
+
else
|
|
1334
|
+
for (let index = 0; index < files.length; index++)
|
|
1335
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1336
|
+
files = [];
|
|
1337
|
+
}
|
|
1338
|
+
const offloaded = offload &&
|
|
1339
|
+
(offload.candidates.length ||
|
|
1340
|
+
offload.cut.length ||
|
|
1341
|
+
offload.body ||
|
|
1342
|
+
offload.attachments?.length)
|
|
1343
|
+
? this.signal === "traces"
|
|
1344
|
+
? this.transport.offloadSpan(redacted, offload, upload, tally)
|
|
1345
|
+
: this.transport.offloadLog(redacted, offload, upload, tally)
|
|
1346
|
+
: undefined;
|
|
1105
1347
|
placed.push({
|
|
1106
1348
|
record,
|
|
1107
1349
|
files,
|
|
1108
1350
|
...(markers ? { markers } : {}),
|
|
1109
|
-
redacted:
|
|
1110
|
-
? this.transport.offloadSpan(redacted, offload, upload, tally)
|
|
1111
|
-
: redacted,
|
|
1351
|
+
redacted: offloaded ?? redacted,
|
|
1112
1352
|
});
|
|
1113
1353
|
}
|
|
1114
1354
|
catch {
|
|
1115
|
-
if (!markers && this.
|
|
1355
|
+
if (!markers && this.transport.offloadsValues()) {
|
|
1116
1356
|
const files = this.transport.mediaAttachments(record);
|
|
1117
1357
|
for (let index = 0; index < files.length; index++)
|
|
1118
|
-
countFallback(tally, "budget", record
|
|
1358
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1119
1359
|
}
|
|
1120
1360
|
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
|
|
1121
1361
|
}
|
|
@@ -1137,20 +1377,19 @@ class ReportingExporter {
|
|
|
1137
1377
|
redacted = { ...redacted, attributes: kept };
|
|
1138
1378
|
}
|
|
1139
1379
|
if (files.length) {
|
|
1140
|
-
const
|
|
1141
|
-
const listed = span.attributes[BLOBS_KEY];
|
|
1380
|
+
const listed = redacted.attributes[BLOBS_KEY];
|
|
1142
1381
|
const baseline = Array.isArray(listed)
|
|
1143
1382
|
? listed.filter((entry) => typeof entry === "string")
|
|
1144
1383
|
: [];
|
|
1145
1384
|
retained = files.slice(0, Math.max(0, MAX_BLOB_ENTRIES - baseline.length));
|
|
1146
1385
|
for (let index = retained.length; index < files.length; index++)
|
|
1147
|
-
countFallback(tally, "budget",
|
|
1386
|
+
countFallback(tally, "budget", recordTraceId(redacted));
|
|
1148
1387
|
if (retained.length) {
|
|
1149
|
-
// Fit the event and its reference together before sending customer bytes.
|
|
1388
|
+
// Fit the event or part and its reference together before sending customer bytes.
|
|
1150
1389
|
redacted = {
|
|
1151
1390
|
...redacted,
|
|
1152
1391
|
attributes: {
|
|
1153
|
-
...
|
|
1392
|
+
...redacted.attributes,
|
|
1154
1393
|
[BLOBS_KEY]: [...baseline, ...retained.map(mediaEntry)],
|
|
1155
1394
|
},
|
|
1156
1395
|
};
|
|
@@ -1166,7 +1405,7 @@ class ReportingExporter {
|
|
|
1166
1405
|
}
|
|
1167
1406
|
catch {
|
|
1168
1407
|
for (let index = 0; index < retained.length; index++)
|
|
1169
|
-
countFallback(tally, "budget", record
|
|
1408
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1170
1409
|
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
|
|
1171
1410
|
}
|
|
1172
1411
|
}
|
|
@@ -1331,15 +1570,16 @@ class ReportingExporter {
|
|
|
1331
1570
|
try {
|
|
1332
1571
|
fitted = await fit(record);
|
|
1333
1572
|
if (fitted && helper) {
|
|
1334
|
-
const
|
|
1335
|
-
const
|
|
1573
|
+
const fittedRecord = fitted.record;
|
|
1574
|
+
const traceId = recordTraceId(fittedRecord);
|
|
1575
|
+
const matched = survivingMedia(remaining, fittedRecord, tally, record);
|
|
1336
1576
|
remaining = matched;
|
|
1337
|
-
const listed =
|
|
1577
|
+
const listed = fittedRecord.attributes[BLOBS_KEY];
|
|
1338
1578
|
const entries = Array.isArray(listed) ? [...listed] : [];
|
|
1339
1579
|
const files = matched.filter((file) => {
|
|
1340
1580
|
const index = entries.indexOf(mediaEntry(file), helper.baseline.length);
|
|
1341
1581
|
if (index < 0) {
|
|
1342
|
-
countFallback(tally, "budget",
|
|
1582
|
+
countFallback(tally, "budget", traceId);
|
|
1343
1583
|
return false;
|
|
1344
1584
|
}
|
|
1345
1585
|
entries.splice(index, 1);
|
|
@@ -1347,7 +1587,7 @@ class ReportingExporter {
|
|
|
1347
1587
|
});
|
|
1348
1588
|
remaining = files;
|
|
1349
1589
|
// Drop provisional entries, retaining only the original references after fitting.
|
|
1350
|
-
const attributes = { ...
|
|
1590
|
+
const attributes = { ...fittedRecord.attributes };
|
|
1351
1591
|
if (Array.isArray(listed)) {
|
|
1352
1592
|
const baseline = listed.slice(0, helper.baseline.length);
|
|
1353
1593
|
if (baseline.length)
|
|
@@ -1355,19 +1595,19 @@ class ReportingExporter {
|
|
|
1355
1595
|
else
|
|
1356
1596
|
delete attributes[BLOBS_KEY];
|
|
1357
1597
|
}
|
|
1358
|
-
attachments = {
|
|
1598
|
+
attachments = { record: { ...fittedRecord, attributes }, files };
|
|
1359
1599
|
remaining = [];
|
|
1360
1600
|
}
|
|
1361
1601
|
}
|
|
1362
1602
|
catch {
|
|
1363
1603
|
for (let index = 0; index < remaining.length; index++)
|
|
1364
|
-
countFallback(tally, "budget", record
|
|
1604
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1365
1605
|
invalid(advisory, "Telemetry record could not be serialized", record);
|
|
1366
1606
|
continue;
|
|
1367
1607
|
}
|
|
1368
1608
|
if (!fitted) {
|
|
1369
1609
|
for (let index = 0; index < remaining.length; index++)
|
|
1370
|
-
countFallback(tally, "budget", record
|
|
1610
|
+
countFallback(tally, "budget", recordTraceId(record));
|
|
1371
1611
|
invalid(advisory, OVER_REQUEST_LIMIT, record);
|
|
1372
1612
|
continue;
|
|
1373
1613
|
}
|
|
@@ -1379,7 +1619,7 @@ class ReportingExporter {
|
|
|
1379
1619
|
if (!attachments)
|
|
1380
1620
|
return fitted;
|
|
1381
1621
|
try {
|
|
1382
|
-
const uploaded = await this.transport.offloadSpan(attachments.
|
|
1622
|
+
const uploaded = await this.transport.offloadSpan(attachments.record, { candidates: [], cut: [], attachments: attachments.files }, upload, tally);
|
|
1383
1623
|
measured.delete(fitted.record);
|
|
1384
1624
|
// Actual references can differ when an existing object supplies its media type.
|
|
1385
1625
|
const final = await probe(uploaded);
|
|
@@ -1397,7 +1637,7 @@ class ReportingExporter {
|
|
|
1397
1637
|
// Preparation includes explicit file uploads. Report their outcome once, before delivery
|
|
1398
1638
|
// failures can consume the diagnostic callback's rate-limit slot. Placeholders never upload.
|
|
1399
1639
|
if (!advisory)
|
|
1400
|
-
this.transport.reportOffload(tally);
|
|
1640
|
+
this.transport.reportOffload(tally, this.signal);
|
|
1401
1641
|
let batch = [];
|
|
1402
1642
|
let batchBytes = 0;
|
|
1403
1643
|
for (const fitted of placed) {
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { Attributes, Context, Span, SpanKind, Tracer, TracerProvider } from "@opentelemetry/api";
|
|
2
|
+
import type { AttributeValue } from "@opentelemetry/api";
|
|
2
3
|
import type { LoggerProvider } from "@opentelemetry/api-logs";
|
|
4
|
+
import type { Attachment } from "./attachment.js";
|
|
3
5
|
/**
|
|
4
6
|
* JSON-compatible data: the shape content has on the wire after the helpers encode it. Helpers
|
|
5
7
|
* accept `unknown` and validate at runtime; values that are not plain JSON are omitted with an
|
|
@@ -8,6 +10,14 @@ import type { LoggerProvider } from "@opentelemetry/api-logs";
|
|
|
8
10
|
export type JsonValue = null | boolean | number | string | JsonValue[] | {
|
|
9
11
|
[key: string]: JsonValue;
|
|
10
12
|
};
|
|
13
|
+
/**
|
|
14
|
+
* JSON data that may hold an {@link Attachment} anywhere: what a span attribute passed to a helper
|
|
15
|
+
* may be besides an OpenTelemetry attribute value. Such a value is recorded as JSON text with each
|
|
16
|
+
* attachment as its digest part.
|
|
17
|
+
*/
|
|
18
|
+
export type AttachmentJson = Attachment | JsonValue | AttachmentJson[] | {
|
|
19
|
+
[key: string]: AttachmentJson;
|
|
20
|
+
};
|
|
11
21
|
/** Telemetry signal named by export reports, issues and processors. */
|
|
12
22
|
export type Signal = "traces" | "logs";
|
|
13
23
|
/** Options accepted by every Hue client and transport, whether enabled or disabled. */
|
|
@@ -124,11 +134,13 @@ export interface ExportReport {
|
|
|
124
134
|
pendingBytes: number;
|
|
125
135
|
/** Helper capture or instrumentation failures that omitted telemetry while preserving application results. */
|
|
126
136
|
instrumentationFailures: number;
|
|
127
|
-
/** Whole span attributes, inline media, and opted-in file helper uploads stored
|
|
128
|
-
* their
|
|
137
|
+
/** Whole span attributes, inline media, attachments and opted-in file helper uploads stored
|
|
138
|
+
* apart from their records (or already held by Hue), each listed under the record's
|
|
139
|
+
* `hue.blobs`. */
|
|
129
140
|
uploadedValues: number;
|
|
130
|
-
/** Whole-attribute, inline-media, and opted-in file helper uploads that could not be
|
|
131
|
-
* Content falls back to truncation, media digests, or file metadata, with a warning
|
|
141
|
+
/** Whole-attribute, inline-media, attachment and opted-in file helper uploads that could not be
|
|
142
|
+
* attached. Content falls back to truncation, media digests, or file metadata, with a warning
|
|
143
|
+
* issue. */
|
|
132
144
|
uploadFallbacks: number;
|
|
133
145
|
}
|
|
134
146
|
/** Caller budget for {@link HueClient.flushSafe} and {@link HueClient.shutdownSafe}. */
|
|
@@ -149,8 +161,15 @@ export interface SafeLifecycleResult {
|
|
|
149
161
|
export interface SpanOptions {
|
|
150
162
|
/** OpenTelemetry span kind; `INTERNAL` by default. */
|
|
151
163
|
kind?: SpanKind;
|
|
152
|
-
/**
|
|
153
|
-
|
|
164
|
+
/**
|
|
165
|
+
* Attributes set when the span starts. A value that is a JSON object, an array OpenTelemetry
|
|
166
|
+
* cannot hold (not of one primitive type) or an {@link Attachment} is recorded as JSON text, with
|
|
167
|
+
* each attachment as its digest part and its bytes uploaded; put files under `metadata` for Hue to
|
|
168
|
+
* list them as the span's metadata files.
|
|
169
|
+
*/
|
|
170
|
+
attributes?: {
|
|
171
|
+
[key: string]: AttributeValue | AttachmentJson | undefined;
|
|
172
|
+
};
|
|
154
173
|
/** Recorded as `gen_ai.conversation.id` on this span and inherited by nested helper spans. */
|
|
155
174
|
sessionId?: string;
|
|
156
175
|
/** Recorded as `user.id` on this span and inherited by nested helper spans. */
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.18.1";
|
package/dist/version.js
CHANGED