@hue-run/sdk 0.16.0 → 0.17.0

package/dist/transport.js

@@ -6,13 +6,46 @@ import { LogsExporterMetricsHelper, ProtobufLogsSerializer, ProtobufTraceSeriali
 import { BatchSpanProcessor, } from "@opentelemetry/sdk-trace";
 import { BatchLogRecordProcessor, } from "@opentelemetry/sdk-logs";
 import { isInsecureOrigin, REDACTION_CONTEXT_UNITS, validateOptions } from "./config.js";
-import { BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, MAX_HELD_BLOB_BYTES, newTally, offloadAttributes, uploadBudgetMillis, } from "./blobs.js";
+import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, countFallback, BlobUploader, FALLBACK_MESSAGES, fallbackAttributes, MAX_HELD_BLOB_BYTES, newTally, offloadAttributes, uploadBudgetMillis, } from "./blobs.js";
 import { advertisedLimits, BATCH_TARGET_BYTES, DEFAULT_RECEIVER_LIMITS, fitsWithoutCompression, } from "./limits.js";
 import { announcesLiveSpan, LIVE_SPAN_INTERVAL_MILLIS, MAX_LIVE_SPANS, pendingPlaceholder, PLACEHOLDERS_HEADER, withoutPlaceholderMarkers, } from "./live-spans.js";
-import { estimateRecordBytes } from "./safety.js";
+import { estimateRecordBytes, spanSource } from "./safety.js";
+import { MediaDigestMemo, isMediaName } from "./inline-files.js";
 import { snapshotLog, snapshotSpan } from "./snapshot.js";
 import { DROPPED_RECORDS_KEY, isContentKey, isTruncatedMarker, redactLog, redactSpan, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys, } from "./privacy.js";
 import { sdkVersion } from "./version.js";
+/** Each surviving event authorizes one staged request, including repeated digests. */
+function survivingMedia(files, span, tally) {
+    const consumed = new Set();
+    const traceId = span.spanContext().traceId;
+    return files.flatMap((file) => {
+        const index = span.events.findIndex((event, index) => !consumed.has(index) &&
+            event.name === "hue.file" &&
+            (!file.eventTime ||
+                (event.time[0] === file.eventTime[0] && event.time[1] === file.eventTime[1])) &&
+            event.attributes?.["hue.file.sha256"] === file.digest.sha256 &&
+            event.attributes?.["hue.file.size"] === file.digest.size &&
+            event.attributes?.["hue.file.media_type"] === file.contentType);
+        if (index < 0) {
+            countFallback(tally, "budget", traceId);
+            return [];
+        }
+        consumed.add(index);
+        const name = span.events[index].attributes?.["hue.file.name"];
+        return [
+            {
+                bytes: file.bytes,
+                digest: file.digest,
+                contentType: file.contentType,
+                ...(file.eventTime ? { eventTime: file.eventTime } : {}),
+                ...(isMediaName(name) ? { name } : {}),
+            },
+        ];
+    });
+}
+function mediaEntry(file) {
+    return blobEntry(`event:hue.file:${file.digest.sha256}`, file.digest.sha256, file.digest.size, file.contentType, file.name);
+}
 /**
  * How long one export holds its records for a receiver that refused them for their rate (HTTP 429)
  * and asked to be retried later than a request's deadline allows: the records stay queued, counted
@@ -125,11 +158,23 @@ export class HueTransport {
     admissionCuts = new WeakMap();
     // The span's own messages whose large inline files each admitted span shrank to their digest.
     admissionDigests = new WeakMap();
+    // The span's own messages each admitted span inspected, or left to export to inspect.
+    admissionInspected = new WeakMap();
     /** Uploads values over the inline limit apart from their spans; absent when disabled. */
     uploader;
     /** Bytes of values over the inline limit queued spans hold whole for upload, and each span's. */
     heldBytes = 0;
     held = new WeakMap();
+    mediaMemo = new MediaDigestMemo();
+    // Context identity also matches safe API wrappers to the provider's ended span. Abandoned
+    // spans retain no span objects, and share the 2048-span / 128 MiB holding bounds.
+    media = new Map();
+    mediaOwners = new Map();
+    mediaTimer;
+    mediaFinalizer = new FinalizationRegistry(({ identity, token }) => {
+        if (this.mediaOwners.get(identity)?.token === token)
+            this.releaseMediaIdentity(identity);
+    });
     uploaded = 0;
     uploadFallbacks = 0;
     constructor(options) {
@@ -137,12 +182,17 @@ export class HueTransport {
         Object.defineProperty(this, "options", { enumerable: false });
         this.liveSpans = this.options.enabled !== false && this.options.liveSpans;
         this.announcesLiveSpans = this.liveSpans;
-        this.traceExporter = new ReportingExporter(this, "traces", ProtobufTraceSerializer, TraceExporterMetricsHelper, (span, cache, offload) => redactSpan(span, this.options, cache, {
-            valueBytes: this.limits.valueBytes,
-            cut: this.admissionCuts.get(span),
-            digested: this.admissionDigests.get(span),
-            ...(offload ? { offload } : {}),
-        }));
+        this.traceExporter = new ReportingExporter(this, "traces", ProtobufTraceSerializer, TraceExporterMetricsHelper, (span, cache, offload) => {
+            // Export inspects only the messages admission inspected within the record's allowance.
+            if (offload)
+                offload.inspected = this.admissionInspected.get(span);
+            return redactSpan(span, this.options, cache, {
+                valueBytes: this.limits.valueBytes,
+                cut: this.admissionCuts.get(span),
+                digested: this.admissionDigests.get(span),
+                ...(offload ? { offload } : {}),
+            });
+        });
         this.logExporter = new ReportingExporter(this, "logs", ProtobufLogsSerializer, LogsExporterMetricsHelper, (log, cache) => redactLog(log, this.options, cache, {
             valueBytes: this.limits.valueBytes,
             cut: this.admissionCuts.get(log),
@@ -187,18 +237,30 @@ export class HueTransport {
                     this.stopLiveTimer();
                 let admitted;
                 try {
-                    if (!(span.spanContext().traceFlags & 1))
+                    if (!(span.spanContext().traceFlags & 1)) {
+                        this.releaseMedia(span);
                         return;
+                    }
                     const queued = this.enqueue("traces", span);
-                    if (!queued)
+                    if (!queued) {
+                        const tally = newTally();
+                        for (const file of this.mediaAttachments(span))
+                            if (file.bytes.byteLength)
+                                countFallback(tally, "budget", span.spanContext().traceId);
+                        this.reportOffload(tally);
+                        this.releaseMedia(span);
                         return;
+                    }
                     admitted = queued;
+                    this.protectMedia(admitted);
                     this.countEnded(admitted);
                     spans.onEnd(admitted);
                 }
                 catch {
                     if (admitted)
                         this.finish("traces", [admitted]);
+                    else
+                        this.releaseMedia(span);
                     this.issue("traces", "invalid", 1, "Telemetry processor could not accept a record", undefined, traceIdsOf([span]));
                 }
             },
@@ -262,12 +324,14 @@ export class HueTransport {
                 }
                 : undefined;
             const snapshot = signal === "traces"
-                ? snapshotSpan(record, remaining, this.options.captureContent, valueUnits, hold)
+                ? snapshotSpan(record, remaining, this.options.captureContent, valueUnits, hold, this.mediaMemo)
                 : snapshotLog(record, remaining, this.options.captureContent, valueUnits);
             if (snapshot.cut.size)
                 this.admissionCuts.set(snapshot.record, snapshot.cut);
             if (snapshot.digested.size)
                 this.admissionDigests.set(snapshot.record, snapshot.digested);
+            if (signal === "traces")
+                this.admissionInspected.set(snapshot.record, snapshot.inspected);
             if (snapshot.held) {
                 this.heldBytes += snapshot.held;
                 this.held.set(snapshot.record, snapshot.held);
@@ -311,7 +375,11 @@ export class HueTransport {
             this.live.delete(span);
             try {
                 // Ended without reaching onEnd: a wrapping processor filtered it, so no real span follows.
-                if (!this.liveSpans || span.ended || !this.batchSpans)
+                if (span.ended) {
+                    this.releaseMedia(span);
+                    continue;
+                }
+                if (!this.liveSpans || !this.batchSpans)
                     continue;
                 const { record, markers } = pendingPlaceholder(span);
                 const admitted = this.enqueue("traces", record, true);
@@ -373,6 +441,7 @@ export class HueTransport {
     /** @internal Exporter callback: releases queued records after an export attempt settles. */
     finish(signal, records) {
         for (const record of records) {
+            this.releaseMedia(record);
             const pending = signal === "traces" ? this.spans : this.logs;
             this.pendingBytes -= pending.get(record) ?? 0;
             pending.delete(record);
@@ -383,7 +452,112 @@ export class HueTransport {
             }
         }
     }
-    /** @internal Whether spans' values over the inline limit are uploaded: content is captured and
+    /** @internal The same bounded digest memo is used by explicit and inline media. */
+    digestMedia(bytes, traceId) {
+        return this.mediaMemo.bytes(bytes, traceId);
+    }
+    /** @internal Retain a defensive copy within the existing held-value budget. */
+    stageMedia(span, file) {
+        if (this.closed || !this.offloadsValues() || file?.bytes.byteLength === 0)
+            return true;
+        if (!file) {
+            const tally = newTally();
+            countFallback(tally, "budget", span.spanContext().traceId);
+            this.reportOffload(tally);
+            return false;
+        }
+        if (!this.holdsValues()) {
+            this.uploadFallbacks++;
+            this.issue("traces", "warning", 1, "File bytes could not be uploaded while the upload service was unavailable; their hue.file metadata was recorded", undefined, [span.spanContext().traceId]);
+            return false;
+        }
+        const identity = this.mediaIdentity(span);
+        this.sweepMedia();
+        const files = this.media.get(identity) ?? [];
+        if (files.length >= 64 ||
+            (!this.media.has(identity) && this.media.size >= 2048) ||
+            this.heldBytes + file.bytes.byteLength > MAX_HELD_BLOB_BYTES) {
+            this.uploadFallbacks++;
+            this.issue("traces", "warning", 1, "File bytes did not fit the SDK's upload budget; their hue.file metadata was recorded", undefined, [span.spanContext().traceId]);
+            return false;
+        }
+        const bytes = Uint8Array.from(file.bytes);
+        files.push({ ...file, bytes });
+        this.media.set(identity, files);
+        if (!this.mediaOwners.has(identity)) {
+            const source = spanSource(span);
+            const token = {};
+            this.mediaOwners.set(identity, { source: new WeakRef(source), token });
+            this.mediaFinalizer.register(source, { identity, token }, token);
+            if (!this.mediaTimer) {
+                this.mediaTimer = setInterval(() => this.sweepMedia(), LIVE_SPAN_INTERVAL_MILLIS);
+                this.mediaTimer.unref();
+            }
+        }
+        this.heldBytes += bytes.byteLength;
+        return true;
+    }
+    mediaIdentity(record) {
+        if (!("spanContext" in record) || typeof record.spanContext !== "function")
+            return undefined;
+        const { traceId, spanId } = record.spanContext();
+        return `${traceId}:${spanId}`;
+    }
+    releaseMedia(record) {
+        const identity = this.mediaIdentity(record);
+        if (!identity)
+            return;
+        this.releaseMediaIdentity(identity);
+    }
+    protectMedia(record) {
+        const identity = this.mediaIdentity(record);
+        if (!identity)
+            return;
+        const owner = this.mediaOwners.get(identity);
+        if (owner)
+            this.mediaFinalizer.unregister(owner.token);
+        this.mediaOwners.delete(identity);
+        if (!this.mediaOwners.size) {
+            clearInterval(this.mediaTimer);
+            this.mediaTimer = undefined;
+        }
+    }
+    releaseMediaIdentity(identity) {
+        const owner = this.mediaOwners.get(identity);
+        if (owner)
+            this.mediaFinalizer.unregister(owner.token);
+        this.mediaOwners.delete(identity);
+        if (!this.mediaOwners.size) {
+            clearInterval(this.mediaTimer);
+            this.mediaTimer = undefined;
+        }
+        const files = this.media.get(identity);
+        if (!files)
+            return;
+        for (const file of files)
+            this.heldBytes -= file.bytes.byteLength;
+        this.media.delete(identity);
+    }
+    sweepMedia() {
+        for (const [identity, owner] of this.mediaOwners) {
+            try {
+                const source = owner.source.deref();
+                if (!source ||
+                    source.ended === true ||
+                    source.isRecording?.() === false)
+                    this.releaseMediaIdentity(identity);
+            }
+            catch {
+                continue;
+            }
+        }
+    }
+    /** @internal Staged requests are matched against surviving events before upload. */
+    mediaAttachments(record) {
+        const identity = this.mediaIdentity(record);
+        return identity ? (this.media.get(identity) ?? []) : [];
+    }
+    /** @internal Whether spans' content and media may be uploaded: content is captured and
      * the key is a project key (setup credentials send metadata only). A value that then cannot be
      * uploaded is cut, reported and counted. */
     offloadsValues() {
@@ -391,7 +565,7 @@ export class HueTransport {
             this.options.captureContent &&
             !this.options.apiKey.startsWith("hue_setup_"));
     }
-    /** @internal Whether values over the inline limit are worth holding whole for upload now: they
+    /** @internal Whether content and media are worth holding whole for upload now: they
      * are uploaded, and the receiver has not shown it lacks the upload route. */
     holdsValues() {
         return this.offloadsValues() && this.uploader?.available() === true;
@@ -416,13 +590,14 @@ export class HueTransport {
                 valueBytes,
                 upload,
                 tally,
+                memo: this.mediaMemo,
             });
             return { ...span, attributes: attributes };
         }
         catch {
             return {
                 ...span,
-                attributes: fallbackAttributes(span.attributes, collector, valueBytes, tally, traceId),
+                attributes: fallbackAttributes(span.attributes, collector, valueBytes, tally, traceId, this.mediaMemo),
             };
         }
     }
@@ -669,6 +844,11 @@ export class HueTransport {
                     this.spanProcessor.shutdown(),
                     this.logRecordProcessor.shutdown(),
                 ]);
+                await Promise.allSettled([this.traceExporter.forceFlush(), this.logExporter.forceFlush()]);
+                this.uploader?.close();
+                for (const identity of this.media.keys())
+                    this.releaseMediaIdentity(identity);
+                this.mediaMemo.clear();
             }
         })();
         return this.shutdownPromise;
@@ -896,6 +1076,7 @@ class ReportingExporter {
             },
         };
         const tally = newTally();
+        const helpers = new Map();
         const placed = [];
         for (const { record, markers } of this.ordered(records)) {
             try {
@@ -918,8 +1099,12 @@ class ReportingExporter {
                     ? { candidates: [], cut: [] }
                     : undefined;
                 const redacted = this.redact(record, cache, offload);
+                const files = offload
+                    ? survivingMedia(this.transport.mediaAttachments(record), redacted, tally)
+                    : [];
                 placed.push({
                     record,
+                    files,
                     ...(markers ? { markers } : {}),
                     redacted: offload && (offload.candidates.length || offload.cut.length)
                         ? this.transport.offloadSpan(redacted, offload, upload, tally)
@@ -927,10 +1112,16 @@ class ReportingExporter {
                 });
             }
             catch {
+                if (!markers && this.signal === "traces" && this.transport.offloadsValues()) {
+                    const files = this.transport.mediaAttachments(record);
+                    for (let index = 0; index < files.length; index++)
+                        countFallback(tally, "budget", record.spanContext().traceId);
+                }
                 invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
             }
         }
-        for (const { record, markers, redacted: placing } of placed) {
+        for (const { record, markers, files, redacted: placing } of placed) {
+            let retained = files;
             try {
                 let redacted = await placing;
                 // Added after redaction, so a redactor cannot alter the markers or the parent identity.
@@ -945,6 +1136,27 @@ class ReportingExporter {
                     if (kept !== attributes)
                         redacted = { ...redacted, attributes: kept };
                 }
+                if (files.length) {
+                    const span = redacted;
+                    const listed = span.attributes[BLOBS_KEY];
+                    const baseline = Array.isArray(listed)
+                        ? listed.filter((entry) => typeof entry === "string")
+                        : [];
+                    retained = files.slice(0, Math.max(0, MAX_BLOB_ENTRIES - baseline.length));
+                    for (let index = retained.length; index < files.length; index++)
+                        countFallback(tally, "budget", span.spanContext().traceId);
+                    if (retained.length) {
+                        // Fit the event and its reference together before sending customer bytes.
+                        redacted = {
+                            ...redacted,
+                            attributes: {
+                                ...span.attributes,
+                                [BLOBS_KEY]: [...baseline, ...retained.map(mediaEntry)],
+                            },
+                        };
+                        helpers.set(redacted, { files: retained, baseline });
+                    }
+                }
                 const bytes = 512 +
                     estimateRecordBytes(recordData(redacted, this.signal), this.transport.options.maxQueueBytes - redactedBytes);
                 if (redactedBytes + bytes > this.transport.options.maxQueueBytes)
@@ -953,6 +1165,8 @@ class ReportingExporter {
                 (markers ? pending : real).push(redacted);
             }
             catch {
+                for (let index = 0; index < retained.length; index++)
+                    countFallback(tally, "budget", record.spanContext().traceId);
                 invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
             }
         }
@@ -1036,7 +1250,6 @@ class ReportingExporter {
             // export may have adopted lower ones meanwhile, and each attempt checks them again.
             return fitsLimits(body, this.transport.receiverLimits()) ? body : undefined;
         };
-        this.transport.reportOffload(tally);
         /** Sends one kind of record (completed records, or placeholders alone: `advisory`) in one
          * request, or in halves when it is over the receiver's limits. A trace's root among completed
          * records carries the trace's dropped-record count as it stands when its request is made, so
@@ -1109,25 +1322,88 @@ class ReportingExporter {
         const pack = async (accepted, advisory) => {
             // Every record is measured, shed and, where it cannot be sent, counted lost before any root
             // is written, so a root carries the losses of children that ended after it.
-            const prepared = [];
+            const admitted = [];
             for (const record of accepted) {
                 let fitted;
+                let attachments;
+                const helper = helpers.get(record);
+                let remaining = helper?.files ?? [];
                 try {
                     fitted = await fit(record);
+                    if (fitted && helper) {
+                        const span = fitted.record;
+                        const matched = survivingMedia(remaining, span, tally);
+                        remaining = matched;
+                        const listed = span.attributes[BLOBS_KEY];
+                        const entries = Array.isArray(listed) ? [...listed] : [];
+                        const files = matched.filter((file) => {
+                            const index = entries.indexOf(mediaEntry(file), helper.baseline.length);
+                            if (index < 0) {
+                                countFallback(tally, "budget", span.spanContext().traceId);
+                                return false;
+                            }
+                            entries.splice(index, 1);
+                            return true;
+                        });
+                        remaining = files;
+                        // Drop provisional entries, retaining only the original references after fitting.
+                        const attributes = { ...span.attributes };
+                        if (Array.isArray(listed)) {
+                            const baseline = listed.slice(0, helper.baseline.length);
+                            if (baseline.length)
+                                attributes[BLOBS_KEY] = baseline;
+                            else
+                                delete attributes[BLOBS_KEY];
+                        }
+                        attachments = { span: { ...span, attributes }, files };
+                        remaining = [];
+                    }
                 }
                 catch {
+                    for (let index = 0; index < remaining.length; index++)
+                        countFallback(tally, "budget", record.spanContext().traceId);
                     invalid(advisory, "Telemetry record could not be serialized", record);
                     continue;
                 }
                 if (!fitted) {
+                    for (let index = 0; index < remaining.length; index++)
+                        countFallback(tally, "budget", record.spanContext().traceId);
                     invalid(advisory, OVER_REQUEST_LIMIT, record);
                     continue;
                 }
-                prepared.push(fitted);
+                admitted.push({ fitted, attachments });
             }
+            // Decide every event/reference admission before starting helper uploads. The shared
+            // uploader's four workers place files across spans under the same upload deadline.
+            const placed = await Promise.all(admitted.map(async ({ fitted, attachments }) => {
+                if (!attachments)
+                    return fitted;
+                try {
+                    const uploaded = await this.transport.offloadSpan(attachments.span, { candidates: [], cut: [], attachments: attachments.files }, upload, tally);
+                    measured.delete(fitted.record);
+                    // Actual references can differ when an existing object supplies its media type.
+                    const final = await probe(uploaded);
+                    if (final?.request)
+                        measured.set(final.record, final.request);
+                    if (!final)
+                        invalid(advisory, OVER_REQUEST_LIMIT, fitted.record);
+                    return final;
+                }
+                catch {
+                    invalid(advisory, "Telemetry record could not be serialized", fitted.record);
+                    return undefined;
+                }
+            }));
+            // Preparation includes explicit file uploads. Report their outcome once, before delivery
+            // failures can consume the diagnostic callback's rate-limit slot. Placeholders never upload.
+            if (!advisory)
+                this.transport.reportOffload(tally);
             let batch = [];
             let batchBytes = 0;
-            for (const { record, bytes: recordBytes } of prepared) {
+            for (const fitted of placed) {
+                if (!fitted)
+                    continue;
+                const { record, bytes: recordBytes } = fitted;
                 const framedBytes = recordBytes + RECORD_FRAMING_BYTES;
                 const target = Math.min(this.transport.receiverLimits().decodedBytes, BATCH_TARGET_BYTES);
                 if (batch.length && batchBytes + framedBytes > target) {

package/dist/types.d.ts

@@ -124,11 +124,11 @@ export interface ExportReport {
     pendingBytes: number;
     /** Helper capture or instrumentation failures that omitted telemetry while preserving application results. */
     instrumentationFailures: number;
-    /** Span values over Hue's inline limit uploaded apart from their spans (or already stored by
-     * Hue), each listed under the span's `hue.blobs` with its first 16 KiB kept inline. */
+    /** Whole span attributes, inline media, and opted-in file helper uploads stored apart from
+     * their spans (or already held by Hue), each listed under the span's `hue.blobs`. */
     uploadedValues: number;
-    /** Span values over Hue's inline limit that could not be uploaded and were exported cut to the
-     * limit (an inline file as its digest), each reported in a warning issue. */
+    /** Whole-attribute, inline-media, and opted-in file helper uploads that could not be attached.
+     * Content falls back to truncation, media digests, or file metadata, with a warning issue. */
     uploadFallbacks: number;
 }
 /** Caller budget for {@link HueClient.flushSafe} and {@link HueClient.shutdownSafe}. */
@@ -237,7 +237,9 @@ export interface FileRecord {
     mediaType: string;
     /** Hex SHA-256 of the file's bytes; computed from `data` when omitted. */
     sha256?: string;
-    /** The file's bytes, only hashed and measured, never exported; a string is hashed as UTF-8.
+    /** Upload supplied bytes on this span when content capture is enabled. Defaults to false. */
+    upload?: boolean;
+    /** The file's bytes, hashed and measured locally by default; a string is hashed as UTF-8.
      * Data larger than 25 MiB is omitted and counted as an instrumentation failure. */
     data?: Uint8Array | string;
     /** Size in bytes; computed from `data` when omitted. */

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export declare const sdkVersion = "0.16.0";
+export declare const sdkVersion = "0.17.0";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Generated by scripts/write-version.mjs from package.json; do not edit by hand.
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export const sdkVersion = "0.16.0";
+export const sdkVersion = "0.17.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hue-run/sdk",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "private": false,
   "license": "MIT",
   "publishConfig": {
@@ -99,7 +99,7 @@
     "@opentelemetry/api": "1.9.1",
     "@opentelemetry/context-async-hooks": "2.11.0",
     "@types/bun": "1.4.2",
-    "@types/node": "26.5.1",
+    "@types/node": "26.6.4",
     "ai": "7.0.99",
     "ajv": "8.20.0",
     "protobufjs": "8.8.0",