@hue-run/sdk 0.18.2 → 0.19.1

package/dist/large-media.d.ts

@@ -24,10 +24,18 @@ export declare const WINDOW_UNITS: number;
  * reused for the next one, so a reader is done with each slice before it asks for the next. */
 export interface ByteSource {
     readonly size: number;
+    /** The strings the source keeps alive while it is held, each whole; absent for one read from
+     * elsewhere. */
+    readonly held?: readonly HeldText[];
     chunks(): Iterable<Uint8Array> | AsyncIterable<Uint8Array>;
     /** Set when the latest read found the source no longer holds the bytes it was recorded with. */
     changed?: boolean;
 }
+/** A string sources keep alive, as its two bytes a code unit: a slice read in place keeps the
+ * whole string, so the sources that read one share its {@link HeldText}, charged once. */
+export interface HeldText {
+    readonly bytes: number;
+}
 /** Whether a value is a {@link ByteSource}. */
 export declare function isByteSource(value: unknown): value is ByteSource;
 /** How the bytes of a long string are read from its text. */
@@ -42,6 +50,8 @@ export interface LongText {
     kind: LongKind;
     /** A `data:` URL's header, between `data:` and its comma. */
     header?: string;
+    /** `text` whole, shared by every long string read from it. */
+    held: HeldText;
 }
 /** A media value read in place, with the long strings its stand-ins name. Its `source` is the
  * skeleton: placing its files edits the skeleton, and {@link placedLargeText} writes the long
@@ -57,7 +67,8 @@ export interface LargeMediaValue extends MediaValue {
  * An attribute's recorded text read for files as `readMediaText` reads it, its long base64 and
  * base64 `data:` URL strings read in place (see {@link skeletonOf}). Undefined for text whose JSON
  * around those strings is over the 8 Mi bound, or that reading finds nothing in. Throws for JSON
- * that does not parse.
+ * that does not parse, except text read by JSON convention that is a `data:` URL, which is read
+ * as the text itself, as `readMediaText` reads it.
  */
 export declare function readLargeMediaText(key: string, text: string, longUnits?: number): Promise<LargeMediaValue | undefined>;
 /**
@@ -67,8 +78,10 @@ export declare function readLargeMediaText(key: string, text: string, longUnits?
  * remainder is never held in two copies.
  */
 export declare class JoinedText {
+    private readonly held;
     private readonly pieces;
-    constructor(pieces: [string, number, number][]);
+    /** `held`: the strings the pieces are cut from, which the text keeps alive whole. */
+    constructor(pieces: [string, number, number][], held: readonly HeldText[]);
     /** The text a window at a time, never between the halves of a surrogate pair. */
     windows(): Generator<string>;
     /** The first `units` code units. */

package/dist/large-media.js

@@ -1,5 +1,5 @@
 import { randomBytes } from "node:crypto";
-import { MAX_INLINE_FILE_TEXT, mediaIn, placedMediaText, readerParses, readingOf, } from "./inline-files.js";
+import { headerMediaType, MAX_INLINE_FILE_TEXT, mediaIn, placedMediaText, readerParses, readingOf, } from "./inline-files.js";
 /**
  * Inline media in a value too long for the media inspection bounds (8 Mi UTF-16 code units for one
  * attribute, 16 Mi for a record), read without parsing the files out of it. A span's own attribute
@@ -63,7 +63,7 @@ async function strictBase64(text, start, end) {
  * is parsed with the rest of the value (a percent-escaped `data:` URL included); a string literal
  * whose header holds a control character, which no JSON does, is left for the parse to refuse.
  */
-async function classify(text, start, end, raw = false) {
+async function classify(text, start, end, held, raw = false) {
     if (text.startsWith("data:", start)) {
         // The comma is looked for within the string and the header's bound only.
         const found = text
@@ -77,11 +77,11 @@ async function classify(text, start, end, raw = false) {
             return undefined;
         if (!(await strictBase64(text, comma + 1, end)))
             return undefined;
-        return { text, start, end, data: comma + 1, kind: "dataBase64", header };
+        return { text, start, end, data: comma + 1, kind: "dataBase64", header, held };
     }
     if (!(await strictBase64(text, start, end)))
         return undefined;
-    return { text, start, end, data: start, kind: "base64" };
+    return { text, start, end, data: start, kind: "base64", held };
 }
 /** The stand-in for a long string in the skeleton: its kind kept (base64 and its first 16
  * characters, which name an audio container; a `data:` URL's header), longer than any label, and
@@ -139,7 +139,7 @@ async function find(text, character, from, to) {
  * unterminated (text that is no JSON) ends the scan; the skeleton then holds the rest as recorded
  * and fails to parse. The scan reads at most a window of the text in one synchronous step.
  */
-async function skeletonOf(text, longUnits) {
+async function skeletonOf(text, longUnits, held) {
     const nonce = randomBytes(16).toString("hex");
     const long = new Map();
     const replaced = [];
@@ -193,7 +193,7 @@ async function skeletonOf(text, longUnits) {
         // Only a value: a key stays as it is.
         const found = escaped || close - open - 1 <= longUnits || text.charCodeAt(space(text, at)) === 0x3a
             ? undefined
-            : await classify(text, open + 1, close);
+            : await classify(text, open + 1, close, held);
         if (found) {
             const stub = standIn(found, `${nonce}${long.size.toString(16).padStart(8, "0")}`);
             long.set(stub, found);
@@ -218,15 +218,17 @@ async function skeletonOf(text, longUnits) {
  * An attribute's recorded text read for files as `readMediaText` reads it, its long base64 and
  * base64 `data:` URL strings read in place (see {@link skeletonOf}). Undefined for text whose JSON
  * around those strings is over the 8 Mi bound, or that reading finds nothing in. Throws for JSON
- * that does not parse.
+ * that does not parse, except text read by JSON convention that is a `data:` URL, which is read
+ * as the text itself, as `readMediaText` reads it.
  */
 export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNITS) {
     const reading = readingOf(key);
+    const held = { bytes: text.length * 2 };
     /** Text that is no JSON is the file itself when it is one long base64 `data:` URL; other text
      * is read as `readMediaText` reads it, within the 8 Mi bound. */
     const whole = async () => {
         const found = text.length > longUnits && text.startsWith("data:")
-            ? await classify(text, 0, text.length, true)
+            ? await classify(text, 0, text.length, held, true)
             : undefined;
         if (!found)
             return text.length > MAX_INLINE_FILE_TEXT
@@ -237,7 +239,7 @@ export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNIT
         return { ...mediaIn(stub), long: new Map([[stub, found]]), nonce, order: [stub] };
     };
     const json = async (check) => {
-        const skeleton = await skeletonOf(text, longUnits);
+        const skeleton = await skeletonOf(text, longUnits, held);
         if (!skeleton)
             return undefined;
         const parsed = JSON.parse(skeleton.text);
@@ -254,7 +256,9 @@ export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNIT
     };
     switch (reading) {
         case "json":
-            return json();
+            // No JSON text starts with `data:`: such text is the text itself, as `readMediaText` reads
+            // text that does not parse, and a whole `data:` URL is the file.
+            return text.startsWith("data:") ? whole() : json();
         case "metadata":
             return /^\s*[[{]/.test(text) ? json() : whole();
         case "parts":
@@ -279,8 +283,11 @@ export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNIT
  * remainder is never held in two copies.
  */
 export class JoinedText {
+    held;
     pieces;
-    constructor(pieces) {
+    /** `held`: the strings the pieces are cut from, which the text keeps alive whole. */
+    constructor(pieces, held) {
+        this.held = held;
         this.pieces = pieces.filter(([, start, end]) => end > start);
     }
     /** The text a window at a time, never between the halves of a surrogate pair. */
@@ -329,6 +336,7 @@ export class JoinedText {
         const windows = () => this.windows();
         return {
             size,
+            held: this.held,
             *chunks() {
                 const buffer = Buffer.allocUnsafe(WINDOW_UNITS * 3);
                 for (const window of windows())
@@ -350,6 +358,7 @@ export async function placedLargeText(media) {
     // One forward scan for the stand-ins' shared nonce finds each one left, in order, by the index
     // its tag carries: linear in the text however many there are, a window at a time.
     const pieces = [];
+    const held = new Set([{ bytes: edited.length * 2 }]);
     let from = 0;
     let turn = 0;
     for (let at = edited.indexOf(media.nonce); at !== -1; at = edited.indexOf(media.nonce, at + 1)) {
@@ -365,21 +374,23 @@ export async function placedLargeText(media) {
             continue;
         const long = media.long.get(stub);
         pieces.push([edited, from, start], [long.text, long.start, long.end]);
+        held.add(long.held);
         from = start + stub.length;
         at = from - 1;
     }
     if (!pieces.length)
         return edited;
     pieces.push([edited, from, edited.length]);
-    return new JoinedText(pieces);
+    return new JoinedText(pieces, [...held]);
 }
 /** Base64 from `start` to `end` of `text` (strict, padded to a multiple of four), decoded a window
- * at a time into one buffer each read reuses. */
-function base64Source(text, start, end) {
+ * at a time into one buffer each read reuses. `held` is `text` whole. */
+function base64Source(text, start, end, held) {
     const length = end - start;
     const pad = (text.charCodeAt(end - 1) === 0x3d ? 1 : 0) + (text.charCodeAt(end - 2) === 0x3d ? 1 : 0);
     return {
         size: (length / 4) * 3 - pad,
+        held: [held],
         *chunks() {
             const buffer = Buffer.allocUnsafe((WINDOW_UNITS / 4) * 3);
             for (let at = start; at < end; at += WINDOW_UNITS) {
@@ -392,7 +403,7 @@ function base64Source(text, start, end) {
 /** The bytes of a file whose content is a long string, as `mediaBytes` reads them: a field its
  * convention declares base64 decoded, base64 text decoded (`bareBase64`), a base64 `data:` URL. */
 export function longFileBytes(file, long) {
-    const source = base64Source(long.text, long.data, long.end);
+    const source = base64Source(long.text, long.data, long.end, long.held);
     if (long.kind === "base64")
         return {
             size: source.size,
@@ -400,7 +411,7 @@ export function longFileBytes(file, long) {
             ...(file.base64 ? {} : { bareBase64: true }),
             source,
         };
-    const mediaType = long.header.split(";")[0].trim();
+    const mediaType = headerMediaType(long.header);
     return {
         size: source.size,
         decoded: true,

package/dist/transport.d.ts

@@ -11,9 +11,11 @@ export type RecordValue = ReadableSpan | ReadableLogRecord;
  * them, reported like any refused request.
  */
 export declare const MAX_RATE_LIMIT_HOLD_MILLIS = 60000;
-/** How long one export may take with requests of `timeoutMillis` each: its uploads of values over
+/** How long one export may take with requests of `timeoutMillis` each: hashing its values over
  * the inline limit, its requests, and a hold for a rate-limited receiver on top. Processors and
- * providers wait this long for a drain. */
+ * providers wait this long for a drain. A drain also waits for the uploads its exports queued,
+ * which run beside the requests and end within six timeouts and a minute of being queued, so they
+ * fit within the same budget. */
 export declare function exportBudgetMillis(timeoutMillis: number): number;
 /**
  * Rejection of {@link HueClient.flush} and {@link HueClient.shutdown}: telemetry was not fully
@@ -67,7 +69,11 @@ export declare class HueTransport {
     private logExporter;
     private closed;
     private shutdownPromise?;
-    private flushPromise?;
+    /** The latest flush's send of its records; the next flush's send starts after it. */
+    private sendPromise?;
+    /** Set while a flush sends its records: the processors leave the uploads to that flush, which
+     * waits for them after its send, so no flush's send waits behind another's uploads. */
+    private sendingRecords;
     private live;
     private liveTimer?;
     private liveSpans;
@@ -89,6 +95,10 @@ export declare class HueTransport {
     /** Bytes of values over the inline limit queued spans hold whole for upload, and each span's. */
     private heldBytes;
     private held;
+    /** Bytes of each record's charge (its own held values and its staged files) that its queued
+     * uploads took over before the record was finished: charged to the uploads alone until they
+     * settle or the record is finished. */
+    private heldByUploads;
     private mediaMemo;
     private media;
     private logParts;
@@ -98,6 +108,11 @@ export declare class HueTransport {
     private mediaFinalizer;
     private uploaded;
     private uploadFallbacks;
+    /** Each export's queued uploads, until they all settle and are reported. */
+    private uploadReports;
+    /** Settles once every upload report registered so far has: each drain awaits this one promise,
+     * so a drain costs the same however many uploads are pending. */
+    private uploadsSettled;
     constructor(options: HueOptions);
     /**
      * Advisory records (placeholders) are admitted only while the queue is under a quarter of its
@@ -113,6 +128,15 @@ export declare class HueTransport {
     /** Holds the staged attachments of a log record being admitted with the queued record, within
      * the held-value budget; those the budget refuses are counted as not uploaded. */
     private admitLogParts;
+    /** What the held-value budget has room for: values queued records hold and bytes unsettled
+     * uploads hold share it. */
+    private heldRoom;
+    /** An upload `record`'s export queued, as it is queued: flushes and the processors wait for it
+     * from now on, and the `shared` bytes it holds of the record's own are charged to the upload
+     * alone, so the held-value budget never counts them twice while the export goes on. A charge
+     * `released` before the record is finished is handed back to the record, which still holds
+     * them. */
+    private uploadQueued;
     /** Registers a span whose staged files are released if it is never exported. */
     private ownMedia;
     private mediaIdentity;
@@ -136,15 +160,22 @@ export declare class HueTransport {
     private endedSpans;
     private countEnded;
     /**
-     * Waits for the processors' and exporters' in-flight work; drain the providers first.
+     * Waits for the processors' and exporters' in-flight work, then for the uploads their exports
+     * queued; drain the providers first. Only the sends are serialized: a flush sends its records
+     * once an earlier flush sent its own, never after that flush's uploads.
      *
      * @throws HueExportError when a new non-warning issue was recorded since the previous observation.
      */
     flush(): Promise<ExportReport>;
-    private flushOnce;
+    /** Sends what the processors and exporters hold, once the previous flush's send ended. */
+    private sendRecords;
+    /** Waits for the uploads queued so far; each ends within its lifetime, so this is bounded. */
+    private drainUploads;
+    private flushReport;
     /**
-     * Flushes and stops the processors; records emitted afterwards are dropped and counted. In attach
-     * mode call it after shutting down the application's providers.
+     * Flushes and stops the processors, waits for queued uploads, then ends the uploader; records
+     * emitted afterwards are dropped and counted. In attach mode call it after shutting down the
+     * application's providers.
      *
      * @throws HueExportError when the final flush observed new failures.
      */