@hue-run/sdk 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +16 -12
- package/EVALUATIONS.md +1 -1
- package/README.md +100 -66
- package/dist/blobs.d.ts +114 -35
- package/dist/blobs.js +354 -159
- package/dist/cli/langfuse.d.ts +21 -5
- package/dist/cli/langfuse.js +123 -16
- package/dist/client.d.ts +11 -7
- package/dist/client.js +35 -26
- package/dist/large-media.d.ts +13 -1
- package/dist/large-media.js +20 -12
- package/dist/transport.d.ts +38 -7
- package/dist/transport.js +240 -63
- package/dist/types.d.ts +5 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/cli/langfuse.d.ts
CHANGED
|
@@ -86,12 +86,15 @@ export declare function readObservation(value: unknown): LangfuseObservation | u
|
|
|
86
86
|
/** A v1 trace from an API answer, or undefined when it lacks an id or timestamp. */
|
|
87
87
|
export declare function readTrace(value: unknown): LangfuseTrace | undefined;
|
|
88
88
|
/** An answer read whole: its status and body, or `tooLarge` for a body past the bound. */
|
|
89
|
-
type Answer = {
|
|
89
|
+
type Answer = ({
|
|
90
90
|
status: number;
|
|
91
91
|
body: Buffer;
|
|
92
92
|
} | {
|
|
93
93
|
status: number;
|
|
94
94
|
tooLarge: true;
|
|
95
|
+
}) & {
|
|
96
|
+
/** A redirect's `Location`, which no request follows itself. */
|
|
97
|
+
location?: string;
|
|
95
98
|
};
|
|
96
99
|
/** Reads one Langfuse project through its public API with Basic authentication. */
|
|
97
100
|
export declare class LangfuseApi {
|
|
@@ -105,9 +108,10 @@ export declare class LangfuseApi {
|
|
|
105
108
|
/**
|
|
106
109
|
* One GET with retries: a rate limit (HTTP 429) is waited out for as long as its `Retry-After`
|
|
107
110
|
* asks (up to 15 minutes, ten times in a row), and a connection failure, a timeout or a 5xx is
|
|
108
|
-
* tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute).
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
+
* tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute). No
|
|
112
|
+
* redirect is followed (its `Location` is answered), so the keys go only to the configured host
|
|
113
|
+
* and a media download reaches only links {@link LangfuseApi.media} checked; `authenticated` is
|
|
114
|
+
* false for a presigned media download, which carries no Langfuse keys.
|
|
111
115
|
*/
|
|
112
116
|
get(url: URL, authenticated?: boolean, maxBytes?: number, timeoutMs?: number): Promise<Answer>;
|
|
113
117
|
/** A GET of `path` under the API, answered with JSON; refusals of the keys are fatal. */
|
|
@@ -131,10 +135,22 @@ export declare class LangfuseApi {
|
|
|
131
135
|
/**
|
|
132
136
|
* A media file's bytes through `GET /api/public/media/{id}` and the download link it answers
|
|
133
137
|
* (fetched without the Langfuse keys): `missing` when Langfuse holds no bytes for it (no record,
|
|
134
|
-
* no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}.
|
|
138
|
+
* no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}. The link
|
|
139
|
+
* and each redirect from it must pass {@link refusedDownload}; one that does not fails the file.
|
|
135
140
|
*/
|
|
136
141
|
media(id: string): Promise<Uint8Array | "missing" | "tooLarge">;
|
|
137
142
|
}
|
|
143
|
+
/** Whether an IP address is one {@link INTERNAL_ADDRESSES} holds; false for anything else. */
|
|
144
|
+
export declare function isInternalAddress(address: string): boolean;
|
|
145
|
+
/**
|
|
146
|
+
* Why a media download link is refused, or undefined when it may be fetched: it must be HTTPS
|
|
147
|
+
* (plain HTTP only to this machine), at most 8192 characters, without credentials or a fragment,
|
|
148
|
+
* and name a host whose every address is public, unless it names the configured Langfuse host or
|
|
149
|
+
* that host is on this machine. Langfuse chooses the link, so this keeps a hostile or compromised
|
|
150
|
+
* deployment from sending requests to this machine's network. The name's lookup gives up after a
|
|
151
|
+
* minute or once `signal` stops the import.
|
|
152
|
+
*/
|
|
153
|
+
export declare function refusedDownload(url: URL, langfuse: URL, signal: AbortSignal): Promise<string | undefined>;
|
|
138
154
|
/** The OpenTelemetry trace id of a Langfuse trace: its own id when it is one (32 lowercase hex
|
|
139
155
|
* digits, as the Langfuse OpenTelemetry SDKs write), otherwise one derived from it by SHA-256, so
|
|
140
156
|
* the same trace always takes the same id. */
|
package/dist/cli/langfuse.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { lookup } from "node:dns/promises";
|
|
3
|
+
import { BlockList, isIP } from "node:net";
|
|
2
4
|
import { SpanKind, SpanStatusCode, TraceFlags, } from "@opentelemetry/api";
|
|
3
|
-
import { MAX_CONTENT_BYTES, MAX_FILE_DATA_BYTES } from "../config.js";
|
|
5
|
+
import { MAX_CONTENT_BYTES, MAX_FILE_DATA_BYTES, isLoopbackHost } from "../config.js";
|
|
4
6
|
import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, newTally, offloadAttributes, } from "../blobs.js";
|
|
5
7
|
import { digestPart, isMediaContentType, mayInlineFiles, mediaIn, placedMediaText, readerParses, } from "../inline-files.js";
|
|
6
8
|
import { sdkVersion } from "../version.js";
|
|
@@ -16,6 +18,10 @@ import { sdkVersion } from "../version.js";
|
|
|
16
18
|
const REQUEST_TIMEOUT_MS = 60_000;
|
|
17
19
|
/** A media file download's deadline: up to 25 MiB over a slow link. */
|
|
18
20
|
const DOWNLOAD_TIMEOUT_MS = 300_000;
|
|
21
|
+
/** Redirects a media download follows, each to a link checked as the first one was. */
|
|
22
|
+
const MAX_DOWNLOAD_REDIRECTS = 5;
|
|
23
|
+
/** The longest media download link accepted, as the Python SDK's media client accepts. */
|
|
24
|
+
const MAX_LINK_LENGTH = 8192;
|
|
19
25
|
/** The largest answer read from the API. Langfuse Cloud answers at most 5 MB; a larger answer is
|
|
20
26
|
* asked for again in smaller pages. */
|
|
21
27
|
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
|
|
@@ -210,9 +216,10 @@ export class LangfuseApi {
|
|
|
210
216
|
/**
|
|
211
217
|
* One GET with retries: a rate limit (HTTP 429) is waited out for as long as its `Retry-After`
|
|
212
218
|
* asks (up to 15 minutes, ten times in a row), and a connection failure, a timeout or a 5xx is
|
|
213
|
-
* tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute).
|
|
214
|
-
*
|
|
215
|
-
*
|
|
219
|
+
* tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute). No
|
|
220
|
+
* redirect is followed (its `Location` is answered), so the keys go only to the configured host
|
|
221
|
+
* and a media download reaches only links {@link LangfuseApi.media} checked; `authenticated` is
|
|
222
|
+
* false for a presigned media download, which carries no Langfuse keys.
|
|
216
223
|
*/
|
|
217
224
|
async get(url, authenticated = true, maxBytes = MAX_RESPONSE_BYTES, timeoutMs = REQUEST_TIMEOUT_MS) {
|
|
218
225
|
let failures = 0;
|
|
@@ -227,9 +234,7 @@ export class LangfuseApi {
|
|
|
227
234
|
"user-agent": `hue-sdk-typescript/${sdkVersion} hue-import`,
|
|
228
235
|
...(authenticated ? { authorization: this.authorization } : {}),
|
|
229
236
|
},
|
|
230
|
-
|
|
231
|
-
// be redirected by its store.
|
|
232
|
-
redirect: authenticated ? "manual" : "follow",
|
|
237
|
+
redirect: "manual",
|
|
233
238
|
signal: AbortSignal.any([this.connection.signal, AbortSignal.timeout(timeoutMs)]),
|
|
234
239
|
});
|
|
235
240
|
}
|
|
@@ -258,6 +263,11 @@ export class LangfuseApi {
|
|
|
258
263
|
failures++;
|
|
259
264
|
continue;
|
|
260
265
|
}
|
|
266
|
+
const location = status >= 300 && status < 400 ? response.headers.get("location") : null;
|
|
267
|
+
if (location !== null) {
|
|
268
|
+
await response.body?.cancel().catch(() => undefined);
|
|
269
|
+
return { status, body: Buffer.alloc(0), location };
|
|
270
|
+
}
|
|
261
271
|
let body;
|
|
262
272
|
try {
|
|
263
273
|
body = await readBounded(response, maxBytes);
|
|
@@ -478,7 +488,8 @@ export class LangfuseApi {
|
|
|
478
488
|
/**
|
|
479
489
|
* A media file's bytes through `GET /api/public/media/{id}` and the download link it answers
|
|
480
490
|
* (fetched without the Langfuse keys): `missing` when Langfuse holds no bytes for it (no record,
|
|
481
|
-
* no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}.
|
|
491
|
+
* no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}. The link
|
|
492
|
+
* and each redirect from it must pass {@link refusedDownload}; one that does not fails the file.
|
|
482
493
|
*/
|
|
483
494
|
async media(id) {
|
|
484
495
|
const answer = await this.json(`/api/public/media/${encodeURIComponent(id)}`, {});
|
|
@@ -499,16 +510,112 @@ export class LangfuseApi {
|
|
|
499
510
|
}
|
|
500
511
|
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
501
512
|
return "missing";
|
|
502
|
-
const
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
513
|
+
const langfuse = new URL(this.connection.baseUrl);
|
|
514
|
+
for (let redirects = 0;; redirects++) {
|
|
515
|
+
const refused = await refusedDownload(url, langfuse, this.connection.signal);
|
|
516
|
+
this.stopped();
|
|
517
|
+
if (refused)
|
|
518
|
+
throw new LangfuseError(`Langfuse named a media download link that ${refused}`);
|
|
519
|
+
const download = await this.get(url, false, MAX_FILE_DATA_BYTES, DOWNLOAD_TIMEOUT_MS);
|
|
520
|
+
if (download.location !== undefined) {
|
|
521
|
+
if (redirects === MAX_DOWNLOAD_REDIRECTS)
|
|
522
|
+
throw new LangfuseError("The media download redirected too many times");
|
|
523
|
+
try {
|
|
524
|
+
url = new URL(download.location, url);
|
|
525
|
+
}
|
|
526
|
+
catch {
|
|
527
|
+
throw new LangfuseError("The media download redirected to something other than a URL");
|
|
528
|
+
}
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
531
|
+
if ("tooLarge" in download)
|
|
532
|
+
return download.status === 200 ? "tooLarge" : "missing";
|
|
533
|
+
if (download.status === 200)
|
|
534
|
+
return new Uint8Array(download.body);
|
|
535
|
+
if ([403, 404, 410].includes(download.status))
|
|
536
|
+
return "missing";
|
|
537
|
+
throw new LangfuseError(`The media download answered HTTP ${download.status}`);
|
|
538
|
+
}
|
|
510
539
|
}
|
|
511
540
|
}
|
|
541
|
+
/** Addresses a media download may not reach: this machine, private networks, link-local
|
|
542
|
+
* (cloud metadata), shared, benchmarking, multicast and reserved ranges. IPv4-mapped IPv6
|
|
543
|
+
* addresses are checked as their IPv4 address. */
|
|
544
|
+
const INTERNAL_ADDRESSES = new BlockList();
|
|
545
|
+
for (const [network, prefix] of [
|
|
546
|
+
["0.0.0.0", 8],
|
|
547
|
+
["10.0.0.0", 8],
|
|
548
|
+
["100.64.0.0", 10],
|
|
549
|
+
["127.0.0.0", 8],
|
|
550
|
+
["169.254.0.0", 16],
|
|
551
|
+
["172.16.0.0", 12],
|
|
552
|
+
["192.0.0.0", 24],
|
|
553
|
+
["192.168.0.0", 16],
|
|
554
|
+
["198.18.0.0", 15],
|
|
555
|
+
["224.0.0.0", 3],
|
|
556
|
+
])
|
|
557
|
+
INTERNAL_ADDRESSES.addSubnet(network, prefix, "ipv4");
|
|
558
|
+
for (const [network, prefix] of [
|
|
559
|
+
["::", 128],
|
|
560
|
+
["::1", 128],
|
|
561
|
+
["64:ff9b::", 96],
|
|
562
|
+
["fc00::", 7],
|
|
563
|
+
["fe80::", 10],
|
|
564
|
+
["fec0::", 10],
|
|
565
|
+
["ff00::", 8],
|
|
566
|
+
])
|
|
567
|
+
INTERNAL_ADDRESSES.addSubnet(network, prefix, "ipv6");
|
|
568
|
+
/** Whether an IP address is one {@link INTERNAL_ADDRESSES} holds; false for anything else. */
|
|
569
|
+
export function isInternalAddress(address) {
|
|
570
|
+
const version = isIP(address);
|
|
571
|
+
return version !== 0 && INTERNAL_ADDRESSES.check(address, version === 4 ? "ipv4" : "ipv6");
|
|
572
|
+
}
|
|
573
|
+
/**
|
|
574
|
+
* Why a media download link is refused, or undefined when it may be fetched: it must be HTTPS
|
|
575
|
+
* (plain HTTP only to this machine), at most 8192 characters, without credentials or a fragment,
|
|
576
|
+
* and name a host whose every address is public, unless it names the configured Langfuse host or
|
|
577
|
+
* that host is on this machine. Langfuse chooses the link, so this keeps a hostile or compromised
|
|
578
|
+
* deployment from sending requests to this machine's network. The name's lookup gives up after a
|
|
579
|
+
* minute or once `signal` stops the import.
|
|
580
|
+
*/
|
|
581
|
+
export async function refusedDownload(url, langfuse, signal) {
|
|
582
|
+
if (url.href.length > MAX_LINK_LENGTH)
|
|
583
|
+
return "is too long";
|
|
584
|
+
if (url.protocol !== "https:" && !(url.protocol === "http:" && isLoopbackHost(url.hostname)))
|
|
585
|
+
return "does not use https";
|
|
586
|
+
if (url.username || url.password || url.hash)
|
|
587
|
+
return "carries credentials or a fragment";
|
|
588
|
+
if (isLoopbackHost(langfuse.hostname) || url.hostname === langfuse.hostname)
|
|
589
|
+
return undefined;
|
|
590
|
+
const host = url.hostname.replace(/^\[|\]$/gu, "");
|
|
591
|
+
const named = host === "localhost" || host.endsWith(".localhost");
|
|
592
|
+
// shortcut: the name is resolved apart from the request's own lookup, so a host that answers a
|
|
593
|
+
// second lookup differently (DNS rebinding) is not caught; pin the lookup if fetch gains a hook.
|
|
594
|
+
let addresses;
|
|
595
|
+
try {
|
|
596
|
+
addresses =
|
|
597
|
+
named || isIP(host)
|
|
598
|
+
? [named ? "127.0.0.1" : host]
|
|
599
|
+
: await resolved(host, AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]));
|
|
600
|
+
}
|
|
601
|
+
catch {
|
|
602
|
+
return "names a host that could not be resolved";
|
|
603
|
+
}
|
|
604
|
+
if (addresses.length === 0 || addresses.some(isInternalAddress))
|
|
605
|
+
return "names an address on this machine or a private network";
|
|
606
|
+
return undefined;
|
|
607
|
+
}
|
|
608
|
+
/** Every address `host` resolves to; rejects once `signal` aborts, since a lookup cannot be
|
|
609
|
+
* cancelled. */
|
|
610
|
+
function resolved(host, signal) {
|
|
611
|
+
return new Promise((resolve, reject) => {
|
|
612
|
+
const stop = () => reject(new Error("The lookup was abandoned"));
|
|
613
|
+
if (signal.aborted)
|
|
614
|
+
return stop();
|
|
615
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
616
|
+
lookup(host, { all: true }).then((entries) => resolve(entries.map((entry) => entry.address)), reject);
|
|
617
|
+
});
|
|
618
|
+
}
|
|
512
619
|
// ---------------------------------------------------------------------------------------------
|
|
513
620
|
// Conversion
|
|
514
621
|
// ---------------------------------------------------------------------------------------------
|
package/dist/client.d.ts
CHANGED
|
@@ -51,8 +51,9 @@ export declare class HueClient {
|
|
|
51
51
|
private ownedProviders?;
|
|
52
52
|
private closed;
|
|
53
53
|
private shutdownPromise?;
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
/** The latest flush while it still sends its records (`sent`, which the next flush's send
|
|
55
|
+
* follows), and the safe result that joined it. */
|
|
56
|
+
private sending?;
|
|
56
57
|
private safeShutdownPromise?;
|
|
57
58
|
constructor(options: HueOptions | ExistingHueProviders);
|
|
58
59
|
/**
|
|
@@ -206,22 +207,25 @@ export declare class HueClient {
|
|
|
206
207
|
/**
|
|
207
208
|
* Safe for finally blocks; preserves the application's result or original exception. When its
|
|
208
209
|
* budget runs out while the client's own transport holds records for a receiver that limited
|
|
209
|
-
* the rate,
|
|
210
|
-
*
|
|
210
|
+
* the rate, or still uploads values its records listed, the hold and the uploads end: those
|
|
211
|
+
* records are reported lost to the rate limit, the uploads counted as failed (their records keep
|
|
212
|
+
* their entries), and neither keeps the process running.
|
|
211
213
|
*/
|
|
212
214
|
shutdownSafe(options?: SafeLifecycleOptions): Promise<SafeLifecycleResult>;
|
|
213
215
|
private safeLifecycle;
|
|
214
216
|
/**
|
|
215
|
-
* Drains the trace and log providers and waits for Hue's acknowledgements
|
|
216
|
-
* fresh
|
|
217
|
+
* Drains the trace and log providers and waits for Hue's acknowledgements, then for the uploads
|
|
218
|
+
* queued so far. Each caller gets a fresh drain that includes records emitted before its call;
|
|
219
|
+
* its records are sent after an earlier drain's records, never after that drain's uploads.
|
|
217
220
|
*
|
|
218
221
|
* @throws HueExportError when this drain observed a new rejection, delivery failure, drop or
|
|
219
222
|
* invalid record; its `issues` and `report` are sanitized counts, never server text.
|
|
220
223
|
*/
|
|
221
224
|
flush(): Promise<ExportReport>;
|
|
222
|
-
private
|
|
225
|
+
private flushProviders;
|
|
223
226
|
/**
|
|
224
227
|
* Flushes, then shuts down the providers this client owns; borrowed providers are left running.
|
|
228
|
+
* Like {@link flush}, it waits for queued uploads after the records are sent.
|
|
225
229
|
* Idempotent: later calls return the same promise, and later helper calls only run their callbacks.
|
|
226
230
|
*
|
|
227
231
|
* @throws HueExportError when the final flush observed new failures; owned providers are still released.
|
package/dist/client.js
CHANGED
|
@@ -219,8 +219,9 @@ export class HueClient {
|
|
|
219
219
|
ownedProviders;
|
|
220
220
|
closed = false;
|
|
221
221
|
shutdownPromise;
|
|
222
|
-
|
|
223
|
-
|
|
222
|
+
/** The latest flush while it still sends its records (`sent`, which the next flush's send
|
|
223
|
+
* follows), and the safe result that joined it. */
|
|
224
|
+
sending;
|
|
224
225
|
safeShutdownPromise;
|
|
225
226
|
constructor(options) {
|
|
226
227
|
if ("transport" in options) {
|
|
@@ -1065,24 +1066,25 @@ export class HueClient {
|
|
|
1065
1066
|
}
|
|
1066
1067
|
/** Production lifecycle path: never rejects; timeouts do not cancel borrowed provider work. */
|
|
1067
1068
|
flushSafe(options = {}) {
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1069
|
+
// A flush still sending its records is joined, so repeated timeouts do not accumulate
|
|
1070
|
+
// promises against a hung borrowed provider. Once it sent them, a new flush sends the
|
|
1071
|
+
// caller's own records instead of waiting on the earlier flush's uploads.
|
|
1072
|
+
if (this.sending?.safe)
|
|
1073
|
+
return this.sending.safe;
|
|
1074
|
+
const work = this.sending?.work ?? this.flush();
|
|
1075
|
+
// An invalid budget returns before the result awaits the flush: its failure is still handled.
|
|
1076
|
+
void work.catch(() => { });
|
|
1071
1077
|
const result = this.safeLifecycle(() => work, options);
|
|
1072
|
-
this.
|
|
1073
|
-
|
|
1074
|
-
this.safeFlushPromise = undefined;
|
|
1075
|
-
};
|
|
1076
|
-
// Retain the timed-out result until its underlying drain settles. Repeated
|
|
1077
|
-
// timeouts must not accumulate promises against a hung borrowed provider.
|
|
1078
|
-
void work.then(clear, clear);
|
|
1078
|
+
if (this.sending?.work === work)
|
|
1079
|
+
this.sending.safe = result;
|
|
1079
1080
|
return result;
|
|
1080
1081
|
}
|
|
1081
1082
|
/**
|
|
1082
1083
|
* Safe for finally blocks; preserves the application's result or original exception. When its
|
|
1083
1084
|
* budget runs out while the client's own transport holds records for a receiver that limited
|
|
1084
|
-
* the rate,
|
|
1085
|
-
*
|
|
1085
|
+
* the rate, or still uploads values its records listed, the hold and the uploads end: those
|
|
1086
|
+
* records are reported lost to the rate limit, the uploads counted as failed (their records keep
|
|
1087
|
+
* their entries), and neither keeps the process running.
|
|
1086
1088
|
*/
|
|
1087
1089
|
shutdownSafe(options = {}) {
|
|
1088
1090
|
if (this.safeShutdownPromise)
|
|
@@ -1130,8 +1132,9 @@ export class HueClient {
|
|
|
1130
1132
|
}
|
|
1131
1133
|
}
|
|
1132
1134
|
/**
|
|
1133
|
-
* Drains the trace and log providers and waits for Hue's acknowledgements
|
|
1134
|
-
* fresh
|
|
1135
|
+
* Drains the trace and log providers and waits for Hue's acknowledgements, then for the uploads
|
|
1136
|
+
* queued so far. Each caller gets a fresh drain that includes records emitted before its call;
|
|
1137
|
+
* its records are sent after an earlier drain's records, never after that drain's uploads.
|
|
1135
1138
|
*
|
|
1136
1139
|
* @throws HueExportError when this drain observed a new rejection, delivery failure, drop or
|
|
1137
1140
|
* invalid record; its `issues` and `report` are sanitized counts, never server text.
|
|
@@ -1139,25 +1142,31 @@ export class HueClient {
|
|
|
1139
1142
|
flush() {
|
|
1140
1143
|
// Each caller needs a drain after its own preceding span/log emissions.
|
|
1141
1144
|
// Joining an earlier drain can acknowledge records that were not in its batch.
|
|
1145
|
+
// Only the sends are serialized: each flush then waits on its own for the uploads queued so
|
|
1146
|
+
// far, so a later flush's records never wait behind an earlier flush's uploads.
|
|
1142
1147
|
const from = this.transport.getFailureSequence();
|
|
1143
|
-
const
|
|
1144
|
-
|
|
1148
|
+
const start = () => this.transport.flushRecords(() => this.flushProviders());
|
|
1149
|
+
const flushing = (this.sending?.sent ?? Promise.resolve()).then(start, start);
|
|
1150
|
+
const sent = flushing.then(({ sent }) => sent);
|
|
1151
|
+
const next = flushing
|
|
1152
|
+
.then(({ report }) => report)
|
|
1153
|
+
.then((report) => {
|
|
1145
1154
|
if (this.transport.getFailureSequence() !== from)
|
|
1146
1155
|
throw new HueExportError(this.transport
|
|
1147
1156
|
.getIssues()
|
|
1148
1157
|
.filter((issue) => issue.sequence > from && issue.kind !== "warning"), report);
|
|
1149
1158
|
return report;
|
|
1150
|
-
};
|
|
1151
|
-
const
|
|
1152
|
-
this.
|
|
1159
|
+
});
|
|
1160
|
+
const sending = { work: next, sent };
|
|
1161
|
+
this.sending = sending;
|
|
1153
1162
|
const clear = () => {
|
|
1154
|
-
if (this.
|
|
1155
|
-
this.
|
|
1163
|
+
if (this.sending === sending)
|
|
1164
|
+
this.sending = undefined;
|
|
1156
1165
|
};
|
|
1157
|
-
void
|
|
1166
|
+
void sent.then(clear, clear);
|
|
1158
1167
|
return next;
|
|
1159
1168
|
}
|
|
1160
|
-
async
|
|
1169
|
+
async flushProviders() {
|
|
1161
1170
|
// Providers this client owns wait as long as an export may take, a rate-limit hold included,
|
|
1162
1171
|
// instead of OpenTelemetry's 30 s; borrowed providers keep their own budget.
|
|
1163
1172
|
const owned = this.ownedProviders;
|
|
@@ -1169,10 +1178,10 @@ export class HueClient {
|
|
|
1169
1178
|
for (const [index, result] of results.entries())
|
|
1170
1179
|
if (result.status === "rejected")
|
|
1171
1180
|
this.transport.issue(index === 0 ? "traces" : "logs", "failed", 0, "OpenTelemetry provider flush failed");
|
|
1172
|
-
return this.transport.flush();
|
|
1173
1181
|
}
|
|
1174
1182
|
/**
|
|
1175
1183
|
* Flushes, then shuts down the providers this client owns; borrowed providers are left running.
|
|
1184
|
+
* Like {@link flush}, it waits for queued uploads after the records are sent.
|
|
1176
1185
|
* Idempotent: later calls return the same promise, and later helper calls only run their callbacks.
|
|
1177
1186
|
*
|
|
1178
1187
|
* @throws HueExportError when the final flush observed new failures; owned providers are still released.
|
package/dist/large-media.d.ts
CHANGED
|
@@ -24,10 +24,18 @@ export declare const WINDOW_UNITS: number;
|
|
|
24
24
|
* reused for the next one, so a reader is done with each slice before it asks for the next. */
|
|
25
25
|
export interface ByteSource {
|
|
26
26
|
readonly size: number;
|
|
27
|
+
/** The strings the source keeps alive while it is held, each whole; absent for one read from
|
|
28
|
+
* elsewhere. */
|
|
29
|
+
readonly held?: readonly HeldText[];
|
|
27
30
|
chunks(): Iterable<Uint8Array> | AsyncIterable<Uint8Array>;
|
|
28
31
|
/** Set when the latest read found the source no longer holds the bytes it was recorded with. */
|
|
29
32
|
changed?: boolean;
|
|
30
33
|
}
|
|
34
|
+
/** A string sources keep alive, as its two bytes a code unit: a slice read in place keeps the
|
|
35
|
+
* whole string, so the sources that read one share its {@link HeldText}, charged once. */
|
|
36
|
+
export interface HeldText {
|
|
37
|
+
readonly bytes: number;
|
|
38
|
+
}
|
|
31
39
|
/** Whether a value is a {@link ByteSource}. */
|
|
32
40
|
export declare function isByteSource(value: unknown): value is ByteSource;
|
|
33
41
|
/** How the bytes of a long string are read from its text. */
|
|
@@ -42,6 +50,8 @@ export interface LongText {
|
|
|
42
50
|
kind: LongKind;
|
|
43
51
|
/** A `data:` URL's header, between `data:` and its comma. */
|
|
44
52
|
header?: string;
|
|
53
|
+
/** `text` whole, shared by every long string read from it. */
|
|
54
|
+
held: HeldText;
|
|
45
55
|
}
|
|
46
56
|
/** A media value read in place, with the long strings its stand-ins name. Its `source` is the
|
|
47
57
|
* skeleton: placing its files edits the skeleton, and {@link placedLargeText} writes the long
|
|
@@ -67,8 +77,10 @@ export declare function readLargeMediaText(key: string, text: string, longUnits?
|
|
|
67
77
|
* remainder is never held in two copies.
|
|
68
78
|
*/
|
|
69
79
|
export declare class JoinedText {
|
|
80
|
+
private readonly held;
|
|
70
81
|
private readonly pieces;
|
|
71
|
-
|
|
82
|
+
/** `held`: the strings the pieces are cut from, which the text keeps alive whole. */
|
|
83
|
+
constructor(pieces: [string, number, number][], held: readonly HeldText[]);
|
|
72
84
|
/** The text a window at a time, never between the halves of a surrogate pair. */
|
|
73
85
|
windows(): Generator<string>;
|
|
74
86
|
/** The first `units` code units. */
|
package/dist/large-media.js
CHANGED
|
@@ -63,7 +63,7 @@ async function strictBase64(text, start, end) {
|
|
|
63
63
|
* is parsed with the rest of the value (a percent-escaped `data:` URL included); a string literal
|
|
64
64
|
* whose header holds a control character, which no JSON does, is left for the parse to refuse.
|
|
65
65
|
*/
|
|
66
|
-
async function classify(text, start, end, raw = false) {
|
|
66
|
+
async function classify(text, start, end, held, raw = false) {
|
|
67
67
|
if (text.startsWith("data:", start)) {
|
|
68
68
|
// The comma is looked for within the string and the header's bound only.
|
|
69
69
|
const found = text
|
|
@@ -77,11 +77,11 @@ async function classify(text, start, end, raw = false) {
|
|
|
77
77
|
return undefined;
|
|
78
78
|
if (!(await strictBase64(text, comma + 1, end)))
|
|
79
79
|
return undefined;
|
|
80
|
-
return { text, start, end, data: comma + 1, kind: "dataBase64", header };
|
|
80
|
+
return { text, start, end, data: comma + 1, kind: "dataBase64", header, held };
|
|
81
81
|
}
|
|
82
82
|
if (!(await strictBase64(text, start, end)))
|
|
83
83
|
return undefined;
|
|
84
|
-
return { text, start, end, data: start, kind: "base64" };
|
|
84
|
+
return { text, start, end, data: start, kind: "base64", held };
|
|
85
85
|
}
|
|
86
86
|
/** The stand-in for a long string in the skeleton: its kind kept (base64 and its first 16
|
|
87
87
|
* characters, which name an audio container; a `data:` URL's header), longer than any label, and
|
|
@@ -139,7 +139,7 @@ async function find(text, character, from, to) {
|
|
|
139
139
|
* unterminated (text that is no JSON) ends the scan; the skeleton then holds the rest as recorded
|
|
140
140
|
* and fails to parse. The scan reads at most a window of the text in one synchronous step.
|
|
141
141
|
*/
|
|
142
|
-
async function skeletonOf(text, longUnits) {
|
|
142
|
+
async function skeletonOf(text, longUnits, held) {
|
|
143
143
|
const nonce = randomBytes(16).toString("hex");
|
|
144
144
|
const long = new Map();
|
|
145
145
|
const replaced = [];
|
|
@@ -193,7 +193,7 @@ async function skeletonOf(text, longUnits) {
|
|
|
193
193
|
// Only a value: a key stays as it is.
|
|
194
194
|
const found = escaped || close - open - 1 <= longUnits || text.charCodeAt(space(text, at)) === 0x3a
|
|
195
195
|
? undefined
|
|
196
|
-
: await classify(text, open + 1, close);
|
|
196
|
+
: await classify(text, open + 1, close, held);
|
|
197
197
|
if (found) {
|
|
198
198
|
const stub = standIn(found, `${nonce}${long.size.toString(16).padStart(8, "0")}`);
|
|
199
199
|
long.set(stub, found);
|
|
@@ -222,11 +222,12 @@ async function skeletonOf(text, longUnits) {
|
|
|
222
222
|
*/
|
|
223
223
|
export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNITS) {
|
|
224
224
|
const reading = readingOf(key);
|
|
225
|
+
const held = { bytes: text.length * 2 };
|
|
225
226
|
/** Text that is no JSON is the file itself when it is one long base64 `data:` URL; other text
|
|
226
227
|
* is read as `readMediaText` reads it, within the 8 Mi bound. */
|
|
227
228
|
const whole = async () => {
|
|
228
229
|
const found = text.length > longUnits && text.startsWith("data:")
|
|
229
|
-
? await classify(text, 0, text.length, true)
|
|
230
|
+
? await classify(text, 0, text.length, held, true)
|
|
230
231
|
: undefined;
|
|
231
232
|
if (!found)
|
|
232
233
|
return text.length > MAX_INLINE_FILE_TEXT
|
|
@@ -237,7 +238,7 @@ export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNIT
|
|
|
237
238
|
return { ...mediaIn(stub), long: new Map([[stub, found]]), nonce, order: [stub] };
|
|
238
239
|
};
|
|
239
240
|
const json = async (check) => {
|
|
240
|
-
const skeleton = await skeletonOf(text, longUnits);
|
|
241
|
+
const skeleton = await skeletonOf(text, longUnits, held);
|
|
241
242
|
if (!skeleton)
|
|
242
243
|
return undefined;
|
|
243
244
|
const parsed = JSON.parse(skeleton.text);
|
|
@@ -279,8 +280,11 @@ export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNIT
|
|
|
279
280
|
* remainder is never held in two copies.
|
|
280
281
|
*/
|
|
281
282
|
export class JoinedText {
|
|
283
|
+
held;
|
|
282
284
|
pieces;
|
|
283
|
-
|
|
285
|
+
/** `held`: the strings the pieces are cut from, which the text keeps alive whole. */
|
|
286
|
+
constructor(pieces, held) {
|
|
287
|
+
this.held = held;
|
|
284
288
|
this.pieces = pieces.filter(([, start, end]) => end > start);
|
|
285
289
|
}
|
|
286
290
|
/** The text a window at a time, never between the halves of a surrogate pair. */
|
|
@@ -329,6 +333,7 @@ export class JoinedText {
|
|
|
329
333
|
const windows = () => this.windows();
|
|
330
334
|
return {
|
|
331
335
|
size,
|
|
336
|
+
held: this.held,
|
|
332
337
|
*chunks() {
|
|
333
338
|
const buffer = Buffer.allocUnsafe(WINDOW_UNITS * 3);
|
|
334
339
|
for (const window of windows())
|
|
@@ -350,6 +355,7 @@ export async function placedLargeText(media) {
|
|
|
350
355
|
// One forward scan for the stand-ins' shared nonce finds each one left, in order, by the index
|
|
351
356
|
// its tag carries: linear in the text however many there are, a window at a time.
|
|
352
357
|
const pieces = [];
|
|
358
|
+
const held = new Set([{ bytes: edited.length * 2 }]);
|
|
353
359
|
let from = 0;
|
|
354
360
|
let turn = 0;
|
|
355
361
|
for (let at = edited.indexOf(media.nonce); at !== -1; at = edited.indexOf(media.nonce, at + 1)) {
|
|
@@ -365,21 +371,23 @@ export async function placedLargeText(media) {
|
|
|
365
371
|
continue;
|
|
366
372
|
const long = media.long.get(stub);
|
|
367
373
|
pieces.push([edited, from, start], [long.text, long.start, long.end]);
|
|
374
|
+
held.add(long.held);
|
|
368
375
|
from = start + stub.length;
|
|
369
376
|
at = from - 1;
|
|
370
377
|
}
|
|
371
378
|
if (!pieces.length)
|
|
372
379
|
return edited;
|
|
373
380
|
pieces.push([edited, from, edited.length]);
|
|
374
|
-
return new JoinedText(pieces);
|
|
381
|
+
return new JoinedText(pieces, [...held]);
|
|
375
382
|
}
|
|
376
383
|
/** Base64 from `start` to `end` of `text` (strict, padded to a multiple of four), decoded a window
|
|
377
|
-
* at a time into one buffer each read reuses. */
|
|
378
|
-
function base64Source(text, start, end) {
|
|
384
|
+
* at a time into one buffer each read reuses. `held` is `text` whole. */
|
|
385
|
+
function base64Source(text, start, end, held) {
|
|
379
386
|
const length = end - start;
|
|
380
387
|
const pad = (text.charCodeAt(end - 1) === 0x3d ? 1 : 0) + (text.charCodeAt(end - 2) === 0x3d ? 1 : 0);
|
|
381
388
|
return {
|
|
382
389
|
size: (length / 4) * 3 - pad,
|
|
390
|
+
held: [held],
|
|
383
391
|
*chunks() {
|
|
384
392
|
const buffer = Buffer.allocUnsafe((WINDOW_UNITS / 4) * 3);
|
|
385
393
|
for (let at = start; at < end; at += WINDOW_UNITS) {
|
|
@@ -392,7 +400,7 @@ function base64Source(text, start, end) {
|
|
|
392
400
|
/** The bytes of a file whose content is a long string, as `mediaBytes` reads them: a field its
|
|
393
401
|
* convention declares base64 decoded, base64 text decoded (`bareBase64`), a base64 `data:` URL. */
|
|
394
402
|
export function longFileBytes(file, long) {
|
|
395
|
-
const source = base64Source(long.text, long.data, long.end);
|
|
403
|
+
const source = base64Source(long.text, long.data, long.end, long.held);
|
|
396
404
|
if (long.kind === "base64")
|
|
397
405
|
return {
|
|
398
406
|
size: source.size,
|