@hue-run/sdk 0.18.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -86,12 +86,15 @@ export declare function readObservation(value: unknown): LangfuseObservation | u
86
86
  /** A v1 trace from an API answer, or undefined when it lacks an id or timestamp. */
87
87
  export declare function readTrace(value: unknown): LangfuseTrace | undefined;
88
88
  /** An answer read whole: its status and body, or `tooLarge` for a body past the bound. */
89
- type Answer = {
89
+ type Answer = ({
90
90
  status: number;
91
91
  body: Buffer;
92
92
  } | {
93
93
  status: number;
94
94
  tooLarge: true;
95
+ }) & {
96
+ /** A redirect's `Location`, which no request follows itself. */
97
+ location?: string;
95
98
  };
96
99
  /** Reads one Langfuse project through its public API with Basic authentication. */
97
100
  export declare class LangfuseApi {
@@ -105,9 +108,10 @@ export declare class LangfuseApi {
105
108
  /**
106
109
  * One GET with retries: a rate limit (HTTP 429) is waited out for as long as its `Retry-After`
107
110
  * asks (up to 15 minutes, ten times in a row), and a connection failure, a timeout or a 5xx is
108
- * tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute). An
109
- * authenticated request never follows a redirect, so the keys go only to the configured host;
110
- * `authenticated` is false for a presigned media download, which carries no Langfuse keys.
111
+ * tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute). No
112
+ * redirect is followed (its `Location` is answered), so the keys go only to the configured host
113
+ * and a media download reaches only links {@link LangfuseApi.media} checked; `authenticated` is
114
+ * false for a presigned media download, which carries no Langfuse keys.
111
115
  */
112
116
  get(url: URL, authenticated?: boolean, maxBytes?: number, timeoutMs?: number): Promise<Answer>;
113
117
  /** A GET of `path` under the API, answered with JSON; refusals of the keys are fatal. */
@@ -131,10 +135,22 @@ export declare class LangfuseApi {
131
135
  /**
132
136
  * A media file's bytes through `GET /api/public/media/{id}` and the download link it answers
133
137
  * (fetched without the Langfuse keys): `missing` when Langfuse holds no bytes for it (no record,
134
- * no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}.
138
+ * no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}. The link
139
+ * and each redirect from it must pass {@link refusedDownload}; one that does not fails the file.
135
140
  */
136
141
  media(id: string): Promise<Uint8Array | "missing" | "tooLarge">;
137
142
  }
143
+ /** Whether an IP address is one {@link INTERNAL_ADDRESSES} holds; false for anything else. */
144
+ export declare function isInternalAddress(address: string): boolean;
145
+ /**
146
+ * Why a media download link is refused, or undefined when it may be fetched: it must be HTTPS
147
+ * (plain HTTP only to this machine), at most 8192 characters, without credentials or a fragment,
148
+ * and name a host whose every address is public, unless it names the configured Langfuse host or
149
+ * that host is on this machine. Langfuse chooses the link, so this keeps a hostile or compromised
150
+ * deployment from sending requests to this machine's network. The name's lookup gives up after a
151
+ * minute or once `signal` stops the import.
152
+ */
153
+ export declare function refusedDownload(url: URL, langfuse: URL, signal: AbortSignal): Promise<string | undefined>;
138
154
  /** The OpenTelemetry trace id of a Langfuse trace: its own id when it is one (32 lowercase hex
139
155
  * digits, as the Langfuse OpenTelemetry SDKs write), otherwise one derived from it by SHA-256, so
140
156
  * the same trace always takes the same id. */
@@ -1,6 +1,8 @@
1
1
  import { createHash } from "node:crypto";
2
+ import { lookup } from "node:dns/promises";
3
+ import { BlockList, isIP } from "node:net";
2
4
  import { SpanKind, SpanStatusCode, TraceFlags, } from "@opentelemetry/api";
3
- import { MAX_CONTENT_BYTES, MAX_FILE_DATA_BYTES } from "../config.js";
5
+ import { MAX_CONTENT_BYTES, MAX_FILE_DATA_BYTES, isLoopbackHost } from "../config.js";
4
6
  import { BLOBS_KEY, MAX_BLOB_ENTRIES, blobEntry, newTally, offloadAttributes, } from "../blobs.js";
5
7
  import { digestPart, isMediaContentType, mayInlineFiles, mediaIn, placedMediaText, readerParses, } from "../inline-files.js";
6
8
  import { sdkVersion } from "../version.js";
@@ -16,6 +18,10 @@ import { sdkVersion } from "../version.js";
16
18
  const REQUEST_TIMEOUT_MS = 60_000;
17
19
  /** A media file download's deadline: up to 25 MiB over a slow link. */
18
20
  const DOWNLOAD_TIMEOUT_MS = 300_000;
21
+ /** Redirects a media download follows, each to a link checked as the first one was. */
22
+ const MAX_DOWNLOAD_REDIRECTS = 5;
23
+ /** The longest media download link accepted, as the Python SDK's media client accepts. */
24
+ const MAX_LINK_LENGTH = 8192;
19
25
  /** The largest answer read from the API. Langfuse Cloud answers at most 5 MB; a larger answer is
20
26
  * asked for again in smaller pages. */
21
27
  const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
@@ -210,9 +216,10 @@ export class LangfuseApi {
210
216
  /**
211
217
  * One GET with retries: a rate limit (HTTP 429) is waited out for as long as its `Retry-After`
212
218
  * asks (up to 15 minutes, ten times in a row), and a connection failure, a timeout or a 5xx is
213
- * tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute). An
214
- * authenticated request never follows a redirect, so the keys go only to the configured host;
215
- * `authenticated` is false for a presigned media download, which carries no Langfuse keys.
219
+ * tried again after 1, 2, 4, 8 and 16 seconds (or its `Retry-After`, up to a minute). No
220
+ * redirect is followed (its `Location` is answered), so the keys go only to the configured host
221
+ * and a media download reaches only links {@link LangfuseApi.media} checked; `authenticated` is
222
+ * false for a presigned media download, which carries no Langfuse keys.
216
223
  */
217
224
  async get(url, authenticated = true, maxBytes = MAX_RESPONSE_BYTES, timeoutMs = REQUEST_TIMEOUT_MS) {
218
225
  let failures = 0;
@@ -227,9 +234,7 @@ export class LangfuseApi {
227
234
  "user-agent": `hue-sdk-typescript/${sdkVersion} hue-import`,
228
235
  ...(authenticated ? { authorization: this.authorization } : {}),
229
236
  },
230
- // The keys go only to the configured host; a presigned download carries none and may
231
- // be redirected by its store.
232
- redirect: authenticated ? "manual" : "follow",
237
+ redirect: "manual",
233
238
  signal: AbortSignal.any([this.connection.signal, AbortSignal.timeout(timeoutMs)]),
234
239
  });
235
240
  }
@@ -258,6 +263,11 @@ export class LangfuseApi {
258
263
  failures++;
259
264
  continue;
260
265
  }
266
+ const location = status >= 300 && status < 400 ? response.headers.get("location") : null;
267
+ if (location !== null) {
268
+ await response.body?.cancel().catch(() => undefined);
269
+ return { status, body: Buffer.alloc(0), location };
270
+ }
261
271
  let body;
262
272
  try {
263
273
  body = await readBounded(response, maxBytes);
@@ -478,7 +488,8 @@ export class LangfuseApi {
478
488
  /**
479
489
  * A media file's bytes through `GET /api/public/media/{id}` and the download link it answers
480
490
  * (fetched without the Langfuse keys): `missing` when Langfuse holds no bytes for it (no record,
481
- * no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}.
491
+ * no link, or a link whose object is gone), `tooLarge` past {@link MAX_FILE_DATA_BYTES}. The link
492
+ * and each redirect from it must pass {@link refusedDownload}; one that does not fails the file.
482
493
  */
483
494
  async media(id) {
484
495
  const answer = await this.json(`/api/public/media/${encodeURIComponent(id)}`, {});
@@ -499,16 +510,112 @@ export class LangfuseApi {
499
510
  }
500
511
  if (url.protocol !== "https:" && url.protocol !== "http:")
501
512
  return "missing";
502
- const download = await this.get(url, false, MAX_FILE_DATA_BYTES, DOWNLOAD_TIMEOUT_MS);
503
- if ("tooLarge" in download)
504
- return download.status === 200 ? "tooLarge" : "missing";
505
- if (download.status === 200)
506
- return new Uint8Array(download.body);
507
- if ([403, 404, 410].includes(download.status))
508
- return "missing";
509
- throw new LangfuseError(`The media download answered HTTP ${download.status}`);
513
+ const langfuse = new URL(this.connection.baseUrl);
514
+ for (let redirects = 0;; redirects++) {
515
+ const refused = await refusedDownload(url, langfuse, this.connection.signal);
516
+ this.stopped();
517
+ if (refused)
518
+ throw new LangfuseError(`Langfuse named a media download link that ${refused}`);
519
+ const download = await this.get(url, false, MAX_FILE_DATA_BYTES, DOWNLOAD_TIMEOUT_MS);
520
+ if (download.location !== undefined) {
521
+ if (redirects === MAX_DOWNLOAD_REDIRECTS)
522
+ throw new LangfuseError("The media download redirected too many times");
523
+ try {
524
+ url = new URL(download.location, url);
525
+ }
526
+ catch {
527
+ throw new LangfuseError("The media download redirected to something other than a URL");
528
+ }
529
+ continue;
530
+ }
531
+ if ("tooLarge" in download)
532
+ return download.status === 200 ? "tooLarge" : "missing";
533
+ if (download.status === 200)
534
+ return new Uint8Array(download.body);
535
+ if ([403, 404, 410].includes(download.status))
536
+ return "missing";
537
+ throw new LangfuseError(`The media download answered HTTP ${download.status}`);
538
+ }
510
539
  }
511
540
  }
541
+ /** Addresses a media download may not reach: this machine, private networks, link-local
542
+ * (cloud metadata), shared, benchmarking, multicast and reserved ranges. IPv4-mapped IPv6
543
+ * addresses are checked as their IPv4 address. */
544
+ const INTERNAL_ADDRESSES = new BlockList();
545
+ for (const [network, prefix] of [
546
+ ["0.0.0.0", 8],
547
+ ["10.0.0.0", 8],
548
+ ["100.64.0.0", 10],
549
+ ["127.0.0.0", 8],
550
+ ["169.254.0.0", 16],
551
+ ["172.16.0.0", 12],
552
+ ["192.0.0.0", 24],
553
+ ["192.168.0.0", 16],
554
+ ["198.18.0.0", 15],
555
+ ["224.0.0.0", 3],
556
+ ])
557
+ INTERNAL_ADDRESSES.addSubnet(network, prefix, "ipv4");
558
+ for (const [network, prefix] of [
559
+ ["::", 128],
560
+ ["::1", 128],
561
+ ["64:ff9b::", 96],
562
+ ["fc00::", 7],
563
+ ["fe80::", 10],
564
+ ["fec0::", 10],
565
+ ["ff00::", 8],
566
+ ])
567
+ INTERNAL_ADDRESSES.addSubnet(network, prefix, "ipv6");
568
+ /** Whether an IP address is one {@link INTERNAL_ADDRESSES} holds; false for anything else. */
569
+ export function isInternalAddress(address) {
570
+ const version = isIP(address);
571
+ return version !== 0 && INTERNAL_ADDRESSES.check(address, version === 4 ? "ipv4" : "ipv6");
572
+ }
573
+ /**
574
+ * Why a media download link is refused, or undefined when it may be fetched: it must be HTTPS
575
+ * (plain HTTP only to this machine), at most 8192 characters, without credentials or a fragment,
576
+ * and name a host whose every address is public, unless it names the configured Langfuse host or
577
+ * that host is on this machine. Langfuse chooses the link, so this keeps a hostile or compromised
578
+ * deployment from sending requests to this machine's network. The name's lookup gives up after a
579
+ * minute or once `signal` stops the import.
580
+ */
581
+ export async function refusedDownload(url, langfuse, signal) {
582
+ if (url.href.length > MAX_LINK_LENGTH)
583
+ return "is too long";
584
+ if (url.protocol !== "https:" && !(url.protocol === "http:" && isLoopbackHost(url.hostname)))
585
+ return "does not use https";
586
+ if (url.username || url.password || url.hash)
587
+ return "carries credentials or a fragment";
588
+ if (isLoopbackHost(langfuse.hostname) || url.hostname === langfuse.hostname)
589
+ return undefined;
590
+ const host = url.hostname.replace(/^\[|\]$/gu, "");
591
+ const named = host === "localhost" || host.endsWith(".localhost");
592
+ // shortcut: the name is resolved apart from the request's own lookup, so a host that answers a
593
+ // second lookup differently (DNS rebinding) is not caught; pin the lookup if fetch gains a hook.
594
+ let addresses;
595
+ try {
596
+ addresses =
597
+ named || isIP(host)
598
+ ? [named ? "127.0.0.1" : host]
599
+ : await resolved(host, AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]));
600
+ }
601
+ catch {
602
+ return "names a host that could not be resolved";
603
+ }
604
+ if (addresses.length === 0 || addresses.some(isInternalAddress))
605
+ return "names an address on this machine or a private network";
606
+ return undefined;
607
+ }
608
+ /** Every address `host` resolves to; rejects once `signal` aborts, since a lookup cannot be
609
+ * cancelled. */
610
+ function resolved(host, signal) {
611
+ return new Promise((resolve, reject) => {
612
+ const stop = () => reject(new Error("The lookup was abandoned"));
613
+ if (signal.aborted)
614
+ return stop();
615
+ signal.addEventListener("abort", stop, { once: true });
616
+ lookup(host, { all: true }).then((entries) => resolve(entries.map((entry) => entry.address)), reject);
617
+ });
618
+ }
512
619
  // ---------------------------------------------------------------------------------------------
513
620
  // Conversion
514
621
  // ---------------------------------------------------------------------------------------------
package/dist/client.d.ts CHANGED
@@ -51,8 +51,9 @@ export declare class HueClient {
51
51
  private ownedProviders?;
52
52
  private closed;
53
53
  private shutdownPromise?;
54
- private flushPromise?;
55
- private safeFlushPromise?;
54
+ /** The latest flush while it still sends its records (`sent`, which the next flush's send
55
+ * follows), and the safe result that joined it. */
56
+ private sending?;
56
57
  private safeShutdownPromise?;
57
58
  constructor(options: HueOptions | ExistingHueProviders);
58
59
  /**
@@ -206,22 +207,25 @@ export declare class HueClient {
206
207
  /**
207
208
  * Safe for finally blocks; preserves the application's result or original exception. When its
208
209
  * budget runs out while the client's own transport holds records for a receiver that limited
209
- * the rate, the hold ends: those records are reported lost to the rate limit, and the hold no
210
- * longer keeps the process running.
210
+ * the rate, or still uploads values its records listed, the hold and the uploads end: those
211
+ * records are reported lost to the rate limit, the uploads counted as failed (their records keep
212
+ * their entries), and neither keeps the process running.
211
213
  */
212
214
  shutdownSafe(options?: SafeLifecycleOptions): Promise<SafeLifecycleResult>;
213
215
  private safeLifecycle;
214
216
  /**
215
- * Drains the trace and log providers and waits for Hue's acknowledgements. Each caller gets a
216
- * fresh serialized drain that includes records emitted before its call.
217
+ * Drains the trace and log providers and waits for Hue's acknowledgements, then for the uploads
218
+ * queued so far. Each caller gets a fresh drain that includes records emitted before its call;
219
+ * its records are sent after an earlier drain's records, never after that drain's uploads.
217
220
  *
218
221
  * @throws HueExportError when this drain observed a new rejection, delivery failure, drop or
219
222
  * invalid record; its `issues` and `report` are sanitized counts, never server text.
220
223
  */
221
224
  flush(): Promise<ExportReport>;
222
- private flushOnce;
225
+ private flushProviders;
223
226
  /**
224
227
  * Flushes, then shuts down the providers this client owns; borrowed providers are left running.
228
+ * Like {@link flush}, it waits for queued uploads after the records are sent.
225
229
  * Idempotent: later calls return the same promise, and later helper calls only run their callbacks.
226
230
  *
227
231
  * @throws HueExportError when the final flush observed new failures; owned providers are still released.
package/dist/client.js CHANGED
@@ -219,8 +219,9 @@ export class HueClient {
219
219
  ownedProviders;
220
220
  closed = false;
221
221
  shutdownPromise;
222
- flushPromise;
223
- safeFlushPromise;
222
+ /** The latest flush while it still sends its records (`sent`, which the next flush's send
223
+ * follows), and the safe result that joined it. */
224
+ sending;
224
225
  safeShutdownPromise;
225
226
  constructor(options) {
226
227
  if ("transport" in options) {
@@ -1065,24 +1066,25 @@ export class HueClient {
1065
1066
  }
1066
1067
  /** Production lifecycle path: never rejects; timeouts do not cancel borrowed provider work. */
1067
1068
  flushSafe(options = {}) {
1068
- if (this.safeFlushPromise)
1069
- return this.safeFlushPromise;
1070
- const work = this.flushPromise ?? this.flush();
1069
+ // A flush still sending its records is joined, so repeated timeouts do not accumulate
1070
+ // promises against a hung borrowed provider. Once it sent them, a new flush sends the
1071
+ // caller's own records instead of waiting on the earlier flush's uploads.
1072
+ if (this.sending?.safe)
1073
+ return this.sending.safe;
1074
+ const work = this.sending?.work ?? this.flush();
1075
+ // An invalid budget returns before the result awaits the flush: its failure is still handled.
1076
+ void work.catch(() => { });
1071
1077
  const result = this.safeLifecycle(() => work, options);
1072
- this.safeFlushPromise = result;
1073
- const clear = () => {
1074
- this.safeFlushPromise = undefined;
1075
- };
1076
- // Retain the timed-out result until its underlying drain settles. Repeated
1077
- // timeouts must not accumulate promises against a hung borrowed provider.
1078
- void work.then(clear, clear);
1078
+ if (this.sending?.work === work)
1079
+ this.sending.safe = result;
1079
1080
  return result;
1080
1081
  }
1081
1082
  /**
1082
1083
  * Safe for finally blocks; preserves the application's result or original exception. When its
1083
1084
  * budget runs out while the client's own transport holds records for a receiver that limited
1084
- * the rate, the hold ends: those records are reported lost to the rate limit, and the hold no
1085
- * longer keeps the process running.
1085
+ * the rate, or still uploads values its records listed, the hold and the uploads end: those
1086
+ * records are reported lost to the rate limit, the uploads counted as failed (their records keep
1087
+ * their entries), and neither keeps the process running.
1086
1088
  */
1087
1089
  shutdownSafe(options = {}) {
1088
1090
  if (this.safeShutdownPromise)
@@ -1130,8 +1132,9 @@ export class HueClient {
1130
1132
  }
1131
1133
  }
1132
1134
  /**
1133
- * Drains the trace and log providers and waits for Hue's acknowledgements. Each caller gets a
1134
- * fresh serialized drain that includes records emitted before its call.
1135
+ * Drains the trace and log providers and waits for Hue's acknowledgements, then for the uploads
1136
+ * queued so far. Each caller gets a fresh drain that includes records emitted before its call;
1137
+ * its records are sent after an earlier drain's records, never after that drain's uploads.
1135
1138
  *
1136
1139
  * @throws HueExportError when this drain observed a new rejection, delivery failure, drop or
1137
1140
  * invalid record; its `issues` and `report` are sanitized counts, never server text.
@@ -1139,25 +1142,31 @@ export class HueClient {
1139
1142
  flush() {
1140
1143
  // Each caller needs a drain after its own preceding span/log emissions.
1141
1144
  // Joining an earlier drain can acknowledge records that were not in its batch.
1145
+ // Only the sends are serialized: each flush then waits on its own for the uploads queued so
1146
+ // far, so a later flush's records never wait behind an earlier flush's uploads.
1142
1147
  const from = this.transport.getFailureSequence();
1143
- const drain = async () => {
1144
- const report = await this.flushOnce();
1148
+ const start = () => this.transport.flushRecords(() => this.flushProviders());
1149
+ const flushing = (this.sending?.sent ?? Promise.resolve()).then(start, start);
1150
+ const sent = flushing.then(({ sent }) => sent);
1151
+ const next = flushing
1152
+ .then(({ report }) => report)
1153
+ .then((report) => {
1145
1154
  if (this.transport.getFailureSequence() !== from)
1146
1155
  throw new HueExportError(this.transport
1147
1156
  .getIssues()
1148
1157
  .filter((issue) => issue.sequence > from && issue.kind !== "warning"), report);
1149
1158
  return report;
1150
- };
1151
- const next = (this.flushPromise ?? Promise.resolve()).then(drain, drain);
1152
- this.flushPromise = next;
1159
+ });
1160
+ const sending = { work: next, sent };
1161
+ this.sending = sending;
1153
1162
  const clear = () => {
1154
- if (this.flushPromise === next)
1155
- this.flushPromise = undefined;
1163
+ if (this.sending === sending)
1164
+ this.sending = undefined;
1156
1165
  };
1157
- void next.then(clear, clear);
1166
+ void sent.then(clear, clear);
1158
1167
  return next;
1159
1168
  }
1160
- async flushOnce() {
1169
+ async flushProviders() {
1161
1170
  // Providers this client owns wait as long as an export may take, a rate-limit hold included,
1162
1171
  // instead of OpenTelemetry's 30 s; borrowed providers keep their own budget.
1163
1172
  const owned = this.ownedProviders;
@@ -1169,10 +1178,10 @@ export class HueClient {
1169
1178
  for (const [index, result] of results.entries())
1170
1179
  if (result.status === "rejected")
1171
1180
  this.transport.issue(index === 0 ? "traces" : "logs", "failed", 0, "OpenTelemetry provider flush failed");
1172
- return this.transport.flush();
1173
1181
  }
1174
1182
  /**
1175
1183
  * Flushes, then shuts down the providers this client owns; borrowed providers are left running.
1184
+ * Like {@link flush}, it waits for queued uploads after the records are sent.
1176
1185
  * Idempotent: later calls return the same promise, and later helper calls only run their callbacks.
1177
1186
  *
1178
1187
  * @throws HueExportError when the final flush observed new failures; owned providers are still released.
@@ -24,10 +24,18 @@ export declare const WINDOW_UNITS: number;
24
24
  * reused for the next one, so a reader is done with each slice before it asks for the next. */
25
25
  export interface ByteSource {
26
26
  readonly size: number;
27
+ /** The strings the source keeps alive while it is held, each whole; absent for one read from
28
+ * elsewhere. */
29
+ readonly held?: readonly HeldText[];
27
30
  chunks(): Iterable<Uint8Array> | AsyncIterable<Uint8Array>;
28
31
  /** Set when the latest read found the source no longer holds the bytes it was recorded with. */
29
32
  changed?: boolean;
30
33
  }
34
+ /** A string sources keep alive, as its two bytes a code unit: a slice read in place keeps the
35
+ * whole string, so the sources that read one share its {@link HeldText}, charged once. */
36
+ export interface HeldText {
37
+ readonly bytes: number;
38
+ }
31
39
  /** Whether a value is a {@link ByteSource}. */
32
40
  export declare function isByteSource(value: unknown): value is ByteSource;
33
41
  /** How the bytes of a long string are read from its text. */
@@ -42,6 +50,8 @@ export interface LongText {
42
50
  kind: LongKind;
43
51
  /** A `data:` URL's header, between `data:` and its comma. */
44
52
  header?: string;
53
+ /** `text` whole, shared by every long string read from it. */
54
+ held: HeldText;
45
55
  }
46
56
  /** A media value read in place, with the long strings its stand-ins name. Its `source` is the
47
57
  * skeleton: placing its files edits the skeleton, and {@link placedLargeText} writes the long
@@ -67,8 +77,10 @@ export declare function readLargeMediaText(key: string, text: string, longUnits?
67
77
  * remainder is never held in two copies.
68
78
  */
69
79
  export declare class JoinedText {
80
+ private readonly held;
70
81
  private readonly pieces;
71
- constructor(pieces: [string, number, number][]);
82
+ /** `held`: the strings the pieces are cut from, which the text keeps alive whole. */
83
+ constructor(pieces: [string, number, number][], held: readonly HeldText[]);
72
84
  /** The text a window at a time, never between the halves of a surrogate pair. */
73
85
  windows(): Generator<string>;
74
86
  /** The first `units` code units. */
@@ -63,7 +63,7 @@ async function strictBase64(text, start, end) {
63
63
  * is parsed with the rest of the value (a percent-escaped `data:` URL included); a string literal
64
64
  * whose header holds a control character, which no JSON does, is left for the parse to refuse.
65
65
  */
66
- async function classify(text, start, end, raw = false) {
66
+ async function classify(text, start, end, held, raw = false) {
67
67
  if (text.startsWith("data:", start)) {
68
68
  // The comma is looked for within the string and the header's bound only.
69
69
  const found = text
@@ -77,11 +77,11 @@ async function classify(text, start, end, raw = false) {
77
77
  return undefined;
78
78
  if (!(await strictBase64(text, comma + 1, end)))
79
79
  return undefined;
80
- return { text, start, end, data: comma + 1, kind: "dataBase64", header };
80
+ return { text, start, end, data: comma + 1, kind: "dataBase64", header, held };
81
81
  }
82
82
  if (!(await strictBase64(text, start, end)))
83
83
  return undefined;
84
- return { text, start, end, data: start, kind: "base64" };
84
+ return { text, start, end, data: start, kind: "base64", held };
85
85
  }
86
86
  /** The stand-in for a long string in the skeleton: its kind kept (base64 and its first 16
87
87
  * characters, which name an audio container; a `data:` URL's header), longer than any label, and
@@ -139,7 +139,7 @@ async function find(text, character, from, to) {
139
139
  * unterminated (text that is no JSON) ends the scan; the skeleton then holds the rest as recorded
140
140
  * and fails to parse. The scan reads at most a window of the text in one synchronous step.
141
141
  */
142
- async function skeletonOf(text, longUnits) {
142
+ async function skeletonOf(text, longUnits, held) {
143
143
  const nonce = randomBytes(16).toString("hex");
144
144
  const long = new Map();
145
145
  const replaced = [];
@@ -193,7 +193,7 @@ async function skeletonOf(text, longUnits) {
193
193
  // Only a value: a key stays as it is.
194
194
  const found = escaped || close - open - 1 <= longUnits || text.charCodeAt(space(text, at)) === 0x3a
195
195
  ? undefined
196
- : await classify(text, open + 1, close);
196
+ : await classify(text, open + 1, close, held);
197
197
  if (found) {
198
198
  const stub = standIn(found, `${nonce}${long.size.toString(16).padStart(8, "0")}`);
199
199
  long.set(stub, found);
@@ -222,11 +222,12 @@ async function skeletonOf(text, longUnits) {
222
222
  */
223
223
  export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNITS) {
224
224
  const reading = readingOf(key);
225
+ const held = { bytes: text.length * 2 };
225
226
  /** Text that is no JSON is the file itself when it is one long base64 `data:` URL; other text
226
227
  * is read as `readMediaText` reads it, within the 8 Mi bound. */
227
228
  const whole = async () => {
228
229
  const found = text.length > longUnits && text.startsWith("data:")
229
- ? await classify(text, 0, text.length, true)
230
+ ? await classify(text, 0, text.length, held, true)
230
231
  : undefined;
231
232
  if (!found)
232
233
  return text.length > MAX_INLINE_FILE_TEXT
@@ -237,7 +238,7 @@ export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNIT
237
238
  return { ...mediaIn(stub), long: new Map([[stub, found]]), nonce, order: [stub] };
238
239
  };
239
240
  const json = async (check) => {
240
- const skeleton = await skeletonOf(text, longUnits);
241
+ const skeleton = await skeletonOf(text, longUnits, held);
241
242
  if (!skeleton)
242
243
  return undefined;
243
244
  const parsed = JSON.parse(skeleton.text);
@@ -279,8 +280,11 @@ export async function readLargeMediaText(key, text, longUnits = LONG_STRING_UNIT
279
280
  * remainder is never held in two copies.
280
281
  */
281
282
  export class JoinedText {
283
+ held;
282
284
  pieces;
283
- constructor(pieces) {
285
+ /** `held`: the strings the pieces are cut from, which the text keeps alive whole. */
286
+ constructor(pieces, held) {
287
+ this.held = held;
284
288
  this.pieces = pieces.filter(([, start, end]) => end > start);
285
289
  }
286
290
  /** The text a window at a time, never between the halves of a surrogate pair. */
@@ -329,6 +333,7 @@ export class JoinedText {
329
333
  const windows = () => this.windows();
330
334
  return {
331
335
  size,
336
+ held: this.held,
332
337
  *chunks() {
333
338
  const buffer = Buffer.allocUnsafe(WINDOW_UNITS * 3);
334
339
  for (const window of windows())
@@ -350,6 +355,7 @@ export async function placedLargeText(media) {
350
355
  // One forward scan for the stand-ins' shared nonce finds each one left, in order, by the index
351
356
  // its tag carries: linear in the text however many there are, a window at a time.
352
357
  const pieces = [];
358
+ const held = new Set([{ bytes: edited.length * 2 }]);
353
359
  let from = 0;
354
360
  let turn = 0;
355
361
  for (let at = edited.indexOf(media.nonce); at !== -1; at = edited.indexOf(media.nonce, at + 1)) {
@@ -365,21 +371,23 @@ export async function placedLargeText(media) {
365
371
  continue;
366
372
  const long = media.long.get(stub);
367
373
  pieces.push([edited, from, start], [long.text, long.start, long.end]);
374
+ held.add(long.held);
368
375
  from = start + stub.length;
369
376
  at = from - 1;
370
377
  }
371
378
  if (!pieces.length)
372
379
  return edited;
373
380
  pieces.push([edited, from, edited.length]);
374
- return new JoinedText(pieces);
381
+ return new JoinedText(pieces, [...held]);
375
382
  }
376
383
  /** Base64 from `start` to `end` of `text` (strict, padded to a multiple of four), decoded a window
377
- * at a time into one buffer each read reuses. */
378
- function base64Source(text, start, end) {
384
+ * at a time into one buffer each read reuses. `held` is `text` whole. */
385
+ function base64Source(text, start, end, held) {
379
386
  const length = end - start;
380
387
  const pad = (text.charCodeAt(end - 1) === 0x3d ? 1 : 0) + (text.charCodeAt(end - 2) === 0x3d ? 1 : 0);
381
388
  return {
382
389
  size: (length / 4) * 3 - pad,
390
+ held: [held],
383
391
  *chunks() {
384
392
  const buffer = Buffer.allocUnsafe((WINDOW_UNITS / 4) * 3);
385
393
  for (let at = start; at < end; at += WINDOW_UNITS) {
@@ -392,7 +400,7 @@ function base64Source(text, start, end) {
392
400
  /** The bytes of a file whose content is a long string, as `mediaBytes` reads them: a field its
393
401
  * convention declares base64 decoded, base64 text decoded (`bareBase64`), a base64 `data:` URL. */
394
402
  export function longFileBytes(file, long) {
395
- const source = base64Source(long.text, long.data, long.end);
403
+ const source = base64Source(long.text, long.data, long.end, long.held);
396
404
  if (long.kind === "base64")
397
405
  return {
398
406
  size: source.size,