@statewalker/webrun-streams-libp2p 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +140 -7
- package/dist/connect-serve.d.ts +27 -1
- package/dist/connect-serve.d.ts.map +1 -1
- package/dist/duplex-over-stream.d.ts +62 -0
- package/dist/duplex-over-stream.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +431 -98
- package/dist/message-target-over-stream.d.ts +3 -0
- package/dist/message-target-over-stream.d.ts.map +1 -1
- package/dist/port-mux.d.ts +89 -0
- package/dist/port-mux.d.ts.map +1 -0
- package/dist/stream-gate.d.ts +10 -0
- package/dist/stream-gate.d.ts.map +1 -0
- package/package.json +8 -6
- package/src/connect-serve.ts +106 -71
- package/src/duplex-over-stream.ts +239 -25
- package/src/index.ts +12 -0
- package/src/message-target-over-stream.ts +36 -31
- package/src/port-mux.ts +238 -0
- package/src/stream-gate.ts +77 -0
|
@@ -52,6 +52,28 @@ export const DEFAULT_DRAIN_TIMEOUT_MS = 300_000;
|
|
|
52
52
|
*/
|
|
53
53
|
export const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
54
54
|
|
|
55
|
+
/**
|
|
56
|
+
* Default for {@link DuplexOverStreamOptions.readHighWaterBytes}: received bytes
|
|
57
|
+
* held for a consumer that is not reading before the stream is paused.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_READ_HIGH_WATER_BYTES = 64 * 1024;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Default for {@link DuplexOverStreamOptions.maxFrameBytes}: the largest frame payload accepted
|
|
63
|
+
* from the peer. Generous, because a sender before this package's outbound chunking sent each
|
|
64
|
+
* producer chunk as one frame; short of letting a peer declare a 4 GiB frame.
|
|
65
|
+
*/
|
|
66
|
+
export const DEFAULT_MAX_FRAME_BYTES = 64 * 1024 * 1024;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The largest payload this side puts in one outbound frame: a larger producer chunk crosses as
|
|
70
|
+
* several frames (they are self-delimiting, so a receiver of any version reassembles the bytes).
|
|
71
|
+
*/
|
|
72
|
+
export const OUTBOUND_FRAME_BYTES = 64 * 1024;
|
|
73
|
+
|
|
74
|
+
/** The peer sent bytes that are not this framing: a malformed or oversized frame header. */
|
|
75
|
+
class FramingError extends Error {}
|
|
76
|
+
|
|
55
77
|
/**
|
|
56
78
|
* The stream ended before the call finished: the peer died, the connection
|
|
57
79
|
* dropped, or the stream was reset or refused before both sides ended it.
|
|
@@ -98,6 +120,25 @@ export interface DuplexOverStreamOptions {
|
|
|
98
120
|
* the peer. Defaults to {@link DEFAULT_DRAIN_TIMEOUT_MS}.
|
|
99
121
|
*/
|
|
100
122
|
drainTimeoutMs?: number;
|
|
123
|
+
/**
|
|
124
|
+
* Received bytes held for a consumer that is not reading before the stream
|
|
125
|
+
* is paused (and resumed once half of them are read). A paused yamux stream
|
|
126
|
+
* stops granting its peer receive window, so the peer's `send()` reports a
|
|
127
|
+
* full buffer and its producer waits: run-ahead is bounded by two receive
|
|
128
|
+
* windows (what libp2p held while paused, and the window granted on resume)
|
|
129
|
+
* plus this — the receiving node's yamux `streamOptions.maxStreamWindowSize`,
|
|
130
|
+
* 16 MiB by default; 1 MiB bounds it at about 2.25 MiB with no loss of
|
|
131
|
+
* one-way throughput measured. Only where the muxer exposes its window.
|
|
132
|
+
* Defaults to {@link DEFAULT_READ_HIGH_WATER_BYTES}.
|
|
133
|
+
*/
|
|
134
|
+
readHighWaterBytes?: number;
|
|
135
|
+
/**
|
|
136
|
+
* The largest frame payload accepted from the peer, in bytes. A frame header past it — or one
|
|
137
|
+
* whose length is not a valid varint — means the peer is not speaking this framing: the call
|
|
138
|
+
* fails and the stream is aborted, rather than buffering without bound. Defaults to
|
|
139
|
+
* {@link DEFAULT_MAX_FRAME_BYTES}.
|
|
140
|
+
*/
|
|
141
|
+
maxFrameBytes?: number;
|
|
101
142
|
}
|
|
102
143
|
|
|
103
144
|
/**
|
|
@@ -212,7 +253,10 @@ export async function* duplexOverStream(
|
|
|
212
253
|
// interleave. Inserting an await above would break that.
|
|
213
254
|
let peerError: Error | undefined;
|
|
214
255
|
try {
|
|
215
|
-
for await (const frame of parseFrames(
|
|
256
|
+
for await (const frame of parseFrames(
|
|
257
|
+
pacedSource(stream, opts.readHighWaterBytes ?? DEFAULT_READ_HIGH_WATER_BYTES),
|
|
258
|
+
opts.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES,
|
|
259
|
+
)) {
|
|
216
260
|
if (frame.type === TYPE_DATA) {
|
|
217
261
|
yield frame.payload;
|
|
218
262
|
} else if (frame.type === TYPE_ERROR) {
|
|
@@ -223,6 +267,11 @@ export async function* duplexOverStream(
|
|
|
223
267
|
}
|
|
224
268
|
} catch (err) {
|
|
225
269
|
if (err === peerError) throw err;
|
|
270
|
+
if (err instanceof FramingError) {
|
|
271
|
+
// the peer is not speaking this framing: fail the call (the stream is aborted below)
|
|
272
|
+
firePeerInputEnd(err);
|
|
273
|
+
throw err;
|
|
274
|
+
}
|
|
226
275
|
// The stream itself failed (reset, aborted, its connection dropped):
|
|
227
276
|
// the call did not finish, whatever libp2p named the error.
|
|
228
277
|
const gone = new PeerGoneError(err);
|
|
@@ -389,7 +438,11 @@ async function* framedOutbound(
|
|
|
389
438
|
for (;;) {
|
|
390
439
|
const next = await iterator.next();
|
|
391
440
|
if (next.done === true) return;
|
|
392
|
-
|
|
441
|
+
// in bounded frames, so a receiver never reassembles one huge frame
|
|
442
|
+
const chunk = normalizeChunk(next.value);
|
|
443
|
+
for (let at = 0; at < chunk.byteLength; at += OUTBOUND_FRAME_BYTES) {
|
|
444
|
+
yield frameData(chunk.subarray(at, Math.min(at + OUTBOUND_FRAME_BYTES, chunk.byteLength)));
|
|
445
|
+
}
|
|
393
446
|
}
|
|
394
447
|
} catch (err) {
|
|
395
448
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
@@ -413,40 +466,201 @@ async function* framedOutbound(
|
|
|
413
466
|
}
|
|
414
467
|
}
|
|
415
468
|
|
|
416
|
-
|
|
469
|
+
/**
|
|
470
|
+
* The stream's inbound bytes, read at the consumer's pace. libp2p's own
|
|
471
|
+
* iterator moves every message into an unbounded queue as it arrives, and
|
|
472
|
+
* yamux grants its peer more receive window on arrival, not on consumption: a
|
|
473
|
+
* consumer that stopped reading let the producer run 22-24 MiB ahead until
|
|
474
|
+
* libp2p's read buffer killed the stream (L6). Here the queue is bounded: past
|
|
475
|
+
* `highWater` bytes the stream is paused (yamux then withholds window updates,
|
|
476
|
+
* libp2p keeps what still arrives in its read buffer, undispatched), and it is
|
|
477
|
+
* resumed once the consumer has read the queue down to half.
|
|
478
|
+
*
|
|
479
|
+
* Ends as libp2p's iterator does: on the peer's `remoteCloseWrite`, or on
|
|
480
|
+
* `close` (throwing its error, if any). Subscribes on the first pull, like it.
|
|
481
|
+
*/
|
|
482
|
+
async function* pacedSource(stream: Stream, highWater: number): AsyncGenerator<Uint8Array> {
|
|
483
|
+
if (stream.readStatus !== "readable" && stream.readStatus !== "paused") return;
|
|
484
|
+
// Pausing bounds anything only where the muxer then stops granting its peer
|
|
485
|
+
// window (yamux, which exposes `maxStreamWindowSize`). Elsewhere a pause just
|
|
486
|
+
// moves the data into libp2p's read buffer, which aborts the stream past
|
|
487
|
+
// `maxReadBufferLength`: there the stream is read as libp2p's own iterator
|
|
488
|
+
// reads it, without a bound.
|
|
489
|
+
const window = (stream as { maxStreamWindowSize?: unknown }).maxStreamWindowSize;
|
|
490
|
+
const paced = typeof window === "number";
|
|
491
|
+
// While paused, libp2p holds what the peer may still send (at most the
|
|
492
|
+
// window granted before the pause) and aborts the stream past
|
|
493
|
+
// `maxReadBufferLength`. yamux's default window (16 MiB) exceeds libp2p's
|
|
494
|
+
// default buffer (4 MiB), so the buffer is raised to the window plus what is
|
|
495
|
+
// queued here; yamux itself refuses a peer that sends past its window.
|
|
496
|
+
if (paced && stream.maxReadBufferLength < window + highWater) {
|
|
497
|
+
stream.maxReadBufferLength = window + highWater;
|
|
498
|
+
}
|
|
499
|
+
const queue: Uint8Array[] = [];
|
|
500
|
+
let queued = 0;
|
|
501
|
+
let ended = false;
|
|
502
|
+
let error: unknown;
|
|
503
|
+
let resuming = false;
|
|
504
|
+
let wake: (() => void) | undefined;
|
|
505
|
+
const notify = (): void => {
|
|
506
|
+
const w = wake;
|
|
507
|
+
wake = undefined;
|
|
508
|
+
w?.();
|
|
509
|
+
};
|
|
510
|
+
const pauseIfFull = (): void => {
|
|
511
|
+
if (paced && queued >= highWater && stream.readStatus === "readable") {
|
|
512
|
+
try {
|
|
513
|
+
stream.pause();
|
|
514
|
+
} catch {
|
|
515
|
+
/* closing: nothing more will be granted anyway */
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
};
|
|
519
|
+
const onMessage = (evt: { data: unknown }): void => {
|
|
520
|
+
const chunk = normalizeChunk(evt.data);
|
|
521
|
+
if (chunk.byteLength === 0) return;
|
|
522
|
+
// In pieces of at most `highWater` (views, not copies): a resume hands over
|
|
523
|
+
// everything libp2p held as one message, up to a window, and a reader that
|
|
524
|
+
// took it whole would hold that window outside this queue's count.
|
|
525
|
+
for (let at = 0; at < chunk.byteLength; at += highWater) {
|
|
526
|
+
queue.push(chunk.subarray(at, Math.min(at + highWater, chunk.byteLength)));
|
|
527
|
+
}
|
|
528
|
+
queued += chunk.byteLength;
|
|
529
|
+
// Never from inside `resume()`: it dispatches what libp2p held first and
|
|
530
|
+
// tells the muxer to grant window after, so a pause here would leave libp2p
|
|
531
|
+
// paused and yamux granting window for every frame (review of #112).
|
|
532
|
+
if (!resuming) pauseIfFull();
|
|
533
|
+
notify();
|
|
534
|
+
};
|
|
535
|
+
const resume = (): void => {
|
|
536
|
+
resuming = true;
|
|
537
|
+
try {
|
|
538
|
+
stream.resume(); // dispatches what libp2p held meanwhile, synchronously
|
|
539
|
+
} catch {
|
|
540
|
+
/* closing */
|
|
541
|
+
} finally {
|
|
542
|
+
resuming = false;
|
|
543
|
+
}
|
|
544
|
+
pauseIfFull(); // after the muxer's own resume, so the pause holds
|
|
545
|
+
};
|
|
546
|
+
const onClose = (evt: StreamCloseEvent): void => {
|
|
547
|
+
ended = true;
|
|
548
|
+
error ??= evt.error;
|
|
549
|
+
notify();
|
|
550
|
+
};
|
|
551
|
+
const onRemoteCloseWrite = (): void => {
|
|
552
|
+
ended = true;
|
|
553
|
+
notify();
|
|
554
|
+
};
|
|
555
|
+
stream.addEventListener("message", onMessage);
|
|
556
|
+
stream.addEventListener("close", onClose);
|
|
557
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
558
|
+
if (stream.readStatus === "paused") resume(); // paused by someone else: nothing here resumes it later
|
|
559
|
+
try {
|
|
560
|
+
for (;;) {
|
|
561
|
+
const chunk = queue.shift();
|
|
562
|
+
if (chunk) {
|
|
563
|
+
queued -= chunk.byteLength;
|
|
564
|
+
if (queued <= highWater / 2 && stream.readStatus === "paused") resume();
|
|
565
|
+
yield chunk;
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
if (ended) {
|
|
569
|
+
if (error) throw error;
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
await new Promise<void>((resolve) => {
|
|
573
|
+
wake = resolve;
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
} finally {
|
|
577
|
+
stream.removeEventListener("message", onMessage);
|
|
578
|
+
stream.removeEventListener("close", onClose);
|
|
579
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Frames out of the stream's chunks, as they complete. Unread bytes sit in a {@link ByteBuffer},
|
|
585
|
+
* which grows by doubling, so a large frame arriving in many chunks is copied a bounded number
|
|
586
|
+
* of times. A header whose length is malformed or past `maxFrameBytes` throws a `FramingError`.
|
|
587
|
+
*/
|
|
588
|
+
export async function* parseFrames(
|
|
417
589
|
source: AsyncIterable<unknown>,
|
|
590
|
+
maxFrameBytes: number = DEFAULT_MAX_FRAME_BYTES,
|
|
418
591
|
): AsyncGenerator<{ type: number; payload: Uint8Array }> {
|
|
419
|
-
|
|
592
|
+
const buf = new ByteBuffer();
|
|
420
593
|
for await (const item of source) {
|
|
421
594
|
const incoming = normalizeChunk(item);
|
|
422
595
|
if (incoming.byteLength === 0) continue;
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
let lenInfo: { value: number; offset: number };
|
|
436
|
-
try {
|
|
437
|
-
lenInfo = decodeVarint(buf, 1);
|
|
438
|
-
} catch {
|
|
439
|
-
break; // varint truncated, need more bytes
|
|
596
|
+
buf.push(incoming);
|
|
597
|
+
for (;;) {
|
|
598
|
+
const bytes = buf.unread();
|
|
599
|
+
if (bytes.byteLength < 2) break; // need at least type + 1 varint byte
|
|
600
|
+
// biome-ignore lint/style/noNonNullAssertion: bytes.byteLength >= 2, checked above, so index 0 exists.
|
|
601
|
+
const type = bytes[0]!;
|
|
602
|
+
const header = readLength(bytes.subarray(1), "frame");
|
|
603
|
+
if (header === undefined) break; // the length is not all here yet
|
|
604
|
+
if (header.value > maxFrameBytes) {
|
|
605
|
+
throw new FramingError(
|
|
606
|
+
`webrun-streams-libp2p: a ${header.value}-byte frame exceeds maxFrameBytes (${maxFrameBytes})`,
|
|
607
|
+
);
|
|
440
608
|
}
|
|
441
|
-
const total =
|
|
442
|
-
if (
|
|
443
|
-
const payload =
|
|
609
|
+
const total = 1 + header.offset + header.value;
|
|
610
|
+
if (bytes.byteLength < total) break;
|
|
611
|
+
const payload = bytes.slice(1 + header.offset, total);
|
|
612
|
+
buf.consume(total);
|
|
444
613
|
yield { type, payload };
|
|
445
|
-
buf = buf.byteLength === total ? new Uint8Array(0) : new Uint8Array(buf.subarray(total));
|
|
446
614
|
}
|
|
447
615
|
}
|
|
448
616
|
}
|
|
449
617
|
|
|
618
|
+
/** Unread bytes `buf[start, end)` of a buffer that grows by doubling. */
|
|
619
|
+
export class ByteBuffer {
|
|
620
|
+
private buf = new Uint8Array(0);
|
|
621
|
+
private start = 0;
|
|
622
|
+
private end = 0;
|
|
623
|
+
|
|
624
|
+
push(incoming: Uint8Array): void {
|
|
625
|
+
if (this.end + incoming.byteLength > this.buf.byteLength) {
|
|
626
|
+
const size = this.end - this.start + incoming.byteLength;
|
|
627
|
+
const next = new Uint8Array(Math.max(size, this.buf.byteLength * 2));
|
|
628
|
+
next.set(this.buf.subarray(this.start, this.end), 0);
|
|
629
|
+
this.buf = next;
|
|
630
|
+
this.end -= this.start;
|
|
631
|
+
this.start = 0;
|
|
632
|
+
}
|
|
633
|
+
this.buf.set(incoming, this.end);
|
|
634
|
+
this.end += incoming.byteLength;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/** a view of the unread bytes, valid until the next `push` */
|
|
638
|
+
unread(): Uint8Array {
|
|
639
|
+
return this.buf.subarray(this.start, this.end);
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
consume(n: number): void {
|
|
643
|
+
this.start += n;
|
|
644
|
+
if (this.start === this.end) this.start = this.end = 0;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* The varint length prefix at the start of `bytes`, or `undefined` while it is incomplete; throws
|
|
650
|
+
* a `FramingError` when it is malformed (no varint of this framing runs past 5 bytes).
|
|
651
|
+
*/
|
|
652
|
+
export function readLength(
|
|
653
|
+
bytes: Uint8Array,
|
|
654
|
+
what: string,
|
|
655
|
+
): { value: number; offset: number } | undefined {
|
|
656
|
+
try {
|
|
657
|
+
return decodeVarint(bytes, 0);
|
|
658
|
+
} catch (err) {
|
|
659
|
+
if (bytes.byteLength < 5) return undefined;
|
|
660
|
+
throw new FramingError(`webrun-streams-libp2p: a malformed ${what} length`, { cause: err });
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
450
664
|
export function normalizeChunk(item: unknown): Uint8Array {
|
|
451
665
|
if (item instanceof Uint8Array) return item;
|
|
452
666
|
const asList = item as { subarray?: () => Uint8Array };
|
package/src/index.ts
CHANGED
|
@@ -11,6 +11,7 @@ export {
|
|
|
11
11
|
export {
|
|
12
12
|
DEFAULT_CLOSE_TIMEOUT_MS,
|
|
13
13
|
DEFAULT_DRAIN_TIMEOUT_MS,
|
|
14
|
+
DEFAULT_MAX_FRAME_BYTES,
|
|
14
15
|
type DuplexOverStreamOptions,
|
|
15
16
|
duplexOverStream,
|
|
16
17
|
PeerGoneError,
|
|
@@ -21,3 +22,14 @@ export {
|
|
|
21
22
|
messageTargetOverStream,
|
|
22
23
|
type StreamMessageTarget,
|
|
23
24
|
} from "./message-target-over-stream.js";
|
|
25
|
+
export {
|
|
26
|
+
DEFAULT_PORT_MAX_MESSAGE_SIZE,
|
|
27
|
+
DEFAULT_PORT_PROTOCOL,
|
|
28
|
+
DEFAULT_PORT_STREAM_LIMIT,
|
|
29
|
+
type InboundPortInfo,
|
|
30
|
+
type Libp2pPortMux,
|
|
31
|
+
type Libp2pPortMuxOptions,
|
|
32
|
+
libp2pPortMux,
|
|
33
|
+
type StreamPort,
|
|
34
|
+
type ValueCodec,
|
|
35
|
+
} from "./port-mux.js";
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import type { Stream, StreamCloseEvent } from "@libp2p/interface";
|
|
2
2
|
import {
|
|
3
|
+
ByteBuffer,
|
|
3
4
|
closeStream,
|
|
4
|
-
decodeVarint,
|
|
5
5
|
encodeVarint,
|
|
6
6
|
normalizeChunk,
|
|
7
7
|
PeerGoneError,
|
|
8
|
+
readLength,
|
|
8
9
|
} from "./duplex-over-stream.js";
|
|
9
10
|
|
|
10
11
|
type Listener = (event: MessageEvent) => void | Promise<void>;
|
|
@@ -18,6 +19,7 @@ export interface StreamMessageTarget {
|
|
|
18
19
|
target: {
|
|
19
20
|
addEventListener(type: "message", listener: Listener): void;
|
|
20
21
|
removeEventListener(type: "message", listener: Listener): void;
|
|
22
|
+
/** Sends one message. Throws `TypeError` for anything but bytes. */
|
|
21
23
|
postMessage(message: unknown): void;
|
|
22
24
|
close(): Promise<void>;
|
|
23
25
|
/** The stream's write buffer, in bytes. */
|
|
@@ -27,6 +29,8 @@ export interface StreamMessageTarget {
|
|
|
27
29
|
* closed; at once when it can now.
|
|
28
30
|
*/
|
|
29
31
|
whenDrained(): Promise<void>;
|
|
32
|
+
/** `signal` again, on the target: `webrun-rpc`'s `getPortCloseSignal` reads it from a port. */
|
|
33
|
+
readonly closeSignal: AbortSignal;
|
|
30
34
|
};
|
|
31
35
|
/**
|
|
32
36
|
* Aborts when the stream is over, after every message the peer sent has
|
|
@@ -75,7 +79,9 @@ export function messageTargetOverStream(
|
|
|
75
79
|
): StreamMessageTarget {
|
|
76
80
|
const listeners = new Set<Listener>();
|
|
77
81
|
const life = new AbortController();
|
|
78
|
-
|
|
82
|
+
// The peer may have finished, or reset the stream, before this target was attached (during
|
|
83
|
+
// protocol negotiation): its events are gone then, and only the stream's state tells.
|
|
84
|
+
let peerClosedWrite = stream.remoteWriteStatus === "closed";
|
|
79
85
|
|
|
80
86
|
const onRemoteCloseWrite = (): void => {
|
|
81
87
|
// The read loop may still hold the peer's last messages; it aborts
|
|
@@ -89,6 +95,9 @@ export function messageTargetOverStream(
|
|
|
89
95
|
};
|
|
90
96
|
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
91
97
|
stream.addEventListener("close", onClose, { once: true });
|
|
98
|
+
// A stream reset or closed before this target was attached (a peer refusing it during protocol
|
|
99
|
+
// negotiation) fired its `close` unseen: report it now, as a dead peer.
|
|
100
|
+
if (stream.status !== undefined && stream.status !== "open") life.abort(new PeerGoneError());
|
|
92
101
|
|
|
93
102
|
const deliver = (data: Uint8Array): void => {
|
|
94
103
|
const event = new MessageEvent("message", { data });
|
|
@@ -101,35 +110,25 @@ export function messageTargetOverStream(
|
|
|
101
110
|
};
|
|
102
111
|
|
|
103
112
|
void (async () => {
|
|
104
|
-
//
|
|
105
|
-
//
|
|
106
|
-
|
|
107
|
-
let start = 0;
|
|
108
|
-
let end = 0;
|
|
113
|
+
// The buffer grows by doubling, so a large message arriving in many chunks
|
|
114
|
+
// is copied a bounded number of times.
|
|
115
|
+
const buf = new ByteBuffer();
|
|
109
116
|
for await (const item of stream) {
|
|
110
|
-
|
|
111
|
-
if (end + incoming.byteLength > buf.byteLength) {
|
|
112
|
-
const size = end - start + incoming.byteLength;
|
|
113
|
-
const next = new Uint8Array(Math.max(size, buf.byteLength * 2));
|
|
114
|
-
next.set(buf.subarray(start, end), 0);
|
|
115
|
-
buf = next;
|
|
116
|
-
end -= start;
|
|
117
|
-
start = 0;
|
|
118
|
-
}
|
|
119
|
-
buf.set(incoming, end);
|
|
120
|
-
end += incoming.byteLength;
|
|
117
|
+
buf.push(normalizeChunk(item));
|
|
121
118
|
for (;;) {
|
|
122
|
-
const
|
|
119
|
+
const bytes = buf.unread();
|
|
120
|
+
const header = readLength(bytes, "message");
|
|
123
121
|
if (header === undefined) break; // the length is not all here yet
|
|
124
122
|
if (header.value > maxMessageBytes) {
|
|
125
123
|
throw new Error(
|
|
126
124
|
`webrun-streams-libp2p: a ${header.value}-byte message exceeds maxMessageBytes (${maxMessageBytes})`,
|
|
127
125
|
);
|
|
128
126
|
}
|
|
129
|
-
const stop =
|
|
130
|
-
if (
|
|
131
|
-
|
|
132
|
-
|
|
127
|
+
const stop = header.offset + header.value;
|
|
128
|
+
if (bytes.byteLength < stop) break;
|
|
129
|
+
const message = bytes.slice(header.offset, stop);
|
|
130
|
+
buf.consume(stop);
|
|
131
|
+
deliver(message);
|
|
133
132
|
}
|
|
134
133
|
}
|
|
135
134
|
if (peerClosedWrite) {
|
|
@@ -147,6 +146,7 @@ export function messageTargetOverStream(
|
|
|
147
146
|
return {
|
|
148
147
|
signal: life.signal,
|
|
149
148
|
target: {
|
|
149
|
+
closeSignal: life.signal,
|
|
150
150
|
addEventListener: (_type, listener) => {
|
|
151
151
|
listeners.add(listener);
|
|
152
152
|
},
|
|
@@ -154,7 +154,12 @@ export function messageTargetOverStream(
|
|
|
154
154
|
listeners.delete(listener);
|
|
155
155
|
},
|
|
156
156
|
postMessage(message) {
|
|
157
|
-
const payload =
|
|
157
|
+
const payload = toBytes(message);
|
|
158
|
+
if (!payload) {
|
|
159
|
+
throw new TypeError(
|
|
160
|
+
`webrun-streams-libp2p: a byte pipe carries Uint8Array, ArrayBuffer or a view, got ${Object.prototype.toString.call(message)}`,
|
|
161
|
+
);
|
|
162
|
+
}
|
|
158
163
|
const header = encodeVarint(payload.byteLength);
|
|
159
164
|
const frame = new Uint8Array(header.byteLength + payload.byteLength);
|
|
160
165
|
frame.set(header, 0);
|
|
@@ -188,12 +193,12 @@ export function messageTargetOverStream(
|
|
|
188
193
|
};
|
|
189
194
|
}
|
|
190
195
|
|
|
191
|
-
/**
|
|
192
|
-
function
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
throw new Error("webrun-streams-libp2p: a malformed message length", { cause: err });
|
|
196
|
+
/** `message` as bytes, or `undefined` when it is not bytes */
|
|
197
|
+
function toBytes(message: unknown): Uint8Array | undefined {
|
|
198
|
+
if (message instanceof Uint8Array) return message;
|
|
199
|
+
if (message instanceof ArrayBuffer) return new Uint8Array(message);
|
|
200
|
+
if (ArrayBuffer.isView(message)) {
|
|
201
|
+
return new Uint8Array(message.buffer, message.byteOffset, message.byteLength);
|
|
198
202
|
}
|
|
203
|
+
return undefined;
|
|
199
204
|
}
|