@statewalker/webrun-streams-libp2p 0.1.8 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -5
- package/dist/connect-serve.d.ts +15 -1
- package/dist/connect-serve.d.ts.map +1 -1
- package/dist/duplex-over-stream.d.ts +32 -0
- package/dist/duplex-over-stream.d.ts.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +525 -53
- package/dist/message-target-over-stream.d.ts +64 -0
- package/dist/message-target-over-stream.d.ts.map +1 -0
- package/dist/port-mux.d.ts +89 -0
- package/dist/port-mux.d.ts.map +1 -0
- package/dist/stream-gate.d.ts +10 -0
- package/dist/stream-gate.d.ts.map +1 -0
- package/package.json +7 -4
- package/src/connect-serve.ts +98 -72
- package/src/duplex-over-stream.ts +219 -11
- package/src/index.ts +18 -0
- package/src/message-target-over-stream.ts +207 -0
- package/src/port-mux.ts +238 -0
- package/src/stream-gate.ts +77 -0
|
@@ -52,6 +52,30 @@ export const DEFAULT_DRAIN_TIMEOUT_MS = 300_000;
|
|
|
52
52
|
*/
|
|
53
53
|
export const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
54
54
|
|
|
55
|
+
/**
|
|
56
|
+
* Default for {@link DuplexOverStreamOptions.readHighWaterBytes}: received bytes
|
|
57
|
+
* held for a consumer that is not reading before the stream is paused.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_READ_HIGH_WATER_BYTES = 64 * 1024;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The stream ended before the call finished: the peer died, the connection
|
|
63
|
+
* dropped, or the stream was reset or refused before both sides ended it.
|
|
64
|
+
* Distinct from a clean end on purpose — a response cut short must never read
|
|
65
|
+
* as a complete one. `cause` is the stream's close error, when libp2p gave one.
|
|
66
|
+
*/
|
|
67
|
+
export class PeerGoneError extends Error {
|
|
68
|
+
constructor(cause?: unknown) {
|
|
69
|
+
super(
|
|
70
|
+
`webrun-streams-libp2p: the stream ended before the call finished${
|
|
71
|
+
cause instanceof Error ? ` (${cause.message})` : ""
|
|
72
|
+
}`,
|
|
73
|
+
{ cause },
|
|
74
|
+
);
|
|
75
|
+
this.name = "PeerGoneError";
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
55
79
|
const textEncoder = new TextEncoder();
|
|
56
80
|
const textDecoder = new TextDecoder();
|
|
57
81
|
|
|
@@ -80,6 +104,18 @@ export interface DuplexOverStreamOptions {
|
|
|
80
104
|
* the peer. Defaults to {@link DEFAULT_DRAIN_TIMEOUT_MS}.
|
|
81
105
|
*/
|
|
82
106
|
drainTimeoutMs?: number;
|
|
107
|
+
/**
|
|
108
|
+
* Received bytes held for a consumer that is not reading before the stream
|
|
109
|
+
* is paused (and resumed once half of them are read). A paused yamux stream
|
|
110
|
+
* stops granting its peer receive window, so the peer's `send()` reports a
|
|
111
|
+
* full buffer and its producer waits: run-ahead is bounded by two receive
|
|
112
|
+
* windows (what libp2p held while paused, and the window granted on resume)
|
|
113
|
+
* plus this — the receiving node's yamux `streamOptions.maxStreamWindowSize`,
|
|
114
|
+
* 16 MiB by default; 1 MiB bounds it at about 2.25 MiB with no loss of
|
|
115
|
+
* one-way throughput measured. Only where the muxer exposes its window.
|
|
116
|
+
* Defaults to {@link DEFAULT_READ_HIGH_WATER_BYTES}.
|
|
117
|
+
*/
|
|
118
|
+
readHighWaterBytes?: number;
|
|
83
119
|
}
|
|
84
120
|
|
|
85
121
|
/**
|
|
@@ -116,6 +152,32 @@ export async function* duplexOverStream(
|
|
|
116
152
|
);
|
|
117
153
|
};
|
|
118
154
|
|
|
155
|
+
// A CLEAN END IS THE PEER'S OWN `remoteCloseWrite`, NOTHING ELSE. When the
|
|
156
|
+
// connection dies (a killed process, a dropped network), libp2p closes the
|
|
157
|
+
// stream WITHOUT an error and its iterator ends exactly as it does after the
|
|
158
|
+
// peer's close — so a truncated response read as a complete one. Listened
|
|
159
|
+
// for before the first pull below, like the iterator's own listener. A
|
|
160
|
+
// generator's body runs only on the consumer's first `next()`, so the peer
|
|
161
|
+
// may have closed already; its state is read for that case.
|
|
162
|
+
let peerClosedWrite = stream.remoteWriteStatus === "closed";
|
|
163
|
+
let closeError: unknown;
|
|
164
|
+
let onStreamClosed!: () => void;
|
|
165
|
+
const streamClosed = new Promise<void>((resolve) => {
|
|
166
|
+
onStreamClosed = resolve;
|
|
167
|
+
});
|
|
168
|
+
const onRemoteCloseWrite = (): void => {
|
|
169
|
+
peerClosedWrite = true;
|
|
170
|
+
};
|
|
171
|
+
const onClose = (evt: StreamCloseEvent): void => {
|
|
172
|
+
closeError = evt.error;
|
|
173
|
+
onStreamClosed();
|
|
174
|
+
};
|
|
175
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
176
|
+
stream.addEventListener("close", onClose);
|
|
177
|
+
|
|
178
|
+
// Set once every input chunk went out. A stream that closes before that lost
|
|
179
|
+
// part of this side's input, which is not a complete call either.
|
|
180
|
+
let inputSent = false;
|
|
119
181
|
const outboundSource = framedOutbound(inputIterator);
|
|
120
182
|
const outbound = (async () => {
|
|
121
183
|
try {
|
|
@@ -134,6 +196,7 @@ export async function* duplexOverStream(
|
|
|
134
196
|
await waitForDrain(stream, opts.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS);
|
|
135
197
|
}
|
|
136
198
|
}
|
|
199
|
+
inputSent = true;
|
|
137
200
|
await closeStream(stream);
|
|
138
201
|
} catch (err) {
|
|
139
202
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
@@ -165,14 +228,31 @@ export async function* duplexOverStream(
|
|
|
165
228
|
// subscribe is lost and this loop would never end. Every current path from
|
|
166
229
|
// dialProtocol/onStream to here is microtask-only, so a socket FIN cannot
|
|
167
230
|
// interleave. Inserting an await above would break that.
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
231
|
+
let peerError: Error | undefined;
|
|
232
|
+
try {
|
|
233
|
+
for await (const frame of parseFrames(
|
|
234
|
+
pacedSource(stream, opts.readHighWaterBytes ?? DEFAULT_READ_HIGH_WATER_BYTES),
|
|
235
|
+
)) {
|
|
236
|
+
if (frame.type === TYPE_DATA) {
|
|
237
|
+
yield frame.payload;
|
|
238
|
+
} else if (frame.type === TYPE_ERROR) {
|
|
239
|
+
peerError = decodeError(frame.payload);
|
|
240
|
+
firePeerInputEnd(peerError);
|
|
241
|
+
throw peerError;
|
|
242
|
+
}
|
|
175
243
|
}
|
|
244
|
+
} catch (err) {
|
|
245
|
+
if (err === peerError) throw err;
|
|
246
|
+
// The stream itself failed (reset, aborted, its connection dropped):
|
|
247
|
+
// the call did not finish, whatever libp2p named the error.
|
|
248
|
+
const gone = new PeerGoneError(err);
|
|
249
|
+
firePeerInputEnd(gone);
|
|
250
|
+
throw gone;
|
|
251
|
+
}
|
|
252
|
+
if (!peerClosedWrite) {
|
|
253
|
+
const gone = new PeerGoneError(closeError);
|
|
254
|
+
firePeerInputEnd(gone);
|
|
255
|
+
throw gone;
|
|
176
256
|
}
|
|
177
257
|
sourceCompleted = true;
|
|
178
258
|
firePeerInputEnd();
|
|
@@ -214,9 +294,23 @@ export async function* duplexOverStream(
|
|
|
214
294
|
}
|
|
215
295
|
void outbound.catch(() => undefined);
|
|
216
296
|
} else {
|
|
217
|
-
|
|
297
|
+
// The peer ended its side; ours may still be sending, waiting on an
|
|
298
|
+
// input that is still open (a tunnel). If the stream dies first, stop
|
|
299
|
+
// waiting for that input: it can never be sent now.
|
|
300
|
+
await Promise.race([outbound, streamClosed]);
|
|
301
|
+
if (!inputSent) {
|
|
302
|
+
cancelInput();
|
|
303
|
+
void Promise.resolve(outboundSource.return?.(undefined)).catch(() => undefined);
|
|
304
|
+
void outbound.catch(() => undefined);
|
|
305
|
+
} else {
|
|
306
|
+
await outbound;
|
|
307
|
+
}
|
|
218
308
|
}
|
|
309
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
310
|
+
stream.removeEventListener("close", onClose);
|
|
219
311
|
}
|
|
312
|
+
// Reached only after a clean end from the peer.
|
|
313
|
+
if (!inputSent) throw new PeerGoneError(closeError);
|
|
220
314
|
}
|
|
221
315
|
|
|
222
316
|
/**
|
|
@@ -339,6 +433,120 @@ async function* framedOutbound(
|
|
|
339
433
|
}
|
|
340
434
|
}
|
|
341
435
|
|
|
436
|
+
/**
|
|
437
|
+
* The stream's inbound bytes, read at the consumer's pace. libp2p's own
|
|
438
|
+
* iterator moves every message into an unbounded queue as it arrives, and
|
|
439
|
+
* yamux grants its peer more receive window on arrival, not on consumption: a
|
|
440
|
+
* consumer that stopped reading let the producer run 22-24 MiB ahead until
|
|
441
|
+
* libp2p's read buffer killed the stream (L6). Here the queue is bounded: past
|
|
442
|
+
* `highWater` bytes the stream is paused (yamux then withholds window updates,
|
|
443
|
+
* libp2p keeps what still arrives in its read buffer, undispatched), and it is
|
|
444
|
+
* resumed once the consumer has read the queue down to half.
|
|
445
|
+
*
|
|
446
|
+
* Ends as libp2p's iterator does: on the peer's `remoteCloseWrite`, or on
|
|
447
|
+
* `close` (throwing its error, if any). Subscribes on the first pull, like it.
|
|
448
|
+
*/
|
|
449
|
+
async function* pacedSource(stream: Stream, highWater: number): AsyncGenerator<Uint8Array> {
|
|
450
|
+
if (stream.readStatus !== "readable" && stream.readStatus !== "paused") return;
|
|
451
|
+
// Pausing bounds anything only where the muxer then stops granting its peer
|
|
452
|
+
// window (yamux, which exposes `maxStreamWindowSize`). Elsewhere a pause just
|
|
453
|
+
// moves the data into libp2p's read buffer, which aborts the stream past
|
|
454
|
+
// `maxReadBufferLength`: there the stream is read as libp2p's own iterator
|
|
455
|
+
// reads it, without a bound.
|
|
456
|
+
const window = (stream as { maxStreamWindowSize?: unknown }).maxStreamWindowSize;
|
|
457
|
+
const paced = typeof window === "number";
|
|
458
|
+
// While paused, libp2p holds what the peer may still send (at most the
|
|
459
|
+
// window granted before the pause) and aborts the stream past
|
|
460
|
+
// `maxReadBufferLength`. yamux's default window (16 MiB) exceeds libp2p's
|
|
461
|
+
// default buffer (4 MiB), so the buffer is raised to the window plus what is
|
|
462
|
+
// queued here; yamux itself refuses a peer that sends past its window.
|
|
463
|
+
if (paced && stream.maxReadBufferLength < window + highWater) {
|
|
464
|
+
stream.maxReadBufferLength = window + highWater;
|
|
465
|
+
}
|
|
466
|
+
const queue: Uint8Array[] = [];
|
|
467
|
+
let queued = 0;
|
|
468
|
+
let ended = false;
|
|
469
|
+
let error: unknown;
|
|
470
|
+
let resuming = false;
|
|
471
|
+
let wake: (() => void) | undefined;
|
|
472
|
+
const notify = (): void => {
|
|
473
|
+
const w = wake;
|
|
474
|
+
wake = undefined;
|
|
475
|
+
w?.();
|
|
476
|
+
};
|
|
477
|
+
const pauseIfFull = (): void => {
|
|
478
|
+
if (paced && queued >= highWater && stream.readStatus === "readable") {
|
|
479
|
+
try {
|
|
480
|
+
stream.pause();
|
|
481
|
+
} catch {
|
|
482
|
+
/* closing: nothing more will be granted anyway */
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
};
|
|
486
|
+
const onMessage = (evt: { data: unknown }): void => {
|
|
487
|
+
const chunk = normalizeChunk(evt.data);
|
|
488
|
+
if (chunk.byteLength === 0) return;
|
|
489
|
+
// In pieces of at most `highWater` (views, not copies): a resume hands over
|
|
490
|
+
// everything libp2p held as one message, up to a window, and a reader that
|
|
491
|
+
// took it whole would hold that window outside this queue's count.
|
|
492
|
+
for (let at = 0; at < chunk.byteLength; at += highWater) {
|
|
493
|
+
queue.push(chunk.subarray(at, Math.min(at + highWater, chunk.byteLength)));
|
|
494
|
+
}
|
|
495
|
+
queued += chunk.byteLength;
|
|
496
|
+
// Never from inside `resume()`: it dispatches what libp2p held first and
|
|
497
|
+
// tells the muxer to grant window after, so a pause here would leave libp2p
|
|
498
|
+
// paused and yamux granting window for every frame (review of #112).
|
|
499
|
+
if (!resuming) pauseIfFull();
|
|
500
|
+
notify();
|
|
501
|
+
};
|
|
502
|
+
const resume = (): void => {
|
|
503
|
+
resuming = true;
|
|
504
|
+
try {
|
|
505
|
+
stream.resume(); // dispatches what libp2p held meanwhile, synchronously
|
|
506
|
+
} catch {
|
|
507
|
+
/* closing */
|
|
508
|
+
} finally {
|
|
509
|
+
resuming = false;
|
|
510
|
+
}
|
|
511
|
+
pauseIfFull(); // after the muxer's own resume, so the pause holds
|
|
512
|
+
};
|
|
513
|
+
const onClose = (evt: StreamCloseEvent): void => {
|
|
514
|
+
ended = true;
|
|
515
|
+
error ??= evt.error;
|
|
516
|
+
notify();
|
|
517
|
+
};
|
|
518
|
+
const onRemoteCloseWrite = (): void => {
|
|
519
|
+
ended = true;
|
|
520
|
+
notify();
|
|
521
|
+
};
|
|
522
|
+
stream.addEventListener("message", onMessage);
|
|
523
|
+
stream.addEventListener("close", onClose);
|
|
524
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
525
|
+
if (stream.readStatus === "paused") resume(); // paused by someone else: nothing here resumes it later
|
|
526
|
+
try {
|
|
527
|
+
for (;;) {
|
|
528
|
+
const chunk = queue.shift();
|
|
529
|
+
if (chunk) {
|
|
530
|
+
queued -= chunk.byteLength;
|
|
531
|
+
if (queued <= highWater / 2 && stream.readStatus === "paused") resume();
|
|
532
|
+
yield chunk;
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
if (ended) {
|
|
536
|
+
if (error) throw error;
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
await new Promise<void>((resolve) => {
|
|
540
|
+
wake = resolve;
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
} finally {
|
|
544
|
+
stream.removeEventListener("message", onMessage);
|
|
545
|
+
stream.removeEventListener("close", onClose);
|
|
546
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
342
550
|
async function* parseFrames(
|
|
343
551
|
source: AsyncIterable<unknown>,
|
|
344
552
|
): AsyncGenerator<{ type: number; payload: Uint8Array }> {
|
|
@@ -373,7 +581,7 @@ async function* parseFrames(
|
|
|
373
581
|
}
|
|
374
582
|
}
|
|
375
583
|
|
|
376
|
-
function normalizeChunk(item: unknown): Uint8Array {
|
|
584
|
+
export function normalizeChunk(item: unknown): Uint8Array {
|
|
377
585
|
if (item instanceof Uint8Array) return item;
|
|
378
586
|
const asList = item as { subarray?: () => Uint8Array };
|
|
379
587
|
if (typeof asList.subarray === "function") {
|
|
@@ -410,7 +618,7 @@ function decodeError(payload: Uint8Array): Error {
|
|
|
410
618
|
}
|
|
411
619
|
}
|
|
412
620
|
|
|
413
|
-
function encodeVarint(value: number): Uint8Array {
|
|
621
|
+
export function encodeVarint(value: number): Uint8Array {
|
|
414
622
|
if (!Number.isInteger(value) || value < 0) {
|
|
415
623
|
throw new RangeError(`encodeVarint: ${value} is not a non-negative integer`);
|
|
416
624
|
}
|
|
@@ -424,7 +632,7 @@ function encodeVarint(value: number): Uint8Array {
|
|
|
424
632
|
return new Uint8Array(out);
|
|
425
633
|
}
|
|
426
634
|
|
|
427
|
-
function decodeVarint(buf: Uint8Array, start: number): { value: number; offset: number } {
|
|
635
|
+
export function decodeVarint(buf: Uint8Array, start: number): { value: number; offset: number } {
|
|
428
636
|
let value = 0;
|
|
429
637
|
let shift = 0;
|
|
430
638
|
let i = start;
|
package/src/index.ts
CHANGED
|
@@ -13,4 +13,22 @@ export {
|
|
|
13
13
|
DEFAULT_DRAIN_TIMEOUT_MS,
|
|
14
14
|
type DuplexOverStreamOptions,
|
|
15
15
|
duplexOverStream,
|
|
16
|
+
PeerGoneError,
|
|
16
17
|
} from "./duplex-over-stream.js";
|
|
18
|
+
export {
|
|
19
|
+
DEFAULT_MAX_MESSAGE_BYTES,
|
|
20
|
+
type MessageTargetOverStreamOptions,
|
|
21
|
+
messageTargetOverStream,
|
|
22
|
+
type StreamMessageTarget,
|
|
23
|
+
} from "./message-target-over-stream.js";
|
|
24
|
+
export {
|
|
25
|
+
DEFAULT_PORT_MAX_MESSAGE_SIZE,
|
|
26
|
+
DEFAULT_PORT_PROTOCOL,
|
|
27
|
+
DEFAULT_PORT_STREAM_LIMIT,
|
|
28
|
+
type InboundPortInfo,
|
|
29
|
+
type Libp2pPortMux,
|
|
30
|
+
type Libp2pPortMuxOptions,
|
|
31
|
+
libp2pPortMux,
|
|
32
|
+
type StreamPort,
|
|
33
|
+
type ValueCodec,
|
|
34
|
+
} from "./port-mux.js";
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import type { Stream, StreamCloseEvent } from "@libp2p/interface";
|
|
2
|
+
import {
|
|
3
|
+
closeStream,
|
|
4
|
+
decodeVarint,
|
|
5
|
+
encodeVarint,
|
|
6
|
+
normalizeChunk,
|
|
7
|
+
PeerGoneError,
|
|
8
|
+
} from "./duplex-over-stream.js";
|
|
9
|
+
|
|
10
|
+
type Listener = (event: MessageEvent) => void | Promise<void>;
|
|
11
|
+
|
|
12
|
+
/** What `messageTargetOverStream` gives: a byte pipe, and its life. */
|
|
13
|
+
export interface StreamMessageTarget {
|
|
14
|
+
/**
|
|
15
|
+
* A `MessageTarget` (as `@statewalker/webrun-rpc` defines it) whose messages
|
|
16
|
+
* are `Uint8Array`s, one length-prefixed frame each.
|
|
17
|
+
*/
|
|
18
|
+
target: {
|
|
19
|
+
addEventListener(type: "message", listener: Listener): void;
|
|
20
|
+
removeEventListener(type: "message", listener: Listener): void;
|
|
21
|
+
postMessage(message: unknown): void;
|
|
22
|
+
close(): Promise<void>;
|
|
23
|
+
/** The stream's write buffer, in bytes. */
|
|
24
|
+
readonly bufferedAmount: number;
|
|
25
|
+
/**
|
|
26
|
+
* Resolves once the stream can take more (libp2p's `drain`), or has
|
|
27
|
+
* closed; at once when it can now.
|
|
28
|
+
*/
|
|
29
|
+
whenDrained(): Promise<void>;
|
|
30
|
+
/** `signal` again, on the target: `webrun-rpc`'s `getPortCloseSignal` reads it from a port. */
|
|
31
|
+
readonly closeSignal: AbortSignal;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Aborts when the stream is over, after every message the peer sent has
|
|
35
|
+
* been delivered. The reason is a {@link PeerGoneError} when the stream
|
|
36
|
+
* ended without the peer finishing it, or an `Error` when the peer closed
|
|
37
|
+
* it or sent bytes that are not this framing.
|
|
38
|
+
* Pass it as `multiplexPort`'s / `overPipe`'s `signal`, so a dead peer
|
|
39
|
+
* closes the multiplexer and every port on it at once.
|
|
40
|
+
*/
|
|
41
|
+
signal: AbortSignal;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Options for {@link messageTargetOverStream}. */
|
|
45
|
+
export interface MessageTargetOverStreamOptions {
|
|
46
|
+
/**
|
|
47
|
+
* The largest message accepted, in bytes. A length prefix past it — or one
|
|
48
|
+
* that is not a valid varint — means the peer is not speaking this framing,
|
|
49
|
+
* so the stream is aborted rather than buffered without bound. Default
|
|
50
|
+
* {@link DEFAULT_MAX_MESSAGE_BYTES}.
|
|
51
|
+
*/
|
|
52
|
+
maxMessageBytes?: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 64 MiB: room for a large unframed message, short of exhausting memory. */
|
|
56
|
+
export const DEFAULT_MAX_MESSAGE_BYTES = 64 * 1024 * 1024;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* One libp2p stream as a pipe of messages, for a multiplexer to run over:
|
|
60
|
+
*
|
|
61
|
+
* ```ts
|
|
62
|
+
* const { target, signal } = messageTargetOverStream(stream);
|
|
63
|
+
* const mux = multiplexPort(target, { codec: msgpackCodec, signal });
|
|
64
|
+
* ```
|
|
65
|
+
*
|
|
66
|
+
* Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
67
|
+
* `message` listener in the same tick: a message with no listener is dropped,
|
|
68
|
+
* as on a `MessagePort`. Every message the peer sent before closing is
|
|
69
|
+
* delivered before `signal` aborts. `postMessage` hands the frame to
|
|
70
|
+
* `stream.send()` and does not wait for room; libp2p buffers up to the
|
|
71
|
+
* stream's `maxWriteBufferLength`, past which it resets the stream. A sender
|
|
72
|
+
* that must wait awaits `whenDrained()`.
|
|
73
|
+
*/
|
|
74
|
+
export function messageTargetOverStream(
|
|
75
|
+
stream: Stream,
|
|
76
|
+
{ maxMessageBytes = DEFAULT_MAX_MESSAGE_BYTES }: MessageTargetOverStreamOptions = {},
|
|
77
|
+
): StreamMessageTarget {
|
|
78
|
+
const listeners = new Set<Listener>();
|
|
79
|
+
const life = new AbortController();
|
|
80
|
+
// The peer may have finished, or reset the stream, before this target was attached (during
|
|
81
|
+
// protocol negotiation): its events are gone then, and only the stream's state tells.
|
|
82
|
+
let peerClosedWrite = stream.remoteWriteStatus === "closed";
|
|
83
|
+
|
|
84
|
+
const onRemoteCloseWrite = (): void => {
|
|
85
|
+
// The read loop may still hold the peer's last messages; it aborts
|
|
86
|
+
// `signal` once it has delivered them.
|
|
87
|
+
peerClosedWrite = true;
|
|
88
|
+
};
|
|
89
|
+
const onClose = (evt: StreamCloseEvent): void => {
|
|
90
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
91
|
+
if (peerClosedWrite && evt.error == null) return; // the read loop ends it
|
|
92
|
+
life.abort(new PeerGoneError(evt.error));
|
|
93
|
+
};
|
|
94
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
95
|
+
stream.addEventListener("close", onClose, { once: true });
|
|
96
|
+
// A stream reset or closed before this target was attached (a peer refusing it during protocol
|
|
97
|
+
// negotiation) fired its `close` unseen: report it now, as a dead peer.
|
|
98
|
+
if (stream.status !== undefined && stream.status !== "open") life.abort(new PeerGoneError());
|
|
99
|
+
|
|
100
|
+
const deliver = (data: Uint8Array): void => {
|
|
101
|
+
const event = new MessageEvent("message", { data });
|
|
102
|
+
for (const listener of [...listeners]) {
|
|
103
|
+
try {
|
|
104
|
+
// One listener's fault — thrown or rejected — must not stop the read loop.
|
|
105
|
+
Promise.resolve(listener(event)).catch(() => undefined);
|
|
106
|
+
} catch {}
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
void (async () => {
|
|
111
|
+
// Unread bytes are `buf[start, end)`; the buffer grows by doubling, so a
|
|
112
|
+
// large message arriving in many chunks is copied a bounded number of times.
|
|
113
|
+
let buf = new Uint8Array(0);
|
|
114
|
+
let start = 0;
|
|
115
|
+
let end = 0;
|
|
116
|
+
for await (const item of stream) {
|
|
117
|
+
const incoming = normalizeChunk(item);
|
|
118
|
+
if (end + incoming.byteLength > buf.byteLength) {
|
|
119
|
+
const size = end - start + incoming.byteLength;
|
|
120
|
+
const next = new Uint8Array(Math.max(size, buf.byteLength * 2));
|
|
121
|
+
next.set(buf.subarray(start, end), 0);
|
|
122
|
+
buf = next;
|
|
123
|
+
end -= start;
|
|
124
|
+
start = 0;
|
|
125
|
+
}
|
|
126
|
+
buf.set(incoming, end);
|
|
127
|
+
end += incoming.byteLength;
|
|
128
|
+
for (;;) {
|
|
129
|
+
const header = readLength(buf.subarray(start, end));
|
|
130
|
+
if (header === undefined) break; // the length is not all here yet
|
|
131
|
+
if (header.value > maxMessageBytes) {
|
|
132
|
+
throw new Error(
|
|
133
|
+
`webrun-streams-libp2p: a ${header.value}-byte message exceeds maxMessageBytes (${maxMessageBytes})`,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
const stop = start + header.offset + header.value;
|
|
137
|
+
if (end < stop) break;
|
|
138
|
+
deliver(buf.slice(start + header.offset, stop));
|
|
139
|
+
start = stop;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (peerClosedWrite) {
|
|
143
|
+
life.abort(new Error("webrun-streams-libp2p: the peer closed the stream"));
|
|
144
|
+
await closeStream(stream);
|
|
145
|
+
}
|
|
146
|
+
})().catch((err: unknown) => {
|
|
147
|
+
// A reset ends the loop with an error, which `close` reports through
|
|
148
|
+
// `signal`; a malformed frame is reported here.
|
|
149
|
+
life.abort(err);
|
|
150
|
+
try {
|
|
151
|
+
stream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
152
|
+
} catch {}
|
|
153
|
+
});
|
|
154
|
+
return {
|
|
155
|
+
signal: life.signal,
|
|
156
|
+
target: {
|
|
157
|
+
closeSignal: life.signal,
|
|
158
|
+
addEventListener: (_type, listener) => {
|
|
159
|
+
listeners.add(listener);
|
|
160
|
+
},
|
|
161
|
+
removeEventListener: (_type, listener) => {
|
|
162
|
+
listeners.delete(listener);
|
|
163
|
+
},
|
|
164
|
+
postMessage(message) {
|
|
165
|
+
const payload = normalizeChunk(message);
|
|
166
|
+
const header = encodeVarint(payload.byteLength);
|
|
167
|
+
const frame = new Uint8Array(header.byteLength + payload.byteLength);
|
|
168
|
+
frame.set(header, 0);
|
|
169
|
+
frame.set(payload, header.byteLength);
|
|
170
|
+
try {
|
|
171
|
+
stream.send(frame);
|
|
172
|
+
} catch {
|
|
173
|
+
// The stream is closed for writing; `signal` says so.
|
|
174
|
+
}
|
|
175
|
+
},
|
|
176
|
+
async close() {
|
|
177
|
+
listeners.clear();
|
|
178
|
+
await closeStream(stream);
|
|
179
|
+
},
|
|
180
|
+
get bufferedAmount() {
|
|
181
|
+
return stream.writeBufferLength;
|
|
182
|
+
},
|
|
183
|
+
whenDrained() {
|
|
184
|
+
if (!stream.writableNeedsDrain || life.signal.aborted) return Promise.resolve();
|
|
185
|
+
return new Promise<void>((resolve) => {
|
|
186
|
+
const done = (): void => {
|
|
187
|
+
stream.removeEventListener("drain", done);
|
|
188
|
+
stream.removeEventListener("close", done);
|
|
189
|
+
resolve();
|
|
190
|
+
};
|
|
191
|
+
stream.addEventListener("drain", done);
|
|
192
|
+
stream.addEventListener("close", done);
|
|
193
|
+
});
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** The length prefix of `bytes`, or `undefined` while it is incomplete; throws when malformed. */
|
|
200
|
+
function readLength(bytes: Uint8Array): { value: number; offset: number } | undefined {
|
|
201
|
+
try {
|
|
202
|
+
return decodeVarint(bytes, 0);
|
|
203
|
+
} catch (err) {
|
|
204
|
+
if (bytes.byteLength < 5) return undefined;
|
|
205
|
+
throw new Error("webrun-streams-libp2p: a malformed message length", { cause: err });
|
|
206
|
+
}
|
|
207
|
+
}
|