@statewalker/webrun-streams-libp2p 0.1.8 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -5
- package/dist/connect-serve.d.ts +15 -1
- package/dist/connect-serve.d.ts.map +1 -1
- package/dist/duplex-over-stream.d.ts +32 -0
- package/dist/duplex-over-stream.d.ts.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +525 -53
- package/dist/message-target-over-stream.d.ts +64 -0
- package/dist/message-target-over-stream.d.ts.map +1 -0
- package/dist/port-mux.d.ts +89 -0
- package/dist/port-mux.d.ts.map +1 -0
- package/dist/stream-gate.d.ts +10 -0
- package/dist/stream-gate.d.ts.map +1 -0
- package/package.json +7 -4
- package/src/connect-serve.ts +98 -72
- package/src/duplex-over-stream.ts +219 -11
- package/src/index.ts +18 -0
- package/src/message-target-over-stream.ts +207 -0
- package/src/port-mux.ts +238 -0
- package/src/stream-gate.ts +77 -0
package/dist/index.js
CHANGED
|
@@ -48,6 +48,18 @@ const DEFAULT_DRAIN_TIMEOUT_MS = 3e5;
|
|
|
48
48
|
* Exported so a deployment that knows its links can lower it deliberately.
|
|
49
49
|
*/
|
|
50
50
|
const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
51
|
+
/**
|
|
52
|
+
* The stream ended before the call finished: the peer died, the connection
|
|
53
|
+
* dropped, or the stream was reset or refused before both sides ended it.
|
|
54
|
+
* Distinct from a clean end on purpose — a response cut short must never read
|
|
55
|
+
* as a complete one. `cause` is the stream's close error, when libp2p gave one.
|
|
56
|
+
*/
|
|
57
|
+
var PeerGoneError = class extends Error {
|
|
58
|
+
constructor(cause) {
|
|
59
|
+
super(`webrun-streams-libp2p: the stream ended before the call finished${cause instanceof Error ? ` (${cause.message})` : ""}`, { cause });
|
|
60
|
+
this.name = "PeerGoneError";
|
|
61
|
+
}
|
|
62
|
+
};
|
|
51
63
|
const textEncoder = new TextEncoder();
|
|
52
64
|
const textDecoder = new TextDecoder();
|
|
53
65
|
/**
|
|
@@ -72,10 +84,27 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
72
84
|
const cancelInput = () => {
|
|
73
85
|
Promise.resolve(inputIterator.return?.(void 0)).catch(() => void 0);
|
|
74
86
|
};
|
|
87
|
+
let peerClosedWrite = stream.remoteWriteStatus === "closed";
|
|
88
|
+
let closeError;
|
|
89
|
+
let onStreamClosed;
|
|
90
|
+
const streamClosed = new Promise((resolve) => {
|
|
91
|
+
onStreamClosed = resolve;
|
|
92
|
+
});
|
|
93
|
+
const onRemoteCloseWrite = () => {
|
|
94
|
+
peerClosedWrite = true;
|
|
95
|
+
};
|
|
96
|
+
const onClose = (evt) => {
|
|
97
|
+
closeError = evt.error;
|
|
98
|
+
onStreamClosed();
|
|
99
|
+
};
|
|
100
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
101
|
+
stream.addEventListener("close", onClose);
|
|
102
|
+
let inputSent = false;
|
|
75
103
|
const outboundSource = framedOutbound(inputIterator);
|
|
76
104
|
const outbound = (async () => {
|
|
77
105
|
try {
|
|
78
106
|
for await (const chunk of outboundSource) if (!stream.send(chunk)) await waitForDrain(stream, opts.drainTimeoutMs ?? 3e5);
|
|
107
|
+
inputSent = true;
|
|
79
108
|
await closeStream(stream);
|
|
80
109
|
} catch (err) {
|
|
81
110
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
@@ -90,11 +119,24 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
90
119
|
})();
|
|
91
120
|
let sourceCompleted = false;
|
|
92
121
|
try {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
122
|
+
let peerError;
|
|
123
|
+
try {
|
|
124
|
+
for await (const frame of parseFrames(pacedSource(stream, opts.readHighWaterBytes ?? 65536))) if (frame.type === TYPE_DATA) yield frame.payload;
|
|
125
|
+
else if (frame.type === TYPE_ERROR) {
|
|
126
|
+
peerError = decodeError(frame.payload);
|
|
127
|
+
firePeerInputEnd(peerError);
|
|
128
|
+
throw peerError;
|
|
129
|
+
}
|
|
130
|
+
} catch (err) {
|
|
131
|
+
if (err === peerError) throw err;
|
|
132
|
+
const gone = new PeerGoneError(err);
|
|
133
|
+
firePeerInputEnd(gone);
|
|
134
|
+
throw gone;
|
|
135
|
+
}
|
|
136
|
+
if (!peerClosedWrite) {
|
|
137
|
+
const gone = new PeerGoneError(closeError);
|
|
138
|
+
firePeerInputEnd(gone);
|
|
139
|
+
throw gone;
|
|
98
140
|
}
|
|
99
141
|
sourceCompleted = true;
|
|
100
142
|
firePeerInputEnd();
|
|
@@ -108,8 +150,18 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
108
150
|
stream.abort(/* @__PURE__ */ new Error("duplexOverStream: consumer cancelled"));
|
|
109
151
|
} catch {}
|
|
110
152
|
outbound.catch(() => void 0);
|
|
111
|
-
} else
|
|
153
|
+
} else {
|
|
154
|
+
await Promise.race([outbound, streamClosed]);
|
|
155
|
+
if (!inputSent) {
|
|
156
|
+
cancelInput();
|
|
157
|
+
Promise.resolve(outboundSource.return?.(void 0)).catch(() => void 0);
|
|
158
|
+
outbound.catch(() => void 0);
|
|
159
|
+
} else await outbound;
|
|
160
|
+
}
|
|
161
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
162
|
+
stream.removeEventListener("close", onClose);
|
|
112
163
|
}
|
|
164
|
+
if (!inputSent) throw new PeerGoneError(closeError);
|
|
113
165
|
}
|
|
114
166
|
/**
|
|
115
167
|
* Wait for `stream` to signal it can accept more data, per the real
|
|
@@ -193,6 +245,93 @@ async function* framedOutbound(iterator) {
|
|
|
193
245
|
Promise.resolve(iterator.return?.(void 0)).catch(() => void 0);
|
|
194
246
|
}
|
|
195
247
|
}
|
|
248
|
+
/**
|
|
249
|
+
* The stream's inbound bytes, read at the consumer's pace. libp2p's own
|
|
250
|
+
* iterator moves every message into an unbounded queue as it arrives, and
|
|
251
|
+
* yamux grants its peer more receive window on arrival, not on consumption: a
|
|
252
|
+
* consumer that stopped reading let the producer run 22-24 MiB ahead until
|
|
253
|
+
* libp2p's read buffer killed the stream (L6). Here the queue is bounded: past
|
|
254
|
+
* `highWater` bytes the stream is paused (yamux then withholds window updates,
|
|
255
|
+
* libp2p keeps what still arrives in its read buffer, undispatched), and it is
|
|
256
|
+
* resumed once the consumer has read the queue down to half.
|
|
257
|
+
*
|
|
258
|
+
* Ends as libp2p's iterator does: on the peer's `remoteCloseWrite`, or on
|
|
259
|
+
* `close` (throwing its error, if any). Subscribes on the first pull, like it.
|
|
260
|
+
*/
|
|
261
|
+
async function* pacedSource(stream, highWater) {
|
|
262
|
+
if (stream.readStatus !== "readable" && stream.readStatus !== "paused") return;
|
|
263
|
+
const window = stream.maxStreamWindowSize;
|
|
264
|
+
const paced = typeof window === "number";
|
|
265
|
+
if (paced && stream.maxReadBufferLength < window + highWater) stream.maxReadBufferLength = window + highWater;
|
|
266
|
+
const queue = [];
|
|
267
|
+
let queued = 0;
|
|
268
|
+
let ended = false;
|
|
269
|
+
let error;
|
|
270
|
+
let resuming = false;
|
|
271
|
+
let wake;
|
|
272
|
+
const notify = () => {
|
|
273
|
+
const w = wake;
|
|
274
|
+
wake = void 0;
|
|
275
|
+
w?.();
|
|
276
|
+
};
|
|
277
|
+
const pauseIfFull = () => {
|
|
278
|
+
if (paced && queued >= highWater && stream.readStatus === "readable") try {
|
|
279
|
+
stream.pause();
|
|
280
|
+
} catch {}
|
|
281
|
+
};
|
|
282
|
+
const onMessage = (evt) => {
|
|
283
|
+
const chunk = normalizeChunk(evt.data);
|
|
284
|
+
if (chunk.byteLength === 0) return;
|
|
285
|
+
for (let at = 0; at < chunk.byteLength; at += highWater) queue.push(chunk.subarray(at, Math.min(at + highWater, chunk.byteLength)));
|
|
286
|
+
queued += chunk.byteLength;
|
|
287
|
+
if (!resuming) pauseIfFull();
|
|
288
|
+
notify();
|
|
289
|
+
};
|
|
290
|
+
const resume = () => {
|
|
291
|
+
resuming = true;
|
|
292
|
+
try {
|
|
293
|
+
stream.resume();
|
|
294
|
+
} catch {} finally {
|
|
295
|
+
resuming = false;
|
|
296
|
+
}
|
|
297
|
+
pauseIfFull();
|
|
298
|
+
};
|
|
299
|
+
const onClose = (evt) => {
|
|
300
|
+
ended = true;
|
|
301
|
+
error ??= evt.error;
|
|
302
|
+
notify();
|
|
303
|
+
};
|
|
304
|
+
const onRemoteCloseWrite = () => {
|
|
305
|
+
ended = true;
|
|
306
|
+
notify();
|
|
307
|
+
};
|
|
308
|
+
stream.addEventListener("message", onMessage);
|
|
309
|
+
stream.addEventListener("close", onClose);
|
|
310
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
311
|
+
if (stream.readStatus === "paused") resume();
|
|
312
|
+
try {
|
|
313
|
+
for (;;) {
|
|
314
|
+
const chunk = queue.shift();
|
|
315
|
+
if (chunk) {
|
|
316
|
+
queued -= chunk.byteLength;
|
|
317
|
+
if (queued <= highWater / 2 && stream.readStatus === "paused") resume();
|
|
318
|
+
yield chunk;
|
|
319
|
+
continue;
|
|
320
|
+
}
|
|
321
|
+
if (ended) {
|
|
322
|
+
if (error) throw error;
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
await new Promise((resolve) => {
|
|
326
|
+
wake = resolve;
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
} finally {
|
|
330
|
+
stream.removeEventListener("message", onMessage);
|
|
331
|
+
stream.removeEventListener("close", onClose);
|
|
332
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
196
335
|
async function* parseFrames(source) {
|
|
197
336
|
let buf = /* @__PURE__ */ new Uint8Array(0);
|
|
198
337
|
for await (const item of source) {
|
|
@@ -283,26 +422,7 @@ function decodeVarint(buf, start) {
|
|
|
283
422
|
throw new Error("decodeVarint: truncated");
|
|
284
423
|
}
|
|
285
424
|
//#endregion
|
|
286
|
-
//#region src/
|
|
287
|
-
const DEFAULT_PROTOCOL = "/webrun-streams/1.0.0";
|
|
288
|
-
/**
|
|
289
|
-
* Builds the options object passed to `node.dialProtocol`/`node.handle`,
|
|
290
|
-
* omitting any field the caller didn't supply rather than setting it to
|
|
291
|
-
* `undefined` — so an unset field falls back to libp2p's own default instead
|
|
292
|
-
* of an explicit `undefined` overriding it.
|
|
293
|
-
*/
|
|
294
|
-
function definedOptions(fields) {
|
|
295
|
-
const options = {};
|
|
296
|
-
let any = false;
|
|
297
|
-
for (const key of Object.keys(fields)) {
|
|
298
|
-
const value = fields[key];
|
|
299
|
-
if (value !== void 0) {
|
|
300
|
-
options[key] = value;
|
|
301
|
-
any = true;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
return any ? options : void 0;
|
|
305
|
-
}
|
|
425
|
+
//#region src/stream-gate.ts
|
|
306
426
|
const streamGates = /* @__PURE__ */ new WeakMap();
|
|
307
427
|
function streamGateFor(node, peer) {
|
|
308
428
|
let gates = streamGates.get(node);
|
|
@@ -334,31 +454,63 @@ function connectionOf(node, stream) {
|
|
|
334
454
|
if (typeof node.getConnections !== "function") return null;
|
|
335
455
|
return node.getConnections().find((connection) => connection.streams.includes(stream)) ?? null;
|
|
336
456
|
}
|
|
337
|
-
|
|
457
|
+
/**
|
|
458
|
+
* Opens a stream to `peer` for `proto`, one at a time until the remote has proven ready: a burst
|
|
459
|
+
* of streams on a connection the remote is still upgrading makes yamux abort the whole muxer
|
|
460
|
+
* (`maxEarlyStreams`). Shared by `connect()` and `libp2pPortMux`, and by every caller dialling
|
|
461
|
+
* the same peer from the same node, since they share its connection.
|
|
462
|
+
*/
|
|
463
|
+
async function openGatedStream(node, peer, proto, dialOptions) {
|
|
464
|
+
const gate = streamGateFor(node, peer);
|
|
465
|
+
while (!isProven(gate)) {
|
|
466
|
+
if (gate.firstOpening == null) {
|
|
467
|
+
const opening = node.dialProtocol(peer, [proto], dialOptions);
|
|
468
|
+
gate.firstOpening = opening;
|
|
469
|
+
try {
|
|
470
|
+
const stream = await opening;
|
|
471
|
+
gate.proven = { connection: connectionOf(node, stream) };
|
|
472
|
+
return stream;
|
|
473
|
+
} finally {
|
|
474
|
+
gate.firstOpening = null;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
await gate.firstOpening.catch(() => void 0);
|
|
478
|
+
}
|
|
479
|
+
return node.dialProtocol(peer, [proto], dialOptions);
|
|
480
|
+
}
|
|
481
|
+
//#endregion
|
|
482
|
+
//#region src/connect-serve.ts
|
|
483
|
+
const DEFAULT_PROTOCOL = "/webrun-streams/1.0.0";
|
|
484
|
+
/**
|
|
485
|
+
* Builds the options object passed to `node.dialProtocol`/`node.handle`,
|
|
486
|
+
* omitting any field the caller didn't supply rather than setting it to
|
|
487
|
+
* `undefined` — so an unset field falls back to libp2p's own default instead
|
|
488
|
+
* of an explicit `undefined` overriding it.
|
|
489
|
+
*/
|
|
490
|
+
function definedOptions(fields) {
|
|
491
|
+
const options = {};
|
|
492
|
+
let any = false;
|
|
493
|
+
for (const key of Object.keys(fields)) {
|
|
494
|
+
const value = fields[key];
|
|
495
|
+
if (value !== void 0) {
|
|
496
|
+
options[key] = value;
|
|
497
|
+
any = true;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
return any ? options : void 0;
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Caller-side: each `call(input)` opens a new libp2p `Stream` via
|
|
504
|
+
* `node.dialProtocol(peer, [protocol])` and runs the call over it.
|
|
505
|
+
*/
|
|
506
|
+
const connect = async ({ node, peer, protocol, drainTimeoutMs, readHighWaterBytes, maxOutboundStreams, runOnLimitedConnection }) => {
|
|
338
507
|
const proto = protocol ?? "/webrun-streams/1.0.0";
|
|
339
508
|
const dialOptions = definedOptions({
|
|
340
509
|
maxOutboundStreams,
|
|
341
510
|
runOnLimitedConnection
|
|
342
511
|
});
|
|
343
512
|
const open = /* @__PURE__ */ new Set();
|
|
344
|
-
const
|
|
345
|
-
const openStream = async () => {
|
|
346
|
-
while (!isProven(gate)) {
|
|
347
|
-
if (gate.firstOpening == null) {
|
|
348
|
-
const opening = node.dialProtocol(peer, [proto], dialOptions);
|
|
349
|
-
gate.firstOpening = opening;
|
|
350
|
-
try {
|
|
351
|
-
const stream = await opening;
|
|
352
|
-
gate.proven = { connection: connectionOf(node, stream) };
|
|
353
|
-
return stream;
|
|
354
|
-
} finally {
|
|
355
|
-
gate.firstOpening = null;
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
await gate.firstOpening.catch(() => void 0);
|
|
359
|
-
}
|
|
360
|
-
return node.dialProtocol(peer, [proto], dialOptions);
|
|
361
|
-
};
|
|
513
|
+
const openStream = () => openGatedStream(node, peer, proto, dialOptions);
|
|
362
514
|
const call = (input) => {
|
|
363
515
|
let streamRef = null;
|
|
364
516
|
const gen = (async function* () {
|
|
@@ -369,6 +521,7 @@ const connect = async ({ node, peer, protocol, drainTimeoutMs, maxOutboundStream
|
|
|
369
521
|
try {
|
|
370
522
|
yield* duplexOverStream(stream, input, {
|
|
371
523
|
drainTimeoutMs,
|
|
524
|
+
readHighWaterBytes,
|
|
372
525
|
onSourceCompleted: () => {
|
|
373
526
|
sourceCompleted = true;
|
|
374
527
|
}
|
|
@@ -406,7 +559,7 @@ const serve = async (params, handler) => serveConnections(params, () => handler)
|
|
|
406
559
|
* Like `serve`, but the handler is built per inbound stream and is told which
|
|
407
560
|
* peer libp2p proved on that connection.
|
|
408
561
|
*/
|
|
409
|
-
async function serveConnections({ node, protocol, drainTimeoutMs, maxInboundStreams, maxOutboundStreams, runOnLimitedConnection }, makeHandler) {
|
|
562
|
+
async function serveConnections({ node, protocol, drainTimeoutMs, readHighWaterBytes, maxInboundStreams, maxOutboundStreams, runOnLimitedConnection }, makeHandler) {
|
|
410
563
|
const proto = protocol ?? "/webrun-streams/1.0.0";
|
|
411
564
|
const handleOptions = definedOptions({
|
|
412
565
|
maxInboundStreams,
|
|
@@ -414,20 +567,26 @@ async function serveConnections({ node, protocol, drainTimeoutMs, maxInboundStre
|
|
|
414
567
|
runOnLimitedConnection
|
|
415
568
|
});
|
|
416
569
|
const onStream = (stream, connection) => {
|
|
570
|
+
let requestComplete = false;
|
|
417
571
|
(async () => {
|
|
418
572
|
const handler = makeHandler({ remotePeer: connection.remotePeer });
|
|
419
|
-
const inputQueue = makeInputQueue();
|
|
420
|
-
const output = handler(inputQueue.iter());
|
|
573
|
+
const inputQueue = makeInputQueue(readHighWaterBytes ?? 65536);
|
|
574
|
+
const output = endsWith(handler(inputQueue.iter()), () => inputQueue.abandon());
|
|
421
575
|
try {
|
|
422
576
|
for await (const chunk of duplexOverStream(stream, output, {
|
|
423
577
|
drainTimeoutMs,
|
|
424
|
-
|
|
425
|
-
|
|
578
|
+
readHighWaterBytes,
|
|
579
|
+
onPeerInputEnd: (err) => {
|
|
580
|
+
if (err === void 0) requestComplete = true;
|
|
581
|
+
inputQueue.done(err);
|
|
582
|
+
}
|
|
583
|
+
})) await inputQueue.push(chunk);
|
|
426
584
|
} finally {
|
|
427
585
|
inputQueue.done();
|
|
428
586
|
await closeStream(stream);
|
|
429
587
|
}
|
|
430
588
|
})().catch((err) => {
|
|
589
|
+
if (requestComplete && err instanceof PeerGoneError) return;
|
|
431
590
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
432
591
|
console.warn(`[webrun-streams-libp2p] serve: inbound stream on protocol ${proto} failed: ${e.message}`, e);
|
|
433
592
|
});
|
|
@@ -440,13 +599,38 @@ async function serveConnections({ node, protocol, drainTimeoutMs, maxInboundStre
|
|
|
440
599
|
await node.unhandle(proto);
|
|
441
600
|
};
|
|
442
601
|
}
|
|
443
|
-
|
|
602
|
+
/** `source`, calling `onEnd` once when it ends, fails or is returned. */
|
|
603
|
+
function endsWith(source, onEnd) {
|
|
604
|
+
return (async function* () {
|
|
605
|
+
try {
|
|
606
|
+
yield* source;
|
|
607
|
+
} finally {
|
|
608
|
+
onEnd();
|
|
609
|
+
}
|
|
610
|
+
})();
|
|
611
|
+
}
|
|
612
|
+
function makeInputQueue(limit) {
|
|
444
613
|
const slots = [];
|
|
445
614
|
let wake = null;
|
|
446
615
|
let closed = false;
|
|
616
|
+
let pulled = false;
|
|
617
|
+
let queued = 0;
|
|
618
|
+
let room = null;
|
|
619
|
+
const makeRoom = () => {
|
|
620
|
+
const r = room;
|
|
621
|
+
room = null;
|
|
622
|
+
r?.();
|
|
623
|
+
};
|
|
624
|
+
const release = () => {
|
|
625
|
+
slots.length = 0;
|
|
626
|
+
queued = 0;
|
|
627
|
+
makeRoom();
|
|
628
|
+
};
|
|
447
629
|
return {
|
|
448
630
|
iter() {
|
|
449
631
|
return (async function* () {
|
|
632
|
+
if (closed) return;
|
|
633
|
+
pulled = true;
|
|
450
634
|
try {
|
|
451
635
|
while (true) {
|
|
452
636
|
if (slots.length === 0) {
|
|
@@ -461,20 +645,28 @@ function makeInputQueue() {
|
|
|
461
645
|
if (s.err) throw s.err;
|
|
462
646
|
return;
|
|
463
647
|
}
|
|
648
|
+
queued -= s.value.byteLength;
|
|
649
|
+
if (queued <= limit) makeRoom();
|
|
464
650
|
yield s.value;
|
|
465
651
|
}
|
|
466
652
|
} finally {
|
|
467
653
|
closed = true;
|
|
654
|
+
release();
|
|
468
655
|
}
|
|
469
656
|
})();
|
|
470
657
|
},
|
|
471
658
|
push(chunk) {
|
|
472
|
-
if (closed) return;
|
|
659
|
+
if (closed) return Promise.resolve();
|
|
473
660
|
slots.push({
|
|
474
661
|
type: "value",
|
|
475
662
|
value: chunk
|
|
476
663
|
});
|
|
664
|
+
queued += chunk.byteLength;
|
|
477
665
|
wake?.();
|
|
666
|
+
if (queued <= limit) return Promise.resolve();
|
|
667
|
+
return new Promise((resolve) => {
|
|
668
|
+
room = resolve;
|
|
669
|
+
});
|
|
478
670
|
},
|
|
479
671
|
done(err) {
|
|
480
672
|
if (closed) return;
|
|
@@ -483,8 +675,288 @@ function makeInputQueue() {
|
|
|
483
675
|
err
|
|
484
676
|
});
|
|
485
677
|
wake?.();
|
|
678
|
+
},
|
|
679
|
+
abandon() {
|
|
680
|
+
if (pulled) return;
|
|
681
|
+
closed = true;
|
|
682
|
+
release();
|
|
683
|
+
wake?.();
|
|
684
|
+
}
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
//#endregion
|
|
688
|
+
//#region src/message-target-over-stream.ts
|
|
689
|
+
/** 64 MiB: room for a large unframed message, short of exhausting memory. */
|
|
690
|
+
const DEFAULT_MAX_MESSAGE_BYTES = 67108864;
|
|
691
|
+
/**
|
|
692
|
+
* One libp2p stream as a pipe of messages, for a multiplexer to run over:
|
|
693
|
+
*
|
|
694
|
+
* ```ts
|
|
695
|
+
* const { target, signal } = messageTargetOverStream(stream);
|
|
696
|
+
* const mux = multiplexPort(target, { codec: msgpackCodec, signal });
|
|
697
|
+
* ```
|
|
698
|
+
*
|
|
699
|
+
* Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
700
|
+
* `message` listener in the same tick: a message with no listener is dropped,
|
|
701
|
+
* as on a `MessagePort`. Every message the peer sent before closing is
|
|
702
|
+
* delivered before `signal` aborts. `postMessage` hands the frame to
|
|
703
|
+
* `stream.send()` and does not wait for room; libp2p buffers up to the
|
|
704
|
+
* stream's `maxWriteBufferLength`, past which it resets the stream. A sender
|
|
705
|
+
* that must wait awaits `whenDrained()`.
|
|
706
|
+
*/
|
|
707
|
+
function messageTargetOverStream(stream, { maxMessageBytes = DEFAULT_MAX_MESSAGE_BYTES } = {}) {
|
|
708
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
709
|
+
const life = new AbortController();
|
|
710
|
+
let peerClosedWrite = stream.remoteWriteStatus === "closed";
|
|
711
|
+
const onRemoteCloseWrite = () => {
|
|
712
|
+
peerClosedWrite = true;
|
|
713
|
+
};
|
|
714
|
+
const onClose = (evt) => {
|
|
715
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
716
|
+
if (peerClosedWrite && evt.error == null) return;
|
|
717
|
+
life.abort(new PeerGoneError(evt.error));
|
|
718
|
+
};
|
|
719
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
720
|
+
stream.addEventListener("close", onClose, { once: true });
|
|
721
|
+
if (stream.status !== void 0 && stream.status !== "open") life.abort(new PeerGoneError());
|
|
722
|
+
const deliver = (data) => {
|
|
723
|
+
const event = new MessageEvent("message", { data });
|
|
724
|
+
for (const listener of [...listeners]) try {
|
|
725
|
+
Promise.resolve(listener(event)).catch(() => void 0);
|
|
726
|
+
} catch {}
|
|
727
|
+
};
|
|
728
|
+
(async () => {
|
|
729
|
+
let buf = /* @__PURE__ */ new Uint8Array(0);
|
|
730
|
+
let start = 0;
|
|
731
|
+
let end = 0;
|
|
732
|
+
for await (const item of stream) {
|
|
733
|
+
const incoming = normalizeChunk(item);
|
|
734
|
+
if (end + incoming.byteLength > buf.byteLength) {
|
|
735
|
+
const size = end - start + incoming.byteLength;
|
|
736
|
+
const next = new Uint8Array(Math.max(size, buf.byteLength * 2));
|
|
737
|
+
next.set(buf.subarray(start, end), 0);
|
|
738
|
+
buf = next;
|
|
739
|
+
end -= start;
|
|
740
|
+
start = 0;
|
|
741
|
+
}
|
|
742
|
+
buf.set(incoming, end);
|
|
743
|
+
end += incoming.byteLength;
|
|
744
|
+
for (;;) {
|
|
745
|
+
const header = readLength(buf.subarray(start, end));
|
|
746
|
+
if (header === void 0) break;
|
|
747
|
+
if (header.value > maxMessageBytes) throw new Error(`webrun-streams-libp2p: a ${header.value}-byte message exceeds maxMessageBytes (${maxMessageBytes})`);
|
|
748
|
+
const stop = start + header.offset + header.value;
|
|
749
|
+
if (end < stop) break;
|
|
750
|
+
deliver(buf.slice(start + header.offset, stop));
|
|
751
|
+
start = stop;
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
if (peerClosedWrite) {
|
|
755
|
+
life.abort(/* @__PURE__ */ new Error("webrun-streams-libp2p: the peer closed the stream"));
|
|
756
|
+
await closeStream(stream);
|
|
757
|
+
}
|
|
758
|
+
})().catch((err) => {
|
|
759
|
+
life.abort(err);
|
|
760
|
+
try {
|
|
761
|
+
stream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
762
|
+
} catch {}
|
|
763
|
+
});
|
|
764
|
+
return {
|
|
765
|
+
signal: life.signal,
|
|
766
|
+
target: {
|
|
767
|
+
closeSignal: life.signal,
|
|
768
|
+
addEventListener: (_type, listener) => {
|
|
769
|
+
listeners.add(listener);
|
|
770
|
+
},
|
|
771
|
+
removeEventListener: (_type, listener) => {
|
|
772
|
+
listeners.delete(listener);
|
|
773
|
+
},
|
|
774
|
+
postMessage(message) {
|
|
775
|
+
const payload = normalizeChunk(message);
|
|
776
|
+
const header = encodeVarint(payload.byteLength);
|
|
777
|
+
const frame = new Uint8Array(header.byteLength + payload.byteLength);
|
|
778
|
+
frame.set(header, 0);
|
|
779
|
+
frame.set(payload, header.byteLength);
|
|
780
|
+
try {
|
|
781
|
+
stream.send(frame);
|
|
782
|
+
} catch {}
|
|
783
|
+
},
|
|
784
|
+
async close() {
|
|
785
|
+
listeners.clear();
|
|
786
|
+
await closeStream(stream);
|
|
787
|
+
},
|
|
788
|
+
get bufferedAmount() {
|
|
789
|
+
return stream.writeBufferLength;
|
|
790
|
+
},
|
|
791
|
+
whenDrained() {
|
|
792
|
+
if (!stream.writableNeedsDrain || life.signal.aborted) return Promise.resolve();
|
|
793
|
+
return new Promise((resolve) => {
|
|
794
|
+
const done = () => {
|
|
795
|
+
stream.removeEventListener("drain", done);
|
|
796
|
+
stream.removeEventListener("close", done);
|
|
797
|
+
resolve();
|
|
798
|
+
};
|
|
799
|
+
stream.addEventListener("drain", done);
|
|
800
|
+
stream.addEventListener("close", done);
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
};
|
|
805
|
+
}
|
|
806
|
+
/** The length prefix of `bytes`, or `undefined` while it is incomplete; throws when malformed. */
|
|
807
|
+
function readLength(bytes) {
|
|
808
|
+
try {
|
|
809
|
+
return decodeVarint(bytes, 0);
|
|
810
|
+
} catch (err) {
|
|
811
|
+
if (bytes.byteLength < 5) return void 0;
|
|
812
|
+
throw new Error("webrun-streams-libp2p: a malformed message length", { cause: err });
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
//#endregion
|
|
816
|
+
//#region src/port-mux.ts
|
|
817
|
+
/** The protocol a port's stream negotiates unless told otherwise. */
|
|
818
|
+
const DEFAULT_PORT_PROTOCOL = "/webrun-ports/1.0.0";
|
|
819
|
+
/**
|
|
820
|
+
* 256 KiB: what one message on a port may carry before `webrun-rpc`'s stream tier chunks a body.
|
|
821
|
+
* A port waits for each chunk's confirmation, so a larger chunk means fewer round trips; well
|
|
822
|
+
* under `messageTargetOverStream`'s 64 MiB frame ceiling.
|
|
823
|
+
*/
|
|
824
|
+
const DEFAULT_PORT_MAX_MESSAGE_SIZE = 262144;
|
|
825
|
+
/**
|
|
826
|
+
* The stream limits a port mux asks libp2p for unless told otherwise: every concurrent call is a
|
|
827
|
+
* stream, and libp2p's own default (32 per protocol) would reset the 33rd.
|
|
828
|
+
*/
|
|
829
|
+
const DEFAULT_PORT_STREAM_LIMIT = 1024;
|
|
830
|
+
/**
|
|
831
|
+
* Native MessagePorts over libp2p: every port is its own libp2p stream, which yamux multiplexes
|
|
832
|
+
* with its own per-stream flow control, framed as messages by `messageTargetOverStream`. Hand it
|
|
833
|
+
* to `webrun-rpc` as a port source and its stream tier runs one call per port, with its tested
|
|
834
|
+
* backpressure — a producer stays one confirmed chunk ahead of its consumer — instead of
|
|
835
|
+
* libp2p's own buffering, which lets a producer run tens of megabytes ahead of a consumer that
|
|
836
|
+
* stopped reading:
|
|
837
|
+
*
|
|
838
|
+
* ```ts
|
|
839
|
+
* const codec = { encode: serialize, decode: deserialize }; // webrun-msgpack
|
|
840
|
+
* await serve({ mux: overPorts((onPort) => libp2pPortMux({ node, codec, onPort })) }, handler);
|
|
841
|
+
* const { call } = await connect({ mux: overPorts(() => libp2pPortMux({ node, codec, peer })) });
|
|
842
|
+
* ```
|
|
843
|
+
*
|
|
844
|
+
* Each port carries `closeSignal`, aborted when its stream ends or its peer goes, so a call on a
|
|
845
|
+
* dead stream fails instead of waiting. Streams are opened through the same early-stream gate as
|
|
846
|
+
* `connect()`. `meta` is not sent: the protocol says what a port is for.
|
|
847
|
+
*/
|
|
848
|
+
async function libp2pPortMux(options) {
|
|
849
|
+
const { node, peer, codec, onPort, maxMessageBytes, runOnLimitedConnection } = options;
|
|
850
|
+
const maxInboundStreams = options.maxInboundStreams ?? 1024;
|
|
851
|
+
const maxOutboundStreams = options.maxOutboundStreams ?? 1024;
|
|
852
|
+
const protocol = options.protocol ?? "/webrun-ports/1.0.0";
|
|
853
|
+
const maxMessageSize = options.maxMessageSize ?? 262144;
|
|
854
|
+
const live = /* @__PURE__ */ new Set();
|
|
855
|
+
let closed = false;
|
|
856
|
+
const wrap = (stream) => {
|
|
857
|
+
const { target: inner } = messageTargetOverStream(stream, { maxMessageBytes });
|
|
858
|
+
const wrapped = /* @__PURE__ */ new Map();
|
|
859
|
+
const local = new AbortController();
|
|
860
|
+
const port = {
|
|
861
|
+
closeSignal: AbortSignal.any([inner.closeSignal, local.signal]),
|
|
862
|
+
get bufferedAmount() {
|
|
863
|
+
return inner.bufferedAmount;
|
|
864
|
+
},
|
|
865
|
+
whenDrained: () => inner.whenDrained(),
|
|
866
|
+
postMessage(message) {
|
|
867
|
+
inner.postMessage(codec.encode(message));
|
|
868
|
+
},
|
|
869
|
+
addEventListener(_type, listener) {
|
|
870
|
+
if (wrapped.has(listener)) return;
|
|
871
|
+
const decoding = (event) => {
|
|
872
|
+
let data;
|
|
873
|
+
try {
|
|
874
|
+
data = codec.decode(event.data);
|
|
875
|
+
} catch (err) {
|
|
876
|
+
stream.abort(new Error("webrun-streams-libp2p: a frame the port's codec cannot decode", { cause: err }));
|
|
877
|
+
return;
|
|
878
|
+
}
|
|
879
|
+
return listener(new MessageEvent("message", { data }));
|
|
880
|
+
};
|
|
881
|
+
wrapped.set(listener, decoding);
|
|
882
|
+
inner.addEventListener("message", decoding);
|
|
883
|
+
},
|
|
884
|
+
removeEventListener(_type, listener) {
|
|
885
|
+
const decoding = wrapped.get(listener);
|
|
886
|
+
if (!decoding) return;
|
|
887
|
+
wrapped.delete(listener);
|
|
888
|
+
inner.removeEventListener("message", decoding);
|
|
889
|
+
},
|
|
890
|
+
async close() {
|
|
891
|
+
live.delete(port);
|
|
892
|
+
if (!local.signal.aborted) local.abort(/* @__PURE__ */ new Error("webrun-streams-libp2p: port closed"));
|
|
893
|
+
await inner.close();
|
|
894
|
+
}
|
|
895
|
+
};
|
|
896
|
+
live.add(port);
|
|
897
|
+
inner.closeSignal.addEventListener("abort", () => live.delete(port), { once: true });
|
|
898
|
+
return port;
|
|
899
|
+
};
|
|
900
|
+
let handled = false;
|
|
901
|
+
if (onPort) {
|
|
902
|
+
const handleOptions = {
|
|
903
|
+
maxInboundStreams,
|
|
904
|
+
maxOutboundStreams
|
|
905
|
+
};
|
|
906
|
+
if (runOnLimitedConnection !== void 0) handleOptions.runOnLimitedConnection = runOnLimitedConnection;
|
|
907
|
+
await node.handle(protocol, (stream, connection) => {
|
|
908
|
+
if (closed) {
|
|
909
|
+
stream.abort(/* @__PURE__ */ new Error("webrun-streams-libp2p: port mux closed"));
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
912
|
+
const port = wrap(stream);
|
|
913
|
+
let accepted = false;
|
|
914
|
+
try {
|
|
915
|
+
accepted = onPort(port, {
|
|
916
|
+
remotePeer: connection.remotePeer,
|
|
917
|
+
connection
|
|
918
|
+
}) !== false;
|
|
919
|
+
} catch {
|
|
920
|
+
accepted = false;
|
|
921
|
+
}
|
|
922
|
+
if (!accepted) {
|
|
923
|
+
live.delete(port);
|
|
924
|
+
stream.abort(/* @__PURE__ */ new Error("webrun-streams-libp2p: port refused"));
|
|
925
|
+
}
|
|
926
|
+
}, handleOptions);
|
|
927
|
+
handled = true;
|
|
928
|
+
}
|
|
929
|
+
const dialOptions = { maxOutboundStreams };
|
|
930
|
+
if (runOnLimitedConnection !== void 0) dialOptions.runOnLimitedConnection = runOnLimitedConnection;
|
|
931
|
+
return {
|
|
932
|
+
maxMessageSize,
|
|
933
|
+
async openPort(_meta, opts) {
|
|
934
|
+
if (closed) throw new Error("webrun-streams-libp2p: port mux closed");
|
|
935
|
+
if (!peer) throw new Error("webrun-streams-libp2p: openPort needs `peer`");
|
|
936
|
+
const stream = await openGatedStream(node, peer, protocol, {
|
|
937
|
+
...dialOptions,
|
|
938
|
+
signal: opts?.signal
|
|
939
|
+
});
|
|
940
|
+
if (closed) {
|
|
941
|
+
await closeStream(stream);
|
|
942
|
+
throw new Error("webrun-streams-libp2p: port mux closed");
|
|
943
|
+
}
|
|
944
|
+
const port = wrap(stream);
|
|
945
|
+
const signal = opts?.signal;
|
|
946
|
+
if (signal) {
|
|
947
|
+
const onAbort = () => void port.close();
|
|
948
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
949
|
+
port.closeSignal.addEventListener("abort", () => signal.removeEventListener("abort", onAbort), { once: true });
|
|
950
|
+
}
|
|
951
|
+
return port;
|
|
952
|
+
},
|
|
953
|
+
async close() {
|
|
954
|
+
if (closed) return;
|
|
955
|
+
closed = true;
|
|
956
|
+
if (handled) await node.unhandle(protocol).catch(() => void 0);
|
|
957
|
+
await Promise.allSettled([...live].map((p) => p.close()));
|
|
486
958
|
}
|
|
487
959
|
};
|
|
488
960
|
}
|
|
489
961
|
//#endregion
|
|
490
|
-
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_PROTOCOL, connect, duplexOverStream, serve, serveConnections };
|
|
962
|
+
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_MAX_MESSAGE_BYTES, DEFAULT_PORT_MAX_MESSAGE_SIZE, DEFAULT_PORT_PROTOCOL, DEFAULT_PORT_STREAM_LIMIT, DEFAULT_PROTOCOL, PeerGoneError, connect, duplexOverStream, libp2pPortMux, messageTargetOverStream, serve, serveConnections };
|