@statewalker/webrun-streams-libp2p 0.1.8 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -5
- package/dist/connect-serve.d.ts +15 -1
- package/dist/connect-serve.d.ts.map +1 -1
- package/dist/duplex-over-stream.d.ts +32 -0
- package/dist/duplex-over-stream.d.ts.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +525 -53
- package/dist/message-target-over-stream.d.ts +64 -0
- package/dist/message-target-over-stream.d.ts.map +1 -0
- package/dist/port-mux.d.ts +89 -0
- package/dist/port-mux.d.ts.map +1 -0
- package/dist/stream-gate.d.ts +10 -0
- package/dist/stream-gate.d.ts.map +1 -0
- package/package.json +7 -4
- package/src/connect-serve.ts +98 -72
- package/src/duplex-over-stream.ts +219 -11
- package/src/index.ts +18 -0
- package/src/message-target-over-stream.ts +207 -0
- package/src/port-mux.ts +238 -0
- package/src/stream-gate.ts +77 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { Stream } from "@libp2p/interface";
|
|
2
|
+
type Listener = (event: MessageEvent) => void | Promise<void>;
|
|
3
|
+
/** What `messageTargetOverStream` gives: a byte pipe, and its life. */
|
|
4
|
+
export interface StreamMessageTarget {
|
|
5
|
+
/**
|
|
6
|
+
* A `MessageTarget` (as `@statewalker/webrun-rpc` defines it) whose messages
|
|
7
|
+
* are `Uint8Array`s, one length-prefixed frame each.
|
|
8
|
+
*/
|
|
9
|
+
target: {
|
|
10
|
+
addEventListener(type: "message", listener: Listener): void;
|
|
11
|
+
removeEventListener(type: "message", listener: Listener): void;
|
|
12
|
+
postMessage(message: unknown): void;
|
|
13
|
+
close(): Promise<void>;
|
|
14
|
+
/** The stream's write buffer, in bytes. */
|
|
15
|
+
readonly bufferedAmount: number;
|
|
16
|
+
/**
|
|
17
|
+
* Resolves once the stream can take more (libp2p's `drain`), or has
|
|
18
|
+
* closed; at once when it can now.
|
|
19
|
+
*/
|
|
20
|
+
whenDrained(): Promise<void>;
|
|
21
|
+
/** `signal` again, on the target: `webrun-rpc`'s `getPortCloseSignal` reads it from a port. */
|
|
22
|
+
readonly closeSignal: AbortSignal;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Aborts when the stream is over, after every message the peer sent has
|
|
26
|
+
* been delivered. The reason is a {@link PeerGoneError} when the stream
|
|
27
|
+
* ended without the peer finishing it, or an `Error` when the peer closed
|
|
28
|
+
* it or sent bytes that are not this framing.
|
|
29
|
+
* Pass it as `multiplexPort`'s / `overPipe`'s `signal`, so a dead peer
|
|
30
|
+
* closes the multiplexer and every port on it at once.
|
|
31
|
+
*/
|
|
32
|
+
signal: AbortSignal;
|
|
33
|
+
}
|
|
34
|
+
/** Options for {@link messageTargetOverStream}. */
|
|
35
|
+
export interface MessageTargetOverStreamOptions {
|
|
36
|
+
/**
|
|
37
|
+
* The largest message accepted, in bytes. A length prefix past it — or one
|
|
38
|
+
* that is not a valid varint — means the peer is not speaking this framing,
|
|
39
|
+
* so the stream is aborted rather than buffered without bound. Default
|
|
40
|
+
* {@link DEFAULT_MAX_MESSAGE_BYTES}.
|
|
41
|
+
*/
|
|
42
|
+
maxMessageBytes?: number;
|
|
43
|
+
}
|
|
44
|
+
/** 64 MiB: room for a large unframed message, short of exhausting memory. */
|
|
45
|
+
export declare const DEFAULT_MAX_MESSAGE_BYTES: number;
|
|
46
|
+
/**
|
|
47
|
+
* One libp2p stream as a pipe of messages, for a multiplexer to run over:
|
|
48
|
+
*
|
|
49
|
+
* ```ts
|
|
50
|
+
* const { target, signal } = messageTargetOverStream(stream);
|
|
51
|
+
* const mux = multiplexPort(target, { codec: msgpackCodec, signal });
|
|
52
|
+
* ```
|
|
53
|
+
*
|
|
54
|
+
* Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
55
|
+
* `message` listener in the same tick: a message with no listener is dropped,
|
|
56
|
+
* as on a `MessagePort`. Every message the peer sent before closing is
|
|
57
|
+
* delivered before `signal` aborts. `postMessage` hands the frame to
|
|
58
|
+
* `stream.send()` and does not wait for room; libp2p buffers up to the
|
|
59
|
+
* stream's `maxWriteBufferLength`, past which it resets the stream. A sender
|
|
60
|
+
* that must wait awaits `whenDrained()`.
|
|
61
|
+
*/
|
|
62
|
+
export declare function messageTargetOverStream(stream: Stream, { maxMessageBytes }?: MessageTargetOverStreamOptions): StreamMessageTarget;
|
|
63
|
+
export {};
|
|
64
|
+
//# sourceMappingURL=message-target-over-stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"message-target-over-stream.d.ts","sourceRoot":"","sources":["../src/message-target-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AASlE,KAAK,QAAQ,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9D,uEAAuE;AACvE,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE;QACN,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC5D,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC/D,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;QACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,2CAA2C;QAC3C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC;;;WAGG;QACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,+FAA+F;QAC/F,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;KACnC,CAAC;IACF;;;;;;;OAOG;IACH,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,8BAA8B;IAC7C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,6EAA6E;AAC7E,eAAO,MAAM,yBAAyB,QAAmB,CAAC;AAE1D;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,EAAE,eAA2C,EAAE,GAAE,8BAAmC,GACnF,mBAAmB,CAwHrB"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import type { Connection, Libp2p, PeerId } from "@libp2p/interface";
|
|
2
|
+
import type { Multiaddr } from "@multiformats/multiaddr";
|
|
3
|
+
import { type StreamMessageTarget } from "./message-target-over-stream.js";
|
|
4
|
+
/** The protocol a port's stream negotiates unless told otherwise. */
|
|
5
|
+
export declare const DEFAULT_PORT_PROTOCOL = "/webrun-ports/1.0.0";
|
|
6
|
+
/**
|
|
7
|
+
* 256 KiB: what one message on a port may carry before `webrun-rpc`'s stream tier chunks a body.
|
|
8
|
+
* A port waits for each chunk's confirmation, so a larger chunk means fewer round trips; well
|
|
9
|
+
* under `messageTargetOverStream`'s 64 MiB frame ceiling.
|
|
10
|
+
*/
|
|
11
|
+
export declare const DEFAULT_PORT_MAX_MESSAGE_SIZE: number;
|
|
12
|
+
/**
|
|
13
|
+
* How a port's messages become bytes on its stream and back: any value the stream tier posts
|
|
14
|
+
* (objects holding `Uint8Array` chunks, serialised errors). `webrun-msgpack`'s `serialize` and
|
|
15
|
+
* `deserialize` fit: `{ encode: serialize, decode: deserialize }`.
|
|
16
|
+
*/
|
|
17
|
+
export interface ValueCodec {
|
|
18
|
+
encode(value: unknown): Uint8Array;
|
|
19
|
+
decode(bytes: Uint8Array): unknown;
|
|
20
|
+
}
|
|
21
|
+
type Listener = (event: MessageEvent) => void | Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* The stream limits a port mux asks libp2p for unless told otherwise: every concurrent call is a
|
|
24
|
+
* stream, and libp2p's own default (32 per protocol) would reset the 33rd.
|
|
25
|
+
*/
|
|
26
|
+
export declare const DEFAULT_PORT_STREAM_LIMIT = 1024;
|
|
27
|
+
/** One port: a libp2p stream as a `MessageTarget` of values, with its close signal on it. */
|
|
28
|
+
export type StreamPort = Omit<StreamMessageTarget["target"], "addEventListener" | "removeEventListener"> & {
|
|
29
|
+
addEventListener(type: "message", listener: Listener): void;
|
|
30
|
+
removeEventListener(type: "message", listener: Listener): void;
|
|
31
|
+
};
|
|
32
|
+
/** What the serving side learns about a port the peer opened. */
|
|
33
|
+
export interface InboundPortInfo {
|
|
34
|
+
/** The peer libp2p's Noise handshake proved for the connection the stream came on. */
|
|
35
|
+
remotePeer: PeerId;
|
|
36
|
+
connection: Connection;
|
|
37
|
+
}
|
|
38
|
+
export interface Libp2pPortMuxOptions {
|
|
39
|
+
node: Libp2p;
|
|
40
|
+
/** Encodes each message as one frame; see {@link ValueCodec}. Both ends must agree. */
|
|
41
|
+
codec: ValueCodec;
|
|
42
|
+
/** The protocol each port's stream negotiates. Default {@link DEFAULT_PORT_PROTOCOL}. */
|
|
43
|
+
protocol?: string;
|
|
44
|
+
/** Whom `openPort` dials. Required to open ports; a serving-only mux leaves it out. */
|
|
45
|
+
peer?: PeerId | Multiaddr;
|
|
46
|
+
/**
|
|
47
|
+
* Called for each port the peer opens (each inbound stream). Return `false` to refuse it: the
|
|
48
|
+
* stream is aborted. Leave it out to accept none (the protocol is not handled).
|
|
49
|
+
*/
|
|
50
|
+
onPort?: (port: StreamPort, info: InboundPortInfo) => boolean | undefined;
|
|
51
|
+
/** Reported as `maxMessageSize`. Default {@link DEFAULT_PORT_MAX_MESSAGE_SIZE}. */
|
|
52
|
+
maxMessageSize?: number;
|
|
53
|
+
/** The largest frame accepted on a port's stream (see `messageTargetOverStream`). */
|
|
54
|
+
maxMessageBytes?: number;
|
|
55
|
+
/** Default {@link DEFAULT_PORT_STREAM_LIMIT}. */
|
|
56
|
+
maxInboundStreams?: number;
|
|
57
|
+
/** Default {@link DEFAULT_PORT_STREAM_LIMIT}. */
|
|
58
|
+
maxOutboundStreams?: number;
|
|
59
|
+
runOnLimitedConnection?: boolean;
|
|
60
|
+
}
|
|
61
|
+
/** A `webrun-rpc` `PortMux` by shape; this package imports nothing from it. */
|
|
62
|
+
export interface Libp2pPortMux {
|
|
63
|
+
openPort(meta?: unknown, options?: {
|
|
64
|
+
signal?: AbortSignal;
|
|
65
|
+
}): Promise<StreamPort>;
|
|
66
|
+
close(reason?: unknown): Promise<void>;
|
|
67
|
+
readonly maxMessageSize: number;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Native MessagePorts over libp2p: every port is its own libp2p stream, which yamux multiplexes
|
|
71
|
+
* with its own per-stream flow control, framed as messages by `messageTargetOverStream`. Hand it
|
|
72
|
+
* to `webrun-rpc` as a port source and its stream tier runs one call per port, with its tested
|
|
73
|
+
* backpressure — a producer stays one confirmed chunk ahead of its consumer — instead of
|
|
74
|
+
* libp2p's own buffering, which lets a producer run tens of megabytes ahead of a consumer that
|
|
75
|
+
* stopped reading:
|
|
76
|
+
*
|
|
77
|
+
* ```ts
|
|
78
|
+
* const codec = { encode: serialize, decode: deserialize }; // webrun-msgpack
|
|
79
|
+
* await serve({ mux: overPorts((onPort) => libp2pPortMux({ node, codec, onPort })) }, handler);
|
|
80
|
+
* const { call } = await connect({ mux: overPorts(() => libp2pPortMux({ node, codec, peer })) });
|
|
81
|
+
* ```
|
|
82
|
+
*
|
|
83
|
+
* Each port carries `closeSignal`, aborted when its stream ends or its peer goes, so a call on a
|
|
84
|
+
* dead stream fails instead of waiting. Streams are opened through the same early-stream gate as
|
|
85
|
+
* `connect()`. `meta` is not sent: the protocol says what a port is for.
|
|
86
|
+
*/
|
|
87
|
+
export declare function libp2pPortMux(options: Libp2pPortMuxOptions): Promise<Libp2pPortMux>;
|
|
88
|
+
export {};
|
|
89
|
+
//# sourceMappingURL=port-mux.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"port-mux.d.ts","sourceRoot":"","sources":["../src/port-mux.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAEV,MAAM,EACN,MAAM,EAGP,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAA2B,KAAK,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAGpG,qEAAqE;AACrE,eAAO,MAAM,qBAAqB,wBAAwB,CAAC;AAE3D;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,QAAa,CAAC;AAExD;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAAC;IACnC,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC;CACpC;AAED,KAAK,QAAQ,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9D;;;GAGG;AACH,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C,6FAA6F;AAC7F,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,mBAAmB,CAAC,QAAQ,CAAC,EAC7B,kBAAkB,GAAG,qBAAqB,CAC3C,GAAG;IACF,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC5D,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;CAChE,CAAC;AAEF,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC9B,sFAAsF;IACtF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,uFAAuF;IACvF,KAAK,EAAE,UAAU,CAAC;IAClB,yFAAyF;IACzF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uFAAuF;IACvF,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B;;;OAGG;IACH,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,KAAK,OAAO,GAAG,SAAS,CAAC;IAC1E,mFAAmF;IACnF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qFAAqF;IACrF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iDAAiD;IACjD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iDAAiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,+EAA+E;AAC/E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAClF,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,aAAa,CAAC,CAmIzF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { DialProtocolOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
|
|
2
|
+
import type { Multiaddr } from "@multiformats/multiaddr";
|
|
3
|
+
/**
|
|
4
|
+
* Opens a stream to `peer` for `proto`, one at a time until the remote has proven ready: a burst
|
|
5
|
+
* of streams on a connection the remote is still upgrading makes yamux abort the whole muxer
|
|
6
|
+
* (`maxEarlyStreams`). Shared by `connect()` and `libp2pPortMux`, and by every caller dialling
|
|
7
|
+
* the same peer from the same node, since they share its connection.
|
|
8
|
+
*/
|
|
9
|
+
export declare function openGatedStream(node: Libp2p, peer: PeerId | Multiaddr, proto: string, dialOptions?: DialProtocolOptions): Promise<Stream>;
|
|
10
|
+
//# sourceMappingURL=stream-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stream-gate.d.ts","sourceRoot":"","sources":["../src/stream-gate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AACjG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AA+CzD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,KAAK,EAAE,MAAM,EACb,WAAW,CAAC,EAAE,mBAAmB,GAChC,OAAO,CAAC,MAAM,CAAC,CAiBjB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/webrun-streams-libp2p",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "libp2p native multi-stream Connect/Serve adapter in the webrun-streams-* family",
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"src"
|
|
31
31
|
],
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@statewalker/webrun-streams": "^0.
|
|
33
|
+
"@statewalker/webrun-streams": "^0.3.0"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
36
|
"@libp2p/interface": "^3.0.0",
|
|
@@ -54,9 +54,12 @@
|
|
|
54
54
|
"libp2p": "^3.0.0",
|
|
55
55
|
"rimraf": "^6.1.3",
|
|
56
56
|
"rolldown": "^1.2.12",
|
|
57
|
+
"tsx": "^4.23.15",
|
|
57
58
|
"typescript": "^7.0.2",
|
|
58
|
-
"vitest": "^5.0.
|
|
59
|
-
"@statewalker/webrun-
|
|
59
|
+
"vitest": "^5.0.5",
|
|
60
|
+
"@statewalker/webrun-rpc": "^0.11.0",
|
|
61
|
+
"@statewalker/webrun-msgpack": "^0.10.0",
|
|
62
|
+
"@statewalker/webrun-streams-conformance": "^0.4.0"
|
|
60
63
|
},
|
|
61
64
|
"sideEffects": false,
|
|
62
65
|
"publishConfig": {
|
package/src/connect-serve.ts
CHANGED
|
@@ -8,7 +8,13 @@ import type {
|
|
|
8
8
|
} from "@libp2p/interface";
|
|
9
9
|
import type { Multiaddr } from "@multiformats/multiaddr";
|
|
10
10
|
import type { Connect, Duplex, Serve } from "@statewalker/webrun-streams";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
closeStream,
|
|
13
|
+
DEFAULT_READ_HIGH_WATER_BYTES,
|
|
14
|
+
duplexOverStream,
|
|
15
|
+
PeerGoneError,
|
|
16
|
+
} from "./duplex-over-stream.js";
|
|
17
|
+
import { openGatedStream } from "./stream-gate.js";
|
|
12
18
|
|
|
13
19
|
export const DEFAULT_PROTOCOL = "/webrun-streams/1.0.0";
|
|
14
20
|
|
|
@@ -22,6 +28,11 @@ export interface ConnectLibp2pParams {
|
|
|
22
28
|
* peer; defaults to `DEFAULT_DRAIN_TIMEOUT_MS` (5 minutes).
|
|
23
29
|
*/
|
|
24
30
|
drainTimeoutMs?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Response bytes held while the caller is not reading before the stream is
|
|
33
|
+
* paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
|
|
34
|
+
*/
|
|
35
|
+
readHighWaterBytes?: number;
|
|
25
36
|
/**
|
|
26
37
|
* How many outgoing streams for this protocol libp2p allows to be open at
|
|
27
38
|
* the same time on one connection; passed through to `node.dialProtocol`.
|
|
@@ -49,6 +60,11 @@ export interface ServeLibp2pParams {
|
|
|
49
60
|
* closing, since the caller-side `.return`/abort escape does not exist here.
|
|
50
61
|
*/
|
|
51
62
|
drainTimeoutMs?: number;
|
|
63
|
+
/**
|
|
64
|
+
* Request bytes held while the handler is not reading before the stream is
|
|
65
|
+
* paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
|
|
66
|
+
*/
|
|
67
|
+
readHighWaterBytes?: number;
|
|
52
68
|
/**
|
|
53
69
|
* How many incoming streams for this protocol libp2p allows to be open at
|
|
54
70
|
* the same time on one connection; passed through to `node.handle`. Left
|
|
@@ -97,56 +113,12 @@ function definedOptions<T extends Record<string, unknown>>(fields: T): Partial<T
|
|
|
97
113
|
* Caller-side: each `call(input)` opens a new libp2p `Stream` via
|
|
98
114
|
* `node.dialProtocol(peer, [protocol])` and runs the call over it.
|
|
99
115
|
*/
|
|
100
|
-
/** The early-stream guard of `connect()`, per local node and remote peer. */
|
|
101
|
-
interface StreamGate {
|
|
102
|
-
/**
|
|
103
|
-
* A stream has negotiated a protocol on an open connection: the remote is ready.
|
|
104
|
-
* `connection` is that connection when the node can tell (`getConnections`), so a
|
|
105
|
-
* burst after it closes is gated again; a node that offers only `dialProtocol`
|
|
106
|
-
* stays proven.
|
|
107
|
-
*/
|
|
108
|
-
proven: { connection: Connection | null } | null;
|
|
109
|
-
/** The stream being opened while no connection is proven; the others wait for it. */
|
|
110
|
-
firstOpening: Promise<Stream> | null;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
const streamGates = new WeakMap<Libp2p, Map<string, StreamGate>>();
|
|
114
|
-
|
|
115
|
-
function streamGateFor(node: Libp2p, peer: PeerId | Multiaddr): StreamGate {
|
|
116
|
-
let gates = streamGates.get(node);
|
|
117
|
-
if (gates == null) {
|
|
118
|
-
gates = new Map();
|
|
119
|
-
streamGates.set(node, gates);
|
|
120
|
-
}
|
|
121
|
-
const key = peer.toString();
|
|
122
|
-
let gate = gates.get(key);
|
|
123
|
-
if (gate == null) {
|
|
124
|
-
gate = { proven: null, firstOpening: null };
|
|
125
|
-
gates.set(key, gate);
|
|
126
|
-
}
|
|
127
|
-
return gate;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
function isProven(gate: StreamGate): boolean {
|
|
131
|
-
if (gate.proven == null) return false;
|
|
132
|
-
return gate.proven.connection == null || gate.proven.connection.status === "open";
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* The connection carrying `stream`, when `node` can list its connections.
|
|
137
|
-
* Callers may pass a narrow stand-in that offers only `dialProtocol` (httpeers
|
|
138
|
-
* routes streams onto a kept relay circuit that way), so nothing else is required.
|
|
139
|
-
*/
|
|
140
|
-
function connectionOf(node: Libp2p, stream: Stream): Connection | null {
|
|
141
|
-
if (typeof node.getConnections !== "function") return null;
|
|
142
|
-
return node.getConnections().find((connection) => connection.streams.includes(stream)) ?? null;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
116
|
export const connect: Connect<ConnectLibp2pParams> = async ({
|
|
146
117
|
node,
|
|
147
118
|
peer,
|
|
148
119
|
protocol,
|
|
149
120
|
drainTimeoutMs,
|
|
121
|
+
readHighWaterBytes,
|
|
150
122
|
maxOutboundStreams,
|
|
151
123
|
runOnLimitedConnection,
|
|
152
124
|
}) => {
|
|
@@ -173,24 +145,7 @@ export const connect: Connect<ConnectLibp2pParams> = async ({
|
|
|
173
145
|
// open concurrently. If that connection closes, the next burst is gated again.
|
|
174
146
|
// The state is shared by every connect() to the same peer from the same node
|
|
175
147
|
// (callers often connect() per request), since they all share its connection.
|
|
176
|
-
const
|
|
177
|
-
const openStream = async (): Promise<Stream> => {
|
|
178
|
-
while (!isProven(gate)) {
|
|
179
|
-
if (gate.firstOpening == null) {
|
|
180
|
-
const opening = node.dialProtocol(peer, [proto], dialOptions);
|
|
181
|
-
gate.firstOpening = opening;
|
|
182
|
-
try {
|
|
183
|
-
const stream = await opening;
|
|
184
|
-
gate.proven = { connection: connectionOf(node, stream) };
|
|
185
|
-
return stream;
|
|
186
|
-
} finally {
|
|
187
|
-
gate.firstOpening = null;
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
await gate.firstOpening.catch(() => undefined);
|
|
191
|
-
}
|
|
192
|
-
return node.dialProtocol(peer, [proto], dialOptions);
|
|
193
|
-
};
|
|
148
|
+
const openStream = (): Promise<Stream> => openGatedStream(node, peer, proto, dialOptions);
|
|
194
149
|
|
|
195
150
|
const call: Duplex = (input) => {
|
|
196
151
|
let streamRef: Stream | null = null;
|
|
@@ -202,6 +157,7 @@ export const connect: Connect<ConnectLibp2pParams> = async ({
|
|
|
202
157
|
try {
|
|
203
158
|
yield* duplexOverStream(stream, input, {
|
|
204
159
|
drainTimeoutMs,
|
|
160
|
+
readHighWaterBytes,
|
|
205
161
|
onSourceCompleted: () => {
|
|
206
162
|
sourceCompleted = true;
|
|
207
163
|
},
|
|
@@ -285,6 +241,7 @@ export async function serveConnections(
|
|
|
285
241
|
node,
|
|
286
242
|
protocol,
|
|
287
243
|
drainTimeoutMs,
|
|
244
|
+
readHighWaterBytes,
|
|
288
245
|
maxInboundStreams,
|
|
289
246
|
maxOutboundStreams,
|
|
290
247
|
runOnLimitedConnection,
|
|
@@ -299,16 +256,29 @@ export async function serveConnections(
|
|
|
299
256
|
});
|
|
300
257
|
|
|
301
258
|
const onStream = (stream: Stream, connection: Connection): void => {
|
|
259
|
+
// The caller sent its whole request; a stream that dies after that is a
|
|
260
|
+
// caller that stopped reading the response — ordinary, not a fault.
|
|
261
|
+
let requestComplete = false;
|
|
302
262
|
void (async () => {
|
|
303
263
|
const handler = makeHandler({ remotePeer: connection.remotePeer });
|
|
304
|
-
const inputQueue = makeInputQueue();
|
|
305
|
-
|
|
264
|
+
const inputQueue = makeInputQueue(readHighWaterBytes ?? DEFAULT_READ_HIGH_WATER_BYTES);
|
|
265
|
+
// once the handler's output has ended, input it never started to read is
|
|
266
|
+
// dropped instead of waited for (input it is reading, perhaps in the
|
|
267
|
+
// background, keeps flowing at its pace)
|
|
268
|
+
const output = endsWith(handler(inputQueue.iter()), () => inputQueue.abandon());
|
|
306
269
|
try {
|
|
307
270
|
for await (const chunk of duplexOverStream(stream, output, {
|
|
308
271
|
drainTimeoutMs,
|
|
309
|
-
|
|
272
|
+
readHighWaterBytes,
|
|
273
|
+
onPeerInputEnd: (err) => {
|
|
274
|
+
if (err === undefined) requestComplete = true;
|
|
275
|
+
inputQueue.done(err);
|
|
276
|
+
},
|
|
310
277
|
})) {
|
|
311
|
-
|
|
278
|
+
// past readHighWaterBytes queued for the handler, the next chunk is
|
|
279
|
+
// read only once it took this one: a handler that stops reading
|
|
280
|
+
// pauses the stream (see readHighWaterBytes)
|
|
281
|
+
await inputQueue.push(chunk);
|
|
312
282
|
}
|
|
313
283
|
} finally {
|
|
314
284
|
inputQueue.done();
|
|
@@ -323,6 +293,7 @@ export async function serveConnections(
|
|
|
323
293
|
// unhandled, and Node's default for `unhandledRejection` is to
|
|
324
294
|
// terminate. Logged rather than rethrown, and never rethrown from here:
|
|
325
295
|
// there is no caller left to receive it.
|
|
296
|
+
if (requestComplete && err instanceof PeerGoneError) return;
|
|
326
297
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
327
298
|
console.warn(
|
|
328
299
|
`[webrun-streams-libp2p] serve: inbound stream on protocol ${proto} failed: ${e.message}`,
|
|
@@ -341,20 +312,61 @@ export async function serveConnections(
|
|
|
341
312
|
};
|
|
342
313
|
}
|
|
343
314
|
|
|
315
|
+
/** `source`, calling `onEnd` once when it ends, fails or is returned. */
|
|
316
|
+
function endsWith(
|
|
317
|
+
source: AsyncIterable<Uint8Array>,
|
|
318
|
+
onEnd: () => void,
|
|
319
|
+
): AsyncGenerator<Uint8Array> {
|
|
320
|
+
return (async function* () {
|
|
321
|
+
try {
|
|
322
|
+
yield* source;
|
|
323
|
+
} finally {
|
|
324
|
+
onEnd();
|
|
325
|
+
}
|
|
326
|
+
})();
|
|
327
|
+
}
|
|
328
|
+
|
|
344
329
|
interface InputQueue {
|
|
345
330
|
iter(): AsyncGenerator<Uint8Array>;
|
|
346
|
-
|
|
331
|
+
/**
|
|
332
|
+
* resolves at once while the bytes queued for the handler stay within the
|
|
333
|
+
* limit, else once the handler has read them back down to it (or will never
|
|
334
|
+
* read on)
|
|
335
|
+
*/
|
|
336
|
+
push(chunk: Uint8Array): Promise<void>;
|
|
347
337
|
done(err?: Error): void;
|
|
338
|
+
/**
|
|
339
|
+
* the handler's output ended: if it never started reading its input, pending
|
|
340
|
+
* and later chunks are dropped (an iterator started already is left to its
|
|
341
|
+
* reader, which may go on reading after the output ended)
|
|
342
|
+
*/
|
|
343
|
+
abandon(): void;
|
|
348
344
|
}
|
|
349
345
|
|
|
350
|
-
function makeInputQueue(): InputQueue {
|
|
346
|
+
function makeInputQueue(limit: number): InputQueue {
|
|
351
347
|
type Slot = { type: "value"; value: Uint8Array } | { type: "done"; err?: Error };
|
|
352
348
|
const slots: Slot[] = [];
|
|
353
349
|
let wake: (() => void) | null = null;
|
|
354
350
|
let closed = false;
|
|
351
|
+
let pulled = false;
|
|
352
|
+
let queued = 0;
|
|
353
|
+
// the one push waiting for room (the serving loop awaits each push)
|
|
354
|
+
let room: (() => void) | null = null;
|
|
355
|
+
const makeRoom = (): void => {
|
|
356
|
+
const r = room;
|
|
357
|
+
room = null;
|
|
358
|
+
r?.();
|
|
359
|
+
};
|
|
360
|
+
const release = (): void => {
|
|
361
|
+
slots.length = 0;
|
|
362
|
+
queued = 0;
|
|
363
|
+
makeRoom();
|
|
364
|
+
};
|
|
355
365
|
return {
|
|
356
366
|
iter(): AsyncGenerator<Uint8Array> {
|
|
357
367
|
return (async function* () {
|
|
368
|
+
if (closed) return; // abandoned before anyone read: the input was dropped
|
|
369
|
+
pulled = true;
|
|
358
370
|
try {
|
|
359
371
|
while (true) {
|
|
360
372
|
if (slots.length === 0) {
|
|
@@ -369,22 +381,36 @@ function makeInputQueue(): InputQueue {
|
|
|
369
381
|
if (s.err) throw s.err;
|
|
370
382
|
return;
|
|
371
383
|
}
|
|
384
|
+
queued -= s.value.byteLength;
|
|
385
|
+
if (queued <= limit) makeRoom();
|
|
372
386
|
yield s.value;
|
|
373
387
|
}
|
|
374
388
|
} finally {
|
|
375
389
|
closed = true;
|
|
390
|
+
release();
|
|
376
391
|
}
|
|
377
392
|
})();
|
|
378
393
|
},
|
|
379
|
-
push(chunk: Uint8Array): void {
|
|
380
|
-
if (closed) return;
|
|
394
|
+
push(chunk: Uint8Array): Promise<void> {
|
|
395
|
+
if (closed) return Promise.resolve();
|
|
381
396
|
slots.push({ type: "value", value: chunk });
|
|
397
|
+
queued += chunk.byteLength;
|
|
382
398
|
wake?.();
|
|
399
|
+
if (queued <= limit) return Promise.resolve();
|
|
400
|
+
return new Promise<void>((resolve) => {
|
|
401
|
+
room = resolve;
|
|
402
|
+
});
|
|
383
403
|
},
|
|
384
404
|
done(err?: Error): void {
|
|
385
405
|
if (closed) return;
|
|
386
406
|
slots.push({ type: "done", err });
|
|
387
407
|
wake?.();
|
|
388
408
|
},
|
|
409
|
+
abandon(): void {
|
|
410
|
+
if (pulled) return;
|
|
411
|
+
closed = true;
|
|
412
|
+
release();
|
|
413
|
+
wake?.();
|
|
414
|
+
},
|
|
389
415
|
};
|
|
390
416
|
}
|