@statewalker/webrun-streams-libp2p 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,6 +9,7 @@ export interface StreamMessageTarget {
9
9
  target: {
10
10
  addEventListener(type: "message", listener: Listener): void;
11
11
  removeEventListener(type: "message", listener: Listener): void;
12
+ /** Sends one message. Throws `TypeError` for anything but bytes. */
12
13
  postMessage(message: unknown): void;
13
14
  close(): Promise<void>;
14
15
  /** The stream's write buffer, in bytes. */
@@ -18,6 +19,8 @@ export interface StreamMessageTarget {
18
19
  * closed; at once when it can now.
19
20
  */
20
21
  whenDrained(): Promise<void>;
22
+ /** `signal` again, on the target: `webrun-rpc`'s `getPortCloseSignal` reads it from a port. */
23
+ readonly closeSignal: AbortSignal;
21
24
  };
22
25
  /**
23
26
  * Aborts when the stream is over, after every message the peer sent has
@@ -1 +1 @@
1
- {"version":3,"file":"message-target-over-stream.d.ts","sourceRoot":"","sources":["../src/message-target-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AASlE,KAAK,QAAQ,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9D,uEAAuE;AACvE,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE;QACN,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC5D,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC/D,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;QACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,2CAA2C;QAC3C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC;;;WAGG;QACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;KAC9B,CAAC;IACF;;;;;;;OAOG;IACH,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,8BAA8B;IAC7C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,6EAA6E;AAC7E,eAAO,MAAM,yBAAyB,QAAmB,CAAC;AAE1D;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,EAAE,eAA2C,EAAE,GAAE,8BAAmC,GACnF,mBAAmB,CAkHrB"}
1
+ {"version":3,"file":"message-target-over-stream.d.ts","sourceRoot":"","sources":["../src/message-target-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AAUlE,KAAK,QAAQ,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9D,uEAAuE;AACvE,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE;QACN,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC5D,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC/D,oEAAoE;QACpE,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;QACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,2CAA2C;QAC3C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC;;;WAGG;QACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,+FAA+F;QAC/F,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;KACnC,CAAC;IACF;;;;;;;OAOG;IACH,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,8BAA8B;IAC7C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,6EAA6E;AAC7E,eAAO,MAAM,yBAAyB,QAAmB,CAAC;AAE1D;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,EAAE,eAA2C,EAAE,GAAE,8BAAmC,GACnF,mBAAmB,CAmHrB"}
@@ -0,0 +1,89 @@
1
+ import type { Connection, Libp2p, PeerId } from "@libp2p/interface";
2
+ import type { Multiaddr } from "@multiformats/multiaddr";
3
+ import { type StreamMessageTarget } from "./message-target-over-stream.js";
4
+ /** The protocol a port's stream negotiates unless told otherwise. */
5
+ export declare const DEFAULT_PORT_PROTOCOL = "/webrun-ports/1.0.0";
6
+ /**
7
+ * 256 KiB: what one message on a port may carry before `webrun-rpc`'s stream tier chunks a body.
8
+ * A port waits for each chunk's confirmation, so a larger chunk means fewer round trips; well
9
+ * under `messageTargetOverStream`'s 64 MiB frame ceiling.
10
+ */
11
+ export declare const DEFAULT_PORT_MAX_MESSAGE_SIZE: number;
12
+ /**
13
+ * How a port's messages become bytes on its stream and back: any value the stream tier posts
14
+ * (objects holding `Uint8Array` chunks, serialised errors). `webrun-msgpack`'s `serialize` and
15
+ * `deserialize` fit: `{ encode: serialize, decode: deserialize }`.
16
+ */
17
+ export interface ValueCodec {
18
+ encode(value: unknown): Uint8Array;
19
+ decode(bytes: Uint8Array): unknown;
20
+ }
21
+ type Listener = (event: MessageEvent) => void | Promise<void>;
22
+ /**
23
+ * The stream limits a port mux asks libp2p for unless told otherwise: every concurrent call is a
24
+ * stream, and libp2p's own default (32 per protocol) would reset the 33rd.
25
+ */
26
+ export declare const DEFAULT_PORT_STREAM_LIMIT = 1024;
27
+ /** One port: a libp2p stream as a `MessageTarget` of values, with its close signal on it. */
28
+ export type StreamPort = Omit<StreamMessageTarget["target"], "addEventListener" | "removeEventListener"> & {
29
+ addEventListener(type: "message", listener: Listener): void;
30
+ removeEventListener(type: "message", listener: Listener): void;
31
+ };
32
+ /** What the serving side learns about a port the peer opened. */
33
+ export interface InboundPortInfo {
34
+ /** The peer libp2p's Noise handshake proved for the connection the stream came on. */
35
+ remotePeer: PeerId;
36
+ connection: Connection;
37
+ }
38
+ export interface Libp2pPortMuxOptions {
39
+ node: Libp2p;
40
+ /** Encodes each message as one frame; see {@link ValueCodec}. Both ends must agree. */
41
+ codec: ValueCodec;
42
+ /** The protocol each port's stream negotiates. Default {@link DEFAULT_PORT_PROTOCOL}. */
43
+ protocol?: string;
44
+ /** Whom `openPort` dials. Required to open ports; a serving-only mux leaves it out. */
45
+ peer?: PeerId | Multiaddr;
46
+ /**
47
+ * Called for each port the peer opens (each inbound stream). Return `false` to refuse it: the
48
+ * stream is aborted. Leave it out to accept none (the protocol is not handled).
49
+ */
50
+ onPort?: (port: StreamPort, info: InboundPortInfo) => boolean | undefined;
51
+ /** Reported as `maxMessageSize`. Default {@link DEFAULT_PORT_MAX_MESSAGE_SIZE}. */
52
+ maxMessageSize?: number;
53
+ /** The largest frame accepted on a port's stream (see `messageTargetOverStream`). */
54
+ maxMessageBytes?: number;
55
+ /** Default {@link DEFAULT_PORT_STREAM_LIMIT}. */
56
+ maxInboundStreams?: number;
57
+ /** Default {@link DEFAULT_PORT_STREAM_LIMIT}. */
58
+ maxOutboundStreams?: number;
59
+ runOnLimitedConnection?: boolean;
60
+ }
61
+ /** A `webrun-rpc` `PortMux` by shape; this package imports nothing from it. */
62
+ export interface Libp2pPortMux {
63
+ openPort(meta?: unknown, options?: {
64
+ signal?: AbortSignal;
65
+ }): Promise<StreamPort>;
66
+ close(reason?: unknown): Promise<void>;
67
+ readonly maxMessageSize: number;
68
+ }
69
+ /**
70
+ * Native MessagePorts over libp2p: every port is its own libp2p stream, which yamux multiplexes
71
+ * with its own per-stream flow control, framed as messages by `messageTargetOverStream`. Hand it
72
+ * to `webrun-rpc` as a port source and its stream tier runs one call per port, with its tested
73
+ * backpressure — a producer stays one confirmed chunk ahead of its consumer — instead of
74
+ * libp2p's own buffering, which lets a producer run tens of megabytes ahead of a consumer that
75
+ * stopped reading:
76
+ *
77
+ * ```ts
78
+ * const codec = { encode: serialize, decode: deserialize }; // webrun-msgpack
79
+ * await serve({ mux: overPorts((onPort) => libp2pPortMux({ node, codec, onPort })) }, handler);
80
+ * const { call } = await connect({ mux: overPorts(() => libp2pPortMux({ node, codec, peer })) });
81
+ * ```
82
+ *
83
+ * Each port carries `closeSignal`, aborted when its stream ends or its peer goes, so a call on a
84
+ * dead stream fails instead of waiting. Streams are opened through the same early-stream gate as
85
+ * `connect()`. `meta` is not sent: the protocol says what a port is for.
86
+ */
87
+ export declare function libp2pPortMux(options: Libp2pPortMuxOptions): Promise<Libp2pPortMux>;
88
+ export {};
89
+ //# sourceMappingURL=port-mux.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"port-mux.d.ts","sourceRoot":"","sources":["../src/port-mux.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAEV,MAAM,EACN,MAAM,EAGP,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAA2B,KAAK,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAGpG,qEAAqE;AACrE,eAAO,MAAM,qBAAqB,wBAAwB,CAAC;AAE3D;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,QAAa,CAAC;AAExD;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAAC;IACnC,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC;CACpC;AAED,KAAK,QAAQ,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9D;;;GAGG;AACH,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C,6FAA6F;AAC7F,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,mBAAmB,CAAC,QAAQ,CAAC,EAC7B,kBAAkB,GAAG,qBAAqB,CAC3C,GAAG;IACF,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC5D,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;CAChE,CAAC;AAEF,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC9B,sFAAsF;IACtF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,uFAAuF;IACvF,KAAK,EAAE,UAAU,CAAC;IAClB,yFAAyF;IACzF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uFAAuF;IACvF,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B;;;OAGG;IACH,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,KAAK,OAAO,GAAG,SAAS,CAAC;IAC1E,mFAAmF;IACnF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qFAAqF;IACrF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iDAAiD;IACjD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iDAAiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,+EAA+E;AAC/E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAClF,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,aAAa,CAAC,CAmIzF"}
@@ -0,0 +1,10 @@
1
+ import type { DialProtocolOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
+ import type { Multiaddr } from "@multiformats/multiaddr";
3
+ /**
4
+ * Opens a stream to `peer` for `proto`, one at a time until the remote has proven ready: a burst
5
+ * of streams on a connection the remote is still upgrading makes yamux abort the whole muxer
6
+ * (`maxEarlyStreams`). Shared by `connect()` and `libp2pPortMux`, and by every caller dialling
7
+ * the same peer from the same node, since they share its connection.
8
+ */
9
+ export declare function openGatedStream(node: Libp2p, peer: PeerId | Multiaddr, proto: string, dialOptions?: DialProtocolOptions): Promise<Stream>;
10
+ //# sourceMappingURL=stream-gate.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"stream-gate.d.ts","sourceRoot":"","sources":["../src/stream-gate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AACjG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AA+CzD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,KAAK,EAAE,MAAM,EACb,WAAW,CAAC,EAAE,mBAAmB,GAChC,OAAO,CAAC,MAAM,CAAC,CAiBjB"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@statewalker/webrun-streams-libp2p",
3
- "version": "0.2.0",
3
+ "version": "0.4.0",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "libp2p native multi-stream Connect/Serve adapter in the webrun-streams-* family",
@@ -30,7 +30,7 @@
30
30
  "src"
31
31
  ],
32
32
  "dependencies": {
33
- "@statewalker/webrun-streams": "^0.2.3"
33
+ "@statewalker/webrun-streams": "^0.4.0"
34
34
  },
35
35
  "peerDependencies": {
36
36
  "@libp2p/interface": "^3.0.0",
@@ -56,10 +56,10 @@
56
56
  "rolldown": "^1.2.12",
57
57
  "tsx": "^4.23.15",
58
58
  "typescript": "^7.0.2",
59
- "vitest": "^5.0.3",
60
- "@statewalker/webrun-msgpack": "^0.6.0",
61
- "@statewalker/webrun-rpc": "^0.7.0",
62
- "@statewalker/webrun-streams-conformance": "^0.3.3"
59
+ "vitest": "^5.0.5",
60
+ "@statewalker/webrun-msgpack": "^0.11.0",
61
+ "@statewalker/webrun-rpc": "^0.12.0",
62
+ "@statewalker/webrun-streams-conformance": "^0.5.0"
63
63
  },
64
64
  "sideEffects": false,
65
65
  "publishConfig": {
@@ -69,6 +69,8 @@
69
69
  "scripts": {
70
70
  "build": "rimraf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
71
71
  "test": "vitest run",
72
+ "typecheck": "tsc --noEmit",
73
+ "typecheck:tests": "tsc -p tsconfig.tests.json",
72
74
  "lint": "biome check src tests"
73
75
  }
74
76
  }
@@ -8,7 +8,13 @@ import type {
8
8
  } from "@libp2p/interface";
9
9
  import type { Multiaddr } from "@multiformats/multiaddr";
10
10
  import type { Connect, Duplex, Serve } from "@statewalker/webrun-streams";
11
- import { closeStream, duplexOverStream, PeerGoneError } from "./duplex-over-stream.js";
11
+ import {
12
+ closeStream,
13
+ DEFAULT_READ_HIGH_WATER_BYTES,
14
+ duplexOverStream,
15
+ PeerGoneError,
16
+ } from "./duplex-over-stream.js";
17
+ import { openGatedStream } from "./stream-gate.js";
12
18
 
13
19
  export const DEFAULT_PROTOCOL = "/webrun-streams/1.0.0";
14
20
 
@@ -22,6 +28,17 @@ export interface ConnectLibp2pParams {
22
28
  * peer; defaults to `DEFAULT_DRAIN_TIMEOUT_MS` (5 minutes).
23
29
  */
24
30
  drainTimeoutMs?: number;
31
+ /**
32
+ * Response bytes held while the caller is not reading before the stream is
33
+ * paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
34
+ */
35
+ readHighWaterBytes?: number;
36
+ /**
37
+ * The largest frame payload accepted from the peer; past it, or on a
38
+ * malformed frame header, the call fails and the stream is aborted. See
39
+ * `DuplexOverStreamOptions.maxFrameBytes` (default 64 MiB).
40
+ */
41
+ maxFrameBytes?: number;
25
42
  /**
26
43
  * How many outgoing streams for this protocol libp2p allows to be open at
27
44
  * the same time on one connection; passed through to `node.dialProtocol`.
@@ -49,6 +66,17 @@ export interface ServeLibp2pParams {
49
66
  * closing, since the caller-side `.return`/abort escape does not exist here.
50
67
  */
51
68
  drainTimeoutMs?: number;
69
+ /**
70
+ * Request bytes held while the handler is not reading before the stream is
71
+ * paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
72
+ */
73
+ readHighWaterBytes?: number;
74
+ /**
75
+ * The largest frame payload accepted from the peer; past it, or on a
76
+ * malformed frame header, the call fails and the stream is aborted. See
77
+ * `DuplexOverStreamOptions.maxFrameBytes` (default 64 MiB).
78
+ */
79
+ maxFrameBytes?: number;
52
80
  /**
53
81
  * How many incoming streams for this protocol libp2p allows to be open at
54
82
  * the same time on one connection; passed through to `node.handle`. Left
@@ -97,56 +125,13 @@ function definedOptions<T extends Record<string, unknown>>(fields: T): Partial<T
97
125
  * Caller-side: each `call(input)` opens a new libp2p `Stream` via
98
126
  * `node.dialProtocol(peer, [protocol])` and runs the call over it.
99
127
  */
100
- /** The early-stream guard of `connect()`, per local node and remote peer. */
101
- interface StreamGate {
102
- /**
103
- * A stream has negotiated a protocol on an open connection: the remote is ready.
104
- * `connection` is that connection when the node can tell (`getConnections`), so a
105
- * burst after it closes is gated again; a node that offers only `dialProtocol`
106
- * stays proven.
107
- */
108
- proven: { connection: Connection | null } | null;
109
- /** The stream being opened while no connection is proven; the others wait for it. */
110
- firstOpening: Promise<Stream> | null;
111
- }
112
-
113
- const streamGates = new WeakMap<Libp2p, Map<string, StreamGate>>();
114
-
115
- function streamGateFor(node: Libp2p, peer: PeerId | Multiaddr): StreamGate {
116
- let gates = streamGates.get(node);
117
- if (gates == null) {
118
- gates = new Map();
119
- streamGates.set(node, gates);
120
- }
121
- const key = peer.toString();
122
- let gate = gates.get(key);
123
- if (gate == null) {
124
- gate = { proven: null, firstOpening: null };
125
- gates.set(key, gate);
126
- }
127
- return gate;
128
- }
129
-
130
- function isProven(gate: StreamGate): boolean {
131
- if (gate.proven == null) return false;
132
- return gate.proven.connection == null || gate.proven.connection.status === "open";
133
- }
134
-
135
- /**
136
- * The connection carrying `stream`, when `node` can list its connections.
137
- * Callers may pass a narrow stand-in that offers only `dialProtocol` (httpeers
138
- * routes streams onto a kept relay circuit that way), so nothing else is required.
139
- */
140
- function connectionOf(node: Libp2p, stream: Stream): Connection | null {
141
- if (typeof node.getConnections !== "function") return null;
142
- return node.getConnections().find((connection) => connection.streams.includes(stream)) ?? null;
143
- }
144
-
145
128
  export const connect: Connect<ConnectLibp2pParams> = async ({
146
129
  node,
147
130
  peer,
148
131
  protocol,
149
132
  drainTimeoutMs,
133
+ readHighWaterBytes,
134
+ maxFrameBytes,
150
135
  maxOutboundStreams,
151
136
  runOnLimitedConnection,
152
137
  }) => {
@@ -173,24 +158,7 @@ export const connect: Connect<ConnectLibp2pParams> = async ({
173
158
  // open concurrently. If that connection closes, the next burst is gated again.
174
159
  // The state is shared by every connect() to the same peer from the same node
175
160
  // (callers often connect() per request), since they all share its connection.
176
- const gate = streamGateFor(node, peer);
177
- const openStream = async (): Promise<Stream> => {
178
- while (!isProven(gate)) {
179
- if (gate.firstOpening == null) {
180
- const opening = node.dialProtocol(peer, [proto], dialOptions);
181
- gate.firstOpening = opening;
182
- try {
183
- const stream = await opening;
184
- gate.proven = { connection: connectionOf(node, stream) };
185
- return stream;
186
- } finally {
187
- gate.firstOpening = null;
188
- }
189
- }
190
- await gate.firstOpening.catch(() => undefined);
191
- }
192
- return node.dialProtocol(peer, [proto], dialOptions);
193
- };
161
+ const openStream = (): Promise<Stream> => openGatedStream(node, peer, proto, dialOptions);
194
162
 
195
163
  const call: Duplex = (input) => {
196
164
  let streamRef: Stream | null = null;
@@ -202,6 +170,8 @@ export const connect: Connect<ConnectLibp2pParams> = async ({
202
170
  try {
203
171
  yield* duplexOverStream(stream, input, {
204
172
  drainTimeoutMs,
173
+ readHighWaterBytes,
174
+ maxFrameBytes,
205
175
  onSourceCompleted: () => {
206
176
  sourceCompleted = true;
207
177
  },
@@ -285,6 +255,8 @@ export async function serveConnections(
285
255
  node,
286
256
  protocol,
287
257
  drainTimeoutMs,
258
+ readHighWaterBytes,
259
+ maxFrameBytes,
288
260
  maxInboundStreams,
289
261
  maxOutboundStreams,
290
262
  runOnLimitedConnection,
@@ -304,17 +276,25 @@ export async function serveConnections(
304
276
  let requestComplete = false;
305
277
  void (async () => {
306
278
  const handler = makeHandler({ remotePeer: connection.remotePeer });
307
- const inputQueue = makeInputQueue();
308
- const output = handler(inputQueue.iter());
279
+ const inputQueue = makeInputQueue(readHighWaterBytes ?? DEFAULT_READ_HIGH_WATER_BYTES);
280
+ // once the handler's output has ended, input it never started to read is
281
+ // dropped instead of waited for (input it is reading, perhaps in the
282
+ // background, keeps flowing at its pace)
283
+ const output = endsWith(handler(inputQueue.iter()), () => inputQueue.abandon());
309
284
  try {
310
285
  for await (const chunk of duplexOverStream(stream, output, {
311
286
  drainTimeoutMs,
287
+ readHighWaterBytes,
288
+ maxFrameBytes,
312
289
  onPeerInputEnd: (err) => {
313
290
  if (err === undefined) requestComplete = true;
314
291
  inputQueue.done(err);
315
292
  },
316
293
  })) {
317
- inputQueue.push(chunk);
294
+ // past readHighWaterBytes queued for the handler, the next chunk is
295
+ // read only once it took this one: a handler that stops reading
296
+ // pauses the stream (see readHighWaterBytes)
297
+ await inputQueue.push(chunk);
318
298
  }
319
299
  } finally {
320
300
  inputQueue.done();
@@ -348,20 +328,61 @@ export async function serveConnections(
348
328
  };
349
329
  }
350
330
 
331
+ /** `source`, calling `onEnd` once when it ends, fails or is returned. */
332
+ function endsWith(
333
+ source: AsyncIterable<Uint8Array>,
334
+ onEnd: () => void,
335
+ ): AsyncGenerator<Uint8Array> {
336
+ return (async function* () {
337
+ try {
338
+ yield* source;
339
+ } finally {
340
+ onEnd();
341
+ }
342
+ })();
343
+ }
344
+
351
345
  interface InputQueue {
352
346
  iter(): AsyncGenerator<Uint8Array>;
353
- push(chunk: Uint8Array): void;
347
+ /**
348
+ * resolves at once while the bytes queued for the handler stay within the
349
+ * limit, else once the handler has read them back down to it (or will never
350
+ * read on)
351
+ */
352
+ push(chunk: Uint8Array): Promise<void>;
354
353
  done(err?: Error): void;
354
+ /**
355
+ * the handler's output ended: if it never started reading its input, pending
356
+ * and later chunks are dropped (an iterator started already is left to its
357
+ * reader, which may go on reading after the output ended)
358
+ */
359
+ abandon(): void;
355
360
  }
356
361
 
357
- function makeInputQueue(): InputQueue {
362
+ function makeInputQueue(limit: number): InputQueue {
358
363
  type Slot = { type: "value"; value: Uint8Array } | { type: "done"; err?: Error };
359
364
  const slots: Slot[] = [];
360
365
  let wake: (() => void) | null = null;
361
366
  let closed = false;
367
+ let pulled = false;
368
+ let queued = 0;
369
+ // the one push waiting for room (the serving loop awaits each push)
370
+ let room: (() => void) | null = null;
371
+ const makeRoom = (): void => {
372
+ const r = room;
373
+ room = null;
374
+ r?.();
375
+ };
376
+ const release = (): void => {
377
+ slots.length = 0;
378
+ queued = 0;
379
+ makeRoom();
380
+ };
362
381
  return {
363
382
  iter(): AsyncGenerator<Uint8Array> {
364
383
  return (async function* () {
384
+ if (closed) return; // abandoned before anyone read: the input was dropped
385
+ pulled = true;
365
386
  try {
366
387
  while (true) {
367
388
  if (slots.length === 0) {
@@ -376,22 +397,36 @@ function makeInputQueue(): InputQueue {
376
397
  if (s.err) throw s.err;
377
398
  return;
378
399
  }
400
+ queued -= s.value.byteLength;
401
+ if (queued <= limit) makeRoom();
379
402
  yield s.value;
380
403
  }
381
404
  } finally {
382
405
  closed = true;
406
+ release();
383
407
  }
384
408
  })();
385
409
  },
386
- push(chunk: Uint8Array): void {
387
- if (closed) return;
410
+ push(chunk: Uint8Array): Promise<void> {
411
+ if (closed) return Promise.resolve();
388
412
  slots.push({ type: "value", value: chunk });
413
+ queued += chunk.byteLength;
389
414
  wake?.();
415
+ if (queued <= limit) return Promise.resolve();
416
+ return new Promise<void>((resolve) => {
417
+ room = resolve;
418
+ });
390
419
  },
391
420
  done(err?: Error): void {
392
421
  if (closed) return;
393
422
  slots.push({ type: "done", err });
394
423
  wake?.();
395
424
  },
425
+ abandon(): void {
426
+ if (pulled) return;
427
+ closed = true;
428
+ release();
429
+ wake?.();
430
+ },
396
431
  };
397
432
  }