@statewalker/webrun-streams-libp2p 0.1.1 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -1
- package/dist/duplex-over-stream.d.ts +29 -0
- package/dist/duplex-over-stream.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -22
- package/package.json +20 -14
- package/src/duplex-over-stream.ts +110 -37
- package/src/index.ts +1 -0
package/README.md
CHANGED
|
@@ -2,6 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
libp2p native multi-stream `Connect` / `Serve` adapter. Each `call(input)` opens a new libp2p `Stream` via `node.dialProtocol(peer, [protocol])`; the responder registers via `node.handle(protocol, ...)`. Default protocol id: `/webrun-streams/1.0.0`. Targets libp2p 3.x.
|
|
4
4
|
|
|
5
|
+
## Why it exists
|
|
6
|
+
|
|
7
|
+
libp2p already solves the hard parts of peer-to-peer — transport negotiation,
|
|
8
|
+
NAT traversal, circuit relaying, and an authenticated peer identity from the
|
|
9
|
+
Noise handshake. What it hands an application is a `Stream`, not a request.
|
|
10
|
+
|
|
11
|
+
This adapter binds that stream to the [`webrun-streams`](../webrun-streams)
|
|
12
|
+
`Duplex` seam, so a handler written for a `MessagePort` or a WebSocket runs
|
|
13
|
+
unchanged across a libp2p network. Because libp2p multiplexes natively, this
|
|
14
|
+
adapter is one of two in the family (with
|
|
15
|
+
[`webrun-streams-webrtc`](../webrun-streams-webrtc)) that needs no `emulateMux`
|
|
16
|
+
— one `call` is one real libp2p stream.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
npm install @statewalker/webrun-streams-libp2p libp2p @libp2p/interface @multiformats/multiaddr
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
`libp2p` (`^3.0.0`), `@libp2p/interface` (`^3.0.0`) and `@multiformats/multiaddr`
|
|
25
|
+
(`^13.0.0`) are **peer dependencies** — you build and own the node, including
|
|
26
|
+
its transports, encryption and muxers.
|
|
27
|
+
|
|
28
|
+
## Getting started
|
|
29
|
+
|
|
5
30
|
```ts
|
|
6
31
|
import { connect, serve } from "@statewalker/webrun-streams-libp2p";
|
|
7
32
|
|
|
@@ -68,6 +93,45 @@ Two bounds keep a misbehaving peer from parking a long-lived server:
|
|
|
68
93
|
|
|
69
94
|
One inbound stream failing never takes down the serving process. `duplexOverStream` rejects on the read side when a peer sends an `ERROR` frame or resets mid-request — which happens in ordinary use, a browser tab closed mid-request being enough — and `serveConnections` catches that per stream and logs it. Callers see their own errors as usual: cancelling a `call` (i.e. `.return()` on the returned generator) sends a reset, and `close()` on the connection aborts every stream it still holds open.
|
|
70
95
|
|
|
96
|
+
## API
|
|
97
|
+
|
|
98
|
+
| Export | Kind | Purpose |
|
|
99
|
+
| --- | --- | --- |
|
|
100
|
+
| `connect(params)` | `Connect<ConnectLibp2pParams>` | Dials `peer` and resolves `{ call, close }`. Each `call` opens one libp2p stream. |
|
|
101
|
+
| `serve(params, handler)` | `Serve<ServeLibp2pParams>` | Registers `handler` on `protocol`. Returns an idempotent teardown. |
|
|
102
|
+
| `serveConnections(params, makeHandler)` | function | Like `serve`, but builds a handler per inbound stream and passes it the authenticated `ConnectionContext`. |
|
|
103
|
+
| `ServeConnectionsHandler` | type | `(context: ConnectionContext) => Duplex` — the factory `serveConnections` takes. |
|
|
104
|
+
| `duplexOverStream(stream, options?)` | function | Wraps one libp2p `Stream` as a `Duplex`, applying the framing and flow control described above. |
|
|
105
|
+
| `closeStream(stream, ...)` | function | The graceful-close-then-abort sequence, with the 5 s close timeout. |
|
|
106
|
+
| `DEFAULT_PROTOCOL` | const | `"/webrun-streams/1.0.0"` — used when `protocol` is unset. |
|
|
107
|
+
| `DEFAULT_DRAIN_TIMEOUT_MS` | const | `300_000` — the default drain bound (5 minutes). |
|
|
108
|
+
|
|
109
|
+
### `ConnectLibp2pParams`
|
|
110
|
+
|
|
111
|
+
| Field | Type | Default | Meaning |
|
|
112
|
+
| --- | --- | --- | --- |
|
|
113
|
+
| `node` | `Libp2p` | — | The local node. |
|
|
114
|
+
| `peer` | `PeerId \| Multiaddr` | — | Who to dial. |
|
|
115
|
+
| `protocol` | `string` | `DEFAULT_PROTOCOL` | libp2p protocol id. |
|
|
116
|
+
| `drainTimeoutMs` | `number` | `300_000` | Backpressure drain bound before dropping the peer. |
|
|
117
|
+
| `maxOutboundStreams` | `number` | libp2p's (64) | Passed through to `dialProtocol`. |
|
|
118
|
+
| `runOnLimitedConnection` | `boolean` | unset | Opt in to relayed / limited connections. |
|
|
119
|
+
|
|
120
|
+
### `ServeLibp2pParams`
|
|
121
|
+
|
|
122
|
+
As above minus `peer`, plus `maxInboundStreams` (libp2p default **32**).
|
|
123
|
+
|
|
124
|
+
### `ConnectionContext`
|
|
125
|
+
|
|
126
|
+
| Field | Type | Meaning |
|
|
127
|
+
| --- | --- | --- |
|
|
128
|
+
| `remotePeer` | `PeerId` | The peer id proved by the Noise handshake. Unforgeable by the request payload. |
|
|
129
|
+
|
|
130
|
+
### `DuplexOverStreamOptions`
|
|
131
|
+
|
|
132
|
+
`onPeerInputEnd(err?)`, `onSourceCompleted()` and `drainTimeoutMs` — see
|
|
133
|
+
[Flow control](#flow-control-and-what-libp2p-3x-does-not-give-you-for-free).
|
|
134
|
+
|
|
71
135
|
## Tests
|
|
72
136
|
|
|
73
137
|
```bash
|
|
@@ -77,6 +141,18 @@ WEBRUN_STREAMS_LIBP2P=1 pnpm test # + the framing/conformance suite
|
|
|
77
141
|
|
|
78
142
|
The conformance suite is opt-in because it spins up two real libp2p TCP nodes in-process.
|
|
79
143
|
|
|
144
|
+
## Dependencies
|
|
145
|
+
|
|
146
|
+
| Dependency | Kind | Why |
|
|
147
|
+
| --- | --- | --- |
|
|
148
|
+
| [`@statewalker/webrun-streams`](../webrun-streams) | runtime | The `Duplex` seam and error serialisation. |
|
|
149
|
+
| `libp2p` | **peer** (`^3.0.0`) | The node you build and own. |
|
|
150
|
+
| `@libp2p/interface` | **peer** (`^3.0.0`) | `Libp2p`, `PeerId`, `Stream` types. |
|
|
151
|
+
| `@multiformats/multiaddr` | **peer** (`^13.0.0`) | `Multiaddr` dial targets. |
|
|
152
|
+
| `@chainsafe/libp2p-noise`, `@chainsafe/libp2p-yamux`, `@libp2p/tcp`, `@libp2p/utils` | dev | Two real in-process nodes for the test suite. |
|
|
153
|
+
|
|
154
|
+
No runtime dependencies outside the workspace. ESM only (`"type": "module"`).
|
|
155
|
+
|
|
80
156
|
## License
|
|
81
157
|
|
|
82
|
-
MIT
|
|
158
|
+
MIT © statewalker — see [LICENSE](../../LICENSE).
|
|
@@ -16,6 +16,35 @@ import type { Stream } from "@libp2p/interface";
|
|
|
16
16
|
* `drainTimeoutMs` when a deployment knows better.
|
|
17
17
|
*/
|
|
18
18
|
export declare const DEFAULT_DRAIN_TIMEOUT_MS = 300000;
|
|
19
|
+
/**
|
|
20
|
+
* Default bound for {@link closeStream}'s wait for a graceful close.
|
|
21
|
+
*
|
|
22
|
+
* WHY THIS IS MINUTES AND NOT SECONDS. `stream.close()` waits for the write
|
|
23
|
+
* queue to DRAIN, and that queue is shared with every other stream on the same
|
|
24
|
+
* muxer. Draining one stream is therefore not a function of that stream alone:
|
|
25
|
+
* with many concurrent transfers it legitimately takes far longer than it would
|
|
26
|
+
* in isolation. When the bound trips, {@link closeStream} falls back to
|
|
27
|
+
* `abort()`, which resets the stream and truncates whatever was still in
|
|
28
|
+
* flight — so a bound tuned for an idle link silently corrupts healthy
|
|
29
|
+
* transfers under load.
|
|
30
|
+
*
|
|
31
|
+
* It did. This was 5000ms, matching libp2p 2.x's `DEFAULT_SEND_CLOSE_WRITE_TIMEOUT`
|
|
32
|
+
* — a number adopted for fidelity rather than reasoned about. Eighteen
|
|
33
|
+
* concurrent 3.5 MB fetches over one connection produced five complete
|
|
34
|
+
* responses out of twenty-two; the rest arrived truncated and rendered as
|
|
35
|
+
* broken images, with nothing but this module's own warning to say why.
|
|
36
|
+
*
|
|
37
|
+
* The bound's actual purpose is to catch a peer that has stopped reading and
|
|
38
|
+
* will never close, so the caller is not parked forever. Minutes serve that
|
|
39
|
+
* purpose exactly as well as seconds, and match {@link DEFAULT_DRAIN_TIMEOUT_MS},
|
|
40
|
+
* whose comment reasons about the identical hazard: *"a bound that is too tight
|
|
41
|
+
* resets a slow-but-alive peer mid-transfer — exactly what backpressure exists
|
|
42
|
+
* to avoid."* The two guard the same thing and should not disagree by a factor
|
|
43
|
+
* of sixty.
|
|
44
|
+
*
|
|
45
|
+
* Exported so a deployment that knows its links can lower it deliberately.
|
|
46
|
+
*/
|
|
47
|
+
export declare const DEFAULT_CLOSE_TIMEOUT_MS = 300000;
|
|
19
48
|
/**
|
|
20
49
|
* Options for {@link duplexOverStream}. The `onPeerInputEnd` hook is the seam
|
|
21
50
|
* that lets the server side close its input queue as soon as the peer's source
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duplex-over-stream.d.ts","sourceRoot":"","sources":["../src/duplex-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"duplex-over-stream.d.ts","sourceRoot":"","sources":["../src/duplex-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AAMlE;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,wBAAwB,SAAU,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,wBAAwB,SAA2B,CAAC;AAKjE;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;OAIG;IACH,cAAc,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,IAAI,IAAI,CAAC;IAC3B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAuB,gBAAgB,CACrC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,EACvD,IAAI,GAAE,uBAA4B,GACjC,cAAc,CAAC,UAAU,CAAC,CAwH5B;AA2CD;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,MAAiC,GAC3C,OAAO,CAAC,IAAI,CAAC,CAsBf"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { type ConnectionContext, type ConnectLibp2pParams, connect, DEFAULT_PROTOCOL, type ServeConnectionsHandler, type ServeLibp2pParams, serve, serveConnections, } from "./connect-serve.js";
|
|
2
|
-
export { DEFAULT_DRAIN_TIMEOUT_MS, type DuplexOverStreamOptions, duplexOverStream, } from "./duplex-over-stream.js";
|
|
2
|
+
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, type DuplexOverStreamOptions, duplexOverStream, } from "./duplex-over-stream.js";
|
|
3
3
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,OAAO,EACP,gBAAgB,EAChB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,EACL,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,KAAK,uBAAuB,EAC5B,gBAAgB,GACjB,MAAM,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,OAAO,EACP,gBAAgB,EAChB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,EACL,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,KAAK,uBAAuB,EAC5B,gBAAgB,GACjB,MAAM,yBAAyB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -3,13 +3,6 @@ import { deserializeError, serializeError } from "@statewalker/webrun-streams";
|
|
|
3
3
|
const TYPE_DATA = 0;
|
|
4
4
|
const TYPE_ERROR = 2;
|
|
5
5
|
/**
|
|
6
|
-
* Default bound for {@link closeStream}'s wait for a graceful close. Matches
|
|
7
|
-
* 2.x's own default (`DEFAULT_SEND_CLOSE_WRITE_TIMEOUT`,
|
|
8
|
-
* `@libp2p/utils@6.7.2/dist/src/abstract-stream.js:7`) — this restores that
|
|
9
|
-
* bound rather than inventing a new number.
|
|
10
|
-
*/
|
|
11
|
-
const DEFAULT_CLOSE_TIMEOUT_MS = 5e3;
|
|
12
|
-
/**
|
|
13
6
|
* Default bound for {@link waitForDrain}'s wait for the peer to make room in
|
|
14
7
|
* its receive window. Without a bound, a peer that requests something and then
|
|
15
8
|
* simply stops reading — alive, so no `close` event ever fires — parks the
|
|
@@ -26,6 +19,35 @@ const DEFAULT_CLOSE_TIMEOUT_MS = 5e3;
|
|
|
26
19
|
* `drainTimeoutMs` when a deployment knows better.
|
|
27
20
|
*/
|
|
28
21
|
const DEFAULT_DRAIN_TIMEOUT_MS = 3e5;
|
|
22
|
+
/**
|
|
23
|
+
* Default bound for {@link closeStream}'s wait for a graceful close.
|
|
24
|
+
*
|
|
25
|
+
* WHY THIS IS MINUTES AND NOT SECONDS. `stream.close()` waits for the write
|
|
26
|
+
* queue to DRAIN, and that queue is shared with every other stream on the same
|
|
27
|
+
* muxer. Draining one stream is therefore not a function of that stream alone:
|
|
28
|
+
* with many concurrent transfers it legitimately takes far longer than it would
|
|
29
|
+
* in isolation. When the bound trips, {@link closeStream} falls back to
|
|
30
|
+
* `abort()`, which resets the stream and truncates whatever was still in
|
|
31
|
+
* flight — so a bound tuned for an idle link silently corrupts healthy
|
|
32
|
+
* transfers under load.
|
|
33
|
+
*
|
|
34
|
+
* It did. This was 5000ms, matching libp2p 2.x's `DEFAULT_SEND_CLOSE_WRITE_TIMEOUT`
|
|
35
|
+
* — a number adopted for fidelity rather than reasoned about. Eighteen
|
|
36
|
+
* concurrent 3.5 MB fetches over one connection produced five complete
|
|
37
|
+
* responses out of twenty-two; the rest arrived truncated and rendered as
|
|
38
|
+
* broken images, with nothing but this module's own warning to say why.
|
|
39
|
+
*
|
|
40
|
+
* The bound's actual purpose is to catch a peer that has stopped reading and
|
|
41
|
+
* will never close, so the caller is not parked forever. Minutes serve that
|
|
42
|
+
* purpose exactly as well as seconds, and match {@link DEFAULT_DRAIN_TIMEOUT_MS},
|
|
43
|
+
* whose comment reasons about the identical hazard: *"a bound that is too tight
|
|
44
|
+
* resets a slow-but-alive peer mid-transfer — exactly what backpressure exists
|
|
45
|
+
* to avoid."* The two guard the same thing and should not disagree by a factor
|
|
46
|
+
* of sixty.
|
|
47
|
+
*
|
|
48
|
+
* Exported so a deployment that knows its links can lower it deliberately.
|
|
49
|
+
*/
|
|
50
|
+
const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
29
51
|
const textEncoder = new TextEncoder();
|
|
30
52
|
const textDecoder = new TextDecoder();
|
|
31
53
|
/**
|
|
@@ -46,7 +68,11 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
46
68
|
peerEndedCalled = true;
|
|
47
69
|
opts.onPeerInputEnd?.(err);
|
|
48
70
|
};
|
|
49
|
-
const
|
|
71
|
+
const inputIterator = input[Symbol.asyncIterator]?.() ?? input[Symbol.iterator]();
|
|
72
|
+
const cancelInput = () => {
|
|
73
|
+
Promise.resolve(inputIterator.return?.(void 0)).catch(() => void 0);
|
|
74
|
+
};
|
|
75
|
+
const outboundSource = framedOutbound(inputIterator);
|
|
50
76
|
const outbound = (async () => {
|
|
51
77
|
try {
|
|
52
78
|
for await (const chunk of outboundSource) if (!stream.send(chunk)) await waitForDrain(stream, opts.drainTimeoutMs ?? 3e5);
|
|
@@ -75,10 +101,14 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
75
101
|
opts.onSourceCompleted?.();
|
|
76
102
|
} finally {
|
|
77
103
|
firePeerInputEnd();
|
|
78
|
-
if (!sourceCompleted)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
104
|
+
if (!sourceCompleted) {
|
|
105
|
+
cancelInput();
|
|
106
|
+
Promise.resolve(outboundSource.return?.(void 0)).catch(() => void 0);
|
|
107
|
+
try {
|
|
108
|
+
stream.abort(/* @__PURE__ */ new Error("duplexOverStream: consumer cancelled"));
|
|
109
|
+
} catch {}
|
|
110
|
+
outbound.catch(() => void 0);
|
|
111
|
+
} else await outbound;
|
|
82
112
|
}
|
|
83
113
|
}
|
|
84
114
|
/**
|
|
@@ -139,11 +169,28 @@ async function closeStream(stream, timeoutMs = DEFAULT_CLOSE_TIMEOUT_MS) {
|
|
|
139
169
|
} catch {}
|
|
140
170
|
}
|
|
141
171
|
}
|
|
142
|
-
|
|
172
|
+
/**
|
|
173
|
+
* Frame the caller's outbound chunks.
|
|
174
|
+
*
|
|
175
|
+
* TAKES AN ITERATOR, NOT AN ITERABLE, so the caller can cancel the PRODUCER
|
|
176
|
+
* directly. Returning this wrapper generator is not enough: while it is
|
|
177
|
+
* parked awaiting the producer's `next()`, a `.return()` on the wrapper is
|
|
178
|
+
* queued behind that pending call and reaches the producer only once the
|
|
179
|
+
* producer yields — which a long-lived session never does. Teardown therefore
|
|
180
|
+
* needs a handle on the producer itself, and acquiring the iterator once (in
|
|
181
|
+
* `duplexOverStream`) is what provides it.
|
|
182
|
+
*/
|
|
183
|
+
async function* framedOutbound(iterator) {
|
|
143
184
|
try {
|
|
144
|
-
for
|
|
185
|
+
for (;;) {
|
|
186
|
+
const next = await iterator.next();
|
|
187
|
+
if (next.done === true) return;
|
|
188
|
+
yield frameData(normalizeChunk(next.value));
|
|
189
|
+
}
|
|
145
190
|
} catch (err) {
|
|
146
191
|
yield frameError(err instanceof Error ? err : new Error(String(err)));
|
|
192
|
+
} finally {
|
|
193
|
+
Promise.resolve(iterator.return?.(void 0)).catch(() => void 0);
|
|
147
194
|
}
|
|
148
195
|
}
|
|
149
196
|
async function* parseFrames(source) {
|
|
@@ -235,13 +282,6 @@ function decodeVarint(buf, start) {
|
|
|
235
282
|
}
|
|
236
283
|
throw new Error("decodeVarint: truncated");
|
|
237
284
|
}
|
|
238
|
-
function toAsyncIterable(input) {
|
|
239
|
-
if (input[Symbol.asyncIterator]) return input;
|
|
240
|
-
const it = input[Symbol.iterator]();
|
|
241
|
-
return { [Symbol.asyncIterator]() {
|
|
242
|
-
return { next: () => Promise.resolve(it.next()) };
|
|
243
|
-
} };
|
|
244
|
-
}
|
|
245
285
|
//#endregion
|
|
246
286
|
//#region src/connect-serve.ts
|
|
247
287
|
const DEFAULT_PROTOCOL = "/webrun-streams/1.0.0";
|
|
@@ -402,4 +442,4 @@ function makeInputQueue() {
|
|
|
402
442
|
};
|
|
403
443
|
}
|
|
404
444
|
//#endregion
|
|
405
|
-
export { DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_PROTOCOL, connect, duplexOverStream, serve, serveConnections };
|
|
445
|
+
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_PROTOCOL, connect, duplexOverStream, serve, serveConnections };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/webrun-streams-libp2p",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "libp2p native multi-stream Connect/Serve adapter in the webrun-streams-* family",
|
|
@@ -15,14 +15,18 @@
|
|
|
15
15
|
"url": "git@github.com:statewalker/webrun-wire.git"
|
|
16
16
|
},
|
|
17
17
|
"exports": {
|
|
18
|
-
".":
|
|
18
|
+
".": {
|
|
19
|
+
"source": "./src/index.ts",
|
|
20
|
+
"types": "./dist/index.d.ts",
|
|
21
|
+
"import": "./dist/index.js"
|
|
22
|
+
}
|
|
19
23
|
},
|
|
20
24
|
"files": [
|
|
21
25
|
"dist",
|
|
22
26
|
"src"
|
|
23
27
|
],
|
|
24
28
|
"dependencies": {
|
|
25
|
-
"@statewalker/webrun-streams": "0.
|
|
29
|
+
"@statewalker/webrun-streams": "^0.2.1"
|
|
26
30
|
},
|
|
27
31
|
"peerDependencies": {
|
|
28
32
|
"@libp2p/interface": "^3.0.0",
|
|
@@ -35,24 +39,26 @@
|
|
|
35
39
|
}
|
|
36
40
|
},
|
|
37
41
|
"devDependencies": {
|
|
38
|
-
"@
|
|
39
|
-
"@chainsafe/libp2p-
|
|
40
|
-
"@libp2p
|
|
41
|
-
"@libp2p/
|
|
42
|
-
"@libp2p/
|
|
43
|
-
"@
|
|
44
|
-
"@
|
|
45
|
-
"
|
|
42
|
+
"@biomejs/biome": "^2.5.15",
|
|
43
|
+
"@chainsafe/libp2p-noise": "^17.0.0",
|
|
44
|
+
"@chainsafe/libp2p-yamux": "^8.0.1",
|
|
45
|
+
"@libp2p/interface": "^3.3.0",
|
|
46
|
+
"@libp2p/tcp": "^11.0.28",
|
|
47
|
+
"@libp2p/utils": "^7.4.1",
|
|
48
|
+
"@multiformats/multiaddr": "^13.0.3",
|
|
49
|
+
"@types/node": "^26.6.4",
|
|
50
|
+
"libp2p": "^3.3.11",
|
|
46
51
|
"rimraf": "^6.1.3",
|
|
47
|
-
"rolldown": "^1.2.
|
|
52
|
+
"rolldown": "^1.2.12",
|
|
48
53
|
"typescript": "^7.0.2",
|
|
49
|
-
"vitest": "^
|
|
50
|
-
"@statewalker/webrun-streams-conformance": "0.
|
|
54
|
+
"vitest": "^5.0.3",
|
|
55
|
+
"@statewalker/webrun-streams-conformance": "^0.3.1"
|
|
51
56
|
},
|
|
52
57
|
"sideEffects": false,
|
|
53
58
|
"publishConfig": {
|
|
54
59
|
"access": "public"
|
|
55
60
|
},
|
|
61
|
+
"types": "./dist/index.d.ts",
|
|
56
62
|
"scripts": {
|
|
57
63
|
"build": "rimraf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
|
|
58
64
|
"test": "vitest run",
|
|
@@ -4,14 +4,6 @@ import { deserializeError, serializeError } from "@statewalker/webrun-streams";
|
|
|
4
4
|
const TYPE_DATA = 0x00;
|
|
5
5
|
const TYPE_ERROR = 0x02;
|
|
6
6
|
|
|
7
|
-
/**
|
|
8
|
-
* Default bound for {@link closeStream}'s wait for a graceful close. Matches
|
|
9
|
-
* 2.x's own default (`DEFAULT_SEND_CLOSE_WRITE_TIMEOUT`,
|
|
10
|
-
* `@libp2p/utils@6.7.2/dist/src/abstract-stream.js:7`) — this restores that
|
|
11
|
-
* bound rather than inventing a new number.
|
|
12
|
-
*/
|
|
13
|
-
const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
|
|
14
|
-
|
|
15
7
|
/**
|
|
16
8
|
* Default bound for {@link waitForDrain}'s wait for the peer to make room in
|
|
17
9
|
* its receive window. Without a bound, a peer that requests something and then
|
|
@@ -30,6 +22,36 @@ const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
|
|
|
30
22
|
*/
|
|
31
23
|
export const DEFAULT_DRAIN_TIMEOUT_MS = 300_000;
|
|
32
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Default bound for {@link closeStream}'s wait for a graceful close.
|
|
27
|
+
*
|
|
28
|
+
* WHY THIS IS MINUTES AND NOT SECONDS. `stream.close()` waits for the write
|
|
29
|
+
* queue to DRAIN, and that queue is shared with every other stream on the same
|
|
30
|
+
* muxer. Draining one stream is therefore not a function of that stream alone:
|
|
31
|
+
* with many concurrent transfers it legitimately takes far longer than it would
|
|
32
|
+
* in isolation. When the bound trips, {@link closeStream} falls back to
|
|
33
|
+
* `abort()`, which resets the stream and truncates whatever was still in
|
|
34
|
+
* flight — so a bound tuned for an idle link silently corrupts healthy
|
|
35
|
+
* transfers under load.
|
|
36
|
+
*
|
|
37
|
+
* It did. This was 5000ms, matching libp2p 2.x's `DEFAULT_SEND_CLOSE_WRITE_TIMEOUT`
|
|
38
|
+
* — a number adopted for fidelity rather than reasoned about. Eighteen
|
|
39
|
+
* concurrent 3.5 MB fetches over one connection produced five complete
|
|
40
|
+
* responses out of twenty-two; the rest arrived truncated and rendered as
|
|
41
|
+
* broken images, with nothing but this module's own warning to say why.
|
|
42
|
+
*
|
|
43
|
+
* The bound's actual purpose is to catch a peer that has stopped reading and
|
|
44
|
+
* will never close, so the caller is not parked forever. Minutes serve that
|
|
45
|
+
* purpose exactly as well as seconds, and match {@link DEFAULT_DRAIN_TIMEOUT_MS},
|
|
46
|
+
* whose comment reasons about the identical hazard: *"a bound that is too tight
|
|
47
|
+
* resets a slow-but-alive peer mid-transfer — exactly what backpressure exists
|
|
48
|
+
* to avoid."* The two guard the same thing and should not disagree by a factor
|
|
49
|
+
* of sixty.
|
|
50
|
+
*
|
|
51
|
+
* Exported so a deployment that knows its links can lower it deliberately.
|
|
52
|
+
*/
|
|
53
|
+
export const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
54
|
+
|
|
33
55
|
const textEncoder = new TextEncoder();
|
|
34
56
|
const textDecoder = new TextDecoder();
|
|
35
57
|
|
|
@@ -83,7 +105,18 @@ export async function* duplexOverStream(
|
|
|
83
105
|
opts.onPeerInputEnd?.(err);
|
|
84
106
|
};
|
|
85
107
|
|
|
86
|
-
|
|
108
|
+
// Acquire the producer's iterator ONCE, so teardown can cancel the producer
|
|
109
|
+
// itself rather than the wrapper around it — see `framedOutbound`.
|
|
110
|
+
const inputIterator: AsyncIterator<Uint8Array> | Iterator<Uint8Array> =
|
|
111
|
+
(input as AsyncIterable<Uint8Array>)[Symbol.asyncIterator]?.() ??
|
|
112
|
+
(input as Iterable<Uint8Array>)[Symbol.iterator]();
|
|
113
|
+
const cancelInput = (): void => {
|
|
114
|
+
void Promise.resolve((inputIterator as AsyncIterator<Uint8Array>).return?.(undefined)).catch(
|
|
115
|
+
() => undefined,
|
|
116
|
+
);
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
const outboundSource = framedOutbound(inputIterator);
|
|
87
120
|
const outbound = (async () => {
|
|
88
121
|
try {
|
|
89
122
|
// libp2p 3.x streams are push-based (`send()` + drain) rather than
|
|
@@ -146,18 +179,43 @@ export async function* duplexOverStream(
|
|
|
146
179
|
opts.onSourceCompleted?.();
|
|
147
180
|
} finally {
|
|
148
181
|
firePeerInputEnd();
|
|
149
|
-
// If the consumer aborted before the source completed,
|
|
150
|
-
// generator
|
|
151
|
-
//
|
|
152
|
-
//
|
|
182
|
+
// If the consumer aborted before the source completed, cut the outbound
|
|
183
|
+
// generator short. On natural source completion we DO NOT: the peer
|
|
184
|
+
// closing its write half does not entitle us to silence our own writes.
|
|
185
|
+
//
|
|
186
|
+
// NEITHER CALL IS AWAITED, AND THAT IS THE FIX.
|
|
187
|
+
//
|
|
188
|
+
// `.return()` on an async generator that is PARKED AWAITING ITS OWN
|
|
189
|
+
// SOURCE is queued behind that pending `next()` — it is not preemptive.
|
|
190
|
+
// For a long-lived session, waiting inside `next()` is where the pump
|
|
191
|
+
// spends its entire life, so the old `await outboundSource.return()`
|
|
192
|
+
// never settled, and the `await outbound` behind it never settled either:
|
|
193
|
+
// tearing down a duplex whose input was still open hung for ever. Two
|
|
194
|
+
// consumers of this package hit it and both worked around it with
|
|
195
|
+
// `close()`, which is a defect report, not a usage pattern.
|
|
196
|
+
//
|
|
197
|
+
// Cancelling without waiting keeps the contract a consumer is entitled to
|
|
198
|
+
// — `.return()` settles promptly, and a well-behaved producer still sees
|
|
199
|
+
// its `finally` — while a producer that cannot be woken is simply
|
|
200
|
+
// abandoned rather than allowed to hold the caller hostage. The stream is
|
|
201
|
+
// aborted below, so the peer is told rather than left guessing.
|
|
153
202
|
if (!sourceCompleted) {
|
|
203
|
+
// The producer first — this is the call that actually runs its
|
|
204
|
+
// `finally` — then the wrapper, which may be parked behind it.
|
|
205
|
+
cancelInput();
|
|
206
|
+
void Promise.resolve(outboundSource.return?.(undefined)).catch(() => undefined);
|
|
207
|
+
// The pump may be parked in `next()` or in `waitForDrain`; aborting the
|
|
208
|
+
// stream is what unblocks the latter and tells the peer this call is
|
|
209
|
+
// over. `closeStream`'s graceful path belongs to a completed call.
|
|
154
210
|
try {
|
|
155
|
-
|
|
211
|
+
stream.abort(new Error("duplexOverStream: consumer cancelled"));
|
|
156
212
|
} catch {
|
|
157
|
-
/*
|
|
213
|
+
/* already gone */
|
|
158
214
|
}
|
|
215
|
+
void outbound.catch(() => undefined);
|
|
216
|
+
} else {
|
|
217
|
+
await outbound;
|
|
159
218
|
}
|
|
160
|
-
await outbound;
|
|
161
219
|
}
|
|
162
220
|
}
|
|
163
221
|
|
|
@@ -239,16 +297,45 @@ export async function closeStream(
|
|
|
239
297
|
}
|
|
240
298
|
}
|
|
241
299
|
|
|
300
|
+
/**
|
|
301
|
+
* Frame the caller's outbound chunks.
|
|
302
|
+
*
|
|
303
|
+
* TAKES AN ITERATOR, NOT AN ITERABLE, so the caller can cancel the PRODUCER
|
|
304
|
+
* directly. Returning this wrapper generator is not enough: while it is
|
|
305
|
+
* parked awaiting the producer's `next()`, a `.return()` on the wrapper is
|
|
306
|
+
* queued behind that pending call and reaches the producer only once the
|
|
307
|
+
* producer yields — which a long-lived session never does. Teardown therefore
|
|
308
|
+
* needs a handle on the producer itself, and acquiring the iterator once (in
|
|
309
|
+
* `duplexOverStream`) is what provides it.
|
|
310
|
+
*/
|
|
242
311
|
async function* framedOutbound(
|
|
243
|
-
|
|
312
|
+
iterator: AsyncIterator<Uint8Array> | Iterator<Uint8Array>,
|
|
244
313
|
): AsyncGenerator<Uint8Array> {
|
|
245
314
|
try {
|
|
246
|
-
for
|
|
247
|
-
|
|
315
|
+
for (;;) {
|
|
316
|
+
const next = await iterator.next();
|
|
317
|
+
if (next.done === true) return;
|
|
318
|
+
yield frameData(normalizeChunk(next.value));
|
|
248
319
|
}
|
|
249
320
|
} catch (err) {
|
|
250
321
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
251
322
|
yield frameError(e);
|
|
323
|
+
} finally {
|
|
324
|
+
// PROPAGATE CANCELLATION TO THE PRODUCER, and do not wait for it.
|
|
325
|
+
//
|
|
326
|
+
// `for await (… of input)` used to do this implicitly: returning this
|
|
327
|
+
// generator ran the loop's cleanup, which returned the producer. Driving
|
|
328
|
+
// the iterator by hand (needed so teardown can hold a handle on the
|
|
329
|
+
// producer) removes that, and removing it silently stopped a server-side
|
|
330
|
+
// handler from ever being told its caller had gone — a regression caught
|
|
331
|
+
// by `cancel-server-handler.test.ts`.
|
|
332
|
+
//
|
|
333
|
+
// Not awaited: `.return()` on a producer parked at an `await` is queued
|
|
334
|
+
// behind it, so awaiting here would reintroduce the very hang this file's
|
|
335
|
+
// teardown was fixed to avoid.
|
|
336
|
+
void Promise.resolve((iterator as AsyncIterator<Uint8Array>).return?.(undefined)).catch(
|
|
337
|
+
() => undefined,
|
|
338
|
+
);
|
|
252
339
|
}
|
|
253
340
|
}
|
|
254
341
|
|
|
@@ -269,7 +356,8 @@ async function* parseFrames(
|
|
|
269
356
|
}
|
|
270
357
|
while (buf.byteLength > 0) {
|
|
271
358
|
if (buf.byteLength < 2) break; // need at least type + 1 varint byte
|
|
272
|
-
|
|
359
|
+
// biome-ignore lint/style/noNonNullAssertion: buf.byteLength >= 2, checked above, so index 0 exists.
|
|
360
|
+
const type = buf[0]!;
|
|
273
361
|
let lenInfo: { value: number; offset: number };
|
|
274
362
|
try {
|
|
275
363
|
lenInfo = decodeVarint(buf, 1);
|
|
@@ -341,7 +429,8 @@ function decodeVarint(buf: Uint8Array, start: number): { value: number; offset:
|
|
|
341
429
|
let shift = 0;
|
|
342
430
|
let i = start;
|
|
343
431
|
while (i < buf.length) {
|
|
344
|
-
|
|
432
|
+
// biome-ignore lint/style/noNonNullAssertion: i < buf.length, checked by the while condition, so this index exists.
|
|
433
|
+
const b = buf[i++]!;
|
|
345
434
|
value |= (b & 0x7f) << shift;
|
|
346
435
|
if ((b & 0x80) === 0) return { value: value >>> 0, offset: i };
|
|
347
436
|
shift += 7;
|
|
@@ -349,19 +438,3 @@ function decodeVarint(buf: Uint8Array, start: number): { value: number; offset:
|
|
|
349
438
|
}
|
|
350
439
|
throw new Error("decodeVarint: truncated");
|
|
351
440
|
}
|
|
352
|
-
|
|
353
|
-
function toAsyncIterable(
|
|
354
|
-
input: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
|
|
355
|
-
): AsyncIterable<Uint8Array> {
|
|
356
|
-
if ((input as AsyncIterable<Uint8Array>)[Symbol.asyncIterator]) {
|
|
357
|
-
return input as AsyncIterable<Uint8Array>;
|
|
358
|
-
}
|
|
359
|
-
const it = (input as Iterable<Uint8Array>)[Symbol.iterator]();
|
|
360
|
-
return {
|
|
361
|
-
[Symbol.asyncIterator]() {
|
|
362
|
-
return {
|
|
363
|
-
next: () => Promise.resolve(it.next()),
|
|
364
|
-
};
|
|
365
|
-
},
|
|
366
|
-
};
|
|
367
|
-
}
|