@statewalker/webrun-streams-libp2p 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -2
- package/dist/connect-serve.d.ts.map +1 -1
- package/dist/duplex-over-stream.d.ts +15 -0
- package/dist/duplex-over-stream.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +191 -8
- package/dist/message-target-over-stream.d.ts +62 -0
- package/dist/message-target-over-stream.d.ts.map +1 -0
- package/package.json +4 -1
- package/src/connect-serve.ts +9 -2
- package/src/duplex-over-stream.ts +85 -11
- package/src/index.ts +7 -0
- package/src/message-target-over-stream.ts +199 -0
package/README.md
CHANGED
|
@@ -55,9 +55,12 @@ runs (Node or browser).
|
|
|
55
55
|
| `serve(params, handler)` | `Serve<ServeLibp2pParams>`. `node.handle(protocol)`; runs `handler` per inbound stream. Resolves to an idempotent teardown that calls `node.unhandle(protocol)`. |
|
|
56
56
|
| `serveConnections(params, makeHandler)` | Like `serve`, but `makeHandler(context)` is called once per inbound stream with `{ remotePeer }`, and returns that stream's `Duplex`. |
|
|
57
57
|
| `duplexOverStream(stream, input, options?)` | The primitive under both: runs one call over one libp2p `Stream`. Returns an `AsyncGenerator<Uint8Array>` of the peer's bytes. |
|
|
58
|
+
| `PeerGoneError` | What a call fails with when the stream ended before both sides finished it: the peer died, the connection dropped, or the stream was reset after the peer's side ended. `cause` is libp2p's close error, if any. |
|
|
59
|
+
| `messageTargetOverStream(stream, options?)` | `{ target, signal }`: one stream as a pipe of `Uint8Array` messages for `multiplexPort` / `overPipe`, and an `AbortSignal` that aborts when the stream is over. `options.maxMessageBytes` caps one message. See below. |
|
|
58
60
|
| `DEFAULT_PROTOCOL` | `"/webrun-streams/1.0.0"`. |
|
|
59
61
|
| `DEFAULT_DRAIN_TIMEOUT_MS` | `300_000` (5 minutes). |
|
|
60
62
|
| `DEFAULT_CLOSE_TIMEOUT_MS` | `300_000` (5 minutes), the bound on a graceful close. |
|
|
63
|
+
| `DEFAULT_MAX_MESSAGE_BYTES` | `67_108_864` (64 MiB), `messageTargetOverStream`'s cap on one message. |
|
|
61
64
|
| `ConnectLibp2pParams`, `ServeLibp2pParams`, `ConnectionContext`, `ServeConnectionsHandler`, `DuplexOverStreamOptions` | Types, below. |
|
|
62
65
|
|
|
63
66
|
`ConnectLibp2pParams`:
|
|
@@ -246,6 +249,21 @@ streams and that an unconfigured server stops at 32.
|
|
|
246
249
|
|
|
247
250
|
- **Normal end.** Both sides finished: the stream is closed gracefully,
|
|
248
251
|
bounded by the close timeout.
|
|
252
|
+
- **The peer dies** (its process is killed, the network drops, the connection
|
|
253
|
+
is aborted). The call's output throws `PeerGoneError`, `webrun-streams-libp2p:
|
|
254
|
+
the stream ended before the call finished`, within milliseconds of libp2p
|
|
255
|
+
noticing, whether or not the caller's input is still open. It never ends
|
|
256
|
+
cleanly: a clean end requires the peer's own close-write
|
|
257
|
+
(`remoteCloseWrite`), because when a connection dies libp2p closes the
|
|
258
|
+
stream with no error and its iterator ends exactly as after a close-write.
|
|
259
|
+
Without that rule a truncated response reads as a complete one. A peer
|
|
260
|
+
that ended its side and then died while this side was still sending fails
|
|
261
|
+
the same way; the input is returned instead of waited on. A peer that
|
|
262
|
+
resets the stream before ending its side throws libp2p's `StreamResetError`;
|
|
263
|
+
one that resets it after (a tunnel whose far end gave up) throws
|
|
264
|
+
`PeerGoneError` with the `StreamResetError` as `cause`. Either way the
|
|
265
|
+
error says the call is incomplete — `PeerGoneError` does not prove the
|
|
266
|
+
peer died, only that the stream ended before both sides finished it.
|
|
249
267
|
- **Caller cancels** (`.return()` on the call's generator). The adapter aborts
|
|
250
268
|
the stream with `call cancelled` before the generator's own cleanup runs,
|
|
251
269
|
because by then the stream is already marked closed and `abort` would do
|
|
@@ -262,6 +280,46 @@ streams and that an unconfigured server stops at 32.
|
|
|
262
280
|
`[webrun-streams-libp2p] serve: inbound stream on protocol <p> failed: <message>`,
|
|
263
281
|
so one failed stream never becomes an unhandled rejection that terminates
|
|
264
282
|
the serving process. The handler's output is returned, so its `finally` runs.
|
|
283
|
+
A caller that sent its whole request and then stopped reading the response
|
|
284
|
+
(`.return()` mid-stream) is ordinary and is not logged.
|
|
285
|
+
|
|
286
|
+
### A multiplexer over one stream closes with it
|
|
287
|
+
|
|
288
|
+
`multiplexPort` (from `@statewalker/webrun-rpc`) can run many ports over one
|
|
289
|
+
stream. A multiplexer cannot see its pipe die, so wire the stream's life to it:
|
|
290
|
+
|
|
291
|
+
```ts
|
|
292
|
+
import { msgpackCodec } from "@statewalker/webrun-msgpack";
|
|
293
|
+
import { multiplexPort } from "@statewalker/webrun-rpc";
|
|
294
|
+
import { messageTargetOverStream } from "@statewalker/webrun-streams-libp2p";
|
|
295
|
+
|
|
296
|
+
const { target, signal } = messageTargetOverStream(await node.dialProtocol(peer, ["/my-mux/1.0.0"]));
|
|
297
|
+
const mux = multiplexPort(target, { codec: msgpackCodec, side: "initiator", signal });
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
`signal` aborts on the stream's `close` (with `PeerGoneError` when the
|
|
301
|
+
connection died) and on the peer's close-write — after the read loop has
|
|
302
|
+
delivered every message the peer sent before it, so a reply followed by a
|
|
303
|
+
close is not lost to a multiplexer that tears down on `signal`. Unwired, every port and every
|
|
304
|
+
call on a dead stream waits forever. Wired, 100 ports on a SIGKILLed peer all
|
|
305
|
+
close within the test's 1 s bound. A stream closes when its connection does,
|
|
306
|
+
so this also covers a peer that dies while no data is flowing.
|
|
307
|
+
|
|
308
|
+
Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
309
|
+
`message` listener in the same tick (`multiplexPort` does). `postMessage`
|
|
310
|
+
does not wait for room, and libp2p resets a stream whose write buffer passes
|
|
311
|
+
`maxWriteBufferLength`. The target reports its state instead:
|
|
312
|
+
`bufferedAmount` is the stream's write buffer, and `whenDrained()` resolves
|
|
313
|
+
on libp2p's `drain` (or the stream's `close`, so no sender is left waiting),
|
|
314
|
+
at once when the stream has room. A sender that must not overrun the buffer
|
|
315
|
+
awaits it before posting more.
|
|
316
|
+
|
|
317
|
+
A length prefix that is not a valid varint, or one past `maxMessageBytes`
|
|
318
|
+
(default `DEFAULT_MAX_MESSAGE_BYTES`, 64 MiB), aborts the stream and `signal`
|
|
319
|
+
with `webrun-streams-libp2p: a malformed message length` or `… exceeds
|
|
320
|
+
maxMessageBytes`: a peer not speaking this framing would otherwise make the
|
|
321
|
+
reader buffer without bound. Raise it with
|
|
322
|
+
`messageTargetOverStream(stream, { maxMessageBytes })` for larger messages.
|
|
265
323
|
|
|
266
324
|
### Constraints
|
|
267
325
|
|
|
@@ -275,7 +333,10 @@ streams and that an unconfigured server stops at 32.
|
|
|
275
333
|
|
|
276
334
|
`pnpm test` runs the backpressure, drain and close timeout, cancellation,
|
|
277
335
|
identity, burst, stream-limit and serving-resilience tests against two real
|
|
278
|
-
in-process TCP nodes.
|
|
336
|
+
in-process TCP nodes. `dead-peer.test.ts` kills the peer two ways, aborting
|
|
337
|
+
its connection in-process and SIGKILLing a forked server
|
|
338
|
+
(`tests/fixtures/greeting-server.ts`, run with `tsx`), with 100 open channels
|
|
339
|
+
each; every channel must fail with an error within 1 s. The shared conformance suite is opt-in because it spins
|
|
279
340
|
up more nodes: `WEBRUN_STREAMS_LIBP2P=1 pnpm test`. It runs with
|
|
280
341
|
`skipHugeBody`, and L6's small window does not apply (there is no
|
|
281
342
|
`emulateMux`), so L6 is an integrity check.
|
|
@@ -287,7 +348,8 @@ up more nodes: `WEBRUN_STREAMS_LIBP2P=1 pnpm test`. It runs with
|
|
|
287
348
|
- `@libp2p/interface`, `@multiformats/multiaddr` (peers, types only) and
|
|
288
349
|
`libp2p` (optional peer). Dev dependencies `@chainsafe/libp2p-noise`,
|
|
289
350
|
`@chainsafe/libp2p-yamux`, `@libp2p/tcp` and `@libp2p/utils` build the test
|
|
290
|
-
nodes
|
|
351
|
+
nodes; `@statewalker/webrun-rpc` and `@statewalker/webrun-msgpack` run the
|
|
352
|
+
multiplexer tests, and `tsx` runs the forked server.
|
|
291
353
|
|
|
292
354
|
## License
|
|
293
355
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connect-serve.d.ts","sourceRoot":"","sources":["../src/connect-serve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,MAAM,EACN,MAAM,EAGP,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AAG1E,eAAO,MAAM,gBAAgB,0BAA0B,CAAC;AAExD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAsED,eAAO,MAAM,OAAO,EAAE,OAAO,CAAC,mBAAmB,CA2GhD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,KAAK,EAAE,KAAK,CAAC,iBAAiB,CACF,CAAC;AAE1C,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,iBAAiB,KAAK,MAAM,CAAC;AAE7E;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,EACE,IAAI,EACJ,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,GACvB,EAAE,iBAAiB,EACpB,WAAW,EAAE,uBAAuB,GACnC,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"connect-serve.d.ts","sourceRoot":"","sources":["../src/connect-serve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,MAAM,EACN,MAAM,EAGP,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AAG1E,eAAO,MAAM,gBAAgB,0BAA0B,CAAC;AAExD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAsED,eAAO,MAAM,OAAO,EAAE,OAAO,CAAC,mBAAmB,CA2GhD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,KAAK,EAAE,KAAK,CAAC,iBAAiB,CACF,CAAC;AAE1C,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,iBAAiB,KAAK,MAAM,CAAC;AAE7E;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,EACE,IAAI,EACJ,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,GACvB,EAAE,iBAAiB,EACpB,WAAW,EAAE,uBAAuB,GACnC,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,CAwD9B"}
|
|
@@ -45,6 +45,15 @@ export declare const DEFAULT_DRAIN_TIMEOUT_MS = 300000;
|
|
|
45
45
|
* Exported so a deployment that knows its links can lower it deliberately.
|
|
46
46
|
*/
|
|
47
47
|
export declare const DEFAULT_CLOSE_TIMEOUT_MS = 300000;
|
|
48
|
+
/**
|
|
49
|
+
* The stream ended before the call finished: the peer died, the connection
|
|
50
|
+
* dropped, or the stream was reset or refused before both sides ended it.
|
|
51
|
+
* Distinct from a clean end on purpose — a response cut short must never read
|
|
52
|
+
* as a complete one. `cause` is the stream's close error, when libp2p gave one.
|
|
53
|
+
*/
|
|
54
|
+
export declare class PeerGoneError extends Error {
|
|
55
|
+
constructor(cause?: unknown);
|
|
56
|
+
}
|
|
48
57
|
/**
|
|
49
58
|
* Options for {@link duplexOverStream}. The `onPeerInputEnd` hook is the seam
|
|
50
59
|
* that lets the server side close its input queue as soon as the peer's source
|
|
@@ -94,4 +103,10 @@ export declare function duplexOverStream(stream: Stream, input: AsyncIterable<Ui
|
|
|
94
103
|
* caller is never left hanging.
|
|
95
104
|
*/
|
|
96
105
|
export declare function closeStream(stream: Stream, timeoutMs?: number): Promise<void>;
|
|
106
|
+
export declare function normalizeChunk(item: unknown): Uint8Array;
|
|
107
|
+
export declare function encodeVarint(value: number): Uint8Array;
|
|
108
|
+
export declare function decodeVarint(buf: Uint8Array, start: number): {
|
|
109
|
+
value: number;
|
|
110
|
+
offset: number;
|
|
111
|
+
};
|
|
97
112
|
//# sourceMappingURL=duplex-over-stream.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duplex-over-stream.d.ts","sourceRoot":"","sources":["../src/duplex-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AAMlE;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,wBAAwB,SAAU,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,wBAAwB,SAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"duplex-over-stream.d.ts","sourceRoot":"","sources":["../src/duplex-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AAMlE;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,wBAAwB,SAAU,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,wBAAwB,SAA2B,CAAC;AAEjE;;;;;GAKG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,YAAY,KAAK,CAAC,EAAE,OAAO,EAQ1B;CACF;AAKD;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;OAIG;IACH,cAAc,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,IAAI,IAAI,CAAC;IAC3B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAuB,gBAAgB,CACrC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,EACvD,IAAI,GAAE,uBAA4B,GACjC,cAAc,CAAC,UAAU,CAAC,CAgL5B;AA2CD;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,MAAiC,GAC3C,OAAO,CAAC,IAAI,CAAC,CAsBf;AA8ED,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,UAAU,CAOxD;AA8BD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,CAYtD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAa9F"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export { type ConnectionContext, type ConnectLibp2pParams, connect, DEFAULT_PROTOCOL, type ServeConnectionsHandler, type ServeLibp2pParams, serve, serveConnections, } from "./connect-serve.js";
|
|
2
|
-
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, type DuplexOverStreamOptions, duplexOverStream, } from "./duplex-over-stream.js";
|
|
2
|
+
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, type DuplexOverStreamOptions, duplexOverStream, PeerGoneError, } from "./duplex-over-stream.js";
|
|
3
|
+
export { DEFAULT_MAX_MESSAGE_BYTES, type MessageTargetOverStreamOptions, messageTargetOverStream, type StreamMessageTarget, } from "./message-target-over-stream.js";
|
|
3
4
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,OAAO,EACP,gBAAgB,EAChB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,EACL,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,KAAK,uBAAuB,EAC5B,gBAAgB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,OAAO,EACP,gBAAgB,EAChB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,EACL,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,KAAK,uBAAuB,EAC5B,gBAAgB,EAChB,aAAa,GACd,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,yBAAyB,EACzB,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,KAAK,mBAAmB,GACzB,MAAM,iCAAiC,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -48,6 +48,18 @@ const DEFAULT_DRAIN_TIMEOUT_MS = 3e5;
|
|
|
48
48
|
* Exported so a deployment that knows its links can lower it deliberately.
|
|
49
49
|
*/
|
|
50
50
|
const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
51
|
+
/**
|
|
52
|
+
* The stream ended before the call finished: the peer died, the connection
|
|
53
|
+
* dropped, or the stream was reset or refused before both sides ended it.
|
|
54
|
+
* Distinct from a clean end on purpose — a response cut short must never read
|
|
55
|
+
* as a complete one. `cause` is the stream's close error, when libp2p gave one.
|
|
56
|
+
*/
|
|
57
|
+
var PeerGoneError = class extends Error {
|
|
58
|
+
constructor(cause) {
|
|
59
|
+
super(`webrun-streams-libp2p: the stream ended before the call finished${cause instanceof Error ? ` (${cause.message})` : ""}`, { cause });
|
|
60
|
+
this.name = "PeerGoneError";
|
|
61
|
+
}
|
|
62
|
+
};
|
|
51
63
|
const textEncoder = new TextEncoder();
|
|
52
64
|
const textDecoder = new TextDecoder();
|
|
53
65
|
/**
|
|
@@ -72,10 +84,27 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
72
84
|
const cancelInput = () => {
|
|
73
85
|
Promise.resolve(inputIterator.return?.(void 0)).catch(() => void 0);
|
|
74
86
|
};
|
|
87
|
+
let peerClosedWrite = stream.remoteWriteStatus === "closed";
|
|
88
|
+
let closeError;
|
|
89
|
+
let onStreamClosed;
|
|
90
|
+
const streamClosed = new Promise((resolve) => {
|
|
91
|
+
onStreamClosed = resolve;
|
|
92
|
+
});
|
|
93
|
+
const onRemoteCloseWrite = () => {
|
|
94
|
+
peerClosedWrite = true;
|
|
95
|
+
};
|
|
96
|
+
const onClose = (evt) => {
|
|
97
|
+
closeError = evt.error;
|
|
98
|
+
onStreamClosed();
|
|
99
|
+
};
|
|
100
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
101
|
+
stream.addEventListener("close", onClose);
|
|
102
|
+
let inputSent = false;
|
|
75
103
|
const outboundSource = framedOutbound(inputIterator);
|
|
76
104
|
const outbound = (async () => {
|
|
77
105
|
try {
|
|
78
106
|
for await (const chunk of outboundSource) if (!stream.send(chunk)) await waitForDrain(stream, opts.drainTimeoutMs ?? 3e5);
|
|
107
|
+
inputSent = true;
|
|
79
108
|
await closeStream(stream);
|
|
80
109
|
} catch (err) {
|
|
81
110
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
@@ -90,11 +119,24 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
90
119
|
})();
|
|
91
120
|
let sourceCompleted = false;
|
|
92
121
|
try {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
122
|
+
let peerError;
|
|
123
|
+
try {
|
|
124
|
+
for await (const frame of parseFrames(stream)) if (frame.type === TYPE_DATA) yield frame.payload;
|
|
125
|
+
else if (frame.type === TYPE_ERROR) {
|
|
126
|
+
peerError = decodeError(frame.payload);
|
|
127
|
+
firePeerInputEnd(peerError);
|
|
128
|
+
throw peerError;
|
|
129
|
+
}
|
|
130
|
+
} catch (err) {
|
|
131
|
+
if (err === peerError) throw err;
|
|
132
|
+
const gone = new PeerGoneError(err);
|
|
133
|
+
firePeerInputEnd(gone);
|
|
134
|
+
throw gone;
|
|
135
|
+
}
|
|
136
|
+
if (!peerClosedWrite) {
|
|
137
|
+
const gone = new PeerGoneError(closeError);
|
|
138
|
+
firePeerInputEnd(gone);
|
|
139
|
+
throw gone;
|
|
98
140
|
}
|
|
99
141
|
sourceCompleted = true;
|
|
100
142
|
firePeerInputEnd();
|
|
@@ -108,8 +150,18 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
108
150
|
stream.abort(/* @__PURE__ */ new Error("duplexOverStream: consumer cancelled"));
|
|
109
151
|
} catch {}
|
|
110
152
|
outbound.catch(() => void 0);
|
|
111
|
-
} else
|
|
153
|
+
} else {
|
|
154
|
+
await Promise.race([outbound, streamClosed]);
|
|
155
|
+
if (!inputSent) {
|
|
156
|
+
cancelInput();
|
|
157
|
+
Promise.resolve(outboundSource.return?.(void 0)).catch(() => void 0);
|
|
158
|
+
outbound.catch(() => void 0);
|
|
159
|
+
} else await outbound;
|
|
160
|
+
}
|
|
161
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
162
|
+
stream.removeEventListener("close", onClose);
|
|
112
163
|
}
|
|
164
|
+
if (!inputSent) throw new PeerGoneError(closeError);
|
|
113
165
|
}
|
|
114
166
|
/**
|
|
115
167
|
* Wait for `stream` to signal it can accept more data, per the real
|
|
@@ -414,6 +466,7 @@ async function serveConnections({ node, protocol, drainTimeoutMs, maxInboundStre
|
|
|
414
466
|
runOnLimitedConnection
|
|
415
467
|
});
|
|
416
468
|
const onStream = (stream, connection) => {
|
|
469
|
+
let requestComplete = false;
|
|
417
470
|
(async () => {
|
|
418
471
|
const handler = makeHandler({ remotePeer: connection.remotePeer });
|
|
419
472
|
const inputQueue = makeInputQueue();
|
|
@@ -421,13 +474,17 @@ async function serveConnections({ node, protocol, drainTimeoutMs, maxInboundStre
|
|
|
421
474
|
try {
|
|
422
475
|
for await (const chunk of duplexOverStream(stream, output, {
|
|
423
476
|
drainTimeoutMs,
|
|
424
|
-
onPeerInputEnd: (err) =>
|
|
477
|
+
onPeerInputEnd: (err) => {
|
|
478
|
+
if (err === void 0) requestComplete = true;
|
|
479
|
+
inputQueue.done(err);
|
|
480
|
+
}
|
|
425
481
|
})) inputQueue.push(chunk);
|
|
426
482
|
} finally {
|
|
427
483
|
inputQueue.done();
|
|
428
484
|
await closeStream(stream);
|
|
429
485
|
}
|
|
430
486
|
})().catch((err) => {
|
|
487
|
+
if (requestComplete && err instanceof PeerGoneError) return;
|
|
431
488
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
432
489
|
console.warn(`[webrun-streams-libp2p] serve: inbound stream on protocol ${proto} failed: ${e.message}`, e);
|
|
433
490
|
});
|
|
@@ -487,4 +544,130 @@ function makeInputQueue() {
|
|
|
487
544
|
};
|
|
488
545
|
}
|
|
489
546
|
//#endregion
|
|
490
|
-
|
|
547
|
+
//#region src/message-target-over-stream.ts
|
|
548
|
+
/** 64 MiB: room for a large unframed message, short of exhausting memory. */
|
|
549
|
+
const DEFAULT_MAX_MESSAGE_BYTES = 67108864;
|
|
550
|
+
/**
|
|
551
|
+
* One libp2p stream as a pipe of messages, for a multiplexer to run over:
|
|
552
|
+
*
|
|
553
|
+
* ```ts
|
|
554
|
+
* const { target, signal } = messageTargetOverStream(stream);
|
|
555
|
+
* const mux = multiplexPort(target, { codec: msgpackCodec, signal });
|
|
556
|
+
* ```
|
|
557
|
+
*
|
|
558
|
+
* Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
559
|
+
* `message` listener in the same tick: a message with no listener is dropped,
|
|
560
|
+
* as on a `MessagePort`. Every message the peer sent before closing is
|
|
561
|
+
* delivered before `signal` aborts. `postMessage` hands the frame to
|
|
562
|
+
* `stream.send()` and does not wait for room; libp2p buffers up to the
|
|
563
|
+
* stream's `maxWriteBufferLength`, past which it resets the stream. A sender
|
|
564
|
+
* that must wait awaits `whenDrained()`.
|
|
565
|
+
*/
|
|
566
|
+
function messageTargetOverStream(stream, { maxMessageBytes = DEFAULT_MAX_MESSAGE_BYTES } = {}) {
|
|
567
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
568
|
+
const life = new AbortController();
|
|
569
|
+
let peerClosedWrite = false;
|
|
570
|
+
const onRemoteCloseWrite = () => {
|
|
571
|
+
peerClosedWrite = true;
|
|
572
|
+
};
|
|
573
|
+
const onClose = (evt) => {
|
|
574
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
575
|
+
if (peerClosedWrite && evt.error == null) return;
|
|
576
|
+
life.abort(new PeerGoneError(evt.error));
|
|
577
|
+
};
|
|
578
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
579
|
+
stream.addEventListener("close", onClose, { once: true });
|
|
580
|
+
const deliver = (data) => {
|
|
581
|
+
const event = new MessageEvent("message", { data });
|
|
582
|
+
for (const listener of [...listeners]) try {
|
|
583
|
+
Promise.resolve(listener(event)).catch(() => void 0);
|
|
584
|
+
} catch {}
|
|
585
|
+
};
|
|
586
|
+
(async () => {
|
|
587
|
+
let buf = /* @__PURE__ */ new Uint8Array(0);
|
|
588
|
+
let start = 0;
|
|
589
|
+
let end = 0;
|
|
590
|
+
for await (const item of stream) {
|
|
591
|
+
const incoming = normalizeChunk(item);
|
|
592
|
+
if (end + incoming.byteLength > buf.byteLength) {
|
|
593
|
+
const size = end - start + incoming.byteLength;
|
|
594
|
+
const next = new Uint8Array(Math.max(size, buf.byteLength * 2));
|
|
595
|
+
next.set(buf.subarray(start, end), 0);
|
|
596
|
+
buf = next;
|
|
597
|
+
end -= start;
|
|
598
|
+
start = 0;
|
|
599
|
+
}
|
|
600
|
+
buf.set(incoming, end);
|
|
601
|
+
end += incoming.byteLength;
|
|
602
|
+
for (;;) {
|
|
603
|
+
const header = readLength(buf.subarray(start, end));
|
|
604
|
+
if (header === void 0) break;
|
|
605
|
+
if (header.value > maxMessageBytes) throw new Error(`webrun-streams-libp2p: a ${header.value}-byte message exceeds maxMessageBytes (${maxMessageBytes})`);
|
|
606
|
+
const stop = start + header.offset + header.value;
|
|
607
|
+
if (end < stop) break;
|
|
608
|
+
deliver(buf.slice(start + header.offset, stop));
|
|
609
|
+
start = stop;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
if (peerClosedWrite) {
|
|
613
|
+
life.abort(/* @__PURE__ */ new Error("webrun-streams-libp2p: the peer closed the stream"));
|
|
614
|
+
await closeStream(stream);
|
|
615
|
+
}
|
|
616
|
+
})().catch((err) => {
|
|
617
|
+
life.abort(err);
|
|
618
|
+
try {
|
|
619
|
+
stream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
620
|
+
} catch {}
|
|
621
|
+
});
|
|
622
|
+
return {
|
|
623
|
+
signal: life.signal,
|
|
624
|
+
target: {
|
|
625
|
+
addEventListener: (_type, listener) => {
|
|
626
|
+
listeners.add(listener);
|
|
627
|
+
},
|
|
628
|
+
removeEventListener: (_type, listener) => {
|
|
629
|
+
listeners.delete(listener);
|
|
630
|
+
},
|
|
631
|
+
postMessage(message) {
|
|
632
|
+
const payload = normalizeChunk(message);
|
|
633
|
+
const header = encodeVarint(payload.byteLength);
|
|
634
|
+
const frame = new Uint8Array(header.byteLength + payload.byteLength);
|
|
635
|
+
frame.set(header, 0);
|
|
636
|
+
frame.set(payload, header.byteLength);
|
|
637
|
+
try {
|
|
638
|
+
stream.send(frame);
|
|
639
|
+
} catch {}
|
|
640
|
+
},
|
|
641
|
+
async close() {
|
|
642
|
+
listeners.clear();
|
|
643
|
+
await closeStream(stream);
|
|
644
|
+
},
|
|
645
|
+
get bufferedAmount() {
|
|
646
|
+
return stream.writeBufferLength;
|
|
647
|
+
},
|
|
648
|
+
whenDrained() {
|
|
649
|
+
if (!stream.writableNeedsDrain || life.signal.aborted) return Promise.resolve();
|
|
650
|
+
return new Promise((resolve) => {
|
|
651
|
+
const done = () => {
|
|
652
|
+
stream.removeEventListener("drain", done);
|
|
653
|
+
stream.removeEventListener("close", done);
|
|
654
|
+
resolve();
|
|
655
|
+
};
|
|
656
|
+
stream.addEventListener("drain", done);
|
|
657
|
+
stream.addEventListener("close", done);
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
};
|
|
662
|
+
}
|
|
663
|
+
/** The length prefix of `bytes`, or `undefined` while it is incomplete; throws when malformed. */
|
|
664
|
+
function readLength(bytes) {
|
|
665
|
+
try {
|
|
666
|
+
return decodeVarint(bytes, 0);
|
|
667
|
+
} catch (err) {
|
|
668
|
+
if (bytes.byteLength < 5) return void 0;
|
|
669
|
+
throw new Error("webrun-streams-libp2p: a malformed message length", { cause: err });
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
//#endregion
|
|
673
|
+
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_MAX_MESSAGE_BYTES, DEFAULT_PROTOCOL, PeerGoneError, connect, duplexOverStream, messageTargetOverStream, serve, serveConnections };
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { Stream } from "@libp2p/interface";
|
|
2
|
+
type Listener = (event: MessageEvent) => void | Promise<void>;
|
|
3
|
+
/** What `messageTargetOverStream` gives: a byte pipe, and its life. */
|
|
4
|
+
export interface StreamMessageTarget {
|
|
5
|
+
/**
|
|
6
|
+
* A `MessageTarget` (as `@statewalker/webrun-rpc` defines it) whose messages
|
|
7
|
+
* are `Uint8Array`s, one length-prefixed frame each.
|
|
8
|
+
*/
|
|
9
|
+
target: {
|
|
10
|
+
addEventListener(type: "message", listener: Listener): void;
|
|
11
|
+
removeEventListener(type: "message", listener: Listener): void;
|
|
12
|
+
postMessage(message: unknown): void;
|
|
13
|
+
close(): Promise<void>;
|
|
14
|
+
/** The stream's write buffer, in bytes. */
|
|
15
|
+
readonly bufferedAmount: number;
|
|
16
|
+
/**
|
|
17
|
+
* Resolves once the stream can take more (libp2p's `drain`), or has
|
|
18
|
+
* closed; at once when it can now.
|
|
19
|
+
*/
|
|
20
|
+
whenDrained(): Promise<void>;
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Aborts when the stream is over, after every message the peer sent has
|
|
24
|
+
* been delivered. The reason is a {@link PeerGoneError} when the stream
|
|
25
|
+
* ended without the peer finishing it, or an `Error` when the peer closed
|
|
26
|
+
* it or sent bytes that are not this framing.
|
|
27
|
+
* Pass it as `multiplexPort`'s / `overPipe`'s `signal`, so a dead peer
|
|
28
|
+
* closes the multiplexer and every port on it at once.
|
|
29
|
+
*/
|
|
30
|
+
signal: AbortSignal;
|
|
31
|
+
}
|
|
32
|
+
/** Options for {@link messageTargetOverStream}. */
|
|
33
|
+
export interface MessageTargetOverStreamOptions {
|
|
34
|
+
/**
|
|
35
|
+
* The largest message accepted, in bytes. A length prefix past it — or one
|
|
36
|
+
* that is not a valid varint — means the peer is not speaking this framing,
|
|
37
|
+
* so the stream is aborted rather than buffered without bound. Default
|
|
38
|
+
* {@link DEFAULT_MAX_MESSAGE_BYTES}.
|
|
39
|
+
*/
|
|
40
|
+
maxMessageBytes?: number;
|
|
41
|
+
}
|
|
42
|
+
/** 64 MiB: room for a large unframed message, short of exhausting memory. */
|
|
43
|
+
export declare const DEFAULT_MAX_MESSAGE_BYTES: number;
|
|
44
|
+
/**
|
|
45
|
+
* One libp2p stream as a pipe of messages, for a multiplexer to run over:
|
|
46
|
+
*
|
|
47
|
+
* ```ts
|
|
48
|
+
* const { target, signal } = messageTargetOverStream(stream);
|
|
49
|
+
* const mux = multiplexPort(target, { codec: msgpackCodec, signal });
|
|
50
|
+
* ```
|
|
51
|
+
*
|
|
52
|
+
* Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
53
|
+
* `message` listener in the same tick: a message with no listener is dropped,
|
|
54
|
+
* as on a `MessagePort`. Every message the peer sent before closing is
|
|
55
|
+
* delivered before `signal` aborts. `postMessage` hands the frame to
|
|
56
|
+
* `stream.send()` and does not wait for room; libp2p buffers up to the
|
|
57
|
+
* stream's `maxWriteBufferLength`, past which it resets the stream. A sender
|
|
58
|
+
* that must wait awaits `whenDrained()`.
|
|
59
|
+
*/
|
|
60
|
+
export declare function messageTargetOverStream(stream: Stream, { maxMessageBytes }?: MessageTargetOverStreamOptions): StreamMessageTarget;
|
|
61
|
+
export {};
|
|
62
|
+
//# sourceMappingURL=message-target-over-stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"message-target-over-stream.d.ts","sourceRoot":"","sources":["../src/message-target-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AASlE,KAAK,QAAQ,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9D,uEAAuE;AACvE,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE;QACN,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC5D,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC/D,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;QACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,2CAA2C;QAC3C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC;;;WAGG;QACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;KAC9B,CAAC;IACF;;;;;;;OAOG;IACH,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,8BAA8B;IAC7C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,6EAA6E;AAC7E,eAAO,MAAM,yBAAyB,QAAmB,CAAC;AAE1D;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,EAAE,eAA2C,EAAE,GAAE,8BAAmC,GACnF,mBAAmB,CAkHrB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/webrun-streams-libp2p",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "libp2p native multi-stream Connect/Serve adapter in the webrun-streams-* family",
|
|
@@ -54,8 +54,11 @@
|
|
|
54
54
|
"libp2p": "^3.0.0",
|
|
55
55
|
"rimraf": "^6.1.3",
|
|
56
56
|
"rolldown": "^1.2.12",
|
|
57
|
+
"tsx": "^4.23.15",
|
|
57
58
|
"typescript": "^7.0.2",
|
|
58
59
|
"vitest": "^5.0.3",
|
|
60
|
+
"@statewalker/webrun-msgpack": "^0.6.0",
|
|
61
|
+
"@statewalker/webrun-rpc": "^0.7.0",
|
|
59
62
|
"@statewalker/webrun-streams-conformance": "^0.3.3"
|
|
60
63
|
},
|
|
61
64
|
"sideEffects": false,
|
package/src/connect-serve.ts
CHANGED
|
@@ -8,7 +8,7 @@ import type {
|
|
|
8
8
|
} from "@libp2p/interface";
|
|
9
9
|
import type { Multiaddr } from "@multiformats/multiaddr";
|
|
10
10
|
import type { Connect, Duplex, Serve } from "@statewalker/webrun-streams";
|
|
11
|
-
import { closeStream, duplexOverStream } from "./duplex-over-stream.js";
|
|
11
|
+
import { closeStream, duplexOverStream, PeerGoneError } from "./duplex-over-stream.js";
|
|
12
12
|
|
|
13
13
|
export const DEFAULT_PROTOCOL = "/webrun-streams/1.0.0";
|
|
14
14
|
|
|
@@ -299,6 +299,9 @@ export async function serveConnections(
|
|
|
299
299
|
});
|
|
300
300
|
|
|
301
301
|
const onStream = (stream: Stream, connection: Connection): void => {
|
|
302
|
+
// The caller sent its whole request; a stream that dies after that is a
|
|
303
|
+
// caller that stopped reading the response — ordinary, not a fault.
|
|
304
|
+
let requestComplete = false;
|
|
302
305
|
void (async () => {
|
|
303
306
|
const handler = makeHandler({ remotePeer: connection.remotePeer });
|
|
304
307
|
const inputQueue = makeInputQueue();
|
|
@@ -306,7 +309,10 @@ export async function serveConnections(
|
|
|
306
309
|
try {
|
|
307
310
|
for await (const chunk of duplexOverStream(stream, output, {
|
|
308
311
|
drainTimeoutMs,
|
|
309
|
-
onPeerInputEnd: (err) =>
|
|
312
|
+
onPeerInputEnd: (err) => {
|
|
313
|
+
if (err === undefined) requestComplete = true;
|
|
314
|
+
inputQueue.done(err);
|
|
315
|
+
},
|
|
310
316
|
})) {
|
|
311
317
|
inputQueue.push(chunk);
|
|
312
318
|
}
|
|
@@ -323,6 +329,7 @@ export async function serveConnections(
|
|
|
323
329
|
// unhandled, and Node's default for `unhandledRejection` is to
|
|
324
330
|
// terminate. Logged rather than rethrown, and never rethrown from here:
|
|
325
331
|
// there is no caller left to receive it.
|
|
332
|
+
if (requestComplete && err instanceof PeerGoneError) return;
|
|
326
333
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
327
334
|
console.warn(
|
|
328
335
|
`[webrun-streams-libp2p] serve: inbound stream on protocol ${proto} failed: ${e.message}`,
|
|
@@ -52,6 +52,24 @@ export const DEFAULT_DRAIN_TIMEOUT_MS = 300_000;
|
|
|
52
52
|
*/
|
|
53
53
|
export const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
54
54
|
|
|
55
|
+
/**
|
|
56
|
+
* The stream ended before the call finished: the peer died, the connection
|
|
57
|
+
* dropped, or the stream was reset or refused before both sides ended it.
|
|
58
|
+
* Distinct from a clean end on purpose — a response cut short must never read
|
|
59
|
+
* as a complete one. `cause` is the stream's close error, when libp2p gave one.
|
|
60
|
+
*/
|
|
61
|
+
export class PeerGoneError extends Error {
|
|
62
|
+
constructor(cause?: unknown) {
|
|
63
|
+
super(
|
|
64
|
+
`webrun-streams-libp2p: the stream ended before the call finished${
|
|
65
|
+
cause instanceof Error ? ` (${cause.message})` : ""
|
|
66
|
+
}`,
|
|
67
|
+
{ cause },
|
|
68
|
+
);
|
|
69
|
+
this.name = "PeerGoneError";
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
55
73
|
const textEncoder = new TextEncoder();
|
|
56
74
|
const textDecoder = new TextDecoder();
|
|
57
75
|
|
|
@@ -116,6 +134,32 @@ export async function* duplexOverStream(
|
|
|
116
134
|
);
|
|
117
135
|
};
|
|
118
136
|
|
|
137
|
+
// A CLEAN END IS THE PEER'S OWN `remoteCloseWrite`, NOTHING ELSE. When the
|
|
138
|
+
// connection dies (a killed process, a dropped network), libp2p closes the
|
|
139
|
+
// stream WITHOUT an error and its iterator ends exactly as it does after the
|
|
140
|
+
// peer's close — so a truncated response read as a complete one. Listened
|
|
141
|
+
// for before the first pull below, like the iterator's own listener. A
|
|
142
|
+
// generator's body runs only on the consumer's first `next()`, so the peer
|
|
143
|
+
// may have closed already; its state is read for that case.
|
|
144
|
+
let peerClosedWrite = stream.remoteWriteStatus === "closed";
|
|
145
|
+
let closeError: unknown;
|
|
146
|
+
let onStreamClosed!: () => void;
|
|
147
|
+
const streamClosed = new Promise<void>((resolve) => {
|
|
148
|
+
onStreamClosed = resolve;
|
|
149
|
+
});
|
|
150
|
+
const onRemoteCloseWrite = (): void => {
|
|
151
|
+
peerClosedWrite = true;
|
|
152
|
+
};
|
|
153
|
+
const onClose = (evt: StreamCloseEvent): void => {
|
|
154
|
+
closeError = evt.error;
|
|
155
|
+
onStreamClosed();
|
|
156
|
+
};
|
|
157
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
158
|
+
stream.addEventListener("close", onClose);
|
|
159
|
+
|
|
160
|
+
// Set once every input chunk went out. A stream that closes before that lost
|
|
161
|
+
// part of this side's input, which is not a complete call either.
|
|
162
|
+
let inputSent = false;
|
|
119
163
|
const outboundSource = framedOutbound(inputIterator);
|
|
120
164
|
const outbound = (async () => {
|
|
121
165
|
try {
|
|
@@ -134,6 +178,7 @@ export async function* duplexOverStream(
|
|
|
134
178
|
await waitForDrain(stream, opts.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS);
|
|
135
179
|
}
|
|
136
180
|
}
|
|
181
|
+
inputSent = true;
|
|
137
182
|
await closeStream(stream);
|
|
138
183
|
} catch (err) {
|
|
139
184
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
@@ -165,14 +210,29 @@ export async function* duplexOverStream(
|
|
|
165
210
|
// subscribe is lost and this loop would never end. Every current path from
|
|
166
211
|
// dialProtocol/onStream to here is microtask-only, so a socket FIN cannot
|
|
167
212
|
// interleave. Inserting an await above would break that.
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
213
|
+
let peerError: Error | undefined;
|
|
214
|
+
try {
|
|
215
|
+
for await (const frame of parseFrames(stream)) {
|
|
216
|
+
if (frame.type === TYPE_DATA) {
|
|
217
|
+
yield frame.payload;
|
|
218
|
+
} else if (frame.type === TYPE_ERROR) {
|
|
219
|
+
peerError = decodeError(frame.payload);
|
|
220
|
+
firePeerInputEnd(peerError);
|
|
221
|
+
throw peerError;
|
|
222
|
+
}
|
|
175
223
|
}
|
|
224
|
+
} catch (err) {
|
|
225
|
+
if (err === peerError) throw err;
|
|
226
|
+
// The stream itself failed (reset, aborted, its connection dropped):
|
|
227
|
+
// the call did not finish, whatever libp2p named the error.
|
|
228
|
+
const gone = new PeerGoneError(err);
|
|
229
|
+
firePeerInputEnd(gone);
|
|
230
|
+
throw gone;
|
|
231
|
+
}
|
|
232
|
+
if (!peerClosedWrite) {
|
|
233
|
+
const gone = new PeerGoneError(closeError);
|
|
234
|
+
firePeerInputEnd(gone);
|
|
235
|
+
throw gone;
|
|
176
236
|
}
|
|
177
237
|
sourceCompleted = true;
|
|
178
238
|
firePeerInputEnd();
|
|
@@ -214,9 +274,23 @@ export async function* duplexOverStream(
|
|
|
214
274
|
}
|
|
215
275
|
void outbound.catch(() => undefined);
|
|
216
276
|
} else {
|
|
217
|
-
|
|
277
|
+
// The peer ended its side; ours may still be sending, waiting on an
|
|
278
|
+
// input that is still open (a tunnel). If the stream dies first, stop
|
|
279
|
+
// waiting for that input: it can never be sent now.
|
|
280
|
+
await Promise.race([outbound, streamClosed]);
|
|
281
|
+
if (!inputSent) {
|
|
282
|
+
cancelInput();
|
|
283
|
+
void Promise.resolve(outboundSource.return?.(undefined)).catch(() => undefined);
|
|
284
|
+
void outbound.catch(() => undefined);
|
|
285
|
+
} else {
|
|
286
|
+
await outbound;
|
|
287
|
+
}
|
|
218
288
|
}
|
|
289
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
290
|
+
stream.removeEventListener("close", onClose);
|
|
219
291
|
}
|
|
292
|
+
// Reached only after a clean end from the peer.
|
|
293
|
+
if (!inputSent) throw new PeerGoneError(closeError);
|
|
220
294
|
}
|
|
221
295
|
|
|
222
296
|
/**
|
|
@@ -373,7 +447,7 @@ async function* parseFrames(
|
|
|
373
447
|
}
|
|
374
448
|
}
|
|
375
449
|
|
|
376
|
-
function normalizeChunk(item: unknown): Uint8Array {
|
|
450
|
+
export function normalizeChunk(item: unknown): Uint8Array {
|
|
377
451
|
if (item instanceof Uint8Array) return item;
|
|
378
452
|
const asList = item as { subarray?: () => Uint8Array };
|
|
379
453
|
if (typeof asList.subarray === "function") {
|
|
@@ -410,7 +484,7 @@ function decodeError(payload: Uint8Array): Error {
|
|
|
410
484
|
}
|
|
411
485
|
}
|
|
412
486
|
|
|
413
|
-
function encodeVarint(value: number): Uint8Array {
|
|
487
|
+
export function encodeVarint(value: number): Uint8Array {
|
|
414
488
|
if (!Number.isInteger(value) || value < 0) {
|
|
415
489
|
throw new RangeError(`encodeVarint: ${value} is not a non-negative integer`);
|
|
416
490
|
}
|
|
@@ -424,7 +498,7 @@ function encodeVarint(value: number): Uint8Array {
|
|
|
424
498
|
return new Uint8Array(out);
|
|
425
499
|
}
|
|
426
500
|
|
|
427
|
-
function decodeVarint(buf: Uint8Array, start: number): { value: number; offset: number } {
|
|
501
|
+
export function decodeVarint(buf: Uint8Array, start: number): { value: number; offset: number } {
|
|
428
502
|
let value = 0;
|
|
429
503
|
let shift = 0;
|
|
430
504
|
let i = start;
|
package/src/index.ts
CHANGED
|
@@ -13,4 +13,11 @@ export {
|
|
|
13
13
|
DEFAULT_DRAIN_TIMEOUT_MS,
|
|
14
14
|
type DuplexOverStreamOptions,
|
|
15
15
|
duplexOverStream,
|
|
16
|
+
PeerGoneError,
|
|
16
17
|
} from "./duplex-over-stream.js";
|
|
18
|
+
export {
|
|
19
|
+
DEFAULT_MAX_MESSAGE_BYTES,
|
|
20
|
+
type MessageTargetOverStreamOptions,
|
|
21
|
+
messageTargetOverStream,
|
|
22
|
+
type StreamMessageTarget,
|
|
23
|
+
} from "./message-target-over-stream.js";
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import type { Stream, StreamCloseEvent } from "@libp2p/interface";
|
|
2
|
+
import {
|
|
3
|
+
closeStream,
|
|
4
|
+
decodeVarint,
|
|
5
|
+
encodeVarint,
|
|
6
|
+
normalizeChunk,
|
|
7
|
+
PeerGoneError,
|
|
8
|
+
} from "./duplex-over-stream.js";
|
|
9
|
+
|
|
10
|
+
type Listener = (event: MessageEvent) => void | Promise<void>;
|
|
11
|
+
|
|
12
|
+
/** What `messageTargetOverStream` gives: a byte pipe, and its life. */
|
|
13
|
+
export interface StreamMessageTarget {
|
|
14
|
+
/**
|
|
15
|
+
* A `MessageTarget` (as `@statewalker/webrun-rpc` defines it) whose messages
|
|
16
|
+
* are `Uint8Array`s, one length-prefixed frame each.
|
|
17
|
+
*/
|
|
18
|
+
target: {
|
|
19
|
+
addEventListener(type: "message", listener: Listener): void;
|
|
20
|
+
removeEventListener(type: "message", listener: Listener): void;
|
|
21
|
+
postMessage(message: unknown): void;
|
|
22
|
+
close(): Promise<void>;
|
|
23
|
+
/** The stream's write buffer, in bytes. */
|
|
24
|
+
readonly bufferedAmount: number;
|
|
25
|
+
/**
|
|
26
|
+
* Resolves once the stream can take more (libp2p's `drain`), or has
|
|
27
|
+
* closed; at once when it can now.
|
|
28
|
+
*/
|
|
29
|
+
whenDrained(): Promise<void>;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Aborts when the stream is over, after every message the peer sent has
|
|
33
|
+
* been delivered. The reason is a {@link PeerGoneError} when the stream
|
|
34
|
+
* ended without the peer finishing it, or an `Error` when the peer closed
|
|
35
|
+
* it or sent bytes that are not this framing.
|
|
36
|
+
* Pass it as `multiplexPort`'s / `overPipe`'s `signal`, so a dead peer
|
|
37
|
+
* closes the multiplexer and every port on it at once.
|
|
38
|
+
*/
|
|
39
|
+
signal: AbortSignal;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Options for {@link messageTargetOverStream}. */
|
|
43
|
+
export interface MessageTargetOverStreamOptions {
|
|
44
|
+
/**
|
|
45
|
+
* The largest message accepted, in bytes. A length prefix past it — or one
|
|
46
|
+
* that is not a valid varint — means the peer is not speaking this framing,
|
|
47
|
+
* so the stream is aborted rather than buffered without bound. Default
|
|
48
|
+
* {@link DEFAULT_MAX_MESSAGE_BYTES}.
|
|
49
|
+
*/
|
|
50
|
+
maxMessageBytes?: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** 64 MiB: room for a large unframed message, short of exhausting memory. */
|
|
54
|
+
export const DEFAULT_MAX_MESSAGE_BYTES = 64 * 1024 * 1024;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* One libp2p stream as a pipe of messages, for a multiplexer to run over:
|
|
58
|
+
*
|
|
59
|
+
* ```ts
|
|
60
|
+
* const { target, signal } = messageTargetOverStream(stream);
|
|
61
|
+
* const mux = multiplexPort(target, { codec: msgpackCodec, signal });
|
|
62
|
+
* ```
|
|
63
|
+
*
|
|
64
|
+
* Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
65
|
+
* `message` listener in the same tick: a message with no listener is dropped,
|
|
66
|
+
* as on a `MessagePort`. Every message the peer sent before closing is
|
|
67
|
+
* delivered before `signal` aborts. `postMessage` hands the frame to
|
|
68
|
+
* `stream.send()` and does not wait for room; libp2p buffers up to the
|
|
69
|
+
* stream's `maxWriteBufferLength`, past which it resets the stream. A sender
|
|
70
|
+
* that must wait awaits `whenDrained()`.
|
|
71
|
+
*/
|
|
72
|
+
export function messageTargetOverStream(
|
|
73
|
+
stream: Stream,
|
|
74
|
+
{ maxMessageBytes = DEFAULT_MAX_MESSAGE_BYTES }: MessageTargetOverStreamOptions = {},
|
|
75
|
+
): StreamMessageTarget {
|
|
76
|
+
const listeners = new Set<Listener>();
|
|
77
|
+
const life = new AbortController();
|
|
78
|
+
let peerClosedWrite = false;
|
|
79
|
+
|
|
80
|
+
const onRemoteCloseWrite = (): void => {
|
|
81
|
+
// The read loop may still hold the peer's last messages; it aborts
|
|
82
|
+
// `signal` once it has delivered them.
|
|
83
|
+
peerClosedWrite = true;
|
|
84
|
+
};
|
|
85
|
+
const onClose = (evt: StreamCloseEvent): void => {
|
|
86
|
+
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
87
|
+
if (peerClosedWrite && evt.error == null) return; // the read loop ends it
|
|
88
|
+
life.abort(new PeerGoneError(evt.error));
|
|
89
|
+
};
|
|
90
|
+
stream.addEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
91
|
+
stream.addEventListener("close", onClose, { once: true });
|
|
92
|
+
|
|
93
|
+
const deliver = (data: Uint8Array): void => {
|
|
94
|
+
const event = new MessageEvent("message", { data });
|
|
95
|
+
for (const listener of [...listeners]) {
|
|
96
|
+
try {
|
|
97
|
+
// One listener's fault — thrown or rejected — must not stop the read loop.
|
|
98
|
+
Promise.resolve(listener(event)).catch(() => undefined);
|
|
99
|
+
} catch {}
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
void (async () => {
|
|
104
|
+
// Unread bytes are `buf[start, end)`; the buffer grows by doubling, so a
|
|
105
|
+
// large message arriving in many chunks is copied a bounded number of times.
|
|
106
|
+
let buf = new Uint8Array(0);
|
|
107
|
+
let start = 0;
|
|
108
|
+
let end = 0;
|
|
109
|
+
for await (const item of stream) {
|
|
110
|
+
const incoming = normalizeChunk(item);
|
|
111
|
+
if (end + incoming.byteLength > buf.byteLength) {
|
|
112
|
+
const size = end - start + incoming.byteLength;
|
|
113
|
+
const next = new Uint8Array(Math.max(size, buf.byteLength * 2));
|
|
114
|
+
next.set(buf.subarray(start, end), 0);
|
|
115
|
+
buf = next;
|
|
116
|
+
end -= start;
|
|
117
|
+
start = 0;
|
|
118
|
+
}
|
|
119
|
+
buf.set(incoming, end);
|
|
120
|
+
end += incoming.byteLength;
|
|
121
|
+
for (;;) {
|
|
122
|
+
const header = readLength(buf.subarray(start, end));
|
|
123
|
+
if (header === undefined) break; // the length is not all here yet
|
|
124
|
+
if (header.value > maxMessageBytes) {
|
|
125
|
+
throw new Error(
|
|
126
|
+
`webrun-streams-libp2p: a ${header.value}-byte message exceeds maxMessageBytes (${maxMessageBytes})`,
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
const stop = start + header.offset + header.value;
|
|
130
|
+
if (end < stop) break;
|
|
131
|
+
deliver(buf.slice(start + header.offset, stop));
|
|
132
|
+
start = stop;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (peerClosedWrite) {
|
|
136
|
+
life.abort(new Error("webrun-streams-libp2p: the peer closed the stream"));
|
|
137
|
+
await closeStream(stream);
|
|
138
|
+
}
|
|
139
|
+
})().catch((err: unknown) => {
|
|
140
|
+
// A reset ends the loop with an error, which `close` reports through
|
|
141
|
+
// `signal`; a malformed frame is reported here.
|
|
142
|
+
life.abort(err);
|
|
143
|
+
try {
|
|
144
|
+
stream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
145
|
+
} catch {}
|
|
146
|
+
});
|
|
147
|
+
return {
|
|
148
|
+
signal: life.signal,
|
|
149
|
+
target: {
|
|
150
|
+
addEventListener: (_type, listener) => {
|
|
151
|
+
listeners.add(listener);
|
|
152
|
+
},
|
|
153
|
+
removeEventListener: (_type, listener) => {
|
|
154
|
+
listeners.delete(listener);
|
|
155
|
+
},
|
|
156
|
+
postMessage(message) {
|
|
157
|
+
const payload = normalizeChunk(message);
|
|
158
|
+
const header = encodeVarint(payload.byteLength);
|
|
159
|
+
const frame = new Uint8Array(header.byteLength + payload.byteLength);
|
|
160
|
+
frame.set(header, 0);
|
|
161
|
+
frame.set(payload, header.byteLength);
|
|
162
|
+
try {
|
|
163
|
+
stream.send(frame);
|
|
164
|
+
} catch {
|
|
165
|
+
// The stream is closed for writing; `signal` says so.
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
async close() {
|
|
169
|
+
listeners.clear();
|
|
170
|
+
await closeStream(stream);
|
|
171
|
+
},
|
|
172
|
+
get bufferedAmount() {
|
|
173
|
+
return stream.writeBufferLength;
|
|
174
|
+
},
|
|
175
|
+
whenDrained() {
|
|
176
|
+
if (!stream.writableNeedsDrain || life.signal.aborted) return Promise.resolve();
|
|
177
|
+
return new Promise<void>((resolve) => {
|
|
178
|
+
const done = (): void => {
|
|
179
|
+
stream.removeEventListener("drain", done);
|
|
180
|
+
stream.removeEventListener("close", done);
|
|
181
|
+
resolve();
|
|
182
|
+
};
|
|
183
|
+
stream.addEventListener("drain", done);
|
|
184
|
+
stream.addEventListener("close", done);
|
|
185
|
+
});
|
|
186
|
+
},
|
|
187
|
+
},
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** The length prefix of `bytes`, or `undefined` while it is incomplete; throws when malformed. */
|
|
192
|
+
function readLength(bytes: Uint8Array): { value: number; offset: number } | undefined {
|
|
193
|
+
try {
|
|
194
|
+
return decodeVarint(bytes, 0);
|
|
195
|
+
} catch (err) {
|
|
196
|
+
if (bytes.byteLength < 5) return undefined;
|
|
197
|
+
throw new Error("webrun-streams-libp2p: a malformed message length", { cause: err });
|
|
198
|
+
}
|
|
199
|
+
}
|