@statewalker/webrun-streams-libp2p 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/dist/connect-serve.d.ts +13 -1
- package/dist/connect-serve.d.ts.map +1 -1
- package/dist/duplex-over-stream.d.ts +45 -0
- package/dist/duplex-over-stream.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +99 -55
- package/dist/message-target-over-stream.d.ts +1 -0
- package/dist/message-target-over-stream.d.ts.map +1 -1
- package/package.json +7 -5
- package/src/connect-serve.ts +16 -0
- package/src/duplex-over-stream.ts +104 -24
- package/src/index.ts +1 -0
- package/src/message-target-over-stream.ts +27 -30
package/README.md
CHANGED
|
@@ -63,6 +63,7 @@ runs (Node or browser).
|
|
|
63
63
|
| `DEFAULT_DRAIN_TIMEOUT_MS` | `300_000` (5 minutes). |
|
|
64
64
|
| `DEFAULT_CLOSE_TIMEOUT_MS` | `300_000` (5 minutes), the bound on a graceful close. |
|
|
65
65
|
| `DEFAULT_MAX_MESSAGE_BYTES` | `67_108_864` (64 MiB), `messageTargetOverStream`'s cap on one message. |
|
|
66
|
+
| `DEFAULT_MAX_FRAME_BYTES` | `67_108_864` (64 MiB), `duplexOverStream`'s default cap on one inbound frame's payload (`maxFrameBytes`). |
|
|
66
67
|
| `ConnectLibp2pParams`, `ServeLibp2pParams`, `ConnectionContext`, `ServeConnectionsHandler`, `DuplexOverStreamOptions` | Types, below. |
|
|
67
68
|
|
|
68
69
|
`ConnectLibp2pParams`:
|
|
@@ -73,10 +74,11 @@ runs (Node or browser).
|
|
|
73
74
|
| `peer` | `PeerId \| Multiaddr` | required | Who to dial. |
|
|
74
75
|
| `protocol` | `string` | `DEFAULT_PROTOCOL` | Protocol id. |
|
|
75
76
|
| `drainTimeoutMs` | `number` | `300_000` | How long a backpressured send waits for `'drain'`. |
|
|
77
|
+
| `maxFrameBytes` | `number` | `DEFAULT_MAX_FRAME_BYTES` | The largest frame payload accepted from the peer. Past it, or on a malformed frame length, the call fails and the stream is aborted. |
|
|
76
78
|
| `maxOutboundStreams` | `number` | libp2p's (64) | Passed to `dialProtocol`. |
|
|
77
79
|
| `runOnLimitedConnection` | `boolean` | unset | Passed to `dialProtocol`. Allows relayed or otherwise limited connections. |
|
|
78
80
|
|
|
79
|
-
`ServeLibp2pParams`: `node`, `protocol`, `drainTimeoutMs`,
|
|
81
|
+
`ServeLibp2pParams`: `node`, `protocol`, `drainTimeoutMs`, `maxFrameBytes`,
|
|
80
82
|
`maxOutboundStreams` and `runOnLimitedConnection` as above (passed to
|
|
81
83
|
`node.handle`), plus `maxInboundStreams` (libp2p default 32).
|
|
82
84
|
|
|
@@ -90,6 +92,7 @@ runs (Node or browser).
|
|
|
90
92
|
| `onPeerInputEnd?(err?)` | Called once when the peer's side ends: closed, `ERROR` frame (passed as `err`) or torn down. `serveConnections` uses it to end the handler's input. |
|
|
91
93
|
| `onSourceCompleted?()` | Called only when the peer's side ended normally. `connect` uses it to choose a graceful close over an abort. |
|
|
92
94
|
| `drainTimeoutMs?` | As above. |
|
|
95
|
+
| `maxFrameBytes?` | As above. |
|
|
93
96
|
|
|
94
97
|
## Examples
|
|
95
98
|
|
|
@@ -175,6 +178,18 @@ from whatever chunks arrive. Normal end of input is libp2p's own
|
|
|
175
178
|
reset carries only "stream reset"; without it, a handler that throws would
|
|
176
179
|
reach the caller as an anonymous reset. Frames of other types are skipped.
|
|
177
180
|
|
|
181
|
+
A producer chunk larger than 64 KiB is sent as several `DATA` frames of at most
|
|
182
|
+
64 KiB each. Frames are self-delimiting and a receiver yields each `DATA`
|
|
183
|
+
payload as it completes, so a receiver of any version reassembles the same
|
|
184
|
+
bytes; only the chunk boundaries the consumer sees change. The receiver keeps
|
|
185
|
+
unread bytes in a buffer that grows by doubling, so a large frame arriving in
|
|
186
|
+
many pieces is copied a bounded number of times. A frame length past
|
|
187
|
+
`maxFrameBytes` (default 64 MiB), or one that is not a valid varint, fails the
|
|
188
|
+
call with `webrun-streams-libp2p: a <n>-byte frame exceeds maxFrameBytes (<cap>)`
|
|
189
|
+
or `webrun-streams-libp2p: a malformed frame length`, and aborts the stream:
|
|
190
|
+
a peer not speaking this framing would otherwise make the receiver buffer
|
|
191
|
+
without bound. On the serving side the handler's input throws that error.
|
|
192
|
+
|
|
178
193
|
### Why identity reaches the handler by closure
|
|
179
194
|
|
|
180
195
|
`Duplex` is bytes-only, so `serveConnections` passes identity to a factory
|
|
@@ -357,7 +372,9 @@ so this also covers a peer that dies while no data is flowing.
|
|
|
357
372
|
|
|
358
373
|
Frames are `[varint length][bytes]`. Reading starts at once, so attach the
|
|
359
374
|
`message` listener in the same tick (`multiplexPort` does). `postMessage`
|
|
360
|
-
|
|
375
|
+
takes a `Uint8Array`, an `ArrayBuffer` or another view, and throws `TypeError`
|
|
376
|
+
for anything else, as the other adapters' byte pipes do. It does not wait for
|
|
377
|
+
room, and libp2p resets a stream whose write buffer passes
|
|
361
378
|
`maxWriteBufferLength`. The target reports its state instead:
|
|
362
379
|
`bufferedAmount` is the stream's write buffer, and `whenDrained()` resolves
|
|
363
380
|
on libp2p's `drain` (or the stream's `close`, so no sender is left waiting),
|
|
@@ -386,8 +403,8 @@ identity, burst, stream-limit and serving-resilience tests against two real
|
|
|
386
403
|
in-process TCP nodes. `dead-peer.test.ts` kills the peer two ways, aborting
|
|
387
404
|
its connection in-process and SIGKILLing a forked server
|
|
388
405
|
(`tests/fixtures/greeting-server.ts`, run with `tsx`), with 100 open channels
|
|
389
|
-
each; every channel must fail with an error within 1 s. The shared conformance
|
|
390
|
-
|
|
406
|
+
each; every channel must fail with an error within 1 s. The shared conformance
|
|
407
|
+
suite (`tests/conformance.test.ts`, about 7 s) runs in the same `pnpm test`, with
|
|
391
408
|
`skipHugeBody`.
|
|
392
409
|
|
|
393
410
|
**Run-ahead is bounded for `connect` / `serve`** (spec D17, L6;
|
package/dist/connect-serve.d.ts
CHANGED
|
@@ -17,6 +17,12 @@ export interface ConnectLibp2pParams {
|
|
|
17
17
|
* paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
|
|
18
18
|
*/
|
|
19
19
|
readHighWaterBytes?: number;
|
|
20
|
+
/**
|
|
21
|
+
* The largest frame payload accepted from the peer; past it, or on a
|
|
22
|
+
* malformed frame header, the call fails and the stream is aborted. See
|
|
23
|
+
* `DuplexOverStreamOptions.maxFrameBytes` (default 64 MiB).
|
|
24
|
+
*/
|
|
25
|
+
maxFrameBytes?: number;
|
|
20
26
|
/**
|
|
21
27
|
* How many outgoing streams for this protocol libp2p allows to be open at
|
|
22
28
|
* the same time on one connection; passed through to `node.dialProtocol`.
|
|
@@ -48,6 +54,12 @@ export interface ServeLibp2pParams {
|
|
|
48
54
|
* paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
|
|
49
55
|
*/
|
|
50
56
|
readHighWaterBytes?: number;
|
|
57
|
+
/**
|
|
58
|
+
* The largest frame payload accepted from the peer; past it, or on a
|
|
59
|
+
* malformed frame header, the call fails and the stream is aborted. See
|
|
60
|
+
* `DuplexOverStreamOptions.maxFrameBytes` (default 64 MiB).
|
|
61
|
+
*/
|
|
62
|
+
maxFrameBytes?: number;
|
|
51
63
|
/**
|
|
52
64
|
* How many incoming streams for this protocol libp2p allows to be open at
|
|
53
65
|
* the same time on one connection; passed through to `node.handle`. Left
|
|
@@ -102,5 +114,5 @@ export type ServeConnectionsHandler = (context: ConnectionContext) => Duplex;
|
|
|
102
114
|
* Like `serve`, but the handler is built per inbound stream and is told which
|
|
103
115
|
* peer libp2p proved on that connection.
|
|
104
116
|
*/
|
|
105
|
-
export declare function serveConnections({ node, protocol, drainTimeoutMs, readHighWaterBytes, maxInboundStreams, maxOutboundStreams, runOnLimitedConnection, }: ServeLibp2pParams, makeHandler: ServeConnectionsHandler): Promise<() => Promise<void>>;
|
|
117
|
+
export declare function serveConnections({ node, protocol, drainTimeoutMs, readHighWaterBytes, maxFrameBytes, maxInboundStreams, maxOutboundStreams, runOnLimitedConnection, }: ServeLibp2pParams, makeHandler: ServeConnectionsHandler): Promise<() => Promise<void>>;
|
|
106
118
|
//# sourceMappingURL=connect-serve.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connect-serve.d.ts","sourceRoot":"","sources":["../src/connect-serve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,MAAM,EACN,MAAM,EAGP,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AAS1E,eAAO,MAAM,gBAAgB,0BAA0B,CAAC;AAExD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAqBD;;;GAGG;AACH,eAAO,MAAM,OAAO,EAAE,OAAO,CAAC,mBAAmB,
|
|
1
|
+
{"version":3,"file":"connect-serve.d.ts","sourceRoot":"","sources":["../src/connect-serve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,MAAM,EACN,MAAM,EAGP,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AAS1E,eAAO,MAAM,gBAAgB,0BAA0B,CAAC;AAExD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAqBD;;;GAGG;AACH,eAAO,MAAM,OAAO,EAAE,OAAO,CAAC,mBAAmB,CA8FhD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,KAAK,EAAE,KAAK,CAAC,iBAAiB,CACF,CAAC;AAE1C,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,iBAAiB,KAAK,MAAM,CAAC;AAE7E;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,EACE,IAAI,EACJ,QAAQ,EACR,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,GACvB,EAAE,iBAAiB,EACpB,WAAW,EAAE,uBAAuB,GACnC,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,CAgE9B"}
|
|
@@ -50,6 +50,17 @@ export declare const DEFAULT_CLOSE_TIMEOUT_MS = 300000;
|
|
|
50
50
|
* held for a consumer that is not reading before the stream is paused.
|
|
51
51
|
*/
|
|
52
52
|
export declare const DEFAULT_READ_HIGH_WATER_BYTES: number;
|
|
53
|
+
/**
|
|
54
|
+
* Default for {@link DuplexOverStreamOptions.maxFrameBytes}: the largest frame payload accepted
|
|
55
|
+
* from the peer. Generous, because a sender before this package's outbound chunking sent each
|
|
56
|
+
* producer chunk as one frame; short of letting a peer declare a 4 GiB frame.
|
|
57
|
+
*/
|
|
58
|
+
export declare const DEFAULT_MAX_FRAME_BYTES: number;
|
|
59
|
+
/**
|
|
60
|
+
* The largest payload this side puts in one outbound frame: a larger producer chunk crosses as
|
|
61
|
+
* several frames (they are self-delimiting, so a receiver of any version reassembles the bytes).
|
|
62
|
+
*/
|
|
63
|
+
export declare const OUTBOUND_FRAME_BYTES: number;
|
|
53
64
|
/**
|
|
54
65
|
* The stream ended before the call finished: the peer died, the connection
|
|
55
66
|
* dropped, or the stream was reset or refused before both sides ended it.
|
|
@@ -96,6 +107,13 @@ export interface DuplexOverStreamOptions {
|
|
|
96
107
|
* Defaults to {@link DEFAULT_READ_HIGH_WATER_BYTES}.
|
|
97
108
|
*/
|
|
98
109
|
readHighWaterBytes?: number;
|
|
110
|
+
/**
|
|
111
|
+
* The largest frame payload accepted from the peer, in bytes. A frame header past it — or one
|
|
112
|
+
* whose length is not a valid varint — means the peer is not speaking this framing: the call
|
|
113
|
+
* fails and the stream is aborted, rather than buffering without bound. Defaults to
|
|
114
|
+
* {@link DEFAULT_MAX_FRAME_BYTES}.
|
|
115
|
+
*/
|
|
116
|
+
maxFrameBytes?: number;
|
|
99
117
|
}
|
|
100
118
|
/**
|
|
101
119
|
* Drive one `Duplex` over one libp2p `Stream` using a small in-band framing
|
|
@@ -120,6 +138,33 @@ export declare function duplexOverStream(stream: Stream, input: AsyncIterable<Ui
|
|
|
120
138
|
* caller is never left hanging.
|
|
121
139
|
*/
|
|
122
140
|
export declare function closeStream(stream: Stream, timeoutMs?: number): Promise<void>;
|
|
141
|
+
/**
|
|
142
|
+
* Frames out of the stream's chunks, as they complete. Unread bytes sit in a {@link ByteBuffer},
|
|
143
|
+
* which grows by doubling, so a large frame arriving in many chunks is copied a bounded number
|
|
144
|
+
* of times. A header whose length is malformed or past `maxFrameBytes` throws a `FramingError`.
|
|
145
|
+
*/
|
|
146
|
+
export declare function parseFrames(source: AsyncIterable<unknown>, maxFrameBytes?: number): AsyncGenerator<{
|
|
147
|
+
type: number;
|
|
148
|
+
payload: Uint8Array;
|
|
149
|
+
}>;
|
|
150
|
+
/** Unread bytes `buf[start, end)` of a buffer that grows by doubling. */
|
|
151
|
+
export declare class ByteBuffer {
|
|
152
|
+
private buf;
|
|
153
|
+
private start;
|
|
154
|
+
private end;
|
|
155
|
+
push(incoming: Uint8Array): void;
|
|
156
|
+
/** a view of the unread bytes, valid until the next `push` */
|
|
157
|
+
unread(): Uint8Array;
|
|
158
|
+
consume(n: number): void;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* The varint length prefix at the start of `bytes`, or `undefined` while it is incomplete; throws
|
|
162
|
+
* a `FramingError` when it is malformed (no varint of this framing runs past 5 bytes).
|
|
163
|
+
*/
|
|
164
|
+
export declare function readLength(bytes: Uint8Array, what: string): {
|
|
165
|
+
value: number;
|
|
166
|
+
offset: number;
|
|
167
|
+
} | undefined;
|
|
123
168
|
export declare function normalizeChunk(item: unknown): Uint8Array;
|
|
124
169
|
export declare function encodeVarint(value: number): Uint8Array;
|
|
125
170
|
export declare function decodeVarint(buf: Uint8Array, start: number): {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duplex-over-stream.d.ts","sourceRoot":"","sources":["../src/duplex-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AAMlE;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,wBAAwB,SAAU,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,wBAAwB,SAA2B,CAAC;AAEjE;;;GAGG;AACH,eAAO,MAAM,6BAA6B,QAAY,CAAC;AAEvD;;;;;GAKG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,YAAY,KAAK,CAAC,EAAE,OAAO,EAQ1B;CACF;AAKD;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;OAIG;IACH,cAAc,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,IAAI,IAAI,CAAC;IAC3B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"duplex-over-stream.d.ts","sourceRoot":"","sources":["../src/duplex-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AAMlE;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,wBAAwB,SAAU,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,wBAAwB,SAA2B,CAAC;AAEjE;;;GAGG;AACH,eAAO,MAAM,6BAA6B,QAAY,CAAC;AAEvD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AAExD;;;GAGG;AACH,eAAO,MAAM,oBAAoB,QAAY,CAAC;AAK9C;;;;;GAKG;AACH,qBAAa,aAAc,SAAQ,KAAK;IACtC,YAAY,KAAK,CAAC,EAAE,OAAO,EAQ1B;CACF;AAKD;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;OAIG;IACH,cAAc,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,IAAI,IAAI,CAAC;IAC3B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,wBAAuB,gBAAgB,CACrC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,EACvD,IAAI,GAAE,uBAA4B,GACjC,cAAc,CAAC,UAAU,CAAC,CAwL5B;AA2CD;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,MAAiC,GAC3C,OAAO,CAAC,IAAI,CAAC,CAsBf;AAkKD;;;;GAIG;AACH,wBAAuB,WAAW,CAChC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAC9B,aAAa,GAAE,MAAgC,GAC9C,cAAc,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,UAAU,CAAA;CAAE,CAAC,CAyBvD;AAED,yEAAyE;AACzE,qBAAa,UAAU;IACrB,OAAO,CAAC,GAAG,CAAqB;IAChC,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,GAAG,CAAK;IAEhB,IAAI,CAAC,QAAQ,EAAE,UAAU,GAAG,IAAI,CAW/B;IAED,8DAA8D;IAC9D,MAAM,IAAI,UAAU,CAEnB;IAED,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAGvB;CACF;AAED;;;GAGG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,MAAM,GACX;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAO/C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,UAAU,CAOxD;AA8BD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,CAYtD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAa9F"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { type ConnectionContext, type ConnectLibp2pParams, connect, DEFAULT_PROTOCOL, type ServeConnectionsHandler, type ServeLibp2pParams, serve, serveConnections, } from "./connect-serve.js";
|
|
2
|
-
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, type DuplexOverStreamOptions, duplexOverStream, PeerGoneError, } from "./duplex-over-stream.js";
|
|
2
|
+
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_MAX_FRAME_BYTES, type DuplexOverStreamOptions, duplexOverStream, PeerGoneError, } from "./duplex-over-stream.js";
|
|
3
3
|
export { DEFAULT_MAX_MESSAGE_BYTES, type MessageTargetOverStreamOptions, messageTargetOverStream, type StreamMessageTarget, } from "./message-target-over-stream.js";
|
|
4
4
|
export { DEFAULT_PORT_MAX_MESSAGE_SIZE, DEFAULT_PORT_PROTOCOL, DEFAULT_PORT_STREAM_LIMIT, type InboundPortInfo, type Libp2pPortMux, type Libp2pPortMuxOptions, libp2pPortMux, type StreamPort, type ValueCodec, } from "./port-mux.js";
|
|
5
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,OAAO,EACP,gBAAgB,EAChB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,EACL,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,KAAK,uBAAuB,EAC5B,gBAAgB,EAChB,aAAa,GACd,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,yBAAyB,EACzB,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,KAAK,mBAAmB,GACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,6BAA6B,EAC7B,qBAAqB,EACrB,yBAAyB,EACzB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,aAAa,EACb,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,OAAO,EACP,gBAAgB,EAChB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,EACL,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,uBAAuB,EACvB,KAAK,uBAAuB,EAC5B,gBAAgB,EAChB,aAAa,GACd,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,yBAAyB,EACzB,KAAK,8BAA8B,EACnC,uBAAuB,EACvB,KAAK,mBAAmB,GACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,6BAA6B,EAC7B,qBAAqB,EACrB,yBAAyB,EACzB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,aAAa,EACb,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,eAAe,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -49,6 +49,19 @@ const DEFAULT_DRAIN_TIMEOUT_MS = 3e5;
|
|
|
49
49
|
*/
|
|
50
50
|
const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
51
51
|
/**
|
|
52
|
+
* Default for {@link DuplexOverStreamOptions.maxFrameBytes}: the largest frame payload accepted
|
|
53
|
+
* from the peer. Generous, because a sender before this package's outbound chunking sent each
|
|
54
|
+
* producer chunk as one frame; short of letting a peer declare a 4 GiB frame.
|
|
55
|
+
*/
|
|
56
|
+
const DEFAULT_MAX_FRAME_BYTES = 67108864;
|
|
57
|
+
/**
|
|
58
|
+
* The largest payload this side puts in one outbound frame: a larger producer chunk crosses as
|
|
59
|
+
* several frames (they are self-delimiting, so a receiver of any version reassembles the bytes).
|
|
60
|
+
*/
|
|
61
|
+
const OUTBOUND_FRAME_BYTES = 65536;
|
|
62
|
+
/** The peer sent bytes that are not this framing: a malformed or oversized frame header. */
|
|
63
|
+
var FramingError = class extends Error {};
|
|
64
|
+
/**
|
|
52
65
|
* The stream ended before the call finished: the peer died, the connection
|
|
53
66
|
* dropped, or the stream was reset or refused before both sides ended it.
|
|
54
67
|
* Distinct from a clean end on purpose — a response cut short must never read
|
|
@@ -121,7 +134,7 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
121
134
|
try {
|
|
122
135
|
let peerError;
|
|
123
136
|
try {
|
|
124
|
-
for await (const frame of parseFrames(pacedSource(stream, opts.readHighWaterBytes ?? 65536))) if (frame.type === TYPE_DATA) yield frame.payload;
|
|
137
|
+
for await (const frame of parseFrames(pacedSource(stream, opts.readHighWaterBytes ?? 65536), opts.maxFrameBytes ?? 67108864)) if (frame.type === TYPE_DATA) yield frame.payload;
|
|
125
138
|
else if (frame.type === TYPE_ERROR) {
|
|
126
139
|
peerError = decodeError(frame.payload);
|
|
127
140
|
firePeerInputEnd(peerError);
|
|
@@ -129,6 +142,10 @@ async function* duplexOverStream(stream, input, opts = {}) {
|
|
|
129
142
|
}
|
|
130
143
|
} catch (err) {
|
|
131
144
|
if (err === peerError) throw err;
|
|
145
|
+
if (err instanceof FramingError) {
|
|
146
|
+
firePeerInputEnd(err);
|
|
147
|
+
throw err;
|
|
148
|
+
}
|
|
132
149
|
const gone = new PeerGoneError(err);
|
|
133
150
|
firePeerInputEnd(gone);
|
|
134
151
|
throw gone;
|
|
@@ -237,7 +254,8 @@ async function* framedOutbound(iterator) {
|
|
|
237
254
|
for (;;) {
|
|
238
255
|
const next = await iterator.next();
|
|
239
256
|
if (next.done === true) return;
|
|
240
|
-
|
|
257
|
+
const chunk = normalizeChunk(next.value);
|
|
258
|
+
for (let at = 0; at < chunk.byteLength; at += OUTBOUND_FRAME_BYTES) yield frameData(chunk.subarray(at, Math.min(at + OUTBOUND_FRAME_BYTES, chunk.byteLength)));
|
|
241
259
|
}
|
|
242
260
|
} catch (err) {
|
|
243
261
|
yield frameError(err instanceof Error ? err : new Error(String(err)));
|
|
@@ -332,37 +350,73 @@ async function* pacedSource(stream, highWater) {
|
|
|
332
350
|
stream.removeEventListener("remoteCloseWrite", onRemoteCloseWrite);
|
|
333
351
|
}
|
|
334
352
|
}
|
|
335
|
-
|
|
336
|
-
|
|
353
|
+
/**
|
|
354
|
+
* Frames out of the stream's chunks, as they complete. Unread bytes sit in a {@link ByteBuffer},
|
|
355
|
+
* which grows by doubling, so a large frame arriving in many chunks is copied a bounded number
|
|
356
|
+
* of times. A header whose length is malformed or past `maxFrameBytes` throws a `FramingError`.
|
|
357
|
+
*/
|
|
358
|
+
async function* parseFrames(source, maxFrameBytes = DEFAULT_MAX_FRAME_BYTES) {
|
|
359
|
+
const buf = new ByteBuffer();
|
|
337
360
|
for await (const item of source) {
|
|
338
361
|
const incoming = normalizeChunk(item);
|
|
339
362
|
if (incoming.byteLength === 0) continue;
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
lenInfo = decodeVarint(buf, 1);
|
|
353
|
-
} catch {
|
|
354
|
-
break;
|
|
355
|
-
}
|
|
356
|
-
const total = lenInfo.offset + lenInfo.value;
|
|
357
|
-
if (buf.byteLength < total) break;
|
|
363
|
+
buf.push(incoming);
|
|
364
|
+
for (;;) {
|
|
365
|
+
const bytes = buf.unread();
|
|
366
|
+
if (bytes.byteLength < 2) break;
|
|
367
|
+
const type = bytes[0];
|
|
368
|
+
const header = readLength(bytes.subarray(1), "frame");
|
|
369
|
+
if (header === void 0) break;
|
|
370
|
+
if (header.value > maxFrameBytes) throw new FramingError(`webrun-streams-libp2p: a ${header.value}-byte frame exceeds maxFrameBytes (${maxFrameBytes})`);
|
|
371
|
+
const total = 1 + header.offset + header.value;
|
|
372
|
+
if (bytes.byteLength < total) break;
|
|
373
|
+
const payload = bytes.slice(1 + header.offset, total);
|
|
374
|
+
buf.consume(total);
|
|
358
375
|
yield {
|
|
359
376
|
type,
|
|
360
|
-
payload
|
|
377
|
+
payload
|
|
361
378
|
};
|
|
362
|
-
buf = buf.byteLength === total ? /* @__PURE__ */ new Uint8Array(0) : new Uint8Array(buf.subarray(total));
|
|
363
379
|
}
|
|
364
380
|
}
|
|
365
381
|
}
|
|
382
|
+
/** Unread bytes `buf[start, end)` of a buffer that grows by doubling. */
|
|
383
|
+
var ByteBuffer = class {
|
|
384
|
+
buf = /* @__PURE__ */ new Uint8Array(0);
|
|
385
|
+
start = 0;
|
|
386
|
+
end = 0;
|
|
387
|
+
push(incoming) {
|
|
388
|
+
if (this.end + incoming.byteLength > this.buf.byteLength) {
|
|
389
|
+
const size = this.end - this.start + incoming.byteLength;
|
|
390
|
+
const next = new Uint8Array(Math.max(size, this.buf.byteLength * 2));
|
|
391
|
+
next.set(this.buf.subarray(this.start, this.end), 0);
|
|
392
|
+
this.buf = next;
|
|
393
|
+
this.end -= this.start;
|
|
394
|
+
this.start = 0;
|
|
395
|
+
}
|
|
396
|
+
this.buf.set(incoming, this.end);
|
|
397
|
+
this.end += incoming.byteLength;
|
|
398
|
+
}
|
|
399
|
+
/** a view of the unread bytes, valid until the next `push` */
|
|
400
|
+
unread() {
|
|
401
|
+
return this.buf.subarray(this.start, this.end);
|
|
402
|
+
}
|
|
403
|
+
consume(n) {
|
|
404
|
+
this.start += n;
|
|
405
|
+
if (this.start === this.end) this.start = this.end = 0;
|
|
406
|
+
}
|
|
407
|
+
};
|
|
408
|
+
/**
|
|
409
|
+
* The varint length prefix at the start of `bytes`, or `undefined` while it is incomplete; throws
|
|
410
|
+
* a `FramingError` when it is malformed (no varint of this framing runs past 5 bytes).
|
|
411
|
+
*/
|
|
412
|
+
function readLength(bytes, what) {
|
|
413
|
+
try {
|
|
414
|
+
return decodeVarint(bytes, 0);
|
|
415
|
+
} catch (err) {
|
|
416
|
+
if (bytes.byteLength < 5) return void 0;
|
|
417
|
+
throw new FramingError(`webrun-streams-libp2p: a malformed ${what} length`, { cause: err });
|
|
418
|
+
}
|
|
419
|
+
}
|
|
366
420
|
function normalizeChunk(item) {
|
|
367
421
|
if (item instanceof Uint8Array) return item;
|
|
368
422
|
const asList = item;
|
|
@@ -503,7 +557,7 @@ function definedOptions(fields) {
|
|
|
503
557
|
* Caller-side: each `call(input)` opens a new libp2p `Stream` via
|
|
504
558
|
* `node.dialProtocol(peer, [protocol])` and runs the call over it.
|
|
505
559
|
*/
|
|
506
|
-
const connect = async ({ node, peer, protocol, drainTimeoutMs, readHighWaterBytes, maxOutboundStreams, runOnLimitedConnection }) => {
|
|
560
|
+
const connect = async ({ node, peer, protocol, drainTimeoutMs, readHighWaterBytes, maxFrameBytes, maxOutboundStreams, runOnLimitedConnection }) => {
|
|
507
561
|
const proto = protocol ?? "/webrun-streams/1.0.0";
|
|
508
562
|
const dialOptions = definedOptions({
|
|
509
563
|
maxOutboundStreams,
|
|
@@ -522,6 +576,7 @@ const connect = async ({ node, peer, protocol, drainTimeoutMs, readHighWaterByte
|
|
|
522
576
|
yield* duplexOverStream(stream, input, {
|
|
523
577
|
drainTimeoutMs,
|
|
524
578
|
readHighWaterBytes,
|
|
579
|
+
maxFrameBytes,
|
|
525
580
|
onSourceCompleted: () => {
|
|
526
581
|
sourceCompleted = true;
|
|
527
582
|
}
|
|
@@ -559,7 +614,7 @@ const serve = async (params, handler) => serveConnections(params, () => handler)
|
|
|
559
614
|
* Like `serve`, but the handler is built per inbound stream and is told which
|
|
560
615
|
* peer libp2p proved on that connection.
|
|
561
616
|
*/
|
|
562
|
-
async function serveConnections({ node, protocol, drainTimeoutMs, readHighWaterBytes, maxInboundStreams, maxOutboundStreams, runOnLimitedConnection }, makeHandler) {
|
|
617
|
+
async function serveConnections({ node, protocol, drainTimeoutMs, readHighWaterBytes, maxFrameBytes, maxInboundStreams, maxOutboundStreams, runOnLimitedConnection }, makeHandler) {
|
|
563
618
|
const proto = protocol ?? "/webrun-streams/1.0.0";
|
|
564
619
|
const handleOptions = definedOptions({
|
|
565
620
|
maxInboundStreams,
|
|
@@ -576,6 +631,7 @@ async function serveConnections({ node, protocol, drainTimeoutMs, readHighWaterB
|
|
|
576
631
|
for await (const chunk of duplexOverStream(stream, output, {
|
|
577
632
|
drainTimeoutMs,
|
|
578
633
|
readHighWaterBytes,
|
|
634
|
+
maxFrameBytes,
|
|
579
635
|
onPeerInputEnd: (err) => {
|
|
580
636
|
if (err === void 0) requestComplete = true;
|
|
581
637
|
inputQueue.done(err);
|
|
@@ -726,29 +782,19 @@ function messageTargetOverStream(stream, { maxMessageBytes = DEFAULT_MAX_MESSAGE
|
|
|
726
782
|
} catch {}
|
|
727
783
|
};
|
|
728
784
|
(async () => {
|
|
729
|
-
|
|
730
|
-
let start = 0;
|
|
731
|
-
let end = 0;
|
|
785
|
+
const buf = new ByteBuffer();
|
|
732
786
|
for await (const item of stream) {
|
|
733
|
-
|
|
734
|
-
if (end + incoming.byteLength > buf.byteLength) {
|
|
735
|
-
const size = end - start + incoming.byteLength;
|
|
736
|
-
const next = new Uint8Array(Math.max(size, buf.byteLength * 2));
|
|
737
|
-
next.set(buf.subarray(start, end), 0);
|
|
738
|
-
buf = next;
|
|
739
|
-
end -= start;
|
|
740
|
-
start = 0;
|
|
741
|
-
}
|
|
742
|
-
buf.set(incoming, end);
|
|
743
|
-
end += incoming.byteLength;
|
|
787
|
+
buf.push(normalizeChunk(item));
|
|
744
788
|
for (;;) {
|
|
745
|
-
const
|
|
789
|
+
const bytes = buf.unread();
|
|
790
|
+
const header = readLength(bytes, "message");
|
|
746
791
|
if (header === void 0) break;
|
|
747
792
|
if (header.value > maxMessageBytes) throw new Error(`webrun-streams-libp2p: a ${header.value}-byte message exceeds maxMessageBytes (${maxMessageBytes})`);
|
|
748
|
-
const stop =
|
|
749
|
-
if (
|
|
750
|
-
|
|
751
|
-
|
|
793
|
+
const stop = header.offset + header.value;
|
|
794
|
+
if (bytes.byteLength < stop) break;
|
|
795
|
+
const message = bytes.slice(header.offset, stop);
|
|
796
|
+
buf.consume(stop);
|
|
797
|
+
deliver(message);
|
|
752
798
|
}
|
|
753
799
|
}
|
|
754
800
|
if (peerClosedWrite) {
|
|
@@ -772,7 +818,8 @@ function messageTargetOverStream(stream, { maxMessageBytes = DEFAULT_MAX_MESSAGE
|
|
|
772
818
|
listeners.delete(listener);
|
|
773
819
|
},
|
|
774
820
|
postMessage(message) {
|
|
775
|
-
const payload =
|
|
821
|
+
const payload = toBytes(message);
|
|
822
|
+
if (!payload) throw new TypeError(`webrun-streams-libp2p: a byte pipe carries Uint8Array, ArrayBuffer or a view, got ${Object.prototype.toString.call(message)}`);
|
|
776
823
|
const header = encodeVarint(payload.byteLength);
|
|
777
824
|
const frame = new Uint8Array(header.byteLength + payload.byteLength);
|
|
778
825
|
frame.set(header, 0);
|
|
@@ -803,14 +850,11 @@ function messageTargetOverStream(stream, { maxMessageBytes = DEFAULT_MAX_MESSAGE
|
|
|
803
850
|
}
|
|
804
851
|
};
|
|
805
852
|
}
|
|
806
|
-
/**
|
|
807
|
-
function
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
if (bytes.byteLength < 5) return void 0;
|
|
812
|
-
throw new Error("webrun-streams-libp2p: a malformed message length", { cause: err });
|
|
813
|
-
}
|
|
853
|
+
/** `message` as bytes, or `undefined` when it is not bytes */
|
|
854
|
+
function toBytes(message) {
|
|
855
|
+
if (message instanceof Uint8Array) return message;
|
|
856
|
+
if (message instanceof ArrayBuffer) return new Uint8Array(message);
|
|
857
|
+
if (ArrayBuffer.isView(message)) return new Uint8Array(message.buffer, message.byteOffset, message.byteLength);
|
|
814
858
|
}
|
|
815
859
|
//#endregion
|
|
816
860
|
//#region src/port-mux.ts
|
|
@@ -959,4 +1003,4 @@ async function libp2pPortMux(options) {
|
|
|
959
1003
|
};
|
|
960
1004
|
}
|
|
961
1005
|
//#endregion
|
|
962
|
-
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_MAX_MESSAGE_BYTES, DEFAULT_PORT_MAX_MESSAGE_SIZE, DEFAULT_PORT_PROTOCOL, DEFAULT_PORT_STREAM_LIMIT, DEFAULT_PROTOCOL, PeerGoneError, connect, duplexOverStream, libp2pPortMux, messageTargetOverStream, serve, serveConnections };
|
|
1006
|
+
export { DEFAULT_CLOSE_TIMEOUT_MS, DEFAULT_DRAIN_TIMEOUT_MS, DEFAULT_MAX_FRAME_BYTES, DEFAULT_MAX_MESSAGE_BYTES, DEFAULT_PORT_MAX_MESSAGE_SIZE, DEFAULT_PORT_PROTOCOL, DEFAULT_PORT_STREAM_LIMIT, DEFAULT_PROTOCOL, PeerGoneError, connect, duplexOverStream, libp2pPortMux, messageTargetOverStream, serve, serveConnections };
|
|
@@ -9,6 +9,7 @@ export interface StreamMessageTarget {
|
|
|
9
9
|
target: {
|
|
10
10
|
addEventListener(type: "message", listener: Listener): void;
|
|
11
11
|
removeEventListener(type: "message", listener: Listener): void;
|
|
12
|
+
/** Sends one message. Throws `TypeError` for anything but bytes. */
|
|
12
13
|
postMessage(message: unknown): void;
|
|
13
14
|
close(): Promise<void>;
|
|
14
15
|
/** The stream's write buffer, in bytes. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"message-target-over-stream.d.ts","sourceRoot":"","sources":["../src/message-target-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"message-target-over-stream.d.ts","sourceRoot":"","sources":["../src/message-target-over-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAoB,MAAM,mBAAmB,CAAC;AAUlE,KAAK,QAAQ,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE9D,uEAAuE;AACvE,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,MAAM,EAAE;QACN,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC5D,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;QAC/D,oEAAoE;QACpE,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;QACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QACvB,2CAA2C;QAC3C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC;;;WAGG;QACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,+FAA+F;QAC/F,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;KACnC,CAAC;IACF;;;;;;;OAOG;IACH,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,8BAA8B;IAC7C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,6EAA6E;AAC7E,eAAO,MAAM,yBAAyB,QAAmB,CAAC;AAE1D;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,EAAE,eAA2C,EAAE,GAAE,8BAAmC,GACnF,mBAAmB,CAmHrB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/webrun-streams-libp2p",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "libp2p native multi-stream Connect/Serve adapter in the webrun-streams-* family",
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"src"
|
|
31
31
|
],
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@statewalker/webrun-streams": "^0.
|
|
33
|
+
"@statewalker/webrun-streams": "^0.4.0"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
36
|
"@libp2p/interface": "^3.0.0",
|
|
@@ -57,9 +57,9 @@
|
|
|
57
57
|
"tsx": "^4.23.15",
|
|
58
58
|
"typescript": "^7.0.2",
|
|
59
59
|
"vitest": "^5.0.5",
|
|
60
|
-
"@statewalker/webrun-
|
|
61
|
-
"@statewalker/webrun-
|
|
62
|
-
"@statewalker/webrun-streams-conformance": "^0.
|
|
60
|
+
"@statewalker/webrun-msgpack": "^0.11.0",
|
|
61
|
+
"@statewalker/webrun-rpc": "^0.12.0",
|
|
62
|
+
"@statewalker/webrun-streams-conformance": "^0.5.0"
|
|
63
63
|
},
|
|
64
64
|
"sideEffects": false,
|
|
65
65
|
"publishConfig": {
|
|
@@ -69,6 +69,8 @@
|
|
|
69
69
|
"scripts": {
|
|
70
70
|
"build": "rimraf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
|
|
71
71
|
"test": "vitest run",
|
|
72
|
+
"typecheck": "tsc --noEmit",
|
|
73
|
+
"typecheck:tests": "tsc -p tsconfig.tests.json",
|
|
72
74
|
"lint": "biome check src tests"
|
|
73
75
|
}
|
|
74
76
|
}
|
package/src/connect-serve.ts
CHANGED
|
@@ -33,6 +33,12 @@ export interface ConnectLibp2pParams {
|
|
|
33
33
|
* paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
|
|
34
34
|
*/
|
|
35
35
|
readHighWaterBytes?: number;
|
|
36
|
+
/**
|
|
37
|
+
* The largest frame payload accepted from the peer; past it, or on a
|
|
38
|
+
* malformed frame header, the call fails and the stream is aborted. See
|
|
39
|
+
* `DuplexOverStreamOptions.maxFrameBytes` (default 64 MiB).
|
|
40
|
+
*/
|
|
41
|
+
maxFrameBytes?: number;
|
|
36
42
|
/**
|
|
37
43
|
* How many outgoing streams for this protocol libp2p allows to be open at
|
|
38
44
|
* the same time on one connection; passed through to `node.dialProtocol`.
|
|
@@ -65,6 +71,12 @@ export interface ServeLibp2pParams {
|
|
|
65
71
|
* paused; see `DuplexOverStreamOptions.readHighWaterBytes` (default 64 KiB).
|
|
66
72
|
*/
|
|
67
73
|
readHighWaterBytes?: number;
|
|
74
|
+
/**
|
|
75
|
+
* The largest frame payload accepted from the peer; past it, or on a
|
|
76
|
+
* malformed frame header, the call fails and the stream is aborted. See
|
|
77
|
+
* `DuplexOverStreamOptions.maxFrameBytes` (default 64 MiB).
|
|
78
|
+
*/
|
|
79
|
+
maxFrameBytes?: number;
|
|
68
80
|
/**
|
|
69
81
|
* How many incoming streams for this protocol libp2p allows to be open at
|
|
70
82
|
* the same time on one connection; passed through to `node.handle`. Left
|
|
@@ -119,6 +131,7 @@ export const connect: Connect<ConnectLibp2pParams> = async ({
|
|
|
119
131
|
protocol,
|
|
120
132
|
drainTimeoutMs,
|
|
121
133
|
readHighWaterBytes,
|
|
134
|
+
maxFrameBytes,
|
|
122
135
|
maxOutboundStreams,
|
|
123
136
|
runOnLimitedConnection,
|
|
124
137
|
}) => {
|
|
@@ -158,6 +171,7 @@ export const connect: Connect<ConnectLibp2pParams> = async ({
|
|
|
158
171
|
yield* duplexOverStream(stream, input, {
|
|
159
172
|
drainTimeoutMs,
|
|
160
173
|
readHighWaterBytes,
|
|
174
|
+
maxFrameBytes,
|
|
161
175
|
onSourceCompleted: () => {
|
|
162
176
|
sourceCompleted = true;
|
|
163
177
|
},
|
|
@@ -242,6 +256,7 @@ export async function serveConnections(
|
|
|
242
256
|
protocol,
|
|
243
257
|
drainTimeoutMs,
|
|
244
258
|
readHighWaterBytes,
|
|
259
|
+
maxFrameBytes,
|
|
245
260
|
maxInboundStreams,
|
|
246
261
|
maxOutboundStreams,
|
|
247
262
|
runOnLimitedConnection,
|
|
@@ -270,6 +285,7 @@ export async function serveConnections(
|
|
|
270
285
|
for await (const chunk of duplexOverStream(stream, output, {
|
|
271
286
|
drainTimeoutMs,
|
|
272
287
|
readHighWaterBytes,
|
|
288
|
+
maxFrameBytes,
|
|
273
289
|
onPeerInputEnd: (err) => {
|
|
274
290
|
if (err === undefined) requestComplete = true;
|
|
275
291
|
inputQueue.done(err);
|
|
@@ -58,6 +58,22 @@ export const DEFAULT_CLOSE_TIMEOUT_MS = DEFAULT_DRAIN_TIMEOUT_MS;
|
|
|
58
58
|
*/
|
|
59
59
|
export const DEFAULT_READ_HIGH_WATER_BYTES = 64 * 1024;
|
|
60
60
|
|
|
61
|
+
/**
|
|
62
|
+
* Default for {@link DuplexOverStreamOptions.maxFrameBytes}: the largest frame payload accepted
|
|
63
|
+
* from the peer. Generous, because a sender before this package's outbound chunking sent each
|
|
64
|
+
* producer chunk as one frame; short of letting a peer declare a 4 GiB frame.
|
|
65
|
+
*/
|
|
66
|
+
export const DEFAULT_MAX_FRAME_BYTES = 64 * 1024 * 1024;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The largest payload this side puts in one outbound frame: a larger producer chunk crosses as
|
|
70
|
+
* several frames (they are self-delimiting, so a receiver of any version reassembles the bytes).
|
|
71
|
+
*/
|
|
72
|
+
export const OUTBOUND_FRAME_BYTES = 64 * 1024;
|
|
73
|
+
|
|
74
|
+
/** The peer sent bytes that are not this framing: a malformed or oversized frame header. */
|
|
75
|
+
class FramingError extends Error {}
|
|
76
|
+
|
|
61
77
|
/**
|
|
62
78
|
* The stream ended before the call finished: the peer died, the connection
|
|
63
79
|
* dropped, or the stream was reset or refused before both sides ended it.
|
|
@@ -116,6 +132,13 @@ export interface DuplexOverStreamOptions {
|
|
|
116
132
|
* Defaults to {@link DEFAULT_READ_HIGH_WATER_BYTES}.
|
|
117
133
|
*/
|
|
118
134
|
readHighWaterBytes?: number;
|
|
135
|
+
/**
|
|
136
|
+
* The largest frame payload accepted from the peer, in bytes. A frame header past it — or one
|
|
137
|
+
* whose length is not a valid varint — means the peer is not speaking this framing: the call
|
|
138
|
+
* fails and the stream is aborted, rather than buffering without bound. Defaults to
|
|
139
|
+
* {@link DEFAULT_MAX_FRAME_BYTES}.
|
|
140
|
+
*/
|
|
141
|
+
maxFrameBytes?: number;
|
|
119
142
|
}
|
|
120
143
|
|
|
121
144
|
/**
|
|
@@ -232,6 +255,7 @@ export async function* duplexOverStream(
|
|
|
232
255
|
try {
|
|
233
256
|
for await (const frame of parseFrames(
|
|
234
257
|
pacedSource(stream, opts.readHighWaterBytes ?? DEFAULT_READ_HIGH_WATER_BYTES),
|
|
258
|
+
opts.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES,
|
|
235
259
|
)) {
|
|
236
260
|
if (frame.type === TYPE_DATA) {
|
|
237
261
|
yield frame.payload;
|
|
@@ -243,6 +267,11 @@ export async function* duplexOverStream(
|
|
|
243
267
|
}
|
|
244
268
|
} catch (err) {
|
|
245
269
|
if (err === peerError) throw err;
|
|
270
|
+
if (err instanceof FramingError) {
|
|
271
|
+
// the peer is not speaking this framing: fail the call (the stream is aborted below)
|
|
272
|
+
firePeerInputEnd(err);
|
|
273
|
+
throw err;
|
|
274
|
+
}
|
|
246
275
|
// The stream itself failed (reset, aborted, its connection dropped):
|
|
247
276
|
// the call did not finish, whatever libp2p named the error.
|
|
248
277
|
const gone = new PeerGoneError(err);
|
|
@@ -409,7 +438,11 @@ async function* framedOutbound(
|
|
|
409
438
|
for (;;) {
|
|
410
439
|
const next = await iterator.next();
|
|
411
440
|
if (next.done === true) return;
|
|
412
|
-
|
|
441
|
+
// in bounded frames, so a receiver never reassembles one huge frame
|
|
442
|
+
const chunk = normalizeChunk(next.value);
|
|
443
|
+
for (let at = 0; at < chunk.byteLength; at += OUTBOUND_FRAME_BYTES) {
|
|
444
|
+
yield frameData(chunk.subarray(at, Math.min(at + OUTBOUND_FRAME_BYTES, chunk.byteLength)));
|
|
445
|
+
}
|
|
413
446
|
}
|
|
414
447
|
} catch (err) {
|
|
415
448
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
@@ -547,40 +580,87 @@ async function* pacedSource(stream: Stream, highWater: number): AsyncGenerator<U
|
|
|
547
580
|
}
|
|
548
581
|
}
|
|
549
582
|
|
|
550
|
-
|
|
583
|
+
/**
|
|
584
|
+
* Frames out of the stream's chunks, as they complete. Unread bytes sit in a {@link ByteBuffer},
|
|
585
|
+
* which grows by doubling, so a large frame arriving in many chunks is copied a bounded number
|
|
586
|
+
* of times. A header whose length is malformed or past `maxFrameBytes` throws a `FramingError`.
|
|
587
|
+
*/
|
|
588
|
+
export async function* parseFrames(
|
|
551
589
|
source: AsyncIterable<unknown>,
|
|
590
|
+
maxFrameBytes: number = DEFAULT_MAX_FRAME_BYTES,
|
|
552
591
|
): AsyncGenerator<{ type: number; payload: Uint8Array }> {
|
|
553
|
-
|
|
592
|
+
const buf = new ByteBuffer();
|
|
554
593
|
for await (const item of source) {
|
|
555
594
|
const incoming = normalizeChunk(item);
|
|
556
595
|
if (incoming.byteLength === 0) continue;
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
let lenInfo: { value: number; offset: number };
|
|
570
|
-
try {
|
|
571
|
-
lenInfo = decodeVarint(buf, 1);
|
|
572
|
-
} catch {
|
|
573
|
-
break; // varint truncated, need more bytes
|
|
596
|
+
buf.push(incoming);
|
|
597
|
+
for (;;) {
|
|
598
|
+
const bytes = buf.unread();
|
|
599
|
+
if (bytes.byteLength < 2) break; // need at least type + 1 varint byte
|
|
600
|
+
// biome-ignore lint/style/noNonNullAssertion: bytes.byteLength >= 2, checked above, so index 0 exists.
|
|
601
|
+
const type = bytes[0]!;
|
|
602
|
+
const header = readLength(bytes.subarray(1), "frame");
|
|
603
|
+
if (header === undefined) break; // the length is not all here yet
|
|
604
|
+
if (header.value > maxFrameBytes) {
|
|
605
|
+
throw new FramingError(
|
|
606
|
+
`webrun-streams-libp2p: a ${header.value}-byte frame exceeds maxFrameBytes (${maxFrameBytes})`,
|
|
607
|
+
);
|
|
574
608
|
}
|
|
575
|
-
const total =
|
|
576
|
-
if (
|
|
577
|
-
const payload =
|
|
609
|
+
const total = 1 + header.offset + header.value;
|
|
610
|
+
if (bytes.byteLength < total) break;
|
|
611
|
+
const payload = bytes.slice(1 + header.offset, total);
|
|
612
|
+
buf.consume(total);
|
|
578
613
|
yield { type, payload };
|
|
579
|
-
buf = buf.byteLength === total ? new Uint8Array(0) : new Uint8Array(buf.subarray(total));
|
|
580
614
|
}
|
|
581
615
|
}
|
|
582
616
|
}
|
|
583
617
|
|
|
618
|
+
/** Unread bytes `buf[start, end)` of a buffer that grows by doubling. */
|
|
619
|
+
export class ByteBuffer {
|
|
620
|
+
private buf = new Uint8Array(0);
|
|
621
|
+
private start = 0;
|
|
622
|
+
private end = 0;
|
|
623
|
+
|
|
624
|
+
push(incoming: Uint8Array): void {
|
|
625
|
+
if (this.end + incoming.byteLength > this.buf.byteLength) {
|
|
626
|
+
const size = this.end - this.start + incoming.byteLength;
|
|
627
|
+
const next = new Uint8Array(Math.max(size, this.buf.byteLength * 2));
|
|
628
|
+
next.set(this.buf.subarray(this.start, this.end), 0);
|
|
629
|
+
this.buf = next;
|
|
630
|
+
this.end -= this.start;
|
|
631
|
+
this.start = 0;
|
|
632
|
+
}
|
|
633
|
+
this.buf.set(incoming, this.end);
|
|
634
|
+
this.end += incoming.byteLength;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/** a view of the unread bytes, valid until the next `push` */
|
|
638
|
+
unread(): Uint8Array {
|
|
639
|
+
return this.buf.subarray(this.start, this.end);
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
consume(n: number): void {
|
|
643
|
+
this.start += n;
|
|
644
|
+
if (this.start === this.end) this.start = this.end = 0;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* The varint length prefix at the start of `bytes`, or `undefined` while it is incomplete; throws
|
|
650
|
+
* a `FramingError` when it is malformed (no varint of this framing runs past 5 bytes).
|
|
651
|
+
*/
|
|
652
|
+
export function readLength(
|
|
653
|
+
bytes: Uint8Array,
|
|
654
|
+
what: string,
|
|
655
|
+
): { value: number; offset: number } | undefined {
|
|
656
|
+
try {
|
|
657
|
+
return decodeVarint(bytes, 0);
|
|
658
|
+
} catch (err) {
|
|
659
|
+
if (bytes.byteLength < 5) return undefined;
|
|
660
|
+
throw new FramingError(`webrun-streams-libp2p: a malformed ${what} length`, { cause: err });
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
584
664
|
export function normalizeChunk(item: unknown): Uint8Array {
|
|
585
665
|
if (item instanceof Uint8Array) return item;
|
|
586
666
|
const asList = item as { subarray?: () => Uint8Array };
|
package/src/index.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import type { Stream, StreamCloseEvent } from "@libp2p/interface";
|
|
2
2
|
import {
|
|
3
|
+
ByteBuffer,
|
|
3
4
|
closeStream,
|
|
4
|
-
decodeVarint,
|
|
5
5
|
encodeVarint,
|
|
6
6
|
normalizeChunk,
|
|
7
7
|
PeerGoneError,
|
|
8
|
+
readLength,
|
|
8
9
|
} from "./duplex-over-stream.js";
|
|
9
10
|
|
|
10
11
|
type Listener = (event: MessageEvent) => void | Promise<void>;
|
|
@@ -18,6 +19,7 @@ export interface StreamMessageTarget {
|
|
|
18
19
|
target: {
|
|
19
20
|
addEventListener(type: "message", listener: Listener): void;
|
|
20
21
|
removeEventListener(type: "message", listener: Listener): void;
|
|
22
|
+
/** Sends one message. Throws `TypeError` for anything but bytes. */
|
|
21
23
|
postMessage(message: unknown): void;
|
|
22
24
|
close(): Promise<void>;
|
|
23
25
|
/** The stream's write buffer, in bytes. */
|
|
@@ -108,35 +110,25 @@ export function messageTargetOverStream(
|
|
|
108
110
|
};
|
|
109
111
|
|
|
110
112
|
void (async () => {
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
|
|
114
|
-
let start = 0;
|
|
115
|
-
let end = 0;
|
|
113
|
+
// The buffer grows by doubling, so a large message arriving in many chunks
|
|
114
|
+
// is copied a bounded number of times.
|
|
115
|
+
const buf = new ByteBuffer();
|
|
116
116
|
for await (const item of stream) {
|
|
117
|
-
|
|
118
|
-
if (end + incoming.byteLength > buf.byteLength) {
|
|
119
|
-
const size = end - start + incoming.byteLength;
|
|
120
|
-
const next = new Uint8Array(Math.max(size, buf.byteLength * 2));
|
|
121
|
-
next.set(buf.subarray(start, end), 0);
|
|
122
|
-
buf = next;
|
|
123
|
-
end -= start;
|
|
124
|
-
start = 0;
|
|
125
|
-
}
|
|
126
|
-
buf.set(incoming, end);
|
|
127
|
-
end += incoming.byteLength;
|
|
117
|
+
buf.push(normalizeChunk(item));
|
|
128
118
|
for (;;) {
|
|
129
|
-
const
|
|
119
|
+
const bytes = buf.unread();
|
|
120
|
+
const header = readLength(bytes, "message");
|
|
130
121
|
if (header === undefined) break; // the length is not all here yet
|
|
131
122
|
if (header.value > maxMessageBytes) {
|
|
132
123
|
throw new Error(
|
|
133
124
|
`webrun-streams-libp2p: a ${header.value}-byte message exceeds maxMessageBytes (${maxMessageBytes})`,
|
|
134
125
|
);
|
|
135
126
|
}
|
|
136
|
-
const stop =
|
|
137
|
-
if (
|
|
138
|
-
|
|
139
|
-
|
|
127
|
+
const stop = header.offset + header.value;
|
|
128
|
+
if (bytes.byteLength < stop) break;
|
|
129
|
+
const message = bytes.slice(header.offset, stop);
|
|
130
|
+
buf.consume(stop);
|
|
131
|
+
deliver(message);
|
|
140
132
|
}
|
|
141
133
|
}
|
|
142
134
|
if (peerClosedWrite) {
|
|
@@ -162,7 +154,12 @@ export function messageTargetOverStream(
|
|
|
162
154
|
listeners.delete(listener);
|
|
163
155
|
},
|
|
164
156
|
postMessage(message) {
|
|
165
|
-
const payload =
|
|
157
|
+
const payload = toBytes(message);
|
|
158
|
+
if (!payload) {
|
|
159
|
+
throw new TypeError(
|
|
160
|
+
`webrun-streams-libp2p: a byte pipe carries Uint8Array, ArrayBuffer or a view, got ${Object.prototype.toString.call(message)}`,
|
|
161
|
+
);
|
|
162
|
+
}
|
|
166
163
|
const header = encodeVarint(payload.byteLength);
|
|
167
164
|
const frame = new Uint8Array(header.byteLength + payload.byteLength);
|
|
168
165
|
frame.set(header, 0);
|
|
@@ -196,12 +193,12 @@ export function messageTargetOverStream(
|
|
|
196
193
|
};
|
|
197
194
|
}
|
|
198
195
|
|
|
199
|
-
/**
|
|
200
|
-
function
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
throw new Error("webrun-streams-libp2p: a malformed message length", { cause: err });
|
|
196
|
+
/** `message` as bytes, or `undefined` when it is not bytes */
|
|
197
|
+
function toBytes(message: unknown): Uint8Array | undefined {
|
|
198
|
+
if (message instanceof Uint8Array) return message;
|
|
199
|
+
if (message instanceof ArrayBuffer) return new Uint8Array(message);
|
|
200
|
+
if (ArrayBuffer.isView(message)) {
|
|
201
|
+
return new Uint8Array(message.buffer, message.byteOffset, message.byteLength);
|
|
206
202
|
}
|
|
203
|
+
return undefined;
|
|
207
204
|
}
|