@statewalker/webrun-streams-libp2p 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,238 @@
1
+ import type {
2
+ Connection,
3
+ DialProtocolOptions,
4
+ Libp2p,
5
+ PeerId,
6
+ Stream,
7
+ StreamHandlerOptions,
8
+ } from "@libp2p/interface";
9
+ import type { Multiaddr } from "@multiformats/multiaddr";
10
+ import { closeStream } from "./duplex-over-stream.js";
11
+ import { messageTargetOverStream, type StreamMessageTarget } from "./message-target-over-stream.js";
12
+ import { openGatedStream } from "./stream-gate.js";
13
+
14
+ /** The protocol a port's stream negotiates unless told otherwise. */
15
+ export const DEFAULT_PORT_PROTOCOL = "/webrun-ports/1.0.0";
16
+
17
+ /**
18
+ * 256 KiB: what one message on a port may carry before `webrun-rpc`'s stream tier chunks a body.
19
+ * A port waits for each chunk's confirmation, so a larger chunk means fewer round trips; well
20
+ * under `messageTargetOverStream`'s 64 MiB frame ceiling.
21
+ */
22
+ export const DEFAULT_PORT_MAX_MESSAGE_SIZE = 256 * 1024;
23
+
24
+ /**
25
+ * How a port's messages become bytes on its stream and back: any value the stream tier posts
26
+ * (objects holding `Uint8Array` chunks, serialised errors). `webrun-msgpack`'s `serialize` and
27
+ * `deserialize` fit: `{ encode: serialize, decode: deserialize }`.
28
+ */
29
+ export interface ValueCodec {
30
+ encode(value: unknown): Uint8Array;
31
+ decode(bytes: Uint8Array): unknown;
32
+ }
33
+
34
+ type Listener = (event: MessageEvent) => void | Promise<void>;
35
+
36
+ /**
37
+ * The stream limits a port mux asks libp2p for unless told otherwise: every concurrent call is a
38
+ * stream, and libp2p's own default (32 per protocol) would reset the 33rd.
39
+ */
40
+ export const DEFAULT_PORT_STREAM_LIMIT = 1024;
41
+
42
+ /** One port: a libp2p stream as a `MessageTarget` of values, with its close signal on it. */
43
+ export type StreamPort = Omit<
44
+ StreamMessageTarget["target"],
45
+ "addEventListener" | "removeEventListener"
46
+ > & {
47
+ addEventListener(type: "message", listener: Listener): void;
48
+ removeEventListener(type: "message", listener: Listener): void;
49
+ };
50
+
51
+ /** What the serving side learns about a port the peer opened. */
52
+ export interface InboundPortInfo {
53
+ /** The peer libp2p's Noise handshake proved for the connection the stream came on. */
54
+ remotePeer: PeerId;
55
+ connection: Connection;
56
+ }
57
+
58
+ export interface Libp2pPortMuxOptions {
59
+ node: Libp2p;
60
+ /** Encodes each message as one frame; see {@link ValueCodec}. Both ends must agree. */
61
+ codec: ValueCodec;
62
+ /** The protocol each port's stream negotiates. Default {@link DEFAULT_PORT_PROTOCOL}. */
63
+ protocol?: string;
64
+ /** Whom `openPort` dials. Required to open ports; a serving-only mux leaves it out. */
65
+ peer?: PeerId | Multiaddr;
66
+ /**
67
+ * Called for each port the peer opens (each inbound stream). Return `false` to refuse it: the
68
+ * stream is aborted. Leave it out to accept none (the protocol is not handled).
69
+ */
70
+ onPort?: (port: StreamPort, info: InboundPortInfo) => boolean | undefined;
71
+ /** Reported as `maxMessageSize`. Default {@link DEFAULT_PORT_MAX_MESSAGE_SIZE}. */
72
+ maxMessageSize?: number;
73
+ /** The largest frame accepted on a port's stream (see `messageTargetOverStream`). */
74
+ maxMessageBytes?: number;
75
+ /** Default {@link DEFAULT_PORT_STREAM_LIMIT}. */
76
+ maxInboundStreams?: number;
77
+ /** Default {@link DEFAULT_PORT_STREAM_LIMIT}. */
78
+ maxOutboundStreams?: number;
79
+ runOnLimitedConnection?: boolean;
80
+ }
81
+
82
+ /** A `webrun-rpc` `PortMux` by shape; this package imports nothing from it. */
83
+ export interface Libp2pPortMux {
84
+ openPort(meta?: unknown, options?: { signal?: AbortSignal }): Promise<StreamPort>;
85
+ close(reason?: unknown): Promise<void>;
86
+ readonly maxMessageSize: number;
87
+ }
88
+
89
+ /**
90
+ * Native MessagePorts over libp2p: every port is its own libp2p stream, which yamux multiplexes
91
+ * with its own per-stream flow control, framed as messages by `messageTargetOverStream`. Hand it
92
+ * to `webrun-rpc` as a port source and its stream tier runs one call per port, with its tested
93
+ * backpressure — a producer stays one confirmed chunk ahead of its consumer — instead of
94
+ * libp2p's own buffering, which lets a producer run tens of megabytes ahead of a consumer that
95
+ * stopped reading:
96
+ *
97
+ * ```ts
98
+ * const codec = { encode: serialize, decode: deserialize }; // webrun-msgpack
99
+ * await serve({ mux: overPorts((onPort) => libp2pPortMux({ node, codec, onPort })) }, handler);
100
+ * const { call } = await connect({ mux: overPorts(() => libp2pPortMux({ node, codec, peer })) });
101
+ * ```
102
+ *
103
+ * Each port carries `closeSignal`, aborted when its stream ends or its peer goes, so a call on a
104
+ * dead stream fails instead of waiting. Streams are opened through the same early-stream gate as
105
+ * `connect()`. `meta` is not sent: the protocol says what a port is for.
106
+ */
107
+ export async function libp2pPortMux(options: Libp2pPortMuxOptions): Promise<Libp2pPortMux> {
108
+ const { node, peer, codec, onPort, maxMessageBytes, runOnLimitedConnection } = options;
109
+ const maxInboundStreams = options.maxInboundStreams ?? DEFAULT_PORT_STREAM_LIMIT;
110
+ const maxOutboundStreams = options.maxOutboundStreams ?? DEFAULT_PORT_STREAM_LIMIT;
111
+ const protocol = options.protocol ?? DEFAULT_PORT_PROTOCOL;
112
+ const maxMessageSize = options.maxMessageSize ?? DEFAULT_PORT_MAX_MESSAGE_SIZE;
113
+ const live = new Set<StreamPort>();
114
+ let closed = false;
115
+
116
+ const wrap = (stream: Stream): StreamPort => {
117
+ const { target: inner } = messageTargetOverStream(stream, { maxMessageBytes });
118
+ const wrapped = new Map<Listener, Listener>();
119
+ // closed here, or the stream ended: a port closed locally is closed at once, not when libp2p
120
+ // finishes closing its stream
121
+ const local = new AbortController();
122
+ const port: StreamPort = {
123
+ closeSignal: AbortSignal.any([inner.closeSignal, local.signal]),
124
+ get bufferedAmount() {
125
+ return inner.bufferedAmount;
126
+ },
127
+ whenDrained: () => inner.whenDrained(),
128
+ postMessage(message: unknown) {
129
+ inner.postMessage(codec.encode(message));
130
+ },
131
+ addEventListener(_type, listener) {
132
+ if (wrapped.has(listener)) return;
133
+ const decoding: Listener = (event) => {
134
+ let data: unknown;
135
+ try {
136
+ data = codec.decode(event.data as Uint8Array);
137
+ } catch (err) {
138
+ // a peer speaking another codec, or a corrupt frame: fail the port, as a malformed
139
+ // frame fails the stream, rather than drop the message and leave its call waiting
140
+ stream.abort(
141
+ new Error("webrun-streams-libp2p: a frame the port's codec cannot decode", {
142
+ cause: err,
143
+ }),
144
+ );
145
+ return;
146
+ }
147
+ return listener(new MessageEvent("message", { data }));
148
+ };
149
+ wrapped.set(listener, decoding);
150
+ inner.addEventListener("message", decoding);
151
+ },
152
+ removeEventListener(_type, listener) {
153
+ const decoding = wrapped.get(listener);
154
+ if (!decoding) return;
155
+ wrapped.delete(listener);
156
+ inner.removeEventListener("message", decoding);
157
+ },
158
+ async close() {
159
+ live.delete(port);
160
+ if (!local.signal.aborted) local.abort(new Error("webrun-streams-libp2p: port closed"));
161
+ await inner.close();
162
+ },
163
+ };
164
+ live.add(port);
165
+ inner.closeSignal.addEventListener("abort", () => live.delete(port), { once: true });
166
+ return port;
167
+ };
168
+
169
+ let handled = false;
170
+ if (onPort) {
171
+ const handleOptions: StreamHandlerOptions = { maxInboundStreams, maxOutboundStreams };
172
+ if (runOnLimitedConnection !== undefined)
173
+ handleOptions.runOnLimitedConnection = runOnLimitedConnection;
174
+ await node.handle(
175
+ protocol,
176
+ (stream: Stream, connection: Connection) => {
177
+ if (closed) {
178
+ stream.abort(new Error("webrun-streams-libp2p: port mux closed"));
179
+ return;
180
+ }
181
+ // The port is wrapped and handed over in this tick: `messageTargetOverStream` starts
182
+ // reading at once, and a message with no listener yet would be dropped.
183
+ const port = wrap(stream);
184
+ let accepted = false;
185
+ try {
186
+ accepted = onPort(port, { remotePeer: connection.remotePeer, connection }) !== false;
187
+ } catch {
188
+ accepted = false;
189
+ }
190
+ if (!accepted) {
191
+ live.delete(port);
192
+ stream.abort(new Error("webrun-streams-libp2p: port refused"));
193
+ }
194
+ },
195
+ handleOptions,
196
+ );
197
+ handled = true;
198
+ }
199
+
200
+ const dialOptions: DialProtocolOptions = { maxOutboundStreams };
201
+ if (runOnLimitedConnection !== undefined)
202
+ dialOptions.runOnLimitedConnection = runOnLimitedConnection;
203
+
204
+ return {
205
+ maxMessageSize,
206
+ async openPort(_meta, opts) {
207
+ if (closed) throw new Error("webrun-streams-libp2p: port mux closed");
208
+ if (!peer) throw new Error("webrun-streams-libp2p: openPort needs `peer`");
209
+ const stream = await openGatedStream(node, peer, protocol, {
210
+ ...dialOptions,
211
+ signal: opts?.signal,
212
+ });
213
+ if (closed) {
214
+ await closeStream(stream);
215
+ throw new Error("webrun-streams-libp2p: port mux closed");
216
+ }
217
+ const port = wrap(stream);
218
+ const signal = opts?.signal;
219
+ if (signal) {
220
+ const onAbort = () => void port.close();
221
+ signal.addEventListener("abort", onAbort, { once: true });
222
+ // the caller's signal may outlive many ports: let go of this one when it closes
223
+ port.closeSignal.addEventListener(
224
+ "abort",
225
+ () => signal.removeEventListener("abort", onAbort),
226
+ { once: true },
227
+ );
228
+ }
229
+ return port;
230
+ },
231
+ async close() {
232
+ if (closed) return;
233
+ closed = true;
234
+ if (handled) await node.unhandle(protocol).catch(() => undefined);
235
+ await Promise.allSettled([...live].map((p) => p.close()));
236
+ },
237
+ };
238
+ }
@@ -0,0 +1,77 @@
1
+ import type { Connection, DialProtocolOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
+ import type { Multiaddr } from "@multiformats/multiaddr";
3
+
4
+ /** The early-stream guard of `connect()`, per local node and remote peer. */
5
+ interface StreamGate {
6
+ /**
7
+ * A stream has negotiated a protocol on an open connection: the remote is ready.
8
+ * `connection` is that connection when the node can tell (`getConnections`), so a
9
+ * burst after it closes is gated again; a node that offers only `dialProtocol`
10
+ * stays proven.
11
+ */
12
+ proven: { connection: Connection | null } | null;
13
+ /** The stream being opened while no connection is proven; the others wait for it. */
14
+ firstOpening: Promise<Stream> | null;
15
+ }
16
+
17
+ const streamGates = new WeakMap<Libp2p, Map<string, StreamGate>>();
18
+
19
+ function streamGateFor(node: Libp2p, peer: PeerId | Multiaddr): StreamGate {
20
+ let gates = streamGates.get(node);
21
+ if (gates == null) {
22
+ gates = new Map();
23
+ streamGates.set(node, gates);
24
+ }
25
+ const key = peer.toString();
26
+ let gate = gates.get(key);
27
+ if (gate == null) {
28
+ gate = { proven: null, firstOpening: null };
29
+ gates.set(key, gate);
30
+ }
31
+ return gate;
32
+ }
33
+
34
+ function isProven(gate: StreamGate): boolean {
35
+ if (gate.proven == null) return false;
36
+ return gate.proven.connection == null || gate.proven.connection.status === "open";
37
+ }
38
+
39
+ /**
40
+ * The connection carrying `stream`, when `node` can list its connections.
41
+ * Callers may pass a narrow stand-in that offers only `dialProtocol` (httpeers
42
+ * routes streams onto a kept relay circuit that way), so nothing else is required.
43
+ */
44
+ function connectionOf(node: Libp2p, stream: Stream): Connection | null {
45
+ if (typeof node.getConnections !== "function") return null;
46
+ return node.getConnections().find((connection) => connection.streams.includes(stream)) ?? null;
47
+ }
48
+
49
+ /**
50
+ * Opens a stream to `peer` for `proto`, one at a time until the remote has proven ready: a burst
51
+ * of streams on a connection the remote is still upgrading makes yamux abort the whole muxer
52
+ * (`maxEarlyStreams`). Shared by `connect()` and `libp2pPortMux`, and by every caller dialling
53
+ * the same peer from the same node, since they share its connection.
54
+ */
55
+ export async function openGatedStream(
56
+ node: Libp2p,
57
+ peer: PeerId | Multiaddr,
58
+ proto: string,
59
+ dialOptions?: DialProtocolOptions,
60
+ ): Promise<Stream> {
61
+ const gate = streamGateFor(node, peer);
62
+ while (!isProven(gate)) {
63
+ if (gate.firstOpening == null) {
64
+ const opening = node.dialProtocol(peer, [proto], dialOptions);
65
+ gate.firstOpening = opening;
66
+ try {
67
+ const stream = await opening;
68
+ gate.proven = { connection: connectionOf(node, stream) };
69
+ return stream;
70
+ } finally {
71
+ gate.firstOpening = null;
72
+ }
73
+ }
74
+ await gate.firstOpening.catch(() => undefined);
75
+ }
76
+ return node.dialProtocol(peer, [proto], dialOptions);
77
+ }