@livestore/webmesh 0.3.0-dev.2 → 0.3.0-dev.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/channel/message-channel-internal.d.ts +26 -0
- package/dist/channel/message-channel-internal.d.ts.map +1 -0
- package/dist/channel/message-channel-internal.js +217 -0
- package/dist/channel/message-channel-internal.js.map +1 -0
- package/dist/channel/message-channel.d.ts +21 -19
- package/dist/channel/message-channel.d.ts.map +1 -1
- package/dist/channel/message-channel.js +132 -162
- package/dist/channel/message-channel.js.map +1 -1
- package/dist/channel/proxy-channel.d.ts +2 -2
- package/dist/channel/proxy-channel.d.ts.map +1 -1
- package/dist/channel/proxy-channel.js +30 -11
- package/dist/channel/proxy-channel.js.map +1 -1
- package/dist/common.d.ts +32 -5
- package/dist/common.d.ts.map +1 -1
- package/dist/common.js +2 -1
- package/dist/common.js.map +1 -1
- package/dist/mesh-schema.d.ts +68 -2
- package/dist/mesh-schema.d.ts.map +1 -1
- package/dist/mesh-schema.js +53 -4
- package/dist/mesh-schema.js.map +1 -1
- package/dist/node.d.ts +31 -9
- package/dist/node.d.ts.map +1 -1
- package/dist/node.js +225 -49
- package/dist/node.js.map +1 -1
- package/dist/node.test.d.ts +1 -1
- package/dist/node.test.d.ts.map +1 -1
- package/dist/node.test.js +384 -149
- package/dist/node.test.js.map +1 -1
- package/dist/websocket-connection.d.ts +5 -6
- package/dist/websocket-connection.d.ts.map +1 -1
- package/dist/websocket-connection.js +21 -26
- package/dist/websocket-connection.js.map +1 -1
- package/dist/websocket-server.d.ts.map +1 -1
- package/dist/websocket-server.js +17 -3
- package/dist/websocket-server.js.map +1 -1
- package/package.json +7 -6
- package/src/channel/message-channel-internal.ts +356 -0
- package/src/channel/message-channel.ts +190 -310
- package/src/channel/proxy-channel.ts +257 -229
- package/src/common.ts +4 -2
- package/src/mesh-schema.ts +60 -4
- package/src/node.test.ts +544 -179
- package/src/node.ts +363 -69
- package/src/websocket-connection.ts +96 -95
- package/src/websocket-server.ts +20 -3
- package/tmp/pack.tgz +0 -0
package/src/node.ts
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
|
|
2
|
-
import type { Scope } from '@livestore/utils/effect'
|
|
1
|
+
import { indent, LS_DEV, shouldNeverHappen } from '@livestore/utils'
|
|
3
2
|
import {
|
|
4
3
|
Cause,
|
|
4
|
+
Deferred,
|
|
5
5
|
Duration,
|
|
6
6
|
Effect,
|
|
7
|
+
Exit,
|
|
7
8
|
Fiber,
|
|
8
9
|
Option,
|
|
9
10
|
PubSub,
|
|
10
11
|
Queue,
|
|
11
12
|
Schema,
|
|
13
|
+
Scope,
|
|
12
14
|
Stream,
|
|
13
15
|
WebChannel,
|
|
14
16
|
} from '@livestore/utils/effect'
|
|
@@ -17,18 +19,24 @@ import { makeMessageChannel } from './channel/message-channel.js'
|
|
|
17
19
|
import { makeProxyChannel } from './channel/proxy-channel.js'
|
|
18
20
|
import type { ChannelKey, MeshNodeName, MessageQueueItem, ProxyQueueItem } from './common.js'
|
|
19
21
|
import { ConnectionAlreadyExistsError, packetAsOtelAttributes } from './common.js'
|
|
20
|
-
import * as
|
|
22
|
+
import * as WebmeshSchema from './mesh-schema.js'
|
|
21
23
|
import { TimeoutSet } from './utils.js'
|
|
22
24
|
|
|
23
|
-
type ConnectionChannel = WebChannel.WebChannel<typeof
|
|
25
|
+
type ConnectionChannel = WebChannel.WebChannel<typeof WebmeshSchema.Packet.Type, typeof WebmeshSchema.Packet.Type>
|
|
24
26
|
|
|
25
|
-
export interface MeshNode {
|
|
26
|
-
nodeName:
|
|
27
|
+
export interface MeshNode<TName extends MeshNodeName = MeshNodeName> {
|
|
28
|
+
nodeName: TName
|
|
27
29
|
|
|
28
30
|
connectionKeys: Effect.Effect<Set<MeshNodeName>>
|
|
29
31
|
|
|
30
32
|
debug: {
|
|
31
|
-
|
|
33
|
+
print: () => void
|
|
34
|
+
/** Sends a ping message to all connected nodes and channels */
|
|
35
|
+
ping: (payload?: string) => void
|
|
36
|
+
/**
|
|
37
|
+
* Requests the topology of the network from all connected nodes
|
|
38
|
+
*/
|
|
39
|
+
requestTopology: (timeoutMs?: number) => Promise<void>
|
|
32
40
|
}
|
|
33
41
|
|
|
34
42
|
/**
|
|
@@ -58,7 +66,16 @@ export interface MeshNode {
|
|
|
58
66
|
/**
|
|
59
67
|
* Tries to broker a MessageChannel connection between the nodes, otherwise will proxy messages via hop-nodes
|
|
60
68
|
*
|
|
61
|
-
* For a channel to successfully open, both sides need to have a connection and call `makeChannel
|
|
69
|
+
* For a channel to successfully open, both sides need to have a connection and call `makeChannel`.
|
|
70
|
+
*
|
|
71
|
+
* Example:
|
|
72
|
+
* ```ts
|
|
73
|
+
* // Code on node A
|
|
74
|
+
* const channel = nodeA.makeChannel({ target: 'B', channelName: 'my-channel', schema: ... })
|
|
75
|
+
*
|
|
76
|
+
* // Code on node B
|
|
77
|
+
* const channel = nodeB.makeChannel({ target: 'A', channelName: 'my-channel', schema: ... })
|
|
78
|
+
* ```
|
|
62
79
|
*/
|
|
63
80
|
makeChannel: <MsgListen, MsgSend>(args: {
|
|
64
81
|
target: MeshNodeName
|
|
@@ -84,9 +101,20 @@ export interface MeshNode {
|
|
|
84
101
|
*/
|
|
85
102
|
timeout?: Duration.DurationInput
|
|
86
103
|
}) => Effect.Effect<WebChannel.WebChannel<MsgListen, MsgSend>, never, Scope.Scope>
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Creates a WebChannel that is broadcasted to all connected nodes.
|
|
107
|
+
* Messages won't be buffered for nodes that join the network after the broadcast channel has been created.
|
|
108
|
+
*/
|
|
109
|
+
makeBroadcastChannel: <Msg>(args: {
|
|
110
|
+
channelName: string
|
|
111
|
+
schema: Schema.Schema<Msg, any>
|
|
112
|
+
}) => Effect.Effect<WebChannel.WebChannel<Msg, Msg>, never, Scope.Scope>
|
|
87
113
|
}
|
|
88
114
|
|
|
89
|
-
export const makeMeshNode =
|
|
115
|
+
export const makeMeshNode = <TName extends MeshNodeName>(
|
|
116
|
+
nodeName: TName,
|
|
117
|
+
): Effect.Effect<MeshNode<TName>, never, Scope.Scope> =>
|
|
90
118
|
Effect.gen(function* () {
|
|
91
119
|
const connectionChannels = new Map<
|
|
92
120
|
MeshNodeName,
|
|
@@ -105,9 +133,27 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
105
133
|
// Effect.acquireRelease(Queue.shutdown),
|
|
106
134
|
// )
|
|
107
135
|
|
|
108
|
-
const channelMap = new Map<
|
|
136
|
+
const channelMap = new Map<
|
|
137
|
+
ChannelKey,
|
|
138
|
+
{
|
|
139
|
+
queue: Queue.Queue<MessageQueueItem | ProxyQueueItem>
|
|
140
|
+
/** This reference is only kept for debugging purposes */
|
|
141
|
+
debugInfo:
|
|
142
|
+
| {
|
|
143
|
+
channel: WebChannel.WebChannel<any, any>
|
|
144
|
+
target: MeshNodeName
|
|
145
|
+
}
|
|
146
|
+
| undefined
|
|
147
|
+
}
|
|
148
|
+
>()
|
|
149
|
+
|
|
150
|
+
type RequestId = string
|
|
151
|
+
const topologyRequestsMap = new Map<RequestId, Map<MeshNodeName, Set<MeshNodeName>>>()
|
|
109
152
|
|
|
110
|
-
|
|
153
|
+
type BroadcastChannelName = string
|
|
154
|
+
const broadcastChannelListenQueueMap = new Map<BroadcastChannelName, Queue.Queue<any>>()
|
|
155
|
+
|
|
156
|
+
const checkTransferableConnections = (packet: typeof WebmeshSchema.MessageChannelPacket.Type) => {
|
|
111
157
|
if (
|
|
112
158
|
(packet._tag === 'MessageChannelRequest' &&
|
|
113
159
|
(connectionChannels.size === 0 ||
|
|
@@ -116,7 +162,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
116
162
|
// ... or if no forward-connections support transferables
|
|
117
163
|
![...connectionChannels.values()].some((c) => c.channel.supportsTransferables === true)
|
|
118
164
|
) {
|
|
119
|
-
return
|
|
165
|
+
return WebmeshSchema.MessageChannelResponseNoTransferables.make({
|
|
120
166
|
reqId: packet.id,
|
|
121
167
|
channelName: packet.channelName,
|
|
122
168
|
// NOTE for now we're "pretending" that the message is coming from the target node
|
|
@@ -131,9 +177,11 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
131
177
|
}
|
|
132
178
|
}
|
|
133
179
|
|
|
134
|
-
const sendPacket = (packet: typeof
|
|
180
|
+
const sendPacket = (packet: typeof WebmeshSchema.Packet.Type) =>
|
|
135
181
|
Effect.gen(function* () {
|
|
136
|
-
|
|
182
|
+
// yield* Effect.log(`${nodeName}: sendPacket:${packet._tag} [${packet.id}]`)
|
|
183
|
+
|
|
184
|
+
if (Schema.is(WebmeshSchema.NetworkConnectionAdded)(packet)) {
|
|
137
185
|
yield* Effect.spanEvent('NetworkConnectionAdded', { packet, nodeName })
|
|
138
186
|
yield* PubSub.publish(newConnectionAvailablePubSub, packet.target)
|
|
139
187
|
|
|
@@ -145,6 +193,89 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
145
193
|
return
|
|
146
194
|
}
|
|
147
195
|
|
|
196
|
+
if (Schema.is(WebmeshSchema.BroadcastChannelPacket)(packet)) {
|
|
197
|
+
const connectionsToForwardTo = Array.from(connectionChannels)
|
|
198
|
+
.filter(([name]) => !packet.hops.includes(name))
|
|
199
|
+
.map(([_, con]) => con.channel)
|
|
200
|
+
|
|
201
|
+
const adjustedPacket = {
|
|
202
|
+
...packet,
|
|
203
|
+
hops: [...packet.hops, nodeName],
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
yield* Effect.forEach(connectionsToForwardTo, (con) => con.send(adjustedPacket), { concurrency: 'unbounded' })
|
|
207
|
+
|
|
208
|
+
// Don't emit the packet to the own node listen queue
|
|
209
|
+
if (packet.source === nodeName) {
|
|
210
|
+
return
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const queue = broadcastChannelListenQueueMap.get(packet.channelName)
|
|
214
|
+
// In case this node is listening to this channel, add the packet to the listen queue
|
|
215
|
+
if (queue !== undefined) {
|
|
216
|
+
yield* Queue.offer(queue, packet)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
return
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (Schema.is(WebmeshSchema.NetworkConnectionTopologyRequest)(packet)) {
|
|
223
|
+
if (packet.source !== nodeName) {
|
|
224
|
+
const backConnectionName =
|
|
225
|
+
packet.hops.at(-1) ?? shouldNeverHappen(`${nodeName}: Expected hops for packet`, packet)
|
|
226
|
+
const backConnectionChannel = connectionChannels.get(backConnectionName)!.channel
|
|
227
|
+
|
|
228
|
+
// Respond with own connection info
|
|
229
|
+
const response = WebmeshSchema.NetworkConnectionTopologyResponse.make({
|
|
230
|
+
reqId: packet.id,
|
|
231
|
+
source: packet.source,
|
|
232
|
+
target: packet.target,
|
|
233
|
+
remainingHops: packet.hops.slice(0, -1),
|
|
234
|
+
nodeName,
|
|
235
|
+
connections: Array.from(connectionChannels.keys()),
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
yield* backConnectionChannel.send(response)
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Forward the packet to all connections except the already visited ones
|
|
242
|
+
const connectionsToForwardTo = Array.from(connectionChannels)
|
|
243
|
+
.filter(([name]) => !packet.hops.includes(name))
|
|
244
|
+
.map(([_, con]) => con.channel)
|
|
245
|
+
|
|
246
|
+
const adjustedPacket = {
|
|
247
|
+
...packet,
|
|
248
|
+
hops: [...packet.hops, nodeName],
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
yield* Effect.forEach(connectionsToForwardTo, (con) => con.send(adjustedPacket), { concurrency: 'unbounded' })
|
|
252
|
+
|
|
253
|
+
return
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
if (Schema.is(WebmeshSchema.NetworkConnectionTopologyResponse)(packet)) {
|
|
257
|
+
if (packet.source === nodeName) {
|
|
258
|
+
const topologyRequestItem = topologyRequestsMap.get(packet.reqId)!
|
|
259
|
+
topologyRequestItem.set(packet.nodeName, new Set(packet.connections))
|
|
260
|
+
} else {
|
|
261
|
+
const remainingHops = packet.remainingHops
|
|
262
|
+
// Forwarding the response to the original sender via the route back
|
|
263
|
+
const routeBack =
|
|
264
|
+
remainingHops.at(-1) ?? shouldNeverHappen(`${nodeName}: Expected remaining hops for packet`, packet)
|
|
265
|
+
const connectionChannel =
|
|
266
|
+
connectionChannels.get(routeBack)?.channel ??
|
|
267
|
+
shouldNeverHappen(
|
|
268
|
+
`${nodeName}: Expected connection channel (${routeBack}) for packet`,
|
|
269
|
+
packet,
|
|
270
|
+
'Available connections:',
|
|
271
|
+
Array.from(connectionChannels.keys()),
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
yield* connectionChannel.send({ ...packet, remainingHops: packet.remainingHops.slice(0, -1) })
|
|
275
|
+
}
|
|
276
|
+
return
|
|
277
|
+
}
|
|
278
|
+
|
|
148
279
|
// We have a direct connection to the target node
|
|
149
280
|
if (connectionChannels.has(packet.target)) {
|
|
150
281
|
const connectionChannel = connectionChannels.get(packet.target)!.channel
|
|
@@ -155,7 +286,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
155
286
|
// eslint-disable-next-line unicorn/no-negated-condition
|
|
156
287
|
else if (packet.remainingHops !== undefined) {
|
|
157
288
|
const hopTarget =
|
|
158
|
-
packet.remainingHops
|
|
289
|
+
packet.remainingHops.at(-1) ?? shouldNeverHappen(`${nodeName}: Expected remaining hops for packet`, packet)
|
|
159
290
|
const connectionChannel = connectionChannels.get(hopTarget)?.channel
|
|
160
291
|
|
|
161
292
|
if (connectionChannel === undefined) {
|
|
@@ -168,7 +299,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
168
299
|
|
|
169
300
|
yield* connectionChannel.send({
|
|
170
301
|
...packet,
|
|
171
|
-
remainingHops: packet.remainingHops.slice(1),
|
|
302
|
+
remainingHops: packet.remainingHops.slice(0, -1),
|
|
172
303
|
hops: [...packet.hops, nodeName],
|
|
173
304
|
})
|
|
174
305
|
}
|
|
@@ -188,6 +319,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
188
319
|
}
|
|
189
320
|
|
|
190
321
|
const packetToSend = { ...packet, hops }
|
|
322
|
+
// console.debug(nodeName, 'sendPacket:forwarding', packetToSend)
|
|
191
323
|
|
|
192
324
|
yield* Effect.forEach(connectionsToForwardTo, (con) => con.send(packetToSend), { concurrency: 'unbounded' })
|
|
193
325
|
}
|
|
@@ -219,57 +351,68 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
219
351
|
Stream.flatten(),
|
|
220
352
|
Stream.tap((message) =>
|
|
221
353
|
Effect.gen(function* () {
|
|
222
|
-
const packet = yield* Schema.decodeUnknown(
|
|
354
|
+
const packet = yield* Schema.decodeUnknown(WebmeshSchema.Packet)(message)
|
|
223
355
|
|
|
224
356
|
// console.debug(nodeName, 'received', packet._tag, packet.source, packet.target)
|
|
225
357
|
|
|
226
358
|
if (handledPacketIds.has(packet.id)) return
|
|
227
359
|
handledPacketIds.add(packet.id)
|
|
228
360
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
361
|
+
switch (packet._tag) {
|
|
362
|
+
case 'NetworkConnectionAdded':
|
|
363
|
+
case 'NetworkConnectionTopologyRequest':
|
|
364
|
+
case 'NetworkConnectionTopologyResponse': {
|
|
365
|
+
yield* sendPacket(packet)
|
|
233
366
|
|
|
234
|
-
|
|
235
|
-
const queue = yield* Queue.unbounded<MessageQueueItem | ProxyQueueItem>().pipe(
|
|
236
|
-
Effect.acquireRelease(Queue.shutdown),
|
|
237
|
-
)
|
|
238
|
-
channelMap.set(channelKey, { queue })
|
|
367
|
+
break
|
|
239
368
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
})
|
|
268
|
-
|
|
369
|
+
default: {
|
|
370
|
+
if (packet.target === nodeName) {
|
|
371
|
+
const channelKey = `target:${packet.source}, channelName:${packet.channelName}` satisfies ChannelKey
|
|
372
|
+
|
|
373
|
+
if (!channelMap.has(channelKey)) {
|
|
374
|
+
const queue = yield* Queue.unbounded<MessageQueueItem | ProxyQueueItem>().pipe(
|
|
375
|
+
Effect.acquireRelease(Queue.shutdown),
|
|
376
|
+
)
|
|
377
|
+
channelMap.set(channelKey, { queue, debugInfo: undefined })
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const queue = channelMap.get(channelKey)!.queue
|
|
381
|
+
|
|
382
|
+
const respondToSender = (outgoingPacket: typeof WebmeshSchema.Packet.Type) =>
|
|
383
|
+
connectionChannel
|
|
384
|
+
.send(outgoingPacket)
|
|
385
|
+
.pipe(
|
|
386
|
+
Effect.withSpan(
|
|
387
|
+
`respondToSender:${outgoingPacket._tag}:${outgoingPacket.source}→${outgoingPacket.target}`,
|
|
388
|
+
{ attributes: packetAsOtelAttributes(outgoingPacket) },
|
|
389
|
+
),
|
|
390
|
+
Effect.orDie,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
if (Schema.is(WebmeshSchema.ProxyChannelPacket)(packet)) {
|
|
394
|
+
yield* Queue.offer(queue, { packet, respondToSender })
|
|
395
|
+
} else if (Schema.is(WebmeshSchema.MessageChannelPacket)(packet)) {
|
|
396
|
+
yield* Queue.offer(queue, { packet, respondToSender })
|
|
397
|
+
}
|
|
398
|
+
} else {
|
|
399
|
+
if (Schema.is(WebmeshSchema.MessageChannelPacket)(packet)) {
|
|
400
|
+
const noTransferableResponse = checkTransferableConnections(packet)
|
|
401
|
+
if (noTransferableResponse !== undefined) {
|
|
402
|
+
yield* Effect.spanEvent(
|
|
403
|
+
`No transferable connections found for ${packet.source}→${packet.target}`,
|
|
404
|
+
)
|
|
405
|
+
return yield* connectionChannel.send(noTransferableResponse).pipe(
|
|
406
|
+
Effect.withSpan(`sendNoTransferableResponse:${packet.source}→${packet.target}`, {
|
|
407
|
+
attributes: packetAsOtelAttributes(noTransferableResponse),
|
|
408
|
+
}),
|
|
409
|
+
)
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
yield* sendPacket(packet)
|
|
269
414
|
}
|
|
270
415
|
}
|
|
271
|
-
|
|
272
|
-
yield* sendPacket(packet)
|
|
273
416
|
}
|
|
274
417
|
}),
|
|
275
418
|
),
|
|
@@ -281,11 +424,11 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
281
424
|
|
|
282
425
|
connectionChannels.set(targetNodeName, { channel: connectionChannel, listenFiber })
|
|
283
426
|
|
|
284
|
-
const connectionAddedPacket =
|
|
427
|
+
const connectionAddedPacket = WebmeshSchema.NetworkConnectionAdded.make({
|
|
285
428
|
source: nodeName,
|
|
286
429
|
target: targetNodeName,
|
|
287
430
|
})
|
|
288
|
-
yield* sendPacket(connectionAddedPacket).pipe(Effect.
|
|
431
|
+
yield* sendPacket(connectionAddedPacket).pipe(Effect.orDie)
|
|
289
432
|
}).pipe(
|
|
290
433
|
Effect.withSpan(`addConnection:${nodeName}→${targetNodeName}`, {
|
|
291
434
|
attributes: { supportsTransferables: connectionChannel.supportsTransferables },
|
|
@@ -315,13 +458,18 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
315
458
|
}) =>
|
|
316
459
|
Effect.gen(function* () {
|
|
317
460
|
const schema = WebChannel.mapSchema(inputSchema)
|
|
318
|
-
const channelKey =
|
|
461
|
+
const channelKey = `target:${target}, channelName:${channelName}` satisfies ChannelKey
|
|
319
462
|
|
|
320
|
-
if (
|
|
463
|
+
if (channelMap.has(channelKey)) {
|
|
464
|
+
const existingChannel = channelMap.get(channelKey)!.debugInfo?.channel
|
|
465
|
+
if (existingChannel) {
|
|
466
|
+
shouldNeverHappen(`Channel ${channelKey} already exists`, existingChannel)
|
|
467
|
+
}
|
|
468
|
+
} else {
|
|
321
469
|
const queue = yield* Queue.unbounded<MessageQueueItem | ProxyQueueItem>().pipe(
|
|
322
470
|
Effect.acquireRelease(Queue.shutdown),
|
|
323
471
|
)
|
|
324
|
-
channelMap.set(channelKey, { queue })
|
|
472
|
+
channelMap.set(channelKey, { queue, debugInfo: undefined })
|
|
325
473
|
}
|
|
326
474
|
|
|
327
475
|
const queue = channelMap.get(channelKey)!.queue as Queue.Queue<any>
|
|
@@ -329,12 +477,23 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
329
477
|
yield* Effect.addFinalizer(() => Effect.sync(() => channelMap.delete(channelKey)))
|
|
330
478
|
|
|
331
479
|
if (mode === 'messagechannel') {
|
|
332
|
-
|
|
480
|
+
const incomingPacketsQueue = yield* Queue.unbounded<any>().pipe(Effect.acquireRelease(Queue.shutdown))
|
|
481
|
+
|
|
482
|
+
// We're we're draining the queue into another new queue.
|
|
483
|
+
// It's a bit of a mystery why this is needed, since the unit tests also work without it.
|
|
484
|
+
// But for the LiveStore devtools to actually work, we need to do this.
|
|
485
|
+
// We should figure out some day why this is needed and further simplify if possible.
|
|
486
|
+
yield* Queue.takeBetween(queue, 1, 10).pipe(
|
|
487
|
+
Effect.tap((_) => Queue.offerAll(incomingPacketsQueue, _)),
|
|
488
|
+
Effect.forever,
|
|
489
|
+
Effect.tapCauseLogPretty,
|
|
490
|
+
Effect.forkScoped,
|
|
491
|
+
)
|
|
333
492
|
|
|
334
493
|
// NOTE already retries internally when transferables are required
|
|
335
|
-
|
|
494
|
+
const { webChannel, initialConnectionDeferred } = yield* makeMessageChannel({
|
|
336
495
|
nodeName,
|
|
337
|
-
|
|
496
|
+
incomingPacketsQueue,
|
|
338
497
|
newConnectionAvailablePubSub,
|
|
339
498
|
target,
|
|
340
499
|
channelName,
|
|
@@ -342,8 +501,14 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
342
501
|
sendPacket,
|
|
343
502
|
checkTransferableConnections,
|
|
344
503
|
})
|
|
504
|
+
|
|
505
|
+
channelMap.set(channelKey, { queue, debugInfo: { channel: webChannel, target } })
|
|
506
|
+
|
|
507
|
+
yield* initialConnectionDeferred
|
|
508
|
+
|
|
509
|
+
return webChannel
|
|
345
510
|
} else {
|
|
346
|
-
|
|
511
|
+
const channel = yield* makeProxyChannel({
|
|
347
512
|
nodeName,
|
|
348
513
|
newConnectionAvailablePubSub,
|
|
349
514
|
target,
|
|
@@ -352,24 +517,153 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
|
|
|
352
517
|
queue,
|
|
353
518
|
sendPacket,
|
|
354
519
|
})
|
|
520
|
+
|
|
521
|
+
channelMap.set(channelKey, { queue, debugInfo: { channel, target } })
|
|
522
|
+
|
|
523
|
+
return channel
|
|
355
524
|
}
|
|
356
525
|
}).pipe(
|
|
526
|
+
// Effect.timeout(timeout),
|
|
357
527
|
Effect.withSpanScoped(`makeChannel:${nodeName}→${target}(${channelName})`, {
|
|
358
528
|
attributes: { target, channelName, mode, timeout },
|
|
359
529
|
}),
|
|
360
530
|
Effect.annotateLogs({ nodeName }),
|
|
361
531
|
)
|
|
362
532
|
|
|
533
|
+
const makeBroadcastChannel: MeshNode['makeBroadcastChannel'] = ({ channelName, schema }) =>
|
|
534
|
+
Effect.scopeWithCloseable((scope) =>
|
|
535
|
+
Effect.gen(function* () {
|
|
536
|
+
if (broadcastChannelListenQueueMap.has(channelName)) {
|
|
537
|
+
return shouldNeverHappen(
|
|
538
|
+
`Broadcast channel ${channelName} already exists`,
|
|
539
|
+
broadcastChannelListenQueueMap.get(channelName),
|
|
540
|
+
)
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
const debugInfo = {}
|
|
544
|
+
|
|
545
|
+
const queue = yield* Queue.unbounded<any>().pipe(Effect.acquireRelease(Queue.shutdown))
|
|
546
|
+
broadcastChannelListenQueueMap.set(channelName, queue)
|
|
547
|
+
|
|
548
|
+
const send = (message: any) =>
|
|
549
|
+
Effect.gen(function* () {
|
|
550
|
+
const payload = yield* Schema.encode(schema)(message)
|
|
551
|
+
const packet = WebmeshSchema.BroadcastChannelPacket.make({
|
|
552
|
+
channelName,
|
|
553
|
+
payload,
|
|
554
|
+
source: nodeName,
|
|
555
|
+
target: '-',
|
|
556
|
+
hops: [],
|
|
557
|
+
})
|
|
558
|
+
|
|
559
|
+
yield* sendPacket(packet)
|
|
560
|
+
})
|
|
561
|
+
|
|
562
|
+
const listen = Stream.fromQueue(queue).pipe(
|
|
563
|
+
Stream.filter(Schema.is(WebmeshSchema.BroadcastChannelPacket)),
|
|
564
|
+
Stream.map((_) => Schema.decodeEither(schema)(_.payload)),
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
|
|
568
|
+
|
|
569
|
+
return {
|
|
570
|
+
[WebChannel.WebChannelSymbol]: WebChannel.WebChannelSymbol,
|
|
571
|
+
send,
|
|
572
|
+
listen,
|
|
573
|
+
closedDeferred,
|
|
574
|
+
supportsTransferables: true,
|
|
575
|
+
schema: { listen: schema, send: schema },
|
|
576
|
+
shutdown: Scope.close(scope, Exit.void),
|
|
577
|
+
debugInfo,
|
|
578
|
+
} satisfies WebChannel.WebChannel<any, any>
|
|
579
|
+
}),
|
|
580
|
+
)
|
|
581
|
+
|
|
363
582
|
const connectionKeys: MeshNode['connectionKeys'] = Effect.sync(() => new Set(connectionChannels.keys()))
|
|
364
583
|
|
|
584
|
+
const runtime = yield* Effect.runtime()
|
|
585
|
+
|
|
365
586
|
const debug: MeshNode['debug'] = {
|
|
366
|
-
|
|
587
|
+
print: () => {
|
|
588
|
+
console.log('Webmesh debug info for node:', nodeName)
|
|
589
|
+
|
|
590
|
+
console.log('Connections:', connectionChannels.size)
|
|
591
|
+
for (const [key, value] of connectionChannels) {
|
|
592
|
+
console.log(` ${key}: supportsTransferables=${value.channel.supportsTransferables}`)
|
|
593
|
+
}
|
|
594
|
+
|
|
367
595
|
console.log('Channels:', channelMap.size)
|
|
368
596
|
for (const [key, value] of channelMap) {
|
|
369
|
-
console.log(
|
|
597
|
+
console.log(
|
|
598
|
+
indent(key, 2),
|
|
599
|
+
'\n',
|
|
600
|
+
Object.entries({
|
|
601
|
+
target: value.debugInfo?.target,
|
|
602
|
+
supportsTransferables: value.debugInfo?.channel.supportsTransferables,
|
|
603
|
+
...value.debugInfo?.channel.debugInfo,
|
|
604
|
+
})
|
|
605
|
+
.map(([key, value]) => indent(`${key}=${value}`, 4))
|
|
606
|
+
.join('\n'),
|
|
607
|
+
' ',
|
|
608
|
+
value.debugInfo?.channel,
|
|
609
|
+
'\n',
|
|
610
|
+
indent(`Queue: ${value.queue.unsafeSize().pipe(Option.getOrUndefined)}`, 4),
|
|
611
|
+
value.queue,
|
|
612
|
+
)
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
console.log('Broadcast channels:', broadcastChannelListenQueueMap.size)
|
|
616
|
+
for (const [key, _value] of broadcastChannelListenQueueMap) {
|
|
617
|
+
console.log(indent(key, 2))
|
|
370
618
|
}
|
|
371
619
|
},
|
|
620
|
+
ping: (payload) => {
|
|
621
|
+
Effect.gen(function* () {
|
|
622
|
+
const msg = (via: string) =>
|
|
623
|
+
WebChannel.DebugPingMessage.make({ message: `ping from ${nodeName} via connection ${via}`, payload })
|
|
624
|
+
|
|
625
|
+
for (const [channelName, con] of connectionChannels) {
|
|
626
|
+
yield* Effect.logDebug(`sending ping via connection ${channelName}`)
|
|
627
|
+
yield* con.channel.send(msg(`connection ${channelName}`) as any)
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
for (const [channelKey, channel] of channelMap) {
|
|
631
|
+
if (channel.debugInfo === undefined) continue
|
|
632
|
+
yield* Effect.logDebug(`sending ping via channel ${channelKey}`)
|
|
633
|
+
yield* channel.debugInfo.channel.send(msg(`channel ${channelKey}`) as any)
|
|
634
|
+
}
|
|
635
|
+
}).pipe(Effect.provide(runtime), Effect.tapCauseLogPretty, Effect.runFork)
|
|
636
|
+
},
|
|
637
|
+
requestTopology: (timeoutMs = 1000) =>
|
|
638
|
+
Effect.gen(function* () {
|
|
639
|
+
const packet = WebmeshSchema.NetworkConnectionTopologyRequest.make({
|
|
640
|
+
source: nodeName,
|
|
641
|
+
target: '-',
|
|
642
|
+
hops: [],
|
|
643
|
+
})
|
|
644
|
+
|
|
645
|
+
const item = new Map<MeshNodeName, Set<MeshNodeName>>()
|
|
646
|
+
item.set(nodeName, new Set(connectionChannels.keys()))
|
|
647
|
+
topologyRequestsMap.set(packet.id, item)
|
|
648
|
+
|
|
649
|
+
yield* sendPacket(packet)
|
|
650
|
+
|
|
651
|
+
yield* Effect.logDebug(`Waiting ${timeoutMs}ms for topology response`)
|
|
652
|
+
yield* Effect.sleep(timeoutMs)
|
|
653
|
+
|
|
654
|
+
for (const [key, value] of item) {
|
|
655
|
+
yield* Effect.logDebug(`node '${key}' is connected to: ${Array.from(value.values()).join(', ')}`)
|
|
656
|
+
}
|
|
657
|
+
}).pipe(Effect.provide(runtime), Effect.tapCauseLogPretty, Effect.runPromise),
|
|
372
658
|
}
|
|
373
659
|
|
|
374
|
-
return {
|
|
660
|
+
return {
|
|
661
|
+
nodeName,
|
|
662
|
+
addConnection,
|
|
663
|
+
removeConnection,
|
|
664
|
+
makeChannel,
|
|
665
|
+
makeBroadcastChannel,
|
|
666
|
+
connectionKeys,
|
|
667
|
+
debug,
|
|
668
|
+
} satisfies MeshNode
|
|
375
669
|
}).pipe(Effect.withSpan(`makeMeshNode:${nodeName}`))
|