@livestore/webmesh 0.3.0-dev.19 → 0.3.0-dev.21

package/src/node.ts

@@ -1,14 +1,16 @@
 import { indent, LS_DEV, shouldNeverHappen } from '@livestore/utils'
-import type { Scope } from '@livestore/utils/effect'
 import {
   Cause,
+  Deferred,
   Duration,
   Effect,
+  Exit,
   Fiber,
   Option,
   PubSub,
   Queue,
   Schema,
+  Scope,
   Stream,
   WebChannel,
 } from '@livestore/utils/effect'
@@ -17,13 +19,13 @@ import { makeMessageChannel } from './channel/message-channel.js'
 import { makeProxyChannel } from './channel/proxy-channel.js'
 import type { ChannelKey, MeshNodeName, MessageQueueItem, ProxyQueueItem } from './common.js'
 import { ConnectionAlreadyExistsError, packetAsOtelAttributes } from './common.js'
-import * as MeshSchema from './mesh-schema.js'
+import * as WebmeshSchema from './mesh-schema.js'
 import { TimeoutSet } from './utils.js'
 
-type ConnectionChannel = WebChannel.WebChannel<typeof MeshSchema.Packet.Type, typeof MeshSchema.Packet.Type>
+type ConnectionChannel = WebChannel.WebChannel<typeof WebmeshSchema.Packet.Type, typeof WebmeshSchema.Packet.Type>
 
-export interface MeshNode {
-  nodeName: MeshNodeName
+export interface MeshNode<TName extends MeshNodeName = MeshNodeName> {
+  nodeName: TName
 
   connectionKeys: Effect.Effect<Set<MeshNodeName>>
 
@@ -31,6 +33,10 @@ export interface MeshNode {
     print: () => void
     /** Sends a ping message to all connected nodes and channels */
     ping: (payload?: string) => void
+    /**
+     * Requests the topology of the network from all connected nodes
+     */
+    requestTopology: (timeoutMs?: number) => Promise<void>
   }
 
   /**
@@ -95,9 +101,20 @@ export interface MeshNode {
      */
     timeout?: Duration.DurationInput
   }) => Effect.Effect<WebChannel.WebChannel<MsgListen, MsgSend>, never, Scope.Scope>
+
+  /**
+   * Creates a WebChannel that is broadcasted to all connected nodes.
+   * Messages won't be buffered for nodes that join the network after the broadcast channel has been created.
+   */
+  makeBroadcastChannel: <Msg>(args: {
+    channelName: string
+    schema: Schema.Schema<Msg, any>
+  }) => Effect.Effect<WebChannel.WebChannel<Msg, Msg>, never, Scope.Scope>
 }
 
-export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, never, Scope.Scope> =>
+export const makeMeshNode = <TName extends MeshNodeName>(
+  nodeName: TName,
+): Effect.Effect<MeshNode<TName>, never, Scope.Scope> =>
   Effect.gen(function* () {
     const connectionChannels = new Map<
       MeshNodeName,
@@ -130,7 +147,13 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
       }
     >()
 
-    const checkTransferableConnections = (packet: typeof MeshSchema.MessageChannelPacket.Type) => {
+    type RequestId = string
+    const topologyRequestsMap = new Map<RequestId, Map<MeshNodeName, Set<MeshNodeName>>>()
+
+    type BroadcastChannelName = string
+    const broadcastChannelListenQueueMap = new Map<BroadcastChannelName, Queue.Queue<any>>()
+
+    const checkTransferableConnections = (packet: typeof WebmeshSchema.MessageChannelPacket.Type) => {
       if (
         (packet._tag === 'MessageChannelRequest' &&
           (connectionChannels.size === 0 ||
@@ -139,7 +162,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
         // ... or if no forward-connections support transferables
         ![...connectionChannels.values()].some((c) => c.channel.supportsTransferables === true)
       ) {
-        return MeshSchema.MessageChannelResponseNoTransferables.make({
+        return WebmeshSchema.MessageChannelResponseNoTransferables.make({
           reqId: packet.id,
           channelName: packet.channelName,
           // NOTE for now we're "pretending" that the message is coming from the target node
@@ -154,11 +177,11 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
       }
     }
 
-    const sendPacket = (packet: typeof MeshSchema.Packet.Type) =>
+    const sendPacket = (packet: typeof WebmeshSchema.Packet.Type) =>
       Effect.gen(function* () {
         // yield* Effect.log(`${nodeName}: sendPacket:${packet._tag} [${packet.id}]`)
 
-        if (Schema.is(MeshSchema.NetworkConnectionAdded)(packet)) {
+        if (Schema.is(WebmeshSchema.NetworkConnectionAdded)(packet)) {
           yield* Effect.spanEvent('NetworkConnectionAdded', { packet, nodeName })
           yield* PubSub.publish(newConnectionAvailablePubSub, packet.target)
 
@@ -170,6 +193,89 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
           return
         }
 
+        if (Schema.is(WebmeshSchema.BroadcastChannelPacket)(packet)) {
+          const connectionsToForwardTo = Array.from(connectionChannels)
+            .filter(([name]) => !packet.hops.includes(name))
+            .map(([_, con]) => con.channel)
+
+          const adjustedPacket = {
+            ...packet,
+            hops: [...packet.hops, nodeName],
+          }
+
+          yield* Effect.forEach(connectionsToForwardTo, (con) => con.send(adjustedPacket), { concurrency: 'unbounded' })
+
+          // Don't emit the packet to the own node listen queue
+          if (packet.source === nodeName) {
+            return
+          }
+
+          const queue = broadcastChannelListenQueueMap.get(packet.channelName)
+          // In case this node is listening to this channel, add the packet to the listen queue
+          if (queue !== undefined) {
+            yield* Queue.offer(queue, packet)
+          }
+
+          return
+        }
+
+        if (Schema.is(WebmeshSchema.NetworkConnectionTopologyRequest)(packet)) {
+          if (packet.source !== nodeName) {
+            const backConnectionName =
+              packet.hops.at(-1) ?? shouldNeverHappen(`${nodeName}: Expected hops for packet`, packet)
+            const backConnectionChannel = connectionChannels.get(backConnectionName)!.channel
+
+            // Respond with own connection info
+            const response = WebmeshSchema.NetworkConnectionTopologyResponse.make({
+              reqId: packet.id,
+              source: packet.source,
+              target: packet.target,
+              remainingHops: packet.hops.slice(0, -1),
+              nodeName,
+              connections: Array.from(connectionChannels.keys()),
+            })
+
+            yield* backConnectionChannel.send(response)
+          }
+
+          // Forward the packet to all connections except the already visited ones
+          const connectionsToForwardTo = Array.from(connectionChannels)
+            .filter(([name]) => !packet.hops.includes(name))
+            .map(([_, con]) => con.channel)
+
+          const adjustedPacket = {
+            ...packet,
+            hops: [...packet.hops, nodeName],
+          }
+
+          yield* Effect.forEach(connectionsToForwardTo, (con) => con.send(adjustedPacket), { concurrency: 'unbounded' })
+
+          return
+        }
+
+        if (Schema.is(WebmeshSchema.NetworkConnectionTopologyResponse)(packet)) {
+          if (packet.source === nodeName) {
+            const topologyRequestItem = topologyRequestsMap.get(packet.reqId)!
+            topologyRequestItem.set(packet.nodeName, new Set(packet.connections))
+          } else {
+            const remainingHops = packet.remainingHops
+            // Forwarding the response to the original sender via the route back
+            const routeBack =
+              remainingHops.at(-1) ?? shouldNeverHappen(`${nodeName}: Expected remaining hops for packet`, packet)
+            const connectionChannel =
+              connectionChannels.get(routeBack)?.channel ??
+              shouldNeverHappen(
+                `${nodeName}: Expected connection channel (${routeBack}) for packet`,
+                packet,
+                'Available connections:',
+                Array.from(connectionChannels.keys()),
+              )
+
+            yield* connectionChannel.send({ ...packet, remainingHops: packet.remainingHops.slice(0, -1) })
+          }
+          return
+        }
+
         // We have a direct connection to the target node
         if (connectionChannels.has(packet.target)) {
           const connectionChannel = connectionChannels.get(packet.target)!.channel
@@ -180,7 +286,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
         // eslint-disable-next-line unicorn/no-negated-condition
         else if (packet.remainingHops !== undefined) {
           const hopTarget =
-            packet.remainingHops[0] ?? shouldNeverHappen(`${nodeName}: Expected remaining hops for packet`, packet)
+            packet.remainingHops.at(-1) ?? shouldNeverHappen(`${nodeName}: Expected remaining hops for packet`, packet)
           const connectionChannel = connectionChannels.get(hopTarget)?.channel
 
           if (connectionChannel === undefined) {
@@ -193,7 +299,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
 
           yield* connectionChannel.send({
             ...packet,
-            remainingHops: packet.remainingHops.slice(1),
+            remainingHops: packet.remainingHops.slice(0, -1),
             hops: [...packet.hops, nodeName],
           })
         }
@@ -245,57 +351,68 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
           Stream.flatten(),
           Stream.tap((message) =>
             Effect.gen(function* () {
-              const packet = yield* Schema.decodeUnknown(MeshSchema.Packet)(message)
+              const packet = yield* Schema.decodeUnknown(WebmeshSchema.Packet)(message)
 
               // console.debug(nodeName, 'received', packet._tag, packet.source, packet.target)
 
               if (handledPacketIds.has(packet.id)) return
               handledPacketIds.add(packet.id)
 
-              if (packet._tag === 'NetworkConnectionAdded') {
-                yield* sendPacket(packet)
-              } else if (packet.target === nodeName) {
-                const channelKey = `${packet.source}-${packet.channelName}` satisfies ChannelKey
+              switch (packet._tag) {
+                case 'NetworkConnectionAdded':
+                case 'NetworkConnectionTopologyRequest':
+                case 'NetworkConnectionTopologyResponse': {
+                  yield* sendPacket(packet)
 
-                if (!channelMap.has(channelKey)) {
-                  const queue = yield* Queue.unbounded<MessageQueueItem | ProxyQueueItem>().pipe(
-                    Effect.acquireRelease(Queue.shutdown),
-                  )
-                  channelMap.set(channelKey, { queue, debugInfo: undefined })
+                  break
                 }
-
-                const queue = channelMap.get(channelKey)!.queue
-
-                const respondToSender = (outgoingPacket: typeof MeshSchema.Packet.Type) =>
-                  connectionChannel
-                    .send(outgoingPacket)
-                    .pipe(
-                      Effect.withSpan(
-                        `respondToSender:${outgoingPacket._tag}:${outgoingPacket.source}→${outgoingPacket.target}`,
-                        { attributes: packetAsOtelAttributes(outgoingPacket) },
-                      ),
-                      Effect.orDie,
-                    )
-
-                if (Schema.is(MeshSchema.ProxyChannelPacket)(packet)) {
-                  yield* Queue.offer(queue, { packet, respondToSender })
-                } else if (Schema.is(MeshSchema.MessageChannelPacket)(packet)) {
-                  yield* Queue.offer(queue, { packet, respondToSender })
-                }
-              } else {
-                if (Schema.is(MeshSchema.MessageChannelPacket)(packet)) {
-                  const noTransferableResponse = checkTransferableConnections(packet)
-                  if (noTransferableResponse !== undefined) {
-                    yield* Effect.spanEvent(`No transferable connections found for ${packet.source}→${packet.target}`)
-                    return yield* connectionChannel.send(noTransferableResponse).pipe(
-                      Effect.withSpan(`sendNoTransferableResponse:${packet.source}→${packet.target}`, {
-                        attributes: packetAsOtelAttributes(noTransferableResponse),
-                      }),
-                    )
+                default: {
+                  if (packet.target === nodeName) {
+                    const channelKey = `target:${packet.source}, channelName:${packet.channelName}` satisfies ChannelKey
+
+                    if (!channelMap.has(channelKey)) {
+                      const queue = yield* Queue.unbounded<MessageQueueItem | ProxyQueueItem>().pipe(
+                        Effect.acquireRelease(Queue.shutdown),
+                      )
+                      channelMap.set(channelKey, { queue, debugInfo: undefined })
+                    }
+
+                    const queue = channelMap.get(channelKey)!.queue
+
+                    const respondToSender = (outgoingPacket: typeof WebmeshSchema.Packet.Type) =>
+                      connectionChannel
+                        .send(outgoingPacket)
+                        .pipe(
+                          Effect.withSpan(
+                            `respondToSender:${outgoingPacket._tag}:${outgoingPacket.source}→${outgoingPacket.target}`,
+                            { attributes: packetAsOtelAttributes(outgoingPacket) },
+                          ),
+                          Effect.orDie,
+                        )
+
+                    if (Schema.is(WebmeshSchema.ProxyChannelPacket)(packet)) {
+                      yield* Queue.offer(queue, { packet, respondToSender })
+                    } else if (Schema.is(WebmeshSchema.MessageChannelPacket)(packet)) {
+                      yield* Queue.offer(queue, { packet, respondToSender })
+                    }
+                  } else {
+                    if (Schema.is(WebmeshSchema.MessageChannelPacket)(packet)) {
+                      const noTransferableResponse = checkTransferableConnections(packet)
+                      if (noTransferableResponse !== undefined) {
+                        yield* Effect.spanEvent(
+                          `No transferable connections found for ${packet.source}→${packet.target}`,
+                        )
+                        return yield* connectionChannel.send(noTransferableResponse).pipe(
+                          Effect.withSpan(`sendNoTransferableResponse:${packet.source}→${packet.target}`, {
+                            attributes: packetAsOtelAttributes(noTransferableResponse),
+                          }),
+                        )
+                      }
+                    }
+
+                    yield* sendPacket(packet)
                   }
                 }
-
-                yield* sendPacket(packet)
               }
             }),
           ),
@@ -307,11 +424,11 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
 
         connectionChannels.set(targetNodeName, { channel: connectionChannel, listenFiber })
 
-        const connectionAddedPacket = MeshSchema.NetworkConnectionAdded.make({
+        const connectionAddedPacket = WebmeshSchema.NetworkConnectionAdded.make({
           source: nodeName,
           target: targetNodeName,
         })
-        yield* sendPacket(connectionAddedPacket).pipe(Effect.ignoreLogged)
+        yield* sendPacket(connectionAddedPacket).pipe(Effect.orDie)
       }).pipe(
         Effect.withSpan(`addConnection:${nodeName}→${targetNodeName}`, {
           attributes: { supportsTransferables: connectionChannel.supportsTransferables },
@@ -341,7 +458,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
     }) =>
       Effect.gen(function* () {
         const schema = WebChannel.mapSchema(inputSchema)
-        const channelKey = `${target}-${channelName}` satisfies ChannelKey
+        const channelKey = `target:${target}, channelName:${channelName}` satisfies ChannelKey
 
         if (channelMap.has(channelKey)) {
           const existingChannel = channelMap.get(channelKey)!.debugInfo?.channel
@@ -413,10 +530,63 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
         Effect.annotateLogs({ nodeName }),
       )
 
+    const makeBroadcastChannel: MeshNode['makeBroadcastChannel'] = ({ channelName, schema }) =>
+      Effect.scopeWithCloseable((scope) =>
+        Effect.gen(function* () {
+          if (broadcastChannelListenQueueMap.has(channelName)) {
+            return shouldNeverHappen(
+              `Broadcast channel ${channelName} already exists`,
+              broadcastChannelListenQueueMap.get(channelName),
+            )
+          }
+
+          const debugInfo = {}
+
+          const queue = yield* Queue.unbounded<any>().pipe(Effect.acquireRelease(Queue.shutdown))
+          broadcastChannelListenQueueMap.set(channelName, queue)
+
+          const send = (message: any) =>
+            Effect.gen(function* () {
+              const payload = yield* Schema.encode(schema)(message)
+              const packet = WebmeshSchema.BroadcastChannelPacket.make({
+                channelName,
+                payload,
+                source: nodeName,
+                target: '-',
+                hops: [],
+              })
+
+              yield* sendPacket(packet)
+            })
+
+          const listen = Stream.fromQueue(queue).pipe(
+            Stream.filter(Schema.is(WebmeshSchema.BroadcastChannelPacket)),
+            Stream.map((_) => Schema.decodeEither(schema)(_.payload)),
+          )
+
+          const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
+
+          return {
+            [WebChannel.WebChannelSymbol]: WebChannel.WebChannelSymbol,
+            send,
+            listen,
+            closedDeferred,
+            supportsTransferables: true,
+            schema: { listen: schema, send: schema },
+            shutdown: Scope.close(scope, Exit.void),
+            debugInfo,
+          } satisfies WebChannel.WebChannel<any, any>
+        }),
+      )
+
     const connectionKeys: MeshNode['connectionKeys'] = Effect.sync(() => new Set(connectionChannels.keys()))
 
+    const runtime = yield* Effect.runtime()
+
     const debug: MeshNode['debug'] = {
       print: () => {
+        console.log('Webmesh debug info for node:', nodeName)
+
         console.log('Connections:', connectionChannels.size)
         for (const [key, value] of connectionChannels) {
           console.log(`  ${key}: supportsTransferables=${value.channel.supportsTransferables}`)
@@ -441,26 +611,59 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
             value.queue,
           )
         }
+
+        console.log('Broadcast channels:', broadcastChannelListenQueueMap.size)
+        for (const [key, _value] of broadcastChannelListenQueueMap) {
+          console.log(indent(key, 2))
+        }
       },
       ping: (payload) => {
         Effect.gen(function* () {
-          const msg = (via: string) => {
-            console.log(`sending message to ${via}`)
-            return WebChannel.DebugPingMessage.make({ message: `ping from ${nodeName} via ${via}`, payload })
-          }
+          const msg = (via: string) =>
+            WebChannel.DebugPingMessage.make({ message: `ping from ${nodeName} via connection ${via}`, payload })
 
-          yield* Effect.forEach(connectionChannels, ([channelName, con]) =>
-            con.channel.send(msg(`connection ${channelName}`) as any),
-          )
+          for (const [channelName, con] of connectionChannels) {
+            yield* Effect.logDebug(`sending ping via connection ${channelName}`)
+            yield* con.channel.send(msg(`connection ${channelName}`) as any)
+          }
 
-          yield* Effect.forEach(
-            channelMap,
-            ([channelKey, channel]) =>
-              channel.debugInfo?.channel.send(msg(`channel ${channelKey}`) as any) ?? Effect.void,
-          )
-        }).pipe(Effect.runFork)
+          for (const [channelKey, channel] of channelMap) {
+            if (channel.debugInfo === undefined) continue
+            yield* Effect.logDebug(`sending ping via channel ${channelKey}`)
+            yield* channel.debugInfo.channel.send(msg(`channel ${channelKey}`) as any)
+          }
+        }).pipe(Effect.provide(runtime), Effect.tapCauseLogPretty, Effect.runFork)
       },
+      requestTopology: (timeoutMs = 1000) =>
+        Effect.gen(function* () {
+          const packet = WebmeshSchema.NetworkConnectionTopologyRequest.make({
+            source: nodeName,
+            target: '-',
+            hops: [],
+          })
+
+          const item = new Map<MeshNodeName, Set<MeshNodeName>>()
+          item.set(nodeName, new Set(connectionChannels.keys()))
+          topologyRequestsMap.set(packet.id, item)
+
+          yield* sendPacket(packet)
+
+          yield* Effect.logDebug(`Waiting ${timeoutMs}ms for topology response`)
+          yield* Effect.sleep(timeoutMs)
+
+          for (const [key, value] of item) {
+            yield* Effect.logDebug(`node '${key}' is connected to: ${Array.from(value.values()).join(', ')}`)
+          }
+        }).pipe(Effect.provide(runtime), Effect.tapCauseLogPretty, Effect.runPromise),
     }
 
-    return { nodeName, addConnection, removeConnection, makeChannel, connectionKeys, debug } satisfies MeshNode
+    return {
+      nodeName,
+      addConnection,
+      removeConnection,
+      makeChannel,
+      makeBroadcastChannel,
+      connectionKeys,
+      debug,
+    } satisfies MeshNode
   }).pipe(Effect.withSpan(`makeMeshNode:${nodeName}`))

package/src/websocket-connection.ts

@@ -82,6 +82,8 @@ export const makeWebSocketConnection = (
         Effect.acquireRelease(Queue.shutdown),
       )
 
+      const schema = WebChannel.mapSchema(WebmeshSchema.Packet)
+
       yield* Stream.fromEventListener<MessageEvent>(socket as any, 'message').pipe(
         Stream.map((msg) => Schema.decodeUnknownEither(MessageMsgPack)(new Uint8Array(msg.data))),
         Stream.flatten(),
@@ -90,7 +92,7 @@ export const makeWebSocketConnection = (
             if (msg._tag === 'WSConnectionInit') {
               yield* Deferred.succeed(fromDeferred, msg.from)
             } else {
-              const decodedPayload = yield* Schema.decode(WebmeshSchema.Packet)(msg.payload)
+              const decodedPayload = yield* Schema.decode(schema.listen)(msg.payload)
               yield* Queue.offer(listenQueue, decodedPayload)
             }
           }),
@@ -133,18 +135,21 @@ export const makeWebSocketConnection = (
       const send = (message: typeof WebmeshSchema.Packet.Type) =>
         Effect.gen(function* () {
           yield* isConnectedLatch.await
-          const payload = yield* Schema.encode(WebmeshSchema.Packet)(message)
+          const payload = yield* Schema.encode(schema.send)(message)
           socket.send(Schema.encodeSync(MessageMsgPack)({ _tag: 'WSConnectionPayload', payload, from }))
         })
 
-      const listen = Stream.fromQueue(listenQueue).pipe(Stream.map(Either.right))
+      const listen = Stream.fromQueue(listenQueue).pipe(
+        Stream.map(Either.right),
+        WebChannel.listenToDebugPing('websocket-connection'),
+      )
 
       const webChannel = {
         [WebChannel.WebChannelSymbol]: WebChannel.WebChannelSymbol,
         send,
         listen,
         closedDeferred,
-        schema: { listen: WebmeshSchema.Packet, send: WebmeshSchema.Packet },
+        schema,
         supportsTransferables: false,
         shutdown: Scope.close(scope, Exit.void),
       } satisfies WebChannel.WebChannel<typeof WebmeshSchema.Packet.Type, typeof WebmeshSchema.Packet.Type>