@livestore/webmesh 0.3.0-dev.10 → 0.3.0-dev.12

package/src/node.ts

@@ -1,4 +1,4 @@
-import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
+import { indent, LS_DEV, shouldNeverHappen } from '@livestore/utils'
 import type { Scope } from '@livestore/utils/effect'
 import {
   Cause,
@@ -29,6 +29,8 @@ export interface MeshNode {
 
   debug: {
     print: () => void
+    /** Sends a ping message to all connected nodes and channels */
+    ping: (payload?: string) => void
   }
 
   /**
@@ -58,7 +60,16 @@ export interface MeshNode {
   /**
    * Tries to broker a MessageChannel connection between the nodes, otherwise will proxy messages via hop-nodes
    *
-   * For a channel to successfully open, both sides need to have a connection and call `makeChannel`
+   * For a channel to successfully open, both sides need to have a connection and call `makeChannel`.
+   *
+   * Example:
+   * ```ts
+   * // Code on node A
+   * const channel = nodeA.makeChannel({ target: 'B', channelName: 'my-channel', schema: ... })
+   *
+   * // Code on node B
+   * const channel = nodeB.makeChannel({ target: 'A', channelName: 'my-channel', schema: ... })
+   * ```
    */
   makeChannel: <MsgListen, MsgSend>(args: {
     target: MeshNodeName
@@ -145,6 +156,8 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
 
     const sendPacket = (packet: typeof MeshSchema.Packet.Type) =>
       Effect.gen(function* () {
+        // yield* Effect.log(`${nodeName}: sendPacket:${packet._tag} [${packet.id}]`)
+
         if (Schema.is(MeshSchema.NetworkConnectionAdded)(packet)) {
           yield* Effect.spanEvent('NetworkConnectionAdded', { packet, nodeName })
           yield* PubSub.publish(newConnectionAvailablePubSub, packet.target)
@@ -330,7 +343,12 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
         const schema = WebChannel.mapSchema(inputSchema)
         const channelKey = `${target}-${channelName}` satisfies ChannelKey
 
-        if (!channelMap.has(channelKey)) {
+        if (channelMap.has(channelKey)) {
+          const existingChannel = channelMap.get(channelKey)!.debugInfo?.channel
+          if (existingChannel) {
+            shouldNeverHappen(`Channel ${channelKey} already exists`, existingChannel)
+          }
+        } else {
           const queue = yield* Queue.unbounded<MessageQueueItem | ProxyQueueItem>().pipe(
             Effect.acquireRelease(Queue.shutdown),
           )
@@ -342,12 +360,23 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
         yield* Effect.addFinalizer(() => Effect.sync(() => channelMap.delete(channelKey)))
 
         if (mode === 'messagechannel') {
-          // console.debug(nodeName, 'message mode', modeRef.current)
+          const incomingPacketsQueue = yield* Queue.unbounded<any>()
+
+          // We're we're draining the queue into another new queue.
+          // It's a bit of a mystery why this is needed, since the unit tests also work without it.
+          // But for the LiveStore devtools to actually work, we need to do this.
+          // We should figure out some day why this is needed and further simplify if possible.
+          yield* Queue.takeBetween(queue, 1, 10).pipe(
+            Effect.tap((_) => Queue.offerAll(incomingPacketsQueue, _)),
+            Effect.forever,
+            Effect.tapCauseLogPretty,
+            Effect.forkScoped,
+          )
 
           // NOTE already retries internally when transferables are required
-          const channel = yield* makeMessageChannel({
+          const { webChannel, initialConnectionDeferred } = yield* makeMessageChannel({
             nodeName,
-            queue,
+            incomingPacketsQueue,
             newConnectionAvailablePubSub,
             target,
             channelName,
@@ -356,9 +385,11 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
             checkTransferableConnections,
           })
 
-          channelMap.set(channelKey, { queue, debugInfo: { channel, target } })
+          channelMap.set(channelKey, { queue, debugInfo: { channel: webChannel, target } })
 
-          return channel
+          yield* initialConnectionDeferred
+
+          return webChannel
         } else {
           const channel = yield* makeProxyChannel({
             nodeName,
@@ -375,6 +406,7 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
           return channel
         }
       }).pipe(
+        // Effect.timeout(timeout),
         Effect.withSpanScoped(`makeChannel:${nodeName}→${target}(${channelName})`, {
           attributes: { target, channelName, mode, timeout },
         }),
@@ -393,15 +425,41 @@ export const makeMeshNode = (nodeName: MeshNodeName): Effect.Effect<MeshNode, ne
         console.log('Channels:', channelMap.size)
         for (const [key, value] of channelMap) {
           console.log(
-            `  ${key}: \n`,
-            `    Queue: ${value.queue.unsafeSize().pipe(Option.getOrUndefined)}`,
-            value.queue,
+            indent(key, 2),
             '\n',
-            `    Channel: target=${value.debugInfo?.target} supportsTransferables=${value.debugInfo?.channel.supportsTransferables}`,
+            Object.entries({
+              target: value.debugInfo?.target,
+              supportsTransferables: value.debugInfo?.channel.supportsTransferables,
+              ...value.debugInfo?.channel.debugInfo,
+            })
+              .map(([key, value]) => indent(`${key}=${value}`, 4))
+              .join('\n'),
+            '    ',
             value.debugInfo?.channel,
+            '\n',
+            indent(`Queue: ${value.queue.unsafeSize().pipe(Option.getOrUndefined)}`, 4),
+            value.queue,
           )
         }
       },
+      ping: (payload) => {
+        Effect.gen(function* () {
+          const msg = (via: string) => {
+            console.log(`sending message to ${via}`)
+            return WebChannel.DebugPingMessage.make({ message: `ping from ${nodeName} via ${via}`, payload })
+          }
+
+          yield* Effect.forEach(connectionChannels, ([channelName, con]) =>
+            con.channel.send(msg(`connection ${channelName}`) as any),
+          )
+
+          yield* Effect.forEach(
+            channelMap,
+            ([channelKey, channel]) =>
+              channel.debugInfo?.channel.send(msg(`channel ${channelKey}`) as any) ?? Effect.void,
+          )
+        }).pipe(Effect.runFork)
+      },
     }
 
     return { nodeName, addConnection, removeConnection, makeChannel, connectionKeys, debug } satisfies MeshNode

package/src/websocket-connection.ts

@@ -1,12 +1,13 @@
-import type { Scope } from '@livestore/utils/effect'
 import {
   Deferred,
   Effect,
   Either,
+  Exit,
   FiberHandle,
   Queue,
   Schedule,
   Schema,
+  Scope,
   Stream,
   WebChannel,
   WebSocket,
@@ -77,82 +78,85 @@ export const makeWebSocketConnection = (
   never,
   Scope.Scope
 > =>
-  Effect.gen(function* () {
-    socket.binaryType = 'arraybuffer'
-
-    const fromDeferred = yield* Deferred.make<string>()
-
-    yield* Stream.fromEventListener<MessageEvent>(socket as any, 'message').pipe(
-      Stream.map((msg) => Schema.decodeUnknownEither(MessageMsgPack)(new Uint8Array(msg.data))),
-      Stream.flatten(),
-      Stream.tap((msg) =>
+  Effect.scopeWithCloseable((scope) =>
+    Effect.gen(function* () {
+      socket.binaryType = 'arraybuffer'
+
+      const fromDeferred = yield* Deferred.make<string>()
+
+      yield* Stream.fromEventListener<MessageEvent>(socket as any, 'message').pipe(
+        Stream.map((msg) => Schema.decodeUnknownEither(MessageMsgPack)(new Uint8Array(msg.data))),
+        Stream.flatten(),
+        Stream.tap((msg) =>
+          Effect.gen(function* () {
+            if (msg._tag === 'WSConnectionInit') {
+              yield* Deferred.succeed(fromDeferred, msg.from)
+            } else {
+              const decodedPayload = yield* Schema.decode(MeshSchema.Packet)(msg.payload)
+              yield* Queue.offer(listenQueue, decodedPayload)
+            }
+          }),
+        ),
+        Stream.runDrain,
+        Effect.tapCauseLogPretty,
+        Effect.forkScoped,
+      )
+
+      const listenQueue = yield* Queue.unbounded<typeof MeshSchema.Packet.Type>().pipe(
+        Effect.acquireRelease(Queue.shutdown),
+      )
+
+      const initHandshake = (from: string) =>
+        socket.send(Schema.encodeSync(MessageMsgPack)({ _tag: 'WSConnectionInit', from }))
+
+      if (socketType._tag === 'leaf') {
+        initHandshake(socketType.from)
+      }
+
+      const deferredResult = yield* fromDeferred
+      const from = socketType._tag === 'leaf' ? socketType.from : deferredResult
+
+      if (socketType._tag === 'relay') {
+        initHandshake(from)
+      }
+
+      const isConnectedLatch = yield* Effect.makeLatch(true)
+
+      const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
+
+      yield* Effect.eventListener<any>(
+        socket,
+        'close',
+        () =>
+          Effect.gen(function* () {
+            yield* isConnectedLatch.close
+            yield* Deferred.succeed(closedDeferred, undefined)
+          }),
+        { once: true },
+      )
+
+      const send = (message: typeof MeshSchema.Packet.Type) =>
         Effect.gen(function* () {
-          if (msg._tag === 'WSConnectionInit') {
-            yield* Deferred.succeed(fromDeferred, msg.from)
-          } else {
-            const decodedPayload = yield* Schema.decode(MeshSchema.Packet)(msg.payload)
-            yield* Queue.offer(listenQueue, decodedPayload)
-          }
-        }),
-      ),
-      Stream.runDrain,
-      Effect.tapCauseLogPretty,
-      Effect.forkScoped,
-    )
-
-    const listenQueue = yield* Queue.unbounded<typeof MeshSchema.Packet.Type>().pipe(
-      Effect.acquireRelease(Queue.shutdown),
-    )
-
-    const initHandshake = (from: string) =>
-      socket.send(Schema.encodeSync(MessageMsgPack)({ _tag: 'WSConnectionInit', from }))
-
-    if (socketType._tag === 'leaf') {
-      initHandshake(socketType.from)
-    }
-
-    const deferredResult = yield* fromDeferred
-    const from = socketType._tag === 'leaf' ? socketType.from : deferredResult
-
-    if (socketType._tag === 'relay') {
-      initHandshake(from)
-    }
-
-    const isConnectedLatch = yield* Effect.makeLatch(true)
-
-    const closedDeferred = yield* Deferred.make<void>()
-
-    yield* Effect.eventListener<any>(
-      socket,
-      'close',
-      () =>
-        Effect.gen(function* () {
-          yield* isConnectedLatch.close
-          yield* Deferred.succeed(closedDeferred, undefined)
-        }),
-      { once: true },
-    )
-
-    const send = (message: typeof MeshSchema.Packet.Type) =>
-      Effect.gen(function* () {
-        yield* isConnectedLatch.await
-        const payload = yield* Schema.encode(MeshSchema.Packet)(message)
-        socket.send(Schema.encodeSync(MessageMsgPack)({ _tag: 'WSConnectionPayload', payload, from }))
-      })
-
-    const listen = Stream.fromQueue(listenQueue).pipe(Stream.map(Either.right))
-
-    const webChannel = {
-      [WebChannel.WebChannelSymbol]: WebChannel.WebChannelSymbol,
-      send,
-      listen,
-      closedDeferred,
-      schema: { listen: MeshSchema.Packet, send: MeshSchema.Packet },
-      supportsTransferables: false,
-    } satisfies WebChannel.WebChannel<typeof MeshSchema.Packet.Type, typeof MeshSchema.Packet.Type>
-
-    return {
-      webChannel: webChannel as WebChannel.WebChannel<typeof MeshSchema.Packet.Type, typeof MeshSchema.Packet.Type>,
-      from,
-    }
-  })
+          yield* isConnectedLatch.await
+          const payload = yield* Schema.encode(MeshSchema.Packet)(message)
+          socket.send(Schema.encodeSync(MessageMsgPack)({ _tag: 'WSConnectionPayload', payload, from }))
+        })
+
+      const listen = Stream.fromQueue(listenQueue).pipe(Stream.map(Either.right))
+
+      const webChannel = {
+        [WebChannel.WebChannelSymbol]: WebChannel.WebChannelSymbol,
+        send,
+        listen,
+        closedDeferred,
+        schema: { listen: MeshSchema.Packet, send: MeshSchema.Packet },
+        supportsTransferables: false,
+        shutdown: Scope.close(scope, Exit.void),
+      } satisfies WebChannel.WebChannel<typeof MeshSchema.Packet.Type, typeof MeshSchema.Packet.Type>
+
+      return {
+        webChannel: webChannel as WebChannel.WebChannel<typeof MeshSchema.Packet.Type, typeof MeshSchema.Packet.Type>,
+        from,
+      }
+    }).pipe(Effect.withSpanScoped('makeWebSocketConnection')),
+  )