@livestore/webmesh 0.3.0-dev.2 → 0.3.0-dev.21

package/src/node.test.ts

@@ -1,4 +1,19 @@
-import { Chunk, Deferred, Effect, identity, Layer, Logger, Schema, Stream, WebChannel } from '@livestore/utils/effect'
+import '@livestore/utils/node-vitest-polyfill'
+
+import { IS_CI } from '@livestore/utils'
+import {
+  Chunk,
+  Deferred,
+  Effect,
+  Exit,
+  identity,
+  Layer,
+  Logger,
+  Schema,
+  Scope,
+  Stream,
+  WebChannel,
+} from '@livestore/utils/effect'
 import { OtelLiveHttp } from '@livestore/utils/node'
 import { Vitest } from '@livestore/utils/node-vitest'
 import { expect } from 'vitest'
@@ -16,35 +31,47 @@ import { makeMeshNode } from './node.js'
 
 const ExampleSchema = Schema.Struct({ message: Schema.String })
 
-const connectNodesViaMessageChannel = (nodeA: MeshNode, nodeB: MeshNode) =>
+const connectNodesViaMessageChannel = (nodeA: MeshNode, nodeB: MeshNode, options?: { replaceIfExists?: boolean }) =>
   Effect.gen(function* () {
     const mc = new MessageChannel()
     const meshChannelAToB = yield* WebChannel.messagePortChannel({ port: mc.port1, schema: Packet })
     const meshChannelBToA = yield* WebChannel.messagePortChannel({ port: mc.port2, schema: Packet })
 
-    yield* nodeA.addConnection({ target: nodeB.nodeName, connectionChannel: meshChannelAToB })
-    yield* nodeB.addConnection({ target: nodeA.nodeName, connectionChannel: meshChannelBToA })
-
-    return mc
+    yield* nodeA.addConnection({
+      target: nodeB.nodeName,
+      connectionChannel: meshChannelAToB,
+      replaceIfExists: options?.replaceIfExists,
+    })
+    yield* nodeB.addConnection({
+      target: nodeA.nodeName,
+      connectionChannel: meshChannelBToA,
+      replaceIfExists: options?.replaceIfExists,
+    })
   }).pipe(Effect.withSpan(`connectNodesViaMessageChannel:${nodeA.nodeName}↔${nodeB.nodeName}`))
 
-const connectNodesViaBroadcastChannel = (nodeA: MeshNode, nodeB: MeshNode) =>
+const connectNodesViaBroadcastChannel = (nodeA: MeshNode, nodeB: MeshNode, options?: { replaceIfExists?: boolean }) =>
   Effect.gen(function* () {
     // Need to instantiate two different channels because they filter out messages they sent themselves
     const broadcastWebChannelA = yield* WebChannel.broadcastChannelWithAck({
       channelName: `${nodeA.nodeName}↔${nodeB.nodeName}`,
-      listenSchema: Packet,
-      sendSchema: Packet,
+      schema: Packet,
     })
 
     const broadcastWebChannelB = yield* WebChannel.broadcastChannelWithAck({
       channelName: `${nodeA.nodeName}↔${nodeB.nodeName}`,
-      listenSchema: Packet,
-      sendSchema: Packet,
+      schema: Packet,
     })
 
-    yield* nodeA.addConnection({ target: nodeB.nodeName, connectionChannel: broadcastWebChannelA })
-    yield* nodeB.addConnection({ target: nodeA.nodeName, connectionChannel: broadcastWebChannelB })
+    yield* nodeA.addConnection({
+      target: nodeB.nodeName,
+      connectionChannel: broadcastWebChannelA,
+      replaceIfExists: options?.replaceIfExists,
+    })
+    yield* nodeB.addConnection({
+      target: nodeA.nodeName,
+      connectionChannel: broadcastWebChannelB,
+      replaceIfExists: options?.replaceIfExists,
+    })
   }).pipe(Effect.withSpan(`connectNodesViaBroadcastChannel:${nodeA.nodeName}↔${nodeB.nodeName}`))
 
 const createChannel = (source: MeshNode, target: string, options?: Partial<Parameters<MeshNode['makeChannel']>[0]>) =>
@@ -73,206 +100,319 @@ const maybeDelay =
       ? effect
       : Effect.sleep(delay).pipe(Effect.withSpan(`${label}:delay(${delay})`), Effect.andThen(effect))
 
+const testTimeout = IS_CI ? 30_000 : 1000
+const propTestTimeout = IS_CI ? 60_000 : 20_000
+
 // TODO also make work without `Vitest.scopedLive` (i.e. with `Vitest.scoped`)
 // probably requires controlling the clocks
-Vitest.describe('webmesh node', { timeout: 1000 }, () => {
-  Vitest.describe('A <> B', () => {
-    Vitest.describe('prop tests', { timeout: 10_000 }, () => {
-      const Delay = Schema.UndefinedOr(Schema.Literal(0, 1, 10, 50))
-      // NOTE for message channels, we test both with and without transferables (i.e. proxying)
-      const ChannelType = Schema.Literal('messagechannel', 'messagechannel.proxy', 'proxy')
-
-      const fromChannelType = (
-        channelType: typeof ChannelType.Type,
-      ): {
-        mode: 'messagechannel' | 'proxy'
-        connectNodes: typeof connectNodesViaMessageChannel | typeof connectNodesViaBroadcastChannel
-      } => {
-        switch (channelType) {
-          case 'proxy': {
-            return { mode: 'proxy', connectNodes: connectNodesViaBroadcastChannel }
-          }
-          case 'messagechannel': {
-            return { mode: 'messagechannel', connectNodes: connectNodesViaMessageChannel }
-          }
-          case 'messagechannel.proxy': {
-            return { mode: 'proxy', connectNodes: connectNodesViaMessageChannel }
-          }
-        }
+Vitest.describe('webmesh node', { timeout: testTimeout }, () => {
+  const Delay = Schema.UndefinedOr(Schema.Literal(0, 1, 10, 50))
+  // NOTE for message channels, we test both with and without transferables (i.e. proxying)
+  const ChannelType = Schema.Literal('messagechannel', 'messagechannel.proxy', 'proxy')
+  const NodeNames = Schema.Union(
+    Schema.Tuple(Schema.Literal('A'), Schema.Literal('B')),
+    Schema.Tuple(Schema.Literal('B'), Schema.Literal('A')),
+  )
+
+  const fromChannelType = (
+    channelType: typeof ChannelType.Type,
+  ): {
+    mode: 'messagechannel' | 'proxy'
+    connectNodes: typeof connectNodesViaMessageChannel | typeof connectNodesViaBroadcastChannel
+  } => {
+    switch (channelType) {
+      case 'proxy': {
+        return { mode: 'proxy', connectNodes: connectNodesViaBroadcastChannel }
+      }
+      case 'messagechannel': {
+        return { mode: 'messagechannel', connectNodes: connectNodesViaMessageChannel }
       }
+      case 'messagechannel.proxy': {
+        return { mode: 'proxy', connectNodes: connectNodesViaMessageChannel }
+      }
+    }
+  }
+
+  const exchangeMessages = ({
+    nodeX,
+    nodeY,
+    channelType,
+    // numberOfMessages = 1,
+    delays,
+  }: {
+    nodeX: MeshNode
+    nodeY: MeshNode
+    channelType: 'messagechannel' | 'proxy' | 'messagechannel.proxy'
+    numberOfMessages?: number
+    delays?: { x?: number; y?: number; connect?: number }
+  }) =>
+    Effect.gen(function* () {
+      const nodeLabel = { x: nodeX.nodeName, y: nodeY.nodeName }
+      const { mode, connectNodes } = fromChannelType(channelType)
+
+      const nodeXCode = Effect.gen(function* () {
+        const channelXToY = yield* createChannel(nodeX, nodeY.nodeName, { mode })
+
+        yield* channelXToY.send({ message: `${nodeLabel.x}1` })
+        // console.log('channelXToY', channelXToY.debugInfo)
+        expect(yield* getFirstMessage(channelXToY)).toEqual({ message: `${nodeLabel.y}1` })
+        // expect(channelXToY.debugInfo.connectCounter).toBe(1)
+      })
+
+      const nodeYCode = Effect.gen(function* () {
+        const channelYToX = yield* createChannel(nodeY, nodeX.nodeName, { mode })
 
+        yield* channelYToX.send({ message: `${nodeLabel.y}1` })
+        // console.log('channelYToX', channelYToX.debugInfo)
+        expect(yield* getFirstMessage(channelYToX)).toEqual({ message: `${nodeLabel.x}1` })
+        // expect(channelYToX.debugInfo.connectCounter).toBe(1)
+      })
+
+      yield* Effect.all(
+        [
+          connectNodes(nodeX, nodeY).pipe(maybeDelay(delays?.connect, 'connectNodes')),
+          nodeXCode.pipe(maybeDelay(delays?.x, `node${nodeLabel.x}Code`)),
+          nodeYCode.pipe(maybeDelay(delays?.y, `node${nodeLabel.y}Code`)),
+        ],
+        { concurrency: 'unbounded' },
+      ).pipe(Effect.withSpan(`exchangeMessages(${nodeLabel.x}↔${nodeLabel.y})`))
+    })
+  Vitest.describe('A <> B', () => {
+    Vitest.describe('prop tests', { timeout: propTestTimeout }, () => {
+      // const delayX = 40
+      // const delayY = undefined
+      // const connectDelay = undefined
+      // const channelType = 'messagechannel'
+      // const nodeNames = ['B', 'A'] as const
+      // Vitest.scopedLive(
+      //   'a / b connect at different times with different channel types',
+      //   (test) =>
       Vitest.scopedLive.prop(
-        // Vitest.scopedLive.only(
         'a / b connect at different times with different channel types',
-        [Delay, Delay, Delay, ChannelType],
-        ([delayA, delayB, connectDelay, channelType], test) =>
-          // (test) =>
+        [Delay, Delay, Delay, ChannelType, NodeNames],
+        ([delayX, delayY, connectDelay, channelType, nodeNames], test) =>
           Effect.gen(function* () {
-            // const delayA = 1
-            // const delayB = 10
-            // const connectDelay = 10
-            // const channelType = 'message.prefer'
-            // console.log('delayA', delayA, 'delayB', delayB, 'connectDelay', connectDelay, 'channelType', channelType)
-
-            const nodeA = yield* makeMeshNode('A')
-            const nodeB = yield* makeMeshNode('B')
+            // console.log({ delayX, delayY, connectDelay, channelType, nodeNames })
 
-            const { mode, connectNodes } = fromChannelType(channelType)
+            const [nodeNameX, nodeNameY] = nodeNames
+            const nodeX = yield* makeMeshNode(nodeNameX)
+            const nodeY = yield* makeMeshNode(nodeNameY)
 
-            const nodeACode = Effect.gen(function* () {
-              const channelAToB = yield* createChannel(nodeA, 'B', { mode })
-
-              yield* channelAToB.send({ message: 'A1' })
-              expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'A2' })
-            })
-
-            const nodeBCode = Effect.gen(function* () {
-              const channelBToA = yield* createChannel(nodeB, 'A', { mode })
-
-              yield* channelBToA.send({ message: 'A2' })
-              expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A1' })
+            yield* exchangeMessages({
+              nodeX,
+              nodeY,
+              channelType,
+              delays: { x: delayX, y: delayY, connect: connectDelay },
             })
 
-            yield* Effect.all(
-              [
-                connectNodes(nodeA, nodeB).pipe(maybeDelay(connectDelay, 'connectNodes')),
-                nodeACode.pipe(maybeDelay(delayA, 'nodeACode')),
-                nodeBCode.pipe(maybeDelay(delayB, 'nodeBCode')),
-              ],
-              { concurrency: 'unbounded' },
-            )
+            yield* Effect.promise(() => nodeX.debug.requestTopology(100))
           }).pipe(
-            withCtx(test, { skipOtel: true, suffix: `delayA=${delayA} delayB=${delayB} channelType=${channelType}` }),
+            withCtx(test, {
+              skipOtel: true,
+              suffix: `delayX=${delayX} delayY=${delayY} connectDelay=${connectDelay} channelType=${channelType} nodeNames=${nodeNames}`,
+            }),
           ),
+        // { fastCheck: { numRuns: 20 } },
       )
 
-      // Vitest.scopedLive.only(
-      //   'reconnects',
-      //   (test) =>
-      Vitest.scopedLive.prop(
-        'b reconnects',
-        [Delay, Delay, ChannelType],
-        ([waitForOfflineDelay, sleepDelay, channelType], test) =>
-          Effect.gen(function* () {
-            // const waitForOfflineDelay = 0
-            // const sleepDelay = 10
-            // const channelType = 'proxy'
-            // console.log(
-            //   'waitForOfflineDelay',
-            //   waitForOfflineDelay,
-            //   'sleepDelay',
-            //   sleepDelay,
-            //   'channelType',
-            //   channelType,
-            // )
-
-            const nodeA = yield* makeMeshNode('A')
-            const nodeB = yield* makeMeshNode('B')
+      {
+        // const waitForOfflineDelay = undefined
+        // const sleepDelay = 0
+        // const channelType = 'messagechannel'
+        // Vitest.scopedLive(
+        //   'b reconnects',
+        //   (test) =>
+        Vitest.scopedLive.prop(
+          'b reconnects',
+          [Delay, Delay, ChannelType],
+          ([waitForOfflineDelay, sleepDelay, channelType], test) =>
+            Effect.gen(function* () {
+              // console.log({ waitForOfflineDelay, sleepDelay, channelType })
 
-            const { mode, connectNodes } = fromChannelType(channelType)
+              if (waitForOfflineDelay === undefined) {
+                // TODO we still need to fix this scenario but it shouldn't really be common in practice
+                return
+              }
 
-            // TODO also optionally delay the connection
-            yield* connectNodes(nodeA, nodeB)
+              const nodeA = yield* makeMeshNode('A')
+              const nodeB = yield* makeMeshNode('B')
 
-            const waitForBToBeOffline =
-              waitForOfflineDelay === undefined ? undefined : yield* Deferred.make<void, never>()
+              const { mode, connectNodes } = fromChannelType(channelType)
 
-            const nodeACode = Effect.gen(function* () {
-              const channelAToB = yield* createChannel(nodeA, 'B', { mode })
+              // TODO also optionally delay the connection
+              yield* connectNodes(nodeA, nodeB)
 
-              yield* channelAToB.send({ message: 'A1' })
-              expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B1' })
+              const waitForBToBeOffline =
+                waitForOfflineDelay === undefined ? undefined : yield* Deferred.make<void, never>()
 
-              if (waitForBToBeOffline !== undefined) {
-                yield* waitForBToBeOffline
-              }
+              const nodeACode = Effect.gen(function* () {
+                const channelAToB = yield* createChannel(nodeA, 'B', { mode })
+                yield* channelAToB.send({ message: 'A1' })
+                expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B1' })
 
-              yield* channelAToB.send({ message: 'A2' })
-              expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B2' })
-            })
+                console.log('nodeACode:waiting for B to be offline')
+                if (waitForBToBeOffline !== undefined) {
+                  yield* waitForBToBeOffline
+                }
 
-            // Simulating node b going offline and then coming back online
-            const nodeBCode = Effect.gen(function* () {
-              yield* Effect.gen(function* () {
-                const channelBToA = yield* createChannel(nodeB, 'A', { mode })
+                yield* channelAToB.send({ message: 'A2' })
+                expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B2' })
+              })
 
-                yield* channelBToA.send({ message: 'B1' })
-                expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A1' })
-              }).pipe(Effect.scoped)
+              // Simulating node b going offline and then coming back online
+              // This test also illustrates why we need a ack-message channel since otherwise
+              // sent messages might get lost
+              const nodeBCode = Effect.gen(function* () {
+                yield* Effect.gen(function* () {
+                  const channelBToA = yield* createChannel(nodeB, 'A', { mode })
 
-              if (waitForBToBeOffline !== undefined) {
-                yield* Deferred.succeed(waitForBToBeOffline, void 0)
-              }
+                  yield* channelBToA.send({ message: 'B1' })
+                  expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A1' })
+                }).pipe(Effect.scoped, Effect.withSpan('nodeBCode:part1'))
 
-              if (sleepDelay !== undefined) {
-                yield* Effect.sleep(sleepDelay).pipe(Effect.withSpan(`B:sleep(${sleepDelay})`))
-              }
+                console.log('nodeBCode:B node going offline')
+                if (waitForBToBeOffline !== undefined) {
+                  yield* Deferred.succeed(waitForBToBeOffline, void 0)
+                }
 
-              yield* Effect.gen(function* () {
-                const channelBToA = yield* createChannel(nodeB, 'A', { mode })
+                if (sleepDelay !== undefined) {
+                  yield* Effect.sleep(sleepDelay).pipe(Effect.withSpan(`B:sleep(${sleepDelay})`))
+                }
+
+                // Recreating the channel
+                yield* Effect.gen(function* () {
+                  const channelBToA = yield* createChannel(nodeB, 'A', { mode })
 
-                yield* channelBToA.send({ message: 'B2' })
-                expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A2' })
-              }).pipe(Effect.scoped)
+                  yield* channelBToA.send({ message: 'B2' })
+                  expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A2' })
+                }).pipe(Effect.scoped, Effect.withSpan('nodeBCode:part2'))
+              })
+
+              yield* Effect.all([nodeACode, nodeBCode], { concurrency: 'unbounded' }).pipe(Effect.withSpan('test'))
+            }).pipe(
+              withCtx(test, {
+                skipOtel: true,
+                suffix: `waitForOfflineDelay=${waitForOfflineDelay} sleepDelay=${sleepDelay} channelType=${channelType}`,
+              }),
+            ),
+          { fastCheck: { numRuns: 20 } },
+        )
+      }
+
+      Vitest.scopedLive('reconnect with re-created node', (test) =>
+        Effect.gen(function* () {
+          const nodeBgen1Scope = yield* Scope.make()
+
+          const nodeA = yield* makeMeshNode('A')
+          const nodeBgen1 = yield* makeMeshNode('B').pipe(Scope.extend(nodeBgen1Scope))
+
+          yield* connectNodesViaMessageChannel(nodeA, nodeBgen1).pipe(Scope.extend(nodeBgen1Scope))
+
+          // yield* Effect.sleep(100)
+
+          const channelAToBOnce = yield* Effect.cached(createChannel(nodeA, 'B'))
+          const nodeACode = Effect.gen(function* () {
+            const channelAToB = yield* channelAToBOnce
+            yield* channelAToB.send({ message: 'A1' })
+            expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B1' })
+            // expect(channelAToB.debugInfo.connectCounter).toBe(1)
+          })
+
+          const nodeBCode = (nodeB: MeshNode) =>
+            Effect.gen(function* () {
+              const channelBToA = yield* createChannel(nodeB, 'A')
+
+              yield* channelBToA.send({ message: 'B1' })
+              expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A1' })
+              // expect(channelBToA.debugInfo.connectCounter).toBe(1)
             })
 
-            yield* Effect.all([nodeACode, nodeBCode], { concurrency: 'unbounded' })
-          }).pipe(
-            withCtx(test, {
-              skipOtel: true,
-              suffix: `waitForOfflineDelay=${waitForOfflineDelay} sleepDelay=${sleepDelay} channelType=${channelType}`,
-            }),
-          ),
+          yield* Effect.all([nodeACode, nodeBCode(nodeBgen1).pipe(Scope.extend(nodeBgen1Scope))], {
+            concurrency: 'unbounded',
+          }).pipe(Effect.withSpan('test1'))
+
+          yield* Scope.close(nodeBgen1Scope, Exit.void)
+
+          const nodeBgen2 = yield* makeMeshNode('B')
+          yield* connectNodesViaMessageChannel(nodeA, nodeBgen2, { replaceIfExists: true })
+
+          yield* Effect.all([nodeACode, nodeBCode(nodeBgen2)], { concurrency: 'unbounded' }).pipe(
+            Effect.withSpan('test2'),
+          )
+        }).pipe(withCtx(test)),
       )
 
       const ChannelTypeWithoutMessageChannelProxy = Schema.Literal('proxy', 'messagechannel')
+      // TODO there seems to be a flaky case here which gets hit sometimes (e.g. 2025-02-28-17:11)
+      // Log output:
+      // test: { seed: -964670352, path: "1", endOnFailure: true }
+      // test: Counterexample: ["messagechannel",["A","B"]]
+      // test: Shrunk 0 time(s)
+      // test: Got AssertionError: expected { _tag: 'MessageChannelPing' } to deeply equal { message: 'A1' }
+      // test:     at next (/Users/schickling/Code/overtone/submodules/livestore/packages/@livestore/webmesh/src/node.test.ts:376:59)
+      // test:     at prop tests:replace connection while keeping the channel:channelType=messagechannel nodeNames=A,B (/Users/schickling/Code/overtone/submodules/livestore/packages/@livestore/webmesh/src/node.test.ts:801:14)
+      // test: Hint: Enable verbose mode in order to have the list of all failing values encountered during the run
+      // test:    ✓ webmesh node > A <> B > prop tests > TODO improve latency > concurrent messages 2110ms
+      // test: ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯
+      // test:  FAIL  src/node.test.ts > webmesh node > A <> B > prop tests > replace connection while keeping the channel
+      // test: Error: Property failed after 2 tests
+      // test: { seed: -964670352, path: "1", endOnFailure: true }
+      // test: Counterexample: ["messagechannel",["A","B"]]
       Vitest.scopedLive.prop(
         'replace connection while keeping the channel',
-        [ChannelTypeWithoutMessageChannelProxy],
-        ([channelType], test) =>
+        [ChannelTypeWithoutMessageChannelProxy, NodeNames],
+        ([channelType, nodeNames], test) =>
           Effect.gen(function* () {
-            const nodeA = yield* makeMeshNode('A')
-            const nodeB = yield* makeMeshNode('B')
+            const [nodeNameX, nodeNameY] = nodeNames
+            const nodeX = yield* makeMeshNode(nodeNameX)
+            const nodeY = yield* makeMeshNode(nodeNameY)
+            const nodeLabel = { x: nodeX.nodeName, y: nodeY.nodeName }
 
             const { mode, connectNodes } = fromChannelType(channelType)
 
-            yield* connectNodes(nodeA, nodeB)
+            yield* connectNodes(nodeX, nodeY)
 
             const waitForConnectionReplacement = yield* Deferred.make<void>()
 
-            const nodeACode = Effect.gen(function* () {
-              const channelAToB = yield* createChannel(nodeA, 'B', { mode })
+            const nodeXCode = Effect.gen(function* () {
+              const channelXToY = yield* createChannel(nodeX, nodeLabel.y, { mode })
 
-              yield* channelAToB.send({ message: 'A1' })
-              expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B1' })
+              yield* channelXToY.send({ message: `${nodeLabel.x}1` })
+              expect(yield* getFirstMessage(channelXToY)).toEqual({ message: `${nodeLabel.y}1` })
 
               yield* waitForConnectionReplacement
 
-              yield* channelAToB.send({ message: 'A2' })
-              expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B2' })
+              yield* channelXToY.send({ message: `${nodeLabel.x}2` })
+              expect(yield* getFirstMessage(channelXToY)).toEqual({ message: `${nodeLabel.y}2` })
             })
 
-            const nodeBCode = Effect.gen(function* () {
-              const channelBToA = yield* createChannel(nodeB, 'A', { mode })
+            const nodeYCode = Effect.gen(function* () {
+              const channelYToX = yield* createChannel(nodeY, nodeLabel.x, { mode })
 
-              yield* channelBToA.send({ message: 'B1' })
-              expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A1' })
+              yield* channelYToX.send({ message: `${nodeLabel.y}1` })
+              expect(yield* getFirstMessage(channelYToX)).toEqual({ message: `${nodeLabel.x}1` })
 
               // Switch out connection while keeping the channel
-              yield* nodeA.removeConnection('B')
-              yield* nodeB.removeConnection('A')
-              yield* connectNodes(nodeA, nodeB)
+              yield* nodeX.removeConnection(nodeLabel.y)
+              yield* nodeY.removeConnection(nodeLabel.x)
+              yield* connectNodes(nodeX, nodeY)
               yield* Deferred.succeed(waitForConnectionReplacement, void 0)
 
-              yield* channelBToA.send({ message: 'B2' })
-              expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A2' })
+              yield* channelYToX.send({ message: `${nodeLabel.y}2` })
+              expect(yield* getFirstMessage(channelYToX)).toEqual({ message: `${nodeLabel.x}2` })
             })
 
-            yield* Effect.all([nodeACode, nodeBCode], { concurrency: 'unbounded' })
-          }).pipe(withCtx(test, { skipOtel: true, suffix: `channelType=${channelType}` })),
+            yield* Effect.all([nodeXCode, nodeYCode], { concurrency: 'unbounded' })
+          }).pipe(
+            withCtx(test, {
+              skipOtel: true,
+              suffix: `channelType=${channelType} nodeNames=${nodeNames}`,
+            }),
+          ),
+        { fastCheck: { numRuns: 10 } },
       )
 
-      Vitest.describe.todo('TODO improve latency', () => {
+      Vitest.describe('TODO improve latency', () => {
         // TODO we need to improve latency when sending messages concurrently
         Vitest.scopedLive.prop(
           'concurrent messages',
@@ -319,12 +459,52 @@ Vitest.describe('webmesh node', { timeout: 1000 }, () => {
               yield* Effect.all([nodeACode, nodeBCode, connectNodes(nodeA, nodeB).pipe(Effect.delay(100))], {
                 concurrency: 'unbounded',
               })
-            }).pipe(withCtx(test, { skipOtel: false, suffix: `channelType=${channelType} count=${count}` })),
-          { timeout: 30_000 },
+            }).pipe(
+              withCtx(test, {
+                skipOtel: true,
+                suffix: `channelType=${channelType} count=${count}`,
+                timeout: testTimeout * 2,
+              }),
+            ),
+          { fastCheck: { numRuns: 10 } },
         )
       })
     })
 
+    Vitest.describe('message channel specific tests', () => {
+      Vitest.scopedLive('differing initial connection counter', (test) =>
+        Effect.gen(function* () {
+          const nodeA = yield* makeMeshNode('A')
+          const nodeB = yield* makeMeshNode('B')
+
+          yield* connectNodesViaMessageChannel(nodeA, nodeB)
+
+          const messageCount = 3
+
+          const bFiber = yield* Effect.gen(function* () {
+            const channelBToA = yield* createChannel(nodeB, 'A')
+            yield* channelBToA.listen.pipe(
+              Stream.flatten(),
+              Stream.tap((msg) => channelBToA.send({ message: `resp:${msg.message}` })),
+              Stream.take(messageCount),
+              Stream.runDrain,
+            )
+          }).pipe(Effect.scoped, Effect.fork)
+
+          // yield* createChannel(nodeA, 'B').pipe(Effect.andThen(WebChannel.shutdown))
+          // // yield* createChannel(nodeA, 'B').pipe(Effect.andThen(WebChannel.shutdown))
+          // // yield* createChannel(nodeA, 'B').pipe(Effect.andThen(WebChannel.shutdown))
+          yield* Effect.gen(function* () {
+            const channelAToB = yield* createChannel(nodeA, 'B')
+            yield* channelAToB.send({ message: 'A' })
+            expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'resp:A' })
+          }).pipe(Effect.scoped, Effect.repeatN(messageCount))
+
+          yield* bFiber
+        }).pipe(withCtx(test)),
+      )
+    })
+
     Vitest.scopedLive('manual debug test', (test) =>
       Effect.gen(function* () {
         const nodeA = yield* makeMeshNode('A')
@@ -382,6 +562,7 @@ Vitest.describe('webmesh node', { timeout: 1000 }, () => {
           yield* channelAToC.send({ message: 'A1' })
           expect(yield* getFirstMessage(channelAToC)).toEqual({ message: 'C1' })
           expect(yield* getFirstMessage(channelAToC)).toEqual({ message: 'C2' })
+          expect(yield* getFirstMessage(channelAToC)).toEqual({ message: 'C3' })
         })
 
         const nodeCCode = Effect.gen(function* () {
@@ -420,9 +601,14 @@ Vitest.describe('webmesh node', { timeout: 1000 }, () => {
           expect(yield* getFirstMessage(channelCToA)).toEqual({ message: 'A1' })
         })
 
-        yield* Effect.all([nodeACode, nodeCCode, connectNodes(nodeB, nodeC).pipe(Effect.delay(100))], {
-          concurrency: 'unbounded',
-        })
+        yield* Effect.all(
+          [
+            nodeACode,
+            nodeCCode,
+            connectNodes(nodeB, nodeC).pipe(Effect.delay(100), Effect.withSpan('connect-nodeB-nodeC-delay(100)')),
+          ],
+          { concurrency: 'unbounded' },
+        )
       }).pipe(withCtx(test)),
     )
 
@@ -451,7 +637,7 @@ Vitest.describe('webmesh node', { timeout: 1000 }, () => {
       }).pipe(withCtx(test)),
     )
 
-    Vitest.scopedLive('should fail', (test) =>
+    Vitest.scopedLive('should fail with timeout due to missing connection', (test) =>
       Effect.gen(function* () {
         const nodeA = yield* makeMeshNode('A')
         const nodeB = yield* makeMeshNode('B')
@@ -473,6 +659,150 @@ Vitest.describe('webmesh node', { timeout: 1000 }, () => {
         yield* Effect.all([nodeACode, nodeCCode], { concurrency: 'unbounded' })
       }).pipe(withCtx(test)),
     )
+
+    Vitest.scopedLive('should fail with timeout due no transferable', (test) =>
+      Effect.gen(function* () {
+        const nodeA = yield* makeMeshNode('A')
+        const nodeB = yield* makeMeshNode('B')
+
+        yield* connectNodesViaBroadcastChannel(nodeA, nodeB)
+
+        const nodeACode = Effect.gen(function* () {
+          const err = yield* createChannel(nodeA, 'B').pipe(Effect.timeout(200), Effect.flip)
+          expect(err._tag).toBe('TimeoutException')
+        })
+
+        const nodeBCode = Effect.gen(function* () {
+          const err = yield* createChannel(nodeB, 'A').pipe(Effect.timeout(200), Effect.flip)
+          expect(err._tag).toBe('TimeoutException')
+        })
+
+        yield* Effect.all([nodeACode, nodeBCode], { concurrency: 'unbounded' })
+      }).pipe(withCtx(test)),
+    )
+
+    Vitest.scopedLive('reconnect with re-created node', (test) =>
+      Effect.gen(function* () {
+        const nodeCgen1Scope = yield* Scope.make()
+
+        const nodeA = yield* makeMeshNode('A')
+        const nodeB = yield* makeMeshNode('B')
+        const nodeCgen1 = yield* makeMeshNode('C').pipe(Scope.extend(nodeCgen1Scope))
+
+        yield* connectNodesViaMessageChannel(nodeA, nodeB)
+        yield* connectNodesViaMessageChannel(nodeB, nodeCgen1).pipe(Scope.extend(nodeCgen1Scope))
+
+        const nodeACode = Effect.gen(function* () {
+          const channelAToB = yield* createChannel(nodeA, 'C')
+
+          yield* channelAToB.send({ message: 'A1' })
+          expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'C1' })
+        })
+
+        const nodeCCode = (nodeB: MeshNode) =>
+          Effect.gen(function* () {
+            const channelBToA = yield* createChannel(nodeB, 'A')
+
+            yield* channelBToA.send({ message: 'C1' })
+            expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A1' })
+          })
+
+        yield* Effect.all([nodeACode, nodeCCode(nodeCgen1)], { concurrency: 'unbounded' }).pipe(
+          Effect.withSpan('test1'),
+          Scope.extend(nodeCgen1Scope),
+        )
+
+        yield* Scope.close(nodeCgen1Scope, Exit.void)
+
+        const nodeCgen2 = yield* makeMeshNode('C')
+        yield* connectNodesViaMessageChannel(nodeB, nodeCgen2, { replaceIfExists: true })
+
+        yield* Effect.all([nodeACode, nodeCCode(nodeCgen2)], { concurrency: 'unbounded' }).pipe(
+          Effect.withSpan('test2'),
+        )
+      }).pipe(withCtx(test)),
+    )
+  })
+
+  /**
+   *    A
+   *   / \
+   *  B   C
+   *   \ /
+   *    D
+   */
+  Vitest.describe('diamond topology', () => {
+    Vitest.scopedLive('should work', (test) =>
+      Effect.gen(function* () {
+        const nodeA = yield* makeMeshNode('A')
+        const nodeB = yield* makeMeshNode('B')
+        const nodeC = yield* makeMeshNode('C')
+        const nodeD = yield* makeMeshNode('D')
+
+        yield* connectNodesViaMessageChannel(nodeA, nodeB)
+        yield* connectNodesViaMessageChannel(nodeA, nodeC)
+        yield* connectNodesViaMessageChannel(nodeB, nodeD)
+        yield* connectNodesViaMessageChannel(nodeC, nodeD)
+
+        const nodeACode = Effect.gen(function* () {
+          const channelAToD = yield* createChannel(nodeA, 'D')
+          yield* channelAToD.send({ message: 'A1' })
+          expect(yield* getFirstMessage(channelAToD)).toEqual({ message: 'D1' })
+        })
+
+        const nodeDCode = Effect.gen(function* () {
+          const channelDToA = yield* createChannel(nodeD, 'A')
+          yield* channelDToA.send({ message: 'D1' })
+          expect(yield* getFirstMessage(channelDToA)).toEqual({ message: 'A1' })
+        })
+
+        yield* Effect.all([nodeACode, nodeDCode], { concurrency: 'unbounded' })
+      }).pipe(withCtx(test)),
+    )
+  })
+
+  /**
+   *    A       E
+   *     \     /
+   *      C---D
+   *     /     \
+   *    B       F
+   *
+   * Topology: Butterfly topology with two connected hubs (C-D) each serving multiple nodes
+   */
+  Vitest.describe('butterfly topology', () => {
+    Vitest.scopedLive('should work', (test) =>
+      Effect.gen(function* () {
+        const nodeA = yield* makeMeshNode('A')
+        const nodeB = yield* makeMeshNode('B')
+        const nodeC = yield* makeMeshNode('C')
+        const nodeD = yield* makeMeshNode('D')
+        const nodeE = yield* makeMeshNode('E')
+        const nodeF = yield* makeMeshNode('F')
+
+        yield* connectNodesViaMessageChannel(nodeA, nodeC)
+        yield* connectNodesViaMessageChannel(nodeB, nodeC)
+        yield* connectNodesViaMessageChannel(nodeC, nodeD)
+        yield* connectNodesViaMessageChannel(nodeD, nodeE)
+        yield* connectNodesViaMessageChannel(nodeD, nodeF)
+
+        yield* Effect.promise(() => nodeA.debug.requestTopology(100))
+
+        const nodeACode = Effect.gen(function* () {
+          const channelAToE = yield* createChannel(nodeA, 'E')
+          yield* channelAToE.send({ message: 'A1' })
+          expect(yield* getFirstMessage(channelAToE)).toEqual({ message: 'E1' })
+        })
+
+        const nodeECode = Effect.gen(function* () {
+          const channelEToA = yield* createChannel(nodeE, 'A')
+          yield* channelEToA.send({ message: 'E1' })
+          expect(yield* getFirstMessage(channelEToA)).toEqual({ message: 'A1' })
+        })
+
+        yield* Effect.all([nodeACode, nodeECode], { concurrency: 'unbounded' })
+      }).pipe(withCtx(test)),
+    )
   })
 
   Vitest.describe('mixture of messagechannel and proxy connections', () => {
@@ -490,43 +820,78 @@ Vitest.describe('webmesh node', { timeout: 1000 }, () => {
       }).pipe(withCtx(test)),
     )
 
-    // TODO this currently fails but should work. probably needs some more guarding internally.
-    Vitest.scopedLive.skip('should work for messagechannels', (test) =>
+    Vitest.scopedLive('should work for messagechannels', (test) =>
       Effect.gen(function* () {
         const nodeA = yield* makeMeshNode('A')
         const nodeB = yield* makeMeshNode('B')
+        const nodeC = yield* makeMeshNode('C')
 
         yield* connectNodesViaMessageChannel(nodeB, nodeA)
-        yield* connectNodesViaBroadcastChannel(nodeA, nodeB)
+        yield* connectNodesViaBroadcastChannel(nodeB, nodeC)
 
         const nodeACode = Effect.gen(function* () {
-          const channelAToB = yield* createChannel(nodeA, 'B', { mode: 'messagechannel' })
-          yield* channelAToB.send({ message: 'A1' })
-          expect(yield* getFirstMessage(channelAToB)).toEqual({ message: 'B1' })
+          const channelAToC = yield* createChannel(nodeA, 'C', { mode: 'proxy' })
+          yield* channelAToC.send({ message: 'A1' })
+          expect(yield* getFirstMessage(channelAToC)).toEqual({ message: 'C1' })
         })
 
-        const nodeBCode = Effect.gen(function* () {
-          const channelBToA = yield* createChannel(nodeB, 'A', { mode: 'messagechannel' })
-          yield* channelBToA.send({ message: 'B1' })
-          expect(yield* getFirstMessage(channelBToA)).toEqual({ message: 'A1' })
+        const nodeCCode = Effect.gen(function* () {
+          const channelCToA = yield* createChannel(nodeC, 'A', { mode: 'proxy' })
+          yield* channelCToA.send({ message: 'C1' })
+          expect(yield* getFirstMessage(channelCToA)).toEqual({ message: 'A1' })
         })
 
-        yield* Effect.all([nodeACode, nodeBCode], { concurrency: 'unbounded' })
+        yield* Effect.all([nodeACode, nodeCCode], { concurrency: 'unbounded' })
       }).pipe(withCtx(test)),
     )
   })
-})
 
-const envTruish = (env: string | undefined) => env !== undefined && env !== 'false' && env !== '0'
-const isCi = envTruish(process.env.CI)
+  Vitest.describe('broadcast channel', () => {
+    Vitest.scopedLive('should work', (test) =>
+      Effect.gen(function* () {
+        const nodeA = yield* makeMeshNode('A')
+        const nodeB = yield* makeMeshNode('B')
+        const nodeC = yield* makeMeshNode('C')
+
+        yield* connectNodesViaMessageChannel(nodeA, nodeB)
+        yield* connectNodesViaMessageChannel(nodeB, nodeC)
+
+        const channelOnA = yield* nodeA.makeBroadcastChannel({ channelName: 'test', schema: Schema.String })
+        const channelOnC = yield* nodeC.makeBroadcastChannel({ channelName: 'test', schema: Schema.String })
+
+        const listenOnAFiber = yield* channelOnA.listen.pipe(
+          Stream.flatten(),
+          Stream.runHead,
+          Effect.flatten,
+          Effect.fork,
+        )
+        const listenOnCFiber = yield* channelOnC.listen.pipe(
+          Stream.flatten(),
+          Stream.runHead,
+          Effect.flatten,
+          Effect.fork,
+        )
+
+        yield* channelOnA.send('A1')
+        yield* channelOnC.send('C1')
+
+        expect(yield* listenOnAFiber).toEqual('C1')
+        expect(yield* listenOnCFiber).toEqual('A1')
+      }).pipe(withCtx(test)),
+    )
+  })
+})
 
-const otelLayer = isCi ? Layer.empty : OtelLiveHttp({ serviceName: 'webmesh-node-test', skipLogUrl: false })
+const otelLayer = IS_CI ? Layer.empty : OtelLiveHttp({ serviceName: 'webmesh-node-test', skipLogUrl: false })
 
 const withCtx =
-  (testContext: Vitest.TaskContext, { suffix, skipOtel = false }: { suffix?: string; skipOtel?: boolean } = {}) =>
+  (
+    testContext: Vitest.TaskContext,
+    { suffix, skipOtel = false, timeout = testTimeout }: { suffix?: string; skipOtel?: boolean; timeout?: number } = {},
+  ) =>
   <A, E, R>(self: Effect.Effect<A, E, R>) =>
     self.pipe(
-      Effect.timeout(isCi ? 10_000 : 500),
+      Effect.timeout(timeout),
       Effect.provide(Logger.pretty),
       Effect.scoped, // We need to scope the effect manually here because otherwise the span is not closed
       Effect.withSpan(`${testContext.task.suite?.name}:${testContext.task.name}${suffix ? `:${suffix}` : ''}`),