@livestore/webmesh 0.4.0-dev.22 → 0.4.0-dev.23

package/src/channel/direct-channel-internal.ts

@@ -4,7 +4,6 @@ import {
   Deferred,
   Effect,
   Exit,
-  OtelTracer,
   Predicate,
   Queue,
   Schema,
@@ -16,11 +15,6 @@ import {
 import { type ChannelName, type MeshNodeName, type MessageQueueItem, packetAsOtelAttributes } from '../common.ts'
 import * as MeshSchema from '../mesh-schema.ts'
 
-// WORKAROUND: @effect/opentelemetry mis-parses `Span.addEvent(name, attributes)` and treats the attributes object as a
-// time input, causing `TypeError: {} is not iterable` at runtime.
-// Upstream: https://github.com/Effect-TS/effect/pull/5929
-// TODO: simplify back to the 2-arg overload once the upstream fix is released and adopted.
-
 export interface MakeDirectChannelArgs {
   nodeName: MeshNodeName
   /** Queue of incoming messages for this channel */
@@ -94,11 +88,6 @@ export const makeDirectChannelInternal = ({
 
     const deferred = yield* makeDeferredResult()
 
-    const span = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
-    // const span = {
-    //   addEvent: (...msg: any[]) => console.log(`${nodeName}→${channelName}→${target}[${channelVersion}]`, ...msg),
-    // }
-
     const schema = {
       send: Schema.Union(schema_.send, MeshSchema.DirectChannelPing, MeshSchema.DirectChannelPong),
       listen: Schema.Union(schema_.listen, MeshSchema.DirectChannelPing, MeshSchema.DirectChannelPong),
@@ -112,16 +101,12 @@ export const makeDirectChannelInternal = ({
       Effect.gen(function* () {
         const channelState = channelStateRef.current
 
-        span?.addEvent(
-          `process:${packet._tag}`,
-          {
-            channelState: channelState._tag,
-            packetId: packet.id,
-            packetReqId: packet.reqId,
-            packetChannelVersion: Predicate.hasProperty('channelVersion')(packet) ? packet.channelVersion : undefined,
-          },
-          undefined,
-        )
+        yield* Effect.spanEvent(`process:${packet._tag}`, {
+          channelState: channelState._tag,
+          packetId: packet.id,
+          packetReqId: packet.reqId,
+          ...(Predicate.hasProperty('channelVersion')(packet) === true ? { packetChannelVersion: packet.channelVersion } : {}),
+        })
 
         // const reqIdStr =
         //   Predicate.hasProperty('reqId')(packet) && packet.reqId !== undefined ? ` for ${packet.reqId}` : ''
@@ -139,7 +124,7 @@ export const makeDirectChannelInternal = ({
         // If the other side has a higher version, we need to close this channel and
         // recreate it with the new version
         if (packet.channelVersion > channelVersion) {
-          span?.addEvent(`incoming packet has higher version (${packet.channelVersion}), closing channel`)
+          yield* Effect.spanEvent(`incoming packet has higher version (${packet.channelVersion}), closing channel`)
           yield* Scope.close(scope, Exit.succeed('higher-version-expected'))
           // TODO include expected version in the error so the channel gets recreated with the new version
           return 'close'
@@ -158,7 +143,7 @@ export const makeDirectChannelInternal = ({
             remainingHops: packet.hops,
             reqId: undefined,
           })
-          span?.addEvent(
+          yield* Effect.spanEvent(
             `incoming packet has lower version (${packet.channelVersion}), sending request to reconnect (${newPacket.id})`,
           )
 
@@ -173,7 +158,7 @@ export const makeDirectChannelInternal = ({
           } else {
             // In case the instance of the source has changed, we need to close the channel
             // and reconnect with a new channel
-            span?.addEvent(`force-new-channel`)
+            yield* Effect.spanEvent(`force-new-channel`)
             yield* Scope.close(scope, Exit.succeed('force-new-channel'))
             return 'close'
           }
@@ -200,15 +185,15 @@ export const makeDirectChannelInternal = ({
                 remainingHops: packet.hops,
                 reqId: packet.id,
               })
-              span?.addEvent(`Re-sending new request (${newRequestPacket.id}) for incoming request (${packet.id})`)
+              yield* Effect.spanEvent(`Re-sending new request (${newRequestPacket.id}) for incoming request (${packet.id})`)
 
               yield* sendPacket(newRequestPacket)
             }
 
             const isWinner = nodeName > target
 
-            if (isWinner) {
-              span?.addEvent(`winner side: creating direct channel and sending response`)
+            if (isWinner === true) {
+              yield* Effect.spanEvent(`winner side: creating direct channel and sending response`)
               const mc = new MessageChannel()
 
               // We're using a direct channel with acks here to make sure messages are not lost
@@ -236,8 +221,6 @@ export const makeDirectChannelInternal = ({
 
               channelStateRef.current = { _tag: 'winner:ResponseSent', channel, otherSourceId: packet.sourceId }
 
-              // span?.addEvent(`winner side: waiting for ping`)
-
               // Now we wait for the other side to respond via the channel
               yield* channel.listen.pipe(
                 Stream.flatten(),
@@ -246,21 +229,19 @@ export const makeDirectChannelInternal = ({
                 Stream.runDrain,
               )
 
-              // span?.addEvent(`winner side: sending pong`)
-
               yield* channel.send(MeshSchema.DirectChannelPong.make({}))
 
-              span?.addEvent(`winner side: established`)
+              yield* Effect.spanEvent(`winner side: established`)
               channelStateRef.current = { _tag: 'Established', otherSourceId: packet.sourceId }
 
               yield* Deferred.succeed(deferred, channel)
             } else {
-              span?.addEvent(`loser side: waiting for response`)
+              yield* Effect.spanEvent(`loser side: waiting for response`)
               // Wait for `DirectChannelResponseSuccess` packet
               channelStateRef.current = { _tag: 'loser:WaitingForResponse', otherSourceId: packet.sourceId }
             }
 
-            break
+            return
           }
           case 'DirectChannelResponseSuccess': {
             if (channelState._tag !== 'loser:WaitingForResponse') {
@@ -284,8 +265,6 @@ export const makeDirectChannelInternal = ({
               Effect.fork,
             )
 
-            // span?.addEvent(`loser side: sending ping`)
-
             // There seems to be some scenario where the initial ping message is lost.
             // As a workaround until we find the root cause, we're retrying the ping a few times.
             // TODO write a test that reproduces this issue and fix the root cause ()
@@ -294,11 +273,9 @@ export const makeDirectChannelInternal = ({
               .send(MeshSchema.DirectChannelPing.make({}))
               .pipe(Effect.timeout(10), Effect.retry({ times: 2 }))
 
-            // span?.addEvent(`loser side: waiting for pong`)
-
             yield* waitForPongFiber
 
-            span?.addEvent(`loser side: established`)
+            yield* Effect.spanEvent(`loser side: established`)
             channelStateRef.current = { _tag: 'Established', otherSourceId: channelState.otherSourceId }
 
             yield* Deferred.succeed(deferred, channel)
@@ -354,7 +331,7 @@ export const makeDirectChannelInternal = ({
       }
 
       yield* sendPacket(packet)
-      span?.addEvent(`initial edge request sent (${packet.id})`)
+      yield* Effect.spanEvent(`initial edge request sent (${packet.id})`)
     })
 
     yield* edgeRequest

package/src/channel/direct-channel.ts

@@ -61,7 +61,7 @@ export const makeDirectChannel = ({
         innerChannelRef: { current: undefined as WebChannel.WebChannel<any, any> | undefined },
       }
 
-      // #region reconnect-loop
+      //#region reconnect-loop
       yield* Effect.gen(function* () {
         const resultDeferred = yield* Deferred.make<{
           channel: WebChannel.WebChannel<any, any>
@@ -133,8 +133,8 @@ export const makeDirectChannel = ({
               yield* Scope.close(makeDirectChannelScope, channelExit)
 
               if (
-                Cause.isFailType(channelExit.cause) &&
-                Schema.is(WebmeshSchema.DirectChannelResponseNoTransferables)(channelExit.cause.error)
+                Cause.isFailType(channelExit.cause) === true &&
+                Schema.is(WebmeshSchema.DirectChannelResponseNoTransferables)(channelExit.cause.error) === true
               ) {
                 // Only retry when there is a new edge available
                 yield* waitForNewEdgeFiber.pipe(Effect.exit)
@@ -193,7 +193,7 @@ export const makeDirectChannel = ({
         Effect.tapCauseLogPretty,
         Effect.forkScoped,
       )
-      // #endregion reconnect-loop
+      //#endregion reconnect-loop
 
       const parentSpan = yield* Effect.currentSpan.pipe(Effect.orDie)
 

package/src/channel/proxy-channel.ts

@@ -26,6 +26,38 @@ import {
 } from '../common.ts'
 import * as MeshSchema from '../mesh-schema.ts'
 
+/**
+ * Simulation parameters for proxy channel operations.
+ * Used for testing race conditions and timing-sensitive behavior.
+ *
+ * Each parameter represents a delay (in ms) injected at a specific point in the code.
+ * Values are bounded 0-500ms to prevent tests from running too long.
+ */
+export const ProxyChannelSimulationParams = Schema.Struct({
+  /**
+   * Delays related to receiving and processing payload messages
+   */
+  onPayload: Schema.Struct({
+    /** Delay before sending the ACK response (simulates slow ACK send) */
+    beforeAckSend: Schema.Int.pipe(Schema.between(0, 500)),
+    /** Delay after forking the ACK send, before adding message to listen queue */
+    afterAckFork: Schema.Int.pipe(Schema.between(0, 500)),
+    /** Delay after adding message to listen queue */
+    afterListenQueueOffer: Schema.Int.pipe(Schema.between(0, 500)),
+  }),
+})
+
+export type ProxyChannelSimulationParams = typeof ProxyChannelSimulationParams.Type
+
+/** Default simulation params with no delays */
+export const defaultSimulationParams: ProxyChannelSimulationParams = {
+  onPayload: {
+    beforeAckSend: 0,
+    afterAckFork: 0,
+    afterListenQueueOffer: 0,
+  },
+}
+
 interface MakeProxyChannelArgs {
   queue: Queue.Queue<ProxyQueueItem>
   nodeName: MeshNodeName
@@ -37,6 +69,8 @@ interface MakeProxyChannelArgs {
     send: Schema.Schema<any, any>
     listen: Schema.Schema<any, any>
   }
+  /** Optional simulation parameters for testing timing-sensitive behavior */
+  simulation?: ProxyChannelSimulationParams
 }
 
 export const makeProxyChannel = ({
@@ -47,9 +81,18 @@ export const makeProxyChannel = ({
   target,
   channelName,
   schema,
+  simulation = defaultSimulationParams,
 }: MakeProxyChannelArgs) =>
   Effect.scopeWithCloseable((scope) =>
     Effect.gen(function* () {
+      /** Helper to inject simulation delays at specific code points */
+      const simSleep = <TKey extends keyof ProxyChannelSimulationParams>(
+        key: TKey,
+        key2: keyof ProxyChannelSimulationParams[TKey],
+      ) => {
+        const delay = (simulation[key]?.[key2] ?? 0) as number
+        return delay > 0 ? Effect.sleep(delay) : Effect.void
+      }
       type ProxiedChannelState =
         | {
             _tag: 'Initial'
@@ -64,7 +107,7 @@ export const makeProxyChannel = ({
         _tag: 'Established'
         listenSchema: Schema.Schema<any, any>
         listenQueue: Queue.Queue<any>
-        ackMap: Map<string, Deferred.Deferred<void, never>>
+        ackMap: Map<string, Deferred.Deferred<void>>
         combinedChannelId: string
       }
 
@@ -117,7 +160,7 @@ export const makeProxyChannel = ({
       )
 
       const getCombinedChannelId = (otherSideChannelIdCandidate: string) =>
-        [channelIdCandidate, otherSideChannelIdCandidate].sort().join('_')
+        [channelIdCandidate, otherSideChannelIdCandidate].toSorted().join('_')
 
       const earlyPayloadBuffer = yield* Queue.unbounded<typeof MeshSchema.ProxyChannelPayload.Type>().pipe(
         Effect.acquireRelease(Queue.shutdown),
@@ -255,34 +298,46 @@ export const makeProxyChannel = ({
                 )
               }
 
-              // yield* Effect.logDebug(`[${nodeName}] Received payload reqId: ${packet.id}. Sending Ack.`)
-              yield* respondToSender(
-                MeshSchema.ProxyChannelPayloadAck.make({
-                  reqId: packet.id,
-                  remainingHops: packet.hops,
-                  hops: [],
-                  target,
-                  source: nodeName,
-                  channelName,
-                  combinedChannelId:
-                    channelState._tag === 'Established' ? channelState.combinedChannelId : packet.combinedChannelId,
-                }),
-              )
+              // Send ACK fire-and-forget to avoid blocking message processing
+              // This is critical because blocking ACK sends can prevent messages from reaching the listen queue
+              // See test: "ACK forkScoped regression tests" for documentation
+
+              // The ACK send effect with optional simulation delay INSIDE the fork
+              // This is key for testing: with forkScoped, the delay happens in background and doesn't block message processing
+              // Without forkScoped (blocking), the delay would block the message from being added to listen queue
+              const ackSendEffect = Effect.gen(function* () {
+                yield* simSleep('onPayload', 'beforeAckSend')
+                yield* respondToSender(
+                  MeshSchema.ProxyChannelPayloadAck.make({
+                    reqId: packet.id,
+                    remainingHops: packet.hops,
+                    hops: [],
+                    target,
+                    source: nodeName,
+                    channelName,
+                    combinedChannelId:
+                      channelState._tag === 'Established' ? channelState.combinedChannelId : packet.combinedChannelId,
+                  }),
+                )
+              })
+
+              yield* ackSendEffect.pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+
+              // Simulation point: delay after forking ACK (before processing message)
+              yield* simSleep('onPayload', 'afterAckFork')
 
               if (channelState._tag === 'Established') {
                 const decodedMessage = yield* Schema.decodeUnknown(channelState.listenSchema)(packet.payload)
                 yield* channelState.listenQueue.pipe(Queue.offer(decodedMessage))
+
+                // Simulation point: delay after adding to listen queue
+                yield* simSleep('onPayload', 'afterListenQueueOffer')
               } else {
-                // yield* Effect.logDebug(
-                //   `[${nodeName}] Buffering early payload reqId: ${packet.id} (state: ${channelState._tag})`,
-                // )
                 yield* Queue.offer(earlyPayloadBuffer, packet)
               }
               return
             }
             case 'ProxyChannelPayloadAck': {
-              // yield* Effect.logDebug(`[${nodeName}] Received Ack for reqId: ${packet.reqId}`)
-
               if (channelState._tag !== 'Established') {
                 yield* Effect.spanEvent(`Not yet connected to ${target}. dropping message`)
                 yield* Effect.logWarning(
@@ -291,9 +346,14 @@ export const makeProxyChannel = ({
                 return
               }
 
-              const ack =
-                channelState.ackMap.get(packet.reqId) ??
-                shouldNeverHappen(`[ProxyChannel[${channelKey}]] Expected ack for ${packet.reqId}`)
+              // Handle missing ACK gracefully - can happen with synthetic ACKs from relay or duplicate ACKs
+              const ack = channelState.ackMap.get(packet.reqId)
+              if (ack === undefined) {
+                yield* Effect.logDebug(
+                  `Received ACK for unknown reqId: ${packet.reqId} (may be synthetic or duplicate)`,
+                )
+                return
+              }
               yield* Deferred.succeed(ack, void 0)
 
               channelState.ackMap.delete(packet.reqId)
@@ -321,7 +381,7 @@ export const makeProxyChannel = ({
 
       yield* Effect.spanEvent(`Connecting`)
 
-      const ackMap = new Map<string, Deferred.Deferred<void, never>>()
+      const ackMap = new Map<string, Deferred.Deferred<void>>()
 
       // check if already established via incoming `ProxyChannelRequest` from other side
       // which indicates we already have a edge to the target node
@@ -366,7 +426,7 @@ export const makeProxyChannel = ({
 
             const innerSend = Effect.gen(function* () {
               // Note we're re-creating new packets every time otherwise they will be skipped because of `handledIds`
-              const ack = yield* Deferred.make<void, never>()
+              const ack = yield* Deferred.make<void>()
               const packet = MeshSchema.ProxyChannelPayload.make({
                 channelName,
                 payload,

package/src/common.ts

@@ -19,19 +19,18 @@ export type ChannelName = string
 export type ChannelKey = `target:${MeshNodeName}, channelName:${ChannelName}`
 
 // TODO actually use this to avoid timeouts in certain cases
-// export class NoConnectionRouteSignal extends Schema.TaggedError<NoConnectionRouteSignal>()(
-//   'NoConnectionRouteSignal',
-//   {},
-// ) {}
+// export class NoConnectionRouteSignal extends Schema.TaggedError<NoConnectionRouteSignal>(
+//   '~@livestore/webmesh/NoConnectionRouteSignal',
+// )('NoConnectionRouteSignal', {}) {}
 
-export class EdgeAlreadyExistsError extends Schema.TaggedError<EdgeAlreadyExistsError>()('EdgeAlreadyExistsError', {
+export class EdgeAlreadyExistsError extends Schema.TaggedError<EdgeAlreadyExistsError>('~@livestore/webmesh/EdgeAlreadyExistsError')('EdgeAlreadyExistsError', {
   target: Schema.String,
 }) {}
 
 export const packetAsOtelAttributes = (packet: typeof Packet.Type) => ({
   packetId: packet.id,
   'span.label':
-    packet.id + (Predicate.hasProperty(packet, 'reqId') && packet.reqId !== undefined ? ` for ${packet.reqId}` : ''),
+    packet.id + (Predicate.hasProperty(packet, 'reqId') === true && packet.reqId !== undefined ? ` for ${packet.reqId}` : ''),
   ...omitUndefineds({
     packet:
       packet._tag !== 'DirectChannelResponseSuccess' && packet._tag !== 'ProxyChannelPayload' ? packet : undefined,