@livestore/webmesh 0.4.0-dev.9 → 0.5.0-dev.0

package/src/channel/proxy-channel.ts

@@ -1,9 +1,9 @@
 import { casesHandled, shouldNeverHappen } from '@livestore/utils'
-import type { PubSub } from '@livestore/utils/effect'
 import {
+  type PubSub,
   Deferred,
   Effect,
-  Either,
+  Result,
   Exit,
   Fiber,
   FiberHandle,
@@ -26,6 +26,38 @@ import {
 } from '../common.ts'
 import * as MeshSchema from '../mesh-schema.ts'
 
+/**
+ * Simulation parameters for proxy channel operations.
+ * Used for testing race conditions and timing-sensitive behavior.
+ *
+ * Each parameter represents a delay (in ms) injected at a specific point in the code.
+ * Values are bounded 0-500ms to prevent tests from running too long.
+ */
+export const ProxyChannelSimulationParams = Schema.Struct({
+  /**
+   * Delays related to receiving and processing payload messages
+   */
+  onPayload: Schema.Struct({
+    /** Delay before sending the ACK response (simulates slow ACK send) */
+    beforeAckSend: Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 500 })),
+    /** Delay after forking the ACK send, before adding message to listen queue */
+    afterAckFork: Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 500 })),
+    /** Delay after adding message to listen queue */
+    afterListenQueueOffer: Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 500 })),
+  }),
+})
+
+export type ProxyChannelSimulationParams = typeof ProxyChannelSimulationParams.Type
+
+/** Default simulation params with no delays */
+export const defaultSimulationParams: ProxyChannelSimulationParams = {
+  onPayload: {
+    beforeAckSend: 0,
+    afterAckFork: 0,
+    afterListenQueueOffer: 0,
+  },
+}
+
 interface MakeProxyChannelArgs {
   queue: Queue.Queue<ProxyQueueItem>
   nodeName: MeshNodeName
@@ -34,9 +66,11 @@ interface MakeProxyChannelArgs {
   channelName: ChannelName
   target: MeshNodeName
   schema: {
-    send: Schema.Schema<any, any>
-    listen: Schema.Schema<any, any>
+    send: Schema.Codec<any, any>
+    listen: Schema.Codec<any, any>
   }
+  /** Optional simulation parameters for testing timing-sensitive behavior */
+  simulation?: ProxyChannelSimulationParams
 }
 
 export const makeProxyChannel = ({
@@ -47,9 +81,18 @@ export const makeProxyChannel = ({
   target,
   channelName,
   schema,
+  simulation = defaultSimulationParams,
 }: MakeProxyChannelArgs) =>
   Effect.scopeWithCloseable((scope) =>
     Effect.gen(function* () {
+      /** Helper to inject simulation delays at specific code points */
+      const simSleep = <TKey extends keyof ProxyChannelSimulationParams>(
+        key: TKey,
+        key2: keyof ProxyChannelSimulationParams[TKey],
+      ) => {
+        const delay = (simulation[key]?.[key2] ?? 0) as number
+        return delay > 0 ? Effect.sleep(delay) : Effect.void
+      }
       type ProxiedChannelState =
         | {
             _tag: 'Initial'
@@ -62,9 +105,9 @@ export const makeProxyChannel = ({
 
       type ProxiedChannelStateEstablished = {
         _tag: 'Established'
-        listenSchema: Schema.Schema<any, any>
+        listenSchema: Schema.Codec<any, any>
         listenQueue: Queue.Queue<any>
-        ackMap: Map<string, Deferred.Deferred<void, never>>
+        ackMap: Map<string, Deferred.Deferred<void>>
         combinedChannelId: string
       }
 
@@ -88,6 +131,10 @@ export const makeProxyChannel = ({
       const channelSpan = yield* Effect.currentSpan.pipe(Effect.orDie)
 
       const connectedStateRef = yield* SubscriptionRef.make<ProxiedChannelStateEstablished | false>(false)
+      // Incoming packets can be processed as soon as the queue-draining fiber starts, so state
+      // referenced by established channels must exist before that processor is forked.
+      const listenQueue = yield* Queue.unbounded<any>()
+      const ackMap = new Map<string, Deferred.Deferred<void>>()
 
       const waitForEstablished = Effect.gen(function* () {
         const state = yield* SubscriptionRef.waitUntil(connectedStateRef, (state) => state !== false)
@@ -117,10 +164,11 @@ export const makeProxyChannel = ({
       )
 
       const getCombinedChannelId = (otherSideChannelIdCandidate: string) =>
-        [channelIdCandidate, otherSideChannelIdCandidate].sort().join('_')
+        [channelIdCandidate, otherSideChannelIdCandidate].toSorted().join('_')
 
-      const earlyPayloadBuffer = yield* Queue.unbounded<typeof MeshSchema.ProxyChannelPayload.Type>().pipe(
-        Effect.acquireRelease(Queue.shutdown),
+      const earlyPayloadBuffer = yield* Effect.acquireRelease(
+        Queue.unbounded<typeof MeshSchema.ProxyChannelPayload.Type>(),
+        Queue.shutdown,
       )
 
       const processProxyPacket = ({ packet, respondToSender }: ProxyQueueItem) =>
@@ -224,7 +272,7 @@ export const makeProxyChannel = ({
               const establishedState = channelStateRef.current
               if (establishedState._tag === 'Established') {
                 //
-                const bufferedPackets = yield* Queue.takeAll(earlyPayloadBuffer)
+                const bufferedPackets = yield* Queue.clear(earlyPayloadBuffer)
                 // yield* Effect.logDebug(
                 //   `[${nodeName}] Draining early payload buffer (${bufferedPackets.length}) after ResponseSuccess`,
                 // )
@@ -235,10 +283,10 @@ export const makeProxyChannel = ({
                     )
                     continue
                   }
-                  const decodedMessage = yield* Schema.decodeUnknown(establishedState.listenSchema)(
+                  const decodedMessage = yield* Schema.decodeUnknownEffect(establishedState.listenSchema)(
                     bufferedPacket.payload,
                   )
-                  yield* establishedState.listenQueue.pipe(Queue.offer(decodedMessage))
+                  yield* Queue.offer(establishedState.listenQueue, decodedMessage)
                 }
               } else {
                 yield* Effect.logError(
@@ -255,34 +303,46 @@ export const makeProxyChannel = ({
                 )
               }
 
-              // yield* Effect.logDebug(`[${nodeName}] Received payload reqId: ${packet.id}. Sending Ack.`)
-              yield* respondToSender(
-                MeshSchema.ProxyChannelPayloadAck.make({
-                  reqId: packet.id,
-                  remainingHops: packet.hops,
-                  hops: [],
-                  target,
-                  source: nodeName,
-                  channelName,
-                  combinedChannelId:
-                    channelState._tag === 'Established' ? channelState.combinedChannelId : packet.combinedChannelId,
-                }),
-              )
+              // Send ACK fire-and-forget to avoid blocking message processing
+              // This is critical because blocking ACK sends can prevent messages from reaching the listen queue
+              // See test: "ACK forkScoped regression tests" for documentation
+
+              // The ACK send effect with optional simulation delay INSIDE the fork
+              // This is key for testing: with forkScoped, the delay happens in background and doesn't block message processing
+              // Without forkScoped (blocking), the delay would block the message from being added to listen queue
+              const ackSendEffect = Effect.gen(function* () {
+                yield* simSleep('onPayload', 'beforeAckSend')
+                yield* respondToSender(
+                  MeshSchema.ProxyChannelPayloadAck.make({
+                    reqId: packet.id,
+                    remainingHops: packet.hops,
+                    hops: [],
+                    target,
+                    source: nodeName,
+                    channelName,
+                    combinedChannelId:
+                      channelState._tag === 'Established' ? channelState.combinedChannelId : packet.combinedChannelId,
+                  }),
+                )
+              })
+
+              yield* ackSendEffect.pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+
+              // Simulation point: delay after forking ACK (before processing message)
+              yield* simSleep('onPayload', 'afterAckFork')
 
               if (channelState._tag === 'Established') {
-                const decodedMessage = yield* Schema.decodeUnknown(channelState.listenSchema)(packet.payload)
-                yield* channelState.listenQueue.pipe(Queue.offer(decodedMessage))
+                const decodedMessage = yield* Schema.decodeUnknownEffect(channelState.listenSchema)(packet.payload)
+                yield* Queue.offer(channelState.listenQueue, decodedMessage)
+
+                // Simulation point: delay after adding to listen queue
+                yield* simSleep('onPayload', 'afterListenQueueOffer')
               } else {
-                // yield* Effect.logDebug(
-                //   `[${nodeName}] Buffering early payload reqId: ${packet.id} (state: ${channelState._tag})`,
-                // )
                 yield* Queue.offer(earlyPayloadBuffer, packet)
               }
               return
             }
             case 'ProxyChannelPayloadAck': {
-              // yield* Effect.logDebug(`[${nodeName}] Received Ack for reqId: ${packet.reqId}`)
-
               if (channelState._tag !== 'Established') {
                 yield* Effect.spanEvent(`Not yet connected to ${target}. dropping message`)
                 yield* Effect.logWarning(
@@ -291,9 +351,14 @@ export const makeProxyChannel = ({
                 return
               }
 
-              const ack =
-                channelState.ackMap.get(packet.reqId) ??
-                shouldNeverHappen(`[ProxyChannel[${channelKey}]] Expected ack for ${packet.reqId}`)
+              // Handle missing ACK gracefully - can happen with synthetic ACKs from relay or duplicate ACKs
+              const ack = channelState.ackMap.get(packet.reqId)
+              if (ack === undefined) {
+                yield* Effect.logDebug(
+                  `Received ACK for unknown reqId: ${packet.reqId} (may be synthetic or duplicate)`,
+                )
+                return
+              }
               yield* Deferred.succeed(ack, void 0)
 
               channelState.ackMap.delete(packet.reqId)
@@ -310,19 +375,8 @@ export const makeProxyChannel = ({
           }),
         )
 
-      yield* Stream.fromQueue(queue).pipe(
-        Stream.tap(processProxyPacket),
-        Stream.runDrain,
-        Effect.tapCauseLogPretty,
-        Effect.forkScoped,
-      )
-
-      const listenQueue = yield* Queue.unbounded<any>()
-
       yield* Effect.spanEvent(`Connecting`)
 
-      const ackMap = new Map<string, Deferred.Deferred<void, never>>()
-
       // check if already established via incoming `ProxyChannelRequest` from other side
       // which indicates we already have a edge to the target node
       // const channelState = channelStateRef.current
@@ -341,6 +395,15 @@ export const makeProxyChannel = ({
           Effect.forkScoped,
         )
 
+        // Drain queued proxy packets only after the local side entered `Pending` and sent its
+        // request. Otherwise an already-buffered response can race initialization.
+        yield* Stream.fromQueue(queue).pipe(
+          Stream.tap(processProxyPacket),
+          Stream.runDrain,
+          Effect.tapCauseLogPretty,
+          Effect.forkScoped,
+        )
+
         const { combinedChannelId: channelId } = yield* waitForEstablished
 
         yield* Fiber.interrupt(retryOnNewEdgeFiber)
@@ -350,7 +413,7 @@ export const makeProxyChannel = ({
 
       const send = (message: any) =>
         Effect.gen(function* () {
-          const payload = yield* Schema.encodeUnknown(schema.send)(message)
+          const payload = yield* Schema.encodeUnknownEffect(schema.send)(message)
           const sendFiberHandle = yield* FiberHandle.make<void, never>()
 
           const sentDeferred = yield* Deferred.make<void>()
@@ -366,7 +429,7 @@ export const makeProxyChannel = ({
 
             const innerSend = Effect.gen(function* () {
               // Note we're re-creating new packets every time otherwise they will be skipped because of `handledIds`
-              const ack = yield* Deferred.make<void, never>()
+              const ack = yield* Deferred.make<void>()
               const packet = MeshSchema.ProxyChannelPayload.make({
                 channelName,
                 payload,
@@ -381,7 +444,7 @@ export const makeProxyChannel = ({
 
               yield* sendPacket(packet)
 
-              yield* ack
+              yield* Deferred.await(ack)
               yield* Deferred.succeed(sentDeferred, void 0)
 
               debugInfo.pendingSends--
@@ -390,18 +453,18 @@ export const makeProxyChannel = ({
             // TODO make this configurable
             // Schedule.exponential(10): 10, 20, 40, 80, 160, 320, ...
             yield* innerSend.pipe(Effect.timeout(100), Effect.retry(Schedule.exponential(10)), Effect.orDie)
-          }).pipe(Effect.tapErrorCause(Effect.logError))
+          }).pipe(Effect.tapCause(Effect.logError))
 
-          const rerunOnNewChannelFiber = yield* connectedStateRef.changes.pipe(
+          const rerunOnNewChannelFiber = yield* SubscriptionRef.changes(connectedStateRef).pipe(
             Stream.filter((_) => _ === false),
             Stream.tap(() => FiberHandle.run(sendFiberHandle, trySend)),
             Stream.runDrain,
-            Effect.fork,
+            Effect.forkChild,
           )
 
           yield* FiberHandle.run(sendFiberHandle, trySend)
 
-          yield* sentDeferred
+          yield* Deferred.await(sentDeferred)
 
           yield* Fiber.interrupt(rerunOnNewChannelFiber)
         }).pipe(
@@ -410,11 +473,11 @@ export const makeProxyChannel = ({
           Effect.withParentSpan(channelSpan),
         )
 
-      const listen = Stream.fromQueue(listenQueue).pipe(Stream.map(Either.right))
+      const listen = Stream.fromQueue(listenQueue).pipe(Stream.map(Result.succeed))
 
-      const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
+      const closedDeferred = yield* Effect.acquireRelease(Deferred.make<void>(), Deferred.done(Exit.void))
 
-      const runtime = yield* Effect.runtime()
+      const services = yield* Effect.context()
 
       const webChannel = {
         [WebChannel.WebChannelSymbol]: WebChannel.WebChannelSymbol,
@@ -429,7 +492,7 @@ export const makeProxyChannel = ({
           debug: {
             ping: (message = 'ping') =>
               send(WebChannel.DebugPingMessage.make({ message })).pipe(
-                Effect.provide(runtime),
+                Effect.provide(services),
                 Effect.tapCauseLogPretty,
                 Effect.runFork,
               ),

package/src/common.ts

@@ -19,19 +19,21 @@ export type ChannelName = string
 export type ChannelKey = `target:${MeshNodeName}, channelName:${ChannelName}`
 
 // TODO actually use this to avoid timeouts in certain cases
-// export class NoConnectionRouteSignal extends Schema.TaggedError<NoConnectionRouteSignal>()(
-//   'NoConnectionRouteSignal',
-//   {},
-// ) {}
+// export class NoConnectionRouteSignal extends Schema.TaggedError<NoConnectionRouteSignal>(
+//   '~@livestore/webmesh/NoConnectionRouteSignal',
+// )('NoConnectionRouteSignal', {}) {}
 
-export class EdgeAlreadyExistsError extends Schema.TaggedError<EdgeAlreadyExistsError>()('EdgeAlreadyExistsError', {
+export class EdgeAlreadyExistsError extends Schema.TaggedError<EdgeAlreadyExistsError>(
+  '~@livestore/webmesh/EdgeAlreadyExistsError',
+)('EdgeAlreadyExistsError', {
   target: Schema.String,
 }) {}
 
 export const packetAsOtelAttributes = (packet: typeof Packet.Type) => ({
   packetId: packet.id,
   'span.label':
-    packet.id + (Predicate.hasProperty(packet, 'reqId') && packet.reqId !== undefined ? ` for ${packet.reqId}` : ''),
+    packet.id +
+    (Predicate.hasProperty(packet, 'reqId') === true && packet.reqId !== undefined ? ` for ${packet.reqId}` : ''),
   ...omitUndefineds({
     packet:
       packet._tag !== 'DirectChannelResponseSuccess' && packet._tag !== 'ProxyChannelPayload' ? packet : undefined,
@@ -41,7 +43,7 @@ export const packetAsOtelAttributes = (packet: typeof Packet.Type) => ({
 export const ListenForChannelResult = Schema.Struct({
   channelName: Schema.String,
   source: Schema.String,
-  mode: Schema.Union(Schema.Literal('proxy'), Schema.Literal('direct')),
+  mode: Schema.Literals(['proxy', 'direct']),
 })
 
 export type ListenForChannelResult = typeof ListenForChannelResult.Type

package/src/mesh-schema.ts

@@ -1,9 +1,9 @@
-import { Schema, Transferable } from '@livestore/utils/effect'
+import { Effect, Schema, Transferable } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
 
 const id = Schema.String.pipe(
-  Schema.optional,
-  Schema.withDefaults({ constructor: () => nanoid(10), decoding: () => nanoid(10) }),
+  Schema.withDecodingDefaultType(Effect.sync(() => nanoid(10))),
+  Schema.withConstructorDefault(Effect.sync(() => nanoid(10))),
 )
 
 const defaultPacketFields = {
@@ -23,10 +23,10 @@ const remainingHopsUndefined = Schema.Undefined.pipe(Schema.optional)
  * We need a clear path back to the sender to avoid this, thus we respond with a separate
  * `DirectChannelResponseSuccess` which contains the `port`.
  */
-export class DirectChannelRequest extends Schema.TaggedStruct('DirectChannelRequest', {
+export const DirectChannelRequest = Schema.TaggedStruct('DirectChannelRequest', {
   ...defaultPacketFields,
   remainingHops: Schema.Array(Schema.String).pipe(Schema.optional),
-  channelVersion: Schema.Number,
+  channelVersion: Schema.Finite,
   /** Only set if the request is in response to an incoming request */
   reqId: Schema.UndefinedOr(Schema.String),
   /**
@@ -34,70 +34,70 @@ export class DirectChannelRequest extends Schema.TaggedStruct('DirectChannelRequ
    * source has changed.
    */
   sourceId: Schema.String,
-}) {}
+})
 
-export class DirectChannelResponseSuccess extends Schema.TaggedStruct('DirectChannelResponseSuccess', {
+export const DirectChannelResponseSuccess = Schema.TaggedStruct('DirectChannelResponseSuccess', {
   ...defaultPacketFields,
   reqId: Schema.String,
   port: Transferable.MessagePort,
   // Since we can't copy this message, we need to follow the exact route back to the sender
   remainingHops: Schema.Array(Schema.String),
-  channelVersion: Schema.Number,
-}) {}
+  channelVersion: Schema.Finite,
+})
 
-export class DirectChannelResponseNoTransferables extends Schema.TaggedStruct('DirectChannelResponseNoTransferables', {
+export const DirectChannelResponseNoTransferables = Schema.TaggedStruct('DirectChannelResponseNoTransferables', {
   ...defaultPacketFields,
   reqId: Schema.String,
   remainingHops: Schema.Array(Schema.String),
-}) {}
+})
 
-export class ProxyChannelRequest extends Schema.TaggedStruct('ProxyChannelRequest', {
+export const ProxyChannelRequest = Schema.TaggedStruct('ProxyChannelRequest', {
   ...defaultPacketFields,
   remainingHops: remainingHopsUndefined,
   channelIdCandidate: Schema.String,
-}) {}
+})
 
-export class ProxyChannelResponseSuccess extends Schema.TaggedStruct('ProxyChannelResponseSuccess', {
+export const ProxyChannelResponseSuccess = Schema.TaggedStruct('ProxyChannelResponseSuccess', {
   ...defaultPacketFields,
   reqId: Schema.String,
   remainingHops: Schema.Array(Schema.String),
   combinedChannelId: Schema.String,
   channelIdCandidate: Schema.String,
-}) {}
+})
 
-export class ProxyChannelPayload extends Schema.TaggedStruct('ProxyChannelPayload', {
+export const ProxyChannelPayload = Schema.TaggedStruct('ProxyChannelPayload', {
   ...defaultPacketFields,
   remainingHops: remainingHopsUndefined,
   payload: Schema.Any,
   combinedChannelId: Schema.String,
-}) {}
+})
 
-export class ProxyChannelPayloadAck extends Schema.TaggedStruct('ProxyChannelPayloadAck', {
+export const ProxyChannelPayloadAck = Schema.TaggedStruct('ProxyChannelPayloadAck', {
   ...defaultPacketFields,
   reqId: Schema.String,
   remainingHops: Schema.Array(Schema.String),
   combinedChannelId: Schema.String,
-}) {}
+})
 
 /**
  * Broadcast to all nodes when a new edge is added.
  * Mostly used for auto-reconnect purposes.
  */
-export class NetworkEdgeAdded extends Schema.TaggedStruct('NetworkEdgeAdded', {
+export const NetworkEdgeAdded = Schema.TaggedStruct('NetworkEdgeAdded', {
   id,
   source: Schema.String,
   target: Schema.String,
-}) {}
+})
 
-export class NetworkTopologyRequest extends Schema.TaggedStruct('NetworkTopologyRequest', {
+export const NetworkTopologyRequest = Schema.TaggedStruct('NetworkTopologyRequest', {
   id,
   hops: Schema.Array(Schema.String),
   /** Always fixed to who requested the topology */
   source: Schema.String,
   target: Schema.Literal('-'),
-}) {}
+})
 
-export class NetworkTopologyResponse extends Schema.TaggedStruct('NetworkTopologyResponse', {
+export const NetworkTopologyResponse = Schema.TaggedStruct('NetworkTopologyResponse', {
   id,
   reqId: Schema.String,
   remainingHops: Schema.Array(Schema.String),
@@ -106,7 +106,7 @@ export class NetworkTopologyResponse extends Schema.TaggedStruct('NetworkTopolog
   /** Always fixed to who requested the topology */
   source: Schema.String,
   target: Schema.Literal('-'),
-}) {}
+})
 
 export const BroadcastChannelPacket = Schema.TaggedStruct('BroadcastChannelPacket', {
   id,
@@ -121,27 +121,28 @@ export const BroadcastChannelPacket = Schema.TaggedStruct('BroadcastChannelPacke
   target: Schema.Literal('-'),
 })
 
-export class DirectChannelPacket extends Schema.Union(
+export const DirectChannelPacket = Schema.Union([
   DirectChannelRequest,
   DirectChannelResponseSuccess,
   DirectChannelResponseNoTransferables,
-) {}
+])
 
-export class ProxyChannelPacket extends Schema.Union(
+export const ProxyChannelPacket = Schema.Union([
   ProxyChannelRequest,
   ProxyChannelResponseSuccess,
   ProxyChannelPayload,
   ProxyChannelPayloadAck,
-) {}
+])
 
-export class Packet extends Schema.Union(
+export const Packet = Schema.Union([
   DirectChannelPacket,
   ProxyChannelPacket,
   NetworkEdgeAdded,
   NetworkTopologyRequest,
   NetworkTopologyResponse,
   BroadcastChannelPacket,
-) {}
+])
+
+export const DirectChannelPing = Schema.TaggedStruct('DirectChannelPing', {})
 
-export class DirectChannelPing extends Schema.TaggedStruct('DirectChannelPing', {}) {}
-export class DirectChannelPong extends Schema.TaggedStruct('DirectChannelPong', {}) {}
+export const DirectChannelPong = Schema.TaggedStruct('DirectChannelPong', {})