@livestore/webmesh 0.3.0-dev.5 → 0.3.0-dev.51

package/src/channel/direct-channel-internal.ts

@@ -0,0 +1,356 @@
+import { casesHandled, shouldNeverHappen } from '@livestore/utils'
+import type { PubSub } from '@livestore/utils/effect'
+import {
+  Deferred,
+  Effect,
+  Exit,
+  OtelTracer,
+  Predicate,
+  Queue,
+  Schema,
+  Scope,
+  Stream,
+  WebChannel,
+} from '@livestore/utils/effect'
+
+import { type ChannelName, type MeshNodeName, type MessageQueueItem, packetAsOtelAttributes } from '../common.js'
+import * as MeshSchema from '../mesh-schema.js'
+
+export interface MakeDirectChannelArgs {
+  nodeName: MeshNodeName
+  /** Queue of incoming messages for this channel */
+  incomingPacketsQueue: Queue.Queue<MessageQueueItem>
+  newEdgeAvailablePubSub: PubSub.PubSub<MeshNodeName>
+  channelName: ChannelName
+  target: MeshNodeName
+  sendPacket: (packet: typeof MeshSchema.DirectChannelPacket.Type) => Effect.Effect<void>
+  checkTransferableEdges: (
+    packet: typeof MeshSchema.DirectChannelPacket.Type,
+  ) => typeof MeshSchema.DirectChannelResponseNoTransferables.Type | undefined
+  schema: WebChannel.OutputSchema<any, any, any, any>
+}
+
+const makeDeferredResult = Deferred.make<
+  WebChannel.WebChannel<any, any>,
+  typeof MeshSchema.DirectChannelResponseNoTransferables.Type
+>
+
+/**
+ * The channel version is important here, as a channel will only be established once both sides have the same version.
+ * The version is used to avoid concurrency issues where both sides have different incompatible message ports.
+ */
+export const makeDirectChannelInternal = ({
+  nodeName,
+  incomingPacketsQueue,
+  target,
+  checkTransferableEdges,
+  channelName,
+  schema: schema_,
+  sendPacket,
+  channelVersion,
+  scope,
+  sourceId,
+}: MakeDirectChannelArgs & {
+  channelVersion: number
+  /** We're passing in the closeable scope from the wrapping direct channel */
+  scope: Scope.CloseableScope
+  sourceId: string
+}): Effect.Effect<
+  WebChannel.WebChannel<any, any>,
+  typeof MeshSchema.DirectChannelResponseNoTransferables.Type,
+  Scope.Scope
+> =>
+  Effect.gen(function* () {
+    // yield* Effect.addFinalizer((exit) =>
+    //   Effect.spanEvent(`shutdown:${exit._tag === 'Success' ? 'Success' : Cause.pretty(exit.cause)}`),
+    // )
+
+    type ChannelState =
+      | {
+          _tag: 'Initial'
+        }
+      | {
+          _tag: 'RequestSent'
+          reqPacketId: string
+        }
+      | {
+          _tag: 'winner:ResponseSent'
+          channel: WebChannel.WebChannel<any, any>
+          otherSourceId: string
+        }
+      | {
+          _tag: 'loser:WaitingForResponse'
+          otherSourceId: string
+        }
+      | {
+          _tag: 'Established'
+          otherSourceId: string
+        }
+
+    const deferred = yield* makeDeferredResult()
+
+    const span = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
+    // const span = {
+    //   addEvent: (...msg: any[]) => console.log(`${nodeName}→${channelName}→${target}[${channelVersion}]`, ...msg),
+    // }
+
+    const schema = {
+      send: Schema.Union(schema_.send, MeshSchema.DirectChannelPing, MeshSchema.DirectChannelPong),
+      listen: Schema.Union(schema_.listen, MeshSchema.DirectChannelPing, MeshSchema.DirectChannelPong),
+    }
+
+    const channelStateRef: { current: ChannelState } = {
+      current: { _tag: 'Initial' },
+    }
+
+    const processMessagePacket = ({ packet, respondToSender }: MessageQueueItem) =>
+      Effect.gen(function* () {
+        const channelState = channelStateRef.current
+
+        span?.addEvent(`process:${packet._tag}`, {
+          channelState: channelState._tag,
+          packetId: packet.id,
+          packetReqId: packet.reqId,
+          packetChannelVersion: Predicate.hasProperty('channelVersion')(packet) ? packet.channelVersion : undefined,
+        })
+
+        // const reqIdStr =
+        //   Predicate.hasProperty('reqId')(packet) && packet.reqId !== undefined ? ` for ${packet.reqId}` : ''
+        // yield* Effect.log(
+        //   `${nodeName}→${channelName}→${target}[${channelVersion}]:process packet ${packet._tag} [${packet.id}${reqIdStr}], channel state: ${channelState._tag}`,
+        // )
+
+        if (channelState._tag === 'Initial') return shouldNeverHappen()
+
+        if (packet._tag === 'DirectChannelResponseNoTransferables') {
+          yield* Deferred.fail(deferred, packet)
+          return 'close'
+        }
+
+        // If the other side has a higher version, we need to close this channel and
+        // recreate it with the new version
+        if (packet.channelVersion > channelVersion) {
+          span?.addEvent(`incoming packet has higher version (${packet.channelVersion}), closing channel`)
+          yield* Scope.close(scope, Exit.succeed('higher-version-expected'))
+          // TODO include expected version in the error so the channel gets recreated with the new version
+          return 'close'
+        }
+
+        // If this channel has a higher version, we need to signal the other side to close
+        // and recreate the channel with the new version
+        if (packet.channelVersion < channelVersion) {
+          const newPacket = MeshSchema.DirectChannelRequest.make({
+            source: nodeName,
+            sourceId,
+            target,
+            channelName,
+            channelVersion,
+            hops: [],
+            remainingHops: packet.hops,
+            reqId: undefined,
+          })
+          span?.addEvent(
+            `incoming packet has lower version (${packet.channelVersion}), sending request to reconnect (${newPacket.id})`,
+          )
+
+          yield* sendPacket(newPacket)
+
+          return
+        }
+
+        if (channelState._tag === 'Established' && packet._tag === 'DirectChannelRequest') {
+          if (packet.sourceId === channelState.otherSourceId) {
+            return
+          } else {
+            // In case the instance of the source has changed, we need to close the channel
+            // and reconnect with a new channel
+            span?.addEvent(`force-new-channel`)
+            yield* Scope.close(scope, Exit.succeed('force-new-channel'))
+            return 'close'
+          }
+        }
+
+        switch (packet._tag) {
+          // Assumption: Each side has sent an initial request and another request as a response for an incoming request
+          case 'DirectChannelRequest': {
+            if (channelState._tag !== 'RequestSent') {
+              // We can safely ignore further incoming requests as we're already creating a channel
+              return
+            }
+
+            if (packet.reqId === channelState.reqPacketId) {
+              // Circuit-breaker: We've already sent a request so we don't need to send another one
+            } else {
+              const newRequestPacket = MeshSchema.DirectChannelRequest.make({
+                source: nodeName,
+                sourceId,
+                target,
+                channelName,
+                channelVersion,
+                hops: [],
+                remainingHops: packet.hops,
+                reqId: packet.id,
+              })
+              span?.addEvent(`Re-sending new request (${newRequestPacket.id}) for incoming request (${packet.id})`)
+
+              yield* sendPacket(newRequestPacket)
+            }
+
+            const isWinner = nodeName > target
+
+            if (isWinner) {
+              span?.addEvent(`winner side: creating direct channel and sending response`)
+              const mc = new MessageChannel()
+
+              // We're using a direct channel with acks here to make sure messages are not lost
+              // which might happen during re-edge scenarios.
+              // Also we need to eagerly start listening since we're using the channel "ourselves"
+              // for the initial ping-pong sequence.
+              const channel = yield* WebChannel.messagePortChannelWithAck({
+                port: mc.port1,
+                schema,
+                debugId: channelVersion,
+              }).pipe(Effect.andThen(WebChannel.toOpenChannel))
+
+              yield* respondToSender(
+                MeshSchema.DirectChannelResponseSuccess.make({
+                  reqId: packet.id,
+                  target,
+                  source: nodeName,
+                  channelName: packet.channelName,
+                  hops: [],
+                  remainingHops: packet.hops.slice(0, -1),
+                  port: mc.port2,
+                  channelVersion,
+                }),
+              )
+
+              channelStateRef.current = { _tag: 'winner:ResponseSent', channel, otherSourceId: packet.sourceId }
+
+              // span?.addEvent(`winner side: waiting for ping`)
+
+              // Now we wait for the other side to respond via the channel
+              yield* channel.listen.pipe(
+                Stream.flatten(),
+                Stream.filter(Schema.is(MeshSchema.DirectChannelPing)),
+                Stream.take(1),
+                Stream.runDrain,
+              )
+
+              // span?.addEvent(`winner side: sending pong`)
+
+              yield* channel.send(MeshSchema.DirectChannelPong.make({}))
+
+              span?.addEvent(`winner side: established`)
+              channelStateRef.current = { _tag: 'Established', otherSourceId: packet.sourceId }
+
+              yield* Deferred.succeed(deferred, channel)
+            } else {
+              span?.addEvent(`loser side: waiting for response`)
+              // Wait for `DirectChannelResponseSuccess` packet
+              channelStateRef.current = { _tag: 'loser:WaitingForResponse', otherSourceId: packet.sourceId }
+            }
+
+            break
+          }
+          case 'DirectChannelResponseSuccess': {
+            if (channelState._tag !== 'loser:WaitingForResponse') {
+              return shouldNeverHappen(
+                `Expected to find direct channel response from ${target}, but was in ${channelState._tag} state`,
+              )
+            }
+
+            // See direct-channel notes above
+            const channel = yield* WebChannel.messagePortChannelWithAck({
+              port: packet.port,
+              schema,
+              debugId: channelVersion,
+            }).pipe(Effect.andThen(WebChannel.toOpenChannel))
+
+            const waitForPongFiber = yield* channel.listen.pipe(
+              Stream.flatten(),
+              Stream.filter(Schema.is(MeshSchema.DirectChannelPong)),
+              Stream.take(1),
+              Stream.runDrain,
+              Effect.fork,
+            )
+
+            // span?.addEvent(`loser side: sending ping`)
+
+            // There seems to be some scenario where the initial ping message is lost.
+            // As a workaround until we find the root cause, we're retrying the ping a few times.
+            // TODO write a test that reproduces this issue and fix the root cause ()
+            // https://github.com/livestorejs/livestore/issues/262
+            yield* channel
+              .send(MeshSchema.DirectChannelPing.make({}))
+              .pipe(Effect.timeout(10), Effect.retry({ times: 2 }))
+
+            // span?.addEvent(`loser side: waiting for pong`)
+
+            yield* waitForPongFiber
+
+            span?.addEvent(`loser side: established`)
+            channelStateRef.current = { _tag: 'Established', otherSourceId: channelState.otherSourceId }
+
+            yield* Deferred.succeed(deferred, channel)
+
+            return
+          }
+          default: {
+            return casesHandled(packet)
+          }
+        }
+      }).pipe(
+        Effect.withSpan(`handleMessagePacket:${packet._tag}:${packet.source}→${packet.target}`, {
+          attributes: packetAsOtelAttributes(packet),
+        }),
+      )
+
+    yield* Effect.gen(function* () {
+      while (true) {
+        const packet = yield* Queue.take(incomingPacketsQueue)
+        const res = yield* processMessagePacket(packet)
+        // We want to give requests another chance to be processed
+        if (res === 'close') {
+          return
+        }
+      }
+    }).pipe(Effect.interruptible, Effect.tapCauseLogPretty, Effect.forkScoped)
+
+    const channelState = channelStateRef.current
+
+    if (channelState._tag !== 'Initial') {
+      return shouldNeverHappen(`Expected channel to be in Initial state, but was in ${channelState._tag} state`)
+    }
+
+    const edgeRequest = Effect.gen(function* () {
+      const packet = MeshSchema.DirectChannelRequest.make({
+        source: nodeName,
+        sourceId,
+        target,
+        channelName,
+        channelVersion,
+        hops: [],
+        reqId: undefined,
+      })
+
+      channelStateRef.current = { _tag: 'RequestSent', reqPacketId: packet.id }
+
+      // yield* Effect.log(`${nodeName}→${channelName}→${target}:edgeRequest [${channelVersion}]`)
+
+      const noTransferableResponse = checkTransferableEdges(packet)
+      if (noTransferableResponse !== undefined) {
+        yield* Effect.spanEvent(`No transferable edges found for ${packet.source}→${packet.target}`)
+        return yield* Effect.fail(noTransferableResponse)
+      }
+
+      yield* sendPacket(packet)
+      span?.addEvent(`initial edge request sent (${packet.id})`)
+    })
+
+    yield* edgeRequest
+
+    const channel = yield* deferred
+
+    return channel
+  }).pipe(Effect.withSpanScoped(`makeDirectChannel:${channelVersion}`))

package/src/channel/direct-channel.ts

@@ -0,0 +1,234 @@
+import {
+  Cause,
+  Deferred,
+  Effect,
+  Either,
+  Exit,
+  Option,
+  Queue,
+  Schema,
+  Scope,
+  Stream,
+  TQueue,
+  WebChannel,
+} from '@livestore/utils/effect'
+import { nanoid } from '@livestore/utils/nanoid'
+
+import * as WebmeshSchema from '../mesh-schema.js'
+import type { MakeDirectChannelArgs } from './direct-channel-internal.js'
+import { makeDirectChannelInternal } from './direct-channel-internal.js'
+
+/**
+ * Behaviour:
+ * - Waits until there is an initial edge
+ * - Automatically reconnects on disconnect
+ *
+ * Implementation notes:
+ * - We've split up the functionality into a wrapper channel and an internal channel.
+ * - The wrapper channel is responsible for:
+ *   - Forwarding send/listen messages to the internal channel (via a queue)
+ *   - Establishing the initial channel and reconnecting on disconnect
+ *     - Listening for new edges as a hint to reconnect if not already connected
+ *     - The wrapper channel maintains a edge counter which is used as the channel version
+ *
+ * If needed we can also implement further functionality (like heartbeat) in this wrapper channel.
+ */
+export const makeDirectChannel = ({
+  schema,
+  newEdgeAvailablePubSub,
+  channelName,
+  checkTransferableEdges,
+  nodeName,
+  incomingPacketsQueue,
+  target,
+  sendPacket,
+}: MakeDirectChannelArgs) =>
+  Effect.scopeWithCloseable((scope) =>
+    Effect.gen(function* () {
+      /** Only used to identify whether a source is the same instance to know when to reconnect */
+      const sourceId = nanoid()
+
+      const listenQueue = yield* Queue.unbounded<any>()
+      const sendQueue = yield* TQueue.unbounded<[msg: any, deferred: Deferred.Deferred<void>]>()
+
+      const initialEdgeDeferred = yield* Deferred.make<void>()
+
+      const debugInfo = {
+        pendingSends: 0,
+        totalSends: 0,
+        connectCounter: 0,
+        isConnected: false,
+        innerChannelRef: { current: undefined as WebChannel.WebChannel<any, any> | undefined },
+      }
+
+      // #region reconnect-loop
+      yield* Effect.gen(function* () {
+        const resultDeferred = yield* Deferred.make<{
+          channel: WebChannel.WebChannel<any, any>
+          channelVersion: number
+          makeDirectChannelScope: Scope.CloseableScope
+        }>()
+
+        while (true) {
+          debugInfo.connectCounter++
+          const channelVersion = debugInfo.connectCounter
+
+          yield* Effect.spanEvent(`Connecting#${channelVersion}`)
+
+          const makeDirectChannelScope = yield* Scope.make()
+          // Attach the new scope to the parent scope
+          yield* Effect.addFinalizer((ex) => Scope.close(makeDirectChannelScope, ex))
+
+          /**
+           * Expected concurrency behaviour:
+           * - We're concurrently running the edge setup and the waitForNewEdgeFiber
+           * - Happy path:
+           *   - The edge setup succeeds and we can interrupt the waitForNewEdgeFiber
+           * - Tricky paths:
+           *   - While a edge is still being setup, we want to re-try when there is a new edge
+           *   - If the edge setup returns a `DirectChannelResponseNoTransferables` error,
+           *     we want to wait for a new edge and then re-try
+           * - Further notes:
+           *   - If the parent scope closes, we want to also interrupt both the edge setup and the waitForNewEdgeFiber
+           *   - We're creating a separate scope for each edge attempt, which
+           *     - we'll use to fork the message channel in which allows us to interrupt it later
+           *   - We need to make sure that "interruption" isn't "bubbling out"
+           */
+          const waitForNewEdgeFiber = yield* Stream.fromPubSub(newEdgeAvailablePubSub).pipe(
+            Stream.tap((edgeName) => Effect.spanEvent(`new-conn:${edgeName}`)),
+            Stream.take(1),
+            Stream.runDrain,
+            Effect.as('new-edge' as const),
+            Effect.fork,
+          )
+
+          const makeChannel = makeDirectChannelInternal({
+            nodeName,
+            sourceId,
+            incomingPacketsQueue,
+            target,
+            checkTransferableEdges,
+            channelName,
+            schema,
+            channelVersion,
+            newEdgeAvailablePubSub,
+            sendPacket,
+            scope: makeDirectChannelScope,
+          }).pipe(
+            Scope.extend(makeDirectChannelScope),
+            Effect.forkIn(makeDirectChannelScope),
+            // Given we only call `Effect.exit` later when joining the fiber,
+            // we don't want Effect to produce a "unhandled error" log message
+            Effect.withUnhandledErrorLogLevel(Option.none()),
+          )
+
+          const raceResult = yield* Effect.raceFirst(makeChannel, waitForNewEdgeFiber.pipe(Effect.disconnect))
+
+          if (raceResult === 'new-edge') {
+            yield* Scope.close(makeDirectChannelScope, Exit.fail('new-edge'))
+            // We'll try again
+          } else {
+            const channelExit = yield* raceResult.pipe(Effect.exit)
+            if (channelExit._tag === 'Failure') {
+              yield* Scope.close(makeDirectChannelScope, channelExit)
+
+              if (
+                Cause.isFailType(channelExit.cause) &&
+                Schema.is(WebmeshSchema.DirectChannelResponseNoTransferables)(channelExit.cause.error)
+              ) {
+                // Only retry when there is a new edge available
+                yield* waitForNewEdgeFiber.pipe(Effect.exit)
+              }
+            } else {
+              const channel = channelExit.value
+
+              yield* Deferred.succeed(resultDeferred, { channel, makeDirectChannelScope, channelVersion })
+              break
+            }
+          }
+        }
+
+        // Now we wait until the first channel is established
+        const { channel, makeDirectChannelScope, channelVersion } = yield* resultDeferred
+
+        yield* Effect.spanEvent(`Connected#${channelVersion}`)
+        debugInfo.isConnected = true
+        debugInfo.innerChannelRef.current = channel
+
+        yield* Deferred.succeed(initialEdgeDeferred, void 0)
+
+        // We'll now forward all incoming messages to the listen queue
+        yield* channel.listen.pipe(
+          Stream.flatten(),
+          // Stream.tap((msg) => Effect.log(`${target}→${channelName}→${nodeName}:message:${msg.message}`)),
+          Stream.tapChunk((chunk) => Queue.offerAll(listenQueue, chunk)),
+          Stream.runDrain,
+          Effect.tapCauseLogPretty,
+          Effect.forkIn(makeDirectChannelScope),
+        )
+
+        yield* Effect.gen(function* () {
+          while (true) {
+            const [msg, deferred] = yield* TQueue.peek(sendQueue)
+            // NOTE we don't need an explicit retry flow here since in case of the channel being closed,
+            // the send will never succeed. Meanwhile the send-loop fiber will be interrupted and
+            // given we only peeked at the queue, the message to send is still there.
+            yield* channel.send(msg)
+            yield* Deferred.succeed(deferred, void 0)
+            yield* TQueue.take(sendQueue) // Remove the message from the queue
+          }
+        }).pipe(Effect.forkIn(makeDirectChannelScope))
+
+        // Wait until the channel is closed and then try to reconnect
+        yield* channel.closedDeferred
+
+        yield* Scope.close(makeDirectChannelScope, Exit.succeed('channel-closed'))
+
+        yield* Effect.spanEvent(`Disconnected#${channelVersion}`)
+        debugInfo.isConnected = false
+        debugInfo.innerChannelRef.current = undefined
+      }).pipe(
+        Effect.scoped, // Additionally scoping here to clean up finalizers after each loop run
+        Effect.forever,
+        Effect.tapCauseLogPretty,
+        Effect.forkScoped,
+      )
+      // #endregion reconnect-loop
+
+      const parentSpan = yield* Effect.currentSpan.pipe(Effect.orDie)
+
+      const send = (message: any) =>
+        Effect.gen(function* () {
+          const sentDeferred = yield* Deferred.make<void>()
+
+          debugInfo.pendingSends++
+          debugInfo.totalSends++
+
+          yield* TQueue.offer(sendQueue, [message, sentDeferred])
+
+          yield* sentDeferred
+
+          debugInfo.pendingSends--
+        }).pipe(Effect.scoped, Effect.withParentSpan(parentSpan))
+
+      const listen = Stream.fromQueue(listenQueue, { maxChunkSize: 1 }).pipe(Stream.map(Either.right))
+
+      const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
+
+      const webChannel = {
+        [WebChannel.WebChannelSymbol]: WebChannel.WebChannelSymbol,
+        send,
+        listen,
+        closedDeferred,
+        supportsTransferables: true,
+        schema,
+        debugInfo,
+        shutdown: Scope.close(scope, Exit.succeed('shutdown')),
+      } satisfies WebChannel.WebChannel<any, any>
+
+      return {
+        webChannel: webChannel as WebChannel.WebChannel<any, any>,
+        initialEdgeDeferred,
+      }
+    }),
+  )