dfx 0.77.3 → 0.79.0

package/src/DiscordGateway/Shard.ts

@@ -3,7 +3,7 @@ import { Tag } from "effect/Context"
 import * as Duration from "effect/Duration"
 import { pipe } from "effect/Function"
 import * as Option from "effect/Option"
-import * as ConfigSecret from "effect/ConfigSecret"
+import * as Secret from "effect/Secret"
 import * as Effect from "effect/Effect"
 import * as PubSub from "effect/PubSub"
 import * as Layer from "effect/Layer"
@@ -17,9 +17,9 @@ import * as Identify from "dfx/DiscordGateway/Shard/identify"
 import * as InvalidSession from "dfx/DiscordGateway/Shard/invalidSession"
 import * as Utils from "dfx/DiscordGateway/Shard/utils"
 import { Reconnect } from "dfx/DiscordGateway/WS"
-import { Log } from "dfx/Log"
 import { RateLimiterLive, RateLimiter } from "dfx/RateLimit"
 import * as Discord from "dfx/types"
+import { Messaging, MesssagingLive } from "dfx/DiscordGateway/Messaging"
 
 const enum Phase {
   Connecting,
@@ -31,19 +31,18 @@ export const make = Effect.gen(function* (_) {
   const { gateway, token } = yield* _(DiscordConfig)
   const limiter = yield* _(RateLimiter)
   const dws = yield* _(DiscordWS)
-  const log = yield* _(Log)
+  const { hub, sendQueue } = yield* _(Messaging)
 
-  const connect = (
-    shard: [id: number, count: number],
-    hub: PubSub.PubSub<Discord.GatewayPayload<Discord.ReceiveEvent>>,
-    sendQueue: Queue.Dequeue<Discord.GatewayPayload<Discord.SendEvent>>,
-  ) =>
+  const connect = (shard: [id: number, count: number]) =>
     Effect.gen(function* (_) {
       const outboundQueue = yield* _(Queue.unbounded<Message>())
       const pendingQueue = yield* _(Queue.unbounded<Message>())
       const phase = yield* _(Ref.make(Phase.Connecting))
       const setPhase = (p: Phase) =>
-        Effect.zipLeft(Ref.set(phase, p), log.debug("Shard", shard, "phase", p))
+        Effect.zipLeft(
+          Ref.set(phase, p),
+          Effect.annotateLogs(Effect.logTrace("phase transition"), "phase", p),
+        )
       const outbound = Effect.zipLeft(
         Queue.take(outboundQueue),
         limiter.maybeWait("dfx.shard.send", Duration.minutes(1), 120),
@@ -115,21 +114,29 @@ export const make = Effect.gen(function* (_) {
           },
         )
 
-      const hellos = yield* _(Queue.unbounded<Discord.GatewayPayload>())
-      const acks = yield* _(Queue.unbounded<Discord.GatewayPayload>())
+      const hellos = yield* _(
+        Effect.acquireRelease(
+          Queue.unbounded<Discord.GatewayPayload>(),
+          Queue.shutdown,
+        ),
+      )
+      const acks = yield* _(
+        Effect.acquireRelease(
+          Queue.unbounded<Discord.GatewayPayload>(),
+          Queue.shutdown,
+        ),
+      )
 
       // heartbeats
-      const heartbeats = Heartbeats.send(
-        hellos,
-        acks,
-        latestSequence,
-        heartbeatSend,
+      yield* _(
+        Heartbeats.send(hellos, acks, latestSequence, heartbeatSend),
+        Effect.forkScoped,
       )
 
       // identify
       const identify = Identify.identifyOrResume(
         {
-          token: ConfigSecret.value(token),
+          token: Secret.value(token),
           shard,
           intents: gateway.intents,
           presence: gateway.presence,
@@ -169,32 +176,43 @@ export const make = Effect.gen(function* (_) {
           }),
         )
 
-      const drainSendQueue = Effect.forever(
-        Effect.tap(Queue.take(sendQueue), send),
+      yield* _(
+        Queue.take(sendQueue),
+        Effect.tap(send),
+        Effect.forever,
+        Effect.forkScoped,
       )
 
-      const run = Effect.all(
-        [
-          Effect.forever(Effect.flatMap(socket.take, onPayload)),
-          heartbeats,
-          drainSendQueue,
-          socket.run,
-        ],
-        { discard: true, concurrency: "unbounded" },
+      yield* _(
+        socket.take,
+        Effect.flatMap(onPayload),
+        Effect.forever,
+        Effect.forkScoped,
       )
 
-      return { id: shard, send, run } as const
-    })
+      return { id: shard, send } as const
+    }).pipe(
+      Effect.annotateLogs({
+        package: "dfx",
+        module: "DiscordGateway/Shard",
+        shard,
+      }),
+    )
 
   return { connect } as const
 })
 
-export interface Shard extends Effect.Effect.Success<typeof make> {}
-export const Shard = Tag<Shard>()
-export const ShardLive = Layer.provide(
-  Layer.effect(Shard, make),
-  Layer.merge(DiscordWSLive, RateLimiterLive),
+type ShardService = Effect.Effect.Success<typeof make>
+
+export interface Shard {
+  readonly _: unique symbol
+}
+export const Shard = Tag<Shard, ShardService>("dfx/DiscordGateway/Shard")
+export const ShardLive = Layer.effect(Shard, make).pipe(
+  Layer.provide(DiscordWSLive),
+  Layer.provide(MesssagingLive),
+  Layer.provide(RateLimiterLive),
 )
 
 export interface RunningShard
-  extends Effect.Effect.Success<ReturnType<Shard["connect"]>> {}
+  extends Effect.Effect.Success<ReturnType<ShardService["connect"]>> {}

package/src/DiscordGateway/ShardStore.ts

@@ -8,17 +8,22 @@ export interface ClaimIdContext {
   totalCount: number
 }
 
-export interface ShardStore {
+export interface ShardStoreService {
   claimId: (
     ctx: ClaimIdContext,
   ) => Effect.Effect<never, never, Option.Option<number>>
   allClaimed: (totalCount: number) => Effect.Effect<never, never, boolean>
   heartbeat?: (shardId: number) => Effect.Effect<never, never, void>
 }
-export const ShardStore = Tag<ShardStore>()
+export interface ShardStore {
+  readonly _: unique symbol
+}
+export const ShardStore = Tag<ShardStore, ShardStoreService>(
+  "dfx/DiscordGateway/ShardStore",
+)
 
 // Very basic shard id store, that does no health checks
-const memoryStore = (): ShardStore => {
+const memoryStore = (): ShardStoreService => {
   let currentId = 0
 
   return {

package/src/DiscordGateway/Sharder.ts

@@ -1,24 +1,19 @@
-import * as Chunk from "effect/Chunk"
+import { DiscordConfig } from "dfx/DiscordConfig"
+import type { RunningShard } from "dfx/DiscordGateway/Shard"
+import { Shard, ShardLive } from "dfx/DiscordGateway/Shard"
+import { ShardStore } from "dfx/DiscordGateway/ShardStore"
+import { DiscordREST, DiscordRESTLive } from "dfx/DiscordREST"
+import { RateLimiter, RateLimiterLive } from "dfx/RateLimit"
+import type * as Discord from "dfx/types"
 import { Tag } from "effect/Context"
 import * as Duration from "effect/Duration"
+import * as Effect from "effect/Effect"
 import { pipe } from "effect/Function"
 import * as HashSet from "effect/HashSet"
-import type * as Option from "effect/Option"
-import * as Deferred from "effect/Deferred"
-import * as Effect from "effect/Effect"
-import type * as PubSub from "effect/PubSub"
 import * as Layer from "effect/Layer"
-import type * as Queue from "effect/Queue"
+import type * as Option from "effect/Option"
 import * as Ref from "effect/Ref"
 import * as Schedule from "effect/Schedule"
-import { DiscordConfig } from "dfx/DiscordConfig"
-import type { RunningShard } from "dfx/DiscordGateway/Shard"
-import { ShardLive, Shard } from "dfx/DiscordGateway/Shard"
-import { ShardStore } from "dfx/DiscordGateway/ShardStore"
-import type { WebSocketCloseError, WebSocketError } from "dfx/DiscordGateway/WS"
-import { DiscordREST } from "dfx/DiscordREST"
-import { RateLimiterLive, RateLimiter } from "dfx/RateLimit"
-import type * as Discord from "dfx/types"
 
 const claimRepeatPolicy = Schedule.spaced("3 minutes").pipe(
   Schedule.whileInput((_: Option.Option<number>) => _._tag === "None"),
@@ -33,29 +28,6 @@ const make = Effect.gen(function* (_) {
   const shard = yield* _(Shard)
   const currentShards = yield* _(Ref.make(HashSet.empty<RunningShard>()))
 
-  const takeConfig = (totalCount: number) =>
-    Effect.gen(function* (_) {
-      const currentCount = yield* _(Ref.make(0))
-
-      const claimId = (
-        sharderCount: number,
-      ): Effect.Effect<never, never, number> =>
-        pipe(
-          store.claimId({
-            totalCount,
-            sharderCount,
-          }),
-          Effect.repeat(claimRepeatPolicy),
-          Effect.map(_ => _.value),
-        )
-
-      return pipe(
-        Ref.getAndUpdate(currentCount, _ => _ + 1),
-        Effect.flatMap(claimId),
-        Effect.map(id => ({ id, totalCount }) as const),
-      )
-    })
-
   const gateway = yield* _(
     rest.getGatewayBot(),
     Effect.flatMap(r => r.json),
@@ -73,68 +45,71 @@ const make = Effect.gen(function* (_) {
     ),
   )
 
-  const run = (
-    hub: PubSub.PubSub<Discord.GatewayPayload<Discord.ReceiveEvent>>,
-    sendQueue: Queue.Dequeue<Discord.GatewayPayload<Discord.SendEvent>>,
-  ) =>
-    Effect.gen(function* (_) {
-      const deferred = yield* _(
-        Deferred.make<WebSocketError | WebSocketCloseError, never>(),
-      )
-      const take = yield* _(takeConfig(config.shardCount ?? gateway.shards))
-
-      const spawner = pipe(
-        take,
-        Effect.map(config => ({
-          ...config,
-          url: gateway.url,
-          concurrency: gateway.session_start_limit.max_concurrency,
-        })),
-        Effect.tap(({ concurrency, id }) =>
-          limiter.maybeWait(
-            `dfx.sharder.${id % concurrency}`,
-            Duration.millis(config.identifyRateLimit[0]),
-            config.identifyRateLimit[1],
-          ),
-        ),
-        Effect.flatMap(c =>
-          shard.connect([c.id, c.totalCount], hub, sendQueue),
-        ),
-        Effect.flatMap(shard =>
-          Effect.acquireUseRelease(
-            Ref.update(currentShards, HashSet.add(shard)),
-            () => shard.run,
-            () => Ref.update(currentShards, HashSet.remove(shard)),
-          ).pipe(
-            Effect.catchAllCause(_ => Deferred.failCause(deferred, _)),
-            Effect.fork,
-          ),
-        ),
-        Effect.forever,
-      )
+  const totalCount = config.shardCount ?? gateway.shards
+  const currentCount = yield* _(Ref.make(0))
+  const claimId = (sharderCount: number): Effect.Effect<never, never, number> =>
+    pipe(
+      store.claimId({
+        totalCount,
+        sharderCount,
+      }),
+      Effect.repeat(claimRepeatPolicy),
+      Effect.map(_ => _.value),
+    )
+  const takeConfig = pipe(
+    Ref.getAndUpdate(currentCount, _ => _ + 1),
+    Effect.flatMap(claimId),
+    Effect.map(id => ({ id, totalCount }) as const),
+  )
 
-      const spawners = Chunk.map(
-        Chunk.range(1, gateway.session_start_limit.max_concurrency),
-        () => spawner,
-      )
+  const spawner = pipe(
+    takeConfig,
+    Effect.map(config => ({
+      ...config,
+      url: gateway.url,
+      concurrency: gateway.session_start_limit.max_concurrency,
+    })),
+    Effect.tap(({ concurrency, id }) =>
+      limiter.maybeWait(
+        `dfx.sharder.${id % concurrency}`,
+        Duration.millis(config.identifyRateLimit[0]),
+        config.identifyRateLimit[1],
+      ),
+    ),
+    Effect.flatMap(c => shard.connect([c.id, c.totalCount])),
+    Effect.flatMap(shard => Ref.update(currentShards, HashSet.add(shard))),
+    Effect.forever,
+  )
 
-      return yield* _(
-        Effect.all(
-          [
-            Effect.all(spawners, { concurrency: "unbounded", discard: true }),
-            Deferred.await(deferred),
-          ],
-          { concurrency: "unbounded", discard: true },
-        ) as Effect.Effect<never, WebSocketError | WebSocketCloseError, never>,
-      )
-    })
+  yield* _(
+    Effect.replicateEffect(
+      spawner,
+      gateway.session_start_limit.max_concurrency,
+      { concurrency: "unbounded", discard: true },
+    ),
+    Effect.scoped,
+    Effect.catchAllCause(Effect.logError),
+    Effect.ensuring(Ref.set(currentShards, HashSet.empty())),
+    Effect.forever,
+    Effect.forkScoped,
+  )
 
-  return { shards: Ref.get(currentShards), run } as const
-})
+  return { shards: Ref.get(currentShards) } as const
+}).pipe(
+  Effect.annotateLogs({
+    package: "dfx",
+    module: "DiscordGateway/Sharder",
+  }),
+)
 
-export interface Sharder extends Effect.Effect.Success<typeof make> {}
-export const Sharder = Tag<Sharder>()
-export const SharedLive = Layer.provide(
-  Layer.effect(Sharder, make),
-  Layer.merge(RateLimiterLive, ShardLive),
+export interface Sharder {
+  readonly _: unique symbol
+}
+export const Sharder = Tag<Sharder, Effect.Effect.Success<typeof make>>(
+  "dfx/DiscordGateway/Sharder",
+)
+export const SharderLive = Layer.scoped(Sharder, make).pipe(
+  Layer.provide(DiscordRESTLive),
+  Layer.provide(RateLimiterLive),
+  Layer.provide(ShardLive),
 )

package/src/DiscordGateway/WS.ts

@@ -1,12 +1,13 @@
 import { Tag } from "effect/Context"
-import * as Duration from "effect/Duration"
-import { pipe } from "effect/Function"
+import type * as Duration from "effect/Duration"
+import { identity, pipe } from "effect/Function"
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
 import * as Queue from "effect/Queue"
 import * as Ref from "effect/Ref"
-import { Log } from "dfx/Log"
 import WebSocket from "isomorphic-ws"
+import type { Predicate } from "effect/Predicate"
+import * as Schedule from "effect/Schedule"
 
 export const Reconnect = Symbol()
 export type Reconnect = typeof Reconnect
@@ -49,16 +50,10 @@ const socket = (urlRef: Ref.Ref<string>) =>
 const offer = (
   ws: globalThis.WebSocket,
   queue: Queue.Enqueue<WebSocket.Data>,
-  log: Log,
 ) =>
   Effect.async<never, WebSocketError | WebSocketCloseError, never>(resume => {
     ws.addEventListener("message", message => {
-      Effect.runFork(
-        Effect.zipRight(
-          log.debug("WS", "receive", message.data),
-          Queue.offer(queue, message.data),
-        ),
-      )
+      Queue.unsafeOffer(queue, message.data)
     })
 
     ws.addEventListener("error", cause => {
@@ -70,7 +65,10 @@ const offer = (
     })
   })
 
-const waitForOpen = (ws: globalThis.WebSocket, timeout: Duration.Duration) =>
+const waitForOpen = (
+  ws: globalThis.WebSocket,
+  timeout: Duration.DurationInput,
+) =>
   Effect.timeoutFail(
     Effect.suspend(() => {
       if (ws.readyState === WebSocket.OPEN) {
@@ -92,11 +90,10 @@ const waitForOpen = (ws: globalThis.WebSocket, timeout: Duration.Duration) =>
 const send = (
   ws: globalThis.WebSocket,
   take: Effect.Effect<never, never, Message>,
-  log: Log,
 ) =>
   pipe(
     take,
-    Effect.tap(data => log.debug("WS", "send", data)),
+    Effect.tap(data => Effect.logTrace(data)),
     Effect.tap(data => {
       if (data === Reconnect) {
         return Effect.failSync(() => {
@@ -110,30 +107,50 @@ const send = (
       })
     }),
     Effect.forever,
+    Effect.annotateLogs("method", "send"),
   )
 
-const make = Effect.gen(function* (_) {
-  const log = yield* _(Log)
-
-  const connect = (
-    url: Ref.Ref<string>,
-    takeOutbound: Effect.Effect<never, never, Message>,
+const wsImpl = {
+  connect: ({
     onConnecting = Effect.unit,
-    openTimeout = Duration.seconds(3),
-  ) =>
+    openTimeout = "3 seconds",
+    reconnectWhen,
+    takeOutbound,
+    urlRef,
+  }: {
+    readonly urlRef: Ref.Ref<string>
+    readonly takeOutbound: Effect.Effect<never, never, Message>
+    readonly onConnecting?: Effect.Effect<never, never, void>
+    readonly openTimeout?: Duration.DurationInput
+    readonly reconnectWhen?: Predicate<WebSocketError | WebSocketCloseError>
+  }) =>
     Effect.gen(function* (_) {
-      const queue = yield* _(Queue.unbounded<WebSocket.Data>())
+      const scope = yield* _(Effect.scope)
+      const queue = yield* _(
+        Effect.acquireRelease(
+          Queue.unbounded<WebSocket.Data>(),
+          Queue.shutdown,
+        ),
+      )
+      const take = Effect.annotateLogs(
+        Effect.tap(Queue.take(queue), data => Effect.logTrace(data)),
+        {
+          package: "dfx",
+          module: "DiscordGateway/WS",
+          method: "take",
+        },
+      )
 
       const run = pipe(
         onConnecting,
-        Effect.zipRight(socket(url)),
+        Effect.zipRight(socket(urlRef)),
         Effect.flatMap(ws =>
           Effect.all(
             [
-              offer(ws, queue, log),
+              offer(ws, queue),
               Effect.zipRight(
                 waitForOpen(ws, openTimeout),
-                send(ws, takeOutbound, log),
+                send(ws, takeOutbound),
               ),
             ],
             { concurrency: "unbounded", discard: true },
@@ -141,14 +158,29 @@ const make = Effect.gen(function* (_) {
         ),
         Effect.scoped,
         Effect.retryWhile(isReconnect),
+        reconnectWhen ? Effect.retryWhile(reconnectWhen) : identity,
+        Effect.catchAllCause(Effect.logError),
+        Effect.repeat(
+          Schedule.exponential("500 millis").pipe(
+            Schedule.union(Schedule.spaced("30 seconds")),
+          ),
+        ),
+        Effect.forkIn(scope),
       )
 
-      return { run, take: Queue.take(queue) } as const
-    })
+      yield* _(run)
 
-  return { connect } as const
-})
+      return { take } as const
+    }).pipe(
+      Effect.annotateLogs({
+        package: "dfx",
+        module: "DiscordGateway/WS",
+      }),
+    ),
+} as const
 
-export interface WS extends Effect.Effect.Success<typeof make> {}
-export const WS = Tag<WS>()
-export const WSLive = Layer.effect(WS, make)
+export interface WS {
+  readonly _: unique symbol
+}
+export const WS = Tag<WS, typeof wsImpl>("dfx/DiscordGateway/WS")
+export const WSLive = Layer.succeed(WS, wsImpl)

package/src/DiscordGateway.ts

@@ -1,42 +1,19 @@
+import { Messaging, MesssagingLive } from "dfx/DiscordGateway/Messaging"
+import type { RunningShard } from "dfx/DiscordGateway/Shard"
+import { Sharder, SharderLive } from "dfx/DiscordGateway/Sharder"
+import type * as Discord from "dfx/types"
 import { Tag } from "effect/Context"
-import type * as HashSet from "effect/HashSet"
 import * as Effect from "effect/Effect"
-import * as PubSub from "effect/PubSub"
+import type * as HashSet from "effect/HashSet"
 import * as Layer from "effect/Layer"
-import * as Queue from "effect/Queue"
-import * as Stream from "effect/Stream"
-import type { RunningShard } from "dfx/DiscordGateway/Shard"
-import { SharedLive, Sharder } from "dfx/DiscordGateway/Sharder"
-import type * as Discord from "dfx/types"
-import * as EffectUtils from "dfx/utils/Effect"
-import * as Schedule from "effect/Schedule"
-
-const fromDispatchFactory =
-  <R, E>(
-    source: Stream.Stream<R, E, Discord.GatewayPayload<Discord.ReceiveEvent>>,
-  ) =>
-  <K extends keyof Discord.ReceiveEvents>(
-    event: K,
-  ): Stream.Stream<R, E, Discord.ReceiveEvents[K]> =>
-    Stream.map(
-      Stream.filter(source, p => p.t === event),
-      p => p.d! as any,
-    )
+import type * as Stream from "effect/Stream"
 
-const handleDispatchFactory =
-  (hub: PubSub.PubSub<Discord.GatewayPayload<Discord.ReceiveEvent>>) =>
-  <K extends keyof Discord.ReceiveEvents, R, E, A>(
-    event: K,
-    handle: (event: Discord.ReceiveEvents[K]) => Effect.Effect<R, E, A>,
-  ): Effect.Effect<R, E, never> =>
-    EffectUtils.subscribeForEachPar(hub, _ => {
-      if (_.t === event) {
-        return handle(_.d as any)
-      }
-      return Effect.unit as any
-    })
+export const TypeId = Symbol.for("dfx/DiscordGateway")
+export type TypeId = typeof TypeId
 
 export interface DiscordGateway {
+  readonly [TypeId]: TypeId
+
   readonly dispatch: Stream.Stream<
     never,
     never,
@@ -54,50 +31,24 @@ export interface DiscordGateway {
   ) => Effect.Effect<never, never, boolean>
   readonly shards: Effect.Effect<never, never, HashSet.HashSet<RunningShard>>
 }
-export const DiscordGateway = Tag<DiscordGateway>()
+
+export const DiscordGateway = Tag<DiscordGateway>(TypeId)
 
 export const make = Effect.gen(function* (_) {
   const sharder = yield* _(Sharder)
-  const hub = yield* _(
-    PubSub.unbounded<Discord.GatewayPayload<Discord.ReceiveEvent>>(),
-  )
-
-  const sendQueue = yield* _(
-    Queue.unbounded<Discord.GatewayPayload<Discord.SendEvent>>(),
-  )
-  const send = (payload: Discord.GatewayPayload<Discord.SendEvent>) =>
-    sendQueue.offer(payload)
-
-  const dispatch = Stream.fromPubSub(hub)
-  const fromDispatch = fromDispatchFactory(dispatch)
-  const handleDispatch = handleDispatchFactory(hub)
-
-  yield* _(
-    sharder.run(hub, sendQueue),
-    Effect.tapErrorCause(_ => Effect.logError("fatal error, restarting", _)),
-    Effect.retry(
-      Schedule.exponential("1 seconds").pipe(
-        Schedule.union(Schedule.spaced("30 seconds")),
-      ),
-    ),
-    Effect.forkScoped,
-  )
+  const messaging = yield* _(Messaging)
 
   return DiscordGateway.of({
-    dispatch,
-    fromDispatch,
-    handleDispatch,
-    send,
+    [TypeId]: TypeId,
+    dispatch: messaging.dispatch,
+    fromDispatch: messaging.fromDispatch,
+    handleDispatch: messaging.handleDispatch,
+    send: messaging.send,
     shards: sharder.shards,
   })
-}).pipe(
-  Effect.annotateLogs({
-    package: "dfx",
-    service: "DiscordGateway",
-  }),
-)
+})
 
-export const DiscordGatewayLive = Layer.provide(
-  Layer.scoped(DiscordGateway, make),
-  SharedLive,
+export const DiscordGatewayLive = Layer.effect(DiscordGateway, make).pipe(
+  Layer.provide(MesssagingLive),
+  Layer.provide(SharderLive),
 )