dfx 1.0.3 → 1.0.5

package/dist/src/DiscordGateway/Shard/identify.ts

@@ -0,0 +1,50 @@
+import * as Option from "effect/Option"
+import * as Effect from "effect/Effect"
+import * as SendEvents from "./sendEvents.ts"
+import type * as Discord from "../../types.ts"
+import type { ShardState } from "./StateStore.ts"
+
+declare const process: { platform?: string | undefined } | undefined
+
+export interface Options {
+  readonly token: string
+  readonly intents: number
+  readonly shard: [number, number]
+  readonly presence?: Discord.GatewayPresenceUpdateData
+}
+
+const identify = ({ intents, presence, shard, token }: Options) =>
+  SendEvents.identify({
+    token,
+    intents,
+    properties: {
+      os:
+        typeof process !== "undefined"
+          ? (process.platform ?? "unknown")
+          : "unknown",
+      browser: "dfx",
+      device: "dfx",
+    },
+    shard,
+    presence,
+  })
+
+const resume = (token: string, session_id: string, seq: number | null) =>
+  SendEvents.resume({
+    token,
+    session_id,
+    seq: seq!,
+  })
+
+export const identifyOrResume = (
+  opts: Options,
+  state: Effect.Effect<Option.Option<ShardState>>,
+): Effect.Effect<Discord.GatewayIdentify | Discord.GatewayResume> =>
+  Effect.map(
+    state,
+    Option.match({
+      onNone: () => identify(opts),
+      onSome: shardState =>
+        resume(opts.token, shardState.sessionId, shardState.sequence),
+    }),
+  )

package/dist/src/DiscordGateway/Shard/sendEvents.ts

@@ -0,0 +1,43 @@
+import * as Discord from "../../types.ts"
+
+export const heartbeat = (
+  d: Discord.GatewayHeartbeatData,
+): Discord.GatewayHeartbeat => ({
+  op: Discord.GatewayOpcodes.Heartbeat,
+  d,
+})
+
+export const identify = (
+  d: Discord.GatewayIdentifyData,
+): Discord.GatewayIdentify => ({
+  op: Discord.GatewayOpcodes.Identify,
+  d,
+})
+
+export const resume = (
+  d: Discord.GatewayResumeData,
+): Discord.GatewayResume => ({
+  op: Discord.GatewayOpcodes.Resume,
+  d,
+})
+
+export const requestGuildMembers = (
+  d: Discord.GatewayRequestGuildMembersData,
+): Discord.GatewayRequestGuildMembers => ({
+  op: Discord.GatewayOpcodes.RequestGuildMembers,
+  d,
+})
+
+export const voiceStateUpdate = (
+  d: Discord.GatewayVoiceStateUpdateData,
+): Discord.GatewayVoiceStateUpdate => ({
+  op: Discord.GatewayOpcodes.VoiceStateUpdate,
+  d,
+})
+
+export const presenceUpdate = (
+  d: Discord.GatewayPresenceUpdateData,
+): Discord.GatewayUpdatePresence => ({
+  op: Discord.GatewayOpcodes.PresenceUpdate,
+  d,
+})

package/dist/src/DiscordGateway/Shard/utils.ts

@@ -0,0 +1,13 @@
+import type * as Discord from "../../types.ts"
+import * as Stream from "effect/Stream"
+
+export const opCode =
+  <R, E>(source: Stream.Stream<Discord.GatewayReceivePayload, E, R>) =>
+  <const Code extends Discord.GatewayOpcodes>(
+    code: Code,
+  ): Stream.Stream<
+    Extract<Discord.GatewayReceivePayload, { readonly op: Code }>,
+    E,
+    R
+  > =>
+    Stream.filter(source, p => p.op === code) as any

package/dist/src/DiscordGateway/Shard.ts

@@ -0,0 +1,205 @@
+import { DiscordConfig } from "../DiscordConfig.ts"
+import type { MessageSend } from "./DiscordWS.ts"
+import { DiscordWS, DiscordWSLive, Reconnect } from "./DiscordWS.ts"
+import { Messaging, MesssagingLive } from "./Messaging.ts"
+import type { ShardState } from "./Shard/StateStore.ts"
+import { ShardStateStore } from "./Shard/StateStore.ts"
+import * as Heartbeats from "./Shard/heartbeats.ts"
+import * as Identify from "./Shard/identify.ts"
+import { RateLimiter, RateLimiterLive } from "../RateLimit.ts"
+import * as Discord from "../types.ts"
+import * as Duration from "effect/Duration"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as Option from "effect/Option"
+import * as Redacted from "effect/Redacted"
+import type * as Types from "effect/Types"
+import * as FiberHandle from "effect/FiberHandle"
+import { constant, constTrue, constVoid } from "effect/Function"
+import * as Queue from "effect/Queue"
+import * as ServiceMap from "effect/ServiceMap"
+import * as PubSub from "effect/PubSub"
+import * as LogLevel from "effect/LogLevel"
+import { MinimumLogLevel } from "effect/References"
+
+const enum Phase {
+  Connecting,
+  Handshake,
+  Connected,
+}
+
+export const make = Effect.gen(function* () {
+  const { gateway, token } = yield* DiscordConfig
+  const limiter = yield* RateLimiter
+  const dws = yield* DiscordWS
+  const { hub, sendMailbox } = yield* Messaging
+  const shardState = yield* ShardStateStore
+
+  const connect = Effect.fnUntraced(
+    function* (shard: [id: number, count: number]) {
+      const traceEnabled = LogLevel.isLessThanOrEqualTo(
+        yield* MinimumLogLevel,
+        "Trace",
+      )
+      const reconnectHandle = yield* FiberHandle.make()
+      let phase = Phase.Connecting
+      const stateStore = shardState.forShard(shard)
+      const resumeState: Types.Mutable<ShardState> = Option.getOrElse(
+        yield* stateStore.get,
+        () => ({
+          resumeUrl: "",
+          sessionId: "",
+          sequence: 0,
+        }),
+      )
+      const setPhase = (p: Phase): Effect.Effect<void> =>
+        Effect.suspend(() => {
+          phase = p
+          if (!traceEnabled) return Effect.void
+          return Effect.annotateLogs(
+            Effect.logTrace("phase transition"),
+            "phase",
+            p,
+          )
+        })
+
+      const heartbeatSend = (p: MessageSend) =>
+        Effect.suspend(() => {
+          if (phase === Phase.Connecting) return Effect.void
+          return write(p)
+        })
+
+      const resume = Effect.andThen(
+        FiberHandle.clear(reconnectHandle),
+        setPhase(Phase.Connected),
+      )
+
+      const onConnecting = setPhase(Phase.Connecting)
+
+      const socket = yield* dws.connect({ onConnecting })
+      const write = (p: MessageSend) =>
+        Effect.andThen(
+          limiter.maybeWait("dfx.shard.send", Duration.minutes(1), 120),
+          socket.write(p),
+        )
+
+      const hellos = yield* Effect.acquireRelease(
+        Queue.make<Discord.GatewayHelloData>(),
+        Queue.shutdown,
+      )
+      const acks = yield* Effect.acquireRelease(
+        Queue.make<void>(),
+        Queue.shutdown,
+      )
+
+      // heartbeats
+      yield* Heartbeats.send(hellos, acks, stateStore.get, heartbeatSend).pipe(
+        Effect.forkScoped,
+        Effect.interruptible,
+      )
+
+      // identify
+      const identify = Identify.identifyOrResume(
+        {
+          token: Redacted.value(token),
+          shard,
+          intents: gateway.intents,
+          presence: gateway.presence,
+        },
+        stateStore.get,
+      )
+
+      // delayed reconnect
+      const delayedReconnect = Effect.delay(socket.write(Reconnect), 30_000)
+
+      function* onPayload(p: Discord.GatewayReceivePayload) {
+        if (typeof p.s === "number") {
+          resumeState.sequence = p.s
+        }
+        if (p.op === Discord.GatewayOpcodes.Dispatch && p.t === "READY") {
+          const payload = p.d
+          resumeState.sessionId = payload.session_id
+          resumeState.resumeUrl = payload.resume_gateway_url
+          yield* stateStore.set(resumeState)
+          yield* socket.setUrl(payload.resume_gateway_url)
+        } else if (
+          resumeState.resumeUrl !== "" &&
+          resumeState.sessionId !== ""
+        ) {
+          yield* stateStore.set(resumeState)
+        }
+
+        switch (p.op) {
+          case Discord.GatewayOpcodes.Hello: {
+            yield* write(yield* identify)
+            yield* setPhase(Phase.Handshake)
+            Queue.offerUnsafe(hellos, p.d)
+            yield* FiberHandle.run(reconnectHandle, delayedReconnect)
+            return
+          }
+          case Discord.GatewayOpcodes.HeartbeatAck: {
+            Queue.offerUnsafe(acks, void 0)
+            return
+          }
+          case Discord.GatewayOpcodes.InvalidSession: {
+            if (!p.d) {
+              resumeState.sessionId = ""
+              yield* stateStore.clear
+            }
+            yield* socket.write(Reconnect)
+            return
+          }
+          case Discord.GatewayOpcodes.Dispatch: {
+            if (p.t === "READY" || p.t === "RESUMED") {
+              yield* resume
+            }
+            PubSub.publishUnsafe(hub, p)
+            return
+          }
+          case Discord.GatewayOpcodes.Reconnect: {
+            yield* socket.write(Reconnect)
+            return
+          }
+        }
+      }
+
+      yield* Effect.whileLoop({
+        while: constTrue,
+        body: constant(Effect.flatMap(Queue.take(sendMailbox), write)),
+        step: constVoid,
+      }).pipe(Effect.forkScoped)
+
+      yield* Effect.gen(function* () {
+        while (true) {
+          yield* onPayload(yield* socket.take)
+        }
+      }).pipe(Effect.forkScoped)
+
+      return { id: shard, write } as const
+    },
+    (effect, shard) =>
+      Effect.annotateLogs(effect, {
+        package: "dfx",
+        module: "DiscordGateway/Shard",
+        shard,
+      }),
+  )
+
+  return { connect } as const
+})
+
+type ShardService = Effect.Success<typeof make>
+
+export class Shard extends ServiceMap.Service<Shard, ShardService>()(
+  "dfx/DiscordGateway/Shard",
+) {}
+export const ShardLive = Layer.effect(Shard, make).pipe(
+  Layer.provide(DiscordWSLive),
+  Layer.provide(MesssagingLive),
+  Layer.provide(RateLimiterLive),
+)
+
+// eslint-disable-next-line @typescript-eslint/no-empty-object-type
+export interface RunningShard extends Effect.Success<
+  ReturnType<ShardService["connect"]>
+> {}

package/dist/src/DiscordGateway/ShardStore.ts

@@ -0,0 +1,42 @@
+import * as Option from "effect/Option"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as ServiceMap from "effect/ServiceMap"
+
+export interface ClaimIdContext {
+  sharderCount: number
+  totalCount: number
+}
+
+export interface ShardStoreService {
+  claimId: (ctx: ClaimIdContext) => Effect.Effect<Option.Option<number>>
+  allClaimed: (totalCount: number) => Effect.Effect<boolean>
+  heartbeat?: (shardId: number) => Effect.Effect<void>
+}
+
+export class ShardStore extends ServiceMap.Service<
+  ShardStore,
+  ShardStoreService
+>()("dfx/DiscordGateway/ShardStore") {}
+
+// Very basic shard id store, that does no health checks
+const memoryStore = (): ShardStoreService => {
+  let currentId = 0
+
+  return {
+    claimId: ({ totalCount }) =>
+      Effect.sync(() => {
+        if (currentId >= totalCount) {
+          return Option.none()
+        }
+
+        const id = currentId
+        currentId++
+        return Option.some(id)
+      }),
+
+    allClaimed: totalCount => Effect.sync(() => currentId >= totalCount),
+  }
+}
+
+export const MemoryShardStoreLive = Layer.sync(ShardStore, memoryStore)

package/dist/src/DiscordGateway/Sharder.ts

@@ -0,0 +1,118 @@
+import { DiscordConfig } from "../DiscordConfig.ts"
+import type { RunningShard } from "./Shard.ts"
+import { Shard, ShardLive } from "./Shard.ts"
+import { ShardStore } from "./ShardStore.ts"
+import { DiscordREST, DiscordRESTLive } from "../DiscordREST.ts"
+import { RateLimiter, RateLimiterLive } from "../RateLimit.ts"
+import type * as Discord from "../types.ts"
+import * as Duration from "effect/Duration"
+import * as Effect from "effect/Effect"
+import { pipe } from "effect/Function"
+import * as Layer from "effect/Layer"
+import type * as Option from "effect/Option"
+import * as Ref from "effect/Ref"
+import * as Schedule from "effect/Schedule"
+import * as ServiceMap from "effect/ServiceMap"
+
+const claimRepeatPolicy = Schedule.identity<Option.Option<number>>().pipe(
+  Schedule.either(Schedule.spaced("3 minutes")),
+  Schedule.while(_ => Effect.succeed(_.input._tag === "None")),
+  Schedule.passthrough,
+) as Schedule.Schedule<Option.Some<number>, Option.Option<number>>
+
+const make = Effect.gen(function* () {
+  const store = yield* ShardStore
+  const rest = yield* DiscordREST
+  const { gateway: gatewayConfig } = yield* DiscordConfig
+  const limiter = yield* RateLimiter
+  const shard = yield* Shard
+  const currentShards = new Set<RunningShard>()
+
+  const gateway = yield* rest.getBotGateway().pipe(
+    Effect.catch(() =>
+      Effect.succeed<Discord.APIGatewayBotInfo>({
+        url: "wss://gateway.discord.gg/",
+        shards: 1,
+        session_start_limit: {
+          total: 0,
+          remaining: 0,
+          reset_after: 0,
+          max_concurrency: 1,
+        },
+      }),
+    ),
+  )
+
+  const totalCount = gatewayConfig.shardCount ?? gateway.shards
+  const currentCount = yield* Ref.make(0)
+  const claimId = (sharderCount: number): Effect.Effect<number> =>
+    pipe(
+      store.claimId({
+        totalCount,
+        sharderCount,
+      }),
+      Effect.repeat(claimRepeatPolicy),
+      Effect.map(_ => _.value),
+    )
+  const takeConfig = pipe(
+    Ref.getAndUpdate(currentCount, _ => _ + 1),
+    Effect.flatMap(claimId),
+    Effect.map(id => ({ id, totalCount }) as const),
+  )
+
+  const spawner = pipe(
+    takeConfig,
+    Effect.map(shardConfig => ({
+      ...shardConfig,
+      url: gateway.url,
+      concurrency: gateway.session_start_limit.max_concurrency,
+    })),
+    Effect.tap(({ concurrency, id }) =>
+      limiter.maybeWait(
+        `dfx.sharder.${id % concurrency}`,
+        Duration.millis(gatewayConfig.identifyRateLimit[0]),
+        gatewayConfig.identifyRateLimit[1],
+      ),
+    ),
+    Effect.flatMap(c => shard.connect([c.id, c.totalCount])),
+    Effect.tap(runningShard => {
+      currentShards.add(runningShard)
+      return Effect.void
+    }),
+    Effect.forever,
+  )
+
+  yield* Effect.replicateEffect(
+    spawner,
+    gateway.session_start_limit.max_concurrency,
+    { concurrency: "unbounded", discard: true },
+  ).pipe(
+    Effect.scoped,
+    Effect.catchCause(Effect.logError),
+    Effect.ensuring(Effect.sync(() => currentShards.clear())),
+    Effect.forever,
+    Effect.forkScoped,
+  )
+
+  return {
+    shards: Effect.sync(() => currentShards as ReadonlySet<RunningShard>),
+  } as const
+}).pipe(
+  Effect.annotateLogs({
+    package: "dfx",
+    module: "DiscordGateway/Sharder",
+  }),
+)
+
+export class Sharder extends ServiceMap.Service<
+  Sharder,
+  {
+    readonly shards: Effect.Effect<ReadonlySet<RunningShard>, never, never>
+  }
+>()("dfx/DiscordGateway/Sharder") {}
+
+export const SharderLive = Layer.effect(Sharder, make).pipe(
+  Layer.provide(DiscordRESTLive),
+  Layer.provide(RateLimiterLive),
+  Layer.provide(ShardLive),
+)

package/dist/src/DiscordGateway.ts

@@ -0,0 +1,79 @@
+import type { WebSocketConstructor } from "effect/unstable/socket/Socket"
+import type { DiscordConfig } from "./DiscordConfig.ts"
+import type { DiscordWSCodec } from "./DiscordGateway/DiscordWS.ts"
+import type { Messsaging } from "./DiscordGateway/Messaging.ts"
+import { Messaging, MesssagingLive } from "./DiscordGateway/Messaging.ts"
+import type { RunningShard } from "./DiscordGateway/Shard.ts"
+import type { ShardStateStore } from "./DiscordGateway/Shard/StateStore.ts"
+import { Sharder, SharderLive } from "./DiscordGateway/Sharder.ts"
+import type { ShardStore } from "./DiscordGateway/ShardStore.ts"
+import type { RateLimitStore } from "./RateLimit.ts"
+import type * as Discord from "./types.ts"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import type * as Stream from "effect/Stream"
+import type { HttpClient } from "effect/unstable/http/HttpClient"
+import * as ServiceMap from "effect/ServiceMap"
+
+export const TypeId = Symbol.for("dfx/DiscordGateway")
+export type TypeId = typeof TypeId
+
+export interface DiscordGateway {
+  readonly [TypeId]: TypeId
+
+  readonly dispatch: Stream.Stream<Discord.GatewayReceivePayload>
+  readonly fromDispatch: <K extends `${Discord.GatewayDispatchEvents}`>(
+    event: K,
+  ) => Stream.Stream<
+    Extract<Discord.DistributedGatewayDispatchPayload, { readonly t: K }>["d"]
+  >
+  readonly handleDispatch: <
+    K extends `${Discord.GatewayDispatchEvents}`,
+    R,
+    E,
+    A,
+  >(
+    event: K,
+    handle: (
+      event: Extract<
+        Discord.DistributedGatewayDispatchPayload,
+        { readonly t: K }
+      >["d"],
+    ) => Effect.Effect<A, E, R>,
+  ) => Effect.Effect<never, E, R>
+  readonly send: (payload: Discord.GatewaySendPayload) => Effect.Effect<boolean>
+  readonly shards: Effect.Effect<ReadonlySet<RunningShard>>
+}
+
+export const DiscordGateway =
+  ServiceMap.Service<DiscordGateway>("dfx/DiscordGateway")
+
+export const make: Effect.Effect<DiscordGateway, never, Messsaging | Sharder> =
+  Effect.gen(function* () {
+    const sharder = yield* Sharder
+    const messaging = yield* Messaging
+
+    return DiscordGateway.of({
+      [TypeId]: TypeId,
+      dispatch: messaging.dispatch,
+      fromDispatch: messaging.fromDispatch as any,
+      handleDispatch: messaging.handleDispatch as any,
+      send: messaging.send,
+      shards: sharder.shards,
+    })
+  })
+
+export const DiscordGatewayLive: Layer.Layer<
+  DiscordGateway,
+  never,
+  | ShardStore
+  | DiscordConfig
+  | RateLimitStore
+  | DiscordWSCodec
+  | ShardStateStore
+  | WebSocketConstructor
+  | HttpClient
+> = Layer.effect(DiscordGateway, make).pipe(
+  Layer.provide(MesssagingLive),
+  Layer.provide(SharderLive),
+)