dfx 0.42.2 → 0.42.3

package/src/DiscordGateway/Sharder.ts

@@ -0,0 +1,102 @@
+import { millis } from "@effect/data/Duration"
+import { DiscordConfig } from "dfx/DiscordConfig"
+import { DiscordREST } from "dfx/DiscordREST"
+import { LiveRateLimiter, RateLimiter } from "../RateLimit.js"
+import { LiveShard, Shard } from "./Shard.js"
+import { ShardStore } from "./ShardStore.js"
+import { WebSocketCloseError, WebSocketError } from "./WS.js"
+
+const make = Do($ => {
+  const store = $(ShardStore)
+  const rest = $(DiscordREST)
+  const { gateway: config } = $(DiscordConfig)
+  const limiter = $(RateLimiter)
+  const shard = $(Shard)
+
+  const takeConfig = (totalCount: number) =>
+    Do($ => {
+      const currentCount = $(Ref.make(0))
+
+      const claimId = (sharderCount: number): Effect<never, never, number> =>
+        store
+          .claimId({
+            totalCount,
+            sharderCount,
+          })
+          .flatMap(a =>
+            a.match(
+              () => claimId(sharderCount).delay(Duration.minutes(3)),
+              id => Effect.succeed(id),
+            ),
+          )
+
+      return currentCount
+        .getAndUpdate(_ => _ + 1)
+        .flatMap(claimId)
+        .map(id => ({ id, totalCount } as const))
+    })
+
+  const gateway = $(
+    rest
+      .getGatewayBot()
+      .flatMap(r => r.json)
+      .catchAll(() =>
+        Effect.succeed<Discord.GetGatewayBotResponse>({
+          url: "wss://gateway.discord.gg/",
+          shards: 1,
+          session_start_limit: {
+            total: 0,
+            remaining: 0,
+            reset_after: 0,
+            max_concurrency: 1,
+          },
+        }),
+      ),
+  )
+
+  const run = (hub: Hub<Discord.GatewayPayload<Discord.ReceiveEvent>>) =>
+    Do($ => {
+      const deferred = $(
+        Deferred.make<WebSocketError | WebSocketCloseError, never>(),
+      )
+      const take = $(takeConfig(config.shardCount ?? gateway.shards))
+
+      const spawner = take
+        .map(config => ({
+          ...config,
+          url: gateway.url,
+          concurrency: gateway.session_start_limit.max_concurrency,
+        }))
+        .tap(({ id, concurrency }) =>
+          limiter.maybeWait(
+            `dfx.sharder.${id % concurrency}`,
+            millis(config.identifyRateLimit[0]),
+            config.identifyRateLimit[1],
+          ),
+        )
+        .flatMap(c => shard.connect([c.id, c.totalCount], hub))
+        .flatMap(
+          shard => shard.run.catchAllCause(_ => deferred.failCause(_)).fork,
+        ).forever
+
+      const spawners = Chunk.range(
+        1,
+        gateway.session_start_limit.max_concurrency,
+      ).map(() => spawner)
+
+      return $(
+        Effect.allParDiscard(spawners).zipParLeft(deferred.await) as Effect<
+          never,
+          WebSocketError | WebSocketCloseError,
+          never
+        >,
+      )
+    })
+
+  return { run } as const
+})
+
+export interface Sharder extends Effect.Success<typeof make> {}
+export const Sharder = Tag<Sharder>()
+export const LiveSharder =
+  (LiveRateLimiter + LiveShard) >> make.toLayer(Sharder)

package/src/DiscordGateway/WS.ts

@@ -0,0 +1,122 @@
+import { Log } from "dfx/Log"
+import WebSocket from "isomorphic-ws"
+
+export const Reconnect = Symbol()
+export type Reconnect = typeof Reconnect
+export type Message = string | Buffer | ArrayBuffer | Reconnect
+
+export class WebSocketError {
+  readonly _tag = "WebSocketError"
+  constructor(
+    readonly reason: "open-timeout" | "error",
+    readonly error?: unknown,
+  ) {}
+}
+
+export class WebSocketCloseError {
+  readonly _tag = "WebSocketCloseError"
+  constructor(readonly code: number, readonly reason: string) {}
+}
+
+const isReconnect = (
+  e: WebSocketError | WebSocketCloseError,
+): e is WebSocketCloseError =>
+  e._tag === "WebSocketCloseError" && e.code === 1012
+
+const socket = (urlRef: Ref<string>) =>
+  urlRef.get
+    .map(_ => new WebSocket(_) as any as globalThis.WebSocket)
+    .acquireRelease(ws =>
+      Effect.sync(() => {
+        ;(ws as any).removeAllListeners?.()
+        ws.close()
+      }),
+    )
+
+const offer = (
+  ws: globalThis.WebSocket,
+  queue: Enqueue<WebSocket.Data>,
+  log: Log,
+) =>
+  Effect.async<never, WebSocketError | WebSocketCloseError, never>(resume => {
+    ws.addEventListener("message", message => {
+      queue.offer(message.data).zipLeft(log.debug("WS", "offer", message.data))
+        .runFork
+    })
+
+    ws.addEventListener("error", cause => {
+      resume(Effect.fail(new WebSocketError("error", cause)))
+    })
+
+    ws.addEventListener("close", e => {
+      resume(Effect.fail(new WebSocketCloseError(e.code, e.reason)))
+    })
+  })
+
+const waitForOpen = (ws: globalThis.WebSocket, timeout: Duration) =>
+  Effect.suspend(() => {
+    if (ws.readyState === WebSocket.OPEN) {
+      return Effect.unit()
+    }
+
+    return Effect.async<never, never, void>(resume => {
+      ws.addEventListener("open", () => resume(Effect.unit()), {
+        once: true,
+      })
+    })
+  }).timeoutFail(() => new WebSocketError("open-timeout"), timeout)
+
+const send = (
+  ws: globalThis.WebSocket,
+  take: Effect<never, never, Message>,
+  log: Log,
+  openTimeout: Duration,
+) => {
+  const loop = take
+    .tap(data => log.debug("WS", "send", data))
+    .tap((data): Effect<never, WebSocketCloseError, void> => {
+      if (data === Reconnect) {
+        return Effect.failSync(() => {
+          ws.close(1012, "reconnecting")
+          return new WebSocketCloseError(1012, "reconnecting")
+        })
+      }
+
+      return Effect.sync(() => {
+        ws.send(data)
+      })
+    }).forever
+
+  return waitForOpen(ws, openTimeout).zipRight(loop)
+}
+
+const make = Do($ => {
+  const log = $(Log)
+
+  const connect = (
+    url: Ref<string>,
+    takeOutbound: Effect<never, never, Message>,
+    onReconnect = Effect.unit(),
+    openTimeout = Duration.seconds(3),
+  ) =>
+    Do($ => {
+      const queue = $(Queue.unbounded<WebSocket.Data>())
+
+      const run = socket(url)
+        .flatMap(ws =>
+          offer(ws, queue, log).zipParLeft(
+            send(ws, takeOutbound, log, openTimeout),
+          ),
+        )
+        .tapError(_ => (isReconnect(_) ? onReconnect : Effect.unit()))
+        .retryWhile(isReconnect).scoped
+
+      return { run, take: queue.take() } as const
+    })
+
+  return { connect } as const
+})
+
+export interface WS extends Effect.Success<typeof make> {}
+export const WS = Tag<WS>()
+export const LiveWS = make.toLayer(WS)

package/src/DiscordGateway.ts

@@ -0,0 +1,43 @@
+import { LiveSharder, Sharder } from "./DiscordGateway/Sharder.js"
+
+const fromDispatchFactory =
+  <R, E>(source: Stream<R, E, Discord.GatewayPayload<Discord.ReceiveEvent>>) =>
+  <K extends keyof Discord.ReceiveEvents>(
+    event: K,
+  ): Stream<R, E, Discord.ReceiveEvents[K]> =>
+    source.filter(p => p.t === event).map(p => p.d! as any)
+
+const handleDispatchFactory =
+  (hub: Hub<Discord.GatewayPayload<Discord.ReceiveEvent>>) =>
+  <K extends keyof Discord.ReceiveEvents, R, E, A>(
+    event: K,
+    handle: (event: Discord.ReceiveEvents[K]) => Effect<R, E, A>,
+  ): Effect<R, E, never> =>
+    hub.subscribeForEachPar(_ => {
+      if (_.t === event) {
+        return handle(_.d as any)
+      }
+      return Effect.unit()
+    })
+
+export const make = Do($ => {
+  const sharder = $(Sharder)
+  const hub = $(Hub.unbounded<Discord.GatewayPayload<Discord.ReceiveEvent>>())
+
+  const dispatch = Stream.fromHub(hub)
+  const fromDispatch = fromDispatchFactory(dispatch)
+  const handleDispatch = handleDispatchFactory(hub)
+
+  const run = sharder.run(hub)
+
+  return {
+    run,
+    dispatch,
+    fromDispatch,
+    handleDispatch,
+  }
+})
+
+export interface DiscordGateway extends Effect.Success<typeof make> {}
+export const DiscordGateway = Tag<DiscordGateway>()
+export const LiveDiscordGateway = LiveSharder >> make.toLayer(DiscordGateway)

package/src/DiscordREST/types.ts

@@ -0,0 +1,13 @@
+import * as Http from "@effect-http/client"
+import { DiscordRESTError } from "dfx/DiscordREST"
+import { Effect } from "dfx/_common"
+
+export interface ResponseWithData<A> extends Http.response.Response {
+  readonly json: Effect<never, Http.ResponseDecodeError, A>
+}
+
+export type RestResponse<T> = Effect<
+  never,
+  DiscordRESTError,
+  ResponseWithData<T>
+>

package/src/DiscordREST/utils.ts

@@ -0,0 +1,33 @@
+const majorResources = ["channels", "guilds", "webhooks"] as const
+
+export const routeFromConfig = (path: string, method: string) => {
+  // Only keep major ID's
+  const routeURL = path
+    .split("?")[0]
+    .replace(/\/([A-Za-z]+)\/(\d{16,21}|@me)/g, (match, resource) =>
+      majorResources.includes(resource) ? match : `/${resource}`,
+    )
+    // Strip reactions
+    .replace(/\/reactions\/(.*)/, "/reactions")
+
+  return `${method}-${routeURL}`
+}
+
+export const numberHeader = (headers: Headers) => (key: string) =>
+  Maybe.fromNullable(headers.get(key))
+    .map(parseFloat)
+    .filter(n => !isNaN(n))
+
+export const retryAfter = (headers: Headers) =>
+  numberHeader(headers)("x-ratelimit-reset-after")
+    .orElse(() => numberHeader(headers)("retry-after"))
+    .map(Duration.seconds)
+
+export const rateLimitFromHeaders = (headers: Headers) =>
+  Maybe.struct({
+    bucket: Maybe.fromNullable(headers.get("x-ratelimit-bucket")),
+    retryAfter: retryAfter(headers),
+    limit: numberHeader(headers)("x-ratelimit-limit"),
+    remaining: numberHeader(headers)("x-ratelimit-remaining"),
+  })
+export type RateLimitDetails = ReturnType<typeof rateLimitFromHeaders>

package/src/DiscordREST.ts

@@ -0,0 +1,203 @@
+import * as Http from "@effect-http/client"
+import { millis } from "@effect/data/Duration"
+import { DiscordConfig } from "./DiscordConfig.js"
+import { ResponseWithData, RestResponse } from "./DiscordREST/types.js"
+import {
+  rateLimitFromHeaders,
+  retryAfter,
+  routeFromConfig,
+} from "./DiscordREST/utils.js"
+import { Log } from "./Log.js"
+import {
+  BucketDetails,
+  LiveRateLimiter,
+  RateLimitStore,
+  RateLimiter,
+} from "./RateLimit.js"
+import Pkg from "./package.json" assert { type: "json" }
+
+export class DiscordRESTError {
+  readonly _tag = "DiscordRESTError"
+  constructor(readonly error: Http.HttpClientError) {}
+}
+
+const make = Do($ => {
+  const { token, rest } = $(DiscordConfig)
+
+  const http = $(Http.HttpRequestExecutor)
+  const log = $(Log)
+  const store = $(RateLimitStore)
+  const { maybeWait } = $(RateLimiter)
+
+  const globalRateLimit = maybeWait(
+    "dfx.rest.global",
+    rest.globalRateLimit.window,
+    rest.globalRateLimit.limit,
+  )
+
+  // Invalid route handling (40x)
+  const badRoutesRef = $(Ref.make(HashSet.empty<string>()))
+  const addBadRoute = (route: string) =>
+    Effect.allParDiscard([
+      log.info("DiscordREST", "addBadRoute", route),
+      badRoutesRef.update(s => s.add(route)),
+      store.incrementCounter(
+        "dfx.rest.invalid",
+        Duration.minutes(10).millis,
+        10000,
+      ),
+    ])
+  const isBadRoute = (route: string) => badRoutesRef.get.map(s => s.has(route))
+  const removeBadRoute = (route: string) =>
+    badRoutesRef.update(s => s.remove(route))
+
+  const invalidRateLimit = (route: string) =>
+    isBadRoute(route).tap(invalid =>
+      invalid
+        ? maybeWait("dfx.rest.invalid", Duration.minutes(10), 10000)
+        : Effect.unit(),
+    ).asUnit
+
+  // Request rate limiting
+  const requestRateLimit = (path: string, request: Http.Request) =>
+    Do($ => {
+      const route = routeFromConfig(path, request.method)
+      const maybeBucket = $(store.getBucketForRoute(route))
+      const bucket = maybeBucket.getOrElse(
+        (): BucketDetails => ({
+          key: `?.${route}`,
+          resetAfter: 5000,
+          limit: 1,
+        }),
+      )
+      const resetAfter = millis(bucket.resetAfter)
+
+      $(invalidRateLimit(route))
+      $(maybeWait(`dfx.rest.${bucket.key}`, resetAfter, bucket.limit))
+    })
+
+  // Update rate limit buckets
+  const updateBuckets = (request: Http.Request, response: Http.Response) =>
+    Do($ => {
+      const route = routeFromConfig(request.url, request.method)
+      const { bucket, retryAfter, limit, remaining } = $(
+        rateLimitFromHeaders(response.headers),
+      )
+
+      const effectsToRun = [
+        removeBadRoute(route),
+        store.putBucketRoute(route, bucket),
+      ]
+
+      const hasBucket = $(store.hasBucket(bucket))
+      if (!hasBucket || limit - 1 === remaining) {
+        effectsToRun.push(
+          store.removeCounter(`dfx.rest.?.${route}`),
+          store.putBucket({
+            key: bucket,
+            resetAfter: retryAfter.millis,
+            limit: !hasBucket && remaining > 0 ? remaining : limit,
+          }),
+        )
+      }
+
+      $(Effect.allParDiscard(effectsToRun))
+    }).ignore
+
+  const httpExecutor = http.execute.filterStatusOk
+    .contramap(_ =>
+      _.updateUrl(_ => `${rest.baseUrl}${_}`).setHeaders({
+        Authorization: `Bot ${token.value}`,
+        "User-Agent": `DiscordBot (https://github.com/tim-smart/dfx, ${Pkg.version})`,
+      }),
+    )
+    .catchAll(_ => Effect.fail(new DiscordRESTError(_)))
+
+  const executor = <A = unknown>(
+    request: Http.Request,
+  ): Effect<never, DiscordRESTError, ResponseWithData<A>> =>
+    Do($ => {
+      $(requestRateLimit(request.url, request))
+      $(globalRateLimit)
+
+      const response = $(httpExecutor(request))
+
+      $(updateBuckets(request, response))
+
+      return response as ResponseWithData<A>
+    }).catchTag("DiscordRESTError", e => {
+      if (e.error._tag !== "StatusCodeError") {
+        return Effect.fail(e)
+      }
+
+      const response = e.error.response
+
+      switch (e.error.status) {
+        case 403:
+          return Do($ => {
+            $(
+              Effect.allParDiscard([
+                log.info("DiscordREST", "403", request.url),
+                addBadRoute(routeFromConfig(request.url, request.method)),
+                updateBuckets(request, response),
+              ]),
+            )
+            return $(Effect.fail(e))
+          })
+
+        case 429:
+          return Do($ => {
+            $(
+              Effect.allParDiscard([
+                log.info("DiscordREST", "429", request.url),
+                addBadRoute(routeFromConfig(request.url, request.method)),
+                updateBuckets(request, response),
+                Effect.sleep(
+                  retryAfter(response.headers).getOrElse(() =>
+                    Duration.seconds(5),
+                  ),
+                ),
+              ]),
+            )
+            return $(executor<A>(request))
+          })
+      }
+
+      return Effect.fail(e)
+    })
+
+  const routes = Discord.createRoutes<Partial<Http.MakeOptions>>(
+    <R, P>({
+      method,
+      url,
+      params,
+      options = {},
+    }: Discord.Route<P, Partial<Http.MakeOptions>>): RestResponse<R> => {
+      const hasBody = method !== "GET" && method !== "DELETE"
+      let request = Http.make(method as any)(url, options)
+
+      if (!hasBody) {
+        if (params) {
+          request = request.appendParams(params as any)
+        }
+      } else if (
+        params &&
+        request.body._tag === "Some" &&
+        request.body.value._tag === "FormDataBody"
+      ) {
+        request.body.value.value.append("payload_json", JSON.stringify(params))
+      } else if (params) {
+        request = request.jsonBody(params)
+      }
+
+      return executor(request)
+    },
+  )
+
+  return { executor, ...routes }
+})
+
+export interface DiscordREST extends Effect.Success<typeof make> {}
+export const DiscordREST = Tag<DiscordREST>()
+export const LiveDiscordREST =
+  LiveRateLimiter >> Layer.effect(DiscordREST, make)

package/src/Helpers/flags.ts

@@ -0,0 +1,68 @@
+export type Flags<T extends number | bigint> = Record<string, T>
+
+/**
+ * Returns all the flags OR'ed together.
+ */
+export function all(flags: Flags<number>): number
+export function all(flags: Flags<bigint>): bigint
+export function all(flags: Flags<any>): any {
+  return Object.values(flags).reduce((acc, flag) => acc | flag)
+}
+
+/**
+ * Returns a function that converts a bitfield to a list of flag names.
+ */
+export function toList<T extends Flags<number>>(
+  flags: T,
+): (bitfield: number) => (keyof T)[]
+export function toList<T extends Flags<bigint>>(
+  flags: T,
+): (bitfield: bigint) => (keyof T)[]
+export function toList<T extends Flags<any>>(
+  flags: T,
+): (bitfield: any) => (keyof T)[] {
+  const entries = Object.entries(flags)
+  return val =>
+    entries.reduce(
+      (acc, [key, flag]) => ((val & flag) === flag ? [...acc, key] : acc),
+      [] as (keyof T)[],
+    )
+}
+
+/**
+ * Returns a function that converts a list of flags names to a bigint bitfield.
+ */
+export const fromListBigint =
+  <T extends Flags<bigint>>(flags: T) =>
+  (list: (keyof T)[]) =>
+    list.reduce((acc, key) => acc | flags[key], BigInt(0))
+
+/**
+ * Returns a function that converts a list of flags names to a bitfield.
+ */
+export const fromList =
+  <T extends Flags<number>>(flags: T) =>
+  (list: (keyof T)[]) =>
+    list.reduce((acc, key) => acc | flags[key], 0)
+
+/**
+ * Checks if a bigint bitfield contains and a flag value.
+ */
+export const hasBigInt = (flag: bigint | string) => {
+  const flagBigInt = BigInt(flag)
+  return (bits: bigint | string) => {
+    const bitsBigInt = BigInt(bits)
+    return (bitsBigInt & flagBigInt) === flagBigInt
+  }
+}
+
+/**
+ * Checks if a bitfield contains and a flag value.
+ */
+export const has = (flag: number | string) => {
+  const flagNumber = +flag
+  return (bits: number | string) => {
+    const bitsNumber = +bits
+    return (bitsNumber & flagNumber) === flagNumber
+  }
+}

package/src/Helpers/intents.ts

@@ -0,0 +1,34 @@
+import * as Flags from "dfx/Helpers/flags"
+
+/**
+ * All the intents
+ */
+export const ALL = Flags.all(Discord.GatewayIntents)
+
+/**
+ * Privileged intents
+ */
+export const PRIVILEGED =
+  Discord.GatewayIntents.GUILD_PRESENCES |
+  Discord.GatewayIntents.GUILD_MEMBERS |
+  Discord.GatewayIntents.MESSAGE_CONTENT
+
+/**
+ * Un-privileged intents
+ */
+export const UNPRIVILEGED = ALL ^ PRIVILEGED
+
+/**
+ * Function that converts a intents bitfield value to a list of intent names.
+ */
+export const toList = Flags.toList(Discord.GatewayIntents)
+
+/**
+ * Function that converts a list of intent names to a bitfield value.
+ */
+export const fromList = Flags.fromList(Discord.GatewayIntents)
+
+/**
+ * Check if an intent flag exists in the permissions.
+ */
+export const has = Flags.has