dfx 1.0.4 → 1.0.5

package/dist/src/DiscordREST/utils.ts

@@ -0,0 +1,42 @@
+import * as Duration from "effect/Duration"
+import { pipe } from "effect/Function"
+import * as Option from "effect/Option"
+import * as Headers from "effect/unstable/http/Headers"
+
+const majorResources = ["channels", "guilds", "webhooks"] as const
+
+export const routeFromConfig = (path: string, method: string) => {
+  // Only keep major ID's
+  const routeURL = path
+    .split("?")[0]
+    .replace(/\/([A-Za-z]+)\/(\d{16,21}|@me)/g, (match, resource) =>
+      majorResources.includes(resource) ? match : `/${resource}`,
+    )
+    // Strip reactions
+    .replace(/\/reactions\/(.*)/, "/reactions")
+
+  return `${method}-${routeURL}`
+}
+
+export const numberHeader = (headers: Headers.Headers) => (key: string) =>
+  pipe(
+    Headers.get(headers, key),
+    Option.fromUndefinedOr,
+    Option.map(parseFloat),
+    Option.filter(n => !isNaN(n)),
+  )
+
+export const retryAfter = (headers: Headers.Headers) =>
+  numberHeader(headers)("x-ratelimit-reset-after").pipe(
+    Option.orElse(() => numberHeader(headers)("retry-after")),
+    Option.map(Duration.seconds),
+  )
+
+export const rateLimitFromHeaders = (headers: Headers.Headers) =>
+  Option.all({
+    bucket: Option.fromUndefinedOr(Headers.get(headers, "x-ratelimit-bucket")),
+    retryAfter: retryAfter(headers),
+    limit: numberHeader(headers)("x-ratelimit-limit"),
+    remaining: numberHeader(headers)("x-ratelimit-remaining"),
+  })
+export type RateLimitDetails = ReturnType<typeof rateLimitFromHeaders>

package/dist/src/DiscordREST.ts

@@ -0,0 +1,256 @@
+import { DiscordConfig } from "./DiscordConfig.ts"
+import {
+  rateLimitFromHeaders,
+  retryAfter,
+  routeFromConfig,
+} from "./DiscordREST/utils.ts"
+import { RateLimitStore, RateLimiter, RateLimiterLive } from "./RateLimit.ts"
+import * as Discord from "./types.ts"
+import { LIB_VERSION } from "./version.ts"
+import * as Duration from "effect/Duration"
+import { millis } from "effect/Duration"
+import * as Effect from "effect/Effect"
+import * as Fiber from "effect/Fiber"
+import { flow } from "effect/Function"
+import * as Layer from "effect/Layer"
+import * as Option from "effect/Option"
+import * as Redacted from "effect/Redacted"
+import * as ServiceMap from "effect/ServiceMap"
+import * as HttpBody from "effect/unstable/http/HttpBody"
+import * as HttpClient from "effect/unstable/http/HttpClient"
+import type { HttpClientError } from "effect/unstable/http/HttpClientError"
+import * as HttpRequest from "effect/unstable/http/HttpClientRequest"
+import type * as HttpResponse from "effect/unstable/http/HttpClientResponse"
+
+const make = Effect.gen(function* () {
+  const { rest, token } = yield* DiscordConfig
+
+  const store = yield* RateLimitStore
+  const { maybeWait } = yield* RateLimiter
+
+  const globalRateLimit = maybeWait(
+    "dfx.rest.global",
+    Duration.fromInputUnsafe(rest.globalRateLimit.window),
+    rest.globalRateLimit.limit,
+  )
+
+  // Invalid route handling (40x)
+  const badRoutes = new Set<string>()
+  const tenMinutes = Duration.minutes(10)
+  const tenMinutesMillis = Duration.toMillis(tenMinutes)
+  const addBadRoute = (route: string) =>
+    Effect.suspend(() => {
+      badRoutes.add(route)
+      return Effect.log("bad route")
+    }).pipe(
+      Effect.andThen(
+        store.incrementCounter("dfx.rest.invalid", tenMinutesMillis, 10000),
+      ),
+      Effect.annotateLogs("route", route),
+    )
+
+  const invalidRateLimit = (route: string) =>
+    Effect.suspend(() =>
+      badRoutes.has(route)
+        ? maybeWait("dfx.rest.invalid", tenMinutes, 10000)
+        : Effect.void,
+    )
+
+  // Request rate limiting
+  const requestRateLimit = Effect.fnUntraced(function* (
+    path: string,
+    request: HttpRequest.HttpClientRequest,
+  ) {
+    const route = routeFromConfig(path, request.method)
+    const maybeBucket = yield* store.getBucketForRoute(route)
+    yield* invalidRateLimit(route)
+    if (Option.isNone(maybeBucket)) return
+    const bucket = maybeBucket.value
+    yield* maybeWait(
+      `dfx.rest.${bucket.key}`,
+      millis(bucket.resetAfter),
+      bucket.limit,
+    )
+  })
+
+  // Update rate limit buckets
+  const updateBuckets = Effect.fnUntraced(function* (
+    request: HttpRequest.HttpClientRequest,
+    response: HttpResponse.HttpClientResponse,
+  ) {
+    const route = routeFromConfig(request.url, request.method)
+    const rateLimitOption = rateLimitFromHeaders(response.headers)
+    if (Option.isNone(rateLimitOption)) return
+    const rateLimit = rateLimitOption.value
+    const hasBucket = yield* store.hasBucket(rateLimit.bucket)
+    const fibers: Array<Fiber.Fiber<unknown, unknown>> = []
+
+    badRoutes.delete(route)
+    fibers.push(
+      yield* Effect.forkChild(store.putBucketRoute(route, rateLimit.bucket)),
+    )
+
+    if (!hasBucket || rateLimit.limit - 1 === rateLimit.remaining) {
+      fibers.push(
+        yield* Effect.forkChild(store.removeCounter(`dfx.rest.?.${route}`)),
+      )
+      fibers.push(
+        yield* Effect.forkChild(
+          store.putBucket({
+            key: rateLimit.bucket,
+            resetAfter: Duration.toMillis(rateLimit.retryAfter),
+            limit:
+              !hasBucket && rateLimit.remaining > 0
+                ? rateLimit.remaining
+                : rateLimit.limit,
+          }),
+        ),
+      )
+    }
+
+    yield* Fiber.awaitAll(fibers)
+  })
+
+  const defaultClient = (yield* HttpClient.HttpClient).pipe(
+    HttpClient.transformResponse(
+      Effect.provideService(HttpClient.TracerPropagationEnabled, false),
+    ),
+  )
+  const rateLimitedClient: HttpClient.HttpClient = defaultClient.pipe(
+    HttpClient.tapRequest(request =>
+      Effect.andThen(requestRateLimit(request.url, request), globalRateLimit),
+    ),
+    HttpClient.transformResponse(
+      flow(
+        Effect.tap(response => updateBuckets(response.request, response)),
+        Effect.flatMap(response => {
+          const request = response.request
+
+          switch (response.status) {
+            case 403:
+              return Effect.as(
+                Effect.all(
+                  [
+                    Effect.annotateLogs(
+                      Effect.logDebug("403"),
+                      "url",
+                      request.url,
+                    ),
+                    addBadRoute(routeFromConfig(request.url, request.method)),
+                    updateBuckets(request, response),
+                  ],
+                  { concurrency: "unbounded", discard: true },
+                ),
+                response,
+              )
+
+            case 429:
+              return Effect.annotateLogs(
+                Effect.logDebug("429"),
+                "url",
+                request.url,
+              ).pipe(
+                Effect.andThen(
+                  addBadRoute(routeFromConfig(request.url, request.method)),
+                ),
+                Effect.andThen(updateBuckets(request, response)),
+                Effect.andThen(
+                  Effect.sleep(
+                    Option.getOrElse(retryAfter(response.headers), () =>
+                      Duration.seconds(5),
+                    ),
+                  ),
+                ),
+                Effect.andThen(rateLimitedClient.execute(request)),
+              )
+          }
+
+          return Effect.succeed(response)
+        }),
+        Effect.annotateLogs({
+          package: "dfx",
+          module: "DiscordREST",
+        }),
+      ),
+    ),
+  )
+
+  const httpClient = HttpClient.mapRequestInputEffect(rateLimitedClient, req =>
+    Effect.sync(() => {
+      const fiber = Fiber.getCurrent()!
+      let request = req.pipe(
+        HttpRequest.prependUrl(rest.baseUrl),
+        HttpRequest.setHeaders({
+          Authorization: `Bot ${Redacted.value(token)}`,
+          "User-Agent": `DiscordBot (https://github.com/tim-smart/dfx, ${LIB_VERSION})`,
+        }),
+      )
+      const formData = ServiceMap.getOption(fiber.services, DiscordFormData)
+      if (Option.isSome(formData)) {
+        if (request.body._tag === "Uint8Array") {
+          formData.value.set(
+            "payload_json",
+            new Blob([request.body.body as Uint8Array<ArrayBuffer>], {
+              type: "application/json",
+            }),
+            "",
+          )
+        }
+        request = HttpRequest.setBody(
+          request,
+          HttpBody.formData(formData.value),
+        )
+      }
+      return request
+    }),
+  )
+
+  return DiscordREST.of({
+    ...Discord.make(httpClient),
+    withFormData(formData) {
+      return Effect.provideService(DiscordFormData, formData)
+    },
+    withFiles(files) {
+      return <A, E, R>(
+        effect: Effect.Effect<A, E, R>,
+      ): Effect.Effect<A, E, R> =>
+        Effect.suspend(() => {
+          const formData = new FormData()
+          for (let i = 0; i < files.length; i++) {
+            formData.append(`files[${i}]`, files[i])
+          }
+          return Effect.provideService(effect, DiscordFormData, formData)
+        })
+    },
+  })
+})
+
+export type DiscordRESTError =
+  | HttpClientError
+  | Discord.DiscordRestError<"RatelimitedResponse", Discord.RatelimitedResponse>
+  | Discord.DiscordRestError<"ErrorResponse", Discord.ErrorResponse>
+
+export class DiscordFormData extends ServiceMap.Service<
+  DiscordFormData,
+  FormData
+>()("dfx/DiscordREST/DiscordFormData") {}
+
+export interface DiscordRestService extends Discord.DiscordRest {
+  withFormData(
+    formData: FormData,
+  ): <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>
+  withFiles(
+    files: ReadonlyArray<File>,
+  ): <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>
+}
+
+export class DiscordREST extends ServiceMap.Service<
+  DiscordREST,
+  DiscordRestService
+>()("dfx/DiscordREST") {}
+
+export const DiscordRESTLive: Layer.Layer<
+  DiscordREST,
+  never,
+  DiscordConfig | HttpClient.HttpClient | RateLimitStore
+> = Layer.effect(DiscordREST, make).pipe(Layer.provide(RateLimiterLive))

package/dist/src/Helpers/flags.ts

@@ -0,0 +1,73 @@
+export type Flags<T extends number | bigint> = Record<string, T>
+
+/**
+ * Returns all the flags OR'ed together.
+ */
+export function all(flags: Flags<number>): number
+export function all(flags: Flags<bigint>): bigint
+export function all(flags: Flags<any>): any {
+  return Object.values(flags).reduce((acc, flag) => acc | flag)
+}
+
+/**
+ * Returns a function that converts a bitfield to a list of flag names.
+ */
+export function toList<T extends Flags<number>>(
+  flags: T,
+): (bitfield: number) => Array<keyof T>
+export function toList<T extends Flags<bigint>>(
+  flags: T,
+): (bitfield: bigint) => Array<keyof T>
+export function toList<T extends Flags<any>>(
+  flags: T,
+): (bitfield: any) => Array<keyof T> {
+  const entries = Object.entries(flags)
+  return val => {
+    const out = []
+    for (let i = 0; i < entries.length; i++) {
+      const [key, flag] = entries[i]
+      if ((val & flag) === flag) {
+        out.push(key)
+      }
+    }
+    return out
+  }
+}
+
+/**
+ * Returns a function that converts a list of flags names to a bigint bitfield.
+ */
+export const fromListBigint =
+  <T extends Flags<bigint>>(flags: T) =>
+  (list: Array<keyof T>) =>
+    list.reduce((acc, key) => acc | flags[key], BigInt(0))
+
+/**
+ * Returns a function that converts a list of flags names to a bitfield.
+ */
+export const fromList =
+  <T extends Flags<number>>(flags: T) =>
+  (list: Array<keyof T>) =>
+    list.reduce((acc, key) => acc | flags[key], 0)
+
+/**
+ * Checks if a bigint bitfield contains and a flag value.
+ */
+export const hasBigInt = (flag: bigint | string) => {
+  const flagBigInt = BigInt(flag)
+  return (bits: bigint | string) => {
+    const bitsBigInt = BigInt(bits)
+    return (bitsBigInt & flagBigInt) === flagBigInt
+  }
+}
+
+/**
+ * Checks if a bitfield contains and a flag value.
+ */
+export const has = (flag: number | string) => {
+  const flagNumber = +flag
+  return (bits: number | string) => {
+    const bitsNumber = +bits
+    return (bitsNumber & flagNumber) === flagNumber
+  }
+}

package/dist/src/Helpers/intents.ts

@@ -0,0 +1,35 @@
+import * as Flags from "./flags.ts"
+import * as Discord from "../types.ts"
+
+/**
+ * All the intents
+ */
+export const ALL = Flags.all(Discord.GatewayIntentBits)
+
+/**
+ * Privileged intents
+ */
+export const PRIVILEGED =
+  Discord.GatewayIntentBits.GuildPresences |
+  Discord.GatewayIntentBits.GuildMembers |
+  Discord.GatewayIntentBits.MessageContent
+
+/**
+ * Un-privileged intents
+ */
+export const UNPRIVILEGED = ALL ^ PRIVILEGED
+
+/**
+ * Function that converts a intents bitfield value to a list of intent names.
+ */
+export const toList = Flags.toList(Discord.GatewayIntentBits)
+
+/**
+ * Function that converts a list of intent names to a bitfield value.
+ */
+export const fromList = Flags.fromList(Discord.GatewayIntentBits)
+
+/**
+ * Check if an intent flag exists in the permissions.
+ */
+export const has = Flags.has