dfx 0.3.1 → 0.5.1

package/src/DiscordGateway/Sharder/index.ts

@@ -0,0 +1,95 @@
+import { millis } from "@fp-ts/data/Duration"
+import { overridePull } from "callbag-effect-ts/Source"
+import { ShardStore } from "../ShardStore/index.js"
+
+const configs = (totalCount: number) =>
+  Do(($) => {
+    const store = $(Effect.service(ShardStore))
+    const claimId = (sharderCount: number) =>
+      store
+        .claimId({
+          totalCount,
+          sharderCount,
+        })
+        .flatMap(
+          (
+            a,
+          ): Effect<
+            never,
+            never,
+            readonly [Maybe<number>, EffectSource<never, never, number>]
+          > =>
+            a.match(
+              () =>
+                Effect.succeed([
+                  Maybe.some(sharderCount),
+                  EffectSource.empty,
+                ] as const).delay(Duration.minutes(3)),
+              (id) =>
+                Effect.succeed([
+                  Maybe.some(sharderCount + 1),
+                  EffectSource.of(id),
+                ]),
+            ),
+        )
+
+    return EffectSource.resource(0, (sharderCount) =>
+      EffectSource.fromEffect(claimId(sharderCount)),
+    ).map((id) => ({
+      id,
+      totalCount,
+    }))
+  })
+
+const spawnEffect = Effect.structPar({
+  gateway: Rest.rest
+    .getGatewayBot()
+    .flatMap((r) => r.json)
+    .catchAll(() =>
+      Effect.succeed<Discord.GetGatewayBotResponse>({
+        url: "wss://gateway.discord.gg/",
+        shards: 1,
+        session_start_limit: {
+          total: 0,
+          remaining: 0,
+          reset_after: 0,
+          max_concurrency: 1,
+        },
+      }),
+    ),
+  config: Config.gateway,
+  limiter: Effect.service(RateLimitStore.RateLimiter),
+})
+  .bind("configs", ({ gateway, config }) =>
+    configs(config.shardCount ?? gateway.shards),
+  )
+  .map(({ gateway, config, configs, limiter }) => {
+    const [source, pull] = overridePull(
+      configs,
+      gateway.session_start_limit.max_concurrency,
+    )
+
+    return source
+      .map((config) => ({
+        ...config,
+        url: gateway.url,
+        concurrency: gateway.session_start_limit.max_concurrency,
+      }))
+      .groupBy((c) => c.id % c.concurrency)
+      .chainPar(([shardConfig, key]) =>
+        shardConfig
+          .tap(() =>
+            limiter.maybeWait(
+              `gateway.sharder.${key}`,
+              millis(config.identifyRateLimit[0]),
+              config.identifyRateLimit[1],
+            ),
+          )
+          .mapEffect((c) => Shard.make([c.id, c.totalCount])),
+      )
+      .tap(() => Effect.sync(pull))
+  })
+
+export const spawn = spawnEffect.unwrap.chainPar((shard) =>
+  EffectSource.of(shard).merge(EffectSource.fromEffect(shard.run).drain),
+)

package/src/DiscordGateway/WS/index.ts

@@ -0,0 +1,99 @@
+import WebSocket from "isomorphic-ws"
+
+export const Reconnect = Symbol()
+export type Reconnect = typeof Reconnect
+export type Message = string | Buffer | ArrayBuffer | Reconnect
+
+const socket = (urlRef: Ref<string>, options?: WebSocket.ClientOptions) =>
+  Do(($) => {
+    const url = $(urlRef.get)
+    return new WebSocket(url, options)
+  }).acquireRelease((ws) =>
+    Effect.sync(() => {
+      ws.close()
+      ws.removeAllListeners()
+    }),
+  )
+
+export class WebSocketError {
+  readonly _tag = "WebSocketError"
+  constructor(readonly reason: unknown) {}
+}
+
+export class WebSocketCloseError {
+  readonly _tag = "WebSocketCloseError"
+  constructor(readonly code: number, readonly reason: string) {}
+}
+
+const recv = (ws: WebSocket) =>
+  EffectSource.async<WebSocketError | WebSocketCloseError, WebSocket.RawData>(
+    (emit) => {
+      ws.on("message", (message) => {
+        return emit.data(message)
+      })
+
+      ws.on("error", (cause) => {
+        emit.fail(new WebSocketError(cause))
+      })
+
+      ws.on("close", (code, reason) => {
+        return emit.fail(new WebSocketCloseError(code, reason.toString("utf8")))
+      })
+    },
+  )
+
+export class WebSocketWriteError {
+  readonly _tag = "WebSocketWriteError"
+  constructor(readonly reason: Error) {}
+}
+
+const send = (ws: WebSocket, out: EffectSource<never, never, Message>) =>
+  Do(($) => {
+    const log = $(Effect.service(Log.Log))
+    return Effect.async<never, never, void>((resume) => {
+      if (ws.readyState & ws.OPEN) {
+        resume(Effect.unit())
+      } else {
+        ws.once("open", () => {
+          resume(Effect.unit())
+        })
+      }
+    })
+      .map(() => out)
+      .unwrap.tap((p) => log.debug("WS", "send", p))
+      .tap((data) =>
+        Effect.async<never, WebSocketWriteError, void>((resume) => {
+          if (data === Reconnect) {
+            ws.close(1012, "reconnecting")
+            resume(Effect.unit())
+          } else {
+            ws.send(data, (err) => {
+              resume(
+                err
+                  ? Effect.fail(new WebSocketWriteError(err!))
+                  : Effect.unit(),
+              )
+            })
+          }
+        }),
+      ).drain
+  })
+
+export const make = (url: Ref<string>, options?: WebSocket.ClientOptions) =>
+  Do(($) => {
+    const [sink, outbound] = EffectSource.asyncSink<never, Message>()
+    const log = $(Effect.service(Log.Log))
+    const withLog = Effect.provideService(Log.Log)(log)
+
+    const source = Do(($) => {
+      const ws = $(socket(url, options))
+      const sendEffect = $(withLog(send(ws, outbound)))
+      return recv(ws).merge(sendEffect)
+    }).unwrapScope.retry(
+      Schedule.recurWhile(
+        (e) => e._tag === "WebSocketCloseError" && e.code === 1012,
+      ),
+    )
+
+    return { source, sink }
+  })

package/src/DiscordGateway/index.ts

@@ -0,0 +1,55 @@
+import { spawn } from "./Sharder/index.js"
+
+const fromDispatchFactory =
+  <R, E>(
+    source: EffectSource<R, E, Discord.GatewayPayload<Discord.ReceiveEvent>>,
+  ) =>
+  <K extends keyof Discord.ReceiveEvents>(
+    event: K,
+  ): EffectSource<R, E, Discord.ReceiveEvents[K]> =>
+    source.filter((p) => p.t === event).map((p) => p.d! as any)
+
+const handleDispatchFactory =
+  <R, E>(
+    source: EffectSource<R, E, Discord.GatewayPayload<Discord.ReceiveEvent>>,
+  ) =>
+  <K extends keyof Discord.ReceiveEvents, R1, E1, A>(
+    event: K,
+    handle: (event: Discord.ReceiveEvents[K]) => Effect<R1, E1, A>,
+  ): Effect<R | R1, E | E1, void> =>
+    source
+      .filter((p) => p.t === event)
+      .chainPar((a) => EffectSource.fromEffect(handle(a.d as any))).runDrain
+
+export const make = Do(($) => {
+  const shards = $(spawn.share)
+  const raw = $(shards.chainPar((s) => s.raw).share)
+  const dispatch = $(shards.chainPar((s) => s.dispatch).share)
+  const fromDispatch = fromDispatchFactory(dispatch)
+  const handleDispatch = handleDispatchFactory(dispatch)
+
+  return {
+    shards,
+    raw,
+    dispatch,
+    fromDispatch,
+    handleDispatch,
+  }
+})
+
+export interface DiscordGateway extends Success<typeof make> {}
+export const DiscordGateway = Tag<DiscordGateway>()
+export const LiveDiscordGateway = Layer.fromEffect(DiscordGateway)(make)
+
+export const handleDispatch = <
+  K extends keyof Discord.ReceiveEvents,
+  R1,
+  E1,
+  A,
+>(
+  event: K,
+  handle: (event: Discord.ReceiveEvents[K]) => Effect<R1, E1, A>,
+) =>
+  Effect.serviceWithEffect(DiscordGateway)((a) =>
+    a.handleDispatch(event, handle),
+  )

package/src/DiscordREST/index.ts

@@ -0,0 +1,132 @@
+import { millis } from "@fp-ts/data/Duration"
+import { BucketDetails } from "dfx/RateLimitStore/index"
+import { ResponseWithData } from "./types.js"
+import { rateLimitFromHeaders, routeFromConfig } from "./utils.js"
+import Pkg from "../../package.json" assert { type: "json" }
+
+const make = Do(($) => {
+  const { token, rest } = $(Effect.service(Config.DiscordConfig))
+
+  const log = $(Effect.service(Log.Log))
+  const store = $(Effect.service(RateLimitStore.RateLimitStore))
+  const { maybeWait } = $(Effect.service(RateLimitStore.RateLimiter))
+
+  const globalRateLimit = maybeWait(
+    "rest.global",
+    rest.globalRateLimit.window,
+    rest.globalRateLimit.limit,
+  )
+
+  const requestRateLimit = (path: string, init: RequestInit) =>
+    Do(($) => {
+      const route = routeFromConfig(path, init)
+      const maybeBucket = $(store.getBucketForRoute(route))
+      const bucket = maybeBucket.getOrElse(
+        (): BucketDetails => ({
+          key: `?.${Equal.hash(route)}`,
+          resetAfter: 5000,
+          limit: 1,
+        }),
+      )
+      const resetAfter = millis(bucket.resetAfter)
+      $(maybeWait(`rest.bucket.${bucket.key}`, resetAfter, bucket.limit))
+    })
+
+  const updateBuckets = (path: string, init: RequestInit, response: Response) =>
+    Do(($) => {
+      const route = routeFromConfig(path, init)
+      const { bucket, retryAfter, limit, remaining } = $(
+        Effect.fromOption(rateLimitFromHeaders(response.headers)),
+      )
+
+      const effectsToRun = [store.putBucketRoute(route, bucket)]
+
+      const hasBucket = $(store.hasBucket(bucket))
+      if (!hasBucket || limit - 1 === remaining) {
+        effectsToRun.push(
+          store.putBucket({
+            key: bucket,
+            resetAfter: retryAfter.millis,
+            limit: !hasBucket && remaining > 0 ? remaining : limit,
+          }),
+        )
+      }
+
+      $(effectsToRun.collectAllParDiscard)
+    }).ignore
+
+  const request = <A = unknown>(
+    path: string,
+    init: RequestInit = {},
+  ): Effect<
+    never,
+    Http.FetchError | Http.StatusCodeError | Http.JsonParseError,
+    ResponseWithData<A>
+  > =>
+    requestRateLimit(path, init)
+      .tap(() => globalRateLimit)
+      .flatMap(() =>
+        Http.requestWithJson<A>(`${rest.baseUrl}${path}`, {
+          ...init,
+          headers: {
+            ...(init?.headers ?? {}),
+            Authorization: `Bot ${token}`,
+            "User-Agent": `DiscordBot (https://github.com/tim-smart/dfx, ${Pkg.version})`,
+          },
+        }),
+      )
+      .catchTag("StatusCodeError", (e) =>
+        e.code === 429
+          ? Do(($) => {
+              $(log.debug("DiscordREST", "429", path))
+              $(updateBuckets(path, init, e.response))
+              return $(request<A>(path, init))
+            })
+          : Effect.fail(e),
+      )
+      .tap(({ response }) => updateBuckets(path, init, response))
+
+  return { request }
+})
+
+export interface DiscordREST extends Success<typeof make> {}
+export const DiscordREST = Tag<DiscordREST>()
+export const LiveDiscordREST = Layer.fromEffect(DiscordREST)(make)
+
+export const rest = Discord.createRoutes<RequestInit>(
+  ({ method, url, params, options = {} }) =>
+    Effect.serviceWithEffect(DiscordREST)(({ request }) => {
+      const hasBody = method !== "GET" && method !== "DELETE"
+      let hasFormData = typeof (options?.body as any)?.append === "function"
+      let body: BodyInit | undefined = undefined
+
+      const headers: Record<string, string> = {}
+      if (hasBody && !hasFormData) {
+        headers["content-type"] = "application/json"
+      }
+
+      const qs = new URLSearchParams()
+      if (!hasBody) {
+        Object.entries((params ?? {}) as Record<string, string>).forEach(
+          ([key, value]) => {
+            qs.append(key, value)
+          },
+        )
+      } else if (hasFormData) {
+        body = options.body!
+        if (params) {
+          ;(body as FormData).append("payload_json", JSON.stringify(params))
+        }
+      } else if (params) {
+        body = JSON.stringify(params)
+      } else {
+        body = options.body!
+      }
+
+      return request(`${url}?${qs.toString()}`, {
+        method,
+        headers,
+        body,
+      })
+    }),
+)

package/src/DiscordREST/types.ts

@@ -0,0 +1,14 @@
+import { DiscordREST } from "./index.js"
+
+export interface ResponseWithData<A> {
+  response: Response
+  json: Effect<never, Http.JsonParseError, A>
+  text: Effect<never, never, string>
+  blob: Effect<never, Http.BlobError, Blob>
+}
+
+export type RestResponse<T> = Effect<
+  DiscordREST,
+  Http.FetchError | Http.StatusCodeError | Http.JsonParseError,
+  ResponseWithData<T>
+>

package/src/DiscordREST/utils.ts

@@ -0,0 +1,35 @@
+const majorResources = ["channels", "guilds", "webhooks"] as const
+
+export const routeFromConfig = (path: string, init: RequestInit) => {
+  const method = (init?.method ?? "get").toLowerCase()
+
+  // Only keep major ID's
+  const routeURL = path
+    .split("?")[0]
+    .replace(/\/([A-Za-z]+)\/(\d{16,21}|@me)/g, (match, resource) =>
+      majorResources.includes(resource) ? match : `/${resource}`,
+    )
+    // Strip reactions
+    .replace(/\/reactions\/(.*)/, "/reactions")
+
+  return `${method}-${routeURL}`
+}
+
+export const numberHeader = (headers: Headers) => (key: string) =>
+  Maybe.fromNullable(headers.get(key))
+    .map(parseFloat)
+    .filter((n) => !isNaN(n))
+
+export const retryAfter = (headers: Headers) =>
+  numberHeader(headers)("x-ratelimit-reset-after")
+    .catchAll(() => numberHeader(headers)("retry-after"))
+    .map(Duration.seconds)
+
+export const rateLimitFromHeaders = (headers: Headers) =>
+  Maybe.struct({
+    bucket: Maybe.fromNullable(headers.get("x-ratelimit-bucket")),
+    retryAfter: retryAfter(headers),
+    limit: numberHeader(headers)("x-ratelimit-limit"),
+    remaining: numberHeader(headers)("x-ratelimit-remaining"),
+  })
+export type RateLimitDetails = ReturnType<typeof rateLimitFromHeaders>

package/src/Helpers/flags.ts

@@ -0,0 +1,68 @@
+export type Flags<T extends number | bigint> = Record<string, T>
+
+/**
+ * Returns all the flags OR'ed together.
+ */
+export function all(flags: Flags<number>): number
+export function all(flags: Flags<bigint>): bigint
+export function all(flags: Flags<any>): any {
+  return Object.values(flags).reduce((acc, flag) => acc | flag)
+}
+
+/**
+ * Returns a function that converts a bitfield to a list of flag names.
+ */
+export function toList<T extends Flags<number>>(
+  flags: T,
+): (bitfield: number) => (keyof T)[]
+export function toList<T extends Flags<bigint>>(
+  flags: T,
+): (bitfield: bigint) => (keyof T)[]
+export function toList<T extends Flags<any>>(
+  flags: T,
+): (bitfield: any) => (keyof T)[] {
+  const entries = Object.entries(flags)
+  return (val) =>
+    entries.reduce(
+      (acc, [key, flag]) => ((val & flag) === flag ? [...acc, key] : acc),
+      [] as (keyof T)[],
+    )
+}
+
+/**
+ * Returns a function that converts a list of flags names to a bigint bitfield.
+ */
+export const fromListBigint =
+  <T extends Flags<bigint>>(flags: T) =>
+  (list: (keyof T)[]) =>
+    list.reduce((acc, key) => acc | flags[key], BigInt(0))
+
+/**
+ * Returns a function that converts a list of flags names to a bitfield.
+ */
+export const fromList =
+  <T extends Flags<number>>(flags: T) =>
+  (list: (keyof T)[]) =>
+    list.reduce((acc, key) => acc | flags[key], 0)
+
+/**
+ * Checks if a bigint bitfield contains and a flag value.
+ */
+export const hasBigInt = (flag: bigint | string) => {
+  const flagBigInt = BigInt(flag)
+  return (bits: bigint | string) => {
+    const bitsBigInt = BigInt(bits)
+    return (bitsBigInt & flagBigInt) === flagBigInt
+  }
+}
+
+/**
+ * Checks if a bitfield contains and a flag value.
+ */
+export const has = (flag: number | string) => {
+  const flagNumber = +flag
+  return (bits: number | string) => {
+    const bitsNumber = +bits
+    return (bitsNumber & flagNumber) === flagNumber
+  }
+}

package/src/Helpers/intents.ts

@@ -0,0 +1,32 @@
+/**
+ * All the intents
+ */
+export const ALL = Flags.all(Discord.GatewayIntents)
+
+/**
+ * Privileged intents
+ */
+export const PRIVILEGED =
+  Discord.GatewayIntents.GUILD_PRESENCES |
+  Discord.GatewayIntents.GUILD_MEMBERS |
+  Discord.GatewayIntents.MESSAGE_CONTENT
+
+/**
+ * Un-privileged intents
+ */
+export const UNPRIVILEGED = ALL ^ PRIVILEGED
+
+/**
+ * Function that converts a intents bitfield value to a list of intent names.
+ */
+export const toList = Flags.toList(Discord.GatewayIntents)
+
+/**
+ * Function that converts a list of intent names to a bitfield value.
+ */
+export const fromList = Flags.fromList(Discord.GatewayIntents)
+
+/**
+ * Check if an intent flag exists in the permissions.
+ */
+export const has = Flags.has