dfx 0.42.2 → 0.42.3

package/src/Cache.ts

@@ -0,0 +1,140 @@
+import { CacheDriver, ParentCacheDriver } from "./Cache/driver.js"
+
+export * from "./Cache/driver.js"
+export {
+  create as memoryDriver,
+  createWithParent as memoryParentDriver,
+} from "./Cache/memory.js"
+export {
+  create as memoryTTLDriver,
+  createWithParent as memoryTTLParentDriver,
+} from "./Cache/memoryTTL.js"
+
+export type ParentCacheOp<T> =
+  | { op: "create"; parentId: string; resourceId: string; resource: T }
+  | { op: "update"; parentId: string; resourceId: string; resource: T }
+  | { op: "delete"; parentId: string; resourceId: string }
+  | { op: "parentDelete"; parentId: string }
+
+export type CacheOp<T> =
+  | { op: "create"; resourceId: string; resource: T }
+  | { op: "update"; resourceId: string; resource: T }
+  | { op: "delete"; resourceId: string }
+
+export const makeWithParent = <EOps, EDriver, EMiss, EPMiss, A>({
+  driver,
+  ops = Stream.empty,
+  id,
+  onMiss,
+  onParentMiss,
+}: {
+  driver: ParentCacheDriver<EDriver, A>
+  ops?: Stream<never, EOps, ParentCacheOp<A>>
+  id: (_: A) => Effect<never, EMiss, readonly [parentId: string, id: string]>
+  onMiss: (parentId: string, id: string) => Effect<never, EMiss, A>
+  onParentMiss: (
+    parentId: string,
+  ) => Effect<never, EPMiss, [id: string, resource: A][]>
+}) => {
+  const sync = ops.tap((op): Effect<never, EDriver, void> => {
+    switch (op.op) {
+      case "create":
+      case "update":
+        return driver.set(op.parentId, op.resourceId, op.resource)
+
+      case "delete":
+        return driver.delete(op.parentId, op.resourceId)
+
+      case "parentDelete":
+        return driver.parentDelete(op.parentId)
+    }
+  }).runDrain
+
+  const get = (parentId: string, id: string) =>
+    driver
+      .get(parentId, id)
+      .someOrElseEffect(() =>
+        onMiss(parentId, id).tap(a => driver.set(parentId, id, a)),
+      )
+
+  const put = (_: A) =>
+    id(_).flatMap(([parentId, id]) => driver.set(parentId, id, _))
+
+  const update = <R, E>(
+    parentId: string,
+    id: string,
+    f: (_: A) => Effect<R, E, A>,
+  ) =>
+    get(parentId, id)
+      .flatMap(f)
+      .tap(_ => driver.set(parentId, id, _))
+
+  return {
+    ...driver,
+
+    get,
+    put,
+    update,
+
+    getForParent: (parentId: string) =>
+      driver.getForParent(parentId).someOrElseEffect(() =>
+        onParentMiss(parentId)
+          .tap(entries =>
+            Effect.allPar(
+              entries.map(([id, a]) => driver.set(parentId, id, a)),
+            ),
+          )
+          .map(entries => new Map(entries) as ReadonlyMap<string, A>),
+      ),
+
+    run: sync.zipParRight(driver.run),
+  }
+}
+
+export const make = <EOps, EDriver, EMiss, A>({
+  driver,
+  ops = Stream.empty,
+  id,
+  onMiss,
+}: {
+  driver: CacheDriver<EDriver, A>
+  ops?: Stream<never, EOps, CacheOp<A>>
+  id: (_: A) => string
+  onMiss: (id: string) => Effect<never, EMiss, A>
+}) => {
+  const sync = ops.tap((op): Effect<never, EDriver, void> => {
+    switch (op.op) {
+      case "create":
+      case "update":
+        return driver.set(op.resourceId, op.resource)
+
+      case "delete":
+        return driver.delete(op.resourceId)
+    }
+  }).runDrain
+
+  const get = (id: string) =>
+    driver
+      .get(id)
+      .someOrElseEffect(() => onMiss(id).tap(a => driver.set(id, a)))
+
+  const put = (_: A) => driver.set(id(_), _)
+
+  const update = <R, E>(id: string, f: (_: A) => Effect<R, E, A>) =>
+    get(id)
+      .flatMap(f)
+      .tap(_ => driver.set(id, _))
+
+  return {
+    ...driver,
+    get,
+    put,
+    update,
+    run: sync.zipParRight(driver.run),
+  }
+}
+
+export class CacheMissError {
+  readonly _tag = "CacheMissError"
+  constructor(readonly cacheName: string, readonly id: string) {}
+}

package/src/DiscordConfig.ts

@@ -0,0 +1,48 @@
+const VERSION = 10
+
+export interface DiscordConfig {
+  token: ConfigSecret
+  rest: {
+    baseUrl: string
+    globalRateLimit: {
+      limit: number
+      window: Duration
+    }
+  }
+  gateway: {
+    intents: number
+    presence?: Discord.UpdatePresence
+    shardCount?: number
+
+    identifyRateLimit: readonly [window: number, limit: number]
+  }
+}
+export const DiscordConfig = Tag<DiscordConfig>()
+
+export interface MakeOpts {
+  token: ConfigSecret
+  rest?: Partial<DiscordConfig["rest"]>
+  gateway?: Partial<DiscordConfig["gateway"]>
+}
+
+export const make = ({ token, rest, gateway }: MakeOpts): DiscordConfig => ({
+  token,
+  rest: {
+    baseUrl: `https://discord.com/api/v${VERSION}`,
+    ...(rest ?? {}),
+    globalRateLimit: {
+      limit: 50,
+      window: Duration.seconds(1),
+      ...(rest?.globalRateLimit ?? {}),
+    },
+  },
+  gateway: {
+    intents: Discord.GatewayIntents.GUILDS,
+    identifyRateLimit: [5000, 1],
+    ...(gateway ?? {}),
+  },
+})
+
+export const makeLayer = flow(make, _ => Layer.succeed(DiscordConfig, _))
+export const makeFromConfig = (_: Config.Wrap<MakeOpts>) =>
+  Config.unwrap(_).config.map(make).toLayer(DiscordConfig)

package/src/DiscordGateway/DiscordWS.ts

@@ -0,0 +1,74 @@
+import {
+  LiveWS,
+  Reconnect,
+  WS,
+  WebSocketCloseError,
+  WebSocketError,
+} from "dfx/DiscordGateway/WS"
+import WebSocket from "isomorphic-ws"
+
+export type Message = Discord.GatewayPayload | Reconnect
+
+export interface OpenOpts {
+  url?: string
+  version?: number
+  encoding?: DiscordWSCodec
+  outbound: Effect<never, never, Message>
+  onReconnect?: Effect<never, never, void>
+}
+
+export interface DiscordWSCodec {
+  type: "json" | "etf"
+  encode: (p: Discord.GatewayPayload) => string
+  decode: (p: WebSocket.Data) => Discord.GatewayPayload
+}
+export const DiscordWSCodec = Tag<DiscordWSCodec>()
+export const LiveJsonDiscordWSCodec = Layer.succeed(DiscordWSCodec, {
+  type: "json",
+  encode: p => JSON.stringify(p),
+  decode: p => JSON.parse(p.toString("utf8")),
+})
+
+const make = Do($ => {
+  const ws = $(WS)
+  const encoding = $(DiscordWSCodec)
+
+  const connect = ({
+    url = "wss://gateway.discord.gg/",
+    version = 10,
+    outbound,
+    onReconnect,
+  }: OpenOpts) =>
+    Do($ => {
+      const urlRef = $(
+        Ref.make(`${url}?v=${version}&encoding=${encoding.type}`),
+      )
+      const setUrl = (url: string) =>
+        urlRef.set(`${url}?v=${version}&encoding=${encoding.type}`)
+      const takeOutbound = outbound.map(a =>
+        a === Reconnect ? a : encoding.encode(a),
+      )
+      const socket = $(ws.connect(urlRef, takeOutbound, onReconnect))
+      const take = socket.take.map(encoding.decode)
+
+      const run = socket.run.retry(
+        Schedule.exponential(Duration.seconds(0.5)).whileInput(
+          (_: WebSocketError | WebSocketCloseError) =>
+            (_._tag === "WebSocketCloseError" && _.code < 2000) ||
+            (_._tag === "WebSocketError" && _.reason === "open-timeout"),
+        ),
+      )
+
+      return {
+        run,
+        take,
+        setUrl,
+      } as const
+    })
+
+  return { connect } as const
+})
+
+export interface DiscordWS extends Effect.Success<typeof make> {}
+export const DiscordWS = Tag<DiscordWS>()
+export const LiveDiscordWS = LiveWS >> make.toLayer(DiscordWS)

package/src/DiscordGateway/Shard/heartbeats.ts

@@ -0,0 +1,42 @@
+import { millis } from "@effect/data/Duration"
+import * as SendEvents from "./sendEvents.js"
+import * as DiscordWS from "dfx/DiscordGateway/DiscordWS"
+import { Reconnect } from "../WS.js"
+
+const payload = (ref: Ref<boolean>, seqRef: Ref<Maybe<number>>) =>
+  seqRef.get
+    .map(a => SendEvents.heartbeat(a.getOrNull))
+    .tap(() => ref.set(false))
+
+const payloadOrReconnect = (ref: Ref<boolean>, seqRef: Ref<Maybe<number>>) =>
+  ref.get.flatMap(
+    (acked): Effect<never, never, DiscordWS.Message> =>
+      acked ? payload(ref, seqRef) : Effect.succeed(Reconnect),
+  )
+
+export const send = (
+  hellos: Dequeue<Discord.GatewayPayload>,
+  acks: Dequeue<Discord.GatewayPayload>,
+  seqRef: Ref<Maybe<number>>,
+  send: (p: DiscordWS.Message) => Effect<never, never, boolean>,
+) =>
+  Do($ => {
+    const ackedRef = $(Ref.make(true))
+
+    const heartbeats = hellos
+      .take()
+      .tap(() => ackedRef.set(true))
+      .foreverSwitch(p =>
+        payloadOrReconnect(ackedRef, seqRef)
+          .tap(send)
+          .schedule(
+            Schedule.duration(
+              millis(p.d!.heartbeat_interval * Math.random()),
+            ).andThen(Schedule.fixed(millis(p.d!.heartbeat_interval))),
+          ),
+      )
+
+    const run = acks.take().tap(() => ackedRef.set(true)).forever
+
+    return $(run.zipParLeft(heartbeats))
+  })

package/src/DiscordGateway/Shard/identify.ts

@@ -0,0 +1,52 @@
+import * as SendEvents from "./sendEvents.js"
+import * as OS from "os"
+
+export interface Options {
+  token: string
+  intents: number
+  shard: [number, number]
+  presence?: Discord.UpdatePresence
+}
+
+export interface Requirements {
+  latestReady: Ref<Maybe<Discord.ReadyEvent>>
+  latestSequence: Ref<Maybe<number>>
+}
+
+const identify = ({ token, intents, shard, presence }: Options) =>
+  SendEvents.identify({
+    token,
+    intents,
+    properties: {
+      os: OS.platform(),
+      browser: "dfx",
+      device: "dfx",
+    },
+    shard,
+    presence,
+  })
+
+const resume = (token: string, ready: Discord.ReadyEvent, seq: number) =>
+  SendEvents.resume({
+    token,
+    session_id: ready.session_id,
+    seq,
+  })
+
+export const identifyOrResume = (
+  opts: Options,
+  ready: Ref<Maybe<Discord.ReadyEvent>>,
+  seq: Ref<Maybe<number>>,
+) =>
+  Do($ => {
+    const readyEvent = $(ready.get)
+    const seqNumber = $(seq.get)
+
+    return Maybe.struct({
+      readyEvent,
+      seqNumber,
+    }).match(
+      () => identify(opts),
+      ({ readyEvent, seqNumber }) => resume(opts.token, readyEvent, seqNumber),
+    )
+  })

package/src/DiscordGateway/Shard/invalidSession.ts

@@ -0,0 +1,10 @@
+import { Message } from "../DiscordWS.js"
+import { Reconnect } from "../WS.js"
+
+export const fromPayload = (
+  p: Discord.GatewayPayload,
+  latestReady: Ref<Maybe<Discord.ReadyEvent>>,
+) =>
+  (p.d ? Effect.unit() : latestReady.set(Maybe.none())).map(
+    (): Message => Reconnect,
+  )

package/src/DiscordGateway/Shard/sendEvents.ts

@@ -0,0 +1,37 @@
+import { Discord } from "dfx/_common"
+
+export const heartbeat = (d: Discord.Heartbeat): Discord.GatewayPayload => ({
+  op: Discord.GatewayOpcode.HEARTBEAT,
+  d,
+})
+
+export const identify = (d: Discord.Identify): Discord.GatewayPayload => ({
+  op: Discord.GatewayOpcode.IDENTIFY,
+  d,
+})
+
+export const resume = (d: Discord.Resume): Discord.GatewayPayload => ({
+  op: Discord.GatewayOpcode.RESUME,
+  d,
+})
+
+export const requestGuildMembers = (
+  d: Discord.RequestGuildMember,
+): Discord.GatewayPayload => ({
+  op: Discord.GatewayOpcode.REQUEST_GUILD_MEMBERS,
+  d,
+})
+
+export const voiceStateUpdate = (
+  d: Discord.UpdateVoiceState,
+): Discord.GatewayPayload => ({
+  op: Discord.GatewayOpcode.VOICE_STATE_UPDATE,
+  d,
+})
+
+export const presenceUpdate = (
+  d: Discord.UpdatePresence,
+): Discord.GatewayPayload => ({
+  op: Discord.GatewayOpcode.PRESENCE_UPDATE,
+  d,
+})

package/src/DiscordGateway/Shard/utils.ts

@@ -0,0 +1,14 @@
+export const opCode =
+  <R, E>(source: Stream<R, E, Discord.GatewayPayload>) =>
+  <T = any>(code: Discord.GatewayOpcode) =>
+    source.filter((p): p is Discord.GatewayPayload<T> => p.op === code)
+
+const maybeUpdateRef = <T>(
+  f: (p: Discord.GatewayPayload) => Maybe<T>,
+  ref: Ref<Maybe<T>>,
+) => flow(f, o => o.match(Effect.unit, a => ref.set(Maybe.some(a))))
+
+export const latest = <T>(f: (p: Discord.GatewayPayload) => Maybe<T>) =>
+  Ref.make<Maybe<T>>(Maybe.none()).map(
+    ref => [ref, maybeUpdateRef(f, ref)] as const,
+  )

package/src/DiscordGateway/Shard.ts

@@ -0,0 +1,152 @@
+import { DiscordConfig } from "dfx/DiscordConfig"
+import { LiveRateLimiter, RateLimiter } from "dfx/RateLimit"
+import { DiscordWS, LiveDiscordWS, Message } from "./DiscordWS.js"
+import * as Heartbeats from "./Shard/heartbeats.js"
+import * as Identify from "./Shard/identify.js"
+import * as InvalidSession from "./Shard/invalidSession.js"
+import * as Utils from "./Shard/utils.js"
+import { Reconnect } from "./WS.js"
+
+export const make = Do($ => {
+  const { token, gateway } = $(DiscordConfig)
+  const limiter = $(RateLimiter)
+  const dws = $(DiscordWS)
+
+  const connect = (
+    shard: [id: number, count: number],
+    hub: Hub<Discord.GatewayPayload<Discord.ReceiveEvent>>,
+  ) =>
+    Do($ => {
+      const outboundQueue = $(Queue.unbounded<Message>())
+      const pendingQueue = $(Queue.unbounded<Message>())
+      const connecting = $(Ref.make(true))
+      const outbound = outboundQueue
+        .take()
+        .tap(() =>
+          limiter.maybeWait("dfx.shard.send", Duration.minutes(1), 120),
+        )
+
+      const send = (p: Message) =>
+        connecting.get.flatMap(_ =>
+          _ ? pendingQueue.offer(p) : outboundQueue.offer(p),
+        )
+
+      const prioritySend = (p: Message) => outboundQueue.offer(p)
+
+      const resume = connecting
+        .set(false)
+        .zipRight(pendingQueue.takeAll())
+        .tap(_ => outboundQueue.offerAll(_)).asUnit
+
+      const onReconnect = outboundQueue
+        .takeAll()
+        .tap(_ =>
+          pendingQueue.offerAll(
+            _.filter(
+              msg =>
+                msg !== Reconnect &&
+                msg.op !== Discord.GatewayOpcode.IDENTIFY &&
+                msg.op !== Discord.GatewayOpcode.RESUME &&
+                msg.op !== Discord.GatewayOpcode.HEARTBEAT,
+            ),
+          ),
+        )
+        .zipRight(connecting.set(true))
+
+      const socket = $(dws.connect({ outbound, onReconnect }))
+
+      const [latestReady, updateLatestReady] = $(
+        Utils.latest(p =>
+          Maybe.some(p)
+            .filter(
+              (p): p is Discord.GatewayPayload<Discord.ReadyEvent> =>
+                p.op === Discord.GatewayOpcode.DISPATCH && p.t === "READY",
+            )
+            .map(p => p.d!),
+        ),
+      )
+      const [latestSequence, updateLatestSequence] = $(
+        Utils.latest(p => Maybe.fromNullable(p.s)),
+      )
+      const maybeUpdateUrl = (p: Discord.GatewayPayload) =>
+        Maybe.some(p)
+          .filter(
+            (p): p is Discord.GatewayPayload<Discord.ReadyEvent> =>
+              p.op === Discord.GatewayOpcode.DISPATCH && p.t === "READY",
+          )
+          .map(p => p.d!)
+          .match(
+            () => Effect.unit(),
+            a => socket.setUrl(a.resume_gateway_url),
+          )
+
+      const hellos = $(Queue.unbounded<Discord.GatewayPayload>())
+      const acks = $(Queue.unbounded<Discord.GatewayPayload>())
+
+      // heartbeats
+      const heartbeats = Heartbeats.send(hellos, acks, latestSequence, send)
+
+      // identify
+      const identify = Identify.identifyOrResume(
+        {
+          token: token.value,
+          shard,
+          intents: gateway.intents,
+          presence: gateway.presence,
+        },
+        latestReady,
+        latestSequence,
+      )
+
+      const onPayload = (p: Discord.GatewayPayload) =>
+        Do($ => {
+          $(
+            updateLatestReady(p)
+              .zipPar(updateLatestSequence(p))
+              .zipPar(maybeUpdateUrl(p)),
+          )
+
+          let effect = Effect.unit()
+
+          switch (p.op) {
+            case Discord.GatewayOpcode.HELLO:
+              effect = identify.tap(prioritySend).zipPar(hellos.offer(p))
+              break
+            case Discord.GatewayOpcode.HEARTBEAT_ACK:
+              effect = acks.offer(p)
+              break
+            case Discord.GatewayOpcode.INVALID_SESSION:
+              effect = InvalidSession.fromPayload(p, latestReady).tap(send)
+              break
+            case Discord.GatewayOpcode.DISPATCH:
+              if (p.t === "READY" || p.t === "RESUMED") {
+                effect = resume.zipRight(hub.publish(p))
+              } else {
+                effect = hub.publish(p)
+              }
+              break
+          }
+
+          $(effect)
+        })
+
+      const run = socket.take
+        .flatMap(onPayload)
+        .forever.zipParLeft(heartbeats)
+        .zipParLeft(socket.run)
+
+      return {
+        run,
+        connected: connecting.get,
+        send: (p: Discord.GatewayPayload) => send(p),
+        reconnect: send(Reconnect),
+      } as const
+    })
+
+  return { connect } as const
+})
+
+export interface Shard extends Effect.Success<typeof make> {}
+export const Shard = Tag<Shard>()
+export const LiveShard =
+  (LiveDiscordWS + LiveRateLimiter) >> make.toLayer(Shard)

package/src/DiscordGateway/ShardStore.ts

@@ -0,0 +1,33 @@
+export interface ClaimIdContext {
+  sharderCount: number
+  totalCount: number
+}
+
+export interface ShardStore {
+  claimId: (ctx: ClaimIdContext) => Effect<never, never, Maybe<number>>
+  allClaimed: (totalCount: number) => Effect<never, never, boolean>
+  heartbeat?: (shardId: number) => Effect<never, never, void>
+}
+export const ShardStore = Tag<ShardStore>()
+
+// Very basic shard id store, that does no health checks
+const memoryStore = (): ShardStore => {
+  let currentId = 0
+
+  return {
+    claimId: ({ totalCount }) =>
+      Effect.sync(() => {
+        if (currentId >= totalCount) {
+          return Maybe.none()
+        }
+
+        const id = currentId
+        currentId++
+        return Maybe.some(id)
+      }),
+
+    allClaimed: totalCount => Effect.sync(() => currentId >= totalCount),
+  }
+}
+
+export const LiveMemoryShardStore = Layer.sync(ShardStore, memoryStore)