dfx 0.73.0 → 0.75.0

package/src/Cache/prelude.ts

@@ -1,4 +1,5 @@
 import * as Effect from "effect/Effect"
+import type * as Scope from "effect/Scope"
 import * as Stream from "effect/Stream"
 import type { DiscordRESTError } from "dfx/DiscordREST"
 import { DiscordREST } from "dfx/DiscordREST"
@@ -138,41 +139,42 @@ export const ops = <E, T>({
 export const guilds = <RM, EM, E>(
   makeDriver: Effect.Effect<RM, EM, CacheDriver<E, Discord.Guild>>,
 ): Effect.Effect<
-  RM | DiscordGateway | DiscordREST,
+  RM | DiscordGateway | DiscordREST | Scope.Scope,
   EM,
-  Cache<never, E, ResponseError | DiscordRESTError, Discord.Guild>
+  Cache<E, ResponseError | DiscordRESTError, Discord.Guild>
 > =>
   Effect.gen(function* (_) {
     const driver = yield* _(makeDriver)
     const gateway = yield* _(DiscordGateway)
     const rest = yield* _(DiscordREST)
 
-    return make({
-      driver,
-      id: _ => _.id,
-      ops: ops({
-        id: (g: Discord.Guild) => g.id,
-        create: Stream.map(gateway.fromDispatch("GUILD_CREATE"), g => ({
-          ...g,
-          channels: [],
-          roles: [],
-          emojis: [],
-          members: [],
-        })),
-        update: gateway.fromDispatch("GUILD_UPDATE"),
-        remove: Stream.map(gateway.fromDispatch("GUILD_DELETE"), a => a.id),
+    return yield* _(
+      make({
+        driver,
+        id: _ => _.id,
+        ops: ops({
+          id: (g: Discord.Guild) => g.id,
+          create: Stream.map(gateway.fromDispatch("GUILD_CREATE"), g => ({
+            ...g,
+            channels: [],
+            roles: [],
+            emojis: [],
+            members: [],
+          })),
+          update: gateway.fromDispatch("GUILD_UPDATE"),
+          remove: Stream.map(gateway.fromDispatch("GUILD_DELETE"), a => a.id),
+        }),
+        onMiss: id => Effect.flatMap(rest.getGuild(id), r => r.json),
       }),
-      onMiss: id => Effect.flatMap(rest.getGuild(id), r => r.json),
-    })
+    )
   })
 
 export const channels = <RM, EM, E>(
   makeDriver: Effect.Effect<RM, EM, ParentCacheDriver<E, Discord.Channel>>,
 ): Effect.Effect<
-  DiscordGateway | DiscordREST | RM,
+  DiscordGateway | DiscordREST | RM | Scope.Scope,
   EM,
   ParentCache<
-    never,
     E,
     ResponseError | DiscordRESTError,
     ResponseError | DiscordRESTError,
@@ -184,90 +186,88 @@ export const channels = <RM, EM, E>(
     const gateway = yield* _(DiscordGateway)
     const rest = yield* _(DiscordREST)
 
-    return makeWithParent({
-      driver,
-      id: _ => Effect.succeed([_.guild_id!, _.id]),
-      ops: opsWithParent({
-        id: (a: Discord.Channel) => a.id,
-        fromParent: Stream.map(gateway.fromDispatch("GUILD_CREATE"), g => [
-          g.id,
-          g.channels.concat(g.threads),
-        ]),
-        create: Stream.merge(
-          gateway.fromDispatch("CHANNEL_CREATE"),
-          gateway.fromDispatch("THREAD_CREATE"),
-        ).pipe(Stream.map(c => [c.guild_id!, c])),
-        update: Stream.merge(
-          gateway.fromDispatch("CHANNEL_UPDATE"),
-          gateway.fromDispatch("THREAD_UPDATE"),
-        ).pipe(Stream.map(c => [c.guild_id!, c])),
-        remove: Stream.merge(
-          gateway.fromDispatch("CHANNEL_DELETE"),
-          gateway.fromDispatch("THREAD_DELETE"),
-        ).pipe(Stream.map(a => [a.guild_id!, a.id])),
-        parentRemove: Stream.map(
-          gateway.fromDispatch("GUILD_DELETE"),
-          g => g.id,
-        ),
+    return yield* _(
+      makeWithParent({
+        driver,
+        id: _ => Effect.succeed([_.guild_id!, _.id]),
+        ops: opsWithParent({
+          id: (a: Discord.Channel) => a.id,
+          fromParent: Stream.map(gateway.fromDispatch("GUILD_CREATE"), g => [
+            g.id,
+            g.channels.concat(g.threads),
+          ]),
+          create: Stream.merge(
+            gateway.fromDispatch("CHANNEL_CREATE"),
+            gateway.fromDispatch("THREAD_CREATE"),
+          ).pipe(Stream.map(c => [c.guild_id!, c])),
+          update: Stream.merge(
+            gateway.fromDispatch("CHANNEL_UPDATE"),
+            gateway.fromDispatch("THREAD_UPDATE"),
+          ).pipe(Stream.map(c => [c.guild_id!, c])),
+          remove: Stream.merge(
+            gateway.fromDispatch("CHANNEL_DELETE"),
+            gateway.fromDispatch("THREAD_DELETE"),
+          ).pipe(Stream.map(a => [a.guild_id!, a.id])),
+          parentRemove: Stream.map(
+            gateway.fromDispatch("GUILD_DELETE"),
+            g => g.id,
+          ),
+        }),
+        onMiss: (_, id) => Effect.flatMap(rest.getChannel(id), r => r.json),
+        onParentMiss: guildId =>
+          rest.getGuildChannels(guildId).pipe(
+            Effect.flatMap(r => r.json),
+            Effect.map(a => a.map(a => [a.id, a])),
+          ),
       }),
-      onMiss: (_, id) => Effect.flatMap(rest.getChannel(id), r => r.json),
-      onParentMiss: guildId =>
-        rest.getGuildChannels(guildId).pipe(
-          Effect.flatMap(r => r.json),
-          Effect.map(a => a.map(a => [a.id, a])),
-        ),
-    })
+    )
   })
 
 export const roles = <RM, EM, E>(
   makeDriver: Effect.Effect<RM, EM, ParentCacheDriver<E, Discord.Role>>,
 ): Effect.Effect<
-  DiscordGateway | DiscordREST | RM,
+  DiscordGateway | DiscordREST | RM | Scope.Scope,
   EM,
-  ParentCache<
-    never,
-    E,
-    CacheMissError,
-    ResponseError | DiscordRESTError,
-    Discord.Role
-  >
+  ParentCache<E, CacheMissError, ResponseError | DiscordRESTError, Discord.Role>
 > =>
   Effect.gen(function* (_) {
     const driver = yield* _(makeDriver)
     const gateway = yield* _(DiscordGateway)
     const rest = yield* _(DiscordREST)
 
-    return makeWithParent({
-      driver,
-      id: _ => Effect.fail(new CacheMissError("RolesCache/id", _.id)),
-      ops: opsWithParent({
-        id: (a: Discord.Role) => a.id,
-        fromParent: Stream.map(gateway.fromDispatch("GUILD_CREATE"), g => [
-          g.id,
-          g.roles,
-        ]),
-        create: Stream.map(gateway.fromDispatch("GUILD_ROLE_CREATE"), r => [
-          r.guild_id,
-          r.role,
-        ]),
-        update: Stream.map(gateway.fromDispatch("GUILD_ROLE_UPDATE"), r => [
-          r.guild_id,
-          r.role,
-        ]),
-        remove: Stream.map(gateway.fromDispatch("GUILD_ROLE_DELETE"), r => [
-          r.guild_id,
-          r.role_id,
-        ]),
-        parentRemove: Stream.map(
-          gateway.fromDispatch("GUILD_DELETE"),
-          g => g.id,
-        ),
+    return yield* _(
+      makeWithParent({
+        driver,
+        id: _ => Effect.fail(new CacheMissError("RolesCache/id", _.id)),
+        ops: opsWithParent({
+          id: (a: Discord.Role) => a.id,
+          fromParent: Stream.map(gateway.fromDispatch("GUILD_CREATE"), g => [
+            g.id,
+            g.roles,
+          ]),
+          create: Stream.map(gateway.fromDispatch("GUILD_ROLE_CREATE"), r => [
+            r.guild_id,
+            r.role,
+          ]),
+          update: Stream.map(gateway.fromDispatch("GUILD_ROLE_UPDATE"), r => [
+            r.guild_id,
+            r.role,
+          ]),
+          remove: Stream.map(gateway.fromDispatch("GUILD_ROLE_DELETE"), r => [
+            r.guild_id,
+            r.role_id,
+          ]),
+          parentRemove: Stream.map(
+            gateway.fromDispatch("GUILD_DELETE"),
+            g => g.id,
+          ),
+        }),
+        onMiss: (_, id) => Effect.fail(new CacheMissError("RolesCache", id)),
+        onParentMiss: guildId =>
+          rest.getGuildRoles(guildId).pipe(
+            Effect.flatMap(r => r.json),
+            Effect.map(_ => _.map(role => [role.id, role])),
+          ),
       }),
-      onMiss: (_, id) => Effect.fail(new CacheMissError("RolesCache", id)),
-      onParentMiss: guildId =>
-        rest.getGuildRoles(guildId).pipe(
-          Effect.flatMap(r => r.json),
-          Effect.map(_ => _.map(role => [role.id, role])),
-        ),
-    })
+    )
   })

package/src/Cache.ts

@@ -1,4 +1,6 @@
 import * as Option from "effect/Option"
+import * as Schedule from "effect/Schedule"
+import type * as Scope from "effect/Scope"
 import * as Effect from "effect/Effect"
 import * as Stream from "effect/Stream"
 import type { CacheDriver, ParentCacheDriver } from "dfx/Cache/driver"
@@ -24,7 +26,11 @@ export type CacheOp<T> =
   | { op: "update"; resourceId: string; resource: T }
   | { op: "delete"; resourceId: string }
 
-export interface ParentCache<EOps, EDriver, EMiss, EPMiss, A> {
+const retryPolicy = Schedule.exponential("500 millis").pipe(
+  Schedule.union(Schedule.spaced("10 seconds")),
+)
+
+export interface ParentCache<EDriver, EMiss, EPMiss, A> {
   readonly get: (
     parentId: string,
     id: string,
@@ -38,7 +44,6 @@ export interface ParentCache<EOps, EDriver, EMiss, EPMiss, A> {
   readonly getForParent: (
     parentId: string,
   ) => Effect.Effect<never, EDriver | EPMiss, ReadonlyMap<string, A>>
-  readonly run: Effect.Effect<never, EOps | EDriver, void>
   readonly size: Effect.Effect<never, EDriver, number>
   readonly sizeForParent: (
     parentId: string,
@@ -77,86 +82,101 @@ export const makeWithParent = <EOps, EDriver, EMiss, EPMiss, A>({
   onParentMiss: (
     parentId: string,
   ) => Effect.Effect<never, EPMiss, Array<[id: string, resource: A]>>
-}): ParentCache<EOps, EDriver, EMiss, EPMiss, A> => {
-  const sync = Stream.runDrain(
-    Stream.tap(ops, (op): Effect.Effect<never, EDriver, void> => {
-      switch (op.op) {
-        case "create":
-        case "update":
-          return driver.set(op.parentId, op.resourceId, op.resource)
-
-        case "delete":
-          return driver.delete(op.parentId, op.resourceId)
-
-        case "parentDelete":
-          return driver.parentDelete(op.parentId)
-      }
-    }),
-  )
+}): Effect.Effect<Scope.Scope, never, ParentCache<EDriver, EMiss, EPMiss, A>> =>
+  Effect.gen(function* (_) {
+    yield* _(
+      Stream.runDrain(
+        Stream.tap(ops, (op): Effect.Effect<never, EDriver, void> => {
+          switch (op.op) {
+            case "create":
+            case "update":
+              return driver.set(op.parentId, op.resourceId, op.resource)
 
-  const get = (parentId: string, id: string) =>
-    Effect.flatMap(
-      driver.get(parentId, id),
-      Option.match({
-        onNone: () =>
-          Effect.tap(onMiss(parentId, id), a => driver.set(parentId, id, a)),
-        onSome: Effect.succeed,
-      }),
-    )
-
-  const put = (_: A) =>
-    Effect.flatMap(id(_), ([parentId, id]) => driver.set(parentId, id, _))
+            case "delete":
+              return driver.delete(op.parentId, op.resourceId)
 
-  const update = <R, E>(
-    parentId: string,
-    id: string,
-    f: (_: A) => Effect.Effect<R, E, A>,
-  ) =>
-    get(parentId, id).pipe(
-      Effect.flatMap(f),
-      Effect.tap(_ => driver.set(parentId, id, _)),
+            case "parentDelete":
+              return driver.parentDelete(op.parentId)
+          }
+        }),
+      ),
+      Effect.tapErrorCause(_ => Effect.logError("ops error, restarting", _)),
+      Effect.retry(retryPolicy),
+      Effect.forkScoped,
+    )
+    yield* _(
+      driver.run,
+      Effect.tapErrorCause(_ =>
+        Effect.logError("cache driver error, restarting", _),
+      ),
+      Effect.retry(retryPolicy),
+      Effect.forkScoped,
     )
 
-  return {
-    ...driver,
-
-    get,
-    put,
-    update,
-
-    getForParent: (parentId: string) =>
+    const get = (parentId: string, id: string) =>
       Effect.flatMap(
-        driver.getForParent(parentId),
+        driver.get(parentId, id),
         Option.match({
           onNone: () =>
-            onParentMiss(parentId).pipe(
-              Effect.tap(entries =>
-                Effect.all(
-                  entries.map(([id, a]) => driver.set(parentId, id, a)),
-                  { concurrency: "unbounded" },
-                ),
-              ),
-              Effect.map(entries => new Map(entries) as ReadonlyMap<string, A>),
-            ),
+            Effect.tap(onMiss(parentId, id), a => driver.set(parentId, id, a)),
           onSome: Effect.succeed,
         }),
-      ),
+      )
+
+    const put = (_: A) =>
+      Effect.flatMap(id(_), ([parentId, id]) => driver.set(parentId, id, _))
+
+    const update = <R, E>(
+      parentId: string,
+      id: string,
+      f: (_: A) => Effect.Effect<R, E, A>,
+    ) =>
+      get(parentId, id).pipe(
+        Effect.flatMap(f),
+        Effect.tap(_ => driver.set(parentId, id, _)),
+      )
+
+    return {
+      ...driver,
+
+      get,
+      put,
+      update,
 
-    run: Effect.all([sync, driver.run], {
-      concurrency: "unbounded",
-      discard: true,
+      getForParent: (parentId: string) =>
+        Effect.flatMap(
+          driver.getForParent(parentId),
+          Option.match({
+            onNone: () =>
+              onParentMiss(parentId).pipe(
+                Effect.tap(entries =>
+                  Effect.all(
+                    entries.map(([id, a]) => driver.set(parentId, id, a)),
+                    { concurrency: "unbounded" },
+                  ),
+                ),
+                Effect.map(
+                  entries => new Map(entries) as ReadonlyMap<string, A>,
+                ),
+              ),
+            onSome: Effect.succeed,
+          }),
+        ),
+    } as const
+  }).pipe(
+    Effect.annotateLogs({
+      package: "dfx",
+      service: "Cache",
     }),
-  } as const
-}
+  )
 
-export interface Cache<EOps, EDriver, EMiss, A> {
+export interface Cache<EDriver, EMiss, A> {
   readonly get: (id: string) => Effect.Effect<never, EDriver | EMiss, A>
   readonly put: (_: A) => Effect.Effect<never, EDriver, void>
   readonly update: <R, E>(
     id: string,
     f: (_: A) => Effect.Effect<R, E, A>,
   ) => Effect.Effect<R, EDriver | EMiss | E, A>
-  readonly run: Effect.Effect<never, EOps | EDriver, void>
   readonly size: Effect.Effect<never, EDriver, number>
   readonly set: (
     resourceId: string,
@@ -178,48 +198,64 @@ export const make = <EOps, EDriver, EMiss, A>({
   ops?: Stream.Stream<never, EOps, CacheOp<A>>
   id: (_: A) => string
   onMiss: (id: string) => Effect.Effect<never, EMiss, A>
-}): Cache<EOps, EDriver, EMiss, A> => {
-  const sync = Stream.runDrain(
-    Stream.tap(ops, (op): Effect.Effect<never, EDriver, void> => {
-      switch (op.op) {
-        case "create":
-        case "update":
-          return driver.set(op.resourceId, op.resource)
-
-        case "delete":
-          return driver.delete(op.resourceId)
-      }
-    }),
-  )
+}): Effect.Effect<Scope.Scope, never, Cache<EDriver, EMiss, A>> =>
+  Effect.gen(function* (_) {
+    yield* _(
+      Stream.runDrain(
+        Stream.tap(ops, (op): Effect.Effect<never, EDriver, void> => {
+          switch (op.op) {
+            case "create":
+            case "update":
+              return driver.set(op.resourceId, op.resource)
 
-  const get = (id: string) =>
-    Effect.flatMap(
-      driver.get(id),
-      Option.match({
-        onNone: () => Effect.tap(onMiss(id), a => driver.set(id, a)),
-        onSome: Effect.succeed,
-      }),
+            case "delete":
+              return driver.delete(op.resourceId)
+          }
+        }),
+      ),
+      Effect.tapErrorCause(_ => Effect.logError("ops error, restarting", _)),
+      Effect.retry(retryPolicy),
+      Effect.forkScoped,
     )
 
-  const put = (_: A) => driver.set(id(_), _)
-
-  const update = <R, E>(id: string, f: (_: A) => Effect.Effect<R, E, A>) =>
-    get(id).pipe(
-      Effect.flatMap(f),
-      Effect.tap(_ => driver.set(id, _)),
+    yield* _(
+      driver.run,
+      Effect.tapErrorCause(_ =>
+        Effect.logError("cache driver error, restarting", _),
+      ),
+      Effect.retry(retryPolicy),
+      Effect.forkScoped,
     )
 
-  return {
-    ...driver,
-    get,
-    put,
-    update,
-    run: Effect.all([sync, driver.run], {
-      concurrency: "unbounded",
-      discard: true,
+    const get = (id: string) =>
+      Effect.flatMap(
+        driver.get(id),
+        Option.match({
+          onNone: () => Effect.tap(onMiss(id), a => driver.set(id, a)),
+          onSome: Effect.succeed,
+        }),
+      )
+
+    const put = (_: A) => driver.set(id(_), _)
+
+    const update = <R, E>(id: string, f: (_: A) => Effect.Effect<R, E, A>) =>
+      get(id).pipe(
+        Effect.flatMap(f),
+        Effect.tap(_ => driver.set(id, _)),
+      )
+
+    return {
+      ...driver,
+      get,
+      put,
+      update,
+    } as const
+  }).pipe(
+    Effect.annotateLogs({
+      package: "dfx",
+      service: "Cache",
     }),
-  } as const
-}
+  )
 
 export class CacheMissError {
   readonly _tag = "CacheMissError"

package/src/DiscordGateway.ts

@@ -7,9 +7,9 @@ import * as Queue from "effect/Queue"
 import * as Stream from "effect/Stream"
 import type { RunningShard } from "dfx/DiscordGateway/Shard"
 import { LiveSharder, Sharder } from "dfx/DiscordGateway/Sharder"
-import type { WebSocketCloseError, WebSocketError } from "dfx/DiscordGateway/WS"
 import type * as Discord from "dfx/types"
 import * as EffectUtils from "dfx/utils/Effect"
+import * as Schedule from "effect/Schedule"
 
 const fromDispatchFactory =
   <R, E>(
@@ -33,15 +33,10 @@ const handleDispatchFactory =
       if (_.t === event) {
         return handle(_.d as any)
       }
-      return Effect.unit
+      return Effect.unit as any
     })
 
 export interface DiscordGateway {
-  readonly run: Effect.Effect<
-    never,
-    WebSocketError | WebSocketCloseError,
-    never
-  >
   readonly dispatch: Stream.Stream<
     never,
     never,
@@ -77,19 +72,32 @@ export const make = Effect.gen(function* (_) {
   const fromDispatch = fromDispatchFactory(dispatch)
   const handleDispatch = handleDispatchFactory(hub)
 
-  const run = sharder.run(hub, sendQueue)
+  yield* _(
+    sharder.run(hub, sendQueue),
+    Effect.tapErrorCause(_ => Effect.logError("fatal error, restarting", _)),
+    Effect.retry(
+      Schedule.exponential("1 seconds").pipe(
+        Schedule.union(Schedule.spaced("30 seconds")),
+      ),
+    ),
+    Effect.forkScoped,
+  )
 
   return DiscordGateway.of({
-    run,
     dispatch,
     fromDispatch,
     handleDispatch,
     send,
     shards: sharder.shards,
   })
-})
+}).pipe(
+  Effect.annotateLogs({
+    package: "dfx",
+    service: "DiscordGateway",
+  }),
+)
 
 export const LiveDiscordGateway = Layer.provide(
   LiveSharder,
-  Layer.effect(DiscordGateway, make),
+  Layer.scoped(DiscordGateway, make),
 )