@effect-app/infra 1.40.0 → 1.41.0

package/src/services/MainFiberSet.ts

@@ -0,0 +1,70 @@
+import { Context, Effect, Fiber, FiberSet, Layer } from "@effect-app/core"
+import type {} from "effect/Scope"
+import type {} from "effect/Context"
+import { InfraLogger } from "../logger.js"
+import { reportNonInterruptedFailureCause } from "./QueueMaker/errors.js"
+import { setRootParentSpan } from "./RequestFiberSet.js"
+
+const make = Effect.gen(function*() {
+  const set = yield* FiberSet.make<unknown, never>()
+  const add = (...fibers: Fiber.RuntimeFiber<never, never>[]) =>
+    Effect.sync(() => fibers.forEach((_) => FiberSet.unsafeAdd(set, _)))
+  const addAll = (fibers: readonly Fiber.RuntimeFiber<never, never>[]) =>
+    Effect.sync(() => fibers.forEach((_) => FiberSet.unsafeAdd(set, _)))
+  const join = FiberSet.size(set).pipe(
+    Effect.andThen((count) => InfraLogger.logDebug(`Joining ${count} current fibers on the MainFiberSet`)),
+    Effect.andThen(FiberSet.join(set))
+  )
+  const run = FiberSet.run(set)
+
+  // const waitUntilEmpty = Effect.gen(function*() {
+  //   const currentSize = yield* FiberSet.size(set)
+  //   if (currentSize === 0) {
+  //     return
+  //   }
+  //   yield* InfraLogger.logInfo("Waiting MainFiberSet to be empty: " + currentSize)
+  //   while ((yield* FiberSet.size(set)) > 0) yield* Effect.sleep("250 millis")
+  //   yield* InfraLogger.logDebug("MainFiberSet is empty")
+  // })
+
+  // TODO: loop and interrupt all fibers in the set continuously?
+  const interrupt = Fiber.interruptAll(set)
+
+  /**
+   * Forks the effect into a new fiber attached to the MainFiberSet scope. Because the
+   * new fiber isn't attached to the parent, when the fiber executing the
+   * returned effect terminates, the forked fiber will continue running.
+   * The fiber will be interrupted when the MainFiberSet scope is closed.
+   *
+   * The parent span is set to the root span of the current fiber.
+   * Reports and then swallows errors.
+   */
+  function fork<A, E, R>(self: Effect<A, E, R>) {
+    return self.pipe(
+      Effect.asVoid,
+      Effect.catchAllCause(reportNonInterruptedFailureCause({})),
+      setRootParentSpan,
+      Effect.uninterruptible,
+      run
+    )
+  }
+  return {
+    interrupt,
+    join,
+    fork,
+    run,
+    add,
+    addAll
+  }
+})
+
+/**
+ * Whenever you fork long running (e.g worker) fibers via e.g `Effect.forkScoped` or `Effect.forkDaemon`
+ * you should register these long running fibers in a FiberSet, and join them at the end of your main program.
+ * This way any errors will blow up the main program instead of fibers dying unknowingly.
+ */
+export class MainFiberSet extends Context.TagMakeId("MainFiberSet", make)<MainFiberSet>() {
+  static readonly Live = this.toLayerScoped()
+  static readonly JoinLive = this.pipe(Effect.andThen((_) => _.join), Layer.effectDiscard, Layer.provide(this.Live))
+  static readonly run = <A, R>(self: Effect<A, never, R>) => this.use((_) => _.run(self))
+}

package/src/services/Operations.ts

@@ -1,6 +1,5 @@
 import { annotateLogscoped } from "@effect-app/core/Effect"
 import { dual, pipe } from "@effect-app/core/Function"
-import { RequestFiberSet } from "@effect-app/infra-adapters/RequestFiberSet"
 import { reportError } from "@effect-app/infra/errorReporter"
 import { NonEmptyString2k } from "@effect-app/schema"
 import { subHours } from "date-fns"
@@ -8,12 +7,12 @@ import type { Fiber } from "effect-app"
 import { Cause, Context, copy, Duration, Effect, Exit, Layer, Option, S, Schedule } from "effect-app"
 import type { OperationProgress } from "effect-app/Operations"
 import { Failure, Operation, OperationId, Success } from "effect-app/Operations"
-import { MainFiberSet } from "effect-app/services/MainFiberSet"
 import * as Scope from "effect/Scope"
-import { forkDaemonReportRequestUnexpected } from "../api/reportError.js"
 import { batch } from "../rateLimit.js"
+import { MainFiberSet } from "./MainFiberSet.js"
 import { OperationsRepo } from "./OperationsRepo.js"
 import { where } from "./query.js"
+import { RequestFiberSet } from "./RequestFiberSet.js"
 
 const reportAppError = reportError("Operations.Cleanup")
 
@@ -97,10 +96,11 @@ const make = Effect.gen(function*() {
             .pipe(
               Scope.extend(scope),
               Effect.flatMap((id) =>
-                forkDaemonReportRequestUnexpected(Scope.use(
-                  self(id).pipe(Effect.withSpan(title)),
-                  scope
-                ))
+                reqFiberSet
+                  .forkDaemonReportRequestUnexpected(Scope.use(
+                    self(id).pipe(Effect.withSpan(title)),
+                    scope
+                  ))
                   .pipe(Effect.map((fiber): RunningOperation<A, E> => ({ fiber, id })))
               ),
               Effect.tap(({ id }) =>
@@ -133,21 +133,58 @@ const make = Effect.gen(function*() {
               Scope.extend(scope),
               Effect
                 .flatMap((id) =>
-                  forkDaemonReportRequestUnexpected(Scope.use(
-                    self(id).pipe(Effect.withSpan(title)),
-                    scope
-                  ))
+                  reqFiberSet
+                    .forkDaemonReportRequestUnexpected(Scope.use(
+                      self(id).pipe(Effect.withSpan(title)),
+                      scope
+                    ))
                     .pipe(Effect.map((fiber): RunningOperation<A, E> => ({ fiber, id })))
                 )
             )
       )
   )
 
+  const forkOperation: {
+    (title: NonEmptyString2k): <R, E, A>(
+      self: Effect<A, E, R>
+    ) => Effect<RunningOperation<A, E>, never, Exclude<R, Scope.Scope>>
+    <R, E, A>(
+      self: Effect<A, E, R>,
+      title: NonEmptyString2k
+    ): Effect<RunningOperation<A, E>, never, Exclude<R, Scope.Scope>>
+  } = dual(
+    2,
+    <R, E, A>(self: Effect<A, E, R>, title: NonEmptyString2k) =>
+      Effect.flatMap(
+        Scope.make(),
+        (scope) =>
+          register(title)
+            .pipe(
+              Scope.extend(scope),
+              Effect
+                .flatMap((id) =>
+                  reqFiberSet
+                    .forkDaemonReportRequestUnexpected(Scope.use(
+                      self.pipe(Effect.withSpan(title)),
+                      scope
+                    ))
+                    .pipe(Effect.map((fiber): RunningOperation<A, E> => ({ fiber, id })))
+                )
+            )
+      )
+  )
+
+  function forkOperationFunction<R, E, A, Inp>(fnc: (inp: Inp) => Effect<A, E, R>, title: NonEmptyString2k) {
+    return (inp: Inp) => fnc(inp).pipe((_) => forkOperation(_, title))
+  }
+
   return {
     cleanup,
     register,
     fork,
     fork2,
+    forkOperation,
+    forkOperationFunction,
     all: repo.all,
     find: findOp,
     update
@@ -181,37 +218,3 @@ export interface RunningOperation<A, E> {
   id: OperationId
   fiber: Fiber.RuntimeFiber<A, E>
 }
-
-export const forkOperation: {
-  (title: NonEmptyString2k): <R, E, A>(
-    self: Effect<A, E, R>
-  ) => Effect<RunningOperation<A, E>, never, Operations | Exclude<R, Scope.Scope>>
-  <R, E, A>(
-    self: Effect<A, E, R>,
-    title: NonEmptyString2k
-  ): Effect<RunningOperation<A, E>, never, Operations | Exclude<R, Scope.Scope>>
-} = dual(
-  2,
-  <R, E, A>(self: Effect<A, E, R>, title: NonEmptyString2k) =>
-    Effect.flatMap(
-      Scope.make(),
-      (scope) =>
-        Operations
-          .register(title)
-          .pipe(
-            Scope.extend(scope),
-            Effect
-              .flatMap((id) =>
-                forkDaemonReportRequestUnexpected(Scope.use(
-                  self.pipe(Effect.withSpan(title)),
-                  scope
-                ))
-                  .pipe(Effect.map((fiber): RunningOperation<A, E> => ({ fiber, id })))
-              )
-          )
-    )
-)
-
-export function forkOperationFunction<R, E, A, Inp>(fnc: (inp: Inp) => Effect<A, E, R>, title: NonEmptyString2k) {
-  return (inp: Inp) => fnc(inp).pipe((_) => forkOperation(_, title))
-}

package/src/services/QueueMaker/errors.ts

@@ -1,38 +1,11 @@
-import { setRootParentSpan } from "@effect-app/infra-adapters/RequestFiberSet"
 import { reportError } from "@effect-app/infra/errorReporter"
 import { Cause, Effect, Exit } from "effect-app"
-import { MainFiberSet } from "effect-app/services/MainFiberSet"
 
 const reportQueueError_ = reportError("Queue")
 
 export const reportQueueError = <E>(cause: Cause<E>, extras?: Record<string, unknown>) =>
   reportQueueError_(cause, extras)
 
-/**
- * Forks the effect into a new fiber attached to the MainFiberSet scope. Because the
- * new fiber isn't attached to the parent, when the fiber executing the
- * returned effect terminates, the forked fiber will continue running.
- * The fiber will be interrupted when the MainFiberSet scope is closed.
- *
- * The parent span is set to the root span of the current fiber.
- * Reports and then swallows errors.
- *
- * @tsplus getter effect/io/Effect forkDaemonReportQueue
- */
-export function forkDaemonReportQueue<A, E, R>(self: Effect<A, E, R>) {
-  return self.pipe(
-    Effect.asVoid,
-    Effect.catchAllCause(reportNonInterruptedFailureCause({})),
-    setRootParentSpan,
-    Effect.uninterruptible,
-    MainFiberSet.run
-  )
-}
-
-export const reportFatalQueueError = reportError(
-  "FatalQueue"
-)
-
 export function reportNonInterruptedFailure(context?: Record<string, unknown>) {
   const report = reportNonInterruptedFailureCause(context)
   return <A, E, R>(inp: Effect<A, E, R>): Effect<Exit<A, E>, never, R> =>

package/src/services/RequestFiberSet.ts

@@ -0,0 +1,104 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import type { Tracer } from "@effect-app/core"
+import { Context, Effect, Fiber, FiberSet, Option } from "@effect-app/core"
+import { reportRequestError, reportUnknownRequestError } from "../api/reportError.js"
+import { InfraLogger } from "../logger.js"
+
+const getRootParentSpan = Effect.gen(function*() {
+  let span: Tracer.AnySpan | null = yield* Effect.currentSpan.pipe(
+    Effect.catchTag("NoSuchElementException", () => Effect.succeed(null))
+  )
+  if (!span) return span
+  while (span._tag === "Span" && Option.isSome(span.parent)) {
+    span = span.parent.value
+  }
+  return span
+})
+
+export const setRootParentSpan = <A, E, R>(self: Effect<A, E, R>) =>
+  getRootParentSpan.pipe(Effect.andThen((span) => span ? Effect.withParentSpan(self, span) : self))
+
+const make = Effect.gen(function*() {
+  const set = yield* FiberSet.make<any, any>()
+  const add = (...fibers: Fiber.RuntimeFiber<any, any>[]) =>
+    Effect.sync(() => fibers.forEach((_) => FiberSet.unsafeAdd(set, _)))
+  const addAll = (fibers: readonly Fiber.RuntimeFiber<any, any>[]) =>
+    Effect.sync(() => fibers.forEach((_) => FiberSet.unsafeAdd(set, _)))
+  const join = FiberSet.size(set).pipe(
+    Effect.andThen((count) => InfraLogger.logInfo(`Joining ${count} current fibers on the RequestFiberSet`)),
+    Effect.andThen(FiberSet.join(set))
+  )
+  const run = FiberSet.run(set)
+  const register = <A, E, R>(self: Effect<A, E, R>) =>
+    self.pipe(Effect.fork, Effect.tap(add), Effect.andThen(Fiber.join))
+
+  // const waitUntilEmpty = Effect.gen(function*() {
+  //   const currentSize = yield* FiberSet.size(set)
+  //   if (currentSize === 0) {
+  //     return
+  //   }
+  //   yield* Effect.logInfo("Waiting RequestFiberSet to be empty: " + currentSize)
+  //   while ((yield* FiberSet.size(set)) > 0) yield* Effect.sleep("250 millis")
+  //   yield* Effect.logDebug("RequestFiberSet is empty")
+  // })
+  // TODO: loop and interrupt all fibers in the set continuously?
+  const interrupt = Fiber.interruptAll(set)
+
+  /**
+   * Forks the effect into a new fiber attached to the RequestFiberSet scope. Because the
+   * new fiber isn't attached to the parent, when the fiber executing the
+   * returned effect terminates, the forked fiber will continue running.
+   * The fiber will be interrupted when the RequestFiberSet scope is closed.
+   *
+   * The parent span is set to the root span of the current fiber.
+   * Reports errors.
+   */
+  function forkDaemonReportRequest<R, E, A>(self: Effect<A, E, R>) {
+    return self.pipe(
+      reportRequestError,
+      setRootParentSpan,
+      Effect.uninterruptible,
+      run
+    )
+  }
+
+  /**
+   * Forks the effect into a new fiber attached to the RequestFiberSet scope. Because the
+   * new fiber isn't attached to the parent, when the fiber executing the
+   * returned effect terminates, the forked fiber will continue running.
+   * The fiber will be interrupted when the RequestFiberSet scope is closed.
+   *
+   * The parent span is set to the root span of the current fiber.
+   * Reports unexpected errors.
+   */
+  function forkDaemonReportRequestUnexpected<R, E, A>(self: Effect<A, E, R>) {
+    return self
+      .pipe(
+        reportUnknownRequestError,
+        setRootParentSpan,
+        Effect.uninterruptible,
+        run
+      )
+  }
+
+  return {
+    interrupt,
+    join,
+    run,
+    add,
+    addAll,
+    register,
+    forkDaemonReportRequest,
+    forkDaemonReportRequestUnexpected
+  }
+})
+
+/**
+ * Whenever you fork a fiber for a Request, and you want to prevent dependent services to close prematurely on interruption,
+ * like the ServiceBus Sender, you should register these fibers in this FiberSet.
+ */
+export class RequestFiberSet extends Context.TagMakeId("RequestFiberSet", make)<RequestFiberSet>() {
+  static readonly Live = this.toLayerScoped()
+  static readonly register = <A, E, R>(self: Effect<A, E, R>) => this.use((_) => _.register(self))
+  static readonly run = <A, E, R>(self: Effect<A, E, R>) => this.use((_) => _.run(self))
+}