@effect-app/infra 4.0.0-beta.98 → 4.0.0

package/src/ClusterServiceBus.ts

@@ -0,0 +1,244 @@
+import { ServiceBusClient, type ServiceBusClientOptions, type ServiceBusMessage, type ServiceBusReceiverOptions, type ServiceBusSenderOptions, type SubscribeOptions } from "@azure/service-bus"
+import * as Context from "effect-app/Context"
+import * as Effect from "effect-app/Effect"
+import * as Layer from "effect-app/Layer"
+import * as ClusterError from "effect/cluster/ClusterError"
+import { Persisted } from "effect/cluster/ClusterSchema"
+import type * as Message from "effect/cluster/Message"
+import type * as MessageStorage from "effect/cluster/MessageStorage"
+import type * as RunnerHealth from "effect/cluster/RunnerHealth"
+import * as Runners from "effect/cluster/Runners"
+import type * as RunnerStorage from "effect/cluster/RunnerStorage"
+import * as Sharding from "effect/cluster/Sharding"
+import type * as ShardingConfig from "effect/cluster/ShardingConfig"
+import * as Snowflake from "effect/cluster/Snowflake"
+import * as Data from "effect/Data"
+import type * as Rpc from "effect/rpc/Rpc"
+import * as Schema from "effect/Schema"
+import * as Scope from "effect/Scope"
+
+export interface ClusterServiceBusSender {
+  readonly sendMessages: (messages: ServiceBusMessage) => Promise<void>
+  readonly close: () => Promise<void>
+}
+
+export interface ClusterServiceBusSubscription {
+  readonly close: () => Promise<void>
+}
+
+export interface ClusterServiceBusReceiver {
+  readonly subscribe: (
+    handlers: {
+      readonly processMessage: () => Promise<void>
+      readonly processError: (args: {
+        readonly error: Error
+        readonly entityPath?: string | undefined
+        readonly errorSource: string
+        readonly fullyQualifiedNamespace: string
+      }) => Promise<void>
+    },
+    options?: SubscribeOptions | undefined
+  ) => ClusterServiceBusSubscription
+  readonly close: () => Promise<void>
+}
+
+export interface ClusterServiceBusClient {
+  readonly createSender: (
+    topicName: string,
+    options?: ServiceBusSenderOptions | undefined
+  ) => ClusterServiceBusSender
+  readonly createReceiver: (
+    topicName: string,
+    subscriptionName: string,
+    options?: ServiceBusReceiverOptions | undefined
+  ) => ClusterServiceBusReceiver
+  readonly close: () => Promise<void>
+}
+
+export class ClusterServiceBus extends Context.Service<ClusterServiceBus, {
+  readonly client: ClusterServiceBusClient
+}>()("@effect-app/infra/ClusterServiceBus") {}
+
+export interface TopicOptions {
+  readonly topicName: string
+  readonly senderOptions?: ServiceBusSenderOptions | undefined
+}
+
+export interface SubscriptionOptions extends TopicOptions {
+  readonly subscriptionName: string
+  readonly receiverOptions?: ServiceBusReceiverOptions | undefined
+  readonly subscribeOptions?: SubscribeOptions | undefined
+}
+
+export interface StorageNotification {
+  readonly _tag: "EffectClusterStorageNotification"
+  readonly envelopeId: string
+  readonly requestId: string
+  readonly shardId: string
+  readonly entityType: string
+  readonly entityId: string
+}
+
+export const layerClient = (
+  connectionString: string,
+  options?: ServiceBusClientOptions | undefined
+): Layer.Layer<ClusterServiceBus> =>
+  Layer.effect(
+    ClusterServiceBus,
+    Effect.acquireRelease(
+      Effect.sync(() => ClusterServiceBus.of({ client: new ServiceBusClient(connectionString, options) })),
+      ({ client }) => Effect.promise(() => client.close()).pipe(Effect.ignore)
+    )
+  )
+
+export const layerClientFrom = (client: ClusterServiceBusClient): Layer.Layer<ClusterServiceBus> =>
+  Layer.succeed(ClusterServiceBus)(ClusterServiceBus.of({ client }))
+
+export const makeRunners: (
+  options: TopicOptions
+) => Effect.Effect<
+  Runners.Runners["Service"],
+  never,
+  | ClusterServiceBus
+  | MessageStorage.MessageStorage
+  | ShardingConfig.ShardingConfig
+  | Snowflake.Generator
+  | Scope.Scope
+> = Effect.fnUntraced(function*(options) {
+  const serviceBus = yield* ClusterServiceBus
+  const scope = yield* Effect.scope
+  const sender = serviceBus.client.createSender(options.topicName, options.senderOptions)
+
+  yield* Scope.addFinalizer(scope, Effect.promise(() => sender.close()).pipe(Effect.ignore))
+
+  const publish = <R extends Rpc.Any>(message: Message.Outgoing<R>) =>
+    Effect
+      .tryPromise({
+        try: () => sender.sendMessages(storageNotification(message)),
+        catch: (cause) => new ServiceBusPublishError({ cause })
+      })
+      .pipe(
+        Effect.catch((cause) => Effect.logDebug("Could not publish cluster storage notification", cause))
+      )
+
+  return yield* Runners.make({
+    codecFor: Schema.toCodecJson,
+    ping: (address) => Effect.fail(new ClusterError.RunnerUnavailable({ address })),
+    send: ({ address, message }) => {
+      const persisted = Context.get(message.rpc.annotations, Persisted)
+      return (persisted ? publish(message) : Effect.void).pipe(
+        Effect.andThen(Effect.fail(new ClusterError.RunnerUnavailable({ address })))
+      )
+    },
+    notify: ({ message }) => publish(message),
+    onRunnerUnavailable: () => Effect.void
+  })
+})
+
+export const layerRunners = (
+  options: TopicOptions
+): Layer.Layer<
+  Runners.Runners,
+  never,
+  ClusterServiceBus | MessageStorage.MessageStorage | ShardingConfig.ShardingConfig
+> =>
+  Layer.effect(Runners.Runners, makeRunners(options)).pipe(
+    Layer.provide(Snowflake.layerGenerator)
+  )
+
+export const makeStoragePoller: (
+  options: SubscriptionOptions
+) => Effect.Effect<void, never, ClusterServiceBus | Sharding.Sharding | Scope.Scope> = Effect.fnUntraced(
+  function*(options) {
+    const serviceBus = yield* ClusterServiceBus
+    const sharding = yield* Sharding.Sharding
+    const scope = yield* Effect.scope
+    const receiver = serviceBus.client.createReceiver(
+      options.topicName,
+      options.subscriptionName,
+      {
+        receiveMode: "receiveAndDelete",
+        ...options.receiverOptions
+      }
+    )
+    const subscription = receiver.subscribe(
+      {
+        processMessage: () =>
+          Effect.runPromise(
+            sharding.pollStorage.pipe(
+              Effect.catchCause((cause) => Effect.logDebug("Could not wake cluster storage poller", cause))
+            )
+          ),
+        processError: (args) =>
+          Effect.runPromise(
+            Effect.logDebug("Error receiving cluster storage notification", args.error).pipe(
+              Effect.annotateLogs({
+                entityPath: args.entityPath,
+                errorSource: args.errorSource,
+                fullyQualifiedNamespace: args.fullyQualifiedNamespace
+              })
+            )
+          )
+      },
+      options.subscribeOptions
+    )
+
+    yield* Scope.addFinalizer(
+      scope,
+      Effect.promise(() => subscription.close()).pipe(
+        Effect.andThen(Effect.promise(() => receiver.close())),
+        Effect.ignore
+      )
+    )
+  }
+)
+
+export const layerStoragePoller = (
+  options: SubscriptionOptions
+): Layer.Layer<never, never, ClusterServiceBus | Sharding.Sharding> => Layer.effectDiscard(makeStoragePoller(options))
+
+export const layer = (
+  options: SubscriptionOptions
+): Layer.Layer<
+  Sharding.Sharding | Runners.Runners,
+  never,
+  | ClusterServiceBus
+  | MessageStorage.MessageStorage
+  | RunnerHealth.RunnerHealth
+  | RunnerStorage.RunnerStorage
+  | ShardingConfig.ShardingConfig
+> => {
+  const sharding = Sharding.layer.pipe(
+    Layer.provideMerge(layerRunners(options))
+  )
+  return Layer.merge(
+    sharding,
+    layerStoragePoller(options).pipe(
+      Layer.provide(sharding)
+    )
+  )
+}
+
+const storageNotification = <R extends Rpc.Any>(message: Message.Outgoing<R>): ServiceBusMessage => {
+  const envelope = message.envelope
+  const envelopeId = envelope._tag === "Request" ? envelope.requestId : envelope.id
+  const body: StorageNotification = {
+    _tag: "EffectClusterStorageNotification",
+    envelopeId: String(envelopeId),
+    requestId: String(envelope.requestId),
+    shardId: envelope.address.shardId.toString(),
+    entityType: envelope.address.entityType,
+    entityId: envelope.address.entityId
+  }
+  return {
+    body,
+    contentType: "application/json",
+    correlationId: body.requestId,
+    messageId: body.envelopeId,
+    subject: body._tag
+  }
+}
+
+class ServiceBusPublishError extends Data.TaggedError("ServiceBusPublishError")<{
+  readonly cause: unknown
+}> {}

package/src/{api/ContextProvider.ts → ContextProvider.ts}

@@ -1,10 +1,14 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { Context, Effect, Layer, type NonEmptyReadonlyArray, pipe, type Scope } from "effect-app"
+import type { NonEmptyReadonlyArray } from "effect-app/Array"
+import * as Context from "effect-app/Context"
+import * as Effect from "effect-app/Effect"
+import * as Layer from "effect-app/Layer"
+import { pipe } from "effect/Function"
+import type * as Scope from "effect/Scope"
 
 import { type HttpRouter } from "effect-app/http"
 import { type EffectGenUtils } from "effect-app/utils/gen"
-import { type Yieldable } from "effect/Effect"
-import { type ContextTagWithDefault, type GetContext, type LayerUtils, mergeContexts } from "./layerUtils.js"
+import { type ContextTagWithDefault, type GetContext, type LayerUtils, mergeContexts } from "./layerUtils.ts"
 
 export interface ContextProviderId {
   _tag: "ContextProvider"
@@ -38,7 +42,7 @@ type TDepsArr<TDeps extends ReadonlyArray<any>> = {
       > // & { _tag: infer _3 }
     ) // [_YW] extends [never] if no yield* is used and just some context is returned
       ? [_YW] extends [never] ? TDeps[K]
-      : [_YW] extends [Yieldable<any, infer _2, never, infer _R>] ? [_R] extends [HttpRouter.Provided] ? TDeps[K]
+      : [_YW] extends [Effect.Effect<any, never, infer _R>] ? [_R] extends [HttpRouter.Provided] ? TDeps[K]
         : `HttpLayerRouter.Provided is the only requirement ${TDeps[K]["Service"][
           "_tag"
         ]}'s returned effect can have`
@@ -97,9 +101,8 @@ export const ContextProvider = <
     effect: Effect.Effect<
       | Effect.Effect<ContextProviderA, never, ContextProviderR>
       | (() => Generator<
-        Yieldable<any, any, never, ContextProviderR>,
-        ContextProviderA,
-        any
+        Effect.Effect<any, never, ContextProviderR>,
+        ContextProviderA
       >),
       MakeContextProviderE,
       MakeContextProviderR | Scope.Scope

package/src/Emailer/Sendgrid.ts

@@ -1,63 +1,85 @@
-import type { EmailData } from "@sendgrid/helpers/classes/email-address.js"
 import type { MailContent } from "@sendgrid/helpers/classes/mail.js"
 import sgMail from "@sendgrid/mail"
-import { Array, Effect, Equivalence, Redacted } from "effect-app"
+import * as Array from "effect-app/Array"
+import * as Effect from "effect-app/Effect"
+import { type EmailData, Emailer, type EmailMsg, type EmailMsgOptionalFrom, SendMailError } from "effect-app/Emailer"
 import { dropUndefinedT } from "effect-app/utils"
+import * as Equivalence from "effect/Equivalence"
+import * as Redacted from "effect/Redacted"
 import { inspect } from "util"
-import { InfraLogger } from "../logger.js"
-import { Emailer, type EmailMsg, type EmailMsgOptionalFrom, type SendgridConfig, SendMailError } from "./service.js"
+import { InfraLogger } from "../logger.ts"
 
-const makeSendgrid = ({ apiKey, defaultFrom, defaultReplyTo, realMail, subjectPrefix }: SendgridConfig) =>
+export interface SendgridConfig {
+  defaultReplyTo?: EmailData
+  subjectPrefix: string
+  realMail: boolean
+  defaultFrom: EmailData
+  apiKey: Redacted.Redacted
+  /**
+   * Email address used for fake/test recipients. Use `{i}` as a placeholder for
+   * an auto-incrementing index to ensure uniqueness.
+   *
+   * @example "test+{i}@example.com"
+   */
+  fakeMailAddress: string
+}
+
+const makeSendgrid = (
+  { apiKey, defaultFrom, defaultReplyTo, fakeMailAddress, realMail, subjectPrefix }: SendgridConfig
+) =>
   Effect.sync(() => {
     sgMail.setApiKey(Redacted.value(apiKey))
 
     return Emailer.of({
-      sendMail: Effect.fn("Sendgrid.sendMail")(function*(msg_: EmailMsgOptionalFrom) {
-        const msg: EmailMsg = dropUndefinedT({
-          ...msg_,
-          from: msg_.from ?? defaultFrom,
-          replyTo: msg_.replyTo ?? (msg_.from ? undefined : defaultReplyTo)
-        })
-        const render = renderMessage(!realMail)
+      sendMail: Effect.fn("Emailer.sendMail", { attributes: { "messaging.system": "sendgrid" } })(
+        function*(msg_: EmailMsgOptionalFrom) {
+          const replyTo = msg_.replyTo ?? (msg_.from ? undefined : defaultReplyTo)
+          const msg = dropUndefinedT({
+            ...msg_,
+            from: msg_.from ?? defaultFrom,
+            replyTo
+          }) satisfies EmailMsg
+          const render = renderMessage(!realMail, fakeMailAddress)
 
-        const renderedMsg_ = render(msg)
-        const renderedMsg = {
-          ...renderedMsg_ as Omit<typeof renderedMsg_, "content">,
-          subject: `${subjectPrefix}${renderedMsg_.subject}`,
-          ..."content" in renderedMsg_
-            ? { content: [...renderedMsg_.content] as [MailContent, ...MailContent[]] }
-            : {}
-        }
-        yield* InfraLogger.logDebug("Sending email").pipe(Effect.annotateLogs("msg", inspect(renderedMsg, false, 5)))
+          const renderedMsg_ = render(msg)
+          const renderedMsg = {
+            ...renderedMsg_ as Omit<typeof renderedMsg_, "content">,
+            subject: `${subjectPrefix}${renderedMsg_.subject}`,
+            ..."content" in renderedMsg_
+              ? { content: [...renderedMsg_.content] as [MailContent, ...MailContent[]] }
+              : {}
+          }
+          yield* InfraLogger.logDebug("Sending email").pipe(Effect.annotateLogs("msg", inspect(renderedMsg, false, 5)))
 
-        const ret = yield* Effect
-          .callback<
-            [sgMail.ClientResponse, Record<string, unknown>],
-            Error | sgMail.ResponseError
-          >(
-            (resume) =>
-              void sgMail.send(
-                renderedMsg as any, // sue me
-                msg.isMultiple ?? true,
-                (err, result) =>
-                  err
-                    ? resume(Effect.fail(err))
-                    : resume(Effect.sync(() => result))
-              )
-          )
-          .pipe(Effect.mapError((raw) => new SendMailError({ raw })))
+          const ret = yield* Effect
+            .callback<
+              [sgMail.ClientResponse, Record<string, unknown>],
+              Error | sgMail.ResponseError
+            >(
+              (resume) =>
+                void sgMail.send(
+                  renderedMsg as any, // sue me
+                  msg.isMultiple ?? true,
+                  (err, result) =>
+                    err
+                      ? resume(Effect.fail(err))
+                      : resume(Effect.sync(() => result))
+                )
+            )
+            .pipe(Effect.mapError((raw) => new SendMailError({ raw })))
 
-        // const event = {
-        //   name: "EmailSent",
-        //   properties: {
-        //     templateId: msg.templateId
-        //   }
-        // }
-        // yield* InfraLogger.logDebug("Tracking email event").annotateLogs("event", event.$$.pretty)
-        // const { trackEvent } = yield* AiContextService
-        // trackEvent(event)
-        return ret
-      })
+          // const event = {
+          //   name: "EmailSent",
+          //   properties: {
+          //     templateId: msg.templateId
+          //   }
+          // }
+          // yield* InfraLogger.logDebug("Tracking email event").annotateLogs("event", event.$$.pretty)
+          // const { trackEvent } = yield* AiContextService
+          // trackEvent(event)
+          return ret
+        }
+      )
     })
   })
 
@@ -68,23 +90,24 @@ export function Sendgrid(config: SendgridConfig) {
 /**
  * @hidden
  */
-export function renderMessage(forceFake: boolean) {
+export function renderMessage(forceFake: boolean, fakeMailAddress: string) {
   let i = 0
   const makeId = () => i++
+  const makeFakeEmail = () => fakeMailAddress.replace("{i}", String(makeId()))
   return forceFake
     ? (msg: EmailMsg) =>
       dropUndefinedT({
         ...msg,
-        to: msg.to && renderFake(msg.to, makeId),
-        cc: msg.cc && renderFake(msg.cc, makeId),
-        bcc: msg.bcc && renderFake(msg.bcc, makeId)
+        to: msg.to && renderFake(msg.to, makeFakeEmail),
+        cc: msg.cc && renderFake(msg.cc, makeFakeEmail),
+        bcc: msg.bcc && renderFake(msg.bcc, makeFakeEmail)
       })
     : (msg: EmailMsg) =>
       dropUndefinedT({
         ...msg,
-        to: msg.to && renderFakeIfTest(msg.to, makeId),
-        cc: msg.cc && renderFakeIfTest(msg.cc, makeId),
-        bcc: msg.bcc && renderFakeIfTest(msg.bcc, makeId)
+        to: msg.to && renderFakeIfTest(msg.to, makeFakeEmail),
+        cc: msg.cc && renderFakeIfTest(msg.cc, makeFakeEmail),
+        bcc: msg.bcc && renderFakeIfTest(msg.bcc, makeFakeEmail)
       })
 }
 
@@ -100,10 +123,10 @@ export function isTestAddress(to: EmailData) {
   )
 }
 
-function renderFake(addr: EmailData | readonly EmailData[], makeId: () => number) {
+function renderFake(addr: EmailData | readonly EmailData[], makeEmail: () => string) {
   return {
     name: renderMailData(addr),
-    email: `test+${makeId()}@nomizz.com`
+    email: makeEmail()
   }
 }
 const eq = Equivalence.mapInput(
@@ -117,14 +140,14 @@ function isEmailDataArray(md: EmailData | readonly EmailData[]): md is readonly
 
 // TODO: should just not add any already added email address
 // https://stackoverflow.com/a/53603076/11595834
-function renderFakeIfTest(addr: EmailData | readonly EmailData[], makeId: () => number) {
+function renderFakeIfTest(addr: EmailData | readonly EmailData[], makeEmail: () => string) {
   if (isEmailDataArray(addr)) {
     return Array.dedupeWith(
-      addr.map((x) => (isTestAddress(x) ? renderFake(x, makeId) : x)),
+      addr.map((x) => (isTestAddress(x) ? renderFake(x, makeEmail) : x)),
       eq
     )
   }
-  return isTestAddress(addr) ? renderFake(addr, makeId) : addr
+  return isTestAddress(addr) ? renderFake(addr, makeEmail) : addr
 }
 
 function renderMailData(md: EmailData | readonly EmailData[]): string {

package/src/Emailer/fake.ts

@@ -1,13 +1,13 @@
-import { Effect } from "effect-app"
+import * as Effect from "effect-app/Effect"
+import { Emailer } from "effect-app/Emailer"
 import { pretty } from "effect-app/utils"
-import { InfraLogger } from "../logger.js"
-import { Emailer } from "./service.js"
+import { InfraLogger } from "../logger.ts"
 
 const makeFake = InfraLogger
   .logInfo("FAKE Emailer Service enabled")
   .pipe(Effect.map(() =>
     Emailer.of({
-      sendMail: Effect.fn("Fake.sendMail")((msg) =>
+      sendMail: Effect.fn("Emailer.sendMail", { attributes: { "messaging.system": "fake" } })((msg) =>
         InfraLogger
           .logDebug(`Fake send mail`)
           .pipe(Effect.annotateLogs("msg", pretty(msg)))

package/src/MainFiberSet.ts

@@ -1,14 +1,34 @@
-import { Context, Effect, Fiber, FiberSet, Layer } from "effect-app"
-import type {} from "effect/Scope"
-import { InfraLogger } from "./logger.js"
-import { reportNonInterruptedFailureCause } from "./QueueMaker/errors.js"
-import { setRootParentSpan } from "./RequestFiberSet.js"
+import * as Context from "effect-app/Context"
+import * as Effect from "effect-app/Effect"
+import * as Layer from "effect-app/Layer"
+import * as Option from "effect-app/Option"
+import * as Fiber from "effect/Fiber"
+import * as FiberSet from "effect/FiberSet"
+import type * as Tracer from "effect/Tracer"
+
+import { InfraLogger } from "./logger.ts"
+import { reportNonInterruptedFailureCause } from "./QueueMaker/errors.ts"
+
+// Walk to the root span so a daemon fiber's spans aren't parented to a span that
+// may already be closing (e.g. the originating request).
+const getRootParentSpan = Effect.gen(function*() {
+  let span: Tracer.AnySpan | null = yield* Effect.currentSpan.pipe(
+    Effect.catchTag("NoSuchElementError", () => Effect.succeed(null))
+  )
+  if (!span) return span
+  while (span._tag === "Span" && Option.isSome(span.parent)) {
+    span = span.parent.value
+  }
+  return span
+})
+
+export const setRootParentSpan = <A, E, R>(self: Effect.Effect<A, E, R>) =>
+  getRootParentSpan.pipe(Effect.andThen((span) => span ? Effect.withParentSpan(self, span) : self))
 
 const make = Effect.gen(function*() {
   const set = yield* FiberSet.make<unknown, never>()
-  const add = (...fibers: Fiber.Fiber<never, never>[]) =>
-    Effect.sync(() => fibers.forEach((_) => FiberSet.addUnsafe(set, _)))
-  const addAll = (fibers: readonly Fiber.Fiber<never, never>[]) =>
+  const add = (...fibers: Fiber.Fiber<never>[]) => Effect.sync(() => fibers.forEach((_) => FiberSet.addUnsafe(set, _)))
+  const addAll = (fibers: readonly Fiber.Fiber<never>[]) =>
     Effect.sync(() => fibers.forEach((_) => FiberSet.addUnsafe(set, _)))
   const join = FiberSet.size(set).pipe(
     Effect.andThen((count) => InfraLogger.logDebug(`Joining ${count} current fibers on the MainFiberSet`)),
@@ -64,13 +84,12 @@ const make = Effect.gen(function*() {
  */
 export class MainFiberSet extends Context.Service<MainFiberSet>()("MainFiberSet", { make }) {
   static readonly Live = Layer.effect(this, this.make)
-  static readonly JoinLive = this.asEffect().pipe(
+  static readonly JoinLive = this.pipe(
     Effect.andThen((_) => _.join),
     Layer.effectDiscard,
     Layer.provide(this.Live)
   )
-  static readonly run = <A, R>(self: Effect.Effect<A, never, R>) =>
-    this.asEffect().pipe(Effect.andThen((_) => _.run(self)))
+  static readonly run = <A, R>(self: Effect.Effect<A, never, R>) => this.pipe(Effect.andThen((_) => _.run(self)))
   static readonly forkDaemonReport = <A, E, R>(self: Effect.Effect<A, E, R>) =>
-    this.asEffect().pipe(Effect.andThen((_) => _.forkDaemonReport(self)))
+    this.pipe(Effect.andThen((_) => _.forkDaemonReport(self)))
 }

package/src/QueueMaker/errors.ts

@@ -1,5 +1,7 @@
 import { reportError } from "@effect-app/infra/errorReporter"
-import { Cause, Effect, Exit } from "effect-app"
+import * as Effect from "effect-app/Effect"
+import * as Cause from "effect/Cause"
+import * as Exit from "effect/Exit"
 
 const reportQueueError_ = reportError("Queue")