@effect-app/infra 4.0.0-beta.31 → 4.0.0-beta.310

package/src/QueueMaker/memQueue.ts

@@ -1,130 +0,0 @@
-import { Cause, Tracer } from "effect"
-import { Effect, Fiber, flow, type NonEmptyReadonlyArray, S } from "effect-app"
-import { pretty } from "effect-app/utils"
-import * as Q from "effect/Queue"
-import { MemQueue } from "../adapters/memQueue.js"
-import { getRequestContext, setupRequestContextWithCustomSpan } from "../api/setupRequest.js"
-import { InfraLogger } from "../logger.js"
-import { reportNonInterruptedFailure, reportNonInterruptedFailureCause } from "./errors.js"
-import { type QueueBase, QueueMeta } from "./service.js"
-
-export function makeMemQueue<
-  Evt extends { id: S.StringId; _tag: string },
-  DrainEvt extends { id: S.StringId; _tag: string },
-  EvtE,
-  DrainEvtE
->(
-  queueName: string,
-  queueDrainName: string,
-  schema: S.Codec<Evt, EvtE>,
-  drainSchema: S.Codec<DrainEvt, DrainEvtE>
-) {
-  return Effect.gen(function*() {
-    const mem = yield* MemQueue
-    const q = yield* mem.getOrCreateQueue(queueName)
-    const qDrain = yield* mem.getOrCreateQueue(queueDrainName)
-
-    const wireSchema = S.Struct({ body: schema, meta: QueueMeta })
-    const wireSchemaJson = S.fromJsonString(wireSchema)
-    const encodePublish = S.encodeEffect(wireSchemaJson)
-    const drainW = S.Struct({ body: drainSchema, meta: QueueMeta })
-    const drainWJson = S.fromJsonString(drainW)
-
-    const parseDrain = flow(S.decodeUnknownEffect(drainWJson), Effect.orDie)
-
-    const queue = {
-      publish: (...messages: NonEmptyReadonlyArray<Evt>) =>
-        getRequestContext
-          .pipe(
-            Effect.flatMap((requestContext) =>
-              Effect
-                .forEach(messages, (m) =>
-                  // we JSON encode, because that is what the wire also does, and it reveals holes in e.g unknown encoders (Date->String)
-                  encodePublish({ body: m, meta: requestContext }).pipe(
-                    Effect.orDie,
-                    // .tap((msg) => info("Publishing Mem Message: " + utils.inspect(msg)))
-                    Effect.flatMap((_) => Q.offer(q, _))
-                  ), { discard: true })
-            ),
-            Effect.withSpan("queue.publish: " + queueName, {
-              kind: "producer",
-              attributes: { "message_tags": messages.map((_) => _._tag) }
-            }, { captureStackTrace: false })
-          ),
-      drain: <DrainE, DrainR>(
-        handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
-        sessionId?: string
-      ) => {
-        const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
-        const reportError = reportNonInterruptedFailureCause({ name: "MemQueue.drain." + queueDrainName })
-        const processMessage = (msg: string) =>
-          // we JSON parse, because that is what the wire also does, and it reveals holes in e.g unknown encoders (Date->String)
-          parseDrain(msg).pipe(
-            Effect.orDie,
-            Effect
-              .flatMap(({ body, meta }) => {
-                let effect = InfraLogger
-                  .logDebug(`[${queueDrainName}] Processing incoming message`)
-                  .pipe(
-                    Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
-                    Effect.andThen(handleEvent(body)),
-                    silenceAndReportError,
-                    (_) =>
-                      setupRequestContextWithCustomSpan(
-                        _,
-                        meta,
-                        `queue.drain: ${queueDrainName}.${body._tag}`,
-                        {
-                          captureStackTrace: false,
-                          kind: "consumer",
-                          attributes: {
-                            "queue.name": queueDrainName,
-                            "queue.sessionId": sessionId,
-                            "queue.input": body
-                          }
-                        }
-                      )
-                  )
-                if (meta.span) {
-                  effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
-                }
-                return effect
-              })
-          )
-        return Q
-          .take(qDrain)
-          .pipe(
-            Effect
-              .flatMap((x) =>
-                processMessage(x).pipe(
-                  Effect.uninterruptible,
-                  Effect.forkChild,
-                  Effect.flatMap(Fiber.join),
-                  // normally a failed item would be returned to the queue and retried up to X times.
-                  Effect.flatMap((_) =>
-                    _._tag === "Failure" && !Cause.hasInterruptsOnly(_.cause)
-                      ? Q.offer(qDrain, x).pipe(
-                        // TODO: retry count tracking and max retries.
-                        Effect.delay("5 seconds"),
-                        Effect.tapCause(reportError),
-                        Effect.forkDetach
-                      )
-                      : Effect.void
-                  )
-                )
-              ),
-            silenceAndReportError,
-            Effect.withSpan(`queue.drain: ${queueDrainName}`, {
-              attributes: {
-                "queue.type": "mem",
-                "queue.name": queueDrainName,
-                "queue.sessionId": sessionId
-              }
-            }),
-            Effect.forever
-          )
-      }
-    }
-    return queue as QueueBase<Evt, DrainEvt>
-  })
-}

package/src/QueueMaker/sbqueue.ts

@@ -1,137 +0,0 @@
-import { Tracer } from "effect"
-import { Cause, Effect, flow, type NonEmptyReadonlyArray, S } from "effect-app"
-import type { StringId } from "effect-app/Schema"
-import { pretty } from "effect-app/utils"
-import { Receiver, Sender } from "../adapters/ServiceBus.js"
-import { getRequestContext, setupRequestContextWithCustomSpan } from "../api/setupRequest.js"
-import { InfraLogger } from "../logger.js"
-import { reportNonInterruptedFailure, reportNonInterruptedFailureCause, reportQueueError } from "./errors.js"
-import { type QueueBase, QueueMeta } from "./service.js"
-
-export function makeServiceBusQueue<
-  Evt extends { id: StringId; _tag: string },
-  DrainEvt extends { id: StringId; _tag: string },
-  EvtE,
-  DrainEvtE
->(
-  schema: S.Codec<Evt, EvtE>,
-  drainSchema: S.Codec<DrainEvt, DrainEvtE>
-) {
-  const wireSchema = S.Struct({
-    body: schema,
-    meta: QueueMeta
-  })
-  const wireSchemaJson = S.fromJsonString(wireSchema)
-  const encodePublish = S.encodeEffect(wireSchemaJson)
-  const drainW = S.Struct({ body: drainSchema, meta: QueueMeta })
-  const drainWJson = S.fromJsonString(drainW)
-  const parseDrain = flow(S.decodeUnknownEffect(drainWJson), Effect.orDie)
-
-  return Effect.gen(function*() {
-    const sender = yield* Sender
-    const receiver = yield* Receiver
-    const silenceAndReportError = reportNonInterruptedFailure({ name: receiver.name })
-    const reportError = reportNonInterruptedFailureCause({ name: receiver.name })
-
-    // TODO: or do async?
-    // This will make sure that the host receives the error (MainFiberSet.join), who will then interrupt everything and commence a shutdown and restart of app
-    // const deferred = yield* Deferred.make<never, ServiceBusError | Error>()
-
-    const queue = {
-      drain: <DrainE, DrainR>(
-        handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
-        sessionId?: string
-      ) => {
-        function processMessage(messageBody: unknown) {
-          return parseDrain(messageBody).pipe(
-            Effect.orDie,
-            Effect
-              .flatMap(({ body, meta }) => {
-                let effect = InfraLogger
-                  .logDebug(`[${receiver.name}] Processing incoming message`)
-                  .pipe(
-                    Effect.annotateLogs({
-                      body: pretty(body),
-                      meta: pretty(meta)
-                    }),
-                    Effect.andThen(handleEvent(body)),
-                    Effect.orDie
-                  )
-                  // we silenceAndReportError here, so that the error is reported, and moves into the Exit.
-                  .pipe(
-                    silenceAndReportError,
-                    (_) =>
-                      setupRequestContextWithCustomSpan(
-                        _,
-                        meta,
-                        `queue.drain: ${receiver.name}${sessionId ? `#${sessionId}` : ""}.${body._tag}`,
-                        {
-                          captureStackTrace: false,
-                          kind: "consumer",
-                          attributes: {
-                            "queue.name": receiver.name,
-                            "queue.sessionId": sessionId,
-                            "queue.input": body
-                          }
-                        }
-                      )
-                  )
-                if (meta.span) {
-                  effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
-                }
-                return effect
-              }),
-            Effect
-              // we reportError here, so that we report the error only, and keep flowing
-              .tapCause(reportError),
-            // we still need to flatten the Exit.
-            Effect.flatMap((_) => _)
-          )
-        }
-
-        return receiver
-          .subscribe({
-            processMessage: (x) => processMessage(x.body).pipe(Effect.uninterruptible),
-            processError: (err) => reportQueueError(Cause.fail(err.error))
-            // Deferred.completeWith(
-            //   deferred,
-            //   reportFatalQueueError(Cause.fail(err.error))
-            //     .pipe(Effect.andThen(Effect.fail(err.error)))
-            // )
-          }, sessionId)
-          // .pipe(Effect.andThen(Deferred.await(deferred).pipe(Effect.orDie))),
-          .pipe(
-            Effect.andThen(Effect.never)
-          )
-      },
-
-      publish: (...messages: NonEmptyReadonlyArray<Evt>) =>
-        getRequestContext
-          .pipe(
-            Effect.flatMap((requestContext) =>
-              Effect
-                .forEach(messages, (m) =>
-                  encodePublish({
-                    body: m,
-                    meta: requestContext
-                  })
-                    .pipe(
-                      Effect.orDie,
-                      Effect.map((body) => ({
-                        body,
-                        messageId: m.id, /* correllationid: requestId */
-                        contentType: "application/json",
-                        sessionId: "sessionId" in m ? m.sessionId as string : undefined as unknown as string // TODO: optional
-                      }))
-                    ))
-                .pipe(Effect.flatMap((msgs) => sender.sendMessages(msgs)))
-            ),
-            Effect.withSpan("queue.publish: " + sender.name, {
-              kind: "producer",
-              attributes: { "message_tags": messages.map((_) => _._tag) }
-            }, { captureStackTrace: false })
-          )
-    }
-    return queue as QueueBase<Evt, DrainEvt>
-  })
-}

package/src/QueueMaker/service.ts

@@ -1,17 +0,0 @@
-import type { Effect, NonEmptyReadonlyArray, Scope } from "effect-app"
-import { RequestContext } from "../RequestContext.js"
-
-export interface QueueBase<Evt, DrainEvt> {
-  drain: <DrainE, DrainR>(
-    makeHandleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
-    sessionId?: string
-  ) => Effect.Effect<never, never, Scope.Scope | DrainR>
-  publish: (
-    ...messages: NonEmptyReadonlyArray<Evt>
-  ) => Effect.Effect<void>
-}
-
-export interface QueueMakerOps {}
-export const QueueMaker: QueueMakerOps = {}
-
-export const QueueMeta = RequestContext

package/src/RequestContext.ts

@@ -1,63 +0,0 @@
-import { S, ServiceMap } from "effect-app"
-import { UserProfileId } from "effect-app/ids"
-import { NonEmptyString255 } from "effect-app/Schema"
-
-export const Locale = S.Literal("en", "de")
-export type Locale = typeof Locale.Type
-
-export class LocaleRef extends ServiceMap.Reference("Locale", { defaultValue: (): Locale => "en" }) {}
-
-export class RequestContext extends S.ExtendedClass<
-  RequestContext,
-  RequestContext.Encoded
->("RequestContext")({
-  span: S.Struct({
-    traceId: S.String,
-    spanId: S.String,
-    sampled: S.Boolean
-  }),
-  name: NonEmptyString255,
-  locale: Locale,
-  sourceId: S.optional(NonEmptyString255), // TODO?
-  namespace: NonEmptyString255,
-  /** @deprecated */
-  userProfile: S.optional(S.Struct({ sub: UserProfileId })) //
-}) {
-  // static Tag = ServiceMap.Tag<RequestContext>()
-
-  static toMonitoring(this: void, self: RequestContext) {
-    return {
-      operationName: self.name,
-      locale: self.locale
-    }
-  }
-}
-
-export const spanAttributes = (ctx: Pick<RequestContext, "locale" | "namespace"> & Partial<RequestContext>) => ({
-  "request.name": ctx.name,
-  "request.locale": ctx.locale,
-  "request.namespace": ctx.namespace,
-  ...ctx.sourceId ? { "request.source.id": ctx.sourceId } : {},
-  ...(ctx.userProfile?.sub
-    ? {
-      "request.user.sub": ctx
-        .userProfile
-        .sub,
-      "request.user.roles": "roles" in ctx
-          .userProfile
-        ? ctx.userProfile.roles
-        : undefined
-    }
-    : {})
-})
-
-// codegen:start {preset: model}
-//
-/* eslint-disable */
-export namespace RequestContext {
-  export interface Encoded extends S.Struct.Encoded<typeof RequestContext["fields"]> {}
-}
-/* eslint-enable */
-//
-// codegen:end
-//

package/src/RequestFiberSet.ts

@@ -1,105 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { Effect, Fiber, FiberSet, Layer, Option, ServiceMap, type Tracer } from "effect-app"
-import { reportRequestError, reportUnknownRequestError } from "./api/reportError.js"
-import { InfraLogger } from "./logger.js"
-
-const getRootParentSpan = Effect.gen(function*() {
-  let span: Tracer.AnySpan | null = yield* Effect.currentSpan.pipe(
-    Effect.catchTag("NoSuchElementError", () => Effect.succeed(null))
-  )
-  if (!span) return span
-  while (span._tag === "Span" && Option.isSome(span.parent)) {
-    span = span.parent.value
-  }
-  return span
-})
-
-export const setRootParentSpan = <A, E, R>(self: Effect.Effect<A, E, R>) =>
-  getRootParentSpan.pipe(Effect.andThen((span) => span ? Effect.withParentSpan(self, span) : self))
-
-const make = Effect.gen(function*() {
-  const set = yield* FiberSet.make<any, any>()
-  const add = (...fibers: Fiber.Fiber<any, any>[]) =>
-    Effect.sync(() => fibers.forEach((_) => FiberSet.addUnsafe(set, _)))
-  const addAll = (fibers: readonly Fiber.Fiber<any, any>[]) =>
-    Effect.sync(() => fibers.forEach((_) => FiberSet.addUnsafe(set, _)))
-  const join = FiberSet.size(set).pipe(
-    Effect.andThen((count) => InfraLogger.logInfo(`Joining ${count} current fibers on the RequestFiberSet`)),
-    Effect.andThen(FiberSet.join(set))
-  )
-  const run = FiberSet.run(set)
-  const register = <A, E, R>(self: Effect.Effect<A, E, R>) =>
-    self.pipe(Effect.forkChild, Effect.tap(add), Effect.andThen(Fiber.join))
-
-  // const waitUntilEmpty = Effect.gen(function*() {
-  //   const currentSize = yield* FiberSet.size(set)
-  //   if (currentSize === 0) {
-  //     return
-  //   }
-  //   yield* Effect.logInfo("Waiting RequestFiberSet to be empty: " + currentSize)
-  //   while ((yield* FiberSet.size(set)) > 0) yield* Effect.sleep("250 millis")
-  //   yield* Effect.logDebug("RequestFiberSet is empty")
-  // })
-  // TODO: loop and interrupt all fibers in the set continuously?
-  const interrupt = Fiber.interruptAll(set)
-
-  /**
-   * Forks the effect into a new fiber attached to the RequestFiberSet scope. Because the
-   * new fiber isn't attached to the parent, when the fiber executing the
-   * returned effect terminates, the forked fiber will continue running.
-   * The fiber will be interrupted when the RequestFiberSet scope is closed.
-   *
-   * Reports errors.
-   */
-  function forkDaemonReport<R, E, A>(self: Effect.Effect<A, E, R>) {
-    return self.pipe(
-      reportRequestError,
-      Effect.uninterruptible,
-      run
-    )
-  }
-
-  /**
-   * Forks the effect into a new fiber attached to the RequestFiberSet scope. Because the
-   * new fiber isn't attached to the parent, when the fiber executing the
-   * returned effect terminates, the forked fiber will continue running.
-   * The fiber will be interrupted when the RequestFiberSet scope is closed.
-   *
-   * Reports unexpected errors.
-   */
-  function forkDaemonReportUnexpected<R, E, A>(self: Effect.Effect<A, E, R>) {
-    return self
-      .pipe(
-        reportUnknownRequestError,
-        Effect.uninterruptible,
-        run
-      )
-  }
-
-  return {
-    interrupt,
-    join,
-    run,
-    add,
-    addAll,
-    register,
-    forkDaemonReport,
-    forkDaemonReportUnexpected
-  }
-})
-
-/**
- * Whenever you fork a fiber for a Request, and you want to prevent dependent services to close prematurely on interruption,
- * like the ServiceBus Sender, you should register these fibers in this FiberSet.
- */
-export class RequestFiberSet extends ServiceMap.Service<RequestFiberSet>()("RequestFiberSet", { make }) {
-  static readonly Live = Layer.effect(this, this.make)
-  static readonly register = <A, E, R>(self: Effect.Effect<A, E, R>) =>
-    this.asEffect().pipe(Effect.andThen((_) => _.register(self)))
-  static readonly run = <A, E, R>(self: Effect.Effect<A, E, R>) =>
-    this.asEffect().pipe(Effect.andThen((_) => _.run(self)))
-  static readonly forkDaemonReport = <R, E, A>(self: Effect.Effect<A, E, R>) =>
-    this.asEffect().pipe(Effect.andThen((_) => _.forkDaemonReport(self)))
-  static readonly forkDaemonReportUnexpected = <R, E, A>(self: Effect.Effect<A, E, R>) =>
-    this.asEffect().pipe(Effect.andThen((_) => _.forkDaemonReportUnexpected(self)))
-}

package/src/Store/ContextMapContainer.ts

@@ -1,20 +0,0 @@
-import { Data, Effect, Layer, ServiceMap } from "effect-app"
-import { ContextMap } from "./service.js"
-
-// TODO: we have to create a new contextmap on every request.
-// we want to share one map during startup
-// but we want to make sure we don't re-use the startup map after startup
-// we can call another start after startup. but it would be even better if we could Die on accessing rootmap
-// we could also make the ContextMap optional, and when missing, issue a warning instead?
-
-export class ContextMapContainer extends ServiceMap.Reference("ContextMapContainer", {
-  defaultValue: (): ContextMap | "root" => "root"
-}) {
-  static readonly layer = Layer.effect(this, ContextMap.make.pipe(Effect.map(ContextMap.of)))
-}
-
-export class ContextMapNotStartedError extends Data.TaggedError("ContextMapNotStartedError") {}
-
-export const getContextMap = ContextMapContainer.asEffect().pipe(
-  Effect.filterOrFail((_) => _ !== "root", () => new ContextMapNotStartedError())
-)

package/src/Store/index.test.ts.bak

@@ -1,112 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import * as Test from "@effect-ts/jest/Test"
-import { describe, expect } from "@jest/globals"
-
-import { OptimisticConcurrencyException } from "../../errors.js"
-
-import { makeMemoryStore } from "./Memory.js"
-
-const { it } = Test.runtime()
-
-describe("Optimistic Concurrency", () => {
-  describe("set", () => {
-    it("works", () =>
-      Effect.gen(function*($) {
-        const existing = new Map<
-          string,
-          { _etag?: string | undefined; id: string; a: string }
-        >()
-        const { make } = makeMemoryStore()
-        const store = yield* $(make("test", Effect.sync(() => existing)))
-
-        const result1 = yield* $(store.set({ id: "1", a: "string", _etag: undefined }))
-        const result2_1 = yield* $(
-          store.set({ id: "2", a: "string", _etag: undefined })
-        )
-        const result2 = yield* $(store.set({ ...result1, a: "another-string" }))
-        const result3 = yield* $(store.set({ ...result2, a: "another-string2" }))
-        const failResult = yield* $(
-          store.set({ ...result1, a: "another-string3" }).result
-        )
-        expect(result1._etag).toBeTruthy()
-        expect(result2_1._etag).toBeTruthy()
-        expect(result2._etag).toBeTruthy()
-        expect(result3._etag).toBeTruthy()
-
-        expect(result2._etag).not.toEqual(result1._etag)
-        expect(result3._etag).not.toEqual(result2._etag)
-
-        expect(Exit.untraced(failResult)).toEqual(
-          Exit.fail(new OptimisticConcurrencyException("test", "1"))
-        )
-        console.log({ result1, result2 })
-      }))
-  })
-
-  describe("bulkSet", () => {
-    it("works", () =>
-      Effect.gen(function*($) {
-        const existing = new Map<
-          string,
-          { _etag?: string | undefined; id: string; a: string }
-        >()
-        const { make } = makeMemoryStore()
-        const store = yield* $(make("test", Effect.sync(() => existing)))
-
-        const [result1] = yield* $(
-          store.bulkSet([{ id: "1", a: "string", _etag: undefined }])
-        )
-        const [result2_1] = yield* $(
-          store.bulkSet([{ id: "2", a: "string", _etag: undefined }])
-        )
-        const [result2] = yield* $(store.bulkSet([{ ...result1, a: "another-string" }]))
-        const [result3] = yield* $(
-          store.bulkSet([{ ...result2, a: "another-string2" }])
-        )
-        const failResult = yield* $(
-          store.bulkSet([{ ...result1, a: "another-string3" }]).result
-        )
-        expect(result1._etag).toBeTruthy()
-        expect(result2_1._etag).toBeTruthy()
-        expect(result2._etag).toBeTruthy()
-        expect(result3._etag).toBeTruthy()
-
-        expect(result2._etag).not.toEqual(result1._etag)
-        expect(result3._etag).not.toEqual(result2._etag)
-
-        expect(Exit.untraced(failResult)).toEqual(
-          Exit.fail(new OptimisticConcurrencyException("test", "1"))
-        )
-        console.log({ result1, result2 })
-      }))
-  })
-
-  describe("batchSet", () => {
-    it("works", () =>
-      Effect.gen(function*($) {
-        const existing = new Map<
-          string,
-          { _etag?: string | undefined; id: string; a: string }
-        >()
-        const { make } = makeMemoryStore()
-        const store = yield* $(make("test", Effect.sync(() => existing)))
-
-        const item1 = { id: "1", a: "string", _etag: undefined }
-        const item2 = { id: "2", a: "string", _etag: undefined }
-
-        yield* $(store.batchSet([item1, item2]))
-
-        const [result1, result2] = yield* $(
-          Effect.collectAll([store.find("1"), store.find("2")])
-        )
-        const failResult = yield* $(
-          store.batchSet([{ id: "1", a: "another-string3", _etag: "abc" }]).result
-        )
-        expect(result1).toEqual(Option.some({ ...item1, _etag: expect.any(String) }))
-        expect(result2).toEqual(Option.some({ ...item2, _etag: expect.any(String) }))
-        expect(Exit.untraced(failResult)).toEqual(
-          Exit.fail(new OptimisticConcurrencyException("test", "1"))
-        )
-      }))
-  })
-})