@effect-app/infra 4.0.0-beta.21 → 4.0.0-beta.210

package/src/QueueMaker/SQLQueue.ts

@@ -3,17 +3,18 @@ import { reportNonInterruptedFailure } from "@effect-app/infra/QueueMaker/errors
 import { type QueueBase, QueueMeta } from "@effect-app/infra/QueueMaker/service"
 import { subMinutes } from "date-fns"
 import { Effect, Fiber, type NonEmptyReadonlyArray, Option, S, Tracer } from "effect-app"
-import type { NonEmptyString255 } from "effect-app/Schema"
+import { type NonEmptyString255 } from "effect-app/Schema"
 import { pretty } from "effect-app/utils"
 import { SqlClient } from "effect/unstable/sql"
 import { SQLModel } from "../adapters/SQL.js"
 import { InfraLogger } from "../logger.js"
+import { messagingSpanArgs } from "../otel.js"
 
-export const QueueId = S.Number.pipe(S.brand("QueueId"))
+export const QueueId = S.Finite.pipe(S.brand("QueueId"))
 export type QueueId = typeof QueueId.Type
 
 // TODO: let the model track and Auto Generate versionColumn on every update instead
-export function makeSQLQueue<
+export const makeSQLQueue = Effect.fnUntraced(function*<
   Evt extends { id: S.StringId; _tag: string },
   DrainEvt extends { id: S.StringId; _tag: string },
   EvtE,
@@ -24,166 +25,157 @@ export function makeSQLQueue<
   schema: S.Codec<Evt, EvtE>,
   drainSchema: S.Codec<DrainEvt, DrainEvtE>
 ) {
-  return Effect.gen(function*() {
-    const base = {
-      id: SQLModel.Generated(QueueId),
-      meta: SQLModel.JsonFromString(QueueMeta),
-      name: S.NonEmptyString255,
-      createdAt: SQLModel.DateTimeInsert,
-      updatedAt: SQLModel.DateTimeUpdate,
-      // TODO: at+owner
-      processingAt: SQLModel.FieldOption(S.Date),
-      finishedAt: SQLModel.FieldOption(S.Date),
-      etag: S.String // TODO: use a SQLModel thing that auto updates it?
-      // TODO: record locking.. / optimistic locking
-      // rowVersion: SQLModel.DateTimeFromNumberWithNow
-    }
-    class Queue extends SQLModel.Class<Queue>("Queue")({
-      body: SQLModel.JsonFromString(schema),
-      ...base
-    }) {}
-    class Drain extends SQLModel.Class<Drain>("Drain")({
-      body: SQLModel.JsonFromString(drainSchema),
-      ...base
-    }) {}
-    const sql = yield* SqlClient.SqlClient
+  const base = {
+    id: SQLModel.Generated(QueueId),
+    meta: SQLModel.JsonFromString(QueueMeta),
+    name: S.NonEmptyString255,
+    createdAt: SQLModel.DateTimeInsert,
+    updatedAt: SQLModel.DateTimeUpdate,
+    // TODO: at+owner
+    processingAt: SQLModel.FieldOption(S.Date),
+    finishedAt: SQLModel.FieldOption(S.Date),
+    etag: S.String // TODO: use a SQLModel thing that auto updates it?
+    // TODO: record locking.. / optimistic locking
+    // rowVersion: SQLModel.DateTimeFromNumberWithNow
+  }
+  class Queue extends SQLModel.Class<Queue>("Queue")({
+    body: SQLModel.JsonFromString(schema),
+    ...base
+  }) {}
+  class Drain extends SQLModel.Class<Drain>("Drain")({
+    body: SQLModel.JsonFromString(drainSchema),
+    ...base
+  }) {}
+  const sql = yield* SqlClient.SqlClient
 
-    const queueRepo = yield* SQLModel.makeRepository(Queue, {
-      tableName: "queue",
-      spanPrefix: "QueueRepo",
-      idColumn: "id",
-      versionColumn: "etag"
-    })
+  const queueRepo = yield* SQLModel.makeRepository(Queue, {
+    tableName: "queue",
+    spanPrefix: "QueueRepo",
+    idColumn: "id",
+    versionColumn: "etag"
+  })
 
-    const drainRepo = yield* SQLModel.makeRepository(Drain, {
-      tableName: "queue",
-      spanPrefix: "DrainRepo",
-      idColumn: "id",
-      versionColumn: "etag"
-    })
+  const drainRepo = yield* SQLModel.makeRepository(Drain, {
+    tableName: "queue",
+    spanPrefix: "DrainRepo",
+    idColumn: "id",
+    versionColumn: "etag"
+  })
 
-    const decodeDrain = S.decodeEffect(Drain)
+  const decodeDrain = S.decodeEffectConcurrently(Drain)
 
-    const drain = Effect
-      .sync(() => subMinutes(new Date(), 15))
-      .pipe(
-        Effect
-          .andThen((limit) =>
-            sql<typeof Drain.Encoded>`SELECT *
+  const drain = Effect.gen(function*() {
+    const limit = subMinutes(new Date(), 15)
+    return yield* sql<typeof Drain.Encoded>`SELECT *
     FROM queue
     WHERE name = ${queueDrainName} AND finishedAt IS NULL AND (processingAt IS NULL OR processingAt < ${limit.getTime()})
     LIMIT 1`
-          )
-      )
+  })
 
-    const q = {
-      offer: Effect.fnUntraced(function*(body: Evt, meta: typeof QueueMeta.Type) {
-        yield* queueRepo.insertVoid({
-          body,
-          meta,
-          name: queueName,
-          processingAt: Option.none(),
-          finishedAt: Option.none(),
-          etag: crypto.randomUUID()
-        })
-      }),
-      take: Effect.gen(function*() {
-        while (true) {
-          const [first] = yield* drain.pipe(Effect.withTracerEnabled(false)) // disable sql tracer otherwise we spam it..
-          if (first) {
-            const dec = yield* decodeDrain(first)
-            const { createdAt, updatedAt, ...rest } = dec
-            return yield* drainRepo.update(
-              { ...rest, processingAt: Option.some(new Date()) } // auto in lib , etag: crypto.randomUUID()
-            )
-          }
-          if (first) return first
-          yield* Effect.sleep(250)
+  const q = {
+    offer: Effect.fnUntraced(function*(body: Evt, meta: typeof QueueMeta.Type) {
+      yield* queueRepo.insertVoid(Queue.insert.make({
+        body,
+        meta,
+        name: queueName,
+        processingAt: Option.none(),
+        finishedAt: Option.none(),
+        etag: crypto.randomUUID()
+      }))
+    }),
+    take: Effect.gen(function*() {
+      while (true) {
+        const [first] = yield* drain.pipe(Effect.withTracerEnabled(false)) // disable sql tracer otherwise we spam it..
+        if (first) {
+          const dec = yield* decodeDrain(first)
+          const { createdAt, updatedAt, ...rest } = dec
+          return yield* drainRepo.update(
+            Drain.update.make({ ...rest, processingAt: Option.some(new Date()) }) // auto in lib , etag: crypto.randomUUID()
+          )
         }
-      }),
-      finish: ({ createdAt, updatedAt, ...q }: Drain) =>
-        drainRepo.updateVoid({ ...q, finishedAt: Option.some(new Date()) }) // auto in lib , etag: crypto.randomUUID()
-    }
-    const queue = {
-      publish: (...messages: NonEmptyReadonlyArray<Evt>) =>
-        getRequestContext
+        if (first) return first
+        yield* Effect.sleep(250)
+      }
+    }),
+    finish: Effect.fn(function*({ createdAt, updatedAt, ...q }: Drain) {
+      return yield* drainRepo.updateVoid(Drain.update.make({ ...q, finishedAt: Option.some(new Date()) })) // auto in lib , etag: crypto.randomUUID()
+    })
+  }
+  const queue = {
+    publish: Effect.fn(`publish ${queueName}`, {
+      kind: "producer",
+      attributes: {
+        "messaging.system": "sql",
+        "messaging.operation.name": "publish",
+        "messaging.destination.name": queueName
+      }
+    })(function*(
+      ...messages: NonEmptyReadonlyArray<Evt>
+    ) {
+      yield* Effect.annotateCurrentSpan({
+        "messaging.batch.message_count": messages.length,
+        "messaging.message.types": messages.map((_) => _._tag)
+      })
+      const requestContext = yield* getRequestContext
+      yield* Effect.forEach(messages, (m) => q.offer(m, requestContext), { discard: true })
+    }),
+    drain: <DrainE, DrainR>(
+      handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
+      sessionId?: string
+    ) => {
+      const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
+      const processMessage = Effect.fnUntraced(function*({ body, meta }: Drain) {
+        let effect = InfraLogger
+          .logDebug(`[${queueDrainName}] Processing incoming message`)
           .pipe(
-            Effect.flatMap((requestContext) =>
-              Effect
-                .forEach(
-                  messages,
-                  (m) => q.offer(m, requestContext),
-                  {
-                    discard: true
-                  }
-                )
-            ),
-            Effect.withSpan("queue.publish: " + queueName, {
-              kind: "producer",
-              attributes: { "message_tags": messages.map((_) => _._tag) }
-            }, { captureStackTrace: false })
-          ),
-      drain: <DrainE, DrainR>(
-        handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
-        sessionId?: string
-      ) => {
-        const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
-        const processMessage = (msg: Drain) =>
-          Effect
-            .succeed(msg)
-            .pipe(Effect
-              .flatMap(({ body, meta }) => {
-                let effect = InfraLogger
-                  .logDebug(`[${queueDrainName}] Processing incoming message`)
-                  .pipe(
-                    Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
-                    Effect.andThen(handleEvent(body)),
-                    silenceAndReportError,
-                    (_) =>
-                      setupRequestContextWithCustomSpan(
-                        _,
-                        meta,
-                        `queue.drain: ${queueDrainName}.${body._tag}`,
-                        {
-                          captureStackTrace: false,
-                          kind: "consumer",
-                          attributes: {
-                            "queue.name": queueDrainName,
-                            "queue.sessionId": sessionId,
-                            "queue.input": body
-                          }
-                        }
-                      )
-                  )
-                if (meta.span) {
-                  effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
+            Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
+            Effect.andThen(handleEvent(body)),
+            silenceAndReportError,
+            (_) => {
+              const args = messagingSpanArgs({
+                operation: "process",
+                system: "sql",
+                destination: queueDrainName,
+                messageId: body.id,
+                conversationId: sessionId,
+                extra: { "messaging.message.type": body._tag, "messaging.message.body": body }
+              }, "consumer")
+              return setupRequestContextWithCustomSpan(
+                _,
+                meta,
+                args.name,
+                {
+                  captureStackTrace: false,
+                  kind: args.kind,
+                  attributes: args.attributes
                 }
-                return effect
-              }))
-
-        return q
-          .take
-          .pipe(
-            Effect.flatMap((x) =>
-              processMessage(x).pipe(
-                Effect.uninterruptible,
-                Effect.forkChild,
-                Effect.flatMap(Fiber.join),
-                Effect.tap(q.finish(x))
               )
-            ),
-            silenceAndReportError,
-            Effect.withSpan(`queue.drain: ${queueDrainName}`, {
-              attributes: {
-                "queue.type": "sql",
-                "queue.name": queueDrainName,
-                "queue.sessionId": sessionId
-              }
-            }),
-            Effect.forever
+            }
           )
-      }
+        if (meta.span) {
+          effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
+        }
+        return yield* effect
+      })
+
+      return Effect.fn(`receive ${queueDrainName}`, {
+        kind: "consumer",
+        attributes: {
+          "messaging.system": "sql",
+          "messaging.operation.name": "receive",
+          "messaging.destination.name": queueDrainName,
+          ...(sessionId !== undefined && { "messaging.message.conversation_id": sessionId })
+        }
+      })(function*() {
+        const x = yield* q.take
+        yield* processMessage(x).pipe(
+          Effect.uninterruptible,
+          Effect.forkChild,
+          Effect.flatMap(Fiber.join),
+          Effect.tap(q.finish(x))
+        )
+      }, (effect) => effect.pipe(silenceAndReportError, Effect.forever))()
     }
-    return queue as QueueBase<Evt, DrainEvt>
-  })
-}
+  }
+  return queue as QueueBase<Evt, DrainEvt>
+})

package/src/QueueMaker/memQueue.ts

@@ -5,10 +5,11 @@ import * as Q from "effect/Queue"
 import { MemQueue } from "../adapters/memQueue.js"
 import { getRequestContext, setupRequestContextWithCustomSpan } from "../api/setupRequest.js"
 import { InfraLogger } from "../logger.js"
+import { messagingSpanArgs } from "../otel.js"
 import { reportNonInterruptedFailure, reportNonInterruptedFailureCause } from "./errors.js"
-import { type QueueBase, QueueMeta } from "./service.js"
+import { QueueMeta } from "./service.js"
 
-export function makeMemQueue<
+export const makeMemQueue = Effect.fnUntraced(function*<
   Evt extends { id: S.StringId; _tag: string },
   DrainEvt extends { id: S.StringId; _tag: string },
   EvtE,
@@ -19,112 +20,112 @@ export function makeMemQueue<
   schema: S.Codec<Evt, EvtE>,
   drainSchema: S.Codec<DrainEvt, DrainEvtE>
 ) {
-  return Effect.gen(function*() {
-    const mem = yield* MemQueue
-    const q = yield* mem.getOrCreateQueue(queueName)
-    const qDrain = yield* mem.getOrCreateQueue(queueDrainName)
+  const mem = yield* MemQueue
+  const q = yield* mem.getOrCreateQueue(queueName)
+  const qDrain = yield* mem.getOrCreateQueue(queueDrainName)
 
-    const wireSchema = S.Struct({ body: schema, meta: QueueMeta })
-    const wireSchemaJson = S.fromJsonString(wireSchema)
-    const encodePublish = S.encodeEffect(wireSchemaJson)
-    const drainW = S.Struct({ body: drainSchema, meta: QueueMeta })
-    const drainWJson = S.fromJsonString(drainW)
+  const wireSchema = S.Struct({ body: schema, meta: QueueMeta })
+  const wireSchemaJson = S.fromJsonString(S.toCodecJson(wireSchema))
+  const encodePublish = S.encodeEffect(wireSchemaJson)
+  const drainW = S.Struct({ body: drainSchema, meta: QueueMeta })
+  const drainWJson = S.fromJsonString(S.toCodecJson(drainW))
 
-    const parseDrain = flow(S.decodeUnknownEffect(drainWJson), Effect.orDie)
+  const parseDrain = flow(S.decodeUnknownEffectConcurrently(drainWJson), Effect.orDie)
 
-    const queue = {
-      publish: (...messages: NonEmptyReadonlyArray<Evt>) =>
-        getRequestContext
-          .pipe(
-            Effect.flatMap((requestContext) =>
-              Effect
-                .forEach(messages, (m) =>
-                  // we JSON encode, because that is what the wire also does, and it reveals holes in e.g unknown encoders (Date->String)
-                  encodePublish({ body: m, meta: requestContext }).pipe(
-                    Effect.orDie,
-                    // .tap((msg) => info("Publishing Mem Message: " + utils.inspect(msg)))
-                    Effect.flatMap((_) => Q.offer(q, _))
-                  ), { discard: true })
-            ),
-            Effect.withSpan("queue.publish: " + queueName, {
-              kind: "producer",
-              attributes: { "message_tags": messages.map((_) => _._tag) }
-            }, { captureStackTrace: false })
-          ),
-      drain: <DrainE, DrainR>(
-        handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
-        sessionId?: string
-      ) => {
-        const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
-        const reportError = reportNonInterruptedFailureCause({ name: "MemQueue.drain." + queueDrainName })
-        const processMessage = (msg: string) =>
-          // we JSON parse, because that is what the wire also does, and it reveals holes in e.g unknown encoders (Date->String)
-          parseDrain(msg).pipe(
+  const queue = {
+    publish: Effect.fn(`publish ${queueName}`, {
+      kind: "producer",
+      attributes: {
+        "messaging.system": "memory",
+        "messaging.operation.name": "publish",
+        "messaging.destination.name": queueName
+      }
+    })(function*(
+      ...messages: NonEmptyReadonlyArray<Evt>
+    ) {
+      yield* Effect.annotateCurrentSpan({
+        "messaging.batch.message_count": messages.length,
+        "messaging.message.types": messages.map((_) => _._tag)
+      })
+      const requestContext = yield* getRequestContext
+      // we JSON encode, because that is what the wire also does, and it reveals holes in e.g unknown encoders (Date->String)
+      yield* Effect.forEach(
+        messages,
+        (m) =>
+          encodePublish({ body: m, meta: requestContext }).pipe(
             Effect.orDie,
-            Effect
-              .flatMap(({ body, meta }) => {
-                let effect = InfraLogger
-                  .logDebug(`[${queueDrainName}] Processing incoming message`)
-                  .pipe(
-                    Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
-                    Effect.andThen(handleEvent(body)),
-                    silenceAndReportError,
-                    (_) =>
-                      setupRequestContextWithCustomSpan(
-                        _,
-                        meta,
-                        `queue.drain: ${queueDrainName}.${body._tag}`,
-                        {
-                          captureStackTrace: false,
-                          kind: "consumer",
-                          attributes: {
-                            "queue.name": queueDrainName,
-                            "queue.sessionId": sessionId,
-                            "queue.input": body
-                          }
-                        }
-                      )
-                  )
-                if (meta.span) {
-                  effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
-                }
-                return effect
-              })
-          )
-        return Q
-          .take(qDrain)
+            Effect.flatMap((_) => Q.offer(q, _))
+          ),
+        { discard: true }
+      )
+    }),
+    drain: <DrainE, DrainR>(
+      handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
+      sessionId?: string
+    ) => {
+      const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
+      const reportError = reportNonInterruptedFailureCause({ name: "MemQueue.drain." + queueDrainName })
+      const processMessage = Effect.fnUntraced(function*(msg: string) {
+        // we JSON parse, because that is what the wire also does, and it reveals holes in e.g unknown encoders (Date->String)
+        const { body, meta } = yield* parseDrain(msg).pipe(Effect.orDie)
+        let effect = InfraLogger
+          .logDebug(`[${queueDrainName}] Processing incoming message`)
           .pipe(
-            Effect
-              .flatMap((x) =>
-                processMessage(x).pipe(
-                  Effect.uninterruptible,
-                  Effect.forkChild,
-                  Effect.flatMap(Fiber.join),
-                  // normally a failed item would be returned to the queue and retried up to X times.
-                  Effect.flatMap((_) =>
-                    _._tag === "Failure" && !Cause.hasInterruptsOnly(_.cause)
-                      ? Q.offer(qDrain, x).pipe(
-                        // TODO: retry count tracking and max retries.
-                        Effect.delay("5 seconds"),
-                        Effect.tapCause(reportError),
-                        Effect.forkDetach
-                      )
-                      : Effect.void
-                  )
-                )
-              ),
+            Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
+            Effect.andThen(handleEvent(body)),
             silenceAndReportError,
-            Effect.withSpan(`queue.drain: ${queueDrainName}`, {
-              attributes: {
-                "queue.type": "mem",
-                "queue.name": queueDrainName,
-                "queue.sessionId": sessionId
-              }
-            }),
-            Effect.forever
+            (_) => {
+              const args = messagingSpanArgs({
+                operation: "process",
+                system: "memory",
+                destination: queueDrainName,
+                messageId: body.id,
+                conversationId: sessionId,
+                extra: { "messaging.message.type": body._tag, "messaging.message.body": body }
+              }, "consumer")
+              return setupRequestContextWithCustomSpan(
+                _,
+                meta,
+                args.name,
+                {
+                  captureStackTrace: false,
+                  kind: args.kind,
+                  attributes: args.attributes
+                }
+              )
+            }
           )
-      }
+        if (meta.span) {
+          effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
+        }
+        return yield* effect
+      })
+      return Effect.fn(`receive ${queueDrainName}`, {
+        kind: "consumer",
+        attributes: {
+          "messaging.system": "memory",
+          "messaging.operation.name": "receive",
+          "messaging.destination.name": queueDrainName,
+          ...(sessionId !== undefined && { "messaging.message.conversation_id": sessionId })
+        }
+      })(function*() {
+        const x = yield* Q.take(qDrain)
+        const exit = yield* processMessage(x).pipe(
+          Effect.uninterruptible,
+          Effect.forkChild,
+          Effect.flatMap(Fiber.join)
+        )
+        if (exit._tag === "Failure" && !Cause.hasInterruptsOnly(exit.cause)) {
+          // normally a failed item would be returned to the queue and retried up to X times.
+          yield* Q.offer(qDrain, x).pipe(
+            // TODO: retry count tracking and max retries.
+            Effect.delay("5 seconds"),
+            Effect.tapCause(reportError),
+            Effect.forkDetach
+          )
+        }
+      }, (effect) => effect.pipe(silenceAndReportError, Effect.forever))()
     }
-    return queue as QueueBase<Evt, DrainEvt>
-  })
-}
+  }
+  return queue
+})