@effect-app/infra 4.0.0-beta.22 → 4.0.0-beta.221

package/src/Model/query/new-kid-interpreter.ts

@@ -1,13 +1,94 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { Array, identity, Match, Option, pipe, S } from "effect-app"
+import * as Array from "effect-app/Array"
 import { toNonEmptyArray } from "effect-app/Array"
+import * as Option from "effect-app/Option"
+import * as S from "effect-app/Schema"
 import { dropUndefinedT } from "effect-app/utils"
+import { identity, pipe } from "effect/Function"
+import * as Match from "effect/Match"
+import * as SchemaAST from "effect/SchemaAST"
 import type { FilterResult } from "../filter/filterApi.js"
 import type { FieldValues } from "../filter/types.js"
 import type { FieldPath } from "../filter/types/path/eager.js"
 import { make, type Q, type QAll } from "../query/dsl.js"
 
+export type ComputedProjectionMathIrExpression =
+  | {
+    readonly _tag: "field"
+    readonly field: string
+  }
+  | {
+    readonly _tag: "mul"
+    readonly left: ComputedProjectionMathIrExpression
+    readonly right: ComputedProjectionMathIrExpression
+  }
+
+export type ComputedProjectionIrExpression =
+  | {
+    readonly _tag: "relation-count"
+    readonly path: string
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-any"
+    readonly path: string
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-every"
+    readonly path: string
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-distinct-count"
+    readonly path: string
+    readonly field: string
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-sum"
+    readonly path: string
+    readonly field: string
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-sum-expr"
+    readonly path: string
+    readonly expression: ComputedProjectionMathIrExpression
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-sum-expr-by"
+    readonly path: string
+    readonly expression: ComputedProjectionMathIrExpression
+    readonly unit: string
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-sum-expr-normalized"
+    readonly path: string
+    readonly expression: ComputedProjectionMathIrExpression
+    readonly unit: string
+    readonly toBase: string
+    readonly factors: Readonly<Record<string, number>>
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-collect"
+    readonly path: string
+    readonly field: string
+    readonly distinct: boolean
+    readonly filter: readonly FilterResult[]
+  }
+  | {
+    readonly _tag: "relation-collect-fields"
+    readonly path: string
+    readonly fields: readonly string[]
+    readonly distinct: boolean
+    readonly filter: readonly FilterResult[]
+  }
+
 type Result<TFieldValues extends FieldValues, A = TFieldValues, R = never> = {
   filter: FilterResult[]
   schema: S.Codec<A, TFieldValues, R> | undefined
@@ -16,6 +97,7 @@ type Result<TFieldValues extends FieldValues, A = TFieldValues, R = never> = {
   order: { key: FieldPath<TFieldValues>; direction: "ASC" | "DESC" }[]
   ttype: "one" | "many" | "count" | undefined
   mode: "collect" | "project" | "transform" | undefined
+  computed: Record<string, ComputedProjectionIrExpression> | undefined
 }
 
 const interpret = <
@@ -33,7 +115,8 @@ const interpret = <
     skip: undefined,
     order: [],
     ttype: undefined,
-    mode: undefined
+    mode: undefined,
+    computed: undefined
   }
 
   const upd = (
@@ -46,6 +129,7 @@ const interpret = <
     if (v.ttype !== undefined) data.ttype = v.ttype
     if (v.schema !== undefined) data.schema = v.schema
     if (v.mode !== undefined) data.mode = v.mode
+    if (v.computed !== undefined) data.computed = v.computed
   }
 
   const applyPath = (path: string) => (_: FilterResult): FilterResult =>
@@ -135,8 +219,84 @@ const interpret = <
       },
       project: (v) => {
         upd(interpret(v.current))
+        if (v.computed && v.mode === "transform") {
+          throw new Error("Computed projections require mode 'project' or 'collect', not 'transform'")
+        }
         data.schema = v.schema
-        data.mode = v.mode
+        data.mode = v.computed
+          ? v.mode === "collect" ? "collect" : "project"
+          : v.mode
+        data.computed = v.computed
+          ? Object.fromEntries(
+            Object.entries(v.computed).map(([key, expression]) => {
+              const e = expression
+              const filter = e.operation ? interpret(e.operation(make())).filter.map(applyPath(e.path)) : []
+              switch (e._tag) {
+                case "relation-count":
+                case "relation-any":
+                case "relation-every":
+                  return [key, { _tag: e._tag, path: e.path, filter } as ComputedProjectionIrExpression]
+                case "relation-distinct-count":
+                case "relation-sum":
+                  return [
+                    key,
+                    { _tag: e._tag, path: e.path, field: e.field, filter } as ComputedProjectionIrExpression
+                  ]
+                case "relation-sum-expr":
+                  return [
+                    key,
+                    { _tag: e._tag, path: e.path, expression: e.expression, filter } as ComputedProjectionIrExpression
+                  ]
+                case "relation-sum-expr-by":
+                  return [
+                    key,
+                    {
+                      _tag: e._tag,
+                      path: e.path,
+                      expression: e.expression,
+                      unit: e.unit,
+                      filter
+                    } as ComputedProjectionIrExpression
+                  ]
+                case "relation-sum-expr-normalized":
+                  return [
+                    key,
+                    {
+                      _tag: e._tag,
+                      path: e.path,
+                      expression: e.expression,
+                      unit: e.unit,
+                      toBase: e.toBase,
+                      factors: e.factors,
+                      filter
+                    } as ComputedProjectionIrExpression
+                  ]
+                case "relation-collect":
+                  return [
+                    key,
+                    {
+                      _tag: e._tag,
+                      path: e.path,
+                      field: e.field,
+                      distinct: e.distinct,
+                      filter
+                    } as ComputedProjectionIrExpression
+                  ]
+                case "relation-collect-fields":
+                  return [
+                    key,
+                    {
+                      _tag: e._tag,
+                      path: e.path,
+                      fields: e.fields,
+                      distinct: e.distinct,
+                      filter
+                    } as ComputedProjectionIrExpression
+                  ]
+              }
+            })
+          )
+          : undefined
       }
     })
   )
@@ -157,15 +317,19 @@ export const toFilter = <
   R,
   TFieldValuesRefined extends TFieldValues = TFieldValues
 >(
-  q: QAll<TFieldValues, TFieldValuesRefined, A, R>
+  q: QAll<TFieldValues, TFieldValuesRefined, A, R>,
+  baseSchema?: S.Schema<unknown>
 ) => {
   // TODO: Native interpreter for each db adapter, instead of the intermediate "new-kid" format
   const a = interpret(q)
   const schema = a.schema
-  let select: (keyof TFieldValues | { key: string; subKeys: string[] })[] = []
+  let select: (keyof TFieldValues | { key: string; subKeys: string[] } | {
+    key: string
+    computed: ComputedProjectionIrExpression
+  })[] = []
   // TODO: support more complex (nested) schemas?
   if (schema) {
-    const t = walkTransformation(schema.ast)
+    const t = walkTransformation(SchemaAST.toEncoded(schema.ast))
     if (S.AST.isObjects(t)) {
       select = t.propertySignatures.map((_) => _.name as string)
       for (const prop of t.propertySignatures) {
@@ -194,12 +358,53 @@ export const toFilter = <
       }
     }
   }
+  const computed = a.computed
+  const getSelectKey = (_: (typeof select)[number]) => {
+    if (typeof _ === "string") {
+      return _
+    }
+    if (typeof _ === "object" && _ !== null && "key" in _) {
+      return _.key
+    }
+    return String(_)
+  }
+  const schemaKeys = select.map(getSelectKey)
+  const nonEncodedSchemaKeys = (() => {
+    if (!baseSchema) {
+      return [] as string[]
+    }
+    const encoded = walkTransformation(SchemaAST.toEncoded(baseSchema.ast))
+    if (!S.AST.isObjects(encoded)) {
+      return [] as string[]
+    }
+    const encodedKeys = encoded.propertySignatures.map((_) => _.name as string)
+    return schemaKeys.filter((key) => !encodedKeys.includes(key))
+  })()
+  const missingComputedKeys = nonEncodedSchemaKeys.filter((key) => !(computed && key in computed))
+
+  if (Array.isArrayNonEmpty(missingComputedKeys)) {
+    throw new Error(`Missing computed projections for schema keys: ${missingComputedKeys.join(", ")}`)
+  }
+
+  if (computed) {
+    const computedKeys = Object.keys(computed)
+    const extraComputedKeys = computedKeys.filter((key) => !schemaKeys.includes(key))
+    if (Array.isArrayNonEmpty(extraComputedKeys)) {
+      throw new Error(`Computed projection keys must exist in projection schema: ${extraComputedKeys.join(", ")}`)
+    }
+    select = select.filter((_) => {
+      const key = getSelectKey(_)
+      return !(key in computed)
+    })
+    select.push(...Object.entries(computed).map(([key, expression]) => ({ key, computed: expression })))
+  }
   return dropUndefinedT({
     t: null as unknown as TFieldValues,
     limit: a.limit,
     skip: a.skip,
     select: Option.getOrUndefined(toNonEmptyArray(select)),
     schema,
+    computed,
     order: Option.getOrUndefined(toNonEmptyArray(a.order)),
     ttype: a.ttype,
     mode: a.mode ?? "transform",

package/src/Model.ts

@@ -1,3 +1,4 @@
 export * from "./Model/dsl.js"
 export * as Q from "./Model/query.js"
 export { makeRepo } from "./Model/Repository.js"
+export { type RegisteredRepository, RepositoryRegistry, RepositoryRegistryLive } from "./Model/Repository.js"

package/src/QueueMaker/SQLQueue.ts

@@ -2,18 +2,24 @@ import { getRequestContext, setupRequestContextWithCustomSpan } from "@effect-ap
 import { reportNonInterruptedFailure } from "@effect-app/infra/QueueMaker/errors"
 import { type QueueBase, QueueMeta } from "@effect-app/infra/QueueMaker/service"
 import { subMinutes } from "date-fns"
-import { Effect, Fiber, type NonEmptyReadonlyArray, Option, S, Tracer } from "effect-app"
-import type { NonEmptyString255 } from "effect-app/Schema"
+import type { NonEmptyReadonlyArray } from "effect-app/Array"
+import * as Effect from "effect-app/Effect"
+import * as Option from "effect-app/Option"
+import * as S from "effect-app/Schema"
+import { type NonEmptyString255 } from "effect-app/Schema"
 import { pretty } from "effect-app/utils"
+import * as Fiber from "effect/Fiber"
+import * as Tracer from "effect/Tracer"
 import { SqlClient } from "effect/unstable/sql"
 import { SQLModel } from "../adapters/SQL.js"
 import { InfraLogger } from "../logger.js"
+import { messagingSpanArgs } from "../otel.js"
 
-export const QueueId = S.Number.pipe(S.brand("QueueId"))
+export const QueueId = S.Finite.pipe(S.brand("QueueId"))
 export type QueueId = typeof QueueId.Type
 
 // TODO: let the model track and Auto Generate versionColumn on every update instead
-export function makeSQLQueue<
+export const makeSQLQueue = Effect.fnUntraced(function*<
   Evt extends { id: S.StringId; _tag: string },
   DrainEvt extends { id: S.StringId; _tag: string },
   EvtE,
@@ -24,166 +30,157 @@ export function makeSQLQueue<
   schema: S.Codec<Evt, EvtE>,
   drainSchema: S.Codec<DrainEvt, DrainEvtE>
 ) {
-  return Effect.gen(function*() {
-    const base = {
-      id: SQLModel.Generated(QueueId),
-      meta: SQLModel.JsonFromString(QueueMeta),
-      name: S.NonEmptyString255,
-      createdAt: SQLModel.DateTimeInsert,
-      updatedAt: SQLModel.DateTimeUpdate,
-      // TODO: at+owner
-      processingAt: SQLModel.FieldOption(S.Date),
-      finishedAt: SQLModel.FieldOption(S.Date),
-      etag: S.String // TODO: use a SQLModel thing that auto updates it?
-      // TODO: record locking.. / optimistic locking
-      // rowVersion: SQLModel.DateTimeFromNumberWithNow
-    }
-    class Queue extends SQLModel.Class<Queue>("Queue")({
-      body: SQLModel.JsonFromString(schema),
-      ...base
-    }) {}
-    class Drain extends SQLModel.Class<Drain>("Drain")({
-      body: SQLModel.JsonFromString(drainSchema),
-      ...base
-    }) {}
-    const sql = yield* SqlClient.SqlClient
+  const base = {
+    id: SQLModel.Generated(QueueId),
+    meta: SQLModel.JsonFromString(QueueMeta),
+    name: S.NonEmptyString255,
+    createdAt: SQLModel.DateTimeInsert,
+    updatedAt: SQLModel.DateTimeUpdate,
+    // TODO: at+owner
+    processingAt: SQLModel.FieldOption(S.Date),
+    finishedAt: SQLModel.FieldOption(S.Date),
+    etag: S.String // TODO: use a SQLModel thing that auto updates it?
+    // TODO: record locking.. / optimistic locking
+    // rowVersion: SQLModel.DateTimeFromNumberWithNow
+  }
+  class Queue extends SQLModel.Class<Queue>("Queue")({
+    body: SQLModel.JsonFromString(schema),
+    ...base
+  }) {}
+  class Drain extends SQLModel.Class<Drain>("Drain")({
+    body: SQLModel.JsonFromString(drainSchema),
+    ...base
+  }) {}
+  const sql = yield* SqlClient.SqlClient
 
-    const queueRepo = yield* SQLModel.makeRepository(Queue, {
-      tableName: "queue",
-      spanPrefix: "QueueRepo",
-      idColumn: "id",
-      versionColumn: "etag"
-    })
+  const queueRepo = yield* SQLModel.makeRepository(Queue, {
+    tableName: "queue",
+    spanPrefix: "QueueRepo",
+    idColumn: "id",
+    versionColumn: "etag"
+  })
 
-    const drainRepo = yield* SQLModel.makeRepository(Drain, {
-      tableName: "queue",
-      spanPrefix: "DrainRepo",
-      idColumn: "id",
-      versionColumn: "etag"
-    })
+  const drainRepo = yield* SQLModel.makeRepository(Drain, {
+    tableName: "queue",
+    spanPrefix: "DrainRepo",
+    idColumn: "id",
+    versionColumn: "etag"
+  })
 
-    const decodeDrain = S.decodeEffect(Drain)
+  const decodeDrain = S.decodeEffectConcurrently(Drain)
 
-    const drain = Effect
-      .sync(() => subMinutes(new Date(), 15))
-      .pipe(
-        Effect
-          .andThen((limit) =>
-            sql<typeof Drain.Encoded>`SELECT *
+  const drain = Effect.gen(function*() {
+    const limit = subMinutes(new Date(), 15)
+    return yield* sql<typeof Drain.Encoded>`SELECT *
     FROM queue
     WHERE name = ${queueDrainName} AND finishedAt IS NULL AND (processingAt IS NULL OR processingAt < ${limit.getTime()})
     LIMIT 1`
-          )
-      )
+  })
 
-    const q = {
-      offer: Effect.fnUntraced(function*(body: Evt, meta: typeof QueueMeta.Type) {
-        yield* queueRepo.insertVoid({
-          body,
-          meta,
-          name: queueName,
-          processingAt: Option.none(),
-          finishedAt: Option.none(),
-          etag: crypto.randomUUID()
-        })
-      }),
-      take: Effect.gen(function*() {
-        while (true) {
-          const [first] = yield* drain.pipe(Effect.withTracerEnabled(false)) // disable sql tracer otherwise we spam it..
-          if (first) {
-            const dec = yield* decodeDrain(first)
-            const { createdAt, updatedAt, ...rest } = dec
-            return yield* drainRepo.update(
-              { ...rest, processingAt: Option.some(new Date()) } // auto in lib , etag: crypto.randomUUID()
-            )
-          }
-          if (first) return first
-          yield* Effect.sleep(250)
+  const q = {
+    offer: Effect.fnUntraced(function*(body: Evt, meta: typeof QueueMeta.Type) {
+      yield* queueRepo.insertVoid(Queue.insert.make({
+        body,
+        meta,
+        name: queueName,
+        processingAt: Option.none(),
+        finishedAt: Option.none(),
+        etag: crypto.randomUUID()
+      }))
+    }),
+    take: Effect.gen(function*() {
+      while (true) {
+        const [first] = yield* drain.pipe(Effect.withTracerEnabled(false)) // disable sql tracer otherwise we spam it..
+        if (first) {
+          const dec = yield* decodeDrain(first)
+          const { createdAt, updatedAt, ...rest } = dec
+          return yield* drainRepo.update(
+            Drain.update.make({ ...rest, processingAt: Option.some(new Date()) }) // auto in lib , etag: crypto.randomUUID()
+          )
         }
-      }),
-      finish: ({ createdAt, updatedAt, ...q }: Drain) =>
-        drainRepo.updateVoid({ ...q, finishedAt: Option.some(new Date()) }) // auto in lib , etag: crypto.randomUUID()
-    }
-    const queue = {
-      publish: (...messages: NonEmptyReadonlyArray<Evt>) =>
-        getRequestContext
+        if (first) return first
+        yield* Effect.sleep(250)
+      }
+    }),
+    finish: Effect.fn(function*({ createdAt, updatedAt, ...q }: Drain) {
+      return yield* drainRepo.updateVoid(Drain.update.make({ ...q, finishedAt: Option.some(new Date()) })) // auto in lib , etag: crypto.randomUUID()
+    })
+  }
+  const queue = {
+    publish: Effect.fn(`publish ${queueName}`, {
+      kind: "producer",
+      attributes: {
+        "messaging.system": "sql",
+        "messaging.operation.name": "publish",
+        "messaging.destination.name": queueName
+      }
+    })(function*(
+      ...messages: NonEmptyReadonlyArray<Evt>
+    ) {
+      yield* Effect.annotateCurrentSpan({
+        "messaging.batch.message_count": messages.length,
+        "messaging.message.types": messages.map((_) => _._tag)
+      })
+      const requestContext = yield* getRequestContext
+      yield* Effect.forEach(messages, (m) => q.offer(m, requestContext), { discard: true })
+    }),
+    drain: <DrainE, DrainR>(
+      handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
+      sessionId?: string
+    ) => {
+      const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
+      const processMessage = Effect.fnUntraced(function*({ body, meta }: Drain) {
+        let effect = InfraLogger
+          .logDebug(`[${queueDrainName}] Processing incoming message`)
           .pipe(
-            Effect.flatMap((requestContext) =>
-              Effect
-                .forEach(
-                  messages,
-                  (m) => q.offer(m, requestContext),
-                  {
-                    discard: true
-                  }
-                )
-            ),
-            Effect.withSpan("queue.publish: " + queueName, {
-              kind: "producer",
-              attributes: { "message_tags": messages.map((_) => _._tag) }
-            }, { captureStackTrace: false })
-          ),
-      drain: <DrainE, DrainR>(
-        handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
-        sessionId?: string
-      ) => {
-        const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
-        const processMessage = (msg: Drain) =>
-          Effect
-            .succeed(msg)
-            .pipe(Effect
-              .flatMap(({ body, meta }) => {
-                let effect = InfraLogger
-                  .logDebug(`[${queueDrainName}] Processing incoming message`)
-                  .pipe(
-                    Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
-                    Effect.andThen(handleEvent(body)),
-                    silenceAndReportError,
-                    (_) =>
-                      setupRequestContextWithCustomSpan(
-                        _,
-                        meta,
-                        `queue.drain: ${queueDrainName}.${body._tag}`,
-                        {
-                          captureStackTrace: false,
-                          kind: "consumer",
-                          attributes: {
-                            "queue.name": queueDrainName,
-                            "queue.sessionId": sessionId,
-                            "queue.input": body
-                          }
-                        }
-                      )
-                  )
-                if (meta.span) {
-                  effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
+            Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
+            Effect.andThen(handleEvent(body)),
+            silenceAndReportError,
+            (_) => {
+              const args = messagingSpanArgs({
+                operation: "process",
+                system: "sql",
+                destination: queueDrainName,
+                messageId: body.id,
+                conversationId: sessionId,
+                extra: { "messaging.message.type": body._tag, "messaging.message.body": body }
+              }, "consumer")
+              return setupRequestContextWithCustomSpan(
+                _,
+                meta,
+                args.name,
+                {
+                  captureStackTrace: false,
+                  kind: args.kind,
+                  attributes: args.attributes
                 }
-                return effect
-              }))
-
-        return q
-          .take
-          .pipe(
-            Effect.flatMap((x) =>
-              processMessage(x).pipe(
-                Effect.uninterruptible,
-                Effect.forkChild,
-                Effect.flatMap(Fiber.join),
-                Effect.tap(q.finish(x))
               )
-            ),
-            silenceAndReportError,
-            Effect.withSpan(`queue.drain: ${queueDrainName}`, {
-              attributes: {
-                "queue.type": "sql",
-                "queue.name": queueDrainName,
-                "queue.sessionId": sessionId
-              }
-            }),
-            Effect.forever
+            }
           )
-      }
+        if (meta.span) {
+          effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
+        }
+        return yield* effect
+      })
+
+      return Effect.fn(`receive ${queueDrainName}`, {
+        kind: "consumer",
+        attributes: {
+          "messaging.system": "sql",
+          "messaging.operation.name": "receive",
+          "messaging.destination.name": queueDrainName,
+          ...(sessionId !== undefined && { "messaging.message.conversation_id": sessionId })
+        }
+      })(function*() {
+        const x = yield* q.take
+        yield* processMessage(x).pipe(
+          Effect.uninterruptible,
+          Effect.forkChild,
+          Effect.flatMap(Fiber.join),
+          Effect.tap(q.finish(x))
+        )
+      }, (effect) => effect.pipe(silenceAndReportError, Effect.forever))()
     }
-    return queue as QueueBase<Evt, DrainEvt>
-  })
-}
+  }
+  return queue as QueueBase<Evt, DrainEvt>
+})

package/src/QueueMaker/errors.ts

@@ -1,5 +1,7 @@
 import { reportError } from "@effect-app/infra/errorReporter"
-import { Cause, Effect, Exit } from "effect-app"
+import * as Effect from "effect-app/Effect"
+import * as Cause from "effect/Cause"
+import * as Exit from "effect/Exit"
 
 const reportQueueError_ = reportError("Queue")