@effect-app/infra 4.0.0-beta.31 → 4.0.0-beta.310

package/src/Model/query/new-kid-interpreter.ts

@@ -1,210 +0,0 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
-/* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { Array, identity, Match, Option, pipe, S } from "effect-app"
-import { toNonEmptyArray } from "effect-app/Array"
-import { dropUndefinedT } from "effect-app/utils"
-import type { FilterResult } from "../filter/filterApi.js"
-import type { FieldValues } from "../filter/types.js"
-import type { FieldPath } from "../filter/types/path/eager.js"
-import { make, type Q, type QAll } from "../query/dsl.js"
-
-type Result<TFieldValues extends FieldValues, A = TFieldValues, R = never> = {
-  filter: FilterResult[]
-  schema: S.Codec<A, TFieldValues, R> | undefined
-  limit: number | undefined
-  skip: number | undefined
-  order: { key: FieldPath<TFieldValues>; direction: "ASC" | "DESC" }[]
-  ttype: "one" | "many" | "count" | undefined
-  mode: "collect" | "project" | "transform" | undefined
-}
-
-const interpret = <
-  TFieldValues extends FieldValues,
-  TFieldValuesRefined extends TFieldValues = TFieldValues,
-  A = TFieldValues,
-  R = never
->(_: QAll<TFieldValues, TFieldValuesRefined, A, R>) => {
-  const a = _ as Q<TFieldValues>
-
-  const data: Result<TFieldValues, any, any> = {
-    filter: [],
-    schema: undefined,
-    limit: undefined,
-    skip: undefined,
-    order: [],
-    ttype: undefined,
-    mode: undefined
-  }
-
-  const upd = (
-    v: Result<TFieldValues, any, any>
-  ) => {
-    data.filter.push(...v.filter)
-    data.order.push(...v.order)
-    if (v.limit !== undefined) data.limit = v.limit
-    if (v.skip !== undefined) data.skip = v.skip
-    if (v.ttype !== undefined) data.ttype = v.ttype
-    if (v.schema !== undefined) data.schema = v.schema
-    if (v.mode !== undefined) data.mode = v.mode
-  }
-
-  const applyPath = (path: string) => (_: FilterResult): FilterResult =>
-    _.t === "where" || _.t === "and" || _.t === "or"
-      ? { ..._, path: `${path}.-1.${_.path}` }
-      : { ..._, result: _.result.map(applyPath(path)) }
-
-  pipe(
-    a,
-    Match.valueTags({
-      value: () => {
-        // data.filter.push(value)
-      },
-      where: ({ current, operation, relation, subPath }) => {
-        upd(interpret(current))
-        if (typeof operation === "function") {
-          data.filter.push(
-            {
-              t: "where-scope",
-              result: interpret(operation(make())).filter.map(subPath ? applyPath(subPath) : identity),
-              relation
-            }
-          )
-        } else {
-          data.filter.push(
-            {
-              t: "where",
-              path: operation[0],
-              op: operation.length === 2 ? "eq" : operation[1],
-              value: operation.length === 2 ? operation[1] : operation[2]
-            }
-          )
-        }
-      },
-      and: ({ current, operation, relation }) => {
-        upd(interpret(current))
-        if (typeof operation === "function") {
-          data.filter.push(
-            { t: "and-scope", result: interpret(operation(make())).filter, relation }
-          )
-        } else {
-          data.filter.push(
-            {
-              t: "and",
-              path: operation[0],
-              op: operation.length === 2 ? "eq" : operation[1],
-              value: operation.length === 2 ? operation[1] : operation[2]
-            }
-          )
-        }
-      },
-      or: ({ current, operation, relation }) => {
-        upd(interpret(current))
-        if (typeof operation === "function") {
-          data.filter.push(
-            { t: "or-scope", result: interpret(operation(make())).filter, relation }
-          )
-        } else {
-          data.filter.push(
-            {
-              t: "or",
-              path: operation[0],
-              op: operation.length === 2 ? "eq" : operation[1],
-              value: operation.length === 2 ? operation[1] : operation[2]
-            }
-          )
-        }
-      },
-      one: ({ current }) => {
-        upd(interpret(current))
-        data.limit = 1
-        data.ttype = "one"
-      },
-      count: ({ current }) => {
-        upd(interpret(current))
-        data.ttype = "count"
-        data.schema = S.Struct({ id: S.String }) as any
-      },
-      order: ({ current, direction, field }) => {
-        upd(interpret(current))
-        data.order.push({ key: field, direction })
-      },
-      page: (v) => {
-        upd(interpret(v.current))
-        data.limit = v.take
-        data.skip = v.skip
-      },
-      project: (v) => {
-        upd(interpret(v.current))
-        data.schema = v.schema
-        data.mode = v.mode
-      }
-    })
-  )
-
-  return data
-}
-
-const walkTransformation = (t: S.AST.AST): S.AST.AST => {
-  if (S.AST.isDeclaration(t) && t.typeParameters.length > 0) {
-    return walkTransformation(t.typeParameters[0]!)
-  }
-  return t
-}
-
-export const toFilter = <
-  TFieldValues extends FieldValues,
-  A,
-  R,
-  TFieldValuesRefined extends TFieldValues = TFieldValues
->(
-  q: QAll<TFieldValues, TFieldValuesRefined, A, R>
-) => {
-  // TODO: Native interpreter for each db adapter, instead of the intermediate "new-kid" format
-  const a = interpret(q)
-  const schema = a.schema
-  let select: (keyof TFieldValues | { key: string; subKeys: string[] })[] = []
-  // TODO: support more complex (nested) schemas?
-  if (schema) {
-    const t = walkTransformation(schema.ast)
-    if (S.AST.isObjects(t)) {
-      select = t.propertySignatures.map((_) => _.name as string)
-      for (const prop of t.propertySignatures) {
-        if (S.AST.isArrays(prop.type)) {
-          // make sure we only select when there are actually type literals in the tuple...
-          // otherwise we might be dealing with strings etc.
-          // TODO; be more strict, can't support arrays with unions that have non TypeLiteral members etc..
-          const arraySelect = {
-            key: prop.name as string,
-            subKeys: Array.flatMap(
-              prop.type.rest,
-              (x) => {
-                const t = walkTransformation(x)
-                return S.AST.isObjects(t) ? t.propertySignatures.map((y) => y.name as string) : []
-              }
-            )
-          }
-          if (arraySelect.subKeys.length > 0) {
-            select.push(arraySelect)
-            // make sure we don't double select?
-            if (select.includes(prop.name as string)) {
-              select.splice(select.indexOf(prop.name as string), 1)
-            }
-          }
-        }
-      }
-    }
-  }
-  return dropUndefinedT({
-    t: null as unknown as TFieldValues,
-    limit: a.limit,
-    skip: a.skip,
-    select: Option.getOrUndefined(toNonEmptyArray(select)),
-    schema,
-    order: Option.getOrUndefined(toNonEmptyArray(a.order)),
-    ttype: a.ttype,
-    mode: a.mode ?? "transform",
-    filter: a.filter.length
-      ? a.filter
-      : undefined
-  })
-}

package/src/Model/query.ts

@@ -1,13 +0,0 @@
-export * from "./query/dsl.js"
-export * from "./query/new-kid-interpreter.js"
-
-export interface RawQuery<Encoded, Out> {
-  cosmos: (vals: { name: string }) => {
-    query: string
-    parameters: {
-      name: string
-      value: any
-    }[]
-  }
-  memory: (t: readonly Encoded[]) => readonly Out[]
-}

package/src/Model.ts

@@ -1,3 +0,0 @@
-export * from "./Model/dsl.js"
-export * as Q from "./Model/query.js"
-export { makeRepo } from "./Model/Repository.js"

package/src/Operations.ts

@@ -1,235 +0,0 @@
-import { reportError } from "@effect-app/infra/errorReporter"
-import { subHours } from "date-fns"
-import { Cause, copy, Duration, Effect, Exit, type Fiber, Layer, Option, S, Schedule, ServiceMap } from "effect-app"
-import { annotateLogscoped } from "effect-app/Effect"
-import { dual, pipe } from "effect-app/Function"
-import { Operation, OperationFailure, OperationId, type OperationProgress, OperationSuccess } from "effect-app/Operations"
-import { NonEmptyString2k } from "effect-app/Schema"
-import * as Scope from "effect/Scope"
-import { setupRequestContextFromCurrent } from "./api/setupRequest.js"
-import { MainFiberSet } from "./MainFiberSet.js"
-import { where } from "./Model/query.js"
-import { OperationsRepo } from "./OperationsRepo.js"
-import { batch } from "./rateLimit.js"
-import { RequestFiberSet } from "./RequestFiberSet.js"
-
-const reportAppError = reportError("Operations.Cleanup")
-
-const make = Effect.gen(function*() {
-  const repo = yield* OperationsRepo
-  const reqFiberSet = yield* RequestFiberSet
-  const makeOp = Effect.sync(() => OperationId.make())
-
-  const register = (title: NonEmptyString2k) =>
-    Effect.tap(
-      makeOp,
-      (id) =>
-        Effect.andThen(
-          annotateLogscoped("operationId", id),
-          Effect.acquireRelease(addOp(id, title), (_, exit) => finishOp(id, exit))
-        )
-    )
-
-  const cleanup = Effect.sync(() => subHours(new Date(), 1)).pipe(
-    Effect.andThen((before) => repo.query(where("updatedAt", "lt", before.toISOString()))),
-    Effect.andThen((ops) => pipe(ops, batch(100, Effect.succeed, (items) => repo.removeAndPublish(items)))),
-    setupRequestContextFromCurrent("Operations.cleanup")
-  )
-
-  function addOp(id: OperationId, title: NonEmptyString2k) {
-    return repo.save(new Operation({ id, title })).pipe(Effect.orDie)
-  }
-  function findOp(id: OperationId) {
-    return repo.find(id)
-  }
-  function finishOp(id: OperationId, exit: Exit.Exit<unknown, unknown>) {
-    return Effect
-      .flatMap(repo.get(id).pipe(Effect.orDie), (_) =>
-        repo
-          .save(
-            copy(_, {
-              updatedAt: new Date(),
-              result: Exit.isSuccess(exit)
-                ? new OperationSuccess()
-                : new OperationFailure({
-                  message: Cause.hasInterruptsOnly(exit.cause)
-                    ? NonEmptyString2k("Interrupted")
-                    : Cause.hasDies(exit.cause)
-                    ? NonEmptyString2k("Unknown error")
-                    : Cause
-                      .findErrorOption(exit.cause)
-                      .pipe(
-                        Option.flatMap((_) =>
-                          typeof _ === "object" && _ !== null && "message" in _ && S.is(NonEmptyString2k)(_.message)
-                            ? Option.some(_.message)
-                            : Option.none()
-                        ),
-                        Option.getOrNull
-                      )
-                })
-            })
-          )
-          .pipe(Effect.orDie))
-  }
-  function update(id: OperationId, progress: OperationProgress) {
-    return Effect.flatMap(
-      repo.get(id).pipe(Effect.orDie),
-      (_) => repo.save(copy(_, { updatedAt: new Date(), progress })).pipe(Effect.orDie)
-    )
-  }
-
-  function fork<R, R2, E, E2, A, A2>(
-    self: (id: OperationId) => Effect.Effect<A, E, R>,
-    fnc: (id: OperationId) => Effect.Effect<A2, E2, R2>,
-    title: NonEmptyString2k
-  ): Effect.Effect<
-    RunningOperation<A, E>,
-    never,
-    Exclude<R, Scope.Scope> | Exclude<R2, Scope.Scope>
-  > {
-    return Effect
-      .flatMap(
-        Scope.make(),
-        (scope) =>
-          register(title)
-            .pipe(
-              Scope.provide(scope),
-              Effect.flatMap((id) =>
-                reqFiberSet
-                  .forkDaemonReportUnexpected(Scope.use(
-                    self(id).pipe(Effect.withSpan(title, {}, { captureStackTrace: false })),
-                    scope
-                  ))
-                  .pipe(Effect.map((fiber): RunningOperation<A, E> => ({ fiber, id })))
-              ),
-              Effect.tap(({ id }) =>
-                Effect.interruptible(fnc(id)).pipe(
-                  Effect.forkScoped,
-                  Scope.provide(scope)
-                )
-              )
-            )
-      )
-  }
-
-  const fork2: {
-    (title: NonEmptyString2k): <R, E, A>(
-      self: (opId: OperationId) => Effect.Effect<A, E, R>
-    ) => Effect.Effect<RunningOperation<A, E>, never, Exclude<R, Scope.Scope>>
-    <R, E, A>(
-      self: (opId: OperationId) => Effect.Effect<A, E, R>,
-      title: NonEmptyString2k
-    ): Effect.Effect<RunningOperation<A, E>, never, Exclude<R, Scope.Scope>>
-  } = dual(
-    2,
-    <R, E, A>(self: (opId: OperationId) => Effect.Effect<A, E, R>, title: NonEmptyString2k) =>
-      Effect.flatMap(
-        Scope.make(),
-        (scope) =>
-          register(title)
-            .pipe(
-              Scope.provide(scope),
-              Effect
-                .flatMap((id) =>
-                  reqFiberSet
-                    .forkDaemonReportUnexpected(Scope.use(
-                      self(id).pipe(Effect.withSpan(title, {}, { captureStackTrace: false })),
-                      scope
-                    ))
-                    .pipe(Effect.map((fiber): RunningOperation<A, E> => ({ fiber, id })))
-                )
-            )
-      )
-  )
-
-  const forkOperation: {
-    (title: NonEmptyString2k): <R, E, A>(
-      self: Effect.Effect<A, E, R>
-    ) => Effect.Effect<RunningOperation<A, E>, never, Exclude<R, Scope.Scope>>
-    <R, E, A>(
-      self: Effect.Effect<A, E, R>,
-      title: NonEmptyString2k
-    ): Effect.Effect<RunningOperation<A, E>, never, Exclude<R, Scope.Scope>>
-  } = dual(
-    2,
-    <R, E, A>(self: Effect.Effect<A, E, R>, title: NonEmptyString2k) =>
-      Effect.flatMap(
-        Scope.make(),
-        (scope) =>
-          register(title)
-            .pipe(
-              Scope.provide(scope),
-              Effect
-                .flatMap((id) =>
-                  reqFiberSet
-                    .forkDaemonReportUnexpected(Scope.use(
-                      self.pipe(Effect.withSpan(title, {}, { captureStackTrace: false })),
-                      scope
-                    ))
-                    .pipe(Effect.map((fiber): RunningOperation<A, E> => ({ fiber, id })))
-                )
-            )
-      )
-  )
-
-  function forkOperationFunction<R, E, A, Inp>(fnc: (inp: Inp) => Effect.Effect<A, E, R>, title: NonEmptyString2k) {
-    return (inp: Inp) => fnc(inp).pipe((_) => forkOperation(_, title))
-  }
-
-  return {
-    cleanup,
-    register,
-    fork,
-    fork2,
-    forkOperation,
-    forkOperationFunction,
-    all: repo.all,
-    find: findOp,
-    update
-  }
-})
-
-export class Operations extends ServiceMap.Opaque<Operations>()("effect-app/Operations", { make }) {
-  private static readonly CleanupLive = this
-    .use((_) =>
-      _.cleanup.pipe(
-        Effect.exit,
-        Effect
-          .flatMap((_) => {
-            if (Exit.isSuccess(_)) {
-              return Effect.void
-            } else {
-              return reportAppError(_.cause)
-            }
-          }),
-        Effect.schedule(Schedule.fixed(Duration.minutes(20))),
-        Effect.map((_) => _ as never),
-        MainFiberSet.run
-      )
-    )
-    .pipe(Layer.effectDiscard, Layer.provide(MainFiberSet.Live))
-
-  static readonly Live = this.CleanupLive.pipe(
-    Layer.provideMerge(this.toLayer(this.make)),
-    Layer.provide(RequestFiberSet.Live)
-  )
-
-  static readonly forkOperation = (title: NonEmptyString2k) => <R, E, A>(self: Effect.Effect<A, E, R>) =>
-    this.use((_) => _.forkOperation(self, title))
-  static readonly forkOperationFunction =
-    <R, E, A, Inp>(fnc: (inp: Inp) => Effect.Effect<A, E, R>, title: NonEmptyString2k) => (inp: Inp) =>
-      this.use((_) => _.forkOperationFunction(fnc, title)(inp))
-  static readonly fork = <R, R2, E, E2, A, A2>(
-    self: (id: OperationId) => Effect.Effect<A, E, R>,
-    fnc: (id: OperationId) => Effect.Effect<A2, E2, R2>,
-    title: NonEmptyString2k
-  ) => this.use((_) => _.fork(self, fnc, title))
-
-  static readonly fork2 = (title: NonEmptyString2k) => <R, E, A>(self: (opId: OperationId) => Effect.Effect<A, E, R>) =>
-    this.use((_) => _.fork2(self, title))
-}
-
-export interface RunningOperation<A, E> {
-  id: OperationId
-  fiber: Fiber.Fiber<A, E>
-}

package/src/OperationsRepo.ts

@@ -1,16 +0,0 @@
-import { Effect, ServiceMap } from "effect-app"
-import { Operation } from "effect-app/Operations"
-import { makeRepo } from "./Model.js"
-
-export class OperationsRepo extends ServiceMap.Service<OperationsRepo>()(
-  "OperationRepo",
-  {
-    make: Effect.gen(function*() {
-      return yield* makeRepo("Operation", Operation, {
-        config: {
-          allowNamespace: () => true
-        }
-      })
-    })
-  }
-) {}

package/src/QueueMaker/SQLQueue.ts

@@ -1,189 +0,0 @@
-import { getRequestContext, setupRequestContextWithCustomSpan } from "@effect-app/infra/api/setupRequest"
-import { reportNonInterruptedFailure } from "@effect-app/infra/QueueMaker/errors"
-import { type QueueBase, QueueMeta } from "@effect-app/infra/QueueMaker/service"
-import { subMinutes } from "date-fns"
-import { Effect, Fiber, type NonEmptyReadonlyArray, Option, S, Tracer } from "effect-app"
-import type { NonEmptyString255 } from "effect-app/Schema"
-import { pretty } from "effect-app/utils"
-import { SqlClient } from "effect/unstable/sql"
-import { SQLModel } from "../adapters/SQL.js"
-import { InfraLogger } from "../logger.js"
-
-export const QueueId = S.Number.pipe(S.brand("QueueId"))
-export type QueueId = typeof QueueId.Type
-
-// TODO: let the model track and Auto Generate versionColumn on every update instead
-export function makeSQLQueue<
-  Evt extends { id: S.StringId; _tag: string },
-  DrainEvt extends { id: S.StringId; _tag: string },
-  EvtE,
-  DrainEvtE
->(
-  queueName: NonEmptyString255,
-  queueDrainName: NonEmptyString255,
-  schema: S.Codec<Evt, EvtE>,
-  drainSchema: S.Codec<DrainEvt, DrainEvtE>
-) {
-  return Effect.gen(function*() {
-    const base = {
-      id: SQLModel.Generated(QueueId),
-      meta: SQLModel.JsonFromString(QueueMeta),
-      name: S.NonEmptyString255,
-      createdAt: SQLModel.DateTimeInsert,
-      updatedAt: SQLModel.DateTimeUpdate,
-      // TODO: at+owner
-      processingAt: SQLModel.FieldOption(S.Date),
-      finishedAt: SQLModel.FieldOption(S.Date),
-      etag: S.String // TODO: use a SQLModel thing that auto updates it?
-      // TODO: record locking.. / optimistic locking
-      // rowVersion: SQLModel.DateTimeFromNumberWithNow
-    }
-    class Queue extends SQLModel.Class<Queue>("Queue")({
-      body: SQLModel.JsonFromString(schema),
-      ...base
-    }) {}
-    class Drain extends SQLModel.Class<Drain>("Drain")({
-      body: SQLModel.JsonFromString(drainSchema),
-      ...base
-    }) {}
-    const sql = yield* SqlClient.SqlClient
-
-    const queueRepo = yield* SQLModel.makeRepository(Queue, {
-      tableName: "queue",
-      spanPrefix: "QueueRepo",
-      idColumn: "id",
-      versionColumn: "etag"
-    })
-
-    const drainRepo = yield* SQLModel.makeRepository(Drain, {
-      tableName: "queue",
-      spanPrefix: "DrainRepo",
-      idColumn: "id",
-      versionColumn: "etag"
-    })
-
-    const decodeDrain = S.decodeEffect(Drain)
-
-    const drain = Effect
-      .sync(() => subMinutes(new Date(), 15))
-      .pipe(
-        Effect
-          .andThen((limit) =>
-            sql<typeof Drain.Encoded>`SELECT *
-    FROM queue
-    WHERE name = ${queueDrainName} AND finishedAt IS NULL AND (processingAt IS NULL OR processingAt < ${limit.getTime()})
-    LIMIT 1`
-          )
-      )
-
-    const q = {
-      offer: Effect.fnUntraced(function*(body: Evt, meta: typeof QueueMeta.Type) {
-        yield* queueRepo.insertVoid({
-          body,
-          meta,
-          name: queueName,
-          processingAt: Option.none(),
-          finishedAt: Option.none(),
-          etag: crypto.randomUUID()
-        })
-      }),
-      take: Effect.gen(function*() {
-        while (true) {
-          const [first] = yield* drain.pipe(Effect.withTracerEnabled(false)) // disable sql tracer otherwise we spam it..
-          if (first) {
-            const dec = yield* decodeDrain(first)
-            const { createdAt, updatedAt, ...rest } = dec
-            return yield* drainRepo.update(
-              { ...rest, processingAt: Option.some(new Date()) } // auto in lib , etag: crypto.randomUUID()
-            )
-          }
-          if (first) return first
-          yield* Effect.sleep(250)
-        }
-      }),
-      finish: ({ createdAt, updatedAt, ...q }: Drain) =>
-        drainRepo.updateVoid({ ...q, finishedAt: Option.some(new Date()) }) // auto in lib , etag: crypto.randomUUID()
-    }
-    const queue = {
-      publish: (...messages: NonEmptyReadonlyArray<Evt>) =>
-        getRequestContext
-          .pipe(
-            Effect.flatMap((requestContext) =>
-              Effect
-                .forEach(
-                  messages,
-                  (m) => q.offer(m, requestContext),
-                  {
-                    discard: true
-                  }
-                )
-            ),
-            Effect.withSpan("queue.publish: " + queueName, {
-              kind: "producer",
-              attributes: { "message_tags": messages.map((_) => _._tag) }
-            }, { captureStackTrace: false })
-          ),
-      drain: <DrainE, DrainR>(
-        handleEvent: (ks: DrainEvt) => Effect.Effect<void, DrainE, DrainR>,
-        sessionId?: string
-      ) => {
-        const silenceAndReportError = reportNonInterruptedFailure({ name: "MemQueue.drain." + queueDrainName })
-        const processMessage = (msg: Drain) =>
-          Effect
-            .succeed(msg)
-            .pipe(Effect
-              .flatMap(({ body, meta }) => {
-                let effect = InfraLogger
-                  .logDebug(`[${queueDrainName}] Processing incoming message`)
-                  .pipe(
-                    Effect.annotateLogs({ body: pretty(body), meta: pretty(meta) }),
-                    Effect.andThen(handleEvent(body)),
-                    silenceAndReportError,
-                    (_) =>
-                      setupRequestContextWithCustomSpan(
-                        _,
-                        meta,
-                        `queue.drain: ${queueDrainName}.${body._tag}`,
-                        {
-                          captureStackTrace: false,
-                          kind: "consumer",
-                          attributes: {
-                            "queue.name": queueDrainName,
-                            "queue.sessionId": sessionId,
-                            "queue.input": body
-                          }
-                        }
-                      )
-                  )
-                if (meta.span) {
-                  effect = Effect.withParentSpan(effect, Tracer.externalSpan(meta.span))
-                }
-                return effect
-              }))
-
-        return q
-          .take
-          .pipe(
-            Effect.flatMap((x) =>
-              processMessage(x).pipe(
-                Effect.uninterruptible,
-                Effect.forkChild,
-                Effect.flatMap(Fiber.join),
-                Effect.tap(q.finish(x))
-              )
-            ),
-            silenceAndReportError,
-            Effect.withSpan(`queue.drain: ${queueDrainName}`, {
-              attributes: {
-                "queue.type": "sql",
-                "queue.name": queueDrainName,
-                "queue.sessionId": sessionId
-              }
-            }),
-            Effect.forever
-          )
-      }
-    }
-    return queue as QueueBase<Evt, DrainEvt>
-  })
-}