effect-app 4.0.0-beta.31 → 4.0.0-beta.311

package/src/Model/Repository/internal/internal.ts

@@ -0,0 +1,905 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import * as Equivalence from "effect/Equivalence"
+import { flow, pipe } from "effect/Function"
+import * as HashMap from "effect/HashMap"
+import * as HashSet from "effect/HashSet"
+import * as Metric from "effect/Metric"
+import * as Pipeable from "effect/Pipeable"
+import * as Ref from "effect/Ref"
+import * as Result from "effect/Result"
+import type * as Scope from "effect/Scope"
+import * as Unify from "effect/Unify"
+import * as Array from "../../../Array.ts"
+import type { NonEmptyReadonlyArray } from "../../../Array.ts"
+import { toNonEmptyArray } from "../../../Array.ts"
+import * as Chunk from "../../../Chunk.ts"
+import { NotFoundError } from "../../../client/errors.ts"
+import * as Context from "../../../Context.ts"
+import * as DataDependencies from "../../../DataDependencies.ts"
+import * as Effect from "../../../Effect.ts"
+import { flatMapOption } from "../../../Effect.ts"
+import * as Option from "../../../Option.ts"
+import * as S from "../../../Schema.ts"
+import { type Codec, NonNegativeInt } from "../../../Schema.ts"
+import * as SchemaAST from "../../../SchemaAST.ts"
+import { setupRequestContextFromCurrent } from "../../../setupRequest.ts"
+import { type FilterArgs, getContextMap, type PersistenceModelType, type StoreConfig, storeId, StoreMaker } from "../../../Store.ts"
+import type { FilterResult } from "../../filter/filterApi.ts"
+import type { FieldValues } from "../../filter/types.ts"
+import * as Q from "../../query.ts"
+import { repositoryDependencyPaths } from "../dependency.ts"
+import type { ChangeFeed, ChangeFeedEvent, Repository } from "../service.ts"
+import { ValidationError, ValidationResult } from "../validation.ts"
+
+const dedupe = Array.dedupeWith(Equivalence.String)
+
+const schemaDurationBoundaries = [0.1, 0.5, 1, 2, 5, 10, 25, 50, 100, 250, 500, 1_000]
+const schemaDecodeDuration = Metric.histogram("app.schema.decode.duration", { boundaries: schemaDurationBoundaries })
+const schemaEncodeDuration = Metric.histogram("app.schema.encode.duration", { boundaries: schemaDurationBoundaries })
+const schemaItemCount = Metric.histogram("app.schema.item_count", {
+  boundaries: [0, 1, 2, 5, 10, 25, 50, 100, 250, 500, 1_000, 5_000]
+})
+
+const timeSchema = (
+  operation: "decode" | "encode",
+  entity: string,
+  queryMode: "aggregate" | "collect" | "project" | "transform" | undefined,
+  itemCount: number
+) =>
+<A, E, R>(self: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+  Effect.clockWith((clock) => {
+    const startedAt = clock.currentTimeNanosUnsafe()
+    const attributes = {
+      "app.entity": entity,
+      "app.schema.operation": operation,
+      ...(queryMode !== undefined && { "app.query.mode": queryMode })
+    }
+    return Effect.onExit(self, () => {
+      const durationMs = Number(clock.currentTimeNanosUnsafe() - startedAt) / 1_000_000
+      return Effect.all([
+        Effect.annotateCurrentSpan({
+          [`app.schema.${operation}.duration_ms`]: durationMs,
+          "app.schema.item_count": itemCount,
+          ...(queryMode !== undefined && { "app.query.mode": queryMode })
+        }),
+        Metric.update(
+          Metric.withAttributes(
+            operation === "decode" ? schemaDecodeDuration : schemaEncodeDuration,
+            attributes
+          ),
+          durationMs
+        ),
+        Metric.update(Metric.withAttributes(schemaItemCount, attributes), itemCount)
+      ], { discard: true })
+    })
+  })
+
+/**
+ * A base implementation to create a repository.
+ */
+export function makeRepoInternal<
+  Evt = never
+>() {
+  return <
+    ItemType extends string,
+    R,
+    Encoded extends FieldValues,
+    T,
+    IdKey extends keyof T & keyof Encoded
+  >(
+    name: ItemType,
+    schema: S.Codec<T, Encoded, R>,
+    mapFrom: (pm: Encoded) => Encoded,
+    mapTo: (e: Encoded, etag: string | undefined) => PersistenceModelType<Encoded>,
+    idKey: IdKey
+  ) => {
+    type PM = PersistenceModelType<Encoded>
+    function mapToPersistenceModel(
+      e: Encoded,
+      getEtag: (id: string) => string | undefined
+    ): PM {
+      return mapTo(e, getEtag(e[idKey]))
+    }
+
+    function mapReverse(
+      { _etag, ...e }: PM,
+      setEtag: (id: string, eTag: string | undefined) => void
+    ): Encoded {
+      setEtag((e as any)[idKey], _etag)
+      return mapFrom(e as unknown as Encoded)
+    }
+
+    const mkStore = makeStore<Encoded>()(name, schema, mapTo, idKey)
+
+    function make<RInitial = never, E = never, RPublish = never, RCtx = never>(
+      args: [Evt] extends [never] ? {
+          schemaContext?: Context.Context<RCtx>
+          makeInitial?: Effect.Effect<readonly T[], E, RInitial> | undefined
+          dependencyIds?: (item: T) => NonEmptyReadonlyArray<string>
+          additionalWriteDependencies?: (item: T) => ReadonlyArray<DataDependencies.DataDependency>
+          config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
+            partitionValue?: (e?: Encoded) => string
+          }
+        }
+        : {
+          schemaContext?: Context.Context<RCtx>
+          publishEvents: (evt: NonEmptyReadonlyArray<Evt>) => Effect.Effect<void, never, RPublish>
+          makeInitial?: Effect.Effect<readonly T[], E, RInitial> | undefined
+          dependencyIds?: (item: T) => NonEmptyReadonlyArray<string>
+          additionalWriteDependencies?: (item: T) => ReadonlyArray<DataDependencies.DataDependency>
+          config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
+            partitionValue?: (e?: Encoded) => string
+          }
+        }
+    ) {
+      return Effect
+        .gen(function*() {
+          const rctx: Context.Context<RCtx> = args.schemaContext ?? Context.empty() as any
+          const provideRctx = Effect.provide(rctx)
+          const encodeMany = (items: readonly T[]) =>
+            S.encodeEffect(S.Array(schema))(items).pipe(
+              provideRctx,
+              timeSchema("encode", name, undefined, items.length)
+            )
+          const decode = flow(S.decodeEffectConcurrently(schema), provideRctx)
+          const decodeMany = flow(
+            S.decodeEffectConcurrently(S.Array(schema)),
+            provideRctx
+          )
+
+          const store = yield* mkStore(args.makeInitial, args.config)
+          const dependencyPaths = repositoryDependencyPaths(schema)
+          const recordRead = DataDependencies.readRepo(name)
+          const entityDependency = (ids: NonEmptyReadonlyArray<T[IdKey]>) =>
+            DataDependencies.repo(name, [String(ids[0]), ...ids.slice(1).map(String)])
+          const valuesAtPath = (value: unknown, path: string): readonly unknown[] => {
+            if (path.length === 0) return [value]
+            const [head, ...tail] = path.split(".")
+            if (head === undefined) return [value]
+            if (head === "-1") {
+              return globalThis.Array.isArray(value)
+                ? value.flatMap((item) => valuesAtPath(item, tail.join(".")))
+                : []
+            }
+            return typeof value === "object" && value !== null
+              ? valuesAtPath((value as Record<string, unknown>)[head], tail.join("."))
+              : []
+          }
+          const dependencyIds = (item: T): NonEmptyReadonlyArray<string> => {
+            const configured = dependencyPaths
+              .flatMap((path) => valuesAtPath(item, String(path)))
+              .filter((value): value is string => typeof value === "string") ?? []
+            const ids = args.dependencyIds?.(item) ?? [String(item[idKey]), ...configured]
+            return [ids[0], ...ids.slice(1)]
+          }
+          const itemDependencies = (item: T) => {
+            const ids = dependencyIds(item)
+            return [
+              DataDependencies.repo(name, ids),
+              ...(args.additionalWriteDependencies?.(item) ?? [])
+            ]
+          }
+          const queryDependencyIds = (filter: readonly FilterResult[] | undefined): readonly string[] => {
+            if (!filter) return []
+            const paths = new Set([String(idKey), ...dependencyPaths])
+            const visit = (items: readonly FilterResult[]): readonly string[] => {
+              if (items.some((item) => item.t === "or" || item.t === "or-scope")) return []
+              return items.flatMap((item) => {
+                if ("result" in item) return visit(item.result)
+                if (!paths.has(item.path)) return []
+                const value: unknown = item.value
+                if (item.op === "eq" && typeof value === "string") return [value]
+                if (item.op === "in" && globalThis.Array.isArray(value)) {
+                  return value.filter((entry): entry is string => typeof entry === "string")
+                }
+                return []
+              })
+            }
+            return [...new Set(visit(filter))]
+          }
+          const recordEntityRead = (id: T[IdKey]) => DataDependencies.read(entityDependency([id]))
+          const recordEntityWrite = (ids: NonEmptyReadonlyArray<T[IdKey]>) =>
+            DataDependencies.write(entityDependency(ids))
+          const recordItemWrite = (items: NonEmptyReadonlyArray<T>) =>
+            Effect.forEach(
+              DataDependencies.merge(new Set(items.flatMap(itemDependencies))),
+              DataDependencies.write,
+              { discard: true }
+            )
+          const cms = Effect.map(getContextMap.pipe(Effect.orDie), (_) => ({
+            get: (id: string) => _.get(`${name}.${id}`),
+            set: (id: string, etag: string | undefined) => _.set(`${name}.${id}`, etag)
+          }))
+
+          const pub = "publishEvents" in args
+            ? args.publishEvents
+            : () => Effect.void
+
+          type ChangeFeedHandler = (evt: ChangeFeedEvent<T>) => Effect.Effect<void>
+          const changeFeedByNamespace = yield* Ref.make(HashMap.empty<string, HashSet.HashSet<ChangeFeedHandler>>())
+          const changeFeedWildcard = yield* Ref.make(HashSet.empty<ChangeFeedHandler>())
+          // clear all handlers when the repository's scope closes
+          yield* Effect.addFinalizer(() =>
+            Effect.all([
+              Ref.set(changeFeedByNamespace, HashMap.empty<string, HashSet.HashSet<ChangeFeedHandler>>()),
+              Ref.set(changeFeedWildcard, HashSet.empty<ChangeFeedHandler>())
+            ], { discard: true })
+          )
+          const changeFeed: ChangeFeed<T> = {
+            publish: ([items, op]) =>
+              Effect.gen(function*() {
+                const ns = yield* storeId
+                const evt: ChangeFeedEvent<T> = [items, op, ns]
+                const map = yield* Ref.get(changeFeedByNamespace)
+                const wild = yield* Ref.get(changeFeedWildcard)
+                const targeted = Option.getOrElse(HashMap.get(map, ns), () => HashSet.empty<ChangeFeedHandler>())
+                const all = HashSet.union(targeted, wild)
+                yield* Effect.forEach(all, (h) => h(evt), { concurrency: "unbounded", discard: true })
+              }),
+            subscribe: (handler, options) => {
+              const ns = options?.namespace
+              if (ns === undefined) {
+                return Effect.acquireRelease(
+                  Ref.update(changeFeedWildcard, HashSet.add(handler)),
+                  () => Ref.update(changeFeedWildcard, HashSet.remove(handler))
+                )
+              }
+              return Effect.acquireRelease(
+                Ref.update(changeFeedByNamespace, (m) => {
+                  const cur = Option.getOrElse(HashMap.get(m, ns), () => HashSet.empty<ChangeFeedHandler>())
+                  return HashMap.set(m, ns, HashSet.add(cur, handler))
+                }),
+                () =>
+                  Ref.update(changeFeedByNamespace, (m) => {
+                    const cur = Option.getOrElse(HashMap.get(m, ns), () => HashSet.empty<ChangeFeedHandler>())
+                    const next = HashSet.remove(cur, handler)
+                    return HashSet.size(next) === 0 ? HashMap.remove(m, ns) : HashMap.set(m, ns, next)
+                  })
+              )
+            }
+          }
+
+          const allE = cms
+            .pipe(Effect.flatMap((cm) => Effect.map(store.all, (_) => _.map((_) => mapReverse(_, cm.set)))))
+
+          const all = Effect
+            .flatMap(
+              allE,
+              (items) =>
+                decodeMany(items).pipe(
+                  Effect.orDie,
+                  timeSchema("decode", name, undefined, items.length)
+                )
+            )
+            .pipe(
+              Effect.tap(() => recordRead),
+              Effect.map((_) => _ as T[]),
+              Effect.withSpan("Repository.all", {
+                kind: "client",
+                attributes: { "app.entity": name }
+              }, { captureStackTrace: false })
+            )
+
+          const fieldsSchema = schema as unknown as { fields: any }
+          // assumes the id field never needs a service...
+          const i = ("fields" in fieldsSchema ? S.Struct(fieldsSchema["fields"]) as unknown as typeof schema : schema)
+            .pipe((_) => {
+              let ast = _.ast
+              if (ast._tag === "Declaration") ast = ast.typeParameters[0]!
+
+              const pickIdFromAst = (a: SchemaAST.AST) => {
+                // Unwrap Declaration (e.g. TaggedClass) to get the underlying Objects AST
+                let inner = a
+                if (inner._tag === "Declaration") inner = inner.typeParameters[0]!
+                // Pick from the original AST to preserve the full encoding chain (e.g. decodeTo transformations).
+                // Using toEncoded would lose transformation info needed to encode Type -> Encoded.
+                if (SchemaAST.isObjects(inner)) {
+                  const field = inner.propertySignatures.find((_) => _.name === idKey)
+                  if (field) {
+                    return S.Struct({ [idKey]: S.make(field.type) }) as unknown as Codec<T, Encoded>
+                  }
+                }
+                return S.make(a) as unknown as Codec<T, Encoded>
+              }
+
+              return ast._tag === "Union"
+                // we need to get the Objects (TypeLiteral), in case of class it has encoding chain...
+                ? S.Union(
+                  ast.types.map((_) => pickIdFromAst(_))
+                )
+                : pickIdFromAst(ast)
+            })
+          const encodeId = flow(S.encodeEffect(i), provideRctx)
+          const encodeIdOnly = (id: string) =>
+            encodeId({ [idKey]: id } as any).pipe(
+              Effect.map((_: Record<string, unknown>) => _[idKey as string] as Encoded[IdKey])
+            )
+          const findEId = Effect.fnUntraced(function*(id: Encoded[IdKey]) {
+            yield* Effect.annotateCurrentSpan({ "app.entity.id": id })
+
+            return yield* Effect.flatMap(
+              store.find(id),
+              (item) =>
+                Effect.gen(function*() {
+                  const { set } = yield* cms
+                  return item.pipe(Option.map((_) => mapReverse(_, set)))
+                })
+            )
+          })
+          // TODO: select the particular field, instead of as struct
+          const findE = Effect.fnUntraced(function*(id: T[IdKey]) {
+            yield* Effect.annotateCurrentSpan({ "app.entity.id": id })
+
+            return yield* pipe(
+              encodeId({ [idKey]: id } as any),
+              Effect.orDie,
+              Effect.map((_) => (_ as any)[idKey]),
+              Effect.flatMap(findEId)
+            )
+          })
+
+          const loadExistingItems = (ids: ReadonlyArray<T[IdKey]>) =>
+            Effect
+              .forEach(
+                ids,
+                (id) =>
+                  findE(id).pipe(
+                    Effect.flatMap(Option.match({
+                      onNone: () => Effect.succeed(Option.none<T>()),
+                      onSome: (encoded) => decode(encoded).pipe(Effect.orDie, Effect.map(Option.some))
+                    }))
+                  )
+              )
+              .pipe(Effect.map(Array.getSomes))
+
+          const find = Effect.fn("Repository.find", {
+            kind: "client",
+            attributes: { "app.entity": name }
+          })(function*(id: T[IdKey]) {
+            yield* recordEntityRead(id)
+            yield* Effect.annotateCurrentSpan({ "app.entity.id": id })
+            return yield* flatMapOption(findE(id), (_) => Effect.orDie(decode(_)))
+          })
+
+          const saveAllE = (a: Iterable<Encoded>) =>
+            flatMapOption(
+              Effect
+                .sync(() => toNonEmptyArray([...a])),
+              (a) =>
+                Effect.gen(function*() {
+                  const { get, set } = yield* cms
+                  const items = a.map((_) => mapToPersistenceModel(_, get))
+                  const ret = yield* store.batchSet(items)
+                  ret.forEach((_) => set(_[idKey], _._etag))
+                })
+            )
+              .pipe(Effect.asVoid)
+
+          const saveAll = (a: Iterable<T>) =>
+            encodeMany(Array.fromIterable(a))
+              .pipe(
+                Effect.orDie,
+                Effect.andThen(saveAllE)
+              )
+
+          const saveAndPublish = Effect.fn("Repository.saveAndPublish", { attributes: { "app.entity": name } })(
+            function*(items: Iterable<T>, events: Iterable<Evt> = []) {
+              const it = Chunk.fromIterable(items)
+              if (Chunk.isNonEmpty(it)) {
+                const values = Chunk.toReadonlyArray(it)
+                const previous = args.additionalWriteDependencies || args.dependencyIds || dependencyPaths.length > 0
+                  ? yield* loadExistingItems(values.map((item) => item[idKey]))
+                  : []
+                yield* recordItemWrite([values[0], ...values.slice(1), ...previous])
+              }
+              const evts = [...events]
+              yield* Effect.annotateCurrentSpan({
+                "app.entity.ids": Chunk.map(it, (_) => _[idKey]),
+                "app.event.count": evts.length
+              })
+              return yield* saveAll(it)
+                .pipe(
+                  Effect.andThen(Effect.sync(() => toNonEmptyArray(evts))),
+                  // TODO: for full consistency the events should be stored within the same database transaction, and then picked up.
+                  (_) => flatMapOption(_, pub),
+                  Effect.andThen(changeFeed.publish([Chunk.toArray(it), "save"] as [T[], "save" | "remove"])),
+                  Effect.asVoid
+                )
+            }
+          )
+
+          const removeAndPublish = Effect.fn("Repository.removeAndPublish", { attributes: { "app.entity": name } })(
+            function*(a: Iterable<T>, events: Iterable<Evt> = []) {
+              const { set } = yield* cms
+              const it = [...a]
+              if (Array.isReadonlyArrayNonEmpty(it)) {
+                yield* recordItemWrite(it)
+              }
+              const evts = [...events]
+              yield* Effect.annotateCurrentSpan({
+                "app.entity.ids": it.map((_) => _[idKey]),
+                "app.event.count": evts.length
+              })
+              const items = yield* encodeMany(it).pipe(Effect.orDie)
+              if (Array.isReadonlyArrayNonEmpty(items)) {
+                yield* store.batchRemove(
+                  items.map((_) => (_[idKey])),
+                  args.config?.partitionValue?.(items[0])
+                )
+                for (const e of items) {
+                  set(e[idKey], undefined)
+                }
+                yield* Effect
+                  .sync(() => toNonEmptyArray(evts))
+                  // TODO: for full consistency the events should be stored within the same database transaction, and then picked up.
+                  .pipe((_) => flatMapOption(_, pub))
+
+                yield* changeFeed.publish([it, "remove"] as [T[], "save" | "remove"])
+              }
+            }
+          )
+
+          const removeById = Effect.fn("Repository.removeById", { attributes: { "app.entity": name } })(
+            function*(idOrIds: T[IdKey] | ReadonlyArray<T[IdKey]>) {
+              const ids = globalThis.Array.isArray(idOrIds)
+                ? idOrIds as readonly T[IdKey][]
+                : [idOrIds as T[IdKey]]
+              if (!Array.isReadonlyArrayNonEmpty(ids)) {
+                return
+              }
+              yield* recordEntityWrite(ids)
+              if (args.additionalWriteDependencies || args.dependencyIds || dependencyPaths.length > 0) {
+                const previous = yield* loadExistingItems(ids)
+                if (Array.isReadonlyArrayNonEmpty(previous)) yield* recordItemWrite(previous)
+              }
+              const { set } = yield* cms
+              const eids = yield* Effect.forEach(ids, (_) => encodeIdOnly(_ as any)).pipe(Effect.orDie)
+              yield* Effect.annotateCurrentSpan({ "app.entity.ids": eids })
+              yield* store.batchRemove(eids)
+              for (const id of eids) {
+                set(id, undefined)
+              }
+              yield* changeFeed.publish([[], "remove"] as [T[], "save" | "remove"])
+            }
+          )
+
+          const parseMany = Effect.fnUntraced(
+            function*(items: readonly PM[]) {
+              const cm = yield* cms
+              return yield* decodeMany(items.map((_) => mapReverse(_, cm.set))).pipe(
+                Effect.orDie,
+                timeSchema("decode", name, "transform", items.length)
+              )
+            }
+          )
+          const decodeManyCache = new WeakMap<
+            S.Codec<any, any, any>,
+            (i: readonly any[]) => Effect.Effect<any, any, any>
+          >()
+          const getDecodeMany = (s: S.Codec<any, Encoded, any>) => {
+            let dec = decodeManyCache.get(s)
+            if (!dec) {
+              dec = S.decodeEffectConcurrently(S.Array(s))
+              decodeManyCache.set(s, dec)
+            }
+            return dec
+          }
+          const parseMany2 = Effect.fnUntraced(
+            function*<A, R>(items: readonly PM[], schema: S.Codec<A, Encoded, R>) {
+              const cm = yield* cms
+              return yield* getDecodeMany(schema)(items.map((_) => mapReverse(_, cm.set))).pipe(
+                Effect.orDie,
+                timeSchema("decode", name, "transform", items.length)
+              )
+            }
+          )
+          const filter = <U extends keyof Encoded = keyof Encoded>(args: FilterArgs<Encoded, U>) =>
+            store
+              .filter(
+                // always enforce id and _etag because they are system fields, required for etag tracking etc
+                {
+                  ...args,
+                  select: args.select
+                    ? dedupe([...args.select, idKey, "_etag" as any])
+                    : undefined
+                } as typeof args
+              )
+              .pipe(
+                Effect.tap((items) =>
+                  Effect.map(cms, ({ set }) => items.forEach((_) => set((_ as Encoded)[idKey], (_ as PM)._etag)))
+                )
+              )
+
+          // TODO: For raw we should use S.from, and drop the R...
+          const query: {
+            <A, R, From extends FieldValues>(
+              q: Q.QueryProjection<Encoded extends From ? From : never, A, R>
+            ): Effect.Effect<readonly A[], S.SchemaError, Exclude<R, RCtx>>
+            <A, R, EncodedRefined extends Encoded = Encoded>(
+              q: Q.QAll<NoInfer<Encoded>, NoInfer<EncodedRefined>, A, R>
+            ): Effect.Effect<readonly A[], never, Exclude<R, RCtx>>
+          } = (<A, R, EncodedRefined extends Encoded = Encoded>(q: Q.QAll<Encoded, EncodedRefined, A, R>) => {
+            const a = Q.toFilter(q, schema)
+            const scopedIds = queryDependencyIds(a.filter)
+            const recordQueryRead = Array.isReadonlyArrayNonEmpty(scopedIds)
+              ? DataDependencies.read(DataDependencies.repo(name, scopedIds))
+              : recordRead
+            // Mode dispatch — see `Q.project` JSDoc for the contract:
+            //   aggregate: GROUP BY + aggregate functions at DB level; decode raw rows with schema; SchemaError surfaces.
+            //   project  : decode raw encoded rows with schema; no PM reverse-mapping; SchemaError surfaces.
+            //   collect  : same as project, but schema yields Option and None rows are dropped.
+            //   transform: PM reverse-map (re-inject _etag/PM state from cms cache) then decode; orDie.
+            const eff = a.mode === "aggregate"
+              ? store
+                // `a.select` contains `{ key, aggregate }` items not expressible in FilterFunc<Encoded, U>'s
+                // `U extends keyof Encoded` generic. Cast is unavoidable until FilterFunc supports aggregate mode.
+                .filter(a as any)
+                // Decode raw aggregate rows directly — no PM reverse-mapping, no id/_etag needed.
+                .pipe(
+                  Effect.andThen(
+                    (items) =>
+                      S.decodeEffectConcurrently(S.Array(a.schema ?? schema))(items).pipe(
+                        provideRctx,
+                        timeSchema("decode", name, "aggregate", items.length)
+                      )
+                  )
+                )
+              : a.mode === "project"
+              ? filter(a)
+                // TODO: mapFrom but need to support per field and dependencies
+                .pipe(
+                  Effect.andThen(
+                    (items) =>
+                      S.decodeEffectConcurrently(S.Array(a.schema ?? schema))(items).pipe(
+                        provideRctx,
+                        timeSchema("decode", name, "project", items.length)
+                      )
+                  )
+                )
+              : a.mode === "collect"
+              ? filter(a)
+                // TODO: mapFrom but need to support per field and dependencies
+                .pipe(
+                  Effect.flatMap((items) =>
+                    S.decodeEffectConcurrently(S.Array(a.schema))(items).pipe(
+                      Effect.map(Array.getSomes),
+                      provideRctx,
+                      timeSchema("decode", name, "collect", items.length)
+                    )
+                  )
+                )
+              : Effect.flatMap(
+                filter(a),
+                (_) =>
+                  Unify.unify(
+                    a.schema
+                      // TODO: partial may not match?
+                      ? parseMany2(_ as any, a.schema as any)
+                      : parseMany(_ as any)
+                  )
+              )
+            return pipe(
+              a.ttype === "one"
+                ? Effect.flatMap(
+                  eff,
+                  flow(
+                    Array.head,
+                    Option.match({
+                      onNone: () => Effect.fail(new NotFoundError({ id: "query", /* TODO */ type: name })),
+                      onSome: Effect.succeed
+                    })
+                  )
+                )
+                : a.ttype === "count"
+                ? Effect
+                  .map(eff, (_) => NonNegativeInt(_.length))
+                  .pipe(Effect.catchTag("SchemaError", (e) => Effect.die(e)))
+                : eff,
+              Effect.tap((r) =>
+                Effect.annotateCurrentSpan({
+                  "app.query.ttype": a.ttype,
+                  "app.query.mode": a.mode,
+                  "db.response.returned_rows": Array.isArray(r) ? r.length : 1
+                })
+              ),
+              Effect.tap(() => recordQueryRead),
+              Effect.withSpan("Repository.query", {
+                kind: "client",
+                attributes: { "app.entity": name }
+              }, { captureStackTrace: false })
+            )
+          }) as any
+
+          const validateSample = Effect.fn("Repository.validateSample", { attributes: { "app.entity": name } })(
+            function*(options?: {
+              percentage?: number
+              maxItems?: number
+            }) {
+              const percentage = options?.percentage ?? 0.1 // default 10%
+              const maxItems = options?.maxItems
+
+              // 1. get all IDs with projection (bypasses main schema decode)
+              const allIds = yield* store
+                .filter({
+                  t: null as unknown as Encoded,
+                  select: [idKey as keyof Encoded]
+                })
+                .pipe(Effect.withSpan("Repository.filter", {
+                  kind: "client",
+                  attributes: { "app.entity": name }
+                }, { captureStackTrace: false }))
+
+              // 2. random subset
+              const shuffled = [...allIds].sort(() => Math.random() - 0.5)
+              const sampleSize = Math.min(
+                maxItems ?? Infinity,
+                Math.ceil(allIds.length * percentage)
+              )
+              const sample = shuffled.slice(0, sampleSize)
+
+              // 3. validate each item
+              const errors: ValidationError[] = []
+
+              for (const item of sample) {
+                const id = item[idKey]
+                const rawResult = yield* store.find(id).pipe(
+                  Effect.withSpan("Repository.find", {
+                    kind: "client",
+                    attributes: { "app.entity": name, "app.entity.id": id }
+                  }, { captureStackTrace: false })
+                )
+
+                if (Option.isNone(rawResult)) continue
+
+                const rawData = rawResult.value as Encoded
+                const jitMResult = mapFrom(rawData) // apply jitM
+
+                const decodeResult = yield* S.decodeEffectConcurrently(schema)(jitMResult).pipe(
+                  Effect.result,
+                  provideRctx
+                )
+
+                if (Result.isFailure(decodeResult)) {
+                  errors.push(
+                    ValidationError.make({
+                      id,
+                      rawData,
+                      jitMResult,
+                      error: decodeResult.failure
+                    })
+                  )
+                }
+              }
+
+              return ValidationResult.make({
+                total: NonNegativeInt(allIds.length),
+                sampled: NonNegativeInt(sample.length),
+                valid: NonNegativeInt(sample.length - errors.length),
+                errors
+              })
+            }
+          )
+
+          const r = {
+            changeFeed,
+            itemType: name,
+            idKey,
+            find,
+            all,
+            saveAndPublish,
+            removeAndPublish,
+            removeById,
+            seedNamespace: (namespace: string) => store.seedNamespace(namespace),
+            validateSample,
+            queryRaw<A, Out, QR>(schema: S.Codec<A, Out, QR>, q: Q.RawQuery<Encoded, Out>) {
+              return store.queryRaw(q).pipe(
+                Effect.flatMap((items) =>
+                  S.decodeEffectConcurrently(S.Array(schema))(items).pipe(
+                    timeSchema("decode", name, undefined, items.length)
+                  )
+                ),
+                Effect.tap(() => recordRead),
+                Effect.withSpan("Repository.queryRaw", {
+                  kind: "client",
+                  attributes: { "app.entity": name }
+                }, { captureStackTrace: false })
+              )
+            },
+            query(q: any) {
+              // eslint-disable-next-line prefer-rest-params
+              return query(typeof q === "function" ? Pipeable.pipeArguments(Q.make(), arguments) : q) as any
+            },
+            /**
+             * @internal
+             */
+            mapped: <A, R>(schema: S.Codec<A, any, R>) => {
+              const dec = S.decodeEffectConcurrently(schema)
+              const encMany = S.encodeEffect(S.Array(schema))
+              const decMany = S.decodeEffectConcurrently(S.Array(schema))
+              const spanAttrs = { kind: "client" as const, attributes: { "app.entity": name } }
+              return {
+                all: allE.pipe(
+                  Effect.flatMap((items) =>
+                    decMany(items).pipe(
+                      timeSchema("decode", name, undefined, items.length)
+                    )
+                  ),
+                  Effect.tap(() => recordRead),
+                  Effect.map((_) => _ as any[]),
+                  Effect.withSpan("Repository.mapped.all", spanAttrs, { captureStackTrace: false })
+                ),
+                find: (id: T[IdKey]) =>
+                  flatMapOption(findE(id), dec).pipe(
+                    Effect.tap(() => recordEntityRead(id)),
+                    Effect.withSpan("Repository.mapped.find", {
+                      ...spanAttrs,
+                      attributes: { ...spanAttrs.attributes, "app.entity.id": id }
+                    }, { captureStackTrace: false })
+                  ),
+                // query: (q: any) => {
+                //   const a = Q.toFilter(q)
+
+                //   return filter(a)
+                //     .pipe(
+                //       Effect.flatMap(decMany),
+                //       Effect.map((_) => _ as any[]),
+                //       Effect.withSpan("Repository.mapped.query [effect-app/infra]", {
+                //  captureStackTrace: false,
+                //         attributes: {
+                //           "repository.model_name": name,
+                //           query: { ...a, schema: a.schema ? "__SCHEMA__" : a.schema, filter: a.filter.build() }
+                //         }
+                //       })
+                //     )
+                // },
+                save: (...xes: any[]) =>
+                  Effect
+                    .flatMap(
+                      encMany(xes).pipe(timeSchema("encode", name, undefined, xes.length)),
+                      (_) => saveAllE(_)
+                    )
+                    .pipe(
+                      Effect.tap(() =>
+                        Array.isReadonlyArrayNonEmpty(xes)
+                          ? recordEntityWrite([xes[0][idKey], ...xes.slice(1).map((_) => _[idKey])])
+                          : Effect.void
+                      ),
+                      Effect.withSpan("Repository.mapped.save", spanAttrs, { captureStackTrace: false })
+                    )
+              }
+            }
+          }
+          return r as Repository<T, Encoded, Evt, ItemType, IdKey, Exclude<R, RCtx>, RPublish, RCtx>
+        })
+        .pipe(Effect
+          // .withSpan("Repository.make [effect-app/infra]", { attributes: { "repository.model_name": name } })
+          .withLogSpan("Repository.make: " + name))
+    }
+
+    return {
+      make,
+      Q: Q.make<Encoded>()
+    }
+  }
+}
+
+const pluralize = (s: string) =>
+  s.endsWith("s")
+    ? s + "es"
+    : s.endsWith("y")
+    ? s.substring(0, s.length - 1) + "ies"
+    : s + "s"
+
+export function makeStore<Encoded extends FieldValues>() {
+  return <
+    ItemType extends string,
+    R,
+    E,
+    T,
+    IdKey extends keyof Encoded
+  >(
+    name: ItemType,
+    schema: S.Codec<T, E, R>,
+    mapTo: (e: E, etag: string | undefined) => Encoded,
+    idKey: IdKey
+  ) => {
+    function makeStore<RInitial = never, EInitial = never>(
+      makeInitial?: Effect.Effect<readonly T[], EInitial, RInitial>,
+      config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
+        partitionValue?: (e?: Encoded) => string
+      }
+    ) {
+      function encodeToEncoded() {
+        const getEtag = () => undefined
+        return (t: T) =>
+          S.encodeEffect(schema)(t).pipe(
+            Effect.orDie,
+            Effect.map((_) => mapToPersistenceModel(_, getEtag))
+          )
+      }
+
+      function mapToPersistenceModel(
+        e: E,
+        getEtag: (id: string) => string | undefined
+      ): Encoded {
+        return mapTo(e, getEtag((e as any)[idKey] as string))
+      }
+
+      return Effect.gen(function*() {
+        const { make } = yield* StoreMaker
+
+        const store = yield* make<IdKey, Encoded, RInitial | R, EInitial>(
+          pluralize(name),
+          idKey,
+          makeInitial
+            ? makeInitial
+              .pipe(
+                Effect.flatMap(Effect.forEach(encodeToEncoded())),
+                setupRequestContextFromCurrent("Repository.makeInitial [effect-app/infra]", {
+                  attributes: { "app.entity": name }
+                })
+              )
+            : undefined,
+          {
+            ...config,
+            partitionValue: config?.partitionValue
+              ?? ((_) => "primary") /*(isIntegrationEvent(r) ? r.companyId : r.id*/
+          }
+        )
+
+        return store
+      })
+    }
+
+    return makeStore
+  }
+}
+
+export interface Repos<
+  T,
+  Encoded extends { id: string },
+  RSchema,
+  Evt,
+  ItemType extends string,
+  IdKey extends keyof T,
+  RPublish
+> {
+  make<RInitial = never, E = never, R2 = never>(
+    args: [Evt] extends [never] ? {
+        makeInitial?: Effect.Effect<readonly T[], E, RInitial> | undefined
+        config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
+          partitionValue?: (e?: Encoded) => string
+        }
+      }
+      : {
+        publishEvents: (evt: NonEmptyReadonlyArray<Evt>) => Effect.Effect<void, never, R2>
+        makeInitial?: Effect.Effect<readonly T[], E, RInitial> | undefined
+        config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
+          partitionValue?: (e?: Encoded) => string
+        }
+      }
+  ): Effect.Effect<
+    Repository<T, Encoded, Evt, ItemType, IdKey, RSchema, RPublish>,
+    E,
+    StoreMaker | RInitial | R2 | Scope.Scope
+  >
+  makeWith<Out, RInitial = never, E = never, R2 = never>(
+    args: [Evt] extends [never] ? {
+        makeInitial?: Effect.Effect<readonly T[], E, RInitial> | undefined
+        config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
+          partitionValue?: (e?: Encoded) => string
+        }
+      }
+      : {
+        publishEvents: (evt: NonEmptyReadonlyArray<Evt>) => Effect.Effect<void, never, R2>
+        makeInitial?: Effect.Effect<readonly T[], E, RInitial> | undefined
+        config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
+          partitionValue?: (e?: Encoded) => string
+        }
+      },
+    f: (r: Repository<T, Encoded, Evt, ItemType, IdKey, RSchema, RPublish>) => Out
+  ): Effect.Effect<Out, E, StoreMaker | RInitial | R2 | Scope.Scope>
+  readonly Q: ReturnType<typeof Q.make<Encoded>>
+  readonly type: Repository<T, Encoded, Evt, ItemType, IdKey, RSchema, RPublish>
+}