@effect-app/infra 2.11.0 → 2.12.1

package/src/Model/Repository/internal/internal.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import type {} from "effect/Equal"
 import type {} from "effect/Hash"
 import type { NonEmptyReadonlyArray } from "effect-app"
@@ -26,9 +27,9 @@ export function makeRepoInternal<
   return <
     ItemType extends string,
     R,
-    Encoded extends { id: string },
+    Encoded extends FieldValues,
     T,
-    IdKey extends keyof T
+    IdKey extends keyof T & keyof Encoded
   >(
     name: ItemType,
     schema: S.Schema<T, Encoded, R>,
@@ -41,18 +42,18 @@ export function makeRepoInternal<
       e: Encoded,
       getEtag: (id: string) => string | undefined
     ): PM {
-      return mapTo(e, getEtag(e.id))
+      return mapTo(e, getEtag(e[idKey]))
     }
 
     function mapReverse(
       { _etag, ...e }: PM,
       setEtag: (id: string, eTag: string | undefined) => void
     ): Encoded {
-      setEtag(e.id, _etag)
+      setEtag((e as any)[idKey], _etag)
       return mapFrom(e as unknown as Encoded)
     }
 
-    const mkStore = makeStore<Encoded>()(name, schema, mapTo)
+    const mkStore = makeStore<Encoded>()(name, schema, mapTo, idKey)
 
     function make<RInitial = never, E = never, RPublish = never, RCtx = never>(
       args: [Evt] extends [never] ? {
@@ -128,7 +129,7 @@ export function makeRepoInternal<
                 : s.pipe(S.pick(idKey as any))
             })
           const encodeId = flow(S.encode(i), provideRctx)
-          function findEId(id: Encoded["id"]) {
+          function findEId(id: Encoded[IdKey]) {
             return Effect.flatMap(
               store.find(id),
               (item) =>
@@ -138,13 +139,12 @@ export function makeRepoInternal<
                 })
             )
           }
+          // TODO: select the particular field, instead of as struct
           function findE(id: T[IdKey]) {
             return pipe(
               encodeId({ [idKey]: id } as any),
               Effect.orDie,
-              // we will have idKey because the transform is undone again by the encode schema mumbo jumbo above
-              // TODO: make reliable. (Security: isin: PrimaryKey(ISIN), idKey: "isin", does end up with "id")
-              Effect.map((_) => (_ as any)[idKey] ?? (_ as any).id),
+              Effect.map((_) => (_ as any)[idKey]),
               Effect.flatMap(findEId)
             )
           }
@@ -163,7 +163,7 @@ export function makeRepoInternal<
                     const { get, set } = yield* cms
                     const items = a.map((_) => mapToPersistenceModel(_, get))
                     const ret = yield* store.batchSet(items)
-                    ret.forEach((_) => set(_.id, _._etag))
+                    ret.forEach((_) => set(_[idKey], _._etag))
                   })
               )
               .pipe(Effect.asVoid)
@@ -199,7 +199,7 @@ export function makeRepoInternal<
               // TODO: we should have a batchRemove on store so the adapter can actually batch...
               for (const e of items) {
                 yield* store.remove(mapToPersistenceModel(e, get))
-                set(e.id, undefined)
+                set(e[idKey], undefined)
               }
               yield* Effect
                 .sync(() => toNonEmptyArray([...events]))
@@ -233,13 +233,13 @@ export function makeRepoInternal<
                 {
                   ...args,
                   select: args.select
-                    ? dedupe([...args.select, "id", "_etag" as any])
+                    ? dedupe([...args.select, idKey, "_etag" as any])
                     : undefined
                 } as typeof args
               )
               .pipe(
                 Effect.tap((items) =>
-                  Effect.map(cms, ({ set }) => items.forEach((_) => set((_ as Encoded).id, (_ as PM)._etag)))
+                  Effect.map(cms, ({ set }) => items.forEach((_) => set((_ as Encoded)[idKey], (_ as PM)._etag)))
                 )
               )
 
@@ -372,46 +372,47 @@ const pluralize = (s: string) =>
     ? s.substring(0, s.length - 1) + "ies"
     : s + "s"
 
-export function makeStore<
-  Encoded extends { id: string }
->() {
+export function makeStore<Encoded extends FieldValues>() {
   return <
     ItemType extends string,
     R,
-    E extends { id: string },
-    T
+    E,
+    T,
+    IdKey extends keyof Encoded
   >(
     name: ItemType,
     schema: S.Schema<T, E, R>,
-    mapTo: (e: E, etag: string | undefined) => Encoded
+    mapTo: (e: E, etag: string | undefined) => Encoded,
+    idKey: IdKey
   ) => {
-    function encodeToEncoded() {
-      const getEtag = () => undefined
-      return (t: T) =>
-        S.encode(schema)(t).pipe(
-          Effect.orDie,
-          Effect.map((_) => mapToPersistenceModel(_, getEtag))
-        )
-    }
-
-    function mapToPersistenceModel(
-      e: E,
-      getEtag: (id: string) => string | undefined
-    ): Encoded {
-      return mapTo(e, getEtag(e.id))
-    }
-
     function makeStore<RInitial = never, EInitial = never>(
       makeInitial?: Effect<readonly T[], EInitial, RInitial>,
       config?: Omit<StoreConfig<Encoded>, "partitionValue"> & {
         partitionValue?: (a: Encoded) => string
       }
     ) {
+      function encodeToEncoded() {
+        const getEtag = () => undefined
+        return (t: T) =>
+          S.encode(schema)(t).pipe(
+            Effect.orDie,
+            Effect.map((_) => mapToPersistenceModel(_, getEtag))
+          )
+      }
+
+      function mapToPersistenceModel(
+        e: E,
+        getEtag: (id: string) => string | undefined
+      ): Encoded {
+        return mapTo(e, getEtag((e as any)[idKey] as string))
+      }
+
       return Effect.gen(function*() {
         const { make } = yield* StoreMaker
 
-        const store = yield* make<Encoded, string, RInitial | R, EInitial>(
+        const store = yield* make<IdKey, Encoded, RInitial | R, EInitial>(
           pluralize(name),
+          idKey,
           makeInitial
             ? makeInitial
               .pipe(

package/src/Model/Repository/legacy.ts

@@ -14,13 +14,13 @@ export interface Mapped2<A, R> {
   all: Effect<A[], ParseResult.ParseError, R>
 }
 
-export interface Mapped<Encoded extends { id: string }> {
+export interface Mapped<Encoded> {
   <A, R, IdKey extends keyof A>(schema: S.Schema<A, Encoded, R>): Mapped1<A, IdKey, R>
   // TODO: constrain on Encoded2 having to contain only fields that fit Encoded
   <A, Encoded2, R>(schema: S.Schema<A, Encoded2, R>): Mapped2<A, R>
 }
 
-export interface MM<Repo, Encoded extends { id: string }> {
+export interface MM<Repo, Encoded> {
   <A, R, IdKey extends keyof A>(schema: S.Schema<A, Encoded, R>): Effect<Mapped1<A, IdKey, R>, never, Repo>
   // TODO: constrain on Encoded2 having to contain only fields that fit Encoded
   <A, Encoded2, R>(schema: S.Schema<A, Encoded2, R>): Effect<Mapped2<A, R>, never, Repo>

package/src/Model/Repository/makeRepo.ts

@@ -10,15 +10,14 @@ import type {} from "effect/Hash"
 import type { Context, NonEmptyReadonlyArray, S } from "effect-app"
 import { Effect } from "effect-app"
 import type { StoreConfig, StoreMaker } from "../../Store.js"
+import type { FieldValues } from "../filter/types.js"
 import type { ExtendedRepository } from "./ext.js"
 import { extendRepo } from "./ext.js"
 import { makeRepoInternal } from "./internal/internal.js"
 
 export interface RepositoryOptions<
-  IdKey extends keyof T,
-  Encoded extends {
-    id: string
-  },
+  IdKey extends keyof T & keyof Encoded,
+  Encoded,
   T,
   Evt = never,
   RPublish = never,
@@ -66,9 +65,9 @@ export const makeRepo: {
   <
     ItemType extends string,
     RSchema,
-    Encoded extends { id: string },
+    Encoded extends FieldValues,
     T,
-    IdKey extends keyof T,
+    IdKey extends keyof T & keyof Encoded,
     E = never,
     Evt = never,
     RInitial = never,
@@ -86,7 +85,7 @@ export const makeRepo: {
   <
     ItemType extends string,
     RSchema,
-    Encoded extends { id: string },
+    Encoded extends FieldValues,
     T extends { id: unknown },
     E = never,
     Evt = never,
@@ -105,9 +104,9 @@ export const makeRepo: {
 } = <
   ItemType extends string,
   R,
-  Encoded extends { id: string },
+  Encoded extends FieldValues,
   T,
-  IdKey extends keyof T,
+  IdKey extends keyof T & keyof Encoded,
   E = never,
   RInitial = never,
   RPublish = never,

package/src/Model/Repository/service.ts

@@ -11,7 +11,7 @@ import type { Mapped } from "./legacy.js"
  */
 export interface Repository<
   T,
-  Encoded extends { id: string },
+  Encoded extends FieldValues,
   Evt,
   ItemType extends string,
   IdKey extends keyof T,

package/src/Store/Cosmos/query.ts

@@ -32,6 +32,7 @@ const arrayContains = (v: any[]) => v.map((_) => JSON.stringify(_)).join(", ")
 const vAsArr = (v: string) => v as unknown as any[]
 
 export function buildWhereCosmosQuery3(
+  idKey: PropertyKey,
   filter: readonly FilterResult[],
   name: string,
   importedMarkerId: string,
@@ -42,10 +43,14 @@ export function buildWhereCosmosQuery3(
   limit?: number
 ) {
   const statement = (x: FilterR, i: number) => {
+    if (x.path === idKey) {
+      x = { ...x, path: "id" }
+    }
     let k = x.path.includes(".-1.")
       ? `${x.path.split(".-1.")[0]}.${x.path.split(".-1.")[1]!}`
       : `f.${x.path}`
 
+    // would have to map id, but shouldnt allow id in defaultValues anyway..
     k = x.path in defaultValues ? `(${k} ?? ${JSON.stringify(defaultValues[x.path])})` : k
 
     const v = "@v" + i
@@ -180,7 +185,7 @@ export function buildWhereCosmosQuery3(
     query: `
     SELECT ${
       select
-        ? `${select.map((_) => `f.${_}`).join(", ")}`
+        ? `${select.map((_) => (_ as any) === idKey ? "id" : _).map((_) => `f.${_}`).join(", ")}`
         : "f"
     }
     FROM ${name} f

package/src/Store/Cosmos.ts

@@ -7,10 +7,21 @@ import { dropUndefinedT } from "effect-app/utils"
 import { CosmosClient, CosmosClientLayer } from "../adapters/cosmos-client.js"
 import { OptimisticConcurrencyException } from "../errors.js"
 import { InfraLogger } from "../logger.js"
+import type { FieldValues } from "../Model/filter/types.js"
 import { buildWhereCosmosQuery3, logQuery } from "./Cosmos/query.js"
 import { StoreMaker } from "./service.js"
 import type { FilterArgs, PersistenceModelType, StorageConfig, Store, StoreConfig } from "./service.js"
 
+const makeMapId =
+  <IdKey extends keyof Encoded, Encoded extends FieldValues>(idKey: IdKey) => ({ [idKey]: id, ...e }: Encoded) => ({
+    ...e,
+    id
+  })
+const makeReverseMapId =
+  <IdKey extends keyof Encoded, Encoded extends FieldValues>(idKey: IdKey) =>
+  ({ id, ...t }: PersistenceModelType<Omit<Encoded, IdKey> & { id: string }>) =>
+    ({ ...t, [idKey]: id }) as any as PersistenceModelType<Encoded>
+
 class CosmosDbOperationError {
   constructor(readonly message: string) {}
 } // TODO: Retry operation when running into RU limit.
@@ -19,13 +30,17 @@ function makeCosmosStore({ prefix }: StorageConfig) {
   return Effect.gen(function*() {
     const { db } = yield* CosmosClient
     return {
-      make: <Id extends string, Encoded extends Record<string, any> & { id: Id }, R = never, E = never>(
+      make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R = never, E = never>(
         name: string,
+        idKey: IdKey,
         seed?: Effect<Iterable<Encoded>, E, R>,
         config?: StoreConfig<Encoded>
       ) =>
         Effect.gen(function*() {
+          const mapId = makeMapId<IdKey, Encoded>(idKey)
+          const mapReverseId = makeReverseMapId<IdKey, Encoded>(idKey)
           type PM = PersistenceModelType<Encoded>
+          type PMCosmos = PersistenceModelType<Omit<Encoded, IdKey> & { id: string }>
           const containerId = `${prefix}${name}`
           yield* Effect.promise(() =>
             db.containers.createIfNotExists(dropUndefinedT({
@@ -47,34 +62,37 @@ function makeCosmosStore({ prefix }: StorageConfig) {
               .gen(function*() {
                 // TODO: disable batching if need atomicity
                 // we delay and batch to keep low amount of RUs
-                const b = [...items].map(
-                  (x) =>
-                    [
-                      x,
-                      Option.match(Option.fromNullable(x._etag), {
-                        onNone: () =>
-                          dropUndefinedT({
-                            operationType: "Create" as const,
-                            resourceBody: {
-                              ...Struct.omit(x, "_etag"),
-                              _partitionKey: config?.partitionValue(x)
-                            },
-                            partitionKey: config?.partitionValue(x)
-                          }),
-                        onSome: (eTag) =>
-                          dropUndefinedT({
-                            operationType: "Replace" as const,
-                            id: x.id,
-                            resourceBody: {
-                              ...Struct.omit(x, "_etag"),
-                              _partitionKey: config?.partitionValue(x)
-                            },
-                            ifMatch: eTag,
-                            partitionKey: config?.partitionValue(x)
-                          })
-                      })
-                    ] as const
-                )
+                const b = [...items]
+                  .map(
+                    (x) =>
+                      [
+                        x,
+                        Option.match(Option.fromNullable(x._etag), {
+                          onNone: () =>
+                            dropUndefinedT({
+                              operationType: "Create" as const,
+                              resourceBody: {
+                                ...Struct.omit(x, "_etag", idKey),
+                                id: x[idKey],
+                                _partitionKey: config?.partitionValue(x)
+                              },
+                              partitionKey: config?.partitionValue(x)
+                            }),
+                          onSome: (eTag) =>
+                            dropUndefinedT({
+                              operationType: "Replace" as const,
+                              id: x[idKey],
+                              resourceBody: {
+                                ...Struct.omit(x, "_etag", idKey),
+                                id: x[idKey],
+                                _partitionKey: config?.partitionValue(x)
+                              },
+                              ifMatch: eTag,
+                              partitionKey: config?.partitionValue(x)
+                            })
+                        })
+                      ] as const
+                  )
                 const batches = Chunk.toReadonlyArray(Array.chunk_(b, config?.maxBulkSize ?? 10))
 
                 const batchResult = yield* Effect.forEach(
@@ -135,15 +153,17 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                         onNone: () => ({
                           operationType: "Create" as const,
                           resourceBody: {
-                            ...Struct.omit(x, "_etag"),
+                            ...Struct.omit(x, "_etag", idKey),
+                            id: x[idKey],
                             _partitionKey: config?.partitionValue(x)
                           }
                         }),
                         onSome: (eTag) => ({
                           operationType: "Replace" as const,
-                          id: x.id,
+                          id: x[idKey],
                           resourceBody: {
-                            ...Struct.omit(x, "_etag"),
+                            ...Struct.omit(x, "_etag", idKey),
+                            id: x[idKey],
                             _partitionKey: config?.partitionValue(x)
                           },
                           ifMatch: eTag
@@ -187,7 +207,7 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                 }))
           }
 
-          const s: Store<Encoded, Id> = {
+          const s: Store<IdKey, Encoded> = {
             all: Effect
               .sync(() => ({
                 query: `SELECT * FROM ${name} f WHERE f.id != @id`,
@@ -199,9 +219,13 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                   Effect.promise(() =>
                     container
                       .items
-                      .query<PM>(q)
+                      .query<PMCosmos>(q)
                       .fetchAll()
-                      .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._ })))
+                      .then(({ resources }) =>
+                        resources.map(
+                          (_) => ({ ...defaultValues, ...mapReverseId(_) })
+                        )
+                      )
                   )
                 ),
                 Effect
@@ -223,6 +247,7 @@ function makeCosmosStore({ prefix }: StorageConfig) {
               return Effect
                 .sync(() =>
                   buildWhereCosmosQuery3(
+                    idKey,
                     filter ?? [],
                     name,
                     importedMarkerId,
@@ -244,13 +269,20 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                             .query<M>(q)
                             .fetchAll()
                             .then(({ resources }) =>
-                              resources.map((_) => ({ ...pipe(defaultValues, Struct.pick(...f.select!)), ..._ }))
+                              resources.map((_) =>
+                                ({
+                                  ...pipe(defaultValues, Struct.pick(...f.select!)),
+                                  ...mapReverseId(_ as any)
+                                }) as any
+                              )
                             )
                           : container
                             .items
                             .query<{ f: M }>(q)
                             .fetchAll()
-                            .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._.f })))
+                            .then(({ resources }) =>
+                              resources.map(({ f }) => ({ ...defaultValues, ...mapReverseId(f as any) }) as any)
+                            )
                       )
                     )
                 )
@@ -263,10 +295,10 @@ function makeCosmosStore({ prefix }: StorageConfig) {
               Effect
                 .promise(() =>
                   container
-                    .item(id, config?.partitionValue({ id } as Encoded))
+                    .item(id, config?.partitionValue({ [idKey]: id } as Encoded))
                     .read<Encoded>()
                     .then(({ resource }) =>
-                      Option.fromNullable(resource).pipe(Option.map((_) => ({ ...defaultValues, ..._ })))
+                      Option.fromNullable(resource).pipe(Option.map((_) => ({ ...defaultValues, ...mapReverseId(_) })))
                     )
                 )
                 .pipe(Effect
@@ -275,7 +307,7 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                     attributes: {
                       "repository.container_id": containerId,
                       "repository.model_name": name,
-                      partitionValue: config?.partitionValue({ id } as Encoded),
+                      partitionValue: config?.partitionValue({ [idKey]: id } as Encoded),
                       id
                     }
                   })),
@@ -288,14 +320,14 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                     onNone: () =>
                       Effect.promise(() =>
                         container.items.create({
-                          ...e,
+                          ...mapId(e),
                           _partitionKey: config?.partitionValue(e)
                         })
                       ),
                     onSome: (eTag) =>
                       Effect.promise(() =>
-                        container.item(e.id, config?.partitionValue(e)).replace(
-                          { ...e, _partitionKey: config?.partitionValue(e) },
+                        container.item(e[idKey], config?.partitionValue(e)).replace(
+                          { ...mapId(e), _partitionKey: config?.partitionValue(e) },
                           {
                             accessCondition: {
                               type: "IfMatch",
@@ -310,7 +342,7 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                   Effect
                     .flatMap((x) => {
                       if (x.statusCode === 412 || x.statusCode === 404) {
-                        return new OptimisticConcurrencyException({ type: name, id: e.id })
+                        return new OptimisticConcurrencyException({ type: name, id: e[idKey] })
                       }
                       if (x.statusCode > 299 || x.statusCode < 200) {
                         return Effect.die(
@@ -327,18 +359,22 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                   Effect
                     .withSpan("Cosmos.set [effect-app/infra/Store]", {
                       captureStackTrace: false,
-                      attributes: { "repository.container_id": containerId, "repository.model_name": name, id: e.id }
+                      attributes: {
+                        "repository.container_id": containerId,
+                        "repository.model_name": name,
+                        id: e[idKey]
+                      }
                     })
                 ),
             batchSet,
             bulkSet,
             remove: (e: Encoded) =>
               Effect
-                .promise(() => container.item(e.id, config?.partitionValue(e)).delete())
+                .promise(() => container.item(e[idKey], config?.partitionValue(e)).delete())
                 .pipe(Effect
                   .withSpan("Cosmos.remove [effect-app/infra/Store]", {
                     captureStackTrace: false,
-                    attributes: { "repository.container_id": containerId, "repository.model_name": name, id: e.id }
+                    attributes: { "repository.container_id": containerId, "repository.model_name": name, id: e[idKey] }
                   }))
           }
 

package/src/Store/Disk.ts

@@ -4,12 +4,14 @@ import * as fu from "../fileUtil.js"
 import fs from "fs"
 
 import { Console, Effect, FiberRef, flow } from "effect-app"
+import type { FieldValues } from "../Model/filter/types.js"
 import { makeMemoryStoreInt, storeId } from "./Memory.js"
 import type { PersistenceModelType, StorageConfig, Store, StoreConfig } from "./service.js"
 import { StoreMaker } from "./service.js"
 
-function makeDiskStoreInt<Id extends string, Encoded extends { id: Id }, R, E>(
+function makeDiskStoreInt<IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
   prefix: string,
+  idKey: IdKey,
   namespace: string,
   dir: string,
   name: string,
@@ -68,8 +70,9 @@ function makeDiskStoreInt<Id extends string, Encoded extends { id: Id }, R, E>(
           )
     }
 
-    const store = yield* makeMemoryStoreInt<Id, Encoded, R, E>(
+    const store = yield* makeMemoryStoreInt<IdKey, Encoded, R, E>(
       name,
+      idKey,
       namespace,
       !fs.existsSync(file)
         ? seed
@@ -107,7 +110,7 @@ function makeDiskStoreInt<Id extends string, Encoded extends { id: Id }, R, E>(
         store.remove,
         Effect.tap(flushToDiskInBackground)
       )
-    } satisfies Store<Encoded, Id>
+    } satisfies Store<IdKey, Encoded>
   })
 }
 
@@ -121,15 +124,16 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) {
       fs.mkdirSync(dir)
     }
     return {
-      make: <Id extends string, Encoded extends { id: Id }, R, E>(
+      make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
         name: string,
+        idKey: IdKey,
         seed?: Effect<Iterable<Encoded>, E, R>,
         config?: StoreConfig<Encoded>
       ) =>
         Effect.gen(function*() {
           const storesSem = Effect.unsafeMakeSemaphore(1)
-          const primary = yield* makeDiskStoreInt(prefix, "primary", dir, name, seed, config?.defaultValues)
-          const stores = new Map<string, Store<Encoded, Id>>([["primary", primary]])
+          const primary = yield* makeDiskStoreInt(prefix, idKey, "primary", dir, name, seed, config?.defaultValues)
+          const stores = new Map<string, Store<IdKey, Encoded>>([["primary", primary]])
           const ctx = yield* Effect.context<R>()
           const getStore = !config?.allowNamespace
             ? Effect.succeed(primary)
@@ -145,7 +149,15 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) {
                 Effect.suspend(() => {
                   const existing = stores.get(namespace)
                   if (existing) return Effect.sync(() => existing)
-                  return makeDiskStoreInt<Id, Encoded, R, E>(prefix, namespace, dir, name, seed, config?.defaultValues)
+                  return makeDiskStoreInt<IdKey, Encoded, R, E>(
+                    prefix,
+                    idKey,
+                    namespace,
+                    dir,
+                    name,
+                    seed,
+                    config?.defaultValues
+                  )
                     .pipe(
                       Effect.orDie,
                       Effect.provide(ctx),
@@ -155,7 +167,7 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) {
               )
             }))
 
-          const s: Store<Encoded, Id> = {
+          const s: Store<IdKey, Encoded> = {
             all: Effect.flatMap(getStore, (_) => _.all),
             find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
             filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),