@effect-app/infra 4.0.0-beta.21 → 4.0.0-beta.211

package/src/Store/Disk.ts

@@ -5,6 +5,7 @@ import fs from "fs"
 
 import { Console, Effect, flow, Semaphore } from "effect-app"
 import type { FieldValues } from "../Model/filter/types.js"
+import { annotateDb } from "../otel.js"
 import { makeMemoryStoreInt, storeId } from "./Memory.js"
 import { type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js"
 
@@ -26,51 +27,83 @@ function makeDiskStoreInt<IdKey extends keyof Encoded, Encoded extends FieldValu
       }
     }
     const file = dir + "/" + prefix + name + ".json"
+    const fileExtra = { "disk.file.path": file }
     const fsStore = {
       get: fu
         .readTextFile(file)
         .pipe(
-          Effect.withSpan("Disk.read.readFile [effect-app/infra/Store]", {}, { captureStackTrace: false }),
+          annotateDb({
+            operation: "read.readFile",
+            system: "disk",
+            collection: name,
+            namespace,
+            entity: name,
+            extra: fileExtra
+          }),
           Effect.flatMap((x) =>
             Effect.sync(() => JSON.parse(x) as PM[]).pipe(
-              Effect.withSpan("Disk.read.parse [effect-app/infra/Store]", {}, { captureStackTrace: false })
+              annotateDb({
+                operation: "read.parse",
+                system: "disk",
+                collection: name,
+                namespace,
+                entity: name,
+                extra: fileExtra
+              })
             )
           ),
           Effect.orDie,
-          Effect.withSpan("Disk.read [effect-app/infra/Store]", {
-            attributes: { "disk.file": file }
-          }, { captureStackTrace: false })
+          annotateDb({
+            operation: "read",
+            system: "disk",
+            collection: name,
+            namespace,
+            entity: name,
+            extra: fileExtra
+          })
         ),
       setRaw: (v: Iterable<PM>) =>
         Effect
           .sync(() => JSON.stringify([...v], undefined, 2))
           .pipe(
-            Effect.withSpan("Disk.stringify [effect-app/infra/Store]", {
-              attributes: { "disk.file": file }
-            }, { captureStackTrace: false }),
+            annotateDb({
+              operation: "stringify",
+              system: "disk",
+              collection: name,
+              namespace,
+              entity: name,
+              extra: fileExtra
+            }),
             Effect
               .flatMap(
                 (json) =>
                   fu
                     .writeTextFile(file, json)
-                    .pipe(Effect
-                      .withSpan("Disk.write.writeFile [effect-app/infra/Store]", {
-                        attributes: { "disk.file_size": json.length }
-                      }, { captureStackTrace: false }))
+                    .pipe(annotateDb({
+                      operation: "write.writeFile",
+                      system: "disk",
+                      collection: name,
+                      namespace,
+                      entity: name,
+                      extra: { ...fileExtra, "disk.file.size": json.length }
+                    }))
               ),
-            Effect
-              .withSpan("Disk.write [effect-app/infra/Store]", {
-                attributes: { "disk.file": file }
-              }, { captureStackTrace: false })
+            annotateDb({
+              operation: "write",
+              system: "disk",
+              collection: name,
+              namespace,
+              entity: name,
+              extra: fileExtra
+            })
           )
     }
 
     // lock file for cross-process coordination during initialization
-    const lockFile = file + ".lock"
 
     // wrap initialization in file lock to prevent race conditions in multi-worker setups
     const store = yield* fu.withFileLock(
-      lockFile,
+      file,
       Effect.gen(function*() {
         const shouldSeed = !(fs.existsSync(file))
 
@@ -133,61 +166,65 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) {
       fs.mkdirSync(dir)
     }
     return {
-      make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
+      make: Effect.fnUntraced(function*<IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
         name: string,
         idKey: IdKey,
         seed?: Effect.Effect<Iterable<Encoded>, E, R>,
         config?: StoreConfig<Encoded>
-      ) =>
-        Effect.gen(function*() {
-          const storesSem = Semaphore.makeUnsafe(1)
-          const primary = yield* makeDiskStoreInt(prefix, idKey, "primary", dir, name, seed, config?.defaultValues)
-          const stores = new Map<string, Store<IdKey, Encoded>>([["primary", primary]])
-          const ctx = yield* Effect.services<R>()
-          const getStore = !config?.allowNamespace
-            ? Effect.succeed(primary)
-            : storeId.asEffect().pipe(Effect.flatMap((namespace) => {
-              const store = stores.get(namespace)
-              if (store) {
-                return Effect.succeed(store)
-              }
-              if (!config.allowNamespace!(namespace)) {
+      ) {
+        const primary = yield* makeDiskStoreInt(prefix, idKey, "primary", dir, name, seed, config?.defaultValues)
+        const stores = new Map<string, Store<IdKey, Encoded>>([["primary", primary]])
+        const ctx = yield* Effect.context<R>()
+        const semaphores = new Map<string, Semaphore.Semaphore>()
+        const getSem = (ns: string) => {
+          let sem = semaphores.get(ns)
+          if (!sem) {
+            sem = Semaphore.makeUnsafe(1)
+            semaphores.set(ns, sem)
+          }
+          return sem
+        }
+        const ensureStore = (namespace: string) =>
+          getSem(namespace).withPermits(1)(
+            Effect.suspend(() => {
+              const existing = stores.get(namespace)
+              if (existing) return Effect.succeed(existing)
+              if (config?.allowNamespace && !config.allowNamespace(namespace)) {
                 throw new Error(`Namespace ${namespace} not allowed!`)
               }
-              return storesSem.withPermits(1)(
-                Effect.suspend(() => {
-                  const existing = stores.get(namespace)
-                  if (existing) return Effect.sync(() => existing)
-                  return makeDiskStoreInt<IdKey, Encoded, R, E>(
-                    prefix,
-                    idKey,
-                    namespace,
-                    dir,
-                    name,
-                    seed,
-                    config?.defaultValues
-                  )
-                    .pipe(
-                      Effect.orDie,
-                      Effect.provide(ctx),
-                      Effect.tap((store) => Effect.sync(() => stores.set(namespace, store)))
-                    )
-                })
+              return makeDiskStoreInt<IdKey, Encoded, R, E>(
+                prefix,
+                idKey,
+                namespace,
+                dir,
+                name,
+                seed,
+                config?.defaultValues
               )
-            }))
+                .pipe(
+                  Effect.orDie,
+                  Effect.provide(ctx),
+                  Effect.tap((store) => Effect.sync(() => stores.set(namespace, store)))
+                )
+            })
+          )
+        const getStore = !config?.allowNamespace
+          ? Effect.succeed(primary)
+          : storeId.asEffect().pipe(Effect.flatMap((namespace) => ensureStore(namespace)))
 
-          const s: Store<IdKey, Encoded> = {
-            all: Effect.flatMap(getStore, (_) => _.all),
-            find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
-            filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
-            set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
-            batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
-            bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),
-            batchRemove: (...args) => Effect.flatMap(getStore, (_) => _.batchRemove(...args)),
-            queryRaw: (...args) => Effect.flatMap(getStore, (_) => _.queryRaw(...args))
-          }
-          return s
-        })
+        const s: Store<IdKey, Encoded> = {
+          seedNamespace: (namespace) => ensureStore(namespace).pipe(Effect.asVoid),
+          all: Effect.flatMap(getStore, (_) => _.all),
+          find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
+          filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
+          set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
+          batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
+          bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),
+          batchRemove: (...args) => Effect.flatMap(getStore, (_) => _.batchRemove(...args)),
+          queryRaw: (...args) => Effect.flatMap(getStore, (_) => _.queryRaw(...args))
+        }
+        return s
+      })
     }
   })
 }

package/src/Store/Memory.ts

@@ -1,14 +1,19 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-import { Array, Effect, flow, type NonEmptyReadonlyArray, Option, Order, pipe, Ref, Result, Semaphore, ServiceMap, Struct } from "effect-app"
+import { Array, Context, Effect, flow, type NonEmptyReadonlyArray, Option, Order, pipe, Ref, Result, Semaphore, Struct } from "effect-app"
 import { NonEmptyString255 } from "effect-app/Schema"
-import { get } from "effect-app/utils"
 import { InfraLogger } from "../logger.js"
 import type { FieldValues } from "../Model/filter/types.js"
+import { annotateDb } from "../otel.js"
 import { codeFilter, codeFilter3_ } from "./codeFilter.js"
 import { type FilterArgs, type PersistenceModelType, type Store, type StoreConfig, StoreMaker } from "./service.js"
 import { makeUpdateETag } from "./utils.js"
 
+/** Traverse an object by a dot-separated path string, e.g. `"a.b.c"`. */
+export function get(obj: any, path: string): any {
+  return path.split(".").reduce((res: any, key: string) => (res != null ? res[key] : res), obj)
+}
+
 export function memFilter<T extends FieldValues, U extends keyof T = never>(f: FilterArgs<T, U>) {
   type M = U extends undefined ? T : Pick<T, U>
   return ((c: T[]): M[] => {
@@ -27,7 +32,7 @@ export function memFilter<T extends FieldValues, U extends keyof T = never>(f: F
           n[subKey.key] = i[subKey.key]!.map(Struct.pick(subKey.subKeys as never[]))
         })
         return n as M
-      }) as any
+      })
     }
     const skip = f?.skip
     const limit = f?.limit
@@ -72,7 +77,7 @@ export function memFilter<T extends FieldValues, U extends keyof T = never>(f: F
 }
 
 const defaultNs: NonEmptyString255 = NonEmptyString255("primary")
-export class storeId extends ServiceMap.Reference("StoreId", { defaultValue: (): NonEmptyString255 => defaultNs }) {}
+export class storeId extends Context.Reference("StoreId", { defaultValue: (): NonEmptyString255 => defaultNs }) {}
 
 function logQuery(f: FilterArgs<any, any>, defaultValues?: any) {
   return InfraLogger
@@ -151,43 +156,55 @@ export function makeMemoryStoreInt<IdKey extends keyof Encoded, Encoded extends
           withPermit
         )
     const s: Store<IdKey, Encoded> = {
+      seedNamespace: () => Effect.void,
+
       queryRaw: (query) =>
         all
           .pipe(
             // Effect.tap(() => logQuery(query, defaultValues)),
             Effect.map(query.memory),
-            Effect.withSpan("Memory.queryRaw [effect-app/infra/Store]", {
-              attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
-            }, { captureStackTrace: false })
+            annotateDb({
+              operation: "queryRaw",
+              system: "memory",
+              collection: modelName,
+              namespace,
+              entity: modelName
+            })
           ),
 
-      all: all.pipe(Effect.withSpan("Memory.all [effect-app/infra/Store]", {
-        attributes: {
-          modelName,
-          namespace
-        }
-      }, { captureStackTrace: false })),
+      all: all.pipe(annotateDb({
+        operation: "all",
+        system: "memory",
+        collection: modelName,
+        namespace,
+        entity: modelName
+      })),
       find: (id) =>
         Ref
           .get(store)
           .pipe(
             Effect.map((_) => Option.fromNullishOr(_.get(id))),
-            Effect
-              .withSpan("Memory.find [effect-app/infra/Store]", {
-                attributes: {
-                  modelName,
-                  namespace
-                }
-              }, { captureStackTrace: false })
+            annotateDb({
+              operation: "find",
+              system: "memory",
+              collection: modelName,
+              namespace,
+              entity: modelName,
+              extra: { "app.entity.id": id as unknown }
+            })
           ),
       filter: (f) =>
         all
           .pipe(
             Effect.tap(() => logQuery(f, defaultValues)),
             Effect.map(memFilter(f)),
-            Effect.withSpan("Memory.filter [effect-app/infra/Store]", {
-              attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
-            }, { captureStackTrace: false })
+            annotateDb({
+              operation: "filter",
+              system: "memory",
+              collection: modelName,
+              namespace,
+              entity: modelName
+            })
           ),
       set: (e) =>
         s
@@ -202,10 +219,14 @@ export function makeMemoryStoreInt<IdKey extends keyof Encoded, Encoded extends
                 )
               ),
             withPermit,
-            Effect
-              .withSpan("Memory.set [effect-app/infra/Store]", {
-                attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
-              }, { captureStackTrace: false })
+            annotateDb({
+              operation: "set",
+              system: "memory",
+              collection: modelName,
+              namespace,
+              entity: modelName,
+              extra: { "app.entity.id": e[idKey] as unknown }
+            })
           ),
       batchRemove: (items: NonEmptyReadonlyArray<Encoded[IdKey]>) =>
         pipe(
@@ -216,10 +237,13 @@ export function makeMemoryStoreInt<IdKey extends keyof Encoded, Encoded extends
               Effect.filterOrFail((_) => _.length <= 100, () => "BatchRemove: a batch may not exceed 100 items"),
               Effect.orDie,
               Effect.andThen(batchRemove),
-              Effect
-                .withSpan("Memory.batchRemove [effect-app/infra/Store]", {
-                  attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
-                }, { captureStackTrace: false })
+              annotateDb({
+                operation: "batchRemove",
+                system: "memory",
+                collection: modelName,
+                namespace,
+                entity: modelName
+              })
             )
         ),
       batchSet: (items: readonly [PM, ...PM[]]) =>
@@ -231,18 +255,25 @@ export function makeMemoryStoreInt<IdKey extends keyof Encoded, Encoded extends
               Effect.filterOrFail((_) => _.length <= 100, () => "BatchSet: a batch may not exceed 100 items"),
               Effect.orDie,
               Effect.andThen(batchSet),
-              Effect
-                .withSpan("Memory.batchSet [effect-app/infra/Store]", {
-                  attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
-                }, { captureStackTrace: false })
+              annotateDb({
+                operation: "batchSet",
+                system: "memory",
+                collection: modelName,
+                namespace,
+                entity: modelName
+              })
             )
         ),
       bulkSet: flow(
         batchSet,
         (_) =>
-          _.pipe(Effect.withSpan("Memory.bulkSet [effect-app/infra/Store]", {
-            attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
-          }, { captureStackTrace: false }))
+          _.pipe(annotateDb({
+            operation: "bulkSet",
+            system: "memory",
+            collection: modelName,
+            namespace,
+            entity: modelName
+          }))
       )
     }
     return s
@@ -250,56 +281,60 @@ export function makeMemoryStoreInt<IdKey extends keyof Encoded, Encoded extends
 }
 
 export const makeMemoryStore = () => ({
-  make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
+  make: Effect.fnUntraced(function*<IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
     modelName: string,
     idKey: IdKey,
     seed?: Effect.Effect<Iterable<Encoded>, E, R>,
     config?: StoreConfig<Encoded>
-  ) =>
-    Effect.gen(function*() {
-      const storesSem = Semaphore.makeUnsafe(1)
-      const primary = yield* makeMemoryStoreInt<IdKey, Encoded, R, E>(
-        modelName,
-        idKey,
-        "primary",
-        seed,
-        config?.defaultValues
-      )
-      const ctx = yield* Effect.services<R>()
-      const stores = new Map([["primary", primary]])
-      const getStore = !config?.allowNamespace
-        ? Effect.succeed(primary)
-        : storeId.asEffect().pipe(Effect.flatMap((namespace) => {
-          const store = stores.get(namespace)
-          if (store) {
-            return Effect.succeed(store)
-          }
-          if (!config.allowNamespace!(namespace)) {
-            throw new Error(`Namespace ${namespace} not allowed!`)
-          }
-          return storesSem.withPermits(1)(Effect.suspend(() => {
-            const store = stores.get(namespace)
-            if (store) return Effect.sync(() => store)
-            return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues)
-              .pipe(
-                Effect.orDie,
-                Effect.provide(ctx),
-                Effect.tap((store) => Effect.sync(() => stores.set(namespace, store)))
-              )
-          }))
-        }))
-      const s: Store<IdKey, Encoded> = {
-        all: Effect.flatMap(getStore, (_) => _.all),
-        queryRaw: (...args) => Effect.flatMap(getStore, (_) => _.queryRaw(...args)),
-        find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
-        filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
-        set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
-        batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
-        bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),
-        batchRemove: (...args) => Effect.flatMap(getStore, (_) => _.batchRemove(...args))
+  ) {
+    const primary = yield* makeMemoryStoreInt<IdKey, Encoded, R, E>(
+      modelName,
+      idKey,
+      "primary",
+      seed,
+      config?.defaultValues
+    )
+    const ctx = yield* Effect.context<R>()
+    const stores = new Map([["primary", primary]])
+    const semaphores = new Map<string, Semaphore.Semaphore>()
+    const getSem = (ns: string) => {
+      let sem = semaphores.get(ns)
+      if (!sem) {
+        sem = Semaphore.makeUnsafe(1)
+        semaphores.set(ns, sem)
       }
-      return s
-    })
+      return sem
+    }
+    const ensureStore = (namespace: string) =>
+      getSem(namespace).withPermits(1)(Effect.suspend(() => {
+        const store = stores.get(namespace)
+        if (store) return Effect.succeed(store)
+        if (config?.allowNamespace && !config.allowNamespace(namespace)) {
+          throw new Error(`Namespace ${namespace} not allowed!`)
+        }
+        return makeMemoryStoreInt(modelName, idKey, namespace, seed, config?.defaultValues)
+          .pipe(
+            Effect.orDie,
+            Effect.provide(ctx),
+            Effect.tap((store) => Effect.sync(() => stores.set(namespace, store)))
+          )
+      }))
+    const getStore = !config?.allowNamespace
+      ? Effect.succeed(primary)
+      : storeId.asEffect().pipe(Effect.flatMap((namespace) => ensureStore(namespace)))
+    const s: Store<IdKey, Encoded> = {
+      seedNamespace: (namespace) => ensureStore(namespace).pipe(Effect.asVoid),
+      all: Effect.flatMap(getStore, (_) => _.all),
+      queryRaw: (...args) => Effect.flatMap(getStore, (_) => _.queryRaw(...args)),
+      find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
+      filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
+      set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
+      batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
+      bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),
+      batchRemove: (...args) => Effect.flatMap(getStore, (_) => _.batchRemove(...args))
+    }
+    return s
+  })
 })
 
 export const MemoryStoreLive = StoreMaker.toLayer(Effect.sync(() => makeMemoryStore()))