@effect-app/infra 4.0.0-beta.33 → 4.0.0-beta.331

package/src/Store/Disk.ts

@@ -1,12 +1,18 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import * as fu from "../fileUtil.js"
+import * as fu from "../fileUtil.ts"
 
 import fs from "fs"
 
-import { Console, Effect, flow, Semaphore } from "effect-app"
-import type { FieldValues } from "../Model/filter/types.js"
-import { makeMemoryStoreInt, storeId } from "./Memory.js"
-import { type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js"
+import * as Effect from "effect-app/Effect"
+import type { FieldValues } from "effect-app/Model/filter/types"
+import { type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, storeId, StoreMaker } from "effect-app/Store"
+import * as Console from "effect/Console"
+import { flow } from "effect/Function"
+import * as Semaphore from "effect/Semaphore"
+import { annotateDb } from "../otel.ts"
+import { decodeStoredMany, makeJsonDocumentCodec, makeStoredDecode } from "./jsonDocument.ts"
+import { makeMemoryStoreInt } from "./Memory.ts"
+import { type JsonLower, makeJsonLower } from "./utils.ts"
 
 function makeDiskStoreInt<IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
   prefix: string,
@@ -15,9 +21,16 @@ function makeDiskStoreInt<IdKey extends keyof Encoded, Encoded extends FieldValu
   dir: string,
   name: string,
   seed?: Effect.Effect<Iterable<Encoded>, E, R>,
-  defaultValues?: Partial<Encoded>
+  defaultValues?: Partial<Encoded>,
+  schema?: StoreConfig<Encoded>["schema"],
+  json?: JsonLower,
+  jitM?: StoreConfig<Encoded>["jitM"]
 ) {
   type PM = PersistenceModelType<Encoded>
+  const codec = makeJsonDocumentCodec<Encoded>(schema)
+  // the file is the raw JSON boundary for this adapter:
+  // stored JSON -> jitM -> JSON→Encoded decode
+  const decodeStored = makeStoredDecode<Encoded>(schema, jitM)
   return Effect.gen(function*() {
     if (namespace !== "primary") {
       dir = dir + "/" + namespace
@@ -26,51 +39,84 @@ function makeDiskStoreInt<IdKey extends keyof Encoded, Encoded extends FieldValu
       }
     }
     const file = dir + "/" + prefix + name + ".json"
+    const fileExtra = { "disk.file.path": file }
     const fsStore = {
       get: fu
         .readTextFile(file)
         .pipe(
-          Effect.withSpan("Disk.read.readFile [effect-app/infra/Store]", {}, { captureStackTrace: false }),
+          annotateDb({
+            operation: "read.readFile",
+            system: "disk",
+            collection: name,
+            namespace,
+            entity: name,
+            extra: fileExtra
+          }),
           Effect.flatMap((x) =>
             Effect.sync(() => JSON.parse(x) as PM[]).pipe(
-              Effect.withSpan("Disk.read.parse [effect-app/infra/Store]", {}, { captureStackTrace: false })
+              Effect.flatMap((docs) => Effect.fromResult(decodeStoredMany(docs, decodeStored))),
+              annotateDb({
+                operation: "read.parse",
+                system: "disk",
+                collection: name,
+                namespace,
+                entity: name,
+                extra: fileExtra
+              })
             )
           ),
           Effect.orDie,
-          Effect.withSpan("Disk.read [effect-app/infra/Store]", {
-            attributes: { "disk.file": file }
-          }, { captureStackTrace: false })
+          annotateDb({
+            operation: "read",
+            system: "disk",
+            collection: name,
+            namespace,
+            entity: name,
+            extra: fileExtra
+          })
         ),
       setRaw: (v: Iterable<PM>) =>
         Effect
-          .sync(() => JSON.stringify([...v], undefined, 2))
+          .sync(() => JSON.stringify([...v].map((row) => codec.encode(row)), undefined, 2))
           .pipe(
-            Effect.withSpan("Disk.stringify [effect-app/infra/Store]", {
-              attributes: { "disk.file": file }
-            }, { captureStackTrace: false }),
+            annotateDb({
+              operation: "stringify",
+              system: "disk",
+              collection: name,
+              namespace,
+              entity: name,
+              extra: fileExtra
+            }),
             Effect
               .flatMap(
                 (json) =>
                   fu
                     .writeTextFile(file, json)
-                    .pipe(Effect
-                      .withSpan("Disk.write.writeFile [effect-app/infra/Store]", {
-                        attributes: { "disk.file_size": json.length }
-                      }, { captureStackTrace: false }))
+                    .pipe(annotateDb({
+                      operation: "write.writeFile",
+                      system: "disk",
+                      collection: name,
+                      namespace,
+                      entity: name,
+                      extra: { ...fileExtra, "disk.file.size": json.length }
+                    }))
               ),
-            Effect
-              .withSpan("Disk.write [effect-app/infra/Store]", {
-                attributes: { "disk.file": file }
-              }, { captureStackTrace: false })
+            annotateDb({
+              operation: "write",
+              system: "disk",
+              collection: name,
+              namespace,
+              entity: name,
+              extra: fileExtra
+            })
           )
     }
 
     // lock file for cross-process coordination during initialization
-    const lockFile = file + ".lock"
 
     // wrap initialization in file lock to prevent race conditions in multi-worker setups
     const store = yield* fu.withFileLock(
-      lockFile,
+      file,
       Effect.gen(function*() {
         const shouldSeed = !(fs.existsSync(file))
 
@@ -81,7 +127,12 @@ function makeDiskStoreInt<IdKey extends keyof Encoded, Encoded extends FieldValu
           shouldSeed
             ? seed
             : fsStore.get,
-          defaultValues
+          defaultValues,
+          schema,
+          json,
+          // no `jitM`: the documents read from disk were already migrated at
+          // the file boundary above, and applying it twice is not safe.
+          undefined
         )
         if (shouldSeed) {
           yield* store.all.pipe(Effect.flatMap(fsStore.setRaw))
@@ -133,61 +184,83 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) {
       fs.mkdirSync(dir)
     }
     return {
-      make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
+      make: Effect.fnUntraced(function*<IdKey extends keyof Encoded, Encoded extends FieldValues, R, E>(
         name: string,
         idKey: IdKey,
         seed?: Effect.Effect<Iterable<Encoded>, E, R>,
         config?: StoreConfig<Encoded>
-      ) =>
-        Effect.gen(function*() {
-          const storesSem = Semaphore.makeUnsafe(1)
-          const primary = yield* makeDiskStoreInt(prefix, idKey, "primary", dir, name, seed, config?.defaultValues)
-          const stores = new Map<string, Store<IdKey, Encoded>>([["primary", primary]])
-          const ctx = yield* Effect.services<R>()
-          const getStore = !config?.allowNamespace
-            ? Effect.succeed(primary)
-            : storeId.asEffect().pipe(Effect.flatMap((namespace) => {
-              const store = stores.get(namespace)
-              if (store) {
-                return Effect.succeed(store)
-              }
-              if (!config.allowNamespace!(namespace)) {
+      ) {
+        const json = makeJsonLower(config)
+        const primary = yield* makeDiskStoreInt(
+          prefix,
+          idKey,
+          "primary",
+          dir,
+          name,
+          seed,
+          config?.defaultValues,
+          config?.schema,
+          json,
+          config?.jitM
+        )
+          .pipe(
+            Effect.orDie
+          )
+        const stores = new Map<string, Store<IdKey, Encoded>>([["primary", primary]])
+        const ctx = yield* Effect.context<R>()
+        const semaphores = new Map<string, Semaphore.Semaphore>()
+        const getSem = (ns: string) => {
+          let sem = semaphores.get(ns)
+          if (!sem) {
+            sem = Semaphore.makeUnsafe(1)
+            semaphores.set(ns, sem)
+          }
+          return sem
+        }
+        const ensureStore = (namespace: string) =>
+          getSem(namespace).withPermits(1)(
+            Effect.suspend(() => {
+              const existing = stores.get(namespace)
+              if (existing) return Effect.succeed(existing)
+              if (config?.allowNamespace && !config.allowNamespace(namespace)) {
                 throw new Error(`Namespace ${namespace} not allowed!`)
               }
-              return storesSem.withPermits(1)(
-                Effect.suspend(() => {
-                  const existing = stores.get(namespace)
-                  if (existing) return Effect.sync(() => existing)
-                  return makeDiskStoreInt<IdKey, Encoded, R, E>(
-                    prefix,
-                    idKey,
-                    namespace,
-                    dir,
-                    name,
-                    seed,
-                    config?.defaultValues
-                  )
-                    .pipe(
-                      Effect.orDie,
-                      Effect.provide(ctx),
-                      Effect.tap((store) => Effect.sync(() => stores.set(namespace, store)))
-                    )
-                })
+              return makeDiskStoreInt<IdKey, Encoded, R, E>(
+                prefix,
+                idKey,
+                namespace,
+                dir,
+                name,
+                seed,
+                config?.defaultValues,
+                config?.schema,
+                json,
+                config?.jitM
               )
-            }))
+                .pipe(
+                  Effect.orDie,
+                  Effect.provide(ctx),
+                  Effect.tap((store) => Effect.sync(() => stores.set(namespace, store)))
+                )
+            })
+          )
+        const getStore = !config?.allowNamespace
+          ? Effect.succeed(primary)
+          : storeId.pipe(Effect.flatMap((namespace) => ensureStore(namespace)))
 
-          const s: Store<IdKey, Encoded> = {
-            all: Effect.flatMap(getStore, (_) => _.all),
-            find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
-            filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
-            set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
-            batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
-            bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),
-            batchRemove: (...args) => Effect.flatMap(getStore, (_) => _.batchRemove(...args)),
-            queryRaw: (...args) => Effect.flatMap(getStore, (_) => _.queryRaw(...args))
-          }
-          return s
-        })
+        const s: Store<IdKey, Encoded> = {
+          seedNamespace: (namespace) => ensureStore(namespace).pipe(Effect.asVoid),
+          all: Effect.flatMap(getStore, (_) => _.all),
+          find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
+          filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
+          set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
+          batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
+          bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),
+          batchRemove: (...args) => Effect.flatMap(getStore, (_) => _.batchRemove(...args)),
+          queryRaw: (...args) => Effect.flatMap(getStore, (_) => _.queryRaw(...args))
+        }
+        return s
+      })
     }
   })
 }