@effect-app/infra 4.0.0-beta.33 → 4.0.0-beta.330

package/src/Store/SQL/Pg.ts

@@ -0,0 +1,412 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import type { NonEmptyReadonlyArray } from "effect-app/Array"
+import { toNonEmptyArray } from "effect-app/Array"
+import * as Effect from "effect-app/Effect"
+import type { FieldValues } from "effect-app/Model/filter/types"
+import type { ComputedProjectionIrExpression } from "effect-app/Model/query"
+import * as Option from "effect-app/Option"
+import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, storeId, StoreMaker } from "effect-app/Store"
+import * as Struct from "effect/Struct"
+import { SqlClient } from "effect/unstable/sql"
+import { DatabaseError, OptimisticConcurrencyException } from "../../errors.ts"
+import { InfraLogger } from "../../logger.ts"
+import { annotateDb } from "../../otel.ts"
+import { decodeStoredMany, decodeStoredOption, makeJsonDocumentCodec, makeStoredDecode } from "../jsonDocument.ts"
+import { makeETag, makeJsonLower, toJsonQueryValue } from "../utils.ts"
+import { buildWhereSQLQuery, logQuery, pgDialect } from "./query.ts"
+
+const sqlErrorMessage = (e: unknown) => (e as any)?.message ? String((e as any).message) : String(e)
+const sqlIsTransient = (e: unknown) =>
+  /timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|connection|deadlock|too many connections/i.test(sqlErrorMessage(e))
+// Map a SqlError into a typed, serializable DatabaseError (instead of `.orDie`,
+// which would turn it into an opaque, non-serializable defect).
+const toDatabaseError = (e: unknown) =>
+  new DatabaseError({ message: `SQL request failed: ${sqlErrorMessage(e)}`, transient: sqlIsTransient(e), cause: e })
+// withTransaction may re-raise setInternal's typed errors or add SqlError on begin/commit.
+// Preserve DatabaseError / OCC by `_tag` (not instanceof — dual package instances break it);
+// map residual SQL failures to DatabaseError.
+const preserveStoreError = (e: unknown): DatabaseError | OptimisticConcurrencyException => {
+  if (e !== null && typeof e === "object" && "_tag" in e) {
+    if (e._tag === "DatabaseError") return e as DatabaseError
+    if (e._tag === "OptimisticConcurrencyException") return e as OptimisticConcurrencyException
+  }
+  return toDatabaseError(e)
+}
+
+/**
+ * Row -> raw stored JSON document. Decoding it is a separate step
+ * (`makeStoredDecode`), so a document `jitM` does not repair fails as a typed
+ * `S.SchemaError` rather than throwing out of here.
+ */
+const parseRow = <Encoded extends FieldValues>(
+  row: { id: string; _etag: string | null; data: unknown },
+  idKey: PropertyKey,
+  defaultValues: Partial<Encoded>
+): PersistenceModelType<Encoded> => {
+  const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object
+  const jsonDefaults = toJsonQueryValue(defaultValues) as Partial<Encoded>
+  return { ...jsonDefaults, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType<Encoded>
+}
+
+const parseSelectRow = (
+  row: Record<string, unknown>,
+  idKey: PropertyKey,
+  defaultValues: Record<string, unknown>
+): any => {
+  const result: Record<string, unknown> = { ...defaultValues }
+  for (const [key, value] of Object.entries(row)) {
+    if (key === "id") {
+      result[idKey as string] = value
+      result["id"] = value
+    } else {
+      result[key] = value
+    }
+  }
+  return result
+}
+
+const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) {
+  const sql = yield* SqlClient.SqlClient
+  return {
+    make: Effect.fnUntraced(function*<IdKey extends keyof Encoded, Encoded extends FieldValues, R = never, E = never>(
+      name: string,
+      idKey: IdKey,
+      seed?: Effect.Effect<Iterable<Encoded>, E, R>,
+      config?: StoreConfig<Encoded>
+    ) {
+      type PM = PersistenceModelType<Encoded>
+      const tableName = `${prefix}${name}`
+      const json = makeJsonLower(config)
+      const defaultValues = json.toJson(config?.defaultValues ?? {}) as Partial<Encoded>
+      const codec = makeJsonDocumentCodec<Encoded>(config?.schema)
+      // read path: stored JSON -> defaultValues -> jitM -> JSON→Encoded decode
+      const decodeStored = makeStoredDecode<Encoded>(config?.schema, config?.jitM)
+      const decodeRow = (row: { id: string; _etag: string | null; data: unknown }) =>
+        decodeStored(parseRow<Encoded>(row, idKey, defaultValues))
+
+      const resolveNamespace = !config?.allowNamespace
+        ? Effect.succeed("primary")
+        : storeId.pipe(Effect.map((namespace) => {
+          if (namespace !== "primary" && !config.allowNamespace!(namespace)) {
+            throw new Error(`Namespace ${namespace} not allowed!`)
+          }
+          return namespace
+        }))
+
+      const ensureTable = sql
+        .unsafe(
+          `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data JSONB NOT NULL, PRIMARY KEY (id, _namespace))`
+        )
+        .pipe(
+          Effect.andThen(
+            sql.unsafe(
+              `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))`
+            )
+          ),
+          Effect.orDie,
+          Effect.asVoid
+        )
+
+      const toRow = (e: PM) => {
+        const newE = makeETag(codec.encode(e))
+        const id = newE[idKey] as string
+        const { _etag, [idKey]: _id, ...rest } = newE as any
+        const data = JSON.stringify(rest)
+        return { id, _etag: newE._etag!, data, item: codec.decode(newE) }
+      }
+
+      const exec = (query: string, params?: readonly unknown[]) =>
+        sql.unsafe(query, params as any).pipe(Effect.mapError(toDatabaseError))
+
+      const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) {
+        const row = toRow(e)
+        if (e._etag) {
+          yield* exec(
+            `UPDATE "${tableName}" SET _etag = $1, data = $2 WHERE id = $3 AND _etag = $4 AND _namespace = $5`,
+            [row._etag, row.data, row.id, e._etag, ns]
+          )
+          const existing = yield* exec(
+            `SELECT _etag FROM "${tableName}" WHERE id = $1 AND _namespace = $2`,
+            [row.id, ns]
+          )
+          const current = (existing as any[])[0]
+          if (!current || current._etag !== row._etag) {
+            if (current) {
+              return yield* new OptimisticConcurrencyException({
+                type: name,
+                id: row.id,
+                current: current._etag,
+                found: e._etag,
+                code: 412
+              })
+            }
+            return yield* new OptimisticConcurrencyException({
+              type: name,
+              id: row.id,
+              current: "",
+              found: e._etag,
+              code: 404
+            })
+          }
+        } else {
+          yield* exec(
+            `INSERT INTO "${tableName}" (id, _namespace, _etag, data) VALUES ($1, $2, $3, $4)`,
+            [row.id, ns, row._etag, row.data]
+          )
+        }
+        return row.item
+      })
+
+      const bulkSetInternal = (items: NonEmptyReadonlyArray<PM>, ns: string) =>
+        sql
+          .withTransaction(Effect.forEach(items, (e) => setInternal(e, ns)))
+          .pipe(
+            Effect.mapError(preserveStoreError),
+            Effect.map((_) => _ as unknown as NonEmptyReadonlyArray<PM>)
+          )
+
+      const ctx = yield* Effect.context<R>()
+      const seedCache = new Map<string, Effect.Effect<void, DatabaseError>>()
+      const makeSeedEffect = Effect.fnUntraced(function*(ns: string) {
+        yield* ensureTable
+        if (!seed) return
+        const existing = yield* exec(
+          `SELECT id FROM "_migrations" WHERE id = $1 AND version = $2`,
+          [`${tableName}::${ns}`, tableName]
+        )
+        if ((existing as any[]).length > 0) return
+        yield* InfraLogger.logInfo(`Seeding data for ${name} (namespace: ${ns})`)
+        const items = yield* seed.pipe(Effect.provide(ctx), Effect.orDie)
+        const ne = toNonEmptyArray([...items])
+        // Seed inserts are not concurrent; OCC here is a programming defect.
+        if (Option.isSome(ne)) {
+          yield* bulkSetInternal(ne.value, ns).pipe(
+            Effect.catchTag("OptimisticConcurrencyException", Effect.die)
+          )
+        }
+        yield* exec(
+          `INSERT INTO "_migrations" (id, version) VALUES ($1, $2)`,
+          [`${tableName}::${ns}`, tableName]
+        )
+      })
+      const seedNamespace = (ns: string) => {
+        let cached = seedCache.get(ns)
+        if (!cached) {
+          cached = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync)
+          seedCache.set(ns, cached)
+        }
+        return cached
+      }
+      const selectAllSql = `SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = $1`
+      const allStored = resolveNamespace.pipe(
+        Effect.flatMap((ns) =>
+          exec(selectAllSql, [ns]).pipe(
+            Effect.map((rows) => (rows as any[]).map((row) => parseRow<Encoded>(row, idKey, defaultValues)))
+          )
+        )
+      )
+
+      const s: Store<IdKey, Encoded> = {
+        seedNamespace: (ns) => seedNamespace(ns),
+
+        all: resolveNamespace.pipe(
+          Effect.flatMap((ns) => {
+            return exec(selectAllSql, [ns])
+              .pipe(
+                Effect.flatMap((rows) => Effect.fromResult(decodeStoredMany(rows as any[], decodeRow))),
+                annotateDb({
+                  operation: "all",
+                  system: "postgresql",
+                  collection: tableName,
+                  namespace: ns,
+                  entity: name,
+                  query: selectAllSql
+                })
+              )
+          })
+        ),
+
+        find: (id) =>
+          resolveNamespace.pipe(Effect
+            .flatMap((ns) => {
+              const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE id = $1 AND _namespace = $2`
+              return exec(sqlText, [id, ns])
+                .pipe(
+                  Effect.flatMap((rows) =>
+                    Effect.fromResult(decodeStoredOption(Option.fromNullishOr((rows as any[])[0]), decodeRow))
+                  ),
+                  annotateDb({
+                    operation: "find",
+                    system: "postgresql",
+                    collection: tableName,
+                    namespace: ns,
+                    entity: name,
+                    query: sqlText,
+                    extra: { "app.entity.id": id }
+                  })
+                )
+            })),
+
+        filter: <U extends keyof Encoded = never>(f: FilterArgs<Encoded, U>) => {
+          const filter = f
+            .filter
+          type M = U extends undefined ? Encoded : Pick<Encoded, U>
+          return resolveNamespace.pipe(Effect.flatMap((ns) =>
+            Effect
+              .sync(() => {
+                const q = buildWhereSQLQuery(
+                  pgDialect,
+                  idKey,
+                  filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [],
+                  tableName,
+                  defaultValues,
+                  f.select as
+                    | NonEmptyReadonlyArray<
+                      string | {
+                        key: string
+                        subKeys: readonly string[]
+                      } | {
+                        key: string
+                        computed: ComputedProjectionIrExpression
+                      }
+                    >
+                    | undefined,
+                  f.order,
+                  f.skip,
+                  f.limit,
+                  ns,
+                  json
+                )
+                const nsPlaceholder = pgDialect.placeholder(q.params.length + 1)
+                const hasWhere = q.sql.includes("WHERE")
+                const nsSql = hasWhere
+                  ? q.sql.replace("WHERE", `WHERE _namespace = ${nsPlaceholder} AND`)
+                  : q.sql.replace(
+                    `FROM "${tableName}"`,
+                    `FROM "${tableName}" WHERE _namespace = ${nsPlaceholder}`
+                  )
+                return { sql: nsSql, params: [...q.params, ns] }
+              })
+              .pipe(
+                Effect.tap((q) => logQuery(q)),
+                Effect.tap((q) => Effect.annotateCurrentSpan({ "db.query.text": q.sql })),
+                Effect.flatMap((q) =>
+                  exec(q.sql, q.params).pipe(
+                    Effect.flatMap((rows) => {
+                      if (f.select) {
+                        return Effect.succeed((rows as any[]).map((r) => {
+                          const selected = parseSelectRow(r, idKey, {})
+                          return {
+                            ...Struct.pick(
+                              defaultValues as any,
+                              f.select!.filter((_) => typeof _ === "string") as never[]
+                            ),
+                            ...selected
+                          } as M
+                        }))
+                      }
+                      return Effect.map(
+                        Effect.fromResult(decodeStoredMany(rows as any[], decodeRow)),
+                        (decoded) => decoded as any as M[]
+                      )
+                    })
+                  )
+                ),
+                annotateDb({
+                  operation: "filter",
+                  system: "postgresql",
+                  collection: tableName,
+                  namespace: ns,
+                  entity: name
+                })
+              )
+          ))
+        },
+
+        set: (e) =>
+          resolveNamespace.pipe(Effect.flatMap((ns) =>
+            setInternal(e, ns).pipe(
+              annotateDb({
+                operation: "set",
+                system: "postgresql",
+                collection: tableName,
+                namespace: ns,
+                entity: name,
+                extra: { "app.entity.id": e[idKey] }
+              })
+            )
+          )),
+
+        batchSet: (items) =>
+          resolveNamespace.pipe(Effect.flatMap((ns) =>
+            bulkSetInternal(items, ns).pipe(
+              annotateDb({
+                operation: "batchSet",
+                system: "postgresql",
+                collection: tableName,
+                namespace: ns,
+                entity: name
+              })
+            )
+          )),
+
+        bulkSet: (items) =>
+          resolveNamespace.pipe(Effect.flatMap((ns) =>
+            bulkSetInternal(items, ns).pipe(
+              annotateDb({
+                operation: "bulkSet",
+                system: "postgresql",
+                collection: tableName,
+                namespace: ns,
+                entity: name
+              })
+            )
+          )),
+
+        batchRemove: (ids) => {
+          const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ")
+          const nsPlaceholder = `$${ids.length + 1}`
+          return resolveNamespace.pipe(Effect.flatMap((ns) => {
+            const sqlText = `DELETE FROM "${tableName}" WHERE id IN (${placeholders}) AND _namespace = ${nsPlaceholder}`
+            return exec(sqlText, [...ids, ns])
+              .pipe(
+                Effect.asVoid,
+                annotateDb({
+                  operation: "batchRemove",
+                  system: "postgresql",
+                  collection: tableName,
+                  namespace: ns,
+                  entity: name,
+                  query: sqlText
+                })
+              )
+          }))
+        },
+
+        queryRaw: (query) =>
+          allStored.pipe(
+            Effect.map(query.memory),
+            annotateDb({
+              operation: "queryRaw",
+              system: "postgresql",
+              collection: tableName,
+              entity: name
+            })
+          )
+      }
+
+      // Eagerly seed primary namespace on initialization. A seed failure at
+      // construction is fatal (orDie); the per-call `seedNamespace` still
+      // surfaces DatabaseError to callers.
+      yield* seedNamespace("primary").pipe(Effect.orDie)
+
+      return s
+    })
+  }
+})
+
+export function PgStoreLayer(cfg: StorageConfig) {
+  return StoreMaker
+    .toLayer(makePgStore(cfg))
+}