@effect-app/infra 4.0.0-beta.8 → 4.0.0-beta.82

package/src/Store/SQL.ts

@@ -0,0 +1,327 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { Effect, type NonEmptyReadonlyArray, Option, Struct } from "effect-app"
+import { toNonEmptyArray } from "effect-app/Array"
+import { SqlClient } from "effect/unstable/sql"
+import { OptimisticConcurrencyException } from "../errors.js"
+import { InfraLogger } from "../logger.js"
+import type { FieldValues } from "../Model/filter/types.js"
+import { storeId } from "./Memory.js"
+import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js"
+import { buildWhereSQLQuery, logQuery, type SQLDialect, sqliteDialect } from "./SQL/query.js"
+import { makeETag } from "./utils.js"
+
+const parseRow = <Encoded extends FieldValues>(
+  row: { id: string; _etag: string | null; data: string },
+  defaultValues: Partial<Encoded>
+): PersistenceModelType<Encoded> => {
+  const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object
+  return { ...defaultValues, ...data, _etag: row._etag ?? undefined } as PersistenceModelType<Encoded>
+}
+
+const parseSelectRow = <Encoded extends FieldValues>(
+  row: Record<string, unknown>,
+  idKey: PropertyKey,
+  defaultValues: Partial<Encoded>
+): any => {
+  const result: Record<string, unknown> = { ...defaultValues }
+  for (const [key, value] of Object.entries(row)) {
+    if (key === "id") {
+      result[idKey as string] = value
+      result["id"] = value
+    } else if (typeof value === "string") {
+      try {
+        result[key] = JSON.parse(value)
+      } catch {
+        result[key] = value
+      }
+    } else {
+      result[key] = value
+    }
+  }
+  return result
+}
+
+function makeSQLStoreInt(dialect: SQLDialect, jsonColumnType: string) {
+  return ({ prefix }: StorageConfig) =>
+    Effect.gen(function*() {
+      const sql = yield* SqlClient.SqlClient
+      return {
+        make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R = never, E = never>(
+          name: string,
+          idKey: IdKey,
+          seed?: Effect.Effect<Iterable<Encoded>, E, R>,
+          config?: StoreConfig<Encoded>
+        ) =>
+          Effect.gen(function*() {
+            type PM = PersistenceModelType<Encoded>
+            const tableName = `${prefix}${name}`
+            const defaultValues = config?.defaultValues ?? {}
+
+            const resolveNamespace = !config?.allowNamespace
+              ? Effect.succeed("primary")
+              : storeId.asEffect().pipe(Effect.map((namespace) => {
+                if (namespace !== "primary" && !config.allowNamespace!(namespace)) {
+                  throw new Error(`Namespace ${namespace} not allowed!`)
+                }
+                return namespace
+              }))
+
+            yield* sql
+              .unsafe(
+                `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data ${jsonColumnType} NOT NULL, PRIMARY KEY (id, _namespace))`
+              )
+              .pipe(Effect.orDie)
+
+            const toRow = (e: PM) => {
+              const newE = makeETag(e)
+              const id = newE[idKey] as string
+              const data = JSON.stringify(newE)
+              return { id, _etag: newE._etag!, data, item: newE }
+            }
+
+            const exec = (query: string, params?: readonly unknown[]) =>
+              sql.unsafe(query, params as any).pipe(Effect.orDie)
+
+            const s: Store<IdKey, Encoded> = {
+              all: resolveNamespace.pipe(Effect.flatMap((ns) =>
+                exec(`SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = ?`, [ns])
+                  .pipe(
+                    Effect.map((rows) => (rows as any[]).map((r) => parseRow<Encoded>(r, defaultValues))),
+                    Effect.withSpan("SQL.all [effect-app/infra/Store]", {
+                      attributes: {
+                        "repository.table_name": tableName,
+                        "repository.model_name": name,
+                        "repository.namespace": ns
+                      }
+                    }, { captureStackTrace: false })
+                  )
+              )),
+
+              find: (id) =>
+                resolveNamespace.pipe(Effect.flatMap((ns) =>
+                  exec(`SELECT id, _etag, data FROM "${tableName}" WHERE id = ? AND _namespace = ?`, [id, ns])
+                    .pipe(
+                      Effect.map((rows) => {
+                        const row = (rows as any[])[0]
+                        return row
+                          ? Option.some(parseRow<Encoded>(row, defaultValues))
+                          : Option.none()
+                      }),
+                      Effect.withSpan("SQL.find [effect-app/infra/Store]", {
+                        attributes: { "repository.table_name": tableName, "repository.model_name": name, id }
+                      }, { captureStackTrace: false })
+                    )
+                )),
+
+              filter: <U extends keyof Encoded = never>(f: FilterArgs<Encoded, U>) => {
+                const filter = f
+                  .filter
+                type M = U extends undefined ? Encoded
+                  : Pick<Encoded, U>
+                return resolveNamespace
+                  .pipe(Effect
+                    .flatMap((ns) =>
+                      Effect
+                        .sync(() => {
+                          const q = buildWhereSQLQuery(
+                            dialect,
+                            idKey,
+                            filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [],
+                            tableName,
+                            defaultValues,
+                            f
+                              .select as
+                                | NonEmptyReadonlyArray<string | { key: string; subKeys: readonly string[] }>
+                                | undefined,
+                            f
+                              .order as NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }> | undefined,
+                            f
+                              .skip,
+                            f
+                              .limit
+                          )
+                          const nsPlaceholder = dialect
+                            .placeholder(
+                              q
+                                .params
+                                .length + 1
+                            )
+                          const hasWhere = q
+                            .sql
+                            .includes("WHERE")
+                          const nsSql = hasWhere
+                            ? q
+                              .sql
+                              .replace("WHERE", `WHERE _namespace = ${nsPlaceholder} AND`)
+                            : q
+                              .sql
+                              .replace(
+                                `FROM "${tableName}"`,
+                                `FROM "${tableName}" WHERE _namespace = ${nsPlaceholder}`
+                              )
+                          return {
+                            sql: nsSql,
+                            params: [
+                              ...q
+                                .params,
+                              ns
+                            ]
+                          }
+                        })
+                        .pipe(
+                          Effect
+                            .tap((q) =>
+                              logQuery(q)
+                            ),
+                          Effect.flatMap((q) =>
+                            exec(q.sql, q.params).pipe(
+                              Effect.map((rows) => {
+                                if (f.select) {
+                                  return (rows as any[]).map((r) => {
+                                    const selected = parseSelectRow(r, idKey, {})
+                                    return {
+                                      ...Struct.pick(
+                                        defaultValues as any,
+                                        f.select!.filter((_) => typeof _ === "string") as never[]
+                                      ),
+                                      ...selected
+                                    } as M
+                                  })
+                                }
+                                return (rows as any[]).map((r) => parseRow<Encoded>(r, defaultValues) as any as M)
+                              })
+                            )
+                          ),
+                          Effect.withSpan("SQL.filter [effect-app/infra/Store]", {
+                            attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                          }, { captureStackTrace: false })
+                        )
+                    ))
+              },
+
+              set: (e) =>
+                resolveNamespace.pipe(Effect.flatMap((ns) =>
+                  Effect
+                    .gen(function*() {
+                      const row = toRow(e)
+                      if (e._etag) {
+                        yield* exec(
+                          `UPDATE "${tableName}" SET _etag = ?, data = ? WHERE id = ? AND _etag = ? AND _namespace = ?`,
+                          [row._etag, row.data, row.id, e._etag, ns]
+                        )
+                        const existing = yield* exec(
+                          `SELECT _etag FROM "${tableName}" WHERE id = ? AND _namespace = ?`,
+                          [row.id, ns]
+                        )
+                        const current = (existing as any[])[0]
+                        if (!current || current._etag !== row._etag) {
+                          if (current) {
+                            return yield* new OptimisticConcurrencyException({
+                              type: name,
+                              id: row.id,
+                              current: current._etag,
+                              found: e._etag,
+                              code: 412
+                            })
+                          }
+                          return yield* new OptimisticConcurrencyException({
+                            type: name,
+                            id: row.id,
+                            current: "",
+                            found: e._etag,
+                            code: 404
+                          })
+                        }
+                      } else {
+                        yield* exec(
+                          `INSERT INTO "${tableName}" (id, _namespace, _etag, data) VALUES (?, ?, ?, ?)`,
+                          [row.id, ns, row._etag, row.data]
+                        )
+                      }
+                      return row.item
+                    })
+                    .pipe(
+                      Effect.withSpan("SQL.set [effect-app/infra/Store]", {
+                        attributes: { "repository.table_name": tableName, "repository.model_name": name, id: e[idKey] }
+                      }, { captureStackTrace: false })
+                    )
+                )),
+
+              batchSet: (items) =>
+                sql
+                  .withTransaction(
+                    Effect.forEach(items, (e) => s.set(e))
+                  )
+                  .pipe(
+                    Effect.orDie,
+                    Effect.map((_) => _ as unknown as NonEmptyReadonlyArray<PM>),
+                    Effect.withSpan("SQL.batchSet [effect-app/infra/Store]", {
+                      attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                    }, { captureStackTrace: false })
+                  ),
+
+              bulkSet: (items) =>
+                sql
+                  .withTransaction(
+                    Effect.forEach(items, (e) => s.set(e))
+                  )
+                  .pipe(
+                    Effect.orDie,
+                    Effect.map((_) => _ as unknown as NonEmptyReadonlyArray<PM>),
+                    Effect.withSpan("SQL.bulkSet [effect-app/infra/Store]", {
+                      attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                    }, { captureStackTrace: false })
+                  ),
+
+              batchRemove: (ids) => {
+                const placeholders = ids.map(() => "?").join(", ")
+                return resolveNamespace.pipe(Effect.flatMap((ns) =>
+                  exec(
+                    `DELETE FROM "${tableName}" WHERE id IN (${placeholders}) AND _namespace = ?`,
+                    [...ids, ns]
+                  )
+                    .pipe(
+                      Effect.asVoid,
+                      Effect.withSpan("SQL.batchRemove [effect-app/infra/Store]", {
+                        attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                      }, { captureStackTrace: false })
+                    )
+                ))
+              },
+
+              queryRaw: (query) =>
+                s.all.pipe(
+                  Effect.map(query.memory),
+                  Effect.withSpan("SQL.queryRaw [effect-app/infra/Store]", {
+                    attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                  }, { captureStackTrace: false })
+                )
+            }
+
+            if (seed) {
+              const existing = yield* exec(
+                `SELECT COUNT(*) as cnt FROM "${tableName}" WHERE _namespace = ?`,
+                ["primary"]
+              )
+              const count = (existing as any[])[0]?.cnt ?? 0
+              if (count === 0) {
+                yield* InfraLogger.logInfo("Seeding data for " + name)
+                const items = yield* seed
+                yield* Effect.flatMapOption(
+                  Effect.succeed(toNonEmptyArray([...items])),
+                  (a) => s.bulkSet(a).pipe(Effect.orDie)
+                )
+              }
+            }
+
+            return s
+          })
+      }
+    })
+}
+
+export function SQLiteStoreLayer(cfg: StorageConfig) {
+  return StoreMaker
+    .toLayer(makeSQLStoreInt(sqliteDialect, "JSON")(cfg))
+}

package/src/Store/index.ts

@@ -5,6 +5,8 @@ import { DiskStoreLayer } from "./Disk.js"
 import { MemoryStoreLive } from "./Memory.js"
 // import { RedisStoreLayer } from "./Redis.js"
 import type { StorageConfig } from "./service.js"
+import { SQLiteStoreLayer } from "./SQL.js"
+import { PgStoreLayer } from "./SQL/Pg.js"
 
 export function StoreMakerLayer(cfg: StorageConfig) {
   return Effect
@@ -19,6 +21,14 @@ export function StoreMakerLayer(cfg: StorageConfig) {
         console.log("Using disk store at " + dir)
         return DiskStoreLayer(cfg, dir)
       }
+      if (storageUrl.startsWith("sqlite://")) {
+        console.log("Using SQLite store")
+        return SQLiteStoreLayer(cfg)
+      }
+      if (storageUrl.startsWith("pg://")) {
+        console.log("Using PostgreSQL store")
+        return PgStoreLayer(cfg)
+      }
       // if (storageUrl.startsWith("redis://")) {
       //   console.log("Using Redis store")
       //   return RedisStoreLayer(cfg)

package/src/Store/service.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import type { UniqueKey } from "@azure/cosmos"
-import { Effect, type NonEmptyReadonlyArray, type Option, type Redacted, ServiceMap } from "effect-app"
+import { Context, Effect, type NonEmptyReadonlyArray, type Option, type Redacted } from "effect-app"
 import type { OptimisticConcurrencyException } from "../errors.js"
 import type { FilterResult } from "../Model/filter/filterApi.js"
 import type { FieldValues } from "../Model/filter/types.js"
@@ -10,8 +10,8 @@ import { type RawQuery } from "../Model/query.js"
 export interface StoreConfig<E> {
   partitionValue: (e?: E) => string
   /**
-   * Primarily used for testing, creating namespaces in the database to separate data e.g to run multiple tests in isolation within the same database
-   * currently only supported in disk/memory. CosmosDB is TODO.
+   * Primarily used for testing, creating namespaces in the database to separate data e.g to run multiple tests in isolation within the same database.
+   * Memory/Disk use separate store instances per namespace. CosmosDB uses namespace-prefixed partition keys. SQL uses a `_namespace` column.
    */
   allowNamespace?: (namespace: string) => boolean
   /**
@@ -89,7 +89,7 @@ export interface Store<
   queryRaw: <Out>(query: RawQuery<Encoded, Out>) => Effect.Effect<readonly Out[]>
 }
 
-export class StoreMaker extends ServiceMap.Opaque<StoreMaker, {
+export class StoreMaker extends Context.Opaque<StoreMaker, {
   make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R = never, E = never>(
     name: string,
     idKey: IdKey,
@@ -161,16 +161,25 @@ export const makeContextMap = () => {
   //   }
   // }
 
+  const store = new Map<symbol, unknown>()
+
   return {
     get: getEtag,
-    set: setEtag
-    // parserEnv
+    set: setEtag,
+    getOrCreateStore: <T>(key: symbol, make: () => T): T => {
+      let value = store.get(key) as T | undefined
+      if (value === undefined) {
+        value = make()
+        store.set(key, value)
+      }
+      return value
+    }
   }
 }
 
 const makeMap = Effect.sync(() => makeContextMap())
 
-export class ContextMap extends ServiceMap.Opaque<ContextMap>()("effect-app/ContextMap", { make: makeMap }) {
+export class ContextMap extends Context.Opaque<ContextMap>()("effect-app/ContextMap", { make: makeMap }) {
 }
 
 export type PersistenceModelType<Encoded extends object> = Encoded & {

package/src/adapters/SQL/Model.ts

@@ -7,10 +7,6 @@
 /**
  * @since 1.0.0
  */
-import * as VariantSchema from "effect/unstable/schema/VariantSchema"
-import { SqlClient } from "effect/unstable/sql/SqlClient"
-import * as SqlResolver from "effect/unstable/sql/SqlResolver"
-import * as SqlSchema from "effect/unstable/sql/SqlSchema"
 import crypto from "crypto" // TODO
 import type { Brand } from "effect/Brand"
 import * as DateTime from "effect/DateTime"
@@ -24,6 +20,10 @@ import * as Schema from "effect/Schema"
 import * as Getter from "effect/SchemaGetter"
 import * as Transformation from "effect/SchemaTransformation"
 import type { Scope } from "effect/Scope"
+import * as VariantSchema from "effect/unstable/schema/VariantSchema"
+import { SqlClient } from "effect/unstable/sql/SqlClient"
+import * as SqlResolver from "effect/unstable/sql/SqlResolver"
+import * as SqlSchema from "effect/unstable/sql/SqlSchema"
 
 const {
   Class,
@@ -190,14 +190,13 @@ export const Generated = <S extends Schema.Top>(
  * @since 1.0.0
  * @category generated
  */
-export interface GeneratedByApp<S extends Schema.Top>
-  extends
-    VariantSchema.Field<{
-      readonly select: S
-      readonly insert: S
-      readonly update: S
-      readonly json: S
-    }>
+export interface GeneratedByApp<S extends Schema.Top> extends
+  VariantSchema.Field<{
+    readonly select: S
+    readonly insert: S
+    readonly update: S
+    readonly json: S
+  }>
 {}
 
 /**
@@ -303,8 +302,7 @@ export const FieldOption: <Field extends VariantSchema.Field<any> | Schema.Top>(
 ) => Field extends Schema.Top ? FieldOption<Field>
   : Field extends VariantSchema.Field<infer S> ? VariantSchema.Field<
       {
-        readonly [K in keyof S]: S[K] extends Schema.Top
-          ? K extends VariantsDatabase ? Schema.OptionFromNullOr<S[K]>
+        readonly [K in keyof S]: S[K] extends Schema.Top ? K extends VariantsDatabase ? Schema.OptionFromNullOr<S[K]>
           : optionalOption<S[K]>
           : never
       }
@@ -545,16 +543,15 @@ export const DateTimeUpdateFromNumber: DateTimeUpdateFromNumber = Field({
  * @since 1.0.0
  * @category json
  */
-export interface JsonFromString<S extends Schema.Top>
-  extends
-    VariantSchema.Field<{
-      readonly select: Schema.fromJsonString<S>
-      readonly insert: Schema.fromJsonString<S>
-      readonly update: Schema.fromJsonString<S>
-      readonly json: S
-      readonly jsonCreate: S
-      readonly jsonUpdate: S
-    }>
+export interface JsonFromString<S extends Schema.Top> extends
+  VariantSchema.Field<{
+    readonly select: Schema.fromJsonString<S>
+    readonly insert: Schema.fromJsonString<S>
+    readonly update: Schema.fromJsonString<S>
+    readonly json: S
+    readonly jsonCreate: S
+    readonly jsonUpdate: S
+  }>
 {}
 
 /**
@@ -825,26 +822,28 @@ export const makeDataLoaders = <
     const idColumn = options.idColumn as string
     const setMaxBatchSize = options.maxBatchSize ? RequestResolver.batchN(options.maxBatchSize) : identity
 
-    const insertResolver = SqlResolver.ordered({
-      Request: Model.insert,
-      Result: Model,
-      execute: (request: any) =>
-        sql.onDialectOrElse({
-          mysql: () =>
-            Effect.forEach(request, (request: any) =>
-              sql`insert into ${sql(options.tableName)} ${sql.insert(request)};
+    const insertResolver = SqlResolver
+      .ordered({
+        Request: Model.insert,
+        Result: Model,
+        execute: (request: any) =>
+          sql.onDialectOrElse({
+            mysql: () =>
+              Effect.forEach(request, (request: any) =>
+                sql`insert into ${sql(options.tableName)} ${sql.insert(request)};
 select * from ${sql(options.tableName)} where ${sql(idColumn)} = LAST_INSERT_ID();`
-                .unprepared
-                .pipe(
-                  Effect.map(([, results]) => results![0] as any)
-                ), { concurrency: 10 }),
-          orElse: () => sql`insert into ${sql(options.tableName)} ${sql.insert(request).returning("*")}`
-        })
-    }).pipe(
-      RequestResolver.setDelay(options.window),
-      setMaxBatchSize,
-      RequestResolver.withSpan(`${options.spanPrefix}.insertResolver`)
-    )
+                  .unprepared
+                  .pipe(
+                    Effect.map(([, results]) => results![0] as any)
+                  ), { concurrency: 10 }),
+            orElse: () => sql`insert into ${sql(options.tableName)} ${sql.insert(request).returning("*")}`
+          })
+      })
+      .pipe(
+        RequestResolver.setDelay(options.window),
+        setMaxBatchSize,
+        RequestResolver.withSpan(`${options.spanPrefix}.insertResolver`)
+      )
     const insertExecute = SqlResolver.request(insertResolver)
     const insert = (
       insert: S["insert"]["Type"]
@@ -860,14 +859,16 @@ select * from ${sql(options.tableName)} where ${sql(idColumn)} = LAST_INSERT_ID(
         })
       ) as any
 
-    const insertVoidResolver = SqlResolver.void({
-      Request: Model.insert,
-      execute: (request: any) => sql`insert into ${sql(options.tableName)} ${sql.insert(request)}`
-    }).pipe(
-      RequestResolver.setDelay(options.window),
-      setMaxBatchSize,
-      RequestResolver.withSpan(`${options.spanPrefix}.insertVoidResolver`)
-    )
+    const insertVoidResolver = SqlResolver
+      .void({
+        Request: Model.insert,
+        execute: (request: any) => sql`insert into ${sql(options.tableName)} ${sql.insert(request)}`
+      })
+      .pipe(
+        RequestResolver.setDelay(options.window),
+        setMaxBatchSize,
+        RequestResolver.withSpan(`${options.spanPrefix}.insertVoidResolver`)
+      )
     const insertVoidExecute = SqlResolver.request(insertVoidResolver)
     const insertVoid = (
       insert: S["insert"]["Type"]
@@ -878,18 +879,20 @@ select * from ${sql(options.tableName)} where ${sql(idColumn)} = LAST_INSERT_ID(
         })
       ) as any
 
-    const findByIdResolver = SqlResolver.findById({
-      Id: idSchema,
-      Result: Model,
-      ResultId(request: any) {
-        return request[idColumn]
-      },
-      execute: (ids: any) => sql`select * from ${sql(options.tableName)} where ${sql.in(idColumn, ids)}`
-    }).pipe(
-      RequestResolver.setDelay(options.window),
-      setMaxBatchSize,
-      RequestResolver.withSpan(`${options.spanPrefix}.findByIdResolver`)
-    )
+    const findByIdResolver = SqlResolver
+      .findById({
+        Id: idSchema,
+        Result: Model,
+        ResultId(request: any) {
+          return request[idColumn]
+        },
+        execute: (ids: any) => sql`select * from ${sql(options.tableName)} where ${sql.in(idColumn, ids)}`
+      })
+      .pipe(
+        RequestResolver.setDelay(options.window),
+        setMaxBatchSize,
+        RequestResolver.withSpan(`${options.spanPrefix}.findByIdResolver`)
+      )
     const findByIdExecute = SqlResolver.request(findByIdResolver)
     const findById = (
       id: S["fields"][Id]["Type"]
@@ -904,14 +907,16 @@ select * from ${sql(options.tableName)} where ${sql(idColumn)} = LAST_INSERT_ID(
         })
       ) as any
 
-    const deleteResolver = SqlResolver.void({
-      Request: idSchema,
-      execute: (ids: any) => sql`delete from ${sql(options.tableName)} where ${sql.in(idColumn, ids)}`
-    }).pipe(
-      RequestResolver.setDelay(options.window),
-      setMaxBatchSize,
-      RequestResolver.withSpan(`${options.spanPrefix}.deleteResolver`)
-    )
+    const deleteResolver = SqlResolver
+      .void({
+        Request: idSchema,
+        execute: (ids: any) => sql`delete from ${sql(options.tableName)} where ${sql.in(idColumn, ids)}`
+      })
+      .pipe(
+        RequestResolver.setDelay(options.window),
+        setMaxBatchSize,
+        RequestResolver.withSpan(`${options.spanPrefix}.deleteResolver`)
+      )
     const deleteExecute = SqlResolver.request(deleteResolver)
     const delete_ = (
       id: S["fields"][Id]["Type"]