@effect-app/infra 4.0.0-beta.81 → 4.0.0-beta.83

package/src/Store/SQL.ts

@@ -0,0 +1,332 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { Effect, type NonEmptyReadonlyArray, Option, Struct } from "effect-app"
+import { toNonEmptyArray } from "effect-app/Array"
+import { SqlClient } from "effect/unstable/sql"
+import { OptimisticConcurrencyException } from "../errors.js"
+import { InfraLogger } from "../logger.js"
+import type { FieldValues } from "../Model/filter/types.js"
+import { storeId } from "./Memory.js"
+import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js"
+import { buildWhereSQLQuery, logQuery, type SQLDialect, sqliteDialect } from "./SQL/query.js"
+import { makeETag } from "./utils.js"
+
+/** @internal */
+export const parseRow = <Encoded extends FieldValues>(
+  row: { id: string; _etag: string | null; data: string },
+  idKey: PropertyKey,
+  defaultValues: Partial<Encoded>
+): PersistenceModelType<Encoded> => {
+  const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object
+  return { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType<Encoded>
+}
+
+const parseSelectRow = <Encoded extends FieldValues>(
+  row: Record<string, unknown>,
+  idKey: PropertyKey,
+  defaultValues: Partial<Encoded>
+): any => {
+  const result: Record<string, unknown> = { ...defaultValues }
+  for (const [key, value] of Object.entries(row)) {
+    if (key === "id") {
+      result[idKey as string] = value
+      result["id"] = value
+    } else if (typeof value === "string") {
+      try {
+        result[key] = JSON.parse(value)
+      } catch {
+        result[key] = value
+      }
+    } else {
+      result[key] = value
+    }
+  }
+  return result
+}
+
+function makeSQLStoreInt(dialect: SQLDialect, jsonColumnType: string) {
+  return ({ prefix }: StorageConfig) =>
+    Effect.gen(function*() {
+      const sql = yield* SqlClient.SqlClient
+      return {
+        make: <IdKey extends keyof Encoded, Encoded extends FieldValues, R = never, E = never>(
+          name: string,
+          idKey: IdKey,
+          seed?: Effect.Effect<Iterable<Encoded>, E, R>,
+          config?: StoreConfig<Encoded>
+        ) =>
+          Effect.gen(function*() {
+            type PM = PersistenceModelType<Encoded>
+            const tableName = `${prefix}${name}`
+            const defaultValues = config?.defaultValues ?? {}
+
+            const resolveNamespace = !config?.allowNamespace
+              ? Effect.succeed("primary")
+              : storeId.asEffect().pipe(Effect.map((namespace) => {
+                if (namespace !== "primary" && !config.allowNamespace!(namespace)) {
+                  throw new Error(`Namespace ${namespace} not allowed!`)
+                }
+                return namespace
+              }))
+
+            yield* sql
+              .unsafe(
+                `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data ${jsonColumnType} NOT NULL, PRIMARY KEY (id, _namespace))`
+              )
+              .pipe(Effect.orDie)
+
+            const toRow = (e: PM) => {
+              const newE = makeETag(e)
+              const id = newE[idKey] as string
+              const { _etag, [idKey]: _id, ...rest } = newE as any
+              const data = JSON.stringify(rest)
+              return { id, _etag: newE._etag!, data, item: newE }
+            }
+
+            const exec = (query: string, params?: readonly unknown[]) =>
+              sql.unsafe(query, params as any).pipe(Effect.orDie)
+
+            const s: Store<IdKey, Encoded> = {
+              all: resolveNamespace.pipe(Effect.flatMap((ns) =>
+                exec(`SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = ?`, [ns])
+                  .pipe(
+                    Effect.map((rows) => (rows as any[]).map((r) => parseRow<Encoded>(r, idKey, defaultValues))),
+                    Effect.withSpan("SQL.all [effect-app/infra/Store]", {
+                      attributes: {
+                        "repository.table_name": tableName,
+                        "repository.model_name": name,
+                        "repository.namespace": ns
+                      }
+                    }, { captureStackTrace: false })
+                  )
+              )),
+
+              find: (id) =>
+                resolveNamespace.pipe(Effect.flatMap((ns) =>
+                  exec(`SELECT id, _etag, data FROM "${tableName}" WHERE id = ? AND _namespace = ?`, [id, ns])
+                    .pipe(
+                      Effect.map((rows) => {
+                        const row = (rows as any[])[0]
+                        return row
+                          ? Option.some(parseRow<Encoded>(row, idKey, defaultValues))
+                          : Option.none()
+                      }),
+                      Effect.withSpan("SQL.find [effect-app/infra/Store]", {
+                        attributes: { "repository.table_name": tableName, "repository.model_name": name, id }
+                      }, { captureStackTrace: false })
+                    )
+                )),
+
+              filter: <U extends keyof Encoded = never>(f: FilterArgs<Encoded, U>) => {
+                const filter = f
+                  .filter
+                type M = U extends undefined ? Encoded
+                  : Pick<Encoded, U>
+                return resolveNamespace
+                  .pipe(Effect
+                    .flatMap((ns) =>
+                      Effect
+                        .sync(() => {
+                          const q = buildWhereSQLQuery(
+                            dialect,
+                            idKey,
+                            filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [],
+                            tableName,
+                            defaultValues,
+                            f
+                              .select as
+                                | NonEmptyReadonlyArray<string | { key: string; subKeys: readonly string[] }>
+                                | undefined,
+                            f
+                              .order as NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }> | undefined,
+                            f
+                              .skip,
+                            f
+                              .limit
+                          )
+                          const nsPlaceholder = dialect
+                            .placeholder(
+                              q
+                                .params
+                                .length + 1
+                            )
+                          const hasWhere = q
+                            .sql
+                            .includes("WHERE")
+                          const nsSql = hasWhere
+                            ? q
+                              .sql
+                              .replace("WHERE", `WHERE _namespace = ${nsPlaceholder} AND`)
+                            : q
+                              .sql
+                              .replace(
+                                `FROM "${tableName}"`,
+                                `FROM "${tableName}" WHERE _namespace = ${nsPlaceholder}`
+                              )
+                          return {
+                            sql: nsSql,
+                            params: [
+                              ...q
+                                .params,
+                              ns
+                            ]
+                          }
+                        })
+                        .pipe(
+                          Effect
+                            .tap((q) =>
+                              logQuery(q)
+                            ),
+                          Effect.flatMap((q) =>
+                            exec(q.sql, q.params).pipe(
+                              Effect.map((rows) => {
+                                if (f.select) {
+                                  return (rows as any[]).map((r) => {
+                                    const selected = parseSelectRow(r, idKey, {})
+                                    return {
+                                      ...Struct.pick(
+                                        defaultValues as any,
+                                        f.select!.filter((_) => typeof _ === "string") as never[]
+                                      ),
+                                      ...selected
+                                    } as M
+                                  })
+                                }
+                                return (rows as any[]).map((r) =>
+                                  parseRow<Encoded>(r, idKey, defaultValues) as any as M
+                                )
+                              })
+                            )
+                          ),
+                          Effect.withSpan("SQL.filter [effect-app/infra/Store]", {
+                            attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                          }, { captureStackTrace: false })
+                        )
+                    ))
+              },
+
+              set: (e) =>
+                resolveNamespace.pipe(Effect.flatMap((ns) =>
+                  Effect
+                    .gen(function*() {
+                      const row = toRow(e)
+                      if (e._etag) {
+                        yield* exec(
+                          `UPDATE "${tableName}" SET _etag = ?, data = ? WHERE id = ? AND _etag = ? AND _namespace = ?`,
+                          [row._etag, row.data, row.id, e._etag, ns]
+                        )
+                        const existing = yield* exec(
+                          `SELECT _etag FROM "${tableName}" WHERE id = ? AND _namespace = ?`,
+                          [row.id, ns]
+                        )
+                        const current = (existing as any[])[0]
+                        if (!current || current._etag !== row._etag) {
+                          if (current) {
+                            return yield* new OptimisticConcurrencyException({
+                              type: name,
+                              id: row.id,
+                              current: current._etag,
+                              found: e._etag,
+                              code: 412
+                            })
+                          }
+                          return yield* new OptimisticConcurrencyException({
+                            type: name,
+                            id: row.id,
+                            current: "",
+                            found: e._etag,
+                            code: 404
+                          })
+                        }
+                      } else {
+                        yield* exec(
+                          `INSERT INTO "${tableName}" (id, _namespace, _etag, data) VALUES (?, ?, ?, ?)`,
+                          [row.id, ns, row._etag, row.data]
+                        )
+                      }
+                      return row.item
+                    })
+                    .pipe(
+                      Effect.withSpan("SQL.set [effect-app/infra/Store]", {
+                        attributes: { "repository.table_name": tableName, "repository.model_name": name, id: e[idKey] }
+                      }, { captureStackTrace: false })
+                    )
+                )),
+
+              batchSet: (items) =>
+                sql
+                  .withTransaction(
+                    Effect.forEach(items, (e) => s.set(e))
+                  )
+                  .pipe(
+                    Effect.orDie,
+                    Effect.map((_) => _ as unknown as NonEmptyReadonlyArray<PM>),
+                    Effect.withSpan("SQL.batchSet [effect-app/infra/Store]", {
+                      attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                    }, { captureStackTrace: false })
+                  ),
+
+              bulkSet: (items) =>
+                sql
+                  .withTransaction(
+                    Effect.forEach(items, (e) => s.set(e))
+                  )
+                  .pipe(
+                    Effect.orDie,
+                    Effect.map((_) => _ as unknown as NonEmptyReadonlyArray<PM>),
+                    Effect.withSpan("SQL.bulkSet [effect-app/infra/Store]", {
+                      attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                    }, { captureStackTrace: false })
+                  ),
+
+              batchRemove: (ids) => {
+                const placeholders = ids.map(() => "?").join(", ")
+                return resolveNamespace.pipe(Effect.flatMap((ns) =>
+                  exec(
+                    `DELETE FROM "${tableName}" WHERE id IN (${placeholders}) AND _namespace = ?`,
+                    [...ids, ns]
+                  )
+                    .pipe(
+                      Effect.asVoid,
+                      Effect.withSpan("SQL.batchRemove [effect-app/infra/Store]", {
+                        attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                      }, { captureStackTrace: false })
+                    )
+                ))
+              },
+
+              queryRaw: (query) =>
+                s.all.pipe(
+                  Effect.map(query.memory),
+                  Effect.withSpan("SQL.queryRaw [effect-app/infra/Store]", {
+                    attributes: { "repository.table_name": tableName, "repository.model_name": name }
+                  }, { captureStackTrace: false })
+                )
+            }
+
+            if (seed) {
+              const existing = yield* exec(
+                `SELECT COUNT(*) as cnt FROM "${tableName}" WHERE _namespace = ?`,
+                ["primary"]
+              )
+              const count = (existing as any[])[0]?.cnt ?? 0
+              if (count === 0) {
+                yield* InfraLogger.logInfo("Seeding data for " + name)
+                const items = yield* seed
+                yield* Effect.flatMapOption(
+                  Effect.succeed(toNonEmptyArray([...items])),
+                  (a) => s.bulkSet(a).pipe(Effect.orDie)
+                )
+              }
+            }
+
+            return s
+          })
+      }
+    })
+}
+
+export function SQLiteStoreLayer(cfg: StorageConfig) {
+  return StoreMaker
+    .toLayer(makeSQLStoreInt(sqliteDialect, "JSON")(cfg))
+}

package/src/Store/index.ts

@@ -5,6 +5,8 @@ import { DiskStoreLayer } from "./Disk.js"
 import { MemoryStoreLive } from "./Memory.js"
 // import { RedisStoreLayer } from "./Redis.js"
 import type { StorageConfig } from "./service.js"
+import { SQLiteStoreLayer } from "./SQL.js"
+import { PgStoreLayer } from "./SQL/Pg.js"
 
 export function StoreMakerLayer(cfg: StorageConfig) {
   return Effect
@@ -19,6 +21,14 @@ export function StoreMakerLayer(cfg: StorageConfig) {
         console.log("Using disk store at " + dir)
         return DiskStoreLayer(cfg, dir)
       }
+      if (storageUrl.startsWith("sqlite://")) {
+        console.log("Using SQLite store")
+        return SQLiteStoreLayer(cfg)
+      }
+      if (storageUrl.startsWith("pg://")) {
+        console.log("Using PostgreSQL store")
+        return PgStoreLayer(cfg)
+      }
       // if (storageUrl.startsWith("redis://")) {
       //   console.log("Using Redis store")
       //   return RedisStoreLayer(cfg)

package/src/Store/service.ts

@@ -10,8 +10,8 @@ import { type RawQuery } from "../Model/query.js"
 export interface StoreConfig<E> {
   partitionValue: (e?: E) => string
   /**
-   * Primarily used for testing, creating namespaces in the database to separate data e.g to run multiple tests in isolation within the same database
-   * currently only supported in disk/memory. CosmosDB is TODO.
+   * Primarily used for testing, creating namespaces in the database to separate data e.g to run multiple tests in isolation within the same database.
+   * Memory/Disk use separate store instances per namespace. CosmosDB uses namespace-prefixed partition keys. SQL uses a `_namespace` column.
    */
   allowNamespace?: (namespace: string) => boolean
   /**

package/src/api/routing/middleware/middleware.ts

@@ -1,7 +1,10 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { Cause, Config, Effect, Layer, Schema } from "effect"
 import { ConfigureInterruptibilityMiddleware, DevMode, DevModeMiddleware, LoggerMiddleware, RequestCacheMiddleware } from "effect-app/middleware"
+import { RpcContextMap, type RpcMiddleware } from "effect-app/rpc"
 import { pretty } from "effect-app/utils"
+import { type Rpc } from "effect/unstable/rpc"
+import { SqlClient } from "effect/unstable/sql"
 import { logError, reportError } from "../../../errorReporter.js"
 import { InfraLogger } from "../../../logger.js"
 import { determineMethod, isCommand } from "../utils.js"
@@ -126,3 +129,43 @@ export const DefaultGenericMiddlewaresLive = Layer.mergeAll(
   LoggerMiddlewareLive,
   DevModeMiddlewareLive
 )
+
+/**
+ * Config entry for `RequestContextMap` that controls per-RPC transaction wrapping.
+ * Defaults to `false` (no transaction). Set `requiresTransaction: true` on a route to enable.
+ *
+ * @example
+ * ```ts
+ * class RequestContextMap extends RpcContextMap.makeMap({
+ *   requiresTransaction: requiresTransactionConfig,
+ *   // ...
+ * }) {}
+ * ```
+ */
+export const requiresTransactionConfig = RpcContextMap.makeCustom()(Schema.Never, false as boolean)
+
+/**
+ * Creates the middleware Effect for SQL transaction wrapping.
+ * Requires `SqlClient` directly (not via serviceOption).
+ * Reads `requiresTransaction` from the RPC config; defaults to `false`.
+ *
+ * @example
+ * ```ts
+ * const SqlTransactionMiddlewareLive = Layer.effect(
+ *   SqlTransactionMiddleware,
+ *   makeSqlTransactionMiddleware(RequestContextMap)
+ * )
+ * ```
+ */
+export const makeSqlTransactionMiddleware = (
+  rcm: { getConfig: (rpc: Rpc.AnyWithProps) => { readonly requiresTransaction?: boolean } }
+) =>
+  Effect.gen(function*() {
+    const sql = yield* SqlClient.SqlClient
+    const mw: RpcMiddleware.RpcMiddlewareV4<never, never, never> = (effect, { rpc }) => {
+      const { requiresTransaction } = rcm.getConfig(rpc)
+      if (requiresTransaction !== true) return effect
+      return sql.withTransaction(effect).pipe(Effect.orDie)
+    }
+    return mw
+  })

package/src/api/setupRequest.ts

@@ -1,9 +1,18 @@
-import { Effect, Layer, Tracer } from "effect-app"
+import { Effect, Layer, Option, Tracer } from "effect-app"
 import { NonEmptyString255 } from "effect-app/Schema"
+import { SqlClient } from "effect/unstable/sql"
 import { LocaleRef, RequestContext, spanAttributes } from "../RequestContext.js"
 import { ContextMapContainer } from "../Store/ContextMapContainer.js"
 import { storeId } from "../Store/Memory.js"
 
+const withSqlTransaction = <R, E, A>(self: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+  Effect.serviceOption(SqlClient.SqlClient).pipe(
+    Effect.flatMap(Option.match({
+      onNone: () => self,
+      onSome: (sql) => sql.withTransaction(self).pipe(Effect.orDie)
+    }))
+  )
+
 export const getRequestContext = Effect
   .all({
     span: Effect.currentSpan.pipe(Effect.orDie),
@@ -43,16 +52,25 @@ const withRequestSpan = (name = "request", options?: Tracer.SpanOptions) => <R,
       )
   )
 
+export interface SetupRequestOptions {
+  readonly withTransaction?: boolean
+}
+
 export const setupRequestContextFromCurrent =
-  (name = "request", options?: Tracer.SpanOptions) => <R, E, A>(self: Effect.Effect<A, E, R>) =>
+  (name = "request", options?: Tracer.SpanOptions & SetupRequestOptions) => <R, E, A>(self: Effect.Effect<A, E, R>) =>
     self
       .pipe(
+        options?.withTransaction === true ? withSqlTransaction : (_) => _,
         withRequestSpan(name, options),
         Effect.provide(ContextMapContainer.layer, { local: true })
       )
 
 // TODO: consider integrating Effect.withParentSpan
-export function setupRequestContext<R, E, A>(self: Effect.Effect<A, E, R>, requestContext: RequestContext) {
+export function setupRequestContext<R, E, A>(
+  self: Effect.Effect<A, E, R>,
+  requestContext: RequestContext,
+  options?: SetupRequestOptions
+) {
   const layer = Layer.mergeAll(
     ContextMapContainer.layer,
     Layer.succeed(LocaleRef, requestContext.locale),
@@ -60,6 +78,7 @@ export function setupRequestContext<R, E, A>(self: Effect.Effect<A, E, R>, reque
   )
   return self
     .pipe(
+      options?.withTransaction === true ? withSqlTransaction : (_) => _,
       withRequestSpan(requestContext.name),
       Effect.provide(layer, { local: true })
     )
@@ -69,7 +88,7 @@ export function setupRequestContextWithCustomSpan<R, E, A>(
   self: Effect.Effect<A, E, R>,
   requestContext: RequestContext,
   name: string,
-  options?: Tracer.SpanOptions
+  options?: Tracer.SpanOptions & SetupRequestOptions
 ) {
   const layer = Layer.mergeAll(
     ContextMapContainer.layer,
@@ -78,6 +97,7 @@ export function setupRequestContextWithCustomSpan<R, E, A>(
   )
   return self
     .pipe(
+      options?.withTransaction === true ? withSqlTransaction : (_) => _,
       withRequestSpan(name, options),
       Effect.provide(layer, { local: true })
     )

package/test/dist/sql-store.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-store.test.d.ts","sourceRoot":"","sources":["../sql-store.test.ts"],"names":[],"mappings":""}