@effect-app/infra 0.7.0 → 0.8.0

package/_src/services/simpledb/cosmosdb.ts

@@ -0,0 +1,134 @@
+import type { IndexingPolicy } from "@azure/cosmos"
+import { typedKeysOf } from "@effect-app/core/utils"
+
+import * as Cosmos from "@effect-app/infra-adapters/cosmos-client"
+import type { CachedRecord, DBRecord } from "./shared.js"
+import { OptimisticLockException } from "./shared.js"
+import * as simpledb from "./simpledb.js"
+import type { Version } from "./simpledb.js"
+
+class CosmosDbOperationError {
+  constructor(readonly message: string) {}
+}
+
+const setup = (type: string, indexingPolicy: IndexingPolicy) =>
+  Cosmos.db.tap(db =>
+    Effect.tryPromise(() =>
+      db.containers
+        .create({ id: type, indexingPolicy })
+        .catch(err => console.warn(err))
+    )
+  )
+// TODO: Error if current indexingPolicy does not match
+// Effect.flatMap((db) => Effect.tryPromise(() => db.container(type).(indexes)))
+export function createContext<TKey extends string, EA, A extends DBRecord<TKey>>() {
+  return <REncode, RDecode, EDecode>(
+    type: string,
+    encode: (record: A) => Effect<REncode, never, EA>,
+    decode: (d: EA) => Effect<RDecode, EDecode, A>,
+    // schemaVersion: string,
+    indexes: IndexingPolicy
+  ) => {
+    return setup(type, indexes).map(() => ({
+      find: simpledb.find(find, decode, type),
+      findBy,
+      save: simpledb.storeDirectly(store, type)
+    }))
+
+    function find(id: string) {
+      return Cosmos.db
+        .flatMap(db => Effect.tryPromise(() => db.container(type).item(id).read<{ data: EA }>()))
+        .map(i => Opt.fromNullable(i.resource))
+        .map(
+          _ =>
+            _.map(
+              ({ _etag, data }) => ({ version: _etag, data } as CachedRecord<EA>)
+            )
+        )
+    }
+
+    function findBy(parameters: Record<string, string>) {
+      return Cosmos.db
+        .flatMap(db =>
+          Effect.tryPromise(() =>
+            db
+              .container(type)
+              .items.query({
+                query: `
+SELECT TOP 1 ${type}.id
+FROM ${type} i
+WHERE (
+  ${
+                  typedKeysOf(parameters)
+                    .map(k => `i.${k} = @${k}`)
+                    .join(" and ")
+                }
+)
+`,
+                parameters: typedKeysOf(parameters).map(p => ({
+                  name: `@${p}`,
+                  value: parameters[p]!
+                }))
+              })
+              .fetchAll()
+          )
+        )
+        .map(x => x.resources.head)
+        .map(_ => _.map(_ => _.id))
+    }
+
+    function store(record: A, currentVersion: Opt<Version>) {
+      return Effect.gen(function*($) {
+        const version = "_etag" // we get this from the etag anyway.
+
+        const db = yield* $(Cosmos.db)
+        const data = yield* $(encode(record))
+
+        yield* $(
+          currentVersion.match(
+            () =>
+              Effect.tryPromise(() =>
+                db.container(type).items.create({
+                  id: record.id,
+                  timestamp: new Date(),
+                  data
+                })
+              ).asUnit.orDie,
+            currentVersion =>
+              Effect.tryPromise(() =>
+                db
+                  .container(type)
+                  .item(record.id)
+                  .replace(
+                    {
+                      id: record.id,
+                      timestamp: new Date(),
+                      data
+                    },
+                    {
+                      accessCondition: {
+                        type: "IfMatch",
+                        condition: currentVersion
+                      }
+                    }
+                  )
+              ).orDie.flatMap(x => {
+                if (x.statusCode === 412) {
+                  return Effect.fail(new OptimisticLockException(type, record.id))
+                }
+                if (x.statusCode > 299 || x.statusCode < 200) {
+                  return Effect.die(
+                    new CosmosDbOperationError(
+                      "not able to update record: " + x.statusCode
+                    )
+                  )
+                }
+                return Effect.unit
+              })
+          )
+        )
+        return { version, data: record } as CachedRecord<A>
+      })
+    }
+  }
+}

package/_src/services/simpledb/diskdb.ts

@@ -0,0 +1,158 @@
+import { flow, pipe } from "@effect-app/core/Function"
+import fs from "fs"
+import * as PLF from "proper-lockfile"
+
+import { pretty } from "@effect-app/core/utils"
+import * as fu from "./fileutil.js"
+import type { CachedRecord, DBRecord, Index } from "./shared.js"
+import { ConnectionException, CouldNotAquireDbLockException, getIndexName, getRecordName } from "./shared.js"
+import * as simpledb from "./simpledb.js"
+import type { Version } from "./simpledb.js"
+
+export function createContext<TKey extends string, EA, A extends DBRecord<TKey>>() {
+  return <REncode, RDecode, EDecode>(
+    type: string,
+    encode: (record: A) => Effect<REncode, never, EA>,
+    decode: (d: EA) => Effect<RDecode, EDecode, A>,
+    schemaVersion: string,
+    makeIndexKey: (r: A) => Index,
+    dir = "./data.js"
+  ) => {
+    initialise(dir)
+    const globalLock = "global.lock"
+    const typeLockKey = getIdxName(type, globalLock)
+    if (!fs.existsSync(typeLockKey)) {
+      fs.writeFileSync(typeLockKey, "", "utf-8")
+    }
+
+    return {
+      find: simpledb.find(find(type), decode, type),
+      findByIndex: getIdx,
+      save: simpledb.store(find(type), store, lockRecordOnDisk(type), type)
+    }
+
+    function store(record: A, currentVersion: Opt<Version>) {
+      const version = currentVersion
+        .map(cv => (parseInt(cv) + 1).toString())
+        .getOrElse(() => "1")
+      const getData = flow(
+        encode,
+        _ => _.map(data => pretty({ version, timestamp: new Date(), data }))
+      )
+
+      const idx = makeIndexKey(record)
+      return currentVersion.isSome()
+        ? lockIndex(record)
+          .zipRight(
+            readIndex(idx).flatMap(x =>
+              x[record.id]
+                ? Effect.fail(() => new Error("Combination already exists, abort"))
+                : getData(record)
+                  .flatMap(serialised => fu.writeTextFile(getFilename(type, record.id), serialised))
+                  .zipRight(writeIndex(idx, { ...x, [idx.key]: record.id }))
+            ).orDie
+          )
+          .scoped.map(() => ({ version, data: record } as CachedRecord<A>))
+        : getData(record)
+          .flatMap(serialised => fu.writeTextFile(getFilename(type, record.id), serialised))
+          .map(() => ({ version, data: record } as CachedRecord<A>))
+    }
+
+    function lockIndex(record: A) {
+      const index = makeIndexKey(record)
+      return lockDiskIndex(index)
+    }
+
+    function lockDiskIndex(_: Index) {
+      /*
+    Disk index locks require a file to exist already, hence for now we use a global index lock.
+    */
+      // const lockKey = getIdxKey(index)
+      const lockKey = globalLock
+      return lockIndexOnDisk(type)(lockKey)
+    }
+
+    function lockRecordOnDisk(type: string) {
+      return (id: string) =>
+        lockFile(getFilename(type, id)).mapBoth(
+          err => new CouldNotAquireDbLockException(type, id, err as Error),
+          release => ({ release })
+        ).acquireRelease(
+          l => l.release
+        )
+    }
+
+    function lockIndexOnDisk(type: string) {
+      return (id: string) =>
+        lockFile(getIdxName(type, id)).mapBoth(
+          err => new CouldNotAquireDbLockException(type, id, err as Error),
+          release => ({ release })
+        ).acquireRelease(
+          l => l.release
+        )
+    }
+
+    function readFile(filePath: string) {
+      return fu
+        .readTextFile(filePath)
+        .catchAll(err => Effect.die(new ConnectionException(err as Error)))
+    }
+
+    function find(type: string) {
+      return (id: string) => {
+        return tryRead(getFilename(type, id)).map(
+          _ => _.map(s => JSON.parse(s) as CachedRecord<EA>)
+        )
+      }
+    }
+
+    function getIdx(index: Index) {
+      return readIndex(index).map(idx => Opt.fromNullable(idx[index.key]))
+    }
+
+    function readIndex(index: Index) {
+      return tryRead(getIdxName(type, index.doc)).map(
+        _ =>
+          _.match(
+            () => ({} as Record<string, TKey>),
+            x => JSON.parse(x) as Record<string, TKey>
+          )
+      )
+    }
+
+    function writeIndex(index: Index, content: Record<string, TKey>) {
+      return pipe(JSON.stringify(content), serialised => fu.writeTextFile(getIdxName(type, index.doc), serialised))
+    }
+
+    function tryRead(filePath: string) {
+      return fu
+        .fileExists(filePath)
+        .flatMap(exists => !exists ? Effect(Opt.none) : readFile(filePath).map(Opt.some))
+    }
+
+    function getFilename(type: string, id: string) {
+      return `${dir}/v${schemaVersion}.${getRecordName(type, id)}.json`
+    }
+
+    function getIdxName(type: string, id: string) {
+      return `${dir}/v${schemaVersion}.${getIndexName(type, id)}.json`
+    }
+  }
+}
+
+function lockFile(fileName: string) {
+  return Effect.tryPromise(() => PLF.lock(fileName).then(flow(Effect.tryPromise, _ => _.orDie)))
+}
+
+// TODO: ugh.
+let initialised = false
+export function initialise(dir: string) {
+  if (initialised) {
+    return
+  }
+
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir)
+  }
+  initialised = true
+}

package/_src/services/simpledb/fileutil.ts

@@ -0,0 +1,28 @@
+import fs from "fs"
+import { promisify } from "util"
+
+const readFile = promisify(fs.readFile)
+const writeFile = promisify(fs.writeFile)
+const rename = promisify(fs.rename)
+const exists = promisify(fs.exists)
+// const mkdir = promisify(fs.mkdir)
+// const unlinkFile = promisify(fs.unlink)
+
+/**
+ * Safe write file to .tmp and then rename
+ */
+export function writeTextFile(fileName: string, content: string) {
+  const tmp = fileName + ".tmp"
+  return (
+    Effect.tryPromise(() => writeFile(tmp, content, "utf-8")) >
+      Effect.tryPromise(() => rename(tmp, fileName))
+  ).orDie
+}
+
+export function fileExists(fileName: string) {
+  return Effect.tryPromise(() => exists(fileName)).orDie
+}
+
+export function readTextFile(fileName: string) {
+  return Effect.tryPromise(() => readFile(fileName, "utf-8"))
+}

package/_src/services/simpledb/index.ts

@@ -0,0 +1,6 @@
+export * as disk from "./diskdb.js"
+export * as mem from "./memdb.js"
+export * as mongo from "./mongodb.js"
+export * as redis from "./redisdb.js"
+export * from "./shared.js"
+export * as SDB from "./simpledb.js"

package/_src/services/simpledb/memdb.ts

@@ -0,0 +1,79 @@
+import { flow } from "@effect-app/core/Function"
+import * as MO from "@effect-app/schema"
+import type * as Eq from "@effect-ts/core/Equal"
+
+import type { CachedRecord, DBRecord } from "./shared.js"
+import { getRecordName, makeMap, SerializedDBRecord } from "./shared.js"
+import * as simpledb from "./simpledb.js"
+import type { Version } from "./simpledb.js"
+
+// When we are in-process, we want to share the same Storage
+// Do not try this at home.
+const storage = makeMap<string, string>()
+
+const parseSDB = SerializedDBRecord.Parser >= MO.condemnFail
+
+export function createContext<TKey extends string, EA, A extends DBRecord<TKey>>() {
+  return <REncode, RDecode, EDecode>(
+    type: string,
+    encode: (record: A) => Effect<REncode, never, EA>,
+    decode: (d: EA) => Effect<RDecode, EDecode, A>
+  ) => {
+    return {
+      find: simpledb.find(find, decode, type),
+      findBy,
+      save: simpledb.store(find, store, bogusLock, type)
+    }
+
+    function find(id: string) {
+      return storage
+        .find(getRecordName(type, id))
+        .map(_ => _.map(s => JSON.parse(s) as unknown))
+        .flatMapOpt(parseSDB)
+        .mapOpt(({ data, version }) => ({
+          data: JSON.parse(data) as EA,
+          version
+        }))
+    }
+
+    function findBy<V extends Partial<A>>(keys: V, eq: Eq.Equal<V>) {
+      // Naive implementation, fine for in memory testing purposes.
+      return Effect.gen(function*($) {
+        for (const [, value] of storage) {
+          const sdb_ = JSON.parse(value) as unknown
+          const sdb = yield* $(parseSDB(sdb_))
+          const cr = { data: JSON.parse(sdb.data) as EA, version: sdb.version }
+          const r = yield* $(
+            decode(cr.data).flatMap(d =>
+              eq.equals(keys, d as unknown as V)
+                ? Effect(d)
+                : Effect.fail("not equals")
+            ).exit
+          )
+          if (r.isSuccess()) {
+            return r.value
+          }
+        }
+        return null
+      }).map(Opt.fromNullable)
+    }
+
+    function store(record: A, currentVersion: Opt<Version>) {
+      const version = currentVersion
+        .map(cv => (parseInt(cv) + 1).toString())
+        .getOrElse(() => "1")
+
+      const getData = flow(
+        encode,
+        _ => _.map(JSON.stringify).map(data => JSON.stringify({ version, timestamp: new Date(), data }))
+      )
+      return getData(record)
+        .flatMap(serialised => storage.set(getRecordName(type, record.id), serialised))
+        .map(() => ({ version, data: record } as CachedRecord<A>))
+    }
+  }
+}
+
+function bogusLock() {
+  return Effect.unit.acquireRelease(() => Effect.unit)
+}

package/_src/services/simpledb/mongodb.ts

@@ -0,0 +1,99 @@
+import type { IndexDescription, InsertOneOptions } from "mongodb"
+
+import * as Mongo from "@effect-app/infra-adapters/mongo-client"
+import type { CachedRecord, DBRecord } from "./shared.js"
+import { OptimisticLockException } from "./shared.js"
+import * as simpledb from "./simpledb.js"
+import type { Version } from "./simpledb.js"
+
+// const makeFromIndexKeys = (indexKeys: string[], unique: boolean) => indexKeys.reduce((prev, cur) => {
+//   prev[cur] = 1
+//   return prev
+// }, {} as Record<string, number>)
+
+const setup = (type: string, indexes: IndexDescription[]) =>
+  Mongo.db
+    .tap(db => Effect.tryPromise(() => db.createCollection(type).catch(err => console.warn(err))))
+    .flatMap(db => Effect.tryPromise(() => db.collection(type).createIndexes(indexes)))
+
+export function createContext<TKey extends string, EA, A extends DBRecord<TKey>>() {
+  return <REncode, RDecode, EDecode>(
+    type: string,
+    encode: (record: A) => Effect<REncode, never, EA>,
+    decode: (d: EA) => Effect<RDecode, EDecode, A>,
+    // schemaVersion: string,
+    indexes: IndexDescription[]
+  ) => {
+    return setup(type, indexes).map(() => ({
+      find: simpledb.find(find, decode, type),
+      findBy,
+      save: simpledb.storeDirectly(store, type)
+    }))
+
+    function find(id: string) {
+      return Mongo.db
+        .flatMap(db =>
+          Effect.tryPromise(() =>
+            db
+              .collection(type)
+              .findOne<{ _id: TKey; version: Version; data: EA }>({ _id: id })
+          )
+        )
+        .map(Opt.fromNullable)
+        .mapOpt(({ data, version }) => ({ version, data } as CachedRecord<EA>))
+    }
+
+    function findBy(keys: Record<string, string>) {
+      return Mongo.db
+        .flatMap(db =>
+          Effect.tryPromise(() => db.collection(type).findOne<{ _id: TKey }>(keys, { projection: { _id: 1 } }))
+        )
+        .map(Opt.fromNullable)
+        .mapOpt(({ _id }) => _id)
+    }
+
+    function store(record: A, currentVersion: Opt<Version>) {
+      return Effect.gen(function*($) {
+        const version = currentVersion
+          .map(cv => (parseInt(cv) + 1).toString())
+          .getOrElse(() => "1")
+
+        const db = yield* $(Mongo.db)
+        const data = yield* $(encode(record))
+        yield* $(
+          currentVersion.match(
+            () =>
+              Effect.tryPromise(() =>
+                db
+                  .collection(type)
+                  .insertOne(
+                    { _id: record.id as any, version, timestamp: new Date(), data },
+                    {
+                      checkKeys: false // support for keys with `.` and `$`. NOTE: you can write them, read them, but NOT query for them.
+                    } as InsertOneOptions
+                  )
+              ).asUnit.orDie,
+            currentVersion =>
+              Effect.tryPromise(() =>
+                db.collection(type).replaceOne(
+                  { _id: record.id, version: currentVersion },
+                  {
+                    version,
+                    timestamp: new Date(),
+                    data
+                  },
+                  { upsert: false }
+                )
+              ).orDie.flatMap(x => {
+                if (!x.modifiedCount) {
+                  return Effect.fail(new OptimisticLockException(type, record.id))
+                }
+                return Effect.unit
+              })
+          )
+        )
+        return { version, data: record } as CachedRecord<A>
+      })
+    }
+  }
+}

package/_src/services/simpledb/redisdb.ts

@@ -0,0 +1,182 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { flow, pipe } from "@effect-app/core/Function"
+import * as MO from "@effect-app/schema"
+import type { Lock } from "redlock"
+
+import * as RED from "@effect-app/infra-adapters/redis-client"
+import type { CachedRecord, DBRecord, Index } from "./shared.js"
+import { ConnectionException, CouldNotAquireDbLockException, getIndexName, getRecordName } from "./shared.js"
+import * as simpledb from "./simpledb.js"
+
+const ttl = 10 * 1000
+
+export function createContext<TKey extends string, EA, A extends DBRecord<TKey>>() {
+  return <REncode, RDecode, EDecode>(
+    type: string,
+    encode: (record: A) => Effect<REncode, never, EA>,
+    decode: (d: EA) => Effect<RDecode, EDecode, A>,
+    schemaVersion: string,
+    makeIndexKey: (r: A) => Index
+  ) => {
+    const getData = flow(encode, _ => _.map(JSON.stringify))
+    return {
+      find: simpledb.find(find, decode, type),
+      findByIndex: getIdx,
+      save: simpledb.store(find, store, lockRedisRecord, type)
+    }
+
+    function find(id: string) {
+      return RED.hmgetAll(getKey(id)).flatMapOpt(v =>
+        pipe(
+          RedisSerializedDBRecord.Parser,
+          MO.condemnFail
+        )(v)
+          .map(({ data, version }) => ({
+            data: JSON.parse(data) as EA,
+            version
+          }))
+          .mapError(e => new ConnectionException(new Error(e.toString())))
+      ).orDie
+    }
+    function store(record: A, currentVersion: Opt<string>) {
+      const version = currentVersion
+        .map(cv => (parseInt(cv) + 1).toString())
+        .getOrElse(() => "1")
+      return currentVersion.match(
+        () =>
+          lockIndex(record).zipRight(
+            getIndex(record)
+              .zipRightOpt(
+                Effect.fail(() => new Error("Combination already exists, abort"))
+              )
+              .zipRight(getData(record))
+              // TODO: instead use MULTI & EXEC to make it in one command?
+              .flatMap(data =>
+                hmSetRec(getKey(record.id), {
+                  version,
+                  timestamp: new Date(),
+                  data
+                })
+              )
+              .zipRight(setIndex(record))
+              .orDie.map(() => ({ version, data: record } as CachedRecord<A>))
+          ).scoped,
+        () =>
+          getData(record)
+            .flatMap(data =>
+              hmSetRec(getKey(record.id), {
+                version,
+                timestamp: new Date(),
+                data
+              })
+            )
+            .orDie.map(() => ({ version, data: record } as CachedRecord<A>))
+      )
+    }
+
+    function getIndex(record: A) {
+      const index = makeIndexKey(record)
+      return getIdx(index)
+    }
+
+    function setIndex(record: A) {
+      const index = makeIndexKey(record)
+      return setIdx(index, record)
+    }
+
+    function lockIndex(record: A) {
+      const index = makeIndexKey(record)
+      return lockRedisIdx(index)
+    }
+
+    function getIdx(index: Index) {
+      return RED.hget(getIdxKey(index), index.key).map(_ => _.map(i => i as TKey))
+    }
+
+    function setIdx(index: Index, r: A) {
+      return RED.hset(getIdxKey(index), index.key, r.id)
+    }
+
+    function lockRedisIdx(index: Index) {
+      const lockKey = getIdxLockKey(index)
+      // acquire
+      return RED.lock
+        .flatMap(lock => Effect.tryPromise(() => lock.lock(lockKey, ttl) as unknown as Promise<Lock>))
+        .mapBoth(
+          err => new CouldNotAquireDbLockException(type, lockKey, err as Error),
+          // release
+          lock => ({
+            release: Effect.tryPromise(() => lock.unlock() as unknown as Promise<void>)
+              .orDie
+          })
+        ).acquireRelease(
+          l => l.release
+        )
+    }
+
+    function lockRedisRecord(id: string) {
+      // acquire
+      return RED.lock
+        .flatMap(lock =>
+          Effect.tryPromise(
+            () => lock.lock(getLockKey(id), ttl) as unknown as Promise<Lock>
+          )
+        )
+        .mapBoth(
+          err => new CouldNotAquireDbLockException(type, id, err as Error),
+          // release
+          lock => ({
+            // TODO
+            release: Effect.tryPromise(() => lock.unlock() as unknown as Promise<void>)
+              .orDie
+          })
+        ).acquireRelease(
+          l => l.release
+        )
+    }
+
+    function getKey(id: string) {
+      return `v${schemaVersion}.${getRecordName(type, id)}`
+    }
+
+    function getLockKey(id: string) {
+      return `v${schemaVersion}.locks.${getRecordName(type, id)}`
+    }
+
+    function getIdxKey(index: Index) {
+      return `v${schemaVersion}.${getIndexName(type, index.doc)}`
+    }
+    function getIdxLockKey(index: Index) {
+      return `v${schemaVersion}.locks.${getIndexName(type, index.doc)}_${index.key}`
+    }
+  }
+
+  function hmSetRec(key: string, val: RedisSerializedDBRecord) {
+    const enc = RedisSerializedDBRecord.Encoder(val)
+    return RED.client.flatMap(
+      client =>
+        Effect.async<never, ConnectionException, void>(res => {
+          client.hmset(
+            key,
+            "version",
+            enc.version,
+            "timestamp",
+            enc.timestamp,
+            "data",
+            enc.data,
+            err =>
+              err
+                ? res(Effect.fail(new ConnectionException(err)))
+                : res(Effect(void 0))
+          )
+        }).uninterruptible
+    )
+  }
+}
+
+export class RedisSerializedDBRecord extends MO.Model<RedisSerializedDBRecord>()({
+  version: MO.prop(MO.string),
+  timestamp: MO.prop(MO.date),
+  data: MO.prop(MO.string)
+}) {}