@naturalcycles/db-lib 8.60.0 → 9.0.0

package/src/adapter/file/file.db.ts

@@ -18,7 +18,15 @@ import {
   Saved,
 } from '@naturalcycles/js-lib'
 import { readableCreate, ReadableTyped, dimGrey } from '@naturalcycles/nodejs-lib'
-import { BaseCommonDB, DBSaveBatchOperation, queryInMemory } from '../..'
+import {
+  BaseCommonDB,
+  commonDBFullSupport,
+  CommonDBSupport,
+  DBOperation,
+  DBSaveBatchOperation,
+  DBTransaction,
+  queryInMemory,
+} from '../..'
 import { CommonDB } from '../../common.db'
 import {
   CommonDBOptions,
@@ -27,7 +35,6 @@ import {
   RunQueryResult,
 } from '../../db.model'
 import { DBQuery } from '../../query/dbQuery'
-import { DBTransaction } from '../../transaction/dbTransaction'
 import { FileDBCfg } from './file.db.model'
 
 /**
@@ -41,6 +48,16 @@ import { FileDBCfg } from './file.db.model'
  * Each save operation saves *whole* file to the persistence layer.
  */
 export class FileDB extends BaseCommonDB implements CommonDB {
+  override support: CommonDBSupport = {
+    ...commonDBFullSupport,
+    bufferValues: false, // todo: implement
+    insertSaveMethod: false,
+    updateSaveMethod: false,
+    updateByQuery: false,
+    createTable: false,
+    transactions: false,
+  }
+
   constructor(cfg: FileDBCfg) {
     super()
     this.cfg = {
@@ -101,72 +118,6 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  /**
-   * Implementation is optimized for loading/saving _whole files_.
-   */
-  override async commitTransaction(tx: DBTransaction, _opt?: CommonDBOptions): Promise<void> {
-    // data[table][id] => row
-    const data: StringMap<StringMap<ObjectWithId>> = {}
-
-    // 1. Load all tables data (concurrently)
-    const tables = _uniq(tx.ops.map(o => o.table))
-
-    await pMap(
-      tables,
-      async table => {
-        const rows = await this.loadFile(table)
-        data[table] = _by(rows, r => r.id)
-      },
-      { concurrency: 16 },
-    )
-
-    const backup = _deepCopy(data)
-
-    // 2. Apply ops one by one (in order)
-    tx.ops.forEach(op => {
-      if (op.type === 'deleteByIds') {
-        op.ids.forEach(id => delete data[op.table]![id])
-      } else if (op.type === 'saveBatch') {
-        op.rows.forEach(r => {
-          if (!r.id) {
-            throw new Error('FileDB: row has an empty id')
-          }
-          data[op.table]![r.id] = r
-        })
-      } else {
-        throw new Error(`DBOperation not supported: ${(op as any).type}`)
-      }
-    })
-
-    // 3. Sort, turn it into ops
-    // Not filtering empty arrays, cause it's already filtered in this.saveFiles()
-    const ops: DBSaveBatchOperation[] = _stringMapEntries(data).map(([table, map]) => {
-      return {
-        type: 'saveBatch',
-        table,
-        rows: this.sortRows(_stringMapValues(map)),
-      }
-    })
-
-    // 4. Save all files
-    try {
-      await this.saveFiles(ops)
-    } catch (err) {
-      const ops: DBSaveBatchOperation[] = _stringMapEntries(backup).map(([table, map]) => {
-        return {
-          type: 'saveBatch',
-          table,
-          rows: this.sortRows(_stringMapValues(map)),
-        }
-      })
-
-      // Rollback, ignore rollback error (if any)
-      await this.saveFiles(ops).catch(_ => {})
-
-      throw err
-    }
-  }
-
   override async runQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     _opt?: CommonDBOptions,
@@ -216,6 +167,27 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     return deleted
   }
 
+  override async deleteByIds(
+    table: string,
+    ids: string[],
+    _opt?: CommonDBOptions,
+  ): Promise<number> {
+    const byId = _by(await this.loadFile(table), r => r.id)
+
+    let deleted = 0
+    ids.forEach(id => {
+      if (!byId[id]) return
+      delete byId[id]
+      deleted++
+    })
+
+    if (deleted > 0) {
+      await this.saveFile(table, _stringMapValues(byId))
+    }
+
+    return deleted
+  }
+
   override async getTableSchema<ROW extends ObjectWithId>(
     table: string,
   ): Promise<JsonSchemaRootObject<ROW>> {
@@ -256,7 +228,11 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     this.logFinished(started, op)
   }
 
-  private sortRows<ROW extends ObjectWithId>(rows: ROW[]): ROW[] {
+  override async createTransaction(): Promise<FileDBTransaction> {
+    return new FileDBTransaction(this)
+  }
+
+  sortRows<ROW extends ObjectWithId>(rows: ROW[]): ROW[] {
     rows = rows.map(r => _filterUndefinedValues(r))
 
     if (this.cfg.sortOnSave) {
@@ -283,3 +259,79 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     this.cfg.logger?.log(`<< ${op} ${dimGrey(`in ${_since(started)}`)}`)
   }
 }
+
+export class FileDBTransaction implements DBTransaction {
+  constructor(private db: FileDB) {}
+
+  ops: DBOperation[] = []
+
+  /**
+   * Implementation is optimized for loading/saving _whole files_.
+   */
+  async commit(): Promise<void> {
+    // data[table][id] => row
+    const data: StringMap<StringMap<ObjectWithId>> = {}
+
+    // 1. Load all tables data (concurrently)
+    const tables = _uniq(this.ops.map(o => o.table))
+
+    await pMap(
+      tables,
+      async table => {
+        const rows = await this.db.loadFile(table)
+        data[table] = _by(rows, r => r.id)
+      },
+      { concurrency: 16 },
+    )
+
+    const backup = _deepCopy(data)
+
+    // 2. Apply ops one by one (in order)
+    this.ops.forEach(op => {
+      if (op.type === 'deleteByIds') {
+        op.ids.forEach(id => delete data[op.table]![id])
+      } else if (op.type === 'saveBatch') {
+        op.rows.forEach(r => {
+          if (!r.id) {
+            throw new Error('FileDB: row has an empty id')
+          }
+          data[op.table]![r.id] = r
+        })
+      } else {
+        throw new Error(`DBOperation not supported: ${(op as any).type}`)
+      }
+    })
+
+    // 3. Sort, turn it into ops
+    // Not filtering empty arrays, cause it's already filtered in this.saveFiles()
+    const ops: DBSaveBatchOperation[] = _stringMapEntries(data).map(([table, map]) => {
+      return {
+        type: 'saveBatch',
+        table,
+        rows: this.db.sortRows(_stringMapValues(map)),
+      }
+    })
+
+    // 4. Save all files
+    try {
+      await this.db.saveFiles(ops)
+    } catch (err) {
+      const ops: DBSaveBatchOperation[] = _stringMapEntries(backup).map(([table, map]) => {
+        return {
+          type: 'saveBatch',
+          table,
+          rows: this.db.sortRows(_stringMapValues(map)),
+        }
+      })
+
+      // Rollback, ignore rollback error (if any)
+      await this.db.saveFiles(ops).catch(_ => {})
+
+      throw err
+    }
+  }
+
+  async rollback(): Promise<void> {
+    this.ops = []
+  }
+}

package/src/adapter/file/localFile.persistence.plugin.ts

@@ -9,8 +9,7 @@ import {
   transformToNDJson,
   writablePushToArray,
   _pipeline,
-  _ensureDir,
-  _pathExists,
+  fs2,
 } from '@naturalcycles/nodejs-lib'
 import { DBSaveBatchOperation } from '../../db.model'
 import { FileDBPersistencePlugin } from './file.db.model'
@@ -50,11 +49,11 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   }
 
   async loadFile<ROW extends ObjectWithId>(table: string): Promise<ROW[]> {
-    await _ensureDir(this.cfg.storagePath)
+    await fs2.ensureDirAsync(this.cfg.storagePath)
     const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
 
-    if (!(await _pathExists(filePath))) return []
+    if (!(await fs2.pathExistsAsync(filePath))) return []
 
     const transformUnzip = this.cfg.gzip ? [createUnzip()] : []
 
@@ -76,7 +75,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   }
 
   async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {
-    await _ensureDir(this.cfg.storagePath)
+    await fs2.ensureDirAsync(this.cfg.storagePath)
     const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
     const transformZip = this.cfg.gzip ? [createGzip()] : []

package/src/adapter/inmemory/inMemory.db.ts

@@ -16,6 +16,7 @@ import {
   CommonLogger,
   _deepCopy,
   _assert,
+  _omit,
 } from '@naturalcycles/js-lib'
 import {
   bufferReviver,
@@ -25,16 +26,24 @@ import {
   transformToNDJson,
   writablePushToArray,
   _pipeline,
-  _emptyDir,
-  _ensureDir,
   dimGrey,
   yellow,
+  fs2,
 } from '@naturalcycles/nodejs-lib'
-import { CommonDB, DBIncrement, DBPatch, DBTransaction, queryInMemory } from '../..'
+import {
+  CommonDB,
+  commonDBFullSupport,
+  CommonDBType,
+  DBIncrement,
+  DBOperation,
+  DBPatch,
+  queryInMemory,
+} from '../..'
 import {
   CommonDBCreateOptions,
   CommonDBOptions,
   CommonDBSaveOptions,
+  DBTransaction,
   RunQueryResult,
 } from '../../db.model'
 import { DBQuery } from '../../query/dbQuery'
@@ -77,6 +86,12 @@ export interface InMemoryDBCfg {
 }
 
 export class InMemoryDB implements CommonDB {
+  dbType = CommonDBType.document
+
+  support = {
+    ...commonDBFullSupport,
+  }
+
   constructor(cfg?: Partial<InMemoryDBCfg>) {
     this.cfg = {
       // defaults
@@ -162,6 +177,17 @@ export class InMemoryDB implements CommonDB {
     rows: ROW[],
     opt: CommonDBSaveOptions<ROW> = {},
   ): Promise<void> {
+    const { tx } = opt
+    if (tx) {
+      ;(tx as InMemoryDBTransaction).ops.push({
+        type: 'saveBatch',
+        table: _table,
+        rows,
+        opt: _omit(opt, ['tx']),
+      })
+      return
+    }
+
     const table = this.cfg.tablesPrefix + _table
     this.data[table] ||= {}
 
@@ -190,16 +216,48 @@ export class InMemoryDB implements CommonDB {
 
   async deleteByQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
-    _opt?: CommonDBOptions,
+    opt: CommonDBOptions = {},
   ): Promise<number> {
     const table = this.cfg.tablesPrefix + q.table
-    this.data[table] ||= {}
+    if (!this.data[table]) return 0
+    const ids = queryInMemory(q, Object.values(this.data[table]!) as ROW[]).map(r => r.id)
+
+    const { tx } = opt
+    if (tx) {
+      ;(tx as InMemoryDBTransaction).ops.push({
+        type: 'deleteByIds',
+        table: q.table,
+        ids,
+        opt: _omit(opt, ['tx']),
+      })
+      return ids.length
+    }
+
+    return await this.deleteByIds(q.table, ids)
+  }
+
+  async deleteByIds(_table: string, ids: string[], opt: CommonDBOptions = {}): Promise<number> {
+    const table = this.cfg.tablesPrefix + _table
+    if (!this.data[table]) return 0
+
+    const { tx } = opt
+    if (tx) {
+      ;(tx as InMemoryDBTransaction).ops.push({
+        type: 'deleteByIds',
+        table: _table,
+        ids,
+        opt: _omit(opt, ['tx']),
+      })
+      return ids.length
+    }
+
     let count = 0
-    queryInMemory(q, Object.values(this.data[table] || {}) as ROW[]).forEach(r => {
-      if (!this.data[table]![r.id]) return
-      delete this.data[table]![r.id]
+    ids.forEach(id => {
+      if (!this.data[table]![id]) return
+      delete this.data[table]![id]
       count++
     })
+
     return count
   }
 
@@ -210,6 +268,8 @@ export class InMemoryDB implements CommonDB {
     const patchEntries = Object.entries(patch)
     if (!patchEntries.length) return 0
 
+    // todo: can we support tx here? :thinking:
+
     const table = this.cfg.tablesPrefix + q.table
     const rows = queryInMemory(q, Object.values(this.data[table] || {}) as ROW[])
     rows.forEach((row: any) => {
@@ -249,29 +309,8 @@ export class InMemoryDB implements CommonDB {
     return Readable.from(queryInMemory(q, Object.values(this.data[table] || {}) as ROW[]))
   }
 
-  async commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void> {
-    const backup = _deepCopy(this.data)
-
-    try {
-      for await (const op of tx.ops) {
-        if (op.type === 'saveBatch') {
-          await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt })
-        } else if (op.type === 'deleteByIds') {
-          await this.deleteByQuery(DBQuery.create(op.table).filter('id', 'in', op.ids), {
-            ...op.opt,
-            ...opt,
-          })
-        } else {
-          throw new Error(`DBOperation not supported: ${(op as any).type}`)
-        }
-      }
-    } catch (err) {
-      // rollback
-      this.data = backup
-      this.cfg.logger!.log('InMemoryDB transaction rolled back')
-
-      throw err
-    }
+  async createTransaction(): Promise<DBTransaction> {
+    return new InMemoryDBTransaction(this)
   }
 
   /**
@@ -283,7 +322,7 @@ export class InMemoryDB implements CommonDB {
 
     const started = Date.now()
 
-    await _emptyDir(persistentStoragePath)
+    await fs2.emptyDirAsync(persistentStoragePath)
 
     const transformZip = persistZip ? [createGzip()] : []
     let tables = 0
@@ -318,7 +357,7 @@ export class InMemoryDB implements CommonDB {
 
     const started = Date.now()
 
-    await _ensureDir(persistentStoragePath)
+    await fs2.ensureDirAsync(persistentStoragePath)
 
     this.data = {} // empty it in the beginning!
 
@@ -349,3 +388,37 @@ export class InMemoryDB implements CommonDB {
     )
   }
 }
+
+export class InMemoryDBTransaction implements DBTransaction {
+  constructor(private db: InMemoryDB) {}
+
+  ops: DBOperation[] = []
+
+  async commit(): Promise<void> {
+    const backup = _deepCopy(this.db.data)
+
+    try {
+      for (const op of this.ops) {
+        if (op.type === 'saveBatch') {
+          await this.db.saveBatch(op.table, op.rows, op.opt)
+        } else if (op.type === 'deleteByIds') {
+          await this.db.deleteByIds(op.table, op.ids, op.opt)
+        } else {
+          throw new Error(`DBOperation not supported: ${(op as any).type}`)
+        }
+      }
+
+      this.ops = []
+    } catch (err) {
+      // rollback
+      this.db.data = backup
+      this.db.cfg.logger!.log('InMemoryDB transaction rolled back')
+
+      throw err
+    }
+  }
+
+  async rollback(): Promise<void> {
+    this.ops = []
+  }
+}

package/src/base.common.db.ts

@@ -1,9 +1,15 @@
 import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { CommonDB } from './common.db'
-import { CommonDBOptions, CommonDBSaveOptions, DBPatch, RunQueryResult } from './db.model'
+import { CommonDB, CommonDBSupport, CommonDBType } from './common.db'
+import {
+  CommonDBOptions,
+  CommonDBSaveOptions,
+  DBPatch,
+  DBTransaction,
+  RunQueryResult,
+} from './db.model'
 import { DBQuery } from './query/dbQuery'
-import { DBTransaction } from './transaction/dbTransaction'
+import { FakeDBTransaction } from './transaction/dbTransaction.util'
 
 /* eslint-disable unused-imports/no-unused-vars */
 
@@ -12,6 +18,10 @@ import { DBTransaction } from './transaction/dbTransaction'
  * To be extended by actual implementations.
  */
 export class BaseCommonDB implements CommonDB {
+  dbType = CommonDBType.document
+
+  support: CommonDBSupport = {}
+
   async ping(): Promise<void> {
     throw new Error('ping is not implemented')
   }
@@ -33,7 +43,7 @@ export class BaseCommonDB implements CommonDB {
     // no-op
   }
 
-  async getByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][]): Promise<ROW[]> {
+  async getByIds<ROW extends ObjectWithId>(table: string, ids: string[]): Promise<ROW[]> {
     throw new Error('getByIds is not implemented')
   }
 
@@ -69,12 +79,11 @@ export class BaseCommonDB implements CommonDB {
     throw new Error('streamQuery is not implemented')
   }
 
-  /**
-   * Naive implementation.
-   * Doesn't support rollback on error, hence doesn't pass dbTest.
-   * To be extended.
-   */
-  async commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void> {
-    throw new Error('commitTransaction is not implemented')
+  async deleteByIds(table: string, ids: string[], opt?: CommonDBOptions): Promise<number> {
+    throw new Error('deleteByIds is not implemented')
+  }
+
+  async createTransaction(): Promise<DBTransaction> {
+    return new FakeDBTransaction(this)
   }
 }

package/src/common.db.ts

@@ -6,12 +6,44 @@ import {
   CommonDBSaveOptions,
   CommonDBStreamOptions,
   DBPatch,
+  DBTransaction,
   RunQueryResult,
 } from './db.model'
 import { DBQuery } from './query/dbQuery'
-import { DBTransaction } from './transaction/dbTransaction'
+
+export enum CommonDBType {
+  'document' = 'document',
+  'relational' = 'relational',
+}
 
 export interface CommonDB {
+  /**
+   * Relational databases are expected to return `null` for all missing properties.
+   */
+  dbType: CommonDBType
+
+  /**
+   * Manifest of supported features.
+   */
+  support: CommonDBSupport
+
+  // Support flags indicate which of the CommonDB features are supported by this implementation.
+  supportsQueries?: boolean
+  supportsDBQueryFilter?: boolean
+  supportsDBQueryFilterIn?: boolean
+  supportsDBQueryOrder?: boolean
+  supportsDBQuerySelectFields?: boolean
+  supportsInsertSaveMethod?: boolean
+  supportsUpdateSaveMethod?: boolean
+  supportsUpdateByQuery?: boolean
+  supportsDBIncrement?: boolean
+  supportsCreateTable?: boolean
+  supportsTableSchemas?: boolean
+  supportsStreaming?: boolean
+  supportsBufferValues?: boolean
+  supportsNullValues?: boolean
+  supportsTransactions?: boolean
+
   /**
    * Checks that connection/credentials/etc is ok.
    * Also acts as a "warmup request" for a DB.
@@ -50,7 +82,7 @@ export interface CommonDB {
    */
   getByIds: <ROW extends ObjectWithId>(
     table: string,
-    ids: ROW['id'][],
+    ids: string[],
     opt?: CommonDBOptions,
   ) => Promise<ROW[]>
 
@@ -84,6 +116,12 @@ export interface CommonDB {
   ) => Promise<void>
 
   // DELETE
+  /**
+   * Returns number of deleted items.
+   * Not supported by all implementations (e.g Datastore will always return same number as number of ids).
+   */
+  deleteByIds: (table: string, ids: string[], opt?: CommonDBOptions) => Promise<number>
+
   /**
    * Returns number of deleted items.
    * Not supported by all implementations (e.g Datastore will always return same number as number of ids).
@@ -122,5 +160,44 @@ export interface CommonDB {
    * Should be implemented as a Transaction (best effort), which means that
    * either ALL or NONE of the operations should be applied.
    */
-  commitTransaction: (tx: DBTransaction, opt?: CommonDBOptions) => Promise<void>
+  createTransaction: () => Promise<DBTransaction>
+}
+
+/**
+ * Manifest of supported features.
+ */
+export interface CommonDBSupport {
+  queries?: boolean
+  dbQueryFilter?: boolean
+  dbQueryFilterIn?: boolean
+  dbQueryOrder?: boolean
+  dbQuerySelectFields?: boolean
+  insertSaveMethod?: boolean
+  updateSaveMethod?: boolean
+  updateByQuery?: boolean
+  dbIncrement?: boolean
+  createTable?: boolean
+  tableSchemas?: boolean
+  streaming?: boolean
+  bufferValues?: boolean
+  nullValues?: boolean
+  transactions?: boolean
+}
+
+export const commonDBFullSupport: CommonDBSupport = {
+  queries: true,
+  dbQueryFilter: true,
+  dbQueryFilterIn: true,
+  dbQueryOrder: true,
+  dbQuerySelectFields: true,
+  insertSaveMethod: true,
+  updateSaveMethod: true,
+  updateByQuery: true,
+  dbIncrement: true,
+  createTable: true,
+  tableSchemas: true,
+  streaming: true,
+  bufferValues: true,
+  nullValues: true,
+  transactions: true,
 }

package/src/commondao/common.dao.model.ts

@@ -274,17 +274,6 @@ export interface CommonDaoOptions extends CommonDBOptions {
    * Useful e.g in AirtableDB where you can have one Dao to control multiple tables.
    */
   table?: string
-
-  /**
-   * If passed - operation will not be performed immediately, but instead "added" to the transaction.
-   * In the end - transaction needs to be committed (by calling `commit`).
-   * This API is inspired by Datastore API.
-   *
-   * Only applicable to save* and delete* operations
-   *
-   * @experimental
-   */
-  tx?: boolean
 }
 
 export interface CommonDaoSaveOptions<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId>