@naturalcycles/db-lib 8.60.1 → 9.1.0

package/src/adapter/file/file.db.ts

@@ -1,24 +1,25 @@
 import {
   generateJsonSchemaFromData,
-  pMap,
-  StringMap,
   _by,
   _deepEquals,
   _since,
   _sortBy,
   _sortObjectDeep,
   _stringMapValues,
-  _uniq,
   JsonSchemaRootObject,
   _filterUndefinedValues,
   ObjectWithId,
   _assert,
-  _deepCopy,
-  _stringMapEntries,
   Saved,
 } from '@naturalcycles/js-lib'
 import { readableCreate, ReadableTyped, dimGrey } from '@naturalcycles/nodejs-lib'
-import { BaseCommonDB, DBSaveBatchOperation, queryInMemory } from '../..'
+import {
+  BaseCommonDB,
+  commonDBFullSupport,
+  CommonDBSupport,
+  DBSaveBatchOperation,
+  queryInMemory,
+} from '../..'
 import { CommonDB } from '../../common.db'
 import {
   CommonDBOptions,
@@ -27,7 +28,6 @@ import {
   RunQueryResult,
 } from '../../db.model'
 import { DBQuery } from '../../query/dbQuery'
-import { DBTransaction } from '../../transaction/dbTransaction'
 import { FileDBCfg } from './file.db.model'
 
 /**
@@ -41,6 +41,16 @@ import { FileDBCfg } from './file.db.model'
  * Each save operation saves *whole* file to the persistence layer.
  */
 export class FileDB extends BaseCommonDB implements CommonDB {
+  override support: CommonDBSupport = {
+    ...commonDBFullSupport,
+    bufferValues: false, // todo: implement
+    insertSaveMethod: false,
+    updateSaveMethod: false,
+    updateByQuery: false,
+    createTable: false,
+    transactions: false, // todo
+  }
+
   constructor(cfg: FileDBCfg) {
     super()
     this.cfg = {
@@ -101,72 +111,6 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  /**
-   * Implementation is optimized for loading/saving _whole files_.
-   */
-  override async commitTransaction(tx: DBTransaction, _opt?: CommonDBOptions): Promise<void> {
-    // data[table][id] => row
-    const data: StringMap<StringMap<ObjectWithId>> = {}
-
-    // 1. Load all tables data (concurrently)
-    const tables = _uniq(tx.ops.map(o => o.table))
-
-    await pMap(
-      tables,
-      async table => {
-        const rows = await this.loadFile(table)
-        data[table] = _by(rows, r => r.id)
-      },
-      { concurrency: 16 },
-    )
-
-    const backup = _deepCopy(data)
-
-    // 2. Apply ops one by one (in order)
-    tx.ops.forEach(op => {
-      if (op.type === 'deleteByIds') {
-        op.ids.forEach(id => delete data[op.table]![id])
-      } else if (op.type === 'saveBatch') {
-        op.rows.forEach(r => {
-          if (!r.id) {
-            throw new Error('FileDB: row has an empty id')
-          }
-          data[op.table]![r.id] = r
-        })
-      } else {
-        throw new Error(`DBOperation not supported: ${(op as any).type}`)
-      }
-    })
-
-    // 3. Sort, turn it into ops
-    // Not filtering empty arrays, cause it's already filtered in this.saveFiles()
-    const ops: DBSaveBatchOperation[] = _stringMapEntries(data).map(([table, map]) => {
-      return {
-        type: 'saveBatch',
-        table,
-        rows: this.sortRows(_stringMapValues(map)),
-      }
-    })
-
-    // 4. Save all files
-    try {
-      await this.saveFiles(ops)
-    } catch (err) {
-      const ops: DBSaveBatchOperation[] = _stringMapEntries(backup).map(([table, map]) => {
-        return {
-          type: 'saveBatch',
-          table,
-          rows: this.sortRows(_stringMapValues(map)),
-        }
-      })
-
-      // Rollback, ignore rollback error (if any)
-      await this.saveFiles(ops).catch(_ => {})
-
-      throw err
-    }
-  }
-
   override async runQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     _opt?: CommonDBOptions,
@@ -216,6 +160,27 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     return deleted
   }
 
+  override async deleteByIds(
+    table: string,
+    ids: string[],
+    _opt?: CommonDBOptions,
+  ): Promise<number> {
+    const byId = _by(await this.loadFile(table), r => r.id)
+
+    let deleted = 0
+    ids.forEach(id => {
+      if (!byId[id]) return
+      delete byId[id]
+      deleted++
+    })
+
+    if (deleted > 0) {
+      await this.saveFile(table, _stringMapValues(byId))
+    }
+
+    return deleted
+  }
+
   override async getTableSchema<ROW extends ObjectWithId>(
     table: string,
   ): Promise<JsonSchemaRootObject<ROW>> {
@@ -256,7 +221,11 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     this.logFinished(started, op)
   }
 
-  private sortRows<ROW extends ObjectWithId>(rows: ROW[]): ROW[] {
+  // override async createTransaction(): Promise<FileDBTransaction> {
+  //   return new FileDBTransaction(this)
+  // }
+
+  sortRows<ROW extends ObjectWithId>(rows: ROW[]): ROW[] {
     rows = rows.map(r => _filterUndefinedValues(r))
 
     if (this.cfg.sortOnSave) {
@@ -283,3 +252,80 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     this.cfg.logger?.log(`<< ${op} ${dimGrey(`in ${_since(started)}`)}`)
   }
 }
+
+// todo: get back and fix it
+// Implementation is optimized for loading/saving _whole files_.
+/*
+export class FileDBTransaction implements DBTransaction {
+  constructor(private db: FileDB) {}
+
+  ops: DBOperation[] = []
+
+  async commit(): Promise<void> {
+    // data[table][id] => row
+    const data: StringMap<StringMap<ObjectWithId>> = {}
+
+    // 1. Load all tables data (concurrently)
+    const tables = _uniq(this.ops.map(o => o.table))
+
+    await pMap(
+      tables,
+      async table => {
+        const rows = await this.db.loadFile(table)
+        data[table] = _by(rows, r => r.id)
+      },
+      { concurrency: 16 },
+    )
+
+    const backup = _deepCopy(data)
+
+    // 2. Apply ops one by one (in order)
+    this.ops.forEach(op => {
+      if (op.type === 'deleteByIds') {
+        op.ids.forEach(id => delete data[op.table]![id])
+      } else if (op.type === 'saveBatch') {
+        op.rows.forEach(r => {
+          if (!r.id) {
+            throw new Error('FileDB: row has an empty id')
+          }
+          data[op.table]![r.id] = r
+        })
+      } else {
+        throw new Error(`DBOperation not supported: ${(op as any).type}`)
+      }
+    })
+
+    // 3. Sort, turn it into ops
+    // Not filtering empty arrays, cause it's already filtered in this.saveFiles()
+    const ops: DBSaveBatchOperation[] = _stringMapEntries(data).map(([table, map]) => {
+      return {
+        type: 'saveBatch',
+        table,
+        rows: this.db.sortRows(_stringMapValues(map)),
+      }
+    })
+
+    // 4. Save all files
+    try {
+      await this.db.saveFiles(ops)
+    } catch (err) {
+      const ops: DBSaveBatchOperation[] = _stringMapEntries(backup).map(([table, map]) => {
+        return {
+          type: 'saveBatch',
+          table,
+          rows: this.db.sortRows(_stringMapValues(map)),
+        }
+      })
+
+      // Rollback, ignore rollback error (if any)
+      await this.db.saveFiles(ops).catch(_ => {})
+
+      throw err
+    }
+  }
+
+  async rollback(): Promise<void> {
+    this.ops = []
+  }
+}
+*/

package/src/adapter/inmemory/inMemory.db.ts

@@ -29,11 +29,21 @@ import {
   yellow,
   fs2,
 } from '@naturalcycles/nodejs-lib'
-import { CommonDB, DBIncrement, DBPatch, DBTransaction, queryInMemory } from '../..'
+import {
+  CommonDB,
+  commonDBFullSupport,
+  CommonDBType,
+  DBIncrement,
+  DBOperation,
+  DBPatch,
+  DBTransactionFn,
+  queryInMemory,
+} from '../..'
 import {
   CommonDBCreateOptions,
   CommonDBOptions,
   CommonDBSaveOptions,
+  DBTransaction,
   RunQueryResult,
 } from '../../db.model'
 import { DBQuery } from '../../query/dbQuery'
@@ -76,6 +86,12 @@ export interface InMemoryDBCfg {
 }
 
 export class InMemoryDB implements CommonDB {
+  dbType = CommonDBType.document
+
+  support = {
+    ...commonDBFullSupport,
+  }
+
   constructor(cfg?: Partial<InMemoryDBCfg>) {
     this.cfg = {
       // defaults
@@ -192,13 +208,22 @@ export class InMemoryDB implements CommonDB {
     _opt?: CommonDBOptions,
   ): Promise<number> {
     const table = this.cfg.tablesPrefix + q.table
-    this.data[table] ||= {}
+    if (!this.data[table]) return 0
+    const ids = queryInMemory(q, Object.values(this.data[table]!) as ROW[]).map(r => r.id)
+    return await this.deleteByIds(q.table, ids)
+  }
+
+  async deleteByIds(_table: string, ids: string[], _opt?: CommonDBOptions): Promise<number> {
+    const table = this.cfg.tablesPrefix + _table
+    if (!this.data[table]) return 0
+
     let count = 0
-    queryInMemory(q, Object.values(this.data[table] || {}) as ROW[]).forEach(r => {
-      if (!this.data[table]![r.id]) return
-      delete this.data[table]![r.id]
+    ids.forEach(id => {
+      if (!this.data[table]![id]) return
+      delete this.data[table]![id]
       count++
     })
+
     return count
   }
 
@@ -248,27 +273,13 @@ export class InMemoryDB implements CommonDB {
     return Readable.from(queryInMemory(q, Object.values(this.data[table] || {}) as ROW[]))
   }
 
-  async commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void> {
-    const backup = _deepCopy(this.data)
-
+  async runInTransaction(fn: DBTransactionFn): Promise<void> {
+    const tx = new InMemoryDBTransaction(this)
     try {
-      for await (const op of tx.ops) {
-        if (op.type === 'saveBatch') {
-          await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt })
-        } else if (op.type === 'deleteByIds') {
-          await this.deleteByQuery(DBQuery.create(op.table).filter('id', 'in', op.ids), {
-            ...op.opt,
-            ...opt,
-          })
-        } else {
-          throw new Error(`DBOperation not supported: ${(op as any).type}`)
-        }
-      }
+      await fn(tx)
+      await tx.commit()
     } catch (err) {
-      // rollback
-      this.data = backup
-      this.cfg.logger!.log('InMemoryDB transaction rolled back')
-
+      await tx.rollback()
       throw err
     }
   }
@@ -348,3 +359,69 @@ export class InMemoryDB implements CommonDB {
     )
   }
 }
+
+export class InMemoryDBTransaction implements DBTransaction {
+  constructor(private db: InMemoryDB) {}
+
+  ops: DBOperation[] = []
+
+  async getByIds<ROW extends ObjectWithId>(
+    table: string,
+    ids: string[],
+    opt?: CommonDBOptions,
+  ): Promise<ROW[]> {
+    return await this.db.getByIds(table, ids, opt)
+  }
+
+  async saveBatch<ROW extends Partial<ObjectWithId>>(
+    table: string,
+    rows: ROW[],
+    opt?: CommonDBSaveOptions<ROW>,
+  ): Promise<void> {
+    this.ops.push({
+      type: 'saveBatch',
+      table,
+      rows,
+      opt,
+    })
+  }
+
+  async deleteByIds(table: string, ids: string[], opt?: CommonDBOptions): Promise<number> {
+    this.ops.push({
+      type: 'deleteByIds',
+      table,
+      ids,
+      opt,
+    })
+    return ids.length
+  }
+
+  async commit(): Promise<void> {
+    const backup = _deepCopy(this.db.data)
+
+    try {
+      for (const op of this.ops) {
+        if (op.type === 'saveBatch') {
+          await this.db.saveBatch(op.table, op.rows, op.opt)
+        } else if (op.type === 'deleteByIds') {
+          await this.db.deleteByIds(op.table, op.ids, op.opt)
+        } else {
+          throw new Error(`DBOperation not supported: ${(op as any).type}`)
+        }
+      }
+
+      this.ops = []
+    } catch (err) {
+      // rollback
+      this.ops = []
+      this.db.data = backup
+      this.db.cfg.logger!.log('InMemoryDB transaction rolled back')
+
+      throw err
+    }
+  }
+
+  async rollback(): Promise<void> {
+    this.ops = []
+  }
+}

package/src/base.common.db.ts

@@ -1,9 +1,15 @@
 import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { CommonDB } from './common.db'
-import { CommonDBOptions, CommonDBSaveOptions, DBPatch, RunQueryResult } from './db.model'
+import { CommonDB, CommonDBSupport, CommonDBType } from './common.db'
+import {
+  CommonDBOptions,
+  CommonDBSaveOptions,
+  DBPatch,
+  DBTransactionFn,
+  RunQueryResult,
+} from './db.model'
 import { DBQuery } from './query/dbQuery'
-import { DBTransaction } from './transaction/dbTransaction'
+import { FakeDBTransaction } from './transaction/dbTransaction.util'
 
 /* eslint-disable unused-imports/no-unused-vars */
 
@@ -12,6 +18,10 @@ import { DBTransaction } from './transaction/dbTransaction'
  * To be extended by actual implementations.
  */
 export class BaseCommonDB implements CommonDB {
+  dbType = CommonDBType.document
+
+  support: CommonDBSupport = {}
+
   async ping(): Promise<void> {
     throw new Error('ping is not implemented')
   }
@@ -33,7 +43,7 @@ export class BaseCommonDB implements CommonDB {
     // no-op
   }
 
-  async getByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][]): Promise<ROW[]> {
+  async getByIds<ROW extends ObjectWithId>(table: string, ids: string[]): Promise<ROW[]> {
     throw new Error('getByIds is not implemented')
   }
 
@@ -69,12 +79,13 @@ export class BaseCommonDB implements CommonDB {
     throw new Error('streamQuery is not implemented')
   }
 
-  /**
-   * Naive implementation.
-   * Doesn't support rollback on error, hence doesn't pass dbTest.
-   * To be extended.
-   */
-  async commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void> {
-    throw new Error('commitTransaction is not implemented')
+  async deleteByIds(table: string, ids: string[], opt?: CommonDBOptions): Promise<number> {
+    throw new Error('deleteByIds is not implemented')
+  }
+
+  async runInTransaction(fn: DBTransactionFn): Promise<void> {
+    const tx = new FakeDBTransaction(this)
+    await fn(tx)
+    // there's no try/catch and rollback, as there's nothing to rollback
   }
 }

package/src/common.db.ts

@@ -1,17 +1,49 @@
 import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import {
   CommonDBCreateOptions,
   CommonDBOptions,
   CommonDBSaveOptions,
   CommonDBStreamOptions,
   DBPatch,
+  DBTransactionFn,
   RunQueryResult,
 } from './db.model'
 import { DBQuery } from './query/dbQuery'
-import { DBTransaction } from './transaction/dbTransaction'
+
+export enum CommonDBType {
+  'document' = 'document',
+  'relational' = 'relational',
+}
 
 export interface CommonDB {
+  /**
+   * Relational databases are expected to return `null` for all missing properties.
+   */
+  dbType: CommonDBType
+
+  /**
+   * Manifest of supported features.
+   */
+  support: CommonDBSupport
+
+  // Support flags indicate which of the CommonDB features are supported by this implementation.
+  supportsQueries?: boolean
+  supportsDBQueryFilter?: boolean
+  supportsDBQueryFilterIn?: boolean
+  supportsDBQueryOrder?: boolean
+  supportsDBQuerySelectFields?: boolean
+  supportsInsertSaveMethod?: boolean
+  supportsUpdateSaveMethod?: boolean
+  supportsUpdateByQuery?: boolean
+  supportsDBIncrement?: boolean
+  supportsCreateTable?: boolean
+  supportsTableSchemas?: boolean
+  supportsStreaming?: boolean
+  supportsBufferValues?: boolean
+  supportsNullValues?: boolean
+  supportsTransactions?: boolean
+
   /**
    * Checks that connection/credentials/etc is ok.
    * Also acts as a "warmup request" for a DB.
@@ -84,6 +116,12 @@ export interface CommonDB {
   ) => Promise<void>
 
   // DELETE
+  /**
+   * Returns number of deleted items.
+   * Not supported by all implementations (e.g Datastore will always return same number as number of ids).
+   */
+  deleteByIds: (table: string, ids: string[], opt?: CommonDBOptions) => Promise<number>
+
   /**
    * Returns number of deleted items.
    * Not supported by all implementations (e.g Datastore will always return same number as number of ids).
@@ -121,6 +159,49 @@ export interface CommonDB {
   /**
    * Should be implemented as a Transaction (best effort), which means that
    * either ALL or NONE of the operations should be applied.
+   *
+   * Transaction is automatically committed if fn resolves normally.
+   * Transaction is rolled back if fn throws, the error is re-thrown in that case.
+   * Graceful rollback is allowed on tx.rollback()
    */
-  commitTransaction: (tx: DBTransaction, opt?: CommonDBOptions) => Promise<void>
+  runInTransaction: (fn: DBTransactionFn) => Promise<void>
+}
+
+/**
+ * Manifest of supported features.
+ */
+export interface CommonDBSupport {
+  queries?: boolean
+  dbQueryFilter?: boolean
+  dbQueryFilterIn?: boolean
+  dbQueryOrder?: boolean
+  dbQuerySelectFields?: boolean
+  insertSaveMethod?: boolean
+  updateSaveMethod?: boolean
+  updateByQuery?: boolean
+  dbIncrement?: boolean
+  createTable?: boolean
+  tableSchemas?: boolean
+  streaming?: boolean
+  bufferValues?: boolean
+  nullValues?: boolean
+  transactions?: boolean
+}
+
+export const commonDBFullSupport: CommonDBSupport = {
+  queries: true,
+  dbQueryFilter: true,
+  dbQueryFilterIn: true,
+  dbQueryOrder: true,
+  dbQuerySelectFields: true,
+  insertSaveMethod: true,
+  updateSaveMethod: true,
+  updateByQuery: true,
+  dbIncrement: true,
+  createTable: true,
+  tableSchemas: true,
+  streaming: true,
+  bufferValues: true,
+  nullValues: true,
+  transactions: true,
 }

package/src/commondao/common.dao.model.ts

@@ -274,17 +274,6 @@ export interface CommonDaoOptions extends CommonDBOptions {
    * Useful e.g in AirtableDB where you can have one Dao to control multiple tables.
    */
   table?: string
-
-  /**
-   * If passed - operation will not be performed immediately, but instead "added" to the transaction.
-   * In the end - transaction needs to be committed (by calling `commit`).
-   * This API is inspired by Datastore API.
-   *
-   * Only applicable to save* and delete* operations
-   *
-   * @experimental
-   */
-  tx?: boolean
 }
 
 export interface CommonDaoSaveOptions<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId>