@naturalcycles/db-lib 8.41.1 → 8.42.0

package/dist/transaction/dbTransaction.util.js

@@ -1,76 +1,76 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.commitDBTransactionSimple = exports.mergeDBOperations = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
+exports.commitDBTransactionSimple = void 0;
 /**
  * Optimizes the Transaction (list of DBOperations) to do less operations.
  * E.g if you save id1 first and then delete it - this function will turn it into a no-op (self-eliminate).
+ * UPD: actually, it will only keep delete, but remove previous ops.
  *
  * Currently only takes into account SaveBatch and DeleteByIds ops.
- * Output ops are maximum 2 (per table) - save and delete (where order actually doesn't matter, cause ids there will not overlap).
+ * Output ops are maximum 1 per entity - save or delete.
  */
-function mergeDBOperations(ops) {
-    if (ops.length <= 1)
-        return ops; // nothing to optimize there
-    // This map will be saved in the end. Null would mean "delete"
-    // saveMap[table][id] => row
-    const saveMapByTable = {};
-    // Merge ops using `saveMap`
-    ops.forEach(op => {
-        saveMapByTable[op.table] = saveMapByTable[op.table] || {};
-        if (op.type === 'saveBatch') {
-            op.rows.forEach(r => (saveMapByTable[op.table][r.id] = r));
-        }
-        else if (op.type === 'deleteByIds') {
-            op.ids.forEach(id => (saveMapByTable[op.table][id] = null));
-        }
-        else {
-            throw new Error(`DBOperation not supported: ${op.type}`);
-        }
-    });
-    const resultOps = [];
-    (0, js_lib_1._stringMapEntries)(saveMapByTable).forEach(([table, saveMap]) => {
-        const rowsToSave = [];
-        const idsToDelete = [];
-        (0, js_lib_1._stringMapEntries)(saveMap).forEach(([id, r]) => {
-            if (r === null) {
-                idsToDelete.push(id);
-            }
-            else {
-                rowsToSave.push(r);
-            }
-        });
-        if (rowsToSave.length) {
-            resultOps.push({
-                type: 'saveBatch',
-                table,
-                rows: rowsToSave,
-            });
-        }
-        if (idsToDelete.length) {
-            resultOps.push({
-                type: 'deleteByIds',
-                table,
-                ids: idsToDelete,
-            });
-        }
-    });
-    return resultOps;
+// Commented out as "overly complicated"
+/*
+export function mergeDBOperations(ops: DBOperation[]): DBOperation[] {
+  if (ops.length <= 1) return ops // nothing to optimize there
+
+  // This map will be saved in the end. Null would mean "delete"
+  // saveMap[table][id] => row
+  const data: StringMap<StringMap<ObjectWithId | null>> = {}
+
+  // Merge ops using `saveMap`
+  ops.forEach(op => {
+    data[op.table] ||= {}
+
+    if (op.type === 'saveBatch') {
+      op.rows.forEach(r => (data[op.table]![r.id] = r))
+    } else if (op.type === 'deleteByIds') {
+      op.ids.forEach(id => (data[op.table]![id] = null))
+    } else {
+      throw new Error(`DBOperation not supported: ${(op as any).type}`)
+    }
+  })
+
+  const resultOps: DBOperation[] = []
+
+  _stringMapEntries(data).forEach(([table, map]) => {
+    const saveOp: DBSaveBatchOperation = {
+      type: 'saveBatch',
+      table,
+      rows: _stringMapValues(map).filter(_isTruthy),
+    }
+
+    if (saveOp.rows.length) {
+      resultOps.push(saveOp)
+    }
+
+    const deleteOp: DBDeleteByIdsOperation = {
+      type: 'deleteByIds',
+      table,
+      ids: _stringMapEntries(map).filter(([id, row]) => row === null).map(([id]) => id),
+    }
+
+    if (deleteOp.ids.length) {
+      resultOps.push(deleteOp)
+    }
+  })
+
+  return resultOps
 }
-exports.mergeDBOperations = mergeDBOperations;
+ */
 /**
  * Naive implementation of "Transaction" which just executes all operations one-by-one.
  * Does NOT actually implement a Transaction, cause partial ops application will happen
  * in case of an error in the middle.
  */
 async function commitDBTransactionSimple(db, tx, opt) {
-    const ops = mergeDBOperations(tx.ops);
-    for await (const op of ops) {
+    // const ops = mergeDBOperations(tx.ops)
+    for await (const op of tx.ops) {
         if (op.type === 'saveBatch') {
-            await db.saveBatch(op.table, op.rows, opt);
+            await db.saveBatch(op.table, op.rows, { ...op.opt, ...opt });
         }
         else if (op.type === 'deleteByIds') {
-            await db.deleteByIds(op.table, op.ids, opt);
+            await db.deleteByIds(op.table, op.ids, { ...op.opt, ...opt });
         }
         else {
             throw new Error(`DBOperation not supported: ${op.type}`);

package/package.json

@@ -41,7 +41,7 @@
   "engines": {
     "node": ">=14.15"
   },
-  "version": "8.41.1",
+  "version": "8.42.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/cachedb/cache.db.ts

@@ -7,8 +7,9 @@ import {
 } from '@naturalcycles/js-lib'
 import { BaseCommonDB } from '../../base.common.db'
 import { CommonDB } from '../../common.db'
-import { RunQueryResult } from '../../db.model'
+import { CommonDBOptions, RunQueryResult } from '../../db.model'
 import { DBQuery } from '../../query/dbQuery'
+import { DBTransaction } from '../../transaction/dbTransaction'
 import {
   CacheDBCfg,
   CacheDBCreateOptions,
@@ -292,4 +293,9 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
     return deletedIds
   }
+
+  override async commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void> {
+    await this.cfg.downstreamDB.commitTransaction(tx, opt)
+    await this.cfg.cacheDB.commitTransaction(tx, opt)
+  }
 }

package/src/adapter/file/file.db.model.ts

@@ -6,7 +6,7 @@ export interface FileDBPersistencePlugin {
   ping(): Promise<void>
   getTables(): Promise<string[]>
   loadFile<ROW extends ObjectWithId>(table: string): Promise<ROW[]>
-  saveFiles(ops: DBSaveBatchOperation[]): Promise<void>
+  saveFiles(ops: DBSaveBatchOperation<any>[]): Promise<void>
 }
 
 export interface FileDBCfg {

package/src/adapter/file/file.db.ts

@@ -12,8 +12,9 @@ import {
   JsonSchemaRootObject,
   _filterUndefinedValues,
   ObjectWithId,
-  AnyObjectWithId,
   _assert,
+  _deepCopy,
+  _stringMapEntries,
 } from '@naturalcycles/js-lib'
 import { readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { dimGrey } from '@naturalcycles/nodejs-lib/dist/colors'
@@ -119,12 +120,19 @@ export class FileDB extends BaseCommonDB implements CommonDB {
       { concurrency: 16 },
     )
 
+    const backup = _deepCopy(data)
+
     // 2. Apply ops one by one (in order)
     tx.ops.forEach(op => {
       if (op.type === 'deleteByIds') {
         op.ids.forEach(id => delete data[op.table]![id])
       } else if (op.type === 'saveBatch') {
-        op.rows.forEach(r => (data[op.table]![r.id] = r))
+        op.rows.forEach(r => {
+          if (!r.id) {
+            throw new Error('FileDB: row has an empty id')
+          }
+          data[op.table]![r.id] = r
+        })
       } else {
         throw new Error(`DBOperation not supported: ${(op as any).type}`)
       }
@@ -132,16 +140,31 @@ export class FileDB extends BaseCommonDB implements CommonDB {
 
     // 3. Sort, turn it into ops
     // Not filtering empty arrays, cause it's already filtered in this.saveFiles()
-    const ops: DBSaveBatchOperation[] = Object.keys(data).map(table => {
+    const ops: DBSaveBatchOperation[] = _stringMapEntries(data).map(([table, map]) => {
       return {
         type: 'saveBatch',
         table,
-        rows: this.sortRows(Object.values(data[table]!) as AnyObjectWithId[]),
+        rows: this.sortRows(_stringMapValues(map)),
       }
     })
 
     // 4. Save all files
-    await this.saveFiles(ops)
+    try {
+      await this.saveFiles(ops)
+    } catch (err) {
+      const ops: DBSaveBatchOperation[] = _stringMapEntries(backup).map(([table, map]) => {
+        return {
+          type: 'saveBatch',
+          table,
+          rows: this.sortRows(_stringMapValues(map)),
+        }
+      })
+
+      // Rollback, ignore rollback error (if any)
+      await this.saveFiles(ops).catch(_ => {})
+
+      throw err
+    }
   }
 
   override async runQuery<ROW extends ObjectWithId>(

package/src/adapter/inmemory/inMemory.db.ts

@@ -12,6 +12,7 @@ import {
   ObjectWithId,
   _stringMapValues,
   CommonLogger,
+  _deepCopy,
 } from '@naturalcycles/js-lib'
 import {
   bufferReviver,
@@ -104,13 +105,13 @@ export class InMemoryDB implements CommonDB {
   async resetCache(_table?: string): Promise<void> {
     if (_table) {
       const table = this.cfg.tablesPrefix + _table
-      this.cfg.logger?.log(`reset ${table}`)
+      this.cfg.logger!.log(`reset ${table}`)
       this.data[table] = {}
     } else {
       ;(await this.getTables()).forEach(table => {
         this.data[table] = {}
       })
-      this.cfg.logger?.log('reset')
+      this.cfg.logger!.log('reset')
     }
   }
 
@@ -161,7 +162,7 @@ export class InMemoryDB implements CommonDB {
 
     rows.forEach(r => {
       if (!r.id) {
-        this.cfg.logger?.warn({ rows })
+        this.cfg.logger!.warn({ rows })
         throw new Error(
           `InMemoryDB doesn't support id auto-generation in saveBatch, row without id was given`,
         )
@@ -234,14 +235,24 @@ export class InMemoryDB implements CommonDB {
   }
 
   async commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void> {
-    for await (const op of tx.ops) {
-      if (op.type === 'saveBatch') {
-        await this.saveBatch(op.table, op.rows, opt)
-      } else if (op.type === 'deleteByIds') {
-        await this.deleteByIds(op.table, op.ids, opt)
-      } else {
-        throw new Error(`DBOperation not supported: ${(op as any).type}`)
+    const backup = _deepCopy(this.data)
+
+    try {
+      for await (const op of tx.ops) {
+        if (op.type === 'saveBatch') {
+          await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt })
+        } else if (op.type === 'deleteByIds') {
+          await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt })
+        } else {
+          throw new Error(`DBOperation not supported: ${(op as any).type}`)
+        }
       }
+    } catch (err) {
+      // rollback
+      this.data = backup
+      this.cfg.logger!.log('InMemoryDB transaction rolled back')
+
+      throw err
     }
   }
 
@@ -277,7 +288,7 @@ export class InMemoryDB implements CommonDB {
       ])
     })
 
-    this.cfg.logger?.log(
+    this.cfg.logger!.log(
       `flushToDisk took ${dimGrey(_since(started))} to save ${yellow(tables)} tables`,
     )
   }
@@ -319,7 +330,7 @@ export class InMemoryDB implements CommonDB {
       this.data[table] = _by(rows, r => r.id)
     })
 
-    this.cfg.logger?.log(
+    this.cfg.logger!.log(
       `restoreFromDisk took ${dimGrey(_since(started))} to read ${yellow(files.length)} tables`,
     )
   }

package/src/base.common.db.ts

@@ -67,6 +67,7 @@ export class BaseCommonDB implements CommonDB {
 
   /**
    * Naive implementation.
+   * Doesn't support rollback on error, hence doesn't pass dbTest.
    * To be extended.
    */
   async commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void> {

package/src/commondao/common.dao.model.ts

@@ -211,6 +211,17 @@ export interface CommonDaoOptions extends CommonDBOptions {
    * @experimental
    */
   timeout?: number
+
+  /**
+   * If passed - operation will not be performed immediately, but instead "added" to the transaction.
+   * In the end - transaction needs to be committed (by calling `commit`).
+   * This API is inspired by Datastore API.
+   *
+   * Only applicable to save* and delete* operations
+   *
+   * @experimental
+   */
+  tx?: boolean
 }
 
 /**

package/src/commondao/common.dao.ts

@@ -35,8 +35,15 @@ import {
   writableVoid,
 } from '@naturalcycles/nodejs-lib'
 import { DBLibError } from '../cnst'
-import { DBModelType, RunQueryResult } from '../db.model'
+import {
+  DBDeleteByIdsOperation,
+  DBModelType,
+  DBOperation,
+  DBSaveBatchOperation,
+  RunQueryResult,
+} from '../db.model'
 import { DBQuery, RunnableDBQuery } from '../query/dbQuery'
+import { DBTransaction } from '../transaction/dbTransaction'
 import {
   CommonDaoCfg,
   CommonDaoCreateOptions,
@@ -606,6 +613,57 @@ export class CommonDao<
     return obj as any
   }
 
+  tx = {
+    save: async (
+      bm: Unsaved<BM>,
+      opt: CommonDaoSaveOptions<DBM> = {},
+    ): Promise<DBSaveBatchOperation> => {
+      const row: DBM = (await this.save(bm, { ...opt, tx: true })) as any
+
+      return {
+        type: 'saveBatch',
+        table: this.cfg.table,
+        rows: [row],
+        opt: {
+          excludeFromIndexes: this.cfg.excludeFromIndexes as any,
+          ...opt,
+        },
+      }
+    },
+    saveBatch: async (
+      bms: Unsaved<BM>[],
+      opt: CommonDaoSaveOptions<DBM> = {},
+    ): Promise<DBSaveBatchOperation> => {
+      const rows: DBM[] = (await this.saveBatch(bms, { ...opt, tx: true })) as any
+
+      return {
+        type: 'saveBatch',
+        table: this.cfg.table,
+        rows,
+        opt: {
+          excludeFromIndexes: this.cfg.excludeFromIndexes as any,
+          ...opt,
+        },
+      }
+    },
+    deleteByIds: async (ids: ID[], opt: CommonDaoOptions = {}): Promise<DBDeleteByIdsOperation> => {
+      return {
+        type: 'deleteByIds',
+        table: this.cfg.table,
+        ids: ids as string[],
+        opt,
+      }
+    },
+    deleteById: async (id: ID, opt: CommonDaoOptions = {}): Promise<DBDeleteByIdsOperation> => {
+      return {
+        type: 'deleteByIds',
+        table: this.cfg.table,
+        ids: [id as string],
+        opt,
+      }
+    },
+  }
+
   // SAVE
   /**
    * Mutates with id, created, updated
@@ -615,6 +673,11 @@ export class CommonDao<
     const idWasGenerated = !bm.id && this.cfg.createId
     this.assignIdCreatedUpdated(bm, opt) // mutates
     const dbm = await this.bmToDBM(bm as BM, opt)
+
+    if (opt.tx) {
+      return dbm as any
+    }
+
     const table = opt.table || this.cfg.table
     if (opt.ensureUniqueId && idWasGenerated) await this.ensureUniqueId(table, dbm)
     if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
@@ -709,6 +772,11 @@ export class CommonDao<
     const table = opt.table || this.cfg.table
     bms.forEach(bm => this.assignIdCreatedUpdated(bm, opt))
     const dbms = await this.bmsToDBM(bms as BM[], opt)
+
+    if (opt.tx) {
+      return dbms as any
+    }
+
     if (opt.ensureUniqueId) throw new AppError('ensureUniqueId is not supported in saveBatch')
     if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
       opt = { ...opt, saveMethod: 'insert' }
@@ -724,7 +792,6 @@ export class CommonDao<
     const started = this.logSaveStarted(op, bms, table)
     const { excludeFromIndexes } = this.cfg
     const assignGeneratedIds = opt.assignGeneratedIds || this.cfg.assignGeneratedIds
-
     await this.cfg.db.saveBatch(table, dbms, {
       excludeFromIndexes,
       assignGeneratedIds,
@@ -1066,9 +1133,13 @@ export class CommonDao<
     await this.cfg.db.ping()
   }
 
-  // transaction(): DBTransaction {
-  //   return this.cfg.db.transaction()
-  // }
+  async runInTransaction(ops: Promise<DBOperation>[]): Promise<void> {
+    if (!ops.length) return
+
+    const resolvedOps = await Promise.all(ops)
+
+    await this.cfg.db.commitTransaction(DBTransaction.create(resolvedOps))
+  }
 
   protected logResult(started: number, op: string, res: any, table: string): void {
     if (!this.cfg.logLevel) return

package/src/db.model.ts

@@ -56,12 +56,14 @@ export interface DBSaveBatchOperation<ROW extends ObjectWithId = AnyObjectWithId
   type: 'saveBatch'
   table: string
   rows: ROW[]
+  opt?: CommonDBSaveOptions<ROW>
 }
 
 export interface DBDeleteByIdsOperation {
   type: 'deleteByIds'
   table: string
   ids: string[]
+  opt?: CommonDBOptions
 }
 
 export enum DBRelation {

package/src/index.ts

@@ -48,7 +48,7 @@ import {
   RunnableDBQuery,
 } from './query/dbQuery'
 import { DBTransaction, RunnableDBTransaction } from './transaction/dbTransaction'
-import { commitDBTransactionSimple, mergeDBOperations } from './transaction/dbTransaction.util'
+import { commitDBTransactionSimple } from './transaction/dbTransaction.util'
 export * from './kv/commonKeyValueDaoMemoCache'
 
 export type {
@@ -104,7 +104,6 @@ export {
   BaseCommonDB,
   DBTransaction,
   RunnableDBTransaction,
-  mergeDBOperations,
   commitDBTransactionSimple,
   CommonKeyValueDao,
 }

package/src/testing/daoTest.ts

@@ -1,4 +1,4 @@
-import { pDelay, _deepCopy, _pick, _sortBy } from '@naturalcycles/js-lib'
+import { pDelay, _deepCopy, _pick, _sortBy, _omit, localTime } from '@naturalcycles/js-lib'
 import { readableToArray, transformNoOp } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel } from '..'
 import { CommonDB } from '../common.db'
@@ -41,6 +41,7 @@ export function runCommonDaoTest(
     streaming = true,
     strongConsistency = true,
     nullValues = true,
+    transactions = true,
   } = features
 
   // const {
@@ -264,4 +265,56 @@ export function runCommonDaoTest(
       )
     })
   }
+
+  if (transactions) {
+    test('transaction happy path', async () => {
+      // cleanup
+      await dao.query().deleteByQuery()
+
+      // Test that id, created, updated are created
+      const now = localTime().unix()
+      await dao.runInTransaction([dao.tx.save(_omit(item1, ['id', 'created', 'updated']))])
+
+      const loaded = await dao.query().runQuery()
+      expect(loaded.length).toBe(1)
+      expect(loaded[0]!.id).toBeDefined()
+      expect(loaded[0]!.created).toBeGreaterThanOrEqual(now)
+      expect(loaded[0]!.updated).toBe(loaded[0]!.created)
+
+      await dao.runInTransaction([dao.tx.deleteById(loaded[0]!.id)])
+
+      // saveBatch [item1, 2, 3]
+      // save item3 with k1: k1_mod
+      // delete item2
+      // remaining: item1, item3_with_k1_mod
+      await dao.runInTransaction([
+        dao.tx.saveBatch(items),
+        dao.tx.save({ ...items[2]!, k1: 'k1_mod' }),
+        dao.tx.deleteById(items[1]!.id),
+      ])
+
+      const rows = await dao.query().runQuery()
+      const expected = [items[0], { ...items[2]!, k1: 'k1_mod' }]
+      expectMatch(expected, rows, quirks)
+    })
+
+    test('transaction rollback', async () => {
+      await expect(
+        dao.runInTransaction([
+          dao.tx.deleteById(items[2]!.id),
+          dao.tx.save({ ...items[0]!, k1: 5 as any }), // it should fail here
+        ]),
+      ).rejects.toThrow()
+
+      const rows = await dao.query().runQuery()
+      const expected = [items[0], { ...items[2]!, k1: 'k1_mod' }]
+      expectMatch(expected, rows, quirks)
+    })
+
+    if (querying) {
+      test('transaction cleanup', async () => {
+        await dao.query().deleteByQuery()
+      })
+    }
+  }
 }