@naturalcycles/db-lib 8.60.0 → 9.0.0

package/src/commondao/common.dao.ts

@@ -18,7 +18,6 @@ import {
   JsonSchemaRootObject,
   ObjectWithId,
   pMap,
-  Promisable,
   Saved,
   SKIP,
   UnixTimestampMillisNumber,
@@ -43,16 +42,8 @@ import {
   writableVoid,
 } from '@naturalcycles/nodejs-lib'
 import { DBLibError } from '../cnst'
-import {
-  DBDeleteByIdsOperation,
-  DBModelType,
-  DBOperation,
-  DBPatch,
-  DBSaveBatchOperation,
-  RunQueryResult,
-} from '../db.model'
+import { DBModelType, DBPatch, DBTransaction, RunQueryResult } from '../db.model'
 import { DBQuery, RunnableDBQuery } from '../query/dbQuery'
-import { DBTransaction } from '../transaction/dbTransaction'
 import {
   CommonDaoCfg,
   CommonDaoCreateOptions,
@@ -698,68 +689,6 @@ export class CommonDao<
     return obj as any
   }
 
-  tx = {
-    save: async (
-      bm: Unsaved<BM>,
-      opt: CommonDaoSaveBatchOptions<DBM> = {},
-    ): Promise<DBSaveBatchOperation | undefined> => {
-      // .save actually returns DBM (not BM) when it detects `opt.tx === true`
-      const row: DBM | null = (await this.save(bm, { ...opt, tx: true })) as any
-      if (row === null) return
-
-      return {
-        type: 'saveBatch',
-        table: this.cfg.table,
-        rows: [row],
-        opt: {
-          excludeFromIndexes: this.cfg.excludeFromIndexes,
-          ...opt,
-        },
-      }
-    },
-    saveBatch: async (
-      bms: Unsaved<BM>[],
-      opt: CommonDaoSaveBatchOptions<DBM> = {},
-    ): Promise<DBSaveBatchOperation | undefined> => {
-      const rows: DBM[] = (await this.saveBatch(bms, { ...opt, tx: true })) as any
-      if (!rows.length) return
-
-      return {
-        type: 'saveBatch',
-        table: this.cfg.table,
-        rows,
-        opt: {
-          excludeFromIndexes: this.cfg.excludeFromIndexes,
-          ...opt,
-        },
-      }
-    },
-    deleteByIds: async (
-      ids: string[],
-      opt: CommonDaoOptions = {},
-    ): Promise<DBDeleteByIdsOperation | undefined> => {
-      if (!ids.length) return
-      return {
-        type: 'deleteByIds',
-        table: this.cfg.table,
-        ids,
-        opt,
-      }
-    },
-    deleteById: async (
-      id: string | null | undefined,
-      opt: CommonDaoOptions = {},
-    ): Promise<DBDeleteByIdsOperation | undefined> => {
-      if (!id) return
-      return {
-        type: 'deleteByIds',
-        table: this.cfg.table,
-        ids: [id],
-        opt,
-      }
-    },
-  }
-
   // SAVE
   /**
    * Mutates with id, created, updated
@@ -778,12 +707,7 @@ export class CommonDao<
 
     if (this.cfg.hooks!.beforeSave) {
       dbm = (await this.cfg.hooks!.beforeSave(dbm))!
-      if (dbm === null && !opt.tx) return bm as any
-    }
-
-    if (opt.tx) {
-      // May return `null`, in which case it'll be skipped
-      return dbm as any
+      if (dbm === null) return bm as any
     }
 
     const table = opt.table || this.cfg.table
@@ -960,10 +884,6 @@ export class CommonDao<
       )
     }
 
-    if (opt.tx) {
-      return dbms as any
-    }
-
     if (opt.ensureUniqueId) throw new AppError('ensureUniqueId is not supported in saveBatch')
     if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
       opt = { ...opt, saveMethod: 'insert' }
@@ -1071,7 +991,7 @@ export class CommonDao<
 
           if (beforeSave) {
             dbm = (await beforeSave(dbm))!
-            if (dbm === null && !opt.tx) return SKIP
+            if (dbm === null) return SKIP
           }
 
           return dbm
@@ -1118,7 +1038,7 @@ export class CommonDao<
     const op = `deleteById(${id})`
     const table = opt.table || this.cfg.table
     const started = this.logStarted(op, table)
-    const count = await this.cfg.db.deleteByQuery(DBQuery.create(table).filterEq('id', id))
+    const count = await this.cfg.db.deleteByIds(table, [id], opt)
     this.logSaveResult(started, op, table)
     return count
   }
@@ -1130,7 +1050,7 @@ export class CommonDao<
     const op = `deleteByIds(${ids.join(', ')})`
     const table = opt.table || this.cfg.table
     const started = this.logStarted(op, table)
-    const count = await this.cfg.db.deleteByQuery(DBQuery.create(table).filterIn('id', ids))
+    const count = await this.cfg.db.deleteByIds(table, ids, opt)
     this.logSaveResult(started, op, table)
     return count
   }
@@ -1410,11 +1330,22 @@ export class CommonDao<
     await this.cfg.db.ping()
   }
 
-  async runInTransaction(ops: Promisable<DBOperation | undefined>[]): Promise<void> {
-    const resolvedOps = (await Promise.all(ops)).filter(_isTruthy)
-    if (!resolvedOps.length) return
+  async useTransaction(fn: (tx: CommonDaoTransaction) => Promise<void>): Promise<void> {
+    const tx = await this.cfg.db.createTransaction()
+    const daoTx = new CommonDaoTransaction(tx)
 
-    await this.cfg.db.commitTransaction(DBTransaction.create(resolvedOps))
+    try {
+      await fn(daoTx)
+      await daoTx.commit()
+    } catch (err) {
+      await daoTx.rollback()
+      throw err
+    }
+  }
+
+  async createTransaction(): Promise<CommonDaoTransaction> {
+    const tx = await this.cfg.db.createTransaction()
+    return new CommonDaoTransaction(tx)
   }
 
   protected logResult(started: number, op: string, res: any, table: string): void {
@@ -1473,3 +1404,86 @@ export class CommonDao<
     return Date.now()
   }
 }
+
+export class CommonDaoTransaction {
+  constructor(private tx: DBTransaction) {}
+
+  async commit(): Promise<void> {
+    await this.tx.commit()
+  }
+  async rollback(): Promise<void> {
+    try {
+      await this.tx.rollback()
+    } catch (err) {
+      console.log(err)
+    }
+  }
+
+  async getById<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId>(
+    dao: CommonDao<BM, DBM, any>,
+    id: string,
+    opt?: CommonDaoOptions,
+  ): Promise<Saved<BM> | null> {
+    return (await this.getByIds(dao, [id], opt))[0] || null
+  }
+
+  async getByIds<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId>(
+    dao: CommonDao<BM, DBM, any>,
+    ids: string[],
+    opt?: CommonDaoOptions,
+  ): Promise<Saved<BM>[]> {
+    try {
+      return await dao.getByIds(ids, { ...opt, tx: this.tx })
+    } catch (err) {
+      await this.rollback()
+      throw err
+    }
+  }
+
+  async runQuery<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId>(
+    dao: CommonDao<BM, DBM, any>,
+    q: DBQuery<DBM>,
+    opt?: CommonDaoOptions,
+  ): Promise<Saved<BM>[]> {
+    try {
+      return await dao.runQuery(q, { ...opt, tx: this.tx })
+    } catch (err) {
+      await this.rollback()
+      throw err
+    }
+  }
+
+  async save<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId>(
+    dao: CommonDao<BM, DBM, any>,
+    bm: Unsaved<BM>,
+    opt?: CommonDaoSaveBatchOptions<DBM>,
+  ): Promise<Saved<BM>> {
+    return (await this.saveBatch(dao, [bm], opt))[0]!
+  }
+
+  async saveBatch<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId>(
+    dao: CommonDao<BM, DBM, any>,
+    bms: Unsaved<BM>[],
+    opt?: CommonDaoSaveBatchOptions<DBM>,
+  ): Promise<Saved<BM>[]> {
+    try {
+      return await dao.saveBatch(bms, { ...opt, tx: this.tx })
+    } catch (err) {
+      await this.rollback()
+      throw err
+    }
+  }
+
+  async deleteById(dao: CommonDao<any>, id: string, opt?: CommonDaoOptions): Promise<number> {
+    return await this.deleteByIds(dao, [id], opt)
+  }
+
+  async deleteByIds(dao: CommonDao<any>, ids: string[], opt?: CommonDaoOptions): Promise<number> {
+    try {
+      return await dao.deleteByIds(ids, { ...opt, tx: this.tx })
+    } catch (err) {
+      await this.rollback()
+      throw err
+    }
+  }
+}

package/src/db.model.ts

@@ -1,4 +1,4 @@
-import { ObjectWithId } from '@naturalcycles/js-lib'
+import type { ObjectWithId } from '@naturalcycles/js-lib'
 
 /**
  * Similar to SQL INSERT, UPDATE.
@@ -10,7 +10,20 @@ import { ObjectWithId } from '@naturalcycles/js-lib'
  */
 export type CommonDBSaveMethod = 'upsert' | 'insert' | 'update'
 
-export interface CommonDBOptions {}
+export interface DBTransaction {
+  commit: () => Promise<void>
+  rollback: () => Promise<void>
+}
+
+export interface CommonDBOptions {
+  /**
+   * If passed - the operation will be performed in the context of that DBTransaction.
+   * Note that not every type of operation supports Transaction
+   * (e.g in Datastore queries cannot be executed inside a Transaction).
+   * Also, not every CommonDB implementation supports Transactions.
+   */
+  tx?: DBTransaction
+}
 
 /**
  * All properties default to undefined.

package/src/index.ts

@@ -13,6 +13,5 @@ export * from './pipeline/dbPipelineBackup'
 export * from './pipeline/dbPipelineCopy'
 export * from './pipeline/dbPipelineRestore'
 export * from './query/dbQuery'
-export * from './transaction/dbTransaction'
 export * from './transaction/dbTransaction.util'
 export * from './kv/commonKeyValueDaoMemoCache'

package/src/pipeline/dbPipelineBackup.ts

@@ -20,14 +20,11 @@ import {
   transformTap,
   transformToNDJson,
   _pipeline,
-  _ensureDirSync,
-  _pathExistsSync,
-  _ensureFileSync,
-  _writeJson,
   boldWhite,
   dimWhite,
   grey,
   yellow,
+  fs2,
 } from '@naturalcycles/nodejs-lib'
 import { CommonDB } from '../common.db'
 import { DBQuery } from '../index'
@@ -180,7 +177,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
 
   console.log(`>> ${dimWhite('dbPipelineBackup')} started in ${grey(outputDirPath)}...`)
 
-  _ensureDirSync(outputDirPath)
+  fs2.ensureDir(outputDirPath)
 
   tables ||= await db.getTables()
 
@@ -213,20 +210,20 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
       const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '')
       const schemaFilePath = `${outputDirPath}/${table}.schema.json`
 
-      if (protectFromOverwrite && _pathExistsSync(filePath)) {
+      if (protectFromOverwrite && fs2.pathExists(filePath)) {
         throw new AppError(`dbPipelineBackup: output file exists: ${filePath}`)
       }
 
       const started = Date.now()
       let rows = 0
 
-      _ensureFileSync(filePath)
+      fs2.ensureFile(filePath)
 
       // console.log(`>> ${grey(filePath)} started...`)
 
       if (emitSchemaFromDB) {
         const schema = await db.getTableSchema(table)
-        await _writeJson(schemaFilePath, schema, { spaces: 2 })
+        await fs2.writeJsonAsync(schemaFilePath, schema, { spaces: 2 })
         console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
       }
 

package/src/pipeline/dbPipelineRestore.ts

@@ -25,12 +25,11 @@ import {
   transformTap,
   writableForEach,
   _pipeline,
-  _ensureDirSync,
-  _readJson,
   boldWhite,
   dimWhite,
   grey,
   yellow,
+  fs2,
 } from '@naturalcycles/nodejs-lib'
 import { CommonDB } from '../common.db'
 import { CommonDBSaveOptions } from '../index'
@@ -145,7 +144,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
     `>> ${dimWhite('dbPipelineRestore')} started in ${grey(inputDirPath)}...${sinceUpdatedStr}`,
   )
 
-  _ensureDirSync(inputDirPath)
+  fs2.ensureDir(inputDirPath)
 
   const tablesToGzip = new Set<string>()
   const sizeByTable: Record<string, number> = {}
@@ -185,7 +184,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
         return
       }
 
-      const schema = await _readJson<JsonSchemaObject<any>>(schemaFilePath)
+      const schema = await fs2.readJsonAsync<JsonSchemaObject<any>>(schemaFilePath)
       await db.createTable(table, schema, { dropIfExists: true })
     })
   }

package/src/testing/daoTest.ts

@@ -1,10 +1,10 @@
 import { Readable } from 'node:stream'
-import { pDelay, _deepCopy, _pick, _sortBy, _omit, localTimeNow } from '@naturalcycles/js-lib'
+import { _deepCopy, _pick, _sortBy, _omit, localTimeNow } from '@naturalcycles/js-lib'
 import { _pipeline, readableToArray, transformNoOp } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel, DBQuery } from '..'
 import { CommonDB } from '../common.db'
 import { CommonDao } from '../commondao/common.dao'
-import { CommonDBImplementationFeatures, CommonDBImplementationQuirks, expectMatch } from './dbTest'
+import { CommonDBImplementationQuirks, expectMatch } from './dbTest'
 import {
   createTestItemsBM,
   testItemBMSchema,
@@ -16,11 +16,8 @@ import {
 } from './test.model'
 import { TestItemBM } from '.'
 
-export function runCommonDaoTest(
-  db: CommonDB,
-  features: CommonDBImplementationFeatures = {},
-  quirks: CommonDBImplementationQuirks = {},
-): void {
+export function runCommonDaoTest(db: CommonDB, quirks: CommonDBImplementationQuirks = {}): void {
+  const { support } = db
   const dao = new CommonDao({
     table: TEST_TABLE,
     db,
@@ -31,26 +28,6 @@ export function runCommonDaoTest(
     logLevel: CommonDaoLogLevel.DATA_FULL,
   })
 
-  const {
-    querying = true,
-    // tableSchemas = true,
-    createTable = true,
-    dbQueryFilter = true,
-    // dbQueryFilterIn = true,
-    dbQueryOrder = true,
-    dbQuerySelectFields = true,
-    streaming = true,
-    strongConsistency = true,
-    nullValues = true,
-    transactions = true,
-  } = features
-
-  // const {
-  // allowExtraPropertiesInResponse,
-  // allowBooleansAsUndefined,
-  // } = quirks
-  const eventualConsistencyDelay = !strongConsistency && quirks.eventualConsistencyDelay
-
   const items = createTestItemsBM(3)
   const itemsClone = _deepCopy(items)
   // deepFreeze(items) // mutation of id/created/updated is allowed now! (even expected)
@@ -66,13 +43,13 @@ export function runCommonDaoTest(
   })
 
   // CREATE TABLE, DROP
-  if (createTable) {
+  if (support.createTable) {
     test('createTable, dropIfExists=true', async () => {
       await dao.createTable(testItemDBMJsonSchema, { dropIfExists: true })
     })
   }
 
-  if (querying) {
+  if (support.queries) {
     // DELETE ALL initially
     test('deleteByIds test items', async () => {
       const rows = await dao.query().select(['id']).runQuery()
@@ -87,7 +64,6 @@ export function runCommonDaoTest(
 
     // QUERY empty
     test('runQuery(all), runQueryCount should return empty', async () => {
-      if (eventualConsistencyDelay) await pDelay(eventualConsistencyDelay)
       expect(await dao.query().runQuery()).toEqual([])
       expect(await dao.query().runQueryCount()).toBe(0)
     })
@@ -109,7 +85,7 @@ export function runCommonDaoTest(
   })
 
   // SAVE
-  if (nullValues) {
+  if (support.nullValues) {
     test('should allow to save and load null values', async () => {
       const item3 = {
         ...createTestItemBM(3),
@@ -168,15 +144,14 @@ export function runCommonDaoTest(
   })
 
   // QUERY
-  if (querying) {
+  if (support.queries) {
     test('runQuery(all) should return all items', async () => {
-      if (eventualConsistencyDelay) await pDelay(eventualConsistencyDelay)
       let rows = await dao.query().runQuery()
       rows = _sortBy(rows, r => r.id)
       expectMatch(expectedItems, rows, quirks)
     })
 
-    if (dbQueryFilter) {
+    if (support.dbQueryFilter) {
       test('query even=true', async () => {
         let rows = await dao.query().filter('even', '==', true).runQuery()
         rows = _sortBy(rows, r => r.id)
@@ -188,14 +163,14 @@ export function runCommonDaoTest(
       })
     }
 
-    if (dbQueryOrder) {
+    if (support.dbQueryOrder) {
       test('query order by k1 desc', async () => {
         const rows = await dao.query().order('k1', true).runQuery()
         expectMatch([...expectedItems].reverse(), rows, quirks)
       })
     }
 
-    if (dbQuerySelectFields) {
+    if (support.dbQuerySelectFields) {
       test('projection query with only ids', async () => {
         let rows = await dao.query().select(['id']).runQuery()
         rows = _sortBy(rows, r => r.id)
@@ -213,7 +188,7 @@ export function runCommonDaoTest(
   }
 
   // STREAM
-  if (streaming) {
+  if (support.streaming) {
     test('streamQueryForEach all', async () => {
       let rows: TestItemBM[] = []
       await dao.query().streamQueryForEach(bm => void rows.push(bm))
@@ -269,11 +244,10 @@ export function runCommonDaoTest(
   }
 
   // DELETE BY
-  if (querying) {
+  if (support.queries) {
     test('deleteByQuery even=false', async () => {
       const deleted = await dao.query().filter('even', '==', false).deleteByQuery()
       expect(deleted).toBe(items.filter(item => !item.even).length)
-      if (eventualConsistencyDelay) await pDelay(eventualConsistencyDelay)
       expect(await dao.query().runQueryCount()).toBe(1)
     })
 
@@ -283,14 +257,18 @@ export function runCommonDaoTest(
     })
   }
 
-  if (transactions) {
+  if (support.transactions) {
     test('transaction happy path', async () => {
       // cleanup
       await dao.query().deleteByQuery()
 
       // Test that id, created, updated are created
       const now = localTimeNow().unix()
-      await dao.runInTransaction([dao.tx.save(_omit(item1, ['id', 'created', 'updated']))])
+
+      await dao.useTransaction(async tx => {
+        const row = _omit(item1, ['id', 'created', 'updated'])
+        await tx.save(dao, row)
+      })
 
       const loaded = await dao.query().runQuery()
       expect(loaded.length).toBe(1)
@@ -298,17 +276,19 @@ export function runCommonDaoTest(
       expect(loaded[0]!.created).toBeGreaterThanOrEqual(now)
       expect(loaded[0]!.updated).toBe(loaded[0]!.created)
 
-      await dao.runInTransaction([dao.tx.deleteById(loaded[0]!.id)])
+      await dao.useTransaction(async tx => {
+        await tx.deleteById(dao, loaded[0]!.id)
+      })
 
       // saveBatch [item1, 2, 3]
       // save item3 with k1: k1_mod
       // delete item2
       // remaining: item1, item3_with_k1_mod
-      await dao.runInTransaction([
-        dao.tx.saveBatch(items),
-        dao.tx.save({ ...items[2]!, k1: 'k1_mod' }),
-        dao.tx.deleteById(items[1]!.id),
-      ])
+      await dao.useTransaction(async tx => {
+        await tx.saveBatch(dao, items)
+        await tx.save(dao, { ...items[2]!, k1: 'k1_mod' })
+        await tx.deleteById(dao, items[1]!.id)
+      })
 
       const rows = await dao.query().runQuery()
       const expected = [items[0], { ...items[2]!, k1: 'k1_mod' }]
@@ -317,10 +297,10 @@ export function runCommonDaoTest(
 
     test('transaction rollback', async () => {
       await expect(
-        dao.runInTransaction([
-          dao.tx.deleteById(items[2]!.id),
-          dao.tx.save({ ...items[0]!, k1: 5 as any }), // it should fail here
-        ]),
+        dao.useTransaction(async tx => {
+          await tx.deleteById(dao, items[2]!.id)
+          await tx.save(dao, { ...items[0]!, k1: 5 as any }) // it should fail here
+        }),
       ).rejects.toThrow()
 
       const rows = await dao.query().runQuery()
@@ -328,7 +308,7 @@ export function runCommonDaoTest(
       expectMatch(expected, rows, quirks)
     })
 
-    if (querying) {
+    if (support.queries) {
       test('transaction cleanup', async () => {
         await dao.query().deleteByQuery()
       })