@naturalcycles/db-lib 8.35.0 → 8.37.0

package/dist/validation/index.d.ts

@@ -1,8 +1,8 @@
 import { CommonDBOptions, CommonDBSaveOptions } from '../db.model';
 import { DBQuery, DBQueryFilter, DBQueryOrder } from '../query/dbQuery';
 export declare const commonDBOptionsSchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<CommonDBOptions, CommonDBOptions>;
-export declare const commonDBSaveOptionsSchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<CommonDBSaveOptions<import("@naturalcycles/js-lib").AnyObjectWithId<string>>, CommonDBSaveOptions<import("@naturalcycles/js-lib").AnyObjectWithId<string>>>;
+export declare const commonDBSaveOptionsSchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<CommonDBSaveOptions<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>, CommonDBSaveOptions<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>>;
 export declare const dbQueryFilterOperatorSchema: import("@naturalcycles/nodejs-lib/dist/validation/joi/string.extensions").ExtendedStringSchema;
-export declare const dbQueryFilterSchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId<string>>, DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId<string>>>;
-export declare const dbQueryOrderSchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId<string>>, DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId<string>>>;
+export declare const dbQueryFilterSchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>, DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>>;
+export declare const dbQueryOrderSchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>, DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId<string | number>>>;
 export declare const dbQuerySchema: import("@naturalcycles/nodejs-lib").ObjectSchemaTyped<DBQuery<any>, DBQuery<any>>;

package/package.json

@@ -42,7 +42,7 @@
   "engines": {
     "node": ">=14.15"
   },
-  "version": "8.35.0",
+  "version": "8.37.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/cachedb/cache.db.model.ts

@@ -45,7 +45,7 @@ export interface CacheDBCfg {
   logger?: CommonLogger
 }
 
-export interface CacheDBOptions<ROW extends ObjectWithId> extends CommonDBSaveOptions<ROW> {
+export interface CacheDBOptions {
   /**
    * @default false
    */
@@ -57,9 +57,10 @@ export interface CacheDBOptions<ROW extends ObjectWithId> extends CommonDBSaveOp
   onlyCache?: boolean
 }
 
-export interface CacheDBStreamOptions<ROW extends ObjectWithId>
-  extends CacheDBOptions<ROW>,
-    CommonDBStreamOptions {}
-export interface CacheDBCreateOptions<ROW extends ObjectWithId>
-  extends CacheDBOptions<ROW>,
-    CommonDBCreateOptions {}
+export interface CacheDBSaveOptions<ROW extends Partial<ObjectWithId>>
+  extends CacheDBOptions,
+    CommonDBSaveOptions<ROW> {}
+
+export interface CacheDBStreamOptions extends CacheDBOptions, CommonDBStreamOptions {}
+
+export interface CacheDBCreateOptions extends CacheDBOptions, CommonDBCreateOptions {}

package/src/adapter/cachedb/cache.db.ts

@@ -13,6 +13,7 @@ import {
   CacheDBCfg,
   CacheDBCreateOptions,
   CacheDBOptions,
+  CacheDBSaveOptions,
   CacheDBStreamOptions,
 } from './cache.db.model'
 
@@ -59,7 +60,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
   override async createTable<ROW extends ObjectWithId>(
     table: string,
     schema: JsonSchemaObject<ROW>,
-    opt: CacheDBCreateOptions<ROW> = {},
+    opt: CacheDBCreateOptions = {},
   ): Promise<void> {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
       await this.cfg.downstreamDB.createTable(table, schema, opt)
@@ -72,11 +73,11 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
   override async getByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
-    opt: CacheDBOptions<ROW> = {},
+    ids: ROW['id'][],
+    opt: CacheDBSaveOptions<ROW> = {},
   ): Promise<ROW[]> {
     const resultMap: StringMap<ROW> = {}
-    const missingIds: string[] = []
+    const missingIds: ROW['id'][] = []
 
     if (!opt.skipCache && !this.cfg.skipCache) {
       const results = await this.cfg.cacheDB.getByIds<ROW>(table, ids, opt)
@@ -118,13 +119,13 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
   override async deleteByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
-    opt: CacheDBOptions<ROW> = {},
+    ids: ROW['id'][],
+    opt: CacheDBOptions = {},
   ): Promise<number> {
     let deletedIds = 0
 
     if (!opt.onlyCache && !this.cfg.onlyCache) {
-      deletedIds = await this.cfg.downstreamDB.deleteByIds(table, ids, opt)
+      deletedIds = await this.cfg.downstreamDB.deleteByIds<ROW>(table, ids, opt)
 
       if (this.cfg.logDownstream) {
         this.cfg.logger?.log(`${table}.deleteByIds ${deletedIds} rows from downstream`)
@@ -132,21 +133,23 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
     }
 
     if (!opt.skipCache && !this.cfg.skipCache) {
-      const cacheResult = this.cfg.cacheDB.deleteByIds(table, ids, opt).then(deletedFromCache => {
-        if (this.cfg.logCached) {
-          this.cfg.logger?.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`)
-        }
-      })
+      const cacheResult = this.cfg.cacheDB
+        .deleteByIds<ROW>(table, ids, opt)
+        .then(deletedFromCache => {
+          if (this.cfg.logCached) {
+            this.cfg.logger?.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`)
+          }
+        })
       if (this.cfg.awaitCache) await cacheResult
     }
 
     return deletedIds
   }
 
-  override async saveBatch<ROW extends ObjectWithId>(
+  override async saveBatch<ROW extends Partial<ObjectWithId>>(
     table: string,
     rows: ROW[],
-    opt: CacheDBOptions<ROW> = {},
+    opt: CacheDBSaveOptions<ROW> = {},
   ): Promise<void> {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
       await this.cfg.downstreamDB.saveBatch(table, rows, opt)
@@ -174,7 +177,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
   override async runQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
-    opt: CacheDBOptions<ROW> = {},
+    opt: CacheDBSaveOptions<ROW> = {},
   ): Promise<RunQueryResult<ROW>> {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
       const { rows, ...queryResult } = await this.cfg.downstreamDB.runQuery(q, opt)
@@ -204,7 +207,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
   override async runQueryCount<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
-    opt: CacheDBOptions<ROW> = {},
+    opt: CacheDBOptions = {},
   ): Promise<number> {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
       return await this.cfg.downstreamDB.runQueryCount(q, opt)
@@ -221,7 +224,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
   override streamQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
-    opt: CacheDBStreamOptions<ROW> = {},
+    opt: CacheDBStreamOptions = {},
   ): Readable {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
       const stream = this.cfg.downstreamDB.streamQuery<ROW>(q, opt)
@@ -260,7 +263,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
   override async deleteByQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
-    opt: CacheDBOptions<ROW> = {},
+    opt: CacheDBOptions = {},
   ): Promise<number> {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
       const deletedIds = await this.cfg.downstreamDB.deleteByQuery(q, opt)

package/src/adapter/file/file.db.ts

@@ -13,6 +13,7 @@ import {
   _filterUndefinedValues,
   ObjectWithId,
   AnyObjectWithId,
+  _assert,
 } from '@naturalcycles/js-lib'
 import { readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { dimGrey } from '@naturalcycles/nodejs-lib/dist/colors'
@@ -64,14 +65,14 @@ export class FileDB extends BaseCommonDB implements CommonDB {
 
   override async getByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
+    ids: ROW['id'][],
     _opt?: CommonDBOptions,
   ): Promise<ROW[]> {
     const byId = _by(await this.loadFile<ROW>(table), r => r.id)
     return ids.map(id => byId[id]!).filter(Boolean)
   }
 
-  override async saveBatch<ROW extends ObjectWithId>(
+  override async saveBatch<ROW extends Partial<ObjectWithId>>(
     table: string,
     rows: ROW[],
     _opt?: CommonDBSaveOptions<ROW>,
@@ -79,13 +80,15 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     if (!rows.length) return // save some api calls
 
     // 1. Load the whole file
-    const byId = _by(await this.loadFile<ROW>(table), r => r.id)
+    const byId = _by(await this.loadFile<ROW & ObjectWithId>(table), r => r.id)
 
     // 2. Merge with new data (using ids)
     let saved = 0
     rows.forEach(r => {
+      _assert(r.id, 'FileDB: row.id is required')
+
       if (!_deepEquals(byId[r.id], r)) {
-        byId[r.id] = r
+        byId[r.id] = r as any
         saved++
       }
     })
@@ -173,7 +176,7 @@ export class FileDB extends BaseCommonDB implements CommonDB {
 
   override async deleteByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
+    ids: ROW['id'][],
     _opt?: CommonDBOptions,
   ): Promise<number> {
     if (!ids.length) return 0

package/src/adapter/inmemory/inMemory.db.ts

@@ -137,49 +137,56 @@ export class InMemoryDB implements CommonDB {
     if (opt.dropIfExists) {
       this.data[table] = {}
     } else {
-      this.data[table] = this.data[table] || {}
+      this.data[table] ||= {}
     }
   }
 
   async getByIds<ROW extends ObjectWithId>(
     _table: string,
-    ids: string[],
+    ids: ROW['id'][],
     _opt?: CommonDBOptions,
   ): Promise<ROW[]> {
     const table = this.cfg.tablesPrefix + _table
-    this.data[table] = this.data[table] || {}
+    this.data[table] ||= {}
     return ids.map(id => this.data[table]![id]).filter(Boolean) as ROW[]
   }
 
-  async saveBatch<ROW extends ObjectWithId>(
+  async saveBatch<ROW extends Partial<ObjectWithId>>(
     _table: string,
     rows: ROW[],
-    _opt?: CommonDBSaveOptions<ROW>,
+    opt: CommonDBSaveOptions<ROW> = {},
   ): Promise<void> {
     const table = this.cfg.tablesPrefix + _table
-    this.data[table] = this.data[table] || {}
+    this.data[table] ||= {}
 
     rows.forEach(r => {
       if (!r.id) {
         this.cfg.logger?.warn({ rows })
-        throw new Error(`InMemoryDB: id doesn't exist for row`)
+        throw new Error(
+          `InMemoryDB doesn't support id auto-generation in saveBatch, row without id was given`,
+        )
+      }
+
+      if (opt.saveMethod === 'insert' && this.data[table]![r.id]) {
+        throw new Error(`InMemoryDB: INSERT failed, entity exists: ${table}.${r.id}`)
+      }
+
+      if (opt.saveMethod === 'update' && !this.data[table]![r.id]) {
+        throw new Error(`InMemoryDB: UPDATE failed, entity doesn't exist: ${table}.${r.id}`)
       }
 
       // JSON parse/stringify (deep clone) is to:
       // 1. Not store values "by reference" (avoid mutation bugs)
       // 2. Simulate real DB that would do something like that in a transport layer anyway
       this.data[table]![r.id] = JSON.parse(JSON.stringify(r), bufferReviver)
-
-      // special treatment for Buffers (assign them raw, without JSON parse/stringify)
-      // Object.entries(r).forEach(([k, v]) => {
-      //   if (Buffer.isBuffer(v)) {
-      //     this.data[table]![r.id]![k] = v
-      //   }
-      // })
     })
   }
 
-  async deleteByIds(_table: string, ids: string[], _opt?: CommonDBOptions): Promise<number> {
+  async deleteByIds<ROW extends ObjectWithId>(
+    _table: string,
+    ids: ROW['id'][],
+    _opt?: CommonDBOptions,
+  ): Promise<number> {
     const table = this.cfg.tablesPrefix + _table
     this.data[table] = this.data[table] || {}
 

package/src/base.common.db.ts

@@ -35,7 +35,7 @@ export class BaseCommonDB implements CommonDB {
     _schema: JsonSchemaObject<ROW>,
   ): Promise<void> {}
 
-  async deleteByIds(_table: string, _ids: string[]): Promise<number> {
+  async deleteByIds<ROW extends ObjectWithId>(_table: string, _ids: ROW['id'][]): Promise<number> {
     return 0
   }
 
@@ -43,7 +43,7 @@ export class BaseCommonDB implements CommonDB {
     return 0
   }
 
-  async getByIds<ROW extends ObjectWithId>(_table: string, _ids: string[]): Promise<ROW[]> {
+  async getByIds<ROW extends ObjectWithId>(_table: string, _ids: ROW['id'][]): Promise<ROW[]> {
     return []
   }
 
@@ -55,7 +55,7 @@ export class BaseCommonDB implements CommonDB {
     return 0
   }
 
-  async saveBatch<ROW extends ObjectWithId>(
+  async saveBatch<ROW extends Partial<ObjectWithId>>(
     _table: string,
     _rows: ROW[],
     _opt?: CommonDBSaveOptions<ROW>,

package/src/common.db.ts

@@ -49,7 +49,7 @@ export interface CommonDB {
    */
   getByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
+    ids: ROW['id'][],
     opt?: CommonDBOptions,
   ): Promise<ROW[]>
 
@@ -70,7 +70,10 @@ export interface CommonDB {
   ): ReadableTyped<ROW>
 
   // SAVE
-  saveBatch<ROW extends ObjectWithId>(
+  /**
+   * rows can have missing ids only if DB supports auto-generating them (like mysql auto_increment).
+   */
+  saveBatch<ROW extends Partial<ObjectWithId>>(
     table: string,
     rows: ROW[],
     opt?: CommonDBSaveOptions<ROW>,
@@ -78,10 +81,14 @@ export interface CommonDB {
 
   // DELETE
   /**
-   * @returns number of deleted items.
+   * Returns number of deleted items.
    * Not supported by all implementations (e.g Datastore will always return same number as number of ids).
    */
-  deleteByIds(table: string, ids: string[], opt?: CommonDBOptions): Promise<number>
+  deleteByIds<ROW extends ObjectWithId>(
+    table: string,
+    ids: ROW['id'][],
+    opt?: CommonDBOptions,
+  ): Promise<number>
 
   deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBOptions): Promise<number>
 

package/src/commondao/common.dao.model.ts

@@ -10,27 +10,21 @@ import {
 import { CommonDB } from '../common.db'
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model'
 
-// Hook DBM, BM, TM types should follow this exact order
-export type CommonDaoCreateIdHook<BM, DBM> = (obj: DBM | BM) => string
-export type CommonDaoParseNaturalIdHook<DBM> = (id: string) => Partial<DBM>
-export type CommonDaoBeforeCreateHook<BM> = (bm: Partial<BM>) => Partial<BM>
-export type CommonDaoBeforeDBMValidateHook<DBM> = (dbm: Partial<DBM>) => Partial<DBM>
-export type CommonDaoBeforeDBMToBMHook<BM, DBM> = (dbm: DBM) => Partial<BM> | Promise<Partial<BM>>
-export type CommonDaoBeforeBMToDBMHook<BM, DBM> = (bm: BM) => Partial<DBM> | Promise<Partial<DBM>>
-export type CommonDaoBeforeTMToBMHook<BM, TM> = (tm: TM) => Partial<BM>
-export type CommonDaoBeforeBMToTMHook<BM, TM> = (bm: BM) => Partial<TM>
-export type CommonDaoAnonymizeHook<DBM> = (dbm: DBM) => DBM
-
-interface CommonDaoHooks<BM, DBM, TM> {
-  createId: CommonDaoCreateIdHook<BM, DBM>
-  parseNaturalId: CommonDaoParseNaturalIdHook<DBM>
-  beforeCreate: CommonDaoBeforeCreateHook<BM>
-  beforeDBMValidate: CommonDaoBeforeDBMValidateHook<DBM>
-  beforeDBMToBM: CommonDaoBeforeDBMToBMHook<BM, DBM>
-  beforeBMToDBM: CommonDaoBeforeBMToDBMHook<BM, DBM>
-  beforeTMToBM: CommonDaoBeforeTMToBMHook<BM, TM>
-  beforeBMToTM: CommonDaoBeforeBMToTMHook<BM, TM>
-  anonymize: CommonDaoAnonymizeHook<DBM>
+export interface CommonDaoHooks<
+  BM extends Partial<ObjectWithId<ID>>,
+  DBM extends ObjectWithId<ID>,
+  TM,
+  ID extends string | number,
+> {
+  createId: (obj: DBM | BM) => ID
+  parseNaturalId: (id: ID) => Partial<DBM>
+  beforeCreate: (bm: Partial<BM>) => Partial<BM>
+  beforeDBMValidate: (dbm: Partial<DBM>) => Partial<DBM>
+  beforeDBMToBM: (dbm: DBM) => Partial<BM> | Promise<Partial<BM>>
+  beforeBMToDBM: (bm: BM) => Partial<DBM> | Promise<Partial<DBM>>
+  beforeTMToBM: (tm: TM) => Partial<BM>
+  beforeBMToTM: (bm: BM) => Partial<TM>
+  anonymize: (dbm: DBM) => DBM
 
   /**
    * If hook is defined - allows to prevent or modify the error thrown.
@@ -61,9 +55,10 @@ export enum CommonDaoLogLevel {
 }
 
 export interface CommonDaoCfg<
-  BM extends Partial<ObjectWithId>,
-  DBM extends ObjectWithId = Saved<BM>,
+  BM extends Partial<ObjectWithId<ID>>,
+  DBM extends ObjectWithId<ID> = Saved<BM>,
   TM = BM,
+  ID extends string | number = DBM['id'],
 > {
   db: CommonDB
   table: string
@@ -109,7 +104,19 @@ export interface CommonDaoCfg<
   logStarted?: boolean
 
   // Hooks are designed with inspiration from got/ky interface
-  hooks?: Partial<CommonDaoHooks<BM, DBM, TM>>
+  hooks?: Partial<CommonDaoHooks<BM, DBM, TM, ID>>
+
+  /**
+   * Defaults to 'string'
+   */
+  idType?: 'string' | 'number'
+
+  /**
+   * Defaults to true.
+   * Set to false to disable auto-generation of `id`.
+   * Useful e.g when your DB is generating ids by itself (e.g mysql auto_increment).
+   */
+  createId?: boolean
 
   /**
    * Defaults to true
@@ -202,7 +209,7 @@ export interface CommonDaoOptions extends CommonDBOptions {
 /**
  * All properties default to undefined.
  */
-export interface CommonDaoSaveOptions<DBM extends ObjectWithId>
+export interface CommonDaoSaveOptions<DBM extends Partial<ObjectWithId>>
   extends CommonDaoOptions,
     CommonDBSaveOptions<DBM> {
   /**