@naturalcycles/db-lib 8.49.0 → 8.50.0

package/src/commondao/common.dao.model.ts

@@ -1,4 +1,4 @@
-import { CommonLogger, ErrorMode, ObjectWithId, Saved } from '@naturalcycles/js-lib'
+import { CommonLogger, ErrorMode, ObjectWithId } from '@naturalcycles/js-lib'
 import {
   AjvSchema,
   AjvValidationError,
@@ -12,24 +12,25 @@ import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../
 
 export interface CommonDaoHooks<
   BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID>,
-  TM,
-  ID extends string | number,
+  DBM extends Partial<ObjectWithId<ID>> = BM,
+  ID extends string | number = NonNullable<BM['id']>,
 > {
-  createRandomId: () => ID
+  createRandomId(): ID
   /**
    * createNaturalId hook is called (tried) first.
    * If it doesn't exist - createRandomId is called.
    */
-  createNaturalId: (obj: DBM | BM) => ID
-  parseNaturalId: (id: ID) => Partial<DBM>
-  beforeCreate: (bm: Partial<BM>) => Partial<BM>
-  beforeDBMValidate: (dbm: Partial<DBM>) => Partial<DBM>
-  beforeDBMToBM: (dbm: DBM) => Partial<BM> | Promise<Partial<BM>>
-  beforeBMToDBM: (bm: BM) => Partial<DBM> | Promise<Partial<DBM>>
-  beforeTMToBM: (tm: TM) => Partial<BM>
-  beforeBMToTM: (bm: BM) => Partial<TM>
-  anonymize: (dbm: DBM) => DBM
+  createNaturalId(obj: DBM | BM): ID
+  parseNaturalId(id: ID): Partial<DBM>
+  beforeCreate(bm: Partial<BM>): Partial<BM>
+  beforeDBMValidate(dbm: Partial<DBM>): Partial<DBM>
+  beforeDBMToBM(dbm: DBM): Partial<BM> | Promise<Partial<BM>>
+  beforeBMToDBM(bm: BM): Partial<DBM> | Promise<Partial<DBM>>
+
+  /**
+   * Applicable to **read** operations, never to **save** operations.
+   */
+  anonymize(dbm: DBM): DBM
 
   /**
    * If hook is defined - allows to prevent or modify the error thrown.
@@ -37,7 +38,7 @@ export interface CommonDaoHooks<
    * Return original `err` to pass the error through (will be thrown in CommonDao).
    * Return modified/new `Error` if needed.
    */
-  onValidationError: (err: JoiValidationError | AjvValidationError) => Error | false
+  onValidationError(err: JoiValidationError | AjvValidationError): Error | false
 }
 
 export enum CommonDaoLogLevel {
@@ -61,9 +62,8 @@ export enum CommonDaoLogLevel {
 
 export interface CommonDaoCfg<
   BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID> = Saved<BM>,
-  TM = BM,
-  ID extends string | number = string,
+  DBM extends Partial<ObjectWithId<ID>> = BM,
+  ID extends string | number = NonNullable<BM['id']>,
 > {
   db: CommonDB
   table: string
@@ -73,7 +73,6 @@ export interface CommonDaoCfg<
    */
   dbmSchema?: ObjectSchemaTyped<DBM> | AjvSchema<DBM>
   bmSchema?: ObjectSchemaTyped<BM> | AjvSchema<BM>
-  tmSchema?: ObjectSchemaTyped<TM> | AjvSchema<TM>
 
   excludeFromIndexes?: (keyof DBM)[]
 
@@ -85,6 +84,8 @@ export interface CommonDaoCfg<
    * `delete*` and `patch` will throw.
    *
    * You can still override saveMethod, or set opt.allowMutability to allow deletion.
+   *
+   * todo: consider merging it with readOnly, as it's almost the same
    */
   immutable?: boolean
 
@@ -110,7 +111,7 @@ export interface CommonDaoCfg<
   logStarted?: boolean
 
   // Hooks are designed with inspiration from got/ky interface
-  hooks?: Partial<CommonDaoHooks<BM, DBM, TM, ID>>
+  hooks?: Partial<CommonDaoHooks<BM, DBM, ID>>
 
   /**
    * Defaults to 'string'
@@ -127,6 +128,7 @@ export interface CommonDaoCfg<
   /**
    * See the same option in CommonDB.
    * Defaults to false normally.
+   * "Generated" means "generated by the underlying DB" (e.g MySQL autoincrement).
    */
   assignGeneratedIds?: boolean
 
@@ -209,14 +211,6 @@ export interface CommonDaoOptions extends CommonDBOptions {
    */
   table?: string
 
-  /**
-   * If set - wraps the method in `pTimeout` with a timeout of given number of milliseconds.
-   * Currently, it is only used to debug an ongoing GCP infra issue.
-   *
-   * @experimental
-   */
-  timeout?: number
-
   /**
    * If passed - operation will not be performed immediately, but instead "added" to the transaction.
    * In the end - transaction needs to be committed (by calling `commit`).
@@ -232,7 +226,7 @@ export interface CommonDaoOptions extends CommonDBOptions {
 /**
  * All properties default to undefined.
  */
-export interface CommonDaoSaveOptions<DBM extends ObjectWithId>
+export interface CommonDaoSaveOptions<DBM extends Partial<ObjectWithId>>
   extends CommonDaoOptions,
     CommonDBSaveOptions<DBM> {
   /**

package/src/commondao/common.dao.ts

@@ -6,7 +6,6 @@ import {
   _since,
   _truncate,
   _uniqBy,
-  AnyObject,
   AppError,
   AsyncMapper,
   ErrorMode,
@@ -14,7 +13,6 @@ import {
   JsonSchemaRootObject,
   ObjectWithId,
   pMap,
-  pTimeout,
   Saved,
   Unsaved,
 } from '@naturalcycles/js-lib'
@@ -64,15 +62,13 @@ const isCI = !!process.env['CI']
  *
  * DBM = Database model (how it's stored in DB)
  * BM = Backend model (optimized for API access)
- * TM = Transport model (optimized to be sent over the wire)
  */
 export class CommonDao<
   BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID> = Saved<BM>,
-  TM extends AnyObject = BM,
+  DBM extends Partial<ObjectWithId<ID>> = BM,
   ID extends string | number = NonNullable<BM['id']>,
 > {
-  constructor(public cfg: CommonDaoCfg<BM, DBM, TM, ID>) {
+  constructor(public cfg: CommonDaoCfg<BM, DBM, ID>) {
     this.cfg = {
       // Default is to NOT log in AppEngine and in CI,
       // otherwise to log Operations
@@ -91,8 +87,6 @@ export class CommonDao<
         beforeDBMValidate: dbm => dbm,
         beforeDBMToBM: dbm => dbm as any,
         beforeBMToDBM: bm => bm as any,
-        beforeTMToBM: tm => tm as any,
-        beforeBMToTM: bm => bm as any,
         anonymize: dbm => dbm,
         onValidationError: err => err,
         ...cfg.hooks,
@@ -128,22 +122,7 @@ export class CommonDao<
     const table = opt.table || this.cfg.table
     const started = this.logStarted(op, table)
 
-    let dbm: DBM | undefined
-
-    if (opt.timeout) {
-      // todo: possibly remove it after debugging is done
-      dbm = await pTimeout(
-        async () => {
-          return (await this.cfg.db.getByIds<DBM>(table, [id]))[0]
-        },
-        {
-          timeout: opt.timeout,
-          name: `getById(${table})`,
-        },
-      )
-    } else {
-      dbm = (await this.cfg.db.getByIds<DBM>(table, [id]))[0]
-    }
+    const dbm = (await this.cfg.db.getByIds<DBM>(table, [id]))[0]
 
     const bm = opt.raw ? (dbm as any) : await this.dbmToBM(dbm, opt)
     this.logResult(started, op, bm, table)
@@ -180,24 +159,6 @@ export class CommonDao<
     return dbm || null
   }
 
-  async getByIdAsTM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
-  async getByIdAsTM(id?: ID | null, opt?: CommonDaoOptions): Promise<TM | null>
-  async getByIdAsTM(id?: ID | null, opt: CommonDaoOptions = {}): Promise<TM | null> {
-    if (!id) return null
-    const op = `getByIdAsTM(${id})`
-    const table = opt.table || this.cfg.table
-    const started = this.logStarted(op, table)
-    const [dbm] = await this.cfg.db.getByIds<DBM>(table, [id])
-    if (opt.raw) {
-      this.logResult(started, op, dbm, table)
-      return (dbm as any) || null
-    }
-    const bm = await this.dbmToBM(dbm, opt)
-    const tm = this.bmToTM(bm, opt)
-    this.logResult(started, op, tm, table)
-    return tm || null
-  }
-
   async getByIds(ids: ID[], opt: CommonDaoOptions = {}): Promise<Saved<BM>[]> {
     const op = `getByIds ${ids.length} id(s) (${_truncate(ids.slice(0, 10).join(', '), 50)})`
     const table = opt.table || this.cfg.table
@@ -295,8 +256,8 @@ export class CommonDao<
   /**
    * Pass `table` to override table
    */
-  query(table?: string): RunnableDBQuery<BM, DBM, TM, ID> {
-    return new RunnableDBQuery<BM, DBM, TM, ID>(this, table)
+  query(table?: string): RunnableDBQuery<BM, DBM, ID> {
+    return new RunnableDBQuery<BM, DBM, ID>(this, table)
   }
 
   async runQuery(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<Saved<BM>[]> {
@@ -364,29 +325,6 @@ export class CommonDao<
     return { rows: dbms, ...queryResult }
   }
 
-  async runQueryAsTM(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<TM[]> {
-    const { rows } = await this.runQueryExtendedAsTM(q, opt)
-    return rows
-  }
-
-  async runQueryExtendedAsTM(
-    q: DBQuery<DBM>,
-    opt: CommonDaoOptions = {},
-  ): Promise<RunQueryResult<TM>> {
-    q.table = opt.table || q.table
-    const op = `runQueryAsTM(${q.pretty()})`
-    const started = this.logStarted(op, q.table)
-    const { rows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
-    const partialQuery = !!q._selectedFieldNames
-    const tms =
-      partialQuery || opt.raw ? (rows as any[]) : this.bmsToTM(await this.dbmsToBM(rows, opt), opt)
-    this.logResult(started, op, tms, q.table)
-    return {
-      rows: tms,
-      ...queryResult,
-    }
-  }
-
   async runQueryCount(q: DBQuery<DBM>, opt: CommonDaoOptions = {}): Promise<number> {
     q.table = opt.table || q.table
     const op = `runQueryCount(${q.pretty()})`
@@ -548,7 +486,7 @@ export class CommonDao<
   async queryIds(q: DBQuery<DBM>, opt: CommonDaoOptions = {}): Promise<ID[]> {
     q.table = opt.table || q.table
     const { rows } = await this.cfg.db.runQuery(q.select(['id']), opt)
-    return rows.map(r => r.id)
+    return rows.map(r => r.id!)
   }
 
   streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions = {}): ReadableTyped<ID> {
@@ -559,7 +497,7 @@ export class CommonDao<
       .streamQuery<DBM>(q.select(['id']), opt)
       .on('error', err => stream.emit('error', err))
       .pipe(
-        transformMapSimple<DBM, ID>(objectWithId => objectWithId.id, {
+        transformMapSimple<DBM, ID>(objectWithId => objectWithId.id!, {
           errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }),
       )
@@ -581,7 +519,7 @@ export class CommonDao<
 
     await _pipeline([
       this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
-      transformMapSimple<DBM, ID>(objectWithId => objectWithId.id),
+      transformMapSimple<DBM, ID>(objectWithId => objectWithId.id!),
       transformTap(() => count++),
       transformMap<ID, void>(mapper, {
         ...opt,
@@ -907,7 +845,7 @@ export class CommonDao<
 
       await _pipeline([
         this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
-        transformMapSimple<DBM, ID>(objectWithId => objectWithId.id, {
+        transformMapSimple<DBM, ID>(objectWithId => objectWithId.id!, {
           errorMode: ErrorMode.SUPPRESS,
         }),
         transformBuffer<string>({ batchSize }),
@@ -971,7 +909,7 @@ export class CommonDao<
 
     // optimization: no need to run full joi DBM validation, cause BM validation will be run
     // const dbm = this.anyToDBM(_dbm, opt)
-    let dbm: DBM = { ..._dbm, ...this.cfg.hooks!.parseNaturalId!(_dbm.id) }
+    let dbm: DBM = { ..._dbm, ...this.cfg.hooks!.parseNaturalId!(_dbm.id!) }
 
     if (opt.anonymize) {
       dbm = this.cfg.hooks!.anonymize!(dbm)
@@ -1027,7 +965,7 @@ export class CommonDao<
     // this shouldn't be happening on load! but should on save!
     // this.assignIdCreatedUpdated(dbm, opt)
 
-    dbm = { ...dbm, ...this.cfg.hooks!.parseNaturalId!(dbm.id) }
+    dbm = { ...dbm, ...this.cfg.hooks!.parseNaturalId!(dbm.id!) }
 
     if (opt.anonymize) {
       dbm = this.cfg.hooks!.anonymize!(dbm)
@@ -1041,50 +979,6 @@ export class CommonDao<
     return entities.map(entity => this.anyToDBM(entity, opt))
   }
 
-  bmToTM(bm: undefined, opt?: CommonDaoOptions): TM | undefined
-  bmToTM(bm?: Saved<BM>, opt?: CommonDaoOptions): TM
-  bmToTM(bm?: Saved<BM>, opt?: CommonDaoOptions): TM | undefined {
-    if (bm === undefined) return
-
-    // optimization: 1 validation is enough
-    // Validate/convert BM
-    // bm gets assigned to the new reference
-    // bm = this.validateAndConvert(bm, this.cfg.bmSchema, DBModelType.BM, opt)
-
-    // BM > TM
-    const tm = this.cfg.hooks!.beforeBMToTM!(bm as any)
-
-    // Validate/convert DBM
-    return this.validateAndConvert(tm, this.cfg.tmSchema, DBModelType.TM, opt)
-  }
-
-  bmsToTM(bms: Saved<BM>[], opt: CommonDaoOptions = {}): TM[] {
-    // try/catch?
-    return bms.map(bm => this.bmToTM(bm, opt))
-  }
-
-  tmToBM(tm: undefined, opt?: CommonDaoOptions): undefined
-  tmToBM(tm?: TM, opt?: CommonDaoOptions): BM
-  tmToBM(tm?: TM, opt: CommonDaoOptions = {}): BM | undefined {
-    if (!tm) return
-
-    // optimization: 1 validation is enough
-    // Validate/convert TM
-    // bm gets assigned to the new reference
-    // tm = this.validateAndConvert(tm, this.cfg.tmSchema, DBModelType.TM, opt)
-
-    // TM > BM
-    const bm = this.cfg.hooks!.beforeTMToBM!(tm) as BM
-
-    // Validate/convert BM
-    return this.validateAndConvert<BM>(bm, this.cfg.bmSchema, DBModelType.BM, opt)
-  }
-
-  tmsToBM(tms: TM[], opt: CommonDaoOptions = {}): BM[] {
-    // try/catch?
-    return tms.map(tm => this.tmToBM(tm, opt))
-  }
-
   /**
    * Returns *converted value*.
    * Validates (unless `skipValidation=true` passed).

package/src/db.model.ts

@@ -74,7 +74,6 @@ export enum DBRelation {
 export enum DBModelType {
   DBM = 'DBM',
   BM = 'BM',
-  TM = 'TM',
 }
 
 /**

package/src/kv/commonKeyValueDao.ts

@@ -32,9 +32,9 @@ export interface CommonKeyValueDaoCfg<T> {
   logStarted?: boolean
 
   hooks?: {
-    mapValueToBuffer?: (v: T) => Promise<Buffer>
-    mapBufferToValue?: (b: Buffer) => Promise<T>
-    beforeCreate?: (v: Partial<T>) => Partial<T>
+    mapValueToBuffer?(v: T): Promise<Buffer>
+    mapBufferToValue?(b: Buffer): Promise<T>
+    beforeCreate?(v: Partial<T>): Partial<T>
   }
 
   /**

package/src/pipeline/dbPipelineBackup.ts

@@ -158,9 +158,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
 
   fs.ensureDirSync(outputDirPath)
 
-  if (!tables) {
-    tables = await db.getTables()
-  }
+  tables ||= await db.getTables()
 
   console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n` + tables.join('\n'))
 

package/src/pipeline/dbPipelineCopy.ts

@@ -106,9 +106,7 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
 
   console.log(`>> ${dimWhite('dbPipelineCopy')} started...${sinceUpdatedStr}`)
 
-  if (!tables) {
-    tables = await dbInput.getTables()
-  }
+  tables ||= await dbInput.getTables()
 
   console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n` + tables.join('\n'))
 

package/src/query/dbQuery.ts

@@ -4,7 +4,6 @@ import {
   AsyncMapper,
   _truncate,
   Saved,
-  AnyObject,
   _objectAssign,
 } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
@@ -61,13 +60,13 @@ export const dbQueryFilterOperatorValues: DBQueryFilterOperator[] = [
   'array-contains-any',
 ]
 
-export interface DBQueryFilter<ROW extends ObjectWithId = AnyObjectWithId> {
+export interface DBQueryFilter<ROW extends Partial<ObjectWithId> = AnyObjectWithId> {
   name: keyof ROW
   op: DBQueryFilterOperator
   val: any
 }
 
-export interface DBQueryOrder<ROW extends ObjectWithId = AnyObjectWithId> {
+export interface DBQueryOrder<ROW extends Partial<ObjectWithId> = AnyObjectWithId> {
   name: keyof ROW
   descending?: boolean
 }
@@ -82,7 +81,7 @@ export interface DBQueryOrder<ROW extends ObjectWithId = AnyObjectWithId> {
  *
  * <DBM> is the type of **queried** object (so e.g `key of DBM` can be used), not **returned** object.
  */
-export class DBQuery<ROW extends ObjectWithId = AnyObjectWithId> {
+export class DBQuery<ROW extends Partial<ObjectWithId> = AnyObjectWithId> {
   constructor(public table: string) {}
 
   /**
@@ -92,7 +91,7 @@ export class DBQuery<ROW extends ObjectWithId = AnyObjectWithId> {
     return new DBQuery(table)
   }
 
-  static fromPlainObject<ROW extends ObjectWithId = AnyObjectWithId>(
+  static fromPlainObject<ROW extends Partial<ObjectWithId> = AnyObjectWithId>(
     obj: Partial<DBQuery<ROW>> & { table: string },
   ): DBQuery<ROW> {
     return Object.assign(new DBQuery<ROW>(obj.table), obj)
@@ -237,14 +236,13 @@ export class DBQuery<ROW extends ObjectWithId = AnyObjectWithId> {
  */
 export class RunnableDBQuery<
   BM extends Partial<ObjectWithId<ID>>,
-  DBM extends ObjectWithId<ID> = Saved<BM>,
-  TM extends AnyObject = BM,
-  ID extends string | number = string,
+  DBM extends Partial<ObjectWithId<ID>> = BM,
+  ID extends string | number = NonNullable<BM['id']>,
 > extends DBQuery<DBM> {
   /**
    * Pass `table` to override table.
    */
-  constructor(public dao: CommonDao<BM, DBM, TM, ID>, table?: string) {
+  constructor(public dao: CommonDao<BM, DBM, ID>, table?: string) {
     super(table || dao.cfg.table)
   }
 
@@ -260,10 +258,6 @@ export class RunnableDBQuery<
     return await this.dao.runQueryAsDBM(this, opt)
   }
 
-  async runQueryAsTM(opt?: CommonDaoOptions): Promise<TM[]> {
-    return await this.dao.runQueryAsTM(this, opt)
-  }
-
   async runQueryExtended(opt?: CommonDaoOptions): Promise<RunQueryResult<Saved<BM>>> {
     return await this.dao.runQueryExtended(this, opt)
   }
@@ -272,10 +266,6 @@ export class RunnableDBQuery<
     return await this.dao.runQueryExtendedAsDBM(this, opt)
   }
 
-  async runQueryExtendedAsTM(opt?: CommonDaoOptions): Promise<RunQueryResult<TM>> {
-    return await this.dao.runQueryExtendedAsTM(this, opt)
-  }
-
   async runQueryCount(opt?: CommonDaoOptions): Promise<number> {
     return await this.dao.runQueryCount(this, opt)
   }

package/src/testing/daoTest.ts

@@ -8,7 +8,6 @@ import {
   createTestItemsBM,
   testItemBMSchema,
   testItemDBMSchema,
-  testItemTMSchema,
   TEST_TABLE,
   createTestItemBM,
   testItemDBMJsonSchema,
@@ -25,7 +24,6 @@ export function runCommonDaoTest(
     db,
     dbmSchema: testItemDBMSchema,
     bmSchema: testItemBMSchema,
-    tmSchema: testItemTMSchema,
     logStarted: true,
     logLevel: CommonDaoLogLevel.DATA_FULL,
   })