@naturalcycles/db-lib 8.21.3 → 8.24.1

package/dist/adapter/cachedb/cache.db.d.ts

@@ -1,10 +1,9 @@
 /// <reference types="node" />
 import { Readable } from 'stream';
-import { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib';
-import { IDebugger } from '@naturalcycles/nodejs-lib';
+import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
 import { BaseCommonDB } from '../../base.common.db';
 import { CommonDB } from '../../common.db';
-import { ObjectWithId, RunQueryResult } from '../../db.model';
+import { RunQueryResult } from '../../db.model';
 import { DBQuery } from '../../query/dbQuery';
 import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBStreamOptions } from './cache.db.model';
 /**
@@ -14,9 +13,8 @@ import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBStreamOptions
  * Queries always hit downstream (unless `onlyCache` is passed)
  */
 export declare class CacheDB extends BaseCommonDB implements CommonDB {
-    cfg: CacheDBCfg;
     constructor(cfg: CacheDBCfg);
-    log: IDebugger;
+    cfg: CacheDBCfg;
     ping(): Promise<void>;
     /**
      * Resets InMemory DB data

package/dist/adapter/cachedb/cache.db.js

@@ -2,8 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CacheDB = void 0;
 const stream_1 = require("stream");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const base_common_db_1 = require("../../base.common.db");
+const isGAE = !!process.env['GAE_INSTANCE'];
 /**
  * CommonDB implementation that proxies requests to downstream CommonDB
  * and does in-memory caching.
@@ -13,8 +13,10 @@ const base_common_db_1 = require("../../base.common.db");
 class CacheDB extends base_common_db_1.BaseCommonDB {
     constructor(cfg) {
         super();
-        this.cfg = cfg;
-        this.log = (0, nodejs_lib_1.Debug)(`nc:db-lib:${cfg.name}`);
+        this.cfg = {
+            logger: isGAE ? undefined : console,
+            ...cfg,
+        };
     }
     async ping() {
         await Promise.all([this.cfg.cacheDB.ping(), this.cfg.downstreamDB.ping()]);
@@ -49,7 +51,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             results.forEach(r => (resultMap[r.id] = r));
             missingIds.push(...ids.filter(id => !resultMap[id]));
             if (this.cfg.logCached) {
-                this.log(`${table}.getByIds ${results.length} rows from cache: [${results
+                this.cfg.logger?.log(`${table}.getByIds ${results.length} rows from cache: [${results
                     .map(r => r.id)
                     .join(', ')}]`);
             }
@@ -58,7 +60,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             const results = await this.cfg.downstreamDB.getByIds(table, missingIds, opt);
             results.forEach(r => (resultMap[r.id] = r));
             if (this.cfg.logDownstream) {
-                this.log(`${table}.getByIds ${results.length} rows from downstream: [${results
+                this.cfg.logger?.log(`${table}.getByIds ${results.length} rows from downstream: [${results
                     .map(r => r.id)
                     .join(', ')}]`);
             }
@@ -76,13 +78,13 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             deletedIds = await this.cfg.downstreamDB.deleteByIds(table, ids, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${table}.deleteByIds ${deletedIds} rows from downstream`);
+                this.cfg.logger?.log(`${table}.deleteByIds ${deletedIds} rows from downstream`);
             }
         }
         if (!opt.skipCache && !this.cfg.skipCache) {
             const cacheResult = this.cfg.cacheDB.deleteByIds(table, ids, opt).then(deletedFromCache => {
                 if (this.cfg.logCached) {
-                    this.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`);
+                    this.cfg.logger?.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`);
                 }
             });
             if (this.cfg.awaitCache)
@@ -94,7 +96,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             await this.cfg.downstreamDB.saveBatch(table, rows, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${table}.saveBatch ${rows.length} rows to downstream: [${rows
+                this.cfg.logger?.log(`${table}.saveBatch ${rows.length} rows to downstream: [${rows
                     .map(r => r.id)
                     .join(', ')}]`);
             }
@@ -102,7 +104,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.skipCache && !this.cfg.skipCache) {
             const cacheResult = this.cfg.cacheDB.saveBatch(table, rows, opt).then(() => {
                 if (this.cfg.logCached) {
-                    this.log(`${table}.saveBatch ${rows.length} rows to cache: [${rows.map(r => r.id).join(', ')}]`);
+                    this.cfg.logger?.log(`${table}.saveBatch ${rows.length} rows to cache: [${rows.map(r => r.id).join(', ')}]`);
                 }
             });
             if (this.cfg.awaitCache)
@@ -113,7 +115,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             const { rows, ...queryResult } = await this.cfg.downstreamDB.runQuery(q, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${q.table}.runQuery ${rows.length} rows from downstream`);
+                this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from downstream`);
             }
             // Don't save to cache if it was a projection query
             if (!opt.skipCache && !opt.skipCache && !q._selectedFieldNames) {
@@ -127,7 +129,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             return { rows: [] };
         const { rows, ...queryResult } = await this.cfg.cacheDB.runQuery(q, opt);
         if (this.cfg.logCached) {
-            this.log(`${q.table}.runQuery ${rows.length} rows from cache`);
+            this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from cache`);
         }
         return { rows, ...queryResult };
     }
@@ -137,7 +139,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         }
         const count = await this.cfg.cacheDB.runQueryCount(q, opt);
         if (this.cfg.logCached) {
-            this.log(`${q.table}.runQueryCount ${count} rows from cache`);
+            this.cfg.logger?.log(`${q.table}.runQueryCount ${count} rows from cache`);
         }
         return count;
     }
@@ -175,7 +177,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             const deletedIds = await this.cfg.downstreamDB.deleteByQuery(q, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${q.table}.deleteByQuery ${deletedIds} rows from downstream and cache`);
+                this.cfg.logger?.log(`${q.table}.deleteByQuery ${deletedIds} rows from downstream and cache`);
             }
             if (!opt.skipCache && !this.cfg.skipCache) {
                 const cacheResult = this.cfg.cacheDB.deleteByQuery(q, opt);
@@ -188,7 +190,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             return 0;
         const deletedIds = await this.cfg.cacheDB.deleteByQuery(q, opt);
         if (this.cfg.logCached) {
-            this.log(`${q.table}.deleteByQuery ${deletedIds} rows from cache`);
+            this.cfg.logger?.log(`${q.table}.deleteByQuery ${deletedIds} rows from cache`);
         }
         return deletedIds;
     }

package/dist/adapter/cachedb/cache.db.model.d.ts

@@ -1,5 +1,6 @@
+import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
 import { CommonDB } from '../../common.db';
-import { CommonDBCreateOptions, CommonDBSaveOptions, CommonDBStreamOptions, ObjectWithId } from '../../db.model';
+import { CommonDBCreateOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model';
 export interface CacheDBCfg {
     name: string;
     cacheDB: CommonDB;
@@ -31,6 +32,13 @@ export interface CacheDBCfg {
      * @default false
      */
     logDownstream?: boolean;
+    /**
+     * Pass noopLogger (or undefined) to skip logging completely.
+     *
+     * Defaults to `console` in dev.
+     * Default to noop in AppEngine.
+     */
+    logger?: CommonLogger;
 }
 export interface CacheDBOptions<ROW extends ObjectWithId> extends CommonDBSaveOptions<ROW> {
     /**

package/dist/adapter/file/file.db.d.ts

@@ -1,6 +1,6 @@
-import { JsonSchemaRootObject } from '@naturalcycles/js-lib';
+import { JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { BaseCommonDB, DBSaveBatchOperation, ObjectWithId } from '../..';
+import { BaseCommonDB, DBSaveBatchOperation } from '../..';
 import { CommonDB } from '../../common.db';
 import { CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, RunQueryResult } from '../../db.model';
 import { DBQuery } from '../../query/dbQuery';

package/dist/adapter/file/file.db.js

@@ -5,7 +5,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const __1 = require("../..");
-const log = (0, nodejs_lib_1.Debug)('nc:db-lib:filedb');
+const isGAE = !!process.env['GAE_INSTANCE'];
 /**
  * Provides barebone implementation for "whole file" based CommonDB.
  * "whole file" means that the persistence layer doesn't allow any querying,
@@ -22,6 +22,7 @@ class FileDB extends __1.BaseCommonDB {
         this.cfg = {
             sortObjects: true,
             logFinished: true,
+            logger: isGAE ? undefined : console,
             ...cfg,
         };
     }
@@ -183,14 +184,14 @@ class FileDB extends __1.BaseCommonDB {
     }
     logStarted(op) {
         if (this.cfg.logStarted) {
-            log(`>> ${op}`);
+            this.cfg.logger?.log(`>> ${op}`);
         }
         return Date.now();
     }
     logFinished(started, op) {
         if (!this.cfg.logFinished)
             return;
-        log(`<< ${op} ${(0, colors_1.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
+        this.cfg.logger?.log(`<< ${op} ${(0, colors_1.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
     }
 }
 exports.FileDB = FileDB;

package/dist/adapter/file/file.db.model.d.ts

@@ -1,4 +1,5 @@
-import { DBSaveBatchOperation, ObjectWithId } from '../../db.model';
+import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
+import { DBSaveBatchOperation } from '../../db.model';
 import type { DBQueryOrder } from '../../query/dbQuery';
 export interface FileDBPersistencePlugin {
     ping(): Promise<void>;
@@ -17,6 +18,11 @@ export interface FileDBCfg {
      * If true - will run `sortObjectDeep()` on each object to achieve deterministic sort
      */
     sortObjects?: boolean;
+    /**
+     * Defaults to `console` in dev.
+     * Default to noop in AppEngine.
+     */
+    logger?: CommonLogger;
     /**
      * @default false
      */

package/dist/adapter/file/inMemory.persistence.plugin.d.ts

@@ -1,5 +1,5 @@
-import { StringMap } from '@naturalcycles/js-lib';
-import { DBSaveBatchOperation, ObjectWithId } from '../../db.model';
+import { StringMap, ObjectWithId } from '@naturalcycles/js-lib';
+import { DBSaveBatchOperation } from '../../db.model';
 import { FileDBPersistencePlugin } from './file.db.model';
 /**
  * Mostly useful for testing.

package/dist/adapter/file/localFile.persistence.plugin.d.ts

@@ -1,4 +1,5 @@
-import { DBSaveBatchOperation, ObjectWithId } from '../../db.model';
+import { ObjectWithId } from '@naturalcycles/js-lib';
+import { DBSaveBatchOperation } from '../../db.model';
 import { FileDBPersistencePlugin } from './file.db.model';
 export interface LocalFilePersistencePluginCfg {
     /**

package/dist/adapter/file/noop.persistence.plugin.d.ts

@@ -1,4 +1,5 @@
-import { DBSaveBatchOperation, ObjectWithId } from '../../db.model';
+import { ObjectWithId } from '@naturalcycles/js-lib';
+import { DBSaveBatchOperation } from '../../db.model';
 import { FileDBPersistencePlugin } from './file.db.model';
 export declare class NoopPersistencePlugin implements FileDBPersistencePlugin {
     ping(): Promise<void>;

package/dist/adapter/inmemory/inMemory.db.d.ts

@@ -1,6 +1,6 @@
-import { JsonSchemaObject, StringMap, JsonSchemaRootObject } from '@naturalcycles/js-lib';
+import { JsonSchemaObject, StringMap, JsonSchemaRootObject, ObjectWithId, CommonLogger } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonDB, DBTransaction, ObjectWithId } from '../..';
+import { CommonDB, DBTransaction } from '../..';
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, RunQueryResult } from '../../db.model';
 import { DBQuery } from '../../query/dbQuery';
 export interface InMemoryDBCfg {
@@ -30,6 +30,11 @@ export interface InMemoryDBCfg {
      * @default true
      */
     persistZip: boolean;
+    /**
+     * Defaults to `console` in dev.
+     * Default to noop in AppEngine.
+     */
+    logger?: CommonLogger;
 }
 export declare class InMemoryDB implements CommonDB {
     constructor(cfg?: Partial<InMemoryDBCfg>);

package/dist/adapter/inmemory/inMemory.db.js

@@ -8,7 +8,7 @@ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const fs = require("fs-extra");
 const __1 = require("../..");
-const log = (0, nodejs_lib_1.Debug)('nc:db-lib:inmemorydb');
+const isGAE = !!process.env['GAE_INSTANCE'];
 class InMemoryDB {
     constructor(cfg) {
         // data[table][id] > {id: 'a', created: ... }
@@ -19,6 +19,7 @@ class InMemoryDB {
             persistenceEnabled: false,
             persistZip: true,
             persistentStoragePath: './tmp/inmemorydb',
+            logger: isGAE ? undefined : console,
             ...cfg,
         };
     }
@@ -36,7 +37,7 @@ class InMemoryDB {
     async resetCache(_table) {
         if (_table) {
             const table = this.cfg.tablesPrefix + _table;
-            log(`reset ${table}`);
+            this.cfg.logger?.log(`reset ${table}`);
             this.data[table] = {};
         }
         else {
@@ -44,7 +45,7 @@ class InMemoryDB {
             (await this.getTables()).forEach(table => {
                 this.data[table] = {};
             });
-            log('reset');
+            this.cfg.logger?.log('reset');
         }
     }
     async getTables() {
@@ -53,7 +54,7 @@ class InMemoryDB {
     async getTableSchema(_table) {
         const table = this.cfg.tablesPrefix + _table;
         return {
-            ...(0, js_lib_1.generateJsonSchemaFromData)(Object.values(this.data[table] || {})),
+            ...(0, js_lib_1.generateJsonSchemaFromData)((0, js_lib_1._stringMapValues)(this.data[table] || {})),
             $id: `${table}.schema.json`,
         };
     }
@@ -76,7 +77,7 @@ class InMemoryDB {
         this.data[table] = this.data[table] || {};
         rows.forEach(r => {
             if (!r.id) {
-                log.warn({ rows });
+                this.cfg.logger?.warn({ rows });
                 throw new Error(`InMemoryDB: id doesn't exist for row`);
             }
             // JSON parse/stringify (deep clone) is to:
@@ -160,7 +161,7 @@ class InMemoryDB {
                 fs.createWriteStream(fname),
             ]);
         });
-        log(`flushToDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, colors_1.yellow)(tables)} tables`);
+        this.cfg.logger?.log(`flushToDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, colors_1.yellow)(tables)} tables`);
     }
     /**
      * Restores all tables (all namespaces) at once.
@@ -189,7 +190,7 @@ class InMemoryDB {
             ]);
             this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
         });
-        log(`restoreFromDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, colors_1.yellow)(files.length)} tables`);
+        this.cfg.logger?.log(`restoreFromDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, colors_1.yellow)(files.length)} tables`);
     }
 }
 exports.InMemoryDB = InMemoryDB;

package/dist/adapter/inmemory/queryInMemory.d.ts

@@ -1,3 +1,3 @@
-import { ObjectWithId } from '../../db.model';
+import { ObjectWithId } from '@naturalcycles/js-lib';
 import { DBQuery } from '../../query/dbQuery';
 export declare function queryInMemory<ROW extends ObjectWithId>(q: DBQuery<ROW>, rows?: ROW[]): ROW[];

package/dist/base.common.db.d.ts

@@ -1,7 +1,7 @@
-import { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib';
+import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from './common.db';
-import { CommonDBOptions, CommonDBSaveOptions, ObjectWithId, RunQueryResult } from './db.model';
+import { CommonDBOptions, CommonDBSaveOptions, RunQueryResult } from './db.model';
 import { DBQuery } from './query/dbQuery';
 import { DBTransaction } from './transaction/dbTransaction';
 /**

package/dist/common.db.d.ts

@@ -1,6 +1,6 @@
-import { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib';
+import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, ObjectWithId, RunQueryResult } from './db.model';
+import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, RunQueryResult } from './db.model';
 import { DBQuery } from './query/dbQuery';
 import { DBTransaction } from './transaction/dbTransaction';
 export interface CommonDB {

package/dist/commondao/common.dao.d.ts

@@ -1,6 +1,6 @@
-import { AsyncMapper, JsonSchemaObject, JsonSchemaRootObject, Saved } from '@naturalcycles/js-lib';
+import { AsyncMapper, JsonSchemaObject, JsonSchemaRootObject, Saved, ObjectWithId } from '@naturalcycles/js-lib';
 import { AjvSchema, ObjectSchemaTyped, ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { DBModelType, ObjectWithId, RunQueryResult } from '../db.model';
+import { DBModelType, RunQueryResult } from '../db.model';
 import { DBQuery, RunnableDBQuery } from '../query/dbQuery';
 import { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions } from './common.dao.model';
 /**
@@ -104,7 +104,14 @@ export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends Obj
     deleteById(id: undefined, opt?: CommonDaoOptions): Promise<0>;
     deleteById(id?: string, opt?: CommonDaoOptions): Promise<number>;
     deleteByIds(ids: string[], opt?: CommonDaoOptions): Promise<number>;
-    deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<number>;
+    /**
+     * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
+     * `deleteByIds` for each batch concurrently (infinite concurrency).
+     * This is expected to be more memory-efficient way of deleting big numbers of rows.
+     */
+    deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamForEachOptions<DBM> & {
+        stream?: boolean;
+    }): Promise<number>;
     dbmToBM(_dbm: undefined, opt?: CommonDaoOptions): Promise<undefined>;
     dbmToBM(_dbm?: DBM, opt?: CommonDaoOptions): Promise<Saved<BM>>;
     dbmsToBM(dbms: DBM[], opt?: CommonDaoOptions): Promise<Saved<BM>[]>;

package/dist/commondao/common.dao.js

@@ -7,8 +7,8 @@ const cnst_1 = require("../cnst");
 const db_model_1 = require("../db.model");
 const dbQuery_1 = require("../query/dbQuery");
 const common_dao_model_1 = require("./common.dao.model");
-const log = (0, nodejs_lib_1.Debug)('nc:db-lib:commondao');
 /* eslint-disable no-dupe-class-members */
+const isGAE = !!process.env['GAE_INSTANCE'];
 /**
  * Lowest common denominator API between supported Databases.
  *
@@ -22,6 +22,7 @@ class CommonDao {
         this.cfg = {
             logLevel: common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
             createdUpdated: true,
+            logger: isGAE ? undefined : console,
             ...cfg,
             hooks: {
                 createId: () => (0, nodejs_lib_1.stringId)(),
@@ -234,7 +235,7 @@ class CommonDao {
         const started = this.logStarted(op, q.table);
         const count = await this.cfg.db.runQueryCount(q, opt);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
         }
         return count;
     }
@@ -269,7 +270,7 @@ class CommonDao {
             (0, nodejs_lib_1.writableVoid)(),
         ]);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
         }
     }
     async streamQueryAsDBMForEach(q, mapper, opt = {}) {
@@ -301,7 +302,7 @@ class CommonDao {
             (0, nodejs_lib_1.writableVoid)(),
         ]);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
         }
     }
     /**
@@ -384,7 +385,7 @@ class CommonDao {
             (0, nodejs_lib_1.writableVoid)(),
         ]);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} id(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} id(s) in ${(0, js_lib_1._since)(started)}`);
         }
     }
     assignIdCreatedUpdated(obj, opt = {}) {
@@ -536,14 +537,45 @@ class CommonDao {
         this.logSaveResult(started, op, table);
         return deletedIds;
     }
+    /**
+     * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
+     * `deleteByIds` for each batch concurrently (infinite concurrency).
+     * This is expected to be more memory-efficient way of deleting big numbers of rows.
+     */
     async deleteByQuery(q, opt = {}) {
         this.requireWriteAccess();
         q.table = opt.table || q.table;
         const op = `deleteByQuery(${q.pretty()})`;
         const started = this.logStarted(op, q.table);
-        const ids = await this.cfg.db.deleteByQuery(q, opt);
+        let deleted = 0;
+        if (opt.stream) {
+            const batchSize = 500;
+            await (0, nodejs_lib_1._pipeline)([
+                this.cfg.db.streamQuery(q.select(['id']), opt),
+                (0, nodejs_lib_1.transformMapSimple)(objectWithId => objectWithId.id, {
+                    errorMode: js_lib_1.ErrorMode.SUPPRESS,
+                }),
+                (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+                (0, nodejs_lib_1.transformMap)(async (ids) => {
+                    deleted += await this.cfg.db.deleteByIds(q.table, ids, opt);
+                }, {
+                    predicate: js_lib_1._passthroughPredicate,
+                }),
+                // LogProgress should be AFTER the mapper, to be able to report correct stats
+                (0, nodejs_lib_1.transformLogProgress)({
+                    metric: q.table,
+                    logEvery: 2,
+                    batchSize,
+                    ...opt,
+                }),
+                (0, nodejs_lib_1.writableVoid)(),
+            ]);
+        }
+        else {
+            deleted = await this.cfg.db.deleteByQuery(q, opt);
+        }
         this.logSaveResult(started, op, q.table);
-        return ids;
+        return deleted;
     }
     async dbmToBM(_dbm, opt = {}) {
         if (!_dbm)
@@ -648,7 +680,12 @@ class CommonDao {
         // obj = _filterNullishValues(obj as any)
         // We still filter `undefined` values here, because `beforeDBMToBM` can return undefined values
         // and they can be annoying with snapshot tests
-        obj = (0, js_lib_1._filterUndefinedValues)(obj);
+        if (this.cfg.filterNullishValues) {
+            obj = (0, js_lib_1._filterNullishValues)(obj);
+        }
+        else {
+            obj = (0, js_lib_1._filterUndefinedValues)(obj);
+        }
         // Pre-validation hooks
         if (modelType === db_model_1.DBModelType.DBM) {
             obj = this.cfg.hooks.beforeDBMValidate(obj);
@@ -721,16 +758,16 @@ class CommonDao {
         else {
             logRes = `undefined`;
         }
-        log(...[`<< ${table}.${op}: ${logRes} in ${(0, js_lib_1._since)(started)}`].concat(args));
+        this.cfg.logger?.log(...[`<< ${table}.${op}: ${logRes} in ${(0, js_lib_1._since)(started)}`].concat(args));
     }
     logSaveResult(started, op, table) {
         if (!this.cfg.logLevel)
             return;
-        log(`<< ${table}.${op} in ${(0, js_lib_1._since)(started)}`);
+        this.cfg.logger?.log(`<< ${table}.${op} in ${(0, js_lib_1._since)(started)}`);
     }
     logStarted(op, table, force = false) {
         if (this.cfg.logStarted || force) {
-            log(`>> ${table}.${op}`);
+            this.cfg.logger?.log(`>> ${table}.${op}`);
         }
         return Date.now();
     }
@@ -750,7 +787,7 @@ class CommonDao {
                     args.push(items);
                 }
             }
-            log(...args);
+            this.cfg.logger?.log(...args);
         }
         return Date.now();
     }

package/dist/commondao/common.dao.model.d.ts

@@ -1,7 +1,7 @@
-import { ErrorMode } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, ObjectWithId } from '@naturalcycles/js-lib';
 import { AjvSchema, AjvValidationError, JoiValidationError, ObjectSchemaTyped, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
-import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, ObjectWithId } from '../db.model';
+import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model';
 export declare type CommonDaoCreateIdHook<BM, DBM> = (obj: DBM | BM) => string;
 export declare type CommonDaoParseNaturalIdHook<DBM> = (id: string) => Partial<DBM>;
 export declare type CommonDaoBeforeCreateHook<BM> = (bm: Partial<BM>) => BM;
@@ -53,6 +53,13 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends Obje
      * Set to true to limit DB writing (will throw an error is such case).
      */
     readOnly?: boolean;
+    /**
+     * Pass undefined (or noopLogger) to disable logging completely.
+     *
+     * Defaults to `console` in dev.
+     * Default to noop in AppEngine.
+     */
+    logger?: CommonLogger;
     /**
      * @default OPERATIONS
      */
@@ -67,6 +74,16 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends Obje
      * Set to false to disable created/updated fields management.
      */
     createdUpdated?: boolean;
+    /**
+     * Default is false.
+     * If true - will run `_filterNullishValues` inside `validateAndConvert` function
+     * (instead of `_filterUndefinedValues`).
+     * This was the old db-lib behavior.
+     * This option allows to keep backwards-compatible behavior.
+     *
+     * @deprecated
+     */
+    filterNullishValues?: boolean;
 }
 /**
  * All properties default to undefined.

package/dist/db.model.d.ts

@@ -1,4 +1,4 @@
-import { AnyObject } from '@naturalcycles/js-lib';
+import { AnyObjectWithId, ObjectWithId } from '@naturalcycles/js-lib';
 import { CommonDB } from './common.db';
 export interface CommonDBOptions {
 }
@@ -40,18 +40,6 @@ export declare enum DBModelType {
     BM = "BM",
     TM = "TM"
 }
-export interface CreatedUpdated {
-    created: number;
-    updated: number;
-}
-export interface CreatedUpdatedId extends CreatedUpdated {
-    id: string;
-}
-export interface ObjectWithId {
-    id: string;
-}
-export interface AnyObjectWithId extends AnyObject, ObjectWithId {
-}
 /**
  * Interface for a module (lib) that implements CommonDB.
  *

package/dist/index.d.ts

@@ -6,16 +6,16 @@ import { DBLibError } from './cnst';
 import { CommonDB } from './common.db';
 import { CommonDao } from './commondao/common.dao';
 import { CommonDaoAnonymizeHook, CommonDaoBeforeBMToDBMHook, CommonDaoBeforeBMToTMHook, CommonDaoBeforeCreateHook, CommonDaoBeforeDBMToBMHook, CommonDaoBeforeDBMValidateHook, CommonDaoBeforeTMToBMHook, CommonDaoCfg, CommonDaoCreateIdHook, CommonDaoCreateOptions, CommonDaoLogLevel, CommonDaoOptions, CommonDaoParseNaturalIdHook, CommonDaoSaveOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions } from './commondao/common.dao.model';
-import { AnyObjectWithId, CommonDBAdapter, CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, CreatedUpdated, CreatedUpdatedId, DBDeleteByIdsOperation, DBModelType, DBOperation, DBRelation, DBSaveBatchOperation, ObjectWithId, RunQueryResult } from './db.model';
+import { CommonDBAdapter, CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, DBDeleteByIdsOperation, DBModelType, DBOperation, DBRelation, DBSaveBatchOperation, RunQueryResult } from './db.model';
 import { getDB } from './getDB';
 import { CommonKeyValueDao, CommonKeyValueDaoCfg } from './kv/commonKeyValueDao';
 import { CommonKeyValueDB, KeyValueDBTuple } from './kv/commonKeyValueDB';
-import { createdUpdatedFields, createdUpdatedIdFields, deserializeJsonField, idField, serializeJsonField } from './model.util';
+import { createdUpdatedFields, createdUpdatedIdFields, deserializeJsonField, serializeJsonField } from './model.util';
 import { dbPipelineBackup, DBPipelineBackupOptions } from './pipeline/dbPipelineBackup';
 import { dbPipelineCopy, DBPipelineCopyOptions } from './pipeline/dbPipelineCopy';
 import { dbPipelineRestore, DBPipelineRestoreOptions } from './pipeline/dbPipelineRestore';
 import { DBQuery, DBQueryFilter, DBQueryFilterOperator, dbQueryFilterOperatorValues, DBQueryOrder, RunnableDBQuery } from './query/dbQuery';
 import { DBTransaction, RunnableDBTransaction } from './transaction/dbTransaction';
 import { commitDBTransactionSimple, mergeDBOperations } from './transaction/dbTransaction.util';
-export type { DBQueryFilterOperator, DBQueryFilter, DBQueryOrder, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, CommonDBCreateOptions, CommonDB, RunQueryResult, CreatedUpdated, CreatedUpdatedId, AnyObjectWithId, ObjectWithId, CommonDaoCfg, CommonDaoCreateIdHook, CommonDaoParseNaturalIdHook, CommonDaoBeforeCreateHook, CommonDaoBeforeDBMValidateHook, CommonDaoBeforeDBMToBMHook, CommonDaoBeforeBMToDBMHook, CommonDaoBeforeTMToBMHook, CommonDaoBeforeBMToTMHook, CommonDaoAnonymizeHook, InMemoryDBCfg, InMemoryKeyValueDBCfg, DBPipelineBackupOptions, DBPipelineRestoreOptions, DBPipelineCopyOptions, CommonDBAdapter, DBOperation, DBSaveBatchOperation, DBDeleteByIdsOperation, CommonKeyValueDB, CommonKeyValueDaoCfg, KeyValueDBTuple, };
-export { DBQuery, dbQueryFilterOperatorValues, RunnableDBQuery, CommonDaoLogLevel, DBRelation, DBModelType, CommonDao, createdUpdatedFields, createdUpdatedIdFields, idField, InMemoryDB, InMemoryKeyValueDB, queryInMemory, serializeJsonField, deserializeJsonField, dbPipelineBackup, dbPipelineRestore, dbPipelineCopy, getDB, DBLibError, BaseCommonDB, DBTransaction, RunnableDBTransaction, mergeDBOperations, commitDBTransactionSimple, CommonKeyValueDao, };
+export type { DBQueryFilterOperator, DBQueryFilter, DBQueryOrder, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, CommonDBCreateOptions, CommonDB, RunQueryResult, CommonDaoCfg, CommonDaoCreateIdHook, CommonDaoParseNaturalIdHook, CommonDaoBeforeCreateHook, CommonDaoBeforeDBMValidateHook, CommonDaoBeforeDBMToBMHook, CommonDaoBeforeBMToDBMHook, CommonDaoBeforeTMToBMHook, CommonDaoBeforeBMToTMHook, CommonDaoAnonymizeHook, InMemoryDBCfg, InMemoryKeyValueDBCfg, DBPipelineBackupOptions, DBPipelineRestoreOptions, DBPipelineCopyOptions, CommonDBAdapter, DBOperation, DBSaveBatchOperation, DBDeleteByIdsOperation, CommonKeyValueDB, CommonKeyValueDaoCfg, KeyValueDBTuple, };
+export { DBQuery, dbQueryFilterOperatorValues, RunnableDBQuery, CommonDaoLogLevel, DBRelation, DBModelType, CommonDao, createdUpdatedFields, createdUpdatedIdFields, InMemoryDB, InMemoryKeyValueDB, queryInMemory, serializeJsonField, deserializeJsonField, dbPipelineBackup, dbPipelineRestore, dbPipelineCopy, getDB, DBLibError, BaseCommonDB, DBTransaction, RunnableDBTransaction, mergeDBOperations, commitDBTransactionSimple, CommonKeyValueDao, };