@naturalcycles/db-lib 8.23.0 → 8.24.3

package/dist/adapter/cachedb/cache.db.d.ts

@@ -1,7 +1,6 @@
 /// <reference types="node" />
 import { Readable } from 'stream';
 import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
-import { IDebugger } from '@naturalcycles/nodejs-lib';
 import { BaseCommonDB } from '../../base.common.db';
 import { CommonDB } from '../../common.db';
 import { RunQueryResult } from '../../db.model';
@@ -14,9 +13,8 @@ import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBStreamOptions
  * Queries always hit downstream (unless `onlyCache` is passed)
  */
 export declare class CacheDB extends BaseCommonDB implements CommonDB {
-    cfg: CacheDBCfg;
     constructor(cfg: CacheDBCfg);
-    log: IDebugger;
+    cfg: CacheDBCfg;
     ping(): Promise<void>;
     /**
      * Resets InMemory DB data

package/dist/adapter/cachedb/cache.db.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CacheDB = void 0;
 const stream_1 = require("stream");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const base_common_db_1 = require("../../base.common.db");
 /**
  * CommonDB implementation that proxies requests to downstream CommonDB
@@ -13,8 +12,10 @@ const base_common_db_1 = require("../../base.common.db");
 class CacheDB extends base_common_db_1.BaseCommonDB {
     constructor(cfg) {
         super();
-        this.cfg = cfg;
-        this.log = (0, nodejs_lib_1.Debug)(`nc:db-lib:${cfg.name}`);
+        this.cfg = {
+            logger: console,
+            ...cfg,
+        };
     }
     async ping() {
         await Promise.all([this.cfg.cacheDB.ping(), this.cfg.downstreamDB.ping()]);
@@ -49,7 +50,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             results.forEach(r => (resultMap[r.id] = r));
             missingIds.push(...ids.filter(id => !resultMap[id]));
             if (this.cfg.logCached) {
-                this.log(`${table}.getByIds ${results.length} rows from cache: [${results
+                this.cfg.logger?.log(`${table}.getByIds ${results.length} rows from cache: [${results
                     .map(r => r.id)
                     .join(', ')}]`);
             }
@@ -58,7 +59,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             const results = await this.cfg.downstreamDB.getByIds(table, missingIds, opt);
             results.forEach(r => (resultMap[r.id] = r));
             if (this.cfg.logDownstream) {
-                this.log(`${table}.getByIds ${results.length} rows from downstream: [${results
+                this.cfg.logger?.log(`${table}.getByIds ${results.length} rows from downstream: [${results
                     .map(r => r.id)
                     .join(', ')}]`);
             }
@@ -76,13 +77,13 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             deletedIds = await this.cfg.downstreamDB.deleteByIds(table, ids, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${table}.deleteByIds ${deletedIds} rows from downstream`);
+                this.cfg.logger?.log(`${table}.deleteByIds ${deletedIds} rows from downstream`);
             }
         }
         if (!opt.skipCache && !this.cfg.skipCache) {
             const cacheResult = this.cfg.cacheDB.deleteByIds(table, ids, opt).then(deletedFromCache => {
                 if (this.cfg.logCached) {
-                    this.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`);
+                    this.cfg.logger?.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`);
                 }
             });
             if (this.cfg.awaitCache)
@@ -94,7 +95,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             await this.cfg.downstreamDB.saveBatch(table, rows, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${table}.saveBatch ${rows.length} rows to downstream: [${rows
+                this.cfg.logger?.log(`${table}.saveBatch ${rows.length} rows to downstream: [${rows
                     .map(r => r.id)
                     .join(', ')}]`);
             }
@@ -102,7 +103,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.skipCache && !this.cfg.skipCache) {
             const cacheResult = this.cfg.cacheDB.saveBatch(table, rows, opt).then(() => {
                 if (this.cfg.logCached) {
-                    this.log(`${table}.saveBatch ${rows.length} rows to cache: [${rows.map(r => r.id).join(', ')}]`);
+                    this.cfg.logger?.log(`${table}.saveBatch ${rows.length} rows to cache: [${rows.map(r => r.id).join(', ')}]`);
                 }
             });
             if (this.cfg.awaitCache)
@@ -113,7 +114,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             const { rows, ...queryResult } = await this.cfg.downstreamDB.runQuery(q, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${q.table}.runQuery ${rows.length} rows from downstream`);
+                this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from downstream`);
             }
             // Don't save to cache if it was a projection query
             if (!opt.skipCache && !opt.skipCache && !q._selectedFieldNames) {
@@ -127,7 +128,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             return { rows: [] };
         const { rows, ...queryResult } = await this.cfg.cacheDB.runQuery(q, opt);
         if (this.cfg.logCached) {
-            this.log(`${q.table}.runQuery ${rows.length} rows from cache`);
+            this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from cache`);
         }
         return { rows, ...queryResult };
     }
@@ -137,7 +138,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         }
         const count = await this.cfg.cacheDB.runQueryCount(q, opt);
         if (this.cfg.logCached) {
-            this.log(`${q.table}.runQueryCount ${count} rows from cache`);
+            this.cfg.logger?.log(`${q.table}.runQueryCount ${count} rows from cache`);
         }
         return count;
     }
@@ -175,7 +176,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
             const deletedIds = await this.cfg.downstreamDB.deleteByQuery(q, opt);
             if (this.cfg.logDownstream) {
-                this.log(`${q.table}.deleteByQuery ${deletedIds} rows from downstream and cache`);
+                this.cfg.logger?.log(`${q.table}.deleteByQuery ${deletedIds} rows from downstream and cache`);
             }
             if (!opt.skipCache && !this.cfg.skipCache) {
                 const cacheResult = this.cfg.cacheDB.deleteByQuery(q, opt);
@@ -188,7 +189,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
             return 0;
         const deletedIds = await this.cfg.cacheDB.deleteByQuery(q, opt);
         if (this.cfg.logCached) {
-            this.log(`${q.table}.deleteByQuery ${deletedIds} rows from cache`);
+            this.cfg.logger?.log(`${q.table}.deleteByQuery ${deletedIds} rows from cache`);
         }
         return deletedIds;
     }

package/dist/adapter/cachedb/cache.db.model.d.ts

@@ -1,4 +1,4 @@
-import { ObjectWithId } from '@naturalcycles/js-lib';
+import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
 import { CommonDB } from '../../common.db';
 import { CommonDBCreateOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model';
 export interface CacheDBCfg {
@@ -32,6 +32,10 @@ export interface CacheDBCfg {
      * @default false
      */
     logDownstream?: boolean;
+    /**
+     * Defaults to `console`.
+     */
+    logger?: CommonLogger;
 }
 export interface CacheDBOptions<ROW extends ObjectWithId> extends CommonDBSaveOptions<ROW> {
     /**

package/dist/adapter/file/file.db.js

@@ -5,7 +5,6 @@ const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const __1 = require("../..");
-const log = (0, nodejs_lib_1.Debug)('nc:db-lib:filedb');
 /**
  * Provides barebone implementation for "whole file" based CommonDB.
  * "whole file" means that the persistence layer doesn't allow any querying,
@@ -22,6 +21,7 @@ class FileDB extends __1.BaseCommonDB {
         this.cfg = {
             sortObjects: true,
             logFinished: true,
+            logger: console,
             ...cfg,
         };
     }
@@ -183,14 +183,14 @@ class FileDB extends __1.BaseCommonDB {
     }
     logStarted(op) {
         if (this.cfg.logStarted) {
-            log(`>> ${op}`);
+            this.cfg.logger?.log(`>> ${op}`);
         }
         return Date.now();
     }
     logFinished(started, op) {
         if (!this.cfg.logFinished)
             return;
-        log(`<< ${op} ${(0, colors_1.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
+        this.cfg.logger?.log(`<< ${op} ${(0, colors_1.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
     }
 }
 exports.FileDB = FileDB;

package/dist/adapter/file/file.db.model.d.ts

@@ -1,4 +1,4 @@
-import { ObjectWithId } from '@naturalcycles/js-lib';
+import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
 import { DBSaveBatchOperation } from '../../db.model';
 import type { DBQueryOrder } from '../../query/dbQuery';
 export interface FileDBPersistencePlugin {
@@ -18,6 +18,10 @@ export interface FileDBCfg {
      * If true - will run `sortObjectDeep()` on each object to achieve deterministic sort
      */
     sortObjects?: boolean;
+    /**
+     * Defaults to `console`.
+     */
+    logger?: CommonLogger;
     /**
      * @default false
      */

package/dist/adapter/inmemory/inMemory.db.d.ts

@@ -1,4 +1,4 @@
-import { JsonSchemaObject, StringMap, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
+import { JsonSchemaObject, StringMap, JsonSchemaRootObject, ObjectWithId, CommonLogger } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonDB, DBTransaction } from '../..';
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, RunQueryResult } from '../../db.model';
@@ -30,6 +30,10 @@ export interface InMemoryDBCfg {
      * @default true
      */
     persistZip: boolean;
+    /**
+     * Defaults to `console`.
+     */
+    logger?: CommonLogger;
 }
 export declare class InMemoryDB implements CommonDB {
     constructor(cfg?: Partial<InMemoryDBCfg>);

package/dist/adapter/inmemory/inMemory.db.js

@@ -8,7 +8,6 @@ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const fs = require("fs-extra");
 const __1 = require("../..");
-const log = (0, nodejs_lib_1.Debug)('nc:db-lib:inmemorydb');
 class InMemoryDB {
     constructor(cfg) {
         // data[table][id] > {id: 'a', created: ... }
@@ -19,6 +18,7 @@ class InMemoryDB {
             persistenceEnabled: false,
             persistZip: true,
             persistentStoragePath: './tmp/inmemorydb',
+            logger: console,
             ...cfg,
         };
     }
@@ -36,7 +36,7 @@ class InMemoryDB {
     async resetCache(_table) {
         if (_table) {
             const table = this.cfg.tablesPrefix + _table;
-            log(`reset ${table}`);
+            this.cfg.logger?.log(`reset ${table}`);
             this.data[table] = {};
         }
         else {
@@ -44,7 +44,7 @@ class InMemoryDB {
             (await this.getTables()).forEach(table => {
                 this.data[table] = {};
             });
-            log('reset');
+            this.cfg.logger?.log('reset');
         }
     }
     async getTables() {
@@ -76,7 +76,7 @@ class InMemoryDB {
         this.data[table] = this.data[table] || {};
         rows.forEach(r => {
             if (!r.id) {
-                log.warn({ rows });
+                this.cfg.logger?.warn({ rows });
                 throw new Error(`InMemoryDB: id doesn't exist for row`);
             }
             // JSON parse/stringify (deep clone) is to:
@@ -160,7 +160,7 @@ class InMemoryDB {
                 fs.createWriteStream(fname),
             ]);
         });
-        log(`flushToDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, colors_1.yellow)(tables)} tables`);
+        this.cfg.logger?.log(`flushToDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, colors_1.yellow)(tables)} tables`);
     }
     /**
      * Restores all tables (all namespaces) at once.
@@ -189,7 +189,7 @@ class InMemoryDB {
             ]);
             this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
         });
-        log(`restoreFromDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, colors_1.yellow)(files.length)} tables`);
+        this.cfg.logger?.log(`restoreFromDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, colors_1.yellow)(files.length)} tables`);
     }
 }
 exports.InMemoryDB = InMemoryDB;

package/dist/commondao/common.dao.d.ts

@@ -1,4 +1,4 @@
-import { AsyncMapper, JsonSchemaObject, JsonSchemaRootObject, Saved, ObjectWithId } from '@naturalcycles/js-lib';
+import { AsyncMapper, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId, Saved } from '@naturalcycles/js-lib';
 import { AjvSchema, ObjectSchemaTyped, ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DBModelType, RunQueryResult } from '../db.model';
 import { DBQuery, RunnableDBQuery } from '../query/dbQuery';
@@ -137,7 +137,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends Obj
      *
      * Does NOT mutate the object.
      */
-    validateAndConvert<IN, OUT = IN>(obj: IN, schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>, modelType?: DBModelType, opt?: CommonDaoOptions): OUT;
+    validateAndConvert<IN, OUT = IN>(obj: Partial<IN>, schema: ObjectSchemaTyped<IN> | AjvSchema<IN> | undefined, modelType: DBModelType, opt?: CommonDaoOptions): OUT;
     getTableSchema(): Promise<JsonSchemaRootObject<DBM>>;
     createTable(schema: JsonSchemaObject<DBM>, opt?: CommonDaoCreateOptions): Promise<void>;
     /**

package/dist/commondao/common.dao.js

@@ -7,8 +7,9 @@ const cnst_1 = require("../cnst");
 const db_model_1 = require("../db.model");
 const dbQuery_1 = require("../query/dbQuery");
 const common_dao_model_1 = require("./common.dao.model");
-const log = (0, nodejs_lib_1.Debug)('nc:db-lib:commondao');
 /* eslint-disable no-dupe-class-members */
+const isGAE = !!process.env['GAE_INSTANCE'];
+const isCI = !!process.env['CI'];
 /**
  * Lowest common denominator API between supported Databases.
  *
@@ -20,8 +21,12 @@ class CommonDao {
     constructor(cfg) {
         this.cfg = cfg;
         this.cfg = {
-            logLevel: common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
+            // Default is to NOT log in AppEngine and in CI,
+            // otherwise to log Operations
+            // e.g in Dev (local machine), Test - it will log operations (useful for debugging)
+            logLevel: isGAE || isCI ? common_dao_model_1.CommonDaoLogLevel.NONE : common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
             createdUpdated: true,
+            logger: console,
             ...cfg,
             hooks: {
                 createId: () => (0, nodejs_lib_1.stringId)(),
@@ -234,7 +239,7 @@ class CommonDao {
         const started = this.logStarted(op, q.table);
         const count = await this.cfg.db.runQueryCount(q, opt);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
         }
         return count;
     }
@@ -269,7 +274,7 @@ class CommonDao {
             (0, nodejs_lib_1.writableVoid)(),
         ]);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
         }
     }
     async streamQueryAsDBMForEach(q, mapper, opt = {}) {
@@ -301,7 +306,7 @@ class CommonDao {
             (0, nodejs_lib_1.writableVoid)(),
         ]);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${(0, js_lib_1._since)(started)}`);
         }
     }
     /**
@@ -384,7 +389,7 @@ class CommonDao {
             (0, nodejs_lib_1.writableVoid)(),
         ]);
         if (this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.OPERATIONS) {
-            log(`<< ${q.table}.${op}: ${count} id(s) in ${(0, js_lib_1._since)(started)}`);
+            this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} id(s) in ${(0, js_lib_1._since)(started)}`);
         }
     }
     assignIdCreatedUpdated(obj, opt = {}) {
@@ -732,8 +737,6 @@ class CommonDao {
     async ping() {
         await this.cfg.db.ping();
     }
-    // todo: logging
-    // todo: bmToDBM, etc. How?
     // transaction(): DBTransaction {
     //   return this.cfg.db.transaction()
     // }
@@ -745,7 +748,7 @@ class CommonDao {
         if (Array.isArray(res)) {
             logRes = `${res.length} row(s)`;
             if (res.length && this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.DATA_FULL) {
-                args.push('\n', res.slice(0, 10)); // max 10 items
+                args.push('\n', ...res.slice(0, 10)); // max 10 items
             }
         }
         else if (res) {
@@ -757,16 +760,16 @@ class CommonDao {
         else {
             logRes = `undefined`;
         }
-        log(...[`<< ${table}.${op}: ${logRes} in ${(0, js_lib_1._since)(started)}`].concat(args));
+        this.cfg.logger?.log(`<< ${table}.${op}: ${logRes} in ${(0, js_lib_1._since)(started)}`, ...args);
     }
     logSaveResult(started, op, table) {
         if (!this.cfg.logLevel)
             return;
-        log(`<< ${table}.${op} in ${(0, js_lib_1._since)(started)}`);
+        this.cfg.logger?.log(`<< ${table}.${op} in ${(0, js_lib_1._since)(started)}`);
     }
     logStarted(op, table, force = false) {
         if (this.cfg.logStarted || force) {
-            log(`>> ${table}.${op}`);
+            this.cfg.logger?.log(`>> ${table}.${op}`);
         }
         return Date.now();
     }
@@ -786,7 +789,7 @@ class CommonDao {
                     args.push(items);
                 }
             }
-            log(...args);
+            this.cfg.logger?.log(...args);
         }
         return Date.now();
     }

package/dist/commondao/common.dao.model.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode, ObjectWithId } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, ObjectWithId, Saved } from '@naturalcycles/js-lib';
 import { AjvSchema, AjvValidationError, JoiValidationError, ObjectSchemaTyped, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model';
@@ -34,11 +34,20 @@ export declare enum CommonDaoLogLevel {
      * Same as undefined
      */
     NONE = 0,
+    /**
+     * Log operations (e.g "getById returned 1 row"), but not data
+     */
     OPERATIONS = 10,
+    /**
+     * Log operations and data for single operations (e.g getById), but not batch operations.
+     */
     DATA_SINGLE = 20,
+    /**
+     * Log EVERYTHING - all data passing in and out (max 10 rows). Very verbose!
+     */
     DATA_FULL = 30
 }
-export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
+export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM = BM> {
     db: CommonDB;
     table: string;
     /**
@@ -53,6 +62,10 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends Obje
      * Set to true to limit DB writing (will throw an error is such case).
      */
     readOnly?: boolean;
+    /**
+     * Defaults to `console`
+     */
+    logger?: CommonLogger;
     /**
      * @default OPERATIONS
      */

package/dist/commondao/common.dao.model.js

@@ -7,7 +7,16 @@ var CommonDaoLogLevel;
      * Same as undefined
      */
     CommonDaoLogLevel[CommonDaoLogLevel["NONE"] = 0] = "NONE";
+    /**
+     * Log operations (e.g "getById returned 1 row"), but not data
+     */
     CommonDaoLogLevel[CommonDaoLogLevel["OPERATIONS"] = 10] = "OPERATIONS";
+    /**
+     * Log operations and data for single operations (e.g getById), but not batch operations.
+     */
     CommonDaoLogLevel[CommonDaoLogLevel["DATA_SINGLE"] = 20] = "DATA_SINGLE";
+    /**
+     * Log EVERYTHING - all data passing in and out (max 10 rows). Very verbose!
+     */
     CommonDaoLogLevel[CommonDaoLogLevel["DATA_FULL"] = 30] = "DATA_FULL";
 })(CommonDaoLogLevel = exports.CommonDaoLogLevel || (exports.CommonDaoLogLevel = {}));

package/dist/pipeline/dbPipelineBackup.js

@@ -8,7 +8,6 @@ const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const time_lib_1 = require("@naturalcycles/time-lib");
 const fs = require("fs-extra");
 const index_1 = require("../index");
-// const log = Debug('nc:db-lib:pipeline')
 /**
  * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
  * File is overwritten (by default).

package/dist/pipeline/dbPipelineRestore.js

@@ -78,7 +78,7 @@ async function dbPipelineRestore(opt) {
                 ...opt,
                 metric: table,
             }),
-            (0, nodejs_lib_1.transformLimit)(limit),
+            (0, nodejs_lib_1.transformLimit)({ limit }),
             ...(sinceUpdated
                 ? [(0, nodejs_lib_1.transformFilterSync)(r => r.updated >= sinceUpdated)]
                 : []),

package/package.json

@@ -42,7 +42,7 @@
   "engines": {
     "node": ">=14.15"
   },
-  "version": "8.23.0",
+  "version": "8.24.3",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/readme.md

@@ -295,11 +295,6 @@ Passing empty array will actually return an array of empty objects (documented e
 .select(['id']) //=> [ { id: 'id1' }, { id: 'id2' }, ... ]
 ```
 
-# DEBUG namespaces
-
-- `nc:db-lib:commondao`
-- `nc:db-lib:inmemorydb`
-
 # Exports
 
 - `/` root
@@ -315,6 +310,6 @@ Passing empty array will actually return an array of empty objects (documented e
 # Packaging
 
 - `engines.node >= LTS`
-- `main: dist/index.js`: commonjs, es2019
+- `main: dist/index.js`: commonjs, es2020
 - `types: dist/index.d.ts`: typescript types
 - `/src` folder with source `*.ts` files included

package/src/adapter/cachedb/cache.db.model.ts

@@ -1,4 +1,4 @@
-import { ObjectWithId } from '@naturalcycles/js-lib'
+import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib'
 import { CommonDB } from '../../common.db'
 import { CommonDBCreateOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model'
 
@@ -38,6 +38,11 @@ export interface CacheDBCfg {
    * @default false
    */
   logDownstream?: boolean
+
+  /**
+   * Defaults to `console`.
+   */
+  logger?: CommonLogger
 }
 
 export interface CacheDBOptions<ROW extends ObjectWithId> extends CommonDBSaveOptions<ROW> {

package/src/adapter/cachedb/cache.db.ts

@@ -5,7 +5,6 @@ import {
   ObjectWithId,
   StringMap,
 } from '@naturalcycles/js-lib'
-import { Debug, IDebugger } from '@naturalcycles/nodejs-lib'
 import { BaseCommonDB } from '../../base.common.db'
 import { CommonDB } from '../../common.db'
 import { RunQueryResult } from '../../db.model'
@@ -24,12 +23,15 @@ import {
  * Queries always hit downstream (unless `onlyCache` is passed)
  */
 export class CacheDB extends BaseCommonDB implements CommonDB {
-  constructor(public cfg: CacheDBCfg) {
+  constructor(cfg: CacheDBCfg) {
     super()
-    this.log = Debug(`nc:db-lib:${cfg.name}`)
+    this.cfg = {
+      logger: console,
+      ...cfg,
+    }
   }
 
-  log!: IDebugger
+  cfg: CacheDBCfg
 
   override async ping(): Promise<void> {
     await Promise.all([this.cfg.cacheDB.ping(), this.cfg.downstreamDB.ping()])
@@ -84,7 +86,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
       missingIds.push(...ids.filter(id => !resultMap[id]))
 
       if (this.cfg.logCached) {
-        this.log(
+        this.cfg.logger?.log(
           `${table}.getByIds ${results.length} rows from cache: [${results
             .map(r => r.id)
             .join(', ')}]`,
@@ -97,7 +99,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
       results.forEach(r => (resultMap[r.id] = r))
 
       if (this.cfg.logDownstream) {
-        this.log(
+        this.cfg.logger?.log(
           `${table}.getByIds ${results.length} rows from downstream: [${results
             .map(r => r.id)
             .join(', ')}]`,
@@ -125,14 +127,14 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
       deletedIds = await this.cfg.downstreamDB.deleteByIds(table, ids, opt)
 
       if (this.cfg.logDownstream) {
-        this.log(`${table}.deleteByIds ${deletedIds} rows from downstream`)
+        this.cfg.logger?.log(`${table}.deleteByIds ${deletedIds} rows from downstream`)
       }
     }
 
     if (!opt.skipCache && !this.cfg.skipCache) {
       const cacheResult = this.cfg.cacheDB.deleteByIds(table, ids, opt).then(deletedFromCache => {
         if (this.cfg.logCached) {
-          this.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`)
+          this.cfg.logger?.log(`${table}.deleteByIds ${deletedFromCache} rows from cache`)
         }
       })
       if (this.cfg.awaitCache) await cacheResult
@@ -150,7 +152,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
       await this.cfg.downstreamDB.saveBatch(table, rows, opt)
 
       if (this.cfg.logDownstream) {
-        this.log(
+        this.cfg.logger?.log(
           `${table}.saveBatch ${rows.length} rows to downstream: [${rows
             .map(r => r.id)
             .join(', ')}]`,
@@ -161,7 +163,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
     if (!opt.skipCache && !this.cfg.skipCache) {
       const cacheResult = this.cfg.cacheDB.saveBatch(table, rows, opt).then(() => {
         if (this.cfg.logCached) {
-          this.log(
+          this.cfg.logger?.log(
             `${table}.saveBatch ${rows.length} rows to cache: [${rows.map(r => r.id).join(', ')}]`,
           )
         }
@@ -178,7 +180,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
       const { rows, ...queryResult } = await this.cfg.downstreamDB.runQuery(q, opt)
 
       if (this.cfg.logDownstream) {
-        this.log(`${q.table}.runQuery ${rows.length} rows from downstream`)
+        this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from downstream`)
       }
 
       // Don't save to cache if it was a projection query
@@ -194,7 +196,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
     const { rows, ...queryResult } = await this.cfg.cacheDB.runQuery(q, opt)
 
     if (this.cfg.logCached) {
-      this.log(`${q.table}.runQuery ${rows.length} rows from cache`)
+      this.cfg.logger?.log(`${q.table}.runQuery ${rows.length} rows from cache`)
     }
 
     return { rows, ...queryResult }
@@ -211,7 +213,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
     const count = await this.cfg.cacheDB.runQueryCount(q, opt)
 
     if (this.cfg.logCached) {
-      this.log(`${q.table}.runQueryCount ${count} rows from cache`)
+      this.cfg.logger?.log(`${q.table}.runQueryCount ${count} rows from cache`)
     }
 
     return count
@@ -264,7 +266,9 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
       const deletedIds = await this.cfg.downstreamDB.deleteByQuery(q, opt)
 
       if (this.cfg.logDownstream) {
-        this.log(`${q.table}.deleteByQuery ${deletedIds} rows from downstream and cache`)
+        this.cfg.logger?.log(
+          `${q.table}.deleteByQuery ${deletedIds} rows from downstream and cache`,
+        )
       }
 
       if (!opt.skipCache && !this.cfg.skipCache) {
@@ -280,7 +284,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
     const deletedIds = await this.cfg.cacheDB.deleteByQuery(q, opt)
 
     if (this.cfg.logCached) {
-      this.log(`${q.table}.deleteByQuery ${deletedIds} rows from cache`)
+      this.cfg.logger?.log(`${q.table}.deleteByQuery ${deletedIds} rows from cache`)
     }
 
     return deletedIds

package/src/adapter/file/file.db.model.ts

@@ -1,4 +1,4 @@
-import { ObjectWithId } from '@naturalcycles/js-lib'
+import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib'
 import { DBSaveBatchOperation } from '../../db.model'
 import type { DBQueryOrder } from '../../query/dbQuery'
 
@@ -23,6 +23,11 @@ export interface FileDBCfg {
    */
   sortObjects?: boolean
 
+  /**
+   * Defaults to `console`.
+   */
+  logger?: CommonLogger
+
   /**
    * @default false
    */

package/src/adapter/file/file.db.ts

@@ -14,7 +14,7 @@ import {
   ObjectWithId,
   AnyObjectWithId,
 } from '@naturalcycles/js-lib'
-import { Debug, readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { dimGrey } from '@naturalcycles/nodejs-lib/dist/colors'
 import { BaseCommonDB, DBSaveBatchOperation, queryInMemory } from '../..'
 import { CommonDB } from '../../common.db'
@@ -28,8 +28,6 @@ import { DBQuery } from '../../query/dbQuery'
 import { DBTransaction } from '../../transaction/dbTransaction'
 import { FileDBCfg } from './file.db.model'
 
-const log = Debug('nc:db-lib:filedb')
-
 /**
  * Provides barebone implementation for "whole file" based CommonDB.
  * "whole file" means that the persistence layer doesn't allow any querying,
@@ -46,6 +44,7 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     this.cfg = {
       sortObjects: true,
       logFinished: true,
+      logger: console,
       ...cfg,
     }
   }
@@ -271,13 +270,13 @@ export class FileDB extends BaseCommonDB implements CommonDB {
 
   private logStarted(op: string): number {
     if (this.cfg.logStarted) {
-      log(`>> ${op}`)
+      this.cfg.logger?.log(`>> ${op}`)
     }
     return Date.now()
   }
 
   private logFinished(started: number, op: string): void {
     if (!this.cfg.logFinished) return
-    log(`<< ${op} ${dimGrey(`in ${_since(started)}`)}`)
+    this.cfg.logger?.log(`<< ${op} ${dimGrey(`in ${_since(started)}`)}`)
   }
 }

package/src/adapter/inmemory/inMemory.db.ts

@@ -11,10 +11,10 @@ import {
   JsonSchemaRootObject,
   ObjectWithId,
   _stringMapValues,
+  CommonLogger,
 } from '@naturalcycles/js-lib'
 import {
   bufferReviver,
-  Debug,
   ReadableTyped,
   transformJsonParse,
   transformSplit,
@@ -63,9 +63,12 @@ export interface InMemoryDBCfg {
    * @default true
    */
   persistZip: boolean
-}
 
-const log = Debug('nc:db-lib:inmemorydb')
+  /**
+   * Defaults to `console`.
+   */
+  logger?: CommonLogger
+}
 
 export class InMemoryDB implements CommonDB {
   constructor(cfg?: Partial<InMemoryDBCfg>) {
@@ -75,6 +78,7 @@ export class InMemoryDB implements CommonDB {
       persistenceEnabled: false,
       persistZip: true,
       persistentStoragePath: './tmp/inmemorydb',
+      logger: console,
       ...cfg,
     }
   }
@@ -100,13 +104,13 @@ export class InMemoryDB implements CommonDB {
   async resetCache(_table?: string): Promise<void> {
     if (_table) {
       const table = this.cfg.tablesPrefix + _table
-      log(`reset ${table}`)
+      this.cfg.logger?.log(`reset ${table}`)
       this.data[table] = {}
     } else {
       ;(await this.getTables()).forEach(table => {
         this.data[table] = {}
       })
-      log('reset')
+      this.cfg.logger?.log('reset')
     }
   }
 
@@ -157,7 +161,7 @@ export class InMemoryDB implements CommonDB {
 
     rows.forEach(r => {
       if (!r.id) {
-        log.warn({ rows })
+        this.cfg.logger?.warn({ rows })
         throw new Error(`InMemoryDB: id doesn't exist for row`)
       }
 
@@ -266,7 +270,9 @@ export class InMemoryDB implements CommonDB {
       ])
     })
 
-    log(`flushToDisk took ${dimGrey(_since(started))} to save ${yellow(tables)} tables`)
+    this.cfg.logger?.log(
+      `flushToDisk took ${dimGrey(_since(started))} to save ${yellow(tables)} tables`,
+    )
   }
 
   /**
@@ -306,6 +312,8 @@ export class InMemoryDB implements CommonDB {
       this.data[table] = _by(rows, r => r.id)
     })
 
-    log(`restoreFromDisk took ${dimGrey(_since(started))} to read ${yellow(files.length)} tables`)
+    this.cfg.logger?.log(
+      `restoreFromDisk took ${dimGrey(_since(started))} to read ${yellow(files.length)} tables`,
+    )
   }
 }

package/src/commondao/common.dao.model.ts

@@ -1,4 +1,4 @@
-import { ErrorMode, ObjectWithId } from '@naturalcycles/js-lib'
+import { CommonLogger, ErrorMode, ObjectWithId, Saved } from '@naturalcycles/js-lib'
 import {
   AjvSchema,
   AjvValidationError,
@@ -46,12 +46,25 @@ export enum CommonDaoLogLevel {
    * Same as undefined
    */
   NONE = 0,
+  /**
+   * Log operations (e.g "getById returned 1 row"), but not data
+   */
   OPERATIONS = 10,
+  /**
+   * Log operations and data for single operations (e.g getById), but not batch operations.
+   */
   DATA_SINGLE = 20,
+  /**
+   * Log EVERYTHING - all data passing in and out (max 10 rows). Very verbose!
+   */
   DATA_FULL = 30,
 }
 
-export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
+export interface CommonDaoCfg<
+  BM extends Partial<ObjectWithId>,
+  DBM extends ObjectWithId = Saved<BM>,
+  TM = BM,
+> {
   db: CommonDB
   table: string
 
@@ -70,6 +83,11 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends Obje
    */
   readOnly?: boolean
 
+  /**
+   * Defaults to `console`
+   */
+  logger?: CommonLogger
+
   /**
    * @default OPERATIONS
    */

package/src/commondao/common.dao.ts

@@ -1,36 +1,35 @@
 import {
+  _filterNullishValues,
+  _filterUndefinedValues,
+  _passthroughPredicate,
+  _since,
+  _truncate,
+  _uniqBy,
   AppError,
   AsyncMapper,
   ErrorMode,
   JsonSchemaObject,
-  _passthroughPredicate,
-  _since,
-  _truncate,
-  pMap,
   JsonSchemaRootObject,
-  Saved,
-  _uniqBy,
   ObjectWithId,
-  _filterUndefinedValues,
-  _filterNullishValues,
+  pMap,
+  Saved,
 } from '@naturalcycles/js-lib'
 import {
+  _pipeline,
   AjvSchema,
   AjvValidationError,
-  Debug,
   getValidationResult,
   JoiValidationError,
   ObjectSchemaTyped,
   ReadableTyped,
   stringId,
+  transformBuffer,
   transformLogProgress,
   transformMap,
   transformMapSimple,
   transformMapSync,
   transformTap,
   writableVoid,
-  _pipeline,
-  transformBuffer,
 } from '@naturalcycles/nodejs-lib'
 import { DBLibError } from '../cnst'
 import { DBModelType, RunQueryResult } from '../db.model'
@@ -45,10 +44,11 @@ import {
   CommonDaoStreamOptions,
 } from './common.dao.model'
 
-const log = Debug('nc:db-lib:commondao')
-
 /* eslint-disable no-dupe-class-members */
 
+const isGAE = !!process.env['GAE_INSTANCE']
+const isCI = !!process.env['CI']
+
 /**
  * Lowest common denominator API between supported Databases.
  *
@@ -63,8 +63,12 @@ export class CommonDao<
 > {
   constructor(public cfg: CommonDaoCfg<BM, DBM, TM>) {
     this.cfg = {
-      logLevel: CommonDaoLogLevel.OPERATIONS,
+      // Default is to NOT log in AppEngine and in CI,
+      // otherwise to log Operations
+      // e.g in Dev (local machine), Test - it will log operations (useful for debugging)
+      logLevel: isGAE || isCI ? CommonDaoLogLevel.NONE : CommonDaoLogLevel.OPERATIONS,
       createdUpdated: true,
+      logger: console,
       ...cfg,
       hooks: {
         createId: () => stringId(),
@@ -323,7 +327,7 @@ export class CommonDao<
     const started = this.logStarted(op, q.table)
     const count = await this.cfg.db.runQueryCount(q, opt)
     if (this.cfg.logLevel! >= CommonDaoLogLevel.OPERATIONS) {
-      log(`<< ${q.table}.${op}: ${count} row(s) in ${_since(started)}`)
+      this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${_since(started)}`)
     }
     return count
   }
@@ -369,7 +373,7 @@ export class CommonDao<
     ])
 
     if (this.cfg.logLevel! >= CommonDaoLogLevel.OPERATIONS) {
-      log(`<< ${q.table}.${op}: ${count} row(s) in ${_since(started)}`)
+      this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${_since(started)}`)
     }
   }
 
@@ -412,7 +416,7 @@ export class CommonDao<
     ])
 
     if (this.cfg.logLevel! >= CommonDaoLogLevel.OPERATIONS) {
-      log(`<< ${q.table}.${op}: ${count} row(s) in ${_since(started)}`)
+      this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} row(s) in ${_since(started)}`)
     }
   }
 
@@ -518,7 +522,7 @@ export class CommonDao<
     ])
 
     if (this.cfg.logLevel! >= CommonDaoLogLevel.OPERATIONS) {
-      log(`<< ${q.table}.${op}: ${count} id(s) in ${_since(started)}`)
+      this.cfg.logger?.log(`<< ${q.table}.${op}: ${count} id(s) in ${_since(started)}`)
     }
   }
 
@@ -899,9 +903,9 @@ export class CommonDao<
    * Does NOT mutate the object.
    */
   validateAndConvert<IN, OUT = IN>(
-    obj: IN,
-    schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>,
-    modelType?: DBModelType,
+    obj: Partial<IN>,
+    schema: ObjectSchemaTyped<IN> | AjvSchema<IN> | undefined,
+    modelType: DBModelType,
     opt: CommonDaoOptions = {},
   ): OUT {
     // `raw` option completely bypasses any processing
@@ -924,12 +928,12 @@ export class CommonDao<
 
     // Pre-validation hooks
     if (modelType === DBModelType.DBM) {
-      obj = this.cfg.hooks!.beforeDBMValidate!(obj as any) as any
+      obj = this.cfg.hooks!.beforeDBMValidate!(obj as any) as IN
     }
 
     // Return as is if no schema is passed or if `skipConversion` is set
     if (!schema || opt.skipConversion) {
-      return obj as any
+      return obj as OUT
     }
 
     // This will Convert and Validate
@@ -943,12 +947,12 @@ export class CommonDao<
       // Ajv schema
       convertedValue = obj // because Ajv mutates original object
 
-      error = schema.getValidationError(obj, {
+      error = schema.getValidationError(obj as IN, {
         objectName,
       })
     } else {
       // Joi
-      const vr = getValidationResult<IN, OUT>(obj, schema, objectName)
+      const vr = getValidationResult<IN, OUT>(obj as IN, schema, objectName)
       error = vr.error
       convertedValue = vr.value
     }
@@ -979,8 +983,6 @@ export class CommonDao<
     await this.cfg.db.ping()
   }
 
-  // todo: logging
-  // todo: bmToDBM, etc. How?
   // transaction(): DBTransaction {
   //   return this.cfg.db.transaction()
   // }
@@ -994,7 +996,7 @@ export class CommonDao<
     if (Array.isArray(res)) {
       logRes = `${res.length} row(s)`
       if (res.length && this.cfg.logLevel >= CommonDaoLogLevel.DATA_FULL) {
-        args.push('\n', res.slice(0, 10)) // max 10 items
+        args.push('\n', ...res.slice(0, 10)) // max 10 items
       }
     } else if (res) {
       logRes = `1 row`
@@ -1005,17 +1007,17 @@ export class CommonDao<
       logRes = `undefined`
     }
 
-    log(...[`<< ${table}.${op}: ${logRes} in ${_since(started)}`].concat(args))
+    this.cfg.logger?.log(`<< ${table}.${op}: ${logRes} in ${_since(started)}`, ...args)
   }
 
   protected logSaveResult(started: number, op: string, table: string): void {
     if (!this.cfg.logLevel) return
-    log(`<< ${table}.${op} in ${_since(started)}`)
+    this.cfg.logger?.log(`<< ${table}.${op} in ${_since(started)}`)
   }
 
   protected logStarted(op: string, table: string, force = false): number {
     if (this.cfg.logStarted || force) {
-      log(`>> ${table}.${op}`)
+      this.cfg.logger?.log(`>> ${table}.${op}`)
     }
     return Date.now()
   }
@@ -1035,7 +1037,7 @@ export class CommonDao<
         }
       }
 
-      log(...args)
+      this.cfg.logger?.log(...args)
     }
 
     return Date.now()

package/src/pipeline/dbPipelineBackup.ts

@@ -115,8 +115,6 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
   sortObjects?: boolean
 }
 
-// const log = Debug('nc:db-lib:pipeline')
-
 /**
  * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
  * File is overwritten (by default).

package/src/pipeline/dbPipelineRestore.ts

@@ -209,7 +209,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
           ...opt,
           metric: table,
         }),
-        transformLimit(limit),
+        transformLimit({ limit }),
         ...(sinceUpdated
           ? [transformFilterSync<SavedDBEntity>(r => r.updated >= sinceUpdated)]
           : []),