@naturalcycles/db-lib 8.24.1 → 8.25.0

package/dist/adapter/cachedb/cache.db.js

@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CacheDB = void 0;
 const stream_1 = require("stream");
 const base_common_db_1 = require("../../base.common.db");
-const isGAE = !!process.env['GAE_INSTANCE'];
 /**
  * CommonDB implementation that proxies requests to downstream CommonDB
  * and does in-memory caching.
@@ -14,7 +13,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
     constructor(cfg) {
         super();
         this.cfg = {
-            logger: isGAE ? undefined : console,
+            logger: console,
             ...cfg,
         };
     }

package/dist/adapter/cachedb/cache.db.model.d.ts

@@ -33,10 +33,7 @@ export interface CacheDBCfg {
      */
     logDownstream?: boolean;
     /**
-     * Pass noopLogger (or undefined) to skip logging completely.
-     *
-     * Defaults to `console` in dev.
-     * Default to noop in AppEngine.
+     * Defaults to `console`.
      */
     logger?: CommonLogger;
 }

package/dist/adapter/file/file.db.js

@@ -5,7 +5,6 @@ const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const __1 = require("../..");
-const isGAE = !!process.env['GAE_INSTANCE'];
 /**
  * Provides barebone implementation for "whole file" based CommonDB.
  * "whole file" means that the persistence layer doesn't allow any querying,
@@ -22,7 +21,7 @@ class FileDB extends __1.BaseCommonDB {
         this.cfg = {
             sortObjects: true,
             logFinished: true,
-            logger: isGAE ? undefined : console,
+            logger: console,
             ...cfg,
         };
     }

package/dist/adapter/file/file.db.model.d.ts

@@ -19,8 +19,7 @@ export interface FileDBCfg {
      */
     sortObjects?: boolean;
     /**
-     * Defaults to `console` in dev.
-     * Default to noop in AppEngine.
+     * Defaults to `console`.
      */
     logger?: CommonLogger;
     /**

package/dist/adapter/inmemory/inMemory.db.d.ts

@@ -31,8 +31,7 @@ export interface InMemoryDBCfg {
      */
     persistZip: boolean;
     /**
-     * Defaults to `console` in dev.
-     * Default to noop in AppEngine.
+     * Defaults to `console`.
      */
     logger?: CommonLogger;
 }

package/dist/adapter/inmemory/inMemory.db.js

@@ -8,7 +8,6 @@ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const fs = require("fs-extra");
 const __1 = require("../..");
-const isGAE = !!process.env['GAE_INSTANCE'];
 class InMemoryDB {
     constructor(cfg) {
         // data[table][id] > {id: 'a', created: ... }
@@ -19,7 +18,7 @@ class InMemoryDB {
             persistenceEnabled: false,
             persistZip: true,
             persistentStoragePath: './tmp/inmemorydb',
-            logger: isGAE ? undefined : console,
+            logger: console,
             ...cfg,
         };
     }

package/dist/commondao/common.dao.d.ts

@@ -1,4 +1,4 @@
-import { AsyncMapper, JsonSchemaObject, JsonSchemaRootObject, Saved, ObjectWithId } from '@naturalcycles/js-lib';
+import { AsyncMapper, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId, Saved } from '@naturalcycles/js-lib';
 import { AjvSchema, ObjectSchemaTyped, ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DBModelType, RunQueryResult } from '../db.model';
 import { DBQuery, RunnableDBQuery } from '../query/dbQuery';
@@ -137,7 +137,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends Obj
      *
      * Does NOT mutate the object.
      */
-    validateAndConvert<IN, OUT = IN>(obj: IN, schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>, modelType?: DBModelType, opt?: CommonDaoOptions): OUT;
+    validateAndConvert<IN, OUT = IN>(obj: Partial<IN>, schema: ObjectSchemaTyped<IN> | AjvSchema<IN> | undefined, modelType: DBModelType, opt?: CommonDaoOptions): OUT;
     getTableSchema(): Promise<JsonSchemaRootObject<DBM>>;
     createTable(schema: JsonSchemaObject<DBM>, opt?: CommonDaoCreateOptions): Promise<void>;
     /**

package/dist/commondao/common.dao.js

@@ -9,6 +9,7 @@ const dbQuery_1 = require("../query/dbQuery");
 const common_dao_model_1 = require("./common.dao.model");
 /* eslint-disable no-dupe-class-members */
 const isGAE = !!process.env['GAE_INSTANCE'];
+const isCI = !!process.env['CI'];
 /**
  * Lowest common denominator API between supported Databases.
  *
@@ -20,9 +21,12 @@ class CommonDao {
     constructor(cfg) {
         this.cfg = cfg;
         this.cfg = {
-            logLevel: common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
+            // Default is to NOT log in AppEngine and in CI,
+            // otherwise to log Operations
+            // e.g in Dev (local machine), Test - it will log operations (useful for debugging)
+            logLevel: isGAE || isCI ? common_dao_model_1.CommonDaoLogLevel.NONE : common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
             createdUpdated: true,
-            logger: isGAE ? undefined : console,
+            logger: console,
             ...cfg,
             hooks: {
                 createId: () => (0, nodejs_lib_1.stringId)(),
@@ -52,7 +56,17 @@ class CommonDao {
         const op = `getById(${id})`;
         const table = opt.table || this.cfg.table;
         const started = this.logStarted(op, table);
-        const [dbm] = await this.cfg.db.getByIds(table, [id]);
+        let dbm;
+        if (opt.timeout) {
+            // todo: possibly remove it after debugging is done
+            dbm = (await (0, js_lib_1.pTimeout)(this.cfg.db.getByIds(table, [id]), {
+                timeout: opt.timeout,
+                name: `getById(${table})`,
+            }))[0];
+        }
+        else {
+            dbm = (await this.cfg.db.getByIds(table, [id]))[0];
+        }
         const bm = opt.raw ? dbm : await this.dbmToBM(dbm, opt);
         this.logResult(started, op, bm, table);
         return bm || null;
@@ -733,8 +747,6 @@ class CommonDao {
     async ping() {
         await this.cfg.db.ping();
     }
-    // todo: logging
-    // todo: bmToDBM, etc. How?
     // transaction(): DBTransaction {
     //   return this.cfg.db.transaction()
     // }
@@ -746,7 +758,7 @@ class CommonDao {
         if (Array.isArray(res)) {
             logRes = `${res.length} row(s)`;
             if (res.length && this.cfg.logLevel >= common_dao_model_1.CommonDaoLogLevel.DATA_FULL) {
-                args.push('\n', res.slice(0, 10)); // max 10 items
+                args.push('\n', ...res.slice(0, 10)); // max 10 items
             }
         }
         else if (res) {
@@ -758,7 +770,7 @@ class CommonDao {
         else {
             logRes = `undefined`;
         }
-        this.cfg.logger?.log(...[`<< ${table}.${op}: ${logRes} in ${(0, js_lib_1._since)(started)}`].concat(args));
+        this.cfg.logger?.log(`<< ${table}.${op}: ${logRes} in ${(0, js_lib_1._since)(started)}`, ...args);
     }
     logSaveResult(started, op, table) {
         if (!this.cfg.logLevel)

package/dist/commondao/common.dao.model.d.ts

@@ -1,4 +1,4 @@
-import { CommonLogger, ErrorMode, ObjectWithId } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, ObjectWithId, Saved } from '@naturalcycles/js-lib';
 import { AjvSchema, AjvValidationError, JoiValidationError, ObjectSchemaTyped, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
 import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model';
@@ -34,11 +34,20 @@ export declare enum CommonDaoLogLevel {
      * Same as undefined
      */
     NONE = 0,
+    /**
+     * Log operations (e.g "getById returned 1 row"), but not data
+     */
     OPERATIONS = 10,
+    /**
+     * Log operations and data for single operations (e.g getById), but not batch operations.
+     */
     DATA_SINGLE = 20,
+    /**
+     * Log EVERYTHING - all data passing in and out (max 10 rows). Very verbose!
+     */
     DATA_FULL = 30
 }
-export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
+export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM = BM> {
     db: CommonDB;
     table: string;
     /**
@@ -54,10 +63,7 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends Obje
      */
     readOnly?: boolean;
     /**
-     * Pass undefined (or noopLogger) to disable logging completely.
-     *
-     * Defaults to `console` in dev.
-     * Default to noop in AppEngine.
+     * Defaults to `console`
      */
     logger?: CommonLogger;
     /**
@@ -129,6 +135,13 @@ export interface CommonDaoOptions extends CommonDBOptions {
      * Useful e.g in AirtableDB where you can have one Dao to control multiple tables.
      */
     table?: string;
+    /**
+     * If set - wraps the method in `pTimeout` with a timeout of given number of milliseconds.
+     * Currently, it is only used to debug an ongoing GCP infra issue.
+     *
+     * @experimental
+     */
+    timeout?: number;
 }
 /**
  * All properties default to undefined.

package/dist/commondao/common.dao.model.js

@@ -7,7 +7,16 @@ var CommonDaoLogLevel;
      * Same as undefined
      */
     CommonDaoLogLevel[CommonDaoLogLevel["NONE"] = 0] = "NONE";
+    /**
+     * Log operations (e.g "getById returned 1 row"), but not data
+     */
     CommonDaoLogLevel[CommonDaoLogLevel["OPERATIONS"] = 10] = "OPERATIONS";
+    /**
+     * Log operations and data for single operations (e.g getById), but not batch operations.
+     */
     CommonDaoLogLevel[CommonDaoLogLevel["DATA_SINGLE"] = 20] = "DATA_SINGLE";
+    /**
+     * Log EVERYTHING - all data passing in and out (max 10 rows). Very verbose!
+     */
     CommonDaoLogLevel[CommonDaoLogLevel["DATA_FULL"] = 30] = "DATA_FULL";
 })(CommonDaoLogLevel = exports.CommonDaoLogLevel || (exports.CommonDaoLogLevel = {}));

package/dist/pipeline/dbPipelineRestore.js

@@ -78,7 +78,7 @@ async function dbPipelineRestore(opt) {
                 ...opt,
                 metric: table,
             }),
-            (0, nodejs_lib_1.transformLimit)(limit),
+            (0, nodejs_lib_1.transformLimit)({ limit }),
             ...(sinceUpdated
                 ? [(0, nodejs_lib_1.transformFilterSync)(r => r.updated >= sinceUpdated)]
                 : []),

package/package.json

@@ -12,7 +12,7 @@
   "devDependencies": {
     "@naturalcycles/bench-lib": "^1.0.0",
     "@naturalcycles/dev-lib": "^12.0.1",
-    "@types/node": "^16.0.0",
+    "@types/node": "^17.0.0",
     "jest": "^27.0.3"
   },
   "files": [
@@ -42,7 +42,7 @@
   "engines": {
     "node": ">=14.15"
   },
-  "version": "8.24.1",
+  "version": "8.25.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/cachedb/cache.db.model.ts

@@ -40,10 +40,7 @@ export interface CacheDBCfg {
   logDownstream?: boolean
 
   /**
-   * Pass noopLogger (or undefined) to skip logging completely.
-   *
-   * Defaults to `console` in dev.
-   * Default to noop in AppEngine.
+   * Defaults to `console`.
    */
   logger?: CommonLogger
 }

package/src/adapter/cachedb/cache.db.ts

@@ -16,8 +16,6 @@ import {
   CacheDBStreamOptions,
 } from './cache.db.model'
 
-const isGAE = !!process.env['GAE_INSTANCE']
-
 /**
  * CommonDB implementation that proxies requests to downstream CommonDB
  * and does in-memory caching.
@@ -28,7 +26,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
   constructor(cfg: CacheDBCfg) {
     super()
     this.cfg = {
-      logger: isGAE ? undefined : console,
+      logger: console,
       ...cfg,
     }
   }

package/src/adapter/file/file.db.model.ts

@@ -24,8 +24,7 @@ export interface FileDBCfg {
   sortObjects?: boolean
 
   /**
-   * Defaults to `console` in dev.
-   * Default to noop in AppEngine.
+   * Defaults to `console`.
    */
   logger?: CommonLogger
 

package/src/adapter/file/file.db.ts

@@ -28,8 +28,6 @@ import { DBQuery } from '../../query/dbQuery'
 import { DBTransaction } from '../../transaction/dbTransaction'
 import { FileDBCfg } from './file.db.model'
 
-const isGAE = !!process.env['GAE_INSTANCE']
-
 /**
  * Provides barebone implementation for "whole file" based CommonDB.
  * "whole file" means that the persistence layer doesn't allow any querying,
@@ -46,7 +44,7 @@ export class FileDB extends BaseCommonDB implements CommonDB {
     this.cfg = {
       sortObjects: true,
       logFinished: true,
-      logger: isGAE ? undefined : console,
+      logger: console,
       ...cfg,
     }
   }

package/src/adapter/inmemory/inMemory.db.ts

@@ -65,14 +65,11 @@ export interface InMemoryDBCfg {
   persistZip: boolean
 
   /**
-   * Defaults to `console` in dev.
-   * Default to noop in AppEngine.
+   * Defaults to `console`.
    */
   logger?: CommonLogger
 }
 
-const isGAE = !!process.env['GAE_INSTANCE']
-
 export class InMemoryDB implements CommonDB {
   constructor(cfg?: Partial<InMemoryDBCfg>) {
     this.cfg = {
@@ -81,7 +78,7 @@ export class InMemoryDB implements CommonDB {
       persistenceEnabled: false,
       persistZip: true,
       persistentStoragePath: './tmp/inmemorydb',
-      logger: isGAE ? undefined : console,
+      logger: console,
       ...cfg,
     }
   }

package/src/commondao/common.dao.model.ts

@@ -1,4 +1,4 @@
-import { CommonLogger, ErrorMode, ObjectWithId } from '@naturalcycles/js-lib'
+import { CommonLogger, ErrorMode, ObjectWithId, Saved } from '@naturalcycles/js-lib'
 import {
   AjvSchema,
   AjvValidationError,
@@ -46,12 +46,25 @@ export enum CommonDaoLogLevel {
    * Same as undefined
    */
   NONE = 0,
+  /**
+   * Log operations (e.g "getById returned 1 row"), but not data
+   */
   OPERATIONS = 10,
+  /**
+   * Log operations and data for single operations (e.g getById), but not batch operations.
+   */
   DATA_SINGLE = 20,
+  /**
+   * Log EVERYTHING - all data passing in and out (max 10 rows). Very verbose!
+   */
   DATA_FULL = 30,
 }
 
-export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
+export interface CommonDaoCfg<
+  BM extends Partial<ObjectWithId>,
+  DBM extends ObjectWithId = Saved<BM>,
+  TM = BM,
+> {
   db: CommonDB
   table: string
 
@@ -71,10 +84,7 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends Obje
   readOnly?: boolean
 
   /**
-   * Pass undefined (or noopLogger) to disable logging completely.
-   *
-   * Defaults to `console` in dev.
-   * Default to noop in AppEngine.
+   * Defaults to `console`
    */
   logger?: CommonLogger
 
@@ -158,6 +168,14 @@ export interface CommonDaoOptions extends CommonDBOptions {
    * Useful e.g in AirtableDB where you can have one Dao to control multiple tables.
    */
   table?: string
+
+  /**
+   * If set - wraps the method in `pTimeout` with a timeout of given number of milliseconds.
+   * Currently, it is only used to debug an ongoing GCP infra issue.
+   *
+   * @experimental
+   */
+  timeout?: number
 }
 
 /**

package/src/commondao/common.dao.ts

@@ -1,20 +1,22 @@
 import {
+  _filterNullishValues,
+  _filterUndefinedValues,
+  _passthroughPredicate,
+  _since,
+  _truncate,
+  _uniqBy,
   AppError,
   AsyncMapper,
   ErrorMode,
   JsonSchemaObject,
-  _passthroughPredicate,
-  _since,
-  _truncate,
-  pMap,
   JsonSchemaRootObject,
-  Saved,
-  _uniqBy,
   ObjectWithId,
-  _filterUndefinedValues,
-  _filterNullishValues,
+  pMap,
+  pTimeout,
+  Saved,
 } from '@naturalcycles/js-lib'
 import {
+  _pipeline,
   AjvSchema,
   AjvValidationError,
   getValidationResult,
@@ -22,14 +24,13 @@ import {
   ObjectSchemaTyped,
   ReadableTyped,
   stringId,
+  transformBuffer,
   transformLogProgress,
   transformMap,
   transformMapSimple,
   transformMapSync,
   transformTap,
   writableVoid,
-  _pipeline,
-  transformBuffer,
 } from '@naturalcycles/nodejs-lib'
 import { DBLibError } from '../cnst'
 import { DBModelType, RunQueryResult } from '../db.model'
@@ -47,6 +48,7 @@ import {
 /* eslint-disable no-dupe-class-members */
 
 const isGAE = !!process.env['GAE_INSTANCE']
+const isCI = !!process.env['CI']
 
 /**
  * Lowest common denominator API between supported Databases.
@@ -62,9 +64,12 @@ export class CommonDao<
 > {
   constructor(public cfg: CommonDaoCfg<BM, DBM, TM>) {
     this.cfg = {
-      logLevel: CommonDaoLogLevel.OPERATIONS,
+      // Default is to NOT log in AppEngine and in CI,
+      // otherwise to log Operations
+      // e.g in Dev (local machine), Test - it will log operations (useful for debugging)
+      logLevel: isGAE || isCI ? CommonDaoLogLevel.NONE : CommonDaoLogLevel.OPERATIONS,
       createdUpdated: true,
-      logger: isGAE ? undefined : console,
+      logger: console,
       ...cfg,
       hooks: {
         createId: () => stringId(),
@@ -99,7 +104,21 @@ export class CommonDao<
     const op = `getById(${id})`
     const table = opt.table || this.cfg.table
     const started = this.logStarted(op, table)
-    const [dbm] = await this.cfg.db.getByIds<DBM>(table, [id])
+
+    let dbm: DBM | undefined
+
+    if (opt.timeout) {
+      // todo: possibly remove it after debugging is done
+      dbm = (
+        await pTimeout(this.cfg.db.getByIds<DBM>(table, [id]), {
+          timeout: opt.timeout,
+          name: `getById(${table})`,
+        })
+      )[0]
+    } else {
+      dbm = (await this.cfg.db.getByIds<DBM>(table, [id]))[0]
+    }
+
     const bm = opt.raw ? (dbm as any) : await this.dbmToBM(dbm, opt)
     this.logResult(started, op, bm, table)
     return bm || null
@@ -899,9 +918,9 @@ export class CommonDao<
    * Does NOT mutate the object.
    */
   validateAndConvert<IN, OUT = IN>(
-    obj: IN,
-    schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>,
-    modelType?: DBModelType,
+    obj: Partial<IN>,
+    schema: ObjectSchemaTyped<IN> | AjvSchema<IN> | undefined,
+    modelType: DBModelType,
     opt: CommonDaoOptions = {},
   ): OUT {
     // `raw` option completely bypasses any processing
@@ -924,12 +943,12 @@ export class CommonDao<
 
     // Pre-validation hooks
     if (modelType === DBModelType.DBM) {
-      obj = this.cfg.hooks!.beforeDBMValidate!(obj as any) as any
+      obj = this.cfg.hooks!.beforeDBMValidate!(obj as any) as IN
     }
 
     // Return as is if no schema is passed or if `skipConversion` is set
     if (!schema || opt.skipConversion) {
-      return obj as any
+      return obj as OUT
     }
 
     // This will Convert and Validate
@@ -943,12 +962,12 @@ export class CommonDao<
       // Ajv schema
       convertedValue = obj // because Ajv mutates original object
 
-      error = schema.getValidationError(obj, {
+      error = schema.getValidationError(obj as IN, {
         objectName,
       })
     } else {
       // Joi
-      const vr = getValidationResult<IN, OUT>(obj, schema, objectName)
+      const vr = getValidationResult<IN, OUT>(obj as IN, schema, objectName)
       error = vr.error
       convertedValue = vr.value
     }
@@ -979,8 +998,6 @@ export class CommonDao<
     await this.cfg.db.ping()
   }
 
-  // todo: logging
-  // todo: bmToDBM, etc. How?
   // transaction(): DBTransaction {
   //   return this.cfg.db.transaction()
   // }
@@ -994,7 +1011,7 @@ export class CommonDao<
     if (Array.isArray(res)) {
       logRes = `${res.length} row(s)`
       if (res.length && this.cfg.logLevel >= CommonDaoLogLevel.DATA_FULL) {
-        args.push('\n', res.slice(0, 10)) // max 10 items
+        args.push('\n', ...res.slice(0, 10)) // max 10 items
       }
     } else if (res) {
       logRes = `1 row`
@@ -1005,7 +1022,7 @@ export class CommonDao<
       logRes = `undefined`
     }
 
-    this.cfg.logger?.log(...[`<< ${table}.${op}: ${logRes} in ${_since(started)}`].concat(args))
+    this.cfg.logger?.log(`<< ${table}.${op}: ${logRes} in ${_since(started)}`, ...args)
   }
 
   protected logSaveResult(started: number, op: string, table: string): void {

package/src/pipeline/dbPipelineRestore.ts

@@ -209,7 +209,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
           ...opt,
           metric: table,
         }),
-        transformLimit(limit),
+        transformLimit({ limit }),
         ...(sinceUpdated
           ? [transformFilterSync<SavedDBEntity>(r => r.updated >= sinceUpdated)]
           : []),