@naturalcycles/datastore-lib 3.16.5 → 3.18.0

package/dist/DatastoreStreamReadable.d.ts

@@ -1,10 +1,12 @@
 /// <reference types="node" />
 import { Readable } from 'stream';
 import { Query } from '@google-cloud/datastore';
+import { CommonLogger } from '@naturalcycles/js-lib';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import type { DatastoreDBStreamOptions } from './datastore.model';
 export declare class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
     private q;
+    private logger;
     private originalLimit;
     private rowsRetrieved;
     private endCursor?;
@@ -12,9 +14,8 @@ export declare class DatastoreStreamReadable<T = any> extends Readable implement
     private done;
     private lastQueryDone?;
     private totalWait;
-    private log;
     private opt;
-    constructor(q: Query, opt: DatastoreDBStreamOptions);
+    constructor(q: Query, opt: DatastoreDBStreamOptions, logger: CommonLogger);
     private runNextQuery;
     count: number;
     _read(): void;

package/dist/DatastoreStreamReadable.js

@@ -4,25 +4,22 @@ exports.DatastoreStreamReadable = void 0;
 const stream_1 = require("stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
 class DatastoreStreamReadable extends stream_1.Readable {
-    constructor(q, opt) {
+    constructor(q, opt, logger) {
         super({ objectMode: true });
         this.q = q;
+        this.logger = logger;
         this.rowsRetrieved = 0;
         this.running = false;
         this.done = false;
         this.totalWait = 0;
-        // private log = (...args: any[]): void => console.log(...args)
-        this.log = console.log.bind(console);
         this.count = 0; // use for debugging
         this.opt = {
             rssLimitMB: 1000,
             batchSize: 1000,
             ...opt,
         };
-        if (!opt.debug)
-            this.log = () => { };
         this.originalLimit = q.limitVal;
-        console.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`);
+        logger.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`);
     }
     async runNextQuery() {
         if (this.done)
@@ -45,7 +42,7 @@ class DatastoreStreamReadable extends stream_1.Readable {
         try {
             const [rows, info] = await q.run();
             this.rowsRetrieved += rows.length;
-            this.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
+            this.logger.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
             this.endCursor = info.endCursor;
             this.running = false; // ready to take more _reads
             this.lastQueryDone = Date.now();
@@ -53,7 +50,7 @@ class DatastoreStreamReadable extends stream_1.Readable {
             if (!info.endCursor ||
                 info.moreResults === 'NO_MORE_RESULTS' ||
                 (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-                this.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
+                this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
                 this.push(null);
                 this.done = true;
             }
@@ -63,7 +60,7 @@ class DatastoreStreamReadable extends stream_1.Readable {
                     void this.runNextQuery();
                 }
                 else {
-                    this.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
+                    this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
                 }
             }
         }
@@ -76,10 +73,10 @@ class DatastoreStreamReadable extends stream_1.Readable {
         // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
         this.count++;
         if (this.running) {
-            this.log(`_read ${this.count}, wasRunning: true`);
+            this.logger.log(`_read ${this.count}, wasRunning: true`);
         }
         if (this.done) {
-            console.warn(`!!! _read was called, but done==true`);
+            this.logger.warn(`!!! _read was called, but done==true`);
             return;
         }
         if (!this.running) {

package/dist/datastore.db.d.ts

@@ -1,6 +1,6 @@
 import type { Datastore, Key, Query } from '@google-cloud/datastore';
 import { BaseCommonDB, CommonDB, DBQuery, DBTransaction, RunQueryResult } from '@naturalcycles/db-lib';
-import { ObjectWithId, JsonSchemaRootObject } from '@naturalcycles/js-lib';
+import { ObjectWithId, JsonSchemaRootObject, CommonLogger } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats } from './datastore.model';
 /**
@@ -9,8 +9,10 @@ import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDB
  * https://cloud.google.com/datastore/docs/datastore-api-tutorial
  */
 export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
-    cfg: DatastoreDBCfg;
     constructor(cfg?: DatastoreDBCfg);
+    cfg: DatastoreDBCfg & {
+        logger: CommonLogger;
+    };
     private cachedDatastore?;
     /**
      * Datastore.KEY

package/dist/datastore.db.js

@@ -21,27 +21,27 @@ const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN'];
 class DatastoreDB extends db_lib_1.BaseCommonDB {
     constructor(cfg = {}) {
         super();
-        this.cfg = cfg;
+        this.cfg = {
+            logger: console,
+            ...cfg,
+        };
     }
     // @memo() // not used to be able to connect to many DBs in the same server instance
     ds() {
+        var _a;
         if (!this.cachedDatastore) {
-            if (process.env['APP_ENV'] === 'test') {
-                throw new Error('DatastoreDB cannot be used in Test env, please use InMemoryDB');
-            }
+            (0, js_lib_1._assert)(process.env['APP_ENV'] !== 'test', 'DatastoreDB cannot be used in Test env, please use InMemoryDB');
             // Lazy-loading
             const datastoreLib = require('@google-cloud/datastore');
             const DS = datastoreLib.Datastore;
-            this.cfg.projectId =
-                this.cfg.projectId ||
-                    this.cfg.credentials?.project_id ||
-                    process.env['GOOGLE_CLOUD_PROJECT'];
-            console.log(`DatastoreDB connected to ${(0, colors_1.boldWhite)(this.cfg.projectId)}`);
+            (_a = this.cfg).projectId || (_a.projectId = this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']);
+            (0, js_lib_1._assert)(this.cfg.projectId, '"projectId" is not set for DatastoreDB');
+            this.cfg.logger.log(`DatastoreDB connected to ${(0, colors_1.boldWhite)(this.cfg.projectId)}`);
             if (this.cfg.useLegacyGRPC) {
                 this.cfg.grpc = require('grpc');
             }
             if (this.cfg.grpc) {
-                console.log('!!! DatastoreDB using custom grpc !!!');
+                this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!');
             }
             this.cachedDatastore = new DS(this.cfg);
             this.KEY = this.cachedDatastore.KEY;
@@ -55,9 +55,20 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         if (!ids.length)
             return [];
         const keys = ids.map(id => this.key(table, id));
-        const [entities] = await this.ds().get(keys);
-        return (entities
-            .map(e => this.mapId(e))
+        let rows;
+        try {
+            rows = (await this.ds().get(keys))[0];
+        }
+        catch (err) {
+            this.cfg.logger.log('datastore recreated on error');
+            // This is to debug "GCP Datastore Timeout issue"
+            const datastoreLib = require('@google-cloud/datastore');
+            const DS = datastoreLib.Datastore;
+            this.cachedDatastore = new DS(this.cfg);
+            throw err;
+        }
+        return (rows
+            .map(r => this.mapId(r))
             // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
             // same ids are not expected here
             .sort((a, b) => (a.id > b.id ? 1 : -1)));
@@ -95,7 +106,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             ..._opt,
         };
         return (opt.experimentalCursorStream
-            ? new DatastoreStreamReadable_1.DatastoreStreamReadable(q, opt)
+            ? new DatastoreStreamReadable_1.DatastoreStreamReadable(q, opt, (0, js_lib_1.commonLoggerMinLevel)(this.cfg.logger, opt.debug ? 'log' : 'warn'))
             : this.ds().runQueryStream(q)).pipe(new stream_1.Transform({
             objectMode: true,
             transform: (chunk, _enc, cb) => {
@@ -119,19 +130,21 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             // Here we retry the GOAWAY errors that are somewhat common for Datastore
             // Currently only retrying them here in .saveBatch(), cause probably they're only thrown when saving
             predicate: err => RETRY_ON.some(s => err?.message.includes(s)),
+            name: `DatastoreLib.saveBatch(${table})`,
             maxAttempts: 5,
             delay: 5000,
             delayMultiplier: 2,
             logFirstAttempt: false,
             logFailures: true,
             // logAll: true,
+            logger: this.cfg.logger,
         });
         try {
             await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(entities, MAX_ITEMS), async (batch) => await save(batch));
         }
         catch (err) {
             // console.log(`datastore.save ${kind}`, { obj, entity })
-            console.error(`error in datastore.save for ${table}`, err);
+            this.cfg.logger.error(`error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`, err);
             // don't throw, because datastore SDK makes it in separate thread, so exception will be unhandled otherwise
             return await Promise.reject(err);
         }

package/dist/datastore.model.d.ts

@@ -1,6 +1,6 @@
 import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore';
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
-import { AnyObjectWithId, ObjectWithId } from '@naturalcycles/js-lib';
+import { AnyObjectWithId, CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
 export interface DatastorePayload<T = any> {
     key: Key;
     data: T;
@@ -30,6 +30,10 @@ export interface DatastoreDBCfg extends DatastoreOptions {
      * e.g you can globally enable `experimentalCursorStream` here, set the batchSize, etc.
      */
     streamOptions?: DatastoreDBStreamOptions;
+    /**
+     * Default to `console`
+     */
+    logger?: CommonLogger;
 }
 export interface DatastoreCredentials {
     type?: string;

package/dist/datastoreKeyValueDB.d.ts

@@ -17,4 +17,5 @@ export declare class DatastoreKeyValueDB implements CommonKeyValueDB {
     streamIds(table: string, limit?: number): ReadableTyped<string>;
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    count(_table: string): Promise<number>;
 }

package/dist/datastoreKeyValueDB.js

@@ -47,5 +47,9 @@ class DatastoreKeyValueDB {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
     }
+    async count(_table) {
+        this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`);
+        return 0;
+    }
 }
 exports.DatastoreKeyValueDB = DatastoreKeyValueDB;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.16.5",
+  "version": "3.18.0",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"
@@ -17,7 +17,7 @@
   "devDependencies": {
     "@google-cloud/datastore": "^6.0.0",
     "@naturalcycles/dev-lib": "^12.0.1",
-    "@types/node": "^16.0.0",
+    "@types/node": "^17.0.8",
     "jest": "^27.0.4"
   },
   "files": [

package/src/DatastoreStreamReadable.ts

@@ -1,6 +1,6 @@
 import { Readable } from 'stream'
 import { Query } from '@google-cloud/datastore'
-import { _ms } from '@naturalcycles/js-lib'
+import { _ms, CommonLogger } from '@naturalcycles/js-lib'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import type { DatastoreDBStreamOptions } from './datastore.model'
 
@@ -13,12 +13,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
   private lastQueryDone?: number
   private totalWait = 0
 
-  // private log = (...args: any[]): void => console.log(...args)
-  private log = console.log.bind(console)
-
   private opt: DatastoreDBStreamOptions & { batchSize: number }
 
-  constructor(private q: Query, opt: DatastoreDBStreamOptions) {
+  constructor(private q: Query, opt: DatastoreDBStreamOptions, private logger: CommonLogger) {
     super({ objectMode: true })
 
     this.opt = {
@@ -27,11 +24,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       ...opt,
     }
 
-    if (!opt.debug) this.log = () => {}
-
     this.originalLimit = q.limitVal
 
-    console.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`)
+    logger.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`)
   }
 
   private async runNextQuery(): Promise<void> {
@@ -61,7 +56,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       const [rows, info] = await q.run()
 
       this.rowsRetrieved += rows.length
-      this.log(
+      this.logger.log(
         `got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
           this.totalWait,
         )}`,
@@ -79,7 +74,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         info.moreResults === 'NO_MORE_RESULTS' ||
         (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
       ) {
-        this.log(
+        this.logger.log(
           `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
         )
         this.push(null)
@@ -90,7 +85,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         if (rssMB <= this.opt.rssLimitMB) {
           void this.runNextQuery()
         } else {
-          this.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
+          this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
         }
       }
     } catch (err) {
@@ -105,11 +100,11 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
     this.count++
     if (this.running) {
-      this.log(`_read ${this.count}, wasRunning: true`)
+      this.logger.log(`_read ${this.count}, wasRunning: true`)
     }
 
     if (this.done) {
-      console.warn(`!!! _read was called, but done==true`)
+      this.logger.warn(`!!! _read was called, but done==true`)
       return
     }
 

package/src/datastore.db.ts

@@ -22,6 +22,8 @@ import {
   _chunk,
   _omit,
   JsonSchemaRootObject,
+  CommonLogger,
+  commonLoggerMinLevel,
 } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { boldWhite } from '@naturalcycles/nodejs-lib/dist/colors'
@@ -51,10 +53,16 @@ const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN']
  * https://cloud.google.com/datastore/docs/datastore-api-tutorial
  */
 export class DatastoreDB extends BaseCommonDB implements CommonDB {
-  constructor(public cfg: DatastoreDBCfg = {}) {
+  constructor(cfg: DatastoreDBCfg = {}) {
     super()
+    this.cfg = {
+      logger: console,
+      ...cfg,
+    }
   }
 
+  public cfg: DatastoreDBCfg & { logger: CommonLogger }
+
   private cachedDatastore?: Datastore
 
   /**
@@ -65,26 +73,26 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   // @memo() // not used to be able to connect to many DBs in the same server instance
   ds(): Datastore {
     if (!this.cachedDatastore) {
-      if (process.env['APP_ENV'] === 'test') {
-        throw new Error('DatastoreDB cannot be used in Test env, please use InMemoryDB')
-      }
+      _assert(
+        process.env['APP_ENV'] !== 'test',
+        'DatastoreDB cannot be used in Test env, please use InMemoryDB',
+      )
 
       // Lazy-loading
       const datastoreLib = require('@google-cloud/datastore')
       const DS = datastoreLib.Datastore as typeof Datastore
-      this.cfg.projectId =
-        this.cfg.projectId ||
-        this.cfg.credentials?.project_id ||
-        process.env['GOOGLE_CLOUD_PROJECT']!
+      this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']
+
+      _assert(this.cfg.projectId, '"projectId" is not set for DatastoreDB')
 
-      console.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`)
+      this.cfg.logger.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`)
 
       if (this.cfg.useLegacyGRPC) {
         this.cfg.grpc = require('grpc')
       }
 
       if (this.cfg.grpc) {
-        console.log('!!! DatastoreDB using custom grpc !!!')
+        this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!')
       }
 
       this.cachedDatastore = new DS(this.cfg)
@@ -105,11 +113,24 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   ): Promise<ROW[]> {
     if (!ids.length) return []
     const keys = ids.map(id => this.key(table, id))
-    const [entities] = await this.ds().get(keys)
+    let rows: any[]
+
+    try {
+      rows = (await this.ds().get(keys))[0]
+    } catch (err) {
+      this.cfg.logger.log('datastore recreated on error')
+
+      // This is to debug "GCP Datastore Timeout issue"
+      const datastoreLib = require('@google-cloud/datastore')
+      const DS = datastoreLib.Datastore as typeof Datastore
+      this.cachedDatastore = new DS(this.cfg)
+
+      throw err
+    }
 
     return (
-      (entities as any[])
-        .map(e => this.mapId<ROW>(e))
+      rows
+        .map(r => this.mapId<ROW>(r))
         // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
         // same ids are not expected here
         .sort((a, b) => (a.id > b.id ? 1 : -1))
@@ -167,7 +188,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
     return (
       opt.experimentalCursorStream
-        ? new DatastoreStreamReadable(q, opt)
+        ? new DatastoreStreamReadable(
+            q,
+            opt,
+            commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
+          )
         : this.ds().runQueryStream(q)
     ).pipe(
       new Transform({
@@ -209,12 +234,14 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         // Here we retry the GOAWAY errors that are somewhat common for Datastore
         // Currently only retrying them here in .saveBatch(), cause probably they're only thrown when saving
         predicate: err => RETRY_ON.some(s => (err as Error)?.message.includes(s)),
+        name: `DatastoreLib.saveBatch(${table})`,
         maxAttempts: 5,
         delay: 5000,
         delayMultiplier: 2,
         logFirstAttempt: false,
         logFailures: true,
         // logAll: true,
+        logger: this.cfg.logger,
       },
     )
 
@@ -222,7 +249,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       await pMap(_chunk(entities, MAX_ITEMS), async batch => await save(batch))
     } catch (err) {
       // console.log(`datastore.save ${kind}`, { obj, entity })
-      console.error(`error in datastore.save for ${table}`, err)
+      this.cfg.logger.error(
+        `error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`,
+        err,
+      )
       // don't throw, because datastore SDK makes it in separate thread, so exception will be unhandled otherwise
       return await Promise.reject(err)
     }

package/src/datastore.model.ts

@@ -1,6 +1,6 @@
 import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore'
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib'
-import { AnyObjectWithId, ObjectWithId } from '@naturalcycles/js-lib'
+import { AnyObjectWithId, CommonLogger, ObjectWithId } from '@naturalcycles/js-lib'
 
 export interface DatastorePayload<T = any> {
   key: Key
@@ -36,6 +36,11 @@ export interface DatastoreDBCfg extends DatastoreOptions {
    * e.g you can globally enable `experimentalCursorStream` here, set the batchSize, etc.
    */
   streamOptions?: DatastoreDBStreamOptions
+
+  /**
+   * Default to `console`
+   */
+  logger?: CommonLogger
 }
 
 export interface DatastoreCredentials {

package/src/datastoreKeyValueDB.ts

@@ -76,4 +76,9 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       }),
     )
   }
+
+  async count(_table: string): Promise<number> {
+    this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`)
+    return 0
+  }
 }