@naturalcycles/datastore-lib 3.17.1 → 3.18.2

package/dist/datastore.db.js

@@ -36,7 +36,12 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             const DS = datastoreLib.Datastore;
             (_a = this.cfg).projectId || (_a.projectId = this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']);
             (0, js_lib_1._assert)(this.cfg.projectId, '"projectId" is not set for DatastoreDB');
-            this.cfg.logger.log(`DatastoreDB connected to ${(0, colors_1.boldWhite)(this.cfg.projectId)}`);
+            if (this.cfg.projectId) {
+                this.cfg.logger.log(`DatastoreDB connected to ${(0, colors_1.boldWhite)(this.cfg.projectId)}`);
+            }
+            else if (process.env['GOOGLE_APPLICATION_CREDENTIALS']) {
+                this.cfg.logger.log(`DatastoreDB connected via GOOGLE_APPLICATION_CREDENTIALS`);
+            }
             if (this.cfg.useLegacyGRPC) {
                 this.cfg.grpc = require('grpc');
             }
@@ -55,9 +60,29 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         if (!ids.length)
             return [];
         const keys = ids.map(id => this.key(table, id));
-        const [entities] = await this.ds().get(keys);
-        return (entities
-            .map(e => this.mapId(e))
+        let rows;
+        try {
+            if (this.cfg.timeout) {
+                const r = await (0, js_lib_1.pTimeout)(this.ds().get(keys), {
+                    timeout: this.cfg.timeout,
+                    name: `datastore.getByIds(${table})`,
+                });
+                rows = r[0];
+            }
+            else {
+                rows = (await this.ds().get(keys))[0];
+            }
+        }
+        catch (err) {
+            this.cfg.logger.log('datastore recreated on error');
+            // This is to debug "GCP Datastore Timeout issue"
+            const datastoreLib = require('@google-cloud/datastore');
+            const DS = datastoreLib.Datastore;
+            this.cachedDatastore = new DS(this.cfg);
+            throw err;
+        }
+        return (rows
+            .map(r => this.mapId(r))
             // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
             // same ids are not expected here
             .sort((a, b) => (a.id > b.id ? 1 : -1)));

package/dist/datastore.model.d.ts

@@ -34,6 +34,13 @@ export interface DatastoreDBCfg extends DatastoreOptions {
      * Default to `console`
      */
     logger?: CommonLogger;
+    /**
+     * Experimental option, currently only applies to `getByIds`.
+     * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
+     *
+     * @experimental
+     */
+    timeout?: number;
 }
 export interface DatastoreCredentials {
     type?: string;

package/dist/datastoreKeyValueDB.d.ts

@@ -17,4 +17,5 @@ export declare class DatastoreKeyValueDB implements CommonKeyValueDB {
     streamIds(table: string, limit?: number): ReadableTyped<string>;
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    count(_table: string): Promise<number>;
 }

package/dist/datastoreKeyValueDB.js

@@ -47,5 +47,9 @@ class DatastoreKeyValueDB {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
     }
+    async count(_table) {
+        this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`);
+        return 0;
+    }
 }
 exports.DatastoreKeyValueDB = DatastoreKeyValueDB;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.17.1",
+  "version": "3.18.2",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"
@@ -17,7 +17,7 @@
   "devDependencies": {
     "@google-cloud/datastore": "^6.0.0",
     "@naturalcycles/dev-lib": "^12.0.1",
-    "@types/node": "^16.0.0",
+    "@types/node": "^17.0.8",
     "jest": "^27.0.4"
   },
   "files": [

package/src/datastore.db.ts

@@ -24,6 +24,7 @@ import {
   JsonSchemaRootObject,
   CommonLogger,
   commonLoggerMinLevel,
+  pTimeout,
 } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { boldWhite } from '@naturalcycles/nodejs-lib/dist/colors'
@@ -85,7 +86,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
       _assert(this.cfg.projectId, '"projectId" is not set for DatastoreDB')
 
-      this.cfg.logger.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`)
+      if (this.cfg.projectId) {
+        this.cfg.logger.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`)
+      } else if (process.env['GOOGLE_APPLICATION_CREDENTIALS']) {
+        this.cfg.logger.log(`DatastoreDB connected via GOOGLE_APPLICATION_CREDENTIALS`)
+      }
 
       if (this.cfg.useLegacyGRPC) {
         this.cfg.grpc = require('grpc')
@@ -113,11 +118,32 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   ): Promise<ROW[]> {
     if (!ids.length) return []
     const keys = ids.map(id => this.key(table, id))
-    const [entities] = await this.ds().get(keys)
+    let rows: any[]
+
+    try {
+      if (this.cfg.timeout) {
+        const r = await pTimeout(this.ds().get(keys), {
+          timeout: this.cfg.timeout,
+          name: `datastore.getByIds(${table})`,
+        })
+        rows = r[0]
+      } else {
+        rows = (await this.ds().get(keys))[0]
+      }
+    } catch (err) {
+      this.cfg.logger.log('datastore recreated on error')
+
+      // This is to debug "GCP Datastore Timeout issue"
+      const datastoreLib = require('@google-cloud/datastore')
+      const DS = datastoreLib.Datastore as typeof Datastore
+      this.cachedDatastore = new DS(this.cfg)
+
+      throw err
+    }
 
     return (
-      (entities as any[])
-        .map(e => this.mapId<ROW>(e))
+      rows
+        .map(r => this.mapId<ROW>(r))
         // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
         // same ids are not expected here
         .sort((a, b) => (a.id > b.id ? 1 : -1))

package/src/datastore.model.ts

@@ -41,6 +41,14 @@ export interface DatastoreDBCfg extends DatastoreOptions {
    * Default to `console`
    */
   logger?: CommonLogger
+
+  /**
+   * Experimental option, currently only applies to `getByIds`.
+   * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
+   *
+   * @experimental
+   */
+  timeout?: number
 }
 
 export interface DatastoreCredentials {

package/src/datastoreKeyValueDB.ts

@@ -76,4 +76,9 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       }),
     )
   }
+
+  async count(_table: string): Promise<number> {
+    this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`)
+    return 0
+  }
 }