@naturalcycles/datastore-lib 3.17.0 → 3.18.1

package/dist/datastore.db.js

@@ -55,9 +55,29 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         if (!ids.length)
             return [];
         const keys = ids.map(id => this.key(table, id));
-        const [entities] = await this.ds().get(keys);
-        return (entities
-            .map(e => this.mapId(e))
+        let rows;
+        try {
+            if (this.cfg.timeout) {
+                const r = await (0, js_lib_1.pTimeout)(this.ds().get(keys), {
+                    timeout: this.cfg.timeout,
+                    name: `datastore.getByIds(${table})`,
+                });
+                rows = r[0];
+            }
+            else {
+                rows = (await this.ds().get(keys))[0];
+            }
+        }
+        catch (err) {
+            this.cfg.logger.log('datastore recreated on error');
+            // This is to debug "GCP Datastore Timeout issue"
+            const datastoreLib = require('@google-cloud/datastore');
+            const DS = datastoreLib.Datastore;
+            this.cachedDatastore = new DS(this.cfg);
+            throw err;
+        }
+        return (rows
+            .map(r => this.mapId(r))
             // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
             // same ids are not expected here
             .sort((a, b) => (a.id > b.id ? 1 : -1)));
@@ -119,19 +139,21 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             // Here we retry the GOAWAY errors that are somewhat common for Datastore
             // Currently only retrying them here in .saveBatch(), cause probably they're only thrown when saving
             predicate: err => RETRY_ON.some(s => err?.message.includes(s)),
+            name: `DatastoreLib.saveBatch(${table})`,
             maxAttempts: 5,
             delay: 5000,
             delayMultiplier: 2,
             logFirstAttempt: false,
             logFailures: true,
             // logAll: true,
+            logger: this.cfg.logger,
         });
         try {
             await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(entities, MAX_ITEMS), async (batch) => await save(batch));
         }
         catch (err) {
             // console.log(`datastore.save ${kind}`, { obj, entity })
-            this.cfg.logger.error(`error in datastore.save for ${table}`, err);
+            this.cfg.logger.error(`error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`, err);
             // don't throw, because datastore SDK makes it in separate thread, so exception will be unhandled otherwise
             return await Promise.reject(err);
         }

package/dist/datastore.model.d.ts

@@ -34,6 +34,13 @@ export interface DatastoreDBCfg extends DatastoreOptions {
      * Default to `console`
      */
     logger?: CommonLogger;
+    /**
+     * Experimental option, currently only applies to `getByIds`.
+     * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
+     *
+     * @experimental
+     */
+    timeout?: number;
 }
 export interface DatastoreCredentials {
     type?: string;

package/dist/datastoreKeyValueDB.d.ts

@@ -17,4 +17,5 @@ export declare class DatastoreKeyValueDB implements CommonKeyValueDB {
     streamIds(table: string, limit?: number): ReadableTyped<string>;
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    count(_table: string): Promise<number>;
 }

package/dist/datastoreKeyValueDB.js

@@ -47,5 +47,9 @@ class DatastoreKeyValueDB {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
     }
+    async count(_table) {
+        this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`);
+        return 0;
+    }
 }
 exports.DatastoreKeyValueDB = DatastoreKeyValueDB;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.17.0",
+  "version": "3.18.1",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"
@@ -17,7 +17,7 @@
   "devDependencies": {
     "@google-cloud/datastore": "^6.0.0",
     "@naturalcycles/dev-lib": "^12.0.1",
-    "@types/node": "^16.0.0",
+    "@types/node": "^17.0.8",
     "jest": "^27.0.4"
   },
   "files": [

package/src/datastore.db.ts

@@ -24,6 +24,7 @@ import {
   JsonSchemaRootObject,
   CommonLogger,
   commonLoggerMinLevel,
+  pTimeout,
 } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { boldWhite } from '@naturalcycles/nodejs-lib/dist/colors'
@@ -113,11 +114,32 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   ): Promise<ROW[]> {
     if (!ids.length) return []
     const keys = ids.map(id => this.key(table, id))
-    const [entities] = await this.ds().get(keys)
+    let rows: any[]
+
+    try {
+      if (this.cfg.timeout) {
+        const r = await pTimeout(this.ds().get(keys), {
+          timeout: this.cfg.timeout,
+          name: `datastore.getByIds(${table})`,
+        })
+        rows = r[0]
+      } else {
+        rows = (await this.ds().get(keys))[0]
+      }
+    } catch (err) {
+      this.cfg.logger.log('datastore recreated on error')
+
+      // This is to debug "GCP Datastore Timeout issue"
+      const datastoreLib = require('@google-cloud/datastore')
+      const DS = datastoreLib.Datastore as typeof Datastore
+      this.cachedDatastore = new DS(this.cfg)
+
+      throw err
+    }
 
     return (
-      (entities as any[])
-        .map(e => this.mapId<ROW>(e))
+      rows
+        .map(r => this.mapId<ROW>(r))
         // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
         // same ids are not expected here
         .sort((a, b) => (a.id > b.id ? 1 : -1))
@@ -221,12 +243,14 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         // Here we retry the GOAWAY errors that are somewhat common for Datastore
         // Currently only retrying them here in .saveBatch(), cause probably they're only thrown when saving
         predicate: err => RETRY_ON.some(s => (err as Error)?.message.includes(s)),
+        name: `DatastoreLib.saveBatch(${table})`,
         maxAttempts: 5,
         delay: 5000,
         delayMultiplier: 2,
         logFirstAttempt: false,
         logFailures: true,
         // logAll: true,
+        logger: this.cfg.logger,
       },
     )
 
@@ -234,7 +258,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       await pMap(_chunk(entities, MAX_ITEMS), async batch => await save(batch))
     } catch (err) {
       // console.log(`datastore.save ${kind}`, { obj, entity })
-      this.cfg.logger.error(`error in datastore.save for ${table}`, err)
+      this.cfg.logger.error(
+        `error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`,
+        err,
+      )
       // don't throw, because datastore SDK makes it in separate thread, so exception will be unhandled otherwise
       return await Promise.reject(err)
     }

package/src/datastore.model.ts

@@ -41,6 +41,14 @@ export interface DatastoreDBCfg extends DatastoreOptions {
    * Default to `console`
    */
   logger?: CommonLogger
+
+  /**
+   * Experimental option, currently only applies to `getByIds`.
+   * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
+   *
+   * @experimental
+   */
+  timeout?: number
 }
 
 export interface DatastoreCredentials {

package/src/datastoreKeyValueDB.ts

@@ -76,4 +76,9 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       }),
     )
   }
+
+  async count(_table: string): Promise<number> {
+    this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`)
+    return 0
+  }
 }