@naturalcycles/datastore-lib 3.18.3 → 3.19.2

package/dist/datastore.db.d.ts

@@ -20,7 +20,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     protected KEY: symbol;
     ds(): Datastore;
     ping(): Promise<void>;
-    getByIds<ROW extends ObjectWithId>(table: string, ids: string[], _opt?: DatastoreDBOptions): Promise<ROW[]>;
+    getByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][], _opt?: DatastoreDBOptions): Promise<ROW[]>;
     getQueryKind(q: Query): string;
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<number>;
@@ -36,7 +36,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
      * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
      * regardless if they were actually deleted or not.
      */
-    deleteByIds(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<number>;
+    deleteByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][], opt?: DatastoreDBOptions): Promise<number>;
     /**
      * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
      */
@@ -50,9 +50,9 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     getTableProperties(table: string): Promise<DatastorePropertyStats[]>;
     mapId<T = any>(o: any, preserveKey?: boolean): T;
     toDatastoreEntity<T = any>(kind: string, o: T & {
-        id?: string;
+        id?: string | number;
     }, excludeFromIndexes?: string[]): DatastorePayload<T>;
-    key(kind: string, id: string): Key;
+    key(kind: string, id: string | number): Key;
     getDsKey(o: any): Key | undefined;
     getKey(key: Key): string | undefined;
     getTables(): Promise<string[]>;

package/dist/datastore.db.js

@@ -60,25 +60,35 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             return [];
         const keys = ids.map(id => this.key(table, id));
         let rows;
-        try {
-            if (this.cfg.timeout) {
+        if (this.cfg.timeout) {
+            // First try
+            try {
                 const r = await (0, js_lib_1.pTimeout)(this.ds().get(keys), {
                     timeout: this.cfg.timeout,
                     name: `datastore.getByIds(${table})`,
                 });
                 rows = r[0];
             }
-            else {
-                rows = (await this.ds().get(keys))[0];
+            catch {
+                this.cfg.logger.log('datastore recreated on error');
+                // This is to debug "GCP Datastore Timeout issue"
+                const datastoreLib = require('@google-cloud/datastore');
+                const DS = datastoreLib.Datastore;
+                this.cachedDatastore = new DS(this.cfg);
+                // Second try (will throw)
+                const r = await (0, js_lib_1.pTimeout)(this.ds().get(keys), {
+                    timeout: this.cfg.timeout,
+                    name: `datastore.getByIds(${table}) second try`,
+                    errorData: {
+                        // This error will be grouped ACROSS all endpoints and usages
+                        fingerprint: ['DATASTORE_TIMEOUT'],
+                    },
+                });
+                rows = r[0];
             }
         }
-        catch (err) {
-            this.cfg.logger.log('datastore recreated on error');
-            // This is to debug "GCP Datastore Timeout issue"
-            const datastoreLib = require('@google-cloud/datastore');
-            const DS = datastoreLib.Datastore;
-            this.cachedDatastore = new DS(this.cfg);
-            throw err;
+        else {
+            rows = (await this.ds().get(keys))[0];
         }
         return (rows
             .map(r => this.mapId(r))
@@ -137,12 +147,12 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
      */
     async saveBatch(table, rows, opt = {}) {
         const entities = rows.map(obj => this.toDatastoreEntity(table, obj, opt.excludeFromIndexes));
-        const save = (0, js_lib_1.pRetry)(async (batch) => {
+        const save = (0, js_lib_1.pRetryFn)(async (batch) => {
             await (opt.tx || this.ds()).save(batch);
         }, {
             // Here we retry the GOAWAY errors that are somewhat common for Datastore
             // Currently only retrying them here in .saveBatch(), cause probably they're only thrown when saving
-            predicate: err => RETRY_ON.some(s => err?.message.includes(s)),
+            predicate: err => RETRY_ON.some(s => err?.message?.includes(s)),
             name: `DatastoreLib.saveBatch(${table})`,
             maxAttempts: 5,
             delay: 5000,
@@ -153,13 +163,19 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             logger: this.cfg.logger,
         });
         try {
-            await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(entities, MAX_ITEMS), async (batch) => await save(batch));
+            const chunks = (0, js_lib_1._chunk)(entities, MAX_ITEMS);
+            if (chunks.length === 1) {
+                // Not using pMap in hope to preserve stack trace
+                await save(chunks[0]);
+            }
+            else {
+                await (0, js_lib_1.pMap)(chunks, async (batch) => await save(batch));
+            }
         }
         catch (err) {
             // console.log(`datastore.save ${kind}`, { obj, entity })
             this.cfg.logger.error(`error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`, err);
-            // don't throw, because datastore SDK makes it in separate thread, so exception will be unhandled otherwise
-            return await Promise.reject(err);
+            throw err;
         }
     }
     async deleteByQuery(q, opt) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.18.3",
+  "version": "3.19.2",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"

package/src/datastore.db.ts

@@ -17,7 +17,6 @@ import {
   JsonSchemaObject,
   JsonSchemaString,
   pMap,
-  pRetry,
   _assert,
   _chunk,
   _omit,
@@ -25,6 +24,7 @@ import {
   CommonLogger,
   commonLoggerMinLevel,
   pTimeout,
+  pRetryFn,
 } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { boldWhite } from '@naturalcycles/nodejs-lib/dist/colors'
@@ -111,32 +111,42 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
   override async getByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
+    ids: ROW['id'][],
     _opt?: DatastoreDBOptions,
   ): Promise<ROW[]> {
     if (!ids.length) return []
     const keys = ids.map(id => this.key(table, id))
     let rows: any[]
 
-    try {
-      if (this.cfg.timeout) {
+    if (this.cfg.timeout) {
+      // First try
+      try {
         const r = await pTimeout(this.ds().get(keys), {
           timeout: this.cfg.timeout,
           name: `datastore.getByIds(${table})`,
         })
         rows = r[0]
-      } else {
-        rows = (await this.ds().get(keys))[0]
-      }
-    } catch (err) {
-      this.cfg.logger.log('datastore recreated on error')
+      } catch {
+        this.cfg.logger.log('datastore recreated on error')
 
-      // This is to debug "GCP Datastore Timeout issue"
-      const datastoreLib = require('@google-cloud/datastore')
-      const DS = datastoreLib.Datastore as typeof Datastore
-      this.cachedDatastore = new DS(this.cfg)
+        // This is to debug "GCP Datastore Timeout issue"
+        const datastoreLib = require('@google-cloud/datastore')
+        const DS = datastoreLib.Datastore as typeof Datastore
+        this.cachedDatastore = new DS(this.cfg)
 
-      throw err
+        // Second try (will throw)
+        const r = await pTimeout(this.ds().get(keys), {
+          timeout: this.cfg.timeout,
+          name: `datastore.getByIds(${table}) second try`,
+          errorData: {
+            // This error will be grouped ACROSS all endpoints and usages
+            fingerprint: ['DATASTORE_TIMEOUT'],
+          },
+        })
+        rows = r[0]
+      }
+    } else {
+      rows = (await this.ds().get(keys))[0]
     }
 
     return (
@@ -237,14 +247,14 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       this.toDatastoreEntity(table, obj, opt.excludeFromIndexes as string[]),
     )
 
-    const save = pRetry(
+    const save = pRetryFn(
       async (batch: DatastorePayload<ROW>[]) => {
         await (opt.tx || this.ds()).save(batch)
       },
       {
         // Here we retry the GOAWAY errors that are somewhat common for Datastore
         // Currently only retrying them here in .saveBatch(), cause probably they're only thrown when saving
-        predicate: err => RETRY_ON.some(s => (err as Error)?.message.includes(s)),
+        predicate: err => RETRY_ON.some(s => err?.message?.includes(s)),
         name: `DatastoreLib.saveBatch(${table})`,
         maxAttempts: 5,
         delay: 5000,
@@ -257,15 +267,21 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     )
 
     try {
-      await pMap(_chunk(entities, MAX_ITEMS), async batch => await save(batch))
+      const chunks = _chunk(entities, MAX_ITEMS)
+      if (chunks.length === 1) {
+        // Not using pMap in hope to preserve stack trace
+        await save(chunks[0]!)
+      } else {
+        await pMap(chunks, async batch => await save(batch))
+      }
     } catch (err) {
       // console.log(`datastore.save ${kind}`, { obj, entity })
       this.cfg.logger.error(
         `error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`,
         err,
       )
-      // don't throw, because datastore SDK makes it in separate thread, so exception will be unhandled otherwise
-      return await Promise.reject(err)
+
+      throw err
     }
   }
 
@@ -286,9 +302,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
    * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
    * regardless if they were actually deleted or not.
    */
-  override async deleteByIds(
+  override async deleteByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
+    ids: ROW['id'][],
     opt: DatastoreDBOptions = {},
   ): Promise<number> {
     const keys = ids.map(id => this.key(table, id))
@@ -370,7 +386,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   // if key field exists on entity, it will be used as key (prevent to duplication of numeric keyed entities)
   toDatastoreEntity<T = any>(
     kind: string,
-    o: T & { id?: string },
+    o: T & { id?: string | number },
     excludeFromIndexes: string[] = [],
   ): DatastorePayload<T> {
     const key = this.getDsKey(o) || this.key(kind, o.id!)
@@ -385,7 +401,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  key(kind: string, id: string): Key {
+  key(kind: string, id: string | number): Key {
     _assert(id, `Cannot save "${kind}" entity without "id"`)
     return this.ds().key([kind, String(id)])
   }

package/src/datastoreKeyValueDB.ts

@@ -50,7 +50,7 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       .limit(limit || 0)
 
     return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<ObjectWithId, string>(objectWithId => objectWithId.id, {
+      transformMapSimple<ObjectWithId<string>, string>(objectWithId => objectWithId.id, {
         errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
       }),
     )