@naturalcycles/datastore-lib 3.25.4 → 3.26.1

package/dist/datastore.db.d.ts

@@ -48,7 +48,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     getStats(table: string): Promise<DatastoreStats | undefined>;
     getStatsCount(table: string): Promise<number | undefined>;
     getTableProperties(table: string): Promise<DatastorePropertyStats[]>;
-    mapId<T = any>(o: any, preserveKey?: boolean): T;
+    mapId<T extends ObjectWithId>(o: any, preserveKey?: boolean): T;
     toDatastoreEntity<T = any>(kind: string, o: T & {
         id?: string | number;
     }, excludeFromIndexes?: string[]): DatastorePayload<T>;

package/dist/datastore.db.js

@@ -10,7 +10,14 @@ const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
 const query_util_1 = require("./query.util");
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
-const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'DEADLINE_EXCEEDED', 'much contention'];
+const RETRY_ON = [
+    'GOAWAY',
+    'UNAVAILABLE',
+    'UNKNOWN',
+    'DEADLINE_EXCEEDED',
+    'much contention',
+    'timeout',
+].map(s => s.toLowerCase());
 // Examples of errors:
 // UNKNOWN: Stream removed
 const methodMap = {
@@ -80,9 +87,10 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                 const DS = datastoreLib.Datastore;
                 this.cachedDatastore = new DS(this.cfg);
                 // Second try (will throw)
-                const r = await (0, js_lib_1.pTimeout)(() => this.ds().get(keys), {
+                const r = await (0, js_lib_1.pRetry)(() => this.ds().get(keys), {
+                    ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
+                    maxAttempts: 3,
                     timeout: this.cfg.timeout,
-                    name: `datastore.getByIds(${table}) second try`,
                     errorData: {
                         // This error will be grouped ACROSS all endpoints and usages
                         fingerprint: ['DATASTORE_TIMEOUT'],
@@ -92,7 +100,9 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             }
         }
         else {
-            rows = (await this.ds().get(keys))[0];
+            rows = await (0, js_lib_1.pRetry)(async () => {
+                return (await this.ds().get(keys))[0];
+            }, this.getPRetryOptions(`datastore.getByIds(${table})`));
         }
         return (rows
             .map(r => this.mapId(r))
@@ -101,7 +111,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             .sort((a, b) => (a.id > b.id ? 1 : -1)));
     }
     getQueryKind(q) {
-        if (!q || !q.kinds || !q.kinds.length)
+        if (!q?.kinds?.length)
             return ''; // should never be the case, but
         return q.kinds[0];
     }
@@ -246,7 +256,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
     }
     async getStatsCount(table) {
         const stats = await this.getStats(table);
-        return stats && stats.count;
+        return stats?.count;
     }
     async getTableProperties(table) {
         const q = this.ds()
@@ -350,11 +360,9 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             }
             else if (dtype === datastore_model_1.DatastoreType.NULL) {
                 // check, maybe we can just skip this type and do nothing?
-                if (!s.properties[name]) {
-                    s.properties[name] = {
-                        type: 'null',
-                    };
-                }
+                s.properties[name] ||= {
+                    type: 'null',
+                };
             }
             else {
                 throw new Error(`Unknown Datastore Type '${stats.property_type}' for ${table}.${name}`);
@@ -364,11 +372,12 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
     }
     getPRetryOptions(name) {
         return {
-            predicate: err => RETRY_ON.some(s => err?.message?.includes(s)),
+            predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
             name,
+            timeout: 10000,
             maxAttempts: 5,
             delay: 5000,
-            delayMultiplier: 2,
+            delayMultiplier: 1.5,
             logFirstAttempt: false,
             logFailures: true,
             // logAll: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.25.4",
+  "version": "3.26.1",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"

package/src/datastore.db.ts

@@ -47,7 +47,14 @@ import { dbQueryToDatastoreQuery } from './query.util'
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500
 
-const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'DEADLINE_EXCEEDED', 'much contention']
+const RETRY_ON = [
+  'GOAWAY',
+  'UNAVAILABLE',
+  'UNKNOWN',
+  'DEADLINE_EXCEEDED',
+  'much contention',
+  'timeout',
+].map(s => s.toLowerCase())
 // Examples of errors:
 // UNKNOWN: Stream removed
 
@@ -144,9 +151,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         this.cachedDatastore = new DS(this.cfg)
 
         // Second try (will throw)
-        const r = await pTimeout(() => this.ds().get(keys), {
+        const r = await pRetry(() => this.ds().get(keys), {
+          ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
+          maxAttempts: 3,
           timeout: this.cfg.timeout,
-          name: `datastore.getByIds(${table}) second try`,
           errorData: {
             // This error will be grouped ACROSS all endpoints and usages
             fingerprint: ['DATASTORE_TIMEOUT'],
@@ -155,7 +163,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         rows = r[0]
       }
     } else {
-      rows = (await this.ds().get(keys))[0]
+      rows = await pRetry(async () => {
+        return (await this.ds().get(keys))[0]
+      }, this.getPRetryOptions(`datastore.getByIds(${table})`))
     }
 
     return (
@@ -168,7 +178,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   }
 
   getQueryKind(q: Query): string {
-    if (!q || !q.kinds || !q.kinds.length) return '' // should never be the case, but
+    if (!q?.kinds?.length) return '' // should never be the case, but
     return q.kinds[0]!
   }
 
@@ -380,7 +390,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
   async getStatsCount(table: string): Promise<number | undefined> {
     const stats = await this.getStats(table)
-    return stats && stats.count
+    return stats?.count
   }
 
   async getTableProperties(table: string): Promise<DatastorePropertyStats[]> {
@@ -391,7 +401,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return stats
   }
 
-  mapId<T = any>(o: any, preserveKey = false): T {
+  mapId<T extends ObjectWithId>(o: any, preserveKey = false): T {
     if (!o) return o
     const r = {
       ...o,
@@ -497,11 +507,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
           s.properties[name] = {} as JsonSchemaAny
         } else if (dtype === DatastoreType.NULL) {
           // check, maybe we can just skip this type and do nothing?
-          if (!s.properties[name]) {
-            s.properties[name] = {
-              type: 'null',
-            } as JsonSchemaNull
-          }
+          s.properties[name] ||= {
+            type: 'null',
+          } as JsonSchemaNull
         } else {
           throw new Error(
             `Unknown Datastore Type '${stats.property_type}' for ${table}.${name as string}`,
@@ -514,11 +522,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
   private getPRetryOptions(name: string): PRetryOptions {
     return {
-      predicate: err => RETRY_ON.some(s => err?.message?.includes(s)),
+      predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
       name,
+      timeout: 10_000,
       maxAttempts: 5,
       delay: 5000,
-      delayMultiplier: 2,
+      delayMultiplier: 1.5,
       logFirstAttempt: false,
       logFailures: true,
       // logAll: true,