@naturalcycles/datastore-lib 3.39.1 → 3.39.3

package/dist/datastore.db.js

@@ -11,7 +11,7 @@ const query_util_1 = require("./query.util");
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
 // It's an empyrical value, but anything less than infinity is better than infinity
-const DATASTORE_RECOMMENDED_CONCURRENCY = 16;
+const DATASTORE_RECOMMENDED_CONCURRENCY = 8;
 const RETRY_ON = [
     'GOAWAY',
     'UNAVAILABLE',
@@ -86,23 +86,33 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                 });
                 rows = r[0];
             }
-            catch {
+            catch (err) {
+                if (!(err instanceof js_lib_1.TimeoutError)) {
+                    // Not a timeout error, re-throw
+                    throw err;
+                }
                 this.cfg.logger.log('datastore recreated on error');
                 // This is to debug "GCP Datastore Timeout issue"
                 const datastoreLib = require('@google-cloud/datastore');
                 const DS = datastoreLib.Datastore;
                 this.cachedDatastore = new DS(this.cfg);
                 // Second try (will throw)
-                const r = await (0, js_lib_1.pRetry)(() => (opt.tx?.tx || this.ds()).get(keys, dsOpt), {
-                    ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
-                    maxAttempts: 3,
-                    timeout: this.cfg.timeout,
-                    errorData: {
-                        // This error will be grouped ACROSS all endpoints and usages
-                        fingerprint: [DATASTORE_TIMEOUT],
-                    },
-                });
-                rows = r[0];
+                try {
+                    const r = await (0, js_lib_1.pRetry)(() => (opt.tx?.tx || this.ds()).get(keys, dsOpt), {
+                        ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
+                        maxAttempts: 3,
+                        timeout: this.cfg.timeout,
+                    });
+                    rows = r[0];
+                }
+                catch (err) {
+                    if (err instanceof js_lib_1.TimeoutError) {
+                        (0, js_lib_1._errorDataAppend)(err, {
+                            fingerprint: [DATASTORE_TIMEOUT],
+                        });
+                    }
+                    throw err;
+                }
             }
         }
         else {
@@ -189,6 +199,11 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             }
         }
         catch (err) {
+            if (err instanceof js_lib_1.TimeoutError) {
+                (0, js_lib_1._errorDataAppend)(err, {
+                    fingerprint: [DATASTORE_TIMEOUT],
+                });
+            }
             // console.log(`datastore.save ${kind}`, { obj, entity })
             this.cfg.logger.error(`error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`, err);
             throw err;
@@ -379,9 +394,10 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             logFailures: true,
             // logAll: true,
             logger: this.cfg.logger,
-            errorData: {
-                fingerprint: [DATASTORE_TIMEOUT],
-            },
+            // not appending fingerprint here, otherwise it would just group all kinds of errors, not just Timeout errors
+            // errorData: {
+            //   fingerprint: [DATASTORE_TIMEOUT],
+            // },
         };
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.39.1",
+  "version": "3.39.3",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky",
@@ -11,15 +11,15 @@
     "lbt": "dev-lib lbt"
   },
   "dependencies": {
-    "@google-cloud/datastore": "^9.0.0",
-    "@naturalcycles/db-lib": "^9.0.0",
-    "@naturalcycles/js-lib": "^14.116.0",
-    "@naturalcycles/nodejs-lib": "^13.1.0"
+    "@google-cloud/datastore": "^9",
+    "@naturalcycles/db-lib": "^9",
+    "@naturalcycles/js-lib": "^14",
+    "@naturalcycles/nodejs-lib": "^13"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "^15.2.0",
-    "@types/node": "^22.0.0",
-    "jest": "^29.0.3"
+    "@naturalcycles/dev-lib": "^15",
+    "@types/node": "^22",
+    "jest": "^29"
   },
   "files": [
     "dist",
@@ -32,7 +32,7 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "engines": {
-    "node": ">=20.13.0"
+    "node": ">=22.12.0"
   },
   "publishConfig": {
     "access": "public"

package/src/datastore.db.ts

@@ -19,6 +19,7 @@ import {
 import {
   _assert,
   _chunk,
+  _errorDataAppend,
   _omit,
   CommonLogger,
   commonLoggerMinLevel,
@@ -35,6 +36,7 @@ import {
   pRetryFn,
   PRetryOptions,
   pTimeout,
+  TimeoutError,
 } from '@naturalcycles/js-lib'
 import { boldWhite, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import {
@@ -54,7 +56,7 @@ import { dbQueryToDatastoreQuery } from './query.util'
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500
 // It's an empyrical value, but anything less than infinity is better than infinity
-const DATASTORE_RECOMMENDED_CONCURRENCY = 16
+const DATASTORE_RECOMMENDED_CONCURRENCY = 8
 
 const RETRY_ON = [
   'GOAWAY',
@@ -160,7 +162,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
           },
         )
         rows = r[0]
-      } catch {
+      } catch (err) {
+        if (!(err instanceof TimeoutError)) {
+          // Not a timeout error, re-throw
+          throw err
+        }
+
         this.cfg.logger.log('datastore recreated on error')
 
         // This is to debug "GCP Datastore Timeout issue"
@@ -169,19 +176,24 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         this.cachedDatastore = new DS(this.cfg)
 
         // Second try (will throw)
-        const r = await pRetry(
-          () => ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).get(keys, dsOpt),
-          {
-            ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
-            maxAttempts: 3,
-            timeout: this.cfg.timeout,
-            errorData: {
-              // This error will be grouped ACROSS all endpoints and usages
-              fingerprint: [DATASTORE_TIMEOUT],
+        try {
+          const r = await pRetry(
+            () => ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).get(keys, dsOpt),
+            {
+              ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
+              maxAttempts: 3,
+              timeout: this.cfg.timeout,
             },
-          },
-        )
-        rows = r[0]
+          )
+          rows = r[0]
+        } catch (err) {
+          if (err instanceof TimeoutError) {
+            _errorDataAppend(err, {
+              fingerprint: [DATASTORE_TIMEOUT],
+            })
+          }
+          throw err
+        }
       }
     } else {
       rows = await pRetry(
@@ -319,6 +331,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         })
       }
     } catch (err) {
+      if (err instanceof TimeoutError) {
+        _errorDataAppend(err, {
+          fingerprint: [DATASTORE_TIMEOUT],
+        })
+      }
+
       // console.log(`datastore.save ${kind}`, { obj, entity })
       this.cfg.logger.error(
         `error in DatastoreLib.saveBatch for ${table} (${rows.length} rows)`,
@@ -556,9 +574,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       logFailures: true,
       // logAll: true,
       logger: this.cfg.logger,
-      errorData: {
-        fingerprint: [DATASTORE_TIMEOUT],
-      },
+      // not appending fingerprint here, otherwise it would just group all kinds of errors, not just Timeout errors
+      // errorData: {
+      //   fingerprint: [DATASTORE_TIMEOUT],
+      // },
     }
   }