@naturalcycles/datastore-lib 3.34.0 → 3.35.1

package/dist/DatastoreStreamReadable.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { Readable } from 'node:stream';
 import { Query } from '@google-cloud/datastore';
 import { CommonLogger } from '@naturalcycles/js-lib';

package/dist/datastore.db.d.ts

@@ -1,7 +1,7 @@
-import { Transaction } from '@google-cloud/datastore';
 import type { Datastore, Key, Query } from '@google-cloud/datastore';
-import { BaseCommonDB, CommonDB, CommonDBOptions, CommonDBSaveOptions, CommonDBSupport, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult, CommonDBTransactionOptions } from '@naturalcycles/db-lib';
-import { JsonSchemaObject, JsonSchemaRootObject, CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
+import { Transaction } from '@google-cloud/datastore';
+import { BaseCommonDB, CommonDB, CommonDBOptions, CommonDBSaveOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
+import { CommonLogger, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats } from './datastore.model';
 /**

package/dist/datastore.db.js

@@ -5,11 +5,13 @@ const datastore_1 = require("@google-cloud/datastore");
 const db_lib_1 = require("@naturalcycles/db-lib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
-const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
 const datastore_model_1 = require("./datastore.model");
+const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
 const query_util_1 = require("./query.util");
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
+// It's an empyrical value, but anything less than infinity is better than infinity
+const DATASTORE_RECOMMENDED_CONCURRENCY = 16;
 const RETRY_ON = [
     'GOAWAY',
     'UNAVAILABLE',
@@ -176,7 +178,9 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                 await save(chunks[0]);
             }
             else {
-                await (0, js_lib_1.pMap)(chunks, async (batch) => await save(batch));
+                await (0, js_lib_1.pMap)(chunks, async (batch) => await save(batch), {
+                    concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
+                });
             }
         }
         catch (err) {
@@ -203,7 +207,9 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         const keys = ids.map(id => this.key(table, id));
         await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(keys, MAX_ITEMS), 
         // eslint-disable-next-line @typescript-eslint/return-await
-        async (batch) => await (opt.tx?.tx || this.ds()).delete(batch));
+        async (batch) => await (opt.tx?.tx || this.ds()).delete(batch), {
+            concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
+        });
         return ids.length;
     }
     async runInTransaction(fn, opt = {}) {

package/dist/datastoreKeyValueDB.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DatastoreDB } from './datastore.db';

package/dist/query.util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.dbQueryToDatastoreQuery = void 0;
+exports.dbQueryToDatastoreQuery = dbQueryToDatastoreQuery;
 const datastore_1 = require("@google-cloud/datastore");
 const FNAME_MAP = {
     id: '__key__',
@@ -41,4 +41,3 @@ function dbQueryToDatastoreQuery(dbQuery, emptyQuery) {
     }
     return q;
 }
-exports.dbQueryToDatastoreQuery = dbQueryToDatastoreQuery;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.34.0",
+  "version": "3.35.1",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky"

package/src/datastore.db.ts

@@ -1,5 +1,5 @@
-import { PropertyFilter, Transaction } from '@google-cloud/datastore'
 import type { Datastore, Key, Query } from '@google-cloud/datastore'
+import { PropertyFilter, Transaction } from '@google-cloud/datastore'
 import {
   BaseCommonDB,
   CommonDB,
@@ -8,34 +8,33 @@ import {
   CommonDBSaveMethod,
   CommonDBSaveOptions,
   CommonDBSupport,
+  CommonDBTransactionOptions,
   DBQuery,
   DBTransaction,
   DBTransactionFn,
   RunQueryResult,
-  CommonDBTransactionOptions,
 } from '@naturalcycles/db-lib'
 import {
+  _assert,
+  _chunk,
+  _omit,
+  CommonLogger,
+  commonLoggerMinLevel,
   JsonSchemaAny,
   JsonSchemaBoolean,
   JsonSchemaNull,
   JsonSchemaNumber,
   JsonSchemaObject,
+  JsonSchemaRootObject,
   JsonSchemaString,
+  ObjectWithId,
   pMap,
-  _assert,
-  _chunk,
-  _omit,
-  JsonSchemaRootObject,
-  CommonLogger,
-  commonLoggerMinLevel,
-  pTimeout,
-  pRetryFn,
   pRetry,
+  pRetryFn,
   PRetryOptions,
-  ObjectWithId,
+  pTimeout,
 } from '@naturalcycles/js-lib'
-import { ReadableTyped, boldWhite } from '@naturalcycles/nodejs-lib'
-import { DatastoreStreamReadable } from './DatastoreStreamReadable'
+import { boldWhite, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import {
   DatastoreDBCfg,
   DatastoreDBOptions,
@@ -46,10 +45,13 @@ import {
   DatastoreStats,
   DatastoreType,
 } from './datastore.model'
+import { DatastoreStreamReadable } from './DatastoreStreamReadable'
 import { dbQueryToDatastoreQuery } from './query.util'
 
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500
+// It's an empyrical value, but anything less than infinity is better than infinity
+const DATASTORE_RECOMMENDED_CONCURRENCY = 16
 
 const RETRY_ON = [
   'GOAWAY',
@@ -300,7 +302,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         // Not using pMap in hope to preserve stack trace
         await save(chunks[0]!)
       } else {
-        await pMap(chunks, async batch => await save(batch))
+        await pMap(chunks, async batch => await save(batch), {
+          concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
+        })
       }
     } catch (err) {
       // console.log(`datastore.save ${kind}`, { obj, entity })
@@ -347,6 +351,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       _chunk(keys, MAX_ITEMS),
       // eslint-disable-next-line @typescript-eslint/return-await
       async batch => await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).delete(batch),
+      {
+        concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
+      },
     )
     return ids.length
   }