@naturalcycles/datastore-lib 3.20.1 → 3.23.0

package/dist/DatastoreStreamReadable.js

@@ -54,6 +54,10 @@ class DatastoreStreamReadable extends stream_1.Readable {
                 this.push(null);
                 this.done = true;
             }
+            else if (this.opt.singleBatchBuffer) {
+                // here we don't start next query until we're asked (via next _read call)
+                // do, let's do nothing
+            }
             else if (this.opt.rssLimitMB) {
                 const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
                 if (rssMB <= this.opt.rssLimitMB) {

package/dist/datastore.db.js

@@ -10,7 +10,7 @@ const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
 const query_util_1 = require("./query.util");
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
-const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN'];
+const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'much contention'];
 // Examples of errors:
 // UNKNOWN: Stream removed
 const methodMap = {
@@ -303,7 +303,8 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         stats
             .filter(s => !s.property_name.includes('.') && s.property_name !== 'id') // filter out objectify's "virtual properties"
             .forEach(stats => {
-            const { property_name: name, property_type: dtype } = stats;
+            const { property_type: dtype } = stats;
+            const name = stats.property_name;
             if (dtype === datastore_model_1.DatastoreType.Blob) {
                 s.properties[name] = {
                     instanceof: 'Buffer',

package/dist/datastore.model.d.ts

@@ -60,7 +60,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
      */
     experimentalCursorStream?: boolean;
     /**
-     * Applicable to `experimentalCursorStream`
+     * Applicable to `experimentalCursorStream`.
+     * Defines the size (limit) of each individual query.
      *
      * @default 1000
      */
@@ -78,6 +79,16 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
      * @default 1000
      */
     rssLimitMB?: number;
+    /**
+     * Applicable to `experimentalCursorStream`
+     * Default false.
+     * If true, stream will pause until consumer requests more data (via _read).
+     * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
+     * There will be gaps in time between "last query loaded" and "next query requested".
+     * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
+     * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+     */
+    singleBatchBuffer?: boolean;
     /**
      * Set to `true` to log additional debug info, when using experimentalCursorStream.
      *

package/dist/query.util.js

@@ -1,12 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbQueryToDatastoreQuery = void 0;
-// import Operator = google.datastore.v1.CompositeFilter.Operator
 const FNAME_MAP = {
     id: '__key__',
 };
 const OP_MAP = {
     '==': '=',
+    in: 'IN',
+    'not-in': 'NOT_IN',
 };
 function dbQueryToDatastoreQuery(dbQuery, emptyQuery) {
     let q = emptyQuery;

package/package.json

@@ -1,22 +1,25 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.20.1",
+  "version": "3.23.0",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"
   },
   "dependencies": {
-    "@google-cloud/datastore": "^6.0.0",
+    "@google-cloud/datastore": "^7.0.0",
     "@naturalcycles/db-lib": "^8.0.0",
     "@naturalcycles/js-lib": "^14.0.0",
     "@naturalcycles/nodejs-lib": "^12.0.0",
     "grpc": "^1.24.2"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "^12.0.1",
-    "@types/node": "^17.0.8",
+    "@naturalcycles/dev-lib": "^13.0.0",
+    "@types/node": "^18.7.2",
     "jest": "^28.1.0"
   },
+  "resolutions": {
+    "long": "^4.0.0"
+  },
   "files": [
     "dist",
     "src",

package/src/DatastoreStreamReadable.ts

@@ -79,6 +79,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         )
         this.push(null)
         this.done = true
+      } else if (this.opt.singleBatchBuffer) {
+        // here we don't start next query until we're asked (via next _read call)
+        // do, let's do nothing
       } else if (this.opt.rssLimitMB) {
         const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
 

package/src/datastore.db.ts

@@ -45,7 +45,7 @@ import { dbQueryToDatastoreQuery } from './query.util'
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500
 
-const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN']
+const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'much contention']
 // Examples of errors:
 // UNKNOWN: Stream removed
 
@@ -448,7 +448,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     stats
       .filter(s => !s.property_name.includes('.') && s.property_name !== 'id') // filter out objectify's "virtual properties"
       .forEach(stats => {
-        const { property_name: name, property_type: dtype } = stats
+        const { property_type: dtype } = stats
+        const name = stats.property_name as keyof ROW
 
         if (dtype === DatastoreType.Blob) {
           s.properties[name] = {
@@ -488,7 +489,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
             } as JsonSchemaNull
           }
         } else {
-          throw new Error(`Unknown Datastore Type '${stats.property_type}' for ${table}.${name}`)
+          throw new Error(
+            `Unknown Datastore Type '${stats.property_type}' for ${table}.${name as string}`,
+          )
         }
       })
 

package/src/datastore.model.ts

@@ -71,7 +71,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
   experimentalCursorStream?: boolean
 
   /**
-   * Applicable to `experimentalCursorStream`
+   * Applicable to `experimentalCursorStream`.
+   * Defines the size (limit) of each individual query.
    *
    * @default 1000
    */
@@ -91,6 +92,17 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
    */
   rssLimitMB?: number
 
+  /**
+   * Applicable to `experimentalCursorStream`
+   * Default false.
+   * If true, stream will pause until consumer requests more data (via _read).
+   * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
+   * There will be gaps in time between "last query loaded" and "next query requested".
+   * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
+   * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+   */
+  singleBatchBuffer?: boolean
+
   /**
    * Set to `true` to log additional debug info, when using experimentalCursorStream.
    *

package/src/query.util.ts

@@ -1,7 +1,6 @@
 import { Query } from '@google-cloud/datastore'
 import { DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
 import { ObjectWithId, StringMap } from '@naturalcycles/js-lib'
-// import Operator = google.datastore.v1.CompositeFilter.Operator
 
 const FNAME_MAP: StringMap = {
   id: '__key__',
@@ -9,6 +8,8 @@ const FNAME_MAP: StringMap = {
 
 const OP_MAP: Partial<Record<DBQueryFilterOperator, string>> = {
   '==': '=',
+  in: 'IN',
+  'not-in': 'NOT_IN',
 }
 
 export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(