@naturalcycles/datastore-lib 3.28.3 → 3.30.0

package/dist/DatastoreStreamReadable.d.ts

@@ -15,9 +15,18 @@ export declare class DatastoreStreamReadable<T = any> extends Readable implement
     private lastQueryDone?;
     private totalWait;
     private table;
+    /**
+     * Used to support maxWait
+     */
+    private lastReadTimestamp;
+    private maxWaitInterval;
     private opt;
     constructor(q: Query, opt: DatastoreDBStreamOptions, logger: CommonLogger);
     private runNextQuery;
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
     count: number;
     _read(): void;
 }

package/dist/DatastoreStreamReadable.js

@@ -12,7 +12,15 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
         this.running = false;
         this.done = false;
         this.totalWait = 0;
-        this.count = 0; // use for debugging
+        /**
+         * Used to support maxWait
+         */
+        this.lastReadTimestamp = 0;
+        /**
+         * Counts how many times _read was called.
+         * For debugging.
+         */
+        this.count = 0;
         this.opt = {
             rssLimitMB: 1000,
             batchSize: 1000,
@@ -21,6 +29,25 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
         this.originalLimit = q.limitVal;
         this.table = q.kinds[0];
         logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`);
+        const { maxWait } = this.opt;
+        if (maxWait) {
+            this.logger.warn(`!! ${this.table} maxWait ${maxWait}`);
+            this.maxWaitInterval = setInterval(() => {
+                const millisSinceLastRead = Date.now() - this.lastReadTimestamp;
+                if (millisSinceLastRead < maxWait * 1000) {
+                    this.logger.warn(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
+                    return;
+                }
+                const { running, rowsRetrieved } = this;
+                this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+                    running,
+                    rowsRetrieved,
+                });
+                // force-trigger _read
+                // regardless of `running` status
+                this._read();
+            }, (maxWait * 1000) / 2);
+        }
     }
     async runNextQuery() {
         if (this.done)
@@ -53,6 +80,7 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
                 delay: 5000,
                 delayMultiplier: 2,
                 logger: this.logger,
+                timeout: 120000, // 2 minutes
             });
         }
         catch (err) {
@@ -61,6 +89,7 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
                 rowsRetrieved: this.rowsRetrieved,
             }, err);
             this.emit('error', err);
+            clearInterval(this.maxWaitInterval);
             return;
         }
         this.rowsRetrieved += rows.length;
@@ -75,6 +104,7 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
             this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
             this.push(null);
             this.done = true;
+            clearInterval(this.maxWaitInterval);
         }
         else if (this.opt.singleBatchBuffer) {
             // here we don't start next query until we're asked (via next _read call)
@@ -91,11 +121,9 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
         }
     }
     _read() {
+        this.lastReadTimestamp = Date.now();
         // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
         this.count++;
-        if (this.running) {
-            this.logger.log(`_read ${this.count}, wasRunning: true`);
-        }
         if (this.done) {
             this.logger.warn(`!!! _read was called, but done==true`);
             return;
@@ -103,6 +131,9 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
         if (!this.running) {
             void this.runNextQuery();
         }
+        else {
+            this.logger.log(`_read ${this.count}, wasRunning: true`);
+        }
     }
 }
 exports.DatastoreStreamReadable = DatastoreStreamReadable;

package/dist/datastore.db.js

@@ -6,8 +6,8 @@ const datastore_1 = require("@google-cloud/datastore");
 const db_lib_1 = require("@naturalcycles/db-lib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
-const datastore_model_1 = require("./datastore.model");
 const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
+const datastore_model_1 = require("./datastore.model");
 const query_util_1 = require("./query.util");
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;

package/dist/datastore.model.d.ts

@@ -1,6 +1,6 @@
 import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore';
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
-import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
+import { CommonLogger, NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib';
 export interface DatastorePayload<T = any> {
     key: Key;
     data: T;
@@ -90,6 +90,14 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
      * @default false
      */
     debug?: boolean;
+    /**
+     * Default is undefined.
+     * If set - sets a "safety timer", which will force call _read after the specified number of seconds.
+     * This is to prevent possible "dead-lock"/race-condition that would make the stream "hang".
+     *
+     * @experimental
+     */
+    maxWait?: NumberOfSeconds;
 }
 export interface DatastoreDBOptions extends CommonDBOptions {
     tx?: Transaction;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.28.3",
+  "version": "3.30.0",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"

package/src/DatastoreStreamReadable.ts

@@ -1,7 +1,7 @@
 import { Readable } from 'node:stream'
-import type { RunQueryInfo } from '@google-cloud/datastore/build/src/query'
 import { Query } from '@google-cloud/datastore'
-import { _ms, CommonLogger, pRetry } from '@naturalcycles/js-lib'
+import type { RunQueryInfo } from '@google-cloud/datastore/build/src/query'
+import { _ms, CommonLogger, pRetry, UnixTimestampMillisNumber } from '@naturalcycles/js-lib'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import type { DatastoreDBStreamOptions } from './datastore.model'
 
@@ -14,6 +14,11 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
   private lastQueryDone?: number
   private totalWait = 0
   private table: string
+  /**
+   * Used to support maxWait
+   */
+  private lastReadTimestamp: UnixTimestampMillisNumber = 0
+  private maxWaitInterval: NodeJS.Timeout | undefined
 
   private opt: DatastoreDBStreamOptions & { batchSize: number }
 
@@ -34,6 +39,35 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     this.table = q.kinds[0]!
 
     logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`)
+
+    const { maxWait } = this.opt
+    if (maxWait) {
+      this.logger.warn(`!! ${this.table} maxWait ${maxWait}`)
+
+      this.maxWaitInterval = setInterval(
+        () => {
+          const millisSinceLastRead = Date.now() - this.lastReadTimestamp
+
+          if (millisSinceLastRead < maxWait * 1000) {
+            this.logger.warn(
+              `!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`,
+            )
+            return
+          }
+
+          const { running, rowsRetrieved } = this
+          this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+            running,
+            rowsRetrieved,
+          })
+
+          // force-trigger _read
+          // regardless of `running` status
+          this._read()
+        },
+        (maxWait * 1000) / 2,
+      )
+    }
   }
 
   private async runNextQuery(): Promise<void> {
@@ -75,6 +109,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
           delay: 5000,
           delayMultiplier: 2,
           logger: this.logger,
+          timeout: 120_000, // 2 minutes
         },
       )
     } catch (err) {
@@ -87,6 +122,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         err,
       )
       this.emit('error', err)
+      clearInterval(this.maxWaitInterval)
       return
     }
 
@@ -114,6 +150,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       )
       this.push(null)
       this.done = true
+      clearInterval(this.maxWaitInterval)
     } else if (this.opt.singleBatchBuffer) {
       // here we don't start next query until we're asked (via next _read call)
       // do, let's do nothing
@@ -128,14 +165,17 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     }
   }
 
-  count = 0 // use for debugging
+  /**
+   * Counts how many times _read was called.
+   * For debugging.
+   */
+  count = 0
 
   override _read(): void {
+    this.lastReadTimestamp = Date.now()
+
     // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
     this.count++
-    if (this.running) {
-      this.logger.log(`_read ${this.count}, wasRunning: true`)
-    }
 
     if (this.done) {
       this.logger.warn(`!!! _read was called, but done==true`)
@@ -144,6 +184,8 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
 
     if (!this.running) {
       void this.runNextQuery()
+    } else {
+      this.logger.log(`_read ${this.count}, wasRunning: true`)
     }
   }
 }

package/src/datastore.db.ts

@@ -30,8 +30,8 @@ import {
   pRetry,
   PRetryOptions,
 } from '@naturalcycles/js-lib'
-import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { boldWhite } from '@naturalcycles/nodejs-lib'
+import { ReadableTyped, boldWhite } from '@naturalcycles/nodejs-lib'
+import { DatastoreStreamReadable } from './DatastoreStreamReadable'
 import {
   DatastoreDBCfg,
   DatastoreDBOptions,
@@ -42,7 +42,6 @@ import {
   DatastoreStats,
   DatastoreType,
 } from './datastore.model'
-import { DatastoreStreamReadable } from './DatastoreStreamReadable'
 import { dbQueryToDatastoreQuery } from './query.util'
 
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.

package/src/datastore.model.ts

@@ -1,6 +1,6 @@
 import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore'
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib'
-import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib'
+import { CommonLogger, NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib'
 
 export interface DatastorePayload<T = any> {
   key: Key
@@ -103,6 +103,15 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
    * @default false
    */
   debug?: boolean
+
+  /**
+   * Default is undefined.
+   * If set - sets a "safety timer", which will force call _read after the specified number of seconds.
+   * This is to prevent possible "dead-lock"/race-condition that would make the stream "hang".
+   *
+   * @experimental
+   */
+  maxWait?: NumberOfSeconds
 }
 
 export interface DatastoreDBOptions extends CommonDBOptions {