@naturalcycles/datastore-lib 4.13.0 → 4.15.0

package/dist/datastore.db.js

@@ -4,7 +4,6 @@ import { _round } from '@naturalcycles/js-lib';
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { _errorDataAppend, TimeoutError } from '@naturalcycles/js-lib/error/error.util.js';
-import { commonLoggerMinLevel } from '@naturalcycles/js-lib/log';
 import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { pRetry, pRetryFn } from '@naturalcycles/js-lib/promise/pRetry.js';
@@ -12,7 +11,7 @@ import { pTimeout } from '@naturalcycles/js-lib/promise/pTimeout.js';
 import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors';
 import { DatastoreType } from './datastore.model.js';
-import { DatastoreStreamReadable } from './DatastoreStreamReadable.js';
+import { DatastoreStreamReadable } from './datastoreStreamReadable.js';
 import { dbQueryToDatastoreQuery } from './query.util.js';
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
@@ -219,11 +218,12 @@ export class DatastoreDB extends BaseCommonDB {
         void this.ds().then(async (ds) => {
             const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table), await this.getPropertyFilter());
             const opt = {
+                logger: this.cfg.logger,
                 ...this.cfg.streamOptions,
                 ..._opt,
             };
             (opt.experimentalCursorStream
-                ? new DatastoreStreamReadable(q, opt, commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'))
+                ? new DatastoreStreamReadable(q, opt)
                 : ds.runQueryStream(q, this.getRunQueryOptions(opt)))
                 .on('error', err => transform.emit('error', err))
                 .pipe(transform);

package/dist/datastore.model.d.ts

@@ -1,7 +1,7 @@
 import type { DatastoreOptions, Key } from '@google-cloud/datastore';
 import type { CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import type { NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib/types';
+import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log';
+import type { NumberOfSeconds, ObjectWithId, PositiveInteger } from '@naturalcycles/js-lib/types';
 export interface DatastorePayload<T = any> {
     key: Key;
     data: T;
@@ -30,6 +30,10 @@ export interface DatastoreDBCfg extends DatastoreOptions {
      * Default to `console`
      */
     logger?: CommonLogger;
+    /**
+     * Defaults to `log`.
+     */
+    logLevel?: CommonLogLevel;
     /**
      * Experimental option, currently only applies to `getByIds`.
      * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
@@ -59,38 +63,23 @@ export interface DatastoreDBStreamOptions extends DatastoreDBReadOptions {
      * Applicable to `experimentalCursorStream`.
      * Defines the size (limit) of each individual query.
      *
-     * @default 1000
-     */
-    batchSize?: number;
-    /**
-     * Applicable to `experimentalCursorStream`
-     *
-     * Set to a value (number of Megabytes) to control the peak RSS size.
-     * If limit is reached - streaming will pause until the stream keeps up, and then
-     * resumes.
-     *
-     * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
-     * when _read is called.
-     *
-     * @default 1000
+     * @default 1_000
      */
-    rssLimitMB?: number;
+    batchSize?: PositiveInteger;
     /**
-     * Applicable to `experimentalCursorStream`
-     * Default false.
-     * If true, stream will pause until consumer requests more data (via _read).
-     * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
-     * There will be gaps in time between "last query loaded" and "next query requested".
-     * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
-     * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+     * Defaults to 3x batchSize.
+     * Default batchSize is 1_000, so default highWaterMark is 3_000.
+     * Controls how many rows to have "buffered".
+     * Should be at least 1x batchSize, otherwise the stream will be "starving"
+     * between the queries.
      */
-    singleBatchBuffer?: boolean;
+    highWaterMark?: PositiveInteger;
+    logger?: CommonLogger;
     /**
-     * Set to `true` to log additional debug info, when using experimentalCursorStream.
-     *
-     * @default false
+     * Defaults to `log`.
+     * Set to `debug` to allow for extra debugging, e.g in experimentalCursorStream.
      */
-    debug?: boolean;
+    logLevel?: CommonLogLevel;
     /**
      * Default is undefined.
      * If set - sets a "safety timer", which will force call _read after the specified number of seconds.

package/dist/{DatastoreStreamReadable.d.ts → datastoreStreamReadable.d.ts}

@@ -1,32 +1,33 @@
 import { Readable } from 'node:stream';
 import type { Query } from '@google-cloud/datastore';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
 import type { DatastoreDBStreamOptions } from './datastore.model.js';
 export declare class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
     private q;
-    private logger;
+    private readonly table;
     private readonly originalLimit;
     private rowsRetrieved;
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    countReads: number;
     private endCursor?;
-    private running;
+    private queryIsRunning;
+    private paused;
     private done;
     private lastQueryDone?;
     private totalWait;
-    private readonly table;
     /**
      * Used to support maxWait
      */
     private lastReadTimestamp;
     private readonly maxWaitInterval;
     private readonly opt;
+    private readonly logger;
     private readonly dsOpt;
-    constructor(q: Query, opt: DatastoreDBStreamOptions, logger: CommonLogger);
-    private runNextQuery;
-    /**
-     * Counts how many times _read was called.
-     * For debugging.
-     */
-    count: number;
+    constructor(q: Query, opt: DatastoreDBStreamOptions);
     _read(): void;
+    private runNextQuery;
+    private runQuery;
 }

package/dist/datastoreStreamReadable.js

@@ -0,0 +1,182 @@
+import { Readable } from 'node:stream';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
+import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
+export class DatastoreStreamReadable extends Readable {
+    q;
+    table;
+    originalLimit;
+    rowsRetrieved = 0;
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    countReads = 0;
+    endCursor;
+    queryIsRunning = false;
+    paused = false;
+    done = false;
+    lastQueryDone;
+    totalWait = 0;
+    /**
+     * Used to support maxWait
+     */
+    lastReadTimestamp = 0;
+    maxWaitInterval;
+    opt;
+    logger;
+    dsOpt;
+    constructor(q, opt) {
+        // 1_000 was optimal in benchmarks
+        const { batchSize = 1000 } = opt;
+        const { highWaterMark = batchSize * 3 } = opt;
+        // Defaulting highWaterMark to 3x batchSize
+        super({ objectMode: true, highWaterMark });
+        this.q = q;
+        this.opt = {
+            ...opt,
+            batchSize,
+            highWaterMark,
+        };
+        this.dsOpt = {};
+        if (opt.readAt) {
+            // Datastore expects UnixTimestamp in milliseconds
+            this.dsOpt.readTime = opt.readAt * 1000;
+        }
+        const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+        this.logger = logger;
+        this.originalLimit = q.limitVal;
+        this.table = q.kinds[0];
+        logger.log(`!! using experimentalCursorStream`, {
+            table: this.table,
+            batchSize,
+            highWaterMark,
+        });
+        const { maxWait } = this.opt;
+        if (maxWait) {
+            logger.log(`!! ${this.table} maxWait ${maxWait}`);
+            this.maxWaitInterval = setInterval(() => {
+                const millisSinceLastRead = Date.now() - this.lastReadTimestamp;
+                if (millisSinceLastRead < maxWait * 1000) {
+                    logger.log(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
+                    return;
+                }
+                const { queryIsRunning, rowsRetrieved } = this;
+                logger.log(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+                    running: queryIsRunning,
+                    rowsRetrieved,
+                });
+                // force-trigger _read
+                // regardless of `running` status
+                this._read();
+            }, (maxWait * 1000) / 2);
+        }
+    }
+    _read() {
+        this.lastReadTimestamp = localTime.nowUnixMillis();
+        // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+        this.countReads++;
+        if (this.done) {
+            this.logger.warn(`!!! _read was called, but done==true`);
+            return;
+        }
+        if (this.paused) {
+            this.logger.log(`_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`);
+            this.paused = false;
+        }
+        if (this.queryIsRunning) {
+            this.logger.debug(`_read #${this.countReads}, queryIsRunning: true, doing nothing`);
+            return;
+        }
+        void this.runNextQuery().catch(err => {
+            this.logger.error('error in runNextQuery', err);
+            this.emit('error', err);
+        });
+    }
+    async runNextQuery() {
+        if (this.done)
+            return;
+        const { logger, table } = this;
+        if (this.lastQueryDone) {
+            const now = Date.now();
+            this.totalWait += now - this.lastQueryDone;
+        }
+        this.queryIsRunning = true;
+        let limit = this.opt.batchSize;
+        if (this.originalLimit) {
+            limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved);
+        }
+        let q = this.q.limit(limit);
+        if (this.endCursor) {
+            q = q.start(this.endCursor);
+        }
+        const started = localTime.nowUnixMillis();
+        const res = await this.runQuery(q);
+        const queryTook = Date.now() - started;
+        if (!res) {
+            // error already emitted in runQuery
+            return;
+        }
+        const rows = res[0];
+        const info = res[1];
+        this.rowsRetrieved += rows.length;
+        logger.log(`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+        this.endCursor = info.endCursor;
+        this.queryIsRunning = false; // ready to take more _reads
+        this.lastQueryDone = Date.now();
+        let shouldContinue = false;
+        for (const row of rows) {
+            shouldContinue = this.push(row);
+        }
+        if (!info.endCursor ||
+            info.moreResults === 'NO_MORE_RESULTS' ||
+            (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+            logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+            this.push(null);
+            this.done = true;
+            this.paused = false;
+            clearInterval(this.maxWaitInterval);
+            return;
+        }
+        if (shouldContinue) {
+            // Keep the stream flowing
+            logger.debug(`${table} continuing the stream`);
+            void this.runNextQuery();
+        }
+        else {
+            // Not starting the next query
+            if (this.paused) {
+                logger.debug(`${table} stream is already paused`);
+            }
+            else {
+                logger.log(`${table} pausing the stream`);
+                this.paused = true;
+            }
+        }
+    }
+    async runQuery(q) {
+        const { table, logger } = this;
+        try {
+            return await pRetry(async () => {
+                return await q.run(this.dsOpt);
+            }, {
+                name: `DatastoreStreamReadable.query(${table})`,
+                maxAttempts: 5,
+                delay: 5000,
+                delayMultiplier: 2,
+                logger,
+                timeout: 120_000, // 2 minutes
+            });
+        }
+        catch (err) {
+            logger.error(`DatastoreStreamReadable error!\n`, {
+                table,
+                rowsRetrieved: this.rowsRetrieved,
+            }, err);
+            this.emit('error', err);
+            clearInterval(this.maxWaitInterval);
+            return;
+        }
+    }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
   "type": "module",
-  "version": "4.13.0",
+  "version": "4.15.0",
   "description": "Opinionated library to work with Google Datastore, implements CommonDB",
   "dependencies": {
     "@google-cloud/datastore": "^10",

package/src/datastore.db.ts

@@ -29,7 +29,6 @@ import type {
   JsonSchemaString,
 } from '@naturalcycles/js-lib/json-schema'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { commonLoggerMinLevel } from '@naturalcycles/js-lib/log'
 import { _omit } from '@naturalcycles/js-lib/object/object.util.js'
 import type { PRetryOptions } from '@naturalcycles/js-lib/promise'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
@@ -54,7 +53,7 @@ import type {
   DatastoreStats,
 } from './datastore.model.js'
 import { DatastoreType } from './datastore.model.js'
-import { DatastoreStreamReadable } from './DatastoreStreamReadable.js'
+import { DatastoreStreamReadable } from './datastoreStreamReadable.js'
 import { dbQueryToDatastoreQuery } from './query.util.js'
 
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
@@ -344,16 +343,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       )
 
       const opt = {
+        logger: this.cfg.logger,
         ...this.cfg.streamOptions,
         ..._opt,
       }
 
       ;(opt.experimentalCursorStream
-        ? new DatastoreStreamReadable<ROW>(
-            q,
-            opt,
-            commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
-          )
+        ? new DatastoreStreamReadable<ROW>(q, opt)
         : (ds.runQueryStream(q, this.getRunQueryOptions(opt)) as ReadableTyped<ROW>)
       )
         .on('error', err => transform.emit('error', err))

package/src/datastore.model.ts

@@ -4,8 +4,8 @@ import type {
   CommonDBReadOptions,
   CommonDBSaveOptions,
 } from '@naturalcycles/db-lib'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import type { NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib/types'
+import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log'
+import type { NumberOfSeconds, ObjectWithId, PositiveInteger } from '@naturalcycles/js-lib/types'
 
 export interface DatastorePayload<T = any> {
   key: Key
@@ -41,6 +41,11 @@ export interface DatastoreDBCfg extends DatastoreOptions {
    */
   logger?: CommonLogger
 
+  /**
+   * Defaults to `log`.
+   */
+  logLevel?: CommonLogLevel
+
   /**
    * Experimental option, currently only applies to `getByIds`.
    * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
@@ -73,41 +78,26 @@ export interface DatastoreDBStreamOptions extends DatastoreDBReadOptions {
    * Applicable to `experimentalCursorStream`.
    * Defines the size (limit) of each individual query.
    *
-   * @default 1000
+   * @default 1_000
    */
-  batchSize?: number
+  batchSize?: PositiveInteger
 
   /**
-   * Applicable to `experimentalCursorStream`
-   *
-   * Set to a value (number of Megabytes) to control the peak RSS size.
-   * If limit is reached - streaming will pause until the stream keeps up, and then
-   * resumes.
-   *
-   * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
-   * when _read is called.
-   *
-   * @default 1000
+   * Defaults to 3x batchSize.
+   * Default batchSize is 1_000, so default highWaterMark is 3_000.
+   * Controls how many rows to have "buffered".
+   * Should be at least 1x batchSize, otherwise the stream will be "starving"
+   * between the queries.
    */
-  rssLimitMB?: number
+  highWaterMark?: PositiveInteger
 
-  /**
-   * Applicable to `experimentalCursorStream`
-   * Default false.
-   * If true, stream will pause until consumer requests more data (via _read).
-   * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
-   * There will be gaps in time between "last query loaded" and "next query requested".
-   * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
-   * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
-   */
-  singleBatchBuffer?: boolean
+  logger?: CommonLogger
 
   /**
-   * Set to `true` to log additional debug info, when using experimentalCursorStream.
-   *
-   * @default false
+   * Defaults to `log`.
+   * Set to `debug` to allow for extra debugging, e.g in experimentalCursorStream.
    */
-  debug?: boolean
+  logLevel?: CommonLogLevel
 
   /**
    * Default is undefined.

package/src/{DatastoreStreamReadable.ts → datastoreStreamReadable.ts}

@@ -1,42 +1,57 @@
 import { Readable } from 'node:stream'
 import type { Query } from '@google-cloud/datastore'
-import type { RunQueryInfo, RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
+import type {
+  RunQueryInfo,
+  RunQueryOptions,
+  RunQueryResponse,
+} from '@google-cloud/datastore/build/src/query.js'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
+import { type CommonLogger, createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
 import type { UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import type { DatastoreDBStreamOptions } from './datastore.model.js'
 
 export class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
+  private readonly table: string
   private readonly originalLimit: number
   private rowsRetrieved = 0
+  /**
+   * Counts how many times _read was called.
+   * For debugging.
+   */
+  countReads = 0
   private endCursor?: string
-  private running = false
+  private queryIsRunning = false
+  private paused = false
   private done = false
   private lastQueryDone?: number
   private totalWait = 0
-  private readonly table: string
   /**
    * Used to support maxWait
    */
   private lastReadTimestamp = 0 as UnixTimestampMillis
   private readonly maxWaitInterval: NodeJS.Timeout | undefined
 
-  private readonly opt: DatastoreDBStreamOptions & { batchSize: number }
+  private readonly opt: DatastoreDBStreamOptions & { batchSize: number; highWaterMark: number }
+  private readonly logger: CommonLogger
   private readonly dsOpt: RunQueryOptions
 
   constructor(
     private q: Query,
     opt: DatastoreDBStreamOptions,
-    private logger: CommonLogger,
   ) {
-    super({ objectMode: true })
+    // 1_000 was optimal in benchmarks
+    const { batchSize = 1000 } = opt
+    const { highWaterMark = batchSize * 3 } = opt
+    // Defaulting highWaterMark to 3x batchSize
+    super({ objectMode: true, highWaterMark })
 
     this.opt = {
-      rssLimitMB: 1000,
-      batchSize: 1000,
       ...opt,
+      batchSize,
+      highWaterMark,
     }
     this.dsOpt = {}
     if (opt.readAt) {
@@ -44,29 +59,35 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       this.dsOpt.readTime = opt.readAt * 1000
     }
 
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+    this.logger = logger
     this.originalLimit = q.limitVal
     this.table = q.kinds[0]!
 
-    logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`)
+    logger.log(`!! using experimentalCursorStream`, {
+      table: this.table,
+      batchSize,
+      highWaterMark,
+    })
 
     const { maxWait } = this.opt
     if (maxWait) {
-      this.logger.warn(`!! ${this.table} maxWait ${maxWait}`)
+      logger.log(`!! ${this.table} maxWait ${maxWait}`)
 
       this.maxWaitInterval = setInterval(
         () => {
           const millisSinceLastRead = Date.now() - this.lastReadTimestamp
 
           if (millisSinceLastRead < maxWait * 1000) {
-            this.logger.warn(
+            logger.log(
               `!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`,
             )
             return
           }
 
-          const { running, rowsRetrieved } = this
-          this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
-            running,
+          const { queryIsRunning, rowsRetrieved } = this
+          logger.log(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+            running: queryIsRunning,
             rowsRetrieved,
           })
 
@@ -79,16 +100,45 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     }
   }
 
+  override _read(): void {
+    this.lastReadTimestamp = localTime.nowUnixMillis()
+
+    // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+    this.countReads++
+
+    if (this.done) {
+      this.logger.warn(`!!! _read was called, but done==true`)
+      return
+    }
+
+    if (this.paused) {
+      this.logger.log(
+        `_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`,
+      )
+      this.paused = false
+    }
+
+    if (this.queryIsRunning) {
+      this.logger.debug(`_read #${this.countReads}, queryIsRunning: true, doing nothing`)
+      return
+    }
+
+    void this.runNextQuery().catch(err => {
+      this.logger.error('error in runNextQuery', err)
+      this.emit('error', err)
+    })
+  }
+
   private async runNextQuery(): Promise<void> {
     if (this.done) return
+    const { logger, table } = this
 
     if (this.lastQueryDone) {
       const now = Date.now()
       this.totalWait += now - this.lastQueryDone
     }
 
-    this.running = true
-    // console.log('running query...')
+    this.queryIsRunning = true
 
     let limit = this.opt.batchSize
 
@@ -96,66 +146,35 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved)
     }
 
-    // console.log(`limit: ${limit}`)
     let q = this.q.limit(limit)
     if (this.endCursor) {
       q = q.start(this.endCursor)
     }
 
-    let rows: T[] = []
-    let info: RunQueryInfo = {}
-
-    try {
-      await pRetry(
-        async () => {
-          const res = await q.run(this.dsOpt)
-          rows = res[0]
-          info = res[1]
-        },
-        {
-          name: `DatastoreStreamReadable.query(${this.table})`,
-          maxAttempts: 5,
-          delay: 5000,
-          delayMultiplier: 2,
-          logger: this.logger,
-          timeout: 120_000, // 2 minutes
-        },
-      )
-    } catch (err) {
-      console.log(
-        `DatastoreStreamReadable error!\n`,
-        {
-          table: this.table,
-          rowsRetrieved: this.rowsRetrieved,
-        },
-        err,
-      )
-      this.emit('error', err)
-      clearInterval(this.maxWaitInterval)
+    const started = localTime.nowUnixMillis()
+    const res = await this.runQuery(q)
+    const queryTook = Date.now() - started
+    if (!res) {
+      // error already emitted in runQuery
       return
     }
+    const rows: T[] = res[0]
+    const info: RunQueryInfo = res[1]
 
     this.rowsRetrieved += rows.length
-    this.logger.log(
-      `${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
+    logger.log(
+      `${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
         this.totalWait,
       )}`,
-      info.moreResults,
     )
 
-    if (!rows.length) {
-      this.logger.warn(
-        `${this.table} got 0 rows, totalWait: ${_ms(this.totalWait)}`,
-        info.moreResults,
-      )
-    }
-
     this.endCursor = info.endCursor
-    this.running = false // ready to take more _reads
+    this.queryIsRunning = false // ready to take more _reads
     this.lastQueryDone = Date.now()
+    let shouldContinue = false
 
     for (const row of rows) {
-      this.push(row)
+      shouldContinue = this.push(row)
     }
 
     if (
@@ -163,52 +182,60 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       info.moreResults === 'NO_MORE_RESULTS' ||
       (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
     ) {
-      this.logger.log(
+      logger.log(
         `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
       )
       this.push(null)
       this.done = true
+      this.paused = false
       clearInterval(this.maxWaitInterval)
-    } else if (this.opt.singleBatchBuffer) {
-      // here we don't start next query until we're asked (via next _read call)
-      // do, let's do nothing
-    } else if (this.opt.rssLimitMB) {
-      const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
-
-      if (rssMB <= this.opt.rssLimitMB) {
-        void this.runNextQuery()
+      return
+    }
+
+    if (shouldContinue) {
+      // Keep the stream flowing
+      logger.debug(`${table} continuing the stream`)
+      void this.runNextQuery()
+    } else {
+      // Not starting the next query
+      if (this.paused) {
+        logger.debug(`${table} stream is already paused`)
       } else {
-        this.logger.warn(
-          `${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`,
-        )
+        logger.log(`${table} pausing the stream`)
+        this.paused = true
       }
     }
   }
 
-  /**
-   * Counts how many times _read was called.
-   * For debugging.
-   */
-  count = 0
-
-  override _read(): void {
-    this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+  private async runQuery(q: Query): Promise<RunQueryResponse | undefined> {
+    const { table, logger } = this
 
-    // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
-    this.count++
-
-    if (this.done) {
-      this.logger.warn(`!!! _read was called, but done==true`)
+    try {
+      return await pRetry(
+        async () => {
+          return await q.run(this.dsOpt)
+        },
+        {
+          name: `DatastoreStreamReadable.query(${table})`,
+          maxAttempts: 5,
+          delay: 5000,
+          delayMultiplier: 2,
+          logger,
+          timeout: 120_000, // 2 minutes
+        },
+      )
+    } catch (err) {
+      logger.error(
+        `DatastoreStreamReadable error!\n`,
+        {
+          table,
+          rowsRetrieved: this.rowsRetrieved,
+        },
+        err,
+      )
+      this.emit('error', err)
+      clearInterval(this.maxWaitInterval)
       return
     }
-
-    if (!this.running) {
-      void this.runNextQuery().catch(err => {
-        console.log('error in runNextQuery', err)
-        this.emit('error', err)
-      })
-    } else {
-      this.logger.log(`_read ${this.count}, wasRunning: true`)
-    }
   }
 }

package/dist/DatastoreStreamReadable.js

@@ -1,158 +0,0 @@
-import { Readable } from 'node:stream';
-import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
-import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
-export class DatastoreStreamReadable extends Readable {
-    q;
-    logger;
-    originalLimit;
-    rowsRetrieved = 0;
-    endCursor;
-    running = false;
-    done = false;
-    lastQueryDone;
-    totalWait = 0;
-    table;
-    /**
-     * Used to support maxWait
-     */
-    lastReadTimestamp = 0;
-    maxWaitInterval;
-    opt;
-    dsOpt;
-    constructor(q, opt, logger) {
-        super({ objectMode: true });
-        this.q = q;
-        this.logger = logger;
-        this.opt = {
-            rssLimitMB: 1000,
-            batchSize: 1000,
-            ...opt,
-        };
-        this.dsOpt = {};
-        if (opt.readAt) {
-            // Datastore expects UnixTimestamp in milliseconds
-            this.dsOpt.readTime = opt.readAt * 1000;
-        }
-        this.originalLimit = q.limitVal;
-        this.table = q.kinds[0];
-        logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`);
-        const { maxWait } = this.opt;
-        if (maxWait) {
-            this.logger.warn(`!! ${this.table} maxWait ${maxWait}`);
-            this.maxWaitInterval = setInterval(() => {
-                const millisSinceLastRead = Date.now() - this.lastReadTimestamp;
-                if (millisSinceLastRead < maxWait * 1000) {
-                    this.logger.warn(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
-                    return;
-                }
-                const { running, rowsRetrieved } = this;
-                this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
-                    running,
-                    rowsRetrieved,
-                });
-                // force-trigger _read
-                // regardless of `running` status
-                this._read();
-            }, (maxWait * 1000) / 2);
-        }
-    }
-    async runNextQuery() {
-        if (this.done)
-            return;
-        if (this.lastQueryDone) {
-            const now = Date.now();
-            this.totalWait += now - this.lastQueryDone;
-        }
-        this.running = true;
-        // console.log('running query...')
-        let limit = this.opt.batchSize;
-        if (this.originalLimit) {
-            limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved);
-        }
-        // console.log(`limit: ${limit}`)
-        let q = this.q.limit(limit);
-        if (this.endCursor) {
-            q = q.start(this.endCursor);
-        }
-        let rows = [];
-        let info = {};
-        try {
-            await pRetry(async () => {
-                const res = await q.run(this.dsOpt);
-                rows = res[0];
-                info = res[1];
-            }, {
-                name: `DatastoreStreamReadable.query(${this.table})`,
-                maxAttempts: 5,
-                delay: 5000,
-                delayMultiplier: 2,
-                logger: this.logger,
-                timeout: 120_000, // 2 minutes
-            });
-        }
-        catch (err) {
-            console.log(`DatastoreStreamReadable error!\n`, {
-                table: this.table,
-                rowsRetrieved: this.rowsRetrieved,
-            }, err);
-            this.emit('error', err);
-            clearInterval(this.maxWaitInterval);
-            return;
-        }
-        this.rowsRetrieved += rows.length;
-        this.logger.log(`${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`, info.moreResults);
-        if (!rows.length) {
-            this.logger.warn(`${this.table} got 0 rows, totalWait: ${_ms(this.totalWait)}`, info.moreResults);
-        }
-        this.endCursor = info.endCursor;
-        this.running = false; // ready to take more _reads
-        this.lastQueryDone = Date.now();
-        for (const row of rows) {
-            this.push(row);
-        }
-        if (!info.endCursor ||
-            info.moreResults === 'NO_MORE_RESULTS' ||
-            (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-            this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
-            this.push(null);
-            this.done = true;
-            clearInterval(this.maxWaitInterval);
-        }
-        else if (this.opt.singleBatchBuffer) {
-            // here we don't start next query until we're asked (via next _read call)
-            // do, let's do nothing
-        }
-        else if (this.opt.rssLimitMB) {
-            const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
-            if (rssMB <= this.opt.rssLimitMB) {
-                void this.runNextQuery();
-            }
-            else {
-                this.logger.warn(`${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
-            }
-        }
-    }
-    /**
-     * Counts how many times _read was called.
-     * For debugging.
-     */
-    count = 0;
-    _read() {
-        this.lastReadTimestamp = Date.now();
-        // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
-        this.count++;
-        if (this.done) {
-            this.logger.warn(`!!! _read was called, but done==true`);
-            return;
-        }
-        if (!this.running) {
-            void this.runNextQuery().catch(err => {
-                console.log('error in runNextQuery', err);
-                this.emit('error', err);
-            });
-        }
-        else {
-            this.logger.log(`_read ${this.count}, wasRunning: true`);
-        }
-    }
-}