@naturalcycles/datastore-lib 4.12.0 → 4.14.0

package/dist/datastore.db.d.ts

@@ -27,7 +27,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     private getDatastoreLib;
     ping(): Promise<void>;
     getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: DatastoreDBReadOptions): Promise<ROW[]>;
-    multiGetByIds<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: DatastoreDBReadOptions): Promise<StringMap<ROW[]>>;
+    multiGet<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: DatastoreDBReadOptions): Promise<StringMap<ROW[]>>;
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<number>;
     private runDatastoreQuery;
@@ -42,7 +42,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
      * regardless if they were actually deleted or not.
      */
     deleteByIds(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<number>;
-    multiDeleteByIds(map: StringMap<string[]>, opt?: DatastoreDBOptions): Promise<number>;
+    multiDelete(map: StringMap<string[]>, opt?: DatastoreDBOptions): Promise<number>;
     createTransaction(opt?: CommonDBTransactionOptions): Promise<DatastoreDBTransaction>;
     runInTransaction(fn: DBTransactionFn, opt?: CommonDBTransactionOptions): Promise<void>;
     getAllStats(): Promise<DatastoreStats[]>;

package/dist/datastore.db.js

@@ -12,7 +12,7 @@ import { pTimeout } from '@naturalcycles/js-lib/promise/pTimeout.js';
 import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors';
 import { DatastoreType } from './datastore.model.js';
-import { DatastoreStreamReadable } from './DatastoreStreamReadable.js';
+import { DatastoreStreamReadable } from './datastoreStreamReadable.js';
 import { dbQueryToDatastoreQuery } from './query.util.js';
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
@@ -148,7 +148,7 @@ export class DatastoreDB extends BaseCommonDB {
             // same ids are not expected here
             .sort(idComparator));
     }
-    async multiGetByIds(map, opt = {}) {
+    async multiGet(map, opt = {}) {
         const result = {};
         const ds = await this.ds();
         const dsOpt = this.getRunQueryOptions(opt);
@@ -296,7 +296,7 @@ export class DatastoreDB extends BaseCommonDB {
         });
         return ids.length;
     }
-    async multiDeleteByIds(map, opt = {}) {
+    async multiDelete(map, opt = {}) {
         const ds = await this.ds();
         const keys = [];
         for (const [table, ids] of _stringMapEntries(map)) {

package/dist/datastore.model.d.ts

@@ -1,7 +1,7 @@
 import type { DatastoreOptions, Key } from '@google-cloud/datastore';
 import type { CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import type { NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib/types';
+import type { NumberOfSeconds, ObjectWithId, PositiveInteger } from '@naturalcycles/js-lib/types';
 export interface DatastorePayload<T = any> {
     key: Key;
     data: T;
@@ -59,32 +59,17 @@ export interface DatastoreDBStreamOptions extends DatastoreDBReadOptions {
      * Applicable to `experimentalCursorStream`.
      * Defines the size (limit) of each individual query.
      *
-     * @default 1000
+     * @default 1_000
      */
-    batchSize?: number;
+    batchSize?: PositiveInteger;
     /**
-     * Applicable to `experimentalCursorStream`
-     *
-     * Set to a value (number of Megabytes) to control the peak RSS size.
-     * If limit is reached - streaming will pause until the stream keeps up, and then
-     * resumes.
-     *
-     * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
-     * when _read is called.
-     *
-     * @default 1000
-     */
-    rssLimitMB?: number;
-    /**
-     * Applicable to `experimentalCursorStream`
-     * Default false.
-     * If true, stream will pause until consumer requests more data (via _read).
-     * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
-     * There will be gaps in time between "last query loaded" and "next query requested".
-     * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
-     * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+     * Defaults to 3x batchSize.
+     * Default batchSize is 1_000, so default highWaterMark is 3_000.
+     * Controls how many rows to have "buffered".
+     * Should be at least 1x batchSize, otherwise the stream will be "starving"
+     * between the queries.
      */
-    singleBatchBuffer?: boolean;
+    highWaterMark?: PositiveInteger;
     /**
      * Set to `true` to log additional debug info, when using experimentalCursorStream.
      *

package/dist/{DatastoreStreamReadable.d.ts → datastoreStreamReadable.d.ts}

@@ -6,14 +6,20 @@ import type { DatastoreDBStreamOptions } from './datastore.model.js';
 export declare class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
     private q;
     private logger;
+    private readonly table;
     private readonly originalLimit;
     private rowsRetrieved;
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    countReads: number;
     private endCursor?;
-    private running;
+    private queryIsRunning;
+    private paused;
     private done;
     private lastQueryDone?;
     private totalWait;
-    private readonly table;
     /**
      * Used to support maxWait
      */
@@ -22,11 +28,7 @@ export declare class DatastoreStreamReadable<T = any> extends Readable implement
     private readonly opt;
     private readonly dsOpt;
     constructor(q: Query, opt: DatastoreDBStreamOptions, logger: CommonLogger);
-    private runNextQuery;
-    /**
-     * Counts how many times _read was called.
-     * For debugging.
-     */
-    count: number;
     _read(): void;
+    private runNextQuery;
+    private runQuery;
 }

package/dist/{DatastoreStreamReadable.js → datastoreStreamReadable.js}

@@ -1,17 +1,24 @@
 import { Readable } from 'node:stream';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
 export class DatastoreStreamReadable extends Readable {
     q;
     logger;
+    table;
     originalLimit;
     rowsRetrieved = 0;
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    countReads = 0;
     endCursor;
-    running = false;
+    queryIsRunning = false;
+    paused = false;
     done = false;
     lastQueryDone;
     totalWait = 0;
-    table;
     /**
      * Used to support maxWait
      */
@@ -20,13 +27,17 @@ export class DatastoreStreamReadable extends Readable {
     opt;
     dsOpt;
     constructor(q, opt, logger) {
-        super({ objectMode: true });
+        // 1_000 was optimal in benchmarks
+        const { batchSize = 1000 } = opt;
+        const { highWaterMark = batchSize * 3 } = opt;
+        // Defaulting highWaterMark to 3x batchSize
+        super({ objectMode: true, highWaterMark });
         this.q = q;
         this.logger = logger;
         this.opt = {
-            rssLimitMB: 1000,
-            batchSize: 1000,
             ...opt,
+            batchSize,
+            highWaterMark,
         };
         this.dsOpt = {};
         if (opt.readAt) {
@@ -35,19 +46,23 @@ export class DatastoreStreamReadable extends Readable {
         }
         this.originalLimit = q.limitVal;
         this.table = q.kinds[0];
-        logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`);
+        logger.warn(`!! using experimentalCursorStream`, {
+            table: this.table,
+            batchSize,
+            highWaterMark,
+        });
         const { maxWait } = this.opt;
         if (maxWait) {
-            this.logger.warn(`!! ${this.table} maxWait ${maxWait}`);
+            logger.warn(`!! ${this.table} maxWait ${maxWait}`);
             this.maxWaitInterval = setInterval(() => {
                 const millisSinceLastRead = Date.now() - this.lastReadTimestamp;
                 if (millisSinceLastRead < maxWait * 1000) {
-                    this.logger.warn(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
+                    logger.warn(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
                     return;
                 }
-                const { running, rowsRetrieved } = this;
-                this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
-                    running,
+                const { queryIsRunning, rowsRetrieved } = this;
+                logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+                    running: queryIsRunning,
                     rowsRetrieved,
                 });
                 // force-trigger _read
@@ -56,103 +71,110 @@ export class DatastoreStreamReadable extends Readable {
             }, (maxWait * 1000) / 2);
         }
     }
+    _read() {
+        this.lastReadTimestamp = Date.now();
+        // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+        this.countReads++;
+        if (this.done) {
+            this.logger.warn(`!!! _read was called, but done==true`);
+            return;
+        }
+        if (this.paused) {
+            this.logger.log(`_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`);
+            this.paused = false;
+        }
+        if (this.queryIsRunning) {
+            this.logger.log(`_read #${this.countReads}, queryIsRunning: true, doing nothing`);
+            return;
+        }
+        void this.runNextQuery().catch(err => {
+            console.log('error in runNextQuery', err);
+            this.emit('error', err);
+        });
+    }
     async runNextQuery() {
         if (this.done)
             return;
+        const { logger, table } = this;
         if (this.lastQueryDone) {
             const now = Date.now();
             this.totalWait += now - this.lastQueryDone;
         }
-        this.running = true;
-        // console.log('running query...')
+        this.queryIsRunning = true;
         let limit = this.opt.batchSize;
         if (this.originalLimit) {
             limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved);
         }
-        // console.log(`limit: ${limit}`)
         let q = this.q.limit(limit);
         if (this.endCursor) {
             q = q.start(this.endCursor);
         }
-        let rows = [];
-        let info = {};
-        try {
-            await pRetry(async () => {
-                const res = await q.run(this.dsOpt);
-                rows = res[0];
-                info = res[1];
-            }, {
-                name: `DatastoreStreamReadable.query(${this.table})`,
-                maxAttempts: 5,
-                delay: 5000,
-                delayMultiplier: 2,
-                logger: this.logger,
-                timeout: 120_000, // 2 minutes
-            });
-        }
-        catch (err) {
-            console.log(`DatastoreStreamReadable error!\n`, {
-                table: this.table,
-                rowsRetrieved: this.rowsRetrieved,
-            }, err);
-            this.emit('error', err);
-            clearInterval(this.maxWaitInterval);
+        const started = localTime.nowUnixMillis();
+        const res = await this.runQuery(q);
+        const queryTook = Date.now() - started;
+        if (!res) {
+            // error already emitted in runQuery
             return;
         }
+        const rows = res[0];
+        const info = res[1];
         this.rowsRetrieved += rows.length;
-        this.logger.log(`${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`, info.moreResults);
-        if (!rows.length) {
-            this.logger.warn(`${this.table} got 0 rows, totalWait: ${_ms(this.totalWait)}`, info.moreResults);
-        }
+        logger.log(`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
         this.endCursor = info.endCursor;
-        this.running = false; // ready to take more _reads
+        this.queryIsRunning = false; // ready to take more _reads
         this.lastQueryDone = Date.now();
+        let shouldContinue = false;
         for (const row of rows) {
-            this.push(row);
+            shouldContinue = this.push(row);
         }
         if (!info.endCursor ||
             info.moreResults === 'NO_MORE_RESULTS' ||
             (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-            this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+            logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
             this.push(null);
             this.done = true;
+            this.paused = false;
             clearInterval(this.maxWaitInterval);
+            return;
         }
-        else if (this.opt.singleBatchBuffer) {
-            // here we don't start next query until we're asked (via next _read call)
-            // do, let's do nothing
+        if (shouldContinue) {
+            // Keep the stream flowing
+            logger.log(`${table} continuing the stream`);
+            void this.runNextQuery();
         }
-        else if (this.opt.rssLimitMB) {
-            const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
-            if (rssMB <= this.opt.rssLimitMB) {
-                void this.runNextQuery();
+        else {
+            // Not starting the next query
+            if (this.paused) {
+                logger.log(`${table} stream is already paused`);
             }
             else {
-                this.logger.warn(`${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
+                logger.log(`${table} pausing the stream`);
+                this.paused = true;
             }
         }
     }
-    /**
-     * Counts how many times _read was called.
-     * For debugging.
-     */
-    count = 0;
-    _read() {
-        this.lastReadTimestamp = Date.now();
-        // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
-        this.count++;
-        if (this.done) {
-            this.logger.warn(`!!! _read was called, but done==true`);
-            return;
-        }
-        if (!this.running) {
-            void this.runNextQuery().catch(err => {
-                console.log('error in runNextQuery', err);
-                this.emit('error', err);
+    async runQuery(q) {
+        const { table, logger } = this;
+        try {
+            return await pRetry(async () => {
+                return await q.run(this.dsOpt);
+            }, {
+                name: `DatastoreStreamReadable.query(${table})`,
+                maxAttempts: 5,
+                delay: 5000,
+                delayMultiplier: 2,
+                logger,
+                timeout: 120_000, // 2 minutes
             });
         }
-        else {
-            this.logger.log(`_read ${this.count}, wasRunning: true`);
+        catch (err) {
+            console.log(`DatastoreStreamReadable error!\n`, {
+                table,
+                rowsRetrieved: this.rowsRetrieved,
+            }, err);
+            this.emit('error', err);
+            clearInterval(this.maxWaitInterval);
+            return;
         }
     }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
   "type": "module",
-  "version": "4.12.0",
+  "version": "4.14.0",
   "description": "Opinionated library to work with Google Datastore, implements CommonDB",
   "dependencies": {
     "@google-cloud/datastore": "^10",

package/src/datastore.db.ts

@@ -54,7 +54,7 @@ import type {
   DatastoreStats,
 } from './datastore.model.js'
 import { DatastoreType } from './datastore.model.js'
-import { DatastoreStreamReadable } from './DatastoreStreamReadable.js'
+import { DatastoreStreamReadable } from './datastoreStreamReadable.js'
 import { dbQueryToDatastoreQuery } from './query.util.js'
 
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
@@ -229,7 +229,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     )
   }
 
-  override async multiGetByIds<ROW extends ObjectWithId>(
+  override async multiGet<ROW extends ObjectWithId>(
     map: StringMap<string[]>,
     opt: DatastoreDBReadOptions = {},
   ): Promise<StringMap<ROW[]>> {
@@ -470,7 +470,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return ids.length
   }
 
-  override async multiDeleteByIds(
+  override async multiDelete(
     map: StringMap<string[]>,
     opt: DatastoreDBOptions = {},
   ): Promise<number> {

package/src/datastore.model.ts

@@ -5,7 +5,7 @@ import type {
   CommonDBSaveOptions,
 } from '@naturalcycles/db-lib'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import type { NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib/types'
+import type { NumberOfSeconds, ObjectWithId, PositiveInteger } from '@naturalcycles/js-lib/types'
 
 export interface DatastorePayload<T = any> {
   key: Key
@@ -73,34 +73,18 @@ export interface DatastoreDBStreamOptions extends DatastoreDBReadOptions {
    * Applicable to `experimentalCursorStream`.
    * Defines the size (limit) of each individual query.
    *
-   * @default 1000
+   * @default 1_000
    */
-  batchSize?: number
+  batchSize?: PositiveInteger
 
   /**
-   * Applicable to `experimentalCursorStream`
-   *
-   * Set to a value (number of Megabytes) to control the peak RSS size.
-   * If limit is reached - streaming will pause until the stream keeps up, and then
-   * resumes.
-   *
-   * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
-   * when _read is called.
-   *
-   * @default 1000
-   */
-  rssLimitMB?: number
-
-  /**
-   * Applicable to `experimentalCursorStream`
-   * Default false.
-   * If true, stream will pause until consumer requests more data (via _read).
-   * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
-   * There will be gaps in time between "last query loaded" and "next query requested".
-   * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
-   * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+   * Defaults to 3x batchSize.
+   * Default batchSize is 1_000, so default highWaterMark is 3_000.
+   * Controls how many rows to have "buffered".
+   * Should be at least 1x batchSize, otherwise the stream will be "starving"
+   * between the queries.
    */
-  singleBatchBuffer?: boolean
+  highWaterMark?: PositiveInteger
 
   /**
    * Set to `true` to log additional debug info, when using experimentalCursorStream.

package/src/{DatastoreStreamReadable.ts → datastoreStreamReadable.ts}

@@ -1,6 +1,11 @@
 import { Readable } from 'node:stream'
 import type { Query } from '@google-cloud/datastore'
-import type { RunQueryInfo, RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
+import type {
+  RunQueryInfo,
+  RunQueryOptions,
+  RunQueryResponse,
+} from '@google-cloud/datastore/build/src/query.js'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
@@ -9,21 +14,27 @@ import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import type { DatastoreDBStreamOptions } from './datastore.model.js'
 
 export class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
+  private readonly table: string
   private readonly originalLimit: number
   private rowsRetrieved = 0
+  /**
+   * Counts how many times _read was called.
+   * For debugging.
+   */
+  countReads = 0
   private endCursor?: string
-  private running = false
+  private queryIsRunning = false
+  private paused = false
   private done = false
   private lastQueryDone?: number
   private totalWait = 0
-  private readonly table: string
   /**
    * Used to support maxWait
    */
   private lastReadTimestamp = 0 as UnixTimestampMillis
   private readonly maxWaitInterval: NodeJS.Timeout | undefined
 
-  private readonly opt: DatastoreDBStreamOptions & { batchSize: number }
+  private readonly opt: DatastoreDBStreamOptions & { batchSize: number; highWaterMark: number }
   private readonly dsOpt: RunQueryOptions
 
   constructor(
@@ -31,12 +42,16 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     opt: DatastoreDBStreamOptions,
     private logger: CommonLogger,
   ) {
-    super({ objectMode: true })
+    // 1_000 was optimal in benchmarks
+    const { batchSize = 1000 } = opt
+    const { highWaterMark = batchSize * 3 } = opt
+    // Defaulting highWaterMark to 3x batchSize
+    super({ objectMode: true, highWaterMark })
 
     this.opt = {
-      rssLimitMB: 1000,
-      batchSize: 1000,
       ...opt,
+      batchSize,
+      highWaterMark,
     }
     this.dsOpt = {}
     if (opt.readAt) {
@@ -47,26 +62,30 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     this.originalLimit = q.limitVal
     this.table = q.kinds[0]!
 
-    logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`)
+    logger.warn(`!! using experimentalCursorStream`, {
+      table: this.table,
+      batchSize,
+      highWaterMark,
+    })
 
     const { maxWait } = this.opt
     if (maxWait) {
-      this.logger.warn(`!! ${this.table} maxWait ${maxWait}`)
+      logger.warn(`!! ${this.table} maxWait ${maxWait}`)
 
       this.maxWaitInterval = setInterval(
         () => {
           const millisSinceLastRead = Date.now() - this.lastReadTimestamp
 
           if (millisSinceLastRead < maxWait * 1000) {
-            this.logger.warn(
+            logger.warn(
               `!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`,
             )
             return
           }
 
-          const { running, rowsRetrieved } = this
-          this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
-            running,
+          const { queryIsRunning, rowsRetrieved } = this
+          logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+            running: queryIsRunning,
             rowsRetrieved,
           })
 
@@ -79,16 +98,45 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     }
   }
 
+  override _read(): void {
+    this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+
+    // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+    this.countReads++
+
+    if (this.done) {
+      this.logger.warn(`!!! _read was called, but done==true`)
+      return
+    }
+
+    if (this.paused) {
+      this.logger.log(
+        `_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`,
+      )
+      this.paused = false
+    }
+
+    if (this.queryIsRunning) {
+      this.logger.log(`_read #${this.countReads}, queryIsRunning: true, doing nothing`)
+      return
+    }
+
+    void this.runNextQuery().catch(err => {
+      console.log('error in runNextQuery', err)
+      this.emit('error', err)
+    })
+  }
+
   private async runNextQuery(): Promise<void> {
     if (this.done) return
+    const { logger, table } = this
 
     if (this.lastQueryDone) {
       const now = Date.now()
       this.totalWait += now - this.lastQueryDone
     }
 
-    this.running = true
-    // console.log('running query...')
+    this.queryIsRunning = true
 
     let limit = this.opt.batchSize
 
@@ -96,66 +144,35 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved)
     }
 
-    // console.log(`limit: ${limit}`)
     let q = this.q.limit(limit)
     if (this.endCursor) {
       q = q.start(this.endCursor)
     }
 
-    let rows: T[] = []
-    let info: RunQueryInfo = {}
-
-    try {
-      await pRetry(
-        async () => {
-          const res = await q.run(this.dsOpt)
-          rows = res[0]
-          info = res[1]
-        },
-        {
-          name: `DatastoreStreamReadable.query(${this.table})`,
-          maxAttempts: 5,
-          delay: 5000,
-          delayMultiplier: 2,
-          logger: this.logger,
-          timeout: 120_000, // 2 minutes
-        },
-      )
-    } catch (err) {
-      console.log(
-        `DatastoreStreamReadable error!\n`,
-        {
-          table: this.table,
-          rowsRetrieved: this.rowsRetrieved,
-        },
-        err,
-      )
-      this.emit('error', err)
-      clearInterval(this.maxWaitInterval)
+    const started = localTime.nowUnixMillis()
+    const res = await this.runQuery(q)
+    const queryTook = Date.now() - started
+    if (!res) {
+      // error already emitted in runQuery
       return
     }
+    const rows: T[] = res[0]
+    const info: RunQueryInfo = res[1]
 
     this.rowsRetrieved += rows.length
-    this.logger.log(
-      `${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
+    logger.log(
+      `${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
         this.totalWait,
       )}`,
-      info.moreResults,
     )
 
-    if (!rows.length) {
-      this.logger.warn(
-        `${this.table} got 0 rows, totalWait: ${_ms(this.totalWait)}`,
-        info.moreResults,
-      )
-    }
-
     this.endCursor = info.endCursor
-    this.running = false // ready to take more _reads
+    this.queryIsRunning = false // ready to take more _reads
     this.lastQueryDone = Date.now()
+    let shouldContinue = false
 
     for (const row of rows) {
-      this.push(row)
+      shouldContinue = this.push(row)
     }
 
     if (
@@ -163,52 +180,60 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       info.moreResults === 'NO_MORE_RESULTS' ||
       (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
     ) {
-      this.logger.log(
+      logger.log(
         `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
       )
       this.push(null)
       this.done = true
+      this.paused = false
       clearInterval(this.maxWaitInterval)
-    } else if (this.opt.singleBatchBuffer) {
-      // here we don't start next query until we're asked (via next _read call)
-      // do, let's do nothing
-    } else if (this.opt.rssLimitMB) {
-      const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
-
-      if (rssMB <= this.opt.rssLimitMB) {
-        void this.runNextQuery()
+      return
+    }
+
+    if (shouldContinue) {
+      // Keep the stream flowing
+      logger.log(`${table} continuing the stream`)
+      void this.runNextQuery()
+    } else {
+      // Not starting the next query
+      if (this.paused) {
+        logger.log(`${table} stream is already paused`)
       } else {
-        this.logger.warn(
-          `${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`,
-        )
+        logger.log(`${table} pausing the stream`)
+        this.paused = true
       }
     }
   }
 
-  /**
-   * Counts how many times _read was called.
-   * For debugging.
-   */
-  count = 0
-
-  override _read(): void {
-    this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+  private async runQuery(q: Query): Promise<RunQueryResponse | undefined> {
+    const { table, logger } = this
 
-    // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
-    this.count++
-
-    if (this.done) {
-      this.logger.warn(`!!! _read was called, but done==true`)
+    try {
+      return await pRetry(
+        async () => {
+          return await q.run(this.dsOpt)
+        },
+        {
+          name: `DatastoreStreamReadable.query(${table})`,
+          maxAttempts: 5,
+          delay: 5000,
+          delayMultiplier: 2,
+          logger,
+          timeout: 120_000, // 2 minutes
+        },
+      )
+    } catch (err) {
+      console.log(
+        `DatastoreStreamReadable error!\n`,
+        {
+          table,
+          rowsRetrieved: this.rowsRetrieved,
+        },
+        err,
+      )
+      this.emit('error', err)
+      clearInterval(this.maxWaitInterval)
       return
     }
-
-    if (!this.running) {
-      void this.runNextQuery().catch(err => {
-        console.log('error in runNextQuery', err)
-        this.emit('error', err)
-      })
-    } else {
-      this.logger.log(`_read ${this.count}, wasRunning: true`)
-    }
   }
 }