@naturalcycles/datastore-lib 3.24.0 → 3.25.0

package/dist/DatastoreStreamReadable.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { Readable } from 'stream';
+import { Readable } from 'node:stream';
 import { Query } from '@google-cloud/datastore';
 import { CommonLogger } from '@naturalcycles/js-lib';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
@@ -14,6 +14,7 @@ export declare class DatastoreStreamReadable<T = any> extends Readable implement
     private done;
     private lastQueryDone?;
     private totalWait;
+    private table;
     private opt;
     constructor(q: Query, opt: DatastoreDBStreamOptions, logger: CommonLogger);
     private runNextQuery;

package/dist/DatastoreStreamReadable.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatastoreStreamReadable = void 0;
-const stream_1 = require("stream");
+const node_stream_1 = require("node:stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
-class DatastoreStreamReadable extends stream_1.Readable {
+class DatastoreStreamReadable extends node_stream_1.Readable {
     constructor(q, opt, logger) {
         super({ objectMode: true });
         this.q = q;
@@ -19,7 +19,8 @@ class DatastoreStreamReadable extends stream_1.Readable {
             ...opt,
         };
         this.originalLimit = q.limitVal;
-        logger.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`);
+        this.table = q.kinds[0];
+        logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`);
     }
     async runNextQuery() {
         if (this.done)
@@ -39,38 +40,54 @@ class DatastoreStreamReadable extends stream_1.Readable {
         if (this.endCursor) {
             q = q.start(this.endCursor);
         }
+        let rows = [];
+        let info = {};
         try {
-            const [rows, info] = await q.run();
-            this.rowsRetrieved += rows.length;
-            this.logger.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
-            this.endCursor = info.endCursor;
-            this.running = false; // ready to take more _reads
-            this.lastQueryDone = Date.now();
-            rows.forEach(row => this.push(row));
-            if (!info.endCursor ||
-                info.moreResults === 'NO_MORE_RESULTS' ||
-                (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-                this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
-                this.push(null);
-                this.done = true;
-            }
-            else if (this.opt.singleBatchBuffer) {
-                // here we don't start next query until we're asked (via next _read call)
-                // do, let's do nothing
-            }
-            else if (this.opt.rssLimitMB) {
-                const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
-                if (rssMB <= this.opt.rssLimitMB) {
-                    void this.runNextQuery();
-                }
-                else {
-                    this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
-                }
-            }
+            await (0, js_lib_1.pRetry)(async () => {
+                const res = await q.run();
+                rows = res[0];
+                info = res[1];
+            }, {
+                name: `DatastoreStreamReadable.query(${this.table})`,
+                maxAttempts: 5,
+                delay: 5000,
+                delayMultiplier: 2,
+                logger: this.logger,
+            });
         }
         catch (err) {
-            console.error('DatastoreStreamReadable error!\n', err);
+            console.error(`DatastoreStreamReadable error!\n`, {
+                table: this.table,
+                rowsRetrieved: this.rowsRetrieved,
+            }, err);
             this.emit('error', err);
+            return;
+        }
+        this.rowsRetrieved += rows.length;
+        this.logger.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
+        this.endCursor = info.endCursor;
+        this.running = false; // ready to take more _reads
+        this.lastQueryDone = Date.now();
+        rows.forEach(row => this.push(row));
+        if (!info.endCursor ||
+            info.moreResults === 'NO_MORE_RESULTS' ||
+            (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+            this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
+            this.push(null);
+            this.done = true;
+        }
+        else if (this.opt.singleBatchBuffer) {
+            // here we don't start next query until we're asked (via next _read call)
+            // do, let's do nothing
+        }
+        else if (this.opt.rssLimitMB) {
+            const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
+            if (rssMB <= this.opt.rssLimitMB) {
+                void this.runNextQuery();
+            }
+            else {
+                this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
+            }
         }
     }
     _read() {

package/dist/datastore.db.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatastoreDB = void 0;
-const stream_1 = require("stream");
+const node_stream_1 = require("node:stream");
 const db_lib_1 = require("@naturalcycles/db-lib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
@@ -133,14 +133,17 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             ...this.cfg.streamOptions,
             ..._opt,
         };
-        return (opt.experimentalCursorStream
+        const stream = (opt.experimentalCursorStream
             ? new DatastoreStreamReadable_1.DatastoreStreamReadable(q, opt, (0, js_lib_1.commonLoggerMinLevel)(this.cfg.logger, opt.debug ? 'log' : 'warn'))
-            : this.ds().runQueryStream(q)).pipe(new stream_1.Transform({
+            : this.ds().runQueryStream(q))
+            .on('error', err => stream.emit('error', err))
+            .pipe(new node_stream_1.Transform({
             objectMode: true,
             transform: (chunk, _enc, cb) => {
                 cb(null, this.mapId(chunk));
             },
         }));
+        return stream;
     }
     streamQuery(dbQuery, opt) {
         const q = (0, query_util_1.dbQueryToDatastoreQuery)(dbQuery, this.ds().createQuery(dbQuery.table));
@@ -199,10 +202,10 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                 const ops = (0, db_lib_1.mergeDBOperations)(_tx.ops);
                 for await (const op of ops) {
                     if (op.type === 'saveBatch') {
-                        await this.saveBatch(op.table, op.rows, { ...opt, tx });
+                        await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt, tx });
                     }
                     else if (op.type === 'deleteByIds') {
-                        await this.deleteByIds(op.table, op.ids, { ...opt, tx });
+                        await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt, tx });
                     }
                     else {
                         throw new Error(`DBOperation not supported: ${op.type}`);

package/dist/datastoreKeyValueDB.js

@@ -30,22 +30,34 @@ class DatastoreKeyValueDB {
         const q = db_lib_1.DBQuery.create(table)
             .select(['id'])
             .limit(limit || 0);
-        return this.db.streamQuery(q).pipe((0, nodejs_lib_1.transformMapSimple)(objectWithId => objectWithId.id, {
+        const stream = this.db
+            .streamQuery(q)
+            .on('error', err => stream.emit('error', err))
+            .pipe((0, nodejs_lib_1.transformMapSimple)(objectWithId => objectWithId.id, {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
+        return stream;
     }
     streamValues(table, limit) {
         // `select v` doesn't work for some reason
         const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
-        return this.db.streamQuery(q).pipe((0, nodejs_lib_1.transformMapSimple)(obj => obj.v, {
+        const stream = this.db
+            .streamQuery(q)
+            .on('error', err => stream.emit('error', err))
+            .pipe((0, nodejs_lib_1.transformMapSimple)(obj => obj.v, {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
+        return stream;
     }
     streamEntries(table, limit) {
         const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
-        return this.db.streamQuery(q).pipe((0, nodejs_lib_1.transformMapSimple)(obj => [obj.id, obj.v], {
+        const stream = this.db
+            .streamQuery(q)
+            .on('error', err => stream.emit('error', err))
+            .pipe((0, nodejs_lib_1.transformMapSimple)(obj => [obj.id, obj.v], {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
+        return stream;
     }
     async count(_table) {
         this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.24.0",
+  "version": "3.25.0",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"

package/src/DatastoreStreamReadable.ts

@@ -1,6 +1,7 @@
-import { Readable } from 'stream'
+import { Readable } from 'node:stream'
+import type { RunQueryInfo } from '@google-cloud/datastore/build/src/query'
 import { Query } from '@google-cloud/datastore'
-import { _ms, CommonLogger } from '@naturalcycles/js-lib'
+import { _ms, CommonLogger, pRetry } from '@naturalcycles/js-lib'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import type { DatastoreDBStreamOptions } from './datastore.model'
 
@@ -12,6 +13,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
   private done = false
   private lastQueryDone?: number
   private totalWait = 0
+  private table: string
 
   private opt: DatastoreDBStreamOptions & { batchSize: number }
 
@@ -25,8 +27,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     }
 
     this.originalLimit = q.limitVal
+    this.table = q.kinds[0]!
 
-    logger.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`)
+    logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`)
   }
 
   private async runNextQuery(): Promise<void> {
@@ -52,48 +55,72 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       q = q.start(this.endCursor)
     }
 
+    let rows: T[] = []
+    let info: RunQueryInfo = {}
+
     try {
-      const [rows, info] = await q.run()
+      await pRetry(
+        async () => {
+          const res = await q.run()
+          rows = res[0]
+          info = res[1]
+        },
+        {
+          name: `DatastoreStreamReadable.query(${this.table})`,
+          maxAttempts: 5,
+          delay: 5000,
+          delayMultiplier: 2,
+          logger: this.logger,
+        },
+      )
+    } catch (err) {
+      console.error(
+        `DatastoreStreamReadable error!\n`,
+        {
+          table: this.table,
+          rowsRetrieved: this.rowsRetrieved,
+        },
+        err,
+      )
+      this.emit('error', err)
+      return
+    }
 
-      this.rowsRetrieved += rows.length
+    this.rowsRetrieved += rows.length
+    this.logger.log(
+      `got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
+        this.totalWait,
+      )}`,
+      info.moreResults,
+    )
+
+    this.endCursor = info.endCursor
+    this.running = false // ready to take more _reads
+    this.lastQueryDone = Date.now()
+
+    rows.forEach(row => this.push(row))
+
+    if (
+      !info.endCursor ||
+      info.moreResults === 'NO_MORE_RESULTS' ||
+      (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
+    ) {
       this.logger.log(
-        `got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
-          this.totalWait,
-        )}`,
-        info.moreResults,
+        `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
       )
-
-      this.endCursor = info.endCursor
-      this.running = false // ready to take more _reads
-      this.lastQueryDone = Date.now()
-
-      rows.forEach(row => this.push(row))
-
-      if (
-        !info.endCursor ||
-        info.moreResults === 'NO_MORE_RESULTS' ||
-        (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
-      ) {
-        this.logger.log(
-          `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
-        )
-        this.push(null)
-        this.done = true
-      } else if (this.opt.singleBatchBuffer) {
-        // here we don't start next query until we're asked (via next _read call)
-        // do, let's do nothing
-      } else if (this.opt.rssLimitMB) {
-        const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
-
-        if (rssMB <= this.opt.rssLimitMB) {
-          void this.runNextQuery()
-        } else {
-          this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
-        }
+      this.push(null)
+      this.done = true
+    } else if (this.opt.singleBatchBuffer) {
+      // here we don't start next query until we're asked (via next _read call)
+      // do, let's do nothing
+    } else if (this.opt.rssLimitMB) {
+      const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
+
+      if (rssMB <= this.opt.rssLimitMB) {
+        void this.runNextQuery()
+      } else {
+        this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
       }
-    } catch (err) {
-      console.error('DatastoreStreamReadable error!\n', err)
-      this.emit('error', err)
     }
   }
 

package/src/datastore.db.ts

@@ -1,4 +1,4 @@
-import { Transform } from 'stream'
+import { Transform } from 'node:stream'
 import type { Datastore, Key, Query } from '@google-cloud/datastore'
 import {
   BaseCommonDB,
@@ -216,7 +216,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       ..._opt,
     }
 
-    return (
+    const stream: ReadableTyped<ROW> = (
       opt.experimentalCursorStream
         ? new DatastoreStreamReadable(
             q,
@@ -224,14 +224,18 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
             commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
           )
         : this.ds().runQueryStream(q)
-    ).pipe(
-      new Transform({
-        objectMode: true,
-        transform: (chunk, _enc, cb) => {
-          cb(null, this.mapId(chunk))
-        },
-      }),
     )
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform: (chunk, _enc, cb) => {
+            cb(null, this.mapId(chunk))
+          },
+        }),
+      )
+
+    return stream
   }
 
   override streamQuery<ROW extends ObjectWithId>(
@@ -327,9 +331,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
         for await (const op of ops) {
           if (op.type === 'saveBatch') {
-            await this.saveBatch(op.table, op.rows, { ...opt, tx })
+            await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt, tx })
           } else if (op.type === 'deleteByIds') {
-            await this.deleteByIds(op.table, op.ids, { ...opt, tx })
+            await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt, tx })
           } else {
             throw new Error(`DBOperation not supported: ${(op as any).type}`)
           }

package/src/datastoreKeyValueDB.ts

@@ -49,32 +49,47 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       .select(['id'])
       .limit(limit || 0)
 
-    return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<ObjectWithId<string>, string>(objectWithId => objectWithId.id, {
-        errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-      }),
-    )
+    const stream: ReadableTyped<string> = this.db
+      .streamQuery<KVObject>(q)
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        transformMapSimple<ObjectWithId<string>, string>(objectWithId => objectWithId.id, {
+          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+        }),
+      )
+
+    return stream
   }
 
   streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
     // `select v` doesn't work for some reason
     const q = DBQuery.create(table).limit(limit || 0)
 
-    return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<{ v: Buffer }, Buffer>(obj => obj.v, {
-        errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-      }),
-    )
+    const stream: ReadableTyped<string> = this.db
+      .streamQuery<KVObject>(q)
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        transformMapSimple<{ v: Buffer }, Buffer>(obj => obj.v, {
+          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+        }),
+      )
+
+    return stream
   }
 
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
     const q = DBQuery.create(table).limit(limit || 0)
 
-    return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<KVObject, KeyValueDBTuple>(obj => [obj.id, obj.v], {
-        errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-      }),
-    )
+    const stream: ReadableTyped<string> = this.db
+      .streamQuery<KVObject>(q)
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        transformMapSimple<KVObject, KeyValueDBTuple>(obj => [obj.id, obj.v], {
+          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+        }),
+      )
+
+    return stream
   }
 
   async count(_table: string): Promise<number> {