@naturalcycles/datastore-lib 3.24.1 → 3.25.1

package/dist/DatastoreStreamReadable.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { Readable } from 'stream';
+import { Readable } from 'node:stream';
 import { Query } from '@google-cloud/datastore';
 import { CommonLogger } from '@naturalcycles/js-lib';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
@@ -14,6 +14,7 @@ export declare class DatastoreStreamReadable<T = any> extends Readable implement
     private done;
     private lastQueryDone?;
     private totalWait;
+    private table;
     private opt;
     constructor(q: Query, opt: DatastoreDBStreamOptions, logger: CommonLogger);
     private runNextQuery;

package/dist/DatastoreStreamReadable.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatastoreStreamReadable = void 0;
-const stream_1 = require("stream");
+const node_stream_1 = require("node:stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
-class DatastoreStreamReadable extends stream_1.Readable {
+class DatastoreStreamReadable extends node_stream_1.Readable {
     constructor(q, opt, logger) {
         super({ objectMode: true });
         this.q = q;
@@ -19,7 +19,8 @@ class DatastoreStreamReadable extends stream_1.Readable {
             ...opt,
         };
         this.originalLimit = q.limitVal;
-        logger.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`);
+        this.table = q.kinds[0];
+        logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`);
     }
     async runNextQuery() {
         if (this.done)
@@ -39,38 +40,54 @@ class DatastoreStreamReadable extends stream_1.Readable {
         if (this.endCursor) {
             q = q.start(this.endCursor);
         }
+        let rows = [];
+        let info = {};
         try {
-            const [rows, info] = await q.run();
-            this.rowsRetrieved += rows.length;
-            this.logger.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
-            this.endCursor = info.endCursor;
-            this.running = false; // ready to take more _reads
-            this.lastQueryDone = Date.now();
-            rows.forEach(row => this.push(row));
-            if (!info.endCursor ||
-                info.moreResults === 'NO_MORE_RESULTS' ||
-                (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-                this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
-                this.push(null);
-                this.done = true;
-            }
-            else if (this.opt.singleBatchBuffer) {
-                // here we don't start next query until we're asked (via next _read call)
-                // do, let's do nothing
-            }
-            else if (this.opt.rssLimitMB) {
-                const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
-                if (rssMB <= this.opt.rssLimitMB) {
-                    void this.runNextQuery();
-                }
-                else {
-                    this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
-                }
-            }
+            await (0, js_lib_1.pRetry)(async () => {
+                const res = await q.run();
+                rows = res[0];
+                info = res[1];
+            }, {
+                name: `DatastoreStreamReadable.query(${this.table})`,
+                maxAttempts: 5,
+                delay: 5000,
+                delayMultiplier: 2,
+                logger: this.logger,
+            });
         }
         catch (err) {
-            console.error('DatastoreStreamReadable error!\n', err);
+            console.error(`DatastoreStreamReadable error!\n`, {
+                table: this.table,
+                rowsRetrieved: this.rowsRetrieved,
+            }, err);
             this.emit('error', err);
+            return;
+        }
+        this.rowsRetrieved += rows.length;
+        this.logger.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
+        this.endCursor = info.endCursor;
+        this.running = false; // ready to take more _reads
+        this.lastQueryDone = Date.now();
+        rows.forEach(row => this.push(row));
+        if (!info.endCursor ||
+            info.moreResults === 'NO_MORE_RESULTS' ||
+            (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+            this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
+            this.push(null);
+            this.done = true;
+        }
+        else if (this.opt.singleBatchBuffer) {
+            // here we don't start next query until we're asked (via next _read call)
+            // do, let's do nothing
+        }
+        else if (this.opt.rssLimitMB) {
+            const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
+            if (rssMB <= this.opt.rssLimitMB) {
+                void this.runNextQuery();
+            }
+            else {
+                this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
+            }
         }
     }
     _read() {

package/dist/datastore.db.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatastoreDB = void 0;
-const stream_1 = require("stream");
+const node_stream_1 = require("node:stream");
 const db_lib_1 = require("@naturalcycles/db-lib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
@@ -33,13 +33,12 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
     }
     // @memo() // not used to be able to connect to many DBs in the same server instance
     ds() {
-        var _a;
         if (!this.cachedDatastore) {
             (0, js_lib_1._assert)(process.env['APP_ENV'] !== 'test', 'DatastoreDB cannot be used in Test env, please use InMemoryDB');
             // Lazy-loading
             const datastoreLib = require('@google-cloud/datastore');
             const DS = datastoreLib.Datastore;
-            (_a = this.cfg).projectId || (_a.projectId = this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']);
+            this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT'];
             if (this.cfg.projectId) {
                 this.cfg.logger.log(`DatastoreDB connected to ${(0, colors_1.boldWhite)(this.cfg.projectId)}`);
             }
@@ -68,7 +67,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         if (this.cfg.timeout) {
             // First try
             try {
-                const r = await (0, js_lib_1.pTimeout)(this.ds().get(keys), {
+                const r = await (0, js_lib_1.pTimeout)(() => this.ds().get(keys), {
                     timeout: this.cfg.timeout,
                     name: `datastore.getByIds(${table})`,
                 });
@@ -81,7 +80,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                 const DS = datastoreLib.Datastore;
                 this.cachedDatastore = new DS(this.cfg);
                 // Second try (will throw)
-                const r = await (0, js_lib_1.pTimeout)(this.ds().get(keys), {
+                const r = await (0, js_lib_1.pTimeout)(() => this.ds().get(keys), {
                     timeout: this.cfg.timeout,
                     name: `datastore.getByIds(${table}) second try`,
                     errorData: {
@@ -133,14 +132,17 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             ...this.cfg.streamOptions,
             ..._opt,
         };
-        return (opt.experimentalCursorStream
+        const stream = (opt.experimentalCursorStream
             ? new DatastoreStreamReadable_1.DatastoreStreamReadable(q, opt, (0, js_lib_1.commonLoggerMinLevel)(this.cfg.logger, opt.debug ? 'log' : 'warn'))
-            : this.ds().runQueryStream(q)).pipe(new stream_1.Transform({
+            : this.ds().runQueryStream(q))
+            .on('error', err => stream.emit('error', err))
+            .pipe(new node_stream_1.Transform({
             objectMode: true,
             transform: (chunk, _enc, cb) => {
                 cb(null, this.mapId(chunk));
             },
         }));
+        return stream;
     }
     streamQuery(dbQuery, opt) {
         const q = (0, query_util_1.dbQueryToDatastoreQuery)(dbQuery, this.ds().createQuery(dbQuery.table));

package/dist/datastoreKeyValueDB.js

@@ -30,22 +30,34 @@ class DatastoreKeyValueDB {
         const q = db_lib_1.DBQuery.create(table)
             .select(['id'])
             .limit(limit || 0);
-        return this.db.streamQuery(q).pipe((0, nodejs_lib_1.transformMapSimple)(objectWithId => objectWithId.id, {
+        const stream = this.db
+            .streamQuery(q)
+            .on('error', err => stream.emit('error', err))
+            .pipe((0, nodejs_lib_1.transformMapSimple)(objectWithId => objectWithId.id, {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
+        return stream;
     }
     streamValues(table, limit) {
         // `select v` doesn't work for some reason
         const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
-        return this.db.streamQuery(q).pipe((0, nodejs_lib_1.transformMapSimple)(obj => obj.v, {
+        const stream = this.db
+            .streamQuery(q)
+            .on('error', err => stream.emit('error', err))
+            .pipe((0, nodejs_lib_1.transformMapSimple)(obj => obj.v, {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
+        return stream;
     }
     streamEntries(table, limit) {
         const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
-        return this.db.streamQuery(q).pipe((0, nodejs_lib_1.transformMapSimple)(obj => [obj.id, obj.v], {
+        const stream = this.db
+            .streamQuery(q)
+            .on('error', err => stream.emit('error', err))
+            .pipe((0, nodejs_lib_1.transformMapSimple)(obj => [obj.id, obj.v], {
             errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }));
+        return stream;
     }
     async count(_table) {
         this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`);

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.24.1",
+  "version": "3.25.1",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"
   },
   "dependencies": {
     "@google-cloud/datastore": "^7.0.0",
-    "@naturalcycles/db-lib": "^8.0.0",
-    "@naturalcycles/js-lib": "^14.0.0",
+    "@naturalcycles/db-lib": "^8.46.1",
+    "@naturalcycles/js-lib": "^14.116.0",
     "@naturalcycles/nodejs-lib": "^12.0.0",
     "grpc": "^1.24.2"
   },
@@ -31,7 +31,7 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "engines": {
-    "node": ">=14.15.0"
+    "node": ">=16.10.0"
   },
   "publishConfig": {
     "access": "public"

package/src/DatastoreStreamReadable.ts

@@ -1,6 +1,7 @@
-import { Readable } from 'stream'
+import { Readable } from 'node:stream'
+import type { RunQueryInfo } from '@google-cloud/datastore/build/src/query'
 import { Query } from '@google-cloud/datastore'
-import { _ms, CommonLogger } from '@naturalcycles/js-lib'
+import { _ms, CommonLogger, pRetry } from '@naturalcycles/js-lib'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import type { DatastoreDBStreamOptions } from './datastore.model'
 
@@ -12,6 +13,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
   private done = false
   private lastQueryDone?: number
   private totalWait = 0
+  private table: string
 
   private opt: DatastoreDBStreamOptions & { batchSize: number }
 
@@ -25,8 +27,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     }
 
     this.originalLimit = q.limitVal
+    this.table = q.kinds[0]!
 
-    logger.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`)
+    logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`)
   }
 
   private async runNextQuery(): Promise<void> {
@@ -52,48 +55,72 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       q = q.start(this.endCursor)
     }
 
+    let rows: T[] = []
+    let info: RunQueryInfo = {}
+
     try {
-      const [rows, info] = await q.run()
+      await pRetry(
+        async () => {
+          const res = await q.run()
+          rows = res[0]
+          info = res[1]
+        },
+        {
+          name: `DatastoreStreamReadable.query(${this.table})`,
+          maxAttempts: 5,
+          delay: 5000,
+          delayMultiplier: 2,
+          logger: this.logger,
+        },
+      )
+    } catch (err) {
+      console.error(
+        `DatastoreStreamReadable error!\n`,
+        {
+          table: this.table,
+          rowsRetrieved: this.rowsRetrieved,
+        },
+        err,
+      )
+      this.emit('error', err)
+      return
+    }
 
-      this.rowsRetrieved += rows.length
+    this.rowsRetrieved += rows.length
+    this.logger.log(
+      `got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
+        this.totalWait,
+      )}`,
+      info.moreResults,
+    )
+
+    this.endCursor = info.endCursor
+    this.running = false // ready to take more _reads
+    this.lastQueryDone = Date.now()
+
+    rows.forEach(row => this.push(row))
+
+    if (
+      !info.endCursor ||
+      info.moreResults === 'NO_MORE_RESULTS' ||
+      (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
+    ) {
       this.logger.log(
-        `got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
-          this.totalWait,
-        )}`,
-        info.moreResults,
+        `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
       )
-
-      this.endCursor = info.endCursor
-      this.running = false // ready to take more _reads
-      this.lastQueryDone = Date.now()
-
-      rows.forEach(row => this.push(row))
-
-      if (
-        !info.endCursor ||
-        info.moreResults === 'NO_MORE_RESULTS' ||
-        (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
-      ) {
-        this.logger.log(
-          `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
-        )
-        this.push(null)
-        this.done = true
-      } else if (this.opt.singleBatchBuffer) {
-        // here we don't start next query until we're asked (via next _read call)
-        // do, let's do nothing
-      } else if (this.opt.rssLimitMB) {
-        const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
-
-        if (rssMB <= this.opt.rssLimitMB) {
-          void this.runNextQuery()
-        } else {
-          this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
-        }
+      this.push(null)
+      this.done = true
+    } else if (this.opt.singleBatchBuffer) {
+      // here we don't start next query until we're asked (via next _read call)
+      // do, let's do nothing
+    } else if (this.opt.rssLimitMB) {
+      const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
+
+      if (rssMB <= this.opt.rssLimitMB) {
+        void this.runNextQuery()
+      } else {
+        this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
       }
-    } catch (err) {
-      console.error('DatastoreStreamReadable error!\n', err)
-      this.emit('error', err)
     }
   }
 

package/src/datastore.db.ts

@@ -1,4 +1,4 @@
-import { Transform } from 'stream'
+import { Transform } from 'node:stream'
 import type { Datastore, Key, Query } from '@google-cloud/datastore'
 import {
   BaseCommonDB,
@@ -130,7 +130,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     if (this.cfg.timeout) {
       // First try
       try {
-        const r = await pTimeout(this.ds().get(keys), {
+        const r = await pTimeout(() => this.ds().get(keys), {
           timeout: this.cfg.timeout,
           name: `datastore.getByIds(${table})`,
         })
@@ -144,7 +144,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         this.cachedDatastore = new DS(this.cfg)
 
         // Second try (will throw)
-        const r = await pTimeout(this.ds().get(keys), {
+        const r = await pTimeout(() => this.ds().get(keys), {
           timeout: this.cfg.timeout,
           name: `datastore.getByIds(${table}) second try`,
           errorData: {
@@ -216,7 +216,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       ..._opt,
     }
 
-    return (
+    const stream: ReadableTyped<ROW> = (
       opt.experimentalCursorStream
         ? new DatastoreStreamReadable(
             q,
@@ -224,14 +224,18 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
             commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
           )
         : this.ds().runQueryStream(q)
-    ).pipe(
-      new Transform({
-        objectMode: true,
-        transform: (chunk, _enc, cb) => {
-          cb(null, this.mapId(chunk))
-        },
-      }),
     )
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform: (chunk, _enc, cb) => {
+            cb(null, this.mapId(chunk))
+          },
+        }),
+      )
+
+    return stream
   }
 
   override streamQuery<ROW extends ObjectWithId>(

package/src/datastoreKeyValueDB.ts

@@ -49,32 +49,47 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       .select(['id'])
       .limit(limit || 0)
 
-    return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<ObjectWithId<string>, string>(objectWithId => objectWithId.id, {
-        errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-      }),
-    )
+    const stream: ReadableTyped<string> = this.db
+      .streamQuery<KVObject>(q)
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        transformMapSimple<ObjectWithId<string>, string>(objectWithId => objectWithId.id, {
+          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+        }),
+      )
+
+    return stream
   }
 
   streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
     // `select v` doesn't work for some reason
     const q = DBQuery.create(table).limit(limit || 0)
 
-    return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<{ v: Buffer }, Buffer>(obj => obj.v, {
-        errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-      }),
-    )
+    const stream: ReadableTyped<string> = this.db
+      .streamQuery<KVObject>(q)
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        transformMapSimple<{ v: Buffer }, Buffer>(obj => obj.v, {
+          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+        }),
+      )
+
+    return stream
   }
 
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
     const q = DBQuery.create(table).limit(limit || 0)
 
-    return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<KVObject, KeyValueDBTuple>(obj => [obj.id, obj.v], {
-        errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-      }),
-    )
+    const stream: ReadableTyped<string> = this.db
+      .streamQuery<KVObject>(q)
+      .on('error', err => stream.emit('error', err))
+      .pipe(
+        transformMapSimple<KVObject, KeyValueDBTuple>(obj => [obj.id, obj.v], {
+          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+        }),
+      )
+
+    return stream
   }
 
   async count(_table: string): Promise<number> {