@naturalcycles/datastore-lib 4.15.0 → 4.16.0

package/dist/datastore.db.d.ts

@@ -4,7 +4,7 @@ import { BaseCommonDB } from '@naturalcycles/db-lib';
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import { type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBReadOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePropertyStats, DatastoreStats } from './datastore.model.js';
 /**
  * Datastore API:
@@ -31,7 +31,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<number>;
     private runDatastoreQuery;
-    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBStreamOptions): Pipeline<ROW>;
     /**
      * Returns saved entities with generated id/updated/created (non-mutating!)
      */

package/dist/datastore.db.js

@@ -1,4 +1,3 @@
-import { Transform } from 'node:stream';
 import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
 import { _round } from '@naturalcycles/js-lib';
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
@@ -10,6 +9,7 @@ import { pRetry, pRetryFn } from '@naturalcycles/js-lib/promise/pRetry.js';
 import { pTimeout } from '@naturalcycles/js-lib/promise/pTimeout.js';
 import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { DatastoreType } from './datastore.model.js';
 import { DatastoreStreamReadable } from './datastoreStreamReadable.js';
 import { dbQueryToDatastoreQuery } from './query.util.js';
@@ -209,26 +209,18 @@ export class DatastoreDB extends BaseCommonDB {
         };
     }
     streamQuery(dbQuery, _opt) {
-        const transform = new Transform({
-            objectMode: true,
-            transform: (chunk, _, cb) => {
-                cb(null, this.mapId(chunk));
-            },
-        });
-        void this.ds().then(async (ds) => {
+        return Pipeline.fromAsyncReadable(async () => {
+            const ds = await this.ds();
             const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table), await this.getPropertyFilter());
             const opt = {
                 logger: this.cfg.logger,
                 ...this.cfg.streamOptions,
                 ..._opt,
             };
-            (opt.experimentalCursorStream
+            return opt.experimentalCursorStream
                 ? new DatastoreStreamReadable(q, opt)
-                : ds.runQueryStream(q, this.getRunQueryOptions(opt)))
-                .on('error', err => transform.emit('error', err))
-                .pipe(transform);
-        });
-        return transform;
+                : ds.runQueryStream(q, this.getRunQueryOptions(opt));
+        }).mapSync(r => this.mapId(r));
     }
     // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js
     /**

package/dist/datastoreKeyValueDB.d.ts

@@ -1,5 +1,5 @@
 import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib/kv';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { DatastoreDB } from './datastore.db.js';
 import type { DatastoreDBCfg } from './datastore.model.js';
 export interface DatastoreKeyValueDBCfg extends DatastoreDBCfg {
@@ -14,9 +14,9 @@ export declare class DatastoreKeyValueDB implements CommonKeyValueDB {
     getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
     deleteByIds(table: string, ids: string[]): Promise<void>;
     saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
-    streamIds(table: string, limit?: number): ReadableTyped<string>;
-    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
-    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    streamIds(table: string, limit?: number): Pipeline<string>;
+    streamValues(table: string, limit?: number): Pipeline<Buffer>;
+    streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple>;
     count(table: string): Promise<number>;
     incrementBatch(_table: string, _entries: IncrementTuple[]): Promise<IncrementTuple[]>;
 }

package/dist/datastoreKeyValueDB.js

@@ -33,10 +33,7 @@ export class DatastoreKeyValueDB {
         const q = DBQuery.create(table)
             .select(['id'])
             .limit(limit || 0);
-        return (this.db
-            .streamQuery(q)
-            // .on('error', err => stream.emit('error', err))
-            .map(r => r.id));
+        return this.db.streamQuery(q).mapSync(r => r.id);
     }
     streamValues(table, limit) {
         // `select v` doesn't work for some reason
@@ -44,14 +41,11 @@ export class DatastoreKeyValueDB {
         return (this.db
             .streamQuery(q)
             // .on('error', err => stream.emit('error', err))
-            .map(r => r.v));
+            .mapSync(r => r.v));
     }
     streamEntries(table, limit) {
         const q = DBQuery.create(table).limit(limit || 0);
-        return (this.db
-            .streamQuery(q)
-            // .on('error', err => stream.emit('error', err))
-            .map(r => [r.id, r.v]));
+        return this.db.streamQuery(q).mapSync(r => [r.id, r.v]);
     }
     async count(table) {
         const q = DBQuery.create(table);

package/dist/datastoreStreamReadable.js

@@ -91,7 +91,7 @@ export class DatastoreStreamReadable extends Readable {
         }
         void this.runNextQuery().catch(err => {
             this.logger.error('error in runNextQuery', err);
-            this.emit('error', err);
+            this.destroy(err);
         });
     }
     async runNextQuery() {
@@ -162,6 +162,7 @@ export class DatastoreStreamReadable extends Readable {
                 return await q.run(this.dsOpt);
             }, {
                 name: `DatastoreStreamReadable.query(${table})`,
+                predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
                 maxAttempts: 5,
                 delay: 5000,
                 delayMultiplier: 2,
@@ -174,9 +175,21 @@ export class DatastoreStreamReadable extends Readable {
                 table,
                 rowsRetrieved: this.rowsRetrieved,
             }, err);
-            this.emit('error', err);
             clearInterval(this.maxWaitInterval);
+            this.destroy(err);
             return;
         }
     }
 }
+// Examples of errors:
+// UNKNOWN: Stream removed
+const RETRY_ON = [
+    'GOAWAY',
+    'UNAVAILABLE',
+    'UNKNOWN',
+    'DEADLINE_EXCEEDED',
+    'ABORTED',
+    'much contention',
+    'try again',
+    'timeout',
+].map(s => s.toLowerCase());

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
   "type": "module",
-  "version": "4.15.0",
+  "version": "4.16.0",
   "description": "Opinionated library to work with Google Datastore, implements CommonDB",
   "dependencies": {
     "@google-cloud/datastore": "^10",
@@ -11,7 +11,7 @@
   },
   "devDependencies": {
     "@types/node": "^24",
-    "@naturalcycles/dev-lib": "18.4.2"
+    "@naturalcycles/dev-lib": "19.37.0"
   },
   "exports": {
     ".": "./dist/index.js"

package/src/datastore.db.ts

@@ -1,4 +1,3 @@
-import { Transform } from 'node:stream'
 import type { Datastore, Key, PropertyFilter, Query, Transaction } from '@google-cloud/datastore'
 import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
 import type {
@@ -41,7 +40,7 @@ import {
   type StringMap,
 } from '@naturalcycles/js-lib/types'
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type {
   DatastoreDBCfg,
   DatastoreDBOptions,
@@ -327,15 +326,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   override streamQuery<ROW extends ObjectWithId>(
     dbQuery: DBQuery<ROW>,
     _opt?: DatastoreDBStreamOptions,
-  ): ReadableTyped<ROW> {
-    const transform = new Transform({
-      objectMode: true,
-      transform: (chunk, _, cb) => {
-        cb(null, this.mapId(chunk))
-      },
-    })
+  ): Pipeline<ROW> {
+    return Pipeline.fromAsyncReadable<ROW>(async () => {
+      const ds = await this.ds()
 
-    void this.ds().then(async ds => {
       const q = dbQueryToDatastoreQuery(
         dbQuery,
         ds.createQuery(dbQuery.table),
@@ -348,15 +342,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         ..._opt,
       }
 
-      ;(opt.experimentalCursorStream
+      return opt.experimentalCursorStream
         ? new DatastoreStreamReadable<ROW>(q, opt)
-        : (ds.runQueryStream(q, this.getRunQueryOptions(opt)) as ReadableTyped<ROW>)
-      )
-        .on('error', err => transform.emit('error', err))
-        .pipe(transform)
-    })
-
-    return transform
+        : ds.runQueryStream(q, this.getRunQueryOptions(opt))
+    }).mapSync(r => this.mapId<ROW>(r))
   }
 
   // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js

package/src/datastoreKeyValueDB.ts

@@ -3,7 +3,7 @@ import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '@natural
 import { commonKeyValueDBFullSupport } from '@naturalcycles/db-lib/kv'
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
 import type { ObjectWithId } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import { DatastoreDB } from './datastore.db.js'
 import type { DatastoreDBCfg } from './datastore.model.js'
 
@@ -52,20 +52,15 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
     )
   }
 
-  streamIds(table: string, limit?: number): ReadableTyped<string> {
+  streamIds(table: string, limit?: number): Pipeline<string> {
     const q = DBQuery.create<ObjectWithId>(table)
       .select(['id'])
       .limit(limit || 0)
 
-    return (
-      this.db
-        .streamQuery(q)
-        // .on('error', err => stream.emit('error', err))
-        .map(r => r.id)
-    )
+    return this.db.streamQuery(q).mapSync(r => r.id)
   }
 
-  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+  streamValues(table: string, limit?: number): Pipeline<Buffer> {
     // `select v` doesn't work for some reason
     const q = DBQuery.create<KVObject>(table).limit(limit || 0)
 
@@ -73,19 +68,14 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       this.db
         .streamQuery(q)
         // .on('error', err => stream.emit('error', err))
-        .map(r => r.v)
+        .mapSync(r => r.v)
     )
   }
 
-  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
+  streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple> {
     const q = DBQuery.create<KVObject>(table).limit(limit || 0)
 
-    return (
-      this.db
-        .streamQuery(q)
-        // .on('error', err => stream.emit('error', err))
-        .map(r => [r.id, r.v])
-    )
+    return this.db.streamQuery(q).mapSync(r => [r.id, r.v])
   }
 
   async count(table: string): Promise<number> {

package/src/datastoreStreamReadable.ts

@@ -125,7 +125,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
 
     void this.runNextQuery().catch(err => {
       this.logger.error('error in runNextQuery', err)
-      this.emit('error', err)
+      this.destroy(err)
     })
   }
 
@@ -217,6 +217,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         },
         {
           name: `DatastoreStreamReadable.query(${table})`,
+          predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
           maxAttempts: 5,
           delay: 5000,
           delayMultiplier: 2,
@@ -233,9 +234,22 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         },
         err,
       )
-      this.emit('error', err)
       clearInterval(this.maxWaitInterval)
+      this.destroy(err as Error)
       return
     }
   }
 }
+
+// Examples of errors:
+// UNKNOWN: Stream removed
+const RETRY_ON = [
+  'GOAWAY',
+  'UNAVAILABLE',
+  'UNKNOWN',
+  'DEADLINE_EXCEEDED',
+  'ABORTED',
+  'much contention',
+  'try again',
+  'timeout',
+].map(s => s.toLowerCase())