@naturalcycles/datastore-lib 4.15.1 → 4.16.1

package/dist/datastore.db.d.ts

@@ -4,7 +4,7 @@ import { BaseCommonDB } from '@naturalcycles/db-lib';
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import { type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBReadOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePropertyStats, DatastoreStats } from './datastore.model.js';
 /**
  * Datastore API:
@@ -31,7 +31,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<number>;
     private runDatastoreQuery;
-    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBStreamOptions): Pipeline<ROW>;
     /**
      * Returns saved entities with generated id/updated/created (non-mutating!)
      */

package/dist/datastore.db.js

@@ -1,7 +1,7 @@
-import { Transform } from 'node:stream';
 import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
 import { _round } from '@naturalcycles/js-lib';
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
+import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { _errorDataAppend, TimeoutError } from '@naturalcycles/js-lib/error/error.util.js';
 import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
@@ -10,6 +10,7 @@ import { pRetry, pRetryFn } from '@naturalcycles/js-lib/promise/pRetry.js';
 import { pTimeout } from '@naturalcycles/js-lib/promise/pTimeout.js';
 import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { DatastoreType } from './datastore.model.js';
 import { DatastoreStreamReadable } from './datastoreStreamReadable.js';
 import { dbQueryToDatastoreQuery } from './query.util.js';
@@ -112,7 +113,7 @@ export class DatastoreDB extends BaseCommonDB {
                     // Not a timeout error, re-throw
                     throw err;
                 }
-                this.cfg.logger.log('datastore recreated on error');
+                this.cfg.logger.log(`datastore recreated on timeout (${_ms(this.cfg.timeout)}) while loading ${table}`);
                 // This is to debug "GCP Datastore Timeout issue"
                 const datastoreLib = await this.getDatastoreLib();
                 const DS = datastoreLib.Datastore;
@@ -209,26 +210,18 @@ export class DatastoreDB extends BaseCommonDB {
         };
     }
     streamQuery(dbQuery, _opt) {
-        const transform = new Transform({
-            objectMode: true,
-            transform: (chunk, _, cb) => {
-                cb(null, this.mapId(chunk));
-            },
-        });
-        void this.ds().then(async (ds) => {
+        return Pipeline.fromAsyncReadable(async () => {
+            const ds = await this.ds();
             const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table), await this.getPropertyFilter());
             const opt = {
                 logger: this.cfg.logger,
                 ...this.cfg.streamOptions,
                 ..._opt,
             };
-            (opt.experimentalCursorStream
+            return opt.experimentalCursorStream
                 ? new DatastoreStreamReadable(q, opt)
-                : ds.runQueryStream(q, this.getRunQueryOptions(opt)))
-                .on('error', err => transform.destroy(err))
-                .pipe(transform);
-        });
-        return transform;
+                : ds.runQueryStream(q, this.getRunQueryOptions(opt));
+        }).mapSync(r => this.mapId(r));
     }
     // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js
     /**

package/dist/datastoreKeyValueDB.d.ts

@@ -1,5 +1,5 @@
 import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib/kv';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { DatastoreDB } from './datastore.db.js';
 import type { DatastoreDBCfg } from './datastore.model.js';
 export interface DatastoreKeyValueDBCfg extends DatastoreDBCfg {
@@ -14,9 +14,9 @@ export declare class DatastoreKeyValueDB implements CommonKeyValueDB {
     getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
     deleteByIds(table: string, ids: string[]): Promise<void>;
     saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
-    streamIds(table: string, limit?: number): ReadableTyped<string>;
-    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
-    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    streamIds(table: string, limit?: number): Pipeline<string>;
+    streamValues(table: string, limit?: number): Pipeline<Buffer>;
+    streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple>;
     count(table: string): Promise<number>;
     incrementBatch(_table: string, _entries: IncrementTuple[]): Promise<IncrementTuple[]>;
 }

package/dist/datastoreKeyValueDB.js

@@ -33,10 +33,7 @@ export class DatastoreKeyValueDB {
         const q = DBQuery.create(table)
             .select(['id'])
             .limit(limit || 0);
-        return (this.db
-            .streamQuery(q)
-            // .on('error', err => stream.emit('error', err))
-            .map(r => r.id));
+        return this.db.streamQuery(q).mapSync(r => r.id);
     }
     streamValues(table, limit) {
         // `select v` doesn't work for some reason
@@ -44,14 +41,11 @@ export class DatastoreKeyValueDB {
         return (this.db
             .streamQuery(q)
             // .on('error', err => stream.emit('error', err))
-            .map(r => r.v));
+            .mapSync(r => r.v));
     }
     streamEntries(table, limit) {
         const q = DBQuery.create(table).limit(limit || 0);
-        return (this.db
-            .streamQuery(q)
-            // .on('error', err => stream.emit('error', err))
-            .map(r => [r.id, r.v]));
+        return this.db.streamQuery(q).mapSync(r => [r.id, r.v]);
     }
     async count(table) {
         const q = DBQuery.create(table);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
   "type": "module",
-  "version": "4.15.1",
+  "version": "4.16.1",
   "description": "Opinionated library to work with Google Datastore, implements CommonDB",
   "dependencies": {
     "@google-cloud/datastore": "^10",

package/src/datastore.db.ts

@@ -1,4 +1,3 @@
-import { Transform } from 'node:stream'
 import type { Datastore, Key, PropertyFilter, Query, Transaction } from '@google-cloud/datastore'
 import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
 import type {
@@ -17,6 +16,7 @@ import type {
 import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib'
 import { _round } from '@naturalcycles/js-lib'
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js'
+import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
 import { _assert } from '@naturalcycles/js-lib/error/assert.js'
 import { _errorDataAppend, TimeoutError } from '@naturalcycles/js-lib/error/error.util.js'
 import type {
@@ -41,7 +41,7 @@ import {
   type StringMap,
 } from '@naturalcycles/js-lib/types'
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type {
   DatastoreDBCfg,
   DatastoreDBOptions,
@@ -183,7 +183,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
           throw err
         }
 
-        this.cfg.logger.log('datastore recreated on error')
+        this.cfg.logger.log(
+          `datastore recreated on timeout (${_ms(this.cfg.timeout)}) while loading ${table}`,
+        )
 
         // This is to debug "GCP Datastore Timeout issue"
         const datastoreLib = await this.getDatastoreLib()
@@ -327,15 +329,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   override streamQuery<ROW extends ObjectWithId>(
     dbQuery: DBQuery<ROW>,
     _opt?: DatastoreDBStreamOptions,
-  ): ReadableTyped<ROW> {
-    const transform = new Transform({
-      objectMode: true,
-      transform: (chunk, _, cb) => {
-        cb(null, this.mapId(chunk))
-      },
-    })
+  ): Pipeline<ROW> {
+    return Pipeline.fromAsyncReadable<ROW>(async () => {
+      const ds = await this.ds()
 
-    void this.ds().then(async ds => {
       const q = dbQueryToDatastoreQuery(
         dbQuery,
         ds.createQuery(dbQuery.table),
@@ -348,15 +345,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         ..._opt,
       }
 
-      ;(opt.experimentalCursorStream
+      return opt.experimentalCursorStream
         ? new DatastoreStreamReadable<ROW>(q, opt)
-        : (ds.runQueryStream(q, this.getRunQueryOptions(opt)) as ReadableTyped<ROW>)
-      )
-        .on('error', err => transform.destroy(err))
-        .pipe(transform)
-    })
-
-    return transform
+        : ds.runQueryStream(q, this.getRunQueryOptions(opt))
+    }).mapSync(r => this.mapId<ROW>(r))
   }
 
   // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js

package/src/datastoreKeyValueDB.ts

@@ -3,7 +3,7 @@ import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '@natural
 import { commonKeyValueDBFullSupport } from '@naturalcycles/db-lib/kv'
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
 import type { ObjectWithId } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import { DatastoreDB } from './datastore.db.js'
 import type { DatastoreDBCfg } from './datastore.model.js'
 
@@ -52,20 +52,15 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
     )
   }
 
-  streamIds(table: string, limit?: number): ReadableTyped<string> {
+  streamIds(table: string, limit?: number): Pipeline<string> {
     const q = DBQuery.create<ObjectWithId>(table)
       .select(['id'])
       .limit(limit || 0)
 
-    return (
-      this.db
-        .streamQuery(q)
-        // .on('error', err => stream.emit('error', err))
-        .map(r => r.id)
-    )
+    return this.db.streamQuery(q).mapSync(r => r.id)
   }
 
-  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+  streamValues(table: string, limit?: number): Pipeline<Buffer> {
     // `select v` doesn't work for some reason
     const q = DBQuery.create<KVObject>(table).limit(limit || 0)
 
@@ -73,19 +68,14 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       this.db
         .streamQuery(q)
         // .on('error', err => stream.emit('error', err))
-        .map(r => r.v)
+        .mapSync(r => r.v)
     )
   }
 
-  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
+  streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple> {
     const q = DBQuery.create<KVObject>(table).limit(limit || 0)
 
-    return (
-      this.db
-        .streamQuery(q)
-        // .on('error', err => stream.emit('error', err))
-        .map(r => [r.id, r.v])
-    )
+    return this.db.streamQuery(q).mapSync(r => [r.id, r.v])
   }
 
   async count(table: string): Promise<number> {