@naturalcycles/firestore-lib 2.12.1 → 2.13.0

package/dist/firestore.db.d.ts

@@ -3,7 +3,7 @@ import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOption
 import { BaseCommonDB } from '@naturalcycles/db-lib';
 import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log';
 import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
     constructor(cfg: FirestoreDBCfg);
     cfg: FirestoreDBCfg & {
@@ -15,7 +15,7 @@ export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
     runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: FirestoreDBOptions): Promise<RunQueryResult<ROW>>;
     runFirestoreQuery<ROW extends ObjectWithId>(q: Query): Promise<ROW[]>;
     runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: FirestoreDBOptions): Promise<number>;
-    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt_?: FirestoreDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt_?: FirestoreDBStreamOptions): Pipeline<ROW>;
     saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
     multiSave<ROW extends ObjectWithId>(map: StringMap<ROW[]>, opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
     patchById<ROW extends ObjectWithId>(table: string, id: string, patch: Partial<ROW>, opt?: FirestoreDBOptions): Promise<void>;

package/dist/firestore.db.js

@@ -6,6 +6,7 @@ import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _stringMapEntries } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { escapeDocId, unescapeDocId } from './firestore.util.js';
 import { FirestoreShardedReadable } from './firestoreShardedReadable.js';
 import { FirestoreStreamReadable } from './firestoreStreamReadable.js';
@@ -101,17 +102,17 @@ export class FirestoreDB extends BaseCommonDB {
             ...opt_,
         };
         if (opt.experimentalCursorStream) {
-            return new FirestoreStreamReadable(firestoreQuery, q, opt);
+            return Pipeline.from(new FirestoreStreamReadable(firestoreQuery, q, opt));
         }
         if (opt.experimentalShardedStream) {
-            return new FirestoreShardedReadable(firestoreQuery, q, opt);
+            return Pipeline.from(new FirestoreShardedReadable(firestoreQuery, q, opt));
         }
-        return firestoreQuery.stream().map(doc => {
+        return Pipeline.from(firestoreQuery.stream().map(doc => {
             return {
                 id: unescapeDocId(doc.id),
                 ...doc.data(),
             };
-        });
+        }));
     }
     // SAVE
     async saveBatch(table, rows, opt = {}) {

package/dist/firestoreStreamReadable.js

@@ -1,5 +1,5 @@
 import { Readable } from 'node:stream';
-import { FieldPath } from '@google-cloud/firestore';
+import { FieldPath, } from '@google-cloud/firestore';
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
@@ -91,9 +91,9 @@ export class FirestoreStreamReadable extends Readable {
             return;
         }
         const rows = [];
-        let lastDocId;
+        let lastDoc;
         for (const doc of qs.docs) {
-            lastDocId = doc.id;
+            lastDoc = doc;
             rows.push({
                 id: unescapeDocId(doc.id),
                 ...doc.data(),
@@ -101,7 +101,7 @@ export class FirestoreStreamReadable extends Readable {
         }
         this.rowsRetrieved += rows.length;
         logger.debug(`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved`);
-        this.endCursor = lastDocId;
+        this.endCursor = lastDoc;
         this.queryIsRunning = false; // ready to take more _reads
         let shouldContinue = false;
         for (const row of rows) {
@@ -146,6 +146,7 @@ export class FirestoreStreamReadable extends Readable {
             });
         }
         catch (err) {
+            console.log(q._queryOptions);
             logger.error(`FirestoreStreamReadable error!\n`, {
                 table,
                 rowsRetrieved: this.rowsRetrieved,

package/package.json

@@ -12,7 +12,7 @@
     "@types/node": "^24",
     "dotenv": "^17",
     "firebase-admin": "^13",
-    "@naturalcycles/dev-lib": "18.4.2"
+    "@naturalcycles/dev-lib": "19.37.0"
   },
   "exports": {
     ".": "./dist/index.js"
@@ -38,7 +38,7 @@
   "engines": {
     "node": ">=22.12.0"
   },
-  "version": "2.12.1",
+  "version": "2.13.0",
   "description": "Firestore implementation of CommonDB interface",
   "author": "Natural Cycles Team",
   "license": "MIT",

package/src/firestore.db.ts

@@ -30,7 +30,7 @@ import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/obje
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types'
 import { _stringMapEntries } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline, type ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import { escapeDocId, unescapeDocId } from './firestore.util.js'
 import { FirestoreShardedReadable } from './firestoreShardedReadable.js'
 import { FirestoreStreamReadable } from './firestoreStreamReadable.js'
@@ -152,7 +152,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
   override streamQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     opt_?: FirestoreDBStreamOptions,
-  ): ReadableTyped<ROW> {
+  ): Pipeline<ROW> {
     const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table))
 
     const opt: FirestoreDBStreamOptions = {
@@ -163,19 +163,21 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
     }
 
     if (opt.experimentalCursorStream) {
-      return new FirestoreStreamReadable(firestoreQuery, q, opt)
+      return Pipeline.from(new FirestoreStreamReadable(firestoreQuery, q, opt))
     }
 
     if (opt.experimentalShardedStream) {
-      return new FirestoreShardedReadable(firestoreQuery, q, opt)
+      return Pipeline.from(new FirestoreShardedReadable(firestoreQuery, q, opt))
     }
 
-    return (firestoreQuery.stream() as ReadableTyped<QueryDocumentSnapshot<any>>).map(doc => {
-      return {
-        id: unescapeDocId(doc.id),
-        ...doc.data(),
-      } as ROW
-    })
+    return Pipeline.from(
+      (firestoreQuery.stream() as ReadableTyped<QueryDocumentSnapshot<any>>).map(doc => {
+        return {
+          id: unescapeDocId(doc.id),
+          ...doc.data(),
+        } as ROW
+      }),
+    )
   }
 
   // SAVE

package/src/firestoreStreamReadable.ts

@@ -1,5 +1,10 @@
 import { Readable } from 'node:stream'
-import { FieldPath, type Query, type QuerySnapshot } from '@google-cloud/firestore'
+import {
+  FieldPath,
+  type Query,
+  type QueryDocumentSnapshot,
+  type QuerySnapshot,
+} from '@google-cloud/firestore'
 import type { DBQuery } from '@naturalcycles/db-lib'
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
@@ -17,7 +22,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
   private readonly table: string
   private readonly originalLimit: number
   private rowsRetrieved = 0
-  private endCursor?: string
+  private endCursor?: QueryDocumentSnapshot
   private queryIsRunning = false
   private paused = false
   private done = false
@@ -121,10 +126,10 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
     }
 
     const rows: T[] = []
-    let lastDocId: string | undefined
+    let lastDoc: QueryDocumentSnapshot | undefined
 
     for (const doc of qs.docs) {
-      lastDocId = doc.id
+      lastDoc = doc
       rows.push({
         id: unescapeDocId(doc.id),
         ...doc.data(),
@@ -136,7 +141,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
       `${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved`,
     )
 
-    this.endCursor = lastDocId
+    this.endCursor = lastDoc
     this.queryIsRunning = false // ready to take more _reads
     let shouldContinue = false
 
@@ -186,6 +191,8 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
         },
       )
     } catch (err) {
+      console.log((q as any)._queryOptions)
+
       logger.error(
         `FirestoreStreamReadable error!\n`,
         {