@naturalcycles/firestore-lib 2.12.0 → 2.13.0

package/dist/firestore.db.d.ts

@@ -3,7 +3,7 @@ import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOption
 import { BaseCommonDB } from '@naturalcycles/db-lib';
 import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log';
 import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
     constructor(cfg: FirestoreDBCfg);
     cfg: FirestoreDBCfg & {
@@ -15,7 +15,7 @@ export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
     runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: FirestoreDBOptions): Promise<RunQueryResult<ROW>>;
     runFirestoreQuery<ROW extends ObjectWithId>(q: Query): Promise<ROW[]>;
     runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: FirestoreDBOptions): Promise<number>;
-    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt_?: FirestoreDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt_?: FirestoreDBStreamOptions): Pipeline<ROW>;
     saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
     multiSave<ROW extends ObjectWithId>(map: StringMap<ROW[]>, opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
     patchById<ROW extends ObjectWithId>(table: string, id: string, patch: Partial<ROW>, opt?: FirestoreDBOptions): Promise<void>;

package/dist/firestore.db.js

@@ -6,6 +6,7 @@ import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _stringMapEntries } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { escapeDocId, unescapeDocId } from './firestore.util.js';
 import { FirestoreShardedReadable } from './firestoreShardedReadable.js';
 import { FirestoreStreamReadable } from './firestoreStreamReadable.js';
@@ -101,17 +102,17 @@ export class FirestoreDB extends BaseCommonDB {
             ...opt_,
         };
         if (opt.experimentalCursorStream) {
-            return new FirestoreStreamReadable(firestoreQuery, q, opt);
+            return Pipeline.from(new FirestoreStreamReadable(firestoreQuery, q, opt));
         }
         if (opt.experimentalShardedStream) {
-            return new FirestoreShardedReadable(firestoreQuery, q, opt);
+            return Pipeline.from(new FirestoreShardedReadable(firestoreQuery, q, opt));
         }
-        return firestoreQuery.stream().map(doc => {
+        return Pipeline.from(firestoreQuery.stream().map(doc => {
             return {
                 id: unescapeDocId(doc.id),
                 ...doc.data(),
             };
-        });
+        }));
     }
     // SAVE
     async saveBatch(table, rows, opt = {}) {

package/dist/firestoreShardedReadable.js

@@ -65,7 +65,7 @@ export class FirestoreShardedReadable extends Readable {
         }
         void this.runNextQuery(shard).catch(err => {
             this.logger.error('error in runNextQuery', err);
-            this.emit('error', err);
+            this.destroy(err);
         });
     }
     async runNextQuery(shard) {
@@ -155,7 +155,7 @@ export class FirestoreShardedReadable extends Readable {
                 table,
                 rowsRetrieved: this.rowsRetrieved,
             }, err);
-            this.emit('error', err);
+            this.destroy(err);
             return;
         }
     }

package/dist/firestoreStreamReadable.js

@@ -1,5 +1,5 @@
 import { Readable } from 'node:stream';
-import { FieldPath } from '@google-cloud/firestore';
+import { FieldPath, } from '@google-cloud/firestore';
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
@@ -62,7 +62,7 @@ export class FirestoreStreamReadable extends Readable {
         }
         void this.runNextQuery().catch(err => {
             this.logger.error('error in runNextQuery', err);
-            this.emit('error', err);
+            this.destroy(err);
         });
     }
     async runNextQuery() {
@@ -91,9 +91,9 @@ export class FirestoreStreamReadable extends Readable {
             return;
         }
         const rows = [];
-        let lastDocId;
+        let lastDoc;
         for (const doc of qs.docs) {
-            lastDocId = doc.id;
+            lastDoc = doc;
             rows.push({
                 id: unescapeDocId(doc.id),
                 ...doc.data(),
@@ -101,7 +101,7 @@ export class FirestoreStreamReadable extends Readable {
         }
         this.rowsRetrieved += rows.length;
         logger.debug(`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved`);
-        this.endCursor = lastDocId;
+        this.endCursor = lastDoc;
         this.queryIsRunning = false; // ready to take more _reads
         let shouldContinue = false;
         for (const row of rows) {
@@ -137,6 +137,7 @@ export class FirestoreStreamReadable extends Readable {
                 return await q.get();
             }, {
                 name: `FirestoreStreamReadable.query(${table})`,
+                predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
                 maxAttempts: 5,
                 delay: 5000,
                 delayMultiplier: 2,
@@ -145,12 +146,25 @@ export class FirestoreStreamReadable extends Readable {
             });
         }
         catch (err) {
+            console.log(q._queryOptions);
             logger.error(`FirestoreStreamReadable error!\n`, {
                 table,
                 rowsRetrieved: this.rowsRetrieved,
             }, err);
-            this.emit('error', err);
+            this.destroy(err);
             return;
         }
     }
 }
+// Examples of errors:
+// UNKNOWN: Stream removed
+const RETRY_ON = [
+    'GOAWAY',
+    'UNAVAILABLE',
+    'UNKNOWN',
+    'DEADLINE_EXCEEDED',
+    'ABORTED',
+    'much contention',
+    'try again',
+    'timeout',
+].map(s => s.toLowerCase());

package/package.json

@@ -12,7 +12,7 @@
     "@types/node": "^24",
     "dotenv": "^17",
     "firebase-admin": "^13",
-    "@naturalcycles/dev-lib": "18.4.2"
+    "@naturalcycles/dev-lib": "19.37.0"
   },
   "exports": {
     ".": "./dist/index.js"
@@ -38,7 +38,7 @@
   "engines": {
     "node": ">=22.12.0"
   },
-  "version": "2.12.0",
+  "version": "2.13.0",
   "description": "Firestore implementation of CommonDB interface",
   "author": "Natural Cycles Team",
   "license": "MIT",

package/src/firestore.db.ts

@@ -30,7 +30,7 @@ import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/obje
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types'
 import { _stringMapEntries } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline, type ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import { escapeDocId, unescapeDocId } from './firestore.util.js'
 import { FirestoreShardedReadable } from './firestoreShardedReadable.js'
 import { FirestoreStreamReadable } from './firestoreStreamReadable.js'
@@ -152,7 +152,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
   override streamQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     opt_?: FirestoreDBStreamOptions,
-  ): ReadableTyped<ROW> {
+  ): Pipeline<ROW> {
     const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table))
 
     const opt: FirestoreDBStreamOptions = {
@@ -163,19 +163,21 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
     }
 
     if (opt.experimentalCursorStream) {
-      return new FirestoreStreamReadable(firestoreQuery, q, opt)
+      return Pipeline.from(new FirestoreStreamReadable(firestoreQuery, q, opt))
     }
 
     if (opt.experimentalShardedStream) {
-      return new FirestoreShardedReadable(firestoreQuery, q, opt)
+      return Pipeline.from(new FirestoreShardedReadable(firestoreQuery, q, opt))
     }
 
-    return (firestoreQuery.stream() as ReadableTyped<QueryDocumentSnapshot<any>>).map(doc => {
-      return {
-        id: unescapeDocId(doc.id),
-        ...doc.data(),
-      } as ROW
-    })
+    return Pipeline.from(
+      (firestoreQuery.stream() as ReadableTyped<QueryDocumentSnapshot<any>>).map(doc => {
+        return {
+          id: unescapeDocId(doc.id),
+          ...doc.data(),
+        } as ROW
+      }),
+    )
   }
 
   // SAVE

package/src/firestoreShardedReadable.ts

@@ -91,7 +91,7 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
     }
     void this.runNextQuery(shard).catch(err => {
       this.logger.error('error in runNextQuery', err)
-      this.emit('error', err)
+      this.destroy(err)
     })
   }
 
@@ -213,7 +213,7 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
         },
         err,
       )
-      this.emit('error', err)
+      this.destroy(err as Error)
       return
     }
   }

package/src/firestoreStreamReadable.ts

@@ -1,5 +1,10 @@
 import { Readable } from 'node:stream'
-import { FieldPath, type Query, type QuerySnapshot } from '@google-cloud/firestore'
+import {
+  FieldPath,
+  type Query,
+  type QueryDocumentSnapshot,
+  type QuerySnapshot,
+} from '@google-cloud/firestore'
 import type { DBQuery } from '@naturalcycles/db-lib'
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
@@ -17,7 +22,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
   private readonly table: string
   private readonly originalLimit: number
   private rowsRetrieved = 0
-  private endCursor?: string
+  private endCursor?: QueryDocumentSnapshot
   private queryIsRunning = false
   private paused = false
   private done = false
@@ -85,7 +90,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
 
     void this.runNextQuery().catch(err => {
       this.logger.error('error in runNextQuery', err)
-      this.emit('error', err)
+      this.destroy(err)
     })
   }
 
@@ -121,10 +126,10 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
     }
 
     const rows: T[] = []
-    let lastDocId: string | undefined
+    let lastDoc: QueryDocumentSnapshot | undefined
 
     for (const doc of qs.docs) {
-      lastDocId = doc.id
+      lastDoc = doc
       rows.push({
         id: unescapeDocId(doc.id),
         ...doc.data(),
@@ -136,7 +141,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
       `${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved`,
     )
 
-    this.endCursor = lastDocId
+    this.endCursor = lastDoc
     this.queryIsRunning = false // ready to take more _reads
     let shouldContinue = false
 
@@ -177,6 +182,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
         },
         {
           name: `FirestoreStreamReadable.query(${table})`,
+          predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
           maxAttempts: 5,
           delay: 5000,
           delayMultiplier: 2,
@@ -185,6 +191,8 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
         },
       )
     } catch (err) {
+      console.log((q as any)._queryOptions)
+
       logger.error(
         `FirestoreStreamReadable error!\n`,
         {
@@ -193,8 +201,21 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
         },
         err,
       )
-      this.emit('error', err)
+      this.destroy(err as Error)
       return
     }
   }
 }
+
+// Examples of errors:
+// UNKNOWN: Stream removed
+const RETRY_ON = [
+  'GOAWAY',
+  'UNAVAILABLE',
+  'UNKNOWN',
+  'DEADLINE_EXCEEDED',
+  'ABORTED',
+  'much contention',
+  'try again',
+  'timeout',
+].map(s => s.toLowerCase())