@naturalcycles/firestore-lib 2.6.0 → 2.8.0

package/dist/firestore.db.d.ts

@@ -1,37 +1,28 @@
-import type { Firestore, Query, Transaction } from '@google-cloud/firestore';
-import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBStreamOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
+import type { Firestore, Query, QuerySnapshot, Transaction } from '@google-cloud/firestore';
+import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
 import { BaseCommonDB } from '@naturalcycles/db-lib';
-import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
+import { type CommonLogger } from '@naturalcycles/js-lib/log';
+import type { NumberOfSeconds, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
-export interface FirestoreDBCfg {
-    firestore: Firestore;
-}
-export interface FirestoreDBOptions extends CommonDBOptions {
-}
-export interface FirestoreDBReadOptions extends CommonDBReadOptions {
-}
-export interface FirestoreDBSaveOptions<ROW extends ObjectWithId> extends CommonDBSaveOptions<ROW> {
-}
-export declare class RollbackError extends Error {
-    constructor();
-}
 export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
-    cfg: FirestoreDBCfg;
     constructor(cfg: FirestoreDBCfg);
+    cfg: FirestoreDBCfg & {
+        logger: CommonLogger;
+    };
     support: CommonDBSupport;
     getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: FirestoreDBReadOptions): Promise<ROW[]>;
-    multiGetByIds<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: CommonDBReadOptions): Promise<StringMap<ROW[]>>;
+    multiGet<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: CommonDBReadOptions): Promise<StringMap<ROW[]>>;
     runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: FirestoreDBOptions): Promise<RunQueryResult<ROW>>;
     runFirestoreQuery<ROW extends ObjectWithId>(q: Query): Promise<ROW[]>;
     runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: FirestoreDBOptions): Promise<number>;
-    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt_?: FirestoreDBStreamOptions): ReadableTyped<ROW>;
     saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
-    multiSaveBatch<ROW extends ObjectWithId>(map: StringMap<ROW[]>, opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
+    multiSave<ROW extends ObjectWithId>(map: StringMap<ROW[]>, opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
     patchById<ROW extends ObjectWithId>(table: string, id: string, patch: Partial<ROW>, opt?: FirestoreDBOptions): Promise<void>;
     deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: FirestoreDBOptions): Promise<number>;
     deleteByIds(table: string, ids: string[], opt?: FirestoreDBOptions): Promise<number>;
-    multiDeleteByIds(map: StringMap<string[]>, opt?: FirestoreDBOptions): Promise<number>;
-    private querySnapshotToArray;
+    multiDelete(map: StringMap<string[]>, opt?: FirestoreDBOptions): Promise<number>;
+    querySnapshotToArray<T = any>(qs: QuerySnapshot): T[];
     runInTransaction(fn: DBTransactionFn, opt?: CommonDBTransactionOptions): Promise<void>;
     /**
      * Caveat: it always returns an empty object, not the actual incrementMap.
@@ -53,3 +44,76 @@ export declare class FirestoreDBTransaction implements DBTransaction {
     saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW>): Promise<void>;
     deleteByIds(table: string, ids: string[], opt?: CommonDBOptions): Promise<number>;
 }
+export interface FirestoreDBCfg {
+    firestore: Firestore;
+    /**
+     * Use it to set default options to stream operations,
+     * e.g you can globally enable `experimentalCursorStream` here, set the batchSize, etc.
+     */
+    streamOptions?: FirestoreDBStreamOptions;
+    /**
+     * Default to `console`
+     */
+    logger?: CommonLogger;
+}
+export declare class RollbackError extends Error {
+    constructor();
+}
+export interface FirestoreDBStreamOptions extends FirestoreDBReadOptions {
+    /**
+     * Set to `true` to stream via experimental "cursor-query based stream".
+     *
+     * Defaults to false
+     */
+    experimentalCursorStream?: boolean;
+    /**
+     * Applicable to `experimentalCursorStream`.
+     * Defines the size (limit) of each individual query.
+     *
+     * Default: 1000
+     */
+    batchSize?: number;
+    /**
+     * Applicable to `experimentalCursorStream`
+     *
+     * Set to a value (number of Megabytes) to control the peak RSS size.
+     * If limit is reached - streaming will pause until the stream keeps up, and then
+     * resumes.
+     *
+     * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
+     * when _read is called.
+     *
+     * @default 1000
+     */
+    rssLimitMB?: number;
+    /**
+     * Applicable to `experimentalCursorStream`
+     * Default false.
+     * If true, stream will pause until consumer requests more data (via _read).
+     * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
+     * There will be gaps in time between "last query loaded" and "next query requested".
+     * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
+     * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+     */
+    singleBatchBuffer?: boolean;
+    /**
+     * Set to `true` to log additional debug info, when using experimentalCursorStream.
+     *
+     * @default false
+     */
+    debug?: boolean;
+    /**
+     * Default is undefined.
+     * If set - sets a "safety timer", which will force call _read after the specified number of seconds.
+     * This is to prevent possible "dead-lock"/race-condition that would make the stream "hang".
+     *
+     * @experimental
+     */
+    maxWait?: NumberOfSeconds;
+}
+export interface FirestoreDBOptions extends CommonDBOptions {
+}
+export interface FirestoreDBReadOptions extends CommonDBReadOptions {
+}
+export interface FirestoreDBSaveOptions<ROW extends ObjectWithId> extends CommonDBSaveOptions<ROW> {
+}

package/dist/firestore.db.js

@@ -3,27 +3,22 @@ import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
 import { _isTruthy } from '@naturalcycles/js-lib';
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
+import { commonLoggerMinLevel } from '@naturalcycles/js-lib/log';
 import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _stringMapEntries } from '@naturalcycles/js-lib/types';
 import { escapeDocId, unescapeDocId } from './firestore.util.js';
+import { FirestoreStreamReadable } from './firestoreStreamReadable.js';
 import { dbQueryToFirestoreQuery } from './query.util.js';
-const methodMap = {
-    insert: 'create',
-    update: 'update',
-    upsert: 'set',
-};
-export class RollbackError extends Error {
-    constructor() {
-        super('rollback');
-    }
-}
 export class FirestoreDB extends BaseCommonDB {
-    cfg;
     constructor(cfg) {
         super();
-        this.cfg = cfg;
+        this.cfg = {
+            logger: console,
+            ...cfg,
+        };
     }
+    cfg;
     support = {
         ...commonDBFullSupport,
         patchByQuery: false, // todo: can be implemented
@@ -33,6 +28,7 @@ export class FirestoreDB extends BaseCommonDB {
     async getByIds(table, ids, opt = {}) {
         if (!ids.length)
             return [];
+        // todo: support PITR: https://firebase.google.com/docs/firestore/enterprise/use-pitr#read-pitr
         const { firestore } = this.cfg;
         const col = firestore.collection(table);
         return (await (opt.tx?.tx || firestore).getAll(...ids.map(id => col.doc(escapeDocId(id)))))
@@ -47,7 +43,7 @@ export class FirestoreDB extends BaseCommonDB {
         })
             .filter(_isTruthy);
     }
-    async multiGetByIds(map, opt = {}) {
+    async multiGet(map, opt = {}) {
         const result = {};
         const { firestore } = this.cfg;
         const refs = [];
@@ -95,8 +91,15 @@ export class FirestoreDB extends BaseCommonDB {
         const r = await firestoreQuery.count().get();
         return r.data().count;
     }
-    streamQuery(q, _opt) {
+    streamQuery(q, opt_) {
         const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table));
+        const opt = {
+            ...this.cfg.streamOptions,
+            ...opt_,
+        };
+        if (opt.experimentalCursorStream) {
+            return new FirestoreStreamReadable(firestoreQuery, q, opt, commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'));
+        }
         return firestoreQuery.stream().map(doc => {
             return {
                 id: unescapeDocId(doc.id),
@@ -132,7 +135,7 @@ export class FirestoreDB extends BaseCommonDB {
             await batch.commit();
         }, { concurrency: FIRESTORE_RECOMMENDED_CONCURRENCY });
     }
-    async multiSaveBatch(map, opt = {}) {
+    async multiSave(map, opt = {}) {
         const { firestore } = this.cfg;
         const method = methodMap[opt.saveMethod] || 'set';
         if (opt.tx) {
@@ -206,7 +209,7 @@ export class FirestoreDB extends BaseCommonDB {
         }, { concurrency: FIRESTORE_RECOMMENDED_CONCURRENCY });
         return ids.length;
     }
-    async multiDeleteByIds(map, opt = {}) {
+    async multiDelete(map, opt = {}) {
         const { firestore } = this.cfg;
         const refs = [];
         for (const [table, ids] of _stringMapEntries(map)) {
@@ -306,3 +309,13 @@ export class FirestoreDBTransaction {
 const MAX_ITEMS = 500;
 // It's an empyrical value, but anything less than infinity is better than infinity
 const FIRESTORE_RECOMMENDED_CONCURRENCY = 8;
+const methodMap = {
+    insert: 'create',
+    update: 'update',
+    upsert: 'set',
+};
+export class RollbackError extends Error {
+    constructor() {
+        super('rollback');
+    }
+}

package/dist/firestoreStreamReadable.d.ts

@@ -0,0 +1,28 @@
+import { Readable } from 'node:stream';
+import { type Query } from '@google-cloud/firestore';
+import type { DBQuery } from '@naturalcycles/db-lib';
+import type { CommonLogger } from '@naturalcycles/js-lib/log';
+import type { ObjectWithId } from '@naturalcycles/js-lib/types';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { FirestoreDBStreamOptions } from './firestore.db.js';
+export declare class FirestoreStreamReadable<T extends ObjectWithId = any> extends Readable implements ReadableTyped<T> {
+    private q;
+    private logger;
+    private readonly table;
+    private readonly originalLimit;
+    private rowsRetrieved;
+    private endCursor?;
+    private running;
+    private done;
+    private lastQueryDone?;
+    private totalWait;
+    private readonly opt;
+    constructor(q: Query, dbQuery: DBQuery<T>, opt: FirestoreDBStreamOptions, logger: CommonLogger);
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    count: number;
+    _read(): void;
+    private runNextQuery;
+}

package/dist/firestoreStreamReadable.js

@@ -0,0 +1,137 @@
+import { Readable } from 'node:stream';
+import { FieldPath } from '@google-cloud/firestore';
+import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
+import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
+import { unescapeDocId } from './firestore.util.js';
+export class FirestoreStreamReadable extends Readable {
+    q;
+    logger;
+    table;
+    originalLimit;
+    rowsRetrieved = 0;
+    endCursor;
+    running = false;
+    done = false;
+    lastQueryDone;
+    totalWait = 0;
+    opt;
+    // private readonly dsOpt: RunQueryOptions
+    constructor(q, dbQuery, opt, logger) {
+        super({ objectMode: true });
+        this.q = q;
+        this.logger = logger;
+        this.opt = {
+            rssLimitMB: 1000,
+            batchSize: 1000,
+            ...opt,
+        };
+        // todo: support PITR!
+        // this.dsOpt = {}
+        // if (opt.readAt) {
+        //   // Datastore expects UnixTimestamp in milliseconds
+        //   this.dsOpt.readTime = opt.readAt * 1000
+        // }
+        this.originalLimit = dbQuery._limitValue;
+        this.table = dbQuery.table;
+        logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${this.opt.batchSize}`);
+    }
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    count = 0;
+    _read() {
+        // this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+        // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+        this.count++;
+        if (this.done) {
+            this.logger.warn(`!!! _read was called, but done==true`);
+            return;
+        }
+        if (!this.running) {
+            void this.runNextQuery().catch(err => {
+                console.log('error in runNextQuery', err);
+                this.emit('error', err);
+            });
+        }
+        else {
+            this.logger.log(`_read ${this.count}, wasRunning: true`);
+        }
+    }
+    async runNextQuery() {
+        if (this.done)
+            return;
+        if (this.lastQueryDone) {
+            const now = Date.now();
+            this.totalWait += now - this.lastQueryDone;
+        }
+        this.running = true;
+        let limit = this.opt.batchSize;
+        if (this.originalLimit) {
+            limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved);
+        }
+        // console.log(`limit: ${limit}`)
+        // We have to orderBy documentId, to be able to use id as a cursor
+        let q = this.q.orderBy(FieldPath.documentId()).limit(limit);
+        if (this.endCursor) {
+            q = q.startAfter(this.endCursor);
+        }
+        let qs;
+        try {
+            await pRetry(async () => {
+                qs = await q.get();
+            }, {
+                name: `FirestoreStreamReadable.query(${this.table})`,
+                maxAttempts: 5,
+                delay: 5000,
+                delayMultiplier: 2,
+                logger: this.logger,
+                timeout: 120_000, // 2 minutes
+            });
+        }
+        catch (err) {
+            console.log(`FirestoreStreamReadable error!\n`, {
+                table: this.table,
+                rowsRetrieved: this.rowsRetrieved,
+            }, err);
+            this.emit('error', err);
+            // clearInterval(this.maxWaitInterval)
+            return;
+        }
+        const rows = [];
+        let lastDocId;
+        for (const doc of qs.docs) {
+            lastDocId = doc.id;
+            rows.push({
+                id: unescapeDocId(doc.id),
+                ...doc.data(),
+            });
+        }
+        this.rowsRetrieved += rows.length;
+        this.logger.log(`${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+        this.endCursor = lastDocId;
+        this.running = false; // ready to take more _reads
+        this.lastQueryDone = Date.now();
+        for (const row of rows) {
+            this.push(row);
+        }
+        if (qs.empty || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+            this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+            this.push(null);
+            this.done = true;
+        }
+        else if (this.opt.singleBatchBuffer) {
+            // here we don't start next query until we're asked (via next _read call)
+            // so, let's do nothing
+        }
+        else {
+            const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
+            if (rssMB <= this.opt.rssLimitMB) {
+                void this.runNextQuery();
+            }
+            else {
+                this.logger.warn(`${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
+            }
+        }
+    }
+}

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 import { Firestore } from '@google-cloud/firestore';
 export * from './firestore.db.js';
+export * from './firestoreStreamReadable.js';
 export * from './query.util.js';
 export { Firestore };

package/dist/index.js

@@ -1,4 +1,5 @@
 import { Firestore } from '@google-cloud/firestore';
 export * from './firestore.db.js';
+export * from './firestoreStreamReadable.js';
 export * from './query.util.js';
 export { Firestore };

package/dist/query.util.js

@@ -12,6 +12,8 @@ export function dbQueryToFirestoreQuery(dbQuery, emptyQuery) {
     }
     // order
     for (const ord of dbQuery._orders) {
+        // todo: support ordering by id like this:
+        // .orderBy(FieldPath.documentId())
         q = q.orderBy(ord.name, ord.descending ? 'desc' : 'asc');
     }
     // limit
@@ -21,5 +23,12 @@ export function dbQueryToFirestoreQuery(dbQuery, emptyQuery) {
         // todo: check if at least id / __key__ is required to be set
         q = q.select(...dbQuery._selectedFieldNames);
     }
+    // cursor
+    if (dbQuery._startCursor) {
+        q = q.startAt(dbQuery._startCursor);
+    }
+    if (dbQuery._endCursor) {
+        q = q.endAt(dbQuery._endCursor);
+    }
     return q;
 }

package/package.json

@@ -12,7 +12,7 @@
     "@types/node": "^24",
     "dotenv": "^17",
     "firebase-admin": "^13",
-    "@naturalcycles/dev-lib": "18.4.2"
+    "@naturalcycles/dev-lib": "19.34.2"
   },
   "exports": {
     ".": "./dist/index.js"
@@ -38,7 +38,7 @@
   "engines": {
     "node": ">=22.12.0"
   },
-  "version": "2.6.0",
+  "version": "2.8.0",
   "description": "Firestore implementation of CommonDB interface",
   "author": "Natural Cycles Team",
   "license": "MIT",

package/src/firestore.db.ts

@@ -14,7 +14,6 @@ import type {
   CommonDBReadOptions,
   CommonDBSaveMethod,
   CommonDBSaveOptions,
-  CommonDBStreamOptions,
   CommonDBSupport,
   CommonDBTransactionOptions,
   DBQuery,
@@ -26,42 +25,27 @@ import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib'
 import { _isTruthy } from '@naturalcycles/js-lib'
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js'
 import { _assert } from '@naturalcycles/js-lib/error/assert.js'
+import { type CommonLogger, commonLoggerMinLevel } from '@naturalcycles/js-lib/log'
 import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/object.util.js'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
-import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
+import type { NumberOfSeconds, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
 import { _stringMapEntries } from '@naturalcycles/js-lib/types'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import { escapeDocId, unescapeDocId } from './firestore.util.js'
+import { FirestoreStreamReadable } from './firestoreStreamReadable.js'
 import { dbQueryToFirestoreQuery } from './query.util.js'
 
-export interface FirestoreDBCfg {
-  firestore: Firestore
-}
-
-export interface FirestoreDBOptions extends CommonDBOptions {}
-export interface FirestoreDBReadOptions extends CommonDBReadOptions {}
-export interface FirestoreDBSaveOptions<ROW extends ObjectWithId>
-  extends CommonDBSaveOptions<ROW> {}
-
-type SaveOp = 'create' | 'update' | 'set'
-
-const methodMap: Record<CommonDBSaveMethod, SaveOp> = {
-  insert: 'create',
-  update: 'update',
-  upsert: 'set',
-}
-
-export class RollbackError extends Error {
-  constructor() {
-    super('rollback')
-  }
-}
-
 export class FirestoreDB extends BaseCommonDB implements CommonDB {
-  constructor(public cfg: FirestoreDBCfg) {
+  constructor(cfg: FirestoreDBCfg) {
     super()
+    this.cfg = {
+      logger: console,
+      ...cfg,
+    }
   }
 
+  cfg: FirestoreDBCfg & { logger: CommonLogger }
+
   override support: CommonDBSupport = {
     ...commonDBFullSupport,
     patchByQuery: false, // todo: can be implemented
@@ -76,6 +60,8 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
   ): Promise<ROW[]> {
     if (!ids.length) return []
 
+    // todo: support PITR: https://firebase.google.com/docs/firestore/enterprise/use-pitr#read-pitr
+
     const { firestore } = this.cfg
     const col = firestore.collection(table)
 
@@ -95,7 +81,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
       .filter(_isTruthy)
   }
 
-  override async multiGetByIds<ROW extends ObjectWithId>(
+  override async multiGet<ROW extends ObjectWithId>(
     map: StringMap<string[]>,
     opt: CommonDBReadOptions = {},
   ): Promise<StringMap<ROW[]>> {
@@ -163,10 +149,24 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
 
   override streamQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
-    _opt?: CommonDBStreamOptions,
+    opt_?: FirestoreDBStreamOptions,
   ): ReadableTyped<ROW> {
     const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table))
 
+    const opt: FirestoreDBStreamOptions = {
+      ...this.cfg.streamOptions,
+      ...opt_,
+    }
+
+    if (opt.experimentalCursorStream) {
+      return new FirestoreStreamReadable(
+        firestoreQuery,
+        q,
+        opt,
+        commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
+      )
+    }
+
     return (firestoreQuery.stream() as ReadableTyped<QueryDocumentSnapshot<any>>).map(doc => {
       return {
         id: unescapeDocId(doc.id),
@@ -230,7 +230,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
     )
   }
 
-  override async multiSaveBatch<ROW extends ObjectWithId>(
+  override async multiSave<ROW extends ObjectWithId>(
     map: StringMap<ROW[]>,
     opt: FirestoreDBSaveOptions<ROW> = {},
   ): Promise<void> {
@@ -361,7 +361,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
     return ids.length
   }
 
-  override async multiDeleteByIds(
+  override async multiDelete(
     map: StringMap<string[]>,
     opt: FirestoreDBOptions = {},
   ): Promise<number> {
@@ -394,7 +394,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
     return refs.length
   }
 
-  private querySnapshotToArray<T = any>(qs: QuerySnapshot): T[] {
+  querySnapshotToArray<T = any>(qs: QuerySnapshot): T[] {
     return qs.docs.map(
       doc =>
         ({
@@ -509,3 +509,95 @@ const MAX_ITEMS = 500
 const FIRESTORE_RECOMMENDED_CONCURRENCY = 8
 
 type TableRow<ROW extends ObjectWithId> = [table: string, row: ROW]
+
+export interface FirestoreDBCfg {
+  firestore: Firestore
+
+  /**
+   * Use it to set default options to stream operations,
+   * e.g you can globally enable `experimentalCursorStream` here, set the batchSize, etc.
+   */
+  streamOptions?: FirestoreDBStreamOptions
+
+  /**
+   * Default to `console`
+   */
+  logger?: CommonLogger
+}
+
+const methodMap: Record<CommonDBSaveMethod, SaveOp> = {
+  insert: 'create',
+  update: 'update',
+  upsert: 'set',
+}
+
+export class RollbackError extends Error {
+  constructor() {
+    super('rollback')
+  }
+}
+
+export interface FirestoreDBStreamOptions extends FirestoreDBReadOptions {
+  /**
+   * Set to `true` to stream via experimental "cursor-query based stream".
+   *
+   * Defaults to false
+   */
+  experimentalCursorStream?: boolean
+
+  /**
+   * Applicable to `experimentalCursorStream`.
+   * Defines the size (limit) of each individual query.
+   *
+   * Default: 1000
+   */
+  batchSize?: number
+
+  /**
+   * Applicable to `experimentalCursorStream`
+   *
+   * Set to a value (number of Megabytes) to control the peak RSS size.
+   * If limit is reached - streaming will pause until the stream keeps up, and then
+   * resumes.
+   *
+   * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
+   * when _read is called.
+   *
+   * @default 1000
+   */
+  rssLimitMB?: number
+
+  /**
+   * Applicable to `experimentalCursorStream`
+   * Default false.
+   * If true, stream will pause until consumer requests more data (via _read).
+   * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
+   * There will be gaps in time between "last query loaded" and "next query requested".
+   * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
+   * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+   */
+  singleBatchBuffer?: boolean
+
+  /**
+   * Set to `true` to log additional debug info, when using experimentalCursorStream.
+   *
+   * @default false
+   */
+  debug?: boolean
+
+  /**
+   * Default is undefined.
+   * If set - sets a "safety timer", which will force call _read after the specified number of seconds.
+   * This is to prevent possible "dead-lock"/race-condition that would make the stream "hang".
+   *
+   * @experimental
+   */
+  maxWait?: NumberOfSeconds
+}
+
+export interface FirestoreDBOptions extends CommonDBOptions {}
+export interface FirestoreDBReadOptions extends CommonDBReadOptions {}
+export interface FirestoreDBSaveOptions<ROW extends ObjectWithId>
+  extends CommonDBSaveOptions<ROW> {}
+
+type SaveOp = 'create' | 'update' | 'set'

package/src/firestoreStreamReadable.ts

@@ -0,0 +1,183 @@
+import { Readable } from 'node:stream'
+import { FieldPath, type Query, type QuerySnapshot } from '@google-cloud/firestore'
+import type { DBQuery } from '@naturalcycles/db-lib'
+import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
+import type { CommonLogger } from '@naturalcycles/js-lib/log'
+import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
+import type { ObjectWithId } from '@naturalcycles/js-lib/types'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { FirestoreDBStreamOptions } from './firestore.db.js'
+import { unescapeDocId } from './firestore.util.js'
+
+export class FirestoreStreamReadable<T extends ObjectWithId = any>
+  extends Readable
+  implements ReadableTyped<T>
+{
+  private readonly table: string
+  private readonly originalLimit: number
+  private rowsRetrieved = 0
+  private endCursor?: string
+  private running = false
+  private done = false
+  private lastQueryDone?: number
+  private totalWait = 0
+
+  private readonly opt: FirestoreDBStreamOptions & { batchSize: number; rssLimitMB: number }
+  // private readonly dsOpt: RunQueryOptions
+
+  constructor(
+    private q: Query,
+    dbQuery: DBQuery<T>,
+    opt: FirestoreDBStreamOptions,
+    private logger: CommonLogger,
+  ) {
+    super({ objectMode: true })
+
+    this.opt = {
+      rssLimitMB: 1000,
+      batchSize: 1000,
+      ...opt,
+    }
+    // todo: support PITR!
+    // this.dsOpt = {}
+    // if (opt.readAt) {
+    //   // Datastore expects UnixTimestamp in milliseconds
+    //   this.dsOpt.readTime = opt.readAt * 1000
+    // }
+
+    this.originalLimit = dbQuery._limitValue
+    this.table = dbQuery.table
+
+    logger.log(
+      `!! using experimentalCursorStream !! ${this.table}, batchSize: ${this.opt.batchSize}`,
+    )
+  }
+
+  /**
+   * Counts how many times _read was called.
+   * For debugging.
+   */
+  count = 0
+
+  override _read(): void {
+    // this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+
+    // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+    this.count++
+
+    if (this.done) {
+      this.logger.warn(`!!! _read was called, but done==true`)
+      return
+    }
+
+    if (!this.running) {
+      void this.runNextQuery().catch(err => {
+        console.log('error in runNextQuery', err)
+        this.emit('error', err)
+      })
+    } else {
+      this.logger.log(`_read ${this.count}, wasRunning: true`)
+    }
+  }
+
+  private async runNextQuery(): Promise<void> {
+    if (this.done) return
+
+    if (this.lastQueryDone) {
+      const now = Date.now()
+      this.totalWait += now - this.lastQueryDone
+    }
+
+    this.running = true
+
+    let limit = this.opt.batchSize
+
+    if (this.originalLimit) {
+      limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved)
+    }
+
+    // console.log(`limit: ${limit}`)
+    // We have to orderBy documentId, to be able to use id as a cursor
+    let q = this.q.orderBy(FieldPath.documentId()).limit(limit)
+    if (this.endCursor) {
+      q = q.startAfter(this.endCursor)
+    }
+
+    let qs: QuerySnapshot
+
+    try {
+      await pRetry(
+        async () => {
+          qs = await q.get()
+        },
+        {
+          name: `FirestoreStreamReadable.query(${this.table})`,
+          maxAttempts: 5,
+          delay: 5000,
+          delayMultiplier: 2,
+          logger: this.logger,
+          timeout: 120_000, // 2 minutes
+        },
+      )
+    } catch (err) {
+      console.log(
+        `FirestoreStreamReadable error!\n`,
+        {
+          table: this.table,
+          rowsRetrieved: this.rowsRetrieved,
+        },
+        err,
+      )
+      this.emit('error', err)
+      // clearInterval(this.maxWaitInterval)
+      return
+    }
+
+    const rows: T[] = []
+    let lastDocId: string | undefined
+
+    for (const doc of qs!.docs) {
+      lastDocId = doc.id
+      rows.push({
+        id: unescapeDocId(doc.id),
+        ...doc.data(),
+      } as T)
+    }
+
+    this.rowsRetrieved += rows.length
+    this.logger.log(
+      `${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
+        this.totalWait,
+      )}`,
+    )
+
+    this.endCursor = lastDocId
+    this.running = false // ready to take more _reads
+    this.lastQueryDone = Date.now()
+
+    for (const row of rows) {
+      this.push(row)
+    }
+
+    if (qs!.empty || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+      this.logger.log(
+        `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
+      )
+      this.push(null)
+      this.done = true
+    } else if (this.opt.singleBatchBuffer) {
+      // here we don't start next query until we're asked (via next _read call)
+      // so, let's do nothing
+    } else {
+      const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
+
+      if (rssMB <= this.opt.rssLimitMB) {
+        void this.runNextQuery()
+      } else {
+        this.logger.warn(
+          `${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`,
+        )
+      }
+    }
+  }
+}

package/src/index.ts

@@ -1,4 +1,5 @@
 import { Firestore } from '@google-cloud/firestore'
 export * from './firestore.db.js'
+export * from './firestoreStreamReadable.js'
 export * from './query.util.js'
 export { Firestore }

package/src/query.util.ts

@@ -22,6 +22,8 @@ export function dbQueryToFirestoreQuery<ROW extends ObjectWithId>(
 
   // order
   for (const ord of dbQuery._orders) {
+    // todo: support ordering by id like this:
+    // .orderBy(FieldPath.documentId())
     q = q.orderBy(ord.name as string, ord.descending ? 'desc' : 'asc')
   }
 
@@ -34,5 +36,14 @@ export function dbQueryToFirestoreQuery<ROW extends ObjectWithId>(
     q = q.select(...(dbQuery._selectedFieldNames as string[]))
   }
 
+  // cursor
+  if (dbQuery._startCursor) {
+    q = q.startAt(dbQuery._startCursor)
+  }
+
+  if (dbQuery._endCursor) {
+    q = q.endAt(dbQuery._endCursor)
+  }
+
   return q
 }