@naturalcycles/firestore-lib 2.9.0 → 2.10.0

package/dist/firestore.db.d.ts

@@ -2,7 +2,7 @@ import type { Firestore, Query, QuerySnapshot, Transaction } from '@google-cloud
 import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
 import { BaseCommonDB } from '@naturalcycles/db-lib';
 import { type CommonLogger } from '@naturalcycles/js-lib/log';
-import type { NumberOfSeconds, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
+import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
 export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
     constructor(cfg: FirestoreDBCfg);
@@ -66,50 +66,28 @@ export interface FirestoreDBStreamOptions extends FirestoreDBReadOptions {
      * Defaults to false
      */
     experimentalCursorStream?: boolean;
+    experimentalShardedStream?: boolean;
     /**
      * Applicable to `experimentalCursorStream`.
      * Defines the size (limit) of each individual query.
      *
-     * Default: 1000
+     * Default: 10_000
      */
-    batchSize?: number;
+    batchSize?: PositiveInteger;
     /**
-     * Applicable to `experimentalCursorStream`
-     *
-     * Set to a value (number of Megabytes) to control the peak RSS size.
-     * If limit is reached - streaming will pause until the stream keeps up, and then
-     * resumes.
-     *
-     * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
-     * when _read is called.
-     *
-     * @default 1000
-     */
-    rssLimitMB?: number;
-    /**
-     * Applicable to `experimentalCursorStream`
-     * Default false.
-     * If true, stream will pause until consumer requests more data (via _read).
-     * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
-     * There will be gaps in time between "last query loaded" and "next query requested".
-     * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
-     * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+     * Defaults to 3x batchSize.
+     * Default batchSize is 10_000, so default highWaterMark is 30_000.
+     * Controls how many rows to have "buffered".
+     * Should be at least 1x batchSize, otherwise the stream will be "starving"
+     * between the queries.
      */
-    singleBatchBuffer?: boolean;
+    highWaterMark?: PositiveInteger;
     /**
      * Set to `true` to log additional debug info, when using experimentalCursorStream.
      *
      * @default false
      */
     debug?: boolean;
-    /**
-     * Default is undefined.
-     * If set - sets a "safety timer", which will force call _read after the specified number of seconds.
-     * This is to prevent possible "dead-lock"/race-condition that would make the stream "hang".
-     *
-     * @experimental
-     */
-    maxWait?: NumberOfSeconds;
 }
 export interface FirestoreDBOptions extends CommonDBOptions {
 }

package/dist/firestore.db.js

@@ -8,6 +8,7 @@ import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/obje
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _stringMapEntries } from '@naturalcycles/js-lib/types';
 import { escapeDocId, unescapeDocId } from './firestore.util.js';
+import { FirestoreShardedReadable } from './firestoreShardedReadable.js';
 import { FirestoreStreamReadable } from './firestoreStreamReadable.js';
 import { dbQueryToFirestoreQuery } from './query.util.js';
 export class FirestoreDB extends BaseCommonDB {
@@ -23,6 +24,7 @@ export class FirestoreDB extends BaseCommonDB {
         ...commonDBFullSupport,
         patchByQuery: false, // todo: can be implemented
         tableSchemas: false,
+        createTransaction: false, // Firestore SDK doesn't support it
     };
     // GET
     async getByIds(table, ids, opt = {}) {
@@ -100,6 +102,9 @@ export class FirestoreDB extends BaseCommonDB {
         if (opt.experimentalCursorStream) {
             return new FirestoreStreamReadable(firestoreQuery, q, opt, commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'));
         }
+        if (opt.experimentalShardedStream) {
+            return new FirestoreShardedReadable(firestoreQuery, q, opt, commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'));
+        }
         return firestoreQuery.stream().map(doc => {
             return {
                 id: unescapeDocId(doc.id),

package/dist/firestoreShardedReadable.d.ts

@@ -0,0 +1,41 @@
+import { Readable } from 'node:stream';
+import { type Query } from '@google-cloud/firestore';
+import type { DBQuery } from '@naturalcycles/db-lib';
+import type { CommonLogger } from '@naturalcycles/js-lib/log';
+import type { ObjectWithId } from '@naturalcycles/js-lib/types';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { FirestoreDBStreamOptions } from './firestore.db.js';
+/**
+ * Highly, HIGHLY experimental!
+ */
+export declare class FirestoreShardedReadable<T extends ObjectWithId = any> extends Readable implements ReadableTyped<T> {
+    private readonly q;
+    readonly dbQuery: DBQuery<T>;
+    private logger;
+    private readonly table;
+    private readonly originalLimit;
+    private rowsRetrieved;
+    /**
+     * Next shard to be used for querying.
+     */
+    private nextShard;
+    private cursorByShard;
+    private queryIsRunningByShard;
+    private paused;
+    private done;
+    private doneShards;
+    private lastQueryDoneByShard;
+    private totalWait;
+    private readonly opt;
+    constructor(q: Query, dbQuery: DBQuery<T>, opt: FirestoreDBStreamOptions, logger: CommonLogger);
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    count: number;
+    _read(): void;
+    private runNextQuery;
+    private runQuery;
+    private findNextFreeShard;
+    private _getNextShardAndMove;
+}

package/dist/firestoreShardedReadable.js

@@ -0,0 +1,173 @@
+import { Readable } from 'node:stream';
+import { FieldPath } from '@google-cloud/firestore';
+import { localTime } from '@naturalcycles/js-lib/datetime';
+import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
+import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
+import { unescapeDocId } from './firestore.util.js';
+const SHARDS = 16;
+const SHARD_COLUMN = 'shard16';
+/**
+ * Highly, HIGHLY experimental!
+ */
+export class FirestoreShardedReadable extends Readable {
+    q;
+    dbQuery;
+    logger;
+    table;
+    originalLimit;
+    rowsRetrieved = 0;
+    /**
+     * Next shard to be used for querying.
+     */
+    nextShard = 1;
+    cursorByShard = {};
+    queryIsRunningByShard = {};
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    paused = false;
+    done = false;
+    doneShards = new Set();
+    lastQueryDoneByShard = {};
+    totalWait = 0;
+    opt;
+    constructor(q, dbQuery, opt, logger) {
+        super({ objectMode: true });
+        this.q = q;
+        this.dbQuery = dbQuery;
+        this.logger = logger;
+        this.opt = {
+            batchSize: 3000,
+            ...opt,
+        };
+        this.originalLimit = dbQuery._limitValue;
+        this.table = dbQuery.table;
+        logger.warn(`!! using experimentalShardedStream !! ${this.table}, batchSize: ${this.opt.batchSize}`);
+    }
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    count = 0;
+    _read() {
+        // this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+        // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+        this.count++;
+        if (this.done) {
+            this.logger.warn(`!!! _read was called, but done==true`);
+            return;
+        }
+        // const shard = this.getNextShardAndMove()
+        const shard = this.findNextFreeShard();
+        if (!shard) {
+            this.logger.log(`_read ${this.count}: all shards are busy, skipping`);
+            return;
+        }
+        void this.runNextQuery(shard).catch(err => {
+            console.log('error in runNextQuery', err);
+            this.emit('error', err);
+        });
+    }
+    async runNextQuery(shard) {
+        if (this.done)
+            return;
+        const { logger, table } = this;
+        if (this.lastQueryDoneByShard[shard]) {
+            this.totalWait += Date.now() - this.lastQueryDoneByShard[shard];
+        }
+        this.queryIsRunningByShard[shard] = true;
+        const limit = this.opt.batchSize;
+        // We have to orderBy documentId, to be able to use id as a cursor
+        let q = this.q.where(SHARD_COLUMN, '==', shard).orderBy(FieldPath.documentId()).limit(limit);
+        if (this.cursorByShard[shard]) {
+            q = q.startAfter(this.cursorByShard[shard]);
+        }
+        console.log(`runNextQuery[${shard}]`, {
+            retrieved: this.rowsRetrieved,
+        });
+        const qs = await this.runQuery(q);
+        if (!qs) {
+            // this means we have already emitted an unrecoverable error
+            return;
+        }
+        const rows = [];
+        let lastDocId;
+        for (const doc of qs.docs) {
+            lastDocId = doc.id;
+            rows.push({
+                id: unescapeDocId(doc.id),
+                ...doc.data(),
+            });
+        }
+        this.rowsRetrieved += rows.length;
+        logger.log(`${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+        this.cursorByShard[shard] = lastDocId;
+        this.queryIsRunningByShard[shard] = false; // ready to take more _reads
+        this.lastQueryDoneByShard[shard] = localTime.nowUnixMillis();
+        for (const row of rows) {
+            this.push(row);
+        }
+        if (qs.empty) {
+            logger.warn(`!!!! Shard ${shard} DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+            this.doneShards.add(shard);
+        }
+        if (this.doneShards.size === SHARDS) {
+            logger.warn(`!!!! DONE: all shards completed, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+            this.push(null);
+            this.paused = false;
+            this.done = true;
+            return;
+        }
+        if (this.originalLimit && this.rowsRetrieved >= this.originalLimit) {
+            logger.warn(`!!!! DONE: reached total limit of ${this.originalLimit}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+            this.push(null);
+            this.paused = false;
+            this.done = true;
+            return;
+        }
+        // if (this.paused) {
+        //   this.paused = false
+        // }
+        const nextShard = this.findNextFreeShard();
+        if (nextShard) {
+            void this.runNextQuery(nextShard);
+        }
+        else {
+            logger.warn(`${table} all shards are busy in runNextQuery, skipping`);
+        }
+    }
+    async runQuery(q) {
+        const { table, logger } = this;
+        try {
+            return await pRetry(async () => {
+                return await q.get();
+            }, {
+                name: `FirestoreStreamReadable.query(${table})`,
+                maxAttempts: 5,
+                delay: 5000,
+                delayMultiplier: 2,
+                logger,
+                timeout: 120_000, // 2 minutes
+            });
+        }
+        catch (err) {
+            console.log(`FirestoreStreamReadable error!\n`, {
+                table,
+                rowsRetrieved: this.rowsRetrieved,
+            }, err);
+            this.emit('error', err);
+            return;
+        }
+    }
+    findNextFreeShard() {
+        for (let shard = 1; shard <= SHARDS; shard++) {
+            if (!this.queryIsRunningByShard[shard] && !this.doneShards.has(shard)) {
+                return shard;
+            }
+        }
+    }
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    _getNextShardAndMove() {
+        const shard = this.nextShard;
+        this.nextShard = shard === SHARDS ? 1 : shard + 1;
+        return shard;
+    }
+}

package/dist/firestoreStreamReadable.d.ts

@@ -15,15 +15,14 @@ export declare class FirestoreStreamReadable<T extends ObjectWithId = any> exten
     private queryIsRunning;
     private paused;
     private done;
-    private lastQueryDone?;
-    private totalWait;
-    private readonly opt;
-    constructor(q: Query, dbQuery: DBQuery<T>, opt: FirestoreDBStreamOptions, logger: CommonLogger);
     /**
      * Counts how many times _read was called.
      * For debugging.
      */
-    count: number;
+    countReads: number;
+    private readonly opt;
+    constructor(q: Query, dbQuery: DBQuery<T>, opt: FirestoreDBStreamOptions, logger: CommonLogger);
     _read(): void;
     private runNextQuery;
+    private runQuery;
 }

package/dist/firestoreStreamReadable.js

@@ -1,6 +1,7 @@
 import { Readable } from 'node:stream';
 import { FieldPath } from '@google-cloud/firestore';
-import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
+import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
 import { unescapeDocId } from './firestore.util.js';
 export class FirestoreStreamReadable extends Readable {
@@ -13,61 +14,60 @@ export class FirestoreStreamReadable extends Readable {
     queryIsRunning = false;
     paused = false;
     done = false;
-    lastQueryDone;
-    totalWait = 0;
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    countReads = 0;
     opt;
-    // private readonly dsOpt: RunQueryOptions
     constructor(q, dbQuery, opt, logger) {
-        super({ objectMode: true });
+        // 10_000 was optimal in benchmarks
+        const { batchSize = 10_000 } = opt;
+        const { highWaterMark = batchSize * 3 } = opt;
+        // Defaulting highWaterMark to 3x batchSize
+        super({ objectMode: true, highWaterMark });
         this.q = q;
         this.logger = logger;
         this.opt = {
-            rssLimitMB: 1000,
-            batchSize: 1000,
             ...opt,
+            batchSize,
+            highWaterMark,
         };
         // todo: support PITR!
-        // this.dsOpt = {}
-        // if (opt.readAt) {
-        //   // Datastore expects UnixTimestamp in milliseconds
-        //   this.dsOpt.readTime = opt.readAt * 1000
-        // }
         this.originalLimit = dbQuery._limitValue;
         this.table = dbQuery.table;
-        logger.warn(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${this.opt.batchSize}`);
+        logger.warn(`!!! using experimentalCursorStream`, {
+            table: this.table,
+            batchSize,
+            highWaterMark,
+        });
     }
-    /**
-     * Counts how many times _read was called.
-     * For debugging.
-     */
-    count = 0;
     _read() {
         // this.lastReadTimestamp = Date.now() as UnixTimestampMillis
         // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
-        this.count++;
+        this.countReads++;
         if (this.done) {
             this.logger.warn(`!!! _read was called, but done==true`);
             return;
         }
-        if (!this.queryIsRunning) {
-            void this.runNextQuery().catch(err => {
-                console.log('error in runNextQuery', err);
-                this.emit('error', err);
-            });
+        if (this.paused) {
+            this.logger.log(`_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`);
+            this.paused = false;
         }
-        else {
-            this.logger.log(`_read ${this.count}, queryIsRunning: true`);
+        if (this.queryIsRunning) {
+            this.logger.log(`_read #${this.countReads}, queryIsRunning: true, doing nothing`);
             // todo: check if this can cause a "hang", if no more _reads would come later and we get stuck?
+            return;
         }
+        void this.runNextQuery().catch(err => {
+            console.log('error in runNextQuery', err);
+            this.emit('error', err);
+        });
     }
     async runNextQuery() {
         if (this.done)
             return;
         const { logger, table } = this;
-        if (this.lastQueryDone) {
-            const now = Date.now();
-            this.totalWait += now - this.lastQueryDone;
-        }
         this.queryIsRunning = true;
         let limit = this.opt.batchSize;
         if (this.originalLimit) {
@@ -79,26 +79,15 @@ export class FirestoreStreamReadable extends Readable {
         if (this.endCursor) {
             q = q.startAfter(this.endCursor);
         }
-        let qs;
-        try {
-            await pRetry(async () => {
-                qs = await q.get();
-            }, {
-                name: `FirestoreStreamReadable.query(${table})`,
-                maxAttempts: 5,
-                delay: 5000,
-                delayMultiplier: 2,
-                logger,
-                timeout: 120_000, // 2 minutes
-            });
-        }
-        catch (err) {
-            console.log(`FirestoreStreamReadable error!\n`, {
-                table,
-                rowsRetrieved: this.rowsRetrieved,
-            }, err);
-            this.emit('error', err);
-            // clearInterval(this.maxWaitInterval)
+        // logger.log(`runNextQuery`, {
+        //   rowsRetrieved: this.rowsRetrieved,
+        //   paused: this.paused,
+        // })
+        const started = localTime.nowUnixMillis();
+        const qs = await this.runQuery(q);
+        logger.log(`${table} query took ${_since(started)}`);
+        if (!qs) {
+            // error already emitted in runQuery
             return;
         }
         const rows = [];
@@ -111,37 +100,57 @@ export class FirestoreStreamReadable extends Readable {
             });
         }
         this.rowsRetrieved += rows.length;
-        logger.log(`${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+        logger.log(`${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved`);
         this.endCursor = lastDocId;
         this.queryIsRunning = false; // ready to take more _reads
-        this.lastQueryDone = Date.now();
+        let shouldContinue = false;
         for (const row of rows) {
-            this.push(row);
+            shouldContinue = this.push(row);
         }
-        if (qs.empty || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-            logger.warn(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+        if (!rows.length || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+            logger.warn(`${table} DONE! ${this.rowsRetrieved} rowsRetrieved`);
             this.push(null);
-            this.paused = false;
             this.done = true;
+            this.paused = false;
             return;
         }
-        if (this.opt.singleBatchBuffer) {
-            // here we don't start next query until we're asked (via next _read call)
-            // so, let's do nothing
-            return;
+        if (shouldContinue) {
+            // Keep the stream flowing
+            logger.log(`${table} continuing the stream`);
+            void this.runNextQuery();
         }
-        const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
-        const { rssLimitMB } = this.opt;
-        if (rssMB <= rssLimitMB) {
+        else {
+            // Not starting the next query
             if (this.paused) {
-                logger.warn(`${table} rssLimitMB is below ${rssMB} < ${rssLimitMB}, unpausing stream`);
-                this.paused = false;
+                logger.log(`${table} stream is already paused`);
+            }
+            else {
+                logger.warn(`${table} pausing the stream`);
+                this.paused = true;
             }
-            void this.runNextQuery();
         }
-        else if (!this.paused) {
-            logger.warn(`${table} rssLimitMB reached ${rssMB} > ${rssLimitMB}, pausing stream`);
-            this.paused = true;
+    }
+    async runQuery(q) {
+        const { table, logger } = this;
+        try {
+            return await pRetry(async () => {
+                return await q.get();
+            }, {
+                name: `FirestoreStreamReadable.query(${table})`,
+                maxAttempts: 5,
+                delay: 5000,
+                delayMultiplier: 2,
+                logger,
+                timeout: 120_000, // 2 minutes
+            });
+        }
+        catch (err) {
+            console.log(`FirestoreStreamReadable error!\n`, {
+                table,
+                rowsRetrieved: this.rowsRetrieved,
+            }, err);
+            this.emit('error', err);
+            return;
         }
     }
 }

package/dist/query.util.d.ts

@@ -1,4 +1,4 @@
-import type { Query } from '@google-cloud/firestore';
+import { type Query } from '@google-cloud/firestore';
 import type { DBQuery } from '@naturalcycles/db-lib';
 import type { ObjectWithId } from '@naturalcycles/js-lib/types';
 export declare function dbQueryToFirestoreQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, emptyQuery: Query): Query;

package/dist/query.util.js

@@ -1,3 +1,4 @@
+import { FieldPath } from '@google-cloud/firestore';
 // Map DBQueryFilterOp to WhereFilterOp
 // Currently it's fully aligned!
 const OP_MAP = {
@@ -8,27 +9,35 @@ export function dbQueryToFirestoreQuery(dbQuery, emptyQuery) {
     let q = emptyQuery;
     // filter
     for (const f of dbQuery._filters) {
-        q = q.where(f.name, OP_MAP[f.op] || f.op, f.val);
+        q = q.where(mapName(f.name), OP_MAP[f.op] || f.op, f.val);
     }
     // order
     for (const ord of dbQuery._orders) {
-        // todo: support ordering by id like this:
-        // .orderBy(FieldPath.documentId())
-        q = q.orderBy(ord.name, ord.descending ? 'desc' : 'asc');
+        q = q.orderBy(mapName(ord.name), ord.descending ? 'desc' : 'asc');
     }
     // limit
     q = q.limit(dbQuery._limitValue);
     // selectedFields
     if (dbQuery._selectedFieldNames) {
-        // todo: check if at least id / __key__ is required to be set
-        q = q.select(...dbQuery._selectedFieldNames);
+        // id is filtered out, because in Firestore it's not a "property",
+        // and doc.id is always returned, even if we request empty set of fields
+        q = q.select(...dbQuery._selectedFieldNames.filter(n => n !== 'id'));
     }
     // cursor
     if (dbQuery._startCursor) {
-        q = q.startAt(dbQuery._startCursor);
+        // Using `startAfter`, not `startAt` here
+        // Why?
+        // Because in Firestore, you can only retrieve "last document id" to be used as Cursor.
+        // That document was already retrieved, so it makes sense to start AFTER it.
+        q = q.startAfter(dbQuery._startCursor);
     }
     if (dbQuery._endCursor) {
         q = q.endAt(dbQuery._endCursor);
     }
     return q;
 }
+function mapName(name) {
+    if (name === 'id')
+        return FieldPath.documentId();
+    return name;
+}

package/package.json

@@ -38,7 +38,7 @@
   "engines": {
     "node": ">=22.12.0"
   },
-  "version": "2.9.0",
+  "version": "2.10.0",
   "description": "Firestore implementation of CommonDB interface",
   "author": "Natural Cycles Team",
   "license": "MIT",

package/src/firestore.db.ts

@@ -28,10 +28,11 @@ import { _assert } from '@naturalcycles/js-lib/error/assert.js'
 import { type CommonLogger, commonLoggerMinLevel } from '@naturalcycles/js-lib/log'
 import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/object.util.js'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
-import type { NumberOfSeconds, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
+import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types'
 import { _stringMapEntries } from '@naturalcycles/js-lib/types'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import { escapeDocId, unescapeDocId } from './firestore.util.js'
+import { FirestoreShardedReadable } from './firestoreShardedReadable.js'
 import { FirestoreStreamReadable } from './firestoreStreamReadable.js'
 import { dbQueryToFirestoreQuery } from './query.util.js'
 
@@ -50,6 +51,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
     ...commonDBFullSupport,
     patchByQuery: false, // todo: can be implemented
     tableSchemas: false,
+    createTransaction: false, // Firestore SDK doesn't support it
   }
 
   // GET
@@ -167,6 +169,15 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
       )
     }
 
+    if (opt.experimentalShardedStream) {
+      return new FirestoreShardedReadable(
+        firestoreQuery,
+        q,
+        opt,
+        commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
+      )
+    }
+
     return (firestoreQuery.stream() as ReadableTyped<QueryDocumentSnapshot<any>>).map(doc => {
       return {
         id: unescapeDocId(doc.id),
@@ -545,38 +556,24 @@ export interface FirestoreDBStreamOptions extends FirestoreDBReadOptions {
    */
   experimentalCursorStream?: boolean
 
+  experimentalShardedStream?: boolean
+
   /**
    * Applicable to `experimentalCursorStream`.
    * Defines the size (limit) of each individual query.
    *
-   * Default: 1000
-   */
-  batchSize?: number
-
-  /**
-   * Applicable to `experimentalCursorStream`
-   *
-   * Set to a value (number of Megabytes) to control the peak RSS size.
-   * If limit is reached - streaming will pause until the stream keeps up, and then
-   * resumes.
-   *
-   * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
-   * when _read is called.
-   *
-   * @default 1000
+   * Default: 10_000
    */
-  rssLimitMB?: number
+  batchSize?: PositiveInteger
 
   /**
-   * Applicable to `experimentalCursorStream`
-   * Default false.
-   * If true, stream will pause until consumer requests more data (via _read).
-   * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
-   * There will be gaps in time between "last query loaded" and "next query requested".
-   * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
-   * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
+   * Defaults to 3x batchSize.
+   * Default batchSize is 10_000, so default highWaterMark is 30_000.
+   * Controls how many rows to have "buffered".
+   * Should be at least 1x batchSize, otherwise the stream will be "starving"
+   * between the queries.
    */
-  singleBatchBuffer?: boolean
+  highWaterMark?: PositiveInteger
 
   /**
    * Set to `true` to log additional debug info, when using experimentalCursorStream.
@@ -584,15 +581,6 @@ export interface FirestoreDBStreamOptions extends FirestoreDBReadOptions {
    * @default false
    */
   debug?: boolean
-
-  /**
-   * Default is undefined.
-   * If set - sets a "safety timer", which will force call _read after the specified number of seconds.
-   * This is to prevent possible "dead-lock"/race-condition that would make the stream "hang".
-   *
-   * @experimental
-   */
-  maxWait?: NumberOfSeconds
 }
 
 export interface FirestoreDBOptions extends CommonDBOptions {}

package/src/firestoreShardedReadable.ts

@@ -0,0 +1,233 @@
+import { Readable } from 'node:stream'
+import { FieldPath, type Query, type QuerySnapshot } from '@google-cloud/firestore'
+import type { DBQuery } from '@naturalcycles/db-lib'
+import { localTime } from '@naturalcycles/js-lib/datetime'
+import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
+import type { CommonLogger } from '@naturalcycles/js-lib/log'
+import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
+import type {
+  ObjectWithId,
+  PositiveInteger,
+  StringMap,
+  UnixTimestampMillis,
+} from '@naturalcycles/js-lib/types'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { FirestoreDBStreamOptions } from './firestore.db.js'
+import { unescapeDocId } from './firestore.util.js'
+
+const SHARDS = 16
+const SHARD_COLUMN = 'shard16'
+
+/**
+ * Highly, HIGHLY experimental!
+ */
+export class FirestoreShardedReadable<T extends ObjectWithId = any>
+  extends Readable
+  implements ReadableTyped<T>
+{
+  private readonly table: string
+  private readonly originalLimit: number
+  private rowsRetrieved = 0
+  /**
+   * Next shard to be used for querying.
+   */
+  private nextShard = 1
+  private cursorByShard: StringMap = {}
+  private queryIsRunningByShard: StringMap<boolean> = {}
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private paused = false
+  private done = false
+  private doneShards = new Set<PositiveInteger>()
+  private lastQueryDoneByShard: StringMap<UnixTimestampMillis> = {}
+  private totalWait = 0
+
+  private readonly opt: FirestoreDBStreamOptions & { batchSize: number }
+
+  constructor(
+    private readonly q: Query,
+    readonly dbQuery: DBQuery<T>,
+    opt: FirestoreDBStreamOptions,
+    private logger: CommonLogger,
+  ) {
+    super({ objectMode: true })
+
+    this.opt = {
+      batchSize: 3000,
+      ...opt,
+    }
+
+    this.originalLimit = dbQuery._limitValue
+    this.table = dbQuery.table
+
+    logger.warn(
+      `!! using experimentalShardedStream !! ${this.table}, batchSize: ${this.opt.batchSize}`,
+    )
+  }
+
+  /**
+   * Counts how many times _read was called.
+   * For debugging.
+   */
+  count = 0
+
+  override _read(): void {
+    // this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+
+    // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+    this.count++
+
+    if (this.done) {
+      this.logger.warn(`!!! _read was called, but done==true`)
+      return
+    }
+
+    // const shard = this.getNextShardAndMove()
+    const shard = this.findNextFreeShard()
+    if (!shard) {
+      this.logger.log(`_read ${this.count}: all shards are busy, skipping`)
+      return
+    }
+    void this.runNextQuery(shard).catch(err => {
+      console.log('error in runNextQuery', err)
+      this.emit('error', err)
+    })
+  }
+
+  private async runNextQuery(shard: PositiveInteger): Promise<void> {
+    if (this.done) return
+    const { logger, table } = this
+
+    if (this.lastQueryDoneByShard[shard]) {
+      this.totalWait += Date.now() - this.lastQueryDoneByShard[shard]
+    }
+
+    this.queryIsRunningByShard[shard] = true
+
+    const limit = this.opt.batchSize
+
+    // We have to orderBy documentId, to be able to use id as a cursor
+
+    let q = this.q.where(SHARD_COLUMN, '==', shard).orderBy(FieldPath.documentId()).limit(limit)
+    if (this.cursorByShard[shard]) {
+      q = q.startAfter(this.cursorByShard[shard])
+    }
+
+    console.log(`runNextQuery[${shard}]`, {
+      retrieved: this.rowsRetrieved,
+    })
+    const qs = await this.runQuery(q)
+    if (!qs) {
+      // this means we have already emitted an unrecoverable error
+      return
+    }
+
+    const rows: T[] = []
+    let lastDocId: string | undefined
+
+    for (const doc of qs.docs) {
+      lastDocId = doc.id
+      rows.push({
+        id: unescapeDocId(doc.id),
+        ...doc.data(),
+      } as T)
+    }
+
+    this.rowsRetrieved += rows.length
+    logger.log(
+      `${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
+        this.totalWait,
+      )}`,
+    )
+
+    this.cursorByShard[shard] = lastDocId
+    this.queryIsRunningByShard[shard] = false // ready to take more _reads
+    this.lastQueryDoneByShard[shard] = localTime.nowUnixMillis()
+
+    for (const row of rows) {
+      this.push(row)
+    }
+
+    if (qs.empty) {
+      logger.warn(
+        `!!!! Shard ${shard} DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
+      )
+      this.doneShards.add(shard)
+    }
+
+    if (this.doneShards.size === SHARDS) {
+      logger.warn(
+        `!!!! DONE: all shards completed, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
+      )
+      this.push(null)
+      this.paused = false
+      this.done = true
+      return
+    }
+
+    if (this.originalLimit && this.rowsRetrieved >= this.originalLimit) {
+      logger.warn(
+        `!!!! DONE: reached total limit of ${this.originalLimit}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
+      )
+      this.push(null)
+      this.paused = false
+      this.done = true
+      return
+    }
+
+    // if (this.paused) {
+    //   this.paused = false
+    // }
+    const nextShard = this.findNextFreeShard()
+    if (nextShard) {
+      void this.runNextQuery(nextShard)
+    } else {
+      logger.warn(`${table} all shards are busy in runNextQuery, skipping`)
+    }
+  }
+
+  private async runQuery(q: Query): Promise<QuerySnapshot | undefined> {
+    const { table, logger } = this
+
+    try {
+      return await pRetry(
+        async () => {
+          return await q.get()
+        },
+        {
+          name: `FirestoreStreamReadable.query(${table})`,
+          maxAttempts: 5,
+          delay: 5000,
+          delayMultiplier: 2,
+          logger,
+          timeout: 120_000, // 2 minutes
+        },
+      )
+    } catch (err) {
+      console.log(
+        `FirestoreStreamReadable error!\n`,
+        {
+          table,
+          rowsRetrieved: this.rowsRetrieved,
+        },
+        err,
+      )
+      this.emit('error', err)
+      return
+    }
+  }
+
+  private findNextFreeShard(): PositiveInteger | undefined {
+    for (let shard = 1; shard <= SHARDS; shard++) {
+      if (!this.queryIsRunningByShard[shard] && !this.doneShards.has(shard)) {
+        return shard
+      }
+    }
+  }
+
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private _getNextShardAndMove(): PositiveInteger {
+    const shard = this.nextShard
+    this.nextShard = shard === SHARDS ? 1 : shard + 1
+    return shard
+  }
+}

package/src/firestoreStreamReadable.ts

@@ -1,7 +1,8 @@
 import { Readable } from 'node:stream'
 import { FieldPath, type Query, type QuerySnapshot } from '@google-cloud/firestore'
 import type { DBQuery } from '@naturalcycles/db-lib'
-import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
+import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
+import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
 import type { ObjectWithId } from '@naturalcycles/js-lib/types'
@@ -20,11 +21,13 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
   private queryIsRunning = false
   private paused = false
   private done = false
-  private lastQueryDone?: number
-  private totalWait = 0
+  /**
+   * Counts how many times _read was called.
+   * For debugging.
+   */
+  countReads = 0
 
-  private readonly opt: FirestoreDBStreamOptions & { batchSize: number; rssLimitMB: number }
-  // private readonly dsOpt: RunQueryOptions
+  private readonly opt: FirestoreDBStreamOptions & { batchSize: number; highWaterMark: number }
 
   constructor(
     private q: Query,
@@ -32,65 +35,63 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
     opt: FirestoreDBStreamOptions,
     private logger: CommonLogger,
   ) {
-    super({ objectMode: true })
+    // 10_000 was optimal in benchmarks
+    const { batchSize = 10_000 } = opt
+    const { highWaterMark = batchSize * 3 } = opt
+    // Defaulting highWaterMark to 3x batchSize
+    super({ objectMode: true, highWaterMark })
 
     this.opt = {
-      rssLimitMB: 1000,
-      batchSize: 1000,
       ...opt,
+      batchSize,
+      highWaterMark,
     }
     // todo: support PITR!
-    // this.dsOpt = {}
-    // if (opt.readAt) {
-    //   // Datastore expects UnixTimestamp in milliseconds
-    //   this.dsOpt.readTime = opt.readAt * 1000
-    // }
 
     this.originalLimit = dbQuery._limitValue
     this.table = dbQuery.table
 
-    logger.warn(
-      `!! using experimentalCursorStream !! ${this.table}, batchSize: ${this.opt.batchSize}`,
-    )
+    logger.warn(`!!! using experimentalCursorStream`, {
+      table: this.table,
+      batchSize,
+      highWaterMark,
+    })
   }
 
-  /**
-   * Counts how many times _read was called.
-   * For debugging.
-   */
-  count = 0
-
   override _read(): void {
     // this.lastReadTimestamp = Date.now() as UnixTimestampMillis
 
     // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
-    this.count++
+    this.countReads++
 
     if (this.done) {
       this.logger.warn(`!!! _read was called, but done==true`)
       return
     }
 
-    if (!this.queryIsRunning) {
-      void this.runNextQuery().catch(err => {
-        console.log('error in runNextQuery', err)
-        this.emit('error', err)
-      })
-    } else {
-      this.logger.log(`_read ${this.count}, queryIsRunning: true`)
+    if (this.paused) {
+      this.logger.log(
+        `_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`,
+      )
+      this.paused = false
+    }
+
+    if (this.queryIsRunning) {
+      this.logger.log(`_read #${this.countReads}, queryIsRunning: true, doing nothing`)
       // todo: check if this can cause a "hang", if no more _reads would come later and we get stuck?
+      return
     }
+
+    void this.runNextQuery().catch(err => {
+      console.log('error in runNextQuery', err)
+      this.emit('error', err)
+    })
   }
 
   private async runNextQuery(): Promise<void> {
     if (this.done) return
     const { logger, table } = this
 
-    if (this.lastQueryDone) {
-      const now = Date.now()
-      this.totalWait += now - this.lastQueryDone
-    }
-
     this.queryIsRunning = true
 
     let limit = this.opt.batchSize
@@ -106,40 +107,23 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
       q = q.startAfter(this.endCursor)
     }
 
-    let qs: QuerySnapshot
+    // logger.log(`runNextQuery`, {
+    //   rowsRetrieved: this.rowsRetrieved,
+    //   paused: this.paused,
+    // })
 
-    try {
-      await pRetry(
-        async () => {
-          qs = await q.get()
-        },
-        {
-          name: `FirestoreStreamReadable.query(${table})`,
-          maxAttempts: 5,
-          delay: 5000,
-          delayMultiplier: 2,
-          logger,
-          timeout: 120_000, // 2 minutes
-        },
-      )
-    } catch (err) {
-      console.log(
-        `FirestoreStreamReadable error!\n`,
-        {
-          table,
-          rowsRetrieved: this.rowsRetrieved,
-        },
-        err,
-      )
-      this.emit('error', err)
-      // clearInterval(this.maxWaitInterval)
+    const started = localTime.nowUnixMillis()
+    const qs = await this.runQuery(q)
+    logger.log(`${table} query took ${_since(started)}`)
+    if (!qs) {
+      // error already emitted in runQuery
       return
     }
 
     const rows: T[] = []
     let lastDocId: string | undefined
 
-    for (const doc of qs!.docs) {
+    for (const doc of qs.docs) {
       lastDocId = doc.id
       rows.push({
         id: unescapeDocId(doc.id),
@@ -148,48 +132,67 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
     }
 
     this.rowsRetrieved += rows.length
-    logger.log(
-      `${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
-        this.totalWait,
-      )}`,
-    )
+    logger.log(`${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved`)
 
     this.endCursor = lastDocId
     this.queryIsRunning = false // ready to take more _reads
-    this.lastQueryDone = Date.now()
+    let shouldContinue = false
 
     for (const row of rows) {
-      this.push(row)
+      shouldContinue = this.push(row)
     }
 
-    if (qs!.empty || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-      logger.warn(
-        `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
-      )
+    if (!rows.length || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+      logger.warn(`${table} DONE! ${this.rowsRetrieved} rowsRetrieved`)
       this.push(null)
-      this.paused = false
       this.done = true
+      this.paused = false
       return
     }
 
-    if (this.opt.singleBatchBuffer) {
-      // here we don't start next query until we're asked (via next _read call)
-      // so, let's do nothing
-      return
+    if (shouldContinue) {
+      // Keep the stream flowing
+      logger.log(`${table} continuing the stream`)
+      void this.runNextQuery()
+    } else {
+      // Not starting the next query
+      if (this.paused) {
+        logger.log(`${table} stream is already paused`)
+      } else {
+        logger.warn(`${table} pausing the stream`)
+        this.paused = true
+      }
     }
+  }
 
-    const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
-    const { rssLimitMB } = this.opt
+  private async runQuery(q: Query): Promise<QuerySnapshot | undefined> {
+    const { table, logger } = this
 
-    if (rssMB <= rssLimitMB) {
-      if (this.paused) {
-        logger.warn(`${table} rssLimitMB is below ${rssMB} < ${rssLimitMB}, unpausing stream`)
-        this.paused = false
-      }
-      void this.runNextQuery()
-    } else if (!this.paused) {
-      logger.warn(`${table} rssLimitMB reached ${rssMB} > ${rssLimitMB}, pausing stream`)
-      this.paused = true
+    try {
+      return await pRetry(
+        async () => {
+          return await q.get()
+        },
+        {
+          name: `FirestoreStreamReadable.query(${table})`,
+          maxAttempts: 5,
+          delay: 5000,
+          delayMultiplier: 2,
+          logger,
+          timeout: 120_000, // 2 minutes
+        },
+      )
+    } catch (err) {
+      console.log(
+        `FirestoreStreamReadable error!\n`,
+        {
+          table,
+          rowsRetrieved: this.rowsRetrieved,
+        },
+        err,
+      )
+      this.emit('error', err)
+      return
     }
   }
 }

package/src/query.util.ts

@@ -1,4 +1,4 @@
-import type { Query, WhereFilterOp } from '@google-cloud/firestore'
+import { FieldPath, type Query, type WhereFilterOp } from '@google-cloud/firestore'
 import type { DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
 import type { ObjectWithId } from '@naturalcycles/js-lib/types'
 
@@ -17,14 +17,12 @@ export function dbQueryToFirestoreQuery<ROW extends ObjectWithId>(
 
   // filter
   for (const f of dbQuery._filters) {
-    q = q.where(f.name as string, OP_MAP[f.op] || (f.op as WhereFilterOp), f.val)
+    q = q.where(mapName(f.name), OP_MAP[f.op] || (f.op as WhereFilterOp), f.val)
   }
 
   // order
   for (const ord of dbQuery._orders) {
-    // todo: support ordering by id like this:
-    // .orderBy(FieldPath.documentId())
-    q = q.orderBy(ord.name as string, ord.descending ? 'desc' : 'asc')
+    q = q.orderBy(mapName(ord.name), ord.descending ? 'desc' : 'asc')
   }
 
   // limit
@@ -32,13 +30,18 @@ export function dbQueryToFirestoreQuery<ROW extends ObjectWithId>(
 
   // selectedFields
   if (dbQuery._selectedFieldNames) {
-    // todo: check if at least id / __key__ is required to be set
-    q = q.select(...(dbQuery._selectedFieldNames as string[]))
+    // id is filtered out, because in Firestore it's not a "property",
+    // and doc.id is always returned, even if we request empty set of fields
+    q = q.select(...(dbQuery._selectedFieldNames as string[]).filter(n => n !== 'id'))
   }
 
   // cursor
   if (dbQuery._startCursor) {
-    q = q.startAt(dbQuery._startCursor)
+    // Using `startAfter`, not `startAt` here
+    // Why?
+    // Because in Firestore, you can only retrieve "last document id" to be used as Cursor.
+    // That document was already retrieved, so it makes sense to start AFTER it.
+    q = q.startAfter(dbQuery._startCursor)
   }
 
   if (dbQuery._endCursor) {
@@ -47,3 +50,8 @@ export function dbQueryToFirestoreQuery<ROW extends ObjectWithId>(
 
   return q
 }
+
+function mapName<ROW extends ObjectWithId>(name: keyof ROW): string | FieldPath {
+  if (name === 'id') return FieldPath.documentId()
+  return name as string
+}