@naturalcycles/db-lib 8.54.5 → 8.55.0

package/dist/adapter/file/localFile.persistence.plugin.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LocalFilePersistencePlugin = void 0;
-const fs = require("node:fs");
+const tslib_1 = require("tslib");
+const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
 const node_stream_1 = require("node:stream");
-const fsp = require("node:fs/promises");
+const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
@@ -20,7 +21,7 @@ class LocalFilePersistencePlugin {
     }
     async ping() { }
     async getTables() {
-        return (await fsp.readdir(this.cfg.storagePath))
+        return (await promises_1.default.readdir(this.cfg.storagePath))
             .filter(f => f.includes('.ndjson'))
             .map(f => f.split('.ndjson')[0]);
     }
@@ -33,7 +34,7 @@ class LocalFilePersistencePlugin {
         const transformUnzip = this.cfg.gzip ? [(0, node_zlib_1.createUnzip)()] : [];
         const rows = [];
         await (0, nodejs_lib_1._pipeline)([
-            fs.createReadStream(filePath),
+            node_fs_1.default.createReadStream(filePath),
             ...transformUnzip,
             (0, nodejs_lib_1.transformSplit)(),
             (0, nodejs_lib_1.transformJsonParse)(),
@@ -53,7 +54,7 @@ class LocalFilePersistencePlugin {
             node_stream_1.Readable.from(rows),
             (0, nodejs_lib_1.transformToNDJson)(),
             ...transformZip,
-            fs.createWriteStream(filePath),
+            node_fs_1.default.createWriteStream(filePath),
         ]);
     }
 }

package/dist/adapter/inmemory/inMemory.db.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryDB = void 0;
-const fs = require("node:fs");
-const fsp = require("node:fs/promises");
+const tslib_1 = require("tslib");
+const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
+const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
 const node_stream_1 = require("node:stream");
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
@@ -181,7 +182,7 @@ class InMemoryDB {
                 node_stream_1.Readable.from(rows),
                 (0, nodejs_lib_1.transformToNDJson)(),
                 ...transformZip,
-                fs.createWriteStream(fname),
+                node_fs_1.default.createWriteStream(fname),
             ]);
         });
         this.cfg.logger.log(`flushToDisk took ${(0, nodejs_lib_2.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, nodejs_lib_2.yellow)(tables)} tables`);
@@ -195,7 +196,7 @@ class InMemoryDB {
         const started = Date.now();
         await (0, nodejs_lib_1._ensureDir)(persistentStoragePath);
         this.data = {}; // empty it in the beginning!
-        const files = (await fsp.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'));
+        const files = (await promises_1.default.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'));
         // infinite concurrency for now
         await (0, js_lib_1.pMap)(files, async (file) => {
             const fname = `${persistentStoragePath}/${file}`;
@@ -203,7 +204,7 @@ class InMemoryDB {
             const transformUnzip = file.endsWith('.gz') ? [(0, node_zlib_1.createUnzip)()] : [];
             const rows = [];
             await (0, nodejs_lib_1._pipeline)([
-                fs.createReadStream(fname),
+                node_fs_1.default.createReadStream(fname),
                 ...transformUnzip,
                 (0, nodejs_lib_1.transformSplit)(),
                 (0, nodejs_lib_1.transformJsonParse)(),

package/dist/commondao/common.dao.d.ts

@@ -1,8 +1,10 @@
+/// <reference types="node" />
+import { Transform } from 'node:stream';
 import { AnyObject, AsyncMapper, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId, Promisable, Saved, UnixTimestampMillisNumber, Unsaved, ZodSchema } from '@naturalcycles/js-lib';
 import { AjvSchema, ObjectSchema, ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DBDeleteByIdsOperation, DBModelType, DBOperation, DBPatch, DBSaveBatchOperation, RunQueryResult } from '../db.model';
 import { DBQuery, RunnableDBQuery } from '../query/dbQuery';
-import { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions } from './common.dao.model';
+import { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions, CommonDaoStreamSaveOptions } from './common.dao.model';
 /**
  * Lowest common denominator API between supported Databases.
  *
@@ -62,7 +64,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
     /**
      * Stream as Readable, to be able to .pipe() it further with support of backpressure.
      */
-    streamQueryAsDBM(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions): ReadableTyped<DBM>;
+    streamQueryAsDBM(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM>;
     /**
      * Stream as Readable, to be able to .pipe() it further with support of backpressure.
      *
@@ -72,9 +74,9 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
      *
      * You can do `.pipe(transformNoOp)` to make it "valid again".
      */
-    streamQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions): ReadableTyped<Saved<BM>>;
+    streamQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<Saved<BM>>): ReadableTyped<Saved<BM>>;
     queryIds(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<ID[]>;
-    streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions): ReadableTyped<ID>;
+    streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID>;
     streamQueryIdsForEach(q: DBQuery<DBM>, mapper: AsyncMapper<ID, void>, opt?: CommonDaoStreamForEachOptions<ID>): Promise<void>;
     /**
      * Mutates!
@@ -106,6 +108,13 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
     saveAsDBM(dbm: DBM, opt?: CommonDaoSaveOptions<DBM>): Promise<DBM>;
     saveBatch(bms: Unsaved<BM>[], opt?: CommonDaoSaveOptions<DBM>): Promise<Saved<BM>[]>;
     saveBatchAsDBM(dbms: DBM[], opt?: CommonDaoSaveOptions<DBM>): Promise<DBM[]>;
+    /**
+     * "Streaming" is implemented by buffering incoming rows into **batches**
+     * (of size opt.batchSize, which defaults to 500),
+     * and then executing db.saveBatch(batch) with the concurrency
+     * of opt.batchConcurrency (which defaults to 16).
+     */
+    streamSaveTransform(opt?: CommonDaoStreamSaveOptions<DBM>): Transform[];
     /**
      * @returns number of deleted items
      */
@@ -117,9 +126,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
      * `deleteByIds` for each batch concurrently (infinite concurrency).
      * This is expected to be more memory-efficient way of deleting big numbers of rows.
      */
-    deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamForEachOptions<DBM> & {
-        stream?: boolean;
-    }): Promise<number>;
+    deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
     updateById(id: ID, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
     updateByIds(ids: ID[], patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
     updateByQuery(q: DBQuery<DBM>, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;

package/dist/commondao/common.dao.js

@@ -718,6 +718,58 @@ class CommonDao {
         this.logSaveResult(started, op, table);
         return rows;
     }
+    /**
+     * "Streaming" is implemented by buffering incoming rows into **batches**
+     * (of size opt.batchSize, which defaults to 500),
+     * and then executing db.saveBatch(batch) with the concurrency
+     * of opt.batchConcurrency (which defaults to 16).
+     */
+    streamSaveTransform(opt = {}) {
+        this.requireWriteAccess();
+        const table = opt.table || this.cfg.table;
+        opt.skipValidation ??= true;
+        opt.skipConversion ??= true;
+        opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
+        if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
+            opt = { ...opt, saveMethod: 'insert' };
+        }
+        const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes;
+        const { beforeSave } = this.cfg.hooks;
+        const { batchSize = 500, batchConcurrency = 16, errorMode } = opt;
+        return [
+            (0, nodejs_lib_1.transformMap)(async (bm) => {
+                this.assignIdCreatedUpdated(bm, opt); // mutates
+                let dbm = await this.bmToDBM(bm, opt);
+                if (beforeSave) {
+                    dbm = (await beforeSave(dbm));
+                    if (dbm === null && !opt.tx)
+                        return js_lib_1.SKIP;
+                }
+                return dbm;
+            }, {
+                errorMode,
+            }),
+            (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+            (0, nodejs_lib_1.transformMap)(async (batch) => {
+                await this.cfg.db.saveBatch(table, batch, {
+                    ...opt,
+                    excludeFromIndexes,
+                });
+                return batch;
+            }, {
+                concurrency: batchConcurrency,
+                errorMode,
+                flattenArrayOutput: true,
+            }),
+            (0, nodejs_lib_1.transformLogProgress)({
+                metric: 'saved',
+                ...opt,
+            }),
+            // just to satisfy and simplify typings
+            // It's easier to return Transform[], rather than (Transform | Writable)[]
+            (0, nodejs_lib_1.writableVoid)(),
+        ];
+    }
     async deleteById(id, opt = {}) {
         if (!id)
             return 0;
@@ -754,8 +806,8 @@ class CommonDao {
         const op = `deleteByQuery(${q.pretty()})`;
         const started = this.logStarted(op, q.table);
         let deleted = 0;
-        if (opt.stream) {
-            const batchSize = 500;
+        if (opt.batchSize) {
+            const { batchSize, batchConcurrency = 16 } = opt;
             await (0, nodejs_lib_1._pipeline)([
                 this.cfg.db.streamQuery(q.select(['id']), opt),
                 (0, nodejs_lib_1.transformMapSimple)(objectWithId => objectWithId.id, {
@@ -766,6 +818,7 @@ class CommonDao {
                     deleted += await this.cfg.db.deleteByQuery(dbQuery_1.DBQuery.create(q.table).filterIn('id', ids), opt);
                 }, {
                     predicate: js_lib_1._passthroughPredicate,
+                    concurrency: batchConcurrency,
                 }),
                 // LogProgress should be AFTER the mapper, to be able to report correct stats
                 (0, nodejs_lib_1.transformLogProgress)({

package/dist/commondao/common.dao.model.d.ts

@@ -252,9 +252,13 @@ export interface CommonDaoSaveOptions<DBM extends ObjectWithId> extends CommonDa
      */
     ensureUniqueId?: boolean;
 }
-export interface CommonDaoStreamForEachOptions<IN> extends CommonDaoStreamOptions, TransformMapOptions<IN, any>, TransformLogProgressOptions<IN> {
+export interface CommonDaoStreamDeleteOptions<DBM extends ObjectWithId> extends CommonDaoStreamOptions<DBM> {
 }
-export interface CommonDaoStreamOptions extends CommonDaoOptions {
+export interface CommonDaoStreamSaveOptions<DBM extends ObjectWithId> extends CommonDaoSaveOptions<DBM>, CommonDaoStreamOptions<DBM> {
+}
+export interface CommonDaoStreamForEachOptions<IN> extends CommonDaoStreamOptions<IN>, TransformMapOptions<IN, any> {
+}
+export interface CommonDaoStreamOptions<IN> extends CommonDaoOptions, TransformLogProgressOptions<IN> {
     /**
      * @default true (for streams)
      */
@@ -268,5 +272,17 @@ export interface CommonDaoStreamOptions extends CommonDaoOptions {
      * @default ErrorMode.SUPPRESS for .forEach() streams as well, but overridable
      */
     errorMode?: ErrorMode;
+    /**
+     * Applicable to some of stream operations, e.g deleteByQuery.
+     * If set - `deleteByQuery` won't execute it "all at once", but in batches.
+     *
+     * Defaults to undefined, so the operation is executed "all at once".
+     */
+    batchSize?: number;
+    /**
+     * When batchSize is set - this option controls how many batches to run concurrently.
+     * Defaults to 16, "the magic number of JavaScript concurrency".
+     */
+    batchConcurrency?: number;
 }
 export type CommonDaoCreateOptions = CommonDBCreateOptions;

package/dist/pipeline/dbPipelineBackup.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbPipelineBackup = void 0;
-const fs = require("node:fs");
-const fsp = require("node:fs/promises");
+const tslib_1 = require("tslib");
+const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
+const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
@@ -65,9 +66,9 @@ async function dbPipelineBackup(opt) {
             }),
             (0, nodejs_lib_1.transformToNDJson)({ strict, sortObjects }),
             ...(gzip ? [(0, node_zlib_1.createGzip)(zlibOptions)] : []),
-            fs.createWriteStream(filePath),
+            node_fs_1.default.createWriteStream(filePath),
         ]);
-        const { size: sizeBytes } = await fsp.stat(filePath);
+        const { size: sizeBytes } = await promises_1.default.stat(filePath);
         const stats = nodejs_lib_1.NDJsonStats.create({
             tookMillis: Date.now() - started,
             rows,

package/dist/pipeline/dbPipelineRestore.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbPipelineRestore = void 0;
-const fs = require("node:fs");
+const tslib_1 = require("tslib");
+const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
 const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
@@ -24,7 +25,7 @@ async function dbPipelineRestore(opt) {
     const sizeByTable = {};
     const statsPerTable = {};
     const tables = [];
-    fs.readdirSync(inputDirPath).forEach(f => {
+    node_fs_1.default.readdirSync(inputDirPath).forEach(f => {
         let table;
         let gzip = false;
         if (f.endsWith('.ndjson')) {
@@ -42,7 +43,7 @@ async function dbPipelineRestore(opt) {
         tables.push(table);
         if (gzip)
             tablesToGzip.add(table);
-        sizeByTable[table] = fs.statSync(`${inputDirPath}/${f}`).size;
+        sizeByTable[table] = node_fs_1.default.statSync(`${inputDirPath}/${f}`).size;
     });
     const sizeStrByTable = (0, js_lib_1._mapValues)(sizeByTable, (_k, b) => (0, js_lib_1._hb)(b));
     console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n`, sizeStrByTable);
@@ -50,7 +51,7 @@ async function dbPipelineRestore(opt) {
     if (recreateTables) {
         await (0, js_lib_1.pMap)(tables, async (table) => {
             const schemaFilePath = `${inputDirPath}/${table}.schema.json`;
-            if (!fs.existsSync(schemaFilePath)) {
+            if (!node_fs_1.default.existsSync(schemaFilePath)) {
                 console.warn(`${schemaFilePath} does not exist!`);
                 return;
             }
@@ -67,7 +68,7 @@ async function dbPipelineRestore(opt) {
         const sizeBytes = sizeByTable[table];
         console.log(`<< ${(0, nodejs_lib_2.grey)(filePath)} ${(0, nodejs_lib_2.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
         await (0, nodejs_lib_1._pipeline)([
-            fs.createReadStream(filePath),
+            node_fs_1.default.createReadStream(filePath),
             ...(gzip ? [(0, node_zlib_1.createUnzip)()] : []),
             (0, nodejs_lib_1.transformSplit)(),
             (0, nodejs_lib_1.transformJsonParse)({ strict }),

package/dist/query/dbQuery.d.ts

@@ -1,6 +1,6 @@
 import { AnyObjectWithId, ObjectWithId, AsyncMapper, Saved, AnyObject } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { CommonDaoOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions, DBPatch } from '..';
+import { CommonDaoOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions, DBPatch } from '..';
 import { CommonDao } from '../commondao/common.dao';
 import { RunQueryResult } from '../db.model';
 /**
@@ -101,12 +101,10 @@ export declare class RunnableDBQuery<BM extends Partial<ObjectWithId<ID>>, DBM e
     updateByQuery(patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
     streamQueryForEach(mapper: AsyncMapper<Saved<BM>, void>, opt?: CommonDaoStreamForEachOptions<Saved<BM>>): Promise<void>;
     streamQueryAsDBMForEach(mapper: AsyncMapper<DBM, void>, opt?: CommonDaoStreamForEachOptions<DBM>): Promise<void>;
-    streamQuery(opt?: CommonDaoStreamOptions): ReadableTyped<Saved<BM>>;
-    streamQueryAsDBM(opt?: CommonDaoStreamOptions): ReadableTyped<DBM>;
+    streamQuery(opt?: CommonDaoStreamOptions<Saved<BM>>): ReadableTyped<Saved<BM>>;
+    streamQueryAsDBM(opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM>;
     queryIds(opt?: CommonDaoOptions): Promise<ID[]>;
-    streamQueryIds(opt?: CommonDaoStreamOptions): ReadableTyped<ID>;
+    streamQueryIds(opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID>;
     streamQueryIdsForEach(mapper: AsyncMapper<ID, void>, opt?: CommonDaoStreamForEachOptions<ID>): Promise<void>;
-    deleteByQuery(opt?: CommonDaoStreamForEachOptions<DBM> & {
-        stream?: boolean;
-    }): Promise<number>;
+    deleteByQuery(opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
 }

package/dist/testing/daoTest.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runCommonDaoTest = void 0;
+const node_stream_1 = require("node:stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const __1 = require("..");
@@ -175,6 +176,15 @@ function runCommonDaoTest(db, features = {}, quirks = {}) {
             ids = ids.sort();
             (0, dbTest_1.expectMatch)(expectedItems.map(i => i.id), ids, quirks);
         });
+        test('streamSaveTransform', async () => {
+            const items2 = (0, test_model_1.createTestItemsBM)(2).map(i => ({ ...i, id: i.id + '_str' }));
+            const ids = items2.map(i => i.id);
+            await (0, nodejs_lib_1._pipeline)([node_stream_1.Readable.from(items2), ...dao.streamSaveTransform()]);
+            const items2Loaded = await dao.getByIds(ids);
+            (0, dbTest_1.expectMatch)(items2, items2Loaded, quirks);
+            // cleanup
+            await dao.query().filterIn('id', ids).deleteByQuery();
+        });
     }
     // DELETE BY
     if (querying) {

package/package.json

@@ -5,7 +5,7 @@
   },
   "dependencies": {
     "@naturalcycles/js-lib": "^14.116.0",
-    "@naturalcycles/nodejs-lib": "^12.0.0"
+    "@naturalcycles/nodejs-lib": "^13.1.1"
   },
   "devDependencies": {
     "@naturalcycles/bench-lib": "^1.0.0",
@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "8.54.5",
+  "version": "8.55.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/file/localFile.persistence.plugin.ts

@@ -1,6 +1,6 @@
-import * as fs from 'node:fs'
+import fs from 'node:fs'
 import { Readable } from 'node:stream'
-import * as fsp from 'node:fs/promises'
+import fsp from 'node:fs/promises'
 import { createGzip, createUnzip } from 'node:zlib'
 import { pMap, ObjectWithId } from '@naturalcycles/js-lib'
 import {

package/src/adapter/inmemory/inMemory.db.ts

@@ -1,5 +1,5 @@
-import * as fs from 'node:fs'
-import * as fsp from 'node:fs/promises'
+import fs from 'node:fs'
+import fsp from 'node:fs/promises'
 import { Readable } from 'node:stream'
 import { createGzip, createUnzip } from 'node:zlib'
 import {

package/src/commondao/common.dao.model.ts

@@ -316,12 +316,20 @@ export interface CommonDaoSaveOptions<DBM extends ObjectWithId>
   ensureUniqueId?: boolean
 }
 
+export interface CommonDaoStreamDeleteOptions<DBM extends ObjectWithId>
+  extends CommonDaoStreamOptions<DBM> {}
+
+export interface CommonDaoStreamSaveOptions<DBM extends ObjectWithId>
+  extends CommonDaoSaveOptions<DBM>,
+    CommonDaoStreamOptions<DBM> {}
+
 export interface CommonDaoStreamForEachOptions<IN>
-  extends CommonDaoStreamOptions,
-    TransformMapOptions<IN, any>,
-    TransformLogProgressOptions<IN> {}
+  extends CommonDaoStreamOptions<IN>,
+    TransformMapOptions<IN, any> {}
 
-export interface CommonDaoStreamOptions extends CommonDaoOptions {
+export interface CommonDaoStreamOptions<IN>
+  extends CommonDaoOptions,
+    TransformLogProgressOptions<IN> {
   /**
    * @default true (for streams)
    */
@@ -337,6 +345,20 @@ export interface CommonDaoStreamOptions extends CommonDaoOptions {
    * @default ErrorMode.SUPPRESS for .forEach() streams as well, but overridable
    */
   errorMode?: ErrorMode
+
+  /**
+   * Applicable to some of stream operations, e.g deleteByQuery.
+   * If set - `deleteByQuery` won't execute it "all at once", but in batches.
+   *
+   * Defaults to undefined, so the operation is executed "all at once".
+   */
+  batchSize?: number
+
+  /**
+   * When batchSize is set - this option controls how many batches to run concurrently.
+   * Defaults to 16, "the magic number of JavaScript concurrency".
+   */
+  batchConcurrency?: number
 }
 
 export type CommonDaoCreateOptions = CommonDBCreateOptions

package/src/commondao/common.dao.ts

@@ -1,3 +1,4 @@
+import { Transform } from 'node:stream'
 import {
   _assert,
   _filterNullishValues,
@@ -57,8 +58,10 @@ import {
   CommonDaoLogLevel,
   CommonDaoOptions,
   CommonDaoSaveOptions,
+  CommonDaoStreamDeleteOptions,
   CommonDaoStreamForEachOptions,
   CommonDaoStreamOptions,
+  CommonDaoStreamSaveOptions,
 } from './common.dao.model'
 
 const isGAE = !!process.env['GAE_INSTANCE']
@@ -529,7 +532,7 @@ export class CommonDao<
   /**
    * Stream as Readable, to be able to .pipe() it further with support of backpressure.
    */
-  streamQueryAsDBM(q: DBQuery<DBM>, opt: CommonDaoStreamOptions = {}): ReadableTyped<DBM> {
+  streamQueryAsDBM(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<DBM> = {}): ReadableTyped<DBM> {
     q.table = opt.table || q.table
     opt.skipValidation = opt.skipValidation !== false // default true
     opt.skipConversion = opt.skipConversion !== false // default true
@@ -568,7 +571,10 @@ export class CommonDao<
    *
    * You can do `.pipe(transformNoOp)` to make it "valid again".
    */
-  streamQuery(q: DBQuery<DBM>, opt: CommonDaoStreamOptions = {}): ReadableTyped<Saved<BM>> {
+  streamQuery(
+    q: DBQuery<DBM>,
+    opt: CommonDaoStreamOptions<Saved<BM>> = {},
+  ): ReadableTyped<Saved<BM>> {
     q.table = opt.table || q.table
     opt.skipValidation = opt.skipValidation !== false // default true
     opt.skipConversion = opt.skipConversion !== false // default true
@@ -611,7 +617,7 @@ export class CommonDao<
     return rows.map(r => r.id)
   }
 
-  streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions = {}): ReadableTyped<ID> {
+  streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<ID> = {}): ReadableTyped<ID> {
     q.table = opt.table || q.table
     opt.errorMode ||= ErrorMode.SUPPRESS
 
@@ -958,6 +964,72 @@ export class CommonDao<
     return rows
   }
 
+  /**
+   * "Streaming" is implemented by buffering incoming rows into **batches**
+   * (of size opt.batchSize, which defaults to 500),
+   * and then executing db.saveBatch(batch) with the concurrency
+   * of opt.batchConcurrency (which defaults to 16).
+   */
+  streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
+    this.requireWriteAccess()
+
+    const table = opt.table || this.cfg.table
+    opt.skipValidation ??= true
+    opt.skipConversion ??= true
+    opt.errorMode ||= ErrorMode.SUPPRESS
+
+    if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
+      opt = { ...opt, saveMethod: 'insert' }
+    }
+
+    const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
+    const { beforeSave } = this.cfg.hooks!
+
+    const { batchSize = 500, batchConcurrency = 16, errorMode } = opt
+
+    return [
+      transformMap<BM, DBM>(
+        async bm => {
+          this.assignIdCreatedUpdated(bm, opt) // mutates
+
+          let dbm = await this.bmToDBM(bm, opt)
+
+          if (beforeSave) {
+            dbm = (await beforeSave(dbm))!
+            if (dbm === null && !opt.tx) return SKIP
+          }
+
+          return dbm
+        },
+        {
+          errorMode,
+        },
+      ),
+      transformBuffer<DBM>({ batchSize }),
+      transformMap<DBM[], DBM[]>(
+        async batch => {
+          await this.cfg.db.saveBatch(table, batch, {
+            ...opt,
+            excludeFromIndexes,
+          })
+          return batch
+        },
+        {
+          concurrency: batchConcurrency,
+          errorMode,
+          flattenArrayOutput: true,
+        },
+      ),
+      transformLogProgress({
+        metric: 'saved',
+        ...opt,
+      }),
+      // just to satisfy and simplify typings
+      // It's easier to return Transform[], rather than (Transform | Writable)[]
+      writableVoid() as Transform,
+    ]
+  }
+
   // DELETE
   /**
    * @returns number of deleted items
@@ -995,7 +1067,7 @@ export class CommonDao<
    */
   async deleteByQuery(
     q: DBQuery<DBM>,
-    opt: CommonDaoStreamForEachOptions<DBM> & { stream?: boolean } = {},
+    opt: CommonDaoStreamDeleteOptions<DBM> = {},
   ): Promise<number> {
     this.requireWriteAccess()
     this.requireObjectMutability(opt)
@@ -1004,8 +1076,8 @@ export class CommonDao<
     const started = this.logStarted(op, q.table)
     let deleted = 0
 
-    if (opt.stream) {
-      const batchSize = 500
+    if (opt.batchSize) {
+      const { batchSize, batchConcurrency = 16 } = opt
 
       await _pipeline([
         this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
@@ -1022,6 +1094,7 @@ export class CommonDao<
           },
           {
             predicate: _passthroughPredicate,
+            concurrency: batchConcurrency,
           },
         ),
         // LogProgress should be AFTER the mapper, to be able to report correct stats

package/src/pipeline/dbPipelineBackup.ts

@@ -1,5 +1,5 @@
-import * as fs from 'node:fs'
-import * as fsp from 'node:fs/promises'
+import fs from 'node:fs'
+import fsp from 'node:fs/promises'
 import { createGzip, ZlibOptions } from 'node:zlib'
 import {
   AppError,

package/src/pipeline/dbPipelineRestore.ts

@@ -1,4 +1,4 @@
-import * as fs from 'node:fs'
+import fs from 'node:fs'
 import { createUnzip } from 'node:zlib'
 import {
   AsyncMapper,

package/src/query/dbQuery.ts

@@ -10,6 +10,7 @@ import {
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import {
   CommonDaoOptions,
+  CommonDaoStreamDeleteOptions,
   CommonDaoStreamForEachOptions,
   CommonDaoStreamOptions,
   DBPatch,
@@ -301,11 +302,11 @@ export class RunnableDBQuery<
     await this.dao.streamQueryAsDBMForEach(this, mapper, opt)
   }
 
-  streamQuery(opt?: CommonDaoStreamOptions): ReadableTyped<Saved<BM>> {
+  streamQuery(opt?: CommonDaoStreamOptions<Saved<BM>>): ReadableTyped<Saved<BM>> {
     return this.dao.streamQuery(this, opt)
   }
 
-  streamQueryAsDBM(opt?: CommonDaoStreamOptions): ReadableTyped<DBM> {
+  streamQueryAsDBM(opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM> {
     return this.dao.streamQueryAsDBM(this, opt)
   }
 
@@ -313,7 +314,7 @@ export class RunnableDBQuery<
     return await this.dao.queryIds(this, opt)
   }
 
-  streamQueryIds(opt?: CommonDaoStreamOptions): ReadableTyped<ID> {
+  streamQueryIds(opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID> {
     return this.dao.streamQueryIds(this, opt)
   }
 
@@ -324,9 +325,7 @@ export class RunnableDBQuery<
     await this.dao.streamQueryIdsForEach(this, mapper, opt)
   }
 
-  async deleteByQuery(
-    opt?: CommonDaoStreamForEachOptions<DBM> & { stream?: boolean },
-  ): Promise<number> {
+  async deleteByQuery(opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number> {
     return await this.dao.deleteByQuery(this, opt)
   }
 }

package/src/testing/daoTest.ts

@@ -1,5 +1,6 @@
+import { Readable } from 'node:stream'
 import { pDelay, _deepCopy, _pick, _sortBy, _omit, localTime } from '@naturalcycles/js-lib'
-import { readableToArray, transformNoOp } from '@naturalcycles/nodejs-lib'
+import { _pipeline, readableToArray, transformNoOp } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel, DBQuery } from '..'
 import { CommonDB } from '../common.db'
 import { CommonDao } from '../commondao/common.dao'
@@ -252,6 +253,19 @@ export function runCommonDaoTest(
         quirks,
       )
     })
+
+    test('streamSaveTransform', async () => {
+      const items2 = createTestItemsBM(2).map(i => ({ ...i, id: i.id + '_str' }))
+      const ids = items2.map(i => i.id)
+
+      await _pipeline([Readable.from(items2), ...dao.streamSaveTransform()])
+
+      const items2Loaded = await dao.getByIds(ids)
+      expectMatch(items2, items2Loaded, quirks)
+
+      // cleanup
+      await dao.query().filterIn('id', ids).deleteByQuery()
+    })
   }
 
   // DELETE BY