@naturalcycles/db-lib 9.12.1 → 9.14.0

package/dist/adapter/file/localFile.persistence.plugin.js

@@ -1,11 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LocalFilePersistencePlugin = void 0;
-const tslib_1 = require("tslib");
-const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
-const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
 const node_stream_1 = require("node:stream");
-const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 /**
@@ -21,7 +17,7 @@ class LocalFilePersistencePlugin {
     }
     async ping() { }
     async getTables() {
-        return (await promises_1.default.readdir(this.cfg.storagePath))
+        return (await nodejs_lib_1.fs2.readdirAsync(this.cfg.storagePath))
             .filter(f => f.includes('.ndjson'))
             .map(f => f.split('.ndjson')[0]);
     }
@@ -31,31 +27,16 @@ class LocalFilePersistencePlugin {
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
         if (!(await nodejs_lib_1.fs2.pathExistsAsync(filePath)))
             return [];
-        const transformUnzip = this.cfg.gzip ? [(0, node_zlib_1.createUnzip)()] : [];
-        const rows = [];
-        await (0, nodejs_lib_1._pipeline)([
-            node_fs_1.default.createReadStream(filePath),
-            ...transformUnzip,
-            (0, nodejs_lib_1.transformSplit)(), // splits by \n
-            (0, nodejs_lib_1.transformJsonParse)(),
-            (0, nodejs_lib_1.writablePushToArray)(rows),
-        ]);
-        return rows;
+        return await nodejs_lib_1.fs2.createReadStreamAsNDJSON(filePath).toArray();
     }
     async saveFiles(ops) {
-        await (0, js_lib_1.pMap)(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 16 });
+        await (0, js_lib_1.pMap)(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 32 });
     }
     async saveFile(table, rows) {
         await nodejs_lib_1.fs2.ensureDirAsync(this.cfg.storagePath);
         const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
         const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
-        const transformZip = this.cfg.gzip ? [(0, node_zlib_1.createGzip)()] : [];
-        await (0, nodejs_lib_1._pipeline)([
-            node_stream_1.Readable.from(rows),
-            (0, nodejs_lib_1.transformToNDJson)(),
-            ...transformZip,
-            node_fs_1.default.createWriteStream(filePath),
-        ]);
+        await (0, nodejs_lib_1._pipeline)([node_stream_1.Readable.from(rows), ...nodejs_lib_1.fs2.createWriteStreamAsNDJSON(filePath)]);
     }
 }
 exports.LocalFilePersistencePlugin = LocalFilePersistencePlugin;

package/dist/adapter/inmemory/inMemory.db.d.ts

@@ -29,7 +29,7 @@ export interface InMemoryDBCfg {
      */
     persistenceEnabled: boolean;
     /**
-     * @default ./tmp/inmemorydb
+     * @default ./tmp/inmemorydb.ndjson.gz
      *
      * Will store one ndjson file per table.
      * Will only flush on demand (see .flushToDisk() and .restoreFromDisk() methods).

package/dist/adapter/inmemory/inMemory.db.js

@@ -1,11 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryDBTransaction = exports.InMemoryDB = void 0;
-const tslib_1 = require("tslib");
-const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
-const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
 const node_stream_1 = require("node:stream");
-const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const __1 = require("../..");
@@ -169,7 +165,6 @@ class InMemoryDB {
         const { persistentStoragePath, persistZip } = this.cfg;
         const started = Date.now();
         await nodejs_lib_1.fs2.emptyDirAsync(persistentStoragePath);
-        const transformZip = persistZip ? [(0, node_zlib_1.createGzip)()] : [];
         let tables = 0;
         // infinite concurrency for now
         await (0, js_lib_1.pMap)(Object.keys(this.data), async (table) => {
@@ -178,12 +173,7 @@ class InMemoryDB {
                 return; // 0 rows
             tables++;
             const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`;
-            await (0, nodejs_lib_1._pipeline)([
-                node_stream_1.Readable.from(rows),
-                (0, nodejs_lib_1.transformToNDJson)(),
-                ...transformZip,
-                node_fs_1.default.createWriteStream(fname),
-            ]);
+            await (0, nodejs_lib_1._pipeline)([node_stream_1.Readable.from(rows), ...nodejs_lib_1.fs2.createWriteStreamAsNDJSON(fname)]);
         });
         this.cfg.logger.log(`flushToDisk took ${(0, nodejs_lib_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, nodejs_lib_1.yellow)(tables)} tables`);
     }
@@ -196,20 +186,12 @@ class InMemoryDB {
         const started = Date.now();
         await nodejs_lib_1.fs2.ensureDirAsync(persistentStoragePath);
         this.data = {}; // empty it in the beginning!
-        const files = (await promises_1.default.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'));
+        const files = (await nodejs_lib_1.fs2.readdirAsync(persistentStoragePath)).filter(f => f.includes('.ndjson'));
         // infinite concurrency for now
         await (0, js_lib_1.pMap)(files, async (file) => {
             const fname = `${persistentStoragePath}/${file}`;
             const table = file.split('.ndjson')[0];
-            const transformUnzip = file.endsWith('.gz') ? [(0, node_zlib_1.createUnzip)()] : [];
-            const rows = [];
-            await (0, nodejs_lib_1._pipeline)([
-                node_fs_1.default.createReadStream(fname),
-                ...transformUnzip,
-                (0, nodejs_lib_1.transformSplit)(), // splits by \n
-                (0, nodejs_lib_1.transformJsonParse)(),
-                (0, nodejs_lib_1.writablePushToArray)(rows),
-            ]);
+            const rows = await nodejs_lib_1.fs2.createReadStreamAsNDJSON(fname).toArray();
             this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
         });
         this.cfg.logger.log(`restoreFromDisk took ${(0, nodejs_lib_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, nodejs_lib_1.yellow)(files.length)} tables`);

package/dist/commondao/common.dao.d.ts

@@ -114,7 +114,7 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
      * "Streaming" is implemented by buffering incoming rows into **batches**
      * (of size opt.chunkSize, which defaults to 500),
      * and then executing db.saveBatch(chunk) with the concurrency
-     * of opt.chunkConcurrency (which defaults to 16).
+     * of opt.chunkConcurrency (which defaults to 32).
      */
     streamSaveTransform(opt?: CommonDaoStreamSaveOptions<DBM>): Transform[];
     /**

package/dist/commondao/common.dao.js

@@ -423,8 +423,7 @@ class CommonDao {
         const started = this.logStarted(op, q.table, true);
         let count = 0;
         await (0, nodejs_lib_1._pipeline)([
-            this.cfg.db.streamQuery(q.select(['id']), opt),
-            (0, nodejs_lib_1.transformMapSimple)(r => {
+            this.cfg.db.streamQuery(q.select(['id']), opt).map(r => {
                 count++;
                 return r.id;
             }),
@@ -699,7 +698,7 @@ class CommonDao {
      * "Streaming" is implemented by buffering incoming rows into **batches**
      * (of size opt.chunkSize, which defaults to 500),
      * and then executing db.saveBatch(chunk) with the concurrency
-     * of opt.chunkConcurrency (which defaults to 16).
+     * of opt.chunkConcurrency (which defaults to 32).
      */
     streamSaveTransform(opt = {}) {
         this.requireWriteAccess();
@@ -711,7 +710,7 @@ class CommonDao {
         }
         const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes;
         const { beforeSave } = this.cfg.hooks;
-        const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt;
+        const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt;
         return [
             (0, nodejs_lib_1.transformMap)(async (bm) => {
                 this.assignIdCreatedUpdated(bm, opt); // mutates
@@ -780,15 +779,13 @@ class CommonDao {
         const started = this.logStarted(op, q.table);
         let deleted = 0;
         if (opt.chunkSize) {
-            const { chunkSize, chunkConcurrency = 16 } = opt;
+            const { chunkSize, chunkConcurrency = 32 } = opt;
             await (0, nodejs_lib_1._pipeline)([
-                this.cfg.db.streamQuery(q.select(['id']), opt),
-                (0, nodejs_lib_1.transformMapSimple)(r => r.id, {
-                    errorMode: js_lib_1.ErrorMode.SUPPRESS,
-                }),
+                this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id),
                 (0, nodejs_lib_1.transformChunk)({ chunkSize }),
                 (0, nodejs_lib_1.transformMap)(async (ids) => {
-                    deleted += await this.cfg.db.deleteByQuery(dbQuery_1.DBQuery.create(q.table).filterIn('id', ids), opt);
+                    await this.cfg.db.deleteByIds(q.table, ids, opt);
+                    deleted += ids.length;
                 }, {
                     predicate: js_lib_1._passthroughPredicate,
                     concurrency: chunkConcurrency,

package/dist/commondao/common.dao.model.d.ts

@@ -257,7 +257,7 @@ export interface CommonDaoStreamOptions<IN> extends CommonDaoOptions, TransformL
     chunkSize?: number;
     /**
      * When chunkSize is set - this option controls how many chunks to run concurrently.
-     * Defaults to 16, "the magic number of JavaScript concurrency".
+     * Defaults to 32.
      */
     chunkConcurrency?: number;
 }

package/dist/kv/commonKeyValueDao.js

@@ -137,7 +137,7 @@ class CommonKeyValueDao {
                 return []; // SKIP
             }
         }, {
-            concurrency: 16,
+            concurrency: 32,
         });
     }
     streamEntries(limit) {
@@ -154,7 +154,7 @@ class CommonKeyValueDao {
                 return []; // SKIP
             }
         }, {
-            concurrency: 16,
+            concurrency: 32,
         });
     }
 }

package/dist/pipeline/dbPipelineBackup.d.ts

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-import { ZlibOptions } from 'node:zlib';
 import { AsyncMapper, ErrorMode, UnixTimestampNumber, StringMap } from '@naturalcycles/js-lib';
 import { NDJsonStats, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
 import { CommonDB } from '../common.db';
@@ -65,8 +63,8 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
     gzip?: boolean;
     /**
      * Only applicable if `gzip` is enabled
+     * Currently not available.
      */
-    zlibOptions?: ZlibOptions;
     /**
      * Optionally you can provide mapper that is going to run for each table.
      *
@@ -96,10 +94,6 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
      * If true - will use CommonDB.getTableSchema() and emit schema.
      */
     emitSchemaFromDB?: boolean;
-    /**
-     * @default false
-     */
-    sortObjects?: boolean;
 }
 /**
  * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).

package/dist/pipeline/dbPipelineBackup.js

@@ -1,10 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbPipelineBackup = void 0;
-const tslib_1 = require("tslib");
-const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
-const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
-const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const index_1 = require("../index");
@@ -18,8 +14,7 @@ const index_1 = require("../index");
  * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
  */
 async function dbPipelineBackup(opt) {
-    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, zlibOptions, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, emitSchemaFromDB = false, sortObjects = false, } = opt;
-    const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
+    const { db, concurrency = 16, limit = 0, outputDirPath, protectFromOverwrite = false, mapperPerTable = {}, queryPerTable = {}, logEveryPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, emitSchemaFromDB = false, } = opt;
     const gzip = opt.gzip !== false; // default to true
     let { tables } = opt;
     console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineBackup')} started in ${(0, nodejs_lib_1.grey)(outputDirPath)}...`);
@@ -74,11 +69,9 @@ async function dbPipelineBackup(opt) {
             (0, nodejs_lib_1.transformTap)(() => {
                 rows++;
             }),
-            (0, nodejs_lib_1.transformToNDJson)({ strict, sortObjects }),
-            ...(gzip ? [(0, node_zlib_1.createGzip)(zlibOptions)] : []), // optional gzip
-            node_fs_1.default.createWriteStream(filePath),
+            ...nodejs_lib_1.fs2.createWriteStreamAsNDJSON(filePath),
         ]);
-        const { size: sizeBytes } = await promises_1.default.stat(filePath);
+        const { size: sizeBytes } = await nodejs_lib_1.fs2.statAsync(filePath);
         const stats = nodejs_lib_1.NDJsonStats.create({
             tookMillis: Date.now() - started,
             rows,

package/dist/pipeline/dbPipelineRestore.js

@@ -1,9 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbPipelineRestore = void 0;
-const tslib_1 = require("tslib");
-const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
-const node_zlib_1 = require("node:zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 /**
@@ -15,7 +12,6 @@ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
  */
 async function dbPipelineRestore(opt) {
     const { db, concurrency = 16, chunkSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
-    const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
     const onlyTables = opt.tables && new Set(opt.tables);
     const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
     console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineRestore')} started in ${(0, nodejs_lib_1.grey)(inputDirPath)}...${sinceUpdatedStr}`);
@@ -24,7 +20,7 @@ async function dbPipelineRestore(opt) {
     const sizeByTable = {};
     const statsPerTable = {};
     const tables = [];
-    node_fs_1.default.readdirSync(inputDirPath).forEach(f => {
+    nodejs_lib_1.fs2.readdir(inputDirPath).forEach(f => {
         let table;
         let gzip = false;
         if (f.endsWith('.ndjson')) {
@@ -42,7 +38,7 @@ async function dbPipelineRestore(opt) {
         tables.push(table);
         if (gzip)
             tablesToGzip.add(table);
-        sizeByTable[table] = node_fs_1.default.statSync(`${inputDirPath}/${f}`).size;
+        sizeByTable[table] = nodejs_lib_1.fs2.stat(`${inputDirPath}/${f}`).size;
     });
     const sizeStrByTable = (0, js_lib_1._mapValues)(sizeByTable, (_k, b) => (0, js_lib_1._hb)(b));
     console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n`, sizeStrByTable);
@@ -50,7 +46,7 @@ async function dbPipelineRestore(opt) {
     if (recreateTables) {
         await (0, js_lib_1.pMap)(tables, async (table) => {
             const schemaFilePath = `${inputDirPath}/${table}.schema.json`;
-            if (!node_fs_1.default.existsSync(schemaFilePath)) {
+            if (!nodejs_lib_1.fs2.pathExists(schemaFilePath)) {
                 console.warn(`${schemaFilePath} does not exist!`);
                 return;
             }
@@ -67,17 +63,13 @@ async function dbPipelineRestore(opt) {
         const sizeBytes = sizeByTable[table];
         console.log(`<< ${(0, nodejs_lib_1.grey)(filePath)} ${(0, nodejs_lib_1.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
         await (0, nodejs_lib_1._pipeline)([
-            node_fs_1.default.createReadStream(filePath),
-            ...(gzip ? [(0, node_zlib_1.createUnzip)()] : []),
-            (0, nodejs_lib_1.transformSplit)(), // splits by \n
-            (0, nodejs_lib_1.transformJsonParse)({ strict }),
+            nodejs_lib_1.fs2.createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
             (0, nodejs_lib_1.transformTap)(() => rows++),
             (0, nodejs_lib_1.transformLogProgress)({
                 logEvery: 1000,
                 ...opt,
                 metric: table,
             }),
-            (0, nodejs_lib_1.transformLimit)({ limit }),
             ...(sinceUpdated
                 ? [(0, nodejs_lib_1.transformFilterSync)(r => r.updated >= sinceUpdated)]
                 : []),

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "9.12.1",
+  "version": "9.14.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/adapter/file/localFile.persistence.plugin.ts

@@ -1,16 +1,6 @@
-import fs from 'node:fs'
-import fsp from 'node:fs/promises'
 import { Readable } from 'node:stream'
-import { createGzip, createUnzip } from 'node:zlib'
 import { ObjectWithId, pMap } from '@naturalcycles/js-lib'
-import {
-  transformJsonParse,
-  transformSplit,
-  transformToNDJson,
-  writablePushToArray,
-  _pipeline,
-  fs2,
-} from '@naturalcycles/nodejs-lib'
+import { _pipeline, fs2 } from '@naturalcycles/nodejs-lib'
 import { DBSaveBatchOperation } from '../../db.model'
 import { FileDBPersistencePlugin } from './file.db.model'
 
@@ -43,7 +33,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   async ping(): Promise<void> {}
 
   async getTables(): Promise<string[]> {
-    return (await fsp.readdir(this.cfg.storagePath))
+    return (await fs2.readdirAsync(this.cfg.storagePath))
       .filter(f => f.includes('.ndjson'))
       .map(f => f.split('.ndjson')[0]!)
   }
@@ -55,36 +45,18 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
 
     if (!(await fs2.pathExistsAsync(filePath))) return []
 
-    const transformUnzip = this.cfg.gzip ? [createUnzip()] : []
-
-    const rows: ROW[] = []
-
-    await _pipeline([
-      fs.createReadStream(filePath),
-      ...transformUnzip,
-      transformSplit(), // splits by \n
-      transformJsonParse(),
-      writablePushToArray(rows),
-    ])
-
-    return rows
+    return await fs2.createReadStreamAsNDJSON(filePath).toArray()
   }
 
   async saveFiles(ops: DBSaveBatchOperation<any>[]): Promise<void> {
-    await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 16 })
+    await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 32 })
   }
 
   async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {
     await fs2.ensureDirAsync(this.cfg.storagePath)
     const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
     const filePath = `${this.cfg.storagePath}/${table}.${ext}`
-    const transformZip = this.cfg.gzip ? [createGzip()] : []
 
-    await _pipeline([
-      Readable.from(rows),
-      transformToNDJson(),
-      ...transformZip,
-      fs.createWriteStream(filePath),
-    ])
+    await _pipeline([Readable.from(rows), ...fs2.createWriteStreamAsNDJSON(filePath)])
   }
 }

package/src/adapter/inmemory/inMemory.db.ts

@@ -1,7 +1,4 @@
-import fs from 'node:fs'
-import fsp from 'node:fs/promises'
 import { Readable } from 'node:stream'
-import { createGzip, createUnzip } from 'node:zlib'
 import {
   generateJsonSchemaFromData,
   JsonSchemaObject,
@@ -20,10 +17,6 @@ import {
 import {
   bufferReviver,
   ReadableTyped,
-  transformJsonParse,
-  transformSplit,
-  transformToNDJson,
-  writablePushToArray,
   _pipeline,
   dimGrey,
   yellow,
@@ -78,7 +71,7 @@ export interface InMemoryDBCfg {
   persistenceEnabled: boolean
 
   /**
-   * @default ./tmp/inmemorydb
+   * @default ./tmp/inmemorydb.ndjson.gz
    *
    * Will store one ndjson file per table.
    * Will only flush on demand (see .flushToDisk() and .restoreFromDisk() methods).
@@ -312,7 +305,6 @@ export class InMemoryDB implements CommonDB {
 
     await fs2.emptyDirAsync(persistentStoragePath)
 
-    const transformZip = persistZip ? [createGzip()] : []
     let tables = 0
 
     // infinite concurrency for now
@@ -323,12 +315,7 @@ export class InMemoryDB implements CommonDB {
       tables++
       const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`
 
-      await _pipeline([
-        Readable.from(rows),
-        transformToNDJson(),
-        ...transformZip,
-        fs.createWriteStream(fname),
-      ])
+      await _pipeline([Readable.from(rows), ...fs2.createWriteStreamAsNDJSON(fname)])
     })
 
     this.cfg.logger!.log(
@@ -349,24 +336,14 @@ export class InMemoryDB implements CommonDB {
 
     this.data = {} // empty it in the beginning!
 
-    const files = (await fsp.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'))
+    const files = (await fs2.readdirAsync(persistentStoragePath)).filter(f => f.includes('.ndjson'))
 
     // infinite concurrency for now
     await pMap(files, async file => {
       const fname = `${persistentStoragePath}/${file}`
       const table = file.split('.ndjson')[0]!
 
-      const transformUnzip = file.endsWith('.gz') ? [createUnzip()] : []
-
-      const rows: any[] = []
-
-      await _pipeline([
-        fs.createReadStream(fname),
-        ...transformUnzip,
-        transformSplit(), // splits by \n
-        transformJsonParse(),
-        writablePushToArray(rows),
-      ])
+      const rows = await fs2.createReadStreamAsNDJSON(fname).toArray()
 
       this.data[table] = _by(rows, r => r.id)
     })

package/src/commondao/common.dao.model.ts

@@ -321,7 +321,7 @@ export interface CommonDaoStreamOptions<IN>
 
   /**
    * When chunkSize is set - this option controls how many chunks to run concurrently.
-   * Defaults to 16, "the magic number of JavaScript concurrency".
+   * Defaults to 32.
    */
   chunkConcurrency?: number
 }

package/src/commondao/common.dao.ts

@@ -40,7 +40,6 @@ import {
   transformChunk,
   transformLogProgress,
   transformMap,
-  transformMapSimple,
   transformNoOp,
   writableVoid,
 } from '@naturalcycles/nodejs-lib'
@@ -580,8 +579,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     let count = 0
 
     await _pipeline([
-      this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
-      transformMapSimple<DBM, string>(r => {
+      this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => {
         count++
         return r.id
       }),
@@ -920,7 +918,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
    * "Streaming" is implemented by buffering incoming rows into **batches**
    * (of size opt.chunkSize, which defaults to 500),
    * and then executing db.saveBatch(chunk) with the concurrency
-   * of opt.chunkConcurrency (which defaults to 16).
+   * of opt.chunkConcurrency (which defaults to 32).
    */
   streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
     this.requireWriteAccess()
@@ -936,7 +934,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
     const { beforeSave } = this.cfg.hooks!
 
-    const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt
+    const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt
 
     return [
       transformMap<BM, DBM>(
@@ -1019,20 +1017,15 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     let deleted = 0
 
     if (opt.chunkSize) {
-      const { chunkSize, chunkConcurrency = 16 } = opt
+      const { chunkSize, chunkConcurrency = 32 } = opt
 
       await _pipeline([
-        this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
-        transformMapSimple<ObjectWithId, string>(r => r.id, {
-          errorMode: ErrorMode.SUPPRESS,
-        }),
+        this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => r.id),
         transformChunk<string>({ chunkSize }),
         transformMap<string[], void>(
           async ids => {
-            deleted += await this.cfg.db.deleteByQuery(
-              DBQuery.create(q.table).filterIn('id', ids),
-              opt,
-            )
+            await this.cfg.db.deleteByIds(q.table, ids, opt)
+            deleted += ids.length
           },
           {
             predicate: _passthroughPredicate,

package/src/kv/commonKeyValueDao.ts

@@ -224,7 +224,7 @@ export class CommonKeyValueDao<T> {
         }
       },
       {
-        concurrency: 16,
+        concurrency: 32,
       },
     )
   }
@@ -248,7 +248,7 @@ export class CommonKeyValueDao<T> {
         }
       },
       {
-        concurrency: 16,
+        concurrency: 32,
       },
     )
   }

package/src/pipeline/dbPipelineBackup.ts

@@ -1,6 +1,3 @@
-import fs from 'node:fs'
-import fsp from 'node:fs/promises'
-import { createGzip, ZlibOptions } from 'node:zlib'
 import {
   AppError,
   AsyncMapper,
@@ -18,7 +15,6 @@ import {
   transformMap,
   TransformMapOptions,
   transformTap,
-  transformToNDJson,
   _pipeline,
   boldWhite,
   dimWhite,
@@ -101,8 +97,9 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
 
   /**
    * Only applicable if `gzip` is enabled
+   * Currently not available.
    */
-  zlibOptions?: ZlibOptions
+  // zlibOptions?: ZlibOptions
 
   /**
    * Optionally you can provide mapper that is going to run for each table.
@@ -138,11 +135,6 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
    * If true - will use CommonDB.getTableSchema() and emit schema.
    */
   emitSchemaFromDB?: boolean
-
-  /**
-   * @default false
-   */
-  sortObjects?: boolean
 }
 
 /**
@@ -161,16 +153,13 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
     limit = 0,
     outputDirPath,
     protectFromOverwrite = false,
-    zlibOptions,
     mapperPerTable = {},
     queryPerTable = {},
     logEveryPerTable = {},
     transformMapOptions,
     errorMode = ErrorMode.SUPPRESS,
     emitSchemaFromDB = false,
-    sortObjects = false,
   } = opt
-  const strict = errorMode !== ErrorMode.SUPPRESS
   const gzip = opt.gzip !== false // default to true
 
   let { tables } = opt
@@ -243,12 +232,10 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         transformTap(() => {
           rows++
         }),
-        transformToNDJson({ strict, sortObjects }),
-        ...(gzip ? [createGzip(zlibOptions)] : []), // optional gzip
-        fs.createWriteStream(filePath),
+        ...fs2.createWriteStreamAsNDJSON(filePath),
       ])
 
-      const { size: sizeBytes } = await fsp.stat(filePath)
+      const { size: sizeBytes } = await fs2.statAsync(filePath)
 
       const stats = NDJsonStats.create({
         tookMillis: Date.now() - started,

package/src/pipeline/dbPipelineRestore.ts

@@ -1,5 +1,3 @@
-import fs from 'node:fs'
-import { createUnzip } from 'node:zlib'
 import {
   AsyncMapper,
   ErrorMode,
@@ -15,13 +13,10 @@ import {
   NDJsonStats,
   transformChunk,
   transformFilterSync,
-  transformJsonParse,
-  transformLimit,
   transformLogProgress,
   TransformLogProgressOptions,
   transformMap,
   TransformMapOptions,
-  transformSplit,
   transformTap,
   writableForEach,
   _pipeline,
@@ -135,7 +130,6 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
     errorMode = ErrorMode.SUPPRESS,
     recreateTables = false,
   } = opt
-  const strict = errorMode !== ErrorMode.SUPPRESS
   const onlyTables = opt.tables && new Set(opt.tables)
 
   const sinceUpdatedStr = sinceUpdated ? ' since ' + grey(localTime(sinceUpdated).toPretty()) : ''
@@ -150,7 +144,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
   const sizeByTable: Record<string, number> = {}
   const statsPerTable: Record<string, NDJsonStats> = {}
   const tables: string[] = []
-  fs.readdirSync(inputDirPath).forEach(f => {
+  fs2.readdir(inputDirPath).forEach(f => {
     let table: string
     let gzip = false
 
@@ -167,7 +161,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
 
     tables.push(table)
     if (gzip) tablesToGzip.add(table)
-    sizeByTable[table] = fs.statSync(`${inputDirPath}/${f}`).size
+    sizeByTable[table] = fs2.stat(`${inputDirPath}/${f}`).size
   })
 
   const sizeStrByTable = _mapValues(sizeByTable, (_k, b) => _hb(b))
@@ -179,7 +173,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
   if (recreateTables) {
     await pMap(tables, async table => {
       const schemaFilePath = `${inputDirPath}/${table}.schema.json`
-      if (!fs.existsSync(schemaFilePath)) {
+      if (!fs2.pathExists(schemaFilePath)) {
         console.warn(`${schemaFilePath} does not exist!`)
         return
       }
@@ -204,17 +198,13 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
       console.log(`<< ${grey(filePath)} ${dimWhite(_hb(sizeBytes))} started...`)
 
       await _pipeline([
-        fs.createReadStream(filePath),
-        ...(gzip ? [createUnzip()] : []),
-        transformSplit(), // splits by \n
-        transformJsonParse({ strict }),
+        fs2.createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
         transformTap(() => rows++),
         transformLogProgress({
           logEvery: 1000,
           ...opt,
           metric: table,
         }),
-        transformLimit({ limit }),
         ...(sinceUpdated
           ? [transformFilterSync<BaseDBEntity>(r => r.updated >= sinceUpdated)]
           : []),