@naturalcycles/db-lib 10.26.1 → 10.27.0

package/dist/adapter/cachedb/cache.db.d.ts

@@ -1,6 +1,6 @@
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { ObjectWithId } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { BaseCommonDB } from '../../commondb/base.common.db.js';
 import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js';
 import type { RunQueryResult } from '../../db.model.js';
@@ -27,7 +27,7 @@ export declare class CacheDB extends BaseCommonDB implements CommonDB {
     saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: CacheDBSaveOptions<ROW>): Promise<void>;
     runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CacheDBSaveOptions<ROW>): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CacheDBOptions): Promise<number>;
-    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CacheDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CacheDBStreamOptions): Pipeline<ROW>;
     deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CacheDBOptions): Promise<number>;
     patchByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, patch: Partial<ROW>, opt?: CacheDBOptions): Promise<number>;
 }

package/dist/adapter/cachedb/cache.db.js

@@ -1,5 +1,5 @@
-import { Readable } from 'node:stream';
 import { _isTruthy } from '@naturalcycles/js-lib';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { BaseCommonDB } from '../../commondb/base.common.db.js';
 import { commonDBFullSupport } from '../../commondb/common.db.js';
 /**
@@ -130,7 +130,7 @@ export class CacheDB extends BaseCommonDB {
     }
     streamQuery(q, opt = {}) {
         if (!opt.onlyCache && !this.cfg.onlyCache) {
-            const stream = this.cfg.downstreamDB.streamQuery(q, opt);
+            const pipeline = this.cfg.downstreamDB.streamQuery(q, opt);
             // Don't save to cache if it was a projection query
             if (!opt.skipCache && !this.cfg.skipCache && !q._selectedFieldNames) {
                 // todo: rethink if we really should download WHOLE stream into memory in order to save it to cache
@@ -141,11 +141,11 @@ export class CacheDB extends BaseCommonDB {
                 //     await this.cfg.cacheDB.saveBatch(q.table, dbms as any)
                 //   })
             }
-            return stream;
+            return pipeline;
         }
         if (opt.skipCache || this.cfg.skipCache)
-            return Readable.from([]);
-        const stream = this.cfg.cacheDB.streamQuery(q, opt);
+            return Pipeline.fromArray([]);
+        const pipeline = this.cfg.cacheDB.streamQuery(q, opt);
         // if (this.cfg.logCached) {
         //   let count = 0
         //
@@ -156,7 +156,7 @@ export class CacheDB extends BaseCommonDB {
         //       this.log(`${q.table}.streamQuery ${length} rows from cache`)
         //     })
         // }
-        return stream;
+        return pipeline;
     }
     async deleteByQuery(q, opt = {}) {
         if (!opt.onlyCache && !this.cfg.onlyCache) {

package/dist/adapter/file/file.db.d.ts

@@ -1,6 +1,6 @@
 import type { JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import { type ObjectWithId } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { BaseCommonDB } from '../../commondb/base.common.db.js';
 import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js';
 import type { CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, DBSaveBatchOperation, RunQueryResult } from '../../db.model.js';
@@ -26,7 +26,7 @@ export declare class FileDB extends BaseCommonDB implements CommonDB {
     saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], _opt?: CommonDBSaveOptions<ROW>): Promise<void>;
     runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
-    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBStreamOptions): Pipeline<ROW>;
     deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
     deleteByIds(table: string, ids: string[], _opt?: CommonDBOptions): Promise<number>;
     getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;

package/dist/adapter/file/file.db.js

@@ -1,3 +1,4 @@
+import { Readable } from 'node:stream';
 import { _by, _sortBy } from '@naturalcycles/js-lib/array';
 import { _since, localTime } from '@naturalcycles/js-lib/datetime';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
@@ -5,7 +6,7 @@ import { generateJsonSchemaFromData } from '@naturalcycles/js-lib/json-schema';
 import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object';
 import { _stringMapValues, } from '@naturalcycles/js-lib/types';
 import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
-import { readableCreate } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { BaseCommonDB } from '../../commondb/base.common.db.js';
 import { commonDBFullSupport } from '../../commondb/common.db.js';
 import { queryInMemory } from '../../inmemory/queryInMemory.js';
@@ -82,12 +83,10 @@ export class FileDB extends BaseCommonDB {
         return (await this.loadFile(q.table)).length;
     }
     streamQuery(q, opt) {
-        const readable = readableCreate();
-        void this.runQuery(q, opt).then(({ rows }) => {
-            rows.forEach(r => readable.push(r));
-            readable.push(null); // done
+        return Pipeline.fromAsyncReadable(async () => {
+            const { rows } = await this.runQuery(q, opt);
+            return Readable.from(rows);
         });
-        return readable;
     }
     async deleteByQuery(q, _opt) {
         const byId = _by(await this.loadFile(q.table), r => r.id);

package/dist/commondao/common.dao.d.ts

@@ -1,11 +1,11 @@
 import type { Transform } from 'node:stream';
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
-import { type AsyncIndexedMapper, type BaseDBEntity, type NonNegativeInteger, type StringMap, type Unsaved } from '@naturalcycles/js-lib/types';
-import { type ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { type BaseDBEntity, type NonNegativeInteger, type StringMap, type Unsaved } from '@naturalcycles/js-lib/types';
+import { type Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBTransactionOptions, RunQueryResult } from '../db.model.js';
 import type { DBQuery } from '../query/dbQuery.js';
 import { RunnableDBQuery } from '../query/dbQuery.js';
-import type { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoPatchByIdOptions, CommonDaoPatchOptions, CommonDaoReadOptions, CommonDaoSaveBatchOptions, CommonDaoSaveOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions, CommonDaoStreamSaveOptions } from './common.dao.model.js';
+import type { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoPatchByIdOptions, CommonDaoPatchOptions, CommonDaoReadOptions, CommonDaoSaveBatchOptions, CommonDaoSaveOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamOptions, CommonDaoStreamSaveOptions } from './common.dao.model.js';
 import { CommonDaoTransaction } from './commonDaoTransaction.js';
 /**
  * Lowest common denominator API between supported Databases.
@@ -45,25 +45,11 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     runQueryAsDBM(q: DBQuery<DBM>, opt?: CommonDaoReadOptions): Promise<DBM[]>;
     runQueryExtendedAsDBM(q: DBQuery<DBM>, opt?: CommonDaoReadOptions): Promise<RunQueryResult<DBM>>;
     runQueryCount(q: DBQuery<DBM>, opt?: CommonDaoReadOptions): Promise<number>;
-    streamQueryForEach(q: DBQuery<DBM>, mapper: AsyncIndexedMapper<BM, void>, opt?: CommonDaoStreamForEachOptions<BM>): Promise<void>;
-    streamQueryAsDBMForEach(q: DBQuery<DBM>, mapper: AsyncIndexedMapper<DBM, void>, opt?: CommonDaoStreamForEachOptions<DBM>): Promise<void>;
-    /**
-     * Stream as Readable, to be able to .pipe() it further with support of backpressure.
-     */
-    streamQueryAsDBM(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM>;
-    /**
-     * Stream as Readable, to be able to .pipe() it further with support of backpressure.
-     *
-     * Please note that this stream is currently not async-iteration friendly, because of
-     * `through2` usage.
-     * Will be migrated/fixed at some point in the future.
-     *
-     * You can do `.pipe(transformNoOp)` to make it "valid again".
-     */
-    streamQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<BM>): ReadableTyped<BM>;
+    streamQueryAsDBM(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<DBM>): Pipeline<DBM>;
+    streamQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<BM>): Pipeline<BM>;
+    private streamQueryRaw;
     queryIds(q: DBQuery<DBM>, opt?: CommonDaoReadOptions): Promise<ID[]>;
-    streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID>;
-    streamQueryIdsForEach(q: DBQuery<DBM>, mapper: AsyncIndexedMapper<ID, void>, opt?: CommonDaoStreamForEachOptions<ID>): Promise<void>;
+    streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<ID>): Pipeline<ID>;
     /**
      * Mutates!
      */

package/dist/commondao/common.dao.js

@@ -8,8 +8,8 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
 import { _passthroughPredicate, _typeCast } from '@naturalcycles/js-lib/types';
 import { stringId } from '@naturalcycles/nodejs-lib';
-import { Pipeline, transformFlatten, transformMapSync, } from '@naturalcycles/nodejs-lib/stream';
-import { transformChunk, transformLogProgress, transformMap, transformNoOp, } from '@naturalcycles/nodejs-lib/stream';
+import { transformFlatten } from '@naturalcycles/nodejs-lib/stream';
+import { transformChunk, transformLogProgress, transformMap, } from '@naturalcycles/nodejs-lib/stream';
 import { DBLibError } from '../cnst.js';
 import { RunnableDBQuery } from '../query/dbQuery.js';
 import { CommonDaoTransaction } from './commonDaoTransaction.js';
@@ -36,7 +36,6 @@ export class CommonDao {
             hooks: {
                 parseNaturalId: () => ({}),
                 beforeCreate: bm => bm,
-                anonymize: dbm => dbm,
                 onValidationError: err => err,
                 ...cfg.hooks,
             },
@@ -161,116 +160,28 @@ export class CommonDao {
         q.table = opt.table || q.table;
         return await this.cfg.db.runQueryCount(q, opt);
     }
-    async streamQueryForEach(q, mapper, opt = {}) {
-        this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
-        q.table = opt.table || q.table;
-        opt.skipValidation = opt.skipValidation !== false; // default true
-        opt.errorMode ||= ErrorMode.SUPPRESS;
+    streamQueryAsDBM(q, opt = {}) {
+        const pipeline = this.streamQueryRaw(q, opt);
         const isPartialQuery = !!q._selectedFieldNames;
-        await Pipeline.from(this.cfg.db.streamQuery(q, opt))
-            .map(async (dbm) => {
-            if (isPartialQuery)
-                return dbm;
-            return await this.dbmToBM(dbm, opt);
-        }, {
-            errorMode: opt.errorMode,
-        })
-            .map(mapper, {
-            ...opt,
-            predicate: _passthroughPredicate, // to be able to logProgress
-        })
-            // LogProgress should be AFTER the mapper, to be able to report correct stats
-            .logProgress({
-            metric: q.table,
-            ...opt,
-        })
-            .run();
-    }
-    async streamQueryAsDBMForEach(q, mapper, opt = {}) {
-        this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
-        q.table = opt.table || q.table;
-        opt.skipValidation = opt.skipValidation !== false; // default true
+        if (isPartialQuery)
+            return pipeline;
+        opt.skipValidation ??= true;
         opt.errorMode ||= ErrorMode.SUPPRESS;
-        const isPartialQuery = !!q._selectedFieldNames;
-        await Pipeline.from(this.cfg.db.streamQuery(q, opt))
-            .mapSync(dbm => {
-            if (isPartialQuery)
-                return dbm;
-            return this.anyToDBM(dbm, opt);
-        }, {
-            errorMode: opt.errorMode,
-        })
-            .map(mapper, {
-            ...opt,
-            predicate: _passthroughPredicate, // to be able to logProgress
-        })
-            .logProgress({
-            metric: q.table,
-            ...opt,
-        })
-            .run();
+        return pipeline.mapSync(dbm => this.anyToDBM(dbm, opt), { errorMode: opt.errorMode });
     }
-    /**
-     * Stream as Readable, to be able to .pipe() it further with support of backpressure.
-     */
-    streamQueryAsDBM(q, opt = {}) {
-        this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
-        q.table = opt.table || q.table;
-        opt.skipValidation = opt.skipValidation !== false; // default true
-        opt.errorMode ||= ErrorMode.SUPPRESS;
+    streamQuery(q, opt = {}) {
+        const pipeline = this.streamQueryRaw(q, opt);
         const isPartialQuery = !!q._selectedFieldNames;
-        const stream = this.cfg.db.streamQuery(q, opt);
         if (isPartialQuery)
-            return stream;
-        return (stream
-            // the commented out line was causing RangeError: Maximum call stack size exceeded
-            // .on('error', err => stream.emit('error', err))
-            .pipe(transformMapSync(dbm => {
-            return this.anyToDBM(dbm, opt);
-        }, {
-            errorMode: opt.errorMode,
-        })));
+            return pipeline;
+        opt.skipValidation ??= true;
+        opt.errorMode ||= ErrorMode.SUPPRESS;
+        return pipeline.map(async (dbm) => await this.dbmToBM(dbm, opt), { errorMode: opt.errorMode });
     }
-    /**
-     * Stream as Readable, to be able to .pipe() it further with support of backpressure.
-     *
-     * Please note that this stream is currently not async-iteration friendly, because of
-     * `through2` usage.
-     * Will be migrated/fixed at some point in the future.
-     *
-     * You can do `.pipe(transformNoOp)` to make it "valid again".
-     */
-    streamQuery(q, opt = {}) {
+    streamQueryRaw(q, opt = {}) {
         this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
         q.table = opt.table || q.table;
-        opt.skipValidation = opt.skipValidation !== false; // default true
-        opt.errorMode ||= ErrorMode.SUPPRESS;
-        const stream = this.cfg.db.streamQuery(q, opt);
-        const isPartialQuery = !!q._selectedFieldNames;
-        if (isPartialQuery)
-            return stream;
-        // This almost works, but hard to implement `errorMode: THROW_AGGREGATED` in this case
-        // return stream.flatMap(async (dbm: DBM) => {
-        //   return [await this.dbmToBM(dbm, opt)] satisfies BM[]
-        // }, {
-        //   concurrency: 16,
-        // })
-        return (stream
-            // optimization: 1 validation is enough
-            // .pipe(transformMap<any, DBM>(dbm => this.anyToDBM(dbm, opt), safeOpt))
-            // .pipe(transformMap<DBM, BM>(dbm => this.dbmToBM(dbm, opt), safeOpt))
-            // the commented out line was causing RangeError: Maximum call stack size exceeded
-            // .on('error', err => stream.emit('error', err))
-            .pipe(transformMap(async (dbm) => {
-            return await this.dbmToBM(dbm, opt);
-        }, {
-            errorMode: opt.errorMode,
-        }))
-            // this can make the stream async-iteration-friendly
-            // but not applying it now for perf reasons
-            // UPD: applying, to be compliant with `.toArray()`, etc.
-            .on('error', err => stream.destroy(err))
-            .pipe(transformNoOp()));
+        return this.cfg.db.streamQuery(q, opt);
     }
     async queryIds(q, opt = {}) {
         this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
@@ -282,36 +193,7 @@ export class CommonDao {
         this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
         q.table = opt.table || q.table;
         opt.errorMode ||= ErrorMode.SUPPRESS;
-        // Experimental: using `.map()`
-        const stream = this.cfg.db
-            .streamQuery(q.select(['id']), opt)
-            // .on('error', err => stream.emit('error', err))
-            .map((r) => r.id);
-        // const stream: ReadableTyped<string> = this.cfg.db
-        //   .streamQuery<DBM>(q.select(['id']), opt)
-        //   .on('error', err => stream.emit('error', err))
-        //   .pipe(
-        //     transformMapSimple<DBM, string>(r => r.id, {
-        //       errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        //     }),
-        //   )
-        return stream;
-    }
-    async streamQueryIdsForEach(q, mapper, opt = {}) {
-        this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
-        q.table = opt.table || q.table;
-        opt.errorMode ||= ErrorMode.SUPPRESS;
-        await Pipeline.from(this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id))
-            .map(mapper, {
-            ...opt,
-            predicate: _passthroughPredicate,
-        })
-            // LogProgress should be AFTER the mapper, to be able to report correct stats
-            .logProgress({
-            metric: q.table,
-            ...opt,
-        })
-            .run();
+        return this.cfg.db.streamQuery(q.select(['id']), opt).mapSync((r) => r.id);
     }
     /**
      * Mutates!
@@ -590,7 +472,9 @@ export class CommonDao {
         let deleted = 0;
         if (opt.chunkSize) {
             const { chunkSize, chunkConcurrency = 8 } = opt;
-            await Pipeline.from(this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id))
+            await this.cfg.db
+                .streamQuery(q.select(['id']), opt)
+                .mapSync(r => r.id)
                 .chunk(chunkSize)
                 .map(async (ids) => {
                 await this.cfg.db.deleteByIds(q.table, ids, opt);
@@ -656,10 +540,7 @@ export class CommonDao {
             return null;
         // optimization: no need to run full joi DBM validation, cause BM validation will be run
         // const dbm = this.anyToDBM(_dbm, opt)
-        let dbm = { ..._dbm, ...this.cfg.hooks.parseNaturalId(_dbm.id) };
-        if (opt.anonymize) {
-            dbm = this.cfg.hooks.anonymize(dbm);
-        }
+        const dbm = { ..._dbm, ...this.cfg.hooks.parseNaturalId(_dbm.id) };
         // DBM > BM
         const bm = ((await this.cfg.hooks.beforeDBMToBM?.(dbm)) || dbm);
         // Validate/convert BM
@@ -680,17 +561,12 @@ export class CommonDao {
         // try/catch?
         return await pMap(bms, async (bm) => await this.bmToDBM(bm, opt));
     }
-    anyToDBM(dbm, opt = {}) {
+    anyToDBM(dbm, _opt = {}) {
         if (!dbm)
             return null;
         // this shouldn't be happening on load! but should on save!
         // this.assignIdCreatedUpdated(dbm, opt)
         dbm = { ...dbm, ...this.cfg.hooks.parseNaturalId(dbm.id) };
-        // todo: is this the right place?
-        // todo: is anyToDBM even needed?
-        if (opt.anonymize) {
-            dbm = this.cfg.hooks.anonymize(dbm);
-        }
         // Validate/convert DBM
         // return this.validateAndConvert(dbm, this.cfg.dbmSchema, DBModelType.DBM, opt)
         return dbm;
@@ -949,13 +825,20 @@ export class CommonDao {
      * Throws if query uses a property that is in `excludeFromIndexes` list.
      */
     validateQueryIndexes(q) {
-        const { excludeFromIndexes } = this.cfg;
-        if (!excludeFromIndexes)
-            return;
-        for (const f of q._filters) {
-            _assert(!excludeFromIndexes.includes(f.name), `cannot query on non-indexed property: ${this.cfg.table}.${f.name}`, {
-                query: q.pretty(),
-            });
+        const { excludeFromIndexes, indexes } = this.cfg;
+        if (excludeFromIndexes) {
+            for (const f of q._filters) {
+                _assert(!excludeFromIndexes.includes(f.name), `cannot query on non-indexed property: ${this.cfg.table}.${f.name}`, {
+                    query: q.pretty(),
+                });
+            }
+        }
+        if (indexes) {
+            for (const f of q._filters) {
+                _assert(f.name === 'id' || indexes.includes(f.name), `cannot query on non-indexed property: ${this.cfg.table}.${f.name}`, {
+                    query: q.pretty(),
+                });
+            }
         }
     }
 }

package/dist/commondao/common.dao.model.d.ts

@@ -2,7 +2,7 @@ import type { ValidationFunction } from '@naturalcycles/js-lib';
 import type { AppError, ErrorMode } from '@naturalcycles/js-lib/error';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import type { TransformLogProgressOptions } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../commondb/common.db.js';
 import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model.js';
 export interface CommonDaoHooks<BM extends BaseDBEntity, DBM extends BaseDBEntity, ID extends string = BM['id']> {
@@ -54,16 +54,6 @@ export interface CommonDaoHooks<BM extends BaseDBEntity, DBM extends BaseDBEntit
      * Or, you can mutate the DBM if needed.
      */
     beforeSave?: (dbm: DBM) => void;
-    /**
-     * Called in:
-     * - dbmToBM (applied before DBM becomes BM)
-     * - anyToDBM
-     *
-     * Hook only allows to apply anonymization to DBM (not to BM).
-     * It still applies to BM "transitively", during dbmToBM
-     * (e.g after loaded from the Database).
-     */
-    anonymize: (dbm: DBM) => DBM;
     /**
      * If hook is defined - allows to prevent or modify the error thrown.
      * Return `false` to prevent throwing an error.
@@ -230,12 +220,6 @@ export interface CommonDaoOptions extends CommonDBOptions {
      * @default false (for streams). Setting to true enables deletion of immutable objects
      */
     allowMutability?: boolean;
-    /**
-     * If true - data will be anonymized (by calling a BaseDao.anonymize() hook that you can extend in your Dao implementation).
-     * Only applicable to loading/querying/streaming_loading operations (n/a for saving).
-     * There is additional validation applied AFTER Anonymization, so your anonymization implementation should keep the object valid.
-     */
-    anonymize?: boolean;
     /**
      * Allows to override the Table that this Dao is connected to, only in the context of this call.
      *
@@ -288,8 +272,6 @@ export interface CommonDaoStreamDeleteOptions<DBM extends BaseDBEntity> extends
 }
 export interface CommonDaoStreamSaveOptions<DBM extends BaseDBEntity> extends CommonDaoSaveBatchOptions<DBM>, CommonDaoStreamOptions<DBM> {
 }
-export interface CommonDaoStreamForEachOptions<IN> extends CommonDaoStreamOptions<IN>, TransformMapOptions<IN, any> {
-}
 export interface CommonDaoStreamOptions<IN> extends CommonDaoReadOptions, TransformLogProgressOptions<IN> {
     /**
      * @default true (for streams)

package/dist/commondb/base.common.db.d.ts

@@ -1,6 +1,6 @@
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBTransaction, DBTransactionFn, RunQueryResult } from '../db.model.js';
 import type { DBQuery } from '../query/dbQuery.js';
 import type { CommonDB, CommonDBSupport } from './common.db.js';
@@ -23,7 +23,7 @@ export declare class BaseCommonDB implements CommonDB {
     runQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(_q: DBQuery<ROW>): Promise<number>;
     saveBatch<ROW extends ObjectWithId>(_table: string, _rows: ROW[], _opt?: CommonDBSaveOptions<ROW>): Promise<void>;
-    streamQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): Pipeline<ROW>;
     deleteByIds(_table: string, _ids: string[], _opt?: CommonDBOptions): Promise<number>;
     runInTransaction(fn: DBTransactionFn, _opt?: CommonDBTransactionOptions): Promise<void>;
     createTransaction(_opt?: CommonDBTransactionOptions): Promise<DBTransaction>;

package/dist/commondb/common.db.d.ts

@@ -1,6 +1,6 @@
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { NonNegativeInteger, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBCreateOptions, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBStreamOptions, CommonDBTransactionOptions, DBTransaction, DBTransactionFn, RunQueryResult } from '../db.model.js';
 import type { DBQuery } from '../query/dbQuery.js';
 export interface CommonDB {
@@ -62,7 +62,7 @@ export interface CommonDB {
      */
     runQuery: <ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBReadOptions) => Promise<RunQueryResult<ROW>>;
     runQueryCount: <ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBReadOptions) => Promise<NonNegativeInteger>;
-    streamQuery: <ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBStreamOptions) => ReadableTyped<ROW>;
+    streamQuery: <ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBStreamOptions) => Pipeline<ROW>;
     /**
      * rows can have missing ids only if DB supports auto-generating them (like mysql auto_increment).
      */

package/dist/inmemory/inMemory.db.d.ts

@@ -1,7 +1,7 @@
 import { type JsonSchemaObject, type JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import { type AnyObjectWithId, type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream/stream.model.js';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB, CommonDBSupport } from '../commondb/common.db.js';
 import { CommonDBType } from '../commondb/common.db.js';
 import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBOperation, DBTransaction, DBTransactionFn, RunQueryResult } from '../db.model.js';
@@ -60,7 +60,7 @@ export declare class InMemoryDB implements CommonDB {
     patchByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, patch: Partial<ROW>): Promise<number>;
     runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
-    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Pipeline<ROW>;
     runInTransaction(fn: DBTransactionFn, opt?: CommonDBTransactionOptions): Promise<void>;
     createTransaction(opt?: CommonDBTransactionOptions): Promise<DBTransaction>;
     incrementBatch(table: string, prop: string, incrementMap: StringMap<number>, _opt?: CommonDBOptions): Promise<StringMap<number>>;

package/dist/inmemory/inMemory.db.js

@@ -1,9 +1,9 @@
-import { Readable } from 'node:stream';
 import { _isEmptyObject } from '@naturalcycles/js-lib';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { generateJsonSchemaFromData, } from '@naturalcycles/js-lib/json-schema';
 import { _deepCopy, _sortObjectDeep } from '@naturalcycles/js-lib/object';
 import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { bufferReviver } from '@naturalcycles/nodejs-lib/stream/ndjson/transformJsonParse.js';
 import { commonDBFullSupport, CommonDBType } from '../commondb/common.db.js';
 import { queryInMemory } from './queryInMemory.js';
@@ -161,7 +161,7 @@ export class InMemoryDB {
     }
     streamQuery(q, _opt) {
         const table = this.cfg.tablesPrefix + q.table;
-        return Readable.from(queryInMemory(q, Object.values(this.data[table] || {})));
+        return Pipeline.fromArray(queryInMemory(q, Object.values(this.data[table] || {})));
     }
     async runInTransaction(fn, opt = {}) {
         const tx = new InMemoryDBTransaction(this, {

package/dist/inmemory/inMemoryKeyValueDB.d.ts

@@ -1,5 +1,5 @@
 import type { StringMap } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBCreateOptions } from '../db.model.js';
 import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '../kv/commonKeyValueDB.js';
 export interface InMemoryKeyValueDBCfg {
@@ -17,9 +17,9 @@ export declare class InMemoryKeyValueDB implements CommonKeyValueDB {
     deleteByIds(table: string, ids: string[]): Promise<void>;
     getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
     saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
-    streamIds(table: string, limit?: number): ReadableTyped<string>;
-    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
-    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    streamIds(table: string, limit?: number): Pipeline<string>;
+    streamValues(table: string, limit?: number): Pipeline<Buffer>;
+    streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple>;
     count(table: string): Promise<number>;
     incrementBatch(table: string, entries: IncrementTuple[]): Promise<IncrementTuple[]>;
 }

package/dist/inmemory/inMemoryKeyValueDB.js

@@ -1,4 +1,4 @@
-import { Readable } from 'node:stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { commonKeyValueDBFullSupport } from '../kv/commonKeyValueDB.js';
 export class InMemoryKeyValueDB {
     cfg;
@@ -29,13 +29,13 @@ export class InMemoryKeyValueDB {
         }
     }
     streamIds(table, limit) {
-        return Readable.from(Object.keys(this.data[table] || {}).slice(0, limit));
+        return Pipeline.fromArray(Object.keys(this.data[table] || {}).slice(0, limit));
     }
     streamValues(table, limit) {
-        return Readable.from(Object.values(this.data[table] || {}).slice(0, limit));
+        return Pipeline.fromArray(Object.values(this.data[table] || {}).slice(0, limit));
     }
     streamEntries(table, limit) {
-        return Readable.from(Object.entries(this.data[table] || {}).slice(0, limit));
+        return Pipeline.fromArray(Object.entries(this.data[table] || {}).slice(0, limit));
     }
     async count(table) {
         this.data[table] ||= {};

package/dist/kv/commonKeyValueDB.d.ts

@@ -1,5 +1,5 @@
 import type { Integer, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBCreateOptions } from '../db.model.js';
 /**
  * Common interface for Key-Value database implementations.
@@ -28,9 +28,9 @@ export interface CommonKeyValueDB {
     getByIds: (table: string, ids: string[]) => Promise<KeyValueDBTuple[]>;
     deleteByIds: (table: string, ids: string[]) => Promise<void>;
     saveBatch: (table: string, entries: KeyValueDBTuple[], opt?: CommonKeyValueDBSaveBatchOptions) => Promise<void>;
-    streamIds: (table: string, limit?: number) => ReadableTyped<string>;
-    streamValues: (table: string, limit?: number) => ReadableTyped<Buffer>;
-    streamEntries: (table: string, limit?: number) => ReadableTyped<KeyValueDBTuple>;
+    streamIds: (table: string, limit?: number) => Pipeline<string>;
+    streamValues: (table: string, limit?: number) => Pipeline<Buffer>;
+    streamEntries: (table: string, limit?: number) => Pipeline<KeyValueDBTuple>;
     count: (table: string) => Promise<number>;
     /**
      * Perform a batch of Increment operations.

package/dist/kv/commonKeyValueDao.d.ts

@@ -1,6 +1,6 @@
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import type { KeyValueTuple } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { type KeyValueTuple } from '@naturalcycles/js-lib/types';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js';
 import type { CommonDBCreateOptions } from '../db.model.js';
 import type { CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, IncrementTuple, KeyValueDBTuple } from './commonKeyValueDB.js';
@@ -49,9 +49,9 @@ export declare class CommonKeyValueDao<K extends string = string, V = Buffer> {
     saveBatch(entries: KeyValueTuple<K, V>[], opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
     deleteByIds(ids: K[]): Promise<void>;
     deleteById(id: K): Promise<void>;
-    streamIds(limit?: number): ReadableTyped<K>;
-    streamValues(limit?: number): ReadableTyped<V>;
-    streamEntries(limit?: number): ReadableTyped<KeyValueTuple<K, V>>;
+    streamIds(limit?: number): Pipeline<K>;
+    streamValues(limit?: number): Pipeline<V>;
+    streamEntries(limit?: number): Pipeline<KeyValueTuple<K, V>>;
     getAllKeys(limit?: number): Promise<K[]>;
     getAllValues(limit?: number): Promise<V[]>;
     getAllEntries(limit?: number): Promise<KeyValueTuple<K, V>[]>;