@naturalcycles/db-lib 10.26.1 → 10.27.0

package/src/commondao/common.dao.ts

@@ -13,7 +13,6 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import {
   _stringMapEntries,
   _stringMapValues,
-  type AsyncIndexedMapper,
   type BaseDBEntity,
   type NonNegativeInteger,
   type ObjectWithId,
@@ -22,17 +21,11 @@ import {
 } from '@naturalcycles/js-lib/types'
 import { _passthroughPredicate, _typeCast } from '@naturalcycles/js-lib/types'
 import { stringId } from '@naturalcycles/nodejs-lib'
-import {
-  Pipeline,
-  type ReadableTyped,
-  transformFlatten,
-  transformMapSync,
-} from '@naturalcycles/nodejs-lib/stream'
+import { type Pipeline, transformFlatten } from '@naturalcycles/nodejs-lib/stream'
 import {
   transformChunk,
   transformLogProgress,
   transformMap,
-  transformNoOp,
 } from '@naturalcycles/nodejs-lib/stream'
 import { DBLibError } from '../cnst.js'
 import type {
@@ -53,7 +46,6 @@ import type {
   CommonDaoSaveBatchOptions,
   CommonDaoSaveOptions,
   CommonDaoStreamDeleteOptions,
-  CommonDaoStreamForEachOptions,
   CommonDaoStreamOptions,
   CommonDaoStreamSaveOptions,
 } from './common.dao.model.js'
@@ -84,7 +76,6 @@ export class CommonDao<
       hooks: {
         parseNaturalId: () => ({}),
         beforeCreate: bm => bm as BM,
-        anonymize: dbm => dbm,
         onValidationError: err => err,
         ...cfg.hooks,
       } satisfies Partial<CommonDaoHooks<BM, DBM, ID>>,
@@ -240,153 +231,34 @@ export class CommonDao<
     return await this.cfg.db.runQueryCount(q, opt)
   }
 
-  async streamQueryForEach(
-    q: DBQuery<DBM>,
-    mapper: AsyncIndexedMapper<BM, void>,
-    opt: CommonDaoStreamForEachOptions<BM> = {},
-  ): Promise<void> {
-    this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
-    q.table = opt.table || q.table
-    opt.skipValidation = opt.skipValidation !== false // default true
-    opt.errorMode ||= ErrorMode.SUPPRESS
+  streamQueryAsDBM(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<DBM> = {}): Pipeline<DBM> {
+    const pipeline = this.streamQueryRaw(q, opt)
 
     const isPartialQuery = !!q._selectedFieldNames
+    if (isPartialQuery) return pipeline
 
-    await Pipeline.from(this.cfg.db.streamQuery<DBM>(q, opt))
-      .map(
-        async dbm => {
-          if (isPartialQuery) return dbm as any
-          return await this.dbmToBM(dbm, opt)
-        },
-        {
-          errorMode: opt.errorMode,
-        },
-      )
-      .map(mapper, {
-        ...opt,
-        predicate: _passthroughPredicate, // to be able to logProgress
-      })
-      // LogProgress should be AFTER the mapper, to be able to report correct stats
-      .logProgress({
-        metric: q.table,
-        ...opt,
-      })
-      .run()
-  }
-
-  async streamQueryAsDBMForEach(
-    q: DBQuery<DBM>,
-    mapper: AsyncIndexedMapper<DBM, void>,
-    opt: CommonDaoStreamForEachOptions<DBM> = {},
-  ): Promise<void> {
-    this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
-    q.table = opt.table || q.table
-    opt.skipValidation = opt.skipValidation !== false // default true
+    opt.skipValidation ??= true
     opt.errorMode ||= ErrorMode.SUPPRESS
 
-    const isPartialQuery = !!q._selectedFieldNames
-
-    await Pipeline.from(this.cfg.db.streamQuery<any>(q, opt))
-      .mapSync(
-        dbm => {
-          if (isPartialQuery) return dbm
-          return this.anyToDBM(dbm, opt)
-        },
-        {
-          errorMode: opt.errorMode,
-        },
-      )
-      .map(mapper, {
-        ...opt,
-        predicate: _passthroughPredicate, // to be able to logProgress
-      })
-      .logProgress({
-        metric: q.table,
-        ...opt,
-      })
-      .run()
+    return pipeline.mapSync(dbm => this.anyToDBM(dbm, opt), { errorMode: opt.errorMode })
   }
 
-  /**
-   * Stream as Readable, to be able to .pipe() it further with support of backpressure.
-   */
-  streamQueryAsDBM(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<DBM> = {}): ReadableTyped<DBM> {
-    this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
-    q.table = opt.table || q.table
-    opt.skipValidation = opt.skipValidation !== false // default true
-    opt.errorMode ||= ErrorMode.SUPPRESS
+  streamQuery(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<BM> = {}): Pipeline<BM> {
+    const pipeline = this.streamQueryRaw(q, opt)
 
     const isPartialQuery = !!q._selectedFieldNames
+    if (isPartialQuery) return pipeline as any as Pipeline<BM>
 
-    const stream = this.cfg.db.streamQuery<DBM>(q, opt)
-    if (isPartialQuery) return stream
-
-    return (
-      stream
-        // the commented out line was causing RangeError: Maximum call stack size exceeded
-        // .on('error', err => stream.emit('error', err))
-        .pipe(
-          transformMapSync<any, DBM>(
-            dbm => {
-              return this.anyToDBM(dbm, opt)
-            },
-            {
-              errorMode: opt.errorMode,
-            },
-          ),
-        )
-    )
+    opt.skipValidation ??= true
+    opt.errorMode ||= ErrorMode.SUPPRESS
+
+    return pipeline.map(async dbm => await this.dbmToBM(dbm, opt), { errorMode: opt.errorMode })
   }
 
-  /**
-   * Stream as Readable, to be able to .pipe() it further with support of backpressure.
-   *
-   * Please note that this stream is currently not async-iteration friendly, because of
-   * `through2` usage.
-   * Will be migrated/fixed at some point in the future.
-   *
-   * You can do `.pipe(transformNoOp)` to make it "valid again".
-   */
-  streamQuery(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<BM> = {}): ReadableTyped<BM> {
+  private streamQueryRaw(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<any> = {}): Pipeline<DBM> {
     this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
     q.table = opt.table || q.table
-    opt.skipValidation = opt.skipValidation !== false // default true
-    opt.errorMode ||= ErrorMode.SUPPRESS
-
-    const stream = this.cfg.db.streamQuery<DBM>(q, opt)
-    const isPartialQuery = !!q._selectedFieldNames
-    if (isPartialQuery) return stream as any
-
-    // This almost works, but hard to implement `errorMode: THROW_AGGREGATED` in this case
-    // return stream.flatMap(async (dbm: DBM) => {
-    //   return [await this.dbmToBM(dbm, opt)] satisfies BM[]
-    // }, {
-    //   concurrency: 16,
-    // })
-
-    return (
-      stream
-        // optimization: 1 validation is enough
-        // .pipe(transformMap<any, DBM>(dbm => this.anyToDBM(dbm, opt), safeOpt))
-        // .pipe(transformMap<DBM, BM>(dbm => this.dbmToBM(dbm, opt), safeOpt))
-        // the commented out line was causing RangeError: Maximum call stack size exceeded
-        // .on('error', err => stream.emit('error', err))
-        .pipe(
-          transformMap<DBM, BM>(
-            async dbm => {
-              return await this.dbmToBM(dbm, opt)
-            },
-            {
-              errorMode: opt.errorMode,
-            },
-          ),
-        )
-        // this can make the stream async-iteration-friendly
-        // but not applying it now for perf reasons
-        // UPD: applying, to be compliant with `.toArray()`, etc.
-        .on('error', err => stream.destroy(err))
-        .pipe(transformNoOp())
-    )
+    return this.cfg.db.streamQuery<DBM>(q, opt)
   }
 
   async queryIds(q: DBQuery<DBM>, opt: CommonDaoReadOptions = {}): Promise<ID[]> {
@@ -396,49 +268,12 @@ export class CommonDao<
     return rows.map(r => r.id as ID)
   }
 
-  streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<ID> = {}): ReadableTyped<ID> {
+  streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<ID> = {}): Pipeline<ID> {
     this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
     q.table = opt.table || q.table
     opt.errorMode ||= ErrorMode.SUPPRESS
 
-    // Experimental: using `.map()`
-    const stream: ReadableTyped<ID> = this.cfg.db
-      .streamQuery<DBM>(q.select(['id']), opt)
-      // .on('error', err => stream.emit('error', err))
-      .map((r: ObjectWithId) => r.id as ID)
-
-    // const stream: ReadableTyped<string> = this.cfg.db
-    //   .streamQuery<DBM>(q.select(['id']), opt)
-    //   .on('error', err => stream.emit('error', err))
-    //   .pipe(
-    //     transformMapSimple<DBM, string>(r => r.id, {
-    //       errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-    //     }),
-    //   )
-
-    return stream
-  }
-
-  async streamQueryIdsForEach(
-    q: DBQuery<DBM>,
-    mapper: AsyncIndexedMapper<ID, void>,
-    opt: CommonDaoStreamForEachOptions<ID> = {},
-  ): Promise<void> {
-    this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
-    q.table = opt.table || q.table
-    opt.errorMode ||= ErrorMode.SUPPRESS
-
-    await Pipeline.from(this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => r.id as ID))
-      .map(mapper, {
-        ...opt,
-        predicate: _passthroughPredicate,
-      })
-      // LogProgress should be AFTER the mapper, to be able to report correct stats
-      .logProgress({
-        metric: q.table,
-        ...opt,
-      })
-      .run()
+    return this.cfg.db.streamQuery(q.select(['id']), opt).mapSync((r: ObjectWithId) => r.id as ID)
   }
 
   /**
@@ -791,7 +626,9 @@ export class CommonDao<
     if (opt.chunkSize) {
       const { chunkSize, chunkConcurrency = 8 } = opt
 
-      await Pipeline.from(this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => r.id))
+      await this.cfg.db
+        .streamQuery<DBM>(q.select(['id']), opt)
+        .mapSync(r => r.id)
         .chunk(chunkSize)
         .map(
           async ids => {
@@ -876,11 +713,7 @@ export class CommonDao<
 
     // optimization: no need to run full joi DBM validation, cause BM validation will be run
     // const dbm = this.anyToDBM(_dbm, opt)
-    let dbm: DBM = { ..._dbm, ...this.cfg.hooks!.parseNaturalId!(_dbm.id as ID) }
-
-    if (opt.anonymize) {
-      dbm = this.cfg.hooks!.anonymize!(dbm)
-    }
+    const dbm: DBM = { ..._dbm, ...this.cfg.hooks!.parseNaturalId!(_dbm.id as ID) }
 
     // DBM > BM
     const bm = ((await this.cfg.hooks!.beforeDBMToBM?.(dbm)) || dbm) as Partial<BM>
@@ -916,7 +749,7 @@ export class CommonDao<
 
   anyToDBM(dbm: undefined, opt?: CommonDaoOptions): null
   anyToDBM(dbm?: any, opt?: CommonDaoOptions): DBM
-  anyToDBM(dbm?: DBM, opt: CommonDaoOptions = {}): DBM | null {
+  anyToDBM(dbm?: DBM, _opt: CommonDaoOptions = {}): DBM | null {
     if (!dbm) return null
 
     // this shouldn't be happening on load! but should on save!
@@ -924,12 +757,6 @@ export class CommonDao<
 
     dbm = { ...dbm, ...this.cfg.hooks!.parseNaturalId!(dbm.id as ID) }
 
-    // todo: is this the right place?
-    // todo: is anyToDBM even needed?
-    if (opt.anonymize) {
-      dbm = this.cfg.hooks!.anonymize!(dbm)
-    }
-
     // Validate/convert DBM
     // return this.validateAndConvert(dbm, this.cfg.dbmSchema, DBModelType.DBM, opt)
     return dbm
@@ -1253,17 +1080,30 @@ export class CommonDao<
    * Throws if query uses a property that is in `excludeFromIndexes` list.
    */
   private validateQueryIndexes(q: DBQuery<DBM>): void {
-    const { excludeFromIndexes } = this.cfg
-    if (!excludeFromIndexes) return
+    const { excludeFromIndexes, indexes } = this.cfg
 
-    for (const f of q._filters) {
-      _assert(
-        !excludeFromIndexes.includes(f.name),
-        `cannot query on non-indexed property: ${this.cfg.table}.${f.name as string}`,
-        {
-          query: q.pretty(),
-        },
-      )
+    if (excludeFromIndexes) {
+      for (const f of q._filters) {
+        _assert(
+          !excludeFromIndexes.includes(f.name),
+          `cannot query on non-indexed property: ${this.cfg.table}.${f.name as string}`,
+          {
+            query: q.pretty(),
+          },
+        )
+      }
+    }
+
+    if (indexes) {
+      for (const f of q._filters) {
+        _assert(
+          f.name === 'id' || indexes.includes(f.name),
+          `cannot query on non-indexed property: ${this.cfg.table}.${f.name as string}`,
+          {
+            query: q.pretty(),
+          },
+        )
+      }
     }
   }
 }

package/src/commondb/base.common.db.ts

@@ -1,6 +1,6 @@
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
 import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type {
   CommonDBOptions,
   CommonDBReadOptions,
@@ -86,7 +86,7 @@ export class BaseCommonDB implements CommonDB {
     throw new Error('saveBatch is not implemented')
   }
 
-  streamQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): ReadableTyped<ROW> {
+  streamQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): Pipeline<ROW> {
     throw new Error('streamQuery is not implemented')
   }
 

package/src/commondb/common.db.ts

@@ -1,6 +1,6 @@
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
 import type { NonNegativeInteger, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type {
   CommonDBCreateOptions,
   CommonDBOptions,
@@ -105,7 +105,7 @@ export interface CommonDB {
   streamQuery: <ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     opt?: CommonDBStreamOptions,
-  ) => ReadableTyped<ROW>
+  ) => Pipeline<ROW>
 
   // SAVE
   /**

package/src/inmemory/inMemory.db.ts

@@ -1,4 +1,3 @@
-import { Readable } from 'node:stream'
 import { _isEmptyObject } from '@naturalcycles/js-lib'
 import { _assert } from '@naturalcycles/js-lib/error/assert.js'
 import {
@@ -15,8 +14,8 @@ import {
   type ObjectWithId,
   type StringMap,
 } from '@naturalcycles/js-lib/types'
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import { bufferReviver } from '@naturalcycles/nodejs-lib/stream/ndjson/transformJsonParse.js'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream/stream.model.js'
 import type { CommonDB, CommonDBSupport } from '../commondb/common.db.js'
 import { commonDBFullSupport, CommonDBType } from '../commondb/common.db.js'
 import type {
@@ -278,12 +277,9 @@ export class InMemoryDB implements CommonDB {
     return queryInMemory<any>(q, Object.values(this.data[table] || {})).length
   }
 
-  streamQuery<ROW extends ObjectWithId>(
-    q: DBQuery<ROW>,
-    _opt?: CommonDBOptions,
-  ): ReadableTyped<ROW> {
+  streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Pipeline<ROW> {
     const table = this.cfg.tablesPrefix + q.table
-    return Readable.from(queryInMemory(q, Object.values(this.data[table] || {}) as ROW[]))
+    return Pipeline.fromArray(queryInMemory(q, Object.values(this.data[table] || {}) as ROW[]))
   }
 
   async runInTransaction(fn: DBTransactionFn, opt: CommonDBTransactionOptions = {}): Promise<void> {

package/src/inmemory/inMemoryKeyValueDB.ts

@@ -1,6 +1,5 @@
-import { Readable } from 'node:stream'
 import type { StringMap } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDBCreateOptions } from '../db.model.js'
 import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '../kv/commonKeyValueDB.js'
 import { commonKeyValueDBFullSupport } from '../kv/commonKeyValueDB.js'
@@ -40,16 +39,16 @@ export class InMemoryKeyValueDB implements CommonKeyValueDB {
     }
   }
 
-  streamIds(table: string, limit?: number): ReadableTyped<string> {
-    return Readable.from(Object.keys(this.data[table] || {}).slice(0, limit))
+  streamIds(table: string, limit?: number): Pipeline<string> {
+    return Pipeline.fromArray(Object.keys(this.data[table] || {}).slice(0, limit))
   }
 
-  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
-    return Readable.from(Object.values(this.data[table] || {}).slice(0, limit))
+  streamValues(table: string, limit?: number): Pipeline<Buffer> {
+    return Pipeline.fromArray(Object.values(this.data[table] || {}).slice(0, limit))
   }
 
-  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
-    return Readable.from(Object.entries(this.data[table] || {}).slice(0, limit))
+  streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple> {
+    return Pipeline.fromArray(Object.entries(this.data[table] || {}).slice(0, limit))
   }
 
   async count(table: string): Promise<number> {

package/src/kv/commonKeyValueDB.ts

@@ -1,5 +1,5 @@
 import type { Integer, UnixTimestamp } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDBCreateOptions } from '../db.model.js'
 
 /**
@@ -39,9 +39,9 @@ export interface CommonKeyValueDB {
     opt?: CommonKeyValueDBSaveBatchOptions,
   ) => Promise<void>
 
-  streamIds: (table: string, limit?: number) => ReadableTyped<string>
-  streamValues: (table: string, limit?: number) => ReadableTyped<Buffer>
-  streamEntries: (table: string, limit?: number) => ReadableTyped<KeyValueDBTuple>
+  streamIds: (table: string, limit?: number) => Pipeline<string>
+  streamValues: (table: string, limit?: number) => Pipeline<Buffer>
+  streamEntries: (table: string, limit?: number) => Pipeline<KeyValueDBTuple>
 
   count: (table: string) => Promise<number>
 

package/src/kv/commonKeyValueDao.ts

@@ -1,8 +1,8 @@
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
-import type { KeyValueTuple } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { type KeyValueTuple, SKIP } from '@naturalcycles/js-lib/types'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip'
 import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js'
 import type { CommonDBCreateOptions } from '../db.model.js'
@@ -159,51 +159,47 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
     await this.cfg.db.deleteByIds(this.cfg.table, [id])
   }
 
-  streamIds(limit?: number): ReadableTyped<K> {
-    return this.cfg.db.streamIds(this.cfg.table, limit) as ReadableTyped<K>
+  streamIds(limit?: number): Pipeline<K> {
+    return this.cfg.db.streamIds(this.cfg.table, limit) as Pipeline<K>
   }
 
-  streamValues(limit?: number): ReadableTyped<V> {
+  streamValues(limit?: number): Pipeline<V> {
     const { transformer } = this.cfg
 
     if (!transformer) {
-      return this.cfg.db.streamValues(this.cfg.table, limit) as ReadableTyped<V>
+      return this.cfg.db.streamValues(this.cfg.table, limit) as Pipeline<V>
     }
 
-    return this.cfg.db.streamValues(this.cfg.table, limit).flatMap(
+    return this.cfg.db.streamValues(this.cfg.table, limit).map(
       async buf => {
         try {
-          return [await transformer.bufferToValue(buf)]
+          return await transformer.bufferToValue(buf)
         } catch (err) {
           this.cfg.logger.error(err)
-          return [] // SKIP
+          return SKIP
         }
       },
-      {
-        concurrency: 32,
-      },
+      { concurrency: 32 },
     )
   }
 
-  streamEntries(limit?: number): ReadableTyped<KeyValueTuple<K, V>> {
+  streamEntries(limit?: number): Pipeline<KeyValueTuple<K, V>> {
     const { transformer } = this.cfg
 
     if (!transformer) {
-      return this.cfg.db.streamEntries(this.cfg.table, limit) as ReadableTyped<KeyValueTuple<K, V>>
+      return this.cfg.db.streamEntries(this.cfg.table, limit) as Pipeline<KeyValueTuple<K, V>>
     }
 
-    return this.cfg.db.streamEntries(this.cfg.table, limit).flatMap(
+    return this.cfg.db.streamEntries(this.cfg.table, limit).map(
       async ([id, buf]) => {
         try {
-          return [[id as K, await transformer.bufferToValue(buf)]]
+          return [id as K, await transformer.bufferToValue(buf)]
         } catch (err) {
           this.cfg.logger.error(err)
-          return [] // SKIP
+          return SKIP
         }
       },
-      {
-        concurrency: 32,
-      },
+      { concurrency: 32 },
     )
   }
 

package/src/pipeline/dbPipelineBackup.ts

@@ -5,10 +5,9 @@ import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-li
 import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import {
-  Pipeline,
-  type TransformLogProgressOptions,
-  type TransformMapOptions,
+import type {
+  TransformLogProgressOptions,
+  TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDB } from '../commondb/common.db.js'
@@ -205,7 +204,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
       }
 
-      await Pipeline.from(db.streamQuery(q))
+      await db
+        .streamQuery(q)
         .logProgress({
           ...opt,
           logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
@@ -217,7 +217,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
           metric: table,
         })
         .flattenIfNeeded()
-        .tap(() => rows++)
+        .tapSync(() => rows++)
         .toNDJsonFile(filePath)
 
       const { size: sizeBytes } = await fs2.statAsync(filePath)

package/src/pipeline/dbPipelineCopy.ts

@@ -4,10 +4,9 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
-import {
-  Pipeline,
-  type TransformLogProgressOptions,
-  type TransformMapOptions,
+import type {
+  TransformLogProgressOptions,
+  TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDB } from '../commondb/common.db.js'
@@ -123,12 +122,11 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
       const saveOptions: CommonDBSaveOptions<any> = saveOptionsPerTable[table] || {}
       const mapper = mapperPerTable[table] || _passthroughMapper
 
-      const stream = dbInput.streamQuery(q)
-
       const started = Date.now()
       let rows = 0
 
-      await Pipeline.from(stream)
+      await dbInput
+        .streamQuery(q)
         .logProgress({
           logEvery: 1000,
           ...opt,
@@ -140,7 +138,7 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
           metric: table,
         })
         .flattenIfNeeded()
-        .tap(() => rows++)
+        .tapSync(() => rows++)
         .chunk(chunkSize)
         .forEach(async dbms => {
           await dbOutput.saveBatch(table, dbms, saveOptions)

package/src/pipeline/dbPipelineRestore.ts

@@ -187,7 +187,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
 
       await Pipeline.fromNDJsonFile<BaseDBEntity>(filePath)
         .limitSource(limit)
-        .tap(() => rows++)
+        .tapSync(() => rows++)
         .logProgress({
           logEvery: 1000,
           ...opt,