@naturalcycles/db-lib 10.26.0 → 10.27.0

package/dist/kv/commonKeyValueDao.js

@@ -1,5 +1,6 @@
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
+import { SKIP } from '@naturalcycles/js-lib/types';
 import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip';
 export const commonKeyValueDaoDeflatedJsonTransformer = {
     valueToBuffer: async (v) => await deflateString(JSON.stringify(v)),
@@ -95,34 +96,30 @@ export class CommonKeyValueDao {
         if (!transformer) {
             return this.cfg.db.streamValues(this.cfg.table, limit);
         }
-        return this.cfg.db.streamValues(this.cfg.table, limit).flatMap(async (buf) => {
+        return this.cfg.db.streamValues(this.cfg.table, limit).map(async (buf) => {
             try {
-                return [await transformer.bufferToValue(buf)];
+                return await transformer.bufferToValue(buf);
             }
             catch (err) {
                 this.cfg.logger.error(err);
-                return []; // SKIP
+                return SKIP;
             }
-        }, {
-            concurrency: 32,
-        });
+        }, { concurrency: 32 });
     }
     streamEntries(limit) {
         const { transformer } = this.cfg;
         if (!transformer) {
             return this.cfg.db.streamEntries(this.cfg.table, limit);
         }
-        return this.cfg.db.streamEntries(this.cfg.table, limit).flatMap(async ([id, buf]) => {
+        return this.cfg.db.streamEntries(this.cfg.table, limit).map(async ([id, buf]) => {
             try {
-                return [[id, await transformer.bufferToValue(buf)]];
+                return [id, await transformer.bufferToValue(buf)];
             }
             catch (err) {
                 this.cfg.logger.error(err);
-                return []; // SKIP
+                return SKIP;
             }
-        }, {
-            concurrency: 32,
-        });
+        }, { concurrency: 32 });
     }
     async getAllKeys(limit) {
         return await this.streamIds(limit).toArray();

package/dist/pipeline/dbPipelineBackup.d.ts

@@ -1,6 +1,6 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error';
 import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../commondb/common.db.js';
 import { DBQuery } from '../query/dbQuery.js';

package/dist/pipeline/dbPipelineBackup.js

@@ -4,7 +4,6 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { Pipeline, } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**
@@ -56,7 +55,8 @@ export async function dbPipelineBackup(opt) {
             await fs2.writeJsonAsync(schemaFilePath, schema, { spaces: 2 });
             console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`);
         }
-        await Pipeline.from(db.streamQuery(q))
+        await db
+            .streamQuery(q)
             .logProgress({
             ...opt,
             logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
@@ -68,7 +68,7 @@ export async function dbPipelineBackup(opt) {
             metric: table,
         })
             .flattenIfNeeded()
-            .tap(() => rows++)
+            .tapSync(() => rows++)
             .toNDJsonFile(filePath);
         const { size: sizeBytes } = await fs2.statAsync(filePath);
         const stats = NDJsonStats.create({

package/dist/pipeline/dbPipelineCopy.d.ts

@@ -1,6 +1,6 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../commondb/common.db.js';
 import type { CommonDBSaveOptions } from '../db.model.js';

package/dist/pipeline/dbPipelineCopy.js

@@ -3,7 +3,6 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
-import { Pipeline, } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**
@@ -27,10 +26,10 @@ export async function dbPipelineCopy(opt) {
         }
         const saveOptions = saveOptionsPerTable[table] || {};
         const mapper = mapperPerTable[table] || _passthroughMapper;
-        const stream = dbInput.streamQuery(q);
         const started = Date.now();
         let rows = 0;
-        await Pipeline.from(stream)
+        await dbInput
+            .streamQuery(q)
             .logProgress({
             logEvery: 1000,
             ...opt,
@@ -42,7 +41,7 @@ export async function dbPipelineCopy(opt) {
             metric: table,
         })
             .flattenIfNeeded()
-            .tap(() => rows++)
+            .tapSync(() => rows++)
             .chunk(chunkSize)
             .forEach(async (dbms) => {
             await dbOutput.saveBatch(table, dbms, saveOptions);

package/dist/pipeline/dbPipelineRestore.js

@@ -69,7 +69,7 @@ export async function dbPipelineRestore(opt) {
         console.log(`<< ${grey(filePath)} ${dimWhite(_hb(sizeBytes))} started...`);
         await Pipeline.fromNDJsonFile(filePath)
             .limitSource(limit)
-            .tap(() => rows++)
+            .tapSync(() => rows++)
             .logProgress({
             logEvery: 1000,
             ...opt,

package/dist/query/dbQuery.d.ts

@@ -1,7 +1,7 @@
-import type { AsyncIndexedMapper, BaseDBEntity, ObjectWithId } from '@naturalcycles/js-lib/types';
-import { Pipeline, type ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { BaseDBEntity, ObjectWithId } from '@naturalcycles/js-lib/types';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDao } from '../commondao/common.dao.js';
-import type { CommonDaoOptions, CommonDaoReadOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions } from '../commondao/common.dao.model.js';
+import type { CommonDaoOptions, CommonDaoReadOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamOptions } from '../commondao/common.dao.model.js';
 import type { RunQueryResult } from '../db.model.js';
 /**
  * Modeled after Firestore operators (WhereFilterOp type)
@@ -100,15 +100,9 @@ export declare class RunnableDBQuery<BM extends BaseDBEntity, DBM extends BaseDB
     runQueryExtendedAsDBM(opt?: CommonDaoReadOptions): Promise<RunQueryResult<DBM>>;
     runQueryCount(opt?: CommonDaoReadOptions): Promise<number>;
     patchByQuery(patch: Partial<DBM>, opt?: CommonDaoOptions): Promise<number>;
-    streamQueryForEach(mapper: AsyncIndexedMapper<BM, void>, opt?: CommonDaoStreamForEachOptions<BM>): Promise<void>;
-    streamQueryAsDBMForEach(mapper: AsyncIndexedMapper<DBM, void>, opt?: CommonDaoStreamForEachOptions<DBM>): Promise<void>;
-    streamQuery(opt?: CommonDaoStreamOptions<BM>): ReadableTyped<BM>;
-    streamQueryAsDBM(opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM>;
-    pipeline(opt?: CommonDaoStreamOptions<BM>): Pipeline<BM>;
-    pipelineAsDBM(opt?: CommonDaoStreamOptions<DBM>): Pipeline<DBM>;
+    streamQuery(opt?: CommonDaoStreamOptions<BM>): Pipeline<BM>;
+    streamQueryAsDBM(opt?: CommonDaoStreamOptions<DBM>): Pipeline<DBM>;
     queryIds(opt?: CommonDaoReadOptions): Promise<ID[]>;
-    streamQueryIds(opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID>;
-    pipelineIds(opt?: CommonDaoStreamOptions<ID>): Pipeline<ID>;
-    streamQueryIdsForEach(mapper: AsyncIndexedMapper<ID, void>, opt?: CommonDaoStreamForEachOptions<ID>): Promise<void>;
+    streamQueryIds(opt?: CommonDaoStreamOptions<ID>): Pipeline<ID>;
     deleteByQuery(opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
 }

package/dist/query/dbQuery.js

@@ -1,6 +1,5 @@
 import { _truncate } from '@naturalcycles/js-lib/string/string.util.js';
 import { _objectAssign } from '@naturalcycles/js-lib/types';
-import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 export const dbQueryFilterOperatorValues = [
     '<',
     '<=',
@@ -176,36 +175,18 @@ export class RunnableDBQuery extends DBQuery {
     async patchByQuery(patch, opt) {
         return await this.dao.patchByQuery(this, patch, opt);
     }
-    async streamQueryForEach(mapper, opt) {
-        await this.dao.streamQueryForEach(this, mapper, opt);
-    }
-    async streamQueryAsDBMForEach(mapper, opt) {
-        await this.dao.streamQueryAsDBMForEach(this, mapper, opt);
-    }
     streamQuery(opt) {
         return this.dao.streamQuery(this, opt);
     }
     streamQueryAsDBM(opt) {
         return this.dao.streamQueryAsDBM(this, opt);
     }
-    pipeline(opt) {
-        return Pipeline.from(this.dao.streamQuery(this, opt));
-    }
-    pipelineAsDBM(opt) {
-        return Pipeline.from(this.dao.streamQueryAsDBM(this, opt));
-    }
     async queryIds(opt) {
         return await this.dao.queryIds(this, opt);
     }
     streamQueryIds(opt) {
         return this.dao.streamQueryIds(this, opt);
     }
-    pipelineIds(opt) {
-        return Pipeline.from(this.dao.streamQueryIds(this, opt));
-    }
-    async streamQueryIdsForEach(mapper, opt) {
-        await this.dao.streamQueryIdsForEach(this, mapper, opt);
-    }
     async deleteByQuery(opt) {
         return await this.dao.deleteByQuery(this, opt);
     }

package/dist/testing/commonDaoTest.js

@@ -179,7 +179,10 @@ export async function runCommonDaoTest(db, quirks = {}) {
     if (support.streaming) {
         test('streamQueryForEach all', async () => {
             let rows = [];
-            await dao.query().streamQueryForEach(bm => void rows.push(bm));
+            await dao
+                .query()
+                .streamQuery()
+                .forEachSync(bm => void rows.push(bm));
             rows = _sortBy(rows, r => r.id);
             expectMatch(expectedItems, rows, quirks);
         });
@@ -190,7 +193,10 @@ export async function runCommonDaoTest(db, quirks = {}) {
         });
         test('streamQueryIdsForEach all', async () => {
             let ids = [];
-            await dao.query().streamQueryIdsForEach(id => void ids.push(id));
+            await dao
+                .query()
+                .streamQueryIds()
+                .forEachSync(id => void ids.push(id));
             ids = ids.sort();
             expectMatch(expectedItems.map(i => i.id), ids, quirks);
         });

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.26.0",
+  "version": "10.27.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "18.4.2"
+    "@naturalcycles/dev-lib": "19.37.0"
   },
   "files": [
     "dist",

package/src/adapter/cachedb/cache.db.ts

@@ -1,8 +1,7 @@
-import { Readable } from 'node:stream'
 import { _isTruthy } from '@naturalcycles/js-lib'
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
 import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import { BaseCommonDB } from '../../commondb/base.common.db.js'
 import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js'
 import { commonDBFullSupport } from '../../commondb/common.db.js'
@@ -201,9 +200,9 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
   override streamQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     opt: CacheDBStreamOptions = {},
-  ): ReadableTyped<ROW> {
+  ): Pipeline<ROW> {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
-      const stream = this.cfg.downstreamDB.streamQuery<ROW>(q, opt)
+      const pipeline = this.cfg.downstreamDB.streamQuery<ROW>(q, opt)
 
       // Don't save to cache if it was a projection query
       if (!opt.skipCache && !this.cfg.skipCache && !q._selectedFieldNames) {
@@ -216,12 +215,12 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
         //   })
       }
 
-      return stream
+      return pipeline
     }
 
-    if (opt.skipCache || this.cfg.skipCache) return Readable.from([])
+    if (opt.skipCache || this.cfg.skipCache) return Pipeline.fromArray([])
 
-    const stream = this.cfg.cacheDB.streamQuery<ROW>(q, opt)
+    const pipeline = this.cfg.cacheDB.streamQuery<ROW>(q, opt)
 
     // if (this.cfg.logCached) {
     //   let count = 0
@@ -234,7 +233,7 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
     //     })
     // }
 
-    return stream
+    return pipeline
   }
 
   override async deleteByQuery<ROW extends ObjectWithId>(

package/src/adapter/file/file.db.ts

@@ -1,3 +1,4 @@
+import { Readable } from 'node:stream'
 import { _by, _sortBy } from '@naturalcycles/js-lib/array'
 import { _since, localTime } from '@naturalcycles/js-lib/datetime'
 import { _assert } from '@naturalcycles/js-lib/error/assert.js'
@@ -10,8 +11,7 @@ import {
   type UnixTimestampMillis,
 } from '@naturalcycles/js-lib/types'
 import { dimGrey } from '@naturalcycles/nodejs-lib/colors'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
-import { readableCreate } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import { BaseCommonDB } from '../../commondb/base.common.db.js'
 import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js'
 import { commonDBFullSupport } from '../../commondb/common.db.js'
@@ -127,15 +127,11 @@ export class FileDB extends BaseCommonDB implements CommonDB {
   override streamQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     opt?: CommonDBStreamOptions,
-  ): ReadableTyped<ROW> {
-    const readable = readableCreate<ROW>()
-
-    void this.runQuery(q, opt).then(({ rows }) => {
-      rows.forEach(r => readable.push(r))
-      readable.push(null) // done
+  ): Pipeline<ROW> {
+    return Pipeline.fromAsyncReadable(async () => {
+      const { rows } = await this.runQuery(q, opt)
+      return Readable.from(rows)
     })
-
-    return readable
   }
 
   override async deleteByQuery<ROW extends ObjectWithId>(

package/src/commondao/common.dao.model.ts

@@ -2,10 +2,7 @@ import type { ValidationFunction } from '@naturalcycles/js-lib'
 import type { AppError, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import type { BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
-import type {
-  TransformLogProgressOptions,
-  TransformMapOptions,
-} from '@naturalcycles/nodejs-lib/stream'
+import type { TransformLogProgressOptions } from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDB } from '../commondb/common.db.js'
 import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model.js'
 
@@ -70,17 +67,6 @@ export interface CommonDaoHooks<
    */
   beforeSave?: (dbm: DBM) => void
 
-  /**
-   * Called in:
-   * - dbmToBM (applied before DBM becomes BM)
-   * - anyToDBM
-   *
-   * Hook only allows to apply anonymization to DBM (not to BM).
-   * It still applies to BM "transitively", during dbmToBM
-   * (e.g after loaded from the Database).
-   */
-  anonymize: (dbm: DBM) => DBM
-
   /**
    * If hook is defined - allows to prevent or modify the error thrown.
    * Return `false` to prevent throwing an error.
@@ -280,13 +266,6 @@ export interface CommonDaoOptions extends CommonDBOptions {
    */
   allowMutability?: boolean
 
-  /**
-   * If true - data will be anonymized (by calling a BaseDao.anonymize() hook that you can extend in your Dao implementation).
-   * Only applicable to loading/querying/streaming_loading operations (n/a for saving).
-   * There is additional validation applied AFTER Anonymization, so your anonymization implementation should keep the object valid.
-   */
-  anonymize?: boolean
-
   /**
    * Allows to override the Table that this Dao is connected to, only in the context of this call.
    *
@@ -363,10 +342,6 @@ export interface CommonDaoStreamSaveOptions<DBM extends BaseDBEntity>
   extends CommonDaoSaveBatchOptions<DBM>,
     CommonDaoStreamOptions<DBM> {}
 
-export interface CommonDaoStreamForEachOptions<IN>
-  extends CommonDaoStreamOptions<IN>,
-    TransformMapOptions<IN, any> {}
-
 export interface CommonDaoStreamOptions<IN>
   extends CommonDaoReadOptions,
     TransformLogProgressOptions<IN> {