@naturalcycles/db-lib 9.9.1 → 9.10.0

package/dist/commondao/common.dao.js

@@ -361,6 +361,17 @@ class CommonDao {
         const partialQuery = !!q._selectedFieldNames;
         if (partialQuery)
             return stream;
+        // This almost works, but hard to implement `errorMode: THROW_AGGREGATED` in this case
+        // return stream.flatMap(async (dbm: DBM) => {
+        //   if (this.cfg.hooks!.afterLoad) {
+        //     dbm = (await this.cfg.hooks!.afterLoad(dbm))!
+        //     if (dbm === null) return [] // SKIP
+        //   }
+        //
+        //   return [await this.dbmToBM(dbm, opt)] satisfies BM[]
+        // }, {
+        //   concurrency: 16,
+        // })
         return (stream
             // optimization: 1 validation is enough
             // .pipe(transformMap<any, DBM>(dbm => this.anyToDBM(dbm, opt), safeOpt))
@@ -376,10 +387,11 @@ class CommonDao {
         }, {
             errorMode: opt.errorMode,
         }))
-        // this can make the stream async-iteration-friendly
-        // but not applying it now for perf reasons
-        // .pipe(transformNoOp())
-        );
+            // this can make the stream async-iteration-friendly
+            // but not applying it now for perf reasons
+            // UPD: applying, to be compliant with `.toArray()`, etc.
+            .on('error', err => stream.emit('error', err))
+            .pipe((0, nodejs_lib_1.transformNoOp)()));
     }
     async queryIds(q, opt = {}) {
         q.table = opt.table || q.table;
@@ -389,12 +401,19 @@ class CommonDao {
     streamQueryIds(q, opt = {}) {
         q.table = opt.table || q.table;
         opt.errorMode ||= js_lib_1.ErrorMode.SUPPRESS;
+        // Experimental: using `.map()`
         const stream = this.cfg.db
             .streamQuery(q.select(['id']), opt)
             .on('error', err => stream.emit('error', err))
-            .pipe((0, nodejs_lib_1.transformMapSimple)(r => r.id, {
-            errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }));
+            .map((r) => r.id);
+        // const stream: ReadableTyped<string> = this.cfg.db
+        //   .streamQuery<DBM>(q.select(['id']), opt)
+        //   .on('error', err => stream.emit('error', err))
+        //   .pipe(
+        //     transformMapSimple<DBM, string>(r => r.id, {
+        //       errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+        //     }),
+        //   )
         return stream;
     }
     async streamQueryIdsForEach(q, mapper, opt = {}) {

package/dist/kv/commonKeyValueDao.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { KeyValueTuple } from '@naturalcycles/js-lib';
+import { CommonLogger, KeyValueTuple } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonDaoLogLevel } from '../commondao/common.dao.model';
 import { CommonDBCreateOptions } from '../db.model';
@@ -12,6 +12,10 @@ export interface CommonKeyValueDaoCfg<T> {
      * Set to true to limit DB writing (will throw an error is such case).
      */
     readOnly?: boolean;
+    /**
+     * Default to console
+     */
+    logger?: CommonLogger;
     /**
      * @default OPERATIONS
      */
@@ -33,8 +37,11 @@ export interface CommonKeyValueDaoCfg<T> {
     deflatedJsonValue?: boolean;
 }
 export declare class CommonKeyValueDao<T> {
-    cfg: CommonKeyValueDaoCfg<T>;
     constructor(cfg: CommonKeyValueDaoCfg<T>);
+    cfg: CommonKeyValueDaoCfg<T> & {
+        hooks: NonNullable<CommonKeyValueDaoCfg<T>['hooks']>;
+        logger: CommonLogger;
+    };
     ping(): Promise<void>;
     createTable(opt?: CommonDBCreateOptions): Promise<void>;
     create(input?: Partial<T>): T;

package/dist/kv/commonKeyValueDao.js

@@ -7,9 +7,13 @@ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 // todo: readonly
 class CommonKeyValueDao {
     constructor(cfg) {
-        this.cfg = cfg;
+        this.cfg = {
+            hooks: {},
+            logger: console,
+            ...cfg,
+        };
         if (cfg.deflatedJsonValue) {
-            cfg.hooks = {
+            this.cfg.hooks = {
                 mapValueToBuffer: async (v) => await (0, nodejs_lib_1.deflateString)(JSON.stringify(v)),
                 mapBufferToValue: async (buf) => JSON.parse(await (0, nodejs_lib_1.inflateToString)(buf)),
                 ...cfg.hooks,
@@ -24,7 +28,7 @@ class CommonKeyValueDao {
     }
     create(input = {}) {
         return {
-            ...this.cfg.hooks?.beforeCreate?.(input),
+            ...this.cfg.hooks.beforeCreate?.(input),
         };
     }
     async getById(id) {
@@ -66,7 +70,7 @@ class CommonKeyValueDao {
         if (r)
             return r[1];
         return {
-            ...this.cfg.hooks?.beforeCreate?.({}),
+            ...this.cfg.hooks.beforeCreate?.({}),
             ...part,
         };
     }
@@ -80,7 +84,7 @@ class CommonKeyValueDao {
     }
     async getByIds(ids) {
         const entries = await this.cfg.db.getByIds(this.cfg.table, ids);
-        if (!this.cfg.hooks?.mapBufferToValue)
+        if (!this.cfg.hooks.mapBufferToValue)
             return entries;
         return await (0, js_lib_1.pMap)(entries, async ([id, buf]) => [
             id,
@@ -97,8 +101,8 @@ class CommonKeyValueDao {
         await this.cfg.db.saveBatch(this.cfg.table, [[id, value]]);
     }
     async saveBatch(entries) {
-        let bufferEntries = [];
-        if (!this.cfg.hooks?.mapValueToBuffer) {
+        let bufferEntries;
+        if (!this.cfg.hooks.mapValueToBuffer) {
             bufferEntries = entries;
         }
         else {
@@ -122,29 +126,41 @@ class CommonKeyValueDao {
         return this.cfg.db.streamIds(this.cfg.table, limit);
     }
     streamValues(limit) {
-        if (!this.cfg.hooks?.mapBufferToValue) {
+        const { mapBufferToValue } = this.cfg.hooks;
+        if (!mapBufferToValue) {
             return this.cfg.db.streamValues(this.cfg.table, limit);
         }
-        // todo: consider it when readableMap supports `errorMode: SUPPRESS`
-        // readableMap(this.cfg.db.streamValues(this.cfg.table, limit), async buf => await this.cfg.hooks!.mapBufferToValue(buf))
         const stream = this.cfg.db
             .streamValues(this.cfg.table, limit)
             .on('error', err => stream.emit('error', err))
-            .pipe((0, nodejs_lib_1.transformMap)(async (buf) => await this.cfg.hooks.mapBufferToValue(buf), {
-            errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe cannot propagate errors
-        }));
+            .flatMap(async (buf) => {
+            try {
+                return [await mapBufferToValue(buf)];
+            }
+            catch (err) {
+                this.cfg.logger.error(err);
+                return []; // SKIP
+            }
+        });
         return stream;
     }
     streamEntries(limit) {
-        if (!this.cfg.hooks?.mapBufferToValue) {
+        const { mapBufferToValue } = this.cfg.hooks;
+        if (!mapBufferToValue) {
             return this.cfg.db.streamEntries(this.cfg.table, limit);
         }
         const stream = this.cfg.db
             .streamEntries(this.cfg.table, limit)
             .on('error', err => stream.emit('error', err))
-            .pipe((0, nodejs_lib_1.transformMap)(async ([id, buf]) => [id, await this.cfg.hooks.mapBufferToValue(buf)], {
-            errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe cannot propagate errors
-        }));
+            .flatMap(async ([id, buf]) => {
+            try {
+                return [[id, await mapBufferToValue(buf)]];
+            }
+            catch (err) {
+                this.cfg.logger.error(err);
+                return []; // SKIP
+            }
+        });
         return stream;
     }
 }

package/dist/kv/commonKeyValueDaoMemoCache.d.ts

@@ -1,4 +1,4 @@
-import { AsyncMemoCache } from '@naturalcycles/js-lib';
+import { AsyncMemoCache, MISS } from '@naturalcycles/js-lib';
 import { CommonKeyValueDao } from './commonKeyValueDao';
 /**
  * AsyncMemoCache implementation, backed by CommonKeyValueDao.
@@ -11,7 +11,7 @@ import { CommonKeyValueDao } from './commonKeyValueDao';
 export declare class CommonKeyValueDaoMemoCache<VALUE = any> implements AsyncMemoCache<string, VALUE> {
     private dao;
     constructor(dao: CommonKeyValueDao<VALUE>);
-    get(k: string): Promise<VALUE | Error | undefined>;
-    set(k: string, v: VALUE | Error): Promise<void>;
+    get(k: string): Promise<VALUE | typeof MISS>;
+    set(k: string, v: VALUE): Promise<void>;
     clear(): Promise<void>;
 }

package/dist/kv/commonKeyValueDaoMemoCache.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CommonKeyValueDaoMemoCache = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
 /**
  * AsyncMemoCache implementation, backed by CommonKeyValueDao.
  *
@@ -14,13 +15,9 @@ class CommonKeyValueDaoMemoCache {
         this.dao = dao;
     }
     async get(k) {
-        return (await this.dao.getById(k)) || undefined;
+        return (await this.dao.getById(k)) || js_lib_1.MISS;
     }
     async set(k, v) {
-        if (v instanceof Error) {
-            // We currently don't persist errors there
-            return;
-        }
         await this.dao.save(k, v);
     }
     async clear() {

package/dist/testing/daoTest.js

@@ -144,9 +144,7 @@ function runCommonDaoTest(db, quirks = {}) {
             (0, dbTest_1.expectMatch)(expectedItems, rows, quirks);
         });
         test('streamQuery all', async () => {
-            // let rows = await readableToArray(dao.query().streamQuery())
-            // todo: remove transformNoOp after `transformMap` learns to be async-iteration-friendly
-            let rows = await (0, nodejs_lib_1.readableToArray)(dao.query().streamQuery().pipe((0, nodejs_lib_1.transformNoOp)()));
+            let rows = await dao.query().streamQuery().toArray();
             rows = (0, js_lib_1._sortBy)(rows, r => r.id);
             (0, dbTest_1.expectMatch)(expectedItems, rows, quirks);
         });

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "9.9.1",
+  "version": "9.10.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/readme.md

@@ -150,7 +150,7 @@ Returns `ReadableTyped` (typed wrapper of Node.js
 
 Streams in Node.js support back-pressure by default (if piped properly by the consumer).
 
-```typescript
+```ts
 const q = DBQuery.create('table1') // "return all items" query
 
 await _pipeline([
@@ -165,6 +165,14 @@ await _pipeline([
 // ...
 ```
 
+Alternative:
+
+```ts
+await db.streamQuery(q).forEach(item => {
+  console.log(item)
+})
+```
+
 ###### saveBatch
 
 `saveBatch<DBM>(table: string, dbms: DBM[]): Promise<void>`

package/src/commondao/common.dao.ts

@@ -40,6 +40,7 @@ import {
   transformLogProgress,
   transformMap,
   transformMapSimple,
+  transformNoOp,
   writableVoid,
 } from '@naturalcycles/nodejs-lib'
 import { DBLibError } from '../cnst'
@@ -496,6 +497,18 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const partialQuery = !!q._selectedFieldNames
     if (partialQuery) return stream
 
+    // This almost works, but hard to implement `errorMode: THROW_AGGREGATED` in this case
+    // return stream.flatMap(async (dbm: DBM) => {
+    //   if (this.cfg.hooks!.afterLoad) {
+    //     dbm = (await this.cfg.hooks!.afterLoad(dbm))!
+    //     if (dbm === null) return [] // SKIP
+    //   }
+    //
+    //   return [await this.dbmToBM(dbm, opt)] satisfies BM[]
+    // }, {
+    //   concurrency: 16,
+    // })
+
     return (
       stream
         // optimization: 1 validation is enough
@@ -517,9 +530,11 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
             },
           ),
         )
-      // this can make the stream async-iteration-friendly
-      // but not applying it now for perf reasons
-      // .pipe(transformNoOp())
+        // this can make the stream async-iteration-friendly
+        // but not applying it now for perf reasons
+        // UPD: applying, to be compliant with `.toArray()`, etc.
+        .on('error', err => stream.emit('error', err))
+        .pipe(transformNoOp())
     )
   }
 
@@ -533,14 +548,20 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     q.table = opt.table || q.table
     opt.errorMode ||= ErrorMode.SUPPRESS
 
+    // Experimental: using `.map()`
     const stream: ReadableTyped<string> = this.cfg.db
       .streamQuery<DBM>(q.select(['id']), opt)
       .on('error', err => stream.emit('error', err))
-      .pipe(
-        transformMapSimple<DBM, string>(r => r.id, {
-          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }),
-      )
+      .map((r: ObjectWithId) => r.id)
+
+    // const stream: ReadableTyped<string> = this.cfg.db
+    //   .streamQuery<DBM>(q.select(['id']), opt)
+    //   .on('error', err => stream.emit('error', err))
+    //   .pipe(
+    //     transformMapSimple<DBM, string>(r => r.id, {
+    //       errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
+    //     }),
+    //   )
 
     return stream
   }

package/src/kv/commonKeyValueDao.ts

@@ -1,10 +1,5 @@
-import { AppError, ErrorMode, KeyValueTuple, pMap } from '@naturalcycles/js-lib'
-import {
-  deflateString,
-  inflateToString,
-  ReadableTyped,
-  transformMap,
-} from '@naturalcycles/nodejs-lib'
+import { AppError, CommonLogger, KeyValueTuple, pMap } from '@naturalcycles/js-lib'
+import { deflateString, inflateToString, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel } from '../commondao/common.dao.model'
 import { CommonDBCreateOptions } from '../db.model'
 import { CommonKeyValueDB, KeyValueDBTuple } from './commonKeyValueDB'
@@ -20,6 +15,11 @@ export interface CommonKeyValueDaoCfg<T> {
    */
   readOnly?: boolean
 
+  /**
+   * Default to console
+   */
+  logger?: CommonLogger
+
   /**
    * @default OPERATIONS
    */
@@ -48,9 +48,15 @@ export interface CommonKeyValueDaoCfg<T> {
 // todo: readonly
 
 export class CommonKeyValueDao<T> {
-  constructor(public cfg: CommonKeyValueDaoCfg<T>) {
+  constructor(cfg: CommonKeyValueDaoCfg<T>) {
+    this.cfg = {
+      hooks: {},
+      logger: console,
+      ...cfg,
+    }
+
     if (cfg.deflatedJsonValue) {
-      cfg.hooks = {
+      this.cfg.hooks = {
         mapValueToBuffer: async v => await deflateString(JSON.stringify(v)),
         mapBufferToValue: async buf => JSON.parse(await inflateToString(buf)),
         ...cfg.hooks,
@@ -58,6 +64,11 @@ export class CommonKeyValueDao<T> {
     }
   }
 
+  cfg: CommonKeyValueDaoCfg<T> & {
+    hooks: NonNullable<CommonKeyValueDaoCfg<T>['hooks']>
+    logger: CommonLogger
+  }
+
   async ping(): Promise<void> {
     await this.cfg.db.ping()
   }
@@ -68,7 +79,7 @@ export class CommonKeyValueDao<T> {
 
   create(input: Partial<T> = {}): T {
     return {
-      ...this.cfg.hooks?.beforeCreate?.(input),
+      ...this.cfg.hooks.beforeCreate?.(input),
     } as T
   }
 
@@ -117,7 +128,7 @@ export class CommonKeyValueDao<T> {
     if (r) return r[1]
 
     return {
-      ...this.cfg.hooks?.beforeCreate?.({}),
+      ...this.cfg.hooks.beforeCreate?.({}),
       ...part,
     } as T
   }
@@ -135,11 +146,11 @@ export class CommonKeyValueDao<T> {
 
   async getByIds(ids: string[]): Promise<KeyValueTuple<string, T>[]> {
     const entries = await this.cfg.db.getByIds(this.cfg.table, ids)
-    if (!this.cfg.hooks?.mapBufferToValue) return entries as any
+    if (!this.cfg.hooks.mapBufferToValue) return entries as any
 
     return await pMap(entries, async ([id, buf]) => [
       id,
-      await this.cfg.hooks!.mapBufferToValue!(buf),
+      await this.cfg.hooks.mapBufferToValue!(buf),
     ])
   }
 
@@ -156,14 +167,14 @@ export class CommonKeyValueDao<T> {
   }
 
   async saveBatch(entries: KeyValueTuple<string, T>[]): Promise<void> {
-    let bufferEntries: KeyValueDBTuple[] = []
+    let bufferEntries: KeyValueDBTuple[]
 
-    if (!this.cfg.hooks?.mapValueToBuffer) {
+    if (!this.cfg.hooks.mapValueToBuffer) {
       bufferEntries = entries as any
     } else {
       bufferEntries = await pMap(entries, async ([id, v]) => [
         id,
-        await this.cfg.hooks!.mapValueToBuffer!(v),
+        await this.cfg.hooks.mapValueToBuffer!(v),
       ])
     }
 
@@ -187,37 +198,45 @@ export class CommonKeyValueDao<T> {
   }
 
   streamValues(limit?: number): ReadableTyped<T> {
-    if (!this.cfg.hooks?.mapBufferToValue) {
+    const { mapBufferToValue } = this.cfg.hooks
+
+    if (!mapBufferToValue) {
       return this.cfg.db.streamValues(this.cfg.table, limit)
     }
 
-    // todo: consider it when readableMap supports `errorMode: SUPPRESS`
-    // readableMap(this.cfg.db.streamValues(this.cfg.table, limit), async buf => await this.cfg.hooks!.mapBufferToValue(buf))
     const stream: ReadableTyped<T> = this.cfg.db
       .streamValues(this.cfg.table, limit)
       .on('error', err => stream.emit('error', err))
-      .pipe(
-        transformMap(async buf => await this.cfg.hooks!.mapBufferToValue!(buf), {
-          errorMode: ErrorMode.SUPPRESS, // cause .pipe cannot propagate errors
-        }),
-      )
+      .flatMap(async (buf: Buffer) => {
+        try {
+          return [await mapBufferToValue(buf)] satisfies T[]
+        } catch (err) {
+          this.cfg.logger.error(err)
+          return [] // SKIP
+        }
+      })
 
     return stream
   }
 
   streamEntries(limit?: number): ReadableTyped<KeyValueTuple<string, T>> {
-    if (!this.cfg.hooks?.mapBufferToValue) {
+    const { mapBufferToValue } = this.cfg.hooks
+
+    if (!mapBufferToValue) {
       return this.cfg.db.streamEntries(this.cfg.table, limit)
     }
 
     const stream: ReadableTyped<KeyValueTuple<string, T>> = this.cfg.db
       .streamEntries(this.cfg.table, limit)
       .on('error', err => stream.emit('error', err))
-      .pipe(
-        transformMap(async ([id, buf]) => [id, await this.cfg.hooks!.mapBufferToValue!(buf)], {
-          errorMode: ErrorMode.SUPPRESS, // cause .pipe cannot propagate errors
-        }),
-      )
+      .flatMap(async ([id, buf]: KeyValueTuple<string, Buffer>) => {
+        try {
+          return [[id, await mapBufferToValue(buf)]] satisfies KeyValueTuple<string, T>[]
+        } catch (err) {
+          this.cfg.logger.error(err)
+          return [] // SKIP
+        }
+      })
 
     return stream
   }

package/src/kv/commonKeyValueDaoMemoCache.ts

@@ -1,4 +1,4 @@
-import { AsyncMemoCache } from '@naturalcycles/js-lib'
+import { AsyncMemoCache, MISS } from '@naturalcycles/js-lib'
 import { CommonKeyValueDao } from './commonKeyValueDao'
 
 /**
@@ -12,16 +12,11 @@ import { CommonKeyValueDao } from './commonKeyValueDao'
 export class CommonKeyValueDaoMemoCache<VALUE = any> implements AsyncMemoCache<string, VALUE> {
   constructor(private dao: CommonKeyValueDao<VALUE>) {}
 
-  async get(k: string): Promise<VALUE | Error | undefined> {
-    return (await this.dao.getById(k)) || undefined
+  async get(k: string): Promise<VALUE | typeof MISS> {
+    return (await this.dao.getById(k)) || MISS
   }
 
-  async set(k: string, v: VALUE | Error): Promise<void> {
-    if (v instanceof Error) {
-      // We currently don't persist errors there
-      return
-    }
-
+  async set(k: string, v: VALUE): Promise<void> {
     await this.dao.save(k, v)
   }
 

package/src/testing/daoTest.ts

@@ -1,6 +1,6 @@
 import { Readable } from 'node:stream'
 import { _deepCopy, _pick, _sortBy, _omit, localTimeNow } from '@naturalcycles/js-lib'
-import { _pipeline, readableToArray, transformNoOp } from '@naturalcycles/nodejs-lib'
+import { _pipeline, readableToArray } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel, DBQuery } from '..'
 import { CommonDB } from '../common.db'
 import { CommonDao } from '../commondao/common.dao'
@@ -194,11 +194,7 @@ export function runCommonDaoTest(db: CommonDB, quirks: CommonDBImplementationQui
     })
 
     test('streamQuery all', async () => {
-      // let rows = await readableToArray(dao.query().streamQuery())
-      // todo: remove transformNoOp after `transformMap` learns to be async-iteration-friendly
-      let rows: TestItemBM[] = await readableToArray(
-        dao.query().streamQuery().pipe(transformNoOp()),
-      )
+      let rows: TestItemBM[] = await dao.query().streamQuery().toArray()
 
       rows = _sortBy(rows, r => r.id)
       expectMatch(expectedItems, rows, quirks)