@naturalcycles/db-lib 10.15.0 → 10.16.0

package/dist/commondao/common.dao.d.ts

@@ -2,7 +2,7 @@ import type { Transform } from 'node:stream';
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { AsyncIndexedMapper, BaseDBEntity, StringMap, UnixTimestampMillis, Unsaved } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { type ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBTransactionOptions, DBTransaction, RunQueryResult } from '../db.model.js';
 import type { DBQuery } from '../query/dbQuery.js';
 import { RunnableDBQuery } from '../query/dbQuery.js';

package/dist/commondao/common.dao.js

@@ -9,6 +9,7 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _truncate } from '@naturalcycles/js-lib/string/string.util.js';
 import { _passthroughPredicate, _typeCast, SKIP } from '@naturalcycles/js-lib/types';
 import { stringId } from '@naturalcycles/nodejs-lib';
+import { transformFlatten } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, transformChunk, transformLogProgress, transformMap, transformNoOp, writableVoid, } from '@naturalcycles/nodejs-lib/stream';
 import { DBLibError } from '../cnst.js';
 import { RunnableDBQuery } from '../query/dbQuery.js';
@@ -346,8 +347,9 @@ export class CommonDao {
         const stream = this.cfg.db.streamQuery(q, opt);
         if (partialQuery)
             return stream;
-        return stream
-            .on('error', err => stream.emit('error', err))
+        return (stream
+            // the commented out line was causing RangeError: Maximum call stack size exceeded
+            // .on('error', err => stream.emit('error', err))
             .pipe(transformMap(async (dbm) => {
             if (this.cfg.hooks.afterLoad) {
                 dbm = (await this.cfg.hooks.afterLoad(dbm));
@@ -357,7 +359,7 @@ export class CommonDao {
             return this.anyToDBM(dbm, opt);
         }, {
             errorMode: opt.errorMode,
-        }));
+        })));
     }
     /**
      * Stream as Readable, to be able to .pipe() it further with support of backpressure.
@@ -392,7 +394,8 @@ export class CommonDao {
             // optimization: 1 validation is enough
             // .pipe(transformMap<any, DBM>(dbm => this.anyToDBM(dbm, opt), safeOpt))
             // .pipe(transformMap<DBM, BM>(dbm => this.dbmToBM(dbm, opt), safeOpt))
-            .on('error', err => stream.emit('error', err))
+            // the commented out line was causing RangeError: Maximum call stack size exceeded
+            // .on('error', err => stream.emit('error', err))
             .pipe(transformMap(async (dbm) => {
             if (this.cfg.hooks.afterLoad) {
                 dbm = (await this.cfg.hooks.afterLoad(dbm));
@@ -471,7 +474,7 @@ export class CommonDao {
             obj.created ||= obj.updated || now;
         }
         if (this.cfg.useUpdatedProperty) {
-            obj.updated = opt.preserveUpdatedCreated && obj.updated ? obj.updated : now;
+            obj.updated = opt.preserveUpdated && obj.updated ? obj.updated : now;
         }
         if (this.cfg.generateId) {
             obj.id ||= (this.cfg.hooks.createNaturalId?.(obj) ||
@@ -764,8 +767,8 @@ export class CommonDao {
             }, {
                 concurrency: chunkConcurrency,
                 errorMode,
-                flattenArrayOutput: true,
             }),
+            transformFlatten(),
             transformLogProgress({
                 metric: 'saved',
                 ...opt,

package/dist/commondao/common.dao.model.d.ts

@@ -202,9 +202,12 @@ export interface CommonDaoOptions extends CommonDBOptions {
      */
     mutateInput?: boolean;
     /**
-     * @default false
+     * Defaults to false.
+     *
+     * If false (default) - will set `updated` property to the current timestamp.
+     * If true - will NOT set it (preserve the original value).
      */
-    preserveUpdatedCreated?: boolean;
+    preserveUpdated?: boolean;
     /**
      * @default false (for streams). Setting to true enables deletion of immutable objects
      */

package/dist/pipeline/dbPipelineBackup.js

@@ -4,7 +4,7 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
+import { createWriteStreamAsNDJSON, transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformLogProgress, transformMap, transformTap, } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**
@@ -65,10 +65,10 @@ export async function dbPipelineBackup(opt) {
             }),
             transformMap(mapperPerTable[table] || _passthroughMapper, {
                 errorMode,
-                flattenArrayOutput: true,
                 ...transformMapOptions,
                 metric: table,
             }),
+            transformFlatten(),
             transformTap(() => {
                 rows++;
             }),

package/dist/pipeline/dbPipelineCopy.d.ts

@@ -1,6 +1,6 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../commondb/common.db.js';
 import type { CommonDBSaveOptions } from '../db.model.js';

package/dist/pipeline/dbPipelineCopy.js

@@ -3,6 +3,7 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
+import { transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformChunk, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
 import { DBQuery } from '../query/dbQuery.js';
 /**
@@ -38,10 +39,10 @@ export async function dbPipelineCopy(opt) {
             }),
             transformMap(mapper, {
                 errorMode,
-                flattenArrayOutput: true,
                 ...transformMapOptions,
                 metric: table,
             }),
+            transformFlatten(),
             transformTap(() => rows++),
             transformChunk({ chunkSize }),
             writableForEach(async (dbms) => {

package/dist/pipeline/dbPipelineRestore.js

@@ -6,7 +6,7 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { createReadStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
+import { createReadStreamAsNDJSON, transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
 import { _pipeline, NDJsonStats, transformChunk, transformFilterSync, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
 /**
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
@@ -80,10 +80,10 @@ export async function dbPipelineRestore(opt) {
                 : []),
             transformMap(mapperPerTable[table] || _passthroughMapper, {
                 errorMode,
-                flattenArrayOutput: true,
                 ...transformMapOptions,
                 metric: table,
             }),
+            transformFlatten(),
             transformChunk({ chunkSize }),
             writableForEach(async (dbms) => {
                 await db.saveBatch(table, dbms, saveOptions);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.15.0",
+  "version": "10.16.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"

package/src/commondao/common.dao.model.ts

@@ -244,9 +244,12 @@ export interface CommonDaoOptions extends CommonDBOptions {
   mutateInput?: boolean
 
   /**
-   * @default false
+   * Defaults to false.
+   *
+   * If false (default) - will set `updated` property to the current timestamp.
+   * If true - will NOT set it (preserve the original value).
    */
-  preserveUpdatedCreated?: boolean
+  preserveUpdated?: boolean
 
   /**
    * @default false (for streams). Setting to true enables deletion of immutable objects

package/src/commondao/common.dao.ts

@@ -24,7 +24,7 @@ import type {
 } from '@naturalcycles/js-lib/types'
 import { _passthroughPredicate, _typeCast, SKIP } from '@naturalcycles/js-lib/types'
 import { stringId } from '@naturalcycles/nodejs-lib'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { type ReadableTyped, transformFlatten } from '@naturalcycles/nodejs-lib/stream'
 import {
   _pipeline,
   transformChunk,
@@ -457,23 +457,26 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
     const stream = this.cfg.db.streamQuery<DBM>(q, opt)
     if (partialQuery) return stream
 
-    return stream
-      .on('error', err => stream.emit('error', err))
-      .pipe(
-        transformMap<any, DBM>(
-          async dbm => {
-            if (this.cfg.hooks!.afterLoad) {
-              dbm = (await this.cfg.hooks!.afterLoad(dbm))!
-              if (dbm === null) return SKIP
-            }
-
-            return this.anyToDBM(dbm, opt)
-          },
-          {
-            errorMode: opt.errorMode,
-          },
-        ),
-      )
+    return (
+      stream
+        // the commented out line was causing RangeError: Maximum call stack size exceeded
+        // .on('error', err => stream.emit('error', err))
+        .pipe(
+          transformMap<any, DBM>(
+            async dbm => {
+              if (this.cfg.hooks!.afterLoad) {
+                dbm = (await this.cfg.hooks!.afterLoad(dbm))!
+                if (dbm === null) return SKIP
+              }
+
+              return this.anyToDBM(dbm, opt)
+            },
+            {
+              errorMode: opt.errorMode,
+            },
+          ),
+        )
+    )
   }
 
   /**
@@ -512,7 +515,8 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
         // optimization: 1 validation is enough
         // .pipe(transformMap<any, DBM>(dbm => this.anyToDBM(dbm, opt), safeOpt))
         // .pipe(transformMap<DBM, BM>(dbm => this.dbmToBM(dbm, opt), safeOpt))
-        .on('error', err => stream.emit('error', err))
+        // the commented out line was causing RangeError: Maximum call stack size exceeded
+        // .on('error', err => stream.emit('error', err))
         .pipe(
           transformMap<DBM, BM>(
             async dbm => {
@@ -613,7 +617,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
     }
 
     if (this.cfg.useUpdatedProperty) {
-      obj.updated = opt.preserveUpdatedCreated && obj.updated ? obj.updated : now
+      obj.updated = opt.preserveUpdated && obj.updated ? obj.updated : now
     }
 
     if (this.cfg.generateId) {
@@ -979,9 +983,9 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, I
         {
           concurrency: chunkConcurrency,
           errorMode,
-          flattenArrayOutput: true,
         },
       ),
+      transformFlatten(),
       transformLogProgress({
         metric: 'saved',
         ...opt,

package/src/pipeline/dbPipelineBackup.ts

@@ -7,6 +7,7 @@ import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/col
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
 import {
   createWriteStreamAsNDJSON,
+  transformFlatten,
   type TransformLogProgressOptions,
   type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
@@ -220,10 +221,10 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         }),
         transformMap(mapperPerTable[table] || _passthroughMapper, {
           errorMode,
-          flattenArrayOutput: true,
           ...transformMapOptions,
           metric: table,
         }),
+        transformFlatten(),
         transformTap(() => {
           rows++
         }),

package/src/pipeline/dbPipelineCopy.ts

@@ -4,9 +4,10 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
 import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
-import type {
-  TransformLogProgressOptions,
-  TransformMapOptions,
+import {
+  transformFlatten,
+  type TransformLogProgressOptions,
+  type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
 import {
   _pipeline,
@@ -144,10 +145,10 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
         }),
         transformMap(mapper, {
           errorMode,
-          flattenArrayOutput: true,
           ...transformMapOptions,
           metric: table,
         }),
+        transformFlatten(),
         transformTap(() => rows++),
         transformChunk({ chunkSize }),
         writableForEach(async dbms => {

package/src/pipeline/dbPipelineRestore.ts

@@ -10,6 +10,7 @@ import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/col
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
 import {
   createReadStreamAsNDJSON,
+  transformFlatten,
   type TransformLogProgressOptions,
   type TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
@@ -207,10 +208,10 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
           : []),
         transformMap(mapperPerTable[table] || _passthroughMapper, {
           errorMode,
-          flattenArrayOutput: true,
           ...transformMapOptions,
           metric: table,
         }),
+        transformFlatten(),
         transformChunk({ chunkSize }),
         writableForEach(async dbms => {
           await db.saveBatch(table, dbms, saveOptions)