@naturalcycles/db-lib 9.12.0 → 9.12.1

package/dist/commondao/common.dao.d.ts

@@ -112,9 +112,9 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     saveBatchAsDBM(dbms: Unsaved<DBM>[], opt?: CommonDaoSaveBatchOptions<DBM>): Promise<DBM[]>;
     /**
      * "Streaming" is implemented by buffering incoming rows into **batches**
-     * (of size opt.batchSize, which defaults to 500),
-     * and then executing db.saveBatch(batch) with the concurrency
-     * of opt.batchConcurrency (which defaults to 16).
+     * (of size opt.chunkSize, which defaults to 500),
+     * and then executing db.saveBatch(chunk) with the concurrency
+     * of opt.chunkConcurrency (which defaults to 16).
      */
     streamSaveTransform(opt?: CommonDaoStreamSaveOptions<DBM>): Transform[];
     /**
@@ -123,9 +123,9 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     deleteById(id?: string | null, opt?: CommonDaoOptions): Promise<number>;
     deleteByIds(ids: string[], opt?: CommonDaoOptions): Promise<number>;
     /**
-     * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
-     * `deleteByIds` for each batch concurrently (infinite concurrency).
-     * This is expected to be more memory-efficient way of deleting big numbers of rows.
+     * Pass `chunkSize: number` (e.g 500) option to use Streaming: it will Stream the query, chunk by 500, and execute
+     * `deleteByIds` for each chunk concurrently (infinite concurrency).
+     * This is expected to be more memory-efficient way of deleting large number of rows.
      */
     deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
     updateById(id: string, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;

package/dist/commondao/common.dao.js

@@ -697,9 +697,9 @@ class CommonDao {
     }
     /**
      * "Streaming" is implemented by buffering incoming rows into **batches**
-     * (of size opt.batchSize, which defaults to 500),
-     * and then executing db.saveBatch(batch) with the concurrency
-     * of opt.batchConcurrency (which defaults to 16).
+     * (of size opt.chunkSize, which defaults to 500),
+     * and then executing db.saveBatch(chunk) with the concurrency
+     * of opt.chunkConcurrency (which defaults to 16).
      */
     streamSaveTransform(opt = {}) {
         this.requireWriteAccess();
@@ -711,7 +711,7 @@ class CommonDao {
         }
         const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes;
         const { beforeSave } = this.cfg.hooks;
-        const { batchSize = 500, batchConcurrency = 16, errorMode } = opt;
+        const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt;
         return [
             (0, nodejs_lib_1.transformMap)(async (bm) => {
                 this.assignIdCreatedUpdated(bm, opt); // mutates
@@ -725,7 +725,7 @@ class CommonDao {
             }, {
                 errorMode,
             }),
-            (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+            (0, nodejs_lib_1.transformChunk)({ chunkSize }),
             (0, nodejs_lib_1.transformMap)(async (batch) => {
                 await this.cfg.db.saveBatch(table, batch, {
                     ...opt,
@@ -733,7 +733,7 @@ class CommonDao {
                 });
                 return batch;
             }, {
-                concurrency: batchConcurrency,
+                concurrency: chunkConcurrency,
                 errorMode,
                 flattenArrayOutput: true,
             }),
@@ -768,9 +768,9 @@ class CommonDao {
         return count;
     }
     /**
-     * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
-     * `deleteByIds` for each batch concurrently (infinite concurrency).
-     * This is expected to be more memory-efficient way of deleting big numbers of rows.
+     * Pass `chunkSize: number` (e.g 500) option to use Streaming: it will Stream the query, chunk by 500, and execute
+     * `deleteByIds` for each chunk concurrently (infinite concurrency).
+     * This is expected to be more memory-efficient way of deleting large number of rows.
      */
     async deleteByQuery(q, opt = {}) {
         this.requireWriteAccess();
@@ -779,25 +779,25 @@ class CommonDao {
         const op = `deleteByQuery(${q.pretty()})`;
         const started = this.logStarted(op, q.table);
         let deleted = 0;
-        if (opt.batchSize) {
-            const { batchSize, batchConcurrency = 16 } = opt;
+        if (opt.chunkSize) {
+            const { chunkSize, chunkConcurrency = 16 } = opt;
             await (0, nodejs_lib_1._pipeline)([
                 this.cfg.db.streamQuery(q.select(['id']), opt),
                 (0, nodejs_lib_1.transformMapSimple)(r => r.id, {
                     errorMode: js_lib_1.ErrorMode.SUPPRESS,
                 }),
-                (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+                (0, nodejs_lib_1.transformChunk)({ chunkSize }),
                 (0, nodejs_lib_1.transformMap)(async (ids) => {
                     deleted += await this.cfg.db.deleteByQuery(dbQuery_1.DBQuery.create(q.table).filterIn('id', ids), opt);
                 }, {
                     predicate: js_lib_1._passthroughPredicate,
-                    concurrency: batchConcurrency,
+                    concurrency: chunkConcurrency,
                 }),
                 // LogProgress should be AFTER the mapper, to be able to report correct stats
                 (0, nodejs_lib_1.transformLogProgress)({
                     metric: q.table,
                     logEvery: 2, // 500 * 2 === 1000
-                    batchSize,
+                    chunkSize,
                     ...opt,
                 }),
                 (0, nodejs_lib_1.writableVoid)(),

package/dist/commondao/common.dao.model.d.ts

@@ -250,15 +250,15 @@ export interface CommonDaoStreamOptions<IN> extends CommonDaoOptions, TransformL
     errorMode?: ErrorMode;
     /**
      * Applicable to some of stream operations, e.g deleteByQuery.
-     * If set - `deleteByQuery` won't execute it "all at once", but in batches.
+     * If set - `deleteByQuery` won't execute it "all at once", but in batches (chunks).
      *
      * Defaults to undefined, so the operation is executed "all at once".
      */
-    batchSize?: number;
+    chunkSize?: number;
     /**
-     * When batchSize is set - this option controls how many batches to run concurrently.
+     * When chunkSize is set - this option controls how many chunks to run concurrently.
      * Defaults to 16, "the magic number of JavaScript concurrency".
      */
-    batchConcurrency?: number;
+    chunkConcurrency?: number;
 }
 export type CommonDaoCreateOptions = CommonDBCreateOptions;

package/dist/pipeline/dbPipelineCopy.d.ts

@@ -21,7 +21,7 @@ export interface DBPipelineCopyOptions extends TransformLogProgressOptions {
      *
      * Determines the size of .saveBatch()
      */
-    batchSize?: number;
+    chunkSize?: number;
     /**
      * @default ErrorMode.SUPPRESS
      *

package/dist/pipeline/dbPipelineCopy.js

@@ -11,7 +11,7 @@ const dbQuery_1 = require("../query/dbQuery");
  * Handles backpressure.
  */
 async function dbPipelineCopy(opt) {
-    const { batchSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
+    const { chunkSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
     let { tables } = opt;
     const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
     console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
@@ -42,7 +42,7 @@ async function dbPipelineCopy(opt) {
                 metric: table,
             }),
             (0, nodejs_lib_1.transformTap)(() => rows++),
-            (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+            (0, nodejs_lib_1.transformChunk)({ chunkSize }),
             (0, nodejs_lib_1.writableForEach)(async (dbms) => {
                 await dbOutput.saveBatch(table, dbms, saveOptions);
             }),

package/dist/pipeline/dbPipelineRestore.js

@@ -14,7 +14,7 @@ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
  * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
  */
 async function dbPipelineRestore(opt) {
-    const { db, concurrency = 16, batchSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
+    const { db, concurrency = 16, chunkSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
     const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
     const onlyTables = opt.tables && new Set(opt.tables);
     const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
@@ -87,7 +87,7 @@ async function dbPipelineRestore(opt) {
                 ...transformMapOptions,
                 metric: table,
             }),
-            (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+            (0, nodejs_lib_1.transformChunk)({ chunkSize }),
             (0, nodejs_lib_1.writableForEach)(async (dbms) => {
                 await db.saveBatch(table, dbms, saveOptions);
             }),

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "9.12.0",
+  "version": "9.12.1",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/commondao/common.dao.model.ts

@@ -313,17 +313,17 @@ export interface CommonDaoStreamOptions<IN>
 
   /**
    * Applicable to some of stream operations, e.g deleteByQuery.
-   * If set - `deleteByQuery` won't execute it "all at once", but in batches.
+   * If set - `deleteByQuery` won't execute it "all at once", but in batches (chunks).
    *
    * Defaults to undefined, so the operation is executed "all at once".
    */
-  batchSize?: number
+  chunkSize?: number
 
   /**
-   * When batchSize is set - this option controls how many batches to run concurrently.
+   * When chunkSize is set - this option controls how many chunks to run concurrently.
    * Defaults to 16, "the magic number of JavaScript concurrency".
    */
-  batchConcurrency?: number
+  chunkConcurrency?: number
 }
 
 export type CommonDaoCreateOptions = CommonDBCreateOptions

package/src/commondao/common.dao.ts

@@ -37,7 +37,7 @@ import {
   ObjectSchema,
   ReadableTyped,
   stringId,
-  transformBuffer,
+  transformChunk,
   transformLogProgress,
   transformMap,
   transformMapSimple,
@@ -918,9 +918,9 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
 
   /**
    * "Streaming" is implemented by buffering incoming rows into **batches**
-   * (of size opt.batchSize, which defaults to 500),
-   * and then executing db.saveBatch(batch) with the concurrency
-   * of opt.batchConcurrency (which defaults to 16).
+   * (of size opt.chunkSize, which defaults to 500),
+   * and then executing db.saveBatch(chunk) with the concurrency
+   * of opt.chunkConcurrency (which defaults to 16).
    */
   streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
     this.requireWriteAccess()
@@ -936,7 +936,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
     const { beforeSave } = this.cfg.hooks!
 
-    const { batchSize = 500, batchConcurrency = 16, errorMode } = opt
+    const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt
 
     return [
       transformMap<BM, DBM>(
@@ -956,7 +956,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
           errorMode,
         },
       ),
-      transformBuffer<DBM>({ batchSize }),
+      transformChunk<DBM>({ chunkSize }),
       transformMap<DBM[], DBM[]>(
         async batch => {
           await this.cfg.db.saveBatch(table, batch, {
@@ -966,7 +966,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
           return batch
         },
         {
-          concurrency: batchConcurrency,
+          concurrency: chunkConcurrency,
           errorMode,
           flattenArrayOutput: true,
         },
@@ -1003,9 +1003,9 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
   }
 
   /**
-   * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
-   * `deleteByIds` for each batch concurrently (infinite concurrency).
-   * This is expected to be more memory-efficient way of deleting big numbers of rows.
+   * Pass `chunkSize: number` (e.g 500) option to use Streaming: it will Stream the query, chunk by 500, and execute
+   * `deleteByIds` for each chunk concurrently (infinite concurrency).
+   * This is expected to be more memory-efficient way of deleting large number of rows.
    */
   async deleteByQuery(
     q: DBQuery<DBM>,
@@ -1018,15 +1018,15 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const started = this.logStarted(op, q.table)
     let deleted = 0
 
-    if (opt.batchSize) {
-      const { batchSize, batchConcurrency = 16 } = opt
+    if (opt.chunkSize) {
+      const { chunkSize, chunkConcurrency = 16 } = opt
 
       await _pipeline([
         this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
         transformMapSimple<ObjectWithId, string>(r => r.id, {
           errorMode: ErrorMode.SUPPRESS,
         }),
-        transformBuffer<string>({ batchSize }),
+        transformChunk<string>({ chunkSize }),
         transformMap<string[], void>(
           async ids => {
             deleted += await this.cfg.db.deleteByQuery(
@@ -1036,14 +1036,14 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
           },
           {
             predicate: _passthroughPredicate,
-            concurrency: batchConcurrency,
+            concurrency: chunkConcurrency,
           },
         ),
         // LogProgress should be AFTER the mapper, to be able to report correct stats
         transformLogProgress({
           metric: q.table,
           logEvery: 2, // 500 * 2 === 1000
-          batchSize,
+          chunkSize,
           ...opt,
         }),
         writableVoid(),

package/src/pipeline/dbPipelineCopy.ts

@@ -8,7 +8,7 @@ import {
 } from '@naturalcycles/js-lib'
 import {
   NDJsonStats,
-  transformBuffer,
+  transformChunk,
   transformLogProgress,
   TransformLogProgressOptions,
   transformMap,
@@ -47,7 +47,7 @@ export interface DBPipelineCopyOptions extends TransformLogProgressOptions {
    *
    * Determines the size of .saveBatch()
    */
-  batchSize?: number
+  chunkSize?: number
 
   /**
    * @default ErrorMode.SUPPRESS
@@ -98,7 +98,7 @@ export interface DBPipelineCopyOptions extends TransformLogProgressOptions {
  */
 export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJsonStats> {
   const {
-    batchSize = 100,
+    chunkSize = 100,
     dbInput,
     dbOutput,
     concurrency = 16,
@@ -153,7 +153,7 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
           metric: table,
         }),
         transformTap(() => rows++),
-        transformBuffer({ batchSize }),
+        transformChunk({ chunkSize }),
         writableForEach(async dbms => {
           await dbOutput.saveBatch(table, dbms, saveOptions)
         }),

package/src/pipeline/dbPipelineRestore.ts

@@ -13,7 +13,7 @@ import {
 } from '@naturalcycles/js-lib'
 import {
   NDJsonStats,
-  transformBuffer,
+  transformChunk,
   transformFilterSync,
   transformJsonParse,
   transformLimit,
@@ -125,7 +125,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
   const {
     db,
     concurrency = 16,
-    batchSize = 100,
+    chunkSize = 100,
     limit,
     sinceUpdated,
     inputDirPath,
@@ -224,7 +224,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
           ...transformMapOptions,
           metric: table,
         }),
-        transformBuffer({ batchSize }),
+        transformChunk({ chunkSize }),
         writableForEach(async dbms => {
           await db.saveBatch(table, dbms, saveOptions)
         }),