@naturalcycles/db-lib 9.11.0 → 9.12.1

package/dist/commondao/common.dao.d.ts

@@ -112,9 +112,9 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     saveBatchAsDBM(dbms: Unsaved<DBM>[], opt?: CommonDaoSaveBatchOptions<DBM>): Promise<DBM[]>;
     /**
      * "Streaming" is implemented by buffering incoming rows into **batches**
-     * (of size opt.batchSize, which defaults to 500),
-     * and then executing db.saveBatch(batch) with the concurrency
-     * of opt.batchConcurrency (which defaults to 16).
+     * (of size opt.chunkSize, which defaults to 500),
+     * and then executing db.saveBatch(chunk) with the concurrency
+     * of opt.chunkConcurrency (which defaults to 16).
      */
     streamSaveTransform(opt?: CommonDaoStreamSaveOptions<DBM>): Transform[];
     /**
@@ -123,9 +123,9 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     deleteById(id?: string | null, opt?: CommonDaoOptions): Promise<number>;
     deleteByIds(ids: string[], opt?: CommonDaoOptions): Promise<number>;
     /**
-     * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
-     * `deleteByIds` for each batch concurrently (infinite concurrency).
-     * This is expected to be more memory-efficient way of deleting big numbers of rows.
+     * Pass `chunkSize: number` (e.g 500) option to use Streaming: it will Stream the query, chunk by 500, and execute
+     * `deleteByIds` for each chunk concurrently (infinite concurrency).
+     * This is expected to be more memory-efficient way of deleting large number of rows.
      */
     deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
     updateById(id: string, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;

package/dist/commondao/common.dao.js

@@ -448,7 +448,7 @@ class CommonDao {
      * "Returns", just to have a type of "Saved"
      */
     assignIdCreatedUpdated(obj, opt = {}) {
-        const now = Math.floor(Date.now() / 1000);
+        const now = (0, js_lib_1.nowUnix)();
         if (this.cfg.useCreatedProperty) {
             obj.created ||= obj.updated || now;
         }
@@ -697,9 +697,9 @@ class CommonDao {
     }
     /**
      * "Streaming" is implemented by buffering incoming rows into **batches**
-     * (of size opt.batchSize, which defaults to 500),
-     * and then executing db.saveBatch(batch) with the concurrency
-     * of opt.batchConcurrency (which defaults to 16).
+     * (of size opt.chunkSize, which defaults to 500),
+     * and then executing db.saveBatch(chunk) with the concurrency
+     * of opt.chunkConcurrency (which defaults to 16).
      */
     streamSaveTransform(opt = {}) {
         this.requireWriteAccess();
@@ -711,7 +711,7 @@ class CommonDao {
         }
         const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes;
         const { beforeSave } = this.cfg.hooks;
-        const { batchSize = 500, batchConcurrency = 16, errorMode } = opt;
+        const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt;
         return [
             (0, nodejs_lib_1.transformMap)(async (bm) => {
                 this.assignIdCreatedUpdated(bm, opt); // mutates
@@ -725,7 +725,7 @@ class CommonDao {
             }, {
                 errorMode,
             }),
-            (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+            (0, nodejs_lib_1.transformChunk)({ chunkSize }),
             (0, nodejs_lib_1.transformMap)(async (batch) => {
                 await this.cfg.db.saveBatch(table, batch, {
                     ...opt,
@@ -733,7 +733,7 @@ class CommonDao {
                 });
                 return batch;
             }, {
-                concurrency: batchConcurrency,
+                concurrency: chunkConcurrency,
                 errorMode,
                 flattenArrayOutput: true,
             }),
@@ -768,9 +768,9 @@ class CommonDao {
         return count;
     }
     /**
-     * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
-     * `deleteByIds` for each batch concurrently (infinite concurrency).
-     * This is expected to be more memory-efficient way of deleting big numbers of rows.
+     * Pass `chunkSize: number` (e.g 500) option to use Streaming: it will Stream the query, chunk by 500, and execute
+     * `deleteByIds` for each chunk concurrently (infinite concurrency).
+     * This is expected to be more memory-efficient way of deleting large number of rows.
      */
     async deleteByQuery(q, opt = {}) {
         this.requireWriteAccess();
@@ -779,25 +779,25 @@ class CommonDao {
         const op = `deleteByQuery(${q.pretty()})`;
         const started = this.logStarted(op, q.table);
         let deleted = 0;
-        if (opt.batchSize) {
-            const { batchSize, batchConcurrency = 16 } = opt;
+        if (opt.chunkSize) {
+            const { chunkSize, chunkConcurrency = 16 } = opt;
             await (0, nodejs_lib_1._pipeline)([
                 this.cfg.db.streamQuery(q.select(['id']), opt),
                 (0, nodejs_lib_1.transformMapSimple)(r => r.id, {
                     errorMode: js_lib_1.ErrorMode.SUPPRESS,
                 }),
-                (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+                (0, nodejs_lib_1.transformChunk)({ chunkSize }),
                 (0, nodejs_lib_1.transformMap)(async (ids) => {
                     deleted += await this.cfg.db.deleteByQuery(dbQuery_1.DBQuery.create(q.table).filterIn('id', ids), opt);
                 }, {
                     predicate: js_lib_1._passthroughPredicate,
-                    concurrency: batchConcurrency,
+                    concurrency: chunkConcurrency,
                 }),
                 // LogProgress should be AFTER the mapper, to be able to report correct stats
                 (0, nodejs_lib_1.transformLogProgress)({
                     metric: q.table,
                     logEvery: 2, // 500 * 2 === 1000
-                    batchSize,
+                    chunkSize,
                     ...opt,
                 }),
                 (0, nodejs_lib_1.writableVoid)(),

package/dist/commondao/common.dao.model.d.ts

@@ -250,15 +250,15 @@ export interface CommonDaoStreamOptions<IN> extends CommonDaoOptions, TransformL
     errorMode?: ErrorMode;
     /**
      * Applicable to some of stream operations, e.g deleteByQuery.
-     * If set - `deleteByQuery` won't execute it "all at once", but in batches.
+     * If set - `deleteByQuery` won't execute it "all at once", but in batches (chunks).
      *
      * Defaults to undefined, so the operation is executed "all at once".
      */
-    batchSize?: number;
+    chunkSize?: number;
     /**
-     * When batchSize is set - this option controls how many batches to run concurrently.
+     * When chunkSize is set - this option controls how many chunks to run concurrently.
      * Defaults to 16, "the magic number of JavaScript concurrency".
      */
-    batchConcurrency?: number;
+    chunkConcurrency?: number;
 }
 export type CommonDaoCreateOptions = CommonDBCreateOptions;

package/dist/kv/commonKeyValueDao.d.ts

@@ -3,7 +3,7 @@ import { CommonLogger, KeyValueTuple } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonDaoLogLevel } from '../commondao/common.dao.model';
 import { CommonDBCreateOptions } from '../db.model';
-import { CommonKeyValueDB, KeyValueDBTuple } from './commonKeyValueDB';
+import { CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, KeyValueDBTuple } from './commonKeyValueDB';
 export interface CommonKeyValueDaoCfg<T> {
     db: CommonKeyValueDB;
     table: string;
@@ -36,6 +36,7 @@ export interface CommonKeyValueDaoCfg<T> {
      */
     deflatedJsonValue?: boolean;
 }
+export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions;
 export declare class CommonKeyValueDao<T> {
     constructor(cfg: CommonKeyValueDaoCfg<T>);
     cfg: CommonKeyValueDaoCfg<T> & {
@@ -50,13 +51,13 @@ export declare class CommonKeyValueDao<T> {
     requireById(id: string): Promise<T>;
     requireByIdAsBuffer(id: string): Promise<Buffer>;
     getByIdOrEmpty(id: string, part?: Partial<T>): Promise<T>;
-    patch(id: string, patch: Partial<T>): Promise<T>;
+    patch(id: string, patch: Partial<T>, opt?: CommonKeyValueDaoSaveOptions): Promise<T>;
     getByIds(ids: string[]): Promise<KeyValueTuple<string, T>[]>;
     getByIdsAsBuffer(ids: string[]): Promise<KeyValueTuple<string, Buffer>[]>;
-    save(id: string, value: T): Promise<void>;
-    saveAsBuffer(id: string, value: Buffer): Promise<void>;
-    saveBatch(entries: KeyValueTuple<string, T>[]): Promise<void>;
-    saveBatchAsBuffer(entries: KeyValueDBTuple[]): Promise<void>;
+    save(id: string, value: T, opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
+    saveAsBuffer(id: string, value: Buffer, opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
+    saveBatch(entries: KeyValueTuple<string, T>[], opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
+    saveBatchAsBuffer(entries: KeyValueDBTuple[], opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
     deleteByIds(ids: string[]): Promise<void>;
     deleteById(id: string): Promise<void>;
     streamIds(limit?: number): ReadableTyped<string>;

package/dist/kv/commonKeyValueDao.js

@@ -74,12 +74,12 @@ class CommonKeyValueDao {
             ...part,
         };
     }
-    async patch(id, patch) {
+    async patch(id, patch, opt) {
         const v = {
             ...(await this.getByIdOrEmpty(id)),
             ...patch,
         };
-        await this.save(id, v);
+        await this.save(id, v, opt);
         return v;
     }
     async getByIds(ids) {
@@ -94,27 +94,25 @@ class CommonKeyValueDao {
     async getByIdsAsBuffer(ids) {
         return await this.cfg.db.getByIds(this.cfg.table, ids);
     }
-    async save(id, value) {
-        await this.saveBatch([[id, value]]);
+    async save(id, value, opt) {
+        await this.saveBatch([[id, value]], opt);
     }
-    async saveAsBuffer(id, value) {
-        await this.cfg.db.saveBatch(this.cfg.table, [[id, value]]);
+    async saveAsBuffer(id, value, opt) {
+        await this.cfg.db.saveBatch(this.cfg.table, [[id, value]], opt);
     }
-    async saveBatch(entries) {
+    async saveBatch(entries, opt) {
+        const { mapValueToBuffer } = this.cfg.hooks;
         let bufferEntries;
-        if (!this.cfg.hooks.mapValueToBuffer) {
+        if (!mapValueToBuffer) {
             bufferEntries = entries;
         }
         else {
-            bufferEntries = await (0, js_lib_1.pMap)(entries, async ([id, v]) => [
-                id,
-                await this.cfg.hooks.mapValueToBuffer(v),
-            ]);
+            bufferEntries = await (0, js_lib_1.pMap)(entries, async ([id, v]) => [id, await mapValueToBuffer(v)]);
         }
-        await this.cfg.db.saveBatch(this.cfg.table, bufferEntries);
+        await this.cfg.db.saveBatch(this.cfg.table, bufferEntries, opt);
     }
-    async saveBatchAsBuffer(entries) {
-        await this.cfg.db.saveBatch(this.cfg.table, entries);
+    async saveBatchAsBuffer(entries, opt) {
+        await this.cfg.db.saveBatch(this.cfg.table, entries, opt);
     }
     async deleteByIds(ids) {
         await this.cfg.db.deleteByIds(this.cfg.table, ids);
@@ -130,10 +128,7 @@ class CommonKeyValueDao {
         if (!mapBufferToValue) {
             return this.cfg.db.streamValues(this.cfg.table, limit);
         }
-        const stream = this.cfg.db
-            .streamValues(this.cfg.table, limit)
-            // .on('error', err => stream.emit('error', err))
-            .flatMap(async (buf) => {
+        return this.cfg.db.streamValues(this.cfg.table, limit).flatMap(async (buf) => {
             try {
                 return [await mapBufferToValue(buf)];
             }
@@ -141,18 +136,16 @@ class CommonKeyValueDao {
                 this.cfg.logger.error(err);
                 return []; // SKIP
             }
+        }, {
+            concurrency: 16,
         });
-        return stream;
     }
     streamEntries(limit) {
         const { mapBufferToValue } = this.cfg.hooks;
         if (!mapBufferToValue) {
             return this.cfg.db.streamEntries(this.cfg.table, limit);
         }
-        const stream = this.cfg.db
-            .streamEntries(this.cfg.table, limit)
-            // .on('error', err => stream.emit('error', err))
-            .flatMap(async ([id, buf]) => {
+        return this.cfg.db.streamEntries(this.cfg.table, limit).flatMap(async ([id, buf]) => {
             try {
                 return [[id, await mapBufferToValue(buf)]];
             }
@@ -160,8 +153,9 @@ class CommonKeyValueDao {
                 this.cfg.logger.error(err);
                 return []; // SKIP
             }
+        }, {
+            concurrency: 16,
         });
-        return stream;
     }
 }
 exports.CommonKeyValueDao = CommonKeyValueDao;

package/dist/kv/commonKeyValueDaoMemoCache.d.ts

@@ -1,5 +1,12 @@
-import { AsyncMemoCache, MISS } from '@naturalcycles/js-lib';
+import { AsyncMemoCache, MISS, NumberOfSeconds } from '@naturalcycles/js-lib';
 import { CommonKeyValueDao } from './commonKeyValueDao';
+export interface CommonKeyValueDaoMemoCacheCfg<VALUE> {
+    dao: CommonKeyValueDao<VALUE>;
+    /**
+     * If set, every `set()` will set `expireAt` (TTL) option.
+     */
+    ttl?: NumberOfSeconds;
+}
 /**
  * AsyncMemoCache implementation, backed by CommonKeyValueDao.
  *
@@ -9,8 +16,8 @@ import { CommonKeyValueDao } from './commonKeyValueDao';
  * clear the whole table/cache.
  */
 export declare class CommonKeyValueDaoMemoCache<VALUE = any> implements AsyncMemoCache<string, VALUE> {
-    private dao;
-    constructor(dao: CommonKeyValueDao<VALUE>);
+    private cfg;
+    constructor(cfg: CommonKeyValueDaoMemoCacheCfg<VALUE>);
     get(k: string): Promise<VALUE | typeof MISS>;
     set(k: string, v: VALUE): Promise<void>;
     clear(): Promise<void>;

package/dist/kv/commonKeyValueDaoMemoCache.js

@@ -11,14 +11,15 @@ const js_lib_1 = require("@naturalcycles/js-lib");
  * clear the whole table/cache.
  */
 class CommonKeyValueDaoMemoCache {
-    constructor(dao) {
-        this.dao = dao;
+    constructor(cfg) {
+        this.cfg = cfg;
     }
     async get(k) {
-        return (await this.dao.getById(k)) || js_lib_1.MISS;
+        return (await this.cfg.dao.getById(k)) || js_lib_1.MISS;
     }
     async set(k, v) {
-        await this.dao.save(k, v);
+        const opt = this.cfg.ttl ? { expireAt: (0, js_lib_1.nowUnix)() + this.cfg.ttl } : undefined;
+        await this.cfg.dao.save(k, v, opt);
     }
     async clear() {
         throw new Error('CommonKeyValueDaoMemoCache.clear is not supported, because cache is expected to be persistent');

package/dist/model.util.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.serializeJsonField = exports.deserializeJsonField = exports.createdUpdatedIdFields = exports.createdUpdatedFields = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 function createdUpdatedFields(existingObject) {
-    const now = Math.floor(Date.now() / 1000);
+    const now = (0, js_lib_1.nowUnix)();
     return {
         created: existingObject?.created || now,
         updated: now,
@@ -11,7 +12,7 @@ function createdUpdatedFields(existingObject) {
 }
 exports.createdUpdatedFields = createdUpdatedFields;
 function createdUpdatedIdFields(existingObject) {
-    const now = Math.floor(Date.now() / 1000);
+    const now = (0, js_lib_1.nowUnix)();
     return {
         created: existingObject?.created || now,
         id: existingObject?.id || (0, nodejs_lib_1.stringId)(),

package/dist/pipeline/dbPipelineCopy.d.ts

@@ -21,7 +21,7 @@ export interface DBPipelineCopyOptions extends TransformLogProgressOptions {
      *
      * Determines the size of .saveBatch()
      */
-    batchSize?: number;
+    chunkSize?: number;
     /**
      * @default ErrorMode.SUPPRESS
      *

package/dist/pipeline/dbPipelineCopy.js

@@ -11,7 +11,7 @@ const dbQuery_1 = require("../query/dbQuery");
  * Handles backpressure.
  */
 async function dbPipelineCopy(opt) {
-    const { batchSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
+    const { chunkSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
     let { tables } = opt;
     const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
     console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
@@ -42,7 +42,7 @@ async function dbPipelineCopy(opt) {
                 metric: table,
             }),
             (0, nodejs_lib_1.transformTap)(() => rows++),
-            (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+            (0, nodejs_lib_1.transformChunk)({ chunkSize }),
             (0, nodejs_lib_1.writableForEach)(async (dbms) => {
                 await dbOutput.saveBatch(table, dbms, saveOptions);
             }),

package/dist/pipeline/dbPipelineRestore.js

@@ -14,7 +14,7 @@ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
  * Optionally you can provide mapperPerTable and @param transformMapOptions (one for all mappers) - it will run for each table.
  */
 async function dbPipelineRestore(opt) {
-    const { db, concurrency = 16, batchSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
+    const { db, concurrency = 16, chunkSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
     const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
     const onlyTables = opt.tables && new Set(opt.tables);
     const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
@@ -87,7 +87,7 @@ async function dbPipelineRestore(opt) {
                 ...transformMapOptions,
                 metric: table,
             }),
-            (0, nodejs_lib_1.transformBuffer)({ batchSize }),
+            (0, nodejs_lib_1.transformChunk)({ chunkSize }),
             (0, nodejs_lib_1.writableForEach)(async (dbms) => {
                 await db.saveBatch(table, dbms, saveOptions);
             }),

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "9.11.0",
+  "version": "9.12.1",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/commondao/common.dao.model.ts

@@ -313,17 +313,17 @@ export interface CommonDaoStreamOptions<IN>
 
   /**
    * Applicable to some of stream operations, e.g deleteByQuery.
-   * If set - `deleteByQuery` won't execute it "all at once", but in batches.
+   * If set - `deleteByQuery` won't execute it "all at once", but in batches (chunks).
    *
    * Defaults to undefined, so the operation is executed "all at once".
    */
-  batchSize?: number
+  chunkSize?: number
 
   /**
-   * When batchSize is set - this option controls how many batches to run concurrently.
+   * When chunkSize is set - this option controls how many chunks to run concurrently.
    * Defaults to 16, "the magic number of JavaScript concurrency".
    */
-  batchConcurrency?: number
+  chunkConcurrency?: number
 }
 
 export type CommonDaoCreateOptions = CommonDBCreateOptions

package/src/commondao/common.dao.ts

@@ -18,6 +18,7 @@ import {
   ErrorMode,
   JsonSchemaObject,
   JsonSchemaRootObject,
+  nowUnix,
   ObjectWithId,
   pMap,
   SKIP,
@@ -36,7 +37,7 @@ import {
   ObjectSchema,
   ReadableTyped,
   stringId,
-  transformBuffer,
+  transformChunk,
   transformLogProgress,
   transformMap,
   transformMapSimple,
@@ -606,7 +607,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
    * "Returns", just to have a type of "Saved"
    */
   assignIdCreatedUpdated<T extends BaseDBEntity>(obj: Partial<T>, opt: CommonDaoOptions = {}): T {
-    const now = Math.floor(Date.now() / 1000)
+    const now = nowUnix()
 
     if (this.cfg.useCreatedProperty) {
       obj.created ||= obj.updated || now
@@ -917,9 +918,9 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
 
   /**
    * "Streaming" is implemented by buffering incoming rows into **batches**
-   * (of size opt.batchSize, which defaults to 500),
-   * and then executing db.saveBatch(batch) with the concurrency
-   * of opt.batchConcurrency (which defaults to 16).
+   * (of size opt.chunkSize, which defaults to 500),
+   * and then executing db.saveBatch(chunk) with the concurrency
+   * of opt.chunkConcurrency (which defaults to 16).
    */
   streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
     this.requireWriteAccess()
@@ -935,7 +936,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
     const { beforeSave } = this.cfg.hooks!
 
-    const { batchSize = 500, batchConcurrency = 16, errorMode } = opt
+    const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt
 
     return [
       transformMap<BM, DBM>(
@@ -955,7 +956,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
           errorMode,
         },
       ),
-      transformBuffer<DBM>({ batchSize }),
+      transformChunk<DBM>({ chunkSize }),
       transformMap<DBM[], DBM[]>(
         async batch => {
           await this.cfg.db.saveBatch(table, batch, {
@@ -965,7 +966,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
           return batch
         },
         {
-          concurrency: batchConcurrency,
+          concurrency: chunkConcurrency,
           errorMode,
           flattenArrayOutput: true,
         },
@@ -1002,9 +1003,9 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
   }
 
   /**
-   * Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
-   * `deleteByIds` for each batch concurrently (infinite concurrency).
-   * This is expected to be more memory-efficient way of deleting big numbers of rows.
+   * Pass `chunkSize: number` (e.g 500) option to use Streaming: it will Stream the query, chunk by 500, and execute
+   * `deleteByIds` for each chunk concurrently (infinite concurrency).
+   * This is expected to be more memory-efficient way of deleting large number of rows.
    */
   async deleteByQuery(
     q: DBQuery<DBM>,
@@ -1017,15 +1018,15 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const started = this.logStarted(op, q.table)
     let deleted = 0
 
-    if (opt.batchSize) {
-      const { batchSize, batchConcurrency = 16 } = opt
+    if (opt.chunkSize) {
+      const { chunkSize, chunkConcurrency = 16 } = opt
 
       await _pipeline([
         this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
         transformMapSimple<ObjectWithId, string>(r => r.id, {
           errorMode: ErrorMode.SUPPRESS,
         }),
-        transformBuffer<string>({ batchSize }),
+        transformChunk<string>({ chunkSize }),
         transformMap<string[], void>(
           async ids => {
             deleted += await this.cfg.db.deleteByQuery(
@@ -1035,14 +1036,14 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
           },
           {
             predicate: _passthroughPredicate,
-            concurrency: batchConcurrency,
+            concurrency: chunkConcurrency,
           },
         ),
         // LogProgress should be AFTER the mapper, to be able to report correct stats
         transformLogProgress({
           metric: q.table,
           logEvery: 2, // 500 * 2 === 1000
-          batchSize,
+          chunkSize,
           ...opt,
         }),
         writableVoid(),

package/src/kv/commonKeyValueDao.ts

@@ -2,7 +2,11 @@ import { AppError, CommonLogger, KeyValueTuple, pMap } from '@naturalcycles/js-l
 import { deflateString, inflateToString, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel } from '../commondao/common.dao.model'
 import { CommonDBCreateOptions } from '../db.model'
-import { CommonKeyValueDB, KeyValueDBTuple } from './commonKeyValueDB'
+import {
+  CommonKeyValueDB,
+  CommonKeyValueDBSaveBatchOptions,
+  KeyValueDBTuple,
+} from './commonKeyValueDB'
 
 export interface CommonKeyValueDaoCfg<T> {
   db: CommonKeyValueDB
@@ -44,6 +48,8 @@ export interface CommonKeyValueDaoCfg<T> {
   deflatedJsonValue?: boolean
 }
 
+export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions
+
 // todo: logging
 // todo: readonly
 
@@ -133,13 +139,13 @@ export class CommonKeyValueDao<T> {
     } as T
   }
 
-  async patch(id: string, patch: Partial<T>): Promise<T> {
+  async patch(id: string, patch: Partial<T>, opt?: CommonKeyValueDaoSaveOptions): Promise<T> {
     const v: T = {
       ...(await this.getByIdOrEmpty(id)),
       ...patch,
     }
 
-    await this.save(id, v)
+    await this.save(id, v, opt)
 
     return v
   }
@@ -158,31 +164,35 @@ export class CommonKeyValueDao<T> {
     return await this.cfg.db.getByIds(this.cfg.table, ids)
   }
 
-  async save(id: string, value: T): Promise<void> {
-    await this.saveBatch([[id, value]])
+  async save(id: string, value: T, opt?: CommonKeyValueDaoSaveOptions): Promise<void> {
+    await this.saveBatch([[id, value]], opt)
   }
 
-  async saveAsBuffer(id: string, value: Buffer): Promise<void> {
-    await this.cfg.db.saveBatch(this.cfg.table, [[id, value]])
+  async saveAsBuffer(id: string, value: Buffer, opt?: CommonKeyValueDaoSaveOptions): Promise<void> {
+    await this.cfg.db.saveBatch(this.cfg.table, [[id, value]], opt)
   }
 
-  async saveBatch(entries: KeyValueTuple<string, T>[]): Promise<void> {
+  async saveBatch(
+    entries: KeyValueTuple<string, T>[],
+    opt?: CommonKeyValueDaoSaveOptions,
+  ): Promise<void> {
+    const { mapValueToBuffer } = this.cfg.hooks
     let bufferEntries: KeyValueDBTuple[]
 
-    if (!this.cfg.hooks.mapValueToBuffer) {
+    if (!mapValueToBuffer) {
       bufferEntries = entries as any
     } else {
-      bufferEntries = await pMap(entries, async ([id, v]) => [
-        id,
-        await this.cfg.hooks.mapValueToBuffer!(v),
-      ])
+      bufferEntries = await pMap(entries, async ([id, v]) => [id, await mapValueToBuffer(v)])
     }
 
-    await this.cfg.db.saveBatch(this.cfg.table, bufferEntries)
+    await this.cfg.db.saveBatch(this.cfg.table, bufferEntries, opt)
   }
 
-  async saveBatchAsBuffer(entries: KeyValueDBTuple[]): Promise<void> {
-    await this.cfg.db.saveBatch(this.cfg.table, entries)
+  async saveBatchAsBuffer(
+    entries: KeyValueDBTuple[],
+    opt?: CommonKeyValueDaoSaveOptions,
+  ): Promise<void> {
+    await this.cfg.db.saveBatch(this.cfg.table, entries, opt)
   }
 
   async deleteByIds(ids: string[]): Promise<void> {
@@ -204,19 +214,19 @@ export class CommonKeyValueDao<T> {
       return this.cfg.db.streamValues(this.cfg.table, limit) as ReadableTyped<T>
     }
 
-    const stream: ReadableTyped<T> = this.cfg.db
-      .streamValues(this.cfg.table, limit)
-      // .on('error', err => stream.emit('error', err))
-      .flatMap(async buf => {
+    return this.cfg.db.streamValues(this.cfg.table, limit).flatMap(
+      async buf => {
         try {
           return [await mapBufferToValue(buf)]
         } catch (err) {
           this.cfg.logger.error(err)
           return [] // SKIP
         }
-      })
-
-    return stream
+      },
+      {
+        concurrency: 16,
+      },
+    )
   }
 
   streamEntries(limit?: number): ReadableTyped<KeyValueTuple<string, T>> {
@@ -228,18 +238,18 @@ export class CommonKeyValueDao<T> {
       >
     }
 
-    const stream: ReadableTyped<KeyValueTuple<string, T>> = this.cfg.db
-      .streamEntries(this.cfg.table, limit)
-      // .on('error', err => stream.emit('error', err))
-      .flatMap(async ([id, buf]) => {
+    return this.cfg.db.streamEntries(this.cfg.table, limit).flatMap(
+      async ([id, buf]) => {
         try {
           return [[id, await mapBufferToValue(buf)]]
         } catch (err) {
           this.cfg.logger.error(err)
           return [] // SKIP
         }
-      })
-
-    return stream
+      },
+      {
+        concurrency: 16,
+      },
+    )
   }
 }

package/src/kv/commonKeyValueDaoMemoCache.ts

@@ -1,6 +1,15 @@
-import { AsyncMemoCache, MISS } from '@naturalcycles/js-lib'
+import { AsyncMemoCache, MISS, nowUnix, NumberOfSeconds } from '@naturalcycles/js-lib'
 import { CommonKeyValueDao } from './commonKeyValueDao'
 
+export interface CommonKeyValueDaoMemoCacheCfg<VALUE> {
+  dao: CommonKeyValueDao<VALUE>
+
+  /**
+   * If set, every `set()` will set `expireAt` (TTL) option.
+   */
+  ttl?: NumberOfSeconds
+}
+
 /**
  * AsyncMemoCache implementation, backed by CommonKeyValueDao.
  *
@@ -10,14 +19,16 @@ import { CommonKeyValueDao } from './commonKeyValueDao'
  * clear the whole table/cache.
  */
 export class CommonKeyValueDaoMemoCache<VALUE = any> implements AsyncMemoCache<string, VALUE> {
-  constructor(private dao: CommonKeyValueDao<VALUE>) {}
+  constructor(private cfg: CommonKeyValueDaoMemoCacheCfg<VALUE>) {}
 
   async get(k: string): Promise<VALUE | typeof MISS> {
-    return (await this.dao.getById(k)) || MISS
+    return (await this.cfg.dao.getById(k)) || MISS
   }
 
   async set(k: string, v: VALUE): Promise<void> {
-    await this.dao.save(k, v)
+    const opt = this.cfg.ttl ? { expireAt: nowUnix() + this.cfg.ttl } : undefined
+
+    await this.cfg.dao.save(k, v, opt)
   }
 
   async clear(): Promise<void> {

package/src/model.util.ts

@@ -1,10 +1,10 @@
-import { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib'
+import { CreatedUpdated, CreatedUpdatedId, nowUnix } from '@naturalcycles/js-lib'
 import { stringId } from '@naturalcycles/nodejs-lib'
 
 export function createdUpdatedFields(
   existingObject?: Partial<CreatedUpdated> | null,
 ): CreatedUpdated {
-  const now = Math.floor(Date.now() / 1000)
+  const now = nowUnix()
   return {
     created: existingObject?.created || now,
     updated: now,
@@ -14,7 +14,7 @@ export function createdUpdatedFields(
 export function createdUpdatedIdFields(
   existingObject?: Partial<CreatedUpdatedId> | null,
 ): CreatedUpdatedId {
-  const now = Math.floor(Date.now() / 1000)
+  const now = nowUnix()
   return {
     created: existingObject?.created || now,
     id: existingObject?.id || stringId(),

package/src/pipeline/dbPipelineCopy.ts

@@ -8,7 +8,7 @@ import {
 } from '@naturalcycles/js-lib'
 import {
   NDJsonStats,
-  transformBuffer,
+  transformChunk,
   transformLogProgress,
   TransformLogProgressOptions,
   transformMap,
@@ -47,7 +47,7 @@ export interface DBPipelineCopyOptions extends TransformLogProgressOptions {
    *
    * Determines the size of .saveBatch()
    */
-  batchSize?: number
+  chunkSize?: number
 
   /**
    * @default ErrorMode.SUPPRESS
@@ -98,7 +98,7 @@ export interface DBPipelineCopyOptions extends TransformLogProgressOptions {
  */
 export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJsonStats> {
   const {
-    batchSize = 100,
+    chunkSize = 100,
     dbInput,
     dbOutput,
     concurrency = 16,
@@ -153,7 +153,7 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
           metric: table,
         }),
         transformTap(() => rows++),
-        transformBuffer({ batchSize }),
+        transformChunk({ chunkSize }),
         writableForEach(async dbms => {
           await dbOutput.saveBatch(table, dbms, saveOptions)
         }),

package/src/pipeline/dbPipelineRestore.ts

@@ -13,7 +13,7 @@ import {
 } from '@naturalcycles/js-lib'
 import {
   NDJsonStats,
-  transformBuffer,
+  transformChunk,
   transformFilterSync,
   transformJsonParse,
   transformLimit,
@@ -125,7 +125,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
   const {
     db,
     concurrency = 16,
-    batchSize = 100,
+    chunkSize = 100,
     limit,
     sinceUpdated,
     inputDirPath,
@@ -224,7 +224,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
           ...transformMapOptions,
           metric: table,
         }),
-        transformBuffer({ batchSize }),
+        transformChunk({ chunkSize }),
         writableForEach(async dbms => {
           await db.saveBatch(table, dbms, saveOptions)
         }),