@naturalcycles/db-lib 9.11.0 → 9.12.0

package/dist/commondao/common.dao.js

@@ -448,7 +448,7 @@ class CommonDao {
      * "Returns", just to have a type of "Saved"
      */
     assignIdCreatedUpdated(obj, opt = {}) {
-        const now = Math.floor(Date.now() / 1000);
+        const now = (0, js_lib_1.nowUnix)();
         if (this.cfg.useCreatedProperty) {
             obj.created ||= obj.updated || now;
         }

package/dist/kv/commonKeyValueDao.d.ts

@@ -3,7 +3,7 @@ import { CommonLogger, KeyValueTuple } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonDaoLogLevel } from '../commondao/common.dao.model';
 import { CommonDBCreateOptions } from '../db.model';
-import { CommonKeyValueDB, KeyValueDBTuple } from './commonKeyValueDB';
+import { CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, KeyValueDBTuple } from './commonKeyValueDB';
 export interface CommonKeyValueDaoCfg<T> {
     db: CommonKeyValueDB;
     table: string;
@@ -36,6 +36,7 @@ export interface CommonKeyValueDaoCfg<T> {
      */
     deflatedJsonValue?: boolean;
 }
+export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions;
 export declare class CommonKeyValueDao<T> {
     constructor(cfg: CommonKeyValueDaoCfg<T>);
     cfg: CommonKeyValueDaoCfg<T> & {
@@ -50,13 +51,13 @@ export declare class CommonKeyValueDao<T> {
     requireById(id: string): Promise<T>;
     requireByIdAsBuffer(id: string): Promise<Buffer>;
     getByIdOrEmpty(id: string, part?: Partial<T>): Promise<T>;
-    patch(id: string, patch: Partial<T>): Promise<T>;
+    patch(id: string, patch: Partial<T>, opt?: CommonKeyValueDaoSaveOptions): Promise<T>;
     getByIds(ids: string[]): Promise<KeyValueTuple<string, T>[]>;
     getByIdsAsBuffer(ids: string[]): Promise<KeyValueTuple<string, Buffer>[]>;
-    save(id: string, value: T): Promise<void>;
-    saveAsBuffer(id: string, value: Buffer): Promise<void>;
-    saveBatch(entries: KeyValueTuple<string, T>[]): Promise<void>;
-    saveBatchAsBuffer(entries: KeyValueDBTuple[]): Promise<void>;
+    save(id: string, value: T, opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
+    saveAsBuffer(id: string, value: Buffer, opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
+    saveBatch(entries: KeyValueTuple<string, T>[], opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
+    saveBatchAsBuffer(entries: KeyValueDBTuple[], opt?: CommonKeyValueDaoSaveOptions): Promise<void>;
     deleteByIds(ids: string[]): Promise<void>;
     deleteById(id: string): Promise<void>;
     streamIds(limit?: number): ReadableTyped<string>;

package/dist/kv/commonKeyValueDao.js

@@ -74,12 +74,12 @@ class CommonKeyValueDao {
             ...part,
         };
     }
-    async patch(id, patch) {
+    async patch(id, patch, opt) {
         const v = {
             ...(await this.getByIdOrEmpty(id)),
             ...patch,
         };
-        await this.save(id, v);
+        await this.save(id, v, opt);
         return v;
     }
     async getByIds(ids) {
@@ -94,27 +94,25 @@ class CommonKeyValueDao {
     async getByIdsAsBuffer(ids) {
         return await this.cfg.db.getByIds(this.cfg.table, ids);
     }
-    async save(id, value) {
-        await this.saveBatch([[id, value]]);
+    async save(id, value, opt) {
+        await this.saveBatch([[id, value]], opt);
     }
-    async saveAsBuffer(id, value) {
-        await this.cfg.db.saveBatch(this.cfg.table, [[id, value]]);
+    async saveAsBuffer(id, value, opt) {
+        await this.cfg.db.saveBatch(this.cfg.table, [[id, value]], opt);
     }
-    async saveBatch(entries) {
+    async saveBatch(entries, opt) {
+        const { mapValueToBuffer } = this.cfg.hooks;
         let bufferEntries;
-        if (!this.cfg.hooks.mapValueToBuffer) {
+        if (!mapValueToBuffer) {
             bufferEntries = entries;
         }
         else {
-            bufferEntries = await (0, js_lib_1.pMap)(entries, async ([id, v]) => [
-                id,
-                await this.cfg.hooks.mapValueToBuffer(v),
-            ]);
+            bufferEntries = await (0, js_lib_1.pMap)(entries, async ([id, v]) => [id, await mapValueToBuffer(v)]);
         }
-        await this.cfg.db.saveBatch(this.cfg.table, bufferEntries);
+        await this.cfg.db.saveBatch(this.cfg.table, bufferEntries, opt);
     }
-    async saveBatchAsBuffer(entries) {
-        await this.cfg.db.saveBatch(this.cfg.table, entries);
+    async saveBatchAsBuffer(entries, opt) {
+        await this.cfg.db.saveBatch(this.cfg.table, entries, opt);
     }
     async deleteByIds(ids) {
         await this.cfg.db.deleteByIds(this.cfg.table, ids);
@@ -130,10 +128,7 @@ class CommonKeyValueDao {
         if (!mapBufferToValue) {
             return this.cfg.db.streamValues(this.cfg.table, limit);
         }
-        const stream = this.cfg.db
-            .streamValues(this.cfg.table, limit)
-            // .on('error', err => stream.emit('error', err))
-            .flatMap(async (buf) => {
+        return this.cfg.db.streamValues(this.cfg.table, limit).flatMap(async (buf) => {
             try {
                 return [await mapBufferToValue(buf)];
             }
@@ -141,18 +136,16 @@ class CommonKeyValueDao {
                 this.cfg.logger.error(err);
                 return []; // SKIP
             }
+        }, {
+            concurrency: 16,
         });
-        return stream;
     }
     streamEntries(limit) {
         const { mapBufferToValue } = this.cfg.hooks;
         if (!mapBufferToValue) {
             return this.cfg.db.streamEntries(this.cfg.table, limit);
         }
-        const stream = this.cfg.db
-            .streamEntries(this.cfg.table, limit)
-            // .on('error', err => stream.emit('error', err))
-            .flatMap(async ([id, buf]) => {
+        return this.cfg.db.streamEntries(this.cfg.table, limit).flatMap(async ([id, buf]) => {
             try {
                 return [[id, await mapBufferToValue(buf)]];
             }
@@ -160,8 +153,9 @@ class CommonKeyValueDao {
                 this.cfg.logger.error(err);
                 return []; // SKIP
             }
+        }, {
+            concurrency: 16,
         });
-        return stream;
     }
 }
 exports.CommonKeyValueDao = CommonKeyValueDao;

package/dist/kv/commonKeyValueDaoMemoCache.d.ts

@@ -1,5 +1,12 @@
-import { AsyncMemoCache, MISS } from '@naturalcycles/js-lib';
+import { AsyncMemoCache, MISS, NumberOfSeconds } from '@naturalcycles/js-lib';
 import { CommonKeyValueDao } from './commonKeyValueDao';
+export interface CommonKeyValueDaoMemoCacheCfg<VALUE> {
+    dao: CommonKeyValueDao<VALUE>;
+    /**
+     * If set, every `set()` will set `expireAt` (TTL) option.
+     */
+    ttl?: NumberOfSeconds;
+}
 /**
  * AsyncMemoCache implementation, backed by CommonKeyValueDao.
  *
@@ -9,8 +16,8 @@ import { CommonKeyValueDao } from './commonKeyValueDao';
  * clear the whole table/cache.
  */
 export declare class CommonKeyValueDaoMemoCache<VALUE = any> implements AsyncMemoCache<string, VALUE> {
-    private dao;
-    constructor(dao: CommonKeyValueDao<VALUE>);
+    private cfg;
+    constructor(cfg: CommonKeyValueDaoMemoCacheCfg<VALUE>);
     get(k: string): Promise<VALUE | typeof MISS>;
     set(k: string, v: VALUE): Promise<void>;
     clear(): Promise<void>;

package/dist/kv/commonKeyValueDaoMemoCache.js

@@ -11,14 +11,15 @@ const js_lib_1 = require("@naturalcycles/js-lib");
  * clear the whole table/cache.
  */
 class CommonKeyValueDaoMemoCache {
-    constructor(dao) {
-        this.dao = dao;
+    constructor(cfg) {
+        this.cfg = cfg;
     }
     async get(k) {
-        return (await this.dao.getById(k)) || js_lib_1.MISS;
+        return (await this.cfg.dao.getById(k)) || js_lib_1.MISS;
     }
     async set(k, v) {
-        await this.dao.save(k, v);
+        const opt = this.cfg.ttl ? { expireAt: (0, js_lib_1.nowUnix)() + this.cfg.ttl } : undefined;
+        await this.cfg.dao.save(k, v, opt);
     }
     async clear() {
         throw new Error('CommonKeyValueDaoMemoCache.clear is not supported, because cache is expected to be persistent');

package/dist/model.util.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.serializeJsonField = exports.deserializeJsonField = exports.createdUpdatedIdFields = exports.createdUpdatedFields = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 function createdUpdatedFields(existingObject) {
-    const now = Math.floor(Date.now() / 1000);
+    const now = (0, js_lib_1.nowUnix)();
     return {
         created: existingObject?.created || now,
         updated: now,
@@ -11,7 +12,7 @@ function createdUpdatedFields(existingObject) {
 }
 exports.createdUpdatedFields = createdUpdatedFields;
 function createdUpdatedIdFields(existingObject) {
-    const now = Math.floor(Date.now() / 1000);
+    const now = (0, js_lib_1.nowUnix)();
     return {
         created: existingObject?.created || now,
         id: existingObject?.id || (0, nodejs_lib_1.stringId)(),

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "9.11.0",
+  "version": "9.12.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/commondao/common.dao.ts

@@ -18,6 +18,7 @@ import {
   ErrorMode,
   JsonSchemaObject,
   JsonSchemaRootObject,
+  nowUnix,
   ObjectWithId,
   pMap,
   SKIP,
@@ -606,7 +607,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
    * "Returns", just to have a type of "Saved"
    */
   assignIdCreatedUpdated<T extends BaseDBEntity>(obj: Partial<T>, opt: CommonDaoOptions = {}): T {
-    const now = Math.floor(Date.now() / 1000)
+    const now = nowUnix()
 
     if (this.cfg.useCreatedProperty) {
       obj.created ||= obj.updated || now

package/src/kv/commonKeyValueDao.ts

@@ -2,7 +2,11 @@ import { AppError, CommonLogger, KeyValueTuple, pMap } from '@naturalcycles/js-l
 import { deflateString, inflateToString, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel } from '../commondao/common.dao.model'
 import { CommonDBCreateOptions } from '../db.model'
-import { CommonKeyValueDB, KeyValueDBTuple } from './commonKeyValueDB'
+import {
+  CommonKeyValueDB,
+  CommonKeyValueDBSaveBatchOptions,
+  KeyValueDBTuple,
+} from './commonKeyValueDB'
 
 export interface CommonKeyValueDaoCfg<T> {
   db: CommonKeyValueDB
@@ -44,6 +48,8 @@ export interface CommonKeyValueDaoCfg<T> {
   deflatedJsonValue?: boolean
 }
 
+export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions
+
 // todo: logging
 // todo: readonly
 
@@ -133,13 +139,13 @@ export class CommonKeyValueDao<T> {
     } as T
   }
 
-  async patch(id: string, patch: Partial<T>): Promise<T> {
+  async patch(id: string, patch: Partial<T>, opt?: CommonKeyValueDaoSaveOptions): Promise<T> {
     const v: T = {
       ...(await this.getByIdOrEmpty(id)),
       ...patch,
     }
 
-    await this.save(id, v)
+    await this.save(id, v, opt)
 
     return v
   }
@@ -158,31 +164,35 @@ export class CommonKeyValueDao<T> {
     return await this.cfg.db.getByIds(this.cfg.table, ids)
   }
 
-  async save(id: string, value: T): Promise<void> {
-    await this.saveBatch([[id, value]])
+  async save(id: string, value: T, opt?: CommonKeyValueDaoSaveOptions): Promise<void> {
+    await this.saveBatch([[id, value]], opt)
   }
 
-  async saveAsBuffer(id: string, value: Buffer): Promise<void> {
-    await this.cfg.db.saveBatch(this.cfg.table, [[id, value]])
+  async saveAsBuffer(id: string, value: Buffer, opt?: CommonKeyValueDaoSaveOptions): Promise<void> {
+    await this.cfg.db.saveBatch(this.cfg.table, [[id, value]], opt)
   }
 
-  async saveBatch(entries: KeyValueTuple<string, T>[]): Promise<void> {
+  async saveBatch(
+    entries: KeyValueTuple<string, T>[],
+    opt?: CommonKeyValueDaoSaveOptions,
+  ): Promise<void> {
+    const { mapValueToBuffer } = this.cfg.hooks
     let bufferEntries: KeyValueDBTuple[]
 
-    if (!this.cfg.hooks.mapValueToBuffer) {
+    if (!mapValueToBuffer) {
       bufferEntries = entries as any
     } else {
-      bufferEntries = await pMap(entries, async ([id, v]) => [
-        id,
-        await this.cfg.hooks.mapValueToBuffer!(v),
-      ])
+      bufferEntries = await pMap(entries, async ([id, v]) => [id, await mapValueToBuffer(v)])
     }
 
-    await this.cfg.db.saveBatch(this.cfg.table, bufferEntries)
+    await this.cfg.db.saveBatch(this.cfg.table, bufferEntries, opt)
   }
 
-  async saveBatchAsBuffer(entries: KeyValueDBTuple[]): Promise<void> {
-    await this.cfg.db.saveBatch(this.cfg.table, entries)
+  async saveBatchAsBuffer(
+    entries: KeyValueDBTuple[],
+    opt?: CommonKeyValueDaoSaveOptions,
+  ): Promise<void> {
+    await this.cfg.db.saveBatch(this.cfg.table, entries, opt)
   }
 
   async deleteByIds(ids: string[]): Promise<void> {
@@ -204,19 +214,19 @@ export class CommonKeyValueDao<T> {
       return this.cfg.db.streamValues(this.cfg.table, limit) as ReadableTyped<T>
     }
 
-    const stream: ReadableTyped<T> = this.cfg.db
-      .streamValues(this.cfg.table, limit)
-      // .on('error', err => stream.emit('error', err))
-      .flatMap(async buf => {
+    return this.cfg.db.streamValues(this.cfg.table, limit).flatMap(
+      async buf => {
         try {
           return [await mapBufferToValue(buf)]
         } catch (err) {
           this.cfg.logger.error(err)
           return [] // SKIP
         }
-      })
-
-    return stream
+      },
+      {
+        concurrency: 16,
+      },
+    )
   }
 
   streamEntries(limit?: number): ReadableTyped<KeyValueTuple<string, T>> {
@@ -228,18 +238,18 @@ export class CommonKeyValueDao<T> {
       >
     }
 
-    const stream: ReadableTyped<KeyValueTuple<string, T>> = this.cfg.db
-      .streamEntries(this.cfg.table, limit)
-      // .on('error', err => stream.emit('error', err))
-      .flatMap(async ([id, buf]) => {
+    return this.cfg.db.streamEntries(this.cfg.table, limit).flatMap(
+      async ([id, buf]) => {
         try {
           return [[id, await mapBufferToValue(buf)]]
         } catch (err) {
           this.cfg.logger.error(err)
           return [] // SKIP
         }
-      })
-
-    return stream
+      },
+      {
+        concurrency: 16,
+      },
+    )
   }
 }

package/src/kv/commonKeyValueDaoMemoCache.ts

@@ -1,6 +1,15 @@
-import { AsyncMemoCache, MISS } from '@naturalcycles/js-lib'
+import { AsyncMemoCache, MISS, nowUnix, NumberOfSeconds } from '@naturalcycles/js-lib'
 import { CommonKeyValueDao } from './commonKeyValueDao'
 
+export interface CommonKeyValueDaoMemoCacheCfg<VALUE> {
+  dao: CommonKeyValueDao<VALUE>
+
+  /**
+   * If set, every `set()` will set `expireAt` (TTL) option.
+   */
+  ttl?: NumberOfSeconds
+}
+
 /**
  * AsyncMemoCache implementation, backed by CommonKeyValueDao.
  *
@@ -10,14 +19,16 @@ import { CommonKeyValueDao } from './commonKeyValueDao'
  * clear the whole table/cache.
  */
 export class CommonKeyValueDaoMemoCache<VALUE = any> implements AsyncMemoCache<string, VALUE> {
-  constructor(private dao: CommonKeyValueDao<VALUE>) {}
+  constructor(private cfg: CommonKeyValueDaoMemoCacheCfg<VALUE>) {}
 
   async get(k: string): Promise<VALUE | typeof MISS> {
-    return (await this.dao.getById(k)) || MISS
+    return (await this.cfg.dao.getById(k)) || MISS
   }
 
   async set(k: string, v: VALUE): Promise<void> {
-    await this.dao.save(k, v)
+    const opt = this.cfg.ttl ? { expireAt: nowUnix() + this.cfg.ttl } : undefined
+
+    await this.cfg.dao.save(k, v, opt)
   }
 
   async clear(): Promise<void> {

package/src/model.util.ts

@@ -1,10 +1,10 @@
-import { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib'
+import { CreatedUpdated, CreatedUpdatedId, nowUnix } from '@naturalcycles/js-lib'
 import { stringId } from '@naturalcycles/nodejs-lib'
 
 export function createdUpdatedFields(
   existingObject?: Partial<CreatedUpdated> | null,
 ): CreatedUpdated {
-  const now = Math.floor(Date.now() / 1000)
+  const now = nowUnix()
   return {
     created: existingObject?.created || now,
     updated: now,
@@ -14,7 +14,7 @@ export function createdUpdatedFields(
 export function createdUpdatedIdFields(
   existingObject?: Partial<CreatedUpdatedId> | null,
 ): CreatedUpdatedId {
-  const now = Math.floor(Date.now() / 1000)
+  const now = nowUnix()
   return {
     created: existingObject?.created || now,
     id: existingObject?.id || stringId(),