@naturalcycles/db-lib 9.10.0 → 9.11.0

package/dist/commondao/common.dao.js

@@ -404,7 +404,7 @@ class CommonDao {
         // Experimental: using `.map()`
         const stream = this.cfg.db
             .streamQuery(q.select(['id']), opt)
-            .on('error', err => stream.emit('error', err))
+            // .on('error', err => stream.emit('error', err))
             .map((r) => r.id);
         // const stream: ReadableTyped<string> = this.cfg.db
         //   .streamQuery<DBM>(q.select(['id']), opt)

package/dist/kv/commonKeyValueDB.d.ts

@@ -1,7 +1,15 @@
 /// <reference types="node" />
+import { UnixTimestampNumber } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { CommonDBCreateOptions } from '../db.model';
 export type KeyValueDBTuple = [key: string, value: Buffer];
+export interface CommonKeyValueDBSaveBatchOptions {
+    /**
+     * If set (and if it's implemented by the driver) - will set expiry TTL for each key of the batch.
+     * E.g EXAT in Redis.
+     */
+    expireAt?: UnixTimestampNumber;
+}
 /**
  * Common interface for Key-Value database implementations.
  *
@@ -24,7 +32,7 @@ export interface CommonKeyValueDB {
      */
     getByIds: (table: string, ids: string[]) => Promise<KeyValueDBTuple[]>;
     deleteByIds: (table: string, ids: string[]) => Promise<void>;
-    saveBatch: (table: string, entries: KeyValueDBTuple[]) => Promise<void>;
+    saveBatch: (table: string, entries: KeyValueDBTuple[], opt?: CommonKeyValueDBSaveBatchOptions) => Promise<void>;
     streamIds: (table: string, limit?: number) => ReadableTyped<string>;
     streamValues: (table: string, limit?: number) => ReadableTyped<Buffer>;
     streamEntries: (table: string, limit?: number) => ReadableTyped<KeyValueDBTuple>;

package/dist/kv/commonKeyValueDao.js

@@ -132,7 +132,7 @@ class CommonKeyValueDao {
         }
         const stream = this.cfg.db
             .streamValues(this.cfg.table, limit)
-            .on('error', err => stream.emit('error', err))
+            // .on('error', err => stream.emit('error', err))
             .flatMap(async (buf) => {
             try {
                 return [await mapBufferToValue(buf)];
@@ -151,7 +151,7 @@ class CommonKeyValueDao {
         }
         const stream = this.cfg.db
             .streamEntries(this.cfg.table, limit)
-            .on('error', err => stream.emit('error', err))
+            // .on('error', err => stream.emit('error', err))
             .flatMap(async ([id, buf]) => {
             try {
                 return [[id, await mapBufferToValue(buf)]];

package/dist/testing/daoTest.js

@@ -155,7 +155,7 @@ function runCommonDaoTest(db, quirks = {}) {
             (0, dbTest_1.expectMatch)(expectedItems.map(i => i.id), ids, quirks);
         });
         test('streamQueryIds all', async () => {
-            let ids = await (0, nodejs_lib_1.readableToArray)(dao.query().streamQueryIds());
+            let ids = await dao.query().streamQueryIds().toArray();
             ids = ids.sort();
             (0, dbTest_1.expectMatch)(expectedItems.map(i => i.id), ids, quirks);
         });

package/dist/testing/dbTest.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.expectMatch = exports.runCommonDBTest = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const common_db_1 = require("../common.db");
 const db_model_1 = require("../db.model");
 const dbQuery_1 = require("../query/dbQuery");
@@ -152,7 +151,7 @@ function runCommonDBTest(db, quirks = {}) {
     // STREAM
     if (support.streaming) {
         test('streamQuery all', async () => {
-            let rows = await (0, nodejs_lib_1.readableToArray)(db.streamQuery(queryAll()));
+            let rows = await db.streamQuery(queryAll()).toArray();
             rows = (0, js_lib_1._sortBy)(rows, r => r.id); // cause order is not specified in DBQuery
             expectMatch(items, rows, quirks);
         });

package/dist/testing/keyValueDBTest.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runCommonKeyValueDBTest = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const test_model_1 = require("./test.model");
 const testIds = (0, js_lib_1._range)(1, 4).map(n => `id${n}`);
 const testEntries = testIds.map(id => [id, Buffer.from(`${id}value`)]);
@@ -33,35 +32,35 @@ function runCommonKeyValueDBTest(db) {
         expect(await db.count(test_model_1.TEST_TABLE)).toBe(3);
     });
     test('streamIds', async () => {
-        const ids = await (0, nodejs_lib_1.readableToArray)(db.streamIds(test_model_1.TEST_TABLE));
+        const ids = await db.streamIds(test_model_1.TEST_TABLE).toArray();
         ids.sort();
         expect(ids).toEqual(testIds);
     });
     test('streamIds limited', async () => {
-        const idsLimited = await (0, nodejs_lib_1.readableToArray)(db.streamIds(test_model_1.TEST_TABLE, 2));
+        const idsLimited = await db.streamIds(test_model_1.TEST_TABLE, 2).toArray();
         // Order is non-deterministic, so, cannot compare values
         // idsLimited.sort()
         // expect(idsLimited).toEqual(testIds.slice(0, 2))
         expect(idsLimited.length).toBe(2);
     });
     test('streamValues', async () => {
-        const values = await (0, nodejs_lib_1.readableToArray)(db.streamValues(test_model_1.TEST_TABLE));
+        const values = await db.streamValues(test_model_1.TEST_TABLE).toArray();
         values.sort();
         expect(values).toEqual(testEntries.map(e => e[1]));
     });
     test('streamValues limited', async () => {
-        const valuesLimited = await (0, nodejs_lib_1.readableToArray)(db.streamValues(test_model_1.TEST_TABLE, 2));
+        const valuesLimited = await db.streamValues(test_model_1.TEST_TABLE, 2).toArray();
         // valuesLimited.sort()
         // expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
         expect(valuesLimited.length).toBe(2);
     });
     test('streamEntries', async () => {
-        const entries = await (0, nodejs_lib_1.readableToArray)(db.streamEntries(test_model_1.TEST_TABLE));
+        const entries = await db.streamEntries(test_model_1.TEST_TABLE).toArray();
         entries.sort();
         expect(entries).toEqual(testEntries);
     });
     test('streamEntries limited', async () => {
-        const entriesLimited = await (0, nodejs_lib_1.readableToArray)(db.streamEntries(test_model_1.TEST_TABLE, 2));
+        const entriesLimited = await db.streamEntries(test_model_1.TEST_TABLE, 2).toArray();
         // entriesLimited.sort()
         // expect(entriesLimited).toEqual(testEntries.slice(0, 2))
         expect(entriesLimited.length).toBe(2);

package/dist/testing/keyValueDaoTest.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runCommonKeyValueDaoTest = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const testIds = (0, js_lib_1._range)(1, 4).map(n => `id${n}`);
 const testEntries = testIds.map(id => [id, Buffer.from(`${id}value`)]);
 function runCommonKeyValueDaoTest(dao) {
@@ -26,35 +25,35 @@ function runCommonKeyValueDaoTest(dao) {
         expect(entries).toEqual(testEntries);
     });
     test('streamIds', async () => {
-        const ids = await (0, nodejs_lib_1.readableToArray)(dao.streamIds());
+        const ids = await dao.streamIds().toArray();
         ids.sort();
         expect(ids).toEqual(testIds);
     });
     test('streamIds limited', async () => {
-        const idsLimited = await (0, nodejs_lib_1.readableToArray)(dao.streamIds(2));
+        const idsLimited = await dao.streamIds(2).toArray();
         // Order is non-deterministic, so, cannot compare values
         // idsLimited.sort()
         // expect(idsLimited).toEqual(testIds.slice(0, 2))
         expect(idsLimited.length).toBe(2);
     });
     test('streamValues', async () => {
-        const values = await (0, nodejs_lib_1.readableToArray)(dao.streamValues());
+        const values = await dao.streamValues().toArray();
         values.sort();
         expect(values).toEqual(testEntries.map(e => e[1]));
     });
     test('streamValues limited', async () => {
-        const valuesLimited = await (0, nodejs_lib_1.readableToArray)(dao.streamValues(2));
+        const valuesLimited = await dao.streamValues(2).toArray();
         // valuesLimited.sort()
         // expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
         expect(valuesLimited.length).toBe(2);
     });
     test('streamEntries', async () => {
-        const entries = await (0, nodejs_lib_1.readableToArray)(dao.streamEntries());
+        const entries = await dao.streamEntries().toArray();
         entries.sort();
         expect(entries).toEqual(testEntries);
     });
     test('streamEntries limited', async () => {
-        const entriesLimited = await (0, nodejs_lib_1.readableToArray)(dao.streamEntries(2));
+        const entriesLimited = await dao.streamEntries(2).toArray();
         // entriesLimited.sort()
         // expect(entriesLimited).toEqual(testEntries.slice(0, 2))
         expect(entriesLimited.length).toBe(2);

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "9.10.0",
+  "version": "9.11.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/src/commondao/common.dao.ts

@@ -495,7 +495,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
 
     const stream = this.cfg.db.streamQuery<DBM>(q, opt)
     const partialQuery = !!q._selectedFieldNames
-    if (partialQuery) return stream
+    if (partialQuery) return stream as any
 
     // This almost works, but hard to implement `errorMode: THROW_AGGREGATED` in this case
     // return stream.flatMap(async (dbm: DBM) => {
@@ -551,7 +551,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     // Experimental: using `.map()`
     const stream: ReadableTyped<string> = this.cfg.db
       .streamQuery<DBM>(q.select(['id']), opt)
-      .on('error', err => stream.emit('error', err))
+      // .on('error', err => stream.emit('error', err))
       .map((r: ObjectWithId) => r.id)
 
     // const stream: ReadableTyped<string> = this.cfg.db

package/src/kv/commonKeyValueDB.ts

@@ -1,8 +1,17 @@
+import { UnixTimestampNumber } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { CommonDBCreateOptions } from '../db.model'
 
 export type KeyValueDBTuple = [key: string, value: Buffer]
 
+export interface CommonKeyValueDBSaveBatchOptions {
+  /**
+   * If set (and if it's implemented by the driver) - will set expiry TTL for each key of the batch.
+   * E.g EXAT in Redis.
+   */
+  expireAt?: UnixTimestampNumber
+}
+
 /**
  * Common interface for Key-Value database implementations.
  *
@@ -29,7 +38,11 @@ export interface CommonKeyValueDB {
 
   deleteByIds: (table: string, ids: string[]) => Promise<void>
 
-  saveBatch: (table: string, entries: KeyValueDBTuple[]) => Promise<void>
+  saveBatch: (
+    table: string,
+    entries: KeyValueDBTuple[],
+    opt?: CommonKeyValueDBSaveBatchOptions,
+  ) => Promise<void>
 
   streamIds: (table: string, limit?: number) => ReadableTyped<string>
   streamValues: (table: string, limit?: number) => ReadableTyped<Buffer>

package/src/kv/commonKeyValueDao.ts

@@ -201,15 +201,15 @@ export class CommonKeyValueDao<T> {
     const { mapBufferToValue } = this.cfg.hooks
 
     if (!mapBufferToValue) {
-      return this.cfg.db.streamValues(this.cfg.table, limit)
+      return this.cfg.db.streamValues(this.cfg.table, limit) as ReadableTyped<T>
     }
 
     const stream: ReadableTyped<T> = this.cfg.db
       .streamValues(this.cfg.table, limit)
-      .on('error', err => stream.emit('error', err))
-      .flatMap(async (buf: Buffer) => {
+      // .on('error', err => stream.emit('error', err))
+      .flatMap(async buf => {
         try {
-          return [await mapBufferToValue(buf)] satisfies T[]
+          return [await mapBufferToValue(buf)]
         } catch (err) {
           this.cfg.logger.error(err)
           return [] // SKIP
@@ -223,15 +223,17 @@ export class CommonKeyValueDao<T> {
     const { mapBufferToValue } = this.cfg.hooks
 
     if (!mapBufferToValue) {
-      return this.cfg.db.streamEntries(this.cfg.table, limit)
+      return this.cfg.db.streamEntries(this.cfg.table, limit) as ReadableTyped<
+        KeyValueTuple<string, T>
+      >
     }
 
     const stream: ReadableTyped<KeyValueTuple<string, T>> = this.cfg.db
       .streamEntries(this.cfg.table, limit)
-      .on('error', err => stream.emit('error', err))
-      .flatMap(async ([id, buf]: KeyValueTuple<string, Buffer>) => {
+      // .on('error', err => stream.emit('error', err))
+      .flatMap(async ([id, buf]) => {
         try {
-          return [[id, await mapBufferToValue(buf)]] satisfies KeyValueTuple<string, T>[]
+          return [[id, await mapBufferToValue(buf)]]
         } catch (err) {
           this.cfg.logger.error(err)
           return [] // SKIP

package/src/testing/daoTest.ts

@@ -1,6 +1,6 @@
 import { Readable } from 'node:stream'
 import { _deepCopy, _pick, _sortBy, _omit, localTimeNow } from '@naturalcycles/js-lib'
-import { _pipeline, readableToArray } from '@naturalcycles/nodejs-lib'
+import { _pipeline } from '@naturalcycles/nodejs-lib'
 import { CommonDaoLogLevel, DBQuery } from '..'
 import { CommonDB } from '../common.db'
 import { CommonDao } from '../commondao/common.dao'
@@ -212,7 +212,7 @@ export function runCommonDaoTest(db: CommonDB, quirks: CommonDBImplementationQui
     })
 
     test('streamQueryIds all', async () => {
-      let ids = await readableToArray(dao.query().streamQueryIds())
+      let ids = await dao.query().streamQueryIds().toArray()
       ids = ids.sort()
       expectMatch(
         expectedItems.map(i => i.id),

package/src/testing/dbTest.ts

@@ -1,5 +1,4 @@
 import { _filterObject, _pick, _sortBy, pMap } from '@naturalcycles/js-lib'
-import { readableToArray } from '@naturalcycles/nodejs-lib'
 import { CommonDB, CommonDBType } from '../common.db'
 import { DBIncrement, DBPatch } from '../db.model'
 import { DBQuery } from '../query/dbQuery'
@@ -220,7 +219,7 @@ export function runCommonDBTest(db: CommonDB, quirks: CommonDBImplementationQuir
   // STREAM
   if (support.streaming) {
     test('streamQuery all', async () => {
-      let rows = await readableToArray(db.streamQuery(queryAll()))
+      let rows = await db.streamQuery(queryAll()).toArray()
 
       rows = _sortBy(rows, r => r.id) // cause order is not specified in DBQuery
       expectMatch(items, rows, quirks)

package/src/testing/keyValueDBTest.ts

@@ -1,5 +1,4 @@
 import { _range, _sortBy } from '@naturalcycles/js-lib'
-import { readableToArray } from '@naturalcycles/nodejs-lib'
 import { CommonKeyValueDB, KeyValueDBTuple } from '../kv/commonKeyValueDB'
 import { TEST_TABLE } from './test.model'
 
@@ -42,13 +41,13 @@ export function runCommonKeyValueDBTest(db: CommonKeyValueDB): void {
   })
 
   test('streamIds', async () => {
-    const ids = await readableToArray(db.streamIds(TEST_TABLE))
+    const ids = await db.streamIds(TEST_TABLE).toArray()
     ids.sort()
     expect(ids).toEqual(testIds)
   })
 
   test('streamIds limited', async () => {
-    const idsLimited = await readableToArray(db.streamIds(TEST_TABLE, 2))
+    const idsLimited = await db.streamIds(TEST_TABLE, 2).toArray()
     // Order is non-deterministic, so, cannot compare values
     // idsLimited.sort()
     // expect(idsLimited).toEqual(testIds.slice(0, 2))
@@ -56,26 +55,26 @@ export function runCommonKeyValueDBTest(db: CommonKeyValueDB): void {
   })
 
   test('streamValues', async () => {
-    const values = await readableToArray(db.streamValues(TEST_TABLE))
+    const values = await db.streamValues(TEST_TABLE).toArray()
     values.sort()
     expect(values).toEqual(testEntries.map(e => e[1]))
   })
 
   test('streamValues limited', async () => {
-    const valuesLimited = await readableToArray(db.streamValues(TEST_TABLE, 2))
+    const valuesLimited = await db.streamValues(TEST_TABLE, 2).toArray()
     // valuesLimited.sort()
     // expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
     expect(valuesLimited.length).toBe(2)
   })
 
   test('streamEntries', async () => {
-    const entries = await readableToArray(db.streamEntries(TEST_TABLE))
+    const entries = await db.streamEntries(TEST_TABLE).toArray()
     entries.sort()
     expect(entries).toEqual(testEntries)
   })
 
   test('streamEntries limited', async () => {
-    const entriesLimited = await readableToArray(db.streamEntries(TEST_TABLE, 2))
+    const entriesLimited = await db.streamEntries(TEST_TABLE, 2).toArray()
     // entriesLimited.sort()
     // expect(entriesLimited).toEqual(testEntries.slice(0, 2))
     expect(entriesLimited.length).toBe(2)

package/src/testing/keyValueDaoTest.ts

@@ -1,5 +1,4 @@
 import { _range, _sortBy } from '@naturalcycles/js-lib'
-import { readableToArray } from '@naturalcycles/nodejs-lib'
 import { KeyValueDBTuple } from '../kv/commonKeyValueDB'
 import { CommonKeyValueDao } from '../kv/commonKeyValueDao'
 
@@ -33,13 +32,13 @@ export function runCommonKeyValueDaoTest(dao: CommonKeyValueDao<Buffer>): void {
   })
 
   test('streamIds', async () => {
-    const ids = await readableToArray(dao.streamIds())
+    const ids = await dao.streamIds().toArray()
     ids.sort()
     expect(ids).toEqual(testIds)
   })
 
   test('streamIds limited', async () => {
-    const idsLimited = await readableToArray(dao.streamIds(2))
+    const idsLimited = await dao.streamIds(2).toArray()
     // Order is non-deterministic, so, cannot compare values
     // idsLimited.sort()
     // expect(idsLimited).toEqual(testIds.slice(0, 2))
@@ -47,26 +46,26 @@ export function runCommonKeyValueDaoTest(dao: CommonKeyValueDao<Buffer>): void {
   })
 
   test('streamValues', async () => {
-    const values = await readableToArray(dao.streamValues())
+    const values = await dao.streamValues().toArray()
     values.sort()
     expect(values).toEqual(testEntries.map(e => e[1]))
   })
 
   test('streamValues limited', async () => {
-    const valuesLimited = await readableToArray(dao.streamValues(2))
+    const valuesLimited = await dao.streamValues(2).toArray()
     // valuesLimited.sort()
     // expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
     expect(valuesLimited.length).toBe(2)
   })
 
   test('streamEntries', async () => {
-    const entries = await readableToArray(dao.streamEntries())
+    const entries = await dao.streamEntries().toArray()
     entries.sort()
     expect(entries).toEqual(testEntries)
   })
 
   test('streamEntries limited', async () => {
-    const entriesLimited = await readableToArray(dao.streamEntries(2))
+    const entriesLimited = await dao.streamEntries(2).toArray()
     // entriesLimited.sort()
     // expect(entriesLimited).toEqual(testEntries.slice(0, 2))
     expect(entriesLimited.length).toBe(2)