@naturalcycles/redis-lib 4.3.1 → 4.4.0

package/dist/redisClient.d.ts

@@ -1,6 +1,6 @@
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { AnyObject, NullableBuffer, NullableString, Promisable, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { Redis, RedisOptions } from 'ioredis';
 import type { ScanStreamOptions } from 'ioredis/built/types.js';
 import type { ChainableCommander } from 'ioredis/built/utils/RedisCommander.js';
@@ -62,13 +62,13 @@ export declare class RedisClient implements CommonClient {
      Convenient type-safe wrapper.
      Returns BATCHES of keys in each iteration (as-is).
      */
-    scanStream(opt?: ScanStreamOptions): ReadableTyped<string[]>;
+    scanStream(opt?: ScanStreamOptions): Pipeline<string[]>;
     /**
      * Like scanStream, but flattens the stream of keys.
      */
-    scanStreamFlat(opt: ScanStreamOptions): ReadableTyped<string>;
+    scanStreamFlat(opt: ScanStreamOptions): Pipeline<string>;
     scanCount(opt: ScanStreamOptions): Promise<number>;
-    hscanStream(key: string, opt?: ScanStreamOptions): ReadableTyped<string[]>;
+    hscanStream(key: string, opt?: ScanStreamOptions): Pipeline<string[]>;
     hscanCount(key: string, opt?: ScanStreamOptions): Promise<number>;
     withPipeline(fn: (pipeline: ChainableCommander) => Promisable<void>): Promise<void>;
     private log;

package/dist/redisClient.js

@@ -1,5 +1,5 @@
-import { Transform } from 'node:stream';
 import { _stringMapEntries } from '@naturalcycles/js-lib/types';
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 /**
  Wraps the redis sdk with unified interface.
  Features:
@@ -174,7 +174,7 @@ export class RedisClient {
         await this.withPipeline(async (pipeline) => {
             await this.scanStream({
                 match: `${table}:*`,
-            }).forEach(keys => {
+            }).forEachSync(keys => {
                 pipeline.del(keys);
                 count += keys.length;
             });
@@ -187,7 +187,7 @@ export class RedisClient {
         await this.withPipeline(async (pipeline) => {
             await this.scanStream({
                 match: `*`,
-            }).forEach(keys => {
+            }).forEachSync(keys => {
                 pipeline.del(keys);
                 count += keys.length;
             });
@@ -199,7 +199,7 @@ export class RedisClient {
      Returns BATCHES of keys in each iteration (as-is).
      */
     scanStream(opt) {
-        return createReadableFromAsync(async () => {
+        return Pipeline.fromAsyncReadable(async () => {
             const redis = await this.redis();
             return redis.scanStream(opt);
         });
@@ -208,20 +208,19 @@ export class RedisClient {
      * Like scanStream, but flattens the stream of keys.
      */
     scanStreamFlat(opt) {
-        // biome-ignore lint/complexity/noFlatMapIdentity: ok
-        return this.scanStream(opt).flatMap(keys => keys);
+        return this.scanStream(opt).flatten();
     }
     async scanCount(opt) {
         const redis = await this.redis();
         // todo: implement more efficiently, e.g via LUA?
         let count = 0;
-        await redis.scanStream(opt).forEach(keys => {
+        await redis.scanStream(opt).forEach((keys) => {
             count += keys.length;
         });
         return count;
     }
     hscanStream(key, opt) {
-        return createReadableFromAsync(async () => {
+        return Pipeline.fromAsyncReadable(async () => {
             const redis = await this.redis();
             return redis.hscanStream(key, opt);
         });
@@ -245,20 +244,3 @@ export class RedisClient {
         this.cfg.logger.log(...args);
     }
 }
-/**
- * Turn async function into Readable.
- */
-function createReadableFromAsync(fn) {
-    const transform = new Transform({
-        objectMode: true,
-        transform: (chunk, _encoding, cb) => {
-            cb(null, chunk);
-        },
-    });
-    void fn()
-        .then(readable => {
-        readable.on('error', err => transform.destroy(err)).pipe(transform);
-    })
-        .catch(err => transform.destroy(err));
-    return transform;
-}

package/dist/redisHashKeyValueDB.d.ts

@@ -1,6 +1,6 @@
 import type { CommonDBCreateOptions } from '@naturalcycles/db-lib';
 import type { CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib/kv';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { RedisKeyValueDBCfg } from './redisKeyValueDB.js';
 /**
  * RedisHashKeyValueDB is a KeyValueDB implementation that uses hash fields to simulate tables.
@@ -23,9 +23,9 @@ export declare class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDispo
     getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
     deleteByIds(table: string, ids: string[]): Promise<void>;
     saveBatch(table: string, entries: KeyValueDBTuple[], opt?: CommonKeyValueDBSaveBatchOptions): Promise<void>;
-    streamIds(table: string, limit?: number): ReadableTyped<string>;
-    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
-    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    streamIds(table: string, limit?: number): Pipeline<string>;
+    streamValues(table: string, limit?: number): Pipeline<Buffer>;
+    streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple>;
     count(table: string): Promise<number>;
     incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]>;
     createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;

package/dist/redisHashKeyValueDB.js

@@ -49,22 +49,22 @@ export class RedisHashKeyValueDB {
         }
     }
     streamIds(table, limit) {
-        const stream = this.cfg.client
+        return this.cfg.client
             .hscanStream(table)
-            .flatMap(keyValueList => {
+            .mapSync(keyValueList => {
             const keys = [];
             for (let i = 0; i < keyValueList.length; i += 2) {
                 keys.push(keyValueList[i]);
             }
             return keys;
         })
-            .take(limit || Infinity);
-        return stream;
+            .flatten()
+            .limit(limit);
     }
     streamValues(table, limit) {
         return this.cfg.client
             .hscanStream(table)
-            .flatMap(keyValueList => {
+            .mapSync(keyValueList => {
             const values = [];
             for (let i = 0; i < keyValueList.length; i += 2) {
                 const value = Buffer.from(keyValueList[i + 1]);
@@ -72,12 +72,13 @@ export class RedisHashKeyValueDB {
             }
             return values;
         })
-            .take(limit || Infinity);
+            .flatten()
+            .limit(limit);
     }
     streamEntries(table, limit) {
         return this.cfg.client
             .hscanStream(table)
-            .flatMap(keyValueList => {
+            .mapSync(keyValueList => {
             const entries = [];
             for (let i = 0; i < keyValueList.length; i += 2) {
                 const key = keyValueList[i];
@@ -86,7 +87,8 @@ export class RedisHashKeyValueDB {
             }
             return entries;
         })
-            .take(limit || Infinity);
+            .flatten()
+            .limit(limit);
     }
     async count(table) {
         return await this.cfg.client.hscanCount(table);

package/dist/redisKeyValueDB.d.ts

@@ -1,6 +1,6 @@
 import type { CommonDBCreateOptions } from '@naturalcycles/db-lib';
 import type { CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib/kv';
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { RedisClient } from './redisClient.js';
 export interface RedisKeyValueDBCfg {
     client: RedisClient;
@@ -14,9 +14,9 @@ export declare class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposabl
     getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
     deleteByIds(table: string, ids: string[]): Promise<void>;
     saveBatch(table: string, entries: KeyValueDBTuple[], opt?: CommonKeyValueDBSaveBatchOptions): Promise<void>;
-    streamIds(table: string, limit?: number): ReadableTyped<string>;
-    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
-    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    streamIds(table: string, limit?: number): Pipeline<string>;
+    streamValues(table: string, limit?: number): Pipeline<Buffer>;
+    streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple>;
     count(table: string): Promise<number>;
     incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]>;
     createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;

package/dist/redisKeyValueDB.js

@@ -45,42 +45,38 @@ export class RedisKeyValueDB {
         }
     }
     streamIds(table, limit) {
-        let stream = this.cfg.client
+        return this.cfg.client
             .scanStream({
             match: `${table}:*`,
             // count: limit, // count is actually a "batchSize", not a limit
         })
-            .flatMap(keys => this.keysToIds(table, keys));
-        if (limit) {
-            stream = stream.take(limit);
-        }
-        return stream;
+            .mapSync(keys => this.keysToIds(table, keys))
+            .flatten()
+            .limit(limit);
     }
     streamValues(table, limit) {
         return this.cfg.client
             .scanStream({
             match: `${table}:*`,
         })
-            .flatMap(async (keys) => {
+            .map(async (keys) => {
             return (await this.cfg.client.mgetBuffer(keys)).filter(_isTruthy);
-        }, {
-            concurrency: 16,
-        })
-            .take(limit || Infinity);
+        }, { concurrency: 16 })
+            .flatten()
+            .limit(limit);
     }
     streamEntries(table, limit) {
         return this.cfg.client
             .scanStream({
             match: `${table}:*`,
         })
-            .flatMap(async (keys) => {
+            .map(async (keys) => {
             // casting as Buffer[], because values are expected to exist for given keys
             const bufs = (await this.cfg.client.mgetBuffer(keys));
             return _zip(this.keysToIds(table, keys), bufs);
-        }, {
-            concurrency: 16,
-        })
-            .take(limit || Infinity);
+        }, { concurrency: 16 })
+            .flatten()
+            .limit(limit);
     }
     async count(table) {
         // todo: implement more efficiently, e.g via LUA?

package/package.json

@@ -10,7 +10,7 @@
   },
   "devDependencies": {
     "@types/node": "^24",
-    "@naturalcycles/dev-lib": "18.4.2"
+    "@naturalcycles/dev-lib": "19.37.0"
   },
   "exports": {
     ".": "./dist/index.js"
@@ -37,7 +37,7 @@
   "engines": {
     "node": ">=22.12.0"
   },
-  "version": "4.3.1",
+  "version": "4.4.0",
   "description": "Redis implementation of CommonKeyValueDB interface",
   "author": "Natural Cycles Team",
   "license": "MIT",

package/src/redisClient.ts

@@ -1,8 +1,6 @@
-import { Transform } from 'node:stream'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import type {
   AnyObject,
-  AsyncFunction,
   NullableBuffer,
   NullableString,
   Promisable,
@@ -10,7 +8,7 @@ import type {
   UnixTimestamp,
 } from '@naturalcycles/js-lib/types'
 import { _stringMapEntries } from '@naturalcycles/js-lib/types'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type { Redis, RedisOptions } from 'ioredis'
 import type { ScanStreamOptions } from 'ioredis/built/types.js'
 import type { ChainableCommander } from 'ioredis/built/utils/RedisCommander.js'
@@ -258,7 +256,7 @@ export class RedisClient implements CommonClient {
     await this.withPipeline(async pipeline => {
       await this.scanStream({
         match: `${table}:*`,
-      }).forEach(keys => {
+      }).forEachSync(keys => {
         pipeline.del(keys)
         count += keys.length
       })
@@ -274,7 +272,7 @@ export class RedisClient implements CommonClient {
     await this.withPipeline(async pipeline => {
       await this.scanStream({
         match: `*`,
-      }).forEach(keys => {
+      }).forEachSync(keys => {
         pipeline.del(keys)
         count += keys.length
       })
@@ -287,8 +285,8 @@ export class RedisClient implements CommonClient {
    Convenient type-safe wrapper.
    Returns BATCHES of keys in each iteration (as-is).
    */
-  scanStream(opt?: ScanStreamOptions): ReadableTyped<string[]> {
-    return createReadableFromAsync(async () => {
+  scanStream(opt?: ScanStreamOptions): Pipeline<string[]> {
+    return Pipeline.fromAsyncReadable(async () => {
       const redis = await this.redis()
       return redis.scanStream(opt)
     })
@@ -297,9 +295,8 @@ export class RedisClient implements CommonClient {
   /**
    * Like scanStream, but flattens the stream of keys.
    */
-  scanStreamFlat(opt: ScanStreamOptions): ReadableTyped<string> {
-    // biome-ignore lint/complexity/noFlatMapIdentity: ok
-    return this.scanStream(opt).flatMap(keys => keys)
+  scanStreamFlat(opt: ScanStreamOptions): Pipeline<string> {
+    return this.scanStream(opt).flatten()
   }
 
   async scanCount(opt: ScanStreamOptions): Promise<number> {
@@ -307,15 +304,15 @@ export class RedisClient implements CommonClient {
     // todo: implement more efficiently, e.g via LUA?
     let count = 0
 
-    await (redis.scanStream(opt) as ReadableTyped<string[]>).forEach(keys => {
+    await redis.scanStream(opt).forEach((keys: string[]) => {
       count += keys.length
     })
 
     return count
   }
 
-  hscanStream(key: string, opt?: ScanStreamOptions): ReadableTyped<string[]> {
-    return createReadableFromAsync(async () => {
+  hscanStream(key: string, opt?: ScanStreamOptions): Pipeline<string[]> {
+    return Pipeline.fromAsyncReadable(async () => {
       const redis = await this.redis()
       return redis.hscanStream(key, opt)
     })
@@ -345,23 +342,3 @@ export class RedisClient implements CommonClient {
     this.cfg.logger.log(...args)
   }
 }
-
-/**
- * Turn async function into Readable.
- */
-function createReadableFromAsync<T>(fn: AsyncFunction<ReadableTyped<T>>): ReadableTyped<T> {
-  const transform = new Transform({
-    objectMode: true,
-    transform: (chunk, _encoding, cb) => {
-      cb(null, chunk)
-    },
-  })
-
-  void fn()
-    .then(readable => {
-      readable.on('error', err => transform.destroy(err)).pipe(transform)
-    })
-    .catch(err => transform.destroy(err))
-
-  return transform
-}

package/src/redisHashKeyValueDB.ts

@@ -6,7 +6,7 @@ import type {
   KeyValueDBTuple,
 } from '@naturalcycles/db-lib/kv'
 import { commonKeyValueDBFullSupport } from '@naturalcycles/db-lib/kv'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type { RedisKeyValueDBCfg } from './redisKeyValueDB.js'
 
 /**
@@ -64,25 +64,24 @@ export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
     }
   }
 
-  streamIds(table: string, limit?: number): ReadableTyped<string> {
-    const stream = this.cfg.client
+  streamIds(table: string, limit?: number): Pipeline<string> {
+    return this.cfg.client
       .hscanStream(table)
-      .flatMap(keyValueList => {
+      .mapSync(keyValueList => {
         const keys: string[] = []
         for (let i = 0; i < keyValueList.length; i += 2) {
           keys.push(keyValueList[i]!)
         }
         return keys
       })
-      .take(limit || Infinity)
-
-    return stream
+      .flatten()
+      .limit(limit)
   }
 
-  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+  streamValues(table: string, limit?: number): Pipeline<Buffer> {
     return this.cfg.client
       .hscanStream(table)
-      .flatMap(keyValueList => {
+      .mapSync(keyValueList => {
         const values: Buffer[] = []
         for (let i = 0; i < keyValueList.length; i += 2) {
           const value = Buffer.from(keyValueList[i + 1]!)
@@ -90,13 +89,14 @@ export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
         }
         return values
       })
-      .take(limit || Infinity)
+      .flatten()
+      .limit(limit)
   }
 
-  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
+  streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple> {
     return this.cfg.client
       .hscanStream(table)
-      .flatMap(keyValueList => {
+      .mapSync(keyValueList => {
         const entries: [string, Buffer][] = []
         for (let i = 0; i < keyValueList.length; i += 2) {
           const key = keyValueList[i]!
@@ -105,7 +105,8 @@ export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
         }
         return entries
       })
-      .take(limit || Infinity)
+      .flatten()
+      .limit(limit)
   }
 
   async count(table: string): Promise<number> {

package/src/redisKeyValueDB.ts

@@ -8,7 +8,7 @@ import type {
 import { commonKeyValueDBFullSupport } from '@naturalcycles/db-lib/kv'
 import { _isTruthy } from '@naturalcycles/js-lib'
 import { _zip } from '@naturalcycles/js-lib/array/array.util.js'
-import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import type { RedisClient } from './redisClient.js'
 
 export interface RedisKeyValueDBCfg {
@@ -65,53 +65,47 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
     }
   }
 
-  streamIds(table: string, limit?: number): ReadableTyped<string> {
-    let stream = this.cfg.client
+  streamIds(table: string, limit?: number): Pipeline<string> {
+    return this.cfg.client
       .scanStream({
         match: `${table}:*`,
         // count: limit, // count is actually a "batchSize", not a limit
       })
-      .flatMap(keys => this.keysToIds(table, keys))
-
-    if (limit) {
-      stream = stream.take(limit)
-    }
-
-    return stream
+      .mapSync(keys => this.keysToIds(table, keys))
+      .flatten()
+      .limit(limit)
   }
 
-  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+  streamValues(table: string, limit?: number): Pipeline<Buffer> {
     return this.cfg.client
       .scanStream({
         match: `${table}:*`,
       })
-      .flatMap(
+      .map(
         async keys => {
           return (await this.cfg.client.mgetBuffer(keys)).filter(_isTruthy)
         },
-        {
-          concurrency: 16,
-        },
+        { concurrency: 16 },
       )
-      .take(limit || Infinity)
+      .flatten()
+      .limit(limit)
   }
 
-  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
+  streamEntries(table: string, limit?: number): Pipeline<KeyValueDBTuple> {
     return this.cfg.client
       .scanStream({
         match: `${table}:*`,
       })
-      .flatMap(
+      .map(
         async keys => {
           // casting as Buffer[], because values are expected to exist for given keys
           const bufs = (await this.cfg.client.mgetBuffer(keys)) as Buffer[]
           return _zip(this.keysToIds(table, keys), bufs)
         },
-        {
-          concurrency: 16,
-        },
+        { concurrency: 16 },
       )
-      .take(limit || Infinity)
+      .flatten()
+      .limit(limit)
   }
 
   async count(table: string): Promise<number> {