@naturalcycles/redis-lib 3.1.1 → 3.3.0

package/dist/redisClient.d.ts

@@ -47,11 +47,13 @@ export declare class RedisClient implements CommonClient {
     hmget(key: string, fields: string[]): Promise<NullableString[]>;
     hmgetBuffer(key: string, fields: string[]): Promise<NullableBuffer[]>;
     hincr(key: string, field: string, increment?: number): Promise<number>;
+    hincrBatch(key: string, incrementTuples: [string, number][]): Promise<[string, number][]>;
     setWithTTL(key: string, value: string | number | Buffer, expireAt: UnixTimestampNumber): Promise<void>;
-    hsetWithTTL(key: string, value: AnyObject, expireAt: UnixTimestampNumber): Promise<void>;
+    hsetWithTTL(_key: string, _value: AnyObject, _expireAt: UnixTimestampNumber): Promise<void>;
     mset(obj: Record<string, string | number>): Promise<void>;
     msetBuffer(obj: Record<string, Buffer>): Promise<void>;
     incr(key: string, by?: number): Promise<number>;
+    incrBatch(incrementTuples: [string, number][]): Promise<[string, number][]>;
     ttl(key: string): Promise<number>;
     dropTable(table: string): Promise<void>;
     clearAll(): Promise<void>;
@@ -59,14 +61,14 @@ export declare class RedisClient implements CommonClient {
      Convenient type-safe wrapper.
      Returns BATCHES of keys in each iteration (as-is).
      */
-    scanStream(opt: ScanStreamOptions): ReadableTyped<string[]>;
+    scanStream(opt?: ScanStreamOptions): ReadableTyped<string[]>;
     /**
      * Like scanStream, but flattens the stream of keys.
      */
     scanStreamFlat(opt: ScanStreamOptions): ReadableTyped<string>;
     scanCount(opt: ScanStreamOptions): Promise<number>;
-    hscanStream(key: string, opt: ScanStreamOptions): ReadableTyped<string[]>;
-    hscanCount(key: string, opt: ScanStreamOptions): Promise<number>;
+    hscanStream(key: string, opt?: ScanStreamOptions): ReadableTyped<string[]>;
+    hscanCount(key: string, opt?: ScanStreamOptions): Promise<number>;
     withPipeline(fn: (pipeline: ChainableCommander) => Promisable<void>): Promise<void>;
     private log;
 }

package/dist/redisClient.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RedisClient = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
 /**
  Wraps the redis sdk with unified interface.
  Features:
@@ -98,17 +99,30 @@ class RedisClient {
     async hincr(key, field, increment = 1) {
         return await this.redis().hincrby(key, field, increment);
     }
+    async hincrBatch(key, incrementTuples) {
+        const results = {};
+        await this.withPipeline(async (pipeline) => {
+            for (const [field, increment] of incrementTuples) {
+                pipeline.hincrby(key, field, increment, (_err, newValue) => {
+                    results[field] = newValue;
+                });
+            }
+        });
+        const validResults = (0, js_lib_1._stringMapEntries)(results).filter(([_, v]) => v !== undefined);
+        return validResults;
+    }
     async setWithTTL(key, value, expireAt) {
         await this.redis().set(key, value, 'EXAT', expireAt);
     }
-    async hsetWithTTL(key, value, expireAt) {
-        const valueKeys = Object.keys(value);
-        const numberOfKeys = valueKeys.length;
-        const keyList = valueKeys.join(' ');
-        const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`;
-        const [command, ...args] = commandString.split(' ');
-        await this.redis().hset(key, value);
-        await this.redis().call(command, args);
+    async hsetWithTTL(_key, _value, _expireAt) {
+        throw new Error('Not supported until Redis 7.4.0');
+        // const valueKeys = Object.keys(value)
+        // const numberOfKeys = valueKeys.length
+        // const keyList = valueKeys.join(' ')
+        // const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`
+        // const [command, ...args] = commandString.split(' ')
+        // await this.redis().hset(key, value)
+        // await this.redis().call(command!, args)
     }
     async mset(obj) {
         await this.redis().mset(obj);
@@ -119,6 +133,18 @@ class RedisClient {
     async incr(key, by = 1) {
         return await this.redis().incrby(key, by);
     }
+    async incrBatch(incrementTuples) {
+        const results = {};
+        await this.withPipeline(async (pipeline) => {
+            for (const [key, increment] of incrementTuples) {
+                pipeline.incrby(key, increment, (_err, newValue) => {
+                    results[key] = newValue;
+                });
+            }
+        });
+        const validResults = (0, js_lib_1._stringMapEntries)(results).filter(([_, v]) => v !== undefined);
+        return validResults;
+    }
     async ttl(key) {
         return await this.redis().ttl(key);
     }

package/dist/redisHashKeyValueDB.d.ts

@@ -1,15 +1,21 @@
-import { CommonDBCreateOptions, CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, KeyValueDBTuple } from '@naturalcycles/db-lib';
-import { StringMap } from '@naturalcycles/js-lib';
+import { CommonDBCreateOptions, CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { RedisClient } from './redisClient';
 import { RedisKeyValueDBCfg } from './redisKeyValueDB';
-export interface RedisHashKeyValueDBCfg extends RedisKeyValueDBCfg {
-    hashKey: string;
-}
-export declare class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
-    client: RedisClient;
-    keyOfHashField: string;
-    constructor(cfg: RedisHashKeyValueDBCfg);
+/**
+ * RedisHashKeyValueDB is a KeyValueDB implementation that uses hash fields to simulate tables.
+ * The value in the `table` arguments points to a hash field in Redis.
+ *
+ * The reason for having this approach and also the traditional RedisKeyValueDB is that
+ * the currently available Redis versions (in Memorystore, or on MacOs) do not support
+ * expiring hash properties.
+ * The expiring fields feature is important, and only available via RedisKeyValueDB.
+ *
+ * Once the available Redis version reaches 7.4.0+,
+ * this implementation can take over for RedisKeyValueDB.
+ */
+export declare class RedishHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
+    cfg: RedisKeyValueDBCfg;
+    constructor(cfg: RedisKeyValueDBCfg);
     support: {
         count?: boolean;
         increment?: boolean;
@@ -23,11 +29,6 @@ export declare class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDispo
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
     count(table: string): Promise<number>;
-    increment(table: string, id: string, by?: number): Promise<number>;
-    incrementBatch(_table: string, _incrementMap: StringMap<number>): Promise<StringMap<number>>;
+    incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]>;
     createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
-    private idsToKeys;
-    private idToKey;
-    private keysToIds;
-    private keyToId;
 }

package/dist/redisHashKeyValueDB.js

@@ -1,121 +1,105 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RedisHashKeyValueDB = void 0;
+exports.RedishHashKeyValueDB = void 0;
 const db_lib_1 = require("@naturalcycles/db-lib");
-const js_lib_1 = require("@naturalcycles/js-lib");
-class RedisHashKeyValueDB {
+/**
+ * RedisHashKeyValueDB is a KeyValueDB implementation that uses hash fields to simulate tables.
+ * The value in the `table` arguments points to a hash field in Redis.
+ *
+ * The reason for having this approach and also the traditional RedisKeyValueDB is that
+ * the currently available Redis versions (in Memorystore, or on MacOs) do not support
+ * expiring hash properties.
+ * The expiring fields feature is important, and only available via RedisKeyValueDB.
+ *
+ * Once the available Redis version reaches 7.4.0+,
+ * this implementation can take over for RedisKeyValueDB.
+ */
+class RedishHashKeyValueDB {
     constructor(cfg) {
+        this.cfg = cfg;
         this.support = {
             ...db_lib_1.commonKeyValueDBFullSupport,
         };
-        this.client = cfg.client;
-        this.keyOfHashField = cfg.hashKey;
     }
     async ping() {
-        await this.client.ping();
+        await this.cfg.client.ping();
     }
     async [Symbol.asyncDispose]() {
-        await this.client.disconnect();
+        await this.cfg.client.disconnect();
     }
     async getByIds(table, ids) {
         if (!ids.length)
             return [];
         // we assume that the order of returned values is the same as order of input ids
-        const bufs = await this.client.hmgetBuffer(this.keyOfHashField, this.idsToKeys(table, ids));
+        const bufs = await this.cfg.client.hmgetBuffer(table, ids);
         return bufs.map((buf, i) => [ids[i], buf]).filter(([_k, v]) => v !== null);
     }
     async deleteByIds(table, ids) {
         if (!ids.length)
             return;
-        await this.client.hdel(this.keyOfHashField, this.idsToKeys(table, ids));
+        await this.cfg.client.hdel(table, ids);
     }
     async saveBatch(table, entries, opt) {
         if (!entries.length)
             return;
-        const entriesWithKey = entries.map(([k, v]) => [this.idToKey(table, k), v]);
-        const map = Object.fromEntries(entriesWithKey);
+        const record = Object.fromEntries(entries);
         if (opt?.expireAt) {
-            await this.client.hsetWithTTL(this.keyOfHashField, map, opt.expireAt);
+            await this.cfg.client.hsetWithTTL(table, record, opt.expireAt);
         }
         else {
-            await this.client.hset(this.keyOfHashField, map);
+            await this.cfg.client.hset(table, record);
         }
     }
     streamIds(table, limit) {
-        let stream = this.client
-            .hscanStream(this.keyOfHashField, {
-            match: `${table}:*`,
-        })
+        const stream = this.cfg.client
+            .hscanStream(table)
             .flatMap(keyValueList => {
             const keys = [];
-            keyValueList.forEach((keyOrValue, index) => {
-                if (index % 2 !== 0)
-                    return;
-                keys.push(keyOrValue);
-            });
-            return this.keysToIds(table, keys);
-        });
-        if (limit) {
-            stream = stream.take(limit);
-        }
+            for (let i = 0; i < keyValueList.length; i += 2) {
+                keys.push(keyValueList[i]);
+            }
+            return keys;
+        })
+            .take(limit || Infinity);
         return stream;
     }
     streamValues(table, limit) {
-        return this.client
-            .hscanStream(this.keyOfHashField, {
-            match: `${table}:*`,
-        })
+        return this.cfg.client
+            .hscanStream(table)
             .flatMap(keyValueList => {
             const values = [];
-            keyValueList.forEach((keyOrValue, index) => {
-                if (index % 2 !== 1)
-                    return;
-                values.push(keyOrValue);
-            });
-            return values.map(v => Buffer.from(v));
+            for (let i = 0; i < keyValueList.length; i += 2) {
+                const value = Buffer.from(keyValueList[i + 1]);
+                values.push(value);
+            }
+            return values;
         })
             .take(limit || Infinity);
     }
     streamEntries(table, limit) {
-        return this.client
-            .hscanStream(this.keyOfHashField, {
-            match: `${table}:*`,
-        })
+        return this.cfg.client
+            .hscanStream(table)
             .flatMap(keyValueList => {
-            const entries = (0, js_lib_1._chunk)(keyValueList, 2);
-            return entries.map(([k, v]) => {
-                return [this.keyToId(table, String(k)), Buffer.from(String(v))];
-            });
+            const entries = [];
+            for (let i = 0; i < keyValueList.length; i += 2) {
+                const key = keyValueList[i];
+                const value = Buffer.from(keyValueList[i + 1]);
+                entries.push([key, value]);
+            }
+            return entries;
         })
             .take(limit || Infinity);
     }
     async count(table) {
-        return await this.client.hscanCount(this.keyOfHashField, {
-            match: `${table}:*`,
-        });
+        return await this.cfg.client.hscanCount(table);
     }
-    async increment(table, id, by = 1) {
-        return await this.client.hincr(this.keyOfHashField, this.idToKey(table, id), by);
-    }
-    async incrementBatch(_table, _incrementMap) {
-        throw new Error('Not implemented');
+    async incrementBatch(table, increments) {
+        return await this.cfg.client.hincrBatch(table, increments);
     }
     async createTable(table, opt) {
         if (!opt?.dropIfExists)
             return;
-        await this.client.dropTable(table);
-    }
-    idsToKeys(table, ids) {
-        return ids.map(id => this.idToKey(table, id));
-    }
-    idToKey(table, id) {
-        return `${table}:${id}`;
-    }
-    keysToIds(table, keys) {
-        return keys.map(key => this.keyToId(table, key));
-    }
-    keyToId(table, key) {
-        return key.slice(table.length + 1);
+        await this.cfg.client.del([table]);
     }
 }
-exports.RedisHashKeyValueDB = RedisHashKeyValueDB;
+exports.RedishHashKeyValueDB = RedishHashKeyValueDB;

package/dist/redisKeyValueDB.d.ts

@@ -1,13 +1,12 @@
-import { CommonDBCreateOptions, CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, KeyValueDBTuple } from '@naturalcycles/db-lib';
-import { StringMap } from '@naturalcycles/js-lib';
+import { CommonDBCreateOptions, CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { RedisClient } from './redisClient';
 export interface RedisKeyValueDBCfg {
     client: RedisClient;
 }
 export declare class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
+    cfg: RedisKeyValueDBCfg;
     constructor(cfg: RedisKeyValueDBCfg);
-    client: RedisClient;
     support: {
         count?: boolean;
         increment?: boolean;
@@ -21,8 +20,7 @@ export declare class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposabl
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
     count(table: string): Promise<number>;
-    increment(table: string, id: string, by?: number): Promise<number>;
-    incrementBatch(_table: string, _incrementMap: StringMap<number>): Promise<StringMap<number>>;
+    incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]>;
     createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
     private idsToKeys;
     private idToKey;

package/dist/redisKeyValueDB.js

@@ -5,28 +5,28 @@ const db_lib_1 = require("@naturalcycles/db-lib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 class RedisKeyValueDB {
     constructor(cfg) {
+        this.cfg = cfg;
         this.support = {
             ...db_lib_1.commonKeyValueDBFullSupport,
         };
-        this.client = cfg.client;
     }
     async ping() {
-        await this.client.ping();
+        await this.cfg.client.ping();
     }
     async [Symbol.asyncDispose]() {
-        await this.client.disconnect();
+        await this.cfg.client.disconnect();
     }
     async getByIds(table, ids) {
         if (!ids.length)
             return [];
         // we assume that the order of returned values is the same as order of input ids
-        const bufs = await this.client.mgetBuffer(this.idsToKeys(table, ids));
+        const bufs = await this.cfg.client.mgetBuffer(this.idsToKeys(table, ids));
         return bufs.map((buf, i) => [ids[i], buf]).filter(([_k, v]) => v !== null);
     }
     async deleteByIds(table, ids) {
         if (!ids.length)
             return;
-        await this.client.del(this.idsToKeys(table, ids));
+        await this.cfg.client.del(this.idsToKeys(table, ids));
     }
     async saveBatch(table, entries, opt) {
         if (!entries.length)
@@ -34,7 +34,7 @@ class RedisKeyValueDB {
         if (opt?.expireAt) {
             // There's no supported mset with TTL: https://stackoverflow.com/questions/16423342/redis-multi-set-with-a-ttl
             // so we gonna use a pipeline instead
-            await this.client.withPipeline(pipeline => {
+            await this.cfg.client.withPipeline(pipeline => {
                 for (const [k, v] of entries) {
                     pipeline.set(this.idToKey(table, k), v, 'EXAT', opt.expireAt);
                 }
@@ -42,11 +42,11 @@ class RedisKeyValueDB {
         }
         else {
             const obj = Object.fromEntries(entries.map(([k, v]) => [this.idToKey(table, k), v]));
-            await this.client.msetBuffer(obj);
+            await this.cfg.client.msetBuffer(obj);
         }
     }
     streamIds(table, limit) {
-        let stream = this.client
+        let stream = this.cfg.client
             .scanStream({
             match: `${table}:*`,
             // count: limit, // count is actually a "batchSize", not a limit
@@ -58,25 +58,25 @@ class RedisKeyValueDB {
         return stream;
     }
     streamValues(table, limit) {
-        return this.client
+        return this.cfg.client
             .scanStream({
             match: `${table}:*`,
         })
             .flatMap(async (keys) => {
-            return (await this.client.mgetBuffer(keys)).filter(js_lib_1._isTruthy);
+            return (await this.cfg.client.mgetBuffer(keys)).filter(js_lib_1._isTruthy);
         }, {
             concurrency: 16,
         })
             .take(limit || Infinity);
     }
     streamEntries(table, limit) {
-        return this.client
+        return this.cfg.client
             .scanStream({
             match: `${table}:*`,
         })
             .flatMap(async (keys) => {
             // casting as Buffer[], because values are expected to exist for given keys
-            const bufs = (await this.client.mgetBuffer(keys));
+            const bufs = (await this.cfg.client.mgetBuffer(keys));
             return (0, js_lib_1._zip)(this.keysToIds(table, keys), bufs);
         }, {
             concurrency: 16,
@@ -85,20 +85,20 @@ class RedisKeyValueDB {
     }
     async count(table) {
         // todo: implement more efficiently, e.g via LUA?
-        return await this.client.scanCount({
+        return await this.cfg.client.scanCount({
             match: `${table}:*`,
         });
     }
-    async increment(table, id, by = 1) {
-        return await this.client.incr(this.idToKey(table, id), by);
-    }
-    async incrementBatch(_table, _incrementMap) {
-        throw new Error('Not implemented');
+    async incrementBatch(table, increments) {
+        const incrementTuplesWithInternalKeys = increments.map(([id, v]) => [this.idToKey(table, id), v]);
+        const resultsWithInternalKeys = await this.cfg.client.incrBatch(incrementTuplesWithInternalKeys);
+        const results = resultsWithInternalKeys.map(([k, v]) => [this.keyToId(table, k), v]);
+        return results;
     }
     async createTable(table, opt) {
         if (!opt?.dropIfExists)
             return;
-        await this.client.dropTable(table);
+        await this.cfg.client.dropTable(table);
     }
     idsToKeys(table, ids) {
         return ids.map(id => this.idToKey(table, id));

package/package.json

@@ -40,7 +40,7 @@
   "engines": {
     "node": ">=20.13"
   },
-  "version": "3.1.1",
+  "version": "3.3.0",
   "description": "Redis implementation of CommonKeyValueDB interface",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/redisClient.ts

@@ -1,9 +1,11 @@
 import {
+  _stringMapEntries,
   AnyObject,
   CommonLogger,
   NullableBuffer,
   NullableString,
   Promisable,
+  StringMap,
   UnixTimestampNumber,
 } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
@@ -157,6 +159,25 @@ export class RedisClient implements CommonClient {
     return await this.redis().hincrby(key, field, increment)
   }
 
+  async hincrBatch(key: string, incrementTuples: [string, number][]): Promise<[string, number][]> {
+    const results: StringMap<number | undefined> = {}
+
+    await this.withPipeline(async pipeline => {
+      for (const [field, increment] of incrementTuples) {
+        pipeline.hincrby(key, field, increment, (_err, newValue) => {
+          results[field] = newValue
+        })
+      }
+    })
+
+    const validResults = _stringMapEntries(results).filter(([_, v]) => v !== undefined) as [
+      string,
+      number,
+    ][]
+
+    return validResults
+  }
+
   async setWithTTL(
     key: string,
     value: string | number | Buffer,
@@ -165,14 +186,19 @@ export class RedisClient implements CommonClient {
     await this.redis().set(key, value, 'EXAT', expireAt)
   }
 
-  async hsetWithTTL(key: string, value: AnyObject, expireAt: UnixTimestampNumber): Promise<void> {
-    const valueKeys = Object.keys(value)
-    const numberOfKeys = valueKeys.length
-    const keyList = valueKeys.join(' ')
-    const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`
-    const [command, ...args] = commandString.split(' ')
-    await this.redis().hset(key, value)
-    await this.redis().call(command!, args)
+  async hsetWithTTL(
+    _key: string,
+    _value: AnyObject,
+    _expireAt: UnixTimestampNumber,
+  ): Promise<void> {
+    throw new Error('Not supported until Redis 7.4.0')
+    // const valueKeys = Object.keys(value)
+    // const numberOfKeys = valueKeys.length
+    // const keyList = valueKeys.join(' ')
+    // const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`
+    // const [command, ...args] = commandString.split(' ')
+    // await this.redis().hset(key, value)
+    // await this.redis().call(command!, args)
   }
 
   async mset(obj: Record<string, string | number>): Promise<void> {
@@ -187,6 +213,25 @@ export class RedisClient implements CommonClient {
     return await this.redis().incrby(key, by)
   }
 
+  async incrBatch(incrementTuples: [string, number][]): Promise<[string, number][]> {
+    const results: StringMap<number | undefined> = {}
+
+    await this.withPipeline(async pipeline => {
+      for (const [key, increment] of incrementTuples) {
+        pipeline.incrby(key, increment, (_err, newValue) => {
+          results[key] = newValue
+        })
+      }
+    })
+
+    const validResults = _stringMapEntries(results).filter(([_, v]) => v !== undefined) as [
+      string,
+      number,
+    ][]
+
+    return validResults
+  }
+
   async ttl(key: string): Promise<number> {
     return await this.redis().ttl(key)
   }
@@ -226,7 +271,7 @@ export class RedisClient implements CommonClient {
    Convenient type-safe wrapper.
    Returns BATCHES of keys in each iteration (as-is).
    */
-  scanStream(opt: ScanStreamOptions): ReadableTyped<string[]> {
+  scanStream(opt?: ScanStreamOptions): ReadableTyped<string[]> {
     return this.redis().scanStream(opt)
   }
 
@@ -249,11 +294,11 @@ export class RedisClient implements CommonClient {
     return count
   }
 
-  hscanStream(key: string, opt: ScanStreamOptions): ReadableTyped<string[]> {
+  hscanStream(key: string, opt?: ScanStreamOptions): ReadableTyped<string[]> {
     return this.redis().hscanStream(key, opt)
   }
 
-  async hscanCount(key: string, opt: ScanStreamOptions): Promise<number> {
+  async hscanCount(key: string, opt?: ScanStreamOptions): Promise<number> {
     let count = 0
 
     const stream = this.redis().hscanStream(key, opt)

package/src/redisHashKeyValueDB.ts

@@ -3,48 +3,49 @@ import {
   CommonKeyValueDB,
   commonKeyValueDBFullSupport,
   CommonKeyValueDBSaveBatchOptions,
+  IncrementTuple,
   KeyValueDBTuple,
 } from '@naturalcycles/db-lib'
-import { _chunk, StringMap } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { RedisClient } from './redisClient'
 import { RedisKeyValueDBCfg } from './redisKeyValueDB'
 
-export interface RedisHashKeyValueDBCfg extends RedisKeyValueDBCfg {
-  hashKey: string
-}
-
-export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
-  client: RedisClient
-  keyOfHashField: string
-
-  constructor(cfg: RedisHashKeyValueDBCfg) {
-    this.client = cfg.client
-    this.keyOfHashField = cfg.hashKey
-  }
+/**
+ * RedisHashKeyValueDB is a KeyValueDB implementation that uses hash fields to simulate tables.
+ * The value in the `table` arguments points to a hash field in Redis.
+ *
+ * The reason for having this approach and also the traditional RedisKeyValueDB is that
+ * the currently available Redis versions (in Memorystore, or on MacOs) do not support
+ * expiring hash properties.
+ * The expiring fields feature is important, and only available via RedisKeyValueDB.
+ *
+ * Once the available Redis version reaches 7.4.0+,
+ * this implementation can take over for RedisKeyValueDB.
+ */
+export class RedishHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
+  constructor(public cfg: RedisKeyValueDBCfg) {}
 
   support = {
     ...commonKeyValueDBFullSupport,
   }
 
   async ping(): Promise<void> {
-    await this.client.ping()
+    await this.cfg.client.ping()
   }
 
   async [Symbol.asyncDispose](): Promise<void> {
-    await this.client.disconnect()
+    await this.cfg.client.disconnect()
   }
 
   async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
     if (!ids.length) return []
     // we assume that the order of returned values is the same as order of input ids
-    const bufs = await this.client.hmgetBuffer(this.keyOfHashField, this.idsToKeys(table, ids))
+    const bufs = await this.cfg.client.hmgetBuffer(table, ids)
     return bufs.map((buf, i) => [ids[i], buf] as KeyValueDBTuple).filter(([_k, v]) => v !== null)
   }
 
   async deleteByIds(table: string, ids: string[]): Promise<void> {
     if (!ids.length) return
-    await this.client.hdel(this.keyOfHashField, this.idsToKeys(table, ids))
+    await this.cfg.client.hdel(table, ids)
   }
 
   async saveBatch(
@@ -54,103 +55,70 @@ export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
   ): Promise<void> {
     if (!entries.length) return
 
-    const entriesWithKey = entries.map(([k, v]) => [this.idToKey(table, k), v])
-    const map: StringMap<any> = Object.fromEntries(entriesWithKey)
+    const record = Object.fromEntries(entries)
 
     if (opt?.expireAt) {
-      await this.client.hsetWithTTL(this.keyOfHashField, map, opt.expireAt)
+      await this.cfg.client.hsetWithTTL(table, record, opt.expireAt)
     } else {
-      await this.client.hset(this.keyOfHashField, map)
+      await this.cfg.client.hset(table, record)
     }
   }
 
   streamIds(table: string, limit?: number): ReadableTyped<string> {
-    let stream = this.client
-      .hscanStream(this.keyOfHashField, {
-        match: `${table}:*`,
-      })
+    const stream = this.cfg.client
+      .hscanStream(table)
       .flatMap(keyValueList => {
         const keys: string[] = []
-        keyValueList.forEach((keyOrValue, index) => {
-          if (index % 2 !== 0) return
-          keys.push(keyOrValue)
-        })
-        return this.keysToIds(table, keys)
+        for (let i = 0; i < keyValueList.length; i += 2) {
+          keys.push(keyValueList[i]!)
+        }
+        return keys
       })
-
-    if (limit) {
-      stream = stream.take(limit)
-    }
+      .take(limit || Infinity)
 
     return stream
   }
 
   streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
-    return this.client
-      .hscanStream(this.keyOfHashField, {
-        match: `${table}:*`,
-      })
+    return this.cfg.client
+      .hscanStream(table)
       .flatMap(keyValueList => {
-        const values: string[] = []
-        keyValueList.forEach((keyOrValue, index) => {
-          if (index % 2 !== 1) return
-          values.push(keyOrValue)
-        })
-        return values.map(v => Buffer.from(v))
+        const values: Buffer[] = []
+        for (let i = 0; i < keyValueList.length; i += 2) {
+          const value = Buffer.from(keyValueList[i + 1]!)
+          values.push(value)
+        }
+        return values
       })
       .take(limit || Infinity)
   }
 
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
-    return this.client
-      .hscanStream(this.keyOfHashField, {
-        match: `${table}:*`,
-      })
+    return this.cfg.client
+      .hscanStream(table)
       .flatMap(keyValueList => {
-        const entries = _chunk(keyValueList, 2)
-        return entries.map(([k, v]) => {
-          return [this.keyToId(table, String(k)), Buffer.from(String(v))] satisfies KeyValueDBTuple
-        })
+        const entries: [string, Buffer][] = []
+        for (let i = 0; i < keyValueList.length; i += 2) {
+          const key = keyValueList[i]!
+          const value = Buffer.from(keyValueList[i + 1]!)
+          entries.push([key, value])
+        }
+        return entries
       })
       .take(limit || Infinity)
   }
 
   async count(table: string): Promise<number> {
-    return await this.client.hscanCount(this.keyOfHashField, {
-      match: `${table}:*`,
-    })
+    return await this.cfg.client.hscanCount(table)
   }
 
-  async increment(table: string, id: string, by = 1): Promise<number> {
-    return await this.client.hincr(this.keyOfHashField, this.idToKey(table, id), by)
-  }
-
-  async incrementBatch(
-    _table: string,
-    _incrementMap: StringMap<number>,
-  ): Promise<StringMap<number>> {
-    throw new Error('Not implemented')
+  async incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]> {
+    return await this.cfg.client.hincrBatch(table, increments)
   }
 
   async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
     if (!opt?.dropIfExists) return
 
-    await this.client.dropTable(table)
-  }
-
-  private idsToKeys(table: string, ids: string[]): string[] {
-    return ids.map(id => this.idToKey(table, id))
-  }
-
-  private idToKey(table: string, id: string): string {
-    return `${table}:${id}`
-  }
-
-  private keysToIds(table: string, keys: string[]): string[] {
-    return keys.map(key => this.keyToId(table, key))
-  }
-
-  private keyToId(table: string, key: string): string {
-    return key.slice(table.length + 1)
+    await this.cfg.client.del([table])
   }
 }

package/src/redisKeyValueDB.ts

@@ -3,9 +3,10 @@ import {
   CommonKeyValueDB,
   commonKeyValueDBFullSupport,
   CommonKeyValueDBSaveBatchOptions,
+  IncrementTuple,
   KeyValueDBTuple,
 } from '@naturalcycles/db-lib'
-import { _isTruthy, _zip, StringMap } from '@naturalcycles/js-lib'
+import { _isTruthy, _zip } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { RedisClient } from './redisClient'
 
@@ -14,34 +15,30 @@ export interface RedisKeyValueDBCfg {
 }
 
 export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
-  constructor(cfg: RedisKeyValueDBCfg) {
-    this.client = cfg.client
-  }
-
-  client: RedisClient
+  constructor(public cfg: RedisKeyValueDBCfg) {}
 
   support = {
     ...commonKeyValueDBFullSupport,
   }
 
   async ping(): Promise<void> {
-    await this.client.ping()
+    await this.cfg.client.ping()
   }
 
   async [Symbol.asyncDispose](): Promise<void> {
-    await this.client.disconnect()
+    await this.cfg.client.disconnect()
   }
 
   async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
     if (!ids.length) return []
     // we assume that the order of returned values is the same as order of input ids
-    const bufs = await this.client.mgetBuffer(this.idsToKeys(table, ids))
+    const bufs = await this.cfg.client.mgetBuffer(this.idsToKeys(table, ids))
     return bufs.map((buf, i) => [ids[i], buf] as KeyValueDBTuple).filter(([_k, v]) => v !== null)
   }
 
   async deleteByIds(table: string, ids: string[]): Promise<void> {
     if (!ids.length) return
-    await this.client.del(this.idsToKeys(table, ids))
+    await this.cfg.client.del(this.idsToKeys(table, ids))
   }
 
   async saveBatch(
@@ -54,7 +51,7 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
     if (opt?.expireAt) {
       // There's no supported mset with TTL: https://stackoverflow.com/questions/16423342/redis-multi-set-with-a-ttl
       // so we gonna use a pipeline instead
-      await this.client.withPipeline(pipeline => {
+      await this.cfg.client.withPipeline(pipeline => {
         for (const [k, v] of entries) {
           pipeline.set(this.idToKey(table, k), v, 'EXAT', opt.expireAt!)
         }
@@ -63,12 +60,12 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
       const obj: Record<string, Buffer> = Object.fromEntries(
         entries.map(([k, v]) => [this.idToKey(table, k), v]) as KeyValueDBTuple[],
       )
-      await this.client.msetBuffer(obj)
+      await this.cfg.client.msetBuffer(obj)
     }
   }
 
   streamIds(table: string, limit?: number): ReadableTyped<string> {
-    let stream = this.client
+    let stream = this.cfg.client
       .scanStream({
         match: `${table}:*`,
         // count: limit, // count is actually a "batchSize", not a limit
@@ -83,13 +80,13 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
   }
 
   streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
-    return this.client
+    return this.cfg.client
       .scanStream({
         match: `${table}:*`,
       })
       .flatMap(
         async keys => {
-          return (await this.client.mgetBuffer(keys)).filter(_isTruthy)
+          return (await this.cfg.client.mgetBuffer(keys)).filter(_isTruthy)
         },
         {
           concurrency: 16,
@@ -99,14 +96,14 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
   }
 
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
-    return this.client
+    return this.cfg.client
       .scanStream({
         match: `${table}:*`,
       })
       .flatMap(
         async keys => {
           // casting as Buffer[], because values are expected to exist for given keys
-          const bufs = (await this.client.mgetBuffer(keys)) as Buffer[]
+          const bufs = (await this.cfg.client.mgetBuffer(keys)) as Buffer[]
           return _zip(this.keysToIds(table, keys), bufs)
         },
         {
@@ -118,26 +115,26 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
 
   async count(table: string): Promise<number> {
     // todo: implement more efficiently, e.g via LUA?
-    return await this.client.scanCount({
+    return await this.cfg.client.scanCount({
       match: `${table}:*`,
     })
   }
 
-  async increment(table: string, id: string, by = 1): Promise<number> {
-    return await this.client.incr(this.idToKey(table, id), by)
-  }
-
-  async incrementBatch(
-    _table: string,
-    _incrementMap: StringMap<number>,
-  ): Promise<StringMap<number>> {
-    throw new Error('Not implemented')
+  async incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]> {
+    const incrementTuplesWithInternalKeys = increments.map(
+      ([id, v]) => [this.idToKey(table, id), v] as [string, number],
+    )
+    const resultsWithInternalKeys = await this.cfg.client.incrBatch(incrementTuplesWithInternalKeys)
+    const results = resultsWithInternalKeys.map(
+      ([k, v]) => [this.keyToId(table, k), v] as IncrementTuple,
+    )
+    return results
   }
 
   async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
     if (!opt?.dropIfExists) return
 
-    await this.client.dropTable(table)
+    await this.cfg.client.dropTable(table)
   }
 
   private idsToKeys(table: string, ids: string[]): string[] {