@naturalcycles/redis-lib 3.0.0 → 3.1.0

package/dist/index.d.ts

@@ -1,2 +1,3 @@
 export * from './redisClient';
 export * from './redisKeyValueDB';
+export * from './redisHashKeyValueDB';

package/dist/index.js

@@ -3,3 +3,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
 tslib_1.__exportStar(require("./redisClient"), exports);
 tslib_1.__exportStar(require("./redisKeyValueDB"), exports);
+tslib_1.__exportStar(require("./redisHashKeyValueDB"), exports);

package/dist/redisClient.d.ts

@@ -1,5 +1,4 @@
-/// <reference types="node" />
-import { CommonLogger, NullableBuffer, NullableString, Promisable, UnixTimestampNumber } from '@naturalcycles/js-lib';
+import { AnyObject, CommonLogger, NullableBuffer, NullableString, Promisable, UnixTimestampNumber } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import type { Redis, RedisOptions } from 'ioredis';
 import type { ScanStreamOptions } from 'ioredis/built/types';
@@ -40,11 +39,19 @@ export declare class RedisClient implements CommonClient {
     getBuffer(key: string): Promise<NullableBuffer>;
     mget(keys: string[]): Promise<NullableString[]>;
     mgetBuffer(keys: string[]): Promise<NullableBuffer[]>;
-    set(key: string, value: string | Buffer): Promise<void>;
-    setWithTTL(key: string, value: string | Buffer, expireAt: UnixTimestampNumber): Promise<void>;
-    mset(obj: Record<string, string>): Promise<void>;
+    set(key: string, value: string | number | Buffer): Promise<void>;
+    hgetall<T extends Record<string, string> = Record<string, string>>(key: string): Promise<T | null>;
+    hget(key: string, field: string): Promise<NullableString>;
+    hset(key: string, value: AnyObject): Promise<void>;
+    hdel(key: string, fields: string[]): Promise<void>;
+    hmget(key: string, fields: string[]): Promise<NullableString[]>;
+    hmgetBuffer(key: string, fields: string[]): Promise<NullableBuffer[]>;
+    hincr(key: string, field: string, increment?: number): Promise<number>;
+    setWithTTL(key: string, value: string | number | Buffer, expireAt: UnixTimestampNumber): Promise<void>;
+    hsetWithTTL(key: string, value: AnyObject, expireAt: UnixTimestampNumber): Promise<void>;
+    mset(obj: Record<string, string | number>): Promise<void>;
     msetBuffer(obj: Record<string, Buffer>): Promise<void>;
-    incr(key: string): Promise<number>;
+    incr(key: string, by?: number): Promise<number>;
     ttl(key: string): Promise<number>;
     dropTable(table: string): Promise<void>;
     clearAll(): Promise<void>;
@@ -58,6 +65,8 @@ export declare class RedisClient implements CommonClient {
      */
     scanStreamFlat(opt: ScanStreamOptions): ReadableTyped<string>;
     scanCount(opt: ScanStreamOptions): Promise<number>;
+    hscanStream(key: string, opt: ScanStreamOptions): ReadableTyped<string[]>;
+    hscanCount(key: string, opt: ScanStreamOptions): Promise<number>;
     withPipeline(fn: (pipeline: ChainableCommander) => Promisable<void>): Promise<void>;
     private log;
 }

package/dist/redisClient.js

@@ -74,17 +74,50 @@ class RedisClient {
     async set(key, value) {
         await this.redis().set(key, value);
     }
+    async hgetall(key) {
+        const result = await this.redis().hgetall(key);
+        if (Object.keys(result).length === 0)
+            return null;
+        return result;
+    }
+    async hget(key, field) {
+        return await this.redis().hget(key, field);
+    }
+    async hset(key, value) {
+        await this.redis().hset(key, value);
+    }
+    async hdel(key, fields) {
+        await this.redis().hdel(key, ...fields);
+    }
+    async hmget(key, fields) {
+        return await this.redis().hmget(key, ...fields);
+    }
+    async hmgetBuffer(key, fields) {
+        return await this.redis().hmgetBuffer(key, ...fields);
+    }
+    async hincr(key, field, increment = 1) {
+        return await this.redis().hincrby(key, field, increment);
+    }
     async setWithTTL(key, value, expireAt) {
         await this.redis().set(key, value, 'EXAT', expireAt);
     }
+    async hsetWithTTL(key, value, expireAt) {
+        const valueKeys = Object.keys(value);
+        const numberOfKeys = valueKeys.length;
+        const keyList = valueKeys.join(' ');
+        const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`;
+        const [command, ...args] = commandString.split(' ');
+        await this.redis().hset(key, value);
+        await this.redis().call(command, args);
+    }
     async mset(obj) {
         await this.redis().mset(obj);
     }
     async msetBuffer(obj) {
         await this.redis().mset(obj);
     }
-    async incr(key) {
-        return await this.redis().incr(key);
+    async incr(key, by = 1) {
+        return await this.redis().incrby(key, by);
     }
     async ttl(key) {
         return await this.redis().ttl(key);
@@ -135,6 +168,17 @@ class RedisClient {
         });
         return count;
     }
+    hscanStream(key, opt) {
+        return this.redis().hscanStream(key, opt);
+    }
+    async hscanCount(key, opt) {
+        let count = 0;
+        const stream = this.redis().hscanStream(key, opt);
+        await stream.forEach((keyValueList) => {
+            count += keyValueList.length / 2;
+        });
+        return count;
+    }
     async withPipeline(fn) {
         const pipeline = this.redis().pipeline();
         await fn(pipeline);

package/dist/redisHashKeyValueDB.d.ts

@@ -0,0 +1,27 @@
+import { CommonKeyValueDBSaveBatchOptions, CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { RedisClient } from './redisClient';
+import { RedisKeyValueDBCfg } from './redisKeyValueDB';
+export interface RedisHashKeyValueDBCfg extends RedisKeyValueDBCfg {
+    hashKey: string;
+}
+export declare class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
+    client: RedisClient;
+    keyOfHashField: string;
+    constructor(cfg: RedisHashKeyValueDBCfg);
+    ping(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+    getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
+    deleteByIds(table: string, ids: string[]): Promise<void>;
+    saveBatch(table: string, entries: KeyValueDBTuple[], opt?: CommonKeyValueDBSaveBatchOptions): Promise<void>;
+    streamIds(table: string, limit?: number): ReadableTyped<string>;
+    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
+    streamEntries(table: string, limit?: number | undefined): ReadableTyped<KeyValueDBTuple>;
+    count(table: string): Promise<number>;
+    increment(table: string, id: string, by?: number): Promise<number>;
+    createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
+    private idsToKeys;
+    private idToKey;
+    private keysToIds;
+    private keyToId;
+}

package/dist/redisHashKeyValueDB.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisHashKeyValueDB = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+class RedisHashKeyValueDB {
+    constructor(cfg) {
+        this.client = cfg.client;
+        this.keyOfHashField = cfg.hashKey;
+    }
+    async ping() {
+        await this.client.ping();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.client.disconnect();
+    }
+    async getByIds(table, ids) {
+        if (!ids.length)
+            return [];
+        // we assume that the order of returned values is the same as order of input ids
+        const bufs = await this.client.hmgetBuffer(this.keyOfHashField, this.idsToKeys(table, ids));
+        return bufs.map((buf, i) => [ids[i], buf]).filter(([_k, v]) => v !== null);
+    }
+    async deleteByIds(table, ids) {
+        if (!ids.length)
+            return;
+        await this.client.hdel(this.keyOfHashField, this.idsToKeys(table, ids));
+    }
+    async saveBatch(table, entries, opt) {
+        if (!entries.length)
+            return;
+        const entriesWithKey = entries.map(([k, v]) => [this.idToKey(table, k), v]);
+        const map = Object.fromEntries(entriesWithKey);
+        if (opt?.expireAt) {
+            await this.client.hsetWithTTL(this.keyOfHashField, map, opt.expireAt);
+        }
+        else {
+            await this.client.hset(this.keyOfHashField, map);
+        }
+    }
+    streamIds(table, limit) {
+        let stream = this.client
+            .hscanStream(this.keyOfHashField, {
+            match: `${table}:*`,
+        })
+            .flatMap(keyValueList => {
+            const keys = [];
+            keyValueList.forEach((keyOrValue, index) => {
+                if (index % 2 !== 0)
+                    return;
+                keys.push(keyOrValue);
+            });
+            return this.keysToIds(table, keys);
+        });
+        if (limit) {
+            stream = stream.take(limit);
+        }
+        return stream;
+    }
+    streamValues(table, limit) {
+        return this.client
+            .hscanStream(this.keyOfHashField, {
+            match: `${table}:*`,
+        })
+            .flatMap(keyValueList => {
+            const values = [];
+            keyValueList.forEach((keyOrValue, index) => {
+                if (index % 2 !== 1)
+                    return;
+                values.push(keyOrValue);
+            });
+            return values.map(v => Buffer.from(v));
+        })
+            .take(limit || Infinity);
+    }
+    streamEntries(table, limit) {
+        return this.client
+            .hscanStream(this.keyOfHashField, {
+            match: `${table}:*`,
+        })
+            .flatMap(keyValueList => {
+            const entries = (0, js_lib_1._chunk)(keyValueList, 2);
+            return entries.map(([k, v]) => {
+                return [this.keyToId(table, String(k)), Buffer.from(String(v))];
+            });
+        })
+            .take(limit || Infinity);
+    }
+    async count(table) {
+        return await this.client.hscanCount(this.keyOfHashField, {
+            match: `${table}:*`,
+        });
+    }
+    async increment(table, id, by = 1) {
+        return await this.client.hincr(this.keyOfHashField, this.idToKey(table, id), by);
+    }
+    async createTable(table, opt) {
+        if (!opt?.dropIfExists)
+            return;
+        await this.client.dropTable(table);
+    }
+    idsToKeys(table, ids) {
+        return ids.map(id => this.idToKey(table, id));
+    }
+    idToKey(table, id) {
+        return `${table}:${id}`;
+    }
+    keysToIds(table, keys) {
+        return keys.map(key => this.keyToId(table, key));
+    }
+    keyToId(table, key) {
+        return key.slice(table.length + 1);
+    }
+}
+exports.RedisHashKeyValueDB = RedisHashKeyValueDB;

package/dist/redisKeyValueDB.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { CommonKeyValueDBSaveBatchOptions, CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { RedisClient } from './redisClient';
@@ -17,6 +16,7 @@ export declare class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposabl
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number | undefined): ReadableTyped<KeyValueDBTuple>;
     count(table: string): Promise<number>;
+    increment(table: string, id: string, by?: number): Promise<number>;
     createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
     private idsToKeys;
     private idToKey;

package/dist/redisKeyValueDB.js

@@ -85,6 +85,9 @@ class RedisKeyValueDB {
             match: `${table}:*`,
         });
     }
+    async increment(table, id, by = 1) {
+        return await this.client.incr(this.idToKey(table, id), by);
+    }
     async createTable(table, opt) {
         if (!opt?.dropIfExists)
             return;

package/package.json

@@ -35,7 +35,7 @@
   "engines": {
     "node": ">=18.12"
   },
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Redis implementation of CommonKeyValueDB interface",
   "author": "Natural Cycles Team",
   "license": "MIT"

package/src/index.ts

@@ -1,2 +1,3 @@
 export * from './redisClient'
 export * from './redisKeyValueDB'
+export * from './redisHashKeyValueDB'

package/src/redisClient.ts

@@ -1,4 +1,5 @@
 import {
+  AnyObject,
   CommonLogger,
   NullableBuffer,
   NullableString,
@@ -120,19 +121,61 @@ export class RedisClient implements CommonClient {
     return await this.redis().mgetBuffer(keys)
   }
 
-  async set(key: string, value: string | Buffer): Promise<void> {
+  async set(key: string, value: string | number | Buffer): Promise<void> {
     await this.redis().set(key, value)
   }
 
+  async hgetall<T extends Record<string, string> = Record<string, string>>(
+    key: string,
+  ): Promise<T | null> {
+    const result = await this.redis().hgetall(key)
+    if (Object.keys(result).length === 0) return null
+    return result as T
+  }
+
+  async hget(key: string, field: string): Promise<NullableString> {
+    return await this.redis().hget(key, field)
+  }
+
+  async hset(key: string, value: AnyObject): Promise<void> {
+    await this.redis().hset(key, value)
+  }
+
+  async hdel(key: string, fields: string[]): Promise<void> {
+    await this.redis().hdel(key, ...fields)
+  }
+
+  async hmget(key: string, fields: string[]): Promise<NullableString[]> {
+    return await this.redis().hmget(key, ...fields)
+  }
+
+  async hmgetBuffer(key: string, fields: string[]): Promise<NullableBuffer[]> {
+    return await this.redis().hmgetBuffer(key, ...fields)
+  }
+
+  async hincr(key: string, field: string, increment: number = 1): Promise<number> {
+    return await this.redis().hincrby(key, field, increment)
+  }
+
   async setWithTTL(
     key: string,
-    value: string | Buffer,
+    value: string | number | Buffer,
     expireAt: UnixTimestampNumber,
   ): Promise<void> {
     await this.redis().set(key, value, 'EXAT', expireAt)
   }
 
-  async mset(obj: Record<string, string>): Promise<void> {
+  async hsetWithTTL(key: string, value: AnyObject, expireAt: UnixTimestampNumber): Promise<void> {
+    const valueKeys = Object.keys(value)
+    const numberOfKeys = valueKeys.length
+    const keyList = valueKeys.join(' ')
+    const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`
+    const [command, ...args] = commandString.split(' ')
+    await this.redis().hset(key, value)
+    await this.redis().call(command!, args)
+  }
+
+  async mset(obj: Record<string, string | number>): Promise<void> {
     await this.redis().mset(obj)
   }
 
@@ -140,8 +183,8 @@ export class RedisClient implements CommonClient {
     await this.redis().mset(obj)
   }
 
-  async incr(key: string): Promise<number> {
-    return await this.redis().incr(key)
+  async incr(key: string, by: number = 1): Promise<number> {
+    return await this.redis().incrby(key, by)
   }
 
   async ttl(key: string): Promise<number> {
@@ -205,6 +248,22 @@ export class RedisClient implements CommonClient {
     return count
   }
 
+  hscanStream(key: string, opt: ScanStreamOptions): ReadableTyped<string[]> {
+    return this.redis().hscanStream(key, opt)
+  }
+
+  async hscanCount(key: string, opt: ScanStreamOptions): Promise<number> {
+    let count = 0
+
+    const stream = this.redis().hscanStream(key, opt)
+
+    await stream.forEach((keyValueList: string[]) => {
+      count += keyValueList.length / 2
+    })
+
+    return count
+  }
+
   async withPipeline(fn: (pipeline: ChainableCommander) => Promisable<void>): Promise<void> {
     const pipeline = this.redis().pipeline()
     await fn(pipeline)

package/src/redisHashKeyValueDB.ts

@@ -0,0 +1,144 @@
+import {
+  CommonKeyValueDBSaveBatchOptions,
+  CommonDBCreateOptions,
+  CommonKeyValueDB,
+  KeyValueDBTuple,
+} from '@naturalcycles/db-lib'
+import { _chunk, StringMap } from '@naturalcycles/js-lib'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { RedisClient } from './redisClient'
+import { RedisKeyValueDBCfg } from './redisKeyValueDB'
+
+export interface RedisHashKeyValueDBCfg extends RedisKeyValueDBCfg {
+  hashKey: string
+}
+
+export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
+  client: RedisClient
+  keyOfHashField: string
+
+  constructor(cfg: RedisHashKeyValueDBCfg) {
+    this.client = cfg.client
+    this.keyOfHashField = cfg.hashKey
+  }
+
+  async ping(): Promise<void> {
+    await this.client.ping()
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.client.disconnect()
+  }
+
+  async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
+    if (!ids.length) return []
+    // we assume that the order of returned values is the same as order of input ids
+    const bufs = await this.client.hmgetBuffer(this.keyOfHashField, this.idsToKeys(table, ids))
+    return bufs.map((buf, i) => [ids[i], buf] as KeyValueDBTuple).filter(([_k, v]) => v !== null)
+  }
+
+  async deleteByIds(table: string, ids: string[]): Promise<void> {
+    if (!ids.length) return
+    await this.client.hdel(this.keyOfHashField, this.idsToKeys(table, ids))
+  }
+
+  async saveBatch(
+    table: string,
+    entries: KeyValueDBTuple[],
+    opt?: CommonKeyValueDBSaveBatchOptions,
+  ): Promise<void> {
+    if (!entries.length) return
+
+    const entriesWithKey = entries.map(([k, v]) => [this.idToKey(table, k), v])
+    const map: StringMap<any> = Object.fromEntries(entriesWithKey)
+
+    if (opt?.expireAt) {
+      await this.client.hsetWithTTL(this.keyOfHashField, map, opt.expireAt)
+    } else {
+      await this.client.hset(this.keyOfHashField, map)
+    }
+  }
+
+  streamIds(table: string, limit?: number): ReadableTyped<string> {
+    let stream = this.client
+      .hscanStream(this.keyOfHashField, {
+        match: `${table}:*`,
+      })
+      .flatMap(keyValueList => {
+        const keys: string[] = []
+        keyValueList.forEach((keyOrValue, index) => {
+          if (index % 2 !== 0) return
+          keys.push(keyOrValue)
+        })
+        return this.keysToIds(table, keys)
+      })
+
+    if (limit) {
+      stream = stream.take(limit)
+    }
+
+    return stream
+  }
+
+  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+    return this.client
+      .hscanStream(this.keyOfHashField, {
+        match: `${table}:*`,
+      })
+      .flatMap(keyValueList => {
+        const values: string[] = []
+        keyValueList.forEach((keyOrValue, index) => {
+          if (index % 2 !== 1) return
+          values.push(keyOrValue)
+        })
+        return values.map(v => Buffer.from(v))
+      })
+      .take(limit || Infinity)
+  }
+
+  streamEntries(table: string, limit?: number | undefined): ReadableTyped<KeyValueDBTuple> {
+    return this.client
+      .hscanStream(this.keyOfHashField, {
+        match: `${table}:*`,
+      })
+      .flatMap(keyValueList => {
+        const entries = _chunk(keyValueList, 2)
+        return entries.map(([k, v]) => {
+          return [this.keyToId(table, String(k)), Buffer.from(String(v))] satisfies KeyValueDBTuple
+        })
+      })
+      .take(limit || Infinity)
+  }
+
+  async count(table: string): Promise<number> {
+    return await this.client.hscanCount(this.keyOfHashField, {
+      match: `${table}:*`,
+    })
+  }
+
+  async increment(table: string, id: string, by: number = 1): Promise<number> {
+    return await this.client.hincr(this.keyOfHashField, this.idToKey(table, id), by)
+  }
+
+  async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
+    if (!opt?.dropIfExists) return
+
+    await this.client.dropTable(table)
+  }
+
+  private idsToKeys(table: string, ids: string[]): string[] {
+    return ids.map(id => this.idToKey(table, id))
+  }
+
+  private idToKey(table: string, id: string): string {
+    return `${table}:${id}`
+  }
+
+  private keysToIds(table: string, keys: string[]): string[] {
+    return keys.map(key => this.keyToId(table, key))
+  }
+
+  private keyToId(table: string, key: string): string {
+    return key.slice(table.length + 1)
+  }
+}

package/src/redisKeyValueDB.ts

@@ -118,6 +118,10 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
     })
   }
 
+  async increment(table: string, id: string, by: number = 1): Promise<number> {
+    return await this.client.incr(this.idToKey(table, id), by)
+  }
+
   async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
     if (!opt?.dropIfExists) return