@naturalcycles/redis-lib 3.0.1 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/redisClient.d.ts +12 -3
- package/dist/redisClient.js +47 -2
- package/dist/redisHashKeyValueDB.d.ts +33 -0
- package/dist/redisHashKeyValueDB.js +121 -0
- package/dist/redisKeyValueDB.d.ts +9 -3
- package/dist/redisKeyValueDB.js +10 -0
- package/package.json +10 -5
- package/src/index.ts +1 -0
- package/src/redisClient.ts +64 -4
- package/src/redisHashKeyValueDB.ts +156 -0
- package/src/redisKeyValueDB.ts +19 -3
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -2,4 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
4
|
tslib_1.__exportStar(require("./redisClient"), exports);
|
|
5
|
+
tslib_1.__exportStar(require("./redisHashKeyValueDB"), exports);
|
|
5
6
|
tslib_1.__exportStar(require("./redisKeyValueDB"), exports);
|
package/dist/redisClient.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
import { CommonLogger, NullableBuffer, NullableString, Promisable, UnixTimestampNumber } from '@naturalcycles/js-lib';
|
|
1
|
+
import { AnyObject, CommonLogger, NullableBuffer, NullableString, Promisable, UnixTimestampNumber } from '@naturalcycles/js-lib';
|
|
3
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
4
3
|
import type { Redis, RedisOptions } from 'ioredis';
|
|
5
4
|
import type { ScanStreamOptions } from 'ioredis/built/types';
|
|
@@ -41,10 +40,18 @@ export declare class RedisClient implements CommonClient {
|
|
|
41
40
|
mget(keys: string[]): Promise<NullableString[]>;
|
|
42
41
|
mgetBuffer(keys: string[]): Promise<NullableBuffer[]>;
|
|
43
42
|
set(key: string, value: string | number | Buffer): Promise<void>;
|
|
43
|
+
hgetall<T extends Record<string, string> = Record<string, string>>(key: string): Promise<T | null>;
|
|
44
|
+
hget(key: string, field: string): Promise<NullableString>;
|
|
45
|
+
hset(key: string, value: AnyObject): Promise<void>;
|
|
46
|
+
hdel(key: string, fields: string[]): Promise<void>;
|
|
47
|
+
hmget(key: string, fields: string[]): Promise<NullableString[]>;
|
|
48
|
+
hmgetBuffer(key: string, fields: string[]): Promise<NullableBuffer[]>;
|
|
49
|
+
hincr(key: string, field: string, increment?: number): Promise<number>;
|
|
44
50
|
setWithTTL(key: string, value: string | number | Buffer, expireAt: UnixTimestampNumber): Promise<void>;
|
|
51
|
+
hsetWithTTL(key: string, value: AnyObject, expireAt: UnixTimestampNumber): Promise<void>;
|
|
45
52
|
mset(obj: Record<string, string | number>): Promise<void>;
|
|
46
53
|
msetBuffer(obj: Record<string, Buffer>): Promise<void>;
|
|
47
|
-
incr(key: string): Promise<number>;
|
|
54
|
+
incr(key: string, by?: number): Promise<number>;
|
|
48
55
|
ttl(key: string): Promise<number>;
|
|
49
56
|
dropTable(table: string): Promise<void>;
|
|
50
57
|
clearAll(): Promise<void>;
|
|
@@ -58,6 +65,8 @@ export declare class RedisClient implements CommonClient {
|
|
|
58
65
|
*/
|
|
59
66
|
scanStreamFlat(opt: ScanStreamOptions): ReadableTyped<string>;
|
|
60
67
|
scanCount(opt: ScanStreamOptions): Promise<number>;
|
|
68
|
+
hscanStream(key: string, opt: ScanStreamOptions): ReadableTyped<string[]>;
|
|
69
|
+
hscanCount(key: string, opt: ScanStreamOptions): Promise<number>;
|
|
61
70
|
withPipeline(fn: (pipeline: ChainableCommander) => Promisable<void>): Promise<void>;
|
|
62
71
|
private log;
|
|
63
72
|
}
|
package/dist/redisClient.js
CHANGED
|
@@ -74,17 +74,50 @@ class RedisClient {
|
|
|
74
74
|
async set(key, value) {
|
|
75
75
|
await this.redis().set(key, value);
|
|
76
76
|
}
|
|
77
|
+
async hgetall(key) {
|
|
78
|
+
const result = await this.redis().hgetall(key);
|
|
79
|
+
if (Object.keys(result).length === 0)
|
|
80
|
+
return null;
|
|
81
|
+
return result;
|
|
82
|
+
}
|
|
83
|
+
async hget(key, field) {
|
|
84
|
+
return await this.redis().hget(key, field);
|
|
85
|
+
}
|
|
86
|
+
async hset(key, value) {
|
|
87
|
+
await this.redis().hset(key, value);
|
|
88
|
+
}
|
|
89
|
+
async hdel(key, fields) {
|
|
90
|
+
await this.redis().hdel(key, ...fields);
|
|
91
|
+
}
|
|
92
|
+
async hmget(key, fields) {
|
|
93
|
+
return await this.redis().hmget(key, ...fields);
|
|
94
|
+
}
|
|
95
|
+
async hmgetBuffer(key, fields) {
|
|
96
|
+
return await this.redis().hmgetBuffer(key, ...fields);
|
|
97
|
+
}
|
|
98
|
+
async hincr(key, field, increment = 1) {
|
|
99
|
+
return await this.redis().hincrby(key, field, increment);
|
|
100
|
+
}
|
|
77
101
|
async setWithTTL(key, value, expireAt) {
|
|
78
102
|
await this.redis().set(key, value, 'EXAT', expireAt);
|
|
79
103
|
}
|
|
104
|
+
async hsetWithTTL(key, value, expireAt) {
|
|
105
|
+
const valueKeys = Object.keys(value);
|
|
106
|
+
const numberOfKeys = valueKeys.length;
|
|
107
|
+
const keyList = valueKeys.join(' ');
|
|
108
|
+
const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`;
|
|
109
|
+
const [command, ...args] = commandString.split(' ');
|
|
110
|
+
await this.redis().hset(key, value);
|
|
111
|
+
await this.redis().call(command, args);
|
|
112
|
+
}
|
|
80
113
|
async mset(obj) {
|
|
81
114
|
await this.redis().mset(obj);
|
|
82
115
|
}
|
|
83
116
|
async msetBuffer(obj) {
|
|
84
117
|
await this.redis().mset(obj);
|
|
85
118
|
}
|
|
86
|
-
async incr(key) {
|
|
87
|
-
return await this.redis().
|
|
119
|
+
async incr(key, by = 1) {
|
|
120
|
+
return await this.redis().incrby(key, by);
|
|
88
121
|
}
|
|
89
122
|
async ttl(key) {
|
|
90
123
|
return await this.redis().ttl(key);
|
|
@@ -125,6 +158,7 @@ class RedisClient {
|
|
|
125
158
|
* Like scanStream, but flattens the stream of keys.
|
|
126
159
|
*/
|
|
127
160
|
scanStreamFlat(opt) {
|
|
161
|
+
// biome-ignore lint/correctness/noFlatMapIdentity: ok
|
|
128
162
|
return this.redis().scanStream(opt).flatMap(keys => keys);
|
|
129
163
|
}
|
|
130
164
|
async scanCount(opt) {
|
|
@@ -135,6 +169,17 @@ class RedisClient {
|
|
|
135
169
|
});
|
|
136
170
|
return count;
|
|
137
171
|
}
|
|
172
|
+
hscanStream(key, opt) {
|
|
173
|
+
return this.redis().hscanStream(key, opt);
|
|
174
|
+
}
|
|
175
|
+
async hscanCount(key, opt) {
|
|
176
|
+
let count = 0;
|
|
177
|
+
const stream = this.redis().hscanStream(key, opt);
|
|
178
|
+
await stream.forEach((keyValueList) => {
|
|
179
|
+
count += keyValueList.length / 2;
|
|
180
|
+
});
|
|
181
|
+
return count;
|
|
182
|
+
}
|
|
138
183
|
async withPipeline(fn) {
|
|
139
184
|
const pipeline = this.redis().pipeline();
|
|
140
185
|
await fn(pipeline);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { CommonDBCreateOptions, CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, KeyValueDBTuple } from '@naturalcycles/db-lib';
|
|
2
|
+
import { StringMap } from '@naturalcycles/js-lib';
|
|
3
|
+
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
4
|
+
import { RedisClient } from './redisClient';
|
|
5
|
+
import { RedisKeyValueDBCfg } from './redisKeyValueDB';
|
|
6
|
+
export interface RedisHashKeyValueDBCfg extends RedisKeyValueDBCfg {
|
|
7
|
+
hashKey: string;
|
|
8
|
+
}
|
|
9
|
+
export declare class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
10
|
+
client: RedisClient;
|
|
11
|
+
keyOfHashField: string;
|
|
12
|
+
constructor(cfg: RedisHashKeyValueDBCfg);
|
|
13
|
+
support: {
|
|
14
|
+
count?: boolean;
|
|
15
|
+
increment?: boolean;
|
|
16
|
+
};
|
|
17
|
+
ping(): Promise<void>;
|
|
18
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
19
|
+
getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
|
|
20
|
+
deleteByIds(table: string, ids: string[]): Promise<void>;
|
|
21
|
+
saveBatch(table: string, entries: KeyValueDBTuple[], opt?: CommonKeyValueDBSaveBatchOptions): Promise<void>;
|
|
22
|
+
streamIds(table: string, limit?: number): ReadableTyped<string>;
|
|
23
|
+
streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
|
|
24
|
+
streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
|
|
25
|
+
count(table: string): Promise<number>;
|
|
26
|
+
increment(table: string, id: string, by?: number): Promise<number>;
|
|
27
|
+
incrementBatch(_table: string, _incrementMap: StringMap<number>): Promise<StringMap<number>>;
|
|
28
|
+
createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
|
|
29
|
+
private idsToKeys;
|
|
30
|
+
private idToKey;
|
|
31
|
+
private keysToIds;
|
|
32
|
+
private keyToId;
|
|
33
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisHashKeyValueDB = void 0;
|
|
4
|
+
const db_lib_1 = require("@naturalcycles/db-lib");
|
|
5
|
+
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
|
+
class RedisHashKeyValueDB {
|
|
7
|
+
constructor(cfg) {
|
|
8
|
+
this.support = {
|
|
9
|
+
...db_lib_1.commonKeyValueDBFullSupport,
|
|
10
|
+
};
|
|
11
|
+
this.client = cfg.client;
|
|
12
|
+
this.keyOfHashField = cfg.hashKey;
|
|
13
|
+
}
|
|
14
|
+
async ping() {
|
|
15
|
+
await this.client.ping();
|
|
16
|
+
}
|
|
17
|
+
async [Symbol.asyncDispose]() {
|
|
18
|
+
await this.client.disconnect();
|
|
19
|
+
}
|
|
20
|
+
async getByIds(table, ids) {
|
|
21
|
+
if (!ids.length)
|
|
22
|
+
return [];
|
|
23
|
+
// we assume that the order of returned values is the same as order of input ids
|
|
24
|
+
const bufs = await this.client.hmgetBuffer(this.keyOfHashField, this.idsToKeys(table, ids));
|
|
25
|
+
return bufs.map((buf, i) => [ids[i], buf]).filter(([_k, v]) => v !== null);
|
|
26
|
+
}
|
|
27
|
+
async deleteByIds(table, ids) {
|
|
28
|
+
if (!ids.length)
|
|
29
|
+
return;
|
|
30
|
+
await this.client.hdel(this.keyOfHashField, this.idsToKeys(table, ids));
|
|
31
|
+
}
|
|
32
|
+
async saveBatch(table, entries, opt) {
|
|
33
|
+
if (!entries.length)
|
|
34
|
+
return;
|
|
35
|
+
const entriesWithKey = entries.map(([k, v]) => [this.idToKey(table, k), v]);
|
|
36
|
+
const map = Object.fromEntries(entriesWithKey);
|
|
37
|
+
if (opt?.expireAt) {
|
|
38
|
+
await this.client.hsetWithTTL(this.keyOfHashField, map, opt.expireAt);
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
await this.client.hset(this.keyOfHashField, map);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
streamIds(table, limit) {
|
|
45
|
+
let stream = this.client
|
|
46
|
+
.hscanStream(this.keyOfHashField, {
|
|
47
|
+
match: `${table}:*`,
|
|
48
|
+
})
|
|
49
|
+
.flatMap(keyValueList => {
|
|
50
|
+
const keys = [];
|
|
51
|
+
keyValueList.forEach((keyOrValue, index) => {
|
|
52
|
+
if (index % 2 !== 0)
|
|
53
|
+
return;
|
|
54
|
+
keys.push(keyOrValue);
|
|
55
|
+
});
|
|
56
|
+
return this.keysToIds(table, keys);
|
|
57
|
+
});
|
|
58
|
+
if (limit) {
|
|
59
|
+
stream = stream.take(limit);
|
|
60
|
+
}
|
|
61
|
+
return stream;
|
|
62
|
+
}
|
|
63
|
+
streamValues(table, limit) {
|
|
64
|
+
return this.client
|
|
65
|
+
.hscanStream(this.keyOfHashField, {
|
|
66
|
+
match: `${table}:*`,
|
|
67
|
+
})
|
|
68
|
+
.flatMap(keyValueList => {
|
|
69
|
+
const values = [];
|
|
70
|
+
keyValueList.forEach((keyOrValue, index) => {
|
|
71
|
+
if (index % 2 !== 1)
|
|
72
|
+
return;
|
|
73
|
+
values.push(keyOrValue);
|
|
74
|
+
});
|
|
75
|
+
return values.map(v => Buffer.from(v));
|
|
76
|
+
})
|
|
77
|
+
.take(limit || Infinity);
|
|
78
|
+
}
|
|
79
|
+
streamEntries(table, limit) {
|
|
80
|
+
return this.client
|
|
81
|
+
.hscanStream(this.keyOfHashField, {
|
|
82
|
+
match: `${table}:*`,
|
|
83
|
+
})
|
|
84
|
+
.flatMap(keyValueList => {
|
|
85
|
+
const entries = (0, js_lib_1._chunk)(keyValueList, 2);
|
|
86
|
+
return entries.map(([k, v]) => {
|
|
87
|
+
return [this.keyToId(table, String(k)), Buffer.from(String(v))];
|
|
88
|
+
});
|
|
89
|
+
})
|
|
90
|
+
.take(limit || Infinity);
|
|
91
|
+
}
|
|
92
|
+
async count(table) {
|
|
93
|
+
return await this.client.hscanCount(this.keyOfHashField, {
|
|
94
|
+
match: `${table}:*`,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
async increment(table, id, by = 1) {
|
|
98
|
+
return await this.client.hincr(this.keyOfHashField, this.idToKey(table, id), by);
|
|
99
|
+
}
|
|
100
|
+
async incrementBatch(_table, _incrementMap) {
|
|
101
|
+
throw new Error('Not implemented');
|
|
102
|
+
}
|
|
103
|
+
async createTable(table, opt) {
|
|
104
|
+
if (!opt?.dropIfExists)
|
|
105
|
+
return;
|
|
106
|
+
await this.client.dropTable(table);
|
|
107
|
+
}
|
|
108
|
+
idsToKeys(table, ids) {
|
|
109
|
+
return ids.map(id => this.idToKey(table, id));
|
|
110
|
+
}
|
|
111
|
+
idToKey(table, id) {
|
|
112
|
+
return `${table}:${id}`;
|
|
113
|
+
}
|
|
114
|
+
keysToIds(table, keys) {
|
|
115
|
+
return keys.map(key => this.keyToId(table, key));
|
|
116
|
+
}
|
|
117
|
+
keyToId(table, key) {
|
|
118
|
+
return key.slice(table.length + 1);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
exports.RedisHashKeyValueDB = RedisHashKeyValueDB;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
import {
|
|
1
|
+
import { CommonDBCreateOptions, CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, KeyValueDBTuple } from '@naturalcycles/db-lib';
|
|
2
|
+
import { StringMap } from '@naturalcycles/js-lib';
|
|
3
3
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
4
4
|
import { RedisClient } from './redisClient';
|
|
5
5
|
export interface RedisKeyValueDBCfg {
|
|
@@ -8,6 +8,10 @@ export interface RedisKeyValueDBCfg {
|
|
|
8
8
|
export declare class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
9
9
|
constructor(cfg: RedisKeyValueDBCfg);
|
|
10
10
|
client: RedisClient;
|
|
11
|
+
support: {
|
|
12
|
+
count?: boolean;
|
|
13
|
+
increment?: boolean;
|
|
14
|
+
};
|
|
11
15
|
ping(): Promise<void>;
|
|
12
16
|
[Symbol.asyncDispose](): Promise<void>;
|
|
13
17
|
getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
|
|
@@ -15,8 +19,10 @@ export declare class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposabl
|
|
|
15
19
|
saveBatch(table: string, entries: KeyValueDBTuple[], opt?: CommonKeyValueDBSaveBatchOptions): Promise<void>;
|
|
16
20
|
streamIds(table: string, limit?: number): ReadableTyped<string>;
|
|
17
21
|
streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
|
|
18
|
-
streamEntries(table: string, limit?: number
|
|
22
|
+
streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
|
|
19
23
|
count(table: string): Promise<number>;
|
|
24
|
+
increment(table: string, id: string, by?: number): Promise<number>;
|
|
25
|
+
incrementBatch(_table: string, _incrementMap: StringMap<number>): Promise<StringMap<number>>;
|
|
20
26
|
createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
|
|
21
27
|
private idsToKeys;
|
|
22
28
|
private idToKey;
|
package/dist/redisKeyValueDB.js
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.RedisKeyValueDB = void 0;
|
|
4
|
+
const db_lib_1 = require("@naturalcycles/db-lib");
|
|
4
5
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
6
|
class RedisKeyValueDB {
|
|
6
7
|
constructor(cfg) {
|
|
8
|
+
this.support = {
|
|
9
|
+
...db_lib_1.commonKeyValueDBFullSupport,
|
|
10
|
+
};
|
|
7
11
|
this.client = cfg.client;
|
|
8
12
|
}
|
|
9
13
|
async ping() {
|
|
@@ -85,6 +89,12 @@ class RedisKeyValueDB {
|
|
|
85
89
|
match: `${table}:*`,
|
|
86
90
|
});
|
|
87
91
|
}
|
|
92
|
+
async increment(table, id, by = 1) {
|
|
93
|
+
return await this.client.incr(this.idToKey(table, id), by);
|
|
94
|
+
}
|
|
95
|
+
async incrementBatch(_table, _incrementMap) {
|
|
96
|
+
throw new Error('Not implemented');
|
|
97
|
+
}
|
|
88
98
|
async createTable(table, opt) {
|
|
89
99
|
if (!opt?.dropIfExists)
|
|
90
100
|
return;
|
package/package.json
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/redis-lib",
|
|
3
3
|
"scripts": {
|
|
4
|
-
"prepare": "husky"
|
|
4
|
+
"prepare": "husky",
|
|
5
|
+
"build": "dev-lib build",
|
|
6
|
+
"test": "dev-lib test",
|
|
7
|
+
"lint": "dev-lib lint",
|
|
8
|
+
"bt": "dev-lib bt",
|
|
9
|
+
"lbt": "dev-lib lbt"
|
|
5
10
|
},
|
|
6
11
|
"dependencies": {
|
|
7
12
|
"@naturalcycles/db-lib": "^9.9.2",
|
|
@@ -10,8 +15,8 @@
|
|
|
10
15
|
"ioredis": "^5.3.2"
|
|
11
16
|
},
|
|
12
17
|
"devDependencies": {
|
|
13
|
-
"@naturalcycles/dev-lib": "^
|
|
14
|
-
"@types/node": "^
|
|
18
|
+
"@naturalcycles/dev-lib": "^15.22.0",
|
|
19
|
+
"@types/node": "^22.7.5",
|
|
15
20
|
"jest": "^29.7.0"
|
|
16
21
|
},
|
|
17
22
|
"files": [
|
|
@@ -33,9 +38,9 @@
|
|
|
33
38
|
"url": "https://github.com/NaturalCycles/redis-lib"
|
|
34
39
|
},
|
|
35
40
|
"engines": {
|
|
36
|
-
"node": ">=
|
|
41
|
+
"node": ">=20.13"
|
|
37
42
|
},
|
|
38
|
-
"version": "3.
|
|
43
|
+
"version": "3.1.1",
|
|
39
44
|
"description": "Redis implementation of CommonKeyValueDB interface",
|
|
40
45
|
"author": "Natural Cycles Team",
|
|
41
46
|
"license": "MIT"
|
package/src/index.ts
CHANGED
package/src/redisClient.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
+
AnyObject,
|
|
2
3
|
CommonLogger,
|
|
3
4
|
NullableBuffer,
|
|
4
5
|
NullableString,
|
|
@@ -6,9 +7,9 @@ import {
|
|
|
6
7
|
UnixTimestampNumber,
|
|
7
8
|
} from '@naturalcycles/js-lib'
|
|
8
9
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib'
|
|
9
|
-
// eslint-disable-next-line import/no-duplicates
|
|
10
|
+
// eslint-disable-next-line import-x/no-duplicates
|
|
10
11
|
import type { Redis, RedisOptions } from 'ioredis'
|
|
11
|
-
// eslint-disable-next-line import/no-duplicates
|
|
12
|
+
// eslint-disable-next-line import-x/no-duplicates
|
|
12
13
|
import type * as RedisLib from 'ioredis'
|
|
13
14
|
import type { ScanStreamOptions } from 'ioredis/built/types'
|
|
14
15
|
import type { ChainableCommander } from 'ioredis/built/utils/RedisCommander'
|
|
@@ -124,6 +125,38 @@ export class RedisClient implements CommonClient {
|
|
|
124
125
|
await this.redis().set(key, value)
|
|
125
126
|
}
|
|
126
127
|
|
|
128
|
+
async hgetall<T extends Record<string, string> = Record<string, string>>(
|
|
129
|
+
key: string,
|
|
130
|
+
): Promise<T | null> {
|
|
131
|
+
const result = await this.redis().hgetall(key)
|
|
132
|
+
if (Object.keys(result).length === 0) return null
|
|
133
|
+
return result as T
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async hget(key: string, field: string): Promise<NullableString> {
|
|
137
|
+
return await this.redis().hget(key, field)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async hset(key: string, value: AnyObject): Promise<void> {
|
|
141
|
+
await this.redis().hset(key, value)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async hdel(key: string, fields: string[]): Promise<void> {
|
|
145
|
+
await this.redis().hdel(key, ...fields)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async hmget(key: string, fields: string[]): Promise<NullableString[]> {
|
|
149
|
+
return await this.redis().hmget(key, ...fields)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async hmgetBuffer(key: string, fields: string[]): Promise<NullableBuffer[]> {
|
|
153
|
+
return await this.redis().hmgetBuffer(key, ...fields)
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async hincr(key: string, field: string, increment = 1): Promise<number> {
|
|
157
|
+
return await this.redis().hincrby(key, field, increment)
|
|
158
|
+
}
|
|
159
|
+
|
|
127
160
|
async setWithTTL(
|
|
128
161
|
key: string,
|
|
129
162
|
value: string | number | Buffer,
|
|
@@ -132,6 +165,16 @@ export class RedisClient implements CommonClient {
|
|
|
132
165
|
await this.redis().set(key, value, 'EXAT', expireAt)
|
|
133
166
|
}
|
|
134
167
|
|
|
168
|
+
async hsetWithTTL(key: string, value: AnyObject, expireAt: UnixTimestampNumber): Promise<void> {
|
|
169
|
+
const valueKeys = Object.keys(value)
|
|
170
|
+
const numberOfKeys = valueKeys.length
|
|
171
|
+
const keyList = valueKeys.join(' ')
|
|
172
|
+
const commandString = `HEXPIREAT ${key} ${expireAt} FIELDS ${numberOfKeys} ${keyList}`
|
|
173
|
+
const [command, ...args] = commandString.split(' ')
|
|
174
|
+
await this.redis().hset(key, value)
|
|
175
|
+
await this.redis().call(command!, args)
|
|
176
|
+
}
|
|
177
|
+
|
|
135
178
|
async mset(obj: Record<string, string | number>): Promise<void> {
|
|
136
179
|
await this.redis().mset(obj)
|
|
137
180
|
}
|
|
@@ -140,8 +183,8 @@ export class RedisClient implements CommonClient {
|
|
|
140
183
|
await this.redis().mset(obj)
|
|
141
184
|
}
|
|
142
185
|
|
|
143
|
-
async incr(key: string): Promise<number> {
|
|
144
|
-
return await this.redis().
|
|
186
|
+
async incr(key: string, by = 1): Promise<number> {
|
|
187
|
+
return await this.redis().incrby(key, by)
|
|
145
188
|
}
|
|
146
189
|
|
|
147
190
|
async ttl(key: string): Promise<number> {
|
|
@@ -191,6 +234,7 @@ export class RedisClient implements CommonClient {
|
|
|
191
234
|
* Like scanStream, but flattens the stream of keys.
|
|
192
235
|
*/
|
|
193
236
|
scanStreamFlat(opt: ScanStreamOptions): ReadableTyped<string> {
|
|
237
|
+
// biome-ignore lint/correctness/noFlatMapIdentity: ok
|
|
194
238
|
return (this.redis().scanStream(opt) as ReadableTyped<string[]>).flatMap(keys => keys)
|
|
195
239
|
}
|
|
196
240
|
|
|
@@ -205,6 +249,22 @@ export class RedisClient implements CommonClient {
|
|
|
205
249
|
return count
|
|
206
250
|
}
|
|
207
251
|
|
|
252
|
+
hscanStream(key: string, opt: ScanStreamOptions): ReadableTyped<string[]> {
|
|
253
|
+
return this.redis().hscanStream(key, opt)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async hscanCount(key: string, opt: ScanStreamOptions): Promise<number> {
|
|
257
|
+
let count = 0
|
|
258
|
+
|
|
259
|
+
const stream = this.redis().hscanStream(key, opt)
|
|
260
|
+
|
|
261
|
+
await stream.forEach((keyValueList: string[]) => {
|
|
262
|
+
count += keyValueList.length / 2
|
|
263
|
+
})
|
|
264
|
+
|
|
265
|
+
return count
|
|
266
|
+
}
|
|
267
|
+
|
|
208
268
|
async withPipeline(fn: (pipeline: ChainableCommander) => Promisable<void>): Promise<void> {
|
|
209
269
|
const pipeline = this.redis().pipeline()
|
|
210
270
|
await fn(pipeline)
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CommonDBCreateOptions,
|
|
3
|
+
CommonKeyValueDB,
|
|
4
|
+
commonKeyValueDBFullSupport,
|
|
5
|
+
CommonKeyValueDBSaveBatchOptions,
|
|
6
|
+
KeyValueDBTuple,
|
|
7
|
+
} from '@naturalcycles/db-lib'
|
|
8
|
+
import { _chunk, StringMap } from '@naturalcycles/js-lib'
|
|
9
|
+
import { ReadableTyped } from '@naturalcycles/nodejs-lib'
|
|
10
|
+
import { RedisClient } from './redisClient'
|
|
11
|
+
import { RedisKeyValueDBCfg } from './redisKeyValueDB'
|
|
12
|
+
|
|
13
|
+
export interface RedisHashKeyValueDBCfg extends RedisKeyValueDBCfg {
|
|
14
|
+
hashKey: string
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
18
|
+
client: RedisClient
|
|
19
|
+
keyOfHashField: string
|
|
20
|
+
|
|
21
|
+
constructor(cfg: RedisHashKeyValueDBCfg) {
|
|
22
|
+
this.client = cfg.client
|
|
23
|
+
this.keyOfHashField = cfg.hashKey
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
support = {
|
|
27
|
+
...commonKeyValueDBFullSupport,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async ping(): Promise<void> {
|
|
31
|
+
await this.client.ping()
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async [Symbol.asyncDispose](): Promise<void> {
|
|
35
|
+
await this.client.disconnect()
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
|
|
39
|
+
if (!ids.length) return []
|
|
40
|
+
// we assume that the order of returned values is the same as order of input ids
|
|
41
|
+
const bufs = await this.client.hmgetBuffer(this.keyOfHashField, this.idsToKeys(table, ids))
|
|
42
|
+
return bufs.map((buf, i) => [ids[i], buf] as KeyValueDBTuple).filter(([_k, v]) => v !== null)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async deleteByIds(table: string, ids: string[]): Promise<void> {
|
|
46
|
+
if (!ids.length) return
|
|
47
|
+
await this.client.hdel(this.keyOfHashField, this.idsToKeys(table, ids))
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async saveBatch(
|
|
51
|
+
table: string,
|
|
52
|
+
entries: KeyValueDBTuple[],
|
|
53
|
+
opt?: CommonKeyValueDBSaveBatchOptions,
|
|
54
|
+
): Promise<void> {
|
|
55
|
+
if (!entries.length) return
|
|
56
|
+
|
|
57
|
+
const entriesWithKey = entries.map(([k, v]) => [this.idToKey(table, k), v])
|
|
58
|
+
const map: StringMap<any> = Object.fromEntries(entriesWithKey)
|
|
59
|
+
|
|
60
|
+
if (opt?.expireAt) {
|
|
61
|
+
await this.client.hsetWithTTL(this.keyOfHashField, map, opt.expireAt)
|
|
62
|
+
} else {
|
|
63
|
+
await this.client.hset(this.keyOfHashField, map)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
streamIds(table: string, limit?: number): ReadableTyped<string> {
|
|
68
|
+
let stream = this.client
|
|
69
|
+
.hscanStream(this.keyOfHashField, {
|
|
70
|
+
match: `${table}:*`,
|
|
71
|
+
})
|
|
72
|
+
.flatMap(keyValueList => {
|
|
73
|
+
const keys: string[] = []
|
|
74
|
+
keyValueList.forEach((keyOrValue, index) => {
|
|
75
|
+
if (index % 2 !== 0) return
|
|
76
|
+
keys.push(keyOrValue)
|
|
77
|
+
})
|
|
78
|
+
return this.keysToIds(table, keys)
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
if (limit) {
|
|
82
|
+
stream = stream.take(limit)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return stream
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
|
|
89
|
+
return this.client
|
|
90
|
+
.hscanStream(this.keyOfHashField, {
|
|
91
|
+
match: `${table}:*`,
|
|
92
|
+
})
|
|
93
|
+
.flatMap(keyValueList => {
|
|
94
|
+
const values: string[] = []
|
|
95
|
+
keyValueList.forEach((keyOrValue, index) => {
|
|
96
|
+
if (index % 2 !== 1) return
|
|
97
|
+
values.push(keyOrValue)
|
|
98
|
+
})
|
|
99
|
+
return values.map(v => Buffer.from(v))
|
|
100
|
+
})
|
|
101
|
+
.take(limit || Infinity)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
|
|
105
|
+
return this.client
|
|
106
|
+
.hscanStream(this.keyOfHashField, {
|
|
107
|
+
match: `${table}:*`,
|
|
108
|
+
})
|
|
109
|
+
.flatMap(keyValueList => {
|
|
110
|
+
const entries = _chunk(keyValueList, 2)
|
|
111
|
+
return entries.map(([k, v]) => {
|
|
112
|
+
return [this.keyToId(table, String(k)), Buffer.from(String(v))] satisfies KeyValueDBTuple
|
|
113
|
+
})
|
|
114
|
+
})
|
|
115
|
+
.take(limit || Infinity)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async count(table: string): Promise<number> {
|
|
119
|
+
return await this.client.hscanCount(this.keyOfHashField, {
|
|
120
|
+
match: `${table}:*`,
|
|
121
|
+
})
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async increment(table: string, id: string, by = 1): Promise<number> {
|
|
125
|
+
return await this.client.hincr(this.keyOfHashField, this.idToKey(table, id), by)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async incrementBatch(
|
|
129
|
+
_table: string,
|
|
130
|
+
_incrementMap: StringMap<number>,
|
|
131
|
+
): Promise<StringMap<number>> {
|
|
132
|
+
throw new Error('Not implemented')
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
|
|
136
|
+
if (!opt?.dropIfExists) return
|
|
137
|
+
|
|
138
|
+
await this.client.dropTable(table)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private idsToKeys(table: string, ids: string[]): string[] {
|
|
142
|
+
return ids.map(id => this.idToKey(table, id))
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private idToKey(table: string, id: string): string {
|
|
146
|
+
return `${table}:${id}`
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
private keysToIds(table: string, keys: string[]): string[] {
|
|
150
|
+
return keys.map(key => this.keyToId(table, key))
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private keyToId(table: string, key: string): string {
|
|
154
|
+
return key.slice(table.length + 1)
|
|
155
|
+
}
|
|
156
|
+
}
|
package/src/redisKeyValueDB.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
|
-
CommonKeyValueDBSaveBatchOptions,
|
|
3
2
|
CommonDBCreateOptions,
|
|
4
3
|
CommonKeyValueDB,
|
|
4
|
+
commonKeyValueDBFullSupport,
|
|
5
|
+
CommonKeyValueDBSaveBatchOptions,
|
|
5
6
|
KeyValueDBTuple,
|
|
6
7
|
} from '@naturalcycles/db-lib'
|
|
7
|
-
import { _isTruthy, _zip } from '@naturalcycles/js-lib'
|
|
8
|
+
import { _isTruthy, _zip, StringMap } from '@naturalcycles/js-lib'
|
|
8
9
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib'
|
|
9
10
|
import { RedisClient } from './redisClient'
|
|
10
11
|
|
|
@@ -19,6 +20,10 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
19
20
|
|
|
20
21
|
client: RedisClient
|
|
21
22
|
|
|
23
|
+
support = {
|
|
24
|
+
...commonKeyValueDBFullSupport,
|
|
25
|
+
}
|
|
26
|
+
|
|
22
27
|
async ping(): Promise<void> {
|
|
23
28
|
await this.client.ping()
|
|
24
29
|
}
|
|
@@ -93,7 +98,7 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
93
98
|
.take(limit || Infinity)
|
|
94
99
|
}
|
|
95
100
|
|
|
96
|
-
streamEntries(table: string, limit?: number
|
|
101
|
+
streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
|
|
97
102
|
return this.client
|
|
98
103
|
.scanStream({
|
|
99
104
|
match: `${table}:*`,
|
|
@@ -118,6 +123,17 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
118
123
|
})
|
|
119
124
|
}
|
|
120
125
|
|
|
126
|
+
async increment(table: string, id: string, by = 1): Promise<number> {
|
|
127
|
+
return await this.client.incr(this.idToKey(table, id), by)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async incrementBatch(
|
|
131
|
+
_table: string,
|
|
132
|
+
_incrementMap: StringMap<number>,
|
|
133
|
+
): Promise<StringMap<number>> {
|
|
134
|
+
throw new Error('Not implemented')
|
|
135
|
+
}
|
|
136
|
+
|
|
121
137
|
async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
|
|
122
138
|
if (!opt?.dropIfExists) return
|
|
123
139
|
|