@naturalcycles/redis-lib 3.2.0 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/redisClient.d.ts +3 -3
- package/dist/redisHashKeyValueDB.d.ts +14 -11
- package/dist/redisHashKeyValueDB.js +47 -63
- package/dist/redisKeyValueDB.d.ts +1 -1
- package/dist/redisKeyValueDB.js +15 -15
- package/package.json +1 -1
- package/src/redisClient.ts +3 -3
- package/src/redisHashKeyValueDB.ts +47 -83
- package/src/redisKeyValueDB.ts +15 -19
package/dist/redisClient.d.ts
CHANGED
|
@@ -61,14 +61,14 @@ export declare class RedisClient implements CommonClient {
|
|
|
61
61
|
Convenient type-safe wrapper.
|
|
62
62
|
Returns BATCHES of keys in each iteration (as-is).
|
|
63
63
|
*/
|
|
64
|
-
scanStream(opt
|
|
64
|
+
scanStream(opt?: ScanStreamOptions): ReadableTyped<string[]>;
|
|
65
65
|
/**
|
|
66
66
|
* Like scanStream, but flattens the stream of keys.
|
|
67
67
|
*/
|
|
68
68
|
scanStreamFlat(opt: ScanStreamOptions): ReadableTyped<string>;
|
|
69
69
|
scanCount(opt: ScanStreamOptions): Promise<number>;
|
|
70
|
-
hscanStream(key: string, opt
|
|
71
|
-
hscanCount(key: string, opt
|
|
70
|
+
hscanStream(key: string, opt?: ScanStreamOptions): ReadableTyped<string[]>;
|
|
71
|
+
hscanCount(key: string, opt?: ScanStreamOptions): Promise<number>;
|
|
72
72
|
withPipeline(fn: (pipeline: ChainableCommander) => Promisable<void>): Promise<void>;
|
|
73
73
|
private log;
|
|
74
74
|
}
|
|
@@ -1,14 +1,21 @@
|
|
|
1
1
|
import { CommonDBCreateOptions, CommonKeyValueDB, CommonKeyValueDBSaveBatchOptions, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib';
|
|
2
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { RedisClient } from './redisClient';
|
|
4
3
|
import { RedisKeyValueDBCfg } from './redisKeyValueDB';
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
/**
|
|
5
|
+
* RedisHashKeyValueDB is a KeyValueDB implementation that uses hash fields to simulate tables.
|
|
6
|
+
* The value in the `table` arguments points to a hash field in Redis.
|
|
7
|
+
*
|
|
8
|
+
* The reason for having this approach and also the traditional RedisKeyValueDB is that
|
|
9
|
+
* the currently available Redis versions (in Memorystore, or on MacOs) do not support
|
|
10
|
+
* expiring hash properties.
|
|
11
|
+
* The expiring fields feature is important, and only available via RedisKeyValueDB.
|
|
12
|
+
*
|
|
13
|
+
* Once the available Redis version reaches 7.4.0+,
|
|
14
|
+
* this implementation can take over for RedisKeyValueDB.
|
|
15
|
+
*/
|
|
8
16
|
export declare class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
constructor(cfg: RedisHashKeyValueDBCfg);
|
|
17
|
+
cfg: RedisKeyValueDBCfg;
|
|
18
|
+
constructor(cfg: RedisKeyValueDBCfg);
|
|
12
19
|
support: {
|
|
13
20
|
count?: boolean;
|
|
14
21
|
increment?: boolean;
|
|
@@ -24,8 +31,4 @@ export declare class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDispo
|
|
|
24
31
|
count(table: string): Promise<number>;
|
|
25
32
|
incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]>;
|
|
26
33
|
createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
|
|
27
|
-
private idsToKeys;
|
|
28
|
-
private idToKey;
|
|
29
|
-
private keysToIds;
|
|
30
|
-
private keyToId;
|
|
31
34
|
}
|
|
@@ -2,120 +2,104 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.RedisHashKeyValueDB = void 0;
|
|
4
4
|
const db_lib_1 = require("@naturalcycles/db-lib");
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* RedisHashKeyValueDB is a KeyValueDB implementation that uses hash fields to simulate tables.
|
|
7
|
+
* The value in the `table` arguments points to a hash field in Redis.
|
|
8
|
+
*
|
|
9
|
+
* The reason for having this approach and also the traditional RedisKeyValueDB is that
|
|
10
|
+
* the currently available Redis versions (in Memorystore, or on MacOs) do not support
|
|
11
|
+
* expiring hash properties.
|
|
12
|
+
* The expiring fields feature is important, and only available via RedisKeyValueDB.
|
|
13
|
+
*
|
|
14
|
+
* Once the available Redis version reaches 7.4.0+,
|
|
15
|
+
* this implementation can take over for RedisKeyValueDB.
|
|
16
|
+
*/
|
|
6
17
|
class RedisHashKeyValueDB {
|
|
7
18
|
constructor(cfg) {
|
|
19
|
+
this.cfg = cfg;
|
|
8
20
|
this.support = {
|
|
9
21
|
...db_lib_1.commonKeyValueDBFullSupport,
|
|
10
22
|
};
|
|
11
|
-
this.client = cfg.client;
|
|
12
|
-
this.keyOfHashField = cfg.hashKey;
|
|
13
23
|
}
|
|
14
24
|
async ping() {
|
|
15
|
-
await this.client.ping();
|
|
25
|
+
await this.cfg.client.ping();
|
|
16
26
|
}
|
|
17
27
|
async [Symbol.asyncDispose]() {
|
|
18
|
-
await this.client.disconnect();
|
|
28
|
+
await this.cfg.client.disconnect();
|
|
19
29
|
}
|
|
20
30
|
async getByIds(table, ids) {
|
|
21
31
|
if (!ids.length)
|
|
22
32
|
return [];
|
|
23
33
|
// we assume that the order of returned values is the same as order of input ids
|
|
24
|
-
const bufs = await this.client.hmgetBuffer(
|
|
34
|
+
const bufs = await this.cfg.client.hmgetBuffer(table, ids);
|
|
25
35
|
return bufs.map((buf, i) => [ids[i], buf]).filter(([_k, v]) => v !== null);
|
|
26
36
|
}
|
|
27
37
|
async deleteByIds(table, ids) {
|
|
28
38
|
if (!ids.length)
|
|
29
39
|
return;
|
|
30
|
-
await this.client.hdel(
|
|
40
|
+
await this.cfg.client.hdel(table, ids);
|
|
31
41
|
}
|
|
32
42
|
async saveBatch(table, entries, opt) {
|
|
33
43
|
if (!entries.length)
|
|
34
44
|
return;
|
|
35
|
-
const
|
|
36
|
-
const map = Object.fromEntries(entriesWithKey);
|
|
45
|
+
const record = Object.fromEntries(entries);
|
|
37
46
|
if (opt?.expireAt) {
|
|
38
|
-
await this.client.hsetWithTTL(
|
|
47
|
+
await this.cfg.client.hsetWithTTL(table, record, opt.expireAt);
|
|
39
48
|
}
|
|
40
49
|
else {
|
|
41
|
-
await this.client.hset(
|
|
50
|
+
await this.cfg.client.hset(table, record);
|
|
42
51
|
}
|
|
43
52
|
}
|
|
44
53
|
streamIds(table, limit) {
|
|
45
|
-
|
|
46
|
-
.hscanStream(
|
|
47
|
-
match: `${table}:*`,
|
|
48
|
-
})
|
|
54
|
+
const stream = this.cfg.client
|
|
55
|
+
.hscanStream(table)
|
|
49
56
|
.flatMap(keyValueList => {
|
|
50
57
|
const keys = [];
|
|
51
|
-
keyValueList.
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
});
|
|
58
|
-
if (limit) {
|
|
59
|
-
stream = stream.take(limit);
|
|
60
|
-
}
|
|
58
|
+
for (let i = 0; i < keyValueList.length; i += 2) {
|
|
59
|
+
keys.push(keyValueList[i]);
|
|
60
|
+
}
|
|
61
|
+
return keys;
|
|
62
|
+
})
|
|
63
|
+
.take(limit || Infinity);
|
|
61
64
|
return stream;
|
|
62
65
|
}
|
|
63
66
|
streamValues(table, limit) {
|
|
64
|
-
return this.client
|
|
65
|
-
.hscanStream(
|
|
66
|
-
match: `${table}:*`,
|
|
67
|
-
})
|
|
67
|
+
return this.cfg.client
|
|
68
|
+
.hscanStream(table)
|
|
68
69
|
.flatMap(keyValueList => {
|
|
69
70
|
const values = [];
|
|
70
|
-
keyValueList.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
return values.map(v => Buffer.from(v));
|
|
71
|
+
for (let i = 0; i < keyValueList.length; i += 2) {
|
|
72
|
+
const value = Buffer.from(keyValueList[i + 1]);
|
|
73
|
+
values.push(value);
|
|
74
|
+
}
|
|
75
|
+
return values;
|
|
76
76
|
})
|
|
77
77
|
.take(limit || Infinity);
|
|
78
78
|
}
|
|
79
79
|
streamEntries(table, limit) {
|
|
80
|
-
return this.client
|
|
81
|
-
.hscanStream(
|
|
82
|
-
match: `${table}:*`,
|
|
83
|
-
})
|
|
80
|
+
return this.cfg.client
|
|
81
|
+
.hscanStream(table)
|
|
84
82
|
.flatMap(keyValueList => {
|
|
85
|
-
const entries =
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
83
|
+
const entries = [];
|
|
84
|
+
for (let i = 0; i < keyValueList.length; i += 2) {
|
|
85
|
+
const key = keyValueList[i];
|
|
86
|
+
const value = Buffer.from(keyValueList[i + 1]);
|
|
87
|
+
entries.push([key, value]);
|
|
88
|
+
}
|
|
89
|
+
return entries;
|
|
89
90
|
})
|
|
90
91
|
.take(limit || Infinity);
|
|
91
92
|
}
|
|
92
93
|
async count(table) {
|
|
93
|
-
return await this.client.hscanCount(
|
|
94
|
-
match: `${table}:*`,
|
|
95
|
-
});
|
|
94
|
+
return await this.cfg.client.hscanCount(table);
|
|
96
95
|
}
|
|
97
96
|
async incrementBatch(table, increments) {
|
|
98
|
-
|
|
99
|
-
const resultsWithInternalKeys = await this.client.hincrBatch(this.keyOfHashField, incrementTuplesWithInternalKeys);
|
|
100
|
-
const results = resultsWithInternalKeys.map(([k, v]) => [this.keyToId(table, k), v]);
|
|
101
|
-
return results;
|
|
97
|
+
return await this.cfg.client.hincrBatch(table, increments);
|
|
102
98
|
}
|
|
103
99
|
async createTable(table, opt) {
|
|
104
100
|
if (!opt?.dropIfExists)
|
|
105
101
|
return;
|
|
106
|
-
await this.client.
|
|
107
|
-
}
|
|
108
|
-
idsToKeys(table, ids) {
|
|
109
|
-
return ids.map(id => this.idToKey(table, id));
|
|
110
|
-
}
|
|
111
|
-
idToKey(table, id) {
|
|
112
|
-
return `${table}:${id}`;
|
|
113
|
-
}
|
|
114
|
-
keysToIds(table, keys) {
|
|
115
|
-
return keys.map(key => this.keyToId(table, key));
|
|
116
|
-
}
|
|
117
|
-
keyToId(table, key) {
|
|
118
|
-
return key.slice(table.length + 1);
|
|
102
|
+
await this.cfg.client.del([table]);
|
|
119
103
|
}
|
|
120
104
|
}
|
|
121
105
|
exports.RedisHashKeyValueDB = RedisHashKeyValueDB;
|
|
@@ -5,8 +5,8 @@ export interface RedisKeyValueDBCfg {
|
|
|
5
5
|
client: RedisClient;
|
|
6
6
|
}
|
|
7
7
|
export declare class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
8
|
+
cfg: RedisKeyValueDBCfg;
|
|
8
9
|
constructor(cfg: RedisKeyValueDBCfg);
|
|
9
|
-
client: RedisClient;
|
|
10
10
|
support: {
|
|
11
11
|
count?: boolean;
|
|
12
12
|
increment?: boolean;
|
package/dist/redisKeyValueDB.js
CHANGED
|
@@ -5,28 +5,28 @@ const db_lib_1 = require("@naturalcycles/db-lib");
|
|
|
5
5
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
6
|
class RedisKeyValueDB {
|
|
7
7
|
constructor(cfg) {
|
|
8
|
+
this.cfg = cfg;
|
|
8
9
|
this.support = {
|
|
9
10
|
...db_lib_1.commonKeyValueDBFullSupport,
|
|
10
11
|
};
|
|
11
|
-
this.client = cfg.client;
|
|
12
12
|
}
|
|
13
13
|
async ping() {
|
|
14
|
-
await this.client.ping();
|
|
14
|
+
await this.cfg.client.ping();
|
|
15
15
|
}
|
|
16
16
|
async [Symbol.asyncDispose]() {
|
|
17
|
-
await this.client.disconnect();
|
|
17
|
+
await this.cfg.client.disconnect();
|
|
18
18
|
}
|
|
19
19
|
async getByIds(table, ids) {
|
|
20
20
|
if (!ids.length)
|
|
21
21
|
return [];
|
|
22
22
|
// we assume that the order of returned values is the same as order of input ids
|
|
23
|
-
const bufs = await this.client.mgetBuffer(this.idsToKeys(table, ids));
|
|
23
|
+
const bufs = await this.cfg.client.mgetBuffer(this.idsToKeys(table, ids));
|
|
24
24
|
return bufs.map((buf, i) => [ids[i], buf]).filter(([_k, v]) => v !== null);
|
|
25
25
|
}
|
|
26
26
|
async deleteByIds(table, ids) {
|
|
27
27
|
if (!ids.length)
|
|
28
28
|
return;
|
|
29
|
-
await this.client.del(this.idsToKeys(table, ids));
|
|
29
|
+
await this.cfg.client.del(this.idsToKeys(table, ids));
|
|
30
30
|
}
|
|
31
31
|
async saveBatch(table, entries, opt) {
|
|
32
32
|
if (!entries.length)
|
|
@@ -34,7 +34,7 @@ class RedisKeyValueDB {
|
|
|
34
34
|
if (opt?.expireAt) {
|
|
35
35
|
// There's no supported mset with TTL: https://stackoverflow.com/questions/16423342/redis-multi-set-with-a-ttl
|
|
36
36
|
// so we gonna use a pipeline instead
|
|
37
|
-
await this.client.withPipeline(pipeline => {
|
|
37
|
+
await this.cfg.client.withPipeline(pipeline => {
|
|
38
38
|
for (const [k, v] of entries) {
|
|
39
39
|
pipeline.set(this.idToKey(table, k), v, 'EXAT', opt.expireAt);
|
|
40
40
|
}
|
|
@@ -42,11 +42,11 @@ class RedisKeyValueDB {
|
|
|
42
42
|
}
|
|
43
43
|
else {
|
|
44
44
|
const obj = Object.fromEntries(entries.map(([k, v]) => [this.idToKey(table, k), v]));
|
|
45
|
-
await this.client.msetBuffer(obj);
|
|
45
|
+
await this.cfg.client.msetBuffer(obj);
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
streamIds(table, limit) {
|
|
49
|
-
let stream = this.client
|
|
49
|
+
let stream = this.cfg.client
|
|
50
50
|
.scanStream({
|
|
51
51
|
match: `${table}:*`,
|
|
52
52
|
// count: limit, // count is actually a "batchSize", not a limit
|
|
@@ -58,25 +58,25 @@ class RedisKeyValueDB {
|
|
|
58
58
|
return stream;
|
|
59
59
|
}
|
|
60
60
|
streamValues(table, limit) {
|
|
61
|
-
return this.client
|
|
61
|
+
return this.cfg.client
|
|
62
62
|
.scanStream({
|
|
63
63
|
match: `${table}:*`,
|
|
64
64
|
})
|
|
65
65
|
.flatMap(async (keys) => {
|
|
66
|
-
return (await this.client.mgetBuffer(keys)).filter(js_lib_1._isTruthy);
|
|
66
|
+
return (await this.cfg.client.mgetBuffer(keys)).filter(js_lib_1._isTruthy);
|
|
67
67
|
}, {
|
|
68
68
|
concurrency: 16,
|
|
69
69
|
})
|
|
70
70
|
.take(limit || Infinity);
|
|
71
71
|
}
|
|
72
72
|
streamEntries(table, limit) {
|
|
73
|
-
return this.client
|
|
73
|
+
return this.cfg.client
|
|
74
74
|
.scanStream({
|
|
75
75
|
match: `${table}:*`,
|
|
76
76
|
})
|
|
77
77
|
.flatMap(async (keys) => {
|
|
78
78
|
// casting as Buffer[], because values are expected to exist for given keys
|
|
79
|
-
const bufs = (await this.client.mgetBuffer(keys));
|
|
79
|
+
const bufs = (await this.cfg.client.mgetBuffer(keys));
|
|
80
80
|
return (0, js_lib_1._zip)(this.keysToIds(table, keys), bufs);
|
|
81
81
|
}, {
|
|
82
82
|
concurrency: 16,
|
|
@@ -85,20 +85,20 @@ class RedisKeyValueDB {
|
|
|
85
85
|
}
|
|
86
86
|
async count(table) {
|
|
87
87
|
// todo: implement more efficiently, e.g via LUA?
|
|
88
|
-
return await this.client.scanCount({
|
|
88
|
+
return await this.cfg.client.scanCount({
|
|
89
89
|
match: `${table}:*`,
|
|
90
90
|
});
|
|
91
91
|
}
|
|
92
92
|
async incrementBatch(table, increments) {
|
|
93
93
|
const incrementTuplesWithInternalKeys = increments.map(([id, v]) => [this.idToKey(table, id), v]);
|
|
94
|
-
const resultsWithInternalKeys = await this.client.incrBatch(incrementTuplesWithInternalKeys);
|
|
94
|
+
const resultsWithInternalKeys = await this.cfg.client.incrBatch(incrementTuplesWithInternalKeys);
|
|
95
95
|
const results = resultsWithInternalKeys.map(([k, v]) => [this.keyToId(table, k), v]);
|
|
96
96
|
return results;
|
|
97
97
|
}
|
|
98
98
|
async createTable(table, opt) {
|
|
99
99
|
if (!opt?.dropIfExists)
|
|
100
100
|
return;
|
|
101
|
-
await this.client.dropTable(table);
|
|
101
|
+
await this.cfg.client.dropTable(table);
|
|
102
102
|
}
|
|
103
103
|
idsToKeys(table, ids) {
|
|
104
104
|
return ids.map(id => this.idToKey(table, id));
|
package/package.json
CHANGED
package/src/redisClient.ts
CHANGED
|
@@ -271,7 +271,7 @@ export class RedisClient implements CommonClient {
|
|
|
271
271
|
Convenient type-safe wrapper.
|
|
272
272
|
Returns BATCHES of keys in each iteration (as-is).
|
|
273
273
|
*/
|
|
274
|
-
scanStream(opt
|
|
274
|
+
scanStream(opt?: ScanStreamOptions): ReadableTyped<string[]> {
|
|
275
275
|
return this.redis().scanStream(opt)
|
|
276
276
|
}
|
|
277
277
|
|
|
@@ -294,11 +294,11 @@ export class RedisClient implements CommonClient {
|
|
|
294
294
|
return count
|
|
295
295
|
}
|
|
296
296
|
|
|
297
|
-
hscanStream(key: string, opt
|
|
297
|
+
hscanStream(key: string, opt?: ScanStreamOptions): ReadableTyped<string[]> {
|
|
298
298
|
return this.redis().hscanStream(key, opt)
|
|
299
299
|
}
|
|
300
300
|
|
|
301
|
-
async hscanCount(key: string, opt
|
|
301
|
+
async hscanCount(key: string, opt?: ScanStreamOptions): Promise<number> {
|
|
302
302
|
let count = 0
|
|
303
303
|
|
|
304
304
|
const stream = this.redis().hscanStream(key, opt)
|
|
@@ -6,46 +6,46 @@ import {
|
|
|
6
6
|
IncrementTuple,
|
|
7
7
|
KeyValueDBTuple,
|
|
8
8
|
} from '@naturalcycles/db-lib'
|
|
9
|
-
import { _chunk, StringMap } from '@naturalcycles/js-lib'
|
|
10
9
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib'
|
|
11
|
-
import { RedisClient } from './redisClient'
|
|
12
10
|
import { RedisKeyValueDBCfg } from './redisKeyValueDB'
|
|
13
11
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
/**
|
|
13
|
+
* RedisHashKeyValueDB is a KeyValueDB implementation that uses hash fields to simulate tables.
|
|
14
|
+
* The value in the `table` arguments points to a hash field in Redis.
|
|
15
|
+
*
|
|
16
|
+
* The reason for having this approach and also the traditional RedisKeyValueDB is that
|
|
17
|
+
* the currently available Redis versions (in Memorystore, or on MacOs) do not support
|
|
18
|
+
* expiring hash properties.
|
|
19
|
+
* The expiring fields feature is important, and only available via RedisKeyValueDB.
|
|
20
|
+
*
|
|
21
|
+
* Once the available Redis version reaches 7.4.0+,
|
|
22
|
+
* this implementation can take over for RedisKeyValueDB.
|
|
23
|
+
*/
|
|
18
24
|
export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
19
|
-
|
|
20
|
-
keyOfHashField: string
|
|
21
|
-
|
|
22
|
-
constructor(cfg: RedisHashKeyValueDBCfg) {
|
|
23
|
-
this.client = cfg.client
|
|
24
|
-
this.keyOfHashField = cfg.hashKey
|
|
25
|
-
}
|
|
25
|
+
constructor(public cfg: RedisKeyValueDBCfg) {}
|
|
26
26
|
|
|
27
27
|
support = {
|
|
28
28
|
...commonKeyValueDBFullSupport,
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
async ping(): Promise<void> {
|
|
32
|
-
await this.client.ping()
|
|
32
|
+
await this.cfg.client.ping()
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
async [Symbol.asyncDispose](): Promise<void> {
|
|
36
|
-
await this.client.disconnect()
|
|
36
|
+
await this.cfg.client.disconnect()
|
|
37
37
|
}
|
|
38
38
|
|
|
39
39
|
async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
|
|
40
40
|
if (!ids.length) return []
|
|
41
41
|
// we assume that the order of returned values is the same as order of input ids
|
|
42
|
-
const bufs = await this.client.hmgetBuffer(
|
|
42
|
+
const bufs = await this.cfg.client.hmgetBuffer(table, ids)
|
|
43
43
|
return bufs.map((buf, i) => [ids[i], buf] as KeyValueDBTuple).filter(([_k, v]) => v !== null)
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
async deleteByIds(table: string, ids: string[]): Promise<void> {
|
|
47
47
|
if (!ids.length) return
|
|
48
|
-
await this.client.hdel(
|
|
48
|
+
await this.cfg.client.hdel(table, ids)
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
async saveBatch(
|
|
@@ -55,106 +55,70 @@ export class RedisHashKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
55
55
|
): Promise<void> {
|
|
56
56
|
if (!entries.length) return
|
|
57
57
|
|
|
58
|
-
const
|
|
59
|
-
const map: StringMap<any> = Object.fromEntries(entriesWithKey)
|
|
58
|
+
const record = Object.fromEntries(entries)
|
|
60
59
|
|
|
61
60
|
if (opt?.expireAt) {
|
|
62
|
-
await this.client.hsetWithTTL(
|
|
61
|
+
await this.cfg.client.hsetWithTTL(table, record, opt.expireAt)
|
|
63
62
|
} else {
|
|
64
|
-
await this.client.hset(
|
|
63
|
+
await this.cfg.client.hset(table, record)
|
|
65
64
|
}
|
|
66
65
|
}
|
|
67
66
|
|
|
68
67
|
streamIds(table: string, limit?: number): ReadableTyped<string> {
|
|
69
|
-
|
|
70
|
-
.hscanStream(
|
|
71
|
-
match: `${table}:*`,
|
|
72
|
-
})
|
|
68
|
+
const stream = this.cfg.client
|
|
69
|
+
.hscanStream(table)
|
|
73
70
|
.flatMap(keyValueList => {
|
|
74
71
|
const keys: string[] = []
|
|
75
|
-
keyValueList.
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return this.keysToIds(table, keys)
|
|
72
|
+
for (let i = 0; i < keyValueList.length; i += 2) {
|
|
73
|
+
keys.push(keyValueList[i]!)
|
|
74
|
+
}
|
|
75
|
+
return keys
|
|
80
76
|
})
|
|
81
|
-
|
|
82
|
-
if (limit) {
|
|
83
|
-
stream = stream.take(limit)
|
|
84
|
-
}
|
|
77
|
+
.take(limit || Infinity)
|
|
85
78
|
|
|
86
79
|
return stream
|
|
87
80
|
}
|
|
88
81
|
|
|
89
82
|
streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
|
|
90
|
-
return this.client
|
|
91
|
-
.hscanStream(
|
|
92
|
-
match: `${table}:*`,
|
|
93
|
-
})
|
|
83
|
+
return this.cfg.client
|
|
84
|
+
.hscanStream(table)
|
|
94
85
|
.flatMap(keyValueList => {
|
|
95
|
-
const values:
|
|
96
|
-
keyValueList.
|
|
97
|
-
|
|
98
|
-
values.push(
|
|
99
|
-
}
|
|
100
|
-
return values
|
|
86
|
+
const values: Buffer[] = []
|
|
87
|
+
for (let i = 0; i < keyValueList.length; i += 2) {
|
|
88
|
+
const value = Buffer.from(keyValueList[i + 1]!)
|
|
89
|
+
values.push(value)
|
|
90
|
+
}
|
|
91
|
+
return values
|
|
101
92
|
})
|
|
102
93
|
.take(limit || Infinity)
|
|
103
94
|
}
|
|
104
95
|
|
|
105
96
|
streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
|
|
106
|
-
return this.client
|
|
107
|
-
.hscanStream(
|
|
108
|
-
match: `${table}:*`,
|
|
109
|
-
})
|
|
97
|
+
return this.cfg.client
|
|
98
|
+
.hscanStream(table)
|
|
110
99
|
.flatMap(keyValueList => {
|
|
111
|
-
const entries =
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
100
|
+
const entries: [string, Buffer][] = []
|
|
101
|
+
for (let i = 0; i < keyValueList.length; i += 2) {
|
|
102
|
+
const key = keyValueList[i]!
|
|
103
|
+
const value = Buffer.from(keyValueList[i + 1]!)
|
|
104
|
+
entries.push([key, value])
|
|
105
|
+
}
|
|
106
|
+
return entries
|
|
115
107
|
})
|
|
116
108
|
.take(limit || Infinity)
|
|
117
109
|
}
|
|
118
110
|
|
|
119
111
|
async count(table: string): Promise<number> {
|
|
120
|
-
return await this.client.hscanCount(
|
|
121
|
-
match: `${table}:*`,
|
|
122
|
-
})
|
|
112
|
+
return await this.cfg.client.hscanCount(table)
|
|
123
113
|
}
|
|
124
114
|
|
|
125
115
|
async incrementBatch(table: string, increments: IncrementTuple[]): Promise<IncrementTuple[]> {
|
|
126
|
-
|
|
127
|
-
([id, v]) => [this.idToKey(table, id), v] as [string, number],
|
|
128
|
-
)
|
|
129
|
-
const resultsWithInternalKeys = await this.client.hincrBatch(
|
|
130
|
-
this.keyOfHashField,
|
|
131
|
-
incrementTuplesWithInternalKeys,
|
|
132
|
-
)
|
|
133
|
-
const results = resultsWithInternalKeys.map(
|
|
134
|
-
([k, v]) => [this.keyToId(table, k), v] as IncrementTuple,
|
|
135
|
-
)
|
|
136
|
-
return results
|
|
116
|
+
return await this.cfg.client.hincrBatch(table, increments)
|
|
137
117
|
}
|
|
138
118
|
|
|
139
119
|
async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
|
|
140
120
|
if (!opt?.dropIfExists) return
|
|
141
121
|
|
|
142
|
-
await this.client.
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
private idsToKeys(table: string, ids: string[]): string[] {
|
|
146
|
-
return ids.map(id => this.idToKey(table, id))
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
private idToKey(table: string, id: string): string {
|
|
150
|
-
return `${table}:${id}`
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
private keysToIds(table: string, keys: string[]): string[] {
|
|
154
|
-
return keys.map(key => this.keyToId(table, key))
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
private keyToId(table: string, key: string): string {
|
|
158
|
-
return key.slice(table.length + 1)
|
|
122
|
+
await this.cfg.client.del([table])
|
|
159
123
|
}
|
|
160
124
|
}
|
package/src/redisKeyValueDB.ts
CHANGED
|
@@ -15,34 +15,30 @@ export interface RedisKeyValueDBCfg {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
18
|
-
constructor(cfg: RedisKeyValueDBCfg) {
|
|
19
|
-
this.client = cfg.client
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
client: RedisClient
|
|
18
|
+
constructor(public cfg: RedisKeyValueDBCfg) {}
|
|
23
19
|
|
|
24
20
|
support = {
|
|
25
21
|
...commonKeyValueDBFullSupport,
|
|
26
22
|
}
|
|
27
23
|
|
|
28
24
|
async ping(): Promise<void> {
|
|
29
|
-
await this.client.ping()
|
|
25
|
+
await this.cfg.client.ping()
|
|
30
26
|
}
|
|
31
27
|
|
|
32
28
|
async [Symbol.asyncDispose](): Promise<void> {
|
|
33
|
-
await this.client.disconnect()
|
|
29
|
+
await this.cfg.client.disconnect()
|
|
34
30
|
}
|
|
35
31
|
|
|
36
32
|
async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
|
|
37
33
|
if (!ids.length) return []
|
|
38
34
|
// we assume that the order of returned values is the same as order of input ids
|
|
39
|
-
const bufs = await this.client.mgetBuffer(this.idsToKeys(table, ids))
|
|
35
|
+
const bufs = await this.cfg.client.mgetBuffer(this.idsToKeys(table, ids))
|
|
40
36
|
return bufs.map((buf, i) => [ids[i], buf] as KeyValueDBTuple).filter(([_k, v]) => v !== null)
|
|
41
37
|
}
|
|
42
38
|
|
|
43
39
|
async deleteByIds(table: string, ids: string[]): Promise<void> {
|
|
44
40
|
if (!ids.length) return
|
|
45
|
-
await this.client.del(this.idsToKeys(table, ids))
|
|
41
|
+
await this.cfg.client.del(this.idsToKeys(table, ids))
|
|
46
42
|
}
|
|
47
43
|
|
|
48
44
|
async saveBatch(
|
|
@@ -55,7 +51,7 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
55
51
|
if (opt?.expireAt) {
|
|
56
52
|
// There's no supported mset with TTL: https://stackoverflow.com/questions/16423342/redis-multi-set-with-a-ttl
|
|
57
53
|
// so we gonna use a pipeline instead
|
|
58
|
-
await this.client.withPipeline(pipeline => {
|
|
54
|
+
await this.cfg.client.withPipeline(pipeline => {
|
|
59
55
|
for (const [k, v] of entries) {
|
|
60
56
|
pipeline.set(this.idToKey(table, k), v, 'EXAT', opt.expireAt!)
|
|
61
57
|
}
|
|
@@ -64,12 +60,12 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
64
60
|
const obj: Record<string, Buffer> = Object.fromEntries(
|
|
65
61
|
entries.map(([k, v]) => [this.idToKey(table, k), v]) as KeyValueDBTuple[],
|
|
66
62
|
)
|
|
67
|
-
await this.client.msetBuffer(obj)
|
|
63
|
+
await this.cfg.client.msetBuffer(obj)
|
|
68
64
|
}
|
|
69
65
|
}
|
|
70
66
|
|
|
71
67
|
streamIds(table: string, limit?: number): ReadableTyped<string> {
|
|
72
|
-
let stream = this.client
|
|
68
|
+
let stream = this.cfg.client
|
|
73
69
|
.scanStream({
|
|
74
70
|
match: `${table}:*`,
|
|
75
71
|
// count: limit, // count is actually a "batchSize", not a limit
|
|
@@ -84,13 +80,13 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
84
80
|
}
|
|
85
81
|
|
|
86
82
|
streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
|
|
87
|
-
return this.client
|
|
83
|
+
return this.cfg.client
|
|
88
84
|
.scanStream({
|
|
89
85
|
match: `${table}:*`,
|
|
90
86
|
})
|
|
91
87
|
.flatMap(
|
|
92
88
|
async keys => {
|
|
93
|
-
return (await this.client.mgetBuffer(keys)).filter(_isTruthy)
|
|
89
|
+
return (await this.cfg.client.mgetBuffer(keys)).filter(_isTruthy)
|
|
94
90
|
},
|
|
95
91
|
{
|
|
96
92
|
concurrency: 16,
|
|
@@ -100,14 +96,14 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
100
96
|
}
|
|
101
97
|
|
|
102
98
|
streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
|
|
103
|
-
return this.client
|
|
99
|
+
return this.cfg.client
|
|
104
100
|
.scanStream({
|
|
105
101
|
match: `${table}:*`,
|
|
106
102
|
})
|
|
107
103
|
.flatMap(
|
|
108
104
|
async keys => {
|
|
109
105
|
// casting as Buffer[], because values are expected to exist for given keys
|
|
110
|
-
const bufs = (await this.client.mgetBuffer(keys)) as Buffer[]
|
|
106
|
+
const bufs = (await this.cfg.client.mgetBuffer(keys)) as Buffer[]
|
|
111
107
|
return _zip(this.keysToIds(table, keys), bufs)
|
|
112
108
|
},
|
|
113
109
|
{
|
|
@@ -119,7 +115,7 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
119
115
|
|
|
120
116
|
async count(table: string): Promise<number> {
|
|
121
117
|
// todo: implement more efficiently, e.g via LUA?
|
|
122
|
-
return await this.client.scanCount({
|
|
118
|
+
return await this.cfg.client.scanCount({
|
|
123
119
|
match: `${table}:*`,
|
|
124
120
|
})
|
|
125
121
|
}
|
|
@@ -128,7 +124,7 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
128
124
|
const incrementTuplesWithInternalKeys = increments.map(
|
|
129
125
|
([id, v]) => [this.idToKey(table, id), v] as [string, number],
|
|
130
126
|
)
|
|
131
|
-
const resultsWithInternalKeys = await this.client.incrBatch(incrementTuplesWithInternalKeys)
|
|
127
|
+
const resultsWithInternalKeys = await this.cfg.client.incrBatch(incrementTuplesWithInternalKeys)
|
|
132
128
|
const results = resultsWithInternalKeys.map(
|
|
133
129
|
([k, v]) => [this.keyToId(table, k), v] as IncrementTuple,
|
|
134
130
|
)
|
|
@@ -138,7 +134,7 @@ export class RedisKeyValueDB implements CommonKeyValueDB, AsyncDisposable {
|
|
|
138
134
|
async createTable(table: string, opt?: CommonDBCreateOptions): Promise<void> {
|
|
139
135
|
if (!opt?.dropIfExists) return
|
|
140
136
|
|
|
141
|
-
await this.client.dropTable(table)
|
|
137
|
+
await this.cfg.client.dropTable(table)
|
|
142
138
|
}
|
|
143
139
|
|
|
144
140
|
private idsToKeys(table: string, ids: string[]): string[] {
|