@owlmeans/redis-resource 0.1.18-rc.8 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +182 -50
- package/agent-meta/manifest.json +3 -3
- package/agent-meta/skills/redis-resource/SKILL.md +96 -31
- package/build/consts.d.ts +20 -1
- package/build/consts.d.ts.map +1 -1
- package/build/consts.js +20 -1
- package/build/consts.js.map +1 -1
- package/build/resource.d.ts +11 -2
- package/build/resource.d.ts.map +1 -1
- package/build/resource.js +263 -221
- package/build/resource.js.map +1 -1
- package/build/types.d.ts +27 -12
- package/build/types.d.ts.map +1 -1
- package/build/types.local.d.ts +4 -0
- package/build/types.local.d.ts.map +1 -0
- package/build/types.local.js +2 -0
- package/build/types.local.js.map +1 -0
- package/build/utils/options.d.ts.map +1 -1
- package/package.json +8 -7
- package/src/consts.ts +27 -2
- package/src/resource.ts +305 -241
- package/src/types.local.ts +5 -0
- package/src/types.ts +28 -18
- package/build/.gitkeep +0 -0
- package/src/utils/index.ts +0 -2
- package/src/utils/options.ts +0 -17
package/src/resource.ts
CHANGED
|
@@ -1,33 +1,42 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import {
|
|
3
|
-
import type {
|
|
4
|
-
import type {
|
|
5
|
-
import {
|
|
1
|
+
import { type Criteria, type FirstOptions, type ListOptions, type ListResult, type ResourceRecord, type SubscribeOptions, type Ttl, type Unsubscribe, type WriteOptions, MisshapedRecord, RecordExists, UnknownRecordError, UnsupportedArgumentError, recordQueryHelper } from '@owlmeans/resource'
|
|
2
|
+
import { logThrottle, logger } from '@owlmeans/log'
|
|
3
|
+
import type { Redis } from 'ioredis'
|
|
4
|
+
import type { RedisDbService, RedisResource } from './types.js'
|
|
5
|
+
import {
|
|
6
|
+
CONSUMER_ID_LENGTH, DEFAULT_DB_ALIAS, DEFAULT_STREAM_BLOCK, READ_BATCH, RECLAIM_COUNT,
|
|
7
|
+
RECLAIM_IDLE, SCAN_BATCH, STREAM_MAX_LENGTH
|
|
8
|
+
} from './consts.js'
|
|
6
9
|
import { appendContextual, assertContext } from '@owlmeans/context'
|
|
7
|
-
import
|
|
8
|
-
import {
|
|
9
|
-
import { createIdOfLength } from '@owlmeans/basic-ids'
|
|
10
|
+
import { createIdOfLength, idHelper } from '@owlmeans/basic-ids'
|
|
11
|
+
import type { Config, Context } from './types.local.js'
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
type Context<C extends Config = Config> = ServerContext<C>
|
|
13
|
+
const log = logger('redis-resource')
|
|
13
14
|
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* A `Resource` over plain redis strings: one JSON document per namespaced key.
|
|
18
|
+
*
|
|
19
|
+
* Reads by id are O(1) GETs. Everything else — a criteria read, a listing, a count, a purge —
|
|
20
|
+
* walks this resource's OWN namespace with SCAN and evaluates the criteria in memory, because
|
|
21
|
+
* redis has no index to ask. That is affordable for a small namespaced set (a session cache, an
|
|
22
|
+
* adapter store) and wrong for anything unbounded or hot; a resource that needs real queries
|
|
23
|
+
* belongs in mongo or postgres.
|
|
24
|
+
*/
|
|
14
25
|
export const makeRedisResource = <
|
|
15
26
|
R extends ResourceRecord, T extends RedisResource<R> = RedisResource<R>
|
|
16
27
|
>(
|
|
17
|
-
alias: string, dbAlias = DEFAULT_DB_ALIAS, serviceAlias = DEFAULT_DB_ALIAS
|
|
18
|
-
makeCustomResource?: ResourceMaker<R, T>
|
|
28
|
+
alias: string, dbAlias = DEFAULT_DB_ALIAS, serviceAlias = DEFAULT_DB_ALIAS
|
|
19
29
|
): T => {
|
|
20
30
|
const location = `redis-resource:${alias}`
|
|
21
31
|
|
|
22
32
|
/**
|
|
23
|
-
* A number
|
|
24
|
-
* expire at.
|
|
33
|
+
* A number is seconds from now; a `Date` is the absolute instant to expire at.
|
|
25
34
|
*
|
|
26
35
|
* The absolute form needs PEXPIREAT, not EXPIREAT: `Date.getTime()` is milliseconds while
|
|
27
36
|
* EXPIREAT reads its argument as seconds, so the timestamp landed tens of thousands of years
|
|
28
37
|
* out and the record never expired.
|
|
29
38
|
*/
|
|
30
|
-
const applyTtl = async (key: string, ttl:
|
|
39
|
+
const applyTtl = async (key: string, ttl: Ttl): Promise<void> => {
|
|
31
40
|
if (ttl instanceof Date) {
|
|
32
41
|
await resource.db.client.pexpireat(key, ttl.getTime())
|
|
33
42
|
} else {
|
|
@@ -35,291 +44,348 @@ export const makeRedisResource = <
|
|
|
35
44
|
}
|
|
36
45
|
}
|
|
37
46
|
|
|
38
|
-
|
|
39
|
-
|
|
47
|
+
/** Close a subscription once its lifetime runs out — seconds from now, or an instant. */
|
|
48
|
+
const armTtl = (close: Unsubscribe, ttl?: Ttl): void => {
|
|
49
|
+
if (ttl == null) {
|
|
50
|
+
return
|
|
51
|
+
}
|
|
52
|
+
setTimeout(() => void close(), ttl instanceof Date ? ttl.getTime() - Date.now() : ttl * 1000)
|
|
53
|
+
}
|
|
40
54
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
if (record == null) {
|
|
44
|
-
throw new UnknownRecordError(id)
|
|
45
|
-
}
|
|
55
|
+
/** The record as it goes into redis: whatever was passed, under the id it is keyed by. */
|
|
56
|
+
const withId = (record: Partial<R>, id: string): R => ({ ...record, id } as unknown as R)
|
|
46
57
|
|
|
47
|
-
|
|
48
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Write the record under its key. SET both creates and replaces, which is what lets `save`
|
|
60
|
+
* upsert without reading first.
|
|
61
|
+
*/
|
|
62
|
+
const put = async (id: string, record: Partial<R>, opts?: WriteOptions): Promise<R> => {
|
|
63
|
+
const key = resource.key(id)
|
|
64
|
+
const stored = withId(record, id)
|
|
65
|
+
await resource.db.client.set(key, JSON.stringify(stored))
|
|
66
|
+
if (opts?.ttl != null) {
|
|
67
|
+
await applyTtl(key, opts.ttl)
|
|
68
|
+
}
|
|
49
69
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
opts = field
|
|
53
|
-
field = field.field
|
|
54
|
-
}
|
|
55
|
-
if (opts != null) {
|
|
56
|
-
throw new UnsupportedArgumentError('load:opts')
|
|
57
|
-
}
|
|
58
|
-
if (field === 'id' || field == null) {
|
|
59
|
-
const val = await resource.db.client.get(resource.key(id))
|
|
60
|
-
if (val == null) {
|
|
61
|
-
return null
|
|
62
|
-
}
|
|
70
|
+
return stored
|
|
71
|
+
}
|
|
63
72
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
73
|
+
/**
|
|
74
|
+
* Every key under this resource's prefix, collected with SCAN.
|
|
75
|
+
*
|
|
76
|
+
* Never KEYS: KEYS blocks the server for the length of the whole sweep, while SCAN walks the
|
|
77
|
+
* keyspace in bounded bites. SCAN may hand the same key back twice across iterations, which is
|
|
78
|
+
* why the keys land in a Set.
|
|
79
|
+
*/
|
|
80
|
+
const scanKeys = async (): Promise<string[]> => {
|
|
81
|
+
const found = new Set<string>()
|
|
82
|
+
let cursor = '0'
|
|
83
|
+
do {
|
|
84
|
+
const [next, batch] = await resource.db.client.scan(
|
|
85
|
+
cursor, 'MATCH', resource.key(), 'COUNT', SCAN_BATCH
|
|
86
|
+
)
|
|
87
|
+
cursor = next
|
|
88
|
+
batch.forEach(key => found.add(key))
|
|
89
|
+
} while (cursor !== '0')
|
|
67
90
|
|
|
68
|
-
|
|
69
|
-
|
|
91
|
+
return [...found]
|
|
92
|
+
}
|
|
70
93
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
94
|
+
/**
|
|
95
|
+
* The namespace as key/record pairs, read in MGET batches.
|
|
96
|
+
*
|
|
97
|
+
* O(N) over the resource's own namespace — every criteria that is not a bare id is answered
|
|
98
|
+
* from this in memory. A key holding something other than a string, this resource's own streams
|
|
99
|
+
* included, reads back as `null` from MGET and is skipped.
|
|
100
|
+
*/
|
|
101
|
+
const readNamespace = async (): Promise<Array<[string, R]>> => {
|
|
102
|
+
const keys = await scanKeys()
|
|
103
|
+
const entries: Array<[string, R]> = []
|
|
104
|
+
for (let from = 0; from < keys.length; from += READ_BATCH) {
|
|
105
|
+
const batch = keys.slice(from, from + READ_BATCH)
|
|
106
|
+
const values = await resource.db.client.mget(batch)
|
|
107
|
+
values.forEach((value, index) => {
|
|
108
|
+
if (typeof value === 'string') {
|
|
109
|
+
entries.push([batch[index], JSON.parse(value)])
|
|
110
|
+
}
|
|
111
|
+
})
|
|
112
|
+
}
|
|
79
113
|
|
|
80
|
-
|
|
81
|
-
|
|
114
|
+
return entries
|
|
115
|
+
}
|
|
82
116
|
|
|
83
|
-
|
|
84
|
-
let field = 'id'
|
|
85
|
-
if (typeof opts === 'string') {
|
|
86
|
-
field = opts
|
|
87
|
-
} else if (typeof opts === 'object') {
|
|
88
|
-
field = opts.field ?? field
|
|
89
|
-
}
|
|
90
|
-
if (field !== 'id' && record[field as keyof typeof record] == null) {
|
|
91
|
-
throw new MisshapedRecord('no-field-value')
|
|
92
|
-
}
|
|
93
|
-
const id = field == 'id' ? record.id : record[field as keyof typeof record] as string
|
|
94
|
-
if (id == null) {
|
|
95
|
-
throw new MisshapedRecord('id')
|
|
96
|
-
}
|
|
97
|
-
const _opts = typeof opts === 'object' ? opts : {}
|
|
98
|
-
const existing = await resource.get(id, field)
|
|
99
|
-
const updated = { ...existing, ...record }
|
|
100
|
-
await resource.db.client.set(resource.key(id), JSON.stringify(updated))
|
|
101
|
-
if (_opts.ttl != null) {
|
|
102
|
-
await applyTtl(resource.key(id), _opts.ttl)
|
|
103
|
-
}
|
|
117
|
+
const readRecords = async (): Promise<R[]> => (await readNamespace()).map(([, record]) => record)
|
|
104
118
|
|
|
105
|
-
|
|
106
|
-
|
|
119
|
+
/** `{ id: 'x' }` is the one criteria redis can answer by key; anything else needs the walk. */
|
|
120
|
+
const soleId = (where: Criteria<any>): string | null => {
|
|
121
|
+
const keys = Object.keys(where)
|
|
107
122
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
throw new UnknownRecordError('id')
|
|
111
|
-
}
|
|
112
|
-
if (await resource.load(record.id) != null) {
|
|
113
|
-
throw new RecordExists(record.id)
|
|
114
|
-
}
|
|
123
|
+
return keys.length === 1 && keys[0] === 'id' && typeof where.id === 'string' ? where.id : null
|
|
124
|
+
}
|
|
115
125
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
126
|
+
const loadOne = async (
|
|
127
|
+
idOrWhere: string | Criteria<any>, opts?: FirstOptions<any>
|
|
128
|
+
): Promise<R | null> => {
|
|
129
|
+
const id = typeof idOrWhere === 'string' ? idOrWhere : soleId(idOrWhere)
|
|
130
|
+
if (id != null) {
|
|
131
|
+
const value = await resource.db.client.get(resource.key(id))
|
|
121
132
|
|
|
122
|
-
return
|
|
123
|
-
}
|
|
133
|
+
return value == null ? null : JSON.parse(value)
|
|
134
|
+
}
|
|
124
135
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
if (typeof id === 'object') {
|
|
128
|
-
if (id.id == null) {
|
|
129
|
-
throw new MisshapedRecord('id')
|
|
130
|
-
}
|
|
131
|
-
record = await resource.load(id.id)
|
|
132
|
-
} else {
|
|
133
|
-
// `opts` is optional in the contract: without this branch `delete(id)` — the plain,
|
|
134
|
-
// by-id form every caller uses — left `record` null and returned without touching redis.
|
|
135
|
-
record = await resource.load(id, opts)
|
|
136
|
-
}
|
|
136
|
+
return recordQueryHelper.firstMatch(await readRecords(), idOrWhere as Criteria<any>, opts)
|
|
137
|
+
}
|
|
137
138
|
|
|
138
|
-
|
|
139
|
-
|
|
139
|
+
/**
|
|
140
|
+
* One dedicated connection per subscription: a redis client in subscriber mode accepts nothing
|
|
141
|
+
* but (un)subscribe commands, so the resource's own client has to stay out of it.
|
|
142
|
+
*/
|
|
143
|
+
const listen = async (
|
|
144
|
+
pattern: string, onMessage: (channel: string, message: string) => void,
|
|
145
|
+
opts?: { once?: boolean, ttl?: Ttl }
|
|
146
|
+
): Promise<Unsubscribe> => {
|
|
147
|
+
const subscriber = (resource.db.client as unknown as { duplicate: () => Redis }).duplicate()
|
|
148
|
+
let closed = false
|
|
149
|
+
const unsubscribe: Unsubscribe = async () => {
|
|
150
|
+
if (closed) {
|
|
151
|
+
return
|
|
140
152
|
}
|
|
153
|
+
closed = true
|
|
154
|
+
try {
|
|
155
|
+
await subscriber.punsubscribe(pattern)
|
|
156
|
+
await subscriber.quit()
|
|
157
|
+
} catch (e) {
|
|
158
|
+
log.error('Redis failed to unsubscribe', { resource: alias, pattern, error: e })
|
|
159
|
+
}
|
|
160
|
+
}
|
|
141
161
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
162
|
+
await subscriber.psubscribe(pattern)
|
|
163
|
+
subscriber.on('pmessage', (_pattern, channel, message) => {
|
|
164
|
+
/**
|
|
165
|
+
* `once` closes before the handler's own work finishes on purpose: the handler may await a
|
|
166
|
+
* read, and a second message arriving meanwhile would fire a subscription that has already
|
|
167
|
+
* delivered its one message.
|
|
168
|
+
*/
|
|
169
|
+
if (opts?.once === true) {
|
|
170
|
+
void unsubscribe()
|
|
148
171
|
}
|
|
172
|
+
onMessage(channel, message)
|
|
173
|
+
})
|
|
174
|
+
armTtl(unsubscribe, opts?.ttl)
|
|
149
175
|
|
|
150
|
-
|
|
176
|
+
return unsubscribe
|
|
177
|
+
}
|
|
151
178
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
}
|
|
179
|
+
const resource: T = appendContextual<T>(alias, {
|
|
180
|
+
key: (key?: string) => `${resource.db.prefix}:${key ?? '*'}`,
|
|
155
181
|
|
|
156
|
-
|
|
157
|
-
|
|
182
|
+
/**
|
|
183
|
+
* @throws {UnknownRecordError}
|
|
184
|
+
*/
|
|
185
|
+
get: async (idOrWhere: string | Criteria<any>, opts?: FirstOptions<any>): Promise<R> => {
|
|
186
|
+
const record = await loadOne(idOrWhere, opts)
|
|
187
|
+
if (record == null) {
|
|
188
|
+
throw new UnknownRecordError(
|
|
189
|
+
typeof idOrWhere === 'string' ? idOrWhere : JSON.stringify(idOrWhere)
|
|
190
|
+
)
|
|
158
191
|
}
|
|
159
192
|
|
|
160
|
-
return
|
|
193
|
+
return record
|
|
161
194
|
},
|
|
162
195
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
196
|
+
load: loadOne,
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Unpaged by default: without a `size` every match comes back. A `page` on its own is a
|
|
200
|
+
* caller error rather than a window over an implied default — there is no default here to
|
|
201
|
+
* take a window of.
|
|
202
|
+
*
|
|
203
|
+
* @throws {UnsupportedArgumentError} on `page` without `size`.
|
|
204
|
+
*/
|
|
205
|
+
list: async (where?: Criteria<any>, opts?: ListOptions<any>): Promise<ListResult<R>> => {
|
|
206
|
+
if (opts?.page != null && opts.size == null) {
|
|
207
|
+
throw new UnsupportedArgumentError('page-without-size')
|
|
172
208
|
}
|
|
173
209
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
210
|
+
return recordQueryHelper.applyQuery(await readRecords(), where, opts)
|
|
211
|
+
},
|
|
177
212
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
213
|
+
count: async (where?: Criteria<any>): Promise<number> =>
|
|
214
|
+
recordQueryHelper.filterRecords(await readRecords(), where).length,
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* SET NX rather than a read followed by a write: two callers racing for the same id both saw
|
|
218
|
+
* an empty key, both "created" the record, and the loser's write silently won.
|
|
219
|
+
*
|
|
220
|
+
* A record with no id gets one — redis needs a key, and `save` relies on this to create.
|
|
221
|
+
*
|
|
222
|
+
* @throws {RecordExists}
|
|
223
|
+
*/
|
|
224
|
+
create: async (record: Partial<R>, opts?: WriteOptions): Promise<R> => {
|
|
225
|
+
const id = record.id ?? idHelper.uuid()
|
|
226
|
+
const key = resource.key(id)
|
|
227
|
+
const stored = withId(record, id)
|
|
228
|
+
if (await resource.db.client.set(key, JSON.stringify(stored), 'NX') == null) {
|
|
229
|
+
throw new RecordExists(id)
|
|
230
|
+
}
|
|
231
|
+
if (opts?.ttl != null) {
|
|
232
|
+
await applyTtl(key, opts.ttl)
|
|
184
233
|
}
|
|
185
234
|
|
|
186
|
-
|
|
235
|
+
return stored
|
|
236
|
+
},
|
|
187
237
|
|
|
188
|
-
|
|
238
|
+
/**
|
|
239
|
+
* Replaces the whole record: what is stored afterwards is exactly what was passed, not a
|
|
240
|
+
* merge over what was there. A renewal has to pass its TTL again — SET drops the old expiry.
|
|
241
|
+
*
|
|
242
|
+
* @throws {MisshapedRecord} without an id, {@link UnknownRecordError} when the id is absent.
|
|
243
|
+
*/
|
|
244
|
+
update: async (record: Partial<R>, opts?: WriteOptions): Promise<R> => {
|
|
245
|
+
if (record.id == null) {
|
|
189
246
|
throw new MisshapedRecord('id')
|
|
190
247
|
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
return record
|
|
248
|
+
if (await resource.db.client.exists(resource.key(record.id)) < 1) {
|
|
249
|
+
throw new UnknownRecordError(record.id)
|
|
194
250
|
}
|
|
195
251
|
|
|
196
|
-
|
|
252
|
+
return put(record.id, record, opts)
|
|
197
253
|
},
|
|
198
254
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
const pager = options.pager ?? {}
|
|
203
|
-
|
|
204
|
-
const keys = await resource.db.client.keys(resource.key())
|
|
205
|
-
pager.total = keys.length
|
|
206
|
-
pager.page = pager.page ?? 0
|
|
207
|
-
pager.size = pager.size ?? DEFAULT_PAGE_SIZE
|
|
208
|
-
|
|
209
|
-
if (keys.length === 0) {
|
|
210
|
-
return { items: [], pager }
|
|
211
|
-
}
|
|
255
|
+
/** Creates when the record carries no id, replaces otherwise — SET does both. */
|
|
256
|
+
save: async (record: Partial<R>, opts?: WriteOptions): Promise<R> =>
|
|
257
|
+
record.id == null ? resource.create(record, opts) : put(record.id, record, opts),
|
|
212
258
|
|
|
213
|
-
|
|
214
|
-
|
|
259
|
+
/**
|
|
260
|
+
* GETDEL — reading the record and dropping the key in one command, so two callers cannot both
|
|
261
|
+
* be handed the same record. Needs redis 6.2 or newer.
|
|
262
|
+
*/
|
|
263
|
+
delete: async (id: string): Promise<R | null> => {
|
|
264
|
+
const value = await resource.db.client.getdel(resource.key(id))
|
|
215
265
|
|
|
216
|
-
return
|
|
217
|
-
pager, items: (await Promise.all(slice.map(
|
|
218
|
-
async key => await resource.db.client.get(key)
|
|
219
|
-
))).filter(val => typeof val === 'string').map(
|
|
220
|
-
val => JSON.parse(val as string)
|
|
221
|
-
)
|
|
222
|
-
}
|
|
266
|
+
return value == null ? null : JSON.parse(value)
|
|
223
267
|
},
|
|
224
268
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
try {
|
|
235
|
-
await subscriber.punsubscribe(resource.key(opts.key)).then(() => subscriber.quit())
|
|
236
|
-
} catch (e) {
|
|
237
|
-
console.error('Try to unsubscribe redis resource', e)
|
|
238
|
-
}
|
|
269
|
+
/**
|
|
270
|
+
* Delete-and-return where absence is an error — the consume-once read.
|
|
271
|
+
*
|
|
272
|
+
* @throws {UnknownRecordError}
|
|
273
|
+
*/
|
|
274
|
+
take: async (id: string): Promise<R> => {
|
|
275
|
+
const record = await resource.delete(id)
|
|
276
|
+
if (record == null) {
|
|
277
|
+
throw new UnknownRecordError(id)
|
|
239
278
|
}
|
|
240
279
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
if (opts.once === true) {
|
|
244
|
-
void unsusbscibe()
|
|
245
|
-
}
|
|
246
|
-
}
|
|
280
|
+
return record
|
|
281
|
+
},
|
|
247
282
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
await subscriber.psubscribe(resource.key(opts.key))
|
|
258
|
-
subscriber.on('pmessage', (_, __, message) => listener(JSON.parse(message)))
|
|
283
|
+
/**
|
|
284
|
+
* Bulk delete over the namespace walk. An empty criteria object is refused: wiping a whole
|
|
285
|
+
* namespace is what FLUSHDB is for, not something a filter that came back empty should do.
|
|
286
|
+
*
|
|
287
|
+
* @throws {UnsupportedArgumentError} on empty criteria.
|
|
288
|
+
*/
|
|
289
|
+
purge: async (where: Criteria<any>): Promise<number> => {
|
|
290
|
+
if (where == null || Object.keys(where).length < 1) {
|
|
291
|
+
throw new UnsupportedArgumentError('purge:empty-criteria')
|
|
259
292
|
}
|
|
293
|
+
const keys = (await readNamespace())
|
|
294
|
+
.filter(([, record]) => recordQueryHelper.matchCriteria(record, where))
|
|
295
|
+
.map(([key]) => key)
|
|
260
296
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
? opts.ttl.getTime() - Date.now()
|
|
265
|
-
: typeof opts.ttl === 'number'
|
|
266
|
-
? opts.ttl * 1000
|
|
267
|
-
: null
|
|
268
|
-
if (ttl == null) {
|
|
269
|
-
throw new UnsupportedArgumentError(`subscribe:ttl:${typeof opts.ttl}`)
|
|
270
|
-
}
|
|
271
|
-
setTimeout(unsusbscibe, ttl)
|
|
297
|
+
let deleted = 0
|
|
298
|
+
for (let from = 0; from < keys.length; from += READ_BATCH) {
|
|
299
|
+
deleted += await resource.db.client.del(keys.slice(from, from + READ_BATCH))
|
|
272
300
|
}
|
|
273
301
|
|
|
274
|
-
return
|
|
302
|
+
return deleted
|
|
275
303
|
},
|
|
276
304
|
|
|
277
|
-
publish: async (
|
|
278
|
-
await resource.db.client.publish(resource.key(
|
|
305
|
+
publish: async (value: R, channel?: string): Promise<void> => {
|
|
306
|
+
await resource.db.client.publish(resource.key(channel), JSON.stringify(value))
|
|
279
307
|
},
|
|
280
308
|
|
|
281
|
-
|
|
309
|
+
/**
|
|
310
|
+
* Channel pub/sub. The channel name is namespaced like a key, and an omitted channel listens
|
|
311
|
+
* to the resource's whole namespace — `key()` yields the glob that PSUBSCRIBE matches on.
|
|
312
|
+
*/
|
|
313
|
+
subscribe: async (
|
|
314
|
+
handler: (value: R) => void | Promise<void>, opts?: SubscribeOptions
|
|
315
|
+
): Promise<Unsubscribe> => listen(
|
|
316
|
+
resource.key(opts?.channel),
|
|
317
|
+
(_channel, message) => void handler(JSON.parse(message)),
|
|
318
|
+
opts
|
|
319
|
+
),
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Keyspace notifications for ONE record: every write, expiry or delete on its key hands the
|
|
323
|
+
* handler the current value, or `null` once the key is gone.
|
|
324
|
+
*
|
|
325
|
+
* Keyspace events are published per database and this listens on db 0, so deployments sharing
|
|
326
|
+
* one instance isolate on the key prefix, never on the db index. The server also has to have
|
|
327
|
+
* `notify-keyspace-events` enabled — redis emits nothing otherwise.
|
|
328
|
+
*/
|
|
329
|
+
watch: async (
|
|
330
|
+
id: string, handler: (value: R | null) => void | Promise<void>,
|
|
331
|
+
opts?: Omit<SubscribeOptions, 'channel'>
|
|
332
|
+
): Promise<Unsubscribe> => listen(
|
|
333
|
+
`__keyspace@0__:${resource.key(id)}`,
|
|
334
|
+
() => void loadOne(id).then(record => handler(record)),
|
|
335
|
+
opts
|
|
336
|
+
),
|
|
337
|
+
|
|
338
|
+
stream: async (key: string, value: R): Promise<void> => {
|
|
282
339
|
await resource.db.client.xadd(
|
|
283
|
-
resource.key(key), 'MAXLEN', '~',
|
|
340
|
+
resource.key(key), 'MAXLEN', '~', STREAM_MAX_LENGTH, '*', 'payload', JSON.stringify(value)
|
|
284
341
|
)
|
|
285
342
|
},
|
|
286
343
|
|
|
287
|
-
consume: async function* (
|
|
288
|
-
|
|
344
|
+
consume: async function* (
|
|
345
|
+
key: string, opts?: { group?: string, consumer?: string, block?: number }
|
|
346
|
+
): AsyncGenerator<R> {
|
|
289
347
|
const streamKey = resource.key(key)
|
|
348
|
+
const group = opts?.group
|
|
349
|
+
const consumer = opts?.consumer ?? createIdOfLength(CONSUMER_ID_LENGTH)
|
|
350
|
+
const block = opts?.block ?? DEFAULT_STREAM_BLOCK
|
|
351
|
+
|
|
290
352
|
if (group == null) {
|
|
291
353
|
let id = '$'
|
|
292
354
|
do {
|
|
293
|
-
const resp = await resource.db.client.xread(
|
|
294
|
-
'BLOCK', 1000, 'STREAMS', streamKey, id
|
|
295
|
-
)
|
|
355
|
+
const resp = await resource.db.client.xread('BLOCK', block, 'STREAMS', streamKey, id)
|
|
296
356
|
if (!resp) continue
|
|
297
357
|
|
|
298
358
|
const [, entries] = resp[0]
|
|
299
359
|
for (const [entryId, fields] of entries) {
|
|
300
360
|
id = entryId
|
|
301
|
-
const line = fields[1]
|
|
302
361
|
try {
|
|
303
|
-
|
|
304
|
-
yield JSON.parse(line)
|
|
362
|
+
yield JSON.parse(fields[1])
|
|
305
363
|
} catch (e) {
|
|
306
|
-
|
|
364
|
+
// Fires per bad entry: one line per window per resource is enough.
|
|
365
|
+
if (logThrottle(`${location}:stream-parse`)) {
|
|
366
|
+
log.warn('Cannot parse redis stream entry', { resource: alias, stream: streamKey, error: e })
|
|
367
|
+
}
|
|
307
368
|
}
|
|
308
369
|
}
|
|
309
370
|
} while (true)
|
|
310
371
|
} else {
|
|
311
|
-
|
|
372
|
+
/**
|
|
373
|
+
* Entries a consumer took and never acknowledged, claimed under THIS consumer's name.
|
|
374
|
+
* Claiming them for a fixed name instead left them owned by a consumer that never reads,
|
|
375
|
+
* so a crashed worker's entries moved once and then stalled for good.
|
|
376
|
+
*/
|
|
377
|
+
const reclaim = async function* (): AsyncGenerator<R> {
|
|
312
378
|
try {
|
|
313
379
|
const [, entries] = await resource.db.client.xautoclaim(
|
|
314
|
-
streamKey, group,
|
|
380
|
+
streamKey, group, consumer, RECLAIM_IDLE, '0-0', 'COUNT', RECLAIM_COUNT
|
|
315
381
|
)
|
|
316
382
|
for (const [id, fields] of entries as [string, string[]][]) {
|
|
317
|
-
console.log('Reclaimed', id)
|
|
318
|
-
const line = fields[1]
|
|
319
383
|
try {
|
|
320
|
-
yield JSON.parse(
|
|
384
|
+
yield JSON.parse(fields[1])
|
|
321
385
|
} catch (e) {
|
|
322
|
-
|
|
386
|
+
if (logThrottle(`${location}:stream-reclaim-parse`)) {
|
|
387
|
+
log.warn('Cannot parse reclaimed redis stream entry', { resource: alias, stream: streamKey, error: e })
|
|
388
|
+
}
|
|
323
389
|
}
|
|
324
390
|
await resource.db.client.xack(streamKey, group, id)
|
|
325
391
|
}
|
|
@@ -328,36 +394,43 @@ export const makeRedisResource = <
|
|
|
328
394
|
try {
|
|
329
395
|
await resource.db.client.xgroup('CREATE', streamKey, group, '$', 'MKSTREAM')
|
|
330
396
|
} catch (e) {
|
|
331
|
-
|
|
397
|
+
// BUSYGROUP is the ordinary answer for a group that already exists.
|
|
398
|
+
if (e instanceof Error && e.message.includes('BUSYGROUP')) {
|
|
399
|
+
log.debug('Redis stream consumer group exists', { resource: alias, stream: streamKey, group })
|
|
400
|
+
} else {
|
|
401
|
+
log.error('Error in redis stream consumer group', { resource: alias, stream: streamKey, group, error: e })
|
|
402
|
+
}
|
|
332
403
|
}
|
|
333
404
|
do {
|
|
334
405
|
const resp = await resource.db.client.xreadgroup(
|
|
335
406
|
'GROUP', group, consumer,
|
|
336
|
-
'BLOCK',
|
|
407
|
+
'BLOCK', block, 'STREAMS', streamKey, '>'
|
|
337
408
|
)
|
|
338
409
|
if (!resp) {
|
|
339
|
-
yield*
|
|
410
|
+
yield* reclaim()
|
|
340
411
|
continue
|
|
341
412
|
}
|
|
342
413
|
|
|
343
414
|
const [, entries] = resp[0] as [string, Array<[string, string[]]>]
|
|
344
415
|
for (const [entryId, fields] of entries) {
|
|
345
|
-
const line = fields[1]
|
|
346
416
|
try {
|
|
347
|
-
yield JSON.parse(
|
|
417
|
+
yield JSON.parse(fields[1])
|
|
348
418
|
await resource.db.client.xack(streamKey, group, entryId)
|
|
349
419
|
} catch (e) {
|
|
350
|
-
|
|
420
|
+
// Fires per bad entry: one line per window per resource is enough.
|
|
421
|
+
if (logThrottle(`${location}:stream-parse`)) {
|
|
422
|
+
log.warn('Cannot parse redis stream entry', { resource: alias, stream: streamKey, error: e })
|
|
423
|
+
}
|
|
351
424
|
}
|
|
352
425
|
}
|
|
353
426
|
} while (true)
|
|
354
427
|
}
|
|
355
428
|
}
|
|
356
|
-
} as Partial<T>)
|
|
429
|
+
} as unknown as Partial<T>)
|
|
357
430
|
|
|
358
431
|
resource.init = async () => {
|
|
359
432
|
const context = assertContext<Config, Context>(resource.ctx as Context, location)
|
|
360
|
-
const redis = context.service<RedisDbService>(serviceAlias
|
|
433
|
+
const redis = context.service<RedisDbService>(serviceAlias)
|
|
361
434
|
await redis.ready()
|
|
362
435
|
const db = await redis.db(dbAlias)
|
|
363
436
|
const _pref = (val: string) => val.replaceAll(/\W+/g, '_')
|
|
@@ -368,14 +441,5 @@ export const makeRedisResource = <
|
|
|
368
441
|
}
|
|
369
442
|
}
|
|
370
443
|
|
|
371
|
-
resource.reinitializeContext = <Type extends Contextual>(context: BasicContext<Config>) => {
|
|
372
|
-
const resource = (makeCustomResource?.(dbAlias, serviceAlias)
|
|
373
|
-
?? makeRedisResource<R, T>(alias, dbAlias, serviceAlias)) as unknown as Type
|
|
374
|
-
|
|
375
|
-
resource.ctx = context
|
|
376
|
-
|
|
377
|
-
return resource as Type
|
|
378
|
-
}
|
|
379
|
-
|
|
380
444
|
return resource
|
|
381
445
|
}
|