@owlmeans/redis-resource 0.1.18-rc.8 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/resource.ts CHANGED
@@ -1,33 +1,42 @@
1
- import type { ResourceMaker, ResourceRecord } from '@owlmeans/resource'
2
- import { MisshapedRecord, prepareListOptions, RecordExists, UnknownRecordError, UnsupportedArgumentError } from '@owlmeans/resource'
3
- import type { RedisClient, RedisDbService, RedisResource } from './types.js'
4
- import type { ServerConfig, ServerContext } from '@owlmeans/server-context'
5
- import { DEFAULT_DB_ALIAS, DEFAULT_PAGE_SIZE } from './consts.js'
1
+ import { type Criteria, type FirstOptions, type ListOptions, type ListResult, type ResourceRecord, type SubscribeOptions, type Ttl, type Unsubscribe, type WriteOptions, MisshapedRecord, RecordExists, UnknownRecordError, UnsupportedArgumentError, recordQueryHelper } from '@owlmeans/resource'
2
+ import { logThrottle, logger } from '@owlmeans/log'
3
+ import type { Redis } from 'ioredis'
4
+ import type { RedisDbService, RedisResource } from './types.js'
5
+ import {
6
+ CONSUMER_ID_LENGTH, DEFAULT_DB_ALIAS, DEFAULT_STREAM_BLOCK, READ_BATCH, RECLAIM_COUNT,
7
+ RECLAIM_IDLE, SCAN_BATCH, STREAM_MAX_LENGTH
8
+ } from './consts.js'
6
9
  import { appendContextual, assertContext } from '@owlmeans/context'
7
- import type { BasicContext, Contextual } from '@owlmeans/context'
8
- import { prepareSubOptions } from './utils/options.js'
9
- import { createIdOfLength } from '@owlmeans/basic-ids'
10
+ import { createIdOfLength, idHelper } from '@owlmeans/basic-ids'
11
+ import type { Config, Context } from './types.local.js'
10
12
 
11
- type Config = ServerConfig
12
- type Context<C extends Config = Config> = ServerContext<C>
13
+ const log = logger('redis-resource')
13
14
 
15
+
16
+ /**
17
+ * A `Resource` over plain redis strings: one JSON document per namespaced key.
18
+ *
19
+ * Reads by id are O(1) GETs. Everything else — a criteria read, a listing, a count, a purge —
20
+ * walks this resource's OWN namespace with SCAN and evaluates the criteria in memory, because
21
+ * redis has no index to ask. That is affordable for a small namespaced set (a session cache, an
22
+ * adapter store) and wrong for anything unbounded or hot; a resource that needs real queries
23
+ * belongs in mongo or postgres.
24
+ */
14
25
  export const makeRedisResource = <
15
26
  R extends ResourceRecord, T extends RedisResource<R> = RedisResource<R>
16
27
  >(
17
- alias: string, dbAlias = DEFAULT_DB_ALIAS, serviceAlias = DEFAULT_DB_ALIAS,
18
- makeCustomResource?: ResourceMaker<R, T>
28
+ alias: string, dbAlias = DEFAULT_DB_ALIAS, serviceAlias = DEFAULT_DB_ALIAS
19
29
  ): T => {
20
30
  const location = `redis-resource:${alias}`
21
31
 
22
32
  /**
23
- * A number (or its string form) is seconds from now; a `Date` is the absolute instant to
24
- * expire at.
33
+ * A number is seconds from now; a `Date` is the absolute instant to expire at.
25
34
  *
26
35
  * The absolute form needs PEXPIREAT, not EXPIREAT: `Date.getTime()` is milliseconds while
27
36
  * EXPIREAT reads its argument as seconds, so the timestamp landed tens of thousands of years
28
37
  * out and the record never expired.
29
38
  */
30
- const applyTtl = async (key: string, ttl: number | string | Date): Promise<void> => {
39
+ const applyTtl = async (key: string, ttl: Ttl): Promise<void> => {
31
40
  if (ttl instanceof Date) {
32
41
  await resource.db.client.pexpireat(key, ttl.getTime())
33
42
  } else {
@@ -35,291 +44,348 @@ export const makeRedisResource = <
35
44
  }
36
45
  }
37
46
 
38
- const resource: T = appendContextual<T>(alias, {
39
- key: key => `${resource.db.prefix}:${key ?? '*'}`,
47
+ /** Close a subscription once its lifetime runs out — seconds from now, or an instant. */
48
+ const armTtl = (close: Unsubscribe, ttl?: Ttl): void => {
49
+ if (ttl == null) {
50
+ return
51
+ }
52
+ setTimeout(() => void close(), ttl instanceof Date ? ttl.getTime() - Date.now() : ttl * 1000)
53
+ }
40
54
 
41
- get: async (id, field, opts) => {
42
- const record = await resource.load(id, field, opts)
43
- if (record == null) {
44
- throw new UnknownRecordError(id)
45
- }
55
+ /** The record as it goes into redis: whatever was passed, under the id it is keyed by. */
56
+ const withId = (record: Partial<R>, id: string): R => ({ ...record, id } as unknown as R)
46
57
 
47
- return record
48
- },
58
+ /**
59
+ * Write the record under its key. SET both creates and replaces, which is what lets `save`
60
+ * upsert without reading first.
61
+ */
62
+ const put = async (id: string, record: Partial<R>, opts?: WriteOptions): Promise<R> => {
63
+ const key = resource.key(id)
64
+ const stored = withId(record, id)
65
+ await resource.db.client.set(key, JSON.stringify(stored))
66
+ if (opts?.ttl != null) {
67
+ await applyTtl(key, opts.ttl)
68
+ }
49
69
 
50
- load: async (id, field, opts) => {
51
- if (typeof field === 'object') {
52
- opts = field
53
- field = field.field
54
- }
55
- if (opts != null) {
56
- throw new UnsupportedArgumentError('load:opts')
57
- }
58
- if (field === 'id' || field == null) {
59
- const val = await resource.db.client.get(resource.key(id))
60
- if (val == null) {
61
- return null
62
- }
70
+ return stored
71
+ }
63
72
 
64
- return JSON.parse(val)
65
- }
66
- const list = await resource.list({ [field]: id }, { pager: { size: 1 } })
73
+ /**
74
+ * Every key under this resource's prefix, collected with SCAN.
75
+ *
76
+ * Never KEYS: KEYS blocks the server for the length of the whole sweep, while SCAN walks the
77
+ * keyspace in bounded bites. SCAN may hand the same key back twice across iterations, which is
78
+ * why the keys land in a Set.
79
+ */
80
+ const scanKeys = async (): Promise<string[]> => {
81
+ const found = new Set<string>()
82
+ let cursor = '0'
83
+ do {
84
+ const [next, batch] = await resource.db.client.scan(
85
+ cursor, 'MATCH', resource.key(), 'COUNT', SCAN_BATCH
86
+ )
87
+ cursor = next
88
+ batch.forEach(key => found.add(key))
89
+ } while (cursor !== '0')
67
90
 
68
- return list.items[0] ?? null
69
- },
91
+ return [...found]
92
+ }
70
93
 
71
- save: async (record, opts) => {
72
- if (record.id == null) {
73
- throw new MisshapedRecord('id')
74
- }
75
- const existing = await resource.load(record.id)
76
- if (existing != null) {
77
- return resource.update(record, opts)
78
- }
94
+ /**
95
+ * The namespace as key/record pairs, read in MGET batches.
96
+ *
97
+ * O(N) over the resource's own namespace — every criteria that is not a bare id is answered
98
+ * from this in memory. A key holding something other than a string, this resource's own streams
99
+ * included, reads back as `null` from MGET and is skipped.
100
+ */
101
+ const readNamespace = async (): Promise<Array<[string, R]>> => {
102
+ const keys = await scanKeys()
103
+ const entries: Array<[string, R]> = []
104
+ for (let from = 0; from < keys.length; from += READ_BATCH) {
105
+ const batch = keys.slice(from, from + READ_BATCH)
106
+ const values = await resource.db.client.mget(batch)
107
+ values.forEach((value, index) => {
108
+ if (typeof value === 'string') {
109
+ entries.push([batch[index], JSON.parse(value)])
110
+ }
111
+ })
112
+ }
79
113
 
80
- return resource.create(record, typeof opts !== 'string' ? opts : undefined)
81
- },
114
+ return entries
115
+ }
82
116
 
83
- update: async (record, opts) => {
84
- let field = 'id'
85
- if (typeof opts === 'string') {
86
- field = opts
87
- } else if (typeof opts === 'object') {
88
- field = opts.field ?? field
89
- }
90
- if (field !== 'id' && record[field as keyof typeof record] == null) {
91
- throw new MisshapedRecord('no-field-value')
92
- }
93
- const id = field == 'id' ? record.id : record[field as keyof typeof record] as string
94
- if (id == null) {
95
- throw new MisshapedRecord('id')
96
- }
97
- const _opts = typeof opts === 'object' ? opts : {}
98
- const existing = await resource.get(id, field)
99
- const updated = { ...existing, ...record }
100
- await resource.db.client.set(resource.key(id), JSON.stringify(updated))
101
- if (_opts.ttl != null) {
102
- await applyTtl(resource.key(id), _opts.ttl)
103
- }
117
+ const readRecords = async (): Promise<R[]> => (await readNamespace()).map(([, record]) => record)
104
118
 
105
- return updated
106
- },
119
+ /** `{ id: 'x' }` is the one criteria redis can answer by key; anything else needs the walk. */
120
+ const soleId = (where: Criteria<any>): string | null => {
121
+ const keys = Object.keys(where)
107
122
 
108
- create: async (record, opts) => {
109
- if (record.id == null) {
110
- throw new UnknownRecordError('id')
111
- }
112
- if (await resource.load(record.id) != null) {
113
- throw new RecordExists(record.id)
114
- }
123
+ return keys.length === 1 && keys[0] === 'id' && typeof where.id === 'string' ? where.id : null
124
+ }
115
125
 
116
- const id = record.id
117
- await resource.db.client.set(resource.key(id), JSON.stringify(record))
118
- if (opts?.ttl != null) {
119
- await applyTtl(resource.key(id), opts.ttl)
120
- }
126
+ const loadOne = async (
127
+ idOrWhere: string | Criteria<any>, opts?: FirstOptions<any>
128
+ ): Promise<R | null> => {
129
+ const id = typeof idOrWhere === 'string' ? idOrWhere : soleId(idOrWhere)
130
+ if (id != null) {
131
+ const value = await resource.db.client.get(resource.key(id))
121
132
 
122
- return record
123
- },
133
+ return value == null ? null : JSON.parse(value)
134
+ }
124
135
 
125
- delete: async (id, opts) => {
126
- let record: R | null = null
127
- if (typeof id === 'object') {
128
- if (id.id == null) {
129
- throw new MisshapedRecord('id')
130
- }
131
- record = await resource.load(id.id)
132
- } else {
133
- // `opts` is optional in the contract: without this branch `delete(id)` — the plain,
134
- // by-id form every caller uses — left `record` null and returned without touching redis.
135
- record = await resource.load(id, opts)
136
- }
136
+ return recordQueryHelper.firstMatch(await readRecords(), idOrWhere as Criteria<any>, opts)
137
+ }
137
138
 
138
- if (record == null) {
139
- return null
139
+ /**
140
+ * One dedicated connection per subscription: a redis client in subscriber mode accepts nothing
141
+ * but (un)subscribe commands, so the resource's own client has to stay out of it.
142
+ */
143
+ const listen = async (
144
+ pattern: string, onMessage: (channel: string, message: string) => void,
145
+ opts?: { once?: boolean, ttl?: Ttl }
146
+ ): Promise<Unsubscribe> => {
147
+ const subscriber = (resource.db.client as unknown as { duplicate: () => Redis }).duplicate()
148
+ let closed = false
149
+ const unsubscribe: Unsubscribe = async () => {
150
+ if (closed) {
151
+ return
140
152
  }
153
+ closed = true
154
+ try {
155
+ await subscriber.punsubscribe(pattern)
156
+ await subscriber.quit()
157
+ } catch (e) {
158
+ log.error('Redis failed to unsubscribe', { resource: alias, pattern, error: e })
159
+ }
160
+ }
141
161
 
142
- let field = 'id'
143
- if (typeof opts === 'string') {
144
- field = opts
145
- opts = undefined
146
- } else if (typeof opts === 'object') {
147
- field = opts.field ?? field
162
+ await subscriber.psubscribe(pattern)
163
+ subscriber.on('pmessage', (_pattern, channel, message) => {
164
+ /**
165
+ * `once` closes before the handler's own work finishes on purpose: the handler may await a
166
+ * read, and a second message arriving meanwhile would fire a subscription that has already
167
+ * delivered its one message.
168
+ */
169
+ if (opts?.once === true) {
170
+ void unsubscribe()
148
171
  }
172
+ onMessage(channel, message)
173
+ })
174
+ armTtl(unsubscribe, opts?.ttl)
149
175
 
150
- id = record.id ?? ''
176
+ return unsubscribe
177
+ }
151
178
 
152
- if (id === '') {
153
- throw new MisshapedRecord('id')
154
- }
179
+ const resource: T = appendContextual<T>(alias, {
180
+ key: (key?: string) => `${resource.db.prefix}:${key ?? '*'}`,
155
181
 
156
- if (await resource.db.client.del(resource.key(id)) > 0) {
157
- return record
182
+ /**
183
+ * @throws {UnknownRecordError}
184
+ */
185
+ get: async (idOrWhere: string | Criteria<any>, opts?: FirstOptions<any>): Promise<R> => {
186
+ const record = await loadOne(idOrWhere, opts)
187
+ if (record == null) {
188
+ throw new UnknownRecordError(
189
+ typeof idOrWhere === 'string' ? idOrWhere : JSON.stringify(idOrWhere)
190
+ )
158
191
  }
159
192
 
160
- return null
193
+ return record
161
194
  },
162
195
 
163
- pick: async (id, opts) => {
164
- let record: R | null = null
165
- if (typeof id === 'object') {
166
- if (id.id == null) {
167
- throw new MisshapedRecord('id')
168
- }
169
- record = await resource.load(id.id)
170
- } else {
171
- record = await resource.load(id, opts)
196
+ load: loadOne,
197
+
198
+ /**
199
+ * Unpaged by default: without a `size` every match comes back. A `page` on its own is a
200
+ * caller error rather than a window over an implied default — there is no default here to
201
+ * take a window of.
202
+ *
203
+ * @throws {UnsupportedArgumentError} on `page` without `size`.
204
+ */
205
+ list: async (where?: Criteria<any>, opts?: ListOptions<any>): Promise<ListResult<R>> => {
206
+ if (opts?.page != null && opts.size == null) {
207
+ throw new UnsupportedArgumentError('page-without-size')
172
208
  }
173
209
 
174
- if (record == null) {
175
- return null
176
- }
210
+ return recordQueryHelper.applyQuery(await readRecords(), where, opts)
211
+ },
177
212
 
178
- let field = 'id'
179
- if (typeof opts === 'string') {
180
- field = opts
181
- opts = undefined
182
- } else if (typeof opts === 'object') {
183
- field = opts.field ?? field
213
+ count: async (where?: Criteria<any>): Promise<number> =>
214
+ recordQueryHelper.filterRecords(await readRecords(), where).length,
215
+
216
+ /**
217
+ * SET NX rather than a read followed by a write: two callers racing for the same id both saw
218
+ * an empty key, both "created" the record, and the loser's write silently won.
219
+ *
220
+ * A record with no id gets one — redis needs a key, and `save` relies on this to create.
221
+ *
222
+ * @throws {RecordExists}
223
+ */
224
+ create: async (record: Partial<R>, opts?: WriteOptions): Promise<R> => {
225
+ const id = record.id ?? idHelper.uuid()
226
+ const key = resource.key(id)
227
+ const stored = withId(record, id)
228
+ if (await resource.db.client.set(key, JSON.stringify(stored), 'NX') == null) {
229
+ throw new RecordExists(id)
230
+ }
231
+ if (opts?.ttl != null) {
232
+ await applyTtl(key, opts.ttl)
184
233
  }
185
234
 
186
- id = record.id ?? ''
235
+ return stored
236
+ },
187
237
 
188
- if (id === '') {
238
+ /**
239
+ * Replaces the whole record: what is stored afterwards is exactly what was passed, not a
240
+ * merge over what was there. A renewal has to pass its TTL again — SET drops the old expiry.
241
+ *
242
+ * @throws {MisshapedRecord} without an id, {@link UnknownRecordError} when the id is absent.
243
+ */
244
+ update: async (record: Partial<R>, opts?: WriteOptions): Promise<R> => {
245
+ if (record.id == null) {
189
246
  throw new MisshapedRecord('id')
190
247
  }
191
-
192
- if (await resource.db.client.del(resource.key(id)) > 0) {
193
- return record
248
+ if (await resource.db.client.exists(resource.key(record.id)) < 1) {
249
+ throw new UnknownRecordError(record.id)
194
250
  }
195
251
 
196
- throw new UnknownRecordError(id)
252
+ return put(record.id, record, opts)
197
253
  },
198
254
 
199
- list: async (criteria, opts) => {
200
- const options = prepareListOptions(DEFAULT_PAGE_SIZE, criteria, opts)
201
- criteria = options.criteria
202
- const pager = options.pager ?? {}
203
-
204
- const keys = await resource.db.client.keys(resource.key())
205
- pager.total = keys.length
206
- pager.page = pager.page ?? 0
207
- pager.size = pager.size ?? DEFAULT_PAGE_SIZE
208
-
209
- if (keys.length === 0) {
210
- return { items: [], pager }
211
- }
255
+ /** Creates when the record carries no id, replaces otherwise — SET does both. */
256
+ save: async (record: Partial<R>, opts?: WriteOptions): Promise<R> =>
257
+ record.id == null ? resource.create(record, opts) : put(record.id, record, opts),
212
258
 
213
- const start = pager.page * pager.size
214
- const slice = keys.toSpliced(start, pager.size)
259
+ /**
260
+ * GETDEL — reading the record and dropping the key in one command, so two callers cannot both
261
+ * be handed the same record. Needs redis 6.2 or newer.
262
+ */
263
+ delete: async (id: string): Promise<R | null> => {
264
+ const value = await resource.db.client.getdel(resource.key(id))
215
265
 
216
- return {
217
- pager, items: (await Promise.all(slice.map(
218
- async key => await resource.db.client.get(key)
219
- ))).filter(val => typeof val === 'string').map(
220
- val => JSON.parse(val as string)
221
- )
222
- }
266
+ return value == null ? null : JSON.parse(value)
223
267
  },
224
268
 
225
- subscribe: async (handler, opts) => {
226
- opts = prepareSubOptions(opts)
227
-
228
- let _unsubascribeCalled = false
229
- const unsusbscibe = async () => {
230
- if (_unsubascribeCalled) {
231
- return
232
- }
233
- _unsubascribeCalled = true
234
- try {
235
- await subscriber.punsubscribe(resource.key(opts.key)).then(() => subscriber.quit())
236
- } catch (e) {
237
- console.error('Try to unsubscribe redis resource', e)
238
- }
269
+ /**
270
+ * Delete-and-return where absence is an error — the consume-once read.
271
+ *
272
+ * @throws {UnknownRecordError}
273
+ */
274
+ take: async (id: string): Promise<R> => {
275
+ const record = await resource.delete(id)
276
+ if (record == null) {
277
+ throw new UnknownRecordError(id)
239
278
  }
240
279
 
241
- const listener = (value: R) => {
242
- handler(value as any)
243
- if (opts.once === true) {
244
- void unsusbscibe()
245
- }
246
- }
280
+ return record
281
+ },
247
282
 
248
- const subscriber = (resource.db.client as RedisClient).duplicate()
249
- if (opts.key != null && null != await resource.load(opts.key)) {
250
- await subscriber.psubscribe(`__keyspace@0__:${resource.key(opts.key)}`)
251
- subscriber.on('pmessage', async (_, channel) => {
252
- const [, , ...key] = (channel as string).split(':')
253
- const record = await resource.load(key.join(':'))
254
- listener(record ?? { id: key, __null: true } as any as R)
255
- })
256
- } else {
257
- await subscriber.psubscribe(resource.key(opts.key))
258
- subscriber.on('pmessage', (_, __, message) => listener(JSON.parse(message)))
283
+ /**
284
+ * Bulk delete over the namespace walk. An empty criteria object is refused: wiping a whole
285
+ * namespace is what FLUSHDB is for, not something a filter that came back empty should do.
286
+ *
287
+ * @throws {UnsupportedArgumentError} on empty criteria.
288
+ */
289
+ purge: async (where: Criteria<any>): Promise<number> => {
290
+ if (where == null || Object.keys(where).length < 1) {
291
+ throw new UnsupportedArgumentError('purge:empty-criteria')
259
292
  }
293
+ const keys = (await readNamespace())
294
+ .filter(([, record]) => recordQueryHelper.matchCriteria(record, where))
295
+ .map(([key]) => key)
260
296
 
261
-
262
- if (opts.ttl != null) {
263
- const ttl = opts.ttl instanceof Date
264
- ? opts.ttl.getTime() - Date.now()
265
- : typeof opts.ttl === 'number'
266
- ? opts.ttl * 1000
267
- : null
268
- if (ttl == null) {
269
- throw new UnsupportedArgumentError(`subscribe:ttl:${typeof opts.ttl}`)
270
- }
271
- setTimeout(unsusbscibe, ttl)
297
+ let deleted = 0
298
+ for (let from = 0; from < keys.length; from += READ_BATCH) {
299
+ deleted += await resource.db.client.del(keys.slice(from, from + READ_BATCH))
272
300
  }
273
301
 
274
- return unsusbscibe
302
+ return deleted
275
303
  },
276
304
 
277
- publish: async (record, key) => {
278
- await resource.db.client.publish(resource.key(key), JSON.stringify(record))
305
+ publish: async (value: R, channel?: string): Promise<void> => {
306
+ await resource.db.client.publish(resource.key(channel), JSON.stringify(value))
279
307
  },
280
308
 
281
- stream: async (key, record) => {
309
+ /**
310
+ * Channel pub/sub. The channel name is namespaced like a key, and an omitted channel listens
311
+ * to the resource's whole namespace — `key()` yields the glob that PSUBSCRIBE matches on.
312
+ */
313
+ subscribe: async (
314
+ handler: (value: R) => void | Promise<void>, opts?: SubscribeOptions
315
+ ): Promise<Unsubscribe> => listen(
316
+ resource.key(opts?.channel),
317
+ (_channel, message) => void handler(JSON.parse(message)),
318
+ opts
319
+ ),
320
+
321
+ /**
322
+ * Keyspace notifications for ONE record: every write, expiry or delete on its key hands the
323
+ * handler the current value, or `null` once the key is gone.
324
+ *
325
+ * Keyspace events are published per database and this listens on db 0, so deployments sharing
326
+ * one instance isolate on the key prefix, never on the db index. The server also has to have
327
+ * `notify-keyspace-events` enabled — redis emits nothing otherwise.
328
+ */
329
+ watch: async (
330
+ id: string, handler: (value: R | null) => void | Promise<void>,
331
+ opts?: Omit<SubscribeOptions, 'channel'>
332
+ ): Promise<Unsubscribe> => listen(
333
+ `__keyspace@0__:${resource.key(id)}`,
334
+ () => void loadOne(id).then(record => handler(record)),
335
+ opts
336
+ ),
337
+
338
+ stream: async (key: string, value: R): Promise<void> => {
282
339
  await resource.db.client.xadd(
283
- resource.key(key), 'MAXLEN', '~', 10000, '*', 'payload', JSON.stringify(record)
340
+ resource.key(key), 'MAXLEN', '~', STREAM_MAX_LENGTH, '*', 'payload', JSON.stringify(value)
284
341
  )
285
342
  },
286
343
 
287
- consume: async function* (key, group, consumer = createIdOfLength(15)) {
288
- console.log(`Establish redis stream consumer: ${group ?? 'no-group'}:${consumer} for ${key}`)
344
+ consume: async function* (
345
+ key: string, opts?: { group?: string, consumer?: string, block?: number }
346
+ ): AsyncGenerator<R> {
289
347
  const streamKey = resource.key(key)
348
+ const group = opts?.group
349
+ const consumer = opts?.consumer ?? createIdOfLength(CONSUMER_ID_LENGTH)
350
+ const block = opts?.block ?? DEFAULT_STREAM_BLOCK
351
+
290
352
  if (group == null) {
291
353
  let id = '$'
292
354
  do {
293
- const resp = await resource.db.client.xread(
294
- 'BLOCK', 1000, 'STREAMS', streamKey, id
295
- )
355
+ const resp = await resource.db.client.xread('BLOCK', block, 'STREAMS', streamKey, id)
296
356
  if (!resp) continue
297
357
 
298
358
  const [, entries] = resp[0]
299
359
  for (const [entryId, fields] of entries) {
300
360
  id = entryId
301
- const line = fields[1]
302
361
  try {
303
- console.log('received', fields)
304
- yield JSON.parse(line)
362
+ yield JSON.parse(fields[1])
305
363
  } catch (e) {
306
- console.error('Cannot parse redis stream entry', e)
364
+ // Fires per bad entry: one line per window per resource is enough.
365
+ if (logThrottle(`${location}:stream-parse`)) {
366
+ log.warn('Cannot parse redis stream entry', { resource: alias, stream: streamKey, error: e })
367
+ }
307
368
  }
308
369
  }
309
370
  } while (true)
310
371
  } else {
311
- const _reclaim = async function* () {
372
+ /**
373
+ * Entries a consumer took and never acknowledged, claimed under THIS consumer's name.
374
+ * Claiming them for a fixed name instead left them owned by a consumer that never reads,
375
+ * so a crashed worker's entries moved once and then stalled for good.
376
+ */
377
+ const reclaim = async function* (): AsyncGenerator<R> {
312
378
  try {
313
379
  const [, entries] = await resource.db.client.xautoclaim(
314
- streamKey, group, 'junitor', 60000, '0-0', 'COUNT', 10
380
+ streamKey, group, consumer, RECLAIM_IDLE, '0-0', 'COUNT', RECLAIM_COUNT
315
381
  )
316
382
  for (const [id, fields] of entries as [string, string[]][]) {
317
- console.log('Reclaimed', id)
318
- const line = fields[1]
319
383
  try {
320
- yield JSON.parse(line)
384
+ yield JSON.parse(fields[1])
321
385
  } catch (e) {
322
- console.error('Cannot parse reclaimed redis stream entry', e)
386
+ if (logThrottle(`${location}:stream-reclaim-parse`)) {
387
+ log.warn('Cannot parse reclaimed redis stream entry', { resource: alias, stream: streamKey, error: e })
388
+ }
323
389
  }
324
390
  await resource.db.client.xack(streamKey, group, id)
325
391
  }
@@ -328,36 +394,43 @@ export const makeRedisResource = <
328
394
  try {
329
395
  await resource.db.client.xgroup('CREATE', streamKey, group, '$', 'MKSTREAM')
330
396
  } catch (e) {
331
- console.error('Error in redis stream consumer group', e)
397
+ // BUSYGROUP is the ordinary answer for a group that already exists.
398
+ if (e instanceof Error && e.message.includes('BUSYGROUP')) {
399
+ log.debug('Redis stream consumer group exists', { resource: alias, stream: streamKey, group })
400
+ } else {
401
+ log.error('Error in redis stream consumer group', { resource: alias, stream: streamKey, group, error: e })
402
+ }
332
403
  }
333
404
  do {
334
405
  const resp = await resource.db.client.xreadgroup(
335
406
  'GROUP', group, consumer,
336
- 'BLOCK', 1000, 'STREAMS', streamKey, '>'
407
+ 'BLOCK', block, 'STREAMS', streamKey, '>'
337
408
  )
338
409
  if (!resp) {
339
- yield* _reclaim()
410
+ yield* reclaim()
340
411
  continue
341
412
  }
342
413
 
343
414
  const [, entries] = resp[0] as [string, Array<[string, string[]]>]
344
415
  for (const [entryId, fields] of entries) {
345
- const line = fields[1]
346
416
  try {
347
- yield JSON.parse(line)
417
+ yield JSON.parse(fields[1])
348
418
  await resource.db.client.xack(streamKey, group, entryId)
349
419
  } catch (e) {
350
- console.error('Cannot parse redis stream entry', e)
420
+ // Fires per bad entry: one line per window per resource is enough.
421
+ if (logThrottle(`${location}:stream-parse`)) {
422
+ log.warn('Cannot parse redis stream entry', { resource: alias, stream: streamKey, error: e })
423
+ }
351
424
  }
352
425
  }
353
426
  } while (true)
354
427
  }
355
428
  }
356
- } as Partial<T>)
429
+ } as unknown as Partial<T>)
357
430
 
358
431
  resource.init = async () => {
359
432
  const context = assertContext<Config, Context>(resource.ctx as Context, location)
360
- const redis = context.service<RedisDbService>(serviceAlias ?? dbAlias)
433
+ const redis = context.service<RedisDbService>(serviceAlias)
361
434
  await redis.ready()
362
435
  const db = await redis.db(dbAlias)
363
436
  const _pref = (val: string) => val.replaceAll(/\W+/g, '_')
@@ -368,14 +441,5 @@ export const makeRedisResource = <
368
441
  }
369
442
  }
370
443
 
371
- resource.reinitializeContext = <Type extends Contextual>(context: BasicContext<Config>) => {
372
- const resource = (makeCustomResource?.(dbAlias, serviceAlias)
373
- ?? makeRedisResource<R, T>(alias, dbAlias, serviceAlias)) as unknown as Type
374
-
375
- resource.ctx = context
376
-
377
- return resource as Type
378
- }
379
-
380
444
  return resource
381
445
  }