@owlmeans/redis-resource 0.1.18-rc.2 → 0.1.18-rc.21

package/src/resource.ts

@@ -1,33 +1,47 @@
-import type { ResourceMaker, ResourceRecord } from '@owlmeans/resource'
-import { MisshapedRecord, prepareListOptions, RecordExists, UnknownRecordError, UnsupportedArgumentError } from '@owlmeans/resource'
+import type {
+  Criteria, FirstOptions, ListOptions, ListResult, ResourceRecord, SubscribeOptions, Ttl,
+  Unsubscribe, WriteOptions
+} from '@owlmeans/resource'
+import {
+  applyQuery, filterRecords, firstMatch, matchCriteria, MisshapedRecord, RecordExists,
+  UnknownRecordError, UnsupportedArgumentError
+} from '@owlmeans/resource'
 import type { RedisClient, RedisDbService, RedisResource } from './types.js'
 import type { ServerConfig, ServerContext } from '@owlmeans/server-context'
-import { DEFAULT_DB_ALIAS, DEFAULT_PAGE_SIZE } from './consts.js'
+import {
+  CONSUMER_ID_LENGTH, DEFAULT_DB_ALIAS, DEFAULT_STREAM_BLOCK, READ_BATCH, RECLAIM_COUNT,
+  RECLAIM_IDLE, SCAN_BATCH, STREAM_MAX_LENGTH
+} from './consts.js'
 import { appendContextual, assertContext } from '@owlmeans/context'
-import type { BasicContext, Contextual } from '@owlmeans/context'
-import { prepareSubOptions } from './utils/options.js'
-import { createIdOfLength } from '@owlmeans/basic-ids'
+import { createIdOfLength, uuid } from '@owlmeans/basic-ids'
 
 type Config = ServerConfig
 type Context<C extends Config = Config> = ServerContext<C>
 
+/**
+ * A `Resource` over plain redis strings: one JSON document per namespaced key.
+ *
+ * Reads by id are O(1) GETs. Everything else — a criteria read, a listing, a count, a purge —
+ * walks this resource's OWN namespace with SCAN and evaluates the criteria in memory, because
+ * redis has no index to ask. That is affordable for a small namespaced set (a session cache, an
+ * adapter store) and wrong for anything unbounded or hot; a resource that needs real queries
+ * belongs in mongo or postgres.
+ */
 export const makeRedisResource = <
   R extends ResourceRecord, T extends RedisResource<R> = RedisResource<R>
 >(
-  alias: string, dbAlias = DEFAULT_DB_ALIAS, serviceAlias = DEFAULT_DB_ALIAS,
-  makeCustomResource?: ResourceMaker<R, T>
+  alias: string, dbAlias = DEFAULT_DB_ALIAS, serviceAlias = DEFAULT_DB_ALIAS
 ): T => {
   const location = `redis-resource:${alias}`
 
   /**
-   * A number (or its string form) is seconds from now; a `Date` is the absolute instant to
-   * expire at.
+   * A number is seconds from now; a `Date` is the absolute instant to expire at.
    *
    * The absolute form needs PEXPIREAT, not EXPIREAT: `Date.getTime()` is milliseconds while
    * EXPIREAT reads its argument as seconds, so the timestamp landed tens of thousands of years
    * out and the record never expired.
    */
-  const applyTtl = async (key: string, ttl: number | string | Date): Promise<void> => {
+  const applyTtl = async (key: string, ttl: Ttl): Promise<void> => {
     if (ttl instanceof Date) {
       await resource.db.client.pexpireat(key, ttl.getTime())
     } else {
@@ -35,289 +49,341 @@ export const makeRedisResource = <
     }
   }
 
-  const resource: T = appendContextual<T>(alias, {
-    key: key => `${resource.db.prefix}:${key ?? '*'}`,
+  /** Close a subscription once its lifetime runs out — seconds from now, or an instant. */
+  const armTtl = (close: Unsubscribe, ttl?: Ttl): void => {
+    if (ttl == null) {
+      return
+    }
+    setTimeout(() => void close(), ttl instanceof Date ? ttl.getTime() - Date.now() : ttl * 1000)
+  }
 
-    get: async (id, field, opts) => {
-      const record = await resource.load(id, field, opts)
-      if (record == null) {
-        throw new UnknownRecordError(id)
-      }
+  /** The record as it goes into redis: whatever was passed, under the id it is keyed by. */
+  const withId = (record: Partial<R>, id: string): R => ({ ...record, id } as unknown as R)
 
-      return record
-    },
+  /**
+   * Write the record under its key. SET both creates and replaces, which is what lets `save`
+   * upsert without reading first.
+   */
+  const put = async (id: string, record: Partial<R>, opts?: WriteOptions): Promise<R> => {
+    const key = resource.key(id)
+    const stored = withId(record, id)
+    await resource.db.client.set(key, JSON.stringify(stored))
+    if (opts?.ttl != null) {
+      await applyTtl(key, opts.ttl)
+    }
 
-    load: async (id, field, opts) => {
-      if (typeof field === 'object') {
-        opts = field
-        field = field.field
-      }
-      if (opts != null) {
-        throw new UnsupportedArgumentError('load:opts')
-      }
-      if (field === 'id' || field == null) {
-        const val = await resource.db.client.get(resource.key(id))
-        if (val == null) {
-          return null
-        }
+    return stored
+  }
 
-        return JSON.parse(val)
-      }
-      const list = await resource.list({ [field]: id }, { pager: { size: 1 } })
+  /**
+   * Every key under this resource's prefix, collected with SCAN.
+   *
+   * Never KEYS: KEYS blocks the server for the length of the whole sweep, while SCAN walks the
+   * keyspace in bounded bites. SCAN may hand the same key back twice across iterations, which is
+   * why the keys land in a Set.
+   */
+  const scanKeys = async (): Promise<string[]> => {
+    const found = new Set<string>()
+    let cursor = '0'
+    do {
+      const [next, batch] = await resource.db.client.scan(
+        cursor, 'MATCH', resource.key(), 'COUNT', SCAN_BATCH
+      )
+      cursor = next
+      batch.forEach(key => found.add(key))
+    } while (cursor !== '0')
 
-      return list.items[0] ?? null
-    },
+    return [...found]
+  }
 
-    save: async (record, opts) => {
-      if (record.id == null) {
-        throw new MisshapedRecord('id')
-      }
-      const existing = await resource.load(record.id)
-      if (existing != null) {
-        return resource.update(record, opts)
-      }
+  /**
+   * The namespace as key/record pairs, read in MGET batches.
+   *
+   * O(N) over the resource's own namespace — every criteria that is not a bare id is answered
+   * from this in memory. A key holding something other than a string, this resource's own streams
+   * included, reads back as `null` from MGET and is skipped.
+   */
+  const readNamespace = async (): Promise<Array<[string, R]>> => {
+    const keys = await scanKeys()
+    const entries: Array<[string, R]> = []
+    for (let from = 0; from < keys.length; from += READ_BATCH) {
+      const batch = keys.slice(from, from + READ_BATCH)
+      const values = await resource.db.client.mget(batch)
+      values.forEach((value, index) => {
+        if (typeof value === 'string') {
+          entries.push([batch[index], JSON.parse(value)])
+        }
+      })
+    }
 
-      return resource.create(record, typeof opts !== 'string' ? opts : undefined)
-    },
+    return entries
+  }
 
-    update: async (record, opts) => {
-      let field = 'id'
-      if (typeof opts === 'string') {
-        field = opts
-      } else if (typeof opts === 'object') {
-        field = opts.field ?? field
-      }
-      if (field !== 'id' && record[field as keyof typeof record] == null) {
-        throw new MisshapedRecord('no-field-value')
-      }
-      const id = field == 'id' ? record.id : record[field as keyof typeof record] as string
-      if (id == null) {
-        throw new MisshapedRecord('id')
-      }
-      const _opts = typeof opts === 'object' ? opts : {}
-      const existing = await resource.get(id, field)
-      const updated = { ...existing, ...record }
-      await resource.db.client.set(resource.key(id), JSON.stringify(updated))
-      if (_opts.ttl != null) {
-        await applyTtl(resource.key(id), _opts.ttl)
-      }
+  const readRecords = async (): Promise<R[]> => (await readNamespace()).map(([, record]) => record)
 
-      return updated
-    },
+  /** `{ id: 'x' }` is the one criteria redis can answer by key; anything else needs the walk. */
+  const soleId = (where: Criteria<any>): string | null => {
+    const keys = Object.keys(where)
 
-    create: async (record, opts) => {
-      if (record.id == null) {
-        throw new UnknownRecordError('id')
-      }
-      if (await resource.load(record.id) != null) {
-        throw new RecordExists(record.id)
-      }
+    return keys.length === 1 && keys[0] === 'id' && typeof where.id === 'string' ? where.id : null
+  }
 
-      const id = record.id
-      await resource.db.client.set(resource.key(id), JSON.stringify(record))
-      if (opts?.ttl != null) {
-        await applyTtl(resource.key(id), opts.ttl)
-      }
+  const loadOne = async (
+    idOrWhere: string | Criteria<any>, opts?: FirstOptions<any>
+  ): Promise<R | null> => {
+    const id = typeof idOrWhere === 'string' ? idOrWhere : soleId(idOrWhere)
+    if (id != null) {
+      const value = await resource.db.client.get(resource.key(id))
 
-      return record
-    },
+      return value == null ? null : JSON.parse(value)
+    }
 
-    delete: async (id, opts) => {
-      let record: R | null = null
-      if (typeof id === 'object') {
-        if (id.id == null) {
-          throw new MisshapedRecord('id')
-        }
-        record = await resource.load(id.id)
-      } else {
-        // `opts` is optional in the contract: without this branch `delete(id)` — the plain,
-        // by-id form every caller uses — left `record` null and returned without touching redis.
-        record = await resource.load(id, opts)
-      }
+    return firstMatch(await readRecords(), idOrWhere as Criteria<any>, opts)
+  }
 
-      if (record == null) {
-        return null
+  /**
+   * One dedicated connection per subscription: a redis client in subscriber mode accepts nothing
+   * but (un)subscribe commands, so the resource's own client has to stay out of it.
+   */
+  const listen = async (
+    pattern: string, onMessage: (channel: string, message: string) => void,
+    opts?: { once?: boolean, ttl?: Ttl }
+  ): Promise<Unsubscribe> => {
+    const subscriber = (resource.db.client as RedisClient).duplicate()
+    let closed = false
+    const unsubscribe: Unsubscribe = async () => {
+      if (closed) {
+        return
+      }
+      closed = true
+      try {
+        await subscriber.punsubscribe(pattern)
+        await subscriber.quit()
+      } catch (e) {
+        console.error(`${location}: failed to unsubscribe ${pattern}`, e)
       }
+    }
 
-      let field = 'id'
-      if (typeof opts === 'string') {
-        field = opts
-        opts = undefined
-      } else if (typeof opts === 'object') {
-        field = opts.field ?? field
+    await subscriber.psubscribe(pattern)
+    subscriber.on('pmessage', (_pattern, channel, message) => {
+      /**
+       * `once` closes before the handler's own work finishes on purpose: the handler may await a
+       * read, and a second message arriving meanwhile would fire a subscription that has already
+       * delivered its one message.
+       */
+      if (opts?.once === true) {
+        void unsubscribe()
       }
+      onMessage(channel, message)
+    })
+    armTtl(unsubscribe, opts?.ttl)
 
-      id = record.id ?? ''
+    return unsubscribe
+  }
 
-      if (id === '') {
-        throw new MisshapedRecord('id')
-      }
+  const resource: T = appendContextual<T>(alias, {
+    key: (key?: string) => `${resource.db.prefix}:${key ?? '*'}`,
 
-      if (await resource.db.client.del(resource.key(id)) > 0) {
-        return record
+    /**
+     * @throws {UnknownRecordError}
+     */
+    get: async (idOrWhere: string | Criteria<any>, opts?: FirstOptions<any>): Promise<R> => {
+      const record = await loadOne(idOrWhere, opts)
+      if (record == null) {
+        throw new UnknownRecordError(
+          typeof idOrWhere === 'string' ? idOrWhere : JSON.stringify(idOrWhere)
+        )
       }
 
-      return null
+      return record
     },
 
-    pick: async (id, opts) => {
-      let record: R | null = null
-      if (typeof id === 'object') {
-        if (id.id == null) {
-          throw new MisshapedRecord('id')
-        }
-        record = await resource.load(id.id)
-      } else {
-        record = await resource.load(id, opts)
+    load: loadOne,
+
+    /**
+     * Unpaged by default: without a `size` every match comes back. A `page` on its own is a
+     * caller error rather than a window over an implied default — there is no default here to
+     * take a window of.
+     *
+     * @throws {UnsupportedArgumentError} on `page` without `size`.
+     */
+    list: async (where?: Criteria<any>, opts?: ListOptions<any>): Promise<ListResult<R>> => {
+      if (opts?.page != null && opts.size == null) {
+        throw new UnsupportedArgumentError('page-without-size')
       }
 
-      if (record == null) {
-        return null
-      }
+      return applyQuery(await readRecords(), where, opts)
+    },
 
-      let field = 'id'
-      if (typeof opts === 'string') {
-        field = opts
-        opts = undefined
-      } else if (typeof opts === 'object') {
-        field = opts.field ?? field
+    count: async (where?: Criteria<any>): Promise<number> =>
+      filterRecords(await readRecords(), where).length,
+
+    /**
+     * SET NX rather than a read followed by a write: two callers racing for the same id both saw
+     * an empty key, both "created" the record, and the loser's write silently won.
+     *
+     * A record with no id gets one — redis needs a key, and `save` relies on this to create.
+     *
+     * @throws {RecordExists}
+     */
+    create: async (record: Partial<R>, opts?: WriteOptions): Promise<R> => {
+      const id = record.id ?? uuid()
+      const key = resource.key(id)
+      const stored = withId(record, id)
+      if (await resource.db.client.set(key, JSON.stringify(stored), 'NX') == null) {
+        throw new RecordExists(id)
+      }
+      if (opts?.ttl != null) {
+        await applyTtl(key, opts.ttl)
       }
 
-      id = record.id ?? ''
+      return stored
+    },
 
-      if (id === '') {
+    /**
+     * Replaces the whole record: what is stored afterwards is exactly what was passed, not a
+     * merge over what was there. A renewal has to pass its TTL again — SET drops the old expiry.
+     *
+     * @throws {MisshapedRecord} without an id, {@link UnknownRecordError} when the id is absent.
+     */
+    update: async (record: Partial<R>, opts?: WriteOptions): Promise<R> => {
+      if (record.id == null) {
         throw new MisshapedRecord('id')
       }
-
-      if (await resource.db.client.del(resource.key(id)) > 0) {
-        return record
+      if (await resource.db.client.exists(resource.key(record.id)) < 1) {
+        throw new UnknownRecordError(record.id)
       }
 
-      throw new UnknownRecordError(id)
+      return put(record.id, record, opts)
     },
 
-    list: async (criteria, opts) => {
-      const options = prepareListOptions(DEFAULT_PAGE_SIZE, criteria, opts)
-      criteria = options.criteria
-      const pager = options.pager ?? {}
+    /** Creates when the record carries no id, replaces otherwise — SET does both. */
+    save: async (record: Partial<R>, opts?: WriteOptions): Promise<R> =>
+      record.id == null ? resource.create(record, opts) : put(record.id, record, opts),
 
-      const keys = await resource.db.client.keys(resource.key())
-      pager.total = keys.length
-      pager.page = pager.page ?? 0
-      pager.size = pager.size ?? DEFAULT_PAGE_SIZE
+    /**
+     * GETDEL — reading the record and dropping the key in one command, so two callers cannot both
+     * be handed the same record. Needs redis 6.2 or newer.
+     */
+    delete: async (id: string): Promise<R | null> => {
+      const value = await resource.db.client.getdel(resource.key(id))
 
-      if (keys.length === 0) {
-        return { items: [], pager }
-      }
-
-      const start = pager.page * pager.size
-      const slice = keys.toSpliced(start, pager.size)
-
-      return {
-        pager, items: (await Promise.all(slice.map(
-          async key => await resource.db.client.get(key)
-        ))).filter(val => typeof val === 'string').map(
-          val => JSON.parse(val as string)
-        )
-      }
+      return value == null ? null : JSON.parse(value)
     },
 
-    subscribe: async (handler, opts) => {
-      opts = prepareSubOptions(opts)
-
-      let _unsubascribeCalled = false
-      const unsusbscibe = async () => {
-        if (_unsubascribeCalled) {
-          return
-        }
-        _unsubascribeCalled = true
-        try {
-          await subscriber.punsubscribe(resource.key(opts.key)).then(() => subscriber.quit())
-        } catch (e) {
-          console.error('Try to unsubscribe redis resource', e)
-        }
+    /**
+     * Delete-and-return where absence is an error — the consume-once read.
+     *
+     * @throws {UnknownRecordError}
+     */
+    take: async (id: string): Promise<R> => {
+      const record = await resource.delete(id)
+      if (record == null) {
+        throw new UnknownRecordError(id)
       }
 
-      const listener = (value: R) => {
-        handler(value as any)
-        if (opts.once === true) {
-          void unsusbscibe()
-        }
-      }
+      return record
+    },
 
-      const subscriber = (resource.db.client as RedisClient).duplicate()
-      if (opts.key != null && null != await resource.load(opts.key)) {
-        await subscriber.psubscribe(`__keyspace@0__:${resource.key(opts.key)}`)
-        subscriber.on('pmessage', async (_, channel) => {
-          const [, , ...key] = (channel as string).split(':')
-          const record = await resource.load(key.join(':'))
-          listener(record ?? { id: key, __null: true } as any as R)
-        })
-      } else {
-        await subscriber.psubscribe(resource.key(opts.key))
-        subscriber.on('pmessage', (_, __, message) => listener(JSON.parse(message)))
+    /**
+     * Bulk delete over the namespace walk. An empty criteria object is refused: wiping a whole
+     * namespace is what FLUSHDB is for, not something a filter that came back empty should do.
+     *
+     * @throws {UnsupportedArgumentError} on empty criteria.
+     */
+    purge: async (where: Criteria<any>): Promise<number> => {
+      if (where == null || Object.keys(where).length < 1) {
+        throw new UnsupportedArgumentError('purge:empty-criteria')
       }
+      const keys = (await readNamespace())
+        .filter(([, record]) => matchCriteria(record, where))
+        .map(([key]) => key)
 
-
-      if (opts.ttl != null) {
-        const ttl = opts.ttl instanceof Date
-          ? opts.ttl.getTime() - Date.now()
-          : typeof opts.ttl === 'number'
-            ? opts.ttl * 1000
-            : null
-        if (ttl == null) {
-          throw new UnsupportedArgumentError(`subscribe:ttl:${typeof opts.ttl}`)
-        }
-        setTimeout(unsusbscibe, ttl)
+      let deleted = 0
+      for (let from = 0; from < keys.length; from += READ_BATCH) {
+        deleted += await resource.db.client.del(keys.slice(from, from + READ_BATCH))
       }
 
-      return unsusbscibe
+      return deleted
     },
 
-    publish: async (record, key) => {
-      await resource.db.client.publish(resource.key(key), JSON.stringify(record))
+    publish: async (value: R, channel?: string): Promise<void> => {
+      await resource.db.client.publish(resource.key(channel), JSON.stringify(value))
     },
 
-    stream: async (key, record) => {
+    /**
+     * Channel pub/sub. The channel name is namespaced like a key, and an omitted channel listens
+     * to the resource's whole namespace — `key()` yields the glob that PSUBSCRIBE matches on.
+     */
+    subscribe: async (
+      handler: (value: R) => void | Promise<void>, opts?: SubscribeOptions
+    ): Promise<Unsubscribe> => listen(
+      resource.key(opts?.channel),
+      (_channel, message) => void handler(JSON.parse(message)),
+      opts
+    ),
+
+    /**
+     * Keyspace notifications for ONE record: every write, expiry or delete on its key hands the
+     * handler the current value, or `null` once the key is gone.
+     *
+     * Keyspace events are published per database and this listens on db 0, so deployments sharing
+     * one instance isolate on the key prefix, never on the db index. The server also has to have
+     * `notify-keyspace-events` enabled — redis emits nothing otherwise.
+     */
+    watch: async (
+      id: string, handler: (value: R | null) => void | Promise<void>,
+      opts?: Omit<SubscribeOptions, 'channel'>
+    ): Promise<Unsubscribe> => listen(
+      `__keyspace@0__:${resource.key(id)}`,
+      () => void loadOne(id).then(record => handler(record)),
+      opts
+    ),
+
+    stream: async (key: string, value: R): Promise<void> => {
       await resource.db.client.xadd(
-        resource.key(key), 'MAXLEN', '~', 10000, '*', 'payload', JSON.stringify(record)
+        resource.key(key), 'MAXLEN', '~', STREAM_MAX_LENGTH, '*', 'payload', JSON.stringify(value)
       )
     },
 
-    consume: async function* (key, group, consumer = createIdOfLength(15)) {
-      console.log(`Establish redis stream consumer: ${group ?? 'no-group'}:${consumer} for ${key}`)
+    consume: async function* (
+      key: string, opts?: { group?: string, consumer?: string, block?: number }
+    ): AsyncGenerator<R> {
       const streamKey = resource.key(key)
+      const group = opts?.group
+      const consumer = opts?.consumer ?? createIdOfLength(CONSUMER_ID_LENGTH)
+      const block = opts?.block ?? DEFAULT_STREAM_BLOCK
+
       if (group == null) {
         let id = '$'
         do {
-          const resp = await resource.db.client.xread(
-            'BLOCK', 1000, 'STREAMS', streamKey, id
-          )
+          const resp = await resource.db.client.xread('BLOCK', block, 'STREAMS', streamKey, id)
           if (!resp) continue
 
           const [, entries] = resp[0]
           for (const [entryId, fields] of entries) {
             id = entryId
-            const line = fields[1]
             try {
-              console.log('received', fields)
-              yield JSON.parse(line)
+              yield JSON.parse(fields[1])
             } catch (e) {
               console.error('Cannot parse redis stream entry', e)
             }
           }
         } while (true)
       } else {
-        const _reclaim = async function* () {
+        /**
+         * Entries a consumer took and never acknowledged, claimed under THIS consumer's name.
+         * Claiming them for a fixed name instead left them owned by a consumer that never reads,
+         * so a crashed worker's entries moved once and then stalled for good.
+         */
+        const reclaim = async function* (): AsyncGenerator<R> {
           try {
             const [, entries] = await resource.db.client.xautoclaim(
-              streamKey, group, 'junitor', 60000, '0-0', 'COUNT', 10
+              streamKey, group, consumer, RECLAIM_IDLE, '0-0', 'COUNT', RECLAIM_COUNT
             )
             for (const [id, fields] of entries as [string, string[]][]) {
-              console.log('Reclaimed', id)
-              const line = fields[1]
               try {
-                yield JSON.parse(line)
+                yield JSON.parse(fields[1])
               } catch (e) {
                 console.error('Cannot parse reclaimed redis stream entry', e)
               }
@@ -333,18 +399,17 @@ export const makeRedisResource = <
         do {
           const resp = await resource.db.client.xreadgroup(
             'GROUP', group, consumer,
-            'BLOCK', 1000, 'STREAMS', streamKey, '>'
+            'BLOCK', block, 'STREAMS', streamKey, '>'
           )
           if (!resp) {
-            yield* _reclaim()
+            yield* reclaim()
             continue
           }
 
           const [, entries] = resp[0] as [string, Array<[string, string[]]>]
           for (const [entryId, fields] of entries) {
-            const line = fields[1]
             try {
-              yield JSON.parse(line)
+              yield JSON.parse(fields[1])
               await resource.db.client.xack(streamKey, group, entryId)
             } catch (e) {
               console.error('Cannot parse redis stream entry', e)
@@ -353,11 +418,11 @@ export const makeRedisResource = <
         } while (true)
       }
     }
-  } as Partial<T>)
+  } as unknown as Partial<T>)
 
   resource.init = async () => {
     const context = assertContext<Config, Context>(resource.ctx as Context, location)
-    const redis = context.service<RedisDbService>(serviceAlias ?? dbAlias)
+    const redis = context.service<RedisDbService>(serviceAlias)
     await redis.ready()
     const db = await redis.db(dbAlias)
     const _pref = (val: string) => val.replaceAll(/\W+/g, '_')
@@ -368,14 +433,5 @@ export const makeRedisResource = <
     }
   }
 
-  resource.reinitializeContext = <Type extends Contextual>(context: BasicContext<Config>) => {
-    const resource = (makeCustomResource?.(dbAlias, serviceAlias)
-      ?? makeRedisResource<R, T>(alias, dbAlias, serviceAlias)) as unknown as Type
-
-    resource.ctx = context
-
-    return resource as Type
-  }
-
   return resource
 }