@ddd-ts-kit/redis 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +699 -2
- package/dist/cache/cache-codec.d.ts +6 -0
- package/dist/cache/cache-codec.d.ts.map +1 -0
- package/dist/cache/cache-codec.js +2 -0
- package/dist/cache/cache-codec.js.map +1 -0
- package/dist/cache/cache-operation.d.ts +3 -0
- package/dist/cache/cache-operation.d.ts.map +1 -0
- package/dist/cache/cache-operation.js +2 -0
- package/dist/cache/cache-operation.js.map +1 -0
- package/dist/cache/json-codec.d.ts +4 -0
- package/dist/cache/json-codec.d.ts.map +1 -0
- package/dist/cache/json-codec.js +9 -0
- package/dist/cache/json-codec.js.map +1 -0
- package/dist/cache/redis-cache.d.ts +65 -0
- package/dist/cache/redis-cache.d.ts.map +1 -0
- package/dist/cache/redis-cache.js +147 -0
- package/dist/cache/redis-cache.js.map +1 -0
- package/dist/connection/close-redis.d.ts +4 -0
- package/dist/connection/close-redis.d.ts.map +1 -0
- package/dist/connection/close-redis.js +20 -0
- package/dist/connection/close-redis.js.map +1 -0
- package/dist/connection/connect-redis.d.ts +10 -0
- package/dist/connection/connect-redis.d.ts.map +1 -0
- package/dist/connection/connect-redis.js +26 -0
- package/dist/connection/connect-redis.js.map +1 -0
- package/dist/connection/fail-fast.d.ts +9 -0
- package/dist/connection/fail-fast.d.ts.map +1 -0
- package/dist/connection/fail-fast.js +8 -0
- package/dist/connection/fail-fast.js.map +1 -0
- package/dist/connection/redis-connection-options.d.ts +7 -0
- package/dist/connection/redis-connection-options.d.ts.map +1 -0
- package/dist/connection/redis-connection-options.js +2 -0
- package/dist/connection/redis-connection-options.js.map +1 -0
- package/dist/connection/redis-health.d.ts +15 -0
- package/dist/connection/redis-health.d.ts.map +1 -0
- package/dist/connection/redis-health.js +31 -0
- package/dist/connection/redis-health.js.map +1 -0
- package/dist/connection/redis-source.d.ts +5 -0
- package/dist/connection/redis-source.d.ts.map +1 -0
- package/dist/connection/redis-source.js +2 -0
- package/dist/connection/redis-source.js.map +1 -0
- package/dist/event-bus.d.ts +22 -0
- package/dist/event-bus.d.ts.map +1 -0
- package/dist/event-bus.js +10 -0
- package/dist/event-bus.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +13 -0
- package/dist/index.js.map +1 -0
- package/dist/keys/namespaced.d.ts +3 -0
- package/dist/keys/namespaced.d.ts.map +1 -0
- package/dist/keys/namespaced.js +5 -0
- package/dist/keys/namespaced.js.map +1 -0
- package/dist/keys/prefix-pattern.d.ts +3 -0
- package/dist/keys/prefix-pattern.d.ts.map +1 -0
- package/dist/keys/prefix-pattern.js +5 -0
- package/dist/keys/prefix-pattern.js.map +1 -0
- package/dist/lock/redis-lock.d.ts +31 -0
- package/dist/lock/redis-lock.d.ts.map +1 -0
- package/dist/lock/redis-lock.js +66 -0
- package/dist/lock/redis-lock.js.map +1 -0
- package/dist/rate-limiter/redis-rate-limiter.d.ts +35 -0
- package/dist/rate-limiter/redis-rate-limiter.d.ts.map +1 -0
- package/dist/rate-limiter/redis-rate-limiter.js +77 -0
- package/dist/rate-limiter/redis-rate-limiter.js.map +1 -0
- package/dist/relay/redis-relay-signal.d.ts +36 -0
- package/dist/relay/redis-relay-signal.d.ts.map +1 -0
- package/dist/relay/redis-relay-signal.js +72 -0
- package/dist/relay/redis-relay-signal.js.map +1 -0
- package/dist/relay/relay-wake-up-middleware.d.ts +11 -0
- package/dist/relay/relay-wake-up-middleware.d.ts.map +1 -0
- package/dist/relay/relay-wake-up-middleware.js +16 -0
- package/dist/relay/relay-wake-up-middleware.js.map +1 -0
- package/dist/replies/list-of.d.ts +3 -0
- package/dist/replies/list-of.d.ts.map +1 -0
- package/dist/replies/list-of.js +8 -0
- package/dist/replies/list-of.js.map +1 -0
- package/dist/secrecy/conceal.d.ts +8 -0
- package/dist/secrecy/conceal.d.ts.map +1 -0
- package/dist/secrecy/conceal.js +12 -0
- package/dist/secrecy/conceal.js.map +1 -0
- package/dist/streams/bury.d.ts +13 -0
- package/dist/streams/bury.d.ts.map +1 -0
- package/dist/streams/bury.js +14 -0
- package/dist/streams/bury.js.map +1 -0
- package/dist/streams/claim-due.d.ts +18 -0
- package/dist/streams/claim-due.d.ts.map +1 -0
- package/dist/streams/claim-due.js +41 -0
- package/dist/streams/claim-due.js.map +1 -0
- package/dist/streams/codec/decode.d.ts +5 -0
- package/dist/streams/codec/decode.d.ts.map +1 -0
- package/dist/streams/codec/decode.js +40 -0
- package/dist/streams/codec/decode.js.map +1 -0
- package/dist/streams/codec/encode.d.ts +7 -0
- package/dist/streams/codec/encode.d.ts.map +1 -0
- package/dist/streams/codec/encode.js +20 -0
- package/dist/streams/codec/encode.js.map +1 -0
- package/dist/streams/codec/entries-by-stream.d.ts +4 -0
- package/dist/streams/codec/entries-by-stream.d.ts.map +1 -0
- package/dist/streams/codec/entries-by-stream.js +14 -0
- package/dist/streams/codec/entries-by-stream.js.map +1 -0
- package/dist/streams/codec/entries-of.d.ts +4 -0
- package/dist/streams/codec/entries-of.d.ts.map +1 -0
- package/dist/streams/codec/entries-of.js +16 -0
- package/dist/streams/codec/entries-of.js.map +1 -0
- package/dist/streams/codec/entry-fields.d.ts +13 -0
- package/dist/streams/codec/entry-fields.d.ts.map +1 -0
- package/dist/streams/codec/entry-fields.js +13 -0
- package/dist/streams/codec/entry-fields.js.map +1 -0
- package/dist/streams/codec/entry.d.ts +6 -0
- package/dist/streams/codec/entry.d.ts.map +1 -0
- package/dist/streams/codec/entry.js +2 -0
- package/dist/streams/codec/entry.js.map +1 -0
- package/dist/streams/codec/fields-with.d.ts +4 -0
- package/dist/streams/codec/fields-with.d.ts.map +1 -0
- package/dist/streams/codec/fields-with.js +6 -0
- package/dist/streams/codec/fields-with.js.map +1 -0
- package/dist/streams/codec/pending-entry.d.ts +7 -0
- package/dist/streams/codec/pending-entry.d.ts.map +1 -0
- package/dist/streams/codec/pending-entry.js +2 -0
- package/dist/streams/codec/pending-entry.js.map +1 -0
- package/dist/streams/codec/pending-of.d.ts +4 -0
- package/dist/streams/codec/pending-of.d.ts.map +1 -0
- package/dist/streams/codec/pending-of.js +11 -0
- package/dist/streams/codec/pending-of.js.map +1 -0
- package/dist/streams/codec/replay-fields.d.ts +4 -0
- package/dist/streams/codec/replay-fields.d.ts.map +1 -0
- package/dist/streams/codec/replay-fields.js +15 -0
- package/dist/streams/codec/replay-fields.js.map +1 -0
- package/dist/streams/consumption.d.ts +22 -0
- package/dist/streams/consumption.d.ts.map +1 -0
- package/dist/streams/consumption.js +46 -0
- package/dist/streams/consumption.js.map +1 -0
- package/dist/streams/delivery.d.ts +8 -0
- package/dist/streams/delivery.d.ts.map +1 -0
- package/dist/streams/delivery.js +2 -0
- package/dist/streams/delivery.js.map +1 -0
- package/dist/streams/errors/broker-unavailable-error.d.ts +8 -0
- package/dist/streams/errors/broker-unavailable-error.d.ts.map +1 -0
- package/dist/streams/errors/broker-unavailable-error.js +12 -0
- package/dist/streams/errors/broker-unavailable-error.js.map +1 -0
- package/dist/streams/errors/consumer-name-required-error.d.ts +5 -0
- package/dist/streams/errors/consumer-name-required-error.d.ts.map +1 -0
- package/dist/streams/errors/consumer-name-required-error.js +8 -0
- package/dist/streams/errors/consumer-name-required-error.js.map +1 -0
- package/dist/streams/errors/event-bus-closed-error.d.ts +5 -0
- package/dist/streams/errors/event-bus-closed-error.d.ts.map +1 -0
- package/dist/streams/errors/event-bus-closed-error.js +8 -0
- package/dist/streams/errors/event-bus-closed-error.js.map +1 -0
- package/dist/streams/errors/failure-code.d.ts +7 -0
- package/dist/streams/errors/failure-code.d.ts.map +1 -0
- package/dist/streams/errors/failure-code.js +28 -0
- package/dist/streams/errors/failure-code.js.map +1 -0
- package/dist/streams/errors/publish-not-confirmed-error.d.ts +9 -0
- package/dist/streams/errors/publish-not-confirmed-error.d.ts.map +1 -0
- package/dist/streams/errors/publish-not-confirmed-error.js +14 -0
- package/dist/streams/errors/publish-not-confirmed-error.js.map +1 -0
- package/dist/streams/group-reader.d.ts +57 -0
- package/dist/streams/group-reader.d.ts.map +1 -0
- package/dist/streams/group-reader.js +187 -0
- package/dist/streams/group-reader.js.map +1 -0
- package/dist/streams/messaging-system.d.ts +3 -0
- package/dist/streams/messaging-system.d.ts.map +1 -0
- package/dist/streams/messaging-system.js +3 -0
- package/dist/streams/messaging-system.js.map +1 -0
- package/dist/streams/options/deduplication-options.d.ts +5 -0
- package/dist/streams/options/deduplication-options.d.ts.map +1 -0
- package/dist/streams/options/deduplication-options.js +2 -0
- package/dist/streams/options/deduplication-options.js.map +1 -0
- package/dist/streams/options/default-prefix.d.ts +2 -0
- package/dist/streams/options/default-prefix.d.ts.map +1 -0
- package/dist/streams/options/default-prefix.js +2 -0
- package/dist/streams/options/default-prefix.js.map +1 -0
- package/dist/streams/options/reconnect-delay.d.ts +4 -0
- package/dist/streams/options/reconnect-delay.d.ts.map +1 -0
- package/dist/streams/options/reconnect-delay.js +5 -0
- package/dist/streams/options/reconnect-delay.js.map +1 -0
- package/dist/streams/options/reconnect-options.d.ts +9 -0
- package/dist/streams/options/reconnect-options.d.ts.map +1 -0
- package/dist/streams/options/reconnect-options.js +2 -0
- package/dist/streams/options/reconnect-options.js.map +1 -0
- package/dist/streams/options/redis-streams-delivery.d.ts +13 -0
- package/dist/streams/options/redis-streams-delivery.d.ts.map +1 -0
- package/dist/streams/options/redis-streams-delivery.js +2 -0
- package/dist/streams/options/redis-streams-delivery.js.map +1 -0
- package/dist/streams/options/redis-streams-event-bus-options.d.ts +48 -0
- package/dist/streams/options/redis-streams-event-bus-options.d.ts.map +1 -0
- package/dist/streams/options/redis-streams-event-bus-options.js +2 -0
- package/dist/streams/options/redis-streams-event-bus-options.js.map +1 -0
- package/dist/streams/options/redis-streams-failure.d.ts +19 -0
- package/dist/streams/options/redis-streams-failure.d.ts.map +1 -0
- package/dist/streams/options/redis-streams-failure.js +2 -0
- package/dist/streams/options/redis-streams-failure.js.map +1 -0
- package/dist/streams/options/retention-options.d.ts +10 -0
- package/dist/streams/options/retention-options.d.ts.map +1 -0
- package/dist/streams/options/retention-options.js +2 -0
- package/dist/streams/options/retention-options.js.map +1 -0
- package/dist/streams/options/retry-options.d.ts +7 -0
- package/dist/streams/options/retry-options.d.ts.map +1 -0
- package/dist/streams/options/retry-options.js +2 -0
- package/dist/streams/options/retry-options.js.map +1 -0
- package/dist/streams/options/settings-of.d.ts +4 -0
- package/dist/streams/options/settings-of.d.ts.map +1 -0
- package/dist/streams/options/settings-of.js +47 -0
- package/dist/streams/options/settings-of.js.map +1 -0
- package/dist/streams/options/settings.d.ts +28 -0
- package/dist/streams/options/settings.d.ts.map +1 -0
- package/dist/streams/options/settings.js +2 -0
- package/dist/streams/options/settings.js.map +1 -0
- package/dist/streams/publications.d.ts +22 -0
- package/dist/streams/publications.d.ts.map +1 -0
- package/dist/streams/publications.js +75 -0
- package/dist/streams/publications.js.map +1 -0
- package/dist/streams/redis-streams-event-bus.d.ts +55 -0
- package/dist/streams/redis-streams-event-bus.d.ts.map +1 -0
- package/dist/streams/redis-streams-event-bus.js +120 -0
- package/dist/streams/redis-streams-event-bus.js.map +1 -0
- package/dist/streams/replay.d.ts +13 -0
- package/dist/streams/replay.d.ts.map +1 -0
- package/dist/streams/replay.js +26 -0
- package/dist/streams/replay.js.map +1 -0
- package/dist/streams/server-link.d.ts +45 -0
- package/dist/streams/server-link.d.ts.map +1 -0
- package/dist/streams/server-link.js +129 -0
- package/dist/streams/server-link.js.map +1 -0
- package/dist/streams/settlement.d.ts +46 -0
- package/dist/streams/settlement.d.ts.map +1 -0
- package/dist/streams/settlement.js +139 -0
- package/dist/streams/settlement.js.map +1 -0
- package/dist/streams/topology/consumer-binding.d.ts +5 -0
- package/dist/streams/topology/consumer-binding.d.ts.map +1 -0
- package/dist/streams/topology/consumer-binding.js +2 -0
- package/dist/streams/topology/consumer-binding.js.map +1 -0
- package/dist/streams/topology/create-group.d.ts +4 -0
- package/dist/streams/topology/create-group.d.ts.map +1 -0
- package/dist/streams/topology/create-group.js +13 -0
- package/dist/streams/topology/create-group.js.map +1 -0
- package/dist/streams/topology/dead-stream-of.d.ts +3 -0
- package/dist/streams/topology/dead-stream-of.d.ts.map +1 -0
- package/dist/streams/topology/dead-stream-of.js +5 -0
- package/dist/streams/topology/dead-stream-of.js.map +1 -0
- package/dist/streams/topology/declare-topology.d.ts +5 -0
- package/dist/streams/topology/declare-topology.d.ts.map +1 -0
- package/dist/streams/topology/declare-topology.js +8 -0
- package/dist/streams/topology/declare-topology.js.map +1 -0
- package/dist/streams/topology/group-description.d.ts +7 -0
- package/dist/streams/topology/group-description.d.ts.map +1 -0
- package/dist/streams/topology/group-description.js +2 -0
- package/dist/streams/topology/group-description.js.map +1 -0
- package/dist/streams/topology/group-of.d.ts +5 -0
- package/dist/streams/topology/group-of.d.ts.map +1 -0
- package/dist/streams/topology/group-of.js +4 -0
- package/dist/streams/topology/group-of.js.map +1 -0
- package/dist/streams/topology/redis-streams-topology.d.ts +8 -0
- package/dist/streams/topology/redis-streams-topology.d.ts.map +1 -0
- package/dist/streams/topology/redis-streams-topology.js +2 -0
- package/dist/streams/topology/redis-streams-topology.js.map +1 -0
- package/dist/streams/topology/stream-description.d.ts +12 -0
- package/dist/streams/topology/stream-description.d.ts.map +1 -0
- package/dist/streams/topology/stream-description.js +2 -0
- package/dist/streams/topology/stream-description.js.map +1 -0
- package/dist/streams/topology/stream-names-of.d.ts +5 -0
- package/dist/streams/topology/stream-names-of.d.ts.map +1 -0
- package/dist/streams/topology/stream-names-of.js +11 -0
- package/dist/streams/topology/stream-names-of.js.map +1 -0
- package/dist/streams/topology/stream-names.d.ts +5 -0
- package/dist/streams/topology/stream-names.d.ts.map +1 -0
- package/dist/streams/topology/stream-names.js +2 -0
- package/dist/streams/topology/stream-names.js.map +1 -0
- package/dist/streams/topology/topology-of.d.ts +5 -0
- package/dist/streams/topology/topology-of.d.ts.map +1 -0
- package/dist/streams/topology/topology-of.js +21 -0
- package/dist/streams/topology/topology-of.js.map +1 -0
- package/dist/streams/topology/topology-settings.d.ts +7 -0
- package/dist/streams/topology/topology-settings.d.ts.map +1 -0
- package/dist/streams/topology/topology-settings.js +2 -0
- package/dist/streams/topology/topology-settings.js.map +1 -0
- package/dist/testing.d.ts +2 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +2 -0
- package/dist/testing.js.map +1 -0
- package/package.json +51 -4
package/README.md
CHANGED
|
@@ -1,3 +1,700 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @ddd-ts-kit/redis
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Adapters of kernel ports on the Redis protocol: the `Cache`, `Lock` and `RateLimiter` ports,
|
|
4
|
+
the wake-up signal of the outbox relay, and an integration event provider on Redis Streams.
|
|
5
|
+
They use a common subset of the protocol, so the same code runs on Redis, Dragonfly and
|
|
6
|
+
Valkey: the choice of server is an operational one, made in the composition root by a
|
|
7
|
+
connection URL.
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
npm i @ddd-ts-kit/redis ioredis
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
| Entry | Holds | Imports |
|
|
14
|
+
|---|---|---|
|
|
15
|
+
| `@ddd-ts-kit/redis` | `connectRedis`, `redisHealth`, `RedisCache`, `RedisLock`, `RedisRateLimiter`, `RedisRelaySignal`, `relayWakeUpMiddleware`; re-exported from the kernel: the `Lock` and `RateLimiter` ports, `InMemoryLock`, `InMemoryRateLimiter` | the kernel, `ioredis` (peer) |
|
|
16
|
+
| `@ddd-ts-kit/redis/event-bus` | `RedisStreamsEventBus`, its options, topology and error types, the names of the entry fields | the kernel, `ioredis` (peer) |
|
|
17
|
+
| `@ddd-ts-kit/redis/testing` | `cacheContract`, `lockContract`, `rateLimiterContract`, re-exported from `@ddd-ts-kit/testing/contracts` | `@ddd-ts-kit/testing` (optional peer), the runner you pass |
|
|
18
|
+
|
|
19
|
+
Nothing here is imported from `domain/` or `application/`. A use case depends on the port
|
|
20
|
+
(`Cache`, `Lock`, `RateLimiter` of the kernel); the adapter is chosen in `platform/` and bound
|
|
21
|
+
in the composition root.
|
|
22
|
+
|
|
23
|
+
With the CLI, `init --cache redis` writes `platform/redis/open-redis.ts` (one connection, the
|
|
24
|
+
cache, the lock, the rate limiter, the relay signal, the readiness probe) and
|
|
25
|
+
`init --broker redis` writes `platform/events/create-event-provider.ts` around
|
|
26
|
+
`RedisStreamsEventBus`. What follows is each piece, wired by hand.
|
|
27
|
+
|
|
28
|
+
## Connection
|
|
29
|
+
|
|
30
|
+
```ts
|
|
31
|
+
// src/platform/redis/redis.ts
|
|
32
|
+
import { connectRedis, redisHealth } from '@ddd-ts-kit/redis';
|
|
33
|
+
|
|
34
|
+
export const redis = connectRedis(config.redisUrl); // or ioredis options, or a client you already have
|
|
35
|
+
|
|
36
|
+
export const ready = () => redisHealth(redis.client); // { status: 'up', latencyMs } | { status: 'down', reason }
|
|
37
|
+
export const shutdown = () => redis.close();
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
- `connectRedis(url | options | client)` answers `{ client, close() }`. A connection it opened
|
|
41
|
+
is closed by `close()`; a client it was handed is used as it is and left open, since its
|
|
42
|
+
owner closes it.
|
|
43
|
+
- A connection opened here adds two settings to the defaults of ioredis (`FAIL_FAST`), which
|
|
44
|
+
your options override: `commandTimeout: 2000` and `maxRetriesPerRequest: 1`. With the plain
|
|
45
|
+
defaults, a command sent while the server is away waits for twenty reconnections before
|
|
46
|
+
failing, and every request of the application waits with it.
|
|
47
|
+
- `redisHealth(client, { timeoutMs? })` sends one `PING`, never throws, and answers within the
|
|
48
|
+
timeout: use it in a readiness probe. A cache being down is rarely a reason to stop
|
|
49
|
+
receiving traffic; a lock or a rate limiter being down may be.
|
|
50
|
+
- Every adapter takes the ioredis client, so several share one connection. The relay signal
|
|
51
|
+
opens a second one, to listen, and the event bus one per consumer name, to block on its
|
|
52
|
+
reads.
|
|
53
|
+
|
|
54
|
+
Local development, with Dragonfly:
|
|
55
|
+
|
|
56
|
+
```yaml
|
|
57
|
+
# compose.yaml
|
|
58
|
+
services:
|
|
59
|
+
dragonfly:
|
|
60
|
+
image: docker.dragonflydb.io/dragonflydb/dragonfly:v2.0.2
|
|
61
|
+
ulimits:
|
|
62
|
+
memlock: -1
|
|
63
|
+
command: ['--maxmemory=512mb', '--proactor_threads=2']
|
|
64
|
+
ports:
|
|
65
|
+
- '127.0.0.1:6379:6379'
|
|
66
|
+
volumes:
|
|
67
|
+
- dragonfly:/data
|
|
68
|
+
volumes:
|
|
69
|
+
dragonfly:
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
`REDIS_URL=redis://127.0.0.1:6379` then suits the three servers alike.
|
|
73
|
+
|
|
74
|
+
**Credentials.** A client of ioredis keeps its options, password included, and shows them to
|
|
75
|
+
whoever inspects it: that cannot be changed from here, so never log a client. The adapters of
|
|
76
|
+
this package (`RedisCache`, `RedisLock`, `RedisRateLimiter`, `RedisRelaySignal`,
|
|
77
|
+
`RedisStreamsEventBus`) hold their client in a field that is not enumerable, and answer their
|
|
78
|
+
name to `JSON.stringify` and `util.inspect`: logging, spreading or serialising an adapter does
|
|
79
|
+
not lead to the client. The object `connectRedis` answers gives its `client` by name
|
|
80
|
+
(`redis.client`, `const { client } = redis`) and leaves it out of a spread or a log.
|
|
81
|
+
|
|
82
|
+
## Cache
|
|
83
|
+
|
|
84
|
+
`RedisCache` implements the kernel `Cache` port with its typed `CacheKey`.
|
|
85
|
+
|
|
86
|
+
```ts
|
|
87
|
+
// src/main.ts
|
|
88
|
+
import { Cache, Lock, Metrics, RateLimiter } from '@ddd-ts-kit/kernel';
|
|
89
|
+
import { contextModule, createApplication, defineContextModule, kernelModule } from '@ddd-ts-kit/di';
|
|
90
|
+
import { RedisCache, RedisLock, RedisRateLimiter } from '@ddd-ts-kit/redis';
|
|
91
|
+
|
|
92
|
+
const platformModule = defineContextModule({
|
|
93
|
+
name: 'platform',
|
|
94
|
+
register(builder) {
|
|
95
|
+
builder.rebind(Cache).toFactory((resolve) => new RedisCache(redis.client, { namespace: 'app', metrics: resolve(Metrics) }));
|
|
96
|
+
builder.rebind(Lock).toValue(new RedisLock(redis.client, { namespace: 'app' }));
|
|
97
|
+
builder.rebind(RateLimiter).toValue(new RedisRateLimiter(redis.client, { namespace: 'app' }));
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
const app = createApplication({ modules: [kernelModule(), platformModule, contextModule('catalog', catalogDeclarations)] });
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
A port of the kernel is an abstract class, and the class is what a binding is keyed by: the
|
|
105
|
+
three adapters `implements` theirs, and a use case that names `Cache`, `Lock` or `RateLimiter`
|
|
106
|
+
in its dependencies receives them. `kernelModule({ cache: (resolve) => new RedisCache(...), lock: ..., rateLimiter: ... })`
|
|
107
|
+
does the same through its options. `cached` of `@ddd-ts-kit/aspects` and everything that
|
|
108
|
+
resolves `Cache` then use it without knowing. A context that wants its keys apart builds its
|
|
109
|
+
own instance with its own `namespace`.
|
|
110
|
+
|
|
111
|
+
| Option | Default | Effect |
|
|
112
|
+
|---|---|---|
|
|
113
|
+
| `namespace` | none | every key is `<namespace>:<name>`: one per bounded context, or per application sharing a server |
|
|
114
|
+
| `codec` | `jsonCodec` | how a value becomes text and back. With JSON, a `Date` comes back as a string and a class instance as a plain object: cache data, not objects |
|
|
115
|
+
| `onError` | `miss` | a failing server costs a miss on read and a dropped write, never an exception. `throw` lets the failure through |
|
|
116
|
+
| `metrics` | none | counts failures as `cache_errors_total`, labels `operation` (`get`, `set`, `delete`) and `context` (the namespace) |
|
|
117
|
+
| `lock` | none | a `Lock` that `getOrLoad` uses across processes |
|
|
118
|
+
|
|
119
|
+
- `set(key, value, { ttlMs })` stores with `PX`, in milliseconds. Without `ttlMs` the entry
|
|
120
|
+
stays until deleted or evicted by the server; `ttlMs: 0` or less deletes it.
|
|
121
|
+
- `deleteByPrefix('price:')` deletes every entry of the namespace whose name starts with the
|
|
122
|
+
prefix and answers how many. It walks the keys with `SCAN` (200 at a time), never `KEYS`,
|
|
123
|
+
and reads the prefix literally (`*`, `?`, `[` are escaped). It is not atomic: an entry
|
|
124
|
+
written during the walk may survive. It is an administrative tool, not something to call per
|
|
125
|
+
request; prefer short time to live values to invalidation.
|
|
126
|
+
- An entry that cannot be decoded is a miss and is counted.
|
|
127
|
+
|
|
128
|
+
**A cache must not take the application down.** With the default `onError: 'miss'`, a server
|
|
129
|
+
that is away, slow beyond the command timeout or answering errors makes the application run
|
|
130
|
+
as if it had no cache. Watch `cache_errors_total`: a cache that silently stopped working is a
|
|
131
|
+
latency incident waiting for load.
|
|
132
|
+
|
|
133
|
+
### Stampede protection
|
|
134
|
+
|
|
135
|
+
When a popular entry expires, every request that misses it computes it again at the same time.
|
|
136
|
+
|
|
137
|
+
- Inside one process, `cached` of `@ddd-ts-kit/aspects` already shares one underlying call
|
|
138
|
+
among concurrent identical calls (single flight). Give it a `RedisCache` and that part is done.
|
|
139
|
+
- Across processes, use `getOrLoad` with a `lock`:
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
const cache = new RedisCache(redis.client, { namespace: 'catalog', lock: new RedisLock(redis.client, { namespace: 'catalog' }) });
|
|
143
|
+
|
|
144
|
+
const quote = await cache.getOrLoad(quoteKey(pair), () => rates.quoteOf(pair), { ttlMs: 30_000 });
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
`getOrLoad` reads the entry; on a miss the callers of this process share one computation, and
|
|
148
|
+
with a lock only one process computes: the others wait for its value up to `waitMs` (default
|
|
149
|
+
2 s), then compute it themselves. It never fails because of the lock or the server: the worst
|
|
150
|
+
case is the unprotected one. A failed computation is not cached.
|
|
151
|
+
|
|
152
|
+
## Lock
|
|
153
|
+
|
|
154
|
+
`RedisLock` implements the kernel `Lock` port:
|
|
155
|
+
|
|
156
|
+
```ts
|
|
157
|
+
abstract class Lock {
|
|
158
|
+
abstract acquire(key: string, options: { ttlMs: number; waitMs?: number }): Promise<LockHandle | undefined>;
|
|
159
|
+
}
|
|
160
|
+
interface LockHandle { readonly key: string; readonly fencingToken: number; release(): Promise<void> }
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
```ts
|
|
164
|
+
const handle = await lock.acquire('nightly-export', { ttlMs: 60_000 });
|
|
165
|
+
if (handle === undefined) { // another instance is already doing it
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
try {
|
|
169
|
+
await exportEverything();
|
|
170
|
+
} finally {
|
|
171
|
+
await handle.release();
|
|
172
|
+
}
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
`RedisLock` is the single-instance algorithm of the Redis documentation: `SET key <random
|
|
176
|
+
owner> NX PX ttl`, and a release that deletes the key only if it still holds that owner
|
|
177
|
+
(compare-and-delete), so a holder whose lock expired cannot free the lock of the next one.
|
|
178
|
+
`waitMs` polls every `retryDelayMs` (default 25 ms). `InMemoryLock` of the kernel is the same
|
|
179
|
+
contract inside one process, for tests and single instances; it is what `kernelModule()` of
|
|
180
|
+
`@ddd-ts-kit/di` binds `Lock` to until the platform rebinds it (`builder.rebind(Lock)`, shown
|
|
181
|
+
under [Cache](#cache)).
|
|
182
|
+
|
|
183
|
+
**What it guarantees, and what it does not.** At most one holder at a time, provided the
|
|
184
|
+
server keeps its data and the holder finishes within `ttlMs`. It is **not a consensus lock**.
|
|
185
|
+
Two holders are possible when:
|
|
186
|
+
|
|
187
|
+
- the holder is paused (garbage collection, a stalled disk, a suspended container) for longer
|
|
188
|
+
than the time to live, and keeps working after the lock went to somebody else;
|
|
189
|
+
- the server fails over to a replica that had not received the key yet;
|
|
190
|
+
- the server restarts without persistence, or evicts the key under memory pressure;
|
|
191
|
+
- clocks, on the server side, jump.
|
|
192
|
+
|
|
193
|
+
So use it to **avoid doing work twice** (a scheduled job on several instances, the
|
|
194
|
+
recomputation of a cache entry, an import): cases where a rare double run costs some waste and
|
|
195
|
+
nothing else. Do **not** use it where two holders would break an invariant. Preventing a
|
|
196
|
+
double booking, a double payment or a duplicated invoice number belongs to a database
|
|
197
|
+
constraint (a unique index, an exclusion constraint, the optimistic version of the aggregate),
|
|
198
|
+
which holds whatever any process believes about a lock.
|
|
199
|
+
|
|
200
|
+
The **fencing token** grows with each acquisition of a key (`INCR`, in the same script as the
|
|
201
|
+
`SET`). A resource that stores the highest token it has seen can refuse a write from a stale
|
|
202
|
+
holder. It narrows the gap, for resources able to compare; its counter lives in the same
|
|
203
|
+
server and starts again if that server loses its data, so it is not a proof either.
|
|
204
|
+
|
|
205
|
+
A server that is away makes `acquire` reject: whether somebody holds the key cannot be known,
|
|
206
|
+
and the caller decides.
|
|
207
|
+
|
|
208
|
+
## Rate limiter
|
|
209
|
+
|
|
210
|
+
`RedisRateLimiter` implements the kernel `RateLimiter` port:
|
|
211
|
+
|
|
212
|
+
```ts
|
|
213
|
+
abstract class RateLimiter {
|
|
214
|
+
abstract consume(key: string, limit: { limit: number; windowMs: number }): Promise<{ allowed: boolean; remaining: number; retryAfterMs: number }>;
|
|
215
|
+
}
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
A sliding window: at any instant, at most `limit` allowed calls during the `windowMs` before
|
|
219
|
+
it. A refused call is not counted. `RedisRateLimiter` keeps one sorted set per key, one member
|
|
220
|
+
per allowed call, and decides in a single script, so concurrent calls from several instances
|
|
221
|
+
cannot both take the last place.
|
|
222
|
+
|
|
223
|
+
In front of the HTTP layer, before authentication cost anything (Fastify shown; the same three
|
|
224
|
+
lines fit an Express middleware):
|
|
225
|
+
|
|
226
|
+
```ts
|
|
227
|
+
// src/platform/http/rate-limit.ts
|
|
228
|
+
const limiter = new RedisRateLimiter(redis.client, { namespace: 'api', onError: 'allow' });
|
|
229
|
+
|
|
230
|
+
server.addHook('onRequest', async (request, reply) => {
|
|
231
|
+
const decision = await limiter.consume(`ip:${request.ip}`, { limit: 120, windowMs: 60_000 });
|
|
232
|
+
reply.header('RateLimit-Remaining', decision.remaining);
|
|
233
|
+
if (decision.allowed) {
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
reply.header('Retry-After', Math.ceil(decision.retryAfterMs / 1000));
|
|
237
|
+
return reply.code(429).send({ type: 'about:blank', title: 'Too Many Requests', status: 429, code: 'RATE_LIMITED' });
|
|
238
|
+
});
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
A use case that must be limited for a business reason (three password resets per hour)
|
|
242
|
+
depends on the `RateLimiter` port of the kernel and receives the adapter by injection
|
|
243
|
+
(`builder.rebind(RateLimiter)`, shown under [Cache](#cache)); `InMemoryRateLimiter` on a
|
|
244
|
+
`FixedClock` is its fake.
|
|
245
|
+
|
|
246
|
+
- **Exact, at a memory cost.** One member per allowed call: a key costs memory in proportion
|
|
247
|
+
to its limit. Fine up to thousands per window; for tens of thousands, limit a coarser key or
|
|
248
|
+
a shorter window.
|
|
249
|
+
- **The clock is the caller's.** Instants come from the `Clock` given to the limiter (the
|
|
250
|
+
system clock by default), not from the server, so every instance must be synchronized (NTP):
|
|
251
|
+
a skew between two instances shifts the window by as much. In exchange the limiter is tested
|
|
252
|
+
on a fixed clock, and the script holds no arithmetic that differs between Lua versions.
|
|
253
|
+
- **`onError`**: `throw` (default) lets the caller decide; `allow` keeps serving without a
|
|
254
|
+
limit while the server is away (the usual choice in front of an API); `deny` refuses
|
|
255
|
+
everything (for the few limits that are a security control, such as login attempts).
|
|
256
|
+
- The key expires by itself one window after its last allowed call.
|
|
257
|
+
|
|
258
|
+
## Relay signal
|
|
259
|
+
|
|
260
|
+
An outbox relay polls. `RedisRelaySignal` lets it poll rarely and still publish at once, across
|
|
261
|
+
processes: the process that committed a command notifies, a relay in any other process wakes.
|
|
262
|
+
|
|
263
|
+
```ts
|
|
264
|
+
// the worker that relays
|
|
265
|
+
const signal = new RedisRelaySignal(redis.client, { channel: 'app:outbox' });
|
|
266
|
+
new OutboxRelay({ store, publisher, clock, wakeOn: signal }).start({ intervalMs: 5_000 });
|
|
267
|
+
|
|
268
|
+
// the API process, before the unit of work so that it notifies after the commit
|
|
269
|
+
commandBus.use(relayWakeUpMiddleware(signal)).use(unitOfWorkMiddleware(unitOfWork));
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
It satisfies the `RelaySignal` port of `@ddd-ts-kit/events` structurally: this package does not
|
|
273
|
+
depend on `@ddd-ts-kit/events`, so the class does not name the port, and it is accepted wherever
|
|
274
|
+
a `RelaySignal` is expected, `builder.bind(RelaySignal).toValue(signal)` included. It has
|
|
275
|
+
the semantics of the in-memory signal: **a hint, never a delivery**. Publish/subscribe keeps
|
|
276
|
+
nothing for a subscriber that is away; a notification sent while the relay reconnects, or
|
|
277
|
+
before it subscribed, is lost, and the event goes out at the next poll. Keep a real
|
|
278
|
+
`intervalMs`. `notify()` never throws. `close()` closes the listening connection the signal
|
|
279
|
+
opened.
|
|
280
|
+
|
|
281
|
+
`relayWakeUpMiddleware` notifies after every command that succeeded, whether it published or
|
|
282
|
+
not: a spurious wake-up costs the relay one empty claim. With PostgreSQL, `LISTEN/NOTIFY` from
|
|
283
|
+
the outbox table does the same without another server.
|
|
284
|
+
|
|
285
|
+
## Event bus on Redis Streams
|
|
286
|
+
|
|
287
|
+
`RedisStreamsEventBus` is an integration event provider: an `EventBus` of the kernel that
|
|
288
|
+
publishes with `XADD` and resolves only when the server acknowledged every entry, so the
|
|
289
|
+
outbox relay can mark a record published, and consumes through one consumer group per
|
|
290
|
+
consumer with a blocking read on a dedicated connection, acknowledgement after the handler, a
|
|
291
|
+
bounded number of delayed redeliveries and a dead stream. It sits next to the RabbitMQ, Kafka
|
|
292
|
+
and NATS providers, passes `eventBusContract` of `@ddd-ts-kit/testing`, and runs unchanged on
|
|
293
|
+
Redis, Dragonfly and Valkey.
|
|
294
|
+
|
|
295
|
+
```
|
|
296
|
+
npm i @ddd-ts-kit/redis @ddd-ts-kit/events ioredis
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
**When to choose this.** You already run and operate Redis, Dragonfly or Valkey, the volume is
|
|
300
|
+
moderate (handlers of a stream run one at a time per instance), and one more server to run is
|
|
301
|
+
worth avoiding. You get consumer groups, redelivery, dead letters and a readable log with one
|
|
302
|
+
dependency. Choose RabbitMQ when you want a broker that owns delivery (publisher confirms
|
|
303
|
+
written to disk, quorum queues replicated before the confirmation, per-message routing), and
|
|
304
|
+
Kafka when you want a durable, replicated log with partitions, per-key ordering and long
|
|
305
|
+
retention. Here durability is what the server is configured for, and retention is bounded by
|
|
306
|
+
memory: see [Guarantees and limits](#guarantees-and-limits).
|
|
307
|
+
|
|
308
|
+
### Composition root
|
|
309
|
+
|
|
310
|
+
Handlers never see this package. They publish to the outbox; the relay publishes to the
|
|
311
|
+
streams; each consuming context subscribes through the bus under its own name.
|
|
312
|
+
|
|
313
|
+
```ts
|
|
314
|
+
// main.ts
|
|
315
|
+
import { SystemClock } from '@ddd-ts-kit/kernel';
|
|
316
|
+
import { OutboxPublisher, OutboxRelay, consume, subscribeAll, validatingPublisher } from '@ddd-ts-kit/events';
|
|
317
|
+
import { RedisStreamsEventBus } from '@ddd-ts-kit/redis/event-bus';
|
|
318
|
+
import { tracedPublisher, tracedSubscriber } from '@ddd-ts-kit/tracing';
|
|
319
|
+
|
|
320
|
+
const bus = new RedisStreamsEventBus({
|
|
321
|
+
connection: config.redisUrl, // or ioredis options, or a client you already have (redis.client)
|
|
322
|
+
prefix: 'integration-events', // the default: one stream per type, `<prefix>:<type>`
|
|
323
|
+
retry: { maxAttempts: 5, delayMs: 5_000 }, // the defaults
|
|
324
|
+
visibilityTimeoutMs: 60_000, // the default: above the worst case of a batch of handlers
|
|
325
|
+
batchSize: 10, // the default
|
|
326
|
+
onFailure: (failure) => logger.warn(failure), // codes only, never an error message
|
|
327
|
+
onDelivery: ({ stream, consumer, outcome }) => deliveries.add(1, { stream, consumer, outcome }),
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
// Publishing side: handlers write to the outbox, in the transaction of the command.
|
|
331
|
+
const publisher = validatingPublisher(new OutboxPublisher(outboxStore), registry);
|
|
332
|
+
|
|
333
|
+
// The relay target is the bus. A record is marked published when the server answered its entry id.
|
|
334
|
+
const relay = new OutboxRelay({ store: outboxStore, publisher: tracedPublisher(bus), clock: new SystemClock() });
|
|
335
|
+
relay.start({ intervalMs: 500 });
|
|
336
|
+
|
|
337
|
+
// Consuming side: one subscriber per consuming context, named after it.
|
|
338
|
+
const billingEvents = tracedSubscriber(bus.forConsumer('billing'));
|
|
339
|
+
subscribeAll(billingEvents, [
|
|
340
|
+
consume(orderPlaced, { consumer: 'billing.ordering-order-placed', processed, unitOfWork }, async (event) => {
|
|
341
|
+
await commandBus.execute(new IssueInvoice(event.payload.orderId));
|
|
342
|
+
}),
|
|
343
|
+
]);
|
|
344
|
+
|
|
345
|
+
await bus.assertTopology(); // fail at startup, not at the first event
|
|
346
|
+
|
|
347
|
+
process.on('SIGTERM', async () => {
|
|
348
|
+
await relay.stop(); // the tick in progress finishes
|
|
349
|
+
await bus.close(); // readers stop, the batch in progress finishes and acknowledges
|
|
350
|
+
});
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
- `publish(events)` sends the batch as one `MULTI` of `XADD` and resolves only when the server
|
|
354
|
+
answered an entry id for **every** event. It rejects otherwise (`PUBLISH_NOT_CONFIRMED` with
|
|
355
|
+
the `eventId` and a `reason` code, `BROKER_UNAVAILABLE`, `EVENT_BUS_CLOSED`). Nothing is
|
|
356
|
+
buffered for later: the outbox is the buffer.
|
|
357
|
+
- A consumer name is required to subscribe: `bus.forConsumer('billing')` gives an
|
|
358
|
+
`IntegrationEventSubscriber` for one context (a modular monolith has several, each with a
|
|
359
|
+
reading connection of its own), and the `consumer` option names the one `bus.subscribe`
|
|
360
|
+
works for (a service that is a single context).
|
|
361
|
+
- With `@ddd-ts-kit/di`, the declared integration event handlers of a context are subscribed on
|
|
362
|
+
`EventSubscribers.forContext(<module name>)`. Give it the bus, and each context reads under
|
|
363
|
+
its own name:
|
|
364
|
+
|
|
365
|
+
```ts
|
|
366
|
+
import { EventSubscribers } from '@ddd-ts-kit/kernel';
|
|
367
|
+
|
|
368
|
+
const app = createApplication({
|
|
369
|
+
modules: [
|
|
370
|
+
kernelModule({ eventSubscribers: () => ({ forContext: (name) => bus.forConsumer(name) }) }),
|
|
371
|
+
contextModule('billing', billingDeclarations),
|
|
372
|
+
],
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
// or, from a later module
|
|
376
|
+
builder.rebind(EventSubscribers).toValue({ forContext: (name) => bus.forConsumer(name) });
|
|
377
|
+
```
|
|
378
|
+
- The tracing wrappers work as with any provider (with `@ddd-ts-kit/di`, the consuming side
|
|
379
|
+
is traced by `kernelModule({ instrumentation: traceApplication() })` and `tracedSubscriber`
|
|
380
|
+
is left out): the bus carries `traceparent` and
|
|
381
|
+
`correlationId` untouched, in the envelope and as fields of the entry, across a retry too.
|
|
382
|
+
Next to the event, each handler receives the delivery (`EventDelivery` of the kernel):
|
|
383
|
+
`{ attempt, maxAttempts, system: 'redis', destination, consumer }`, where `destination` is the
|
|
384
|
+
stream. `@ddd-ts-kit/tracing` reads it to make one CONSUMER span per attempt; the bus names
|
|
385
|
+
itself `messagingSystem: 'redis'`.
|
|
386
|
+
- The correlation makes the round trip. Publish through `scopedPublisher(publisher, scope)` of
|
|
387
|
+
the kernel and consume with `consume(definition, { ..., scope }, handler)`: the consumer runs
|
|
388
|
+
in an execution context whose `correlationId` is the one of the request that published,
|
|
389
|
+
whose `causationId` is the event, and a redelivery runs in the same one. Or give the bus the
|
|
390
|
+
`scope` option: every handler then runs inside it, under the name of its consumer.
|
|
391
|
+
- A context extracted to its own service keeps its consumer name and so its groups: nothing
|
|
392
|
+
changes on the server.
|
|
393
|
+
- The publishing connection is the one you give, shared with the other adapters of this
|
|
394
|
+
package if you hand over `redis.client`. Every consumer name reads on a duplicate of it
|
|
395
|
+
(`client.duplicate()`), because a blocking read holds its connection. `close()` closes what
|
|
396
|
+
the bus opened and leaves a client it was handed open.
|
|
397
|
+
|
|
398
|
+
| Option | Default | Effect |
|
|
399
|
+
|---|---|---|
|
|
400
|
+
| `connection` | required | URL, ioredis options or client. A connection opened here takes `FAIL_FAST` and a reconnection delay from `reconnect` |
|
|
401
|
+
| `prefix`, `streamOf` | `integration-events` | the stream of a type is `<prefix>:<type>`, or what `streamOf(type)` answers |
|
|
402
|
+
| `consumer` | none | the consumer `bus.subscribe` reads for |
|
|
403
|
+
| `instance` | host, process id, random suffix | the name of this process inside its groups; unique per process |
|
|
404
|
+
| `batchSize` | 10 | entries read at once per stream (`COUNT`), then handled in order, one at a time |
|
|
405
|
+
| `blockMs` | 1000 | how long a read waits (`BLOCK`): it bounds `close()` and the precision of the retry delay |
|
|
406
|
+
| `retry` | 5 attempts, 5000 ms | deliveries before the dead stream, and the wait after a failure |
|
|
407
|
+
| `visibilityTimeoutMs` | 60000 | idle time after which the pending entries of another instance are taken over; never less than `retry.delayMs` |
|
|
408
|
+
| `startFrom` | `new` | where a group this bus creates starts: what is published from then on, or `beginning` for the whole stream |
|
|
409
|
+
| `retention` | none | `{ maxLength }` or `{ maxAgeMs }`: trimming applied by each publication. See the limits |
|
|
410
|
+
| `deduplication` | none | `{ windowMs }`: the publish-side guard below |
|
|
411
|
+
| `scope` | none | an `ExecutionScope` every handler runs inside |
|
|
412
|
+
| `reconnect` | 200 ms doubling up to 30 s | delay between reconnections, and between the attempts of a reader that fails |
|
|
413
|
+
|
|
414
|
+
### Topology
|
|
415
|
+
|
|
416
|
+
`bus.describeTopology()` returns the streams and the groups as data, for a review or a
|
|
417
|
+
generated document; `bus.assertTopology()` creates them, idempotently. The bus also creates
|
|
418
|
+
its groups before it reads or publishes, and again after any failure of a reader (a server
|
|
419
|
+
that came back without its data has lost them). For `forConsumer('billing')` subscribed to
|
|
420
|
+
`ordering.order.placed`:
|
|
421
|
+
|
|
422
|
+
| Object | Name | Properties |
|
|
423
|
+
|---|---|---|
|
|
424
|
+
| events stream | `integration-events:ordering.order.placed` | one per event type, shared by every consumer; created empty by the first group (`MKSTREAM`) or by the first publication |
|
|
425
|
+
| consumer group | `billing`, on that stream | one per consumer name; holds its position and its pending entries; created at `$` (or `0` with `startFrom: 'beginning'`) |
|
|
426
|
+
| group member | the `instance` name | one per process; removed at `close()` when it leaves nothing pending |
|
|
427
|
+
| dead stream | `integration-events:ordering.order.placed:billing:dead` | one per consumer and type, created by the first dead letter; no group reads it |
|
|
428
|
+
|
|
429
|
+
An entry holds the envelope as JSON in the field `envelope` (`id`, `type`, `version`,
|
|
430
|
+
`occurredAt`, `payload`, `traceparent`, `correlationId`) and, beside it, the fields `id`,
|
|
431
|
+
`type`, `version`, `traceparent` and `correlation-id`: an operator reads them with `XRANGE`
|
|
432
|
+
without parsing the body, and any client of the protocol can consume the stream.
|
|
433
|
+
|
|
434
|
+
Two consumers of one type have a group each and both receive every event. Two instances of one
|
|
435
|
+
consumer are two members of its group: each event goes to one of them.
|
|
436
|
+
|
|
437
|
+
**Ordering.** One instance receives the events of a type in the order of the stream, which is
|
|
438
|
+
the order the server took them in. That order is broken by a redelivery (the failed event
|
|
439
|
+
comes back after later ones were handled) and by competing instances (two members of a group
|
|
440
|
+
progress independently). There is no order across types, and none per aggregate: carry a
|
|
441
|
+
version in the payload when it matters.
|
|
442
|
+
|
|
443
|
+
### Retry and dead-letter behaviour
|
|
444
|
+
|
|
445
|
+
1. An entry is acknowledged (`XACK`) after every handler subscribed to its type resolved. Use
|
|
446
|
+
`consume` of `@ddd-ts-kit/events`: the acknowledgement then follows the commit of the unit
|
|
447
|
+
of work, and a redelivery is deduplicated.
|
|
448
|
+
2. When a handler throws, the entry is not acknowledged: it stays in the pending entry list of
|
|
449
|
+
the group, owned by this instance, and its idle time restarts at the failure. Once it has
|
|
450
|
+
been idle for `retry.delayMs`, the instance claims it again (`XPENDING` with `IDLE`, then
|
|
451
|
+
`XCLAIM`). A failed entry is never handled again on the spot: no hot loop.
|
|
452
|
+
3. The attempt number is the delivery counter the server keeps in the pending entry list: 1
|
|
453
|
+
for the first read, incremented by each claim. At the `retry.maxAttempts`-th failed
|
|
454
|
+
delivery the entry is copied to the dead stream with `dead-reason: max-attempts`,
|
|
455
|
+
`error-code`, `attempt`, `consumer` and `entry-id` (its id in the events stream), and
|
|
456
|
+
acknowledged, in one script: never one without the other.
|
|
457
|
+
4. The pending entries of an instance that died, or that is stuck, are taken over by another
|
|
458
|
+
instance of the consumer once they have been idle for `visibilityTimeoutMs`. That delivery
|
|
459
|
+
counts as an attempt. An entry that used all its attempts this way, never answered by a
|
|
460
|
+
handler, goes to the dead stream with `error-code: ATTEMPTS_EXHAUSTED` without being
|
|
461
|
+
handled again: an event that kills its consumers does not kill them forever.
|
|
462
|
+
5. An entry that does not hold an envelope (no `envelope` field, unparseable, or without `id`,
|
|
463
|
+
`type`, `version`, `occurredAt`) is poison: it goes straight to the dead stream with
|
|
464
|
+
`dead-reason: poison` and `error-code: POISON_ENTRY`, and no handler sees it.
|
|
465
|
+
6. Only codes travel: `error-code` is the `code` of the error or its class name. The message
|
|
466
|
+
of an error is never logged, reported or attached, because it may quote payload data.
|
|
467
|
+
|
|
468
|
+
Delivery is at least once. An acknowledgement lost to a disconnection, a claim racing a slow
|
|
469
|
+
handler, a relay retry: each delivers an event twice, never zero times.
|
|
470
|
+
|
|
471
|
+
On a lost connection ioredis reconnects with the delay of `reconnect`; `publish` rejects at
|
|
472
|
+
once between two attempts (`BROKER_UNAVAILABLE`), so the outbox relay keeps the records and
|
|
473
|
+
retries them, and the readers resume where their groups are. Entries that were being handled
|
|
474
|
+
stay pending and are claimed again.
|
|
475
|
+
|
|
476
|
+
### Publishing twice
|
|
477
|
+
|
|
478
|
+
Redis Streams have **no publish-side deduplication**: an `XADD` that succeeded but whose
|
|
479
|
+
answer was lost is appended again by the retry of the relay, and both entries are delivered.
|
|
480
|
+
The answer is the same as for every provider: the consumer deduplicates, with a
|
|
481
|
+
processed-event store in the transaction of its handler (`consume`).
|
|
482
|
+
|
|
483
|
+
A cheap guard narrows the window, as an option:
|
|
484
|
+
|
|
485
|
+
```ts
|
|
486
|
+
new RedisStreamsEventBus({ connection, deduplication: { windowMs: 3_600_000 } });
|
|
487
|
+
```
|
|
488
|
+
|
|
489
|
+
Each event is then appended by a script that first looks for the key
|
|
490
|
+
`<stream>:<event id>:published`, appends only if it is absent, and writes it with the window
|
|
491
|
+
as its time to live. A relay retry within the window appends nothing and is acknowledged. It
|
|
492
|
+
is a guard, not a guarantee: the mark is one more key, evicted or lost with the server like
|
|
493
|
+
any other, and it costs one key per event for the duration of the window. Consumers still
|
|
494
|
+
deduplicate. Verified on Dragonfly v2.0.2, Valkey 9.1.2 and Redis 8.10.2. (Redis 8.10.2
|
|
495
|
+
reports a server-side idempotent append in `XINFO STREAM`; it is not used, the two other
|
|
496
|
+
servers do not have it.)
|
|
497
|
+
|
|
498
|
+
### Operations
|
|
499
|
+
|
|
500
|
+
What to watch, with the commands of any client (`<stream>` is `<prefix>:<type>`):
|
|
501
|
+
|
|
502
|
+
| Signal | Command | Meaning |
|
|
503
|
+
|---|---|---|
|
|
504
|
+
| outbox `stats()` (`pending`, `failed`, `dead`) | the outbox store | the publishing side: see `@ddd-ts-kit/events` |
|
|
505
|
+
| lag of a consumer | `XINFO GROUPS <stream>`: `lag`, `last-delivered-id` | entries not read yet: the consumer is late or stopped |
|
|
506
|
+
| pending entries | `XINFO GROUPS <stream>`: `pending`; detail with `XPENDING <stream> <consumer> - + 10` (owner, idle time, deliveries) | being handled, or waiting for a redelivery; a plateau means a handler that keeps failing, an idle time above the visibility timeout means nobody reads |
|
|
507
|
+
| depth of the dead stream | `XLEN <stream>:<consumer>:dead` | must be 0: alert on anything else |
|
|
508
|
+
| what died and why | `XRANGE <stream>:<consumer>:dead - + COUNT 10`: `error-code`, `dead-reason`, `attempt` | the code to look for in the logs of the consumer |
|
|
509
|
+
| members of a group | `XINFO CONSUMERS <stream> <consumer>` | instances; one that crashed stays listed until removed |
|
|
510
|
+
| memory of a stream | `XLEN <stream>`, `MEMORY USAGE <stream>` | streams only grow unless trimmed |
|
|
511
|
+
| `onDelivery` counted by `outcome` | this bus | rate of `retried`, `dead`, `poison` against `acknowledged` |
|
|
512
|
+
| `onFailure` with `kind: 'connection'` | this bus | reconnections (`CONNECTION_LOST`, `ECONNREFUSED`) and what the server refused to a reader (`REDIS_NOGROUP`) |
|
|
513
|
+
| `bus.deliveriesInFlight` | this bus | entries read and not settled; stuck above 0 means a handler that hangs |
|
|
514
|
+
|
|
515
|
+
Replaying dead entries, once the cause is fixed:
|
|
516
|
+
|
|
517
|
+
```ts
|
|
518
|
+
const replayed = await bus.replayDead({ consumer: 'billing', type: 'ordering.order.placed' }, 100);
|
|
519
|
+
```
|
|
520
|
+
|
|
521
|
+
It moves up to that many entries, oldest first, from the dead stream back to the events
|
|
522
|
+
stream, each in one script (appended, then deleted from the dead stream). The new entry has a
|
|
523
|
+
fresh attempt budget and carries `replay-for: billing`: the other consumers of the type
|
|
524
|
+
acknowledge it without handling it. A client of your own that reads the stream should skip an
|
|
525
|
+
entry whose `replay-for` names somebody else. Look at `dead-reason` and `error-code` first:
|
|
526
|
+
replaying a poison entry only kills it again.
|
|
527
|
+
|
|
528
|
+
Housekeeping:
|
|
529
|
+
|
|
530
|
+
- A consumer stopped for good: `XGROUP DESTROY <stream> <consumer>` and delete its dead
|
|
531
|
+
stream. Unsubscribing only stops reading; the group keeps its position and its lag grows.
|
|
532
|
+
- An instance that crashed stays a member of its groups. Its pending entries are taken over
|
|
533
|
+
after the visibility timeout; remove the name with `XGROUP DELCONSUMER <stream> <consumer>
|
|
534
|
+
<instance>` once `XINFO CONSUMERS` shows it with 0 pending (deleting a member drops its
|
|
535
|
+
pending entries).
|
|
536
|
+
- Trimming by hand, when no `retention` is configured: `XTRIM <stream> MINID ~ <id>` with an
|
|
537
|
+
id every group has passed (`last-delivered-id` of `XINFO GROUPS`) and nothing pending below.
|
|
538
|
+
|
|
539
|
+
### Guarantees and limits
|
|
540
|
+
|
|
541
|
+
- **Durability is the server's, and it is a setting.** An entry is acknowledged when the
|
|
542
|
+
server holds it in memory. What survives a crash depends on persistence: with Redis or
|
|
543
|
+
Valkey and `appendonly yes`, `appendfsync everysec` (the usual setting) can lose about the
|
|
544
|
+
last second of acknowledged entries and `always` none, at a cost on every write; with
|
|
545
|
+
snapshots only (RDB, and Dragonfly, which persists by snapshots), everything since the last
|
|
546
|
+
snapshot is lost; without persistence a restart starts empty. Replication is asynchronous:
|
|
547
|
+
a failover can lose what the replica had not received. For an event log this means an event
|
|
548
|
+
the relay marked published can vanish, and no consumer will ever see it. A broker that
|
|
549
|
+
confirms after writing to a replicated log does not have that window. Decide whether your
|
|
550
|
+
events tolerate it, set persistence accordingly, and do not purge published outbox records
|
|
551
|
+
before your backup horizon.
|
|
552
|
+
- **The positions are in the same server.** Groups, pending entries and delivery counters are
|
|
553
|
+
lost or rewound with the data. A server restored from an older snapshot delivers again what
|
|
554
|
+
was handled since (consumers deduplicate); a server that lost everything gets its groups
|
|
555
|
+
created again at the end of empty streams.
|
|
556
|
+
- **No publish-side deduplication**, as said above.
|
|
557
|
+
- **Retention is bounded by memory.** A stream lives in memory, entirely. By default nothing
|
|
558
|
+
is trimmed. `retention: { maxLength }` (`MAXLEN ~`) and `retention: { maxAgeMs }`
|
|
559
|
+
(`MINID ~`, by the clock of the publisher) trim at each publication, approximately, and
|
|
560
|
+
**without regard for the groups**: an entry no consumer has read yet, or one waiting for
|
|
561
|
+
its redelivery, is dropped with the rest and silently leaves the pending list at its next
|
|
562
|
+
claim. Size the cap far above the worst lag you accept, or trim by hand. Dead streams are
|
|
563
|
+
never trimmed.
|
|
564
|
+
- **Eviction applies to everything.** Under `allkeys-*` a whole stream can be evicted. Run the
|
|
565
|
+
event bus on a server with `noeviction` (a publication then fails when memory is full, and
|
|
566
|
+
the relay retries), not on the one that is your cache.
|
|
567
|
+
- **One at a time per stream and instance.** Throughput is that of your handlers in sequence;
|
|
568
|
+
scale with instances, at the price of ordering. Types are handled in parallel inside a
|
|
569
|
+
batch, but a turn ends when its slowest stream does.
|
|
570
|
+
- **The visibility timeout must exceed the worst case of a batch** (`batchSize` times the
|
|
571
|
+
slowest handler): beyond it another instance takes the entries over while the first is still
|
|
572
|
+
working, and they are handled twice. With slow handlers, lower `batchSize`.
|
|
573
|
+
- **One delay.** The same wait before every retry, no exponential backoff, precise to
|
|
574
|
+
`blockMs`. A failed entry whose instance stops is retried by another one after the
|
|
575
|
+
visibility timeout, not after the retry delay.
|
|
576
|
+
- **A redelivery runs every handler of the type again**, those that succeeded included, when
|
|
577
|
+
several handlers of one consumer subscribe to the same type. Each must be idempotent.
|
|
578
|
+
- **A batch is one transaction on one server**: all its entries are appended, or the
|
|
579
|
+
publication rejects. A rejection does not prove nothing was appended (the answer may have
|
|
580
|
+
been lost); the relay retries and consumers deduplicate.
|
|
581
|
+
- **A new group starts at the end of the stream** by default: what was published before a
|
|
582
|
+
consumer first subscribed is not delivered to it. `startFrom: 'beginning'` reads what the
|
|
583
|
+
stream still holds.
|
|
584
|
+
- **A subscription added while the consumer is reading** is read from the next turn, up to
|
|
585
|
+
`blockMs` later. Nothing is missed: its group exists before the bus publishes.
|
|
586
|
+
- **One server or a primary with replicas.** A read covers the streams of a consumer in one
|
|
587
|
+
command and a dead letter touches two keys in one script: on a Redis Cluster the keys must
|
|
588
|
+
share a slot, which a `prefix` in braces gives (`{events}`). Not tested on a cluster.
|
|
589
|
+
- **A crashed process leaves its name** in its groups, and a deduplication mark per event for
|
|
590
|
+
the window, when the guard is on: housekeeping above.
|
|
591
|
+
|
|
592
|
+
## What is not here: a processed-event store
|
|
593
|
+
|
|
594
|
+
There is deliberately no `RedisProcessedEventStore`. The idempotency of a consumer only holds
|
|
595
|
+
if "this event was processed" commits **with** the state change it stands for
|
|
596
|
+
(`events/consumer-idempotent`). A mark in Redis and a change in PostgreSQL are two systems: a
|
|
597
|
+
crash between the two either loses the event (mark kept, change rolled back) or processes it
|
|
598
|
+
twice. Use the store of your database adapter (`KnexProcessedEventStore`,
|
|
599
|
+
`DrizzleProcessedEventStore`), in the unit of work of the consumer.
|
|
600
|
+
|
|
601
|
+
The same reasoning applies to the outbox itself and to anything that must be atomic with
|
|
602
|
+
business data: it lives in the database.
|
|
603
|
+
|
|
604
|
+
## Compatibility: the commands used
|
|
605
|
+
|
|
606
|
+
| Adapter | Commands |
|
|
607
|
+
|---|---|
|
|
608
|
+
| `RedisCache` | `GET`, `SET` (with `PX`), `DEL`, `SCAN` (with `MATCH`, `COUNT`) |
|
|
609
|
+
| `RedisLock` | `EVAL` of two scripts calling `SET` (with `NX`, `PX`), `INCR`, `GET`, `DEL` |
|
|
610
|
+
| `RedisRateLimiter` | `EVAL` of one script calling `ZREMRANGEBYSCORE`, `ZCARD`, `ZADD`, `PEXPIRE`, `ZRANGE` (with `WITHSCORES`) |
|
|
611
|
+
| `RedisRelaySignal` | `PUBLISH`, `SUBSCRIBE` |
|
|
612
|
+
| `RedisStreamsEventBus`, publishing | `MULTI`, `EXEC`, `XADD` (with `MAXLEN ~` or `MINID ~` under `retention`); under `deduplication`, `EVAL` of one script calling `GET`, `XADD`, `SET` (with `PX`) |
|
|
613
|
+
| `RedisStreamsEventBus`, consuming | `XGROUP CREATE` (with `MKSTREAM`), `XREADGROUP` (with `GROUP`, `COUNT`, `BLOCK`, several streams), `XACK`, `XPENDING` (extended form, with `IDLE` and with a consumer), `XCLAIM` (with a minimum idle time, and with `JUSTID`), `XGROUP DELCONSUMER`, `EVAL` of one script calling `XADD`, `XACK` |
|
|
614
|
+
| `RedisStreamsEventBus`, `replayDead` | `XRANGE` (with `COUNT`), `EVAL` of one script calling `XRANGE`, `XADD`, `XDEL` |
|
|
615
|
+
| `redisHealth`, `close` | `PING`, `QUIT` |
|
|
616
|
+
| ioredis itself, at connection | `HELLO 3` (ioredis 6; replies are mapped to their classic shape), `INFO`, `CLIENT SETINFO`, and `AUTH`, `SELECT` when the URL asks |
|
|
617
|
+
|
|
618
|
+
Choices made for portability:
|
|
619
|
+
|
|
620
|
+
- Scripts are sent with `EVAL`, not `EVALSHA`: no script cache to warm or to lose behind a
|
|
621
|
+
proxy or after a failover. They are a few lines each.
|
|
622
|
+
- Every key a script touches is declared in `KEYS`: Dragonfly refuses a script that reaches
|
|
623
|
+
an undeclared key (Valkey runs it), and a cluster needs the keys for routing. The two keys of the lock share a hash tag (`{key}`).
|
|
624
|
+
- Lua is kept to commands and comparisons. No arithmetic on scores, no `redis.call('TIME')`,
|
|
625
|
+
no `cjson`, no `redis.setresp`: Redis and Valkey embed Lua 5.1, Dragonfly Lua 5.4, and they
|
|
626
|
+
differ on numbers. Measured on the two servers above: a script returning `10/4` answers `2`
|
|
627
|
+
on Valkey and `2.5` on Dragonfly, and `tostring(10/2)` is `5` on one and `5.0` on the other.
|
|
628
|
+
- No `KEYS`, no `UNLINK`, no keyspace notifications, no modules, no functions. Streams are used
|
|
629
|
+
by the event bus only.
|
|
630
|
+
- The scripts of the event bus pass a variable number of fields with `unpack(ARGV, n)` and
|
|
631
|
+
test an empty reply with `#`: both behave the same on Lua 5.1 and 5.4, checked on the three
|
|
632
|
+
servers. A dead letter is a script, not a `MULTI`: a transaction would acknowledge the entry
|
|
633
|
+
even if its copy to the dead stream had failed.
|
|
634
|
+
- Redeliveries use `XPENDING` then `XCLAIM`, not `XAUTOCLAIM`. `XAUTOCLAIM` works on the three
|
|
635
|
+
servers, but answers no delivery counter, and the bounded retries need it.
|
|
636
|
+
- The blocking read goes through the named command of ioredis (`client.xreadgroup`), never
|
|
637
|
+
`client.call('XREADGROUP')`: under RESP3, which ioredis 6 speaks, only the commands it knows
|
|
638
|
+
have their reply mapped back to the classic shape.
|
|
639
|
+
- `deleteByPrefix` deletes one key per `DEL`, since a cluster refuses a `DEL` whose keys live
|
|
640
|
+
in different slots.
|
|
641
|
+
|
|
642
|
+
What differs between the servers, as far as the event bus is concerned, and is absorbed:
|
|
643
|
+
|
|
644
|
+
- `XREADGROUP` on several streams answers every stream on Dragonfly, those without a new
|
|
645
|
+
entry with an empty list; Redis and Valkey leave them out.
|
|
646
|
+
- A command on a stream that no longer exists is refused as `NOGROUP` by Redis and Valkey;
|
|
647
|
+
Dragonfly answers `ERR no such key` to `XPENDING`. Both make the reader create its groups
|
|
648
|
+
again; the code reported is `REDIS_NOGROUP` or `REDIS_ERR`.
|
|
649
|
+
- An empty blocking read lasted its `BLOCK` time on Dragonfly and Redis, and was measured at
|
|
650
|
+
about 100 ms more on Valkey.
|
|
651
|
+
|
|
652
|
+
Tested against Dragonfly v2.0.2, Valkey 9.1.2 and Redis 8.10.2 with ioredis 6, same suite,
|
|
653
|
+
same results. The suite needs a server and is skipped without `DDD_TS_KIT_TEST_REDIS_URL`:
|
|
654
|
+
|
|
655
|
+
```
|
|
656
|
+
docker run -d --rm --name ddd-ts-kit-test-redis -p 127.0.0.1:6390:6379 docker.dragonflydb.io/dragonflydb/dragonfly:v2.0.2
|
|
657
|
+
DDD_TS_KIT_TEST_REDIS_URL=redis://127.0.0.1:6390 npx vitest run packages/redis
|
|
658
|
+
```
|
|
659
|
+
|
|
660
|
+
The specs write under a random namespace per test and never flush the server. Those of the
|
|
661
|
+
event bus run the contract suite (with its defaults, then with the publication guard and capped
|
|
662
|
+
streams), the behaviours of the provider (acknowledged publication cut mid-flight, redelivery,
|
|
663
|
+
dead-lettering, replay, poison, competing and separate consumers, takeover after the visibility
|
|
664
|
+
timeout, groups lost and created again, reconnection through a TCP relay the test cuts,
|
|
665
|
+
graceful close) and the whole chain from the outbox to an idempotent consumer. Each test works
|
|
666
|
+
under a prefix of its own and deletes its keys.
|
|
667
|
+
|
|
668
|
+
## Contract suites
|
|
669
|
+
|
|
670
|
+
Each port has a suite that its in-memory implementation and its adapter both pass. An adapter
|
|
671
|
+
of your own (another client library, another store) is held to the same:
|
|
672
|
+
|
|
673
|
+
```ts
|
|
674
|
+
import { cacheContract, lockContract, rateLimiterContract } from '@ddd-ts-kit/testing/contracts';
|
|
675
|
+
|
|
676
|
+
const runner = { describe, it, expect, beforeEach };
|
|
677
|
+
cacheContract(runner, { name: 'MyCache', createCache: () => new MyCache(), elapse: (ms) => sleep(ms) });
|
|
678
|
+
lockContract(runner, { name: 'MyLock', createLock: () => new MyLock(), elapse: (ms) => sleep(ms) });
|
|
679
|
+
rateLimiterContract(runner, { name: 'MyLimiter', createLimiter: () => new MyLimiter(clock), elapse: (ms) => clock.advance(ms) });
|
|
680
|
+
```
|
|
681
|
+
|
|
682
|
+
`elapse` makes time pass for the subject: advance the clock it reads, or really wait when the
|
|
683
|
+
time is the server's. The suites live in `@ddd-ts-kit/testing/contracts`, with those of the
|
|
684
|
+
other kernel ports, where the in-memory implementations of the kernel are run against them;
|
|
685
|
+
`@ddd-ts-kit/redis/testing` re-exports them.
|
|
686
|
+
|
|
687
|
+
## Limits
|
|
688
|
+
|
|
689
|
+
- **One server or a primary with replicas.** A Redis Cluster works for the cache (one key per
|
|
690
|
+
command) and the lock (hash tag); `deleteByPrefix` would have to scan every node, which it
|
|
691
|
+
does not. Not tested on a cluster.
|
|
692
|
+
- **Eviction applies to everything.** A server configured to evict (`allkeys-lru`) may evict
|
|
693
|
+
a lock or a rate-limit key as well as a cache entry. Use a `volatile-*` policy, or a second
|
|
694
|
+
server for what is not a cache.
|
|
695
|
+
- **No client-side caching, no pipelining of `get`**: each read is a round trip.
|
|
696
|
+
- **Values are JSON by default**: `undefined`, `Date`, `Map`, `bigint` and class instances do
|
|
697
|
+
not survive. Give a codec, or cache plain data.
|
|
698
|
+
- **The lock is not re-entrant and is not renewed**: choose `ttlMs` above the worst case of
|
|
699
|
+
the work, or split the work.
|
|
700
|
+
- **ioredis 5** is accepted as a peer; the suites were only run with ioredis 6.
|