@nestjs-redis/streams-transporter 1.0.0-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +176 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/lib/redis.events.d.ts +34 -0
- package/dist/lib/redis.events.d.ts.map +1 -0
- package/dist/lib/redis.events.js +20 -0
- package/dist/lib/streams-transporter.client.d.ts +29 -0
- package/dist/lib/streams-transporter.client.d.ts.map +1 -0
- package/dist/lib/streams-transporter.client.js +212 -0
- package/dist/lib/streams-transporter.context.d.ts +14 -0
- package/dist/lib/streams-transporter.context.d.ts.map +1 -0
- package/dist/lib/streams-transporter.context.js +25 -0
- package/dist/lib/streams-transporter.options.d.ts +15 -0
- package/dist/lib/streams-transporter.options.d.ts.map +1 -0
- package/dist/lib/streams-transporter.options.js +24 -0
- package/dist/lib/streams-transporter.server.d.ts +46 -0
- package/dist/lib/streams-transporter.server.d.ts.map +1 -0
- package/dist/lib/streams-transporter.server.js +256 -0
- package/dist/lib/types.d.ts +21 -0
- package/dist/lib/types.d.ts.map +1 -0
- package/dist/lib/types.js +14 -0
- package/package.json +53 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Saba Pochkhua
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<img src="https://raw.githubusercontent.com/CSenshi/nestjs-redis/main/docs/images/logo.png" alt="NestJS Redis Toolkit Logo" width="200" height="200">
|
|
4
|
+
|
|
5
|
+
# @nestjs-redis/streams-transporter
|
|
6
|
+
|
|
7
|
+
Custom NestJS microservices transporter using Redis Streams with event and request/response patterns
|
|
8
|
+
|
|
9
|
+
[](https://www.npmjs.com/package/@nestjs-redis/streams-transporter)
|
|
10
|
+
[](https://www.npmjs.com/package/@nestjs-redis/streams-transporter)
|
|
11
|
+
[](https://opensource.org/licenses/MIT)
|
|
12
|
+
[](https://www.typescriptlang.org/)
|
|
13
|
+
[](https://nestjs.com/) [](https://redis.io/)
|
|
14
|
+
|
|
15
|
+
</div>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Features
|
|
20
|
+
|
|
21
|
+
- **Redis Streams–based transport**: Events and requests stored as stream entries; replies written to per-client reply streams
|
|
22
|
+
- **Event and request/response**: Fire-and-forget events (`dispatchEvent`) and request/response via `send()` with routing callbacks
|
|
23
|
+
- **Consumer groups**: Server uses `XREADGROUP` + `XACK` for at-least-once delivery and horizontal scaling
|
|
24
|
+
- **Configurable options**: Stream prefix, consumer group/name, block timeout, batch size, max stream length (MAXLEN trim), retry delay
|
|
25
|
+
- **NestJS integration**: `RedisStreamsContext` (stream name, message id, consumer group/name) passed to handlers; optional `onProcessingStartHook` / `onProcessingEndHook`
|
|
26
|
+
- **Type-safe**: Event/request/response type guards and resolved options
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
npm install @nestjs-redis/streams-transporter redis
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Quick Start
|
|
35
|
+
|
|
36
|
+
### Server (microservice)
|
|
37
|
+
|
|
38
|
+
```typescript
|
|
39
|
+
// main.ts
|
|
40
|
+
import { NestFactory } from '@nestjs/core';
|
|
41
|
+
import { MicroserviceOptions } from '@nestjs/microservices';
|
|
42
|
+
import { RedisStreamServer } from '@nestjs-redis/streams-transporter';
|
|
43
|
+
import { AppModule } from './app.module';
|
|
44
|
+
|
|
45
|
+
async function bootstrap() {
|
|
46
|
+
const app = await NestFactory.createMicroservice<MicroserviceOptions>(
|
|
47
|
+
AppModule,
|
|
48
|
+
{
|
|
49
|
+
strategy: new RedisStreamServer({
|
|
50
|
+
url: 'redis://localhost:6379',
|
|
51
|
+
streamPrefix: '_microservices',
|
|
52
|
+
consumerGroup: 'nestjs-streams',
|
|
53
|
+
consumerName: 'my-consumer',
|
|
54
|
+
}),
|
|
55
|
+
},
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
await app.listen();
|
|
59
|
+
}
|
|
60
|
+
bootstrap();
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Client (hybrid app or separate service)
|
|
64
|
+
|
|
65
|
+
```typescript
|
|
66
|
+
// app.module.ts
|
|
67
|
+
import { Module } from '@nestjs/common';
|
|
68
|
+
import { ClientsModule, Transport } from '@nestjs/microservices';
|
|
69
|
+
import { RedisStreamClient } from '@nestjs-redis/streams-transporter';
|
|
70
|
+
import { AppController } from './app.controller';
|
|
71
|
+
import { AppService } from './app.service';
|
|
72
|
+
|
|
73
|
+
@Module({
|
|
74
|
+
imports: [
|
|
75
|
+
ClientsModule.register([
|
|
76
|
+
{
|
|
77
|
+
name: 'STREAMS_SERVICE',
|
|
78
|
+
customClass: RedisStreamClient,
|
|
79
|
+
options: {
|
|
80
|
+
url: 'redis://localhost:6379',
|
|
81
|
+
streamPrefix: '_microservices',
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
]),
|
|
85
|
+
],
|
|
86
|
+
controllers: [AppController],
|
|
87
|
+
providers: [AppService],
|
|
88
|
+
})
|
|
89
|
+
export class AppModule {}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
```typescript
|
|
93
|
+
// app.controller.ts
|
|
94
|
+
import { Controller, Get, Inject } from '@nestjs/common';
|
|
95
|
+
import { ClientProxy } from '@nestjs/microservices';
|
|
96
|
+
import { firstValueFrom } from 'rxjs';
|
|
97
|
+
|
|
98
|
+
@Controller()
|
|
99
|
+
export class AppController {
|
|
100
|
+
constructor(
|
|
101
|
+
@Inject('STREAMS_SERVICE') private readonly client: ClientProxy,
|
|
102
|
+
) {}
|
|
103
|
+
|
|
104
|
+
@Get('echo')
|
|
105
|
+
async echo() {
|
|
106
|
+
return firstValueFrom(this.client.send('user.echo', { hello: 'world' }));
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Event handlers (server)
|
|
112
|
+
|
|
113
|
+
```typescript
|
|
114
|
+
// app.controller.ts
|
|
115
|
+
import { Controller } from '@nestjs/common';
|
|
116
|
+
import { EventPattern, MessagePattern, Payload } from '@nestjs/microservices';
|
|
117
|
+
import { Ctx } from '@nestjs/microservices';
|
|
118
|
+
import { RedisStreamsContext } from '@nestjs-redis/streams-transporter';
|
|
119
|
+
|
|
120
|
+
@Controller()
|
|
121
|
+
export class AppController {
|
|
122
|
+
@MessagePattern('user.echo')
|
|
123
|
+
echo(@Payload() data: object, @Ctx() ctx: RedisStreamsContext) {
|
|
124
|
+
return { ok: true, data };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
@EventPattern('user.created')
|
|
128
|
+
onUserCreated(@Payload() data: object, @Ctx() ctx: RedisStreamsContext) {
|
|
129
|
+
// fire-and-forget; no reply
|
|
130
|
+
console.log('User created', data, ctx.getStreamName(), ctx.getMessageId());
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## Options
|
|
136
|
+
|
|
137
|
+
`RedisStreamServer` and `RedisStreamClient` accept `RedisStreamsOptions`, which extends Redis client options and adds:
|
|
138
|
+
|
|
139
|
+
| Option | Default | Description |
|
|
140
|
+
| ----------------- | ------------------------------------- | ------------------------------------------------------------------------- |
|
|
141
|
+
| `streamPrefix` | `'_microservices'` | Prefix for request streams (e.g. `{prefix}:user.echo`) and reply streams. |
|
|
142
|
+
| `consumerGroup` | `'nestjs-streams'` | Consumer group name for server `XREADGROUP`. |
|
|
143
|
+
| `consumerName` | `''` (then `consumer-${process.pid}`) | Consumer name in the group. |
|
|
144
|
+
| `blockTimeout` | `100` | Block timeout (ms) for `XREAD` / `XREADGROUP`. |
|
|
145
|
+
| `batchSize` | `50` | Max entries per read (`COUNT`). |
|
|
146
|
+
| `maxStreamLength` | `10000` | Max length for streams; `XADD ... TRIM MAXLEN ~` is used on add. |
|
|
147
|
+
| `retryDelay` | `250` | Delay (ms) before retrying after a read/connection error. |
|
|
148
|
+
|
|
149
|
+
Use `resolveRedisStreamsOptions(options)` to get a fully resolved options object (all optional fields filled with defaults).
|
|
150
|
+
|
|
151
|
+
## API
|
|
152
|
+
|
|
153
|
+
- **`RedisStreamClient`** – NestJS `ClientProxy` implementation. Connects to Redis, publishes events/requests to streams, listens for replies on a dedicated reply stream and dispatches to `routingMap` callbacks. `close()` flushes callbacks with an error, deletes the reply stream, and quits the client.
|
|
154
|
+
- **`RedisStreamServer`** – NestJS `CustomTransportStrategy`. Creates consumer groups, consumes via `XREADGROUP`, ACKs with `XACK`, invokes message/event handlers with `RedisStreamsContext`, and writes replies to the client’s reply stream.
|
|
155
|
+
- **`RedisStreamsContext`** – Context passed to handlers (like Nest’s RPC context). Methods: `getStreamName()`, `getMessageId()`, `getConsumerGroup()`, `getConsumerName()`.
|
|
156
|
+
- **`RedisStreamsOptions`** / **`RedisStreamsResolvedOptions`** – Options and resolved type; **`resolveRedisStreamsOptions(options)`** – returns resolved options.
|
|
157
|
+
|
|
158
|
+
Stream entry shape:
|
|
159
|
+
|
|
160
|
+
- **Events**: `e: '1'`, `data` (JSON).
|
|
161
|
+
- **Requests**: `e: '0'`, `id`, `replyTo`, `data` (JSON).
|
|
162
|
+
- **Responses**: `id`, `data` or `err` (JSON), `isDisposed: '1'`.
|
|
163
|
+
|
|
164
|
+
## Links
|
|
165
|
+
|
|
166
|
+
- Root repo: [CSenshi/nestjs-redis](https://github.com/CSenshi/nestjs-redis)
|
|
167
|
+
- Issues: [GitHub Issues](https://github.com/CSenshi/nestjs-redis/issues)
|
|
168
|
+
- Discussions: [GitHub Discussions](https://github.com/CSenshi/nestjs-redis/discussions)
|
|
169
|
+
|
|
170
|
+
## Contributing
|
|
171
|
+
|
|
172
|
+
Please see the [root contributing guidelines](https://github.com/CSenshi/nestjs-redis#contributing).
|
|
173
|
+
|
|
174
|
+
## License
|
|
175
|
+
|
|
176
|
+
MIT © [CSenshi](https://github.com/CSenshi)
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export * from './lib/redis.events';
|
|
2
|
+
export * from './lib/streams-transporter.client';
|
|
3
|
+
export * from './lib/streams-transporter.context';
|
|
4
|
+
export * from './lib/streams-transporter.options';
|
|
5
|
+
export * from './lib/streams-transporter.server';
|
|
6
|
+
export * from './lib/types';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAClD,cAAc,mCAAmC,CAAC;AAClD,cAAc,kCAAkC,CAAC;AACjD,cAAc,aAAa,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const tslib_1 = require("tslib");
|
|
4
|
+
tslib_1.__exportStar(require("./lib/redis.events"), exports);
|
|
5
|
+
tslib_1.__exportStar(require("./lib/streams-transporter.client"), exports);
|
|
6
|
+
tslib_1.__exportStar(require("./lib/streams-transporter.context"), exports);
|
|
7
|
+
tslib_1.__exportStar(require("./lib/streams-transporter.options"), exports);
|
|
8
|
+
tslib_1.__exportStar(require("./lib/streams-transporter.server"), exports);
|
|
9
|
+
tslib_1.__exportStar(require("./lib/types"), exports);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
type VoidCallback = () => void;
|
|
2
|
+
type OnErrorCallback = (error: Error) => void;
|
|
3
|
+
type OnWarningCallback = (warning: any) => void;
|
|
4
|
+
export declare const enum RedisStatus {
|
|
5
|
+
CONNECT = "connect",
|
|
6
|
+
DISCONNECTED = "disconnected",
|
|
7
|
+
RECONNECTING = "reconnecting",
|
|
8
|
+
CONNECTED = "connected"
|
|
9
|
+
}
|
|
10
|
+
export declare const enum RedisEventsMap {
|
|
11
|
+
CONNECT = "connect",
|
|
12
|
+
READY = "ready",
|
|
13
|
+
ERROR = "error",
|
|
14
|
+
CLOSE = "close",
|
|
15
|
+
RECONNECTING = "reconnecting",
|
|
16
|
+
END = "end",
|
|
17
|
+
WARNING = "warning"
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Redis events map for the Redis client.
|
|
21
|
+
* Key is the event name and value is the corresponding callback function.
|
|
22
|
+
* @publicApi
|
|
23
|
+
*/
|
|
24
|
+
export type RedisEvents = {
|
|
25
|
+
connect: VoidCallback;
|
|
26
|
+
ready: VoidCallback;
|
|
27
|
+
error: OnErrorCallback;
|
|
28
|
+
close: VoidCallback;
|
|
29
|
+
reconnecting: VoidCallback;
|
|
30
|
+
end: VoidCallback;
|
|
31
|
+
warning: OnWarningCallback;
|
|
32
|
+
};
|
|
33
|
+
export {};
|
|
34
|
+
//# sourceMappingURL=redis.events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.events.d.ts","sourceRoot":"","sources":["../../src/lib/redis.events.ts"],"names":[],"mappings":"AAAA,KAAK,YAAY,GAAG,MAAM,IAAI,CAAC;AAC/B,KAAK,eAAe,GAAG,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;AAC9C,KAAK,iBAAiB,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,CAAC;AAEhD,0BAAkB,WAAW;IAC3B,OAAO,YAAY;IACnB,YAAY,iBAAiB;IAC7B,YAAY,iBAAiB;IAC7B,SAAS,cAAc;CACxB;AAED,0BAAkB,cAAc;IAC9B,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,YAAY,iBAAiB;IAC7B,GAAG,QAAQ;IACX,OAAO,YAAY;CACpB;AACD;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,eAAe,CAAC;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,YAAY,EAAE,YAAY,CAAC;IAC3B,GAAG,EAAE,YAAY,CAAC;IAClB,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAAC;AACF,OAAO,EAAE,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisEventsMap = exports.RedisStatus = void 0;
|
|
4
|
+
var RedisStatus;
|
|
5
|
+
(function (RedisStatus) {
|
|
6
|
+
RedisStatus["CONNECT"] = "connect";
|
|
7
|
+
RedisStatus["DISCONNECTED"] = "disconnected";
|
|
8
|
+
RedisStatus["RECONNECTING"] = "reconnecting";
|
|
9
|
+
RedisStatus["CONNECTED"] = "connected";
|
|
10
|
+
})(RedisStatus || (exports.RedisStatus = RedisStatus = {}));
|
|
11
|
+
var RedisEventsMap;
|
|
12
|
+
(function (RedisEventsMap) {
|
|
13
|
+
RedisEventsMap["CONNECT"] = "connect";
|
|
14
|
+
RedisEventsMap["READY"] = "ready";
|
|
15
|
+
RedisEventsMap["ERROR"] = "error";
|
|
16
|
+
RedisEventsMap["CLOSE"] = "close";
|
|
17
|
+
RedisEventsMap["RECONNECTING"] = "reconnecting";
|
|
18
|
+
RedisEventsMap["END"] = "end";
|
|
19
|
+
RedisEventsMap["WARNING"] = "warning";
|
|
20
|
+
})(RedisEventsMap || (exports.RedisEventsMap = RedisEventsMap = {}));
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { Logger } from '@nestjs/common';
|
|
2
|
+
import { ClientProxy, ReadPacket, WritePacket } from '@nestjs/microservices';
|
|
3
|
+
import { RedisClientType, createClient } from 'redis';
|
|
4
|
+
import { type RedisEvents, RedisStatus } from './redis.events';
|
|
5
|
+
import { RedisStreamsOptions } from './streams-transporter.options';
|
|
6
|
+
export declare class RedisStreamClient extends ClientProxy<RedisEvents, RedisStatus> {
|
|
7
|
+
protected readonly logger: Logger;
|
|
8
|
+
private client;
|
|
9
|
+
protected connectionPromise: Promise<ReturnType<typeof createClient>> | null;
|
|
10
|
+
private readonly clientId;
|
|
11
|
+
private replyStreamName;
|
|
12
|
+
private isListening;
|
|
13
|
+
private replyListenerPromise;
|
|
14
|
+
private readonly options;
|
|
15
|
+
private pendingEventListeners;
|
|
16
|
+
constructor(options?: RedisStreamsOptions);
|
|
17
|
+
connect(): Promise<ReturnType<typeof createClient>>;
|
|
18
|
+
private registerEventListeners;
|
|
19
|
+
close(): Promise<void>;
|
|
20
|
+
dispatchEvent(packet: ReadPacket): Promise<any>;
|
|
21
|
+
publish(packet: ReadPacket, callback: (packet: WritePacket) => void): () => void;
|
|
22
|
+
getRequestPattern(pattern: string): string;
|
|
23
|
+
on<EventKey extends keyof RedisEvents = keyof RedisEvents, EventCallback extends RedisEvents[EventKey] = RedisEvents[EventKey]>(event: EventKey, callback: EventCallback): void;
|
|
24
|
+
private listenForReplies;
|
|
25
|
+
private handleReplyMessage;
|
|
26
|
+
private parseMessage;
|
|
27
|
+
unwrap<T = RedisClientType>(): T;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=streams-transporter.client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streams-transporter.client.d.ts","sourceRoot":"","sources":["../../src/lib/streams-transporter.client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAE7E,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,OAAO,CAAC;AACtD,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EACL,mBAAmB,EAGpB,MAAM,+BAA+B,CAAC;AAIvC,qBAAa,iBAAkB,SAAQ,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC;IAC1E,SAAS,CAAC,QAAQ,CAAC,MAAM,SAAsC;IAC/D,OAAO,CAAC,MAAM,CACP;IACP,SAAS,CAAC,iBAAiB,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,GAAG,IAAI,CACrE;IACP,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4B;IACrD,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,oBAAoB,CAA8B;IAC1D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8B;IACtD,OAAO,CAAC,qBAAqB,CAGrB;gBAEI,OAAO,GAAE,mBAAwB;IAQvC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;IAgBzD,OAAO,CAAC,sBAAsB;IA2BxB,KAAK;IAsBL,aAAa,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC;IAuBrD,OAAO,CACL,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GACtC,MAAM,IAAI;IA6CN,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAIjC,EAAE,CAChB,QAAQ,SAAS,MAAM,WAAW,GAAG,MAAM,WAAW,EACtD,aAAa,SAAS,WAAW,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,EACnE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,GAAG,IAAI;YAQnC,gBAAgB;IAiC9B,OAAO,CAAC,kBAAkB;IAwB1B,OAAO,CAAC,YAAY;IAQpB,MAAM,CAAC,CAAC,GAAG,eAAe,KAAK,CAAC;CASjC"}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisStreamClient = void 0;
|
|
4
|
+
const common_1 = require("@nestjs/common");
|
|
5
|
+
const microservices_1 = require("@nestjs/microservices");
|
|
6
|
+
const crypto_1 = require("crypto");
|
|
7
|
+
const redis_1 = require("redis");
|
|
8
|
+
const redis_events_1 = require("./redis.events");
|
|
9
|
+
const streams_transporter_options_1 = require("./streams-transporter.options");
|
|
10
|
+
const types_1 = require("./types");
|
|
11
|
+
class RedisStreamClient extends microservices_1.ClientProxy {
|
|
12
|
+
constructor(options = {}) {
|
|
13
|
+
super();
|
|
14
|
+
this.logger = new common_1.Logger(RedisStreamClient.name);
|
|
15
|
+
this.client = null;
|
|
16
|
+
this.connectionPromise = null;
|
|
17
|
+
this.clientId = `client-${(0, crypto_1.randomUUID)()}`;
|
|
18
|
+
this.replyStreamName = '';
|
|
19
|
+
this.isListening = false;
|
|
20
|
+
this.replyListenerPromise = null;
|
|
21
|
+
this.pendingEventListeners = [];
|
|
22
|
+
this.options = (0, streams_transporter_options_1.resolveRedisStreamsOptions)(options);
|
|
23
|
+
this.replyStreamName = `${this.options.streamPrefix}:reply:${this.clientId}`;
|
|
24
|
+
this.initializeSerializer({});
|
|
25
|
+
this.initializeDeserializer({});
|
|
26
|
+
}
|
|
27
|
+
async connect() {
|
|
28
|
+
if (this.connectionPromise) {
|
|
29
|
+
return this.connectionPromise;
|
|
30
|
+
}
|
|
31
|
+
this.client = (0, redis_1.createClient)(this.options);
|
|
32
|
+
this.registerEventListeners();
|
|
33
|
+
this.connectionPromise = this.client.connect();
|
|
34
|
+
await this.connectionPromise;
|
|
35
|
+
this.isListening = true;
|
|
36
|
+
this.replyListenerPromise = this.listenForReplies();
|
|
37
|
+
return this.connectionPromise;
|
|
38
|
+
}
|
|
39
|
+
registerEventListeners() {
|
|
40
|
+
if (!this.client)
|
|
41
|
+
return;
|
|
42
|
+
this.client.on('error', (err) => this.logger.error(err));
|
|
43
|
+
this.client.on('connect', () => {
|
|
44
|
+
this._status$.next(redis_events_1.RedisStatus.CONNECT);
|
|
45
|
+
});
|
|
46
|
+
this.client.on('ready', () => {
|
|
47
|
+
this._status$.next(redis_events_1.RedisStatus.CONNECTED);
|
|
48
|
+
});
|
|
49
|
+
this.client.on('reconnecting', () => {
|
|
50
|
+
this._status$.next(redis_events_1.RedisStatus.RECONNECTING);
|
|
51
|
+
});
|
|
52
|
+
this.client.on('end', () => {
|
|
53
|
+
this._status$.next(redis_events_1.RedisStatus.DISCONNECTED);
|
|
54
|
+
});
|
|
55
|
+
if (this.pendingEventListeners.length > 0) {
|
|
56
|
+
this.pendingEventListeners.forEach(({ event, callback }) => {
|
|
57
|
+
if (!this.client)
|
|
58
|
+
return;
|
|
59
|
+
this.client.on(event, callback);
|
|
60
|
+
});
|
|
61
|
+
this.pendingEventListeners = [];
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async close() {
|
|
65
|
+
this.isListening = false;
|
|
66
|
+
if (this.replyListenerPromise) {
|
|
67
|
+
await this.replyListenerPromise;
|
|
68
|
+
}
|
|
69
|
+
if (this.client) {
|
|
70
|
+
for (const callback of this.routingMap.values()) {
|
|
71
|
+
callback({ err: new Error('Client closed'), isDisposed: true });
|
|
72
|
+
}
|
|
73
|
+
this.routingMap.clear();
|
|
74
|
+
try {
|
|
75
|
+
await this.client.del(this.replyStreamName);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
// ignore
|
|
79
|
+
}
|
|
80
|
+
await this.client.quit();
|
|
81
|
+
}
|
|
82
|
+
this.client = null;
|
|
83
|
+
this.connectionPromise = null;
|
|
84
|
+
this.pendingEventListeners = [];
|
|
85
|
+
}
|
|
86
|
+
async dispatchEvent(packet) {
|
|
87
|
+
if (!this.client) {
|
|
88
|
+
throw new Error('Client not connected. Call connect() first.');
|
|
89
|
+
}
|
|
90
|
+
const pattern = this.normalizePattern(packet.pattern);
|
|
91
|
+
const streamName = this.getRequestPattern(pattern);
|
|
92
|
+
const serializedPacket = this.serializer.serialize(packet);
|
|
93
|
+
const data = {
|
|
94
|
+
e: '1',
|
|
95
|
+
data: JSON.stringify(serializedPacket.data),
|
|
96
|
+
};
|
|
97
|
+
await this.client.xAdd(streamName, '*', data, {
|
|
98
|
+
TRIM: {
|
|
99
|
+
strategy: 'MAXLEN',
|
|
100
|
+
strategyModifier: '~',
|
|
101
|
+
threshold: this.options.maxStreamLength,
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
publish(packet, callback) {
|
|
106
|
+
if (!this.client) {
|
|
107
|
+
throw new Error('Client not connected. Call connect() first.');
|
|
108
|
+
}
|
|
109
|
+
const requestPacket = this.assignPacketId(packet);
|
|
110
|
+
const cleanup = () => this.routingMap.delete(requestPacket.id);
|
|
111
|
+
try {
|
|
112
|
+
const pattern = this.normalizePattern(requestPacket.pattern);
|
|
113
|
+
const streamName = this.getRequestPattern(pattern);
|
|
114
|
+
const serializedPacket = this.serializer.serialize(requestPacket);
|
|
115
|
+
this.routingMap.set(requestPacket.id, callback);
|
|
116
|
+
void this.client
|
|
117
|
+
.xAdd(streamName, '*', {
|
|
118
|
+
e: '0',
|
|
119
|
+
id: requestPacket.id,
|
|
120
|
+
replyTo: this.replyStreamName,
|
|
121
|
+
data: JSON.stringify(serializedPacket.data),
|
|
122
|
+
}, {
|
|
123
|
+
TRIM: {
|
|
124
|
+
strategy: 'MAXLEN',
|
|
125
|
+
strategyModifier: '~',
|
|
126
|
+
threshold: this.options.maxStreamLength,
|
|
127
|
+
},
|
|
128
|
+
})
|
|
129
|
+
.catch((err) => {
|
|
130
|
+
cleanup();
|
|
131
|
+
callback({ err });
|
|
132
|
+
});
|
|
133
|
+
return cleanup;
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
callback({ err });
|
|
137
|
+
return () => void 0;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
getRequestPattern(pattern) {
|
|
141
|
+
return `${this.options.streamPrefix}:${pattern}`;
|
|
142
|
+
}
|
|
143
|
+
on(event, callback) {
|
|
144
|
+
if (this.client) {
|
|
145
|
+
this.client.on(event, callback);
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
this.pendingEventListeners.push({ event, callback });
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
async listenForReplies() {
|
|
152
|
+
if (!this.client)
|
|
153
|
+
return;
|
|
154
|
+
let lastId = '0';
|
|
155
|
+
while (this.isListening && this.client) {
|
|
156
|
+
try {
|
|
157
|
+
const result = await this.client.xRead({ key: this.replyStreamName, id: lastId }, { BLOCK: this.options.blockTimeout, COUNT: this.options.batchSize });
|
|
158
|
+
if (Array.isArray(result) && result.length > 0) {
|
|
159
|
+
const streams = result;
|
|
160
|
+
for (const streamData of streams) {
|
|
161
|
+
for (const message of streamData.messages) {
|
|
162
|
+
lastId = message.id;
|
|
163
|
+
this.handleReplyMessage(message.message);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
if (this.isListening) {
|
|
170
|
+
this.logger.error(err);
|
|
171
|
+
await new Promise((resolve) => setTimeout(resolve, this.options.retryDelay));
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
handleReplyMessage(rawMessage) {
|
|
177
|
+
if (!(0, types_1.isResponsePacket)(rawMessage))
|
|
178
|
+
return;
|
|
179
|
+
const id = rawMessage.id;
|
|
180
|
+
const callback = this.routingMap.get(id);
|
|
181
|
+
if (!callback)
|
|
182
|
+
return;
|
|
183
|
+
if (rawMessage.err) {
|
|
184
|
+
this.routingMap.delete(id);
|
|
185
|
+
callback({ err: this.parseMessage(rawMessage.err) });
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
const response = rawMessage.data !== undefined
|
|
189
|
+
? this.parseMessage(rawMessage.data)
|
|
190
|
+
: undefined;
|
|
191
|
+
const isDisposed = rawMessage.isDisposed === '1';
|
|
192
|
+
callback({ response, isDisposed });
|
|
193
|
+
if (isDisposed) {
|
|
194
|
+
this.routingMap.delete(id);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
parseMessage(content) {
|
|
198
|
+
try {
|
|
199
|
+
return JSON.parse(content);
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
return content;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
unwrap() {
|
|
206
|
+
if (!this.client) {
|
|
207
|
+
throw new Error('Not initialized. Please call the "connect" method first.');
|
|
208
|
+
}
|
|
209
|
+
return this.client;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
exports.RedisStreamClient = RedisStreamClient;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { BaseRpcContext } from '@nestjs/microservices';
|
|
2
|
+
type RedisStreamsContextArgs = [string, string, string, string];
|
|
3
|
+
/**
|
|
4
|
+
* @publicApi
|
|
5
|
+
*/
|
|
6
|
+
export declare class RedisStreamsContext extends BaseRpcContext<RedisStreamsContextArgs> {
|
|
7
|
+
constructor(args: RedisStreamsContextArgs);
|
|
8
|
+
getStreamName(): string;
|
|
9
|
+
getMessageId(): string;
|
|
10
|
+
getConsumerGroup(): string;
|
|
11
|
+
getConsumerName(): string;
|
|
12
|
+
}
|
|
13
|
+
export {};
|
|
14
|
+
//# sourceMappingURL=streams-transporter.context.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streams-transporter.context.d.ts","sourceRoot":"","sources":["../../src/lib/streams-transporter.context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,KAAK,uBAAuB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhE;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,cAAc,CAAC,uBAAuB,CAAC;gBAClE,IAAI,EAAE,uBAAuB;IAIzC,aAAa;IAIb,YAAY;IAIZ,gBAAgB;IAIhB,eAAe;CAGhB"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisStreamsContext = void 0;
|
|
4
|
+
const microservices_1 = require("@nestjs/microservices");
|
|
5
|
+
/**
|
|
6
|
+
* @publicApi
|
|
7
|
+
*/
|
|
8
|
+
class RedisStreamsContext extends microservices_1.BaseRpcContext {
|
|
9
|
+
constructor(args) {
|
|
10
|
+
super(args);
|
|
11
|
+
}
|
|
12
|
+
getStreamName() {
|
|
13
|
+
return this.args[0];
|
|
14
|
+
}
|
|
15
|
+
getMessageId() {
|
|
16
|
+
return this.args[1];
|
|
17
|
+
}
|
|
18
|
+
getConsumerGroup() {
|
|
19
|
+
return this.args[2];
|
|
20
|
+
}
|
|
21
|
+
getConsumerName() {
|
|
22
|
+
return this.args[3];
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
exports.RedisStreamsContext = RedisStreamsContext;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { RedisClientOptions } from 'redis';
|
|
2
|
+
export interface RedisStreamsOptions extends RedisClientOptions {
|
|
3
|
+
streamPrefix?: string;
|
|
4
|
+
consumerGroup?: string;
|
|
5
|
+
consumerName?: string;
|
|
6
|
+
blockTimeout?: number;
|
|
7
|
+
batchSize?: number;
|
|
8
|
+
replyStreamTTL?: number;
|
|
9
|
+
maxStreamLength?: number;
|
|
10
|
+
retryDelay?: number;
|
|
11
|
+
}
|
|
12
|
+
export declare const REDIS_STREAMS_DEFAULT_OPTIONS: Required<Pick<RedisStreamsOptions, 'streamPrefix' | 'consumerGroup' | 'consumerName' | 'blockTimeout' | 'batchSize' | 'maxStreamLength' | 'retryDelay'>>;
|
|
13
|
+
export type RedisStreamsResolvedOptions = RedisStreamsOptions & Required<Pick<RedisStreamsOptions, keyof typeof REDIS_STREAMS_DEFAULT_OPTIONS>>;
|
|
14
|
+
export declare const resolveRedisStreamsOptions: (options?: RedisStreamsOptions) => RedisStreamsResolvedOptions;
|
|
15
|
+
//# sourceMappingURL=streams-transporter.options.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streams-transporter.options.d.ts","sourceRoot":"","sources":["../../src/lib/streams-transporter.options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAEhD,MAAM,WAAW,mBAAoB,SAAQ,kBAAkB;IAC7D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAClD,IAAI,CACF,mBAAmB,EACjB,cAAc,GACd,eAAe,GACf,cAAc,GACd,cAAc,GACd,WAAW,GACX,iBAAiB,GACjB,YAAY,CACf,CASF,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,mBAAmB,GAC3D,QAAQ,CACN,IAAI,CAAC,mBAAmB,EAAE,MAAM,OAAO,6BAA6B,CAAC,CACtE,CAAC;AAEJ,eAAO,MAAM,0BAA0B,GACrC,UAAS,mBAAwB,KAChC,2BAeD,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolveRedisStreamsOptions = exports.REDIS_STREAMS_DEFAULT_OPTIONS = void 0;
|
|
4
|
+
exports.REDIS_STREAMS_DEFAULT_OPTIONS = {
|
|
5
|
+
streamPrefix: '_microservices',
|
|
6
|
+
consumerGroup: 'nestjs-streams',
|
|
7
|
+
consumerName: '',
|
|
8
|
+
blockTimeout: 100,
|
|
9
|
+
batchSize: 50,
|
|
10
|
+
maxStreamLength: 10000,
|
|
11
|
+
retryDelay: 250,
|
|
12
|
+
};
|
|
13
|
+
const resolveRedisStreamsOptions = (options = {}) => ({
|
|
14
|
+
...options,
|
|
15
|
+
...exports.REDIS_STREAMS_DEFAULT_OPTIONS,
|
|
16
|
+
streamPrefix: options.streamPrefix ?? exports.REDIS_STREAMS_DEFAULT_OPTIONS.streamPrefix,
|
|
17
|
+
consumerGroup: options.consumerGroup ?? exports.REDIS_STREAMS_DEFAULT_OPTIONS.consumerGroup,
|
|
18
|
+
consumerName: options.consumerName ?? exports.REDIS_STREAMS_DEFAULT_OPTIONS.consumerName,
|
|
19
|
+
blockTimeout: options.blockTimeout ?? exports.REDIS_STREAMS_DEFAULT_OPTIONS.blockTimeout,
|
|
20
|
+
batchSize: options.batchSize ?? exports.REDIS_STREAMS_DEFAULT_OPTIONS.batchSize,
|
|
21
|
+
maxStreamLength: options.maxStreamLength ?? exports.REDIS_STREAMS_DEFAULT_OPTIONS.maxStreamLength,
|
|
22
|
+
retryDelay: options.retryDelay ?? exports.REDIS_STREAMS_DEFAULT_OPTIONS.retryDelay,
|
|
23
|
+
});
|
|
24
|
+
exports.resolveRedisStreamsOptions = resolveRedisStreamsOptions;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { CustomTransportStrategy, Server } from '@nestjs/microservices';
|
|
2
|
+
import { type RedisClientType } from 'redis';
|
|
3
|
+
import { RedisEvents, RedisStatus } from './redis.events';
|
|
4
|
+
import { RedisStreamsOptions } from './streams-transporter.options';
|
|
5
|
+
export declare class RedisStreamServer extends Server<RedisEvents, RedisStatus> implements CustomTransportStrategy {
|
|
6
|
+
private client;
|
|
7
|
+
private isConsuming;
|
|
8
|
+
private consumePromise;
|
|
9
|
+
private lastIds;
|
|
10
|
+
private consumerGroup;
|
|
11
|
+
private consumerName;
|
|
12
|
+
transportId: symbol;
|
|
13
|
+
private readonly options;
|
|
14
|
+
constructor(options?: RedisStreamsOptions);
|
|
15
|
+
connect(): Promise<void>;
|
|
16
|
+
private registerEventListeners;
|
|
17
|
+
/**
|
|
18
|
+
* Triggered on application shutdown.
|
|
19
|
+
*/
|
|
20
|
+
close(): Promise<void>;
|
|
21
|
+
/**
|
|
22
|
+
* Triggered when you run "app.listen()".
|
|
23
|
+
*/
|
|
24
|
+
listen(callback: (err?: unknown) => void): Promise<void>;
|
|
25
|
+
/**
|
|
26
|
+
* You can ignore this method if you don't want transporter users
|
|
27
|
+
* to be able to register event listeners. Most custom implementations
|
|
28
|
+
* will not need this.
|
|
29
|
+
*/
|
|
30
|
+
on<EventKey extends keyof RedisEvents = keyof RedisEvents, EventCallback extends RedisEvents[EventKey] = RedisEvents[EventKey]>(event: EventKey, callback: EventCallback): void;
|
|
31
|
+
/**
|
|
32
|
+
* You can ignore this method if you don't want transporter users
|
|
33
|
+
* to be able to retrieve the underlying native server. Most custom implementations
|
|
34
|
+
* will not need this.
|
|
35
|
+
*/
|
|
36
|
+
unwrap<T = RedisClientType>(): T;
|
|
37
|
+
private bindEvents;
|
|
38
|
+
private createConsumerGroup;
|
|
39
|
+
private consumeMessages;
|
|
40
|
+
private handleStreamMessage;
|
|
41
|
+
private acknowledgeMessage;
|
|
42
|
+
parseMessage(content: any): Record<string, any>;
|
|
43
|
+
getRequestPattern(pattern: string): string;
|
|
44
|
+
private handleRequest;
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=streams-transporter.server.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streams-transporter.server.d.ts","sourceRoot":"","sources":["../../src/lib/streams-transporter.server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,KAAK,eAAe,EAAgB,MAAM,OAAO,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE1D,OAAO,EACL,mBAAmB,EAGpB,MAAM,+BAA+B,CAAC;AAGvC,qBAAa,iBACX,SAAQ,MAAM,CAAC,WAAW,EAAE,WAAW,CACvC,YAAW,uBAAuB;IAElC,OAAO,CAAC,MAAM,CACP;IACP,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,aAAa,CAAM;IAC3B,OAAO,CAAC,YAAY,CAA6B;IACjC,WAAW,SAA2B;IACtD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8B;gBAE1C,OAAO,GAAE,mBAAwB;IASvC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B,OAAO,CAAC,sBAAsB;IAmB9B;;OAEG;IACG,KAAK;IAUX;;OAEG;IACG,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,KAAK,IAAI;IAY9C;;;;OAIG;IACH,EAAE,CACA,QAAQ,SAAS,MAAM,WAAW,GAAG,MAAM,WAAW,EACtD,aAAa,SAAS,WAAW,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,EACnE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,GAAG,IAAI;IAIjD;;;;OAIG;IACH,MAAM,CAAC,CAAC,GAAG,eAAe,KAAK,CAAC;YAUlB,UAAU;YAoBV,mBAAmB;YAanB,eAAe;YAuDf,mBAAmB;YAqCnB,kBAAkB;IAYzB,YAAY,CAAC,OAAO,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAQ/C,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;YAInC,aAAa;CA6D5B"}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisStreamServer = void 0;
|
|
4
|
+
const microservices_1 = require("@nestjs/microservices");
|
|
5
|
+
const redis_1 = require("redis");
|
|
6
|
+
const rxjs_1 = require("rxjs");
|
|
7
|
+
const redis_events_1 = require("./redis.events");
|
|
8
|
+
const streams_transporter_context_1 = require("./streams-transporter.context");
|
|
9
|
+
const streams_transporter_options_1 = require("./streams-transporter.options");
|
|
10
|
+
const types_1 = require("./types");
|
|
11
|
+
class RedisStreamServer extends microservices_1.Server {
|
|
12
|
+
constructor(options = {}) {
|
|
13
|
+
super();
|
|
14
|
+
this.client = null;
|
|
15
|
+
this.isConsuming = false;
|
|
16
|
+
this.consumePromise = null;
|
|
17
|
+
this.lastIds = new Map();
|
|
18
|
+
this.consumerGroup = '';
|
|
19
|
+
this.consumerName = `consumer-${process.pid}`;
|
|
20
|
+
this.transportId = Symbol('REDIS_STREAMS');
|
|
21
|
+
this.options = (0, streams_transporter_options_1.resolveRedisStreamsOptions)(options);
|
|
22
|
+
this.consumerGroup = this.options.consumerGroup;
|
|
23
|
+
this.consumerName = this.options.consumerName || `consumer-${process.pid}`;
|
|
24
|
+
this.initializeSerializer({});
|
|
25
|
+
this.initializeDeserializer({});
|
|
26
|
+
}
|
|
27
|
+
async connect() {
|
|
28
|
+
this.client = (0, redis_1.createClient)(this.options);
|
|
29
|
+
this.registerEventListeners();
|
|
30
|
+
await this.client.connect();
|
|
31
|
+
}
|
|
32
|
+
registerEventListeners() {
|
|
33
|
+
if (!this.client)
|
|
34
|
+
return;
|
|
35
|
+
this.client.on('error', (err) => this.logger.error(err));
|
|
36
|
+
this.client.on('connect', () => {
|
|
37
|
+
this._status$.next(redis_events_1.RedisStatus.CONNECT);
|
|
38
|
+
});
|
|
39
|
+
this.client.on('ready', () => {
|
|
40
|
+
this._status$.next(redis_events_1.RedisStatus.CONNECTED);
|
|
41
|
+
});
|
|
42
|
+
this.client.on('reconnecting', () => {
|
|
43
|
+
this._status$.next(redis_events_1.RedisStatus.RECONNECTING);
|
|
44
|
+
});
|
|
45
|
+
this.client.on('end', () => {
|
|
46
|
+
this._status$.next(redis_events_1.RedisStatus.DISCONNECTED);
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Triggered on application shutdown.
|
|
51
|
+
*/
|
|
52
|
+
async close() {
|
|
53
|
+
this.isConsuming = false;
|
|
54
|
+
if (this.consumePromise) {
|
|
55
|
+
await this.consumePromise;
|
|
56
|
+
}
|
|
57
|
+
if (this.client)
|
|
58
|
+
await this.client.quit();
|
|
59
|
+
this.client = null;
|
|
60
|
+
this.lastIds.clear();
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Triggered when you run "app.listen()".
|
|
64
|
+
*/
|
|
65
|
+
async listen(callback) {
|
|
66
|
+
try {
|
|
67
|
+
if (!this.client) {
|
|
68
|
+
await this.connect();
|
|
69
|
+
}
|
|
70
|
+
await this.bindEvents();
|
|
71
|
+
callback();
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
callback(err);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* You can ignore this method if you don't want transporter users
|
|
79
|
+
* to be able to register event listeners. Most custom implementations
|
|
80
|
+
* will not need this.
|
|
81
|
+
*/
|
|
82
|
+
on(event, callback) {
|
|
83
|
+
throw new Error('Method not implemented.' + event + callback);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* You can ignore this method if you don't want transporter users
|
|
87
|
+
* to be able to retrieve the underlying native server. Most custom implementations
|
|
88
|
+
* will not need this.
|
|
89
|
+
*/
|
|
90
|
+
unwrap() {
|
|
91
|
+
if (!this.client) {
|
|
92
|
+
throw new Error('Redis client is not initialized. Make sure to call "connect()" first.');
|
|
93
|
+
}
|
|
94
|
+
return this.client;
|
|
95
|
+
}
|
|
96
|
+
async bindEvents() {
|
|
97
|
+
const patterns = Array.from(this.messageHandlers.keys());
|
|
98
|
+
if (patterns.length === 0) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
for (const pattern of patterns) {
|
|
102
|
+
const streamName = this.getRequestPattern(pattern);
|
|
103
|
+
if (!this.lastIds.has(streamName)) {
|
|
104
|
+
this.lastIds.set(streamName, '0');
|
|
105
|
+
}
|
|
106
|
+
await this.createConsumerGroup(streamName);
|
|
107
|
+
}
|
|
108
|
+
if (!this.isConsuming) {
|
|
109
|
+
this.isConsuming = true;
|
|
110
|
+
this.consumePromise = this.consumeMessages();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async createConsumerGroup(streamName) {
|
|
114
|
+
if (!this.client)
|
|
115
|
+
return;
|
|
116
|
+
try {
|
|
117
|
+
await this.client.xGroupCreate(streamName, this.consumerGroup, '$', {
|
|
118
|
+
MKSTREAM: true,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
if (!error?.message?.includes('BUSYGROUP')) {
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
async consumeMessages() {
|
|
128
|
+
if (!this.client)
|
|
129
|
+
return;
|
|
130
|
+
while (this.isConsuming && this.client) {
|
|
131
|
+
const streams = Array.from(this.lastIds.keys()).map((key) => ({
|
|
132
|
+
key,
|
|
133
|
+
id: '>',
|
|
134
|
+
}));
|
|
135
|
+
if (streams.length === 0) {
|
|
136
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
try {
|
|
140
|
+
const result = (await this.client.xReadGroup(this.consumerGroup, this.consumerName, streams, {
|
|
141
|
+
BLOCK: this.options.blockTimeout,
|
|
142
|
+
COUNT: this.options.batchSize,
|
|
143
|
+
}));
|
|
144
|
+
if (!result) {
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
for (const streamData of result) {
|
|
148
|
+
for (const message of streamData.messages) {
|
|
149
|
+
await this.handleStreamMessage(streamData.name, message.id, message.message);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
if (this.isConsuming) {
|
|
155
|
+
this.logger.error(error);
|
|
156
|
+
await new Promise((resolve) => setTimeout(resolve, this.options.retryDelay));
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
async handleStreamMessage(streamName, messageId, rawMessage) {
|
|
162
|
+
try {
|
|
163
|
+
const pattern = streamName.replace(`${this.options.streamPrefix}:`, '');
|
|
164
|
+
const data = this.parseMessage(rawMessage.data);
|
|
165
|
+
const context = new streams_transporter_context_1.RedisStreamsContext([
|
|
166
|
+
streamName,
|
|
167
|
+
messageId,
|
|
168
|
+
this.consumerGroup,
|
|
169
|
+
this.consumerName,
|
|
170
|
+
]);
|
|
171
|
+
if ((0, types_1.isRequestPacket)(rawMessage)) {
|
|
172
|
+
await this.handleRequest(pattern, data, rawMessage, context);
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
if ((0, types_1.isEventPacket)(rawMessage)) {
|
|
176
|
+
this.onProcessingStartHook(this.transportId, context, async () => void 0);
|
|
177
|
+
try {
|
|
178
|
+
await this.handleEvent(pattern, { pattern, data }, context);
|
|
179
|
+
}
|
|
180
|
+
finally {
|
|
181
|
+
this.onProcessingEndHook?.(this.transportId, context);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
await this.acknowledgeMessage(streamName, messageId);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
async acknowledgeMessage(streamName, messageId) {
|
|
190
|
+
if (!this.client)
|
|
191
|
+
return;
|
|
192
|
+
try {
|
|
193
|
+
await this.client.xAck(streamName, this.consumerGroup, messageId);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
this.logger.error(error);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
parseMessage(content) {
|
|
200
|
+
try {
|
|
201
|
+
return JSON.parse(content);
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return content;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
getRequestPattern(pattern) {
|
|
208
|
+
return `${this.options.streamPrefix}:${pattern}`;
|
|
209
|
+
}
|
|
210
|
+
async handleRequest(pattern, data, rawMessage, context) {
|
|
211
|
+
if (!this.client)
|
|
212
|
+
return;
|
|
213
|
+
const handler = this.getHandlerByPattern(pattern);
|
|
214
|
+
if (!handler)
|
|
215
|
+
return;
|
|
216
|
+
const replyTo = rawMessage.replyTo;
|
|
217
|
+
const id = rawMessage.id;
|
|
218
|
+
if (!replyTo || !id)
|
|
219
|
+
return;
|
|
220
|
+
this.onProcessingStartHook?.(this.transportId, context, async () => void 0);
|
|
221
|
+
try {
|
|
222
|
+
const response$ = this.transformToObservable(await handler(data, context));
|
|
223
|
+
const response = await (0, rxjs_1.firstValueFrom)(response$);
|
|
224
|
+
await this.client.xAdd(replyTo, '*', {
|
|
225
|
+
id,
|
|
226
|
+
data: JSON.stringify(response),
|
|
227
|
+
isDisposed: '1',
|
|
228
|
+
}, {
|
|
229
|
+
TRIM: {
|
|
230
|
+
strategy: 'MAXLEN',
|
|
231
|
+
strategyModifier: '~',
|
|
232
|
+
threshold: this.options.maxStreamLength,
|
|
233
|
+
},
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
catch (error) {
|
|
237
|
+
await this.client.xAdd(replyTo, '*', {
|
|
238
|
+
id,
|
|
239
|
+
err: JSON.stringify(error instanceof Error
|
|
240
|
+
? { message: error.message, name: error.name }
|
|
241
|
+
: error),
|
|
242
|
+
isDisposed: '1',
|
|
243
|
+
}, {
|
|
244
|
+
TRIM: {
|
|
245
|
+
strategy: 'MAXLEN',
|
|
246
|
+
strategyModifier: '~',
|
|
247
|
+
threshold: this.options.maxStreamLength,
|
|
248
|
+
},
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
finally {
|
|
252
|
+
this.onProcessingEndHook?.(this.transportId, context);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
exports.RedisStreamServer = RedisStreamServer;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export type EventType = {
|
|
2
|
+
e: '0' | '1';
|
|
3
|
+
data: string;
|
|
4
|
+
};
|
|
5
|
+
export type RequestType = {
|
|
6
|
+
e: '0' | '1';
|
|
7
|
+
data: string;
|
|
8
|
+
id: string;
|
|
9
|
+
replyTo: string;
|
|
10
|
+
};
|
|
11
|
+
export type ResponseType = {
|
|
12
|
+
id: string;
|
|
13
|
+
data?: string;
|
|
14
|
+
err?: string;
|
|
15
|
+
isDisposed?: '0' | '1';
|
|
16
|
+
};
|
|
17
|
+
export type RedisPacket = EventType | RequestType;
|
|
18
|
+
export declare function isEventPacket(packet: Record<string, string>): packet is EventType;
|
|
19
|
+
export declare function isRequestPacket(packet: Record<string, string>): packet is RequestType;
|
|
20
|
+
export declare function isResponsePacket(packet: Record<string, string>): packet is ResponseType;
|
|
21
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,GAAG;IACtB,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,WAAW,CAAC;AAElD,wBAAgB,aAAa,CAC3B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,MAAM,IAAI,SAAS,CAErB;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,MAAM,IAAI,WAAW,CAEvB;AAED,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,MAAM,IAAI,YAAY,CAExB"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isEventPacket = isEventPacket;
|
|
4
|
+
exports.isRequestPacket = isRequestPacket;
|
|
5
|
+
exports.isResponsePacket = isResponsePacket;
|
|
6
|
+
function isEventPacket(packet) {
|
|
7
|
+
return packet.e === '1';
|
|
8
|
+
}
|
|
9
|
+
function isRequestPacket(packet) {
|
|
10
|
+
return packet.e === '0';
|
|
11
|
+
}
|
|
12
|
+
function isResponsePacket(packet) {
|
|
13
|
+
return typeof packet.id === 'string' && 'isDisposed' in packet;
|
|
14
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@nestjs-redis/streams-transporter",
|
|
3
|
+
"version": "1.0.0-0",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"author": "Saba Pochkhua <saba.pochkhua@gmail.com> (https://github.com/CSenshi)",
|
|
6
|
+
"description": "Redis Streams-based transporter for NestJS microservices, enabling message passing via Redis Streams",
|
|
7
|
+
"keywords": [
|
|
8
|
+
"nestjs",
|
|
9
|
+
"redis",
|
|
10
|
+
"streams",
|
|
11
|
+
"transporter",
|
|
12
|
+
"microservices",
|
|
13
|
+
"typescript",
|
|
14
|
+
"node-redis"
|
|
15
|
+
],
|
|
16
|
+
"homepage": "https://github.com/CSenshi/nestjs-redis/tree/main/packages/streams-transporter",
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "https://github.com/CSenshi/nestjs-redis",
|
|
20
|
+
"directory": "packages/streams-transporter"
|
|
21
|
+
},
|
|
22
|
+
"engines": {
|
|
23
|
+
"node": ">=18.0.0",
|
|
24
|
+
"npm": ">=8.0.0"
|
|
25
|
+
},
|
|
26
|
+
"main": "./dist/index.js",
|
|
27
|
+
"module": "./dist/index.js",
|
|
28
|
+
"types": "./dist/index.d.ts",
|
|
29
|
+
"exports": {
|
|
30
|
+
"./package.json": "./package.json",
|
|
31
|
+
".": {
|
|
32
|
+
"development": "./src/index.ts",
|
|
33
|
+
"types": "./dist/index.d.ts",
|
|
34
|
+
"import": "./dist/index.js",
|
|
35
|
+
"default": "./dist/index.js"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"files": [
|
|
39
|
+
"dist",
|
|
40
|
+
"!**/*.tsbuildinfo"
|
|
41
|
+
],
|
|
42
|
+
"nx": {
|
|
43
|
+
"name": "streams-transporter"
|
|
44
|
+
},
|
|
45
|
+
"dependencies": {
|
|
46
|
+
"@nestjs/microservices": "^11.1.12",
|
|
47
|
+
"redis": "^5.0.0",
|
|
48
|
+
"tslib": "^2.3.0"
|
|
49
|
+
},
|
|
50
|
+
"peerDependencies": {
|
|
51
|
+
"@nestjs/common": "^9.0.0 || ^10.0.0 || ^11.0.0"
|
|
52
|
+
}
|
|
53
|
+
}
|