@uploadista/event-broadcaster-redis 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/LICENSE +21 -0
- package/README.md +643 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1 -0
- package/dist/redis-event-broadcaster.d.ts +29 -0
- package/dist/redis-event-broadcaster.d.ts.map +1 -0
- package/dist/redis-event-broadcaster.js +54 -0
- package/package.json +30 -0
- package/src/index.ts +1 -0
- package/src/redis-event-broadcaster.ts +85 -0
- package/tsconfig.json +14 -0
- package/tsconfig.tsbuildinfo +1 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 uploadista
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
# @uploadista/event-broadcaster-redis
|
|
2
|
+
|
|
3
|
+
Redis-backed event broadcaster for Uploadista. Distributes events across multiple server instances using Redis Pub/Sub.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
The Redis event broadcaster uses Redis Pub/Sub to broadcast events across distributed systems. Perfect for:
|
|
8
|
+
|
|
9
|
+
- **Distributed Servers**: Share events across multiple instances
|
|
10
|
+
- **Horizontal Scaling**: Add more servers without reconfiguration
|
|
11
|
+
- **Real-Time Updates**: Sub-millisecond event propagation
|
|
12
|
+
- **Production Deployments**: Battle-tested Redis infrastructure
|
|
13
|
+
- **Load Balancing**: Events reach any server instance
|
|
14
|
+
|
|
15
|
+
Events published to a channel are delivered to all subscribers on any server instance.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
npm install @uploadista/event-broadcaster-redis @redis/client
|
|
21
|
+
# or
|
|
22
|
+
pnpm add @uploadista/event-broadcaster-redis @redis/client
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
### Prerequisites
|
|
26
|
+
|
|
27
|
+
- Node.js 18+
|
|
28
|
+
- Redis 5.0+ server running and accessible
|
|
29
|
+
- Two Redis connections (one for publish, one for subscribe)
|
|
30
|
+
|
|
31
|
+
## Quick Start
|
|
32
|
+
|
|
33
|
+
```typescript
|
|
34
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
35
|
+
import { createClient } from "@redis/client";
|
|
36
|
+
import { Effect } from "effect";
|
|
37
|
+
|
|
38
|
+
// Create Redis clients (one for pub, one for sub)
|
|
39
|
+
const redisPublisher = createClient({
|
|
40
|
+
url: "redis://localhost:6379",
|
|
41
|
+
});
|
|
42
|
+
const redisSubscriber = createClient({
|
|
43
|
+
url: "redis://localhost:6379",
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
await redisPublisher.connect();
|
|
47
|
+
await redisSubscriber.connect();
|
|
48
|
+
|
|
49
|
+
const program = Effect.gen(function* () {
|
|
50
|
+
// Event broadcaster is automatically available
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
Effect.runSync(
|
|
54
|
+
program.pipe(
|
|
55
|
+
Effect.provide(
|
|
56
|
+
redisEventBroadcaster({
|
|
57
|
+
redis: redisPublisher,
|
|
58
|
+
subscriberRedis: redisSubscriber,
|
|
59
|
+
})
|
|
60
|
+
),
|
|
61
|
+
// ... other layers
|
|
62
|
+
)
|
|
63
|
+
);
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Features
|
|
67
|
+
|
|
68
|
+
- ✅ **Distributed Broadcasting**: Events reach all servers
|
|
69
|
+
- ✅ **Scalable**: Add servers without reconfiguration
|
|
70
|
+
- ✅ **High Performance**: Redis optimized for Pub/Sub
|
|
71
|
+
- ✅ **Multiple Channels**: Independent event streams
|
|
72
|
+
- ✅ **Reliable**: Redis persistence optional
|
|
73
|
+
- ✅ **Type Safe**: Full TypeScript support
|
|
74
|
+
|
|
75
|
+
## API Reference
|
|
76
|
+
|
|
77
|
+
### Main Exports
|
|
78
|
+
|
|
79
|
+
#### `redisEventBroadcaster(config: RedisEventBroadcasterConfig): Layer<EventBroadcasterService>`
|
|
80
|
+
|
|
81
|
+
Creates an Effect layer providing the `EventBroadcasterService` backed by Redis Pub/Sub.
|
|
82
|
+
|
|
83
|
+
```typescript
|
|
84
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
85
|
+
import { createClient } from "@redis/client";
|
|
86
|
+
|
|
87
|
+
const redis = createClient({ url: "redis://localhost:6379" });
|
|
88
|
+
const subscriberRedis = createClient({ url: "redis://localhost:6379" });
|
|
89
|
+
|
|
90
|
+
await redis.connect();
|
|
91
|
+
await subscriberRedis.connect();
|
|
92
|
+
|
|
93
|
+
const layer = redisEventBroadcaster({
|
|
94
|
+
redis,
|
|
95
|
+
subscriberRedis,
|
|
96
|
+
});
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
**Configuration**:
|
|
100
|
+
|
|
101
|
+
```typescript
|
|
102
|
+
interface RedisEventBroadcasterConfig {
|
|
103
|
+
redis: RedisClientType; // Connection for publishing
|
|
104
|
+
subscriberRedis: RedisClientType; // Connection for subscribing
|
|
105
|
+
}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
#### `createRedisEventBroadcaster(config: RedisEventBroadcasterConfig): EventBroadcaster`
|
|
109
|
+
|
|
110
|
+
Factory function to create a broadcaster instance.
|
|
111
|
+
|
|
112
|
+
```typescript
|
|
113
|
+
import { createRedisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
114
|
+
|
|
115
|
+
const broadcaster = createRedisEventBroadcaster({
|
|
116
|
+
redis,
|
|
117
|
+
subscriberRedis,
|
|
118
|
+
});
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Available Operations
|
|
122
|
+
|
|
123
|
+
The Redis broadcaster implements the `EventBroadcaster` interface:
|
|
124
|
+
|
|
125
|
+
#### `publish(channel: string, message: string): Effect<void>`
|
|
126
|
+
|
|
127
|
+
Broadcast a message to all subscribers on a channel (across all server instances).
|
|
128
|
+
|
|
129
|
+
```typescript
|
|
130
|
+
const program = Effect.gen(function* () {
|
|
131
|
+
yield* broadcaster.publish("uploads:complete", JSON.stringify({
|
|
132
|
+
uploadId: "abc123",
|
|
133
|
+
duration: 45000,
|
|
134
|
+
}));
|
|
135
|
+
// Delivered to all subscribers on all servers
|
|
136
|
+
});
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
#### `subscribe(channel: string, handler: (message: string) => void): Effect<void>`
|
|
140
|
+
|
|
141
|
+
Subscribe to a channel and receive messages from this and other servers.
|
|
142
|
+
|
|
143
|
+
```typescript
|
|
144
|
+
const program = Effect.gen(function* () {
|
|
145
|
+
yield* broadcaster.subscribe("uploads:complete", (message: string) => {
|
|
146
|
+
const event = JSON.parse(message);
|
|
147
|
+
console.log(`Upload complete: ${event.uploadId}`);
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
#### `unsubscribe(channel: string): Effect<void>`
|
|
153
|
+
|
|
154
|
+
Unsubscribe from a channel.
|
|
155
|
+
|
|
156
|
+
```typescript
|
|
157
|
+
const program = Effect.gen(function* () {
|
|
158
|
+
yield* broadcaster.unsubscribe("uploads:complete");
|
|
159
|
+
});
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
## Configuration
|
|
163
|
+
|
|
164
|
+
### Basic Setup
|
|
165
|
+
|
|
166
|
+
```typescript
|
|
167
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
168
|
+
import { createClient } from "@redis/client";
|
|
169
|
+
|
|
170
|
+
const redis = createClient({
|
|
171
|
+
url: "redis://localhost:6379",
|
|
172
|
+
});
|
|
173
|
+
const subscriberRedis = createClient({
|
|
174
|
+
url: "redis://localhost:6379",
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
await redis.connect();
|
|
178
|
+
await subscriberRedis.connect();
|
|
179
|
+
|
|
180
|
+
const layer = redisEventBroadcaster({
|
|
181
|
+
redis,
|
|
182
|
+
subscriberRedis,
|
|
183
|
+
});
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### Environment-Based Configuration
|
|
187
|
+
|
|
188
|
+
```typescript
|
|
189
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
190
|
+
import { createClient } from "@redis/client";
|
|
191
|
+
|
|
192
|
+
const redisUrl = process.env.REDIS_URL || "redis://localhost:6379";
|
|
193
|
+
|
|
194
|
+
const redis = createClient({ url: redisUrl });
|
|
195
|
+
const subscriberRedis = createClient({ url: redisUrl });
|
|
196
|
+
|
|
197
|
+
await redis.connect();
|
|
198
|
+
await subscriberRedis.connect();
|
|
199
|
+
|
|
200
|
+
const layer = redisEventBroadcaster({
|
|
201
|
+
redis,
|
|
202
|
+
subscriberRedis,
|
|
203
|
+
});
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
### Production with Replication
|
|
207
|
+
|
|
208
|
+
```typescript
|
|
209
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
210
|
+
import { createClient } from "@redis/client";
|
|
211
|
+
|
|
212
|
+
// Use replicas for distribution
|
|
213
|
+
const redis = createClient({
|
|
214
|
+
url: process.env.REDIS_PRIMARY,
|
|
215
|
+
password: process.env.REDIS_PASSWORD,
|
|
216
|
+
});
|
|
217
|
+
const subscriberRedis = createClient({
|
|
218
|
+
url: process.env.REDIS_REPLICA,
|
|
219
|
+
password: process.env.REDIS_PASSWORD,
|
|
220
|
+
readonly: true,
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
await redis.connect();
|
|
224
|
+
await subscriberRedis.connect();
|
|
225
|
+
|
|
226
|
+
const layer = redisEventBroadcaster({
|
|
227
|
+
redis,
|
|
228
|
+
subscriberRedis,
|
|
229
|
+
});
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
## Examples
|
|
233
|
+
|
|
234
|
+
### Example 1: Distributed Upload Server
|
|
235
|
+
|
|
236
|
+
Multiple server instances broadcasting upload events:
|
|
237
|
+
|
|
238
|
+
```typescript
|
|
239
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
240
|
+
import { uploadServer } from "@uploadista/server";
|
|
241
|
+
import { createClient } from "@redis/client";
|
|
242
|
+
import { Effect } from "effect";
|
|
243
|
+
|
|
244
|
+
const redis = createClient({ url: "redis://redis-cluster:6379" });
|
|
245
|
+
const subscriberRedis = createClient({ url: "redis://redis-cluster:6379" });
|
|
246
|
+
|
|
247
|
+
await redis.connect();
|
|
248
|
+
await subscriberRedis.connect();
|
|
249
|
+
|
|
250
|
+
const program = Effect.gen(function* () {
|
|
251
|
+
// Subscribe on this server
|
|
252
|
+
yield* broadcaster.subscribe("uploads:complete", (message: string) => {
|
|
253
|
+
const event = JSON.parse(message);
|
|
254
|
+
console.log(`[Server] Upload complete: ${event.uploadId}`);
|
|
255
|
+
// Trigger downstream processing on this server
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
// Any server can publish
|
|
259
|
+
yield* broadcaster.publish("uploads:complete", JSON.stringify({
|
|
260
|
+
uploadId: "abc123",
|
|
261
|
+
source: "server-2",
|
|
262
|
+
}));
|
|
263
|
+
// ALL servers (including this one) receive the event
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
Effect.runSync(
|
|
267
|
+
program.pipe(
|
|
268
|
+
Effect.provide(
|
|
269
|
+
redisEventBroadcaster({
|
|
270
|
+
redis,
|
|
271
|
+
subscriberRedis,
|
|
272
|
+
})
|
|
273
|
+
)
|
|
274
|
+
)
|
|
275
|
+
);
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
### Example 2: Flow Job Notifications
|
|
279
|
+
|
|
280
|
+
Notify all servers when a flow job completes:
|
|
281
|
+
|
|
282
|
+
```typescript
|
|
283
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
284
|
+
import { Effect } from "effect";
|
|
285
|
+
|
|
286
|
+
const broadcaster = createRedisEventBroadcaster({
|
|
287
|
+
redis,
|
|
288
|
+
subscriberRedis,
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
interface FlowCompletedEvent {
|
|
292
|
+
jobId: string;
|
|
293
|
+
uploadId: string;
|
|
294
|
+
status: "success" | "failed";
|
|
295
|
+
duration: number;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const notifyFlowComplete = (event: FlowCompletedEvent) =>
|
|
299
|
+
Effect.gen(function* () {
|
|
300
|
+
yield* broadcaster.publish(
|
|
301
|
+
"flows:completed",
|
|
302
|
+
JSON.stringify(event)
|
|
303
|
+
);
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
const program = Effect.gen(function* () {
|
|
307
|
+
// Subscribe on all servers
|
|
308
|
+
yield* broadcaster.subscribe("flows:completed", (message: string) => {
|
|
309
|
+
const event: FlowCompletedEvent = JSON.parse(message);
|
|
310
|
+
|
|
311
|
+
// Update metrics on this server
|
|
312
|
+
console.log(`Job ${event.jobId}: ${event.status} (${event.duration}ms)`);
|
|
313
|
+
|
|
314
|
+
// Cleanup local resources
|
|
315
|
+
if (event.status === "success") {
|
|
316
|
+
console.log(`Archiving results for upload ${event.uploadId}`);
|
|
317
|
+
}
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
// When processing completes
|
|
321
|
+
yield* notifyFlowComplete({
|
|
322
|
+
jobId: "job_xyz",
|
|
323
|
+
uploadId: "upl_abc",
|
|
324
|
+
status: "success",
|
|
325
|
+
duration: 12000,
|
|
326
|
+
});
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
Effect.runSync(program);
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
### Example 3: Cross-Server Cache Invalidation
|
|
333
|
+
|
|
334
|
+
Invalidate cached data across all servers:
|
|
335
|
+
|
|
336
|
+
```typescript
|
|
337
|
+
import { redisEventBroadcaster } from "@uploadista/event-broadcaster-redis";
|
|
338
|
+
import { Effect } from "effect";
|
|
339
|
+
|
|
340
|
+
const broadcaster = createRedisEventBroadcaster({
|
|
341
|
+
redis,
|
|
342
|
+
subscriberRedis,
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
// Local cache (in each server)
|
|
346
|
+
const localCache = new Map<string, any>();
|
|
347
|
+
|
|
348
|
+
const program = Effect.gen(function* () {
|
|
349
|
+
// Subscribe to cache invalidation
|
|
350
|
+
yield* broadcaster.subscribe("cache:invalidate", (message: string) => {
|
|
351
|
+
const { key } = JSON.parse(message);
|
|
352
|
+
localCache.delete(key);
|
|
353
|
+
console.log(`Cache invalidated: ${key}`);
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
// When data changes, notify all servers
|
|
357
|
+
yield* broadcaster.publish(
|
|
358
|
+
"cache:invalidate",
|
|
359
|
+
JSON.stringify({ key: "upload:abc123" })
|
|
360
|
+
);
|
|
361
|
+
// All servers clear their local cache for "upload:abc123"
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
Effect.runSync(program);
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
## Performance Characteristics
|
|
368
|
+
|
|
369
|
+
| Operation | Latency | Distribution |
|
|
370
|
+
|-----------|---------|--------------|
|
|
371
|
+
| publish() | 1-2ms | All servers |
|
|
372
|
+
| subscribe() | 2-5ms | Immediate |
|
|
373
|
+
| unsubscribe() | 1-2ms | Immediate |
|
|
374
|
+
| Event Delivery | 2-10ms | Global |
|
|
375
|
+
|
|
376
|
+
Events are delivered to all subscribers globally within milliseconds.
|
|
377
|
+
|
|
378
|
+
## Architecture
|
|
379
|
+
|
|
380
|
+
### Single Redis Instance
|
|
381
|
+
|
|
382
|
+
```
|
|
383
|
+
Server 1 ─┐
|
|
384
|
+
Server 2 ├──→ Redis ──→ Broadcast
|
|
385
|
+
Server 3 ─┘ to subscribers
|
|
386
|
+
```
|
|
387
|
+
|
|
388
|
+
### Redis Replication
|
|
389
|
+
|
|
390
|
+
```
|
|
391
|
+
Write: Server → Master Redis ─→ Replicate → Replicas
|
|
392
|
+
Read: Server → Replica Redis (for subscribe connections)
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
### Redis Cluster
|
|
396
|
+
|
|
397
|
+
```
|
|
398
|
+
Server 1 ──→ Cluster Node 1
|
|
399
|
+
Server 2 ──→ Cluster Node 2 ──→ Auto-replicated
|
|
400
|
+
Server 3 ──→ Cluster Node 3 across cluster
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
## Scaling Patterns
|
|
404
|
+
|
|
405
|
+
### 2-3 Servers
|
|
406
|
+
|
|
407
|
+
Use single Redis instance or master-replica:
|
|
408
|
+
|
|
409
|
+
```
|
|
410
|
+
App 1 ──┐
|
|
411
|
+
App 2 ├──→ Redis Master ──→ Replica (optional)
|
|
412
|
+
App 3 ──┘
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
### 5-10 Servers
|
|
416
|
+
|
|
417
|
+
Use Redis Sentinel for automatic failover:
|
|
418
|
+
|
|
419
|
+
```
|
|
420
|
+
Apps ──→ Sentinel ──→ Master Redis
|
|
421
|
+
(monitors) + Replicas
|
|
422
|
+
monitors
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
### 50+ Servers
|
|
426
|
+
|
|
427
|
+
Use Redis Cluster:
|
|
428
|
+
|
|
429
|
+
```
|
|
430
|
+
Apps ──→ Redis Cluster (auto-distributed)
|
|
431
|
+
16+ shards with replicas
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
## Best Practices
|
|
435
|
+
|
|
436
|
+
### 1. Use Two Connections
|
|
437
|
+
|
|
438
|
+
Always use separate connections for pub and sub:
|
|
439
|
+
|
|
440
|
+
```typescript
|
|
441
|
+
// ✅ Correct
|
|
442
|
+
const publisher = createClient({ url: redis_url });
|
|
443
|
+
const subscriber = createClient({ url: redis_url });
|
|
444
|
+
await publisher.connect();
|
|
445
|
+
await subscriber.connect();
|
|
446
|
+
|
|
447
|
+
const broadcaster = redisEventBroadcaster({
|
|
448
|
+
redis: publisher,
|
|
449
|
+
subscriberRedis: subscriber,
|
|
450
|
+
});
|
|
451
|
+
|
|
452
|
+
// ❌ Wrong (will deadlock)
|
|
453
|
+
const redis = createClient({ url: redis_url });
|
|
454
|
+
const broadcaster = redisEventBroadcaster({
|
|
455
|
+
redis,
|
|
456
|
+
subscriberRedis: redis, // Same connection!
|
|
457
|
+
});
|
|
458
|
+
```
|
|
459
|
+
|
|
460
|
+
### 2. Structured Event Format
|
|
461
|
+
|
|
462
|
+
Use consistent JSON structure:
|
|
463
|
+
|
|
464
|
+
```typescript
|
|
465
|
+
// Good: Type-safe events
|
|
466
|
+
interface UploadEvent {
|
|
467
|
+
type: "started" | "progress" | "completed";
|
|
468
|
+
uploadId: string;
|
|
469
|
+
timestamp: string;
|
|
470
|
+
data?: Record<string, any>;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
yield* broadcaster.publish("uploads", JSON.stringify(event));
|
|
474
|
+
|
|
475
|
+
// Handle with parsing
|
|
476
|
+
yield* broadcaster.subscribe("uploads", (message: string) => {
|
|
477
|
+
const event: UploadEvent = JSON.parse(message);
|
|
478
|
+
// Fully typed
|
|
479
|
+
});
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
### 3. Channel Naming Convention
|
|
483
|
+
|
|
484
|
+
Organize channels hierarchically:
|
|
485
|
+
|
|
486
|
+
```typescript
|
|
487
|
+
// Good: Clear hierarchy
|
|
488
|
+
"uploads:started"
|
|
489
|
+
"uploads:completed"
|
|
490
|
+
"flows:job:123:status"
|
|
491
|
+
"cache:invalidate"
|
|
492
|
+
|
|
493
|
+
// Avoid: Flat or unclear
|
|
494
|
+
"event", "update", "msg"
|
|
495
|
+
```
|
|
496
|
+
|
|
497
|
+
## Deployment
|
|
498
|
+
|
|
499
|
+
### Docker Compose
|
|
500
|
+
|
|
501
|
+
```yaml
|
|
502
|
+
version: "3"
|
|
503
|
+
services:
|
|
504
|
+
app1:
|
|
505
|
+
build: .
|
|
506
|
+
environment:
|
|
507
|
+
REDIS_URL: redis://redis:6379
|
|
508
|
+
depends_on:
|
|
509
|
+
- redis
|
|
510
|
+
app2:
|
|
511
|
+
build: .
|
|
512
|
+
environment:
|
|
513
|
+
REDIS_URL: redis://redis:6379
|
|
514
|
+
depends_on:
|
|
515
|
+
- redis
|
|
516
|
+
redis:
|
|
517
|
+
image: redis:7-alpine
|
|
518
|
+
ports:
|
|
519
|
+
- "6379:6379"
|
|
520
|
+
volumes:
|
|
521
|
+
- redis_data:/data
|
|
522
|
+
command: redis-server --appendonly yes
|
|
523
|
+
|
|
524
|
+
volumes:
|
|
525
|
+
redis_data:
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
### Kubernetes
|
|
529
|
+
|
|
530
|
+
```yaml
|
|
531
|
+
apiVersion: apps/v1
|
|
532
|
+
kind: Deployment
|
|
533
|
+
metadata:
|
|
534
|
+
name: uploadista-app
|
|
535
|
+
spec:
|
|
536
|
+
replicas: 3
|
|
537
|
+
template:
|
|
538
|
+
spec:
|
|
539
|
+
containers:
|
|
540
|
+
- name: app
|
|
541
|
+
env:
|
|
542
|
+
- name: REDIS_URL
|
|
543
|
+
value: redis://redis-service.default.svc.cluster.local:6379
|
|
544
|
+
```
|
|
545
|
+
|
|
546
|
+
## Monitoring
|
|
547
|
+
|
|
548
|
+
### Check Active Subscriptions
|
|
549
|
+
|
|
550
|
+
```bash
|
|
551
|
+
redis-cli PUBSUB CHANNELS
|
|
552
|
+
# Shows all active channels
|
|
553
|
+
|
|
554
|
+
redis-cli PUBSUB NUMSUB uploads:complete
|
|
555
|
+
# Shows number of subscribers per channel
|
|
556
|
+
```
|
|
557
|
+
|
|
558
|
+
### Monitor Published Events
|
|
559
|
+
|
|
560
|
+
```bash
|
|
561
|
+
redis-cli
|
|
562
|
+
> SUBSCRIBE uploads:complete
|
|
563
|
+
```
|
|
564
|
+
|
|
565
|
+
## Related Packages
|
|
566
|
+
|
|
567
|
+
- [@uploadista/core](../../core) - Core types
|
|
568
|
+
- [@uploadista/event-broadcaster-ioredis](../ioredis) - IORedis broadcaster with clustering
|
|
569
|
+
- [@uploadista/event-broadcaster-memory](../memory) - Single-process broadcaster
|
|
570
|
+
- [@uploadista/event-emitter-websocket](../../event-emitters/websocket) - WebSocket real-time
|
|
571
|
+
- [@uploadista/kv-store-redis](../../kv-stores/redis) - Redis KV store
|
|
572
|
+
- [@uploadista/server](../../servers/server) - Upload server
|
|
573
|
+
|
|
574
|
+
## Troubleshooting
|
|
575
|
+
|
|
576
|
+
### "Pub/Sub connection blocked" Error
|
|
577
|
+
|
|
578
|
+
Using same connection for pub and sub:
|
|
579
|
+
|
|
580
|
+
```typescript
|
|
581
|
+
// ❌ Wrong
|
|
582
|
+
const redis = createClient();
|
|
583
|
+
const broadcaster = redisEventBroadcaster({
|
|
584
|
+
redis,
|
|
585
|
+
subscriberRedis: redis, // Same!
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
// ✅ Fix
|
|
589
|
+
const pubRedis = createClient();
|
|
590
|
+
const subRedis = createClient();
|
|
591
|
+
const broadcaster = redisEventBroadcaster({
|
|
592
|
+
redis: pubRedis,
|
|
593
|
+
subscriberRedis: subRedis,
|
|
594
|
+
});
|
|
595
|
+
```
|
|
596
|
+
|
|
597
|
+
### Events Not Delivered
|
|
598
|
+
|
|
599
|
+
Verify subscription is active before publishing:
|
|
600
|
+
|
|
601
|
+
```typescript
|
|
602
|
+
// ✅ Subscribe first
|
|
603
|
+
yield* broadcaster.subscribe("channel", handler);
|
|
604
|
+
yield* broadcaster.publish("channel", "message");
|
|
605
|
+
|
|
606
|
+
// ❌ Publish without subscribers
|
|
607
|
+
yield* broadcaster.publish("channel", "message"); // Lost!
|
|
608
|
+
yield* broadcaster.subscribe("channel", handler); // Never receives
|
|
609
|
+
```
|
|
610
|
+
|
|
611
|
+
### High Latency
|
|
612
|
+
|
|
613
|
+
Check Redis connection and network:
|
|
614
|
+
|
|
615
|
+
```bash
|
|
616
|
+
# Monitor Redis latency
|
|
617
|
+
redis-cli --latency
|
|
618
|
+
|
|
619
|
+
# Check network between app and Redis
|
|
620
|
+
ping redis-host
|
|
621
|
+
```
|
|
622
|
+
|
|
623
|
+
### Memory Growth
|
|
624
|
+
|
|
625
|
+
Redis stores subscriptions in memory. Clean up when done:
|
|
626
|
+
|
|
627
|
+
```typescript
|
|
628
|
+
yield* broadcaster.subscribe("channel", handler);
|
|
629
|
+
// Do work...
|
|
630
|
+
yield* broadcaster.unsubscribe("channel");
|
|
631
|
+
```
|
|
632
|
+
|
|
633
|
+
## License
|
|
634
|
+
|
|
635
|
+
See [LICENSE](../../../LICENSE) in the main repository.
|
|
636
|
+
|
|
637
|
+
## See Also
|
|
638
|
+
|
|
639
|
+
- [EVENT_SYSTEM.md](./EVENT_SYSTEM.md) - Architecture and patterns
|
|
640
|
+
- [Server Setup Guide](../../../SERVER_SETUP.md#redis-events) - Redis in servers
|
|
641
|
+
- [Redis Pub/Sub Documentation](https://redis.io/topics/pubsub) - Official Redis Pub/Sub
|
|
642
|
+
- [IORedis Broadcaster](../ioredis/README.md) - For clustering
|
|
643
|
+
- [WebSocket Event Emitter](../../event-emitters/websocket/README.md) - Real-time WebSocket
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./redis-event-broadcaster";
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { RedisClientType } from "@redis/client";
|
|
2
|
+
import type { EventBroadcaster } from "@uploadista/core/types";
|
|
3
|
+
import { EventBroadcasterService } from "@uploadista/core/types";
|
|
4
|
+
import { Layer } from "effect";
|
|
5
|
+
/**
|
|
6
|
+
* Configuration for Redis event broadcaster
|
|
7
|
+
*/
|
|
8
|
+
export interface RedisEventBroadcasterConfig {
|
|
9
|
+
/**
|
|
10
|
+
* Redis client for publishing messages
|
|
11
|
+
*/
|
|
12
|
+
redis: RedisClientType;
|
|
13
|
+
/**
|
|
14
|
+
* Separate Redis client for subscribing to messages
|
|
15
|
+
* (Redis requires a dedicated connection for pub/sub)
|
|
16
|
+
*/
|
|
17
|
+
subscriberRedis: RedisClientType;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Redis-based event broadcaster for distributed deployments.
|
|
21
|
+
* Uses Redis Pub/Sub to broadcast events across multiple instances.
|
|
22
|
+
* Requires two separate Redis connections (one for pub, one for sub).
|
|
23
|
+
*/
|
|
24
|
+
export declare function createRedisEventBroadcaster(config: RedisEventBroadcasterConfig): EventBroadcaster;
|
|
25
|
+
/**
|
|
26
|
+
* Layer factory for Redis event broadcaster
|
|
27
|
+
*/
|
|
28
|
+
export declare const redisEventBroadcaster: (config: RedisEventBroadcasterConfig) => Layer.Layer<EventBroadcasterService, never, never>;
|
|
29
|
+
//# sourceMappingURL=redis-event-broadcaster.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis-event-broadcaster.d.ts","sourceRoot":"","sources":["../src/redis-event-broadcaster.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAU,KAAK,EAAE,MAAM,QAAQ,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C;;OAEG;IACH,KAAK,EAAE,eAAe,CAAC;IACvB;;;OAGG;IACH,eAAe,EAAE,eAAe,CAAC;CAClC;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,2BAA2B,GAClC,gBAAgB,CAkDlB;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,GAAI,QAAQ,2BAA2B,uDACI,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { UploadistaError } from "@uploadista/core/errors";
|
|
2
|
+
import { EventBroadcasterService } from "@uploadista/core/types";
|
|
3
|
+
import { Effect, Layer } from "effect";
|
|
4
|
+
/**
|
|
5
|
+
* Redis-based event broadcaster for distributed deployments.
|
|
6
|
+
* Uses Redis Pub/Sub to broadcast events across multiple instances.
|
|
7
|
+
* Requires two separate Redis connections (one for pub, one for sub).
|
|
8
|
+
*/
|
|
9
|
+
export function createRedisEventBroadcaster(config) {
|
|
10
|
+
const { redis, subscriberRedis } = config;
|
|
11
|
+
subscriberRedis.on("error", (error) => {
|
|
12
|
+
console.error(`[Redis] Subscriber Error:`, error);
|
|
13
|
+
});
|
|
14
|
+
redis.on("error", (error) => {
|
|
15
|
+
console.error(`[Redis] Error:`, error);
|
|
16
|
+
});
|
|
17
|
+
return {
|
|
18
|
+
publish: (channel, message) => Effect.tryPromise({
|
|
19
|
+
try: async () => {
|
|
20
|
+
const result = await redis.publish(channel, message);
|
|
21
|
+
return result;
|
|
22
|
+
},
|
|
23
|
+
catch: (cause) => {
|
|
24
|
+
console.error(`[Redis] Failed to publish to ${channel}:`, cause);
|
|
25
|
+
return UploadistaError.fromCode("UNKNOWN_ERROR", {
|
|
26
|
+
cause,
|
|
27
|
+
});
|
|
28
|
+
},
|
|
29
|
+
}).pipe(Effect.asVoid),
|
|
30
|
+
subscribe: (channel, handler) => Effect.tryPromise({
|
|
31
|
+
try: async () => {
|
|
32
|
+
await subscriberRedis.subscribe(channel, (message, _channel) => {
|
|
33
|
+
handler(message);
|
|
34
|
+
});
|
|
35
|
+
},
|
|
36
|
+
catch: (cause) => {
|
|
37
|
+
console.error(`[Redis] Failed to subscribe to ${channel}:`, cause);
|
|
38
|
+
return UploadistaError.fromCode("UNKNOWN_ERROR", {
|
|
39
|
+
cause,
|
|
40
|
+
});
|
|
41
|
+
},
|
|
42
|
+
}).pipe(Effect.asVoid),
|
|
43
|
+
unsubscribe: (channel) => Effect.tryPromise({
|
|
44
|
+
try: () => subscriberRedis.unsubscribe(channel),
|
|
45
|
+
catch: (cause) => UploadistaError.fromCode("UNKNOWN_ERROR", {
|
|
46
|
+
cause,
|
|
47
|
+
}),
|
|
48
|
+
}).pipe(Effect.asVoid),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Layer factory for Redis event broadcaster
|
|
53
|
+
*/
|
|
54
|
+
export const redisEventBroadcaster = (config) => Layer.succeed(EventBroadcasterService, createRedisEventBroadcaster(config));
|
package/package.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@uploadista/event-broadcaster-redis",
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "0.0.3",
|
|
5
|
+
"description": "Redis event broadcaster for Uploadista",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Uploadista",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dependencies": {
|
|
16
|
+
"@redis/client": "5.8.3",
|
|
17
|
+
"effect": "3.18.4",
|
|
18
|
+
"@uploadista/core": "0.0.3"
|
|
19
|
+
},
|
|
20
|
+
"devDependencies": {
|
|
21
|
+
"@types/node": "24.8.1",
|
|
22
|
+
"@uploadista/typescript-config": "0.0.3"
|
|
23
|
+
},
|
|
24
|
+
"scripts": {
|
|
25
|
+
"build": "tsc -b",
|
|
26
|
+
"format": "biome format --write ./src",
|
|
27
|
+
"lint": "biome lint --write ./src",
|
|
28
|
+
"check": "biome check --write ./src"
|
|
29
|
+
}
|
|
30
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./redis-event-broadcaster";
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { RedisClientType } from "@redis/client";
|
|
2
|
+
import { UploadistaError } from "@uploadista/core/errors";
|
|
3
|
+
import type { EventBroadcaster } from "@uploadista/core/types";
|
|
4
|
+
import { EventBroadcasterService } from "@uploadista/core/types";
|
|
5
|
+
import { Effect, Layer } from "effect";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Configuration for Redis event broadcaster
|
|
9
|
+
*/
|
|
10
|
+
export interface RedisEventBroadcasterConfig {
|
|
11
|
+
/**
|
|
12
|
+
* Redis client for publishing messages
|
|
13
|
+
*/
|
|
14
|
+
redis: RedisClientType;
|
|
15
|
+
/**
|
|
16
|
+
* Separate Redis client for subscribing to messages
|
|
17
|
+
* (Redis requires a dedicated connection for pub/sub)
|
|
18
|
+
*/
|
|
19
|
+
subscriberRedis: RedisClientType;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Redis-based event broadcaster for distributed deployments.
|
|
24
|
+
* Uses Redis Pub/Sub to broadcast events across multiple instances.
|
|
25
|
+
* Requires two separate Redis connections (one for pub, one for sub).
|
|
26
|
+
*/
|
|
27
|
+
export function createRedisEventBroadcaster(
|
|
28
|
+
config: RedisEventBroadcasterConfig,
|
|
29
|
+
): EventBroadcaster {
|
|
30
|
+
const { redis, subscriberRedis } = config;
|
|
31
|
+
|
|
32
|
+
subscriberRedis.on("error", (error) => {
|
|
33
|
+
console.error(`[Redis] Subscriber Error:`, error);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
redis.on("error", (error) => {
|
|
37
|
+
console.error(`[Redis] Error:`, error);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
publish: (channel: string, message: string) =>
|
|
42
|
+
Effect.tryPromise({
|
|
43
|
+
try: async () => {
|
|
44
|
+
const result = await redis.publish(channel, message);
|
|
45
|
+
return result;
|
|
46
|
+
},
|
|
47
|
+
catch: (cause) => {
|
|
48
|
+
console.error(`[Redis] Failed to publish to ${channel}:`, cause);
|
|
49
|
+
return UploadistaError.fromCode("UNKNOWN_ERROR", {
|
|
50
|
+
cause,
|
|
51
|
+
});
|
|
52
|
+
},
|
|
53
|
+
}).pipe(Effect.asVoid),
|
|
54
|
+
|
|
55
|
+
subscribe: (channel: string, handler: (message: string) => void) =>
|
|
56
|
+
Effect.tryPromise({
|
|
57
|
+
try: async () => {
|
|
58
|
+
await subscriberRedis.subscribe(channel, (message, _channel) => {
|
|
59
|
+
handler(message);
|
|
60
|
+
});
|
|
61
|
+
},
|
|
62
|
+
catch: (cause) => {
|
|
63
|
+
console.error(`[Redis] Failed to subscribe to ${channel}:`, cause);
|
|
64
|
+
return UploadistaError.fromCode("UNKNOWN_ERROR", {
|
|
65
|
+
cause,
|
|
66
|
+
});
|
|
67
|
+
},
|
|
68
|
+
}).pipe(Effect.asVoid),
|
|
69
|
+
|
|
70
|
+
unsubscribe: (channel: string) =>
|
|
71
|
+
Effect.tryPromise({
|
|
72
|
+
try: () => subscriberRedis.unsubscribe(channel),
|
|
73
|
+
catch: (cause) =>
|
|
74
|
+
UploadistaError.fromCode("UNKNOWN_ERROR", {
|
|
75
|
+
cause,
|
|
76
|
+
}),
|
|
77
|
+
}).pipe(Effect.asVoid),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Layer factory for Redis event broadcaster
|
|
83
|
+
*/
|
|
84
|
+
export const redisEventBroadcaster = (config: RedisEventBroadcasterConfig) =>
|
|
85
|
+
Layer.succeed(EventBroadcasterService, createRedisEventBroadcaster(config));
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "@uploadista/typescript-config/server.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"baseUrl": "./",
|
|
5
|
+
"paths": {
|
|
6
|
+
"@/*": ["./src/*"]
|
|
7
|
+
},
|
|
8
|
+
"outDir": "./dist",
|
|
9
|
+
"rootDir": "./src",
|
|
10
|
+
"typeRoots": ["../../../../node_modules/@types"],
|
|
11
|
+
"lib": ["ES2022"]
|
|
12
|
+
},
|
|
13
|
+
"include": ["src"]
|
|
14
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"root":["./src/index.ts","./src/redis-event-broadcaster.ts"],"version":"5.9.3"}
|