@emanuelepifani/nexo-client 0.1.1 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -140
- package/dist/brokers/queue.d.ts +51 -1
- package/dist/brokers/queue.js +111 -6
- package/dist/brokers/stream.d.ts +1 -1
- package/dist/brokers/stream.js +0 -2
- package/dist/codec.js +18 -16
- package/dist/connection.js +8 -7
- package/package.json +7 -2
- package/dist/performance-test.d.ts +0 -2
- package/dist/performance-test.js +0 -58
- package/dist/ringbuffer.d.ts +0 -27
- package/dist/ringbuffer.js +0 -109
package/README.md
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# Nexo Client SDK
|
|
2
2
|
|
|
3
|
-
High-performance TypeScript client for [Nexo](https://
|
|
3
|
+
High-performance TypeScript client for [Nexo](https://nexo-docs-hub.vercel.app/).
|
|
4
|
+
|
|
5
|
+
|
|
4
6
|
|
|
5
7
|
## Quick Start
|
|
6
8
|
|
|
@@ -19,6 +21,7 @@ This exposes:
|
|
|
19
21
|
npm install @emanuelepifani/nexo-client
|
|
20
22
|
```
|
|
21
23
|
|
|
24
|
+
|
|
22
25
|
### Connection
|
|
23
26
|
```typescript
|
|
24
27
|
const client = await NexoClient.connect({ host: 'localhost', port: 7654 });
|
|
@@ -48,57 +51,6 @@ await mailQ.subscribe((msg) => console.log(msg));
|
|
|
48
51
|
await mailQ.delete();
|
|
49
52
|
```
|
|
50
53
|
|
|
51
|
-
<details>
|
|
52
|
-
<summary><strong>Advanced Features (Persistence, Retry, Delay, Priority)</strong></summary>
|
|
53
|
-
|
|
54
|
-
```typescript
|
|
55
|
-
// -------------------------------------------------------------
|
|
56
|
-
// 1. CREATION (Default behavior for all messages in this queue)
|
|
57
|
-
// -------------------------------------------------------------
|
|
58
|
-
interface CriticalTask { type: string; payload: any; }
|
|
59
|
-
|
|
60
|
-
const criticalQueue = await client.queue<CriticalTask>('critical-tasks').create({
|
|
61
|
-
// RELIABILITY:
|
|
62
|
-
visibilityTimeoutMs: 10000, // Retry delivery if not ACKed within 10s (default=30s)
|
|
63
|
-
maxRetries: 5, // Move to DLQ after 5 failures (default=5)
|
|
64
|
-
ttlMs: 60000, // Message expires if not consumed in 60s (default=7days)
|
|
65
|
-
|
|
66
|
-
// PERSISTENCE:
|
|
67
|
-
// - 'memory': Volatile (Fastest, lost on restart)
|
|
68
|
-
// - 'file_sync': Save every message (Safest, Slowest)
|
|
69
|
-
// - 'file_async': Flush periodically (Fast & Durable) -
|
|
70
|
-
// DEFAULT: 'file_async': Flush every 50ms or 5000 msgs
|
|
71
|
-
persistence: 'file_sync',
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
// ---------------------------------------------------------
|
|
76
|
-
// 2. PRODUCING (Override specific behaviors per message)
|
|
77
|
-
// ---------------------------------------------------------
|
|
78
|
-
|
|
79
|
-
// PRIORITY: Higher value (255) delivered before lower values (0)
|
|
80
|
-
// This message jumps ahead of all priority < 255 messages sent previously and still not consumed.
|
|
81
|
-
await criticalQueue.push({ type: 'urgent' }, { priority: 255 });
|
|
82
|
-
|
|
83
|
-
// SCHEDULING: Delay visibility
|
|
84
|
-
// This message is hidden for 1 hour (default delayMs: 0, instant)
|
|
85
|
-
await criticalQueue.push({ type: 'scheduled' }, { delayMs: 3600000 });
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
// ---------------------------------------------------------
|
|
90
|
-
// 3. CONSUMING (Worker Tuning to optimize throughput and latency)
|
|
91
|
-
// ---------------------------------------------------------
|
|
92
|
-
await criticalQueue.subscribe(
|
|
93
|
-
async (task) => { await processTask(task); },
|
|
94
|
-
{
|
|
95
|
-
batchSize: 100, // Network: Fetch 100 messages in one request
|
|
96
|
-
concurrency: 10, // Local: Process 10 messages concurrently (useful for I/O tasks)
|
|
97
|
-
waitMs: 5000 // Polling: If empty, wait 5s for new messages before retrying
|
|
98
|
-
}
|
|
99
|
-
);
|
|
100
|
-
```
|
|
101
|
-
</details>
|
|
102
54
|
|
|
103
55
|
### 3. PUB/SUB
|
|
104
56
|
|
|
@@ -111,35 +63,6 @@ await alerts.subscribe((msg) => console.log(msg));
|
|
|
111
63
|
await alerts.publish({ level: "high" });
|
|
112
64
|
```
|
|
113
65
|
|
|
114
|
-
<details>
|
|
115
|
-
<summary><strong>Wildcards & Retained Messages</strong></summary>
|
|
116
|
-
|
|
117
|
-
```typescript
|
|
118
|
-
// WILDCARD SUBSCRIPTIONS
|
|
119
|
-
// ----------------------
|
|
120
|
-
|
|
121
|
-
// 1. Single-Level Wildcard (+)
|
|
122
|
-
// Matches: 'home/kitchen/light', 'home/garage/light'
|
|
123
|
-
const roomLights = client.pubsub<LightStatus>('home/+/light');
|
|
124
|
-
await roomLights.subscribe((status) => console.log('Light is:', status.state));
|
|
125
|
-
|
|
126
|
-
// 2. Multi-Level Wildcard (#)
|
|
127
|
-
// Matches all topics under 'sensors/'
|
|
128
|
-
const allSensors = client.pubsub<SensorData>('sensors/#');
|
|
129
|
-
await allSensors.subscribe((data) => console.log('Sensor value:', data.value));
|
|
130
|
-
|
|
131
|
-
// PUBLISHING (No wildcards allowed!)
|
|
132
|
-
// ---------------------------------
|
|
133
|
-
// You must publish to concrete topics with matching types
|
|
134
|
-
await client.pubsub<LightStatus>('home/kitchen/light').publish({ state: 'ON' });
|
|
135
|
-
await client.pubsub<SensorData>('sensors/kitchen/temp').publish({ value: 22.5, unit: 'C' });
|
|
136
|
-
|
|
137
|
-
// RETAINED MESSAGES
|
|
138
|
-
// -----------------
|
|
139
|
-
// Last value is stored and immediately sent to new subscribers
|
|
140
|
-
await client.pubsub<string>('config/theme').publish('dark', { retain: true });
|
|
141
|
-
```
|
|
142
|
-
</details>
|
|
143
66
|
|
|
144
67
|
### 4. STREAM
|
|
145
68
|
|
|
@@ -154,60 +77,14 @@ await stream.subscribe('analytics', (msg) => {console.log(`User ${msg.userId} pe
|
|
|
154
77
|
await stream.delete();
|
|
155
78
|
```
|
|
156
79
|
|
|
157
|
-
<details>
|
|
158
|
-
<summary><strong>Advanced Features (Consumer Groups, Persistence, Retention)</strong></summary>
|
|
159
|
-
|
|
160
|
-
```typescript
|
|
161
|
-
// ---------------------------------------------------------
|
|
162
|
-
// 1. STREAM CREATION & POLICY
|
|
163
|
-
// ---------------------------------------------------------
|
|
164
|
-
const orders = await client.stream<Order>('orders').create({
|
|
165
|
-
// SCALING
|
|
166
|
-
partitions: 4, // Max concurrent consumers per group on same topic (default=8)
|
|
167
|
-
|
|
168
|
-
// PERSISTENCE:
|
|
169
|
-
// - 'memory': Volatile (Fastest, lost on restart)
|
|
170
|
-
// - 'file_sync': Save every message (Safest, Slowest)
|
|
171
|
-
// - 'file_async': Flush periodically (Fast & Durable) -
|
|
172
|
-
// DEFAULT: 'file_async': Flush every 50ms or 5000 msgs
|
|
173
|
-
persistence: 'file_sync',
|
|
174
|
-
|
|
175
|
-
// RETENTION (Cleanup Policy)
|
|
176
|
-
// --------------------------
|
|
177
|
-
// Delete old data when EITHER limit is reached:
|
|
178
|
-
retention: {
|
|
179
|
-
maxAgeMs: 86400000, // 1 Day (Default: 7 Days)
|
|
180
|
-
maxBytes: 536870912 // 512 MB (Default: 1 GB)
|
|
181
|
-
},
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
// ---------------------------------------------------------
|
|
185
|
-
// 2. CONSUMING (Scaling & Broadcast patterns)
|
|
186
|
-
// ---------------------------------------------------------
|
|
187
|
-
|
|
188
|
-
// SCALING (Microservices Replicas / K8s Pods)
|
|
189
|
-
// Same Group ('workers') -> Automatic Load Balancing & Rebalancing
|
|
190
|
-
// Partitions are distributed among workers.
|
|
191
|
-
await orders.subscribe('workers', (order) => process(order));
|
|
192
|
-
await orders.subscribe('workers', (order) => process(order));
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
// BROADCAST (Independent Domains)
|
|
196
|
-
// Different Groups -> Each group gets a full copy of the stream.
|
|
197
|
-
// Useful for independent services reacting to the same event.
|
|
198
|
-
await orders.subscribe('analytics', (order) => trackMetrics(order));
|
|
199
|
-
await orders.subscribe('audit-log', (order) => saveAudit(order));
|
|
200
|
-
```
|
|
201
|
-
</details>
|
|
202
|
-
|
|
203
80
|
|
|
204
81
|
|
|
205
82
|
---
|
|
206
83
|
|
|
207
|
-
### Binary Payloads
|
|
84
|
+
### Binary Payloads
|
|
208
85
|
|
|
209
86
|
All Nexo brokers (**Store, Queue, Stream, PubSub**) natively support raw binary data (`Buffer`).
|
|
210
|
-
Bypassing JSON serialization drastically reduces Latency, increases Throughput, and saves Bandwidth
|
|
87
|
+
Bypassing JSON serialization drastically reduces Latency, increases Throughput, and saves Bandwidth.
|
|
211
88
|
|
|
212
89
|
**Perfect for:** Video chunks, Images, Protobuf/MsgPack, Encrypted blobs.
|
|
213
90
|
|
|
@@ -215,16 +92,13 @@ Bypassing JSON serialization drastically reduces Latency, increases Throughput,
|
|
|
215
92
|
// Send 1MB raw buffer (30% smaller than JSON/Base64)
|
|
216
93
|
const heavyPayload = Buffer.alloc(1024 * 1024);
|
|
217
94
|
|
|
218
|
-
// 1. STREAM
|
|
95
|
+
// 1. STREAM
|
|
219
96
|
await client.stream('cctv-archive').publish(heavyPayload);
|
|
220
|
-
|
|
221
|
-
// 2. PUBSUB: Ephemeral Live Data (e.g. VoIP, Real-time Sensor)
|
|
97
|
+
// 2. PUBSUB
|
|
222
98
|
await client.pubsub('live-audio-call').publish(heavyPayload);
|
|
223
|
-
|
|
224
|
-
// 3. STORE (Cache Images)
|
|
99
|
+
// 3. STORE
|
|
225
100
|
await client.store.map.set('user:avatar:1', heavyPayload);
|
|
226
|
-
|
|
227
|
-
// 4. QUEUE (Process Files)
|
|
101
|
+
// 4. QUEUE
|
|
228
102
|
await client.queue('pdf-processing').push(heavyPayload);
|
|
229
103
|
```
|
|
230
104
|
|
|
@@ -237,10 +111,8 @@ MIT
|
|
|
237
111
|
|
|
238
112
|
## Links
|
|
239
113
|
|
|
240
|
-
-
|
|
241
|
-
-
|
|
242
|
-
- **SDK Source:** [sdk/ts](https://github.com/emanuel-epifani/nexo/tree/main/sdk/ts)
|
|
243
|
-
- **Docker Image:** [emanuelepifani/nexo](https://hub.docker.com/r/emanuelepifani/nexo)
|
|
114
|
+
- **📚 Full Documentation:** [Nexo Docs](https://nexo-docs-hub.vercel.app/)
|
|
115
|
+
- **🐳 Docker Image:** [emanuelepifani/nexo](https://hub.docker.com/r/emanuelepifani/nexo)
|
|
244
116
|
|
|
245
117
|
## Author
|
|
246
118
|
|
package/dist/brokers/queue.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { NexoConnection } from '../connection';
|
|
2
2
|
import { Logger } from '../utils/logger';
|
|
3
|
-
export type PersistenceStrategy = '
|
|
3
|
+
export type PersistenceStrategy = 'file_sync' | 'file_async';
|
|
4
4
|
export interface QueueConfig {
|
|
5
5
|
visibilityTimeoutMs?: number;
|
|
6
6
|
maxRetries?: number;
|
|
@@ -16,12 +16,61 @@ export interface QueuePushOptions {
|
|
|
16
16
|
priority?: number;
|
|
17
17
|
delayMs?: number;
|
|
18
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Dead Letter Queue (DLQ) management for a queue.
|
|
21
|
+
* Provides methods to inspect, replay, delete, and purge failed messages.
|
|
22
|
+
*/
|
|
23
|
+
export declare class NexoDLQ<T = any> {
|
|
24
|
+
private conn;
|
|
25
|
+
private queueName;
|
|
26
|
+
private logger;
|
|
27
|
+
constructor(conn: NexoConnection, queueName: string, logger: Logger);
|
|
28
|
+
/**
|
|
29
|
+
* Peek messages in the DLQ without consuming them.
|
|
30
|
+
* @param limit Maximum number of messages to return (default: 10)
|
|
31
|
+
* @param offset Pagination offset (default: 0)
|
|
32
|
+
* @returns Object containing total count and array of messages
|
|
33
|
+
*/
|
|
34
|
+
peek(limit?: number, offset?: number): Promise<{
|
|
35
|
+
total: number;
|
|
36
|
+
items: {
|
|
37
|
+
id: string;
|
|
38
|
+
data: T;
|
|
39
|
+
attempts: number;
|
|
40
|
+
failureReason: string;
|
|
41
|
+
}[];
|
|
42
|
+
}>;
|
|
43
|
+
/**
|
|
44
|
+
* Move a message from DLQ back to the main queue (replay/retry).
|
|
45
|
+
* The message will be reset with attempts = 0 and become available for consumption.
|
|
46
|
+
* @param messageId ID of the message to move
|
|
47
|
+
* @returns true if the message was moved, false if not found
|
|
48
|
+
*/
|
|
49
|
+
moveToQueue(messageId: string): Promise<boolean>;
|
|
50
|
+
/**
|
|
51
|
+
* Delete a specific message from the DLQ.
|
|
52
|
+
* @param messageId ID of the message to delete
|
|
53
|
+
* @returns true if the message was deleted, false if not found
|
|
54
|
+
*/
|
|
55
|
+
delete(messageId: string): Promise<boolean>;
|
|
56
|
+
/**
|
|
57
|
+
* Purge all messages from the DLQ.
|
|
58
|
+
* @returns Number of messages purged
|
|
59
|
+
*/
|
|
60
|
+
purge(): Promise<number>;
|
|
61
|
+
}
|
|
19
62
|
export declare class NexoQueue<T = any> {
|
|
20
63
|
private conn;
|
|
21
64
|
readonly name: string;
|
|
22
65
|
private logger;
|
|
23
66
|
private isSubscribed;
|
|
67
|
+
private _dlq;
|
|
24
68
|
constructor(conn: NexoConnection, name: string, logger: Logger);
|
|
69
|
+
/**
|
|
70
|
+
* Access the Dead Letter Queue (DLQ) for this queue.
|
|
71
|
+
* Use this to inspect, replay, delete, or purge failed messages.
|
|
72
|
+
*/
|
|
73
|
+
get dlq(): NexoDLQ<T>;
|
|
25
74
|
create(config?: QueueConfig): Promise<this>;
|
|
26
75
|
exists(): Promise<boolean>;
|
|
27
76
|
delete(): Promise<void>;
|
|
@@ -30,4 +79,5 @@ export declare class NexoQueue<T = any> {
|
|
|
30
79
|
stop: () => void;
|
|
31
80
|
}>;
|
|
32
81
|
private ack;
|
|
82
|
+
private nack;
|
|
33
83
|
}
|
package/dist/brokers/queue.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.NexoQueue = void 0;
|
|
3
|
+
exports.NexoQueue = exports.NexoDLQ = void 0;
|
|
4
4
|
const codec_1 = require("../codec");
|
|
5
5
|
const errors_1 = require("../errors");
|
|
6
6
|
var QueueOpcode;
|
|
@@ -11,6 +11,11 @@ var QueueOpcode;
|
|
|
11
11
|
QueueOpcode[QueueOpcode["Q_ACK"] = 19] = "Q_ACK";
|
|
12
12
|
QueueOpcode[QueueOpcode["Q_EXISTS"] = 20] = "Q_EXISTS";
|
|
13
13
|
QueueOpcode[QueueOpcode["Q_DELETE"] = 21] = "Q_DELETE";
|
|
14
|
+
QueueOpcode[QueueOpcode["Q_PEEK_DLQ"] = 22] = "Q_PEEK_DLQ";
|
|
15
|
+
QueueOpcode[QueueOpcode["Q_MOVE_TO_QUEUE"] = 23] = "Q_MOVE_TO_QUEUE";
|
|
16
|
+
QueueOpcode[QueueOpcode["Q_DELETE_DLQ"] = 24] = "Q_DELETE_DLQ";
|
|
17
|
+
QueueOpcode[QueueOpcode["Q_PURGE_DLQ"] = 25] = "Q_PURGE_DLQ";
|
|
18
|
+
QueueOpcode[QueueOpcode["Q_NACK"] = 26] = "Q_NACK";
|
|
14
19
|
})(QueueOpcode || (QueueOpcode = {}));
|
|
15
20
|
const QueueCommands = {
|
|
16
21
|
create: (conn, name, config) => conn.send(QueueOpcode.Q_CREATE, codec_1.FrameCodec.string(name), codec_1.FrameCodec.string(JSON.stringify(config || {}))),
|
|
@@ -41,6 +46,36 @@ const QueueCommands = {
|
|
|
41
46
|
return messages;
|
|
42
47
|
},
|
|
43
48
|
ack: (conn, name, id) => conn.send(QueueOpcode.Q_ACK, codec_1.FrameCodec.uuid(id), codec_1.FrameCodec.string(name)),
|
|
49
|
+
nack: (conn, name, id, reason) => conn.send(QueueOpcode.Q_NACK, codec_1.FrameCodec.uuid(id), codec_1.FrameCodec.string(name), codec_1.FrameCodec.string(reason)),
|
|
50
|
+
// DLQ Commands
|
|
51
|
+
peekDLQ: async (conn, name, limit, offset) => {
|
|
52
|
+
const res = await conn.send(QueueOpcode.Q_PEEK_DLQ, codec_1.FrameCodec.string(name), codec_1.FrameCodec.u32(limit), codec_1.FrameCodec.u32(offset));
|
|
53
|
+
const total = res.cursor.readU32();
|
|
54
|
+
const count = res.cursor.readU32();
|
|
55
|
+
const items = [];
|
|
56
|
+
for (let i = 0; i < count; i++) {
|
|
57
|
+
const idHex = res.cursor.readUUID();
|
|
58
|
+
const payloadLen = res.cursor.readU32();
|
|
59
|
+
const payloadBuf = res.cursor.readBuffer(payloadLen);
|
|
60
|
+
const data = codec_1.FrameCodec.decodeAny(new codec_1.Cursor(payloadBuf));
|
|
61
|
+
const attempts = res.cursor.readU32();
|
|
62
|
+
const failureReason = res.cursor.readString();
|
|
63
|
+
items.push({ id: idHex, data, attempts, failureReason });
|
|
64
|
+
}
|
|
65
|
+
return { total, items };
|
|
66
|
+
},
|
|
67
|
+
moveToQueue: async (conn, name, messageId) => {
|
|
68
|
+
const res = await conn.send(QueueOpcode.Q_MOVE_TO_QUEUE, codec_1.FrameCodec.string(name), codec_1.FrameCodec.uuid(messageId));
|
|
69
|
+
return res.cursor.readU8() === 1;
|
|
70
|
+
},
|
|
71
|
+
deleteDLQ: async (conn, name, messageId) => {
|
|
72
|
+
const res = await conn.send(QueueOpcode.Q_DELETE_DLQ, codec_1.FrameCodec.string(name), codec_1.FrameCodec.uuid(messageId));
|
|
73
|
+
return res.cursor.readU8() === 1;
|
|
74
|
+
},
|
|
75
|
+
purgeDLQ: async (conn, name) => {
|
|
76
|
+
const res = await conn.send(QueueOpcode.Q_PURGE_DLQ, codec_1.FrameCodec.string(name));
|
|
77
|
+
return res.cursor.readU32();
|
|
78
|
+
},
|
|
44
79
|
};
|
|
45
80
|
async function runConcurrent(items, concurrency, fn) {
|
|
46
81
|
if (concurrency === 1) {
|
|
@@ -55,12 +90,69 @@ async function runConcurrent(items, concurrency, fn) {
|
|
|
55
90
|
});
|
|
56
91
|
await Promise.all(workers);
|
|
57
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* Dead Letter Queue (DLQ) management for a queue.
|
|
95
|
+
* Provides methods to inspect, replay, delete, and purge failed messages.
|
|
96
|
+
*/
|
|
97
|
+
class NexoDLQ {
|
|
98
|
+
constructor(conn, queueName, logger) {
|
|
99
|
+
this.conn = conn;
|
|
100
|
+
this.queueName = queueName;
|
|
101
|
+
this.logger = logger;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Peek messages in the DLQ without consuming them.
|
|
105
|
+
* @param limit Maximum number of messages to return (default: 10)
|
|
106
|
+
* @param offset Pagination offset (default: 0)
|
|
107
|
+
* @returns Object containing total count and array of messages
|
|
108
|
+
*/
|
|
109
|
+
async peek(limit = 10, offset = 0) {
|
|
110
|
+
this.logger.debug(`[DLQ:${this.queueName}] Peeking ${limit} messages at offset ${offset}`);
|
|
111
|
+
return QueueCommands.peekDLQ(this.conn, this.queueName, limit, offset);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Move a message from DLQ back to the main queue (replay/retry).
|
|
115
|
+
* The message will be reset with attempts = 0 and become available for consumption.
|
|
116
|
+
* @param messageId ID of the message to move
|
|
117
|
+
* @returns true if the message was moved, false if not found
|
|
118
|
+
*/
|
|
119
|
+
async moveToQueue(messageId) {
|
|
120
|
+
this.logger.debug(`[DLQ:${this.queueName}] Moving message ${messageId} to main queue`);
|
|
121
|
+
return QueueCommands.moveToQueue(this.conn, this.queueName, messageId);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Delete a specific message from the DLQ.
|
|
125
|
+
* @param messageId ID of the message to delete
|
|
126
|
+
* @returns true if the message was deleted, false if not found
|
|
127
|
+
*/
|
|
128
|
+
async delete(messageId) {
|
|
129
|
+
this.logger.debug(`[DLQ:${this.queueName}] Deleting message ${messageId}`);
|
|
130
|
+
return QueueCommands.deleteDLQ(this.conn, this.queueName, messageId);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Purge all messages from the DLQ.
|
|
134
|
+
* @returns Number of messages purged
|
|
135
|
+
*/
|
|
136
|
+
async purge() {
|
|
137
|
+
this.logger.debug(`[DLQ:${this.queueName}] Purging all messages`);
|
|
138
|
+
return QueueCommands.purgeDLQ(this.conn, this.queueName);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
exports.NexoDLQ = NexoDLQ;
|
|
58
142
|
class NexoQueue {
|
|
59
143
|
constructor(conn, name, logger) {
|
|
60
144
|
this.conn = conn;
|
|
61
145
|
this.name = name;
|
|
62
146
|
this.logger = logger;
|
|
63
147
|
this.isSubscribed = false;
|
|
148
|
+
this._dlq = new NexoDLQ(conn, name, logger);
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Access the Dead Letter Queue (DLQ) for this queue.
|
|
152
|
+
* Use this to inspect, replay, delete, or purge failed messages.
|
|
153
|
+
*/
|
|
154
|
+
get dlq() {
|
|
155
|
+
return this._dlq;
|
|
64
156
|
}
|
|
65
157
|
async create(config = {}) {
|
|
66
158
|
await QueueCommands.create(this.conn, this.name, config);
|
|
@@ -93,8 +185,6 @@ class NexoQueue {
|
|
|
93
185
|
await new Promise(r => setTimeout(r, 500));
|
|
94
186
|
continue;
|
|
95
187
|
}
|
|
96
|
-
// Add small jitter to avoid thundering herd on reconnect
|
|
97
|
-
await new Promise(r => setTimeout(r, Math.random() * 500));
|
|
98
188
|
try {
|
|
99
189
|
// Double check before sending
|
|
100
190
|
if (!this.conn.isConnected)
|
|
@@ -103,8 +193,13 @@ class NexoQueue {
|
|
|
103
193
|
if (messages.length === 0)
|
|
104
194
|
continue;
|
|
105
195
|
await runConcurrent(messages, concurrency, async (msg) => {
|
|
106
|
-
if (!active)
|
|
196
|
+
if (!active) {
|
|
197
|
+
if (this.conn.isConnected) {
|
|
198
|
+
// If stopped while consuming, release message immediately so it's available for others
|
|
199
|
+
await this.nack(msg.id, "Consumer stopped").catch(() => { });
|
|
200
|
+
}
|
|
107
201
|
return;
|
|
202
|
+
}
|
|
108
203
|
try {
|
|
109
204
|
await callback(msg.data);
|
|
110
205
|
await this.ack(msg.id);
|
|
@@ -112,7 +207,9 @@ class NexoQueue {
|
|
|
112
207
|
catch (e) {
|
|
113
208
|
if (!this.conn.isConnected)
|
|
114
209
|
return;
|
|
115
|
-
|
|
210
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
211
|
+
this.logger.error(`[Queue:${this.name}] Consumer error, sending NACK. Reason: ${reason}`);
|
|
212
|
+
await this.nack(msg.id, reason);
|
|
116
213
|
}
|
|
117
214
|
});
|
|
118
215
|
}
|
|
@@ -133,10 +230,18 @@ class NexoQueue {
|
|
|
133
230
|
loop().catch(err => {
|
|
134
231
|
this.logger.error(`[CRITICAL] Queue loop crashed for ${this.name}`, err);
|
|
135
232
|
});
|
|
136
|
-
return {
|
|
233
|
+
return {
|
|
234
|
+
stop: () => {
|
|
235
|
+
active = false;
|
|
236
|
+
this.isSubscribed = false;
|
|
237
|
+
}
|
|
238
|
+
};
|
|
137
239
|
}
|
|
138
240
|
async ack(id) {
|
|
139
241
|
await QueueCommands.ack(this.conn, this.name, id);
|
|
140
242
|
}
|
|
243
|
+
async nack(id, reason) {
|
|
244
|
+
await QueueCommands.nack(this.conn, this.name, id, reason);
|
|
245
|
+
}
|
|
141
246
|
}
|
|
142
247
|
exports.NexoQueue = NexoQueue;
|
package/dist/brokers/stream.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { NexoConnection } from '../connection';
|
|
2
2
|
import { Logger } from '../utils/logger';
|
|
3
|
-
export type PersistenceStrategy = '
|
|
3
|
+
export type PersistenceStrategy = 'file_sync' | 'file_async';
|
|
4
4
|
export interface RetentionOptions {
|
|
5
5
|
maxAgeMs: number;
|
|
6
6
|
maxBytes: number;
|
package/dist/brokers/stream.js
CHANGED
package/dist/codec.js
CHANGED
|
@@ -51,7 +51,10 @@ class FrameCodec {
|
|
|
51
51
|
return result;
|
|
52
52
|
}
|
|
53
53
|
static uuid(hex) {
|
|
54
|
-
|
|
54
|
+
// Remove dashes from UUID format (e.g., "550e8400-e29b-41d4-a716-446655440000")
|
|
55
|
+
// This handles both formats: with dashes (from crypto.randomUUID()) and without (from server)
|
|
56
|
+
const cleanHex = hex.replace(/-/g, '');
|
|
57
|
+
return Buffer.from(cleanHex, 'hex');
|
|
55
58
|
}
|
|
56
59
|
static any(data) {
|
|
57
60
|
let type = protocol_1.DataType.RAW;
|
|
@@ -74,25 +77,24 @@ class FrameCodec {
|
|
|
74
77
|
return result;
|
|
75
78
|
}
|
|
76
79
|
static packRequest(id, opcode, ...parts) {
|
|
77
|
-
// 1. Calcolo dimensione PAYLOAD
|
|
78
|
-
let payloadSize =
|
|
80
|
+
// 1. Calcolo dimensione PAYLOAD
|
|
81
|
+
let payloadSize = 0;
|
|
79
82
|
for (const part of parts)
|
|
80
83
|
payloadSize += part.length;
|
|
81
|
-
// 2. Allocazione buffer TOTALE (Header [
|
|
82
|
-
// Header
|
|
83
|
-
const result = Buffer.allocUnsafe(
|
|
84
|
-
// --- SCRITTURA HEADER (Bytes 0-
|
|
84
|
+
// 2. Allocazione buffer TOTALE (Header [10 bytes] + Payload)
|
|
85
|
+
// Header: [Type:1][Opcode:1][ID:4][Len:4]
|
|
86
|
+
const result = Buffer.allocUnsafe(10 + payloadSize);
|
|
87
|
+
// --- SCRITTURA HEADER (Bytes 0-9) ---
|
|
85
88
|
// Byte 0: Tipo di Frame (REQUEST = 0x01)
|
|
86
89
|
result.writeUInt8(protocol_1.FrameType.REQUEST, 0);
|
|
87
|
-
//
|
|
88
|
-
result.
|
|
89
|
-
// Bytes 5
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
//
|
|
94
|
-
|
|
95
|
-
// Bytes 10+: Copia degli argomenti (dati effettivi)
|
|
90
|
+
// Byte 1: Opcode
|
|
91
|
+
result.writeUInt8(opcode, 1);
|
|
92
|
+
// Bytes 2-5: ID della richiesta (UInt32 Big Endian)
|
|
93
|
+
result.writeUInt32BE(id, 2);
|
|
94
|
+
// Bytes 6-9: Lunghezza del Payload (UInt32 Big Endian)
|
|
95
|
+
result.writeUInt32BE(payloadSize, 6);
|
|
96
|
+
// --- SCRITTURA PAYLOAD (Bytes 10+) ---
|
|
97
|
+
// Copia degli argomenti (dati effettivi)
|
|
96
98
|
let offset = 10;
|
|
97
99
|
for (const part of parts) {
|
|
98
100
|
part.copy(result, offset);
|
package/dist/connection.js
CHANGED
|
@@ -54,7 +54,7 @@ class NexoConnection extends events_1.EventEmitter {
|
|
|
54
54
|
this.port = options.port;
|
|
55
55
|
this.logger = logger;
|
|
56
56
|
this.socket = new net.Socket();
|
|
57
|
-
this.requestTimeoutMs = options.requestTimeoutMs ??
|
|
57
|
+
this.requestTimeoutMs = options.requestTimeoutMs ?? 15000;
|
|
58
58
|
}
|
|
59
59
|
async connect() {
|
|
60
60
|
this.shouldReconnect = true;
|
|
@@ -126,11 +126,11 @@ class NexoConnection extends events_1.EventEmitter {
|
|
|
126
126
|
this.chunks = [];
|
|
127
127
|
}
|
|
128
128
|
while (true) {
|
|
129
|
-
// Need at least header (
|
|
130
|
-
if (this.buffer.length <
|
|
129
|
+
// Need at least header (10 bytes): [Type:1][Opcode:1][ID:4][Len:4]
|
|
130
|
+
if (this.buffer.length < 10)
|
|
131
131
|
break;
|
|
132
|
-
const payloadLen = this.buffer.readUInt32BE(
|
|
133
|
-
const totalFrameLen =
|
|
132
|
+
const payloadLen = this.buffer.readUInt32BE(6);
|
|
133
|
+
const totalFrameLen = 10 + payloadLen;
|
|
134
134
|
if (this.buffer.length < totalFrameLen)
|
|
135
135
|
break;
|
|
136
136
|
// 2. Slice Frame
|
|
@@ -142,6 +142,7 @@ class NexoConnection extends events_1.EventEmitter {
|
|
|
142
142
|
handleFrame(frame) {
|
|
143
143
|
const cursor = new codec_1.Cursor(frame);
|
|
144
144
|
const type = cursor.readU8();
|
|
145
|
+
const opcodeOrStatus = cursor.readU8(); // Opcode for requests, Status for responses
|
|
145
146
|
const id = cursor.readU32();
|
|
146
147
|
cursor.readU32(); // Skip payloadLen
|
|
147
148
|
const payload = cursor.buf.subarray(cursor.offset);
|
|
@@ -150,8 +151,8 @@ class NexoConnection extends events_1.EventEmitter {
|
|
|
150
151
|
const req = this.pending.get(id);
|
|
151
152
|
if (req) {
|
|
152
153
|
this.pending.delete(id);
|
|
153
|
-
//
|
|
154
|
-
req.resolve({ status:
|
|
154
|
+
// Status is in header (byte 1), payload is clean data
|
|
155
|
+
req.resolve({ status: opcodeOrStatus, data: payload });
|
|
155
156
|
}
|
|
156
157
|
break;
|
|
157
158
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@emanuelepifani/nexo-client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.8",
|
|
4
4
|
"description": "High-performance TypeScript Client for Nexo Broker",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
],
|
|
14
14
|
"scripts": {
|
|
15
15
|
"build": "tsc",
|
|
16
|
+
"test": "vitest run",
|
|
17
|
+
"test:watch": "vitest",
|
|
16
18
|
"prepublishOnly": "npm run build"
|
|
17
19
|
},
|
|
18
20
|
"repository": {
|
|
@@ -37,6 +39,9 @@
|
|
|
37
39
|
],
|
|
38
40
|
"devDependencies": {
|
|
39
41
|
"@types/node": "^20.10.0",
|
|
40
|
-
"typescript": "^5.3.0"
|
|
42
|
+
"typescript": "^5.3.0",
|
|
43
|
+
"vitest": "^1.2.0",
|
|
44
|
+
"tsx": "^4.7.0",
|
|
45
|
+
"zod": "^4.2.1"
|
|
41
46
|
}
|
|
42
47
|
}
|
package/dist/performance-test.js
DELETED
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.performanceTest = performanceTest;
|
|
4
|
-
const ringbuffer_1 = require("./ringbuffer");
|
|
5
|
-
// Simple performance test for RingBuffer vs Buffer.concat
|
|
6
|
-
function performanceTest() {
|
|
7
|
-
const iterations = 10000;
|
|
8
|
-
const chunkSize = 1024;
|
|
9
|
-
const totalData = iterations * chunkSize;
|
|
10
|
-
console.log(`Testing performance with ${iterations} chunks of ${chunkSize} bytes each...`);
|
|
11
|
-
// Test 1: Buffer.concat (old method)
|
|
12
|
-
console.time('Buffer.concat');
|
|
13
|
-
const chunks = [];
|
|
14
|
-
let totalBuffer = Buffer.alloc(0);
|
|
15
|
-
for (let i = 0; i < iterations; i++) {
|
|
16
|
-
const chunk = Buffer.allocUnsafe(chunkSize);
|
|
17
|
-
chunk.fill(Math.floor(Math.random() * 256));
|
|
18
|
-
chunks.push(chunk);
|
|
19
|
-
// Simulate periodic concatenation (like the old code)
|
|
20
|
-
if (chunks.length > 10) {
|
|
21
|
-
totalBuffer = Buffer.concat([totalBuffer, ...chunks]);
|
|
22
|
-
chunks.length = 0;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
// Final concat
|
|
26
|
-
if (chunks.length > 0) {
|
|
27
|
-
totalBuffer = Buffer.concat([totalBuffer, ...chunks]);
|
|
28
|
-
}
|
|
29
|
-
console.timeEnd('Buffer.concat');
|
|
30
|
-
// Test 2: RingBuffer (new method)
|
|
31
|
-
console.time('RingBuffer');
|
|
32
|
-
const ringBuffer = new ringbuffer_1.RingBuffer();
|
|
33
|
-
for (let i = 0; i < iterations; i++) {
|
|
34
|
-
const chunk = Buffer.allocUnsafe(chunkSize);
|
|
35
|
-
chunk.fill(Math.floor(Math.random() * 256));
|
|
36
|
-
ringBuffer.write(chunk);
|
|
37
|
-
}
|
|
38
|
-
// Read all data
|
|
39
|
-
const result = ringBuffer.consume(ringBuffer.available);
|
|
40
|
-
console.timeEnd('RingBuffer');
|
|
41
|
-
console.log(`Total data processed: ${(totalData / 1024 / 1024).toFixed(2)} MB`);
|
|
42
|
-
console.log(`Buffer.concat result length: ${totalBuffer.length}`);
|
|
43
|
-
console.log(`RingBuffer result length: ${result.length}`);
|
|
44
|
-
// Memory usage comparison
|
|
45
|
-
const memBefore = process.memoryUsage();
|
|
46
|
-
// Test memory efficiency
|
|
47
|
-
const largeRingBuffer = new ringbuffer_1.RingBuffer(1024 * 1024); // 1MB initial
|
|
48
|
-
for (let i = 0; i < 1000; i++) {
|
|
49
|
-
const chunk = Buffer.allocUnsafe(4096);
|
|
50
|
-
largeRingBuffer.write(chunk);
|
|
51
|
-
}
|
|
52
|
-
const memAfter = process.memoryUsage();
|
|
53
|
-
console.log(`Memory usage increase: ${((memAfter.heapUsed - memBefore.heapUsed) / 1024 / 1024).toFixed(2)} MB`);
|
|
54
|
-
}
|
|
55
|
-
// Run the test
|
|
56
|
-
if (require.main === module) {
|
|
57
|
-
performanceTest();
|
|
58
|
-
}
|
package/dist/ringbuffer.d.ts
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
/** @internal */
|
|
2
|
-
export declare class RingBuffer {
|
|
3
|
-
private buffer;
|
|
4
|
-
private head;
|
|
5
|
-
private tail;
|
|
6
|
-
private size;
|
|
7
|
-
private capacity;
|
|
8
|
-
constructor(initialCapacity?: number);
|
|
9
|
-
/** Write data to ring buffer */
|
|
10
|
-
write(data: Buffer): number;
|
|
11
|
-
/** Read data from ring buffer without consuming it */
|
|
12
|
-
peek(length: number): Buffer;
|
|
13
|
-
/** Consume data from ring buffer */
|
|
14
|
-
consume(length: number): Buffer;
|
|
15
|
-
/** Get current data size */
|
|
16
|
-
get available(): number;
|
|
17
|
-
/** Get available space for writing */
|
|
18
|
-
get freeSpace(): number;
|
|
19
|
-
/** Check if buffer is empty */
|
|
20
|
-
get isEmpty(): boolean;
|
|
21
|
-
/** Clear all data */
|
|
22
|
-
clear(): void;
|
|
23
|
-
/** Ensure capacity for new data */
|
|
24
|
-
private ensureCapacity;
|
|
25
|
-
/** Get next power of 2 */
|
|
26
|
-
private nextPowerOf2;
|
|
27
|
-
}
|
package/dist/ringbuffer.js
DELETED
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.RingBuffer = void 0;
|
|
4
|
-
/** @internal */
|
|
5
|
-
class RingBuffer {
|
|
6
|
-
constructor(initialCapacity = 8192) {
|
|
7
|
-
this.head = 0; // write position
|
|
8
|
-
this.tail = 0; // read position
|
|
9
|
-
this.size = 0; // current data size
|
|
10
|
-
this.capacity = this.nextPowerOf2(initialCapacity);
|
|
11
|
-
this.buffer = Buffer.allocUnsafe(this.capacity);
|
|
12
|
-
}
|
|
13
|
-
/** Write data to ring buffer */
|
|
14
|
-
write(data) {
|
|
15
|
-
const dataLen = data.length;
|
|
16
|
-
// Ensure we have enough space
|
|
17
|
-
this.ensureCapacity(this.size + dataLen);
|
|
18
|
-
// Write data in potentially two chunks (if wrapping around)
|
|
19
|
-
const availableToEnd = this.capacity - this.head;
|
|
20
|
-
if (dataLen <= availableToEnd) {
|
|
21
|
-
// Fits in one chunk
|
|
22
|
-
data.copy(this.buffer, this.head);
|
|
23
|
-
this.head = (this.head + dataLen) & (this.capacity - 1);
|
|
24
|
-
}
|
|
25
|
-
else {
|
|
26
|
-
// Needs to wrap around
|
|
27
|
-
const firstChunk = data.subarray(0, availableToEnd);
|
|
28
|
-
const secondChunk = data.subarray(availableToEnd);
|
|
29
|
-
firstChunk.copy(this.buffer, this.head);
|
|
30
|
-
secondChunk.copy(this.buffer, 0);
|
|
31
|
-
this.head = secondChunk.length;
|
|
32
|
-
}
|
|
33
|
-
this.size += dataLen;
|
|
34
|
-
return dataLen;
|
|
35
|
-
}
|
|
36
|
-
/** Read data from ring buffer without consuming it */
|
|
37
|
-
peek(length) {
|
|
38
|
-
if (length > this.size) {
|
|
39
|
-
length = this.size;
|
|
40
|
-
}
|
|
41
|
-
if (length === 0)
|
|
42
|
-
return Buffer.alloc(0);
|
|
43
|
-
const availableToEnd = this.capacity - this.tail;
|
|
44
|
-
if (length <= availableToEnd) {
|
|
45
|
-
// Data is contiguous
|
|
46
|
-
return this.buffer.subarray(this.tail, this.tail + length);
|
|
47
|
-
}
|
|
48
|
-
else {
|
|
49
|
-
// Data wraps around, need to concatenate
|
|
50
|
-
const firstChunk = this.buffer.subarray(this.tail, this.capacity);
|
|
51
|
-
const secondChunk = this.buffer.subarray(0, length - availableToEnd);
|
|
52
|
-
return Buffer.concat([firstChunk, secondChunk]);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
/** Consume data from ring buffer */
|
|
56
|
-
consume(length) {
|
|
57
|
-
const data = this.peek(length);
|
|
58
|
-
this.tail = (this.tail + length) & (this.capacity - 1);
|
|
59
|
-
this.size -= length;
|
|
60
|
-
return data;
|
|
61
|
-
}
|
|
62
|
-
/** Get current data size */
|
|
63
|
-
get available() {
|
|
64
|
-
return this.size;
|
|
65
|
-
}
|
|
66
|
-
/** Get available space for writing */
|
|
67
|
-
get freeSpace() {
|
|
68
|
-
return this.capacity - this.size;
|
|
69
|
-
}
|
|
70
|
-
/** Check if buffer is empty */
|
|
71
|
-
get isEmpty() {
|
|
72
|
-
return this.size === 0;
|
|
73
|
-
}
|
|
74
|
-
/** Clear all data */
|
|
75
|
-
clear() {
|
|
76
|
-
this.head = 0;
|
|
77
|
-
this.tail = 0;
|
|
78
|
-
this.size = 0;
|
|
79
|
-
}
|
|
80
|
-
/** Ensure capacity for new data */
|
|
81
|
-
ensureCapacity(required) {
|
|
82
|
-
if (required <= this.capacity)
|
|
83
|
-
return;
|
|
84
|
-
const newCapacity = this.nextPowerOf2(required);
|
|
85
|
-
const newBuffer = Buffer.allocUnsafe(newCapacity);
|
|
86
|
-
// Copy existing data to new buffer
|
|
87
|
-
if (this.size > 0) {
|
|
88
|
-
const data = this.peek(this.size);
|
|
89
|
-
data.copy(newBuffer, 0);
|
|
90
|
-
}
|
|
91
|
-
this.buffer = newBuffer;
|
|
92
|
-
this.capacity = newCapacity;
|
|
93
|
-
this.head = this.size;
|
|
94
|
-
this.tail = 0;
|
|
95
|
-
}
|
|
96
|
-
/** Get next power of 2 */
|
|
97
|
-
nextPowerOf2(n) {
|
|
98
|
-
if (n <= 0)
|
|
99
|
-
return 1;
|
|
100
|
-
n--;
|
|
101
|
-
n |= n >> 1;
|
|
102
|
-
n |= n >> 2;
|
|
103
|
-
n |= n >> 4;
|
|
104
|
-
n |= n >> 8;
|
|
105
|
-
n |= n >> 16;
|
|
106
|
-
return n + 1;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
exports.RingBuffer = RingBuffer;
|