@emanuelepifani/nexo-client 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +219 -0
- package/dist/brokers/pubsub.d.ts +41 -0
- package/dist/brokers/pubsub.js +81 -0
- package/dist/brokers/queue.d.ts +66 -0
- package/dist/brokers/queue.js +121 -0
- package/dist/brokers/store.d.ts +32 -0
- package/dist/brokers/store.js +42 -0
- package/dist/brokers/stream.d.ts +66 -0
- package/dist/brokers/stream.js +201 -0
- package/dist/client.d.ts +27 -0
- package/dist/client.js +71 -0
- package/dist/codec.d.ts +23 -0
- package/dist/codec.js +116 -0
- package/dist/connection.d.ts +28 -0
- package/dist/connection.js +183 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +15 -0
- package/dist/protocol.d.ts +22 -0
- package/dist/protocol.js +28 -0
- package/dist/utils/logger.d.ts +12 -0
- package/dist/utils/logger.js +60 -0
- package/package.json +34 -0
package/README.md
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# Nexo Client SDK
|
|
2
|
+
|
|
3
|
+
High-performance TypeScript client for [Nexo Broker](https://github.com/emanuel-epifani/nexo).
|
|
4
|
+
|
|
5
|
+
## Quick Start
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### Run NEXO server
|
|
9
|
+
```bash
|
|
10
|
+
docker run -p 7654:7654 -p 8080:8080 emanuelepifani/nexo:latest
|
|
11
|
+
```
|
|
12
|
+
This exposes:
|
|
13
|
+
- Port 7654 (TCP): Main server socket for SDK clients.
|
|
14
|
+
- Port 8080 (HTTP): Web Dashboard with status of all brokers.
|
|
15
|
+
|
|
16
|
+
### Install SDK
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npm install @emanuelepifani/nexo-client
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### Connection
|
|
23
|
+
```typescript
|
|
24
|
+
const client = await NexoClient.connect({ host: 'localhost', port: 7654 });
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
### 1. STORE (Key-Value Map)
|
|
28
|
+
|
|
29
|
+
```typescript
|
|
30
|
+
// SET key
|
|
31
|
+
await client.store.map.set("user:1", { name: "Max", role: "admin" });
|
|
32
|
+
|
|
33
|
+
// GET key
|
|
34
|
+
const user = await client.store.map.get<User>("user:1");
|
|
35
|
+
|
|
36
|
+
// DEL key
|
|
37
|
+
await client.store.map.del("user:1");
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### 2. QUEUE
|
|
41
|
+
|
|
42
|
+
```typescript
|
|
43
|
+
// Create queue
|
|
44
|
+
const mailQ = await client.queue<MailJob>("emails").create();
|
|
45
|
+
// Push message
|
|
46
|
+
await mailQ.push({ to: "test@test.com" });
|
|
47
|
+
// Subscribe
|
|
48
|
+
await mailQ.subscribe((msg) => console.log(msg));
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
<details>
|
|
52
|
+
<summary><strong>Advanced Queue Features (Retry, Delay, Priority, Concurrency)</strong></summary>
|
|
53
|
+
|
|
54
|
+
```typescript
|
|
55
|
+
// -------------------------------------------------------------
|
|
56
|
+
// 1. CREATION (Default behavior for all messages in this queue)
|
|
57
|
+
// -------------------------------------------------------------
|
|
58
|
+
interface CriticalTask { type: string; payload: any; }
|
|
59
|
+
|
|
60
|
+
const criticalQueue = await client.queue<CriticalTask>('critical-tasks').create({
|
|
61
|
+
// RELIABILITY:
|
|
62
|
+
visibilityTimeoutMs: 10000, // Retry delivery if not ACKed within 10s (default=30s)
|
|
63
|
+
maxRetries: 5, // Move to DLQ after 5 failures (default=5)
|
|
64
|
+
ttlMs: 60000, // Message expires if not consumed in 60s (default=7days)
|
|
65
|
+
|
|
66
|
+
// PERSISTENCE:
|
|
67
|
+
// - strategy: 'memory' -> Volatile (Fastest, lost on restart)
|
|
68
|
+
// - strategy: 'file_sync' -> Save every message (Safest, Slowest)
|
|
69
|
+
// - strategy: 'file_async' -> Buffer & flush periodically (Fast & Durable) -> DEFAULT
|
|
70
|
+
// DEFAULT: { strategy: 'file_async', flushIntervalMs: 1000 }
|
|
71
|
+
persistence: {
|
|
72
|
+
strategy: 'file_async',
|
|
73
|
+
flushIntervalMs: 100
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
// ---------------------------------------------------------
|
|
79
|
+
// 2. PRODUCING (Override specific behaviors per message)
|
|
80
|
+
// ---------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
// PRIORITY: Higher value (255) delivered before lower values (0)
|
|
83
|
+
// This message jumps ahead of all priority < 255 messages sent previously and still not consumed.
|
|
84
|
+
await criticalQueue.push({ type: 'urgent' }, { priority: 255 });
|
|
85
|
+
|
|
86
|
+
// SCHEDULING: Delay visibility
|
|
87
|
+
// This message is hidden for 1 hour (default delayMs: 0, instant)
|
|
88
|
+
await criticalQueue.push({ type: 'scheduled' }, { delayMs: 3600000 });
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
// ---------------------------------------------------------
|
|
93
|
+
// 3. CONSUMING (Worker Tuning to optimize throughput and latency)
|
|
94
|
+
// ---------------------------------------------------------
|
|
95
|
+
await criticalQueue.subscribe(
|
|
96
|
+
async (task) => { await processTask(task); },
|
|
97
|
+
{
|
|
98
|
+
batchSize: 100, // Network: Fetch 100 messages in one request
|
|
99
|
+
concurrency: 10, // Local: Process 10 messages concurrently (useful for I/O tasks)
|
|
100
|
+
waitMs: 5000 // Polling: If empty, wait 5s for new messages before retrying
|
|
101
|
+
}
|
|
102
|
+
);
|
|
103
|
+
```
|
|
104
|
+
</details>
|
|
105
|
+
|
|
106
|
+
### 3. PUB/SUB
|
|
107
|
+
|
|
108
|
+
```typescript
|
|
109
|
+
// Define topic (not need to create, auto-created on first publish)
|
|
110
|
+
const alerts = client.pubsub<AlertMsg>("system-alerts");
|
|
111
|
+
// Subscribe
|
|
112
|
+
await alerts.subscribe((msg) => console.log(msg));
|
|
113
|
+
// Publish
|
|
114
|
+
await alerts.publish({ level: "high" });
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
<details>
|
|
118
|
+
<summary><strong>Wildcards & Retained Messages</strong></summary>
|
|
119
|
+
|
|
120
|
+
```typescript
|
|
121
|
+
// WILDCARD SUBSCRIPTIONS
|
|
122
|
+
// ----------------------
|
|
123
|
+
|
|
124
|
+
// 1. Single-Level Wildcard (+)
|
|
125
|
+
// Matches: 'home/kitchen/light', 'home/garage/light'
|
|
126
|
+
const roomLights = client.pubsub<LightStatus>('home/+/light');
|
|
127
|
+
await roomLights.subscribe((status) => console.log('Light is:', status.state));
|
|
128
|
+
|
|
129
|
+
// 2. Multi-Level Wildcard (#)
|
|
130
|
+
// Matches all topics under 'sensors/'
|
|
131
|
+
const allSensors = client.pubsub<SensorData>('sensors/#');
|
|
132
|
+
await allSensors.subscribe((data) => console.log('Sensor value:', data.value));
|
|
133
|
+
|
|
134
|
+
// PUBLISHING (No wildcards allowed!)
|
|
135
|
+
// ---------------------------------
|
|
136
|
+
// You must publish to concrete topics with matching types
|
|
137
|
+
await client.pubsub<LightStatus>('home/kitchen/light').publish({ state: 'ON' });
|
|
138
|
+
await client.pubsub<SensorData>('sensors/kitchen/temp').publish({ value: 22.5, unit: 'C' });
|
|
139
|
+
|
|
140
|
+
// RETAINED MESSAGES
|
|
141
|
+
// -----------------
|
|
142
|
+
// Last value is stored and immediately sent to new subscribers
|
|
143
|
+
await client.pubsub<string>('config/theme').publish('dark', { retain: true });
|
|
144
|
+
```
|
|
145
|
+
</details>
|
|
146
|
+
|
|
147
|
+
### 4. STREAM
|
|
148
|
+
|
|
149
|
+
```typescript
|
|
150
|
+
// Create topic
|
|
151
|
+
const stream = await client.stream<UserEvent>('user-events').create();
|
|
152
|
+
// Publisher
|
|
153
|
+
await stream.publish({ type: 'login', userId: 'u1' });
|
|
154
|
+
// Consumer (must specify group)
|
|
155
|
+
await stream.subscribe('analytics', (msg) => {console.log(`User ${msg.userId} performed ${msg.type}`); });
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
<details>
|
|
159
|
+
<summary><strong>Consumer Groups & Scaling</strong></summary>
|
|
160
|
+
|
|
161
|
+
```typescript
|
|
162
|
+
// ---------------------------------------------------------
|
|
163
|
+
// 1. STREAM CREATION & POLICY
|
|
164
|
+
// ---------------------------------------------------------
|
|
165
|
+
const orders = await client.stream<Order>('orders').create({
|
|
166
|
+
// SCALING
|
|
167
|
+
partitions: 4, // Max concurrent consumers per group on same topic (default=8)
|
|
168
|
+
|
|
169
|
+
// PERSISTENCE:
|
|
170
|
+
// - strategy: 'memory' -> Volatile (Fastest, lost on restart)
|
|
171
|
+
// - strategy: 'file_sync' -> Save every message (Safest, Slowest)
|
|
172
|
+
// - strategy: 'file_async' -> Buffer & flush periodically (Fast & Durable) -> DEFAULT
|
|
173
|
+
// DEFAULT: { strategy: 'file_async', flushIntervalMs: 1000 }
|
|
174
|
+
persistence: {
|
|
175
|
+
strategy: 'file_async',
|
|
176
|
+
flushIntervalMs: 100
|
|
177
|
+
}
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
// ---------------------------------------------------------
|
|
181
|
+
// 2. CONSUMING (Scaling & Broadcast patterns)
|
|
182
|
+
// ---------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
// SCALING (Microservices Replicas / K8s Pods)
|
|
185
|
+
// Same Group ('workers') -> Automatic Load Balancing & Rebalancing
|
|
186
|
+
// Partitions are distributed among workers.
|
|
187
|
+
await orders.subscribe('workers', (order) => process(order));
|
|
188
|
+
await orders.subscribe('workers', (order) => process(order));
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
// BROADCAST (Independent Domains)
|
|
192
|
+
// Different Groups -> Each group gets a full copy of the stream.
|
|
193
|
+
// Useful for independent services reacting to the same event.
|
|
194
|
+
await orders.subscribe('analytics', (order) => trackMetrics(order));
|
|
195
|
+
await orders.subscribe('audit-log', (order) => saveAudit(order));
|
|
196
|
+
```
|
|
197
|
+
</details>
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
---
|
|
201
|
+
|
|
202
|
+
## License
|
|
203
|
+
|
|
204
|
+
MIT
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
## Links
|
|
208
|
+
|
|
209
|
+
- **Nexo Broker (Server):** [GitHub Repository](https://github.com/emanuel-epifani/nexo)
|
|
210
|
+
- **Server Docs:** [Nexo Internals & Architecture](https://github.com/emanuel-epifani/nexo/tree/main/docs)
|
|
211
|
+
- **SDK Source:** [sdk/ts](https://github.com/emanuel-epifani/nexo/tree/main/sdk/ts)
|
|
212
|
+
- **Docker Image:** [emanuelepifani/nexo](https://hub.docker.com/r/emanuelepifani/nexo)
|
|
213
|
+
|
|
214
|
+
## Author
|
|
215
|
+
|
|
216
|
+
Built by **Emanuel Epifani**.
|
|
217
|
+
|
|
218
|
+
- [LinkedIn](https://www.linkedin.com/in/emanuel-epifani/)
|
|
219
|
+
- [GitHub](https://github.com/emanuel-epifani)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { NexoConnection } from '../connection';
|
|
2
|
+
export declare enum PubSubOpcode {
|
|
3
|
+
PUB = 33,
|
|
4
|
+
SUB = 34,
|
|
5
|
+
UNSUB = 35
|
|
6
|
+
}
|
|
7
|
+
export declare const PubSubCommands: {
|
|
8
|
+
publish: (conn: NexoConnection, topic: string, data: any, options: PublishOptions) => Promise<{
|
|
9
|
+
status: import("../protocol").ResponseStatus;
|
|
10
|
+
cursor: import("../codec").Cursor;
|
|
11
|
+
}>;
|
|
12
|
+
subscribe: (conn: NexoConnection, topic: string) => Promise<{
|
|
13
|
+
status: import("../protocol").ResponseStatus;
|
|
14
|
+
cursor: import("../codec").Cursor;
|
|
15
|
+
}>;
|
|
16
|
+
unsubscribe: (conn: NexoConnection, topic: string) => Promise<{
|
|
17
|
+
status: import("../protocol").ResponseStatus;
|
|
18
|
+
cursor: import("../codec").Cursor;
|
|
19
|
+
}>;
|
|
20
|
+
};
|
|
21
|
+
export interface PublishOptions {
|
|
22
|
+
retain?: boolean;
|
|
23
|
+
}
|
|
24
|
+
export declare class NexoTopic<T = any> {
|
|
25
|
+
private broker;
|
|
26
|
+
readonly name: string;
|
|
27
|
+
constructor(broker: NexoPubSub, name: string);
|
|
28
|
+
publish(data: T, options?: PublishOptions): Promise<void>;
|
|
29
|
+
subscribe(cb: (data: T) => void): Promise<void>;
|
|
30
|
+
unsubscribe(): Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
export declare class NexoPubSub {
|
|
33
|
+
private conn;
|
|
34
|
+
private handlers;
|
|
35
|
+
constructor(conn: NexoConnection);
|
|
36
|
+
publish(topic: string, data: any, options?: PublishOptions): Promise<void>;
|
|
37
|
+
subscribe(topic: string, callback: (data: any) => void): Promise<void>;
|
|
38
|
+
unsubscribe(topic: string): Promise<void>;
|
|
39
|
+
private dispatch;
|
|
40
|
+
private matches;
|
|
41
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NexoPubSub = exports.NexoTopic = exports.PubSubCommands = exports.PubSubOpcode = void 0;
|
|
4
|
+
const codec_1 = require("../codec");
|
|
5
|
+
var PubSubOpcode;
|
|
6
|
+
(function (PubSubOpcode) {
|
|
7
|
+
PubSubOpcode[PubSubOpcode["PUB"] = 33] = "PUB";
|
|
8
|
+
PubSubOpcode[PubSubOpcode["SUB"] = 34] = "SUB";
|
|
9
|
+
PubSubOpcode[PubSubOpcode["UNSUB"] = 35] = "UNSUB";
|
|
10
|
+
})(PubSubOpcode || (exports.PubSubOpcode = PubSubOpcode = {}));
|
|
11
|
+
exports.PubSubCommands = {
|
|
12
|
+
publish: (conn, topic, data, options) => conn.send(PubSubOpcode.PUB, codec_1.FrameCodec.string(topic), codec_1.FrameCodec.string(JSON.stringify(options || {})), codec_1.FrameCodec.any(data)),
|
|
13
|
+
subscribe: (conn, topic) => conn.send(PubSubOpcode.SUB, codec_1.FrameCodec.string(topic)),
|
|
14
|
+
unsubscribe: (conn, topic) => conn.send(PubSubOpcode.UNSUB, codec_1.FrameCodec.string(topic)),
|
|
15
|
+
};
|
|
16
|
+
class NexoTopic {
|
|
17
|
+
constructor(broker, name) {
|
|
18
|
+
this.broker = broker;
|
|
19
|
+
this.name = name;
|
|
20
|
+
}
|
|
21
|
+
async publish(data, options) { return this.broker.publish(this.name, data, options); }
|
|
22
|
+
async subscribe(cb) { return this.broker.subscribe(this.name, cb); }
|
|
23
|
+
async unsubscribe() { return this.broker.unsubscribe(this.name); }
|
|
24
|
+
}
|
|
25
|
+
exports.NexoTopic = NexoTopic;
|
|
26
|
+
class NexoPubSub {
|
|
27
|
+
constructor(conn) {
|
|
28
|
+
this.conn = conn;
|
|
29
|
+
this.handlers = new Map();
|
|
30
|
+
conn.onPush = (topic, data) => this.dispatch(topic, data);
|
|
31
|
+
}
|
|
32
|
+
async publish(topic, data, options) {
|
|
33
|
+
await exports.PubSubCommands.publish(this.conn, topic, data, options || {});
|
|
34
|
+
}
|
|
35
|
+
async subscribe(topic, callback) {
|
|
36
|
+
if (!this.handlers.has(topic))
|
|
37
|
+
this.handlers.set(topic, []);
|
|
38
|
+
this.handlers.get(topic).push(callback);
|
|
39
|
+
if (this.handlers.get(topic).length === 1) {
|
|
40
|
+
await exports.PubSubCommands.subscribe(this.conn, topic);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
async unsubscribe(topic) {
|
|
44
|
+
if (this.handlers.has(topic)) {
|
|
45
|
+
this.handlers.delete(topic);
|
|
46
|
+
await exports.PubSubCommands.unsubscribe(this.conn, topic);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
dispatch(topic, data) {
|
|
50
|
+
this.handlers.get(topic)?.forEach(cb => { try {
|
|
51
|
+
cb(data);
|
|
52
|
+
}
|
|
53
|
+
catch (e) {
|
|
54
|
+
console.error(e);
|
|
55
|
+
} });
|
|
56
|
+
for (const [pattern, cbs] of this.handlers) {
|
|
57
|
+
if (pattern === topic)
|
|
58
|
+
continue;
|
|
59
|
+
if (this.matches(pattern, topic)) {
|
|
60
|
+
cbs.forEach(cb => { try {
|
|
61
|
+
cb(data);
|
|
62
|
+
}
|
|
63
|
+
catch (e) {
|
|
64
|
+
console.error(e);
|
|
65
|
+
} });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
matches(pattern, topic) {
|
|
70
|
+
const pParts = pattern.split('/');
|
|
71
|
+
const tParts = topic.split('/');
|
|
72
|
+
for (let i = 0; i < pParts.length; i++) {
|
|
73
|
+
if (pParts[i] === '#')
|
|
74
|
+
return true;
|
|
75
|
+
if (i >= tParts.length || (pParts[i] !== '+' && pParts[i] !== tParts[i]))
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
return pParts.length === tParts.length;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
exports.NexoPubSub = NexoPubSub;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { NexoConnection } from '../connection';
|
|
2
|
+
import { Cursor } from '../codec';
|
|
3
|
+
export declare enum QueueOpcode {
|
|
4
|
+
Q_CREATE = 16,
|
|
5
|
+
Q_PUSH = 17,
|
|
6
|
+
Q_CONSUME = 18,
|
|
7
|
+
Q_ACK = 19,
|
|
8
|
+
Q_EXISTS = 20
|
|
9
|
+
}
|
|
10
|
+
export declare const QueueCommands: {
|
|
11
|
+
create: (conn: NexoConnection, name: string, config: QueueConfig) => Promise<{
|
|
12
|
+
status: import("../protocol").ResponseStatus;
|
|
13
|
+
cursor: Cursor;
|
|
14
|
+
}>;
|
|
15
|
+
exists: (conn: NexoConnection, name: string) => Promise<boolean>;
|
|
16
|
+
push: (conn: NexoConnection, name: string, data: any, options: QueuePushOptions) => Promise<{
|
|
17
|
+
status: import("../protocol").ResponseStatus;
|
|
18
|
+
cursor: Cursor;
|
|
19
|
+
}>;
|
|
20
|
+
consume: <T>(conn: NexoConnection, name: string, batchSize: number, waitMs: number) => Promise<{
|
|
21
|
+
id: string;
|
|
22
|
+
data: T;
|
|
23
|
+
}[]>;
|
|
24
|
+
ack: (conn: NexoConnection, name: string, id: string) => Promise<{
|
|
25
|
+
status: import("../protocol").ResponseStatus;
|
|
26
|
+
cursor: Cursor;
|
|
27
|
+
}>;
|
|
28
|
+
};
|
|
29
|
+
export type PersistenceStrategy = 'memory' | 'file_sync' | 'file_async';
|
|
30
|
+
export type PersistenceOptions = {
|
|
31
|
+
strategy: 'memory';
|
|
32
|
+
} | {
|
|
33
|
+
strategy: 'file_sync';
|
|
34
|
+
} | {
|
|
35
|
+
strategy: 'file_async';
|
|
36
|
+
flushIntervalMs?: number;
|
|
37
|
+
};
|
|
38
|
+
export interface QueueConfig {
|
|
39
|
+
visibilityTimeoutMs?: number;
|
|
40
|
+
maxRetries?: number;
|
|
41
|
+
ttlMs?: number;
|
|
42
|
+
persistence?: PersistenceOptions;
|
|
43
|
+
}
|
|
44
|
+
export interface QueueSubscribeOptions {
|
|
45
|
+
batchSize?: number;
|
|
46
|
+
waitMs?: number;
|
|
47
|
+
concurrency?: number;
|
|
48
|
+
}
|
|
49
|
+
export interface QueuePushOptions {
|
|
50
|
+
priority?: number;
|
|
51
|
+
delayMs?: number;
|
|
52
|
+
}
|
|
53
|
+
export declare class NexoQueue<T = any> {
|
|
54
|
+
private conn;
|
|
55
|
+
readonly name: string;
|
|
56
|
+
private _config?;
|
|
57
|
+
private isSubscribed;
|
|
58
|
+
constructor(conn: NexoConnection, name: string, _config?: QueueConfig | undefined);
|
|
59
|
+
create(config?: QueueConfig): Promise<this>;
|
|
60
|
+
exists(): Promise<boolean>;
|
|
61
|
+
push(data: T, options?: QueuePushOptions): Promise<void>;
|
|
62
|
+
subscribe(callback: (data: T) => Promise<any> | any, options?: QueueSubscribeOptions): Promise<{
|
|
63
|
+
stop: () => void;
|
|
64
|
+
}>;
|
|
65
|
+
private ack;
|
|
66
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NexoQueue = exports.QueueCommands = exports.QueueOpcode = void 0;
|
|
4
|
+
const codec_1 = require("../codec");
|
|
5
|
+
const logger_1 = require("../utils/logger");
|
|
6
|
+
var QueueOpcode;
|
|
7
|
+
(function (QueueOpcode) {
|
|
8
|
+
QueueOpcode[QueueOpcode["Q_CREATE"] = 16] = "Q_CREATE";
|
|
9
|
+
QueueOpcode[QueueOpcode["Q_PUSH"] = 17] = "Q_PUSH";
|
|
10
|
+
QueueOpcode[QueueOpcode["Q_CONSUME"] = 18] = "Q_CONSUME";
|
|
11
|
+
QueueOpcode[QueueOpcode["Q_ACK"] = 19] = "Q_ACK";
|
|
12
|
+
QueueOpcode[QueueOpcode["Q_EXISTS"] = 20] = "Q_EXISTS";
|
|
13
|
+
})(QueueOpcode || (exports.QueueOpcode = QueueOpcode = {}));
|
|
14
|
+
exports.QueueCommands = {
|
|
15
|
+
create: (conn, name, config) => conn.send(QueueOpcode.Q_CREATE, codec_1.FrameCodec.string(name), codec_1.FrameCodec.string(JSON.stringify(config || {}))),
|
|
16
|
+
exists: async (conn, name) => {
|
|
17
|
+
try {
|
|
18
|
+
const res = await conn.send(QueueOpcode.Q_EXISTS, codec_1.FrameCodec.string(name));
|
|
19
|
+
return res.status === 0x00;
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
push: (conn, name, data, options) => conn.send(QueueOpcode.Q_PUSH, codec_1.FrameCodec.string(name), codec_1.FrameCodec.string(JSON.stringify(options || {})), codec_1.FrameCodec.any(data)),
|
|
26
|
+
consume: async (conn, name, batchSize, waitMs) => {
|
|
27
|
+
const res = await conn.send(QueueOpcode.Q_CONSUME, codec_1.FrameCodec.string(name), codec_1.FrameCodec.string(JSON.stringify({ batchSize, waitMs })));
|
|
28
|
+
const count = res.cursor.readU32();
|
|
29
|
+
if (count === 0)
|
|
30
|
+
return [];
|
|
31
|
+
const messages = [];
|
|
32
|
+
for (let i = 0; i < count; i++) {
|
|
33
|
+
const idHex = res.cursor.readUUID();
|
|
34
|
+
const payloadLen = res.cursor.readU32();
|
|
35
|
+
const payloadBuf = res.cursor.readBuffer(payloadLen);
|
|
36
|
+
const data = codec_1.FrameCodec.decodeAny(new codec_1.Cursor(payloadBuf));
|
|
37
|
+
messages.push({ id: idHex, data });
|
|
38
|
+
}
|
|
39
|
+
return messages;
|
|
40
|
+
},
|
|
41
|
+
ack: (conn, name, id) => conn.send(QueueOpcode.Q_ACK, codec_1.FrameCodec.uuid(id), codec_1.FrameCodec.string(name)),
|
|
42
|
+
};
|
|
43
|
+
async function runConcurrent(items, concurrency, fn) {
|
|
44
|
+
if (concurrency === 1) {
|
|
45
|
+
for (const item of items)
|
|
46
|
+
await fn(item);
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const queue = [...items];
|
|
50
|
+
const workers = Array(Math.min(concurrency, items.length)).fill(null).map(async () => {
|
|
51
|
+
while (queue.length)
|
|
52
|
+
await fn(queue.shift());
|
|
53
|
+
});
|
|
54
|
+
await Promise.all(workers);
|
|
55
|
+
}
|
|
56
|
+
class NexoQueue {
|
|
57
|
+
constructor(conn, name, _config) {
|
|
58
|
+
this.conn = conn;
|
|
59
|
+
this.name = name;
|
|
60
|
+
this._config = _config;
|
|
61
|
+
this.isSubscribed = false;
|
|
62
|
+
}
|
|
63
|
+
async create(config = {}) {
|
|
64
|
+
await exports.QueueCommands.create(this.conn, this.name, config);
|
|
65
|
+
return this;
|
|
66
|
+
}
|
|
67
|
+
async exists() {
|
|
68
|
+
return exports.QueueCommands.exists(this.conn, this.name);
|
|
69
|
+
}
|
|
70
|
+
async push(data, options = {}) {
|
|
71
|
+
await exports.QueueCommands.push(this.conn, this.name, data, options);
|
|
72
|
+
}
|
|
73
|
+
async subscribe(callback, options = {}) {
|
|
74
|
+
if (this.isSubscribed)
|
|
75
|
+
throw new Error(`Queue '${this.name}' already subscribed.`);
|
|
76
|
+
// Fail Fast: Check existence first
|
|
77
|
+
if (!(await this.exists())) {
|
|
78
|
+
throw new Error(`Queue '${this.name}' not found`);
|
|
79
|
+
}
|
|
80
|
+
this.isSubscribed = true;
|
|
81
|
+
const batchSize = options.batchSize ?? 50;
|
|
82
|
+
const waitMs = options.waitMs ?? 20000;
|
|
83
|
+
const concurrency = options.concurrency ?? 5;
|
|
84
|
+
let active = true;
|
|
85
|
+
const loop = async () => {
|
|
86
|
+
while (active && this.conn.isConnected) {
|
|
87
|
+
try {
|
|
88
|
+
const messages = await exports.QueueCommands.consume(this.conn, this.name, batchSize, waitMs);
|
|
89
|
+
if (messages.length === 0)
|
|
90
|
+
continue;
|
|
91
|
+
await runConcurrent(messages, concurrency, async (msg) => {
|
|
92
|
+
if (!active)
|
|
93
|
+
return;
|
|
94
|
+
try {
|
|
95
|
+
await callback(msg.data);
|
|
96
|
+
await this.ack(msg.id);
|
|
97
|
+
}
|
|
98
|
+
catch (e) {
|
|
99
|
+
if (!this.conn.isConnected)
|
|
100
|
+
return;
|
|
101
|
+
logger_1.logger.error(`Callback error in queue ${this.name}:`, e);
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
catch (e) {
|
|
106
|
+
if (!active || !this.conn.isConnected)
|
|
107
|
+
return;
|
|
108
|
+
logger_1.logger.error(`Queue consume error in ${this.name}:`, e);
|
|
109
|
+
await new Promise(r => setTimeout(r, 1000));
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
this.isSubscribed = false;
|
|
113
|
+
};
|
|
114
|
+
loop();
|
|
115
|
+
return { stop: () => { active = false; } };
|
|
116
|
+
}
|
|
117
|
+
async ack(id) {
|
|
118
|
+
await exports.QueueCommands.ack(this.conn, this.name, id);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
exports.NexoQueue = NexoQueue;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { NexoConnection } from '../connection';
|
|
2
|
+
import { ResponseStatus } from '../protocol';
|
|
3
|
+
export declare enum StoreOpcode {
|
|
4
|
+
MAP_SET = 2,
|
|
5
|
+
MAP_GET = 3,
|
|
6
|
+
MAP_DEL = 4
|
|
7
|
+
}
|
|
8
|
+
export declare const StoreCommands: {
|
|
9
|
+
mapSet: (conn: NexoConnection, key: string, value: any, options: MapSetOptions) => Promise<{
|
|
10
|
+
status: ResponseStatus;
|
|
11
|
+
cursor: import("../codec").Cursor;
|
|
12
|
+
}>;
|
|
13
|
+
mapGet: (conn: NexoConnection, key: string) => Promise<any>;
|
|
14
|
+
mapDel: (conn: NexoConnection, key: string) => Promise<{
|
|
15
|
+
status: ResponseStatus;
|
|
16
|
+
cursor: import("../codec").Cursor;
|
|
17
|
+
}>;
|
|
18
|
+
};
|
|
19
|
+
export interface MapSetOptions {
|
|
20
|
+
ttl?: number;
|
|
21
|
+
}
|
|
22
|
+
export declare class NexoMap {
|
|
23
|
+
private conn;
|
|
24
|
+
constructor(conn: NexoConnection);
|
|
25
|
+
set(key: string, value: any, options?: MapSetOptions): Promise<void>;
|
|
26
|
+
get<T = any>(key: string): Promise<T | null>;
|
|
27
|
+
del(key: string): Promise<void>;
|
|
28
|
+
}
|
|
29
|
+
export declare class NexoStore {
|
|
30
|
+
readonly map: NexoMap;
|
|
31
|
+
constructor(conn: NexoConnection);
|
|
32
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NexoStore = exports.NexoMap = exports.StoreCommands = exports.StoreOpcode = void 0;
|
|
4
|
+
const protocol_1 = require("../protocol");
|
|
5
|
+
const codec_1 = require("../codec");
|
|
6
|
+
var StoreOpcode;
|
|
7
|
+
(function (StoreOpcode) {
|
|
8
|
+
StoreOpcode[StoreOpcode["MAP_SET"] = 2] = "MAP_SET";
|
|
9
|
+
StoreOpcode[StoreOpcode["MAP_GET"] = 3] = "MAP_GET";
|
|
10
|
+
StoreOpcode[StoreOpcode["MAP_DEL"] = 4] = "MAP_DEL";
|
|
11
|
+
})(StoreOpcode || (exports.StoreOpcode = StoreOpcode = {}));
|
|
12
|
+
exports.StoreCommands = {
|
|
13
|
+
mapSet: (conn, key, value, options) => conn.send(StoreOpcode.MAP_SET, codec_1.FrameCodec.string(key), codec_1.FrameCodec.string(JSON.stringify(options || {})), codec_1.FrameCodec.any(value)),
|
|
14
|
+
mapGet: async (conn, key) => {
|
|
15
|
+
const res = await conn.send(StoreOpcode.MAP_GET, codec_1.FrameCodec.string(key));
|
|
16
|
+
if (res.status === protocol_1.ResponseStatus.NULL)
|
|
17
|
+
return null;
|
|
18
|
+
return codec_1.FrameCodec.decodeAny(res.cursor);
|
|
19
|
+
},
|
|
20
|
+
mapDel: (conn, key) => conn.send(StoreOpcode.MAP_DEL, codec_1.FrameCodec.string(key)),
|
|
21
|
+
};
|
|
22
|
+
class NexoMap {
|
|
23
|
+
constructor(conn) {
|
|
24
|
+
this.conn = conn;
|
|
25
|
+
}
|
|
26
|
+
async set(key, value, options = {}) {
|
|
27
|
+
await exports.StoreCommands.mapSet(this.conn, key, value, options);
|
|
28
|
+
}
|
|
29
|
+
async get(key) {
|
|
30
|
+
return exports.StoreCommands.mapGet(this.conn, key);
|
|
31
|
+
}
|
|
32
|
+
async del(key) {
|
|
33
|
+
await exports.StoreCommands.mapDel(this.conn, key);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
exports.NexoMap = NexoMap;
|
|
37
|
+
class NexoStore {
|
|
38
|
+
constructor(conn) {
|
|
39
|
+
this.map = new NexoMap(conn);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
exports.NexoStore = NexoStore;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { NexoConnection } from '../connection';
|
|
2
|
+
import { Cursor } from '../codec';
|
|
3
|
+
export declare enum StreamOpcode {
|
|
4
|
+
S_CREATE = 48,
|
|
5
|
+
S_PUB = 49,
|
|
6
|
+
S_FETCH = 50,
|
|
7
|
+
S_JOIN = 51,
|
|
8
|
+
S_COMMIT = 52,
|
|
9
|
+
S_EXISTS = 53
|
|
10
|
+
}
|
|
11
|
+
export type PersistenceStrategy = 'memory' | 'file_sync' | 'file_async';
|
|
12
|
+
export type PersistenceOptions = {
|
|
13
|
+
strategy: 'memory';
|
|
14
|
+
} | {
|
|
15
|
+
strategy: 'file_sync';
|
|
16
|
+
} | {
|
|
17
|
+
strategy: 'file_async';
|
|
18
|
+
flushIntervalMs?: number;
|
|
19
|
+
};
|
|
20
|
+
export interface StreamCreateOptions {
|
|
21
|
+
partitions?: number;
|
|
22
|
+
persistence?: PersistenceOptions;
|
|
23
|
+
}
|
|
24
|
+
export interface StreamPublishOptions {
|
|
25
|
+
key?: string;
|
|
26
|
+
}
|
|
27
|
+
export declare const StreamCommands: {
|
|
28
|
+
create: (conn: NexoConnection, name: string, options: StreamCreateOptions) => Promise<{
|
|
29
|
+
status: import("../protocol").ResponseStatus;
|
|
30
|
+
cursor: Cursor;
|
|
31
|
+
}>;
|
|
32
|
+
exists: (conn: NexoConnection, name: string) => Promise<boolean>;
|
|
33
|
+
publish: (conn: NexoConnection, name: string, data: any, options: StreamPublishOptions) => Promise<{
|
|
34
|
+
status: import("../protocol").ResponseStatus;
|
|
35
|
+
cursor: Cursor;
|
|
36
|
+
}>;
|
|
37
|
+
join: (conn: NexoConnection, stream: string, group: string) => Promise<{
|
|
38
|
+
generationId: bigint;
|
|
39
|
+
partitions: {
|
|
40
|
+
id: number;
|
|
41
|
+
offset: bigint;
|
|
42
|
+
}[];
|
|
43
|
+
}>;
|
|
44
|
+
fetch: <T>(conn: NexoConnection, stream: string, group: string, partition: number, offset: bigint, generationId: bigint, batchSize: number) => Promise<{
|
|
45
|
+
offset: bigint;
|
|
46
|
+
data: T;
|
|
47
|
+
}[]>;
|
|
48
|
+
commit: (conn: NexoConnection, stream: string, group: string, partition: number, offset: bigint, generationId: bigint) => Promise<{
|
|
49
|
+
status: import("../protocol").ResponseStatus;
|
|
50
|
+
cursor: Cursor;
|
|
51
|
+
}>;
|
|
52
|
+
};
|
|
53
|
+
export interface StreamSubscribeOptions {
|
|
54
|
+
batchSize?: number;
|
|
55
|
+
}
|
|
56
|
+
export declare class NexoStream<T = any> {
|
|
57
|
+
private conn;
|
|
58
|
+
readonly name: string;
|
|
59
|
+
constructor(conn: NexoConnection, name: string);
|
|
60
|
+
create(options?: StreamCreateOptions): Promise<this>;
|
|
61
|
+
exists(): Promise<boolean>;
|
|
62
|
+
publish(data: T, options?: StreamPublishOptions): Promise<void>;
|
|
63
|
+
subscribe(group: string, callback: (data: T) => Promise<any> | any, options?: StreamSubscribeOptions): Promise<{
|
|
64
|
+
stop: () => void;
|
|
65
|
+
}>;
|
|
66
|
+
}
|