bunqueue 1.2.2 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +358 -702
- package/dist/client/events.d.ts +19 -0
- package/dist/client/events.d.ts.map +1 -0
- package/dist/client/events.js +66 -0
- package/dist/client/events.js.map +1 -0
- package/dist/client/index.d.ts +26 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +25 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client/manager.d.ts +9 -0
- package/dist/client/manager.d.ts.map +1 -0
- package/dist/client/manager.js +18 -0
- package/dist/client/manager.js.map +1 -0
- package/dist/client/queue.d.ts +42 -0
- package/dist/client/queue.d.ts.map +1 -0
- package/dist/client/queue.js +82 -0
- package/dist/client/queue.js.map +1 -0
- package/dist/client/types.d.ts +49 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/client/types.js +34 -0
- package/dist/client/types.js.map +1 -0
- package/dist/client/worker.d.ts +29 -0
- package/dist/client/worker.d.ts.map +1 -0
- package/dist/client/worker.js +134 -0
- package/dist/client/worker.js.map +1 -0
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -10,254 +10,241 @@
|
|
|
10
10
|
|
|
11
11
|
<p align="center">
|
|
12
12
|
<a href="#features">Features</a> •
|
|
13
|
-
<a href="#sdk">SDK</a> •
|
|
14
13
|
<a href="#quick-start">Quick Start</a> •
|
|
15
|
-
<a href="#
|
|
14
|
+
<a href="#embedded-mode">Embedded</a> •
|
|
15
|
+
<a href="#server-mode">Server</a> •
|
|
16
16
|
<a href="#api-reference">API</a> •
|
|
17
17
|
<a href="#docker">Docker</a>
|
|
18
18
|
</p>
|
|
19
19
|
|
|
20
20
|
<p align="center">
|
|
21
21
|
<a href="https://www.npmjs.com/package/bunqueue"><img src="https://img.shields.io/npm/v/bunqueue?label=bunqueue" alt="bunqueue npm"></a>
|
|
22
|
-
<a href="https://www.npmjs.com/package/
|
|
23
|
-
<a href="https://www.npmjs.com/package/flashq"><img src="https://img.shields.io/npm/dm/flashq" alt="npm downloads"></a>
|
|
22
|
+
<a href="https://www.npmjs.com/package/bunqueue"><img src="https://img.shields.io/npm/dm/bunqueue" alt="npm downloads"></a>
|
|
24
23
|
</p>
|
|
25
24
|
|
|
26
25
|
---
|
|
27
26
|
|
|
28
|
-
##
|
|
27
|
+
## Why bunqueue?
|
|
29
28
|
|
|
30
|
-
bunqueue requires
|
|
29
|
+
> ⚠️ **Bun only** — bunqueue requires [Bun](https://bun.sh) runtime. Node.js is not supported.
|
|
31
30
|
|
|
32
|
-
|
|
33
|
-
# Install both packages
|
|
34
|
-
bun add bunqueue flashq
|
|
35
|
-
```
|
|
31
|
+
**Every other job queue requires external infrastructure.** bunqueue doesn't.
|
|
36
32
|
|
|
37
|
-
|
|
|
38
|
-
|
|
39
|
-
|
|
|
40
|
-
|
|
|
33
|
+
| Library | Requires |
|
|
34
|
+
|---------|----------|
|
|
35
|
+
| BullMQ | ❌ Redis |
|
|
36
|
+
| Agenda | ❌ MongoDB |
|
|
37
|
+
| Bee-Queue | ❌ Redis |
|
|
38
|
+
| pg-boss | ❌ PostgreSQL |
|
|
39
|
+
| Celery | ❌ Redis/RabbitMQ |
|
|
40
|
+
| **bunqueue** | ✅ **Nothing. Zero. Nada.** |
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
bunqueue is the **only** job queue with:
|
|
43
|
+
- **BullMQ-compatible API** — Same `Queue`, `Worker`, `QueueEvents` you know
|
|
44
|
+
- **Zero external dependencies** — No Redis, no MongoDB, no nothing
|
|
45
|
+
- **Persistent storage** — SQLite survives restarts, no data loss
|
|
46
|
+
- **100K+ jobs/sec** — Faster than Redis-based queues
|
|
47
|
+
- **Single file deployment** — Just your app, that's it
|
|
43
48
|
|
|
44
49
|
```bash
|
|
45
|
-
#
|
|
46
|
-
|
|
50
|
+
# Others: Install Redis, configure connection, manage infrastructure...
|
|
51
|
+
# bunqueue:
|
|
52
|
+
bun add bunqueue
|
|
53
|
+
```
|
|
47
54
|
|
|
48
|
-
|
|
49
|
-
|
|
55
|
+
```typescript
|
|
56
|
+
import { Queue, Worker } from 'bunqueue/client';
|
|
57
|
+
// That's it. You're done. Start queuing.
|
|
58
|
+
```
|
|
50
59
|
|
|
51
|
-
|
|
52
|
-
./node_modules/.bin/bunqueue
|
|
60
|
+
---
|
|
53
61
|
|
|
54
|
-
|
|
55
|
-
bun add -g bunqueue
|
|
56
|
-
bunqueue
|
|
62
|
+
## Quick Install
|
|
57
63
|
|
|
58
|
-
|
|
59
|
-
|
|
64
|
+
```bash
|
|
65
|
+
# Requires Bun runtime (https://bun.sh)
|
|
66
|
+
bun add bunqueue
|
|
60
67
|
```
|
|
61
68
|
|
|
62
|
-
|
|
69
|
+
bunqueue works in **two modes**:
|
|
63
70
|
|
|
64
|
-
|
|
71
|
+
| Mode | Description | Use Case |
|
|
72
|
+
|------|-------------|----------|
|
|
73
|
+
| **Embedded** | In-process, no server needed | Monolith, scripts, serverless |
|
|
74
|
+
| **Server** | Standalone TCP/HTTP server | Microservices, multi-process |
|
|
65
75
|
|
|
66
|
-
|
|
76
|
+
---
|
|
67
77
|
|
|
68
|
-
|
|
69
|
-
# With environment variables
|
|
70
|
-
DATA_PATH=./data/bunqueue.db AUTH_TOKENS=your-secret-token bunqueue
|
|
78
|
+
## Quick Start
|
|
71
79
|
|
|
72
|
-
|
|
73
|
-
TCP_PORT=6789 HTTP_PORT=6790 DATA_PATH=./data/bunqueue.db AUTH_TOKENS=token1,token2 bunqueue
|
|
74
|
-
```
|
|
80
|
+
### Embedded Mode (Recommended)
|
|
75
81
|
|
|
76
|
-
|
|
82
|
+
No server required. BullMQ-compatible API.
|
|
77
83
|
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
TCP_PORT=6789
|
|
81
|
-
HTTP_PORT=6790
|
|
82
|
-
HOST=0.0.0.0
|
|
84
|
+
```typescript
|
|
85
|
+
import { Queue, Worker } from 'bunqueue/client';
|
|
83
86
|
|
|
84
|
-
|
|
85
|
-
|
|
87
|
+
// Create queue
|
|
88
|
+
const queue = new Queue('emails');
|
|
86
89
|
|
|
87
|
-
|
|
88
|
-
|
|
90
|
+
// Create worker
|
|
91
|
+
const worker = new Worker('emails', async (job) => {
|
|
92
|
+
console.log('Sending email to:', job.data.to);
|
|
93
|
+
await job.updateProgress(50);
|
|
94
|
+
return { sent: true };
|
|
95
|
+
}, { concurrency: 5 });
|
|
89
96
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
97
|
+
// Handle events
|
|
98
|
+
worker.on('completed', (job, result) => {
|
|
99
|
+
console.log(`Job ${job.id} completed:`, result);
|
|
100
|
+
});
|
|
93
101
|
|
|
94
|
-
|
|
102
|
+
worker.on('failed', (job, err) => {
|
|
103
|
+
console.error(`Job ${job.id} failed:`, err.message);
|
|
104
|
+
});
|
|
95
105
|
|
|
96
|
-
|
|
97
|
-
|
|
106
|
+
// Add jobs
|
|
107
|
+
await queue.add('send-welcome', { to: 'user@example.com' });
|
|
98
108
|
```
|
|
99
109
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
110
|
+
### Server Mode
|
|
111
|
+
|
|
112
|
+
For multi-process or microservice architectures.
|
|
103
113
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
Auth: enabled (2 tokens)
|
|
114
|
+
**Terminal 1 - Start server:**
|
|
115
|
+
```bash
|
|
116
|
+
bunqueue start
|
|
108
117
|
```
|
|
109
118
|
|
|
110
|
-
|
|
119
|
+
<img src=".github/terminal.png" alt="bunqueue server running" width="600" />
|
|
111
120
|
|
|
121
|
+
**Terminal 2 - Producer:**
|
|
112
122
|
```typescript
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const worker = new Worker('emails', async (job) => {
|
|
121
|
-
console.log('Sending email to:', job.data.to);
|
|
122
|
-
return { sent: true };
|
|
123
|
+
const res = await fetch('http://localhost:6790/push', {
|
|
124
|
+
method: 'POST',
|
|
125
|
+
headers: { 'Content-Type': 'application/json' },
|
|
126
|
+
body: JSON.stringify({
|
|
127
|
+
queue: 'emails',
|
|
128
|
+
data: { to: 'user@example.com' }
|
|
129
|
+
})
|
|
123
130
|
});
|
|
124
131
|
```
|
|
125
132
|
|
|
133
|
+
**Terminal 3 - Consumer:**
|
|
134
|
+
```typescript
|
|
135
|
+
while (true) {
|
|
136
|
+
const res = await fetch('http://localhost:6790/pull', {
|
|
137
|
+
method: 'POST',
|
|
138
|
+
body: JSON.stringify({ queue: 'emails', timeout: 5000 })
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
const job = await res.json();
|
|
142
|
+
if (job.id) {
|
|
143
|
+
console.log('Processing:', job.data);
|
|
144
|
+
await fetch('http://localhost:6790/ack', {
|
|
145
|
+
method: 'POST',
|
|
146
|
+
body: JSON.stringify({ id: job.id })
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
126
152
|
---
|
|
127
153
|
|
|
128
154
|
## Features
|
|
129
155
|
|
|
130
|
-
- **Blazing Fast** —
|
|
131
|
-
- **
|
|
132
|
-
- **
|
|
133
|
-
- **
|
|
134
|
-
- **
|
|
156
|
+
- **Blazing Fast** — 500K+ jobs/sec, built on Bun runtime
|
|
157
|
+
- **Dual Mode** — Embedded (in-process) or Server (TCP/HTTP)
|
|
158
|
+
- **BullMQ-Compatible API** — Easy migration with `Queue`, `Worker`, `QueueEvents`
|
|
159
|
+
- **Persistent Storage** — SQLite with WAL mode
|
|
160
|
+
- **Priority Queues** — FIFO, LIFO, and priority-based ordering
|
|
161
|
+
- **Delayed Jobs** — Schedule jobs for later
|
|
162
|
+
- **Cron Scheduling** — Recurring jobs with cron expressions
|
|
135
163
|
- **Retry & Backoff** — Automatic retries with exponential backoff
|
|
136
|
-
- **Dead Letter Queue** — Failed jobs preserved for inspection
|
|
137
|
-
- **Job Dependencies** —
|
|
138
|
-
- **Progress Tracking** — Real-time progress updates
|
|
139
|
-
- **Rate Limiting** — Per-queue rate limits
|
|
164
|
+
- **Dead Letter Queue** — Failed jobs preserved for inspection
|
|
165
|
+
- **Job Dependencies** — Parent-child relationships
|
|
166
|
+
- **Progress Tracking** — Real-time progress updates
|
|
167
|
+
- **Rate Limiting** — Per-queue rate limits
|
|
140
168
|
- **Webhooks** — HTTP callbacks on job events
|
|
141
|
-
- **Real-time Events** — WebSocket and
|
|
142
|
-
- **Prometheus Metrics** — Built-in
|
|
143
|
-
- **
|
|
144
|
-
- **Dual Protocol** — TCP (high performance) and HTTP/REST (compatibility)
|
|
145
|
-
- **Full-Featured CLI** — Manage queues, jobs, cron, and more from the command line
|
|
146
|
-
|
|
147
|
-
## CLI
|
|
169
|
+
- **Real-time Events** — WebSocket and SSE support
|
|
170
|
+
- **Prometheus Metrics** — Built-in monitoring
|
|
171
|
+
- **Full CLI** — Manage queues from command line
|
|
148
172
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
### Server Mode
|
|
152
|
-
|
|
153
|
-
```bash
|
|
154
|
-
# Start server with defaults
|
|
155
|
-
bunqueue
|
|
156
|
-
|
|
157
|
-
# Start with options
|
|
158
|
-
bunqueue start --tcp-port 6789 --http-port 6790 --data-path ./data/queue.db
|
|
159
|
-
```
|
|
173
|
+
---
|
|
160
174
|
|
|
161
|
-
|
|
175
|
+
## Embedded Mode
|
|
162
176
|
|
|
163
|
-
|
|
164
|
-
# Push a job
|
|
165
|
-
bunqueue push emails '{"to":"user@test.com","subject":"Hello"}'
|
|
166
|
-
bunqueue push tasks '{"action":"sync"}' --priority 10 --delay 5000
|
|
167
|
-
|
|
168
|
-
# Pull and process jobs
|
|
169
|
-
bunqueue pull emails --timeout 5000
|
|
170
|
-
bunqueue ack 12345 --result '{"sent":true}'
|
|
171
|
-
bunqueue fail 12345 --error "SMTP timeout"
|
|
177
|
+
### Queue API
|
|
172
178
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
bunqueue job progress 12345 50 --message "Processing..."
|
|
176
|
-
bunqueue job cancel 12345
|
|
177
|
-
|
|
178
|
-
# Queue control
|
|
179
|
-
bunqueue queue list
|
|
180
|
-
bunqueue queue pause emails
|
|
181
|
-
bunqueue queue resume emails
|
|
182
|
-
bunqueue queue drain emails
|
|
179
|
+
```typescript
|
|
180
|
+
import { Queue } from 'bunqueue/client';
|
|
183
181
|
|
|
184
|
-
|
|
185
|
-
bunqueue cron list
|
|
186
|
-
bunqueue cron add hourly-cleanup -q maintenance -d '{"task":"cleanup"}' -s "0 * * * *"
|
|
187
|
-
bunqueue cron delete hourly-cleanup
|
|
182
|
+
const queue = new Queue('my-queue');
|
|
188
183
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
bunqueue dlq retry emails
|
|
192
|
-
bunqueue dlq purge emails
|
|
184
|
+
// Add job
|
|
185
|
+
const job = await queue.add('task-name', { data: 'value' });
|
|
193
186
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
187
|
+
// Add with options
|
|
188
|
+
await queue.add('task', { data: 'value' }, {
|
|
189
|
+
priority: 10, // Higher = processed first
|
|
190
|
+
delay: 5000, // Delay in ms
|
|
191
|
+
attempts: 3, // Max retries
|
|
192
|
+
backoff: 1000, // Backoff base (ms)
|
|
193
|
+
timeout: 30000, // Processing timeout
|
|
194
|
+
jobId: 'unique-id', // Custom ID
|
|
195
|
+
removeOnComplete: true,
|
|
196
|
+
removeOnFail: false,
|
|
197
|
+
});
|
|
199
198
|
|
|
200
|
-
|
|
199
|
+
// Bulk add
|
|
200
|
+
await queue.addBulk([
|
|
201
|
+
{ name: 'task1', data: { id: 1 } },
|
|
202
|
+
{ name: 'task2', data: { id: 2 } },
|
|
203
|
+
]);
|
|
201
204
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
-p, --port <port> # TCP port (default: 6789)
|
|
205
|
-
-t, --token <token> # Authentication token
|
|
206
|
-
--json # Output as JSON
|
|
207
|
-
--help # Show help
|
|
208
|
-
--version # Show version
|
|
209
|
-
```
|
|
205
|
+
// Get job
|
|
206
|
+
const job = await queue.getJob('job-id');
|
|
210
207
|
|
|
211
|
-
|
|
208
|
+
// Remove job
|
|
209
|
+
await queue.remove('job-id');
|
|
212
210
|
|
|
213
|
-
|
|
211
|
+
// Get counts
|
|
212
|
+
const counts = await queue.getJobCounts();
|
|
213
|
+
// { waiting: 10, active: 2, completed: 100, failed: 5 }
|
|
214
214
|
|
|
215
|
-
|
|
216
|
-
|
|
215
|
+
// Queue control
|
|
216
|
+
await queue.pause();
|
|
217
|
+
await queue.resume();
|
|
218
|
+
await queue.drain(); // Remove waiting jobs
|
|
219
|
+
await queue.obliterate(); // Remove ALL data
|
|
217
220
|
```
|
|
218
221
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
### Basic Usage
|
|
222
|
+
### Worker API
|
|
222
223
|
|
|
223
224
|
```typescript
|
|
224
|
-
import {
|
|
225
|
+
import { Worker } from 'bunqueue/client';
|
|
225
226
|
|
|
226
|
-
// Create a queue
|
|
227
|
-
const queue = new Queue('my-queue', {
|
|
228
|
-
connection: { host: 'localhost', port: 6789 }
|
|
229
|
-
});
|
|
230
|
-
|
|
231
|
-
// Add a job
|
|
232
|
-
await queue.add('process-data', { userId: 123, action: 'sync' });
|
|
233
|
-
|
|
234
|
-
// Add with options
|
|
235
|
-
await queue.add('send-email',
|
|
236
|
-
{ to: 'user@example.com', subject: 'Hello' },
|
|
237
|
-
{
|
|
238
|
-
priority: 10,
|
|
239
|
-
delay: 5000, // 5 seconds
|
|
240
|
-
attempts: 3,
|
|
241
|
-
backoff: { type: 'exponential', delay: 1000 }
|
|
242
|
-
}
|
|
243
|
-
);
|
|
244
|
-
|
|
245
|
-
// Create a worker
|
|
246
227
|
const worker = new Worker('my-queue', async (job) => {
|
|
247
228
|
console.log('Processing:', job.name, job.data);
|
|
248
229
|
|
|
249
230
|
// Update progress
|
|
250
|
-
await job.updateProgress(50);
|
|
231
|
+
await job.updateProgress(50, 'Halfway done');
|
|
251
232
|
|
|
252
|
-
//
|
|
233
|
+
// Add log
|
|
234
|
+
await job.log('Processing step completed');
|
|
253
235
|
|
|
236
|
+
// Return result
|
|
254
237
|
return { success: true };
|
|
255
238
|
}, {
|
|
256
|
-
|
|
257
|
-
|
|
239
|
+
concurrency: 10, // Parallel jobs
|
|
240
|
+
autorun: true, // Start automatically
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
// Events
|
|
244
|
+
worker.on('active', (job) => {
|
|
245
|
+
console.log(`Job ${job.id} started`);
|
|
258
246
|
});
|
|
259
247
|
|
|
260
|
-
// Handle events
|
|
261
248
|
worker.on('completed', (job, result) => {
|
|
262
249
|
console.log(`Job ${job.id} completed:`, result);
|
|
263
250
|
});
|
|
@@ -265,667 +252,336 @@ worker.on('completed', (job, result) => {
|
|
|
265
252
|
worker.on('failed', (job, err) => {
|
|
266
253
|
console.error(`Job ${job.id} failed:`, err.message);
|
|
267
254
|
});
|
|
268
|
-
```
|
|
269
|
-
|
|
270
|
-
### Cron Jobs
|
|
271
255
|
|
|
272
|
-
|
|
273
|
-
|
|
256
|
+
worker.on('progress', (job, progress) => {
|
|
257
|
+
console.log(`Job ${job.id} progress:`, progress);
|
|
258
|
+
});
|
|
274
259
|
|
|
275
|
-
|
|
276
|
-
|
|
260
|
+
worker.on('error', (err) => {
|
|
261
|
+
console.error('Worker error:', err);
|
|
277
262
|
});
|
|
278
263
|
|
|
279
|
-
//
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
);
|
|
284
|
-
|
|
285
|
-
// Every 5 minutes
|
|
286
|
-
await queue.upsertJobScheduler('health-check',
|
|
287
|
-
{ every: 300000 },
|
|
288
|
-
{ name: 'ping', data: {} }
|
|
289
|
-
);
|
|
264
|
+
// Control
|
|
265
|
+
worker.pause();
|
|
266
|
+
worker.resume();
|
|
267
|
+
await worker.close(); // Graceful shutdown
|
|
268
|
+
await worker.close(true); // Force close
|
|
290
269
|
```
|
|
291
270
|
|
|
292
|
-
###
|
|
271
|
+
### QueueEvents
|
|
272
|
+
|
|
273
|
+
Listen to queue events without processing jobs.
|
|
293
274
|
|
|
294
275
|
```typescript
|
|
295
|
-
import {
|
|
276
|
+
import { QueueEvents } from 'bunqueue/client';
|
|
296
277
|
|
|
297
|
-
const
|
|
298
|
-
connection: { host: 'localhost', port: 6789 }
|
|
299
|
-
});
|
|
278
|
+
const events = new QueueEvents('my-queue');
|
|
300
279
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
name: 'final-step',
|
|
304
|
-
queueName: 'pipeline',
|
|
305
|
-
data: { step: 'aggregate' },
|
|
306
|
-
children: [
|
|
307
|
-
{
|
|
308
|
-
name: 'step-1',
|
|
309
|
-
queueName: 'pipeline',
|
|
310
|
-
data: { step: 'fetch' }
|
|
311
|
-
},
|
|
312
|
-
{
|
|
313
|
-
name: 'step-2',
|
|
314
|
-
queueName: 'pipeline',
|
|
315
|
-
data: { step: 'transform' }
|
|
316
|
-
}
|
|
317
|
-
]
|
|
280
|
+
events.on('waiting', ({ jobId }) => {
|
|
281
|
+
console.log(`Job ${jobId} waiting`);
|
|
318
282
|
});
|
|
319
|
-
```
|
|
320
|
-
|
|
321
|
-
### Real-time Events
|
|
322
|
-
|
|
323
|
-
```typescript
|
|
324
|
-
import { QueueEvents } from 'flashq';
|
|
325
283
|
|
|
326
|
-
|
|
327
|
-
|
|
284
|
+
events.on('active', ({ jobId }) => {
|
|
285
|
+
console.log(`Job ${jobId} active`);
|
|
328
286
|
});
|
|
329
287
|
|
|
330
288
|
events.on('completed', ({ jobId, returnvalue }) => {
|
|
331
|
-
console.log(`Job ${jobId} completed
|
|
289
|
+
console.log(`Job ${jobId} completed:`, returnvalue);
|
|
332
290
|
});
|
|
333
291
|
|
|
334
292
|
events.on('failed', ({ jobId, failedReason }) => {
|
|
335
|
-
console.
|
|
293
|
+
console.log(`Job ${jobId} failed:`, failedReason);
|
|
336
294
|
});
|
|
337
295
|
|
|
338
296
|
events.on('progress', ({ jobId, data }) => {
|
|
339
297
|
console.log(`Job ${jobId} progress:`, data);
|
|
340
298
|
});
|
|
341
|
-
```
|
|
342
|
-
|
|
343
|
-
For more examples, see the [SDK documentation](https://www.npmjs.com/package/flashq).
|
|
344
|
-
|
|
345
|
-
## Quick Start
|
|
346
299
|
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
```bash
|
|
350
|
-
# Using Bun directly
|
|
351
|
-
bun run src/main.ts
|
|
352
|
-
|
|
353
|
-
# Or with Docker
|
|
354
|
-
docker run -p 6789:6789 -p 6790:6790 ghcr.io/egeominotti/bunqueue
|
|
355
|
-
```
|
|
356
|
-
|
|
357
|
-
### Push a Job (HTTP)
|
|
358
|
-
|
|
359
|
-
```bash
|
|
360
|
-
curl -X POST http://localhost:6790/queues/emails/jobs \
|
|
361
|
-
-H "Content-Type: application/json" \
|
|
362
|
-
-d '{"data": {"to": "user@example.com", "subject": "Hello"}}'
|
|
363
|
-
```
|
|
364
|
-
|
|
365
|
-
### Pull a Job (HTTP)
|
|
366
|
-
|
|
367
|
-
```bash
|
|
368
|
-
curl http://localhost:6790/queues/emails/jobs
|
|
369
|
-
```
|
|
370
|
-
|
|
371
|
-
### Acknowledge Completion
|
|
372
|
-
|
|
373
|
-
```bash
|
|
374
|
-
curl -X POST http://localhost:6790/jobs/1/ack \
|
|
375
|
-
-H "Content-Type: application/json" \
|
|
376
|
-
-d '{"result": {"sent": true}}'
|
|
300
|
+
await events.close();
|
|
377
301
|
```
|
|
378
302
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
### Server + SDK
|
|
382
|
-
|
|
383
|
-
bunqueue is composed of two packages:
|
|
384
|
-
|
|
385
|
-
| Package | Description | Install |
|
|
386
|
-
|---------|-------------|---------|
|
|
387
|
-
| **bunqueue** | Job queue server | `bun add bunqueue` |
|
|
388
|
-
| **flashq** | TypeScript SDK | `bun add flashq` |
|
|
389
|
-
|
|
390
|
-
```bash
|
|
391
|
-
# Install both
|
|
392
|
-
bun add bunqueue flashq
|
|
393
|
-
```
|
|
394
|
-
|
|
395
|
-
### Quick Setup
|
|
396
|
-
|
|
397
|
-
```bash
|
|
398
|
-
# 1. Start the server
|
|
399
|
-
bunqueue
|
|
400
|
-
|
|
401
|
-
# 2. Use the SDK in your app
|
|
402
|
-
```
|
|
303
|
+
### Shutdown
|
|
403
304
|
|
|
404
305
|
```typescript
|
|
405
|
-
import {
|
|
406
|
-
|
|
407
|
-
const queue = new Queue('tasks');
|
|
408
|
-
await queue.add('my-job', { data: 'hello' });
|
|
306
|
+
import { shutdownManager } from 'bunqueue/client';
|
|
409
307
|
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
return { done: true };
|
|
413
|
-
});
|
|
308
|
+
// Cleanup when done
|
|
309
|
+
shutdownManager();
|
|
414
310
|
```
|
|
415
311
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
```bash
|
|
419
|
-
git clone https://github.com/egeominotti/bunqueue.git
|
|
420
|
-
cd bunqueue
|
|
421
|
-
bun install
|
|
422
|
-
bun run start
|
|
423
|
-
```
|
|
424
|
-
|
|
425
|
-
### Build Binary
|
|
312
|
+
---
|
|
426
313
|
|
|
427
|
-
|
|
428
|
-
bun run build
|
|
429
|
-
./dist/bunqueue
|
|
430
|
-
```
|
|
314
|
+
## Server Mode
|
|
431
315
|
|
|
432
|
-
###
|
|
316
|
+
### Start Server
|
|
433
317
|
|
|
434
318
|
```bash
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
-p 6789:6789 \
|
|
438
|
-
-p 6790:6790 \
|
|
439
|
-
-v bunqueue-data:/app/data \
|
|
440
|
-
ghcr.io/egeominotti/bunqueue
|
|
441
|
-
```
|
|
442
|
-
|
|
443
|
-
### Docker Compose
|
|
319
|
+
# Basic
|
|
320
|
+
bunqueue start
|
|
444
321
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
services:
|
|
448
|
-
bunqueue:
|
|
449
|
-
image: ghcr.io/egeominotti/bunqueue
|
|
450
|
-
ports:
|
|
451
|
-
- "6789:6789"
|
|
452
|
-
- "6790:6790"
|
|
453
|
-
volumes:
|
|
454
|
-
- bunqueue-data:/app/data
|
|
455
|
-
environment:
|
|
456
|
-
- AUTH_TOKENS=your-secret-token
|
|
322
|
+
# With options
|
|
323
|
+
bunqueue start --tcp-port 6789 --http-port 6790 --data-path ./data/queue.db
|
|
457
324
|
|
|
458
|
-
|
|
459
|
-
|
|
325
|
+
# With environment variables
|
|
326
|
+
DATA_PATH=./data/bunqueue.db AUTH_TOKENS=secret bunqueue start
|
|
460
327
|
```
|
|
461
328
|
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
### TCP Protocol (High Performance)
|
|
465
|
-
|
|
466
|
-
Connect via TCP for maximum throughput. Commands are newline-delimited JSON.
|
|
467
|
-
|
|
468
|
-
```bash
|
|
469
|
-
# Connect with netcat
|
|
470
|
-
nc localhost 6789
|
|
471
|
-
|
|
472
|
-
# Push a job
|
|
473
|
-
{"cmd":"PUSH","queue":"tasks","data":{"action":"process"}}
|
|
474
|
-
|
|
475
|
-
# Pull a job
|
|
476
|
-
{"cmd":"PULL","queue":"tasks"}
|
|
329
|
+
### Environment Variables
|
|
477
330
|
|
|
478
|
-
|
|
479
|
-
|
|
331
|
+
```env
|
|
332
|
+
TCP_PORT=6789
|
|
333
|
+
HTTP_PORT=6790
|
|
334
|
+
HOST=0.0.0.0
|
|
335
|
+
DATA_PATH=./data/bunqueue.db
|
|
336
|
+
AUTH_TOKENS=token1,token2
|
|
480
337
|
```
|
|
481
338
|
|
|
482
|
-
### HTTP
|
|
339
|
+
### HTTP API
|
|
483
340
|
|
|
484
341
|
```bash
|
|
485
342
|
# Push job
|
|
486
|
-
curl -X POST http://localhost:6790/
|
|
343
|
+
curl -X POST http://localhost:6790/push \
|
|
487
344
|
-H "Content-Type: application/json" \
|
|
488
|
-
-d '{
|
|
489
|
-
"data": {"action": "process"},
|
|
490
|
-
"priority": 10,
|
|
491
|
-
"delay": 5000,
|
|
492
|
-
"maxAttempts": 5
|
|
493
|
-
}'
|
|
345
|
+
-d '{"queue":"emails","data":{"to":"user@test.com"},"priority":10}'
|
|
494
346
|
|
|
495
|
-
# Pull job
|
|
496
|
-
curl
|
|
347
|
+
# Pull job
|
|
348
|
+
curl -X POST http://localhost:6790/pull \
|
|
349
|
+
-H "Content-Type: application/json" \
|
|
350
|
+
-d '{"queue":"emails","timeout":5000}'
|
|
497
351
|
|
|
498
|
-
#
|
|
499
|
-
curl http://localhost:6790/
|
|
352
|
+
# Acknowledge
|
|
353
|
+
curl -X POST http://localhost:6790/ack \
|
|
354
|
+
-H "Content-Type: application/json" \
|
|
355
|
+
-d '{"id":"job-id","result":{"sent":true}}'
|
|
500
356
|
|
|
501
|
-
# Fail
|
|
502
|
-
curl -X POST http://localhost:6790/
|
|
357
|
+
# Fail
|
|
358
|
+
curl -X POST http://localhost:6790/fail \
|
|
503
359
|
-H "Content-Type: application/json" \
|
|
504
|
-
-d '{"error":
|
|
360
|
+
-d '{"id":"job-id","error":"Failed to send"}'
|
|
505
361
|
|
|
506
|
-
#
|
|
362
|
+
# Stats
|
|
507
363
|
curl http://localhost:6790/stats
|
|
508
|
-
```
|
|
509
|
-
|
|
510
|
-
### WebSocket (Real-time)
|
|
511
|
-
|
|
512
|
-
```javascript
|
|
513
|
-
const ws = new WebSocket('ws://localhost:6790/ws');
|
|
514
|
-
|
|
515
|
-
ws.onmessage = (event) => {
|
|
516
|
-
const job = JSON.parse(event.data);
|
|
517
|
-
console.log('Job event:', job);
|
|
518
|
-
};
|
|
519
|
-
|
|
520
|
-
// Subscribe to specific queue
|
|
521
|
-
const wsQueue = new WebSocket('ws://localhost:6790/ws/queues/emails');
|
|
522
|
-
```
|
|
523
|
-
|
|
524
|
-
### Server-Sent Events (SSE)
|
|
525
|
-
|
|
526
|
-
```javascript
|
|
527
|
-
const events = new EventSource('http://localhost:6790/events');
|
|
528
364
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
console.log('Event:', data);
|
|
532
|
-
};
|
|
533
|
-
|
|
534
|
-
// Filter by queue
|
|
535
|
-
const queueEvents = new EventSource('http://localhost:6790/events/queues/emails');
|
|
536
|
-
```
|
|
537
|
-
|
|
538
|
-
### Job Options
|
|
539
|
-
|
|
540
|
-
| Option | Type | Default | Description |
|
|
541
|
-
|--------|------|---------|-------------|
|
|
542
|
-
| `data` | any | required | Job payload |
|
|
543
|
-
| `priority` | number | 0 | Higher = processed first |
|
|
544
|
-
| `delay` | number | 0 | Delay in milliseconds |
|
|
545
|
-
| `maxAttempts` | number | 3 | Max retry attempts |
|
|
546
|
-
| `backoff` | number | 1000 | Initial backoff (ms), doubles each retry |
|
|
547
|
-
| `ttl` | number | null | Time-to-live in milliseconds |
|
|
548
|
-
| `timeout` | number | null | Job processing timeout |
|
|
549
|
-
| `uniqueKey` | string | null | Deduplication key |
|
|
550
|
-
| `jobId` | string | null | Custom job identifier |
|
|
551
|
-
| `dependsOn` | string[] | [] | Job IDs that must complete first |
|
|
552
|
-
| `tags` | string[] | [] | Tags for filtering |
|
|
553
|
-
| `groupId` | string | null | Group identifier |
|
|
554
|
-
| `lifo` | boolean | false | Last-in-first-out ordering |
|
|
555
|
-
| `removeOnComplete` | boolean | false | Auto-delete on completion |
|
|
556
|
-
| `removeOnFail` | boolean | false | Auto-delete on failure |
|
|
557
|
-
|
|
558
|
-
### Cron Jobs
|
|
559
|
-
|
|
560
|
-
```bash
|
|
561
|
-
# Cron expression (every hour)
|
|
562
|
-
curl -X POST http://localhost:6790/cron \
|
|
563
|
-
-d '{"cmd":"Cron","name":"hourly-cleanup","queue":"maintenance","data":{"task":"cleanup"},"schedule":"0 * * * *"}'
|
|
564
|
-
|
|
565
|
-
# Fixed interval (every 5 minutes)
|
|
566
|
-
curl -X POST http://localhost:6790/cron \
|
|
567
|
-
-d '{"cmd":"Cron","name":"health-check","queue":"monitoring","data":{"check":"ping"},"repeatEvery":300000}'
|
|
365
|
+
# Health
|
|
366
|
+
curl http://localhost:6790/health
|
|
568
367
|
|
|
569
|
-
#
|
|
570
|
-
curl
|
|
571
|
-
-d '{"cmd":"Cron","name":"one-time-migration","queue":"migrations","data":{},"repeatEvery":0,"maxLimit":1}'
|
|
368
|
+
# Prometheus metrics
|
|
369
|
+
curl http://localhost:6790/prometheus
|
|
572
370
|
```
|
|
573
371
|
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
### Core Operations
|
|
577
|
-
|
|
578
|
-
| Command | Description |
|
|
579
|
-
|---------|-------------|
|
|
580
|
-
| `PUSH` | Add a job to a queue |
|
|
581
|
-
| `PUSHB` | Batch push multiple jobs |
|
|
582
|
-
| `PULL` | Get the next job from a queue |
|
|
583
|
-
| `PULLB` | Batch pull multiple jobs |
|
|
584
|
-
| `ACK` | Mark job as completed |
|
|
585
|
-
| `ACKB` | Batch acknowledge jobs |
|
|
586
|
-
| `FAIL` | Mark job as failed |
|
|
587
|
-
|
|
588
|
-
### Query Operations
|
|
589
|
-
|
|
590
|
-
| Command | Description |
|
|
591
|
-
|---------|-------------|
|
|
592
|
-
| `GetJob` | Get job by ID |
|
|
593
|
-
| `GetState` | Get job state |
|
|
594
|
-
| `GetResult` | Get job result |
|
|
595
|
-
| `GetJobs` | List jobs with filters |
|
|
596
|
-
| `GetJobCounts` | Count jobs by state |
|
|
597
|
-
| `GetJobByCustomId` | Find job by custom ID |
|
|
598
|
-
| `GetProgress` | Get job progress |
|
|
599
|
-
| `GetLogs` | Get job logs |
|
|
600
|
-
|
|
601
|
-
### Job Management
|
|
602
|
-
|
|
603
|
-
| Command | Description |
|
|
604
|
-
|---------|-------------|
|
|
605
|
-
| `Cancel` | Cancel a pending job |
|
|
606
|
-
| `Progress` | Update job progress |
|
|
607
|
-
| `Update` | Update job data |
|
|
608
|
-
| `ChangePriority` | Change job priority |
|
|
609
|
-
| `Promote` | Move delayed job to waiting |
|
|
610
|
-
| `MoveToDelayed` | Delay a waiting job |
|
|
611
|
-
| `Discard` | Discard a job |
|
|
612
|
-
| `Heartbeat` | Send job heartbeat |
|
|
613
|
-
| `AddLog` | Add log entry to job |
|
|
614
|
-
|
|
615
|
-
### Queue Control
|
|
616
|
-
|
|
617
|
-
| Command | Description |
|
|
618
|
-
|---------|-------------|
|
|
619
|
-
| `Pause` | Pause queue processing |
|
|
620
|
-
| `Resume` | Resume queue processing |
|
|
621
|
-
| `IsPaused` | Check if queue is paused |
|
|
622
|
-
| `Drain` | Remove all waiting jobs |
|
|
623
|
-
| `Obliterate` | Remove all queue data |
|
|
624
|
-
| `Clean` | Remove old jobs |
|
|
625
|
-
| `ListQueues` | List all queues |
|
|
626
|
-
| `RateLimit` | Set queue rate limit |
|
|
627
|
-
| `SetConcurrency` | Set max concurrent jobs |
|
|
628
|
-
|
|
629
|
-
### Dead Letter Queue
|
|
630
|
-
|
|
631
|
-
| Command | Description |
|
|
632
|
-
|---------|-------------|
|
|
633
|
-
| `Dlq` | Get failed jobs |
|
|
634
|
-
| `RetryDlq` | Retry failed jobs |
|
|
635
|
-
| `PurgeDlq` | Clear failed jobs |
|
|
636
|
-
|
|
637
|
-
### Scheduling
|
|
638
|
-
|
|
639
|
-
| Command | Description |
|
|
640
|
-
|---------|-------------|
|
|
641
|
-
| `Cron` | Create/update cron job |
|
|
642
|
-
| `CronDelete` | Delete cron job |
|
|
643
|
-
| `CronList` | List all cron jobs |
|
|
644
|
-
|
|
645
|
-
### Workers & Webhooks
|
|
646
|
-
|
|
647
|
-
| Command | Description |
|
|
648
|
-
|---------|-------------|
|
|
649
|
-
| `RegisterWorker` | Register a worker |
|
|
650
|
-
| `UnregisterWorker` | Unregister a worker |
|
|
651
|
-
| `ListWorkers` | List active workers |
|
|
652
|
-
| `AddWebhook` | Add webhook endpoint |
|
|
653
|
-
| `RemoveWebhook` | Remove webhook |
|
|
654
|
-
| `ListWebhooks` | List webhooks |
|
|
655
|
-
|
|
656
|
-
### Monitoring
|
|
657
|
-
|
|
658
|
-
| Command | Description |
|
|
659
|
-
|---------|-------------|
|
|
660
|
-
| `Stats` | Get server statistics |
|
|
661
|
-
| `Metrics` | Get job metrics |
|
|
662
|
-
| `Prometheus` | Prometheus format metrics |
|
|
663
|
-
|
|
664
|
-
## Configuration
|
|
665
|
-
|
|
666
|
-
### Environment Variables
|
|
372
|
+
### TCP Protocol
|
|
667
373
|
|
|
668
|
-
| Variable | Default | Description |
|
|
669
|
-
|----------|---------|-------------|
|
|
670
|
-
| `TCP_PORT` | 6789 | TCP protocol port |
|
|
671
|
-
| `HTTP_PORT` | 6790 | HTTP/WebSocket port |
|
|
672
|
-
| `HOST` | 0.0.0.0 | Bind address |
|
|
673
|
-
| `AUTH_TOKENS` | - | Comma-separated auth tokens |
|
|
674
|
-
| `DATA_PATH` | - | SQLite database path (in-memory if not set) |
|
|
675
|
-
| `CORS_ALLOW_ORIGIN` | * | Allowed CORS origins |
|
|
676
|
-
|
|
677
|
-
### S3 Backup Configuration
|
|
678
|
-
|
|
679
|
-
| Variable | Default | Description |
|
|
680
|
-
|----------|---------|-------------|
|
|
681
|
-
| `S3_BACKUP_ENABLED` | 0 | Enable automated S3 backups (1/true) |
|
|
682
|
-
| `S3_ACCESS_KEY_ID` | - | S3 access key |
|
|
683
|
-
| `S3_SECRET_ACCESS_KEY` | - | S3 secret key |
|
|
684
|
-
| `S3_BUCKET` | - | S3 bucket name |
|
|
685
|
-
| `S3_REGION` | us-east-1 | S3 region |
|
|
686
|
-
| `S3_ENDPOINT` | - | Custom endpoint for S3-compatible services |
|
|
687
|
-
| `S3_BACKUP_INTERVAL` | 21600000 | Backup interval in ms (default: 6 hours) |
|
|
688
|
-
| `S3_BACKUP_RETENTION` | 7 | Number of backups to retain |
|
|
689
|
-
| `S3_BACKUP_PREFIX` | backups/ | Prefix for backup files |
|
|
690
|
-
|
|
691
|
-
**Supported S3 Providers:**
|
|
692
|
-
- AWS S3
|
|
693
|
-
- Cloudflare R2: `S3_ENDPOINT=https://<account-id>.r2.cloudflarestorage.com`
|
|
694
|
-
- MinIO: `S3_ENDPOINT=http://localhost:9000`
|
|
695
|
-
- DigitalOcean Spaces: `S3_ENDPOINT=https://<region>.digitaloceanspaces.com`
|
|
696
|
-
|
|
697
|
-
**CLI Commands:**
|
|
698
374
|
```bash
|
|
699
|
-
|
|
700
|
-
bunqueue backup now
|
|
701
|
-
|
|
702
|
-
# List available backups
|
|
703
|
-
bunqueue backup list
|
|
704
|
-
|
|
705
|
-
# Restore from backup (requires --force)
|
|
706
|
-
bunqueue backup restore backups/bunqueue-2024-01-15.db --force
|
|
375
|
+
nc localhost 6789
|
|
707
376
|
|
|
708
|
-
#
|
|
709
|
-
|
|
377
|
+
# Commands (JSON)
|
|
378
|
+
{"cmd":"PUSH","queue":"tasks","data":{"action":"process"}}
|
|
379
|
+
{"cmd":"PULL","queue":"tasks","timeout":5000}
|
|
380
|
+
{"cmd":"ACK","id":"1","result":{"done":true}}
|
|
381
|
+
{"cmd":"FAIL","id":"1","error":"Something went wrong"}
|
|
710
382
|
```
|
|
711
383
|
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
Enable authentication by setting `AUTH_TOKENS`:
|
|
384
|
+
---
|
|
715
385
|
|
|
716
|
-
|
|
717
|
-
AUTH_TOKENS=token1,token2 bun run start
|
|
718
|
-
```
|
|
386
|
+
## CLI
|
|
719
387
|
|
|
720
|
-
**HTTP:**
|
|
721
388
|
```bash
|
|
722
|
-
|
|
723
|
-
|
|
389
|
+
# Server
|
|
390
|
+
bunqueue start
|
|
391
|
+
bunqueue start --tcp-port 6789 --http-port 6790
|
|
724
392
|
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
{"
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
```json
|
|
732
|
-
{"cmd":"Auth","token":"token1"}
|
|
733
|
-
```
|
|
734
|
-
|
|
735
|
-
## Monitoring
|
|
736
|
-
|
|
737
|
-
### Health Check
|
|
393
|
+
# Jobs
|
|
394
|
+
bunqueue push emails '{"to":"user@test.com"}'
|
|
395
|
+
bunqueue push tasks '{"action":"sync"}' --priority 10 --delay 5000
|
|
396
|
+
bunqueue pull emails --timeout 5000
|
|
397
|
+
bunqueue ack <job-id>
|
|
398
|
+
bunqueue fail <job-id> --error "Failed"
|
|
738
399
|
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
400
|
+
# Job management
|
|
401
|
+
bunqueue job get <id>
|
|
402
|
+
bunqueue job progress <id> 50 --message "Processing"
|
|
403
|
+
bunqueue job cancel <id>
|
|
743
404
|
|
|
744
|
-
|
|
405
|
+
# Queue control
|
|
406
|
+
bunqueue queue list
|
|
407
|
+
bunqueue queue pause emails
|
|
408
|
+
bunqueue queue resume emails
|
|
409
|
+
bunqueue queue drain emails
|
|
745
410
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
411
|
+
# Cron
|
|
412
|
+
bunqueue cron list
|
|
413
|
+
bunqueue cron add cleanup -q maintenance -d '{}' -s "0 * * * *"
|
|
414
|
+
bunqueue cron delete cleanup
|
|
749
415
|
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
- `bunqueue_queue_latency_seconds{queue}` — Processing latency
|
|
416
|
+
# DLQ
|
|
417
|
+
bunqueue dlq list emails
|
|
418
|
+
bunqueue dlq retry emails
|
|
419
|
+
bunqueue dlq purge emails
|
|
755
420
|
|
|
756
|
-
|
|
421
|
+
# Monitoring
|
|
422
|
+
bunqueue stats
|
|
423
|
+
bunqueue metrics
|
|
424
|
+
bunqueue health
|
|
757
425
|
|
|
758
|
-
|
|
759
|
-
|
|
426
|
+
# Backup (S3)
|
|
427
|
+
bunqueue backup now
|
|
428
|
+
bunqueue backup list
|
|
429
|
+
bunqueue backup restore <key> --force
|
|
760
430
|
```
|
|
761
431
|
|
|
762
|
-
|
|
763
|
-
{
|
|
764
|
-
"ok": true,
|
|
765
|
-
"stats": {
|
|
766
|
-
"waiting": 150,
|
|
767
|
-
"active": 10,
|
|
768
|
-
"delayed": 25,
|
|
769
|
-
"completed": 10000,
|
|
770
|
-
"failed": 50,
|
|
771
|
-
"dlq": 5,
|
|
772
|
-
"totalPushed": 10235,
|
|
773
|
-
"totalPulled": 10085,
|
|
774
|
-
"totalCompleted": 10000,
|
|
775
|
-
"totalFailed": 50
|
|
776
|
-
}
|
|
777
|
-
}
|
|
778
|
-
```
|
|
432
|
+
---
|
|
779
433
|
|
|
780
434
|
## Docker
|
|
781
435
|
|
|
782
|
-
### Build
|
|
783
|
-
|
|
784
436
|
```bash
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
### Run
|
|
789
|
-
|
|
790
|
-
```bash
|
|
791
|
-
# Basic
|
|
792
|
-
docker run -p 6789:6789 -p 6790:6790 bunqueue
|
|
437
|
+
# Run
|
|
438
|
+
docker run -p 6789:6789 -p 6790:6790 ghcr.io/egeominotti/bunqueue
|
|
793
439
|
|
|
794
440
|
# With persistence
|
|
795
441
|
docker run -p 6789:6789 -p 6790:6790 \
|
|
796
442
|
-v bunqueue-data:/app/data \
|
|
797
443
|
-e DATA_PATH=/app/data/bunqueue.db \
|
|
798
|
-
bunqueue
|
|
444
|
+
ghcr.io/egeominotti/bunqueue
|
|
799
445
|
|
|
800
|
-
# With
|
|
446
|
+
# With auth
|
|
801
447
|
docker run -p 6789:6789 -p 6790:6790 \
|
|
802
|
-
-e AUTH_TOKENS=
|
|
803
|
-
bunqueue
|
|
448
|
+
-e AUTH_TOKENS=secret \
|
|
449
|
+
ghcr.io/egeominotti/bunqueue
|
|
804
450
|
```
|
|
805
451
|
|
|
806
452
|
### Docker Compose
|
|
807
453
|
|
|
808
|
-
```
|
|
809
|
-
|
|
810
|
-
|
|
454
|
+
```yaml
|
|
455
|
+
version: "3.8"
|
|
456
|
+
services:
|
|
457
|
+
bunqueue:
|
|
458
|
+
image: ghcr.io/egeominotti/bunqueue
|
|
459
|
+
ports:
|
|
460
|
+
- "6789:6789"
|
|
461
|
+
- "6790:6790"
|
|
462
|
+
volumes:
|
|
463
|
+
- bunqueue-data:/app/data
|
|
464
|
+
environment:
|
|
465
|
+
- DATA_PATH=/app/data/bunqueue.db
|
|
466
|
+
- AUTH_TOKENS=your-secret-token
|
|
811
467
|
|
|
812
|
-
|
|
813
|
-
|
|
468
|
+
volumes:
|
|
469
|
+
bunqueue-data:
|
|
814
470
|
```
|
|
815
471
|
|
|
816
|
-
|
|
472
|
+
---
|
|
817
473
|
|
|
818
|
-
|
|
474
|
+
## S3 Backup
|
|
819
475
|
|
|
820
|
-
|
|
476
|
+
```env
|
|
477
|
+
S3_BACKUP_ENABLED=1
|
|
478
|
+
S3_ACCESS_KEY_ID=your-key
|
|
479
|
+
S3_SECRET_ACCESS_KEY=your-secret
|
|
480
|
+
S3_BUCKET=my-bucket
|
|
481
|
+
S3_REGION=us-east-1
|
|
482
|
+
S3_BACKUP_INTERVAL=21600000 # 6 hours
|
|
483
|
+
S3_BACKUP_RETENTION=7
|
|
484
|
+
```
|
|
821
485
|
|
|
822
|
-
|
|
823
|
-
|----------|:---:|:------:|:---:|-------|
|
|
824
|
-
| [Fly.io](https://fly.io) | ✅ | ✅ | ✅ | Recommended - persistent volumes, global deployment |
|
|
825
|
-
| [Railway](https://railway.app) | ✅ | ✅ | ✅ | Easy deploy from GitHub |
|
|
826
|
-
| [Render](https://render.com) | ✅ | ✅ | ✅ | Docker support, persistent disks |
|
|
827
|
-
| [DigitalOcean](https://digitalocean.com) | ✅ | ✅ | ✅ | App Platform or Droplets |
|
|
828
|
-
| Any VPS | ✅ | ✅ | ✅ | Full control |
|
|
486
|
+
Supported providers: AWS S3, Cloudflare R2, MinIO, DigitalOcean Spaces.
|
|
829
487
|
|
|
830
|
-
|
|
488
|
+
---
|
|
831
489
|
|
|
832
|
-
|
|
833
|
-
# Install flyctl
|
|
834
|
-
curl -L https://fly.io/install.sh | sh
|
|
490
|
+
## When to Use What?
|
|
835
491
|
|
|
836
|
-
|
|
837
|
-
|
|
492
|
+
| Scenario | Mode |
|
|
493
|
+
|----------|------|
|
|
494
|
+
| Single app, monolith | **Embedded** |
|
|
495
|
+
| Scripts, CLI tools | **Embedded** |
|
|
496
|
+
| Serverless (with persistence) | **Embedded** |
|
|
497
|
+
| Microservices | **Server** |
|
|
498
|
+
| Multiple languages | **Server** (HTTP API) |
|
|
499
|
+
| Horizontal scaling | **Server** |
|
|
838
500
|
|
|
839
|
-
|
|
840
|
-
fly volumes create bunqueue_data --size 1
|
|
501
|
+
### Server Mode SDK
|
|
841
502
|
|
|
842
|
-
|
|
843
|
-
fly secrets set AUTH_TOKENS=your-secret-token
|
|
503
|
+
For communicating with bunqueue server from **separate processes**, use the [flashq](https://www.npmjs.com/package/flashq) SDK:
|
|
844
504
|
|
|
845
|
-
|
|
846
|
-
|
|
505
|
+
```bash
|
|
506
|
+
bun add flashq
|
|
847
507
|
```
|
|
848
508
|
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
[mounts]
|
|
852
|
-
source = "bunqueue_data"
|
|
853
|
-
destination = "/app/data"
|
|
854
|
-
|
|
855
|
-
[env]
|
|
856
|
-
DATA_PATH = "/app/data/bunqueue.db"
|
|
857
|
-
```
|
|
509
|
+
```typescript
|
|
510
|
+
import { FlashQ } from 'flashq';
|
|
858
511
|
|
|
859
|
-
|
|
512
|
+
const client = new FlashQ({ host: 'localhost', port: 6789 });
|
|
860
513
|
|
|
861
|
-
|
|
514
|
+
// Push job
|
|
515
|
+
await client.push('emails', { to: 'user@test.com' });
|
|
862
516
|
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
517
|
+
// Pull and process
|
|
518
|
+
const job = await client.pull('emails');
|
|
519
|
+
if (job) {
|
|
520
|
+
console.log('Processing:', job.data);
|
|
521
|
+
await client.ack(job.id);
|
|
522
|
+
}
|
|
868
523
|
```
|
|
869
524
|
|
|
870
|
-
|
|
525
|
+
| Package | Use Case |
|
|
526
|
+
|---------|----------|
|
|
527
|
+
| `bunqueue/client` | Same process (embedded) |
|
|
528
|
+
| `flashq` | Different process (TCP client) |
|
|
871
529
|
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
530
|
+
```
|
|
531
|
+
┌─────────────────┐ ┌─────────────────┐
|
|
532
|
+
│ Your App │ │ Your App │
|
|
533
|
+
│ │ │ │
|
|
534
|
+
│ bunqueue/client│ │ flashq │
|
|
535
|
+
│ (embedded) │ │ (TCP client) │
|
|
536
|
+
└────────┬────────┘ └────────┬────────┘
|
|
537
|
+
│ │
|
|
538
|
+
│ ▼
|
|
539
|
+
│ ┌─────────────────┐
|
|
540
|
+
│ │ bunqueue server │
|
|
541
|
+
│ │ (port 6789) │
|
|
542
|
+
│ └─────────────────┘
|
|
543
|
+
│
|
|
544
|
+
Same process Different process
|
|
545
|
+
```
|
|
546
|
+
|
|
547
|
+
---
|
|
878
548
|
|
|
879
549
|
## Architecture
|
|
880
550
|
|
|
881
551
|
```
|
|
882
552
|
┌─────────────────────────────────────────────────────────────┐
|
|
883
|
-
│ bunqueue
|
|
553
|
+
│ bunqueue │
|
|
884
554
|
├─────────────────────────────────────────────────────────────┤
|
|
885
|
-
│
|
|
555
|
+
│ Embedded Mode │ Server Mode │
|
|
556
|
+
│ (bunqueue/client) │ (bunqueue start) │
|
|
557
|
+
│ │ │
|
|
558
|
+
│ Queue, Worker │ TCP (6789) + HTTP (6790) │
|
|
559
|
+
│ in-process │ multi-process │
|
|
886
560
|
├─────────────────────────────────────────────────────────────┤
|
|
887
|
-
│ Core Engine
|
|
888
|
-
│ ┌──────────┐ ┌──────────┐ ┌───────────┐ ┌──────────┐
|
|
889
|
-
│ │ Queues │ │ Workers │ │ Scheduler │ │ DLQ │
|
|
890
|
-
│ │
|
|
891
|
-
│ └──────────┘ └──────────┘ └───────────┘ └──────────┘
|
|
561
|
+
│ Core Engine │
|
|
562
|
+
│ ┌──────────┐ ┌──────────┐ ┌───────────┐ ┌──────────┐ │
|
|
563
|
+
│ │ Queues │ │ Workers │ │ Scheduler │ │ DLQ │ │
|
|
564
|
+
│ │(32 shards)│ │ │ │ (Cron) │ │ │ │
|
|
565
|
+
│ └──────────┘ └──────────┘ └───────────┘ └──────────┘ │
|
|
892
566
|
├─────────────────────────────────────────────────────────────┤
|
|
893
|
-
│ SQLite (WAL mode, 256MB mmap)
|
|
567
|
+
│ SQLite (WAL mode, 256MB mmap) │
|
|
894
568
|
└─────────────────────────────────────────────────────────────┘
|
|
895
569
|
```
|
|
896
570
|
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
- **32 Shards** — Lock contention minimized with FNV-1a hash distribution
|
|
900
|
-
- **WAL Mode** — Concurrent reads during writes
|
|
901
|
-
- **Memory-mapped I/O** — 256MB mmap for fast access
|
|
902
|
-
- **Batch Operations** — Bulk inserts and updates
|
|
903
|
-
- **Bounded Collections** — Automatic memory cleanup
|
|
571
|
+
---
|
|
904
572
|
|
|
905
573
|
## Contributing
|
|
906
574
|
|
|
907
|
-
Contributions are welcome! Please read our contributing guidelines before submitting PRs.
|
|
908
|
-
|
|
909
575
|
```bash
|
|
910
|
-
# Install dependencies
|
|
911
576
|
bun install
|
|
912
|
-
|
|
913
|
-
# Run tests
|
|
914
577
|
bun test
|
|
915
|
-
|
|
916
|
-
# Run linter
|
|
917
578
|
bun run lint
|
|
918
|
-
|
|
919
|
-
# Format code
|
|
920
579
|
bun run format
|
|
921
|
-
|
|
922
|
-
# Type check
|
|
923
|
-
bun run typecheck
|
|
924
|
-
|
|
925
|
-
# Run all checks
|
|
926
580
|
bun run check
|
|
927
581
|
```
|
|
928
582
|
|
|
583
|
+
---
|
|
584
|
+
|
|
929
585
|
## License
|
|
930
586
|
|
|
931
587
|
MIT License — see [LICENSE](LICENSE) for details.
|