@bitclaw/jobs 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +355 -31
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/otel.d.ts +39 -0
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +45 -0
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -1,18 +1,6 @@
|
|
|
1
1
|
# @bitclaw/jobs
|
|
2
2
|
|
|
3
|
-
SQLite-backed background job queue for Bun.
|
|
4
|
-
|
|
5
|
-
## Features
|
|
6
|
-
|
|
7
|
-
- **Typed** Generic `JobQueue<TMap>` with per-type payload validation
|
|
8
|
-
- **Priority** Jobs ordered by priority DESC then created_at ASC
|
|
9
|
-
- **Retries** Configurable `maxRetries` with automatic dead-letter after exhaustion
|
|
10
|
-
- **Dependencies** Blocked jobs auto-unblock when all dependencies complete
|
|
11
|
-
- **Batches** Group jobs, track progress, fire `then`/`finally` callbacks on completion
|
|
12
|
-
- **Cron** 5-field cron parser with `nextCronOccurrence` and overlap control
|
|
13
|
-
- **Scheduler** Persistent `schedules` table with upsert semantics and cleanup
|
|
14
|
-
- **Rate Limiter** In-memory sliding window per-worker throttling
|
|
15
|
-
- **Worker** `setTimeout`-based poll loop with graceful shutdown and per-job timeout
|
|
3
|
+
SQLite-backed background job queue for Bun. Typed generics, multi-process lease safety, priority aging, middleware pipeline, workflow engine with saga compensation, and zero runtime dependencies.
|
|
16
4
|
|
|
17
5
|
## Installation
|
|
18
6
|
|
|
@@ -20,6 +8,36 @@ SQLite-backed background job queue for Bun. Features priority ordering, retries
|
|
|
20
8
|
bun add @bitclaw/jobs
|
|
21
9
|
```
|
|
22
10
|
|
|
11
|
+
Requires Bun ≥ 1.3.0. Uses `bun:sqlite` — no native build step, no extra packages.
|
|
12
|
+
|
|
13
|
+
## Feature Overview
|
|
14
|
+
|
|
15
|
+
| Feature | Details |
|
|
16
|
+
|---|---|
|
|
17
|
+
| **Typed payloads** | `JobQueue<TMap>` — per-type payload inference, no `any` |
|
|
18
|
+
| **Priority + aging** | Jobs ordered by priority DESC; workers can boost old jobs per minute |
|
|
19
|
+
| **Retries + backoff** | `exponential`, `fixed`, `jitter`, `fibonacci`; `retryIf` predicate to skip retries |
|
|
20
|
+
| **Dead-letter** | Exhausted jobs moved to `failed_jobs`, retryable via `retryFailedJob` |
|
|
21
|
+
| **Dependencies** | Blocked jobs auto-unblock when all deps complete |
|
|
22
|
+
| **Multi-process safety** | Lease column (`claimed_until`) prevents double-claim across processes |
|
|
23
|
+
| **Lease renewal** | Long-running handlers call `ctx.renewLease()` to extend their claim |
|
|
24
|
+
| **Batches** | Group jobs, track progress, fire `then`/`finally` callbacks on completion |
|
|
25
|
+
| **Cron scheduler** | 5-field cron parser, persistent `schedules` table, overlap control |
|
|
26
|
+
| **Rate limiter** | Per-worker sliding-window throttle (`maxRate`) |
|
|
27
|
+
| **Middleware** | Onion-style `queue.use(fn)` wraps all executions (logging, OTel, timing) |
|
|
28
|
+
| **Job graph API** | `getJobGraph(id)` traverses dependency DAG via recursive CTE |
|
|
29
|
+
| **Dedup** | `uniqueKey` + `dedup: 'ignore' | 'replace'` — state-aware, key reusable after completion |
|
|
30
|
+
| **TTL** | `expireAt` — expired jobs silently skipped and purgeable |
|
|
31
|
+
| **Result storage** | Handler return value persisted; `getJobResult<T>(id)` to read it |
|
|
32
|
+
| **Webhook on completion** | `onComplete: { url }` fires a detached POST after job finishes |
|
|
33
|
+
| **Typed events** | `queue.on('job:done' | 'job:failed' | 'job:dead' | ...)` |
|
|
34
|
+
| **Pause / resume** | `worker.pause()` / `worker.resume()` — in-flight job finishes, no new claims |
|
|
35
|
+
| **Admin handler** | `queue.mountAdminHandler()` returns a zero-dep `Request → Response` handler |
|
|
36
|
+
| **Workflow engine** | `WorkflowEngine` — typed DAG of steps, saga compensation, restart-safe `reconcile()` |
|
|
37
|
+
| **OpenTelemetry** | `createOtelMiddleware(tracer)` — zero-dep, structural tracer interface |
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
23
41
|
## Quick Start
|
|
24
42
|
|
|
25
43
|
```typescript
|
|
@@ -27,58 +45,364 @@ import { JobQueue } from '@bitclaw/jobs'
|
|
|
27
45
|
|
|
28
46
|
type AppJobs = {
|
|
29
47
|
'email:send': { to: string; subject: string }
|
|
48
|
+
'report:generate': { type: string }
|
|
30
49
|
}
|
|
31
50
|
|
|
32
51
|
const queue = new JobQueue<AppJobs>('./jobs.db')
|
|
33
|
-
queue.add('email:send', { to: 'user@test.com', subject: 'Hello' })
|
|
34
|
-
```
|
|
35
52
|
|
|
36
|
-
|
|
53
|
+
// Enqueue
|
|
54
|
+
queue.add('email:send', { to: 'user@example.com', subject: 'Welcome' })
|
|
37
55
|
|
|
38
|
-
|
|
56
|
+
// Worker
|
|
39
57
|
const worker = queue.createWorker({
|
|
40
58
|
type: 'email:send',
|
|
41
59
|
handler: async (job, ctx) => {
|
|
42
60
|
ctx.reportProgress(50)
|
|
43
|
-
await sendEmail(job.data
|
|
61
|
+
await sendEmail(job.data)
|
|
62
|
+
return { sent: true } // stored in job.result
|
|
44
63
|
},
|
|
45
64
|
pollIntervalMs: 1000,
|
|
46
|
-
maxRate: { count:
|
|
65
|
+
maxRate: { count: 20, windowMs: 1000 }
|
|
47
66
|
})
|
|
48
67
|
|
|
49
68
|
worker.start()
|
|
50
|
-
// ... later
|
|
51
|
-
await worker.stop()
|
|
52
69
|
```
|
|
53
70
|
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Retries and Backoff
|
|
74
|
+
|
|
75
|
+
```typescript
|
|
76
|
+
queue.add('email:send', data, {
|
|
77
|
+
maxRetries: 5,
|
|
78
|
+
backoff: { type: 'exponential', delayMs: 1000 }
|
|
79
|
+
// types: 'exponential' | 'fixed' | 'jitter' | 'fibonacci'
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
// Skip retries for permanent errors
|
|
83
|
+
queue.createWorker({
|
|
84
|
+
type: 'email:send',
|
|
85
|
+
handler: async job => { /* ... */ },
|
|
86
|
+
retryIf: (err, job) => !(err instanceof ValidationError)
|
|
87
|
+
})
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
---
|
|
91
|
+
|
|
92
|
+
## Dependencies
|
|
93
|
+
|
|
94
|
+
```typescript
|
|
95
|
+
const depA = queue.add('data:fetch', { source: 'api' })
|
|
96
|
+
const depB = queue.add('data:fetch', { source: 'db' })
|
|
97
|
+
|
|
98
|
+
// Blocked until both depA and depB complete
|
|
99
|
+
queue.add('report:generate', { type: 'combined' }, {
|
|
100
|
+
dependsOn: [depA, depB]
|
|
101
|
+
})
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
## Multi-Process Lease Safety
|
|
107
|
+
|
|
108
|
+
Two workers against the same DB file cannot claim the same job. `pollAndClaim` atomically sets `claimed_until`. If a worker crashes mid-job, any other worker reclaims after lease expiry.
|
|
109
|
+
|
|
110
|
+
```typescript
|
|
111
|
+
// Default lease: 5 minutes. Override per worker:
|
|
112
|
+
queue.createWorker({
|
|
113
|
+
type: 'server:provision',
|
|
114
|
+
handler: async (job, ctx) => {
|
|
115
|
+
await longStep()
|
|
116
|
+
ctx.renewLease() // extend before expiry
|
|
117
|
+
await anotherStep()
|
|
118
|
+
},
|
|
119
|
+
leaseMs: 600_000 // 10 min
|
|
120
|
+
})
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## Middleware
|
|
126
|
+
|
|
127
|
+
```typescript
|
|
128
|
+
// Logging
|
|
129
|
+
queue.use(async (job, next) => {
|
|
130
|
+
console.info(`[job] ${job.type} #${job.id} start`)
|
|
131
|
+
const result = await next()
|
|
132
|
+
console.info(`[job] ${job.type} #${job.id} done`)
|
|
133
|
+
return result
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
// Multiple middlewares run in registration order (onion)
|
|
137
|
+
queue.use(timingMiddleware)
|
|
138
|
+
queue.use(loggingMiddleware)
|
|
139
|
+
// execution: timing → logging → handler → logging → timing
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## OpenTelemetry
|
|
145
|
+
|
|
146
|
+
No peer dependency. Pass any OTel-compatible tracer — the interface is structural.
|
|
147
|
+
|
|
148
|
+
```typescript
|
|
149
|
+
import { trace } from '@opentelemetry/api'
|
|
150
|
+
import { createOtelMiddleware } from '@bitclaw/jobs/otel'
|
|
151
|
+
|
|
152
|
+
const tracer = trace.getTracer('my-app', '1.0.0')
|
|
153
|
+
queue.use(createOtelMiddleware(tracer))
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Each job execution becomes a span named `job.<type>` with attributes:
|
|
157
|
+
|
|
158
|
+
| Attribute | Value |
|
|
159
|
+
|---|---|
|
|
160
|
+
| `job.id` | numeric job ID |
|
|
161
|
+
| `job.type` | type string |
|
|
162
|
+
| `job.priority` | priority |
|
|
163
|
+
| `job.retry` | retry count at execution time |
|
|
164
|
+
| `job.error` | error message (failure only) |
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## Priority Aging
|
|
169
|
+
|
|
170
|
+
Prevents starvation of low-priority jobs under sustained high-priority load.
|
|
171
|
+
|
|
172
|
+
```typescript
|
|
173
|
+
queue.createWorker({
|
|
174
|
+
type: 'email:send',
|
|
175
|
+
handler: async job => { /* ... */ },
|
|
176
|
+
aging: {
|
|
177
|
+
boostPerMinute: 10, // +10 priority per minute of wait
|
|
178
|
+
maxBoost: 100 // cap
|
|
179
|
+
}
|
|
180
|
+
})
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## Job Graph API
|
|
186
|
+
|
|
187
|
+
```typescript
|
|
188
|
+
const nodes = queue.getJobGraph(rootJobId)
|
|
189
|
+
// nodes: Array<{ id, type, status, result, dependsOn: number[], dependents: number[] }>
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Traverses the full dependency DAG (ancestors + descendants) via SQLite recursive CTE.
|
|
193
|
+
|
|
194
|
+
---
|
|
195
|
+
|
|
196
|
+
## Deduplication
|
|
197
|
+
|
|
198
|
+
```typescript
|
|
199
|
+
// Ignore: silently re-uses existing pending job
|
|
200
|
+
queue.add('report:generate', data, { uniqueKey: 'daily-2026-06-27', dedup: 'ignore' })
|
|
201
|
+
|
|
202
|
+
// Replace: updates data on existing pending job
|
|
203
|
+
queue.add('report:generate', freshData, { uniqueKey: 'daily-2026-06-27', dedup: 'replace' })
|
|
204
|
+
|
|
205
|
+
// Cancel pending job by key
|
|
206
|
+
queue.cancelByUniqueKey('report:generate', 'daily-2026-06-27')
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## Result Storage
|
|
212
|
+
|
|
213
|
+
```typescript
|
|
214
|
+
const id = queue.add('data:process', payload)
|
|
215
|
+
|
|
216
|
+
// In handler — return value is stored automatically
|
|
217
|
+
queue.createWorker({
|
|
218
|
+
type: 'data:process',
|
|
219
|
+
handler: async job => {
|
|
220
|
+
return { count: 42, processed: true }
|
|
221
|
+
}
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
// Read later
|
|
225
|
+
const result = queue.getJobResult<{ count: number; processed: boolean }>(id)
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
## Typed Events
|
|
231
|
+
|
|
232
|
+
```typescript
|
|
233
|
+
queue.on('job:done', job => console.info('done', job.id))
|
|
234
|
+
queue.on('job:failed', (job, err) => console.warn('retry', err))
|
|
235
|
+
queue.on('job:dead', (job, err) => alert.send(err))
|
|
236
|
+
queue.on('job:progress', (job, pct) => ws.send({ id: job.id, pct }))
|
|
237
|
+
queue.on('batch:complete', batch => notify(batch.id))
|
|
238
|
+
|
|
239
|
+
const unsub = queue.on('job:done', cb)
|
|
240
|
+
unsub() // remove listener
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
---
|
|
244
|
+
|
|
245
|
+
## Workflow Engine
|
|
246
|
+
|
|
247
|
+
Typed DAG of steps where each step is a real job. Dependencies handled by the existing `job_dependencies` mechanism — no separate scheduler. Saga compensation runs in reverse topological order if any step fails permanently.
|
|
248
|
+
|
|
249
|
+
```typescript
|
|
250
|
+
import { WorkflowEngine } from '@bitclaw/jobs'
|
|
251
|
+
|
|
252
|
+
const engine = new WorkflowEngine(queue)
|
|
253
|
+
|
|
254
|
+
const { instanceId, jobIds } = engine
|
|
255
|
+
.workflow('order-flow')
|
|
256
|
+
.step('charge', 'payment:charge', { amount: 100 })
|
|
257
|
+
.step('provision', 'server:provision', { serverId: 'srv-1' }, { dependsOn: ['charge'] })
|
|
258
|
+
.step('notify', 'email:welcome', { userId: 'u1' }, { dependsOn: ['provision'] })
|
|
259
|
+
.onFail('charge', {
|
|
260
|
+
compensate: 'payment:refund',
|
|
261
|
+
compensateData: { amount: 100, reason: 'provision-failed' }
|
|
262
|
+
})
|
|
263
|
+
.run()
|
|
264
|
+
|
|
265
|
+
// Reconcile on startup — resumes any interrupted workflows
|
|
266
|
+
engine.reconcile()
|
|
267
|
+
|
|
268
|
+
// Read a step's result from within a dependent handler
|
|
269
|
+
const chargeResult = engine.getStepResult<ChargeResult>(instanceId, 'charge')
|
|
270
|
+
|
|
271
|
+
// Query executions
|
|
272
|
+
engine.listExecutions({ status: 'running', name: 'order-flow' })
|
|
273
|
+
engine.getExecution(instanceId)
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
**Saga semantics:** `onFail(stepName, { compensate, compensateData })` registers a compensation job for a step that completed successfully. If a later step fails permanently, `reconcile()` enqueues compensation jobs (in reverse topological order) for all completed steps that registered one, then transitions the execution to `compensating`. When all compensation jobs finish, the execution is marked `failed`.
|
|
277
|
+
|
|
278
|
+
**Restart safety:** Call `engine.reconcile()` on boot. It finds all `running`/`compensating` executions and advances their state without re-running completed steps.
|
|
279
|
+
|
|
280
|
+
---
|
|
281
|
+
|
|
54
282
|
## Cron Scheduler
|
|
55
283
|
|
|
56
284
|
```typescript
|
|
57
285
|
import { Scheduler } from '@bitclaw/jobs'
|
|
58
286
|
|
|
59
287
|
const scheduler = new Scheduler(queue)
|
|
288
|
+
|
|
60
289
|
scheduler.register('daily-report', 'report:generate', '0 2 * * *', {
|
|
61
|
-
data: { type: 'daily' }
|
|
290
|
+
data: { type: 'daily' },
|
|
291
|
+
timezone: 'America/New_York',
|
|
292
|
+
overlap: false // skip if previous run still processing
|
|
293
|
+
})
|
|
294
|
+
|
|
295
|
+
scheduler.start() // ticks every 60s by default
|
|
296
|
+
await scheduler.stop()
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
---
|
|
300
|
+
|
|
301
|
+
## Admin Handler
|
|
302
|
+
|
|
303
|
+
Zero-dependency `Request → Response` handler. Mount in any framework.
|
|
304
|
+
|
|
305
|
+
```typescript
|
|
306
|
+
// TanStack Start / Hono / bare Bun.serve
|
|
307
|
+
const adminHandler = queue.mountAdminHandler('/admin/jobs')
|
|
308
|
+
|
|
309
|
+
// Routes:
|
|
310
|
+
// GET /admin/jobs/stats
|
|
311
|
+
// GET /admin/jobs/jobs
|
|
312
|
+
// GET /admin/jobs/jobs/:id
|
|
313
|
+
// GET /admin/jobs/jobs/:id/graph
|
|
314
|
+
// POST /admin/jobs/jobs/:id/cancel
|
|
315
|
+
// POST /admin/jobs/jobs/:id/force-retry
|
|
316
|
+
// GET /admin/jobs/failed
|
|
317
|
+
// POST /admin/jobs/failed/:id/retry
|
|
318
|
+
// POST /admin/jobs/failed/retry-by-type
|
|
319
|
+
// GET /admin/jobs/jobs/types
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
---
|
|
323
|
+
|
|
324
|
+
## Batch Processing
|
|
325
|
+
|
|
326
|
+
```typescript
|
|
327
|
+
const { batchId, jobIds } = queue.addBatch('nightly-sync', [
|
|
328
|
+
{ type: 'user:sync', data: { userId: 'u1' } },
|
|
329
|
+
{ type: 'user:sync', data: { userId: 'u2' } }
|
|
330
|
+
], {
|
|
331
|
+
thenType: 'report:generate',
|
|
332
|
+
thenData: { trigger: 'batch-done' }
|
|
62
333
|
})
|
|
63
|
-
|
|
334
|
+
|
|
335
|
+
const batch = queue.getBatch(batchId)
|
|
336
|
+
// { totalJobs: 2, pendingJobs: 1, failedJobs: 0, ... }
|
|
64
337
|
```
|
|
65
338
|
|
|
339
|
+
---
|
|
340
|
+
|
|
66
341
|
## Subpath Exports
|
|
67
342
|
|
|
68
343
|
```typescript
|
|
69
|
-
import { JobQueue }
|
|
70
|
-
import {
|
|
71
|
-
import {
|
|
72
|
-
import { Scheduler }
|
|
73
|
-
import { parseCron }
|
|
74
|
-
import { initializeSchema }
|
|
75
|
-
import { SlidingWindowRateLimiter }
|
|
344
|
+
import { JobQueue, WorkflowEngine } from '@bitclaw/jobs'
|
|
345
|
+
import { createOtelMiddleware } from '@bitclaw/jobs/otel'
|
|
346
|
+
import { WorkflowBuilder } from '@bitclaw/jobs/workflow'
|
|
347
|
+
import { Scheduler } from '@bitclaw/jobs/scheduler'
|
|
348
|
+
import { parseCron } from '@bitclaw/jobs/cron'
|
|
349
|
+
import { initializeSchema } from '@bitclaw/jobs/schema'
|
|
350
|
+
import { SlidingWindowRateLimiter } from '@bitclaw/jobs/rate-limiter'
|
|
351
|
+
import { JobWorker } from '@bitclaw/jobs/worker'
|
|
76
352
|
```
|
|
77
353
|
|
|
354
|
+
---
|
|
355
|
+
|
|
356
|
+
## Competitor Comparison
|
|
357
|
+
|
|
358
|
+
Analysis against every actively-maintained SQLite job queue as of 2026-06.
|
|
359
|
+
|
|
360
|
+
| Feature | **@bitclaw/jobs** | bunqueue | plainjob | workmatic | liteq (Go) | apalis-sqlite (Rust) |
|
|
361
|
+
|---|:---:|:---:|:---:|:---:|:---:|:---:|
|
|
362
|
+
| Typed generics | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
|
363
|
+
| Priority ordering | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
364
|
+
| Priority aging (starvation prevention) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
365
|
+
| Exponential / jitter / fibonacci backoff | ✅ | ✅ (exp only) | ✅ (exp only) | ❌ | ✅ (fixed) | ✅ (exp only) |
|
|
366
|
+
| `retryIf` predicate | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
367
|
+
| Dead-letter table | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ |
|
|
368
|
+
| Job dependencies (DAG) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
369
|
+
| Job graph API (recursive CTE) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
370
|
+
| Multi-process lease safety | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ |
|
|
371
|
+
| Lease renewal (`ctx.renewLease`) | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
372
|
+
| Dedup (ignore + replace) | ✅ | ✅ (ignore) | ❌ | ✅ (ignore) | ❌ | ❌ |
|
|
373
|
+
| Job TTL / `expireAt` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
374
|
+
| Result storage | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
|
375
|
+
| Typed events | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
376
|
+
| Middleware pipeline | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
377
|
+
| OpenTelemetry helper | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
378
|
+
| Pause / resume per worker | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
379
|
+
| Webhook on completion | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
380
|
+
| Batch processing | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ |
|
|
381
|
+
| Cron scheduler | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ |
|
|
382
|
+
| Admin HTTP handler | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ |
|
|
383
|
+
| Workflow engine | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
384
|
+
| Saga compensation | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
385
|
+
| Restart-safe reconcile | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
386
|
+
| Concurrency per worker | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
387
|
+
| Rate limiting per worker | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
388
|
+
| Zero runtime dependencies | ✅ | ✅ | ✅ | ❌ | n/a | n/a |
|
|
389
|
+
|
|
390
|
+
**Key differentiators:**
|
|
391
|
+
|
|
392
|
+
- **Only queue with a workflow engine.** `WorkflowEngine` composes real jobs into typed DAGs with saga compensation and restart-safe reconciliation. Competitors require external orchestrators (Temporal, Inngest) for this.
|
|
393
|
+
- **Only queue with middleware.** `queue.use(fn)` enables logging, tracing, and auth-checking without modifying job handlers. Every other queue buries these concerns inside worker configuration.
|
|
394
|
+
- **Only queue with lease renewal.** Long-running jobs (provisioning, ML inference) can extend their claim without crashing. Competitors force you to size `leaseMs` conservatively.
|
|
395
|
+
- **Only queue with priority aging.** High-throughput workloads can starve low-priority jobs indefinitely. Aging prevents it.
|
|
396
|
+
- **Only queue with `retryIf`.** Permanent errors (bad config, invalid input) should not consume retry budget. All others retry blindly.
|
|
397
|
+
- **Job graph API.** `getJobGraph(id)` returns the full dependency graph via recursive CTE — useful for admin UIs and debugging complex pipelines. No competitor exposes this.
|
|
398
|
+
- **OTel helper.** `createOtelMiddleware(tracer)` instruments all job executions with zero library coupling. No peer dependency — the tracer interface is structural.
|
|
399
|
+
|
|
400
|
+
---
|
|
401
|
+
|
|
78
402
|
## Testing
|
|
79
403
|
|
|
80
404
|
```bash
|
|
81
405
|
bun test
|
|
82
406
|
```
|
|
83
407
|
|
|
84
|
-
|
|
408
|
+
204 tests across 9 files.
|
package/dist/index.d.ts
CHANGED
|
@@ -2,6 +2,8 @@ export type { ParsedCron } from './cron';
|
|
|
2
2
|
export { cronMatches, nextCronOccurrence, parseCron } from './cron';
|
|
3
3
|
export type { JobQueueEventMap } from './events';
|
|
4
4
|
export { JobQueueEmitter } from './events';
|
|
5
|
+
export type { OtelSpan, OtelTracer } from './otel';
|
|
6
|
+
export { createOtelMiddleware } from './otel';
|
|
5
7
|
export { JobQueue } from './queue';
|
|
6
8
|
export { SlidingWindowRateLimiter } from './rate-limiter';
|
|
7
9
|
export { Scheduler } from './scheduler';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACpE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC1D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,SAAS,EACT,GAAG,EACH,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,SAAS,EACT,QAAQ,EACR,aAAa,EACb,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,iBAAiB,EAClB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACpE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC1D,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,SAAS,EACT,GAAG,EACH,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,SAAS,EACT,QAAQ,EACR,aAAa,EACb,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,iBAAiB,EAClB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// Barrel export for @bitclaw/jobs
|
|
3
3
|
export { cronMatches, nextCronOccurrence, parseCron } from './cron';
|
|
4
4
|
export { JobQueueEmitter } from './events';
|
|
5
|
+
export { createOtelMiddleware } from './otel';
|
|
5
6
|
export { JobQueue } from './queue';
|
|
6
7
|
export { SlidingWindowRateLimiter } from './rate-limiter';
|
|
7
8
|
export { Scheduler } from './scheduler';
|
package/dist/otel.d.ts
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { MiddlewareFn } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* Minimal interface matching @opentelemetry/api Tracer.
|
|
4
|
+
* Typed structurally so @bitclaw/jobs stays dependency-free —
|
|
5
|
+
* pass any OTel-compatible tracer without adding it as a peer dep.
|
|
6
|
+
*/
|
|
7
|
+
export type OtelTracer = {
|
|
8
|
+
startActiveSpan<T>(name: string, fn: (span: OtelSpan) => T): T;
|
|
9
|
+
};
|
|
10
|
+
export type OtelSpan = {
|
|
11
|
+
setAttribute(key: string, value: string | number | boolean): void;
|
|
12
|
+
setStatus(status: {
|
|
13
|
+
code: number;
|
|
14
|
+
message?: string;
|
|
15
|
+
}): void;
|
|
16
|
+
recordException(error: unknown): void;
|
|
17
|
+
end(): void;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Returns a middleware that wraps every job execution in an OTel span.
|
|
21
|
+
*
|
|
22
|
+
* Usage:
|
|
23
|
+
* ```ts
|
|
24
|
+
* import { trace } from '@opentelemetry/api'
|
|
25
|
+
* import { createOtelMiddleware } from '@bitclaw/jobs/otel'
|
|
26
|
+
*
|
|
27
|
+
* const tracer = trace.getTracer('my-app')
|
|
28
|
+
* queue.use(createOtelMiddleware(tracer))
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* Each span is named `job.<type>` and carries these attributes:
|
|
32
|
+
* - `job.id` — numeric job ID
|
|
33
|
+
* - `job.type` — job type string
|
|
34
|
+
* - `job.priority` — job priority
|
|
35
|
+
* - `job.retry` — current retry count
|
|
36
|
+
* - `job.error` — error message (only on failure)
|
|
37
|
+
*/
|
|
38
|
+
export declare function createOtelMiddleware(tracer: OtelTracer): MiddlewareFn;
|
|
39
|
+
//# sourceMappingURL=otel.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,eAAe,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,CAAC,GAAG,CAAC,CAAC;CAChE,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IAClE,SAAS,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC5D,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,GAAG,IAAI,IAAI,CAAC;CACb,CAAC;AAMF;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAqBrE"}
|
package/dist/otel.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** SpanStatusCode.OK = 1, SpanStatusCode.ERROR = 2 (OTel spec) */
|
|
2
|
+
const STATUS_OK = 1;
|
|
3
|
+
const STATUS_ERROR = 2;
|
|
4
|
+
/**
|
|
5
|
+
* Returns a middleware that wraps every job execution in an OTel span.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* ```ts
|
|
9
|
+
* import { trace } from '@opentelemetry/api'
|
|
10
|
+
* import { createOtelMiddleware } from '@bitclaw/jobs/otel'
|
|
11
|
+
*
|
|
12
|
+
* const tracer = trace.getTracer('my-app')
|
|
13
|
+
* queue.use(createOtelMiddleware(tracer))
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* Each span is named `job.<type>` and carries these attributes:
|
|
17
|
+
* - `job.id` — numeric job ID
|
|
18
|
+
* - `job.type` — job type string
|
|
19
|
+
* - `job.priority` — job priority
|
|
20
|
+
* - `job.retry` — current retry count
|
|
21
|
+
* - `job.error` — error message (only on failure)
|
|
22
|
+
*/
|
|
23
|
+
export function createOtelMiddleware(tracer) {
|
|
24
|
+
return (job, next) => tracer.startActiveSpan(`job.${job.type}`, async (span) => {
|
|
25
|
+
span.setAttribute('job.id', job.id);
|
|
26
|
+
span.setAttribute('job.type', job.type);
|
|
27
|
+
span.setAttribute('job.priority', job.priority);
|
|
28
|
+
span.setAttribute('job.retry', job.retryCount);
|
|
29
|
+
try {
|
|
30
|
+
const result = await next();
|
|
31
|
+
span.setStatus({ code: STATUS_OK });
|
|
32
|
+
return result;
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
36
|
+
span.setAttribute('job.error', message);
|
|
37
|
+
span.recordException(err);
|
|
38
|
+
span.setStatus({ code: STATUS_ERROR, message });
|
|
39
|
+
throw err;
|
|
40
|
+
}
|
|
41
|
+
finally {
|
|
42
|
+
span.end();
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bitclaw/jobs",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "SQLite-backed background job queue using bun:sqlite",
|
|
5
5
|
"files": [
|
|
6
6
|
"dist",
|
|
@@ -42,6 +42,10 @@
|
|
|
42
42
|
"./workflow": {
|
|
43
43
|
"types": "./dist/workflow.d.ts",
|
|
44
44
|
"default": "./dist/workflow.js"
|
|
45
|
+
},
|
|
46
|
+
"./otel": {
|
|
47
|
+
"types": "./dist/otel.d.ts",
|
|
48
|
+
"default": "./dist/otel.js"
|
|
45
49
|
}
|
|
46
50
|
},
|
|
47
51
|
"scripts": {
|