bunqueue 2.8.35 → 2.8.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +14 -2
  2. package/dist/application/backgroundTasks.js +28 -5
  3. package/dist/application/contextFactory.js +5 -0
  4. package/dist/application/dlqManager.d.ts +6 -0
  5. package/dist/application/dlqManager.js +35 -14
  6. package/dist/application/lockManager.js +22 -8
  7. package/dist/application/operations/customId.d.ts +31 -0
  8. package/dist/application/operations/customId.js +59 -0
  9. package/dist/application/operations/jobClaim.d.ts +17 -0
  10. package/dist/application/operations/jobClaim.js +20 -0
  11. package/dist/application/operations/jobManagement.d.ts +4 -5
  12. package/dist/application/operations/jobManagement.js +15 -116
  13. package/dist/application/operations/jobMoveOperations.d.ts +9 -0
  14. package/dist/application/operations/jobMoveOperations.js +94 -0
  15. package/dist/application/operations/jobStateTransitions.js +3 -0
  16. package/dist/application/operations/pull.js +5 -0
  17. package/dist/application/operations/push.js +8 -103
  18. package/dist/application/operations/pushInsert.d.ts +12 -0
  19. package/dist/application/operations/pushInsert.js +25 -0
  20. package/dist/application/operations/pushLocks.d.ts +14 -0
  21. package/dist/application/operations/pushLocks.js +43 -0
  22. package/dist/application/operations/queueControl.d.ts +1 -1
  23. package/dist/application/operations/queueControl.js +1 -1
  24. package/dist/application/queueManager.d.ts +1 -1
  25. package/dist/application/queueManager.js +17 -6
  26. package/dist/application/stallDetection.js +17 -8
  27. package/dist/client/queue/helpers.js +2 -0
  28. package/dist/client/queue/operations/counts.d.ts +1 -0
  29. package/dist/client/queue/operations/counts.js +3 -0
  30. package/dist/domain/queue/dependencyTracker.d.ts +5 -0
  31. package/dist/domain/queue/dependencyTracker.js +20 -0
  32. package/dist/domain/queue/shard.d.ts +1 -1
  33. package/dist/domain/queue/shard.js +6 -0
  34. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  35. package/dist/infrastructure/persistence/schema.js +25 -2
  36. package/dist/infrastructure/persistence/sqlite.d.ts +11 -0
  37. package/dist/infrastructure/persistence/sqlite.js +59 -5
  38. package/dist/infrastructure/persistence/sqliteBatch.js +8 -6
  39. package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -0
  40. package/dist/infrastructure/persistence/sqliteSerializer.js +9 -1
  41. package/dist/infrastructure/persistence/statements.d.ts +6 -1
  42. package/dist/infrastructure/persistence/statements.js +7 -5
  43. package/package.json +4 -1
@@ -4,10 +4,10 @@
4
4
  /** SQLite PRAGMA settings for optimal performance */
5
5
  export declare const PRAGMA_SETTINGS = "\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA cache_size = -64000;\nPRAGMA temp_store = MEMORY;\nPRAGMA mmap_size = 268435456;\nPRAGMA page_size = 4096;\nPRAGMA busy_timeout = 5000;\n";
6
6
  /** Main schema creation */
7
- export declare const SCHEMA = "\n-- Jobs table (using UUIDv7 for job IDs)\n-- Uses BLOB for data fields (MessagePack serialization for ~2-3x faster than JSON)\nCREATE TABLE IF NOT EXISTS jobs (\n id TEXT PRIMARY KEY,\n queue TEXT NOT NULL,\n data BLOB NOT NULL,\n priority INTEGER NOT NULL DEFAULT 0,\n created_at INTEGER NOT NULL,\n run_at INTEGER NOT NULL,\n started_at INTEGER,\n completed_at INTEGER,\n attempts INTEGER NOT NULL DEFAULT 0,\n max_attempts INTEGER NOT NULL DEFAULT 3,\n backoff INTEGER NOT NULL DEFAULT 1000,\n ttl INTEGER,\n timeout INTEGER,\n unique_key TEXT,\n custom_id TEXT,\n depends_on BLOB,\n parent_id TEXT,\n children_ids BLOB,\n tags BLOB,\n state TEXT NOT NULL DEFAULT 'waiting',\n lifo INTEGER NOT NULL DEFAULT 0,\n group_id TEXT,\n progress INTEGER DEFAULT 0,\n progress_msg TEXT,\n remove_on_complete INTEGER DEFAULT 0,\n remove_on_fail INTEGER DEFAULT 0,\n stall_timeout INTEGER,\n last_heartbeat INTEGER,\n timeline BLOB,\n stacktrace BLOB\n);\n\n-- Indexes for common queries\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state\n ON jobs(queue, state);\n-- Stable createdAt/id pagination for unfiltered and logical-state queue views\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_created\n ON jobs(queue, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state_created\n ON jobs(queue, state, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_run_at\n ON jobs(run_at) WHERE state IN ('waiting', 'delayed');\nCREATE INDEX IF NOT EXISTS idx_jobs_unique\n ON jobs(queue, unique_key) WHERE unique_key IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_custom_id\n ON jobs(custom_id) WHERE custom_id IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_parent\n ON jobs(parent_id) WHERE parent_id IS NOT NULL;\n\n-- Job results storage (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS job_results (\n job_id TEXT PRIMARY KEY,\n result BLOB,\n completed_at INTEGER NOT NULL\n);\n\n-- Dead letter queue (BLOB for MessagePack - stores full DlqEntry)\nCREATE TABLE IF NOT EXISTS dlq (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n job_id TEXT NOT NULL,\n queue TEXT NOT NULL,\n entry BLOB NOT NULL,\n entered_at INTEGER NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_dlq_queue ON dlq(queue);\nCREATE INDEX IF NOT EXISTS idx_dlq_job_id ON dlq(job_id);\nCREATE INDEX IF NOT EXISTS idx_dlq_entered_at ON dlq(entered_at);\n\n-- Performance indexes for high-throughput operations\n-- Stall detection: runs every 5s, needs fast lookup of active jobs by started_at\nCREATE INDEX IF NOT EXISTS idx_jobs_state_started\n ON jobs(state, started_at) WHERE state = 'active';\n\n-- Group operations: fast lookup by group_id\nCREATE INDEX IF NOT EXISTS idx_jobs_group_id\n ON jobs(group_id) WHERE group_id IS NOT NULL;\n\n-- Pending jobs: compound index for priority-ordered retrieval\nCREATE INDEX IF NOT EXISTS idx_jobs_pending_priority\n ON jobs(queue, state, priority DESC, run_at ASC) WHERE state IN ('waiting', 'delayed');\n\n-- Completed jobs: index for recovery ordering (issue #84)\nCREATE INDEX IF NOT EXISTS idx_jobs_completed_order\n ON jobs(completed_at DESC) WHERE state = 'completed';\n\n-- Cron jobs (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS cron_jobs (\n name TEXT PRIMARY KEY,\n queue TEXT NOT NULL,\n data BLOB NOT NULL,\n schedule TEXT,\n repeat_every INTEGER,\n priority INTEGER NOT NULL DEFAULT 0,\n next_run INTEGER NOT NULL,\n executions INTEGER NOT NULL DEFAULT 0,\n max_limit INTEGER,\n timezone TEXT,\n unique_key TEXT,\n dedup BLOB,\n skip_missed_on_restart INTEGER NOT NULL DEFAULT 0,\n skip_if_no_worker INTEGER NOT NULL DEFAULT 0,\n prevent_overlap INTEGER NOT NULL DEFAULT 1,\n job_options BLOB\n);\n\n-- Queue state persistence (optional)\nCREATE TABLE IF NOT EXISTS queue_state (\n name TEXT PRIMARY KEY,\n paused INTEGER NOT NULL DEFAULT 0,\n rate_limit INTEGER,\n concurrency_limit INTEGER,\n rate_limit_duration INTEGER,\n rate_limit_expires_at INTEGER\n);\n";
7
+ export declare const SCHEMA = "\n-- Jobs table (using UUIDv7 for job IDs)\n-- Uses BLOB for data fields (MessagePack serialization for ~2-3x faster than JSON)\nCREATE TABLE IF NOT EXISTS jobs (\n id TEXT PRIMARY KEY,\n queue TEXT NOT NULL,\n data BLOB NOT NULL,\n priority INTEGER NOT NULL DEFAULT 0,\n created_at INTEGER NOT NULL,\n run_at INTEGER NOT NULL,\n started_at INTEGER,\n completed_at INTEGER,\n attempts INTEGER NOT NULL DEFAULT 0,\n max_attempts INTEGER NOT NULL DEFAULT 3,\n backoff INTEGER NOT NULL DEFAULT 1000,\n ttl INTEGER,\n timeout INTEGER,\n unique_key TEXT,\n custom_id TEXT,\n depends_on BLOB,\n parent_id TEXT,\n children_ids BLOB,\n tags BLOB,\n state TEXT NOT NULL DEFAULT 'waiting',\n lifo INTEGER NOT NULL DEFAULT 0,\n group_id TEXT,\n progress INTEGER DEFAULT 0,\n progress_msg TEXT,\n remove_on_complete INTEGER DEFAULT 0,\n remove_on_fail INTEGER DEFAULT 0,\n stall_timeout INTEGER,\n last_heartbeat INTEGER,\n stall_count INTEGER NOT NULL DEFAULT 0,\n timeline BLOB,\n stacktrace BLOB\n);\n\n-- Indexes for common queries\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state\n ON jobs(queue, state);\n-- Stable createdAt/id pagination for unfiltered and logical-state queue views\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_created\n ON jobs(queue, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state_created\n ON jobs(queue, state, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_run_at\n ON jobs(run_at) WHERE state IN ('waiting', 'delayed');\nCREATE INDEX IF NOT EXISTS idx_jobs_unique\n ON jobs(queue, unique_key) WHERE unique_key IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_custom_id\n ON jobs(custom_id) WHERE custom_id IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_parent\n ON jobs(parent_id) WHERE parent_id IS NOT NULL;\n\n-- Job results storage (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS job_results (\n job_id TEXT PRIMARY KEY,\n result BLOB,\n completed_at INTEGER NOT NULL\n);\n\n-- Dead letter queue (BLOB for MessagePack - stores full DlqEntry)\nCREATE TABLE IF NOT EXISTS dlq (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n job_id TEXT NOT NULL,\n queue TEXT NOT NULL,\n entry BLOB NOT NULL,\n entered_at INTEGER NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_dlq_queue ON dlq(queue);\nCREATE INDEX IF NOT EXISTS idx_dlq_job_id ON dlq(job_id);\nCREATE INDEX IF NOT EXISTS idx_dlq_entered_at ON dlq(entered_at);\n\n-- Performance indexes for high-throughput operations\n-- Stall detection: runs every 5s, needs fast lookup of active jobs by started_at\nCREATE INDEX IF NOT EXISTS idx_jobs_state_started\n ON jobs(state, started_at) WHERE state = 'active';\n\n-- Group operations: fast lookup by group_id\nCREATE INDEX IF NOT EXISTS idx_jobs_group_id\n ON jobs(group_id) WHERE group_id IS NOT NULL;\n\n-- Pending jobs: compound index for priority-ordered retrieval\nCREATE INDEX IF NOT EXISTS idx_jobs_pending_priority\n ON jobs(queue, state, priority DESC, run_at ASC) WHERE state IN ('waiting', 'delayed');\n\n-- Completed jobs: index for recovery ordering (issue #84)\nCREATE INDEX IF NOT EXISTS idx_jobs_completed_order\n ON jobs(completed_at DESC) WHERE state = 'completed';\n\n-- Cron jobs (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS cron_jobs (\n name TEXT PRIMARY KEY,\n queue TEXT NOT NULL,\n data BLOB NOT NULL,\n schedule TEXT,\n repeat_every INTEGER,\n priority INTEGER NOT NULL DEFAULT 0,\n next_run INTEGER NOT NULL,\n executions INTEGER NOT NULL DEFAULT 0,\n max_limit INTEGER,\n timezone TEXT,\n unique_key TEXT,\n dedup BLOB,\n skip_missed_on_restart INTEGER NOT NULL DEFAULT 0,\n skip_if_no_worker INTEGER NOT NULL DEFAULT 0,\n prevent_overlap INTEGER NOT NULL DEFAULT 1,\n job_options BLOB\n);\n\n-- Queue state persistence (optional)\nCREATE TABLE IF NOT EXISTS queue_state (\n name TEXT PRIMARY KEY,\n paused INTEGER NOT NULL DEFAULT 0,\n rate_limit INTEGER,\n concurrency_limit INTEGER,\n rate_limit_duration INTEGER,\n rate_limit_expires_at INTEGER,\n stall_enabled INTEGER,\n stall_interval INTEGER,\n max_stalls INTEGER,\n stall_grace_period INTEGER\n);\n";
8
8
  /** Migration version table */
9
9
  export declare const MIGRATION_TABLE = "\nCREATE TABLE IF NOT EXISTS migrations (\n version INTEGER PRIMARY KEY,\n applied_at INTEGER NOT NULL\n);\n";
10
10
  /** Current schema version */
11
- export declare const SCHEMA_VERSION = 16;
11
+ export declare const SCHEMA_VERSION = 21;
12
12
  /** All migrations in order */
13
13
  export declare const MIGRATIONS: Record<number, string>;
@@ -44,6 +44,7 @@ CREATE TABLE IF NOT EXISTS jobs (
44
44
  remove_on_fail INTEGER DEFAULT 0,
45
45
  stall_timeout INTEGER,
46
46
  last_heartbeat INTEGER,
47
+ stall_count INTEGER NOT NULL DEFAULT 0,
47
48
  timeline BLOB,
48
49
  stacktrace BLOB
49
50
  );
@@ -129,7 +130,11 @@ CREATE TABLE IF NOT EXISTS queue_state (
129
130
  rate_limit INTEGER,
130
131
  concurrency_limit INTEGER,
131
132
  rate_limit_duration INTEGER,
132
- rate_limit_expires_at INTEGER
133
+ rate_limit_expires_at INTEGER,
134
+ stall_enabled INTEGER,
135
+ stall_interval INTEGER,
136
+ max_stalls INTEGER,
137
+ stall_grace_period INTEGER
133
138
  );
134
139
  `;
135
140
  /** Migration version table */
@@ -140,7 +145,7 @@ CREATE TABLE IF NOT EXISTS migrations (
140
145
  );
141
146
  `;
142
147
  /** Current schema version */
143
- export const SCHEMA_VERSION = 16;
148
+ export const SCHEMA_VERSION = 21;
144
149
  /** All migrations in order */
145
150
  export const MIGRATIONS = {
146
151
  1: SCHEMA,
@@ -212,5 +217,23 @@ ALTER TABLE queue_state ADD COLUMN rate_limit_duration INTEGER;
212
217
  `,
213
218
  16: `
214
219
  ALTER TABLE queue_state ADD COLUMN rate_limit_expires_at INTEGER;
220
+ `,
221
+ // Migration 17: preserve the cumulative stall bound across restarts.
222
+ 17: `
223
+ ALTER TABLE jobs ADD COLUMN stall_count INTEGER NOT NULL DEFAULT 0;
224
+ `,
225
+ // Migrations 18-21: persist the complete per-queue stall policy. One ALTER
226
+ // per version keeps interrupted upgrades retryable (same rule as 15-16).
227
+ 18: `
228
+ ALTER TABLE queue_state ADD COLUMN stall_enabled INTEGER;
229
+ `,
230
+ 19: `
231
+ ALTER TABLE queue_state ADD COLUMN stall_interval INTEGER;
232
+ `,
233
+ 20: `
234
+ ALTER TABLE queue_state ADD COLUMN max_stalls INTEGER;
235
+ `,
236
+ 21: `
237
+ ALTER TABLE queue_state ADD COLUMN stall_grace_period INTEGER;
215
238
  `,
216
239
  };
@@ -4,6 +4,7 @@
4
4
  * Uses MessagePack for ~2-3x faster serialization than JSON
5
5
  */
6
6
  import type { Job, JobId, JobTimelineEntry } from '../../domain/types/job';
7
+ import type { StallConfig } from '../../domain/types/stall';
7
8
  import type { CronJob } from '../../domain/types/cron';
8
9
  import { type DlqEntry } from '../../domain/types/dlq';
9
10
  /** Critical-loss callback: invoked when WriteBuffer drops jobs after exhausting retries. */
@@ -101,12 +102,20 @@ export declare class SqliteStorage {
101
102
  deleteDlqEntry(jobId: JobId): void;
102
103
  /** Clear all DLQ entries for a queue */
103
104
  clearDlqQueue(queue: string): void;
105
+ /**
106
+ * Permanently purge terminal DLQ jobs in one transaction.
107
+ * Pending buffered inserts are removed first so a purged job cannot be
108
+ * written back after the transaction commits.
109
+ */
110
+ purgeDlqEntries(queue: string, dlqJobIds: readonly JobId[], terminalJobIds: readonly JobId[], clearQueue: boolean): void;
104
111
  /** Load all DLQ entries */
105
112
  loadDlq(): Map<string, DlqEntry[]>;
106
113
  updateForRetry(job: Job): void;
107
114
  deleteJob(jobId: JobId): void;
108
115
  /** Update a job's data blob (e.g. after adding __parentId) */
109
116
  updateJobData(jobId: JobId, data: unknown): void;
117
+ /** Persist priority ordering, including its LIFO tie-break, across recovery. */
118
+ updateJobPriority(jobId: JobId, priority: number, lifo: boolean): void;
110
119
  /**
111
120
  * Persist a delay change (moveToDelayed / changeDelay) so the new `run_at`
112
121
  * survives a restart. Without this the delay lives only in the in-memory heap:
@@ -210,6 +219,7 @@ export declare class SqliteStorage {
210
219
  concurrencyLimit: number | null;
211
220
  rateLimitDuration?: number | null;
212
221
  rateLimitExpiresAt?: number | null;
222
+ stallConfig?: StallConfig | null;
213
223
  }): void;
214
224
  /** Load all persisted queue control-state rows (used by recover() on boot). */
215
225
  loadQueueState(): Array<{
@@ -219,6 +229,7 @@ export declare class SqliteStorage {
219
229
  concurrencyLimit: number | null;
220
230
  rateLimitDuration: number | null;
221
231
  rateLimitExpiresAt: number | null;
232
+ stallConfig: StallConfig | null;
222
233
  }>;
223
234
  /** Drop a queue's persisted control-state (e.g. on obliterate). */
224
235
  deleteQueueState(name: string): void;
@@ -7,7 +7,7 @@ import { Database } from 'bun:sqlite';
7
7
  import { createDlqEntry } from '../../domain/types/dlq';
8
8
  import { PRAGMA_SETTINGS, SCHEMA, MIGRATION_TABLE, SCHEMA_VERSION, MIGRATIONS } from './schema';
9
9
  import { prepareStatements, } from './statements';
10
- import { pack, unpack, rowToJob, reconstructDlqEntry } from './sqliteSerializer';
10
+ import { pack, persistedStallCount, reconstructDlqEntry, rowToJob, unpack, } from './sqliteSerializer';
11
11
  import { BatchInsertManager, WriteBuffer } from './sqliteBatch';
12
12
  import { storageLog } from '../../shared/logger';
13
13
  /** Check if an error is a SQLITE_FULL (disk full) error */
@@ -251,7 +251,7 @@ export class SqliteStorage {
251
251
  runInsertJobStmt(job) {
252
252
  this.statements
253
253
  .get('insertJob')
254
- .run(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, job.runAt > Date.now() ? 'delayed' : 'waiting', job.lifo ? 1 : 0, job.groupId, job.removeOnComplete ? 1 : 0, job.removeOnFail ? 1 : 0, job.stallTimeout, job.timeline.length > 0 ? pack(job.timeline) : null);
254
+ .run(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, job.runAt > Date.now() ? 'delayed' : 'waiting', job.lifo ? 1 : 0, job.groupId, job.removeOnComplete ? 1 : 0, job.removeOnFail ? 1 : 0, job.stallTimeout, persistedStallCount(job), job.timeline.length > 0 ? pack(job.timeline) : null);
255
255
  }
256
256
  /**
257
257
  * Ensure a job's buffered INSERT has been written to disk before issuing a
@@ -315,6 +315,35 @@ export class SqliteStorage {
315
315
  this.statements.get('clearDlqQueue').run(queue);
316
316
  });
317
317
  }
318
+ /**
319
+ * Permanently purge terminal DLQ jobs in one transaction.
320
+ * Pending buffered inserts are removed first so a purged job cannot be
321
+ * written back after the transaction commits.
322
+ */
323
+ purgeDlqEntries(queue, dlqJobIds, terminalJobIds, clearQueue) {
324
+ for (const jobId of terminalJobIds)
325
+ this.writeBuffer.removePending(jobId);
326
+ this.safeWrite(() => {
327
+ const deleteDlq = this.statements.get('deleteDlqEntryForQueue');
328
+ const clearDlq = this.statements.get('clearDlqQueue');
329
+ const deleteJob = this.statements.get('deleteJob');
330
+ const deleteResult = this.statements.get('deleteJobResult');
331
+ const tx = this.db.transaction(() => {
332
+ if (clearQueue) {
333
+ clearDlq.run(queue);
334
+ }
335
+ else {
336
+ for (const jobId of dlqJobIds)
337
+ deleteDlq.run(jobId, queue);
338
+ }
339
+ for (const jobId of terminalJobIds) {
340
+ deleteJob.run(jobId);
341
+ deleteResult.run(jobId);
342
+ }
343
+ });
344
+ tx();
345
+ });
346
+ }
318
347
  /** Load all DLQ entries */
319
348
  loadDlq() {
320
349
  const rows = this.statements.get('loadDlq').all();
@@ -336,8 +365,8 @@ export class SqliteStorage {
336
365
  updateForRetry(job) {
337
366
  this.safeWrite(() => {
338
367
  this.db
339
- .prepare('UPDATE jobs SET attempts = ?, run_at = ?, state = ?, timeline = ?, stacktrace = ? WHERE id = ?')
340
- .run(job.attempts, job.runAt, 'waiting', job.timeline.length > 0 ? pack(job.timeline) : null, job.stacktrace && job.stacktrace.length > 0 ? pack(job.stacktrace) : null, job.id);
368
+ .prepare('UPDATE jobs SET attempts = ?, stall_count = ?, run_at = ?, state = ?, timeline = ?, stacktrace = ? WHERE id = ?')
369
+ .run(job.attempts, persistedStallCount(job), job.runAt, 'waiting', job.timeline.length > 0 ? pack(job.timeline) : null, job.stacktrace && job.stacktrace.length > 0 ? pack(job.stacktrace) : null, job.id);
341
370
  });
342
371
  }
343
372
  deleteJob(jobId) {
@@ -360,10 +389,20 @@ export class SqliteStorage {
360
389
  }
361
390
  /** Update a job's data blob (e.g. after adding __parentId) */
362
391
  updateJobData(jobId, data) {
392
+ this.flushIfBuffered(jobId);
363
393
  this.safeWrite(() => {
364
394
  this.db.prepare('UPDATE jobs SET data = ? WHERE id = ?').run(pack(data), jobId);
365
395
  });
366
396
  }
397
+ /** Persist priority ordering, including its LIFO tie-break, across recovery. */
398
+ updateJobPriority(jobId, priority, lifo) {
399
+ this.flushIfBuffered(jobId);
400
+ this.safeWrite(() => {
401
+ this.db
402
+ .prepare('UPDATE jobs SET priority = ?, lifo = ? WHERE id = ?')
403
+ .run(priority, lifo ? 1 : 0, jobId);
404
+ });
405
+ }
367
406
  /**
368
407
  * Persist a delay change (moveToDelayed / changeDelay) so the new `run_at`
369
408
  * survives a restart. Without this the delay lives only in the in-memory heap:
@@ -654,7 +693,11 @@ export class SqliteStorage {
654
693
  this.safeWrite(() => {
655
694
  this.statements
656
695
  .get('upsertQueueState')
657
- .run(name, state.paused ? 1 : 0, state.rateLimit, state.concurrencyLimit, state.rateLimitDuration ?? null, state.rateLimitExpiresAt ?? null);
696
+ .run(name, state.paused ? 1 : 0, state.rateLimit, state.concurrencyLimit, state.rateLimitDuration ?? null, state.rateLimitExpiresAt ?? null, state.stallConfig === null || state.stallConfig === undefined
697
+ ? null
698
+ : state.stallConfig.enabled
699
+ ? 1
700
+ : 0, state.stallConfig?.stallInterval ?? null, state.stallConfig?.maxStalls ?? null, state.stallConfig?.gracePeriod ?? null);
658
701
  });
659
702
  }
660
703
  /** Load all persisted queue control-state rows (used by recover() on boot). */
@@ -667,6 +710,17 @@ export class SqliteStorage {
667
710
  concurrencyLimit: row.concurrency_limit,
668
711
  rateLimitDuration: row.rate_limit_duration ?? null,
669
712
  rateLimitExpiresAt: row.rate_limit_expires_at ?? null,
713
+ stallConfig: row.stall_enabled === null ||
714
+ row.stall_interval === null ||
715
+ row.max_stalls === null ||
716
+ row.stall_grace_period === null
717
+ ? null
718
+ : {
719
+ enabled: row.stall_enabled === 1,
720
+ stallInterval: row.stall_interval,
721
+ maxStalls: row.max_stalls,
722
+ gracePeriod: row.stall_grace_period,
723
+ },
670
724
  }));
671
725
  }
672
726
  /** Drop a queue's persisted control-state (e.g. on obliterate). */
@@ -2,8 +2,8 @@
2
2
  * SQLite Batch Operations
3
3
  * High-performance batch insert with prepared statement caching
4
4
  */
5
- import { pack } from './sqliteSerializer';
6
- const COLS_PER_ROW = 24;
5
+ import { pack, persistedStallCount } from './sqliteSerializer';
6
+ const COLS_PER_ROW = 25;
7
7
  // SQLite has a limit of ~999 variables, so batch in chunks
8
8
  const MAX_ROWS_PER_INSERT = Math.floor(999 / COLS_PER_ROW);
9
9
  /**
@@ -75,7 +75,7 @@ export class BatchInsertManager {
75
75
  getBatchInsertStmt(size) {
76
76
  let stmt = this.cache.get(size);
77
77
  if (!stmt) {
78
- const rowPlaceholder = '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
78
+ const rowPlaceholder = '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
79
79
  const placeholders = Array(size).fill(rowPlaceholder).join(', ');
80
80
  // ON CONFLICT(id) DO UPDATE (upsert): a colliding id overwrites in place instead
81
81
  // of failing the whole flush. Without it, one stale ORPHAN row (left when
@@ -86,7 +86,8 @@ export class BatchInsertManager {
86
86
  const sql = `INSERT INTO jobs (
87
87
  id, queue, data, priority, created_at, run_at, attempts, max_attempts,
88
88
  backoff, ttl, timeout, unique_key, custom_id, depends_on, parent_id,
89
- children_ids, tags, state, lifo, group_id, remove_on_complete, remove_on_fail, stall_timeout, timeline
89
+ children_ids, tags, state, lifo, group_id, remove_on_complete, remove_on_fail,
90
+ stall_timeout, stall_count, timeline
90
91
  ) VALUES ${placeholders}
91
92
  ON CONFLICT(id) DO UPDATE SET
92
93
  queue=excluded.queue, data=excluded.data, priority=excluded.priority,
@@ -96,7 +97,8 @@ export class BatchInsertManager {
96
97
  depends_on=excluded.depends_on, parent_id=excluded.parent_id, children_ids=excluded.children_ids,
97
98
  tags=excluded.tags, state=excluded.state, lifo=excluded.lifo, group_id=excluded.group_id,
98
99
  remove_on_complete=excluded.remove_on_complete, remove_on_fail=excluded.remove_on_fail,
99
- stall_timeout=excluded.stall_timeout, timeline=excluded.timeline,
100
+ stall_timeout=excluded.stall_timeout, stall_count=excluded.stall_count,
101
+ timeline=excluded.timeline,
100
102
  started_at=excluded.started_at, completed_at=excluded.completed_at,
101
103
  progress=excluded.progress, progress_msg=excluded.progress_msg,
102
104
  last_heartbeat=excluded.last_heartbeat, stacktrace=excluded.stacktrace`;
@@ -114,7 +116,7 @@ export class BatchInsertManager {
114
116
  // Flatten all values
115
117
  const values = [];
116
118
  for (const job of jobs) {
117
- values.push(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, job.runAt > now ? 'delayed' : 'waiting', job.lifo ? 1 : 0, job.groupId, job.removeOnComplete ? 1 : 0, job.removeOnFail ? 1 : 0, job.stallTimeout, job.timeline.length > 0 ? pack(job.timeline) : null);
119
+ values.push(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, job.runAt > now ? 'delayed' : 'waiting', job.lifo ? 1 : 0, job.groupId, job.removeOnComplete ? 1 : 0, job.removeOnFail ? 1 : 0, job.stallTimeout, persistedStallCount(job), job.timeline.length > 0 ? pack(job.timeline) : null);
118
120
  }
119
121
  stmt.run(...values);
120
122
  }
@@ -10,6 +10,12 @@ import type { DbJob } from './statements';
10
10
  export declare function pack(data: unknown): Uint8Array;
11
11
  /** Decode MessagePack buffer to data */
12
12
  export declare function unpack<T>(buffer: Uint8Array | null, fallback: T, context: string): T;
13
+ /**
14
+ * Normalize jobs constructed by pre-stallCount integrations at the persistence
15
+ * boundary. The database keeps its NOT NULL invariant while an omitted legacy
16
+ * field receives the same zero default as a freshly created Job.
17
+ */
18
+ export declare function persistedStallCount(job: Pick<Job, 'stallCount'>): number;
13
19
  /**
14
20
  * Symbol marker stamped on a Job whose `depends_on` blob failed to decode.
15
21
  *
@@ -22,6 +22,14 @@ export function unpack(buffer, fallback, context) {
22
22
  return fallback;
23
23
  }
24
24
  }
25
+ /**
26
+ * Normalize jobs constructed by pre-stallCount integrations at the persistence
27
+ * boundary. The database keeps its NOT NULL invariant while an omitted legacy
28
+ * field receives the same zero default as a freshly created Job.
29
+ */
30
+ export function persistedStallCount(job) {
31
+ return job.stallCount ?? 0;
32
+ }
25
33
  /**
26
34
  * Symbol marker stamped on a Job whose `depends_on` blob failed to decode.
27
35
  *
@@ -104,7 +112,7 @@ export function rowToJob(row) {
104
112
  repeat: null,
105
113
  lastHeartbeat: row.last_heartbeat ?? row.created_at,
106
114
  stallTimeout: row.stall_timeout,
107
- stallCount: 0,
115
+ stallCount: row.stall_count ?? 0,
108
116
  // BullMQ v5 additional fields (not persisted in DB, use defaults)
109
117
  stackTraceLimit: 10,
110
118
  keepLogs: null,
@@ -4,7 +4,7 @@
4
4
  */
5
5
  import type { Database } from 'bun:sqlite';
6
6
  /** Statement names */
7
- export type StatementName = 'insertJob' | 'updateJobState' | 'completeJob' | 'deleteJob' | 'deleteJobResult' | 'getJob' | 'insertResult' | 'getResult' | 'insertDlq' | 'loadDlq' | 'deleteDlqEntry' | 'clearDlqQueue' | 'insertCron' | 'updateCron' | 'upsertQueueState' | 'loadQueueState' | 'deleteQueueState';
7
+ export type StatementName = 'insertJob' | 'updateJobState' | 'completeJob' | 'deleteJob' | 'deleteJobResult' | 'getJob' | 'insertResult' | 'getResult' | 'insertDlq' | 'loadDlq' | 'deleteDlqEntry' | 'deleteDlqEntryForQueue' | 'clearDlqQueue' | 'insertCron' | 'updateCron' | 'upsertQueueState' | 'loadQueueState' | 'deleteQueueState';
8
8
  /** SQL statements */
9
9
  export declare const SQL_STATEMENTS: Record<StatementName, string>;
10
10
  /** Prepare all statements */
@@ -39,6 +39,7 @@ export interface DbJob {
39
39
  remove_on_fail: number;
40
40
  stall_timeout: number | null;
41
41
  last_heartbeat: number | null;
42
+ stall_count: number;
42
43
  timeline: Uint8Array | null;
43
44
  stacktrace: Uint8Array | null;
44
45
  }
@@ -69,4 +70,8 @@ export interface DbQueueState {
69
70
  concurrency_limit: number | null;
70
71
  rate_limit_duration: number | null;
71
72
  rate_limit_expires_at: number | null;
73
+ stall_enabled: number | null;
74
+ stall_interval: number | null;
75
+ max_stalls: number | null;
76
+ stall_grace_period: number | null;
72
77
  }
@@ -15,12 +15,12 @@ export const SQL_STATEMENTS = {
15
15
  id, queue, data, priority, created_at, run_at, attempts,
16
16
  max_attempts, backoff, ttl, timeout, unique_key, custom_id,
17
17
  depends_on, parent_id, children_ids, tags, state, lifo, group_id,
18
- remove_on_complete, remove_on_fail, stall_timeout, timeline
18
+ remove_on_complete, remove_on_fail, stall_timeout, stall_count, timeline
19
19
  ) VALUES (
20
20
  ?, ?, ?, ?, ?, ?, ?,
21
21
  ?, ?, ?, ?, ?, ?,
22
22
  ?, ?, ?, ?, ?, ?, ?,
23
- ?, ?, ?, ?
23
+ ?, ?, ?, ?, ?
24
24
  )
25
25
  ON CONFLICT(id) DO UPDATE SET
26
26
  queue=excluded.queue, data=excluded.data, priority=excluded.priority,
@@ -30,7 +30,8 @@ export const SQL_STATEMENTS = {
30
30
  depends_on=excluded.depends_on, parent_id=excluded.parent_id, children_ids=excluded.children_ids,
31
31
  tags=excluded.tags, state=excluded.state, lifo=excluded.lifo, group_id=excluded.group_id,
32
32
  remove_on_complete=excluded.remove_on_complete, remove_on_fail=excluded.remove_on_fail,
33
- stall_timeout=excluded.stall_timeout, timeline=excluded.timeline,
33
+ stall_timeout=excluded.stall_timeout, stall_count=excluded.stall_count,
34
+ timeline=excluded.timeline,
34
35
  -- Per-execution columns are NOT in the INSERT column list, so excluded.<col>
35
36
  -- resolves to each column's DEFAULT (NULL / 0) — i.e. the fresh-job value. Reset
36
37
  -- them too, otherwise an upsert over an ORPHAN row would leave a brand-new job
@@ -49,6 +50,7 @@ export const SQL_STATEMENTS = {
49
50
  insertDlq: 'INSERT INTO dlq (job_id, queue, entry, entered_at) VALUES (?, ?, ?, ?)',
50
51
  loadDlq: 'SELECT * FROM dlq ORDER BY entered_at',
51
52
  deleteDlqEntry: 'DELETE FROM dlq WHERE job_id = ?',
53
+ deleteDlqEntryForQueue: 'DELETE FROM dlq WHERE job_id = ? AND queue = ?',
52
54
  clearDlqQueue: 'DELETE FROM dlq WHERE queue = ?',
53
55
  insertCron: `
54
56
  INSERT OR REPLACE INTO cron_jobs
@@ -57,8 +59,8 @@ export const SQL_STATEMENTS = {
57
59
  `,
58
60
  updateCron: 'UPDATE cron_jobs SET executions = ?, next_run = ? WHERE name = ?',
59
61
  // Queue control-state persistence (#100): paused / rate-limit / concurrency.
60
- upsertQueueState: 'INSERT OR REPLACE INTO queue_state (name, paused, rate_limit, concurrency_limit, rate_limit_duration, rate_limit_expires_at) VALUES (?, ?, ?, ?, ?, ?)',
61
- loadQueueState: 'SELECT name, paused, rate_limit, concurrency_limit, rate_limit_duration, rate_limit_expires_at FROM queue_state',
62
+ upsertQueueState: 'INSERT OR REPLACE INTO queue_state (name, paused, rate_limit, concurrency_limit, rate_limit_duration, rate_limit_expires_at, stall_enabled, stall_interval, max_stalls, stall_grace_period) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
63
+ loadQueueState: 'SELECT name, paused, rate_limit, concurrency_limit, rate_limit_duration, rate_limit_expires_at, stall_enabled, stall_interval, max_stalls, stall_grace_period FROM queue_state',
62
64
  deleteQueueState: 'DELETE FROM queue_state WHERE name = ?',
63
65
  };
64
66
  /** Prepare all statements */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bunqueue",
3
- "version": "2.8.35",
3
+ "version": "2.8.36",
4
4
  "description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external infrastructure.",
5
5
  "type": "module",
6
6
  "main": "dist/main.js",
@@ -57,7 +57,9 @@
57
57
  "build": "bun build --compile --minify src/main.ts --outfile dist/bunqueue",
58
58
  "build:lib": "tsc -p tsconfig.build.json",
59
59
  "test": "BUNQUEUE_EMBEDDED=1 bun test",
60
+ "test:model": "BUNQUEUE_EMBEDDED=1 bun test test/model-based/queue-model.test.ts",
60
61
  "test:sandbox": "bun run scripts/test-sandbox.ts",
62
+ "test:sandbox:sdk": "bun run scripts/test-sdk-sandbox.ts",
61
63
  "bench": "bun run bench/throughput.ts && bun run bench/latency.ts",
62
64
  "bench:throughput": "bun run bench/throughput.ts",
63
65
  "bench:latency": "bun run bench/latency.ts",
@@ -82,6 +84,7 @@
82
84
  "@types/bun": "^1.3.9",
83
85
  "bullmq": "^5.79.3",
84
86
  "elysia": "^1.4.25",
87
+ "fast-check": "^4.9.0",
85
88
  "ioredis": "^5.11.1",
86
89
  "typescript": "^5.9.3",
87
90
  "zod": "^4.3.6"