bunqueue 2.8.4 → 2.8.5

package/dist/application/contextFactory.js

@@ -80,6 +80,8 @@ export class ContextFactory {
             shards: this.deps.shards,
             shardLocks: this.deps.shardLocks,
             completedJobs: this.deps.completedJobs,
+            completedJobsData: this.deps.completedJobsData,
+            jobResults: this.deps.jobResults,
             customIdMap: this.deps.customIdMap,
             jobIndex: this.deps.jobIndex,
             totalPushed: this.deps.metrics.totalPushed,

package/dist/application/operations/push.d.ts

@@ -14,6 +14,8 @@ export interface PushContext {
     shards: Shard[];
     shardLocks: RWLock[];
     completedJobs: SetLike<JobId>;
+    completedJobsData: MapLike<JobId, Job>;
+    jobResults: MapLike<JobId, unknown>;
     customIdMap: MapLike<string, JobId>;
     jobIndex: Map<JobId, JobLocation>;
     totalPushed: {

package/dist/application/operations/push.js

@@ -17,19 +17,29 @@ function handleCustomId(input, shard, ctx) {
     }
     const id = jobId(input.customId);
     const existing = ctx.customIdMap.get(input.customId);
-    // No existing mapping - register and proceed
-    if (!existing) {
-        ctx.customIdMap.set(input.customId, id);
-        return { skip: false, id };
+    // If the existing job is still queued, the add is idempotent — return it.
+    if (existing) {
+        const location = ctx.jobIndex.get(existing);
+        const existingJob = location?.type === 'queue' ? shard.getQueue(location.queueName).find(existing) : null;
+        if (existingJob) {
+            return { skip: true, existingJob };
+        }
     }
-    // Check if existing job is still in queue
-    const location = ctx.jobIndex.get(existing);
-    const existingJob = location?.type === 'queue' ? shard.getQueue(location.queueName).find(existing) : null;
-    if (existingJob) {
-        return { skip: true, existingJob };
+    // Reuse path: the id is free in the queue (no mapping, or the prior job is
+    // processing/completed). If the prior job COMPLETED, its row survives on disk
+    // (markCompleted does an UPDATE, not a DELETE) and it is still in completedJobs.
+    // Reusing the same deterministic id would then (a) make getJobState return
+    // 'completed' for the brand-new job and (b) collide on the `jobs.id` PRIMARY KEY
+    // at flush time. Evict the stale completed job so the reused id starts fresh as
+    // 'waiting' (#92). Checked regardless of customIdMap state — the mapping may have
+    // been cleared on completion, which would otherwise skip this path entirely.
+    if (ctx.completedJobs.has(id)) {
+        ctx.completedJobs.delete(id);
+        ctx.completedJobsData.delete(id);
+        ctx.jobResults.delete(id);
+        ctx.jobIndex.delete(id);
+        ctx.storage?.deleteJob(id); // removes the surviving row + result + any buffered insert
     }
-    // Job gone (processing/completed) - allow reuse of customId
-    ctx.customIdMap.delete(input.customId);
     ctx.customIdMap.set(input.customId, id);
     return { skip: false, id };
 }

package/dist/infrastructure/persistence/sqlite.js

@@ -56,11 +56,24 @@ export class SqliteStorage {
             if (isSqliteFullError(err)) {
                 this.setDiskFull(err.message);
             }
-            storageLog.error('Write buffer flush failed', {
-                jobCount,
-                error: err.message,
-                diskFull: this._diskFull,
-            });
+            // A constraint violation (e.g. duplicate jobs.id) is a PERMANENT per-row
+            // rejection that the WriteBuffer isolated and dropped — sibling valid jobs
+            // in the same flush were still persisted. Log it distinctly from a
+            // transient flush failure (and never route it to the DLQ, which would
+            // resurrect a duplicate).
+            if (/constraint failed/i.test(err.message)) {
+                storageLog.error('Write buffer rejected jobs (constraint violation, dropped)', {
+                    rejectedJobCount: jobCount,
+                    error: err.message,
+                });
+            }
+            else {
+                storageLog.error('Write buffer flush failed', {
+                    jobCount,
+                    error: err.message,
+                    diskFull: this._diskFull,
+                });
+            }
         }, (jobs, lastError, attempts) => {
             this.handleCriticalLoss(jobs, lastError, attempts);
         });

package/dist/infrastructure/persistence/sqliteBatch.d.ts

@@ -4,13 +4,33 @@
  */
 import type { Database } from 'bun:sqlite';
 import type { Job } from '../../domain/types/job';
+/** Outcome of a batch insert after isolating per-row failures. */
+export interface BatchInsertResult {
+    /** Jobs that hit a transient (non-constraint) error — caller should retry. */
+    transient: Job[];
+    /** Jobs rejected by a permanent constraint (e.g. duplicate id) — drop, never retry. */
+    conflicts: Job[];
+    /** The originating error from the fast-path failure, for logging. */
+    error?: Error;
+}
 /** Batch insert manager with prepared statement caching */
 export declare class BatchInsertManager {
     private readonly db;
     private readonly cache;
     constructor(db: Database);
-    /** Insert batch of jobs using multi-row INSERT for 50-100x speedup */
-    insertJobsBatch(jobs: Job[]): void;
+    /**
+     * Insert a batch of jobs using a single multi-row INSERT (50-100x speedup).
+     * On the common path every row succeeds and an empty result is returned.
+     *
+     * If the atomic batch fails (e.g. a single duplicate `jobs.id`), the rows are
+     * re-inserted ONE AT A TIME so a single bad row can no longer drop the rest:
+     * valid jobs persist, constraint violations are isolated as `conflicts` (drop,
+     * never retry — they would poison every future flush), and any transient
+     * failures are returned so the caller can retry just those. Never throws.
+     */
+    insertJobsBatch(jobs: Job[]): BatchInsertResult;
+    /** Fallback path: insert each job independently, isolating per-row failures. */
+    private insertRowByRow;
     /** Get or create cached prepared statement for batch insert */
     private getBatchInsertStmt;
     /** Insert a chunk of jobs with single multi-row INSERT */
@@ -52,7 +72,16 @@ export declare class WriteBuffer {
     add(job: Job): void;
     /** Add multiple jobs to buffer */
     addBatch(jobs: Job[]): void;
-    /** Flush buffer to disk using double-buffering. Returns number of jobs flushed. */
+    /**
+     * Flush buffer to disk using double-buffering. Returns number of jobs persisted.
+     *
+     * Per-row isolation (see BatchInsertManager.insertJobsBatch): valid jobs are
+     * persisted even if a sibling row violates a constraint. Constraint conflicts
+     * (e.g. duplicate id) are dropped+reported and NEVER re-buffered — re-buffering
+     * them would poison every future flush and silently drop unrelated valid jobs
+     * (the #92-class data-loss bug). Transient failures are re-buffered and retried
+     * with exponential backoff, exactly as before.
+     */
     flush(): number;
     /** Schedule a retry with exponential backoff */
     private scheduleBackoffRetry;

package/dist/infrastructure/persistence/sqliteBatch.js

@@ -3,6 +3,20 @@
  * High-performance batch insert with prepared statement caching
  */
 import { pack } from './sqliteSerializer';
+const COLS_PER_ROW = 24;
+// SQLite has a limit of ~999 variables, so batch in chunks
+const MAX_ROWS_PER_INSERT = Math.floor(999 / COLS_PER_ROW);
+/**
+ * A constraint violation (e.g. a duplicate `jobs.id` PRIMARY KEY) is PERMANENT
+ * for that row — retrying never succeeds. It must be isolated from the rest of
+ * the batch, otherwise one bad row poisons the whole atomic flush and drops
+ * unrelated valid jobs (data loss, #92-class). Everything else (disk I/O, busy,
+ * full) is treated as transient and retried.
+ */
+function isConstraintError(err) {
+    const code = err.code ?? '';
+    return code.startsWith('SQLITE_CONSTRAINT') || /constraint failed/i.test(err.message);
+}
 /** Batch insert manager with prepared statement caching */
 export class BatchInsertManager {
     db;
@@ -10,20 +24,52 @@ export class BatchInsertManager {
     constructor(db) {
         this.db = db;
     }
-    /** Insert batch of jobs using multi-row INSERT for 50-100x speedup */
+    /**
+     * Insert a batch of jobs using a single multi-row INSERT (50-100x speedup).
+     * On the common path every row succeeds and an empty result is returned.
+     *
+     * If the atomic batch fails (e.g. a single duplicate `jobs.id`), the rows are
+     * re-inserted ONE AT A TIME so a single bad row can no longer drop the rest:
+     * valid jobs persist, constraint violations are isolated as `conflicts` (drop,
+     * never retry — they would poison every future flush), and any transient
+     * failures are returned so the caller can retry just those. Never throws.
+     */
     insertJobsBatch(jobs) {
         if (jobs.length === 0)
-            return;
+            return { transient: [], conflicts: [] };
         const now = Date.now();
-        const COLS_PER_ROW = 24;
-        // SQLite has a limit of ~999 variables, so batch in chunks
-        const MAX_ROWS_PER_INSERT = Math.floor(999 / COLS_PER_ROW);
-        this.db.transaction(() => {
-            for (let offset = 0; offset < jobs.length; offset += MAX_ROWS_PER_INSERT) {
-                const chunk = jobs.slice(offset, offset + MAX_ROWS_PER_INSERT);
-                this.insertJobsChunk(chunk, now);
+        try {
+            this.db.transaction(() => {
+                for (let offset = 0; offset < jobs.length; offset += MAX_ROWS_PER_INSERT) {
+                    const chunk = jobs.slice(offset, offset + MAX_ROWS_PER_INSERT);
+                    this.insertJobsChunk(chunk, now);
+                }
+            })();
+            return { transient: [], conflicts: [] };
+        }
+        catch (err) {
+            const batchError = err instanceof Error ? err : new Error(String(err));
+            return this.insertRowByRow(jobs, now, batchError);
+        }
+    }
+    /** Fallback path: insert each job independently, isolating per-row failures. */
+    insertRowByRow(jobs, now, batchError) {
+        const transient = [];
+        const conflicts = [];
+        for (const job of jobs) {
+            try {
+                // Single-row INSERT, auto-committed: succeeds/fails independently.
+                this.insertJobsChunk([job], now);
+            }
+            catch (e) {
+                const err = e instanceof Error ? e : new Error(String(e));
+                if (isConstraintError(err))
+                    conflicts.push(job);
+                else
+                    transient.push(job);
             }
-        })();
+        }
+        return { transient, conflicts, error: batchError };
     }
     /** Get or create cached prepared statement for batch insert */
     getBatchInsertStmt(size) {
@@ -112,7 +158,16 @@ export class WriteBuffer {
             this.flush();
         }
     }
-    /** Flush buffer to disk using double-buffering. Returns number of jobs flushed. */
+    /**
+     * Flush buffer to disk using double-buffering. Returns number of jobs persisted.
+     *
+     * Per-row isolation (see BatchInsertManager.insertJobsBatch): valid jobs are
+     * persisted even if a sibling row violates a constraint. Constraint conflicts
+     * (e.g. duplicate id) are dropped+reported and NEVER re-buffered — re-buffering
+     * them would poison every future flush and silently drop unrelated valid jobs
+     * (the #92-class data-loss bug). Transient failures are re-buffered and retried
+     * with exponential backoff, exactly as before.
+     */
     flush() {
         // Prevent flush after stop or concurrent flushes
         if (this.stopped || this.flushing)
@@ -125,53 +180,68 @@ export class WriteBuffer {
         this.activeBuffer = [];
         const jobCount = this.flushBuffer.length;
         try {
-            this.batchManager.insertJobsBatch(this.flushBuffer);
-            this.flushBuffer = []; // Clear after successful write
-            // Reset retry state on success
-            this.retryCount = 0;
-            this.currentBackoffMs = this.initialBackoffMs;
-            this.lastError = null;
-            return jobCount;
-        }
-        catch (err) {
-            const error = err instanceof Error ? err : new Error(String(err));
-            this.lastError = error;
-            this.retryCount++;
-            // On failure, prepend failed jobs back to active buffer
-            // This preserves order: failed jobs first, then new jobs
-            this.activeBuffer = this.flushBuffer.concat(this.activeBuffer);
+            // BatchInsertManager.insertJobsBatch isolates per-row failures and never
+            // throws. Stay defensive: a manager that throws (or returns nothing) is
+            // treated as a transient failure of the whole batch, preserving the
+            // re-buffer/retry/critical-loss semantics.
+            let result;
+            try {
+                // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- defensive against a non-conforming batch manager (e.g. a test double returning undefined)
+                result = this.batchManager.insertJobsBatch(this.flushBuffer) ?? {
+                    transient: [],
+                    conflicts: [],
+                };
+            }
+            catch (err) {
+                result = {
+                    transient: this.flushBuffer,
+                    conflicts: [],
+                    error: err instanceof Error ? err : new Error(String(err)),
+                };
+            }
+            const { transient, conflicts, error } = result;
             this.flushBuffer = [];
-            // Check if we've exceeded max retries
-            if (this.retryCount >= this.maxRetries) {
-                // Move jobs to dead letter / emit critical error
-                const lostJobs = [...this.activeBuffer];
-                this.activeBuffer = [];
-                if (this.onCriticalError) {
-                    this.onCriticalError(lostJobs, error, this.retryCount);
+            // Permanent constraint conflicts: drop + report once, never retry.
+            if (conflicts.length > 0) {
+                this.onError(error ?? new Error('Constraint violation'), conflicts.length);
+            }
+            // Transient failures: re-buffer just those and retry with backoff.
+            if (transient.length > 0) {
+                const error2 = error ?? new Error('Write buffer flush failed');
+                this.lastError = error2;
+                this.retryCount++;
+                // Prepend failed jobs back to active buffer (failed first, then new).
+                this.activeBuffer = transient.concat(this.activeBuffer);
+                if (this.retryCount >= this.maxRetries) {
+                    const lostJobs = [...this.activeBuffer];
+                    this.activeBuffer = [];
+                    if (this.onCriticalError)
+                        this.onCriticalError(lostJobs, error2, this.retryCount);
+                    this.onError(error2, lostJobs.length, {
+                        retryCount: this.retryCount,
+                        nextBackoffMs: 0,
+                        maxRetries: this.maxRetries,
+                    });
+                    this.retryCount = 0;
+                    this.currentBackoffMs = this.initialBackoffMs;
+                    this.lastError = null;
+                }
+                else {
+                    const nextBackoffMs = Math.min(this.currentBackoffMs * 2, this.maxBackoffMs);
+                    this.onError(error2, transient.length, {
+                        retryCount: this.retryCount,
+                        nextBackoffMs,
+                        maxRetries: this.maxRetries,
+                    });
+                    this.scheduleBackoffRetry();
                 }
-                // Also call onError with retry info for logging
-                this.onError(error, lostJobs.length, {
-                    retryCount: this.retryCount,
-                    nextBackoffMs: 0, // No more retries
-                    maxRetries: this.maxRetries,
-                });
-                // Reset retry state
-                this.retryCount = 0;
-                this.currentBackoffMs = this.initialBackoffMs;
-                this.lastError = null;
-                throw err;
+                return jobCount - transient.length - conflicts.length;
             }
-            // Calculate next backoff with exponential increase
-            const nextBackoffMs = Math.min(this.currentBackoffMs * 2, this.maxBackoffMs);
-            // Call error callback with retry information
-            this.onError(error, jobCount, {
-                retryCount: this.retryCount,
-                nextBackoffMs: nextBackoffMs,
-                maxRetries: this.maxRetries,
-            });
-            // Schedule backoff retry
-            this.scheduleBackoffRetry();
-            throw err;
+            // Success (or success-modulo-dropped-conflicts): reset retry state.
+            this.retryCount = 0;
+            this.currentBackoffMs = this.initialBackoffMs;
+            this.lastError = null;
+            return jobCount - conflicts.length;
         }
         finally {
             this.flushing = false;
@@ -287,6 +357,10 @@ export class WriteBuffer {
                 const flushed = this.flush();
                 clearTimeout(timeout);
                 this.stopped = true;
+                // flush() no longer throws on transient failure (it re-buffers); report
+                // anything that could not be persisted so it isn't silently lost.
+                if (this.pendingCount > 0)
+                    this.reportLostJobs();
                 resolve(flushed);
             }
             catch {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bunqueue",
-  "version": "2.8.4",
+  "version": "2.8.5",
   "description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external dependencies.",
   "type": "module",
   "main": "dist/main.js",