bunqueue 2.8.31 → 2.8.33

package/dist/domain/queue/temporalManager.js

@@ -5,29 +5,15 @@
  * - Temporal index (SkipList) for efficient cleanQueue range queries
  * - Delayed heap (MinHeap) for O(k) delayed job refresh
  */
-import { SkipList } from '../../shared/skipList';
 import { MinHeap } from '../../shared/minHeap';
+import { TemporalIndex } from './temporalIndex';
+const DELAYED_COMPACTION_MIN_STALE = 256;
 /**
  * Manages temporal indexing for efficient job cleanup
  * and delayed job tracking for O(k) refresh
  */
 export class TemporalManager {
-    /**
-     * Temporal index: Skip List for O(log n) insert/delete
-     * Ordered by createdAt for efficient cleanQueue range queries
-     * Uses equality check on jobId to prevent duplicate entries for the same job
-     */
-    // Total-order comparator: createdAt first, then jobId as a tie-break. Without
-    // the tie-break, a batch of jobs sharing one createdAt (the addBulk case —
-    // `now` is captured once) makes every node compare-equal, which (a) turns
-    // SkipList.insert's duplicate-check scan into O(n) per insert => O(n²) for the
-    // batch, and (b) makes SkipList.delete remove the WRONG same-createdAt node
-    // (it stops at the first compare-equal node). A total order fixes both: the
-    // dedup scan and delete both resolve to the exact (createdAt, jobId) node in
-    // O(log n). jobId is a string (UUIDv7 by default, or any custom id), so its
-    // lexicographic comparison is a valid total order in every case. Equality is
-    // still by jobId (now reached only for a true duplicate).
-    temporalIndex = new SkipList((a, b) => a.createdAt - b.createdAt || (a.jobId < b.jobId ? -1 : a.jobId > b.jobId ? 1 : 0), 16, 0.5, (a, b) => a.jobId === b.jobId);
+    temporalIndex = new TemporalIndex();
     /** Set of delayed job IDs for tracking when they become ready */
     delayedJobIds = new Set();
     /**
@@ -40,7 +26,7 @@ export class TemporalManager {
     // ============ Temporal Index Operations ============
     /** Add job to temporal index - O(log n) */
     addToIndex(createdAt, jobId, queue) {
-        this.temporalIndex.insert({ createdAt, jobId, queue });
+        this.temporalIndex.add(createdAt, jobId, queue);
     }
     /**
      * Get old jobs from temporal index - O(log n + k)
@@ -49,36 +35,22 @@ export class TemporalManager {
     getOldJobs(queue, thresholdMs, limit) {
         const now = Date.now();
         const threshold = now - thresholdMs;
-        const result = [];
-        for (const entry of this.temporalIndex.values()) {
-            if (entry.createdAt > threshold)
-                break;
-            if (entry.queue === queue) {
-                result.push({ jobId: entry.jobId, createdAt: entry.createdAt });
-                if (result.length >= limit)
-                    break;
-            }
-        }
-        return result;
+        return this.temporalIndex.getOldJobs(queue, threshold, limit);
     }
     /** Remove job from temporal index */
     removeFromIndex(jobId) {
-        this.temporalIndex.deleteWhere((e) => e.jobId === jobId);
+        this.temporalIndex.remove(jobId);
     }
     /** Clear temporal index for a queue */
     clearIndexForQueue(queue) {
-        this.temporalIndex.removeAll((e) => e.queue === queue);
+        this.temporalIndex.clearQueue(queue);
     }
     /**
      * Clean orphaned temporal index entries.
      * Removes entries for jobs that no longer exist.
      */
     cleanOrphaned(validJobIds) {
-        if (this.temporalIndex.size === 0)
-            return 0;
-        const beforeSize = this.temporalIndex.size;
-        this.temporalIndex.removeAll((e) => !validJobIds.has(e.jobId));
-        return beforeSize - this.temporalIndex.size;
+        return this.temporalIndex.cleanOrphaned(validJobIds);
     }
     /** Get temporal index size */
     get indexSize() {
@@ -94,15 +66,15 @@ export class TemporalManager {
         this.delayedJobIds.add(jobId);
         this.delayedHeap.push({ jobId, runAt });
         this.delayedRunAt.set(jobId, runAt);
+        this.maybeCompactDelayedHeap();
     }
     /** Remove a delayed job (lazy removal from heap) */
     removeDelayed(jobId) {
-        if (this.delayedJobIds.has(jobId)) {
-            this.delayedJobIds.delete(jobId);
-            this.delayedRunAt.delete(jobId);
-            return true;
-        }
-        return false;
+        if (!this.delayedJobIds.delete(jobId))
+            return false;
+        this.delayedRunAt.delete(jobId);
+        this.maybeCompactDelayedHeap();
+        return true;
     }
     /**
      * Refresh delayed jobs that have become ready
@@ -127,8 +99,20 @@ export class TemporalManager {
             this.delayedRunAt.delete(top.jobId);
             count++;
         }
+        this.maybeCompactDelayedHeap();
         return count;
     }
+    maybeCompactDelayedHeap() {
+        const liveCount = this.delayedRunAt.size;
+        if (liveCount === 0) {
+            this.delayedHeap.clear();
+            return;
+        }
+        const staleCount = this.delayedHeap.size - liveCount;
+        if (staleCount < DELAYED_COMPACTION_MIN_STALE || staleCount < liveCount)
+            return;
+        this.delayedHeap.buildFrom(Array.from(this.delayedRunAt, ([jobId, runAt]) => ({ jobId, runAt })));
+    }
     /** Get delayed job count */
     get delayedCount() {
         return this.delayedJobIds.size;
@@ -143,8 +127,7 @@ export class TemporalManager {
     /** Clear all data */
     clear() {
         this.clearDelayed();
-        // Note: SkipList doesn't have a clear method, so we recreate by removing all
-        this.temporalIndex.removeAll(() => true);
+        this.temporalIndex.clear();
     }
     // ============ Debug Info ============
     /** Get internal structure sizes for memory debugging */

package/dist/domain/queue/waiterManager.d.ts

@@ -1,20 +1,18 @@
 /**
- * WaiterManager - Manages job availability notifications
- * Handles worker polling with timeout-based waiting
+ * WaiterManager - Manages queue-scoped job availability notifications.
+ * Handles worker polling with timeout-based waiting.
  */
 export declare class WaiterManager {
-    /** Waiter entries with cancellation flag for O(1) cleanup */
-    private readonly waiters;
-    /** Pending notification counter - incremented when notify() is called with no waiters */
-    private pendingNotifications;
-    /** Notify that multiple jobs are available - wakes up to `count` waiters */
+    private readonly queues;
+    private activeWaiters;
+    notify(queue?: string): void;
     notifyBatch(count: number): void;
-    /** Notify that jobs are available - wakes first non-cancelled waiter */
-    notify(): void;
-    /** Wait for a job to become available (with timeout) */
+    notifyBatch(queue: string, count: number): void;
     waitForJob(timeoutMs: number): Promise<void>;
-    /** Remove all cancelled waiters from the array */
-    private cleanupWaiters;
-    /** Current number of waiters */
+    waitForJob(queue: string, timeoutMs: number): Promise<void>;
+    private notifyQueue;
+    private createQueue;
+    private compact;
+    private deleteIdleQueue;
     get length(): number;
 }

package/dist/domain/queue/waiterManager.js

@@ -1,69 +1,100 @@
 /**
- * WaiterManager - Manages job availability notifications
- * Handles worker polling with timeout-based waiting
+ * WaiterManager - Manages queue-scoped job availability notifications.
+ * Handles worker polling with timeout-based waiting.
  */
-/** Threshold for triggering full waiters cleanup */
-const WAITERS_CLEANUP_THRESHOLD = 1000;
+const DEFAULT_QUEUE = '';
+const COMPACT_MIN_HEAD = 1024;
 export class WaiterManager {
-    /** Waiter entries with cancellation flag for O(1) cleanup */
-    waiters = [];
-    /** Pending notification counter - incremented when notify() is called with no waiters */
-    pendingNotifications = 0;
-    /** Notify that multiple jobs are available - wakes up to `count` waiters */
-    notifyBatch(count) {
-        for (let i = 0; i < count; i++) {
-            this.notify();
-        }
+    queues = new Map();
+    activeWaiters = 0;
+    notify(queue) {
+        this.notifyQueue(queue ?? DEFAULT_QUEUE, 1);
     }
-    /** Notify that jobs are available - wakes first non-cancelled waiter */
-    notify() {
-        // Clean up leading cancelled waiters first
-        while (this.waiters.length > 0 && this.waiters[0].cancelled) {
-            this.waiters.shift();
-        }
-        // Wake the first active waiter
-        const waiter = this.waiters.shift();
-        if (waiter && !waiter.cancelled) {
-            waiter.resolve();
-        }
-        else {
-            // No active waiter - increment pending counter so next waitForJob returns immediately
-            this.pendingNotifications++;
-        }
-        // Periodic full cleanup when array grows too large
-        if (this.waiters.length > WAITERS_CLEANUP_THRESHOLD) {
-            this.cleanupWaiters();
-        }
+    notifyBatch(queueOrCount, maybeCount) {
+        const queue = typeof queueOrCount === 'string' ? queueOrCount : DEFAULT_QUEUE;
+        const count = typeof queueOrCount === 'number' ? queueOrCount : (maybeCount ?? 0);
+        if (count <= 0)
+            return;
+        this.notifyQueue(queue, count);
     }
-    /** Wait for a job to become available (with timeout) */
-    waitForJob(timeoutMs) {
+    waitForJob(queueOrTimeout, maybeTimeout) {
+        const queue = typeof queueOrTimeout === 'string' ? queueOrTimeout : DEFAULT_QUEUE;
+        const timeoutMs = typeof queueOrTimeout === 'number' ? queueOrTimeout : (maybeTimeout ?? 0);
         if (timeoutMs <= 0)
             return Promise.resolve();
-        // Check for pending notifications - if any, decrement and return immediately
-        if (this.pendingNotifications > 0) {
-            this.pendingNotifications--;
+        const state = this.queues.get(queue);
+        if (state?.pending) {
+            state.pending = false;
+            this.deleteIdleQueue(queue, state);
             return Promise.resolve();
         }
         return new Promise((resolve) => {
-            const waiter = { resolve, cancelled: false };
-            const cleanup = () => {
+            const queueState = state ?? this.createQueue(queue);
+            const waiter = {
+                resolve,
+                cancelled: false,
+                timer: undefined,
+            };
+            waiter.timer = setTimeout(() => {
                 if (waiter.cancelled)
                     return;
                 waiter.cancelled = true;
+                queueState.active--;
+                this.activeWaiters--;
                 resolve();
-            };
-            this.waiters.push(waiter);
-            setTimeout(cleanup, timeoutMs);
+                this.compact(queue, queueState);
+            }, timeoutMs);
+            queueState.entries.push(waiter);
+            queueState.active++;
+            this.activeWaiters++;
         });
     }
-    /** Remove all cancelled waiters from the array */
-    cleanupWaiters() {
-        const active = this.waiters.filter((w) => !w.cancelled);
-        this.waiters.length = 0;
-        this.waiters.push(...active);
+    notifyQueue(queue, count) {
+        const state = this.queues.get(queue);
+        if (!state) {
+            this.createQueue(queue).pending = true;
+            return;
+        }
+        let remaining = count;
+        while (remaining > 0 && state.active > 0) {
+            const waiter = state.entries[state.head++];
+            if (!waiter || waiter.cancelled)
+                continue;
+            waiter.cancelled = true;
+            clearTimeout(waiter.timer);
+            state.active--;
+            this.activeWaiters--;
+            remaining--;
+            waiter.resolve();
+        }
+        // Availability is edge-triggered: surplus notifications coalesce into one
+        // retry hint instead of accumulating an unbounded notification debt.
+        if (remaining > 0)
+            state.pending = true;
+        this.compact(queue, state);
+    }
+    createQueue(queue) {
+        const state = { entries: [], head: 0, active: 0, pending: false };
+        this.queues.set(queue, state);
+        return state;
+    }
+    compact(queue, state) {
+        if (state.active === 0) {
+            state.entries = [];
+            state.head = 0;
+            this.deleteIdleQueue(queue, state);
+            return;
+        }
+        if (state.head >= COMPACT_MIN_HEAD && state.head * 2 >= state.entries.length) {
+            state.entries = state.entries.slice(state.head);
+            state.head = 0;
+        }
+    }
+    deleteIdleQueue(queue, state) {
+        if (state.active === 0 && !state.pending)
+            this.queues.delete(queue);
     }
-    /** Current number of waiters */
     get length() {
-        return this.waiters.length;
+        return this.activeWaiters;
     }
 }

package/dist/infrastructure/cloud/commands.js

@@ -311,8 +311,11 @@ export const COMMANDS = {
     // --- Queue list ---
     'queue:list': (qm) => {
         const queueNames = qm.listQueues();
-        const queues = queueNames.map((name) => {
-            const counts = qm.getQueueJobCounts(name);
+        const countsByQueue = qm.getAllQueueJobCounts();
+        const queues = queueNames.flatMap((name) => {
+            const counts = countsByQueue.get(name);
+            if (!counts)
+                return [];
             return {
                 name,
                 waiting: counts.waiting,

package/dist/infrastructure/cloud/snapshotCollector.js

@@ -85,8 +85,11 @@ export async function collectSnapshot(params) {
     const storage = queueManager.getStorageStatus();
     const mem = process.memoryUsage();
     const queueNames = queueManager.listQueues();
-    const queues = queueNames.map((name) => {
-        const counts = queueManager.getQueueJobCounts(name);
+    const countsByQueue = queueManager.getAllQueueJobCounts();
+    const queues = queueNames.flatMap((name) => {
+        const counts = countsByQueue.get(name);
+        if (!counts)
+            return [];
         return {
             name,
             waiting: counts.waiting,

package/dist/infrastructure/cloud/statsRefresh.js

@@ -26,8 +26,11 @@ export function buildStatsRefresh(qm) {
     const mem = process.memoryUsage();
     const workerStats = qm.workerManager.getStats();
     const queueNames = qm.listQueues();
-    const queues = queueNames.map((name) => {
-        const counts = qm.getQueueJobCounts(name);
+    const countsByQueue = qm.getAllQueueJobCounts();
+    const queues = queueNames.flatMap((name) => {
+        const counts = countsByQueue.get(name);
+        if (!counts)
+            return [];
         return {
             name,
             waiting: counts.waiting,

package/dist/infrastructure/cloud/statsUpdate.js

@@ -11,8 +11,11 @@ export function buildStatsUpdate(qm) {
     const s = qm.getStats();
     const rates = throughputTracker.getRates();
     const queueNames = qm.listQueues();
-    const queues = queueNames.map((name) => {
-        const c = qm.getQueueJobCounts(name);
+    const countsByQueue = qm.getAllQueueJobCounts();
+    const queues = queueNames.flatMap((name) => {
+        const c = countsByQueue.get(name);
+        if (!c)
+            return [];
         return {
             name,
             waiting: c.waiting,

package/dist/infrastructure/persistence/schema.d.ts

@@ -4,10 +4,10 @@
 /** SQLite PRAGMA settings for optimal performance */
 export declare const PRAGMA_SETTINGS = "\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA cache_size = -64000;\nPRAGMA temp_store = MEMORY;\nPRAGMA mmap_size = 268435456;\nPRAGMA page_size = 4096;\nPRAGMA busy_timeout = 5000;\n";
 /** Main schema creation */
-export declare const SCHEMA = "\n-- Jobs table (using UUIDv7 for job IDs)\n-- Uses BLOB for data fields (MessagePack serialization for ~2-3x faster than JSON)\nCREATE TABLE IF NOT EXISTS jobs (\n    id TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    data BLOB NOT NULL,\n    priority INTEGER NOT NULL DEFAULT 0,\n    created_at INTEGER NOT NULL,\n    run_at INTEGER NOT NULL,\n    started_at INTEGER,\n    completed_at INTEGER,\n    attempts INTEGER NOT NULL DEFAULT 0,\n    max_attempts INTEGER NOT NULL DEFAULT 3,\n    backoff INTEGER NOT NULL DEFAULT 1000,\n    ttl INTEGER,\n    timeout INTEGER,\n    unique_key TEXT,\n    custom_id TEXT,\n    depends_on BLOB,\n    parent_id TEXT,\n    children_ids BLOB,\n    tags BLOB,\n    state TEXT NOT NULL DEFAULT 'waiting',\n    lifo INTEGER NOT NULL DEFAULT 0,\n    group_id TEXT,\n    progress INTEGER DEFAULT 0,\n    progress_msg TEXT,\n    remove_on_complete INTEGER DEFAULT 0,\n    remove_on_fail INTEGER DEFAULT 0,\n    stall_timeout INTEGER,\n    last_heartbeat INTEGER,\n    timeline BLOB,\n    stacktrace BLOB\n);\n\n-- Indexes for common queries\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state\n    ON jobs(queue, state);\nCREATE INDEX IF NOT EXISTS idx_jobs_run_at\n    ON jobs(run_at) WHERE state IN ('waiting', 'delayed');\nCREATE INDEX IF NOT EXISTS idx_jobs_unique\n    ON jobs(queue, unique_key) WHERE unique_key IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_custom_id\n    ON jobs(custom_id) WHERE custom_id IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_parent\n    ON jobs(parent_id) WHERE parent_id IS NOT NULL;\n\n-- Job results storage (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS job_results (\n    job_id TEXT PRIMARY KEY,\n    result BLOB,\n    completed_at INTEGER NOT NULL\n);\n\n-- Dead letter queue (BLOB for MessagePack - stores full DlqEntry)\nCREATE TABLE IF NOT EXISTS dlq (\n    id INTEGER PRIMARY KEY AUTOINCREMENT,\n    job_id TEXT NOT NULL,\n    queue TEXT NOT NULL,\n    entry BLOB NOT NULL,\n    entered_at INTEGER NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_dlq_queue ON dlq(queue);\nCREATE INDEX IF NOT EXISTS idx_dlq_job_id ON dlq(job_id);\nCREATE INDEX IF NOT EXISTS idx_dlq_entered_at ON dlq(entered_at);\n\n-- Performance indexes for high-throughput operations\n-- Stall detection: runs every 5s, needs fast lookup of active jobs by started_at\nCREATE INDEX IF NOT EXISTS idx_jobs_state_started\n    ON jobs(state, started_at) WHERE state = 'active';\n\n-- Group operations: fast lookup by group_id\nCREATE INDEX IF NOT EXISTS idx_jobs_group_id\n    ON jobs(group_id) WHERE group_id IS NOT NULL;\n\n-- Pending jobs: compound index for priority-ordered retrieval\nCREATE INDEX IF NOT EXISTS idx_jobs_pending_priority\n    ON jobs(queue, state, priority DESC, run_at ASC) WHERE state IN ('waiting', 'delayed');\n\n-- Completed jobs: index for recovery ordering (issue #84)\nCREATE INDEX IF NOT EXISTS idx_jobs_completed_order\n    ON jobs(completed_at DESC) WHERE state = 'completed';\n\n-- Cron jobs (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS cron_jobs (\n    name TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    data BLOB NOT NULL,\n    schedule TEXT,\n    repeat_every INTEGER,\n    priority INTEGER NOT NULL DEFAULT 0,\n    next_run INTEGER NOT NULL,\n    executions INTEGER NOT NULL DEFAULT 0,\n    max_limit INTEGER,\n    timezone TEXT,\n    unique_key TEXT,\n    dedup BLOB,\n    skip_missed_on_restart INTEGER NOT NULL DEFAULT 0,\n    skip_if_no_worker INTEGER NOT NULL DEFAULT 0,\n    prevent_overlap INTEGER NOT NULL DEFAULT 1,\n    job_options BLOB\n);\n\n-- Queue state persistence (optional)\nCREATE TABLE IF NOT EXISTS queue_state (\n    name TEXT PRIMARY KEY,\n    paused INTEGER NOT NULL DEFAULT 0,\n    rate_limit INTEGER,\n    concurrency_limit INTEGER\n);\n";
+export declare const SCHEMA = "\n-- Jobs table (using UUIDv7 for job IDs)\n-- Uses BLOB for data fields (MessagePack serialization for ~2-3x faster than JSON)\nCREATE TABLE IF NOT EXISTS jobs (\n    id TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    data BLOB NOT NULL,\n    priority INTEGER NOT NULL DEFAULT 0,\n    created_at INTEGER NOT NULL,\n    run_at INTEGER NOT NULL,\n    started_at INTEGER,\n    completed_at INTEGER,\n    attempts INTEGER NOT NULL DEFAULT 0,\n    max_attempts INTEGER NOT NULL DEFAULT 3,\n    backoff INTEGER NOT NULL DEFAULT 1000,\n    ttl INTEGER,\n    timeout INTEGER,\n    unique_key TEXT,\n    custom_id TEXT,\n    depends_on BLOB,\n    parent_id TEXT,\n    children_ids BLOB,\n    tags BLOB,\n    state TEXT NOT NULL DEFAULT 'waiting',\n    lifo INTEGER NOT NULL DEFAULT 0,\n    group_id TEXT,\n    progress INTEGER DEFAULT 0,\n    progress_msg TEXT,\n    remove_on_complete INTEGER DEFAULT 0,\n    remove_on_fail INTEGER DEFAULT 0,\n    stall_timeout INTEGER,\n    last_heartbeat INTEGER,\n    timeline BLOB,\n    stacktrace BLOB\n);\n\n-- Indexes for common queries\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state\n    ON jobs(queue, state);\n-- Stable createdAt/id pagination for unfiltered and logical-state queue views\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_created\n    ON jobs(queue, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state_created\n    ON jobs(queue, state, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_run_at\n    ON jobs(run_at) WHERE state IN ('waiting', 'delayed');\nCREATE INDEX IF NOT EXISTS idx_jobs_unique\n    ON jobs(queue, unique_key) WHERE unique_key IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_custom_id\n    ON jobs(custom_id) WHERE custom_id IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_parent\n    ON jobs(parent_id) WHERE parent_id IS NOT NULL;\n\n-- Job results storage (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS job_results (\n    job_id TEXT PRIMARY KEY,\n    result BLOB,\n    completed_at INTEGER NOT NULL\n);\n\n-- Dead letter queue (BLOB for MessagePack - stores full DlqEntry)\nCREATE TABLE IF NOT EXISTS dlq (\n    id INTEGER PRIMARY KEY AUTOINCREMENT,\n    job_id TEXT NOT NULL,\n    queue TEXT NOT NULL,\n    entry BLOB NOT NULL,\n    entered_at INTEGER NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_dlq_queue ON dlq(queue);\nCREATE INDEX IF NOT EXISTS idx_dlq_job_id ON dlq(job_id);\nCREATE INDEX IF NOT EXISTS idx_dlq_entered_at ON dlq(entered_at);\n\n-- Performance indexes for high-throughput operations\n-- Stall detection: runs every 5s, needs fast lookup of active jobs by started_at\nCREATE INDEX IF NOT EXISTS idx_jobs_state_started\n    ON jobs(state, started_at) WHERE state = 'active';\n\n-- Group operations: fast lookup by group_id\nCREATE INDEX IF NOT EXISTS idx_jobs_group_id\n    ON jobs(group_id) WHERE group_id IS NOT NULL;\n\n-- Pending jobs: compound index for priority-ordered retrieval\nCREATE INDEX IF NOT EXISTS idx_jobs_pending_priority\n    ON jobs(queue, state, priority DESC, run_at ASC) WHERE state IN ('waiting', 'delayed');\n\n-- Completed jobs: index for recovery ordering (issue #84)\nCREATE INDEX IF NOT EXISTS idx_jobs_completed_order\n    ON jobs(completed_at DESC) WHERE state = 'completed';\n\n-- Cron jobs (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS cron_jobs (\n    name TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    data BLOB NOT NULL,\n    schedule TEXT,\n    repeat_every INTEGER,\n    priority INTEGER NOT NULL DEFAULT 0,\n    next_run INTEGER NOT NULL,\n    executions INTEGER NOT NULL DEFAULT 0,\n    max_limit INTEGER,\n    timezone TEXT,\n    unique_key TEXT,\n    dedup BLOB,\n    skip_missed_on_restart INTEGER NOT NULL DEFAULT 0,\n    skip_if_no_worker INTEGER NOT NULL DEFAULT 0,\n    prevent_overlap INTEGER NOT NULL DEFAULT 1,\n    job_options BLOB\n);\n\n-- Queue state persistence (optional)\nCREATE TABLE IF NOT EXISTS queue_state (\n    name TEXT PRIMARY KEY,\n    paused INTEGER NOT NULL DEFAULT 0,\n    rate_limit INTEGER,\n    concurrency_limit INTEGER\n);\n";
 /** Migration version table */
 export declare const MIGRATION_TABLE = "\nCREATE TABLE IF NOT EXISTS migrations (\n    version INTEGER PRIMARY KEY,\n    applied_at INTEGER NOT NULL\n);\n";
 /** Current schema version */
-export declare const SCHEMA_VERSION = 13;
+export declare const SCHEMA_VERSION = 14;
 /** All migrations in order */
 export declare const MIGRATIONS: Record<number, string>;

package/dist/infrastructure/persistence/schema.js

@@ -51,6 +51,11 @@ CREATE TABLE IF NOT EXISTS jobs (
 -- Indexes for common queries
 CREATE INDEX IF NOT EXISTS idx_jobs_queue_state
     ON jobs(queue, state);
+-- Stable createdAt/id pagination for unfiltered and logical-state queue views
+CREATE INDEX IF NOT EXISTS idx_jobs_queue_created
+    ON jobs(queue, created_at, id);
+CREATE INDEX IF NOT EXISTS idx_jobs_queue_state_created
+    ON jobs(queue, state, created_at, id);
 CREATE INDEX IF NOT EXISTS idx_jobs_run_at
     ON jobs(run_at) WHERE state IN ('waiting', 'delayed');
 CREATE INDEX IF NOT EXISTS idx_jobs_unique
@@ -133,7 +138,7 @@ CREATE TABLE IF NOT EXISTS migrations (
 );
 `;
 /** Current schema version */
-export const SCHEMA_VERSION = 13;
+export const SCHEMA_VERSION = 14;
 /** All migrations in order */
 export const MIGRATIONS = {
     1: SCHEMA,
@@ -187,5 +192,12 @@ ALTER TABLE cron_jobs ADD COLUMN job_options BLOB;
     // Migration 13: Persist the last failure's stacktrace on jobs (issue #74)
     13: `
 ALTER TABLE jobs ADD COLUMN stacktrace BLOB;
+`,
+    // Migration 14: Stable, index-backed getJobs ordering and pagination
+    14: `
+CREATE INDEX IF NOT EXISTS idx_jobs_queue_created
+    ON jobs(queue, created_at, id);
+CREATE INDEX IF NOT EXISTS idx_jobs_queue_state_created
+    ON jobs(queue, state, created_at, id);
 `,
 };

package/dist/infrastructure/persistence/sqlite.d.ts

@@ -142,7 +142,8 @@ export declare class SqliteStorage {
     insertJobsBatch(jobs: Job[], durable?: boolean): void;
     /**
      * Query jobs by queue with optional state filter and pagination.
-     * Uses idx_jobs_queue_state index for O(log n) lookups.
+     * This is the raw persisted-state query. Logical waiting/prioritized/delayed
+     * views should use queryJobsByLogicalStates().
      */
     queryJobs(queue: string, options: {
         state?: string;
@@ -151,9 +152,24 @@ export declare class SqliteStorage {
         offset: number;
         asc: boolean;
     }): Job[];
+    /**
+     * Query public/logical job states before applying pagination.
+     *
+     * SQLite persists pending jobs as waiting/delayed, while the public state is
+     * time-dependent: a delayed row becomes waiting or prioritized once run_at
+     * is reached. Keeping these predicates in SQL makes OFFSET count the logical
+     * result rather than the unfiltered persisted rows.
+     */
+    queryJobsByLogicalStates(queue: string, states: string[], options: {
+        limit: number;
+        offset: number;
+        asc: boolean;
+        now: number;
+    }): Job[];
     /**
      * Load pending jobs with pagination for efficient recovery.
-     * Orders by priority (desc) and run_at (asc) to process urgent jobs first.
+     * Orders by priority (desc), run_at (asc), and id (asc) so page boundaries
+     * remain deterministic when scheduling fields compare equal.
      * @param limit Max jobs to return (default: 10000)
      * @param offset Skip first N jobs (default: 0)
      */

package/dist/infrastructure/persistence/sqlite.js

@@ -482,7 +482,8 @@ export class SqliteStorage {
     // ============ Query Operations ============
     /**
      * Query jobs by queue with optional state filter and pagination.
-     * Uses idx_jobs_queue_state index for O(log n) lookups.
+     * This is the raw persisted-state query. Logical waiting/prioritized/delayed
+     * views should use queryJobsByLogicalStates().
      */
     queryJobs(queue, options) {
         const order = options.asc ? 'ASC' : 'DESC';
@@ -490,30 +491,72 @@ export class SqliteStorage {
         if (options.states && options.states.length > 0) {
             const placeholders = options.states.map(() => '?').join(',');
             rows = this.db
-                .query(`SELECT * FROM jobs WHERE queue = ? AND state IN (${placeholders}) ORDER BY created_at ${order} LIMIT ? OFFSET ?`)
+                .query(`SELECT * FROM jobs WHERE queue = ? AND state IN (${placeholders}) ORDER BY created_at ${order}, id ${order} LIMIT ? OFFSET ?`)
                 .all(queue, ...options.states, options.limit, options.offset);
         }
         else if (options.state) {
             rows = this.db
-                .query(`SELECT * FROM jobs WHERE queue = ? AND state = ? ORDER BY created_at ${order} LIMIT ? OFFSET ?`)
+                .query(`SELECT * FROM jobs WHERE queue = ? AND state = ? ORDER BY created_at ${order}, id ${order} LIMIT ? OFFSET ?`)
                 .all(queue, options.state, options.limit, options.offset);
         }
         else {
             rows = this.db
-                .query(`SELECT * FROM jobs WHERE queue = ? ORDER BY created_at ${order} LIMIT ? OFFSET ?`)
+                .query(`SELECT * FROM jobs WHERE queue = ? ORDER BY created_at ${order}, id ${order} LIMIT ? OFFSET ?`)
                 .all(queue, options.limit, options.offset);
         }
         return rows.map((row) => rowToJob(row));
     }
+    /**
+     * Query public/logical job states before applying pagination.
+     *
+     * SQLite persists pending jobs as waiting/delayed, while the public state is
+     * time-dependent: a delayed row becomes waiting or prioritized once run_at
+     * is reached. Keeping these predicates in SQL makes OFFSET count the logical
+     * result rather than the unfiltered persisted rows.
+     */
+    queryJobsByLogicalStates(queue, states, options) {
+        const uniqueStates = Array.from(new Set(states));
+        if (uniqueStates.length === 0)
+            return [];
+        const predicates = [];
+        const predicateParams = [];
+        const requested = new Set(uniqueStates);
+        if (requested.has('waiting')) {
+            predicates.push("(state IN ('waiting', 'delayed') AND run_at <= ? AND priority <= 0)");
+            predicateParams.push(options.now);
+        }
+        if (requested.has('prioritized')) {
+            predicates.push("(state IN ('waiting', 'delayed') AND run_at <= ? AND priority > 0)");
+            predicateParams.push(options.now);
+        }
+        if (requested.has('delayed')) {
+            predicates.push("(state IN ('waiting', 'delayed') AND run_at > ?)");
+            predicateParams.push(options.now);
+        }
+        const persistedStates = uniqueStates.filter((state) => state !== 'waiting' && state !== 'prioritized' && state !== 'delayed');
+        if (persistedStates.length > 0) {
+            predicates.push(`state IN (${persistedStates.map(() => '?').join(',')})`);
+            predicateParams.push(...persistedStates);
+        }
+        const order = options.asc ? 'ASC' : 'DESC';
+        const rows = this.db
+            .query(`SELECT * FROM jobs INDEXED BY idx_jobs_queue_created
+         WHERE queue = ? AND (${predicates.join(' OR ')})
+         ORDER BY created_at ${order}, id ${order}
+         LIMIT ? OFFSET ?`)
+            .all(queue, ...predicateParams, options.limit, options.offset);
+        return rows.map((row) => rowToJob(row));
+    }
     /**
      * Load pending jobs with pagination for efficient recovery.
-     * Orders by priority (desc) and run_at (asc) to process urgent jobs first.
+     * Orders by priority (desc), run_at (asc), and id (asc) so page boundaries
+     * remain deterministic when scheduling fields compare equal.
      * @param limit Max jobs to return (default: 10000)
      * @param offset Skip first N jobs (default: 0)
      */
     loadPendingJobs(limit = 10000, offset = 0) {
         const rows = this.db
-            .query("SELECT * FROM jobs WHERE state IN ('waiting', 'delayed') ORDER BY priority DESC, run_at ASC LIMIT ? OFFSET ?")
+            .query("SELECT * FROM jobs WHERE state IN ('waiting', 'delayed') ORDER BY priority DESC, run_at ASC, id ASC LIMIT ? OFFSET ?")
             .all(limit, offset);
         return rows.map((row) => rowToJob(row));
     }

package/dist/infrastructure/server/handlers/core.js

@@ -5,6 +5,7 @@
 import * as resp from '../../../domain/types/response';
 import { jobId } from '../../../domain/types/job';
 import { validateQueueName, validateJobData, validateJobOptions, validateNumericField, } from '../protocol';
+import { validatePushBatchJobs } from './pushBatchValidation';
 /** Handle PUSH command */
 export async function handlePush(cmd, ctx, reqId) {
     const queueError = validateQueueName(cmd.queue);
@@ -86,11 +87,11 @@ export async function handlePushBatch(cmd, ctx, reqId) {
     const queueError = validateQueueName(cmd.queue);
     if (queueError)
         return resp.error(queueError, reqId);
-    for (const job of cmd.jobs) {
-        const dataError = validateJobData(job.data);
-        if (dataError)
-            return resp.error(dataError, reqId);
-    }
+    // PUSH parity: per-job data/option bounds + dependsOn existence gate
+    // (extended to earlier same-batch custom ids for intra-batch chains).
+    const jobsError = validatePushBatchJobs(cmd.jobs, ctx);
+    if (jobsError)
+        return resp.error(jobsError, reqId);
     const ids = await ctx.queueManager.pushBatch(cmd.queue, cmd.jobs);
     return resp.batch(ids, reqId);
 }
@@ -128,6 +129,10 @@ export async function handlePullBatch(cmd, ctx, reqId) {
     const countError = validateNumericField(cmd.count, 'count', { min: 1, max: 1000 });
     if (countError)
         return resp.error(countError, reqId);
+    // Validate timeout (same bound as PULL)
+    const timeoutError = validateNumericField(cmd.timeout, 'timeout', { min: 0, max: 60000 });
+    if (timeoutError)
+        return resp.error(timeoutError, reqId);
     // If owner is provided, use lock-based pull
     if (cmd.owner) {
         const { jobs, tokens } = await ctx.queueManager.pullBatchWithLock(cmd.queue, cmd.count, cmd.owner, cmd.timeout ?? 0, cmd.lockTtl);
@@ -139,8 +144,9 @@ export async function handlePullBatch(cmd, ctx, reqId) {
         }
         return resp.pulledJobs(jobs, tokens, reqId);
     }
-    // Standard pull (no locks, but still track for client release)
-    const jobs = await ctx.queueManager.pullBatch(cmd.queue, cmd.count, 0);
+    // Standard pull (no locks, but still track for client release) — the
+    // non-owner branch honors cmd.timeout exactly like the owner branch and PULL.
+    const jobs = await ctx.queueManager.pullBatch(cmd.queue, cmd.count, cmd.timeout ?? 0);
     if (ctx.clientId) {
         for (const job of jobs) {
             ctx.queueManager.registerClientJob(ctx.clientId, job.id);

package/dist/infrastructure/server/handlers/pushBatchValidation.d.ts

@@ -0,0 +1,23 @@
+/**
+ * PUSHB Validation
+ * Per-job validation for batch pushes with PUSH parity: the same
+ * validateJobData/validateJobOptions bounds and the same dependsOn
+ * existence gate handlePush enforces (core.ts).
+ */
+import type { JobInput } from '../../../domain/types/job';
+import type { HandlerContext } from '../types';
+/**
+ * Validate every job of a PUSHB batch. Returns an error message naming the
+ * offending index, or null when the whole batch is valid.
+ *
+ * The dependsOn gate is EXTENDED beyond the PUSH one: a dependency may also
+ * reference ANY OTHER job of the same batch via its custom id (the only ids
+ * a client can know before the batch is applied). Order within the batch is
+ * deliberately irrelevant: the default-on auto-batcher groups concurrent
+ * add() calls in arbitrary order, so a child can legitimately precede its
+ * parent in the array, and the readiness layer (waitingDeps) resolves the
+ * chain once the whole batch is applied. Self-references are rejected, they
+ * can never resolve. jobIndex/completedJobs/depCompletions are hoisted out
+ * of the loop because PUSHB is the bulk hot path.
+ */
+export declare function validatePushBatchJobs(jobs: JobInput[], ctx: HandlerContext): string | null;