@platformatic/job-queue 0.0.1 → 0.1.0-alpha.3

package/dist/reaper.js

@@ -0,0 +1,376 @@
+import { EventEmitter } from 'node:events';
+import { randomUUID } from 'node:crypto';
+import { createJsonSerde } from "./serde/index.js";
+import { parseState } from "./utils/state.js";
+const LOCK_KEY = 'reaper:lock';
+const DEFAULT_LOCK_TTL = 30000;
+const DEFAULT_RENEWAL_INTERVAL = 10000;
+const DEFAULT_ACQUIRE_RETRY_INTERVAL = 5000;
+/**
+ * Reaper monitors for stalled jobs and requeues them.
+ *
+ * A job is considered stalled if it has been in "processing" state
+ * longer than the visibility timeout.
+ *
+ * When leader election is enabled, multiple Reaper instances can run
+ * for high availability, with only one active at a time.
+ */
+export class Reaper extends EventEmitter {
+    #storage;
+    #payloadSerde;
+    #visibilityTimeout;
+    #leaderElection;
+    #running = false;
+    #unsubscribe = null;
+    #processingTimers = new Map();
+    // Leader election state
+    #reaperId;
+    #isLeader = false;
+    #leadershipTimer = null;
+    constructor(config) {
+        super();
+        this.#storage = config.storage;
+        this.#payloadSerde = config.payloadSerde ?? createJsonSerde();
+        this.#visibilityTimeout = config.visibilityTimeout ?? 30000;
+        this.#leaderElection = config.leaderElection ?? { enabled: false };
+        this.#reaperId = randomUUID();
+    }
+    /**
+     * Get the unique identifier for this reaper instance
+     */
+    get reaperId() {
+        return this.#reaperId;
+    }
+    /**
+     * Check if this reaper is currently the leader
+     */
+    get isLeader() {
+        return this.#isLeader;
+    }
+    /**
+     * Start the reaper
+     */
+    async start() {
+        if (this.#running)
+            return;
+        this.#running = true;
+        if (this.#leaderElection.enabled) {
+            await this.#startLeadershipLoop();
+        }
+        else {
+            // No leader election, become active immediately
+            await this.#becomeActive();
+        }
+    }
+    /**
+     * Stop the reaper gracefully
+     */
+    async stop() {
+        if (!this.#running)
+            return;
+        this.#running = false;
+        // Stop leadership loop
+        if (this.#leadershipTimer) {
+            clearInterval(this.#leadershipTimer);
+            this.#leadershipTimer = null;
+        }
+        // Release lock if we're leader
+        if (this.#isLeader && this.#leaderElection.enabled) {
+            await this.#releaseLeadership();
+            this.#isLeader = false;
+        }
+        // Become inactive (cleanup)
+        await this.#becomeInactive();
+    }
+    /**
+     * Become active: subscribe to events and do initial scan
+     */
+    async #becomeActive() {
+        // Subscribe to job events
+        this.#unsubscribe = await this.#storage.subscribeToEvents((id, event) => {
+            this.#handleEvent(id, event);
+        });
+        // Do an initial scan for any jobs that were processing before we started
+        await this.#checkStalledJobs();
+    }
+    /**
+     * Become inactive: clear timers and unsubscribe
+     */
+    async #becomeInactive() {
+        // Clear all processing timers
+        for (const timer of this.#processingTimers.values()) {
+            clearTimeout(timer);
+        }
+        this.#processingTimers.clear();
+        // Unsubscribe from events
+        if (this.#unsubscribe) {
+            await this.#unsubscribe();
+            this.#unsubscribe = null;
+        }
+    }
+    /**
+     * Start the leadership acquisition loop
+     */
+    async #startLeadershipLoop() {
+        const lockTTL = this.#leaderElection.lockTTL ?? DEFAULT_LOCK_TTL;
+        // Try to acquire lock immediately
+        const acquired = await this.#tryAcquireLock(lockTTL);
+        if (acquired) {
+            this.#isLeader = true;
+            await this.#becomeActive();
+            this.emit('leadershipAcquired');
+        }
+        // Start the appropriate timer based on current role
+        this.#scheduleLeadershipCheck();
+    }
+    /**
+     * Schedule the next leadership check (renewal or acquisition)
+     */
+    #scheduleLeadershipCheck() {
+        const lockTTL = this.#leaderElection.lockTTL ?? DEFAULT_LOCK_TTL;
+        const renewalInterval = this.#leaderElection.renewalInterval ?? DEFAULT_RENEWAL_INTERVAL;
+        const acquireRetryInterval = this.#leaderElection.acquireRetryInterval ?? DEFAULT_ACQUIRE_RETRY_INTERVAL;
+        // Clear any existing timer
+        if (this.#leadershipTimer) {
+            clearInterval(this.#leadershipTimer);
+        }
+        const interval = this.#isLeader ? renewalInterval : acquireRetryInterval;
+        this.#leadershipTimer = setInterval(async () => {
+            if (!this.#running)
+                return;
+            try {
+                if (this.#isLeader) {
+                    // Renew lock
+                    const renewed = await this.#tryRenewLock(lockTTL);
+                    if (!renewed) {
+                        // Lost leadership
+                        await this.#transitionToFollower();
+                    }
+                }
+                else {
+                    // Try to acquire lock
+                    const acquired = await this.#tryAcquireLock(lockTTL);
+                    if (acquired) {
+                        await this.#transitionToLeader();
+                    }
+                }
+            }
+            catch (err) {
+                this.emit('error', err);
+            }
+        }, interval);
+    }
+    /**
+     * Try to acquire the leader lock
+     */
+    async #tryAcquireLock(ttlMs) {
+        if (!this.#storage.acquireLeaderLock) {
+            // Storage doesn't support leader election
+            this.emit('error', new Error('Storage does not support leader election'));
+            return false;
+        }
+        return this.#storage.acquireLeaderLock(LOCK_KEY, this.#reaperId, ttlMs);
+    }
+    /**
+     * Try to renew the leader lock
+     */
+    async #tryRenewLock(ttlMs) {
+        if (!this.#storage.renewLeaderLock) {
+            return false;
+        }
+        return this.#storage.renewLeaderLock(LOCK_KEY, this.#reaperId, ttlMs);
+    }
+    /**
+     * Release the leader lock
+     */
+    async #releaseLeadership() {
+        if (!this.#storage.releaseLeaderLock) {
+            return;
+        }
+        await this.#storage.releaseLeaderLock(LOCK_KEY, this.#reaperId);
+    }
+    /**
+     * Transition from follower to leader
+     */
+    async #transitionToLeader() {
+        this.#isLeader = true;
+        // Become active
+        await this.#becomeActive();
+        // Update interval timing (leaders use renewalInterval)
+        this.#scheduleLeadershipCheck();
+        this.emit('leadershipAcquired');
+    }
+    /**
+     * Transition from leader to follower
+     */
+    async #transitionToFollower() {
+        this.#isLeader = false;
+        // Become inactive
+        await this.#becomeInactive();
+        // Update interval timing (followers use acquireRetryInterval)
+        this.#scheduleLeadershipCheck();
+        this.emit('leadershipLost');
+    }
+    /**
+     * Handle a job state change event
+     */
+    #handleEvent(id, event) {
+        if (event === 'processing') {
+            // Start a timer for this job
+            this.#startTimer(id);
+        }
+        else if (event === 'completed' || event === 'failed' || event === 'cancelled') {
+            // Cancel any existing timer
+            this.#cancelTimer(id);
+        }
+    }
+    /**
+     * Start a visibility timeout timer for a job
+     */
+    #startTimer(id) {
+        // Cancel any existing timer first
+        this.#cancelTimer(id);
+        const timer = setTimeout(() => {
+            this.#processingTimers.delete(id);
+            this.#checkJob(id).catch((err) => {
+                this.emit('error', err);
+            });
+        }, this.#visibilityTimeout);
+        this.#processingTimers.set(id, timer);
+    }
+    /**
+     * Cancel the timer for a job
+     */
+    #cancelTimer(id) {
+        const timer = this.#processingTimers.get(id);
+        if (timer) {
+            clearTimeout(timer);
+            this.#processingTimers.delete(id);
+        }
+    }
+    /**
+     * Check if a job is stalled and requeue if necessary
+     */
+    async #checkJob(id) {
+        if (!this.#running)
+            return;
+        if (this.#leaderElection.enabled && !this.#isLeader)
+            return;
+        const state = await this.#storage.getJobState(id);
+        if (!state)
+            return;
+        const { status, timestamp, workerId } = parseState(state);
+        if (status !== 'processing') {
+            // Job is no longer processing, nothing to do
+            return;
+        }
+        // Check if visibility timeout has elapsed
+        const elapsed = Date.now() - timestamp;
+        if (elapsed < this.#visibilityTimeout) {
+            // Not yet stalled, restart timer for remaining time
+            const remaining = this.#visibilityTimeout - elapsed;
+            const timer = setTimeout(() => {
+                this.#processingTimers.delete(id);
+                this.#checkJob(id).catch((err) => {
+                    this.emit('error', err);
+                });
+            }, remaining);
+            this.#processingTimers.set(id, timer);
+            return;
+        }
+        // Job is stalled - need to recover it
+        await this.#recoverStalledJob(id, workerId);
+    }
+    /**
+     * Recover a stalled job by requeueing it
+     */
+    async #recoverStalledJob(id, workerId) {
+        if (!workerId) {
+            this.emit('error', new Error(`Cannot recover stalled job ${id}: no workerId in state`));
+            return;
+        }
+        // Get the job from the worker's processing queue
+        const processingJobs = await this.#storage.getProcessingJobs(workerId);
+        // Find the message for this job
+        let jobMessage = null;
+        for (const message of processingJobs) {
+            try {
+                const queueMessage = this.#payloadSerde.deserialize(message);
+                if (queueMessage.id === id) {
+                    jobMessage = message;
+                    break;
+                }
+            }
+            catch {
+                // Ignore deserialization errors
+            }
+        }
+        if (!jobMessage) {
+            // Job not found in processing queue - it may have already been processed
+            // Clear the state to prevent future checks
+            return;
+        }
+        // Requeue the job
+        await this.#storage.requeue(id, jobMessage, workerId);
+        // Update state to reflect retry
+        const queueMessage = this.#payloadSerde.deserialize(jobMessage);
+        const newState = `failing:${Date.now()}:${queueMessage.attempts + 1}`;
+        await this.#storage.setJobState(id, newState);
+        // Emit stalled event
+        this.emit('stalled', id);
+    }
+    /**
+     * Periodically check all workers' processing queues for stalled jobs
+     */
+    async #checkStalledJobs() {
+        if (!this.#running)
+            return;
+        if (this.#leaderElection.enabled && !this.#isLeader)
+            return;
+        const workers = await this.#storage.getWorkers();
+        for (const workerId of workers) {
+            await this.#checkWorkerProcessingQueue(workerId);
+        }
+    }
+    /**
+     * Check a single worker's processing queue for stalled jobs
+     */
+    async #checkWorkerProcessingQueue(workerId) {
+        if (!this.#running)
+            return;
+        if (this.#leaderElection.enabled && !this.#isLeader)
+            return;
+        const processingJobs = await this.#storage.getProcessingJobs(workerId);
+        for (const message of processingJobs) {
+            try {
+                const queueMessage = this.#payloadSerde.deserialize(message);
+                const state = await this.#storage.getJobState(queueMessage.id);
+                if (!state)
+                    continue;
+                const { status, timestamp } = parseState(state);
+                if (status === 'processing') {
+                    const elapsed = Date.now() - timestamp;
+                    if (elapsed >= this.#visibilityTimeout) {
+                        // Job is stalled
+                        await this.#recoverStalledJob(queueMessage.id, workerId);
+                    }
+                    else if (!this.#processingTimers.has(queueMessage.id)) {
+                        // Start a timer for remaining time
+                        const remaining = this.#visibilityTimeout - elapsed;
+                        const timer = setTimeout(() => {
+                            this.#processingTimers.delete(queueMessage.id);
+                            this.#checkJob(queueMessage.id).catch((err) => {
+                                this.emit('error', err);
+                            });
+                        }, remaining);
+                        this.#processingTimers.set(queueMessage.id, timer);
+                    }
+                }
+            }
+            catch {
+                // Ignore deserialization errors
+            }
+        }
+    }
+}
+//# sourceMappingURL=reaper.js.map

package/dist/reaper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reaper.js","sourceRoot":"","sources":["../src/reaper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAIxC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAuB7C,MAAM,QAAQ,GAAG,aAAa,CAAA;AAC9B,MAAM,gBAAgB,GAAG,KAAK,CAAA;AAC9B,MAAM,wBAAwB,GAAG,KAAK,CAAA;AACtC,MAAM,8BAA8B,GAAG,IAAI,CAAA;AAE3C;;;;;;;;GAQG;AACH,MAAM,OAAO,MAAiB,SAAQ,YAA0B;IAC9D,QAAQ,CAAS;IACjB,aAAa,CAAiB;IAC9B,kBAAkB,CAAQ;IAC1B,eAAe,CAAsB;IAErC,QAAQ,GAAG,KAAK,CAAA;IAChB,YAAY,GAAiC,IAAI,CAAA;IACjD,iBAAiB,GAA+C,IAAI,GAAG,EAAE,CAAA;IAEzE,wBAAwB;IACxB,SAAS,CAAQ;IACjB,SAAS,GAAG,KAAK,CAAA;IACjB,gBAAgB,GAA0C,IAAI,CAAA;IAE9D,YAAa,MAA8B;QACzC,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAA;QAC9B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,YAAY,IAAI,eAAe,EAAY,CAAA;QACvE,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,IAAI,KAAK,CAAA;QAC3D,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,cAAc,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;QAClE,IAAI,CAAC,SAAS,GAAG,UAAU,EAAE,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED;;OAEG;IACH,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAA;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAM;QAEzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAA;QAEpB,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;QACnC,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAM;QAE1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAA;QAErB,uBAAuB;QACvB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YACpC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAA;QAC9B,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;YACnD,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;YAC/B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;QACxB,CAAC;QAED,4BAA4B;QAC5B,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACjB,0BAA0B;QAC1B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;YACtE,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;QAC9B,CAAC,CAAC,CAAA;QAEF,yEAAyE;QACzE,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACnB,8BAA8B;QAC9B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YACpD,YAAY,CAAC,KAAK,CAAC,CAAA;QACrB,CAAC;QACD,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;QAE9B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAA;QAC1B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,gBAAgB,CAAA;QAEhE,kCAAkC;QAClC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;QACpD,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;YACrB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;YAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;QACjC,CAAC;QAED,oDAAoD;QACpD,IAAI,CAAC,wBAAwB,EAAE,CAAA;IACjC,CAAC;IAED;;OAEG;IACH,wBAAwB;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,gBAAgB,CAAA;QAChE,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,eAAe,IAAI,wBAAwB,CAAA;QACxF,MAAM,oBAAoB,GAAG,IAAI,CAAC,eAAe,CAAC,oBAAoB,IAAI,8BAA8B,CAAA;QAExG,2BAA2B;QAC3B,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QACtC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,oBAAoB,CAAA;QAExE,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC7C,IAAI,CAAC,IAAI,CAAC,QAAQ;gBAAE,OAAM;YAE1B,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACnB,aAAa;oBACb,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;oBACjD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,kBAAkB;wBAClB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAA;oBACpC,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,sBAAsB;oBACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;oBACpD,IAAI,QAAQ,EAAE,CAAC;wBACb,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAA;oBAClC,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAY,CAAC,CAAA;YAClC,CAAC;QACH,CAAC,EAAE,QAAQ,CAAC,CAAA;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAE,KAAa;QAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YACrC,0CAA0C;YAC1C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,CAAA;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;IACzE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAE,KAAa;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC;YACnC,OAAO,KAAK,CAAA;QACd,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;IACvE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YACrC,OAAM;QACR,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACjE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB;QACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QAErB,gBAAgB;QAChB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;QAE1B,uDAAuD;QACvD,IAAI,CAAC,wBAAwB,EAAE,CAAA;QAE/B,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;IACjC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;QAEtB,kBAAkB;QAClB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;QAE5B,8DAA8D;QAC9D,IAAI,CAAC,wBAAwB,EAAE,CAAA;QAE/B,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC7B,CAAC;IAED;;OAEG;IACH,YAAY,CAAE,EAAU,EAAE,KAAa;QACrC,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,6BAA6B;YAC7B,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtB,CAAC;aAAM,IAAI,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YAChF,4BAA4B;YAC5B,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,WAAW,CAAE,EAAU;QACrB,kCAAkC;QAClC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;QAErB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACjC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YACzB,CAAC,CAAC,CAAA;QACJ,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAA;QAE3B,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;IACvC,CAAC;IAED;;OAEG;IACH,YAAY,CAAE,EAAU;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC5C,IAAI,KAAK,EAAE,CAAC;YACV,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAE,EAAU;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAM;QAC1B,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAM;QAE3D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACjD,IAAI,CAAC,KAAK;YAAE,OAAM;QAElB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAEzD,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;YAC5B,6CAA6C;YAC7C,OAAM;QACR,CAAC;QAED,0CAA0C;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;QACtC,IAAI,OAAO,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACtC,oDAAoD;YACpD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAA;YACnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBACjC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;gBACzB,CAAC,CAAC,CAAA;YACJ,CAAC,EAAE,SAAS,CAAC,CAAA;YACb,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;YACrC,OAAM;QACR,CAAC;QAED,sCAAsC;QACtC,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAE,EAAU,EAAE,QAAiB;QACrD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,8BAA8B,EAAE,wBAAwB,CAAC,CAAC,CAAA;YACvF,OAAM;QACR,CAAC;QAED,iDAAiD;QACjD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAEtE,gCAAgC;QAChC,IAAI,UAAU,GAAkB,IAAI,CAAA;QACpC,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,OAAO,CAAsC,CAAA;gBACjG,IAAI,YAAY,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;oBAC3B,UAAU,GAAG,OAAO,CAAA;oBACpB,MAAK;gBACP,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,yEAAyE;YACzE,2CAA2C;YAC3C,OAAM;QACR,CAAC;QAED,kBAAkB;QAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAA;QAErD,gCAAgC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,UAAU,CAAsC,CAAA;QACpG,MAAM,QAAQ,GAAG,WAAW,IAAI,CAAC,GAAG,EAAE,IAAI,YAAY,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAA;QACrE,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QAE7C,qBAAqB;QACrB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAM;QAC1B,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAM;QAE3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;QAEhD,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAA;QAClD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,2BAA2B,CAAE,QAAgB;QACjD,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAM;QAC1B,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAM;QAE3D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAEtE,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,OAAO,CAAsC,CAAA;gBACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;gBAE9D,IAAI,CAAC,KAAK;oBAAE,SAAQ;gBAEpB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;gBAE/C,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;oBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;oBACtC,IAAI,OAAO,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBACvC,iBAAiB;wBACjB,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;oBAC1D,CAAC;yBAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,CAAC;wBACxD,mCAAmC;wBACnC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAA;wBACnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;4BAC5B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;4BAC9C,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gCAC5C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;4BACzB,CAAC,CAAC,CAAA;wBACJ,CAAC,EAAE,SAAS,CAAC,CAAA;wBACb,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;oBACpD,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/serde/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Serialization/deserialization interface for queue messages
+ */
+export interface Serde<T> {
+    serialize(value: T): Buffer;
+    deserialize(buffer: Buffer): T;
+}
+/**
+ * JSON serializer - default implementation
+ */
+export declare class JsonSerde<T> implements Serde<T> {
+    serialize(value: T): Buffer;
+    deserialize(buffer: Buffer): T;
+}
+/**
+ * Create a JSON serde instance
+ */
+export declare function createJsonSerde<T>(): Serde<T>;
+//# sourceMappingURL=index.d.ts.map

package/dist/serde/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/serde/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,KAAK,CAAC,CAAC;IACtB,SAAS,CAAE,KAAK,EAAE,CAAC,GAAG,MAAM,CAAA;IAC5B,WAAW,CAAE,MAAM,EAAE,MAAM,GAAG,CAAC,CAAA;CAChC;AAED;;GAEG;AACH,qBAAa,SAAS,CAAC,CAAC,CAAE,YAAW,KAAK,CAAC,CAAC,CAAC;IAC3C,SAAS,CAAE,KAAK,EAAE,CAAC,GAAG,MAAM;IAI5B,WAAW,CAAE,MAAM,EAAE,MAAM,GAAG,CAAC;CAGhC;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,CAAC,KAAM,KAAK,CAAC,CAAC,CAAC,CAE9C"}

package/dist/serde/index.js

@@ -0,0 +1,18 @@
+/**
+ * JSON serializer - default implementation
+ */
+export class JsonSerde {
+    serialize(value) {
+        return Buffer.from(JSON.stringify(value));
+    }
+    deserialize(buffer) {
+        return JSON.parse(buffer.toString());
+    }
+}
+/**
+ * Create a JSON serde instance
+ */
+export function createJsonSerde() {
+    return new JsonSerde();
+}
+//# sourceMappingURL=index.js.map

package/dist/serde/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/serde/index.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,OAAO,SAAS;IACpB,SAAS,CAAE,KAAQ;QACjB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;IAC3C,CAAC;IAED,WAAW,CAAE,MAAc;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAM,CAAA;IAC3C,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,OAAO,IAAI,SAAS,EAAK,CAAA;AAC3B,CAAC"}

package/dist/storage/file.d.ts

@@ -0,0 +1,44 @@
+import type { Storage } from './types.ts';
+interface FileStorageConfig {
+    basePath: string;
+}
+/**
+ * File-based storage implementation
+ * Uses the filesystem for persistence with atomic writes
+ */
+export declare class FileStorage implements Storage {
+    #private;
+    constructor(config: FileStorageConfig);
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    enqueue(id: string, message: Buffer, timestamp: number): Promise<string | null>;
+    dequeue(workerId: string, timeout: number): Promise<Buffer | null>;
+    requeue(id: string, message: Buffer, workerId: string): Promise<void>;
+    ack(id: string, message: Buffer, workerId: string): Promise<void>;
+    getJobState(id: string): Promise<string | null>;
+    setJobState(id: string, state: string): Promise<void>;
+    deleteJob(id: string): Promise<boolean>;
+    getJobStates(ids: string[]): Promise<Map<string, string | null>>;
+    setResult(id: string, result: Buffer, ttlMs: number): Promise<void>;
+    getResult(id: string): Promise<Buffer | null>;
+    setError(id: string, error: Buffer, ttlMs: number): Promise<void>;
+    getError(id: string): Promise<Buffer | null>;
+    registerWorker(workerId: string, ttlMs: number): Promise<void>;
+    refreshWorker(workerId: string, ttlMs: number): Promise<void>;
+    unregisterWorker(workerId: string): Promise<void>;
+    getWorkers(): Promise<string[]>;
+    getProcessingJobs(workerId: string): Promise<Buffer[]>;
+    subscribeToJob(id: string, handler: (status: 'completed' | 'failed' | 'failing') => void): Promise<() => Promise<void>>;
+    notifyJobComplete(id: string, status: 'completed' | 'failed' | 'failing'): Promise<void>;
+    subscribeToEvents(handler: (id: string, event: string) => void): Promise<() => Promise<void>>;
+    publishEvent(id: string, event: string): Promise<void>;
+    completeJob(id: string, message: Buffer, workerId: string, result: Buffer, resultTtlMs: number): Promise<void>;
+    failJob(id: string, message: Buffer, workerId: string, error: Buffer, errorTtlMs: number): Promise<void>;
+    retryJob(id: string, message: Buffer, workerId: string, attempts: number): Promise<void>;
+    /**
+     * Clear all data (useful for testing)
+     */
+    clear(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=file.d.ts.map

package/dist/storage/file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../src/storage/file.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAIzC,UAAU,iBAAiB;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAQD;;;GAGG;AACH,qBAAa,WAAY,YAAW,OAAO;;gBA8B5B,MAAM,EAAE,iBAAiB;IAchC,OAAO,IAAK,OAAO,CAAC,IAAI,CAAC;IA8BzB,UAAU,IAAK,OAAO,CAAC,IAAI,CAAC;IA0L5B,OAAO,CACX,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IA2CnB,OAAO,CAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAqBnE,OAAO,CAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBtE,GAAG,CAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlE,WAAW,CAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAQhD,WAAW,CAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItD,SAAS,CAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUxC,YAAY,CAAE,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;IAUjE,SAAS,CAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpE,SAAS,CAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAoB9C,QAAQ,CAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWlE,QAAQ,CAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAmB7C,cAAc,CAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/D,aAAa,CAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D,gBAAgB,CAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD,UAAU,IAAK,OAAO,CAAC,MAAM,EAAE,CAAC;IA6BhC,iBAAiB,CAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAsBvD,cAAc,CAClB,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,KAAK,IAAI,GAC5D,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IASzB,iBAAiB,CACrB,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,GACzC,OAAO,CAAC,IAAI,CAAC;IASV,iBAAiB,CACrB,OAAO,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAC3C,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAQzB,YAAY,CAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvD,WAAW,CACf,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC;IAmBV,OAAO,CACX,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAmBV,QAAQ,CACZ,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC;IAyFhB;;OAEG;IACG,KAAK,IAAK,OAAO,CAAC,IAAI,CAAC;CAsB9B"}