bunqueue 2.8.54 → 2.8.56

package/dist/client/worker/runtime/control.js

@@ -0,0 +1,175 @@
+import { hostname } from 'os';
+import { jobId } from '../../../domain/types/job';
+import { getSharedManager } from '../../manager';
+import { TcpEventSubscription } from '../../queue-events/tcpSubscription';
+import { startHeartbeat } from '../workerHeartbeat';
+import { WorkerState } from './state';
+export class WorkerControl extends WorkerState {
+    run() {
+        if (this.running || this.closed)
+            return;
+        this.running = true;
+        this.paused = false;
+        this._closing = false;
+        this._forceClose = false;
+        this._closingPromise = null;
+        queueMicrotask(() => {
+            if (!this.closed)
+                this.emit('ready');
+        });
+        if (!this.stalledUnsubscribe && !this.opts.skipStalledCheck) {
+            this.subscribeToStalledEvents();
+        }
+        if (this.embedded && !this.registered) {
+            getSharedManager().registerWorker(this.queueKey, [this.queueKey], this.opts.concurrency, {
+                workerId: this.workerId,
+                hostname: hostname(),
+                pid: process.pid,
+                startedAt: this.startedAt,
+            });
+            this.registered = true;
+        }
+        else if (this.tcp && !this.registered) {
+            this.registerWithServer();
+        }
+        if (this.opts.heartbeatInterval > 0 && !this.opts.skipLockRenewal && !this.heartbeatTimer) {
+            if (this.embedded) {
+                this.heartbeatTimer = setInterval(() => {
+                    const manager = getSharedManager();
+                    for (const id of this.pulledJobIds) {
+                        const token = this.opts.useLocks ? this.jobTokens.get(id) : undefined;
+                        manager.jobHeartbeat(jobId(id), token);
+                    }
+                }, this.opts.heartbeatInterval);
+            }
+            else {
+                this.heartbeatTimer = startHeartbeat(this.getHeartbeatDeps(), this.opts.heartbeatInterval);
+            }
+        }
+        if (this.opts.heartbeatInterval > 0)
+            this.startWorkerHeartbeat();
+        if (this.stalledSubscription) {
+            void this.stalledSubscription.waitUntilReady().then(() => this.poll(), () => this.poll());
+        }
+        else {
+            this.poll();
+        }
+    }
+    subscribeToStalledEvents() {
+        const handleEvent = (event) => {
+            if (event.queue !== this.queueKey)
+                return;
+            if (event.eventType === "stalled" /* EventType.Stalled */) {
+                this.emit('stalled', event.jobId, 'active');
+            }
+        };
+        if (this.embedded) {
+            this.stalledUnsubscribe = getSharedManager().subscribe(handleEvent);
+            return;
+        }
+        const subscription = new TcpEventSubscription({
+            connection: this.opts.connection,
+            queue: this.queueKey,
+            onEvent: handleEvent,
+            onError: (error) => {
+                if (this.listenerCount('error') > 0)
+                    this.emit('error', error);
+            },
+        });
+        this.stalledSubscription = subscription;
+        this.stalledUnsubscribe = () => {
+            subscription.close();
+            if (this.stalledSubscription === subscription)
+                this.stalledSubscription = null;
+        };
+    }
+    pause() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.paused = true;
+        if (this.pollTimer) {
+            clearTimeout(this.pollTimer);
+            this.pollTimer = null;
+        }
+    }
+    resume() {
+        if (this.closed)
+            return;
+        this.paused = false;
+        this.run();
+    }
+    isRunning() {
+        return this.running;
+    }
+    isPaused() {
+        return this.paused && !this.closed;
+    }
+    isClosed() {
+        return this.closed;
+    }
+    get concurrency() {
+        return this.opts.concurrency;
+    }
+    set concurrency(value) {
+        const clamped = Math.max(1, value);
+        const previous = this.opts.concurrency;
+        this.opts.concurrency = clamped;
+        if (clamped > previous && this.running && !this._closing)
+            this.poll();
+    }
+    get closing() {
+        return this._closingPromise;
+    }
+    async waitUntilReady() {
+        if (this.embedded)
+            return;
+        if (this.tcpPool)
+            await this.tcpPool.send({ cmd: 'Ping' });
+        if (this.stalledSubscription)
+            await this.stalledSubscription.waitUntilReady();
+    }
+    cancelJob(jobId, reason) {
+        if (this.hasActiveDelivery(jobId)) {
+            this.cancelledJobs.add(jobId);
+            this.emit('cancelled', { jobId, reason: reason ?? 'Job cancelled by worker' });
+            return true;
+        }
+        return false;
+    }
+    cancelAllJobs(reason) {
+        for (const jobId of this.activeDeliveryIds()) {
+            this.cancelledJobs.add(jobId);
+            this.emit('cancelled', { jobId, reason: reason ?? 'All jobs cancelled' });
+        }
+    }
+    isJobCancelled(jobId) {
+        return this.cancelledJobs.has(jobId);
+    }
+    getRateLimiterInfo() {
+        return this.rateLimiter.getRateLimiterInfo();
+    }
+    rateLimit(expireTimeMs) {
+        this.rateLimiter.rateLimit(expireTimeMs);
+    }
+    isRateLimited() {
+        return this.rateLimiter.isRateLimited();
+    }
+    async startStalledCheckTimer() {
+        // No-op for API compatibility; stall detection is automatic.
+    }
+    // biome-ignore lint/suspicious/useAwait: public compatibility contract is asynchronous
+    async delay(milliseconds = 0, abortController) {
+        if (milliseconds <= 0)
+            return;
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(resolve, milliseconds);
+            if (abortController) {
+                abortController.signal.addEventListener('abort', () => {
+                    clearTimeout(timeout);
+                    reject(new Error('Delay aborted'));
+                });
+            }
+        });
+    }
+}

package/dist/client/worker/runtime/execution.d.ts

@@ -0,0 +1,11 @@
+import type { HeartbeatDeps } from '../workerHeartbeat';
+import type { PullConfig } from '../workerPull';
+import { WorkerPolling } from './polling';
+import type { WorkerDelivery } from './state';
+export declare class WorkerExecution<T = unknown, R = unknown> extends WorkerPolling<T, R> {
+    protected startJob(delivery: WorkerDelivery): boolean;
+    protected registerWithServer(): void;
+    protected startWorkerHeartbeat(): void;
+    protected getHeartbeatDeps(): HeartbeatDeps;
+    protected getPullConfig(): PullConfig;
+}

package/dist/client/worker/runtime/execution.js

@@ -0,0 +1,123 @@
+import { hostname } from 'os';
+import { getSharedManager } from '../../manager';
+import { processJob } from '../processor';
+import { WorkerPolling } from './polling';
+export class WorkerExecution extends WorkerPolling {
+    startJob(delivery) {
+        const { job, token } = delivery;
+        if (this.isActiveDelivery(delivery))
+            return true;
+        if (this.groupLimiter && !this.groupLimiter.canProcess(job))
+            return false;
+        if (!this.rateLimiter.tryAcquire())
+            return false;
+        this.activeJobs++;
+        this.beginDelivery(delivery);
+        if (this.groupLimiter)
+            this.groupLimiter.increment(job);
+        const tokenForProcess = this.opts.useLocks ? token : undefined;
+        void processJob(job, {
+            name: this.queueKey,
+            processor: this.processor,
+            embedded: this.embedded,
+            tcp: this.tcp,
+            ackBatcher: this.ackBatcher,
+            emitter: this,
+            token: tokenForProcess,
+            removeOnComplete: this.opts.removeOnComplete === true ? true : undefined,
+            removeOnFail: this.opts.removeOnFail === true ? true : undefined,
+            shouldAbandonOutcome: () => this._forceClose || !this.isCurrentDelivery(delivery),
+            onOutcome: (ok) => {
+                if (ok)
+                    this.processedCount++;
+                else
+                    this.failedCount++;
+            },
+        }).finally(() => {
+            this.activeJobs--;
+            this.finishDelivery(delivery);
+            if (this.groupLimiter)
+                this.groupLimiter.decrement(job);
+            if (this.running && !this._closing)
+                this.poll();
+        });
+        if (this.activeJobs < this.opts.concurrency && !this._closing && !this.processingScheduled) {
+            this.processingScheduled = true;
+            setImmediate(() => {
+                this.processingScheduled = false;
+                void this.tryProcess();
+            });
+        }
+        return true;
+    }
+    registerWithServer() {
+        if (!this.tcp || this.registered)
+            return;
+        void this.tcp
+            .send({
+            cmd: 'RegisterWorker',
+            name: this.queueKey,
+            queues: [this.queueKey],
+            concurrency: this.opts.concurrency,
+            workerId: this.workerId,
+            hostname: hostname(),
+            pid: process.pid,
+            startedAt: this.startedAt,
+        })
+            .then(() => {
+            this.registered = true;
+        })
+            .catch((errorValue) => {
+            const error = errorValue instanceof Error ? errorValue : new Error(String(errorValue));
+            this.emit('error', Object.assign(error, { context: 'worker-register' }));
+        });
+    }
+    startWorkerHeartbeat() {
+        if (this.workerHeartbeatTimer)
+            return;
+        this.workerHeartbeatTimer = setInterval(() => {
+            if (!this.registered)
+                return;
+            if (this.embedded) {
+                getSharedManager().workerManager.heartbeat(this.workerId, {
+                    activeJobs: this.activeJobs,
+                    processed: this.processedCount,
+                    failed: this.failedCount,
+                });
+                return;
+            }
+            if (!this.tcp)
+                return;
+            void this.tcp
+                .send({
+                cmd: 'Heartbeat',
+                id: this.workerId,
+                activeJobs: this.activeJobs,
+                processed: this.processedCount,
+                failed: this.failedCount,
+            })
+                .catch((errorValue) => {
+                const error = errorValue instanceof Error ? errorValue : new Error(String(errorValue));
+                this.emit('error', Object.assign(error, { context: 'worker-heartbeat' }));
+            });
+        }, this.opts.heartbeatInterval);
+    }
+    getHeartbeatDeps() {
+        return {
+            pulledJobIds: this.pulledJobIds,
+            jobTokens: this.jobTokens,
+            tcp: this.tcp,
+            useLocks: this.opts.useLocks,
+            emitter: this,
+        };
+    }
+    getPullConfig() {
+        return {
+            name: this.queueKey,
+            workerId: this.workerId,
+            useLocks: this.opts.useLocks,
+            pollTimeout: this.opts.pollTimeout,
+            lockDuration: this.opts.lockDuration,
+        };
+    }
+}

package/dist/client/worker/runtime/lifecycle.d.ts

@@ -0,0 +1,7 @@
+import { WorkerManual } from './manual';
+export declare abstract class WorkerLifecycle<T = unknown, R = unknown> extends WorkerManual<T, R> {
+    close(force?: boolean): Promise<void>;
+    protected _doClose(force: boolean): Promise<void>;
+    protected releaseBufferedJobs(): Promise<void>;
+    private releasePulledJob;
+}

package/dist/client/worker/runtime/lifecycle.js

@@ -0,0 +1,114 @@
+import { jobId } from '../../../domain/types/job';
+import { getSharedManager } from '../../manager';
+import { WorkerManual } from './manual';
+export class WorkerLifecycle extends WorkerManual {
+    // biome-ignore lint/suspicious/useAwait: preserves the public async close contract
+    async close(force = false) {
+        if (this.closed)
+            return;
+        if (this._closingPromise) {
+            if (force)
+                this._forceClose = true;
+            return this._closingPromise;
+        }
+        this._forceClose = force;
+        this._closingPromise = this._doClose(force);
+        return this._closingPromise;
+    }
+    async _doClose(force) {
+        this._closing = true;
+        this.running = false;
+        this.paused = false;
+        if (this.pollTimer) {
+            clearTimeout(this.pollTimer);
+            this.pollTimer = null;
+        }
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+        if (this.workerHeartbeatTimer) {
+            clearInterval(this.workerHeartbeatTimer);
+            this.workerHeartbeatTimer = null;
+        }
+        await this.releaseBufferedJobs();
+        if (!force) {
+            while (this.activeJobs > 0 && !this._forceClose)
+                await Bun.sleep(50);
+        }
+        await this.ackBatcher.flush();
+        await this.ackBatcher.waitForInFlight();
+        this.ackBatcher.stop();
+        if (this.registered) {
+            if (this.embedded) {
+                getSharedManager().unregisterWorker(this.workerId);
+            }
+            else if (this.tcp) {
+                try {
+                    await this.tcp.send({ cmd: 'UnregisterWorker', workerId: this.workerId });
+                }
+                catch {
+                    // Best-effort unregister — server will cleanup via stale timeout.
+                }
+            }
+            this.registered = false;
+        }
+        await Bun.sleep(100);
+        if (this.stalledUnsubscribe) {
+            this.stalledUnsubscribe();
+            this.stalledUnsubscribe = null;
+        }
+        this.clearDeliveries();
+        this.pulledJobIds.clear();
+        this.jobTokens.clear();
+        this.cancelledJobs.clear();
+        this.pendingJobs = [];
+        this.pendingJobsHead = 0;
+        if (this.groupLimiter)
+            this.groupLimiter.clear();
+        if (this.tcpPool)
+            this.tcpPool.close();
+        this.closed = true;
+        this._closing = false;
+        this.emit('closed');
+    }
+    async releaseBufferedJobs() {
+        const buffered = this.pendingJobs.slice(this.pendingJobsHead);
+        this.pendingJobs = [];
+        this.pendingJobsHead = 0;
+        for (const delivery of buffered) {
+            if (this.isCurrentDelivery(delivery))
+                await this.releasePulledJob(delivery);
+        }
+        for (const id of [...this.pulledJobIds]) {
+            if (this.hasActiveDelivery(id))
+                continue;
+            const delivery = this.currentDelivery(id);
+            if (delivery)
+                await this.releasePulledJob(delivery);
+        }
+    }
+    async releasePulledJob(delivery) {
+        const id = String(delivery.job.id);
+        const { token } = delivery;
+        try {
+            if (this.embedded) {
+                const manager = getSharedManager();
+                await manager.moveActiveToWait(jobId(id), token ?? undefined);
+            }
+            else if (this.tcp) {
+                await this.tcp.send({
+                    cmd: 'MoveToWait',
+                    id,
+                    ...(token ? { token } : {}),
+                });
+            }
+        }
+        catch {
+            // Lock expiration recovers anything that could not be released.
+        }
+        finally {
+            this.forgetDelivery(delivery);
+        }
+    }
+}

package/dist/client/worker/runtime/manual.d.ts

@@ -0,0 +1,10 @@
+import type { ManualJob } from '../types';
+import { WorkerControl } from './control';
+export declare abstract class WorkerManual<T = unknown, R = unknown> extends WorkerControl<T, R> {
+    private registerManualJob;
+    getNextJob(token?: string, _opts?: {
+        block?: boolean;
+    }): Promise<ManualJob<T> | undefined>;
+    processJobManually(job: ManualJob<T>, token?: string, fetchNextCallback?: () => Promise<ManualJob<T> | undefined>): Promise<ManualJob<T> | undefined>;
+    extendJobLocks(jobIds: string[], tokens: string[], duration: number): Promise<number>;
+}

package/dist/client/worker/runtime/manual.js

@@ -0,0 +1,137 @@
+import { resolvePublicJobPayload } from '../../jobHelpers';
+import { getSharedManager } from '../../manager';
+import { parseJobFromResponse } from '../jobParser';
+import { processJob } from '../processor';
+import { WorkerControl } from './control';
+export class WorkerManual extends WorkerControl {
+    registerManualJob(job, token) {
+        const payload = resolvePublicJobPayload(job);
+        const manualJob = {
+            ...job,
+            name: payload.name,
+            data: payload.data,
+            ...(token ? { token } : {}),
+        };
+        this.trackDelivery({ job: manualJob, token });
+        return manualJob;
+    }
+    async getNextJob(token, _opts) {
+        if (this.closed)
+            return undefined;
+        if (this.embedded) {
+            const manager = getSharedManager();
+            if (this.opts.useLocks) {
+                const { job, token: lockToken } = await manager.pullWithLock(this.queueKey, this.workerId, 0, this.opts.lockDuration);
+                return job ? this.registerManualJob(job, lockToken) : undefined;
+            }
+            const job = await manager.pull(this.queueKey, 0);
+            return job ? this.registerManualJob(job, null) : undefined;
+        }
+        if (!this.tcp)
+            return undefined;
+        const command = {
+            cmd: 'PULL',
+            queue: this.queueKey,
+            timeout: 0,
+        };
+        if (this.opts.useLocks) {
+            command.owner = this.workerId;
+            if (this.opts.lockDuration !== undefined)
+                command.lockTtl = this.opts.lockDuration;
+            if (token)
+                command.token = token;
+        }
+        const response = await this.tcp.send(command);
+        if (!response.ok || !response.job)
+            return undefined;
+        const job = parseJobFromResponse(response.job, this.queueKey);
+        const lockToken = this.opts.useLocks ? (response.token ?? null) : null;
+        return this.registerManualJob(job, lockToken);
+    }
+    async processJobManually(job, token, fetchNextCallback) {
+        if (this.closed)
+            return undefined;
+        const jobIdStr = String(job.id);
+        let delivery = this.currentDelivery(jobIdStr);
+        const expectedToken = this.opts.useLocks ? (token ?? job.token ?? null) : null;
+        if (delivery) {
+            if (token !== undefined && token !== delivery.token) {
+                throw new Error(`Invalid or expired lock token for job ${jobIdStr}`);
+            }
+            if (delivery.job !== job && expectedToken !== delivery.token)
+                return undefined;
+        }
+        else {
+            if (this.opts.useLocks && expectedToken === null) {
+                throw new Error(`Lock token required for manually processed job ${jobIdStr}`);
+            }
+            delivery = this.trackDelivery({ job, token: expectedToken });
+        }
+        const processingJob = delivery.job;
+        while (!this.closed && !this._closing) {
+            if (!this.isCurrentDelivery(delivery))
+                return undefined;
+            if (this.isActiveDelivery(delivery))
+                return undefined;
+            const concurrencyBlocked = this.activeJobs >= this.opts.concurrency;
+            const groupBlocked = this.groupLimiter !== null && !this.groupLimiter.canProcess(processingJob);
+            if (!concurrencyBlocked && !groupBlocked && this.rateLimiter.tryAcquire())
+                break;
+            const waitTime = concurrencyBlocked || groupBlocked ? 10 : this.rateLimiter.getTimeUntilNextSlot();
+            await Bun.sleep(Math.min(Math.max(waitTime, 10), 100));
+        }
+        if (this.closed || this._closing || !this.isCurrentDelivery(delivery))
+            return undefined;
+        if (this.groupLimiter)
+            this.groupLimiter.increment(processingJob);
+        this.activeJobs++;
+        this.beginDelivery(delivery);
+        try {
+            await processJob(processingJob, {
+                name: this.queueKey,
+                processor: this.processor,
+                embedded: this.embedded,
+                tcp: this.tcp,
+                ackBatcher: this.ackBatcher,
+                emitter: this,
+                token: this.opts.useLocks ? delivery.token : undefined,
+                shouldAbandonOutcome: () => this._forceClose || !this.isCurrentDelivery(delivery),
+            });
+            if (fetchNextCallback)
+                return await fetchNextCallback();
+        }
+        finally {
+            this.activeJobs--;
+            this.finishDelivery(delivery);
+            if (this.groupLimiter)
+                this.groupLimiter.decrement(processingJob);
+        }
+    }
+    async extendJobLocks(jobIds, tokens, duration) {
+        if (this.closed || jobIds.length === 0)
+            return 0;
+        if (jobIds.length !== tokens.length) {
+            throw new Error('jobIds and tokens arrays must have the same length');
+        }
+        if (this.embedded) {
+            const manager = getSharedManager();
+            let extended = 0;
+            for (let index = 0; index < jobIds.length; index++) {
+                const success = await manager.extendLock(jobIds[index], tokens[index], duration);
+                if (success)
+                    extended++;
+            }
+            return extended;
+        }
+        if (!this.tcp)
+            return 0;
+        const response = await this.tcp.send({
+            cmd: 'ExtendLocks',
+            ids: jobIds,
+            tokens,
+            durations: jobIds.map(() => duration),
+        });
+        const extended = response.count;
+        return extended ?? 0;
+    }
+}

package/dist/client/worker/runtime/options.d.ts

@@ -0,0 +1,5 @@
+import type { WorkerOptions } from '../../types';
+import { TcpConnectionPool } from '../../tcpPool';
+import type { ExtendedWorkerOptions } from '../types';
+export declare function resolveWorkerOptions(options: WorkerOptions, embedded: boolean): ExtendedWorkerOptions;
+export declare function createTcpPool(options: WorkerOptions, concurrency: number): TcpConnectionPool;

package/dist/client/worker/runtime/options.js

@@ -0,0 +1,39 @@
+import { resolveToken } from '../../resolveToken';
+import { TcpConnectionPool } from '../../tcpPool';
+import { WORKER_CONSTANTS } from '../constants';
+export function resolveWorkerOptions(options, embedded) {
+    return {
+        concurrency: options.concurrency ?? 1,
+        autorun: options.autorun ?? true,
+        heartbeatInterval: options.heartbeatInterval ?? 10000,
+        batchSize: Math.min(options.batchSize ?? 10, 1000),
+        pollTimeout: Math.min(options.pollTimeout ?? 0, WORKER_CONSTANTS.MAX_POLL_TIMEOUT),
+        embedded,
+        useLocks: options.useLocks ?? true,
+        skipLockRenewal: options.skipLockRenewal ?? false,
+        skipStalledCheck: options.skipStalledCheck ?? false,
+        drainDelay: options.drainDelay ?? 50,
+        lockDuration: options.lockDuration ?? 30000,
+        maxStalledCount: options.maxStalledCount ?? 1,
+        removeOnComplete: options.removeOnComplete,
+        removeOnFail: options.removeOnFail,
+        connection: options.connection,
+    };
+}
+export function createTcpPool(options, concurrency) {
+    const connection = options.connection ?? {};
+    const poolSize = connection.poolSize ?? Math.min(concurrency, 8);
+    const token = resolveToken(connection.token);
+    return new TcpConnectionPool({
+        host: connection.host ?? 'localhost',
+        port: connection.port ?? 6789,
+        token,
+        tls: connection.tls,
+        poolSize,
+        pingInterval: connection.pingInterval,
+        commandTimeout: connection.commandTimeout,
+        maxCommandTimeouts: connection.maxCommandTimeouts,
+        pipelining: connection.pipelining,
+        maxInFlight: connection.maxInFlight,
+    });
+}

package/dist/client/worker/runtime/polling.d.ts

@@ -0,0 +1,12 @@
+import { type PullConfig } from '../workerPull';
+import { WorkerBuffer } from './buffer';
+import type { PulledJob, WorkerDelivery } from './state';
+export declare abstract class WorkerPolling<T = unknown, R = unknown> extends WorkerBuffer<T, R> {
+    protected poll(): void;
+    protected tryProcess(): Promise<void>;
+    protected doPullBatch(): Promise<PulledJob[]>;
+    private scheduleRateLimitPoll;
+    protected handlePullError(errorValue: unknown): void;
+    protected abstract startJob(delivery: WorkerDelivery): boolean;
+    protected abstract getPullConfig(): PullConfig;
+}