bunqueue 1.9.6 → 1.9.8

package/dist/application/backgroundTasks.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Background Tasks - Periodic maintenance operations
+ * Orchestrates stall detection, cleanup, recovery, DLQ maintenance
+ */
+import { processPendingDependencies } from './dependencyProcessor';
+import type { BackgroundContext } from './types';
+import type { CronScheduler } from '../infrastructure/scheduler/cronScheduler';
+/** Background task handles for cleanup */
+export interface BackgroundTaskHandles {
+    cleanupInterval: ReturnType<typeof setInterval>;
+    timeoutInterval: ReturnType<typeof setInterval>;
+    depCheckInterval: ReturnType<typeof setInterval>;
+    stallCheckInterval: ReturnType<typeof setInterval>;
+    dlqMaintenanceInterval: ReturnType<typeof setInterval>;
+    lockCheckInterval: ReturnType<typeof setInterval>;
+    cronScheduler: CronScheduler;
+}
+/**
+ * Start all background tasks
+ * Returns handles that can be used to stop tasks later
+ */
+export declare function startBackgroundTasks(ctx: BackgroundContext, cronScheduler: CronScheduler): BackgroundTaskHandles;
+/**
+ * Stop all background tasks
+ */
+export declare function stopBackgroundTasks(handles: BackgroundTaskHandles): void;
+export declare function recover(ctx: BackgroundContext): void;
+export { processPendingDependencies };
+//# sourceMappingURL=backgroundTasks.d.ts.map

package/dist/application/backgroundTasks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backgroundTasks.d.ts","sourceRoot":"","sources":["../../src/application/backgroundTasks.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAe,MAAM,SAAS,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2CAA2C,CAAC;AAE/E,0CAA0C;AAC1C,MAAM,WAAW,qBAAqB;IACpC,eAAe,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IAChD,eAAe,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IAChD,gBAAgB,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IACjD,kBAAkB,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IACnD,sBAAsB,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IACvD,iBAAiB,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IAClD,aAAa,EAAE,aAAa,CAAC;CAC9B;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,iBAAiB,EACtB,aAAa,EAAE,aAAa,GAC3B,qBAAqB,CA2CvB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,IAAI,CAQxE;AA8DD,wBAAgB,OAAO,CAAC,GAAG,EAAE,iBAAiB,GAAG,IAAI,CAmCpD;AAGD,OAAO,EAAE,0BAA0B,EAAE,CAAC"}

package/dist/application/backgroundTasks.js

@@ -0,0 +1,155 @@
+/**
+ * Background Tasks - Periodic maintenance operations
+ * Orchestrates stall detection, cleanup, recovery, DLQ maintenance
+ */
+import { queueLog } from '../shared/logger';
+import { shardIndex } from '../shared/hash';
+import * as dlqOps from './dlqManager';
+import { checkExpiredLocks } from './lockManager';
+import { cleanup } from './cleanupTasks';
+import { checkStalledJobs } from './stallDetection';
+import { processPendingDependencies } from './dependencyProcessor';
+/**
+ * Start all background tasks
+ * Returns handles that can be used to stop tasks later
+ */
+export function startBackgroundTasks(ctx, cronScheduler) {
+    const cleanupInterval = setInterval(() => {
+        cleanup(ctx).catch((err) => {
+            queueLog.error('Cleanup task failed', { error: String(err) });
+        });
+    }, ctx.config.cleanupIntervalMs);
+    const timeoutInterval = setInterval(() => {
+        checkJobTimeouts(ctx);
+    }, ctx.config.jobTimeoutCheckMs);
+    const depCheckInterval = setInterval(() => {
+        processPendingDependencies(ctx).catch((err) => {
+            queueLog.error('Dependency check failed', { error: String(err) });
+        });
+    }, ctx.config.dependencyCheckMs);
+    const stallCheckInterval = setInterval(() => {
+        checkStalledJobs(ctx);
+    }, ctx.config.stallCheckMs);
+    const dlqMaintenanceInterval = setInterval(() => {
+        performDlqMaintenance(ctx);
+    }, ctx.config.dlqMaintenanceMs);
+    // Lock expiration check runs at same interval as stall check
+    const lockCheckInterval = setInterval(() => {
+        checkExpiredLocks(getLockContext(ctx)).catch((err) => {
+            queueLog.error('Lock expiration check failed', { error: String(err) });
+        });
+    }, ctx.config.stallCheckMs);
+    cronScheduler.start();
+    return {
+        cleanupInterval,
+        timeoutInterval,
+        depCheckInterval,
+        stallCheckInterval,
+        dlqMaintenanceInterval,
+        lockCheckInterval,
+        cronScheduler,
+    };
+}
+/**
+ * Stop all background tasks
+ */
+export function stopBackgroundTasks(handles) {
+    clearInterval(handles.cleanupInterval);
+    clearInterval(handles.timeoutInterval);
+    clearInterval(handles.depCheckInterval);
+    clearInterval(handles.stallCheckInterval);
+    clearInterval(handles.dlqMaintenanceInterval);
+    clearInterval(handles.lockCheckInterval);
+    handles.cronScheduler.stop();
+}
+/** Extract lock context from background context */
+function getLockContext(ctx) {
+    return {
+        jobIndex: ctx.jobIndex,
+        jobLocks: ctx.jobLocks,
+        clientJobs: ctx.clientJobs,
+        processingShards: ctx.processingShards,
+        processingLocks: ctx.processingLocks,
+        shards: ctx.shards,
+        shardLocks: ctx.shardLocks,
+        eventsManager: ctx.eventsManager,
+    };
+}
+// ============ Job Timeouts ============
+function checkJobTimeouts(ctx) {
+    const now = Date.now();
+    for (const procShard of ctx.processingShards) {
+        for (const [jobId, job] of procShard) {
+            if (job.timeout && job.startedAt && now - job.startedAt > job.timeout) {
+                ctx.fail(jobId, 'Job timeout exceeded').catch((err) => {
+                    queueLog.error('Failed to mark timed out job as failed', {
+                        jobId: String(jobId),
+                        error: String(err),
+                    });
+                });
+            }
+        }
+    }
+}
+// ============ DLQ Maintenance ============
+function performDlqMaintenance(ctx) {
+    const dlqCtx = {
+        shards: ctx.shards,
+        jobIndex: ctx.jobIndex,
+        storage: ctx.storage,
+    };
+    for (const queueName of ctx.queueNamesCache) {
+        try {
+            const retried = dlqOps.processAutoRetry(queueName, dlqCtx);
+            if (retried > 0) {
+                queueLog.info('DLQ auto-retry completed', { queue: queueName, retried });
+            }
+            const purged = dlqOps.purgeExpiredDlq(queueName, dlqCtx);
+            if (purged > 0) {
+                queueLog.info('DLQ purge completed', { queue: queueName, purged });
+            }
+        }
+        catch (err) {
+            queueLog.error('DLQ maintenance failed', { queue: queueName, error: String(err) });
+        }
+    }
+}
+// ============ Recovery ============
+export function recover(ctx) {
+    if (!ctx.storage)
+        return;
+    const now = Date.now();
+    for (const job of ctx.storage.loadPendingJobs()) {
+        const idx = shardIndex(job.queue);
+        const shard = ctx.shards[idx];
+        shard.getQueue(job.queue).push(job);
+        ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: job.queue });
+        const isDelayed = job.runAt > now;
+        shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
+        ctx.registerQueueName(job.queue);
+    }
+    // Load DLQ entries
+    const dlqEntries = ctx.storage.loadDlq();
+    let dlqCount = 0;
+    for (const [queue, entries] of dlqEntries) {
+        const idx = shardIndex(queue);
+        const shard = ctx.shards[idx];
+        for (const entry of entries) {
+            let dlq = shard.dlq.get(queue);
+            if (!dlq) {
+                dlq = [];
+                shard.dlq.set(queue, dlq);
+            }
+            dlq.push(entry);
+            shard.incrementDlq();
+            dlqCount++;
+        }
+        ctx.registerQueueName(queue);
+    }
+    if (dlqCount > 0) {
+        queueLog.info('Loaded DLQ entries', { count: dlqCount });
+    }
+}
+// Re-export for backward compatibility
+export { processPendingDependencies };
+//# sourceMappingURL=backgroundTasks.js.map

package/dist/application/backgroundTasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backgroundTasks.js","sourceRoot":"","sources":["../../src/application/backgroundTasks.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,KAAK,MAAM,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAenE;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,GAAsB,EACtB,aAA4B;IAE5B,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;QACvC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YAClC,QAAQ,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IAEjC,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;QACvC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IAEjC,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,EAAE;QACxC,0BAA0B,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACrD,QAAQ,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IAEjC,MAAM,kBAAkB,GAAG,WAAW,CAAC,GAAG,EAAE;QAC1C,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAE5B,MAAM,sBAAsB,GAAG,WAAW,CAAC,GAAG,EAAE;QAC9C,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAEhC,6DAA6D;IAC7D,MAAM,iBAAiB,GAAG,WAAW,CAAC,GAAG,EAAE;QACzC,iBAAiB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YAC5D,QAAQ,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAE5B,aAAa,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO;QACL,eAAe;QACf,eAAe;QACf,gBAAgB;QAChB,kBAAkB;QAClB,sBAAsB;QACtB,iBAAiB;QACjB,aAAa;KACd,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAA8B;IAChE,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IACvC,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IACvC,aAAa,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACxC,aAAa,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1C,aAAa,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC9C,aAAa,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACzC,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED,mDAAmD;AACnD,SAAS,cAAc,CAAC,GAAsB;IAC5C,OAAO;QACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,eAAe,EAAE,GAAG,CAAC,eAAe;QACpC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,aAAa,EAAE,GAAG,CAAC,aAAa;KACjC,CAAC;AACJ,CAAC;AAED,yCAAyC;AAEzC,SAAS,gBAAgB,CAAC,GAAsB;IAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,SAAS,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC;gBACtE,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;oBAC7D,QAAQ,CAAC,KAAK,CAAC,wCAAwC,EAAE;wBACvD,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;wBACpB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;qBACnB,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,4CAA4C;AAE5C,SAAS,qBAAqB,CAAC,GAAsB;IACnD,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,OAAO,EAAE,GAAG,CAAC,OAAO;KACrB,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,eAAe,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3E,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACzD,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,QAAQ,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;AACH,CAAC;AAED,qCAAqC;AAErC,MAAM,UAAU,OAAO,CAAC,GAAsB;IAC5C,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO;IAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;QAChD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACjF,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAClC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9E,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,mBAAmB;IACnB,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,GAAG,GAAG,EAAE,CAAC;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC5B,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,QAAQ,EAAE,CAAC;QACb,CAAC;QACD,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,uCAAuC;AACvC,OAAO,EAAE,0BAA0B,EAAE,CAAC"}

package/dist/application/cleanupTasks.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Cleanup Tasks - Periodic maintenance and garbage collection
+ * Handles orphaned entries, stale data, and memory management
+ */
+import type { BackgroundContext } from './types';
+/**
+ * Main cleanup function - called periodically to maintain system health
+ * Cleans orphaned entries, stale data, and manages memory
+ */
+export declare function cleanup(ctx: BackgroundContext): Promise<void>;
+//# sourceMappingURL=cleanupTasks.d.ts.map

package/dist/application/cleanupTasks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanupTasks.d.ts","sourceRoot":"","sources":["../../src/application/cleanupTasks.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAEjD;;;GAGG;AACH,wBAAsB,OAAO,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBnE"}

package/dist/application/cleanupTasks.js

@@ -0,0 +1,216 @@
+/**
+ * Cleanup Tasks - Periodic maintenance and garbage collection
+ * Handles orphaned entries, stale data, and memory management
+ */
+import { queueLog } from '../shared/logger';
+import { processingShardIndex, SHARD_COUNT } from '../shared/hash';
+import { withWriteLock } from '../shared/lock';
+/**
+ * Main cleanup function - called periodically to maintain system health
+ * Cleans orphaned entries, stale data, and manages memory
+ */
+export async function cleanup(ctx) {
+    const now = Date.now();
+    const stallTimeout = 30 * 60 * 1000; // 30 minutes max for processing
+    // Refresh delayed counters
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        ctx.shards[i].refreshDelayedCount(now);
+    }
+    // Compact priority queues if stale ratio > 20%
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        for (const q of ctx.shards[i].queues.values()) {
+            if (q.needsCompaction(0.2)) {
+                q.compact();
+            }
+        }
+    }
+    await cleanOrphanedProcessingEntries(ctx, now, stallTimeout);
+    cleanStaleWaitingDependencies(ctx, now);
+    cleanUniqueKeysAndGroups(ctx);
+    cleanStalledCandidates(ctx);
+    await cleanOrphanedJobIndex(ctx);
+    cleanOrphanedJobLocks(ctx);
+    cleanEmptyQueues(ctx);
+}
+async function cleanOrphanedProcessingEntries(ctx, now, stallTimeout) {
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        // Phase 1: Collect candidates (read-only, no lock needed)
+        const orphaned = [];
+        for (const [jobId, job] of ctx.processingShards[i]) {
+            if (job.startedAt && now - job.startedAt > stallTimeout) {
+                orphaned.push(jobId);
+            }
+        }
+        if (orphaned.length === 0)
+            continue;
+        // Phase 2: Delete with lock to prevent race conditions
+        await withWriteLock(ctx.processingLocks[i], () => {
+            for (const jobId of orphaned) {
+                const job = ctx.processingShards[i].get(jobId);
+                if (job) {
+                    ctx.processingShards[i].delete(jobId);
+                    ctx.jobIndex.delete(jobId);
+                    queueLog.warn('Cleaned orphaned processing job', { jobId: String(jobId) });
+                }
+            }
+        });
+    }
+}
+function cleanStaleWaitingDependencies(ctx, now) {
+    const depTimeout = 60 * 60 * 1000; // 1 hour
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        const shard = ctx.shards[i];
+        const stale = [];
+        for (const [_id, job] of shard.waitingDeps) {
+            if (now - job.createdAt > depTimeout) {
+                stale.push(job);
+            }
+        }
+        for (const job of stale) {
+            shard.waitingDeps.delete(job.id);
+            shard.unregisterDependencies(job.id, job.dependsOn);
+            ctx.jobIndex.delete(job.id);
+            queueLog.warn('Cleaned stale waiting dependency', { jobId: String(job.id) });
+        }
+    }
+}
+function cleanUniqueKeysAndGroups(ctx) {
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        const shard = ctx.shards[i];
+        // Clean expired unique keys
+        const expiredCleaned = shard.cleanExpiredUniqueKeys();
+        if (expiredCleaned > 0) {
+            queueLog.info('Cleaned expired unique keys', { shard: i, removed: expiredCleaned });
+        }
+        // Trim if too many keys remain
+        for (const [queueName, keys] of shard.uniqueKeys) {
+            if (keys.size > 1000) {
+                const toRemove = Math.floor(keys.size / 2);
+                const iter = keys.keys();
+                for (let j = 0; j < toRemove; j++) {
+                    const { value, done } = iter.next();
+                    if (done)
+                        break;
+                    keys.delete(value);
+                }
+                queueLog.info('Trimmed unique keys', { queue: queueName, removed: toRemove });
+            }
+        }
+        // Clean orphaned active groups
+        for (const [queueName, groups] of shard.activeGroups) {
+            if (groups.size > 1000) {
+                const toRemove = Math.floor(groups.size / 2);
+                const iter = groups.values();
+                for (let j = 0; j < toRemove; j++) {
+                    const { value, done } = iter.next();
+                    if (done)
+                        break;
+                    groups.delete(value);
+                }
+                queueLog.info('Trimmed active groups', { queue: queueName, removed: toRemove });
+            }
+        }
+    }
+}
+function cleanStalledCandidates(ctx) {
+    for (const jobId of ctx.stalledCandidates) {
+        const loc = ctx.jobIndex.get(jobId);
+        if (loc?.type !== 'processing') {
+            ctx.stalledCandidates.delete(jobId);
+        }
+    }
+}
+async function cleanOrphanedJobIndex(ctx) {
+    // Expensive operation - only run when index is large
+    if (ctx.jobIndex.size <= 100_000)
+        return;
+    // Phase 1: Collect candidates grouped by shard (read-only)
+    const processingCandidates = new Map();
+    const queueCandidates = new Map();
+    for (const [jobId, loc] of ctx.jobIndex) {
+        if (loc.type === 'processing') {
+            const procIdx = processingShardIndex(String(jobId));
+            let list = processingCandidates.get(procIdx);
+            if (!list) {
+                list = [];
+                processingCandidates.set(procIdx, list);
+            }
+            list.push(jobId);
+        }
+        else if (loc.type === 'queue') {
+            let list = queueCandidates.get(loc.shardIdx);
+            if (!list) {
+                list = [];
+                queueCandidates.set(loc.shardIdx, list);
+            }
+            list.push({ jobId, queueName: loc.queueName });
+        }
+    }
+    let orphanedCount = 0;
+    // Phase 2: Check and delete processing entries with locks
+    for (const [procIdx, candidates] of processingCandidates) {
+        await withWriteLock(ctx.processingLocks[procIdx], () => {
+            for (const jobId of candidates) {
+                if (!ctx.processingShards[procIdx].has(jobId)) {
+                    ctx.jobIndex.delete(jobId);
+                    orphanedCount++;
+                }
+            }
+        });
+    }
+    // Phase 3: Check and delete queue entries with locks
+    for (const [shardIdx, candidates] of queueCandidates) {
+        await withWriteLock(ctx.shardLocks[shardIdx], () => {
+            const shard = ctx.shards[shardIdx];
+            for (const { jobId, queueName } of candidates) {
+                if (!shard.getQueue(queueName).has(jobId)) {
+                    ctx.jobIndex.delete(jobId);
+                    orphanedCount++;
+                }
+            }
+        });
+    }
+    if (orphanedCount > 0) {
+        queueLog.info('Cleaned orphaned jobIndex entries', { count: orphanedCount });
+    }
+}
+function cleanOrphanedJobLocks(ctx) {
+    for (const jobId of ctx.jobLocks.keys()) {
+        const loc = ctx.jobIndex.get(jobId);
+        if (loc?.type !== 'processing') {
+            ctx.jobLocks.delete(jobId);
+        }
+    }
+}
+function cleanEmptyQueues(ctx) {
+    for (let i = 0; i < SHARD_COUNT; i++) {
+        const shard = ctx.shards[i];
+        const emptyQueues = [];
+        for (const [queueName, queue] of shard.queues) {
+            const dlqEntries = shard.dlq.get(queueName);
+            if (queue.size === 0 && (!dlqEntries || dlqEntries.length === 0)) {
+                emptyQueues.push(queueName);
+            }
+        }
+        for (const queueName of emptyQueues) {
+            shard.queues.delete(queueName);
+            shard.dlq.delete(queueName);
+            shard.uniqueKeys.delete(queueName);
+            shard.queueState.delete(queueName);
+            shard.activeGroups.delete(queueName);
+            shard.clearQueueLimiters(queueName);
+            shard.stallConfig.delete(queueName);
+            shard.dlqConfig.delete(queueName);
+            ctx.unregisterQueueName(queueName);
+        }
+        if (emptyQueues.length > 0) {
+            queueLog.info('Removed empty queues', { shard: i, count: emptyQueues.length });
+        }
+        // Clean orphaned temporal index entries
+        const cleanedTemporal = shard.cleanOrphanedTemporalEntries();
+        if (cleanedTemporal > 0) {
+            queueLog.info('Cleaned orphaned temporal entries', { shard: i, count: cleanedTemporal });
+        }
+    }
+}
+//# sourceMappingURL=cleanupTasks.js.map

package/dist/application/cleanupTasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanupTasks.js","sourceRoot":"","sources":["../../src/application/cleanupTasks.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,GAAsB;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,gCAAgC;IAErE,2BAA2B;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,+CAA+C;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,8BAA8B,CAAC,GAAG,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;IAC7D,6BAA6B,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,wBAAwB,CAAC,GAAG,CAAC,CAAC;IAC9B,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACjC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC3B,gBAAgB,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,KAAK,UAAU,8BAA8B,CAC3C,GAAsB,EACtB,GAAW,EACX,YAAoB;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,0DAA0D;QAC1D,MAAM,QAAQ,GAAY,EAAE,CAAC;QAC7B,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,YAAY,EAAE,CAAC;gBACxD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEpC,uDAAuD;QACvD,MAAM,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE;YAC/C,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC/C,IAAI,GAAG,EAAE,CAAC;oBACR,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACtC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC3B,QAAQ,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,6BAA6B,CAAC,GAAsB,EAAE,GAAW;IACxE,MAAM,UAAU,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,KAAK,GAAU,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC;gBACrC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;YACpD,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,GAAsB;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,4BAA4B;QAC5B,MAAM,cAAc,GAAG,KAAK,CAAC,sBAAsB,EAAE,CAAC;QACtD,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,+BAA+B;QAC/B,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;oBAClC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;oBACpC,IAAI,IAAI;wBAAE,MAAM;oBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAChF,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,KAAK,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YACrD,IAAI,MAAM,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;gBACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;oBAClC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;oBACpC,IAAI,IAAI;wBAAE,MAAM;oBAChB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACvB,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,GAAsB;IACpD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;YAC/B,GAAG,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,GAAsB;IACzD,qDAAqD;IACrD,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO;QAAE,OAAO;IAEzC,2DAA2D;IAC3D,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAsD,CAAC;IAEtF,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACxC,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,oBAAoB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACpD,IAAI,IAAI,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC7C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,IAAI,GAAG,EAAE,CAAC;gBACV,oBAAoB,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,IAAI,GAAG,EAAE,CAAC;gBACV,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,0DAA0D;IAC1D,KAAK,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,oBAAoB,EAAE,CAAC;QACzD,MAAM,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE;YACrD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9C,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC3B,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,eAAe,EAAE,CAAC;QACrD,MAAM,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE;YACjD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACnC,KAAK,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,UAAU,EAAE,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC1C,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC3B,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAsB;IACnD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;YAC/B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAsB;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;gBACjE,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC/B,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5B,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACnC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACnC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACrC,KAAK,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YACpC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACpC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,GAAG,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,wCAAwC;QACxC,MAAM,eAAe,GAAG,KAAK,CAAC,4BAA4B,EAAE,CAAC;QAC7D,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,QAAQ,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/application/clientTracking.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Client Tracking - Client-job relationship management
+ * Handles job ownership and release on client disconnect
+ */
+import type { JobId } from '../domain/types/job';
+import type { LockContext } from './types';
+/**
+ * Register a job as owned by a client (called on PULL).
+ */
+export declare function registerClientJob(clientId: string, jobId: JobId, ctx: LockContext): void;
+/**
+ * Unregister a job from a client (called on ACK/FAIL).
+ */
+export declare function unregisterClientJob(clientId: string | undefined, jobId: JobId, ctx: LockContext): void;
+/**
+ * Release all jobs owned by a client back to queue (called on TCP disconnect).
+ * Returns the number of jobs released.
+ *
+ * Uses proper locking to prevent race conditions.
+ */
+export declare function releaseClientJobs(clientId: string, ctx: LockContext): Promise<number>;
+//# sourceMappingURL=clientTracking.d.ts.map

package/dist/application/clientTracking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clientTracking.d.ts","sourceRoot":"","sources":["../../src/application/clientTracking.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAO,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAItD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,GAAG,IAAI,CAOxF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,WAAW,GACf,IAAI,CASN;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAuE3F"}

package/dist/application/clientTracking.js

@@ -0,0 +1,122 @@
+/**
+ * Client Tracking - Client-job relationship management
+ * Handles job ownership and release on client disconnect
+ */
+import { queueLog } from '../shared/logger';
+import { shardIndex } from '../shared/hash';
+import { withWriteLock } from '../shared/lock';
+/**
+ * Register a job as owned by a client (called on PULL).
+ */
+export function registerClientJob(clientId, jobId, ctx) {
+    let jobs = ctx.clientJobs.get(clientId);
+    if (!jobs) {
+        jobs = new Set();
+        ctx.clientJobs.set(clientId, jobs);
+    }
+    jobs.add(jobId);
+}
+/**
+ * Unregister a job from a client (called on ACK/FAIL).
+ */
+export function unregisterClientJob(clientId, jobId, ctx) {
+    if (!clientId)
+        return;
+    const jobs = ctx.clientJobs.get(clientId);
+    if (jobs) {
+        jobs.delete(jobId);
+        if (jobs.size === 0) {
+            ctx.clientJobs.delete(clientId);
+        }
+    }
+}
+/**
+ * Release all jobs owned by a client back to queue (called on TCP disconnect).
+ * Returns the number of jobs released.
+ *
+ * Uses proper locking to prevent race conditions.
+ */
+export async function releaseClientJobs(clientId, ctx) {
+    const jobs = ctx.clientJobs.get(clientId);
+    if (!jobs || jobs.size === 0) {
+        ctx.clientJobs.delete(clientId);
+        return 0;
+    }
+    // Phase 1: Collect jobs to release (read-only, no locks needed)
+    const jobsToRelease = [];
+    for (const jobId of jobs) {
+        const loc = ctx.jobIndex.get(jobId);
+        if (loc?.type !== 'processing')
+            continue;
+        const procIdx = loc.shardIdx;
+        const job = ctx.processingShards[procIdx].get(jobId);
+        if (!job)
+            continue;
+        jobsToRelease.push({
+            jobId,
+            procIdx,
+            queueShardIdx: shardIndex(job.queue),
+        });
+    }
+    if (jobsToRelease.length === 0) {
+        ctx.clientJobs.delete(clientId);
+        return 0;
+    }
+    // Phase 2: Group by processing shard for efficient locking
+    const byProcShard = new Map();
+    for (const item of jobsToRelease) {
+        let list = byProcShard.get(item.procIdx);
+        if (!list) {
+            list = [];
+            byProcShard.set(item.procIdx, list);
+        }
+        list.push(item);
+    }
+    let released = 0;
+    const now = Date.now();
+    // Phase 3: Process each processing shard with proper locking
+    for (const [procIdx, items] of byProcShard) {
+        await withWriteLock(ctx.processingLocks[procIdx], async () => {
+            for (const { jobId, queueShardIdx } of items) {
+                const job = ctx.processingShards[procIdx].get(jobId);
+                if (!job)
+                    continue;
+                // Acquire shard lock for queue modifications
+                await withWriteLock(ctx.shardLocks[queueShardIdx], () => {
+                    released += releaseJobToQueue({ jobId, job, procIdx, queueShardIdx, ctx, now });
+                });
+            }
+        });
+    }
+    // Clear client tracking
+    ctx.clientJobs.delete(clientId);
+    if (released > 0) {
+        queueLog.info('Released client jobs', { clientId: clientId.substring(0, 8), released });
+    }
+    return released;
+}
+/** Release a single job back to queue */
+function releaseJobToQueue(opts) {
+    const { jobId, job, procIdx, queueShardIdx, ctx, now } = opts;
+    const shard = ctx.shards[queueShardIdx];
+    // Remove from processing
+    ctx.processingShards[procIdx].delete(jobId);
+    // Release lock if exists
+    ctx.jobLocks.delete(jobId);
+    // Release concurrency
+    shard.releaseConcurrency(job.queue);
+    // Release group if active
+    if (job.groupId) {
+        shard.releaseGroup(job.queue, job.groupId);
+    }
+    // Reset job state for retry
+    job.startedAt = null;
+    job.lastHeartbeat = now;
+    // Re-queue the job
+    shard.getQueue(job.queue).push(job);
+    const isDelayed = job.runAt > now;
+    shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
+    ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: queueShardIdx, queueName: job.queue });
+    return 1;
+}
+//# sourceMappingURL=clientTracking.js.map

package/dist/application/clientTracking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clientTracking.js","sourceRoot":"","sources":["../../src/application/clientTracking.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,KAAY,EAAE,GAAgB;IAChF,IAAI,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACjB,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAA4B,EAC5B,KAAY,EACZ,GAAgB;IAEhB,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,IAAI,EAAE,CAAC;QACT,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACpB,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB,EAAE,GAAgB;IACxE,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,gEAAgE;IAChE,MAAM,aAAa,GAId,EAAE,CAAC;IAER,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY;YAAE,SAAS;QAEzC,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,aAAa,CAAC,IAAI,CAAC;YACjB,KAAK;YACL,OAAO;YACP,aAAa,EAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;SACrC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,2DAA2D;IAC3D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC5D,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,GAAG,EAAE,CAAC;YACV,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,6DAA6D;IAC7D,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QAC3C,MAAM,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,KAAK,IAAI,EAAE;YAC3D,KAAK,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,KAAK,EAAE,CAAC;gBAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACrD,IAAI,CAAC,GAAG;oBAAE,SAAS;gBAEnB,6CAA6C;gBAC7C,MAAM,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE;oBACtD,QAAQ,IAAI,iBAAiB,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;gBAClF,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEhC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAYD,yCAAyC;AACzC,SAAS,iBAAiB,CAAC,IAAuB;IAChD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC9D,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAExC,yBAAyB;IACzB,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE5C,yBAAyB;IACzB,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE3B,sBAAsB;IACtB,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAEpC,0BAA0B;IAC1B,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAChB,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,4BAA4B;IAC5B,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;IACrB,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC;IAExB,mBAAmB;IACnB,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;IAClC,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7E,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IAE1F,OAAO,CAAC,CAAC;AACX,CAAC"}