@open-mercato/queue 0.6.7-develop.6744.1.0483b18565 → 0.6.7-develop.6750.1.14497517d0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/factory.js +4 -1
- package/dist/factory.js.map +2 -2
- package/dist/strategies/async.js +12 -2
- package/dist/strategies/async.js.map +2 -2
- package/dist/worker/runner.js +5 -1
- package/dist/worker/runner.js.map +2 -2
- package/package.json +2 -2
- package/src/__tests__/async.strategy.test.ts +32 -0
- package/src/factory.ts +4 -1
- package/src/strategies/async.ts +17 -2
- package/src/types.ts +14 -0
- package/src/worker/runner.ts +8 -0
package/dist/factory.js
CHANGED
|
@@ -15,7 +15,10 @@ function createModuleQueue(name, options) {
|
|
|
15
15
|
if (strategy === "async") {
|
|
16
16
|
return createAsyncQueue(name, {
|
|
17
17
|
connection: { url: getRedisUrlOrThrow("QUEUE") },
|
|
18
|
-
concurrency: options?.concurrency
|
|
18
|
+
concurrency: options?.concurrency,
|
|
19
|
+
attempts: options?.attempts,
|
|
20
|
+
lockDuration: options?.lockDuration,
|
|
21
|
+
maxStalledCount: options?.maxStalledCount
|
|
19
22
|
});
|
|
20
23
|
}
|
|
21
24
|
return createLocalQueue(name, { concurrency: options?.concurrency });
|
package/dist/factory.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/factory.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Queue, LocalQueueOptions, AsyncQueueOptions, QueueStrategyType } from './types'\nimport { createLocalQueue } from './strategies/local'\nimport { createAsyncQueue } from './strategies/async'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\n\n/**\n * Creates a queue instance with the specified strategy.\n *\n * @template T - The payload type for jobs in this queue\n * @param name - Unique name for the queue\n * @param strategy - Queue strategy: 'local' for file-based, 'async' for BullMQ\n * @param options - Strategy-specific options\n * @returns A Queue instance\n *\n * @example\n * ```typescript\n * // Local file-based queue\n * const localQueue = createQueue<MyJobData>('my-queue', 'local')\n *\n * // BullMQ-based queue\n * const asyncQueue = createQueue<MyJobData>('my-queue', 'async', {\n * connection: { url: 'redis://localhost:6379' },\n * concurrency: 5\n * })\n * ```\n */\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local',\n options?: LocalQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'async',\n options?: AsyncQueueOptions\n): Queue<T>\n\n// General overload for dynamic strategy (union type)\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T> {\n if (strategy === 'async') {\n return createAsyncQueue<T>(name, options as AsyncQueueOptions)\n }\n\n return createLocalQueue<T>(name, options as LocalQueueOptions)\n}\n\n/**\n * Resolve the queue strategy from `QUEUE_STRATEGY`. Defaults to `'local'`.\n */\nexport function resolveQueueStrategy(): QueueStrategyType {\n return process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local'\n}\n\n/**\n * Create a module-owned queue using the strategy declared in `QUEUE_STRATEGY`.\n *\n * - When `QUEUE_STRATEGY=async`, builds a BullMQ queue and resolves the\n * Redis URL via `getRedisUrlOrThrow('QUEUE')` so missing config fails loudly.\n * - Otherwise builds a local file-based queue.\n *\n * Replaces the boilerplate `process.env.QUEUE_STRATEGY === 'async' ? ... : ...`\n * pattern that every module queue helper used to repeat. Concurrency applies\n * to both strategies so the same number means the same thing in dev and prod.\n *\n * @example\n * ```typescript\n * export function getDataSyncQueue(name: string) {\n * return createModuleQueue<MyJob>(name, { concurrency: 5 })\n * }\n * ```\n */\nexport function createModuleQueue<T = unknown>(\n name: string,\n options?:
|
|
5
|
-
"mappings": "AACA,SAAS,wBAAwB;AACjC,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AA0C5B,SAAS,YACd,MACA,UACA,SACU;AACV,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM,OAA4B;AAAA,EAC/D;AAEA,SAAO,iBAAoB,MAAM,OAA4B;AAC/D;AAKO,SAAS,uBAA0C;AACxD,SAAO,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAC5D;AAoBO,SAAS,kBACd,MACA,SACU;AACV,QAAM,WAAW,qBAAqB;AACtC,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM;AAAA,MAC/B,YAAY,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAAA,MAC/C,aAAa,SAAS;AAAA,
|
|
4
|
+
"sourcesContent": ["import type { Queue, LocalQueueOptions, AsyncQueueOptions, QueueStrategyType } from './types'\nimport { createLocalQueue } from './strategies/local'\nimport { createAsyncQueue } from './strategies/async'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\n\n/**\n * Creates a queue instance with the specified strategy.\n *\n * @template T - The payload type for jobs in this queue\n * @param name - Unique name for the queue\n * @param strategy - Queue strategy: 'local' for file-based, 'async' for BullMQ\n * @param options - Strategy-specific options\n * @returns A Queue instance\n *\n * @example\n * ```typescript\n * // Local file-based queue\n * const localQueue = createQueue<MyJobData>('my-queue', 'local')\n *\n * // BullMQ-based queue\n * const asyncQueue = createQueue<MyJobData>('my-queue', 'async', {\n * connection: { url: 'redis://localhost:6379' },\n * concurrency: 5\n * })\n * ```\n */\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local',\n options?: LocalQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'async',\n options?: AsyncQueueOptions\n): Queue<T>\n\n// General overload for dynamic strategy (union type)\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T> {\n if (strategy === 'async') {\n return createAsyncQueue<T>(name, options as AsyncQueueOptions)\n }\n\n return createLocalQueue<T>(name, options as LocalQueueOptions)\n}\n\n/**\n * Resolve the queue strategy from `QUEUE_STRATEGY`. Defaults to `'local'`.\n */\nexport function resolveQueueStrategy(): QueueStrategyType {\n return process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local'\n}\n\n/**\n * Create a module-owned queue using the strategy declared in `QUEUE_STRATEGY`.\n *\n * - When `QUEUE_STRATEGY=async`, builds a BullMQ queue and resolves the\n * Redis URL via `getRedisUrlOrThrow('QUEUE')` so missing config fails loudly.\n * - Otherwise builds a local file-based queue.\n *\n * Replaces the boilerplate `process.env.QUEUE_STRATEGY === 'async' ? ... : ...`\n * pattern that every module queue helper used to repeat. Concurrency applies\n * to both strategies so the same number means the same thing in dev and prod.\n *\n * @example\n * ```typescript\n * export function getDataSyncQueue(name: string) {\n * return createModuleQueue<MyJob>(name, { concurrency: 5 })\n * }\n * ```\n */\nexport function createModuleQueue<T = unknown>(\n name: string,\n options?: Pick<AsyncQueueOptions, 'attempts' | 'concurrency' | 'lockDuration' | 'maxStalledCount'>,\n): Queue<T> {\n const strategy = resolveQueueStrategy()\n if (strategy === 'async') {\n return createAsyncQueue<T>(name, {\n connection: { url: getRedisUrlOrThrow('QUEUE') },\n concurrency: options?.concurrency,\n attempts: options?.attempts,\n lockDuration: options?.lockDuration,\n maxStalledCount: options?.maxStalledCount,\n })\n }\n return createLocalQueue<T>(name, { concurrency: options?.concurrency })\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,wBAAwB;AACjC,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AA0C5B,SAAS,YACd,MACA,UACA,SACU;AACV,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM,OAA4B;AAAA,EAC/D;AAEA,SAAO,iBAAoB,MAAM,OAA4B;AAC/D;AAKO,SAAS,uBAA0C;AACxD,SAAO,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAC5D;AAoBO,SAAS,kBACd,MACA,SACU;AACV,QAAM,WAAW,qBAAqB;AACtC,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM;AAAA,MAC/B,YAAY,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAAA,MAC/C,aAAa,SAAS;AAAA,MACtB,UAAU,SAAS;AAAA,MACnB,cAAc,SAAS;AAAA,MACvB,iBAAiB,SAAS;AAAA,IAC5B,CAAC;AAAA,EACH;AACA,SAAO,iBAAoB,MAAM,EAAE,aAAa,SAAS,YAAY,CAAC;AACxE;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/strategies/async.js
CHANGED
|
@@ -33,6 +33,9 @@ function resolveConnection(options) {
|
|
|
33
33
|
function createAsyncQueue(name, options) {
|
|
34
34
|
const connection = resolveConnection(options?.connection);
|
|
35
35
|
const concurrency = options?.concurrency ?? 1;
|
|
36
|
+
const attempts = options?.attempts ?? 3;
|
|
37
|
+
const lockDuration = options?.lockDuration;
|
|
38
|
+
const maxStalledCount = options?.maxStalledCount;
|
|
36
39
|
const logger = packageLogger.child({ queue: name });
|
|
37
40
|
let bullQueue = null;
|
|
38
41
|
let bullWorker = null;
|
|
@@ -67,7 +70,7 @@ function createAsyncQueue(name, options) {
|
|
|
67
70
|
delay: options2?.delayMs && options2.delayMs > 0 ? options2.delayMs : void 0,
|
|
68
71
|
removeOnComplete: true,
|
|
69
72
|
removeOnFail: 1e3,
|
|
70
|
-
attempts
|
|
73
|
+
attempts,
|
|
71
74
|
backoff: { type: "exponential", delay: 1e3 }
|
|
72
75
|
});
|
|
73
76
|
return job.id ?? jobData.id;
|
|
@@ -86,7 +89,9 @@ function createAsyncQueue(name, options) {
|
|
|
86
89
|
},
|
|
87
90
|
{
|
|
88
91
|
connection,
|
|
89
|
-
concurrency
|
|
92
|
+
concurrency,
|
|
93
|
+
...lockDuration !== void 0 ? { lockDuration } : {},
|
|
94
|
+
...maxStalledCount !== void 0 ? { maxStalledCount } : {}
|
|
90
95
|
}
|
|
91
96
|
);
|
|
92
97
|
bullWorker.on("completed", (job) => {
|
|
@@ -98,6 +103,11 @@ function createAsyncQueue(name, options) {
|
|
|
98
103
|
const error = err;
|
|
99
104
|
logger.error("Job failed", { jobId: jobWithId?.id, err: error });
|
|
100
105
|
});
|
|
106
|
+
bullWorker.on("stalled", (jobId) => {
|
|
107
|
+
logger.warn("Job stalled and will be redelivered \u2014 the handler may run concurrently with a previous delivery", {
|
|
108
|
+
jobId: typeof jobId === "string" ? jobId : null
|
|
109
|
+
});
|
|
110
|
+
});
|
|
101
111
|
bullWorker.on("error", (err) => {
|
|
102
112
|
const error = err;
|
|
103
113
|
logger.error("Worker error", { err: error });
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/async.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst packageLogger = createLogger('queue')\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n url?: string\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n data?: T\n remove: () => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: { connection: ConnectionOptions; concurrency: number }\n ) => BullWorkerInterface\n}\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects an ioredis-compatible connection object. Preserve the full\n * Redis URL under the `url` key so rediss://, username, database, and query\n * params are not lost in translation.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return { url: options.url }\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n }\n }\n\n return { url: getRedisUrlOrThrow('QUEUE') }\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n const logger = packageLogger.child({ queue: name })\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts
|
|
5
|
-
"mappings": "AACA,SAAS,0BAA0B;AACnC,SAAS,oBAAoB;AAE7B,MAAM,gBAAgB,aAAa,OAAO;AAiD1C,MAAM,uBAAuB,CAAC,WAAW,WAAW,eAAe,UAAU,kBAAkB;AAE/F,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AASA,SAAS,kBAAkB,SAA8D;AACvF,MAAI,SAAS,KAAK;AAChB,WAAO,EAAE,KAAK,QAAQ,IAAI;AAAA,EAC5B;AAEA,MAAI,SAAS,MAAM;AACjB,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ;AAAA,MACtB,UAAU,QAAQ;AAAA,MAClB,UAAU,QAAQ;AAAA,MAClB,IAAI,QAAQ;AAAA,MACZ,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAEA,SAAO,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAC5C;AAeO,SAAS,iBACd,MACA,SACU;AACV,QAAM,aAAa,kBAAkB,SAAS,UAAU;AACxD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,MAAI,YAAqD;AACzD,MAAI,aAAyC;AAC7C,MAAI,eAAoC;AAMxC,iBAAe,YAAmC;AAChD,QAAI,CAAC,cAAc;AACjB,UAAI;AACF,uBAAe,MAAM,OAAO,QAAQ;AAAA,MACtC,QAAQ;AACN,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,WAAsD;AACnE,QAAI,CAAC,WAAW;AACd,YAAM,EAAE,OAAO,eAAe,IAAI,MAAM,UAAU;AAClD,kBAAY,IAAI,eAA6B,MAAM,EAAE,WAAW,CAAC;AAAA,IACnE;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,UAAwB;AAAA,MAC5B,IAAI,OAAO,WAAW;AAAA,MACtB,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IACpC;AAEA,UAAM,MAAM,MAAM,MAAM,IAAI,QAAQ,IAAI,SAAS;AAAA,MAC/C,OAAOA,UAAS,WAAWA,SAAQ,UAAU,IAAIA,SAAQ,UAAU;AAAA,MACnE,kBAAkB;AAAA,MAClB,cAAc;AAAA,MACd
|
|
4
|
+
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst packageLogger = createLogger('queue')\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n url?: string\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n data?: T\n remove: () => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: { connection: ConnectionOptions; concurrency: number; lockDuration?: number; maxStalledCount?: number }\n ) => BullWorkerInterface\n}\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects an ioredis-compatible connection object. Preserve the full\n * Redis URL under the `url` key so rediss://, username, database, and query\n * params are not lost in translation.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return { url: options.url }\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n }\n }\n\n return { url: getRedisUrlOrThrow('QUEUE') }\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n const attempts = options?.attempts ?? 3\n const lockDuration = options?.lockDuration\n const maxStalledCount = options?.maxStalledCount\n const logger = packageLogger.child({ queue: name })\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n await handler(jobData, {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n })\n },\n {\n connection,\n concurrency,\n ...(lockDuration !== undefined ? { lockDuration } : {}),\n ...(maxStalledCount !== undefined ? { maxStalledCount } : {}),\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n logger.info('Job completed', { jobId: jobWithId.id })\n })\n\n bullWorker.on('failed', (job, err) => {\n const jobWithId = job as { id?: string } | undefined\n const error = err as Error\n logger.error('Job failed', { jobId: jobWithId?.id, err: error })\n })\n\n // A stalled job is redelivered under the same id while the previous worker\n // may still be running it, so this is the signal that a handler is about to\n // be executed twice. BullMQ's docs require surfacing it: without this line\n // duplicate processing is invisible.\n bullWorker.on('stalled', (jobId) => {\n logger.warn('Job stalled and will be redelivered \u2014 the handler may run concurrently with a previous delivery', {\n jobId: typeof jobId === 'string' ? jobId : null,\n })\n })\n\n bullWorker.on('error', (err) => {\n const error = err as Error\n logger.error('Worker error', { err: error })\n })\n\n logger.info('Worker started', { concurrency })\n\n // For async strategy, return a sentinel result indicating worker mode\n // processed=-1 signals that this is a continuous worker, not a batch process\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const queue = await getQueue()\n\n // Obliterate removes all jobs from the queue\n await queue.obliterate({ force: true })\n\n return { removed: -1 } // BullMQ obliterate doesn't return count\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n const queue = await getQueue()\n const jobs = await queue.getJobs(REMOVABLE_JOB_STATES, 0, -1)\n let removed = 0\n\n for (const job of jobs) {\n if (!payloadMatchesScope(job.data?.payload, scope)) continue\n try {\n await job.remove()\n removed++\n } catch {\n // The job may have started between enumeration and removal. In-flight\n // cancellation is handled by the caller's lock/heartbeat contract.\n }\n }\n\n return { removed }\n }\n\n async function close(): Promise<void> {\n if (bullWorker) {\n await bullWorker.close()\n bullWorker = null\n }\n if (bullQueue) {\n await bullQueue.close()\n bullQueue = null\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const queue = await getQueue()\n const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')\n return {\n waiting: counts.waiting ?? 0,\n active: counts.active ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'async',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,0BAA0B;AACnC,SAAS,oBAAoB;AAE7B,MAAM,gBAAgB,aAAa,OAAO;AAiD1C,MAAM,uBAAuB,CAAC,WAAW,WAAW,eAAe,UAAU,kBAAkB;AAE/F,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AASA,SAAS,kBAAkB,SAA8D;AACvF,MAAI,SAAS,KAAK;AAChB,WAAO,EAAE,KAAK,QAAQ,IAAI;AAAA,EAC5B;AAEA,MAAI,SAAS,MAAM;AACjB,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ;AAAA,MACtB,UAAU,QAAQ;AAAA,MAClB,UAAU,QAAQ;AAAA,MAClB,IAAI,QAAQ;AAAA,MACZ,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAEA,SAAO,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAC5C;AAeO,SAAS,iBACd,MACA,SACU;AACV,QAAM,aAAa,kBAAkB,SAAS,UAAU;AACxD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,eAAe,SAAS;AAC9B,QAAM,kBAAkB,SAAS;AACjC,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,MAAI,YAAqD;AACzD,MAAI,aAAyC;AAC7C,MAAI,eAAoC;AAMxC,iBAAe,YAAmC;AAChD,QAAI,CAAC,cAAc;AACjB,UAAI;AACF,uBAAe,MAAM,OAAO,QAAQ;AAAA,MACtC,QAAQ;AACN,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,WAAsD;AACnE,QAAI,CAAC,WAAW;AACd,YAAM,EAAE,OAAO,eAAe,IAAI,MAAM,UAAU;AAClD,kBAAY,IAAI,eAA6B,MAAM,EAAE,WAAW,CAAC;AAAA,IACnE;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,UAAwB;AAAA,MAC5B,IAAI,OAAO,WAAW;AAAA,MACtB,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IACpC;AAEA,UAAM,MAAM,MAAM,MAAM,IAAI,QAAQ,IAAI,SAAS;AAAA,MAC/C,OAAOA,UAAS,WAAWA,SAAQ,UAAU,IAAIA,SAAQ,UAAU;AAAA,MACnE,kBAAkB;AAAA,MAClB,cAAc;AAAA,MACd;AAAA,MACA,SAAS,EAAE,MAAM,eAAe,OAAO,IAAK;AAAA,IAC9C,CAAC;AAED,WAAO,IAAI,MAAM,QAAQ;AAAA,EAC3B;AAEA,iBAAe,QAAQ,SAAgD;AACrE,UAAM,EAAE,OAAO,IAAI,MAAM,UAAU;AAGnC,iBAAa,IAAI;AAAA,MACf;AAAA,MACA,OAAO,QAAQ;AACb,cAAM,UAAU,IAAI;AACpB,cAAM,QAAQ,SAAS;AAAA,UACrB,OAAO,IAAI,MAAM,QAAQ;AAAA,UACzB,eAAe,IAAI,eAAe;AAAA,UAClC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA,GAAI,iBAAiB,SAAY,EAAE,aAAa,IAAI,CAAC;AAAA,QACrD,GAAI,oBAAoB,SAAY,EAAE,gBAAgB,IAAI,CAAC;AAAA,MAC7D;AAAA,IACF;AAGA,eAAW,GAAG,aAAa,CAAC,QAAQ;AAClC,YAAM,YAAY;AAClB,aAAO,KAAK,iBAAiB,EAAE,OAAO,UAAU,GAAG,CAAC;AAAA,IACtD,CAAC;AAED,eAAW,GAAG,UAAU,CAAC,KAAK,QAAQ;AACpC,YAAM,YAAY;AAClB,YAAM,QAAQ;AACd,aAAO,MAAM,cAAc,EAAE,OAAO,WAAW,IAAI,KAAK,MAAM,CAAC;AAAA,IACjE,CAAC;AAMD,eAAW,GAAG,WAAW,CAAC,UAAU;AAClC,aAAO,KAAK,wGAAmG;AAAA,QAC7G,OAAO,OAAO,UAAU,WAAW,QAAQ;AAAA,MAC7C,CAAC;AAAA,IACH,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,QAAQ;AAC9B,YAAM,QAAQ;AACd,aAAO,MAAM,gBAAgB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC7C,CAAC;AAED,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAI7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,UAAM,QAAQ,MAAM,SAAS;AAG7B,UAAM,MAAM,WAAW,EAAE,OAAO,KAAK,CAAC;AAEtC,WAAO,EAAE,SAAS,GAAG;AAAA,EACvB;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,OAAO,MAAM,MAAM,QAAQ,sBAAsB,GAAG,EAAE;AAC5D,QAAI,UAAU;AAEd,eAAW,OAAO,MAAM;AACtB,UAAI,CAAC,oBAAoB,IAAI,MAAM,SAAS,KAAK,EAAG;AACpD,UAAI;AACF,cAAM,IAAI,OAAO;AACjB;AAAA,MACF,QAAQ;AAAA,MAGR;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ;AAAA,EACnB;AAEA,iBAAe,QAAuB;AACpC,QAAI,YAAY;AACd,YAAM,WAAW,MAAM;AACvB,mBAAa;AAAA,IACf;AACA,QAAI,WAAW;AACb,YAAM,UAAU,MAAM;AACtB,kBAAY;AAAA,IACd;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,SAAS,MAAM,MAAM,aAAa,WAAW,UAAU,aAAa,QAAQ;AAClF,WAAO;AAAA,MACL,SAAS,OAAO,WAAW;AAAA,MAC3B,QAAQ,OAAO,UAAU;AAAA,MACzB,WAAW,OAAO,aAAa;AAAA,MAC/B,QAAQ,OAAO,UAAU;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/dist/worker/runner.js
CHANGED
|
@@ -62,6 +62,8 @@ async function runWorker(options) {
|
|
|
62
62
|
handler,
|
|
63
63
|
connection,
|
|
64
64
|
concurrency = 1,
|
|
65
|
+
lockDuration,
|
|
66
|
+
maxStalledCount,
|
|
65
67
|
gracefulShutdown = true,
|
|
66
68
|
background = false,
|
|
67
69
|
strategy: strategyOption
|
|
@@ -70,7 +72,9 @@ async function runWorker(options) {
|
|
|
70
72
|
logger.info("Starting worker for queue", { queueName, strategy });
|
|
71
73
|
const queue = createQueue(queueName, strategy, {
|
|
72
74
|
connection,
|
|
73
|
-
concurrency
|
|
75
|
+
concurrency,
|
|
76
|
+
lockDuration,
|
|
77
|
+
maxStalledCount
|
|
74
78
|
});
|
|
75
79
|
if (gracefulShutdown) {
|
|
76
80
|
managedQueues.add(queue);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/worker/runner.ts"],
|
|
4
|
-
"sourcesContent": ["import { createQueue } from '../factory'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, JobHandler, AsyncQueueOptions, QueueStrategyType } from '../types'\n\nconst logger = createLogger('queue').child({ component: 'worker' })\n\n/**\n * Options for running a queue worker.\n */\nexport type WorkerRunnerOptions<T = unknown> = {\n /** Name of the queue to process */\n queueName: string\n /** Handler function to process each job */\n handler: JobHandler<T>\n /** Redis connection options (only used for async strategy) */\n connection?: AsyncQueueOptions['connection']\n /** Number of concurrent jobs to process */\n concurrency?: number\n /** Whether to set up graceful shutdown handlers */\n gracefulShutdown?: boolean\n /** If true, don't block - return immediately after starting processing (for multi-queue mode) */\n background?: boolean\n /** Queue strategy to use. Defaults to QUEUE_STRATEGY env var or 'local' */\n strategy?: QueueStrategyType\n}\n\nconst managedQueues = new Set<Queue<unknown>>()\nconst managedShutdownHooks = new Set<() => Promise<void> | void>()\nlet shutdownHandlersRegistered = false\nlet shutdownInProgress = false\n\nfunction unregisterShutdownHandlers(sigtermHandler: () => void, sigintHandler: () => void): void {\n process.off('SIGTERM', sigtermHandler)\n process.off('SIGINT', sigintHandler)\n shutdownHandlersRegistered = false\n}\n\nfunction registerShutdownHandlers(): void {\n if (shutdownHandlersRegistered) return\n\n const shutdown = async (signal: string) => {\n if (shutdownInProgress) return\n shutdownInProgress = true\n\n logger.info('Received signal, shutting down gracefully', { signal })\n\n let hasError = false\n for (const queue of managedQueues) {\n try {\n await queue.close()\n } catch (error) {\n hasError = true\n logger.error('Error during shutdown', { err: error })\n }\n }\n\n managedQueues.clear()\n for (const hook of managedShutdownHooks) {\n try {\n await hook()\n } catch (error) {\n hasError = true\n logger.error('Error during shutdown hook', { err: error })\n }\n }\n managedShutdownHooks.clear()\n unregisterShutdownHandlers(sigtermHandler, sigintHandler)\n shutdownInProgress = false\n\n if (!hasError) {\n logger.info('Worker closed successfully')\n }\n\n process.exit(hasError ? 1 : 0)\n }\n\n const sigtermHandler = () => {\n void shutdown('SIGTERM')\n }\n\n const sigintHandler = () => {\n void shutdown('SIGINT')\n }\n\n process.on('SIGTERM', sigtermHandler)\n process.on('SIGINT', sigintHandler)\n shutdownHandlersRegistered = true\n}\n\n/**\n * Register a process-local service that must stop before a worker exits.\n * The returned callback removes the hook when the service is stopped early.\n */\nexport function registerWorkerShutdownHook(hook: () => Promise<void> | void): () => void {\n managedShutdownHooks.add(hook)\n return () => managedShutdownHooks.delete(hook)\n}\n\n/**\n * Runs a queue worker that processes jobs continuously.\n *\n * This function:\n * 1. Creates an async queue instance\n * 2. Starts a BullMQ worker\n * 3. Sets up graceful shutdown on SIGTERM/SIGINT\n * 4. Keeps the process running until shutdown\n *\n * @template T - The job payload type\n * @param options - Worker configuration\n *\n * @example\n * ```typescript\n * import { runWorker } from '@open-mercato/queue/worker'\n *\n * await runWorker({\n * queueName: 'events',\n * handler: async (job, ctx) => {\n * console.log(`Processing ${ctx.jobId}:`, job.payload)\n * },\n * connection: { url: process.env.REDIS_URL },\n * concurrency: 5,\n * })\n * ```\n */\nexport async function runWorker<T = unknown>(\n options: WorkerRunnerOptions<T>\n): Promise<void> {\n const {\n queueName,\n handler,\n connection,\n concurrency = 1,\n gracefulShutdown = true,\n background = false,\n strategy: strategyOption,\n } = options\n\n // Determine queue strategy from option, env var, or default to 'local'\n const strategy: QueueStrategyType = strategyOption\n ?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')\n\n logger.info('Starting worker for queue', { queueName, strategy })\n\n const queue = createQueue<T>(queueName, strategy, {\n connection,\n concurrency,\n })\n\n // Set up graceful shutdown\n if (gracefulShutdown) {\n managedQueues.add(queue as Queue<unknown>)\n registerShutdownHandlers()\n }\n\n // Start processing\n await queue.process(handler)\n\n logger.info('Worker running', { concurrency })\n\n if (background) {\n // Return immediately for multi-queue mode\n return\n }\n\n logger.info('Press Ctrl+C to stop')\n\n // Keep the process alive (single-queue mode)\n await new Promise(() => {\n // This promise never resolves, keeping the worker running\n })\n}\n\n/**\n * Creates a worker handler that routes jobs to specific handlers based on job type.\n *\n * @template T - Base job payload type (must include a 'type' field)\n * @param handlers - Map of job types to their handlers\n *\n * @example\n * ```typescript\n * const handler = createRoutedHandler({\n * 'user.created': async (job) => { ... },\n * 'order.placed': async (job) => { ... },\n * })\n *\n * await runWorker({ queueName: 'events', handler })\n * ```\n */\nexport function createRoutedHandler<T extends { type: string }>(\n handlers: Record<string, JobHandler<T>>\n): JobHandler<T> {\n return async (job, ctx) => {\n const type = job.payload.type\n const handler = handlers[type]\n\n if (!handler) {\n logger.warn('No handler registered for job type', { type })\n return\n }\n\n await handler(job, ctx)\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAG7B,MAAM,SAAS,aAAa,OAAO,EAAE,MAAM,EAAE,WAAW,SAAS,CAAC;
|
|
4
|
+
"sourcesContent": ["import { createQueue } from '../factory'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, JobHandler, AsyncQueueOptions, QueueStrategyType } from '../types'\n\nconst logger = createLogger('queue').child({ component: 'worker' })\n\n/**\n * Options for running a queue worker.\n */\nexport type WorkerRunnerOptions<T = unknown> = {\n /** Name of the queue to process */\n queueName: string\n /** Handler function to process each job */\n handler: JobHandler<T>\n /** Redis connection options (only used for async strategy) */\n connection?: AsyncQueueOptions['connection']\n /** Number of concurrent jobs to process */\n concurrency?: number\n /** How long a job lock is held before the job counts as stalled, in ms. */\n lockDuration?: number\n /** Number of stalled-job recoveries BullMQ permits before failing a job. */\n maxStalledCount?: number\n /** Whether to set up graceful shutdown handlers */\n gracefulShutdown?: boolean\n /** If true, don't block - return immediately after starting processing (for multi-queue mode) */\n background?: boolean\n /** Queue strategy to use. Defaults to QUEUE_STRATEGY env var or 'local' */\n strategy?: QueueStrategyType\n}\n\nconst managedQueues = new Set<Queue<unknown>>()\nconst managedShutdownHooks = new Set<() => Promise<void> | void>()\nlet shutdownHandlersRegistered = false\nlet shutdownInProgress = false\n\nfunction unregisterShutdownHandlers(sigtermHandler: () => void, sigintHandler: () => void): void {\n process.off('SIGTERM', sigtermHandler)\n process.off('SIGINT', sigintHandler)\n shutdownHandlersRegistered = false\n}\n\nfunction registerShutdownHandlers(): void {\n if (shutdownHandlersRegistered) return\n\n const shutdown = async (signal: string) => {\n if (shutdownInProgress) return\n shutdownInProgress = true\n\n logger.info('Received signal, shutting down gracefully', { signal })\n\n let hasError = false\n for (const queue of managedQueues) {\n try {\n await queue.close()\n } catch (error) {\n hasError = true\n logger.error('Error during shutdown', { err: error })\n }\n }\n\n managedQueues.clear()\n for (const hook of managedShutdownHooks) {\n try {\n await hook()\n } catch (error) {\n hasError = true\n logger.error('Error during shutdown hook', { err: error })\n }\n }\n managedShutdownHooks.clear()\n unregisterShutdownHandlers(sigtermHandler, sigintHandler)\n shutdownInProgress = false\n\n if (!hasError) {\n logger.info('Worker closed successfully')\n }\n\n process.exit(hasError ? 1 : 0)\n }\n\n const sigtermHandler = () => {\n void shutdown('SIGTERM')\n }\n\n const sigintHandler = () => {\n void shutdown('SIGINT')\n }\n\n process.on('SIGTERM', sigtermHandler)\n process.on('SIGINT', sigintHandler)\n shutdownHandlersRegistered = true\n}\n\n/**\n * Register a process-local service that must stop before a worker exits.\n * The returned callback removes the hook when the service is stopped early.\n */\nexport function registerWorkerShutdownHook(hook: () => Promise<void> | void): () => void {\n managedShutdownHooks.add(hook)\n return () => managedShutdownHooks.delete(hook)\n}\n\n/**\n * Runs a queue worker that processes jobs continuously.\n *\n * This function:\n * 1. Creates an async queue instance\n * 2. Starts a BullMQ worker\n * 3. Sets up graceful shutdown on SIGTERM/SIGINT\n * 4. Keeps the process running until shutdown\n *\n * @template T - The job payload type\n * @param options - Worker configuration\n *\n * @example\n * ```typescript\n * import { runWorker } from '@open-mercato/queue/worker'\n *\n * await runWorker({\n * queueName: 'events',\n * handler: async (job, ctx) => {\n * console.log(`Processing ${ctx.jobId}:`, job.payload)\n * },\n * connection: { url: process.env.REDIS_URL },\n * concurrency: 5,\n * })\n * ```\n */\nexport async function runWorker<T = unknown>(\n options: WorkerRunnerOptions<T>\n): Promise<void> {\n const {\n queueName,\n handler,\n connection,\n concurrency = 1,\n lockDuration,\n maxStalledCount,\n gracefulShutdown = true,\n background = false,\n strategy: strategyOption,\n } = options\n\n // Determine queue strategy from option, env var, or default to 'local'\n const strategy: QueueStrategyType = strategyOption\n ?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')\n\n logger.info('Starting worker for queue', { queueName, strategy })\n\n const queue = createQueue<T>(queueName, strategy, {\n connection,\n concurrency,\n lockDuration,\n maxStalledCount,\n })\n\n // Set up graceful shutdown\n if (gracefulShutdown) {\n managedQueues.add(queue as Queue<unknown>)\n registerShutdownHandlers()\n }\n\n // Start processing\n await queue.process(handler)\n\n logger.info('Worker running', { concurrency })\n\n if (background) {\n // Return immediately for multi-queue mode\n return\n }\n\n logger.info('Press Ctrl+C to stop')\n\n // Keep the process alive (single-queue mode)\n await new Promise(() => {\n // This promise never resolves, keeping the worker running\n })\n}\n\n/**\n * Creates a worker handler that routes jobs to specific handlers based on job type.\n *\n * @template T - Base job payload type (must include a 'type' field)\n * @param handlers - Map of job types to their handlers\n *\n * @example\n * ```typescript\n * const handler = createRoutedHandler({\n * 'user.created': async (job) => { ... },\n * 'order.placed': async (job) => { ... },\n * })\n *\n * await runWorker({ queueName: 'events', handler })\n * ```\n */\nexport function createRoutedHandler<T extends { type: string }>(\n handlers: Record<string, JobHandler<T>>\n): JobHandler<T> {\n return async (job, ctx) => {\n const type = job.payload.type\n const handler = handlers[type]\n\n if (!handler) {\n logger.warn('No handler registered for job type', { type })\n return\n }\n\n await handler(job, ctx)\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAG7B,MAAM,SAAS,aAAa,OAAO,EAAE,MAAM,EAAE,WAAW,SAAS,CAAC;AA0BlE,MAAM,gBAAgB,oBAAI,IAAoB;AAC9C,MAAM,uBAAuB,oBAAI,IAAgC;AACjE,IAAI,6BAA6B;AACjC,IAAI,qBAAqB;AAEzB,SAAS,2BAA2B,gBAA4B,eAAiC;AAC/F,UAAQ,IAAI,WAAW,cAAc;AACrC,UAAQ,IAAI,UAAU,aAAa;AACnC,+BAA6B;AAC/B;AAEA,SAAS,2BAAiC;AACxC,MAAI,2BAA4B;AAEhC,QAAM,WAAW,OAAO,WAAmB;AACzC,QAAI,mBAAoB;AACxB,yBAAqB;AAErB,WAAO,KAAK,6CAA6C,EAAE,OAAO,CAAC;AAEnE,QAAI,WAAW;AACf,eAAW,SAAS,eAAe;AACjC,UAAI;AACF,cAAM,MAAM,MAAM;AAAA,MACpB,SAAS,OAAO;AACd,mBAAW;AACX,eAAO,MAAM,yBAAyB,EAAE,KAAK,MAAM,CAAC;AAAA,MACtD;AAAA,IACF;AAEA,kBAAc,MAAM;AACpB,eAAW,QAAQ,sBAAsB;AACvC,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAO;AACd,mBAAW;AACX,eAAO,MAAM,8BAA8B,EAAE,KAAK,MAAM,CAAC;AAAA,MAC3D;AAAA,IACF;AACA,yBAAqB,MAAM;AAC3B,+BAA2B,gBAAgB,aAAa;AACxD,yBAAqB;AAErB,QAAI,CAAC,UAAU;AACb,aAAO,KAAK,4BAA4B;AAAA,IAC1C;AAEA,YAAQ,KAAK,WAAW,IAAI,CAAC;AAAA,EAC/B;AAEA,QAAM,iBAAiB,MAAM;AAC3B,SAAK,SAAS,SAAS;AAAA,EACzB;AAEA,QAAM,gBAAgB,MAAM;AAC1B,SAAK,SAAS,QAAQ;AAAA,EACxB;AAEA,UAAQ,GAAG,WAAW,cAAc;AACpC,UAAQ,GAAG,UAAU,aAAa;AAClC,+BAA6B;AAC/B;AAMO,SAAS,2BAA2B,MAA8C;AACvF,uBAAqB,IAAI,IAAI;AAC7B,SAAO,MAAM,qBAAqB,OAAO,IAAI;AAC/C;AA4BA,eAAsB,UACpB,SACe;AACf,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA,mBAAmB;AAAA,IACnB,aAAa;AAAA,IACb,UAAU;AAAA,EACZ,IAAI;AAGJ,QAAM,WAA8B,mBAC9B,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAEzD,SAAO,KAAK,6BAA6B,EAAE,WAAW,SAAS,CAAC;AAEhE,QAAM,QAAQ,YAAe,WAAW,UAAU;AAAA,IAChD;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AAGD,MAAI,kBAAkB;AACpB,kBAAc,IAAI,KAAuB;AACzC,6BAAyB;AAAA,EAC3B;AAGA,QAAM,MAAM,QAAQ,OAAO;AAE3B,SAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAE7C,MAAI,YAAY;AAEd;AAAA,EACF;AAEA,SAAO,KAAK,sBAAsB;AAGlC,QAAM,IAAI,QAAQ,MAAM;AAAA,EAExB,CAAC;AACH;AAkBO,SAAS,oBACd,UACe;AACf,SAAO,OAAO,KAAK,QAAQ;AACzB,UAAM,OAAO,IAAI,QAAQ;AACzB,UAAM,UAAU,SAAS,IAAI;AAE7B,QAAI,CAAC,SAAS;AACZ,aAAO,KAAK,sCAAsC,EAAE,KAAK,CAAC;AAC1D;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,GAAG;AAAA,EACxB;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/queue",
|
|
3
|
-
"version": "0.6.7-develop.
|
|
3
|
+
"version": "0.6.7-develop.6750.1.14497517d0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Multi-strategy job queue with local and BullMQ support",
|
|
6
6
|
"type": "module",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"access": "public"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@open-mercato/shared": "0.6.7-develop.
|
|
55
|
+
"@open-mercato/shared": "0.6.7-develop.6750.1.14497517d0"
|
|
56
56
|
},
|
|
57
57
|
"repository": {
|
|
58
58
|
"type": "git",
|
|
@@ -114,6 +114,38 @@ describe('Queue - async strategy', () => {
|
|
|
114
114
|
await queue.close()
|
|
115
115
|
})
|
|
116
116
|
|
|
117
|
+
it('threads queue retry, lock-duration and stalled-job options to BullMQ', async () => {
|
|
118
|
+
const queue = createQueue<{ value: number }>('test-queue', 'async', {
|
|
119
|
+
attempts: 5,
|
|
120
|
+
lockDuration: 120_000,
|
|
121
|
+
maxStalledCount: 10,
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
await queue.enqueue({ value: 42 })
|
|
125
|
+
await queue.process(async () => {})
|
|
126
|
+
|
|
127
|
+
expect(queueAdd).toHaveBeenCalledWith(
|
|
128
|
+
expect.any(String),
|
|
129
|
+
expect.objectContaining({ payload: { value: 42 } }),
|
|
130
|
+
expect.objectContaining({ attempts: 5 }),
|
|
131
|
+
)
|
|
132
|
+
expect(workerCtor).toHaveBeenCalledWith(
|
|
133
|
+
'test-queue',
|
|
134
|
+
expect.any(Function),
|
|
135
|
+
expect.objectContaining({ lockDuration: 120_000, maxStalledCount: 10 }),
|
|
136
|
+
)
|
|
137
|
+
})
|
|
138
|
+
|
|
139
|
+
it('leaves BullMQ on its own lock and stall defaults when the options are unset', async () => {
|
|
140
|
+
const queue = createQueue<{ value: number }>('test-queue', 'async', {})
|
|
141
|
+
|
|
142
|
+
await queue.process(async () => {})
|
|
143
|
+
|
|
144
|
+
const workerOptions = workerCtor.mock.calls[0]?.[2] as Record<string, unknown>
|
|
145
|
+
expect(workerOptions).not.toHaveProperty('lockDuration')
|
|
146
|
+
expect(workerOptions).not.toHaveProperty('maxStalledCount')
|
|
147
|
+
})
|
|
148
|
+
|
|
117
149
|
it('removeQueuedJobsByScope removes only queued jobs matching tenant scope', async () => {
|
|
118
150
|
const removeMatching = jest.fn(async () => {})
|
|
119
151
|
const removeAutoIndex = jest.fn(async () => {})
|
package/src/factory.ts
CHANGED
|
@@ -82,13 +82,16 @@ export function resolveQueueStrategy(): QueueStrategyType {
|
|
|
82
82
|
*/
|
|
83
83
|
export function createModuleQueue<T = unknown>(
|
|
84
84
|
name: string,
|
|
85
|
-
options?:
|
|
85
|
+
options?: Pick<AsyncQueueOptions, 'attempts' | 'concurrency' | 'lockDuration' | 'maxStalledCount'>,
|
|
86
86
|
): Queue<T> {
|
|
87
87
|
const strategy = resolveQueueStrategy()
|
|
88
88
|
if (strategy === 'async') {
|
|
89
89
|
return createAsyncQueue<T>(name, {
|
|
90
90
|
connection: { url: getRedisUrlOrThrow('QUEUE') },
|
|
91
91
|
concurrency: options?.concurrency,
|
|
92
|
+
attempts: options?.attempts,
|
|
93
|
+
lockDuration: options?.lockDuration,
|
|
94
|
+
maxStalledCount: options?.maxStalledCount,
|
|
92
95
|
})
|
|
93
96
|
}
|
|
94
97
|
return createLocalQueue<T>(name, { concurrency: options?.concurrency })
|
package/src/strategies/async.ts
CHANGED
|
@@ -47,7 +47,7 @@ interface BullMQModule {
|
|
|
47
47
|
Worker: new <T>(
|
|
48
48
|
name: string,
|
|
49
49
|
processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,
|
|
50
|
-
opts: { connection: ConnectionOptions; concurrency: number }
|
|
50
|
+
opts: { connection: ConnectionOptions; concurrency: number; lockDuration?: number; maxStalledCount?: number }
|
|
51
51
|
) => BullWorkerInterface
|
|
52
52
|
}
|
|
53
53
|
|
|
@@ -111,6 +111,9 @@ export function createAsyncQueue<T = unknown>(
|
|
|
111
111
|
): Queue<T> {
|
|
112
112
|
const connection = resolveConnection(options?.connection)
|
|
113
113
|
const concurrency = options?.concurrency ?? 1
|
|
114
|
+
const attempts = options?.attempts ?? 3
|
|
115
|
+
const lockDuration = options?.lockDuration
|
|
116
|
+
const maxStalledCount = options?.maxStalledCount
|
|
114
117
|
const logger = packageLogger.child({ queue: name })
|
|
115
118
|
|
|
116
119
|
let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
|
|
@@ -158,7 +161,7 @@ export function createAsyncQueue<T = unknown>(
|
|
|
158
161
|
delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,
|
|
159
162
|
removeOnComplete: true,
|
|
160
163
|
removeOnFail: 1000,
|
|
161
|
-
attempts
|
|
164
|
+
attempts,
|
|
162
165
|
backoff: { type: 'exponential', delay: 1000 },
|
|
163
166
|
})
|
|
164
167
|
|
|
@@ -182,6 +185,8 @@ export function createAsyncQueue<T = unknown>(
|
|
|
182
185
|
{
|
|
183
186
|
connection,
|
|
184
187
|
concurrency,
|
|
188
|
+
...(lockDuration !== undefined ? { lockDuration } : {}),
|
|
189
|
+
...(maxStalledCount !== undefined ? { maxStalledCount } : {}),
|
|
185
190
|
}
|
|
186
191
|
)
|
|
187
192
|
|
|
@@ -197,6 +202,16 @@ export function createAsyncQueue<T = unknown>(
|
|
|
197
202
|
logger.error('Job failed', { jobId: jobWithId?.id, err: error })
|
|
198
203
|
})
|
|
199
204
|
|
|
205
|
+
// A stalled job is redelivered under the same id while the previous worker
|
|
206
|
+
// may still be running it, so this is the signal that a handler is about to
|
|
207
|
+
// be executed twice. BullMQ's docs require surfacing it: without this line
|
|
208
|
+
// duplicate processing is invisible.
|
|
209
|
+
bullWorker.on('stalled', (jobId) => {
|
|
210
|
+
logger.warn('Job stalled and will be redelivered — the handler may run concurrently with a previous delivery', {
|
|
211
|
+
jobId: typeof jobId === 'string' ? jobId : null,
|
|
212
|
+
})
|
|
213
|
+
})
|
|
214
|
+
|
|
200
215
|
bullWorker.on('error', (err) => {
|
|
201
216
|
const error = err as Error
|
|
202
217
|
logger.error('Worker error', { err: error })
|
package/src/types.ts
CHANGED
|
@@ -91,6 +91,12 @@ export type AsyncQueueOptions = {
|
|
|
91
91
|
connection?: RedisConnectionOptions
|
|
92
92
|
/** Number of concurrent job processors. Defaults to 1 */
|
|
93
93
|
concurrency?: number
|
|
94
|
+
/** Number of attempts for newly enqueued jobs. Defaults to 3. */
|
|
95
|
+
attempts?: number
|
|
96
|
+
/** How long a job lock is held before the job counts as stalled, in ms. Defaults to 30000. */
|
|
97
|
+
lockDuration?: number
|
|
98
|
+
/** Number of stalled-job recoveries BullMQ permits before failing a job. Defaults to 1. */
|
|
99
|
+
maxStalledCount?: number
|
|
94
100
|
}
|
|
95
101
|
|
|
96
102
|
/**
|
|
@@ -250,6 +256,10 @@ export type WorkerMeta = {
|
|
|
250
256
|
id?: string
|
|
251
257
|
/** Worker concurrency (default: 1) */
|
|
252
258
|
concurrency?: number
|
|
259
|
+
/** How long a job lock is held before the job counts as stalled, in ms. */
|
|
260
|
+
lockDuration?: number
|
|
261
|
+
/** Number of stalled-job recoveries BullMQ permits before failing a job. */
|
|
262
|
+
maxStalledCount?: number
|
|
253
263
|
}
|
|
254
264
|
|
|
255
265
|
/**
|
|
@@ -265,4 +275,8 @@ export type WorkerDescriptor<T = unknown> = {
|
|
|
265
275
|
handler: JobHandler<T>
|
|
266
276
|
/** Concurrency level */
|
|
267
277
|
concurrency: number
|
|
278
|
+
/** How long a job lock is held before the job counts as stalled, in ms. */
|
|
279
|
+
lockDuration?: number
|
|
280
|
+
/** Number of stalled-job recoveries BullMQ permits before failing a job. */
|
|
281
|
+
maxStalledCount?: number
|
|
268
282
|
}
|
package/src/worker/runner.ts
CHANGED
|
@@ -16,6 +16,10 @@ export type WorkerRunnerOptions<T = unknown> = {
|
|
|
16
16
|
connection?: AsyncQueueOptions['connection']
|
|
17
17
|
/** Number of concurrent jobs to process */
|
|
18
18
|
concurrency?: number
|
|
19
|
+
/** How long a job lock is held before the job counts as stalled, in ms. */
|
|
20
|
+
lockDuration?: number
|
|
21
|
+
/** Number of stalled-job recoveries BullMQ permits before failing a job. */
|
|
22
|
+
maxStalledCount?: number
|
|
19
23
|
/** Whether to set up graceful shutdown handlers */
|
|
20
24
|
gracefulShutdown?: boolean
|
|
21
25
|
/** If true, don't block - return immediately after starting processing (for multi-queue mode) */
|
|
@@ -130,6 +134,8 @@ export async function runWorker<T = unknown>(
|
|
|
130
134
|
handler,
|
|
131
135
|
connection,
|
|
132
136
|
concurrency = 1,
|
|
137
|
+
lockDuration,
|
|
138
|
+
maxStalledCount,
|
|
133
139
|
gracefulShutdown = true,
|
|
134
140
|
background = false,
|
|
135
141
|
strategy: strategyOption,
|
|
@@ -144,6 +150,8 @@ export async function runWorker<T = unknown>(
|
|
|
144
150
|
const queue = createQueue<T>(queueName, strategy, {
|
|
145
151
|
connection,
|
|
146
152
|
concurrency,
|
|
153
|
+
lockDuration,
|
|
154
|
+
maxStalledCount,
|
|
147
155
|
})
|
|
148
156
|
|
|
149
157
|
// Set up graceful shutdown
|