@open-mercato/queue 0.6.7-develop.6860.1.f796b22e80 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/factory.js +1 -4
- package/dist/factory.js.map +2 -2
- package/dist/pending-probe.js +2 -4
- package/dist/pending-probe.js.map +2 -2
- package/dist/strategies/async.js +10 -50
- package/dist/strategies/async.js.map +2 -2
- package/dist/strategies/local.js +8 -15
- package/dist/strategies/local.js.map +2 -2
- package/dist/worker/runner.js +1 -18
- package/dist/worker/runner.js.map +2 -2
- package/jest.config.cjs +0 -1
- package/package.json +6 -12
- package/src/__tests__/async.strategy.test.ts +5 -62
- package/src/__tests__/factory.test.ts +1 -5
- package/src/__tests__/pending-probe.test.ts +1 -40
- package/src/factory.ts +1 -4
- package/src/pending-probe.ts +2 -4
- package/src/strategies/async.ts +13 -83
- package/src/strategies/local.ts +10 -24
- package/src/types.ts +0 -16
- package/src/worker/runner.ts +0 -32
- package/dist/tracing.js +0 -27
- package/dist/tracing.js.map +0 -7
- package/src/__tests__/async.telemetry.test.ts +0 -123
- package/src/__tests__/tracing.test.ts +0 -123
- package/src/__tests__/worker-shutdown-telemetry.test.ts +0 -80
- package/src/tracing.ts +0 -58
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
[build:queue] found
|
|
1
|
+
[build:queue] found 8 entry points
|
|
2
2
|
[build:queue] built successfully
|
package/dist/factory.js
CHANGED
|
@@ -15,10 +15,7 @@ function createModuleQueue(name, options) {
|
|
|
15
15
|
if (strategy === "async") {
|
|
16
16
|
return createAsyncQueue(name, {
|
|
17
17
|
connection: { url: getRedisUrlOrThrow("QUEUE") },
|
|
18
|
-
concurrency: options?.concurrency
|
|
19
|
-
attempts: options?.attempts,
|
|
20
|
-
lockDuration: options?.lockDuration,
|
|
21
|
-
maxStalledCount: options?.maxStalledCount
|
|
18
|
+
concurrency: options?.concurrency
|
|
22
19
|
});
|
|
23
20
|
}
|
|
24
21
|
return createLocalQueue(name, { concurrency: options?.concurrency });
|
package/dist/factory.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/factory.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Queue, LocalQueueOptions, AsyncQueueOptions, QueueStrategyType } from './types'\nimport { createLocalQueue } from './strategies/local'\nimport { createAsyncQueue } from './strategies/async'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\n\n/**\n * Creates a queue instance with the specified strategy.\n *\n * @template T - The payload type for jobs in this queue\n * @param name - Unique name for the queue\n * @param strategy - Queue strategy: 'local' for file-based, 'async' for BullMQ\n * @param options - Strategy-specific options\n * @returns A Queue instance\n *\n * @example\n * ```typescript\n * // Local file-based queue\n * const localQueue = createQueue<MyJobData>('my-queue', 'local')\n *\n * // BullMQ-based queue\n * const asyncQueue = createQueue<MyJobData>('my-queue', 'async', {\n * connection: { url: 'redis://localhost:6379' },\n * concurrency: 5\n * })\n * ```\n */\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local',\n options?: LocalQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'async',\n options?: AsyncQueueOptions\n): Queue<T>\n\n// General overload for dynamic strategy (union type)\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T> {\n if (strategy === 'async') {\n return createAsyncQueue<T>(name, options as AsyncQueueOptions)\n }\n\n return createLocalQueue<T>(name, options as LocalQueueOptions)\n}\n\n/**\n * Resolve the queue strategy from `QUEUE_STRATEGY`. Defaults to `'local'`.\n */\nexport function resolveQueueStrategy(): QueueStrategyType {\n return process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local'\n}\n\n/**\n * Create a module-owned queue using the strategy declared in `QUEUE_STRATEGY`.\n *\n * - When `QUEUE_STRATEGY=async`, builds a BullMQ queue and resolves the\n * Redis URL via `getRedisUrlOrThrow('QUEUE')` so missing config fails loudly.\n * - Otherwise builds a local file-based queue.\n *\n * Replaces the boilerplate `process.env.QUEUE_STRATEGY === 'async' ? ... : ...`\n * pattern that every module queue helper used to repeat. Concurrency applies\n * to both strategies so the same number means the same thing in dev and prod.\n *\n * @example\n * ```typescript\n * export function getDataSyncQueue(name: string) {\n * return createModuleQueue<MyJob>(name, { concurrency: 5 })\n * }\n * ```\n */\nexport function createModuleQueue<T = unknown>(\n name: string,\n options?:
|
|
5
|
-
"mappings": "AACA,SAAS,wBAAwB;AACjC,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AA0C5B,SAAS,YACd,MACA,UACA,SACU;AACV,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM,OAA4B;AAAA,EAC/D;AAEA,SAAO,iBAAoB,MAAM,OAA4B;AAC/D;AAKO,SAAS,uBAA0C;AACxD,SAAO,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAC5D;AAoBO,SAAS,kBACd,MACA,SACU;AACV,QAAM,WAAW,qBAAqB;AACtC,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM;AAAA,MAC/B,YAAY,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAAA,MAC/C,aAAa,SAAS;AAAA,
|
|
4
|
+
"sourcesContent": ["import type { Queue, LocalQueueOptions, AsyncQueueOptions, QueueStrategyType } from './types'\nimport { createLocalQueue } from './strategies/local'\nimport { createAsyncQueue } from './strategies/async'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\n\n/**\n * Creates a queue instance with the specified strategy.\n *\n * @template T - The payload type for jobs in this queue\n * @param name - Unique name for the queue\n * @param strategy - Queue strategy: 'local' for file-based, 'async' for BullMQ\n * @param options - Strategy-specific options\n * @returns A Queue instance\n *\n * @example\n * ```typescript\n * // Local file-based queue\n * const localQueue = createQueue<MyJobData>('my-queue', 'local')\n *\n * // BullMQ-based queue\n * const asyncQueue = createQueue<MyJobData>('my-queue', 'async', {\n * connection: { url: 'redis://localhost:6379' },\n * concurrency: 5\n * })\n * ```\n */\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local',\n options?: LocalQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'async',\n options?: AsyncQueueOptions\n): Queue<T>\n\n// General overload for dynamic strategy (union type)\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T>\n\nexport function createQueue<T = unknown>(\n name: string,\n strategy: 'local' | 'async',\n options?: LocalQueueOptions | AsyncQueueOptions\n): Queue<T> {\n if (strategy === 'async') {\n return createAsyncQueue<T>(name, options as AsyncQueueOptions)\n }\n\n return createLocalQueue<T>(name, options as LocalQueueOptions)\n}\n\n/**\n * Resolve the queue strategy from `QUEUE_STRATEGY`. Defaults to `'local'`.\n */\nexport function resolveQueueStrategy(): QueueStrategyType {\n return process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local'\n}\n\n/**\n * Create a module-owned queue using the strategy declared in `QUEUE_STRATEGY`.\n *\n * - When `QUEUE_STRATEGY=async`, builds a BullMQ queue and resolves the\n * Redis URL via `getRedisUrlOrThrow('QUEUE')` so missing config fails loudly.\n * - Otherwise builds a local file-based queue.\n *\n * Replaces the boilerplate `process.env.QUEUE_STRATEGY === 'async' ? ... : ...`\n * pattern that every module queue helper used to repeat. Concurrency applies\n * to both strategies so the same number means the same thing in dev and prod.\n *\n * @example\n * ```typescript\n * export function getDataSyncQueue(name: string) {\n * return createModuleQueue<MyJob>(name, { concurrency: 5 })\n * }\n * ```\n */\nexport function createModuleQueue<T = unknown>(\n name: string,\n options?: { concurrency?: number },\n): Queue<T> {\n const strategy = resolveQueueStrategy()\n if (strategy === 'async') {\n return createAsyncQueue<T>(name, {\n connection: { url: getRedisUrlOrThrow('QUEUE') },\n concurrency: options?.concurrency,\n })\n }\n return createLocalQueue<T>(name, { concurrency: options?.concurrency })\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,wBAAwB;AACjC,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AA0C5B,SAAS,YACd,MACA,UACA,SACU;AACV,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM,OAA4B;AAAA,EAC/D;AAEA,SAAO,iBAAoB,MAAM,OAA4B;AAC/D;AAKO,SAAS,uBAA0C;AACxD,SAAO,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAC5D;AAoBO,SAAS,kBACd,MACA,SACU;AACV,QAAM,WAAW,qBAAqB;AACtC,MAAI,aAAa,SAAS;AACxB,WAAO,iBAAoB,MAAM;AAAA,MAC/B,YAAY,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAAA,MAC/C,aAAa,SAAS;AAAA,IACxB,CAAC;AAAA,EACH;AACA,SAAO,iBAAoB,MAAM,EAAE,aAAa,SAAS,YAAY,CAAC;AACxE;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/pending-probe.js
CHANGED
|
@@ -75,16 +75,14 @@ async function probeAsyncQueue(queueName, options) {
|
|
|
75
75
|
if (!bullmq) {
|
|
76
76
|
return errorResult(queueName, "async", new Error("bullmq is not installed"));
|
|
77
77
|
}
|
|
78
|
-
const { getRedisUrl
|
|
78
|
+
const { getRedisUrl } = await import("@open-mercato/shared/lib/redis/connection");
|
|
79
79
|
let connection = options?.connection;
|
|
80
80
|
if (!connection) {
|
|
81
81
|
const url = getRedisUrl("QUEUE");
|
|
82
82
|
if (!url) {
|
|
83
83
|
return errorResult(queueName, "async", new Error("QUEUE Redis URL is not configured"));
|
|
84
84
|
}
|
|
85
|
-
connection =
|
|
86
|
-
} else if (connection.url) {
|
|
87
|
-
connection = parseRedisUrl(connection.url);
|
|
85
|
+
connection = { url };
|
|
88
86
|
}
|
|
89
87
|
let queue = null;
|
|
90
88
|
try {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/pending-probe.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * Lightweight pending-job probe helpers.\n *\n * Used by the lazy worker supervisor to detect whether a queue has a\n * ready-to-process job before spawning a long-lived worker process for it.\n *\n * The probes MUST NOT:\n * - call `queue.process()` or otherwise install handlers\n * - import any module worker handler code\n * - create BullMQ `Worker` instances\n *\n * Probes are best-effort and fail-soft: a probe error returns \"no pending\"\n * so the supervisor can keep polling instead of crashing the runtime.\n */\n\nimport fs from 'node:fs'\nimport path from 'node:path'\nimport type { QueueStrategyType, RedisConnectionOptions } from './types'\n\nexport type QueuePendingProbeOptions = {\n /** Async strategy: Redis connection override (mirrors `AsyncQueueOptions['connection']`). */\n connection?: RedisConnectionOptions\n /** Local strategy: override `QUEUE_BASE_DIR` for test isolation. */\n baseDir?: string\n}\n\nexport type QueuePendingProbeResult = {\n queueName: string\n strategy: QueueStrategyType\n /** Number of jobs ready to process now (no delay or delay already elapsed). */\n ready: number\n /** Number of jobs scheduled for the future (still waiting on `availableAt`). */\n delayedFuture: number\n /** Number of jobs currently being processed. May be unavailable for some strategies. */\n active: number\n /**\n * True when the probe could not query the underlying storage at all\n * (filesystem error, Redis unreachable, optional dependency missing, etc.).\n * The supervisor treats `error: true` as \"do not start worker yet\".\n */\n error: boolean\n errorMessage?: string\n}\n\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\n\nconst fsp = fs.promises\n\nfunction emptyResult(queueName: string, strategy: QueueStrategyType): QueuePendingProbeResult {\n return { queueName, strategy, ready: 0, delayedFuture: 0, active: 0, error: false }\n}\n\nfunction errorResult(\n queueName: string,\n strategy: QueueStrategyType,\n err: unknown,\n): QueuePendingProbeResult {\n const message = err instanceof Error ? err.message : String(err)\n return {\n queueName,\n strategy,\n ready: 0,\n delayedFuture: 0,\n active: 0,\n error: true,\n errorMessage: message,\n }\n}\n\nasync function probeLocalQueue(\n queueName: string,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const envBaseDir = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(envBaseDir || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueFile = path.join(baseDir, queueName, 'queue.json')\n\n let raw: string\n try {\n raw = await fsp.readFile(queueFile, 'utf8')\n } catch (err) {\n const fsErr = err as NodeJS.ErrnoException\n if (fsErr?.code === 'ENOENT') {\n return emptyResult(queueName, 'local')\n }\n return errorResult(queueName, 'local', err)\n }\n\n let parsed: unknown\n try {\n parsed = JSON.parse(raw)\n } catch (err) {\n return errorResult(queueName, 'local', err)\n }\n\n if (!Array.isArray(parsed)) {\n return emptyResult(queueName, 'local')\n }\n\n const now = Date.now()\n let ready = 0\n let delayedFuture = 0\n\n for (const entry of parsed) {\n if (!entry || typeof entry !== 'object') continue\n const availableAt = (entry as { availableAt?: unknown }).availableAt\n if (typeof availableAt !== 'string' || availableAt.length === 0) {\n ready++\n continue\n }\n const ts = Date.parse(availableAt)\n if (!Number.isFinite(ts) || ts <= now) {\n ready++\n } else {\n delayedFuture++\n }\n }\n\n return { queueName, strategy: 'local', ready, delayedFuture, active: 0, error: false }\n}\n\ntype BullMQModuleShape = {\n Queue: new <T>(name: string, opts: { connection: RedisConnectionOptions }) => {\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n close: () => Promise<void>\n }\n}\n\nlet cachedBullMQ: BullMQModuleShape | null | undefined\n\nasync function loadBullMQ(): Promise<BullMQModuleShape | null> {\n if (cachedBullMQ !== undefined) return cachedBullMQ\n try {\n cachedBullMQ = (await import('bullmq')) as unknown as BullMQModuleShape\n } catch {\n cachedBullMQ = null\n }\n return cachedBullMQ\n}\n\nasync function probeAsyncQueue(\n queueName: string,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const bullmq = await loadBullMQ()\n if (!bullmq) {\n return errorResult(queueName, 'async', new Error('bullmq is not installed'))\n }\n\n const { getRedisUrl
|
|
5
|
-
"mappings": "AAeA,OAAO,QAAQ;AACf,OAAO,UAAU;AA4BjB,MAAM,+BAA+B;AAErC,MAAM,MAAM,GAAG;AAEf,SAAS,YAAY,WAAmB,UAAsD;AAC5F,SAAO,EAAE,WAAW,UAAU,OAAO,GAAG,eAAe,GAAG,QAAQ,GAAG,OAAO,MAAM;AACpF;AAEA,SAAS,YACP,WACA,UACA,KACyB;AACzB,QAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,cAAc;AAAA,EAChB;AACF;AAEA,eAAe,gBACb,WACA,SACkC;AAClC,QAAM,cAAe,WAAgE;AACrF,QAAM,aAAa,aAAa,KAAK;AACrC,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,cAAc,4BAA4B;AAC5D,QAAM,YAAY,KAAK,KAAK,SAAS,WAAW,YAAY;AAE5D,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,EAC5C,SAAS,KAAK;AACZ,UAAM,QAAQ;AACd,QAAI,OAAO,SAAS,UAAU;AAC5B,aAAO,YAAY,WAAW,OAAO;AAAA,IACvC;AACA,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;AAAA,EACzB,SAAS,KAAK;AACZ,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C;AAEA,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,WAAO,YAAY,WAAW,OAAO;AAAA,EACvC;AAEA,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,QAAQ;AACZ,MAAI,gBAAgB;AAEpB,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,SAAS,OAAO,UAAU,SAAU;AACzC,UAAM,cAAe,MAAoC;AACzD,QAAI,OAAO,gBAAgB,YAAY,YAAY,WAAW,GAAG;AAC/D;AACA;AAAA,IACF;AACA,UAAM,KAAK,KAAK,MAAM,WAAW;AACjC,QAAI,CAAC,OAAO,SAAS,EAAE,KAAK,MAAM,KAAK;AACrC;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,UAAU,SAAS,OAAO,eAAe,QAAQ,GAAG,OAAO,MAAM;AACvF;AASA,IAAI;AAEJ,eAAe,aAAgD;AAC7D,MAAI,iBAAiB,OAAW,QAAO;AACvC,MAAI;AACF,mBAAgB,MAAM,OAAO,QAAQ;AAAA,EACvC,QAAQ;AACN,mBAAe;AAAA,EACjB;AACA,SAAO;AACT;AAEA,eAAe,gBACb,WACA,SACkC;AAClC,QAAM,SAAS,MAAM,WAAW;AAChC,MAAI,CAAC,QAAQ;AACX,WAAO,YAAY,WAAW,SAAS,IAAI,MAAM,yBAAyB,CAAC;AAAA,EAC7E;AAEA,QAAM,EAAE,
|
|
4
|
+
"sourcesContent": ["/**\n * Lightweight pending-job probe helpers.\n *\n * Used by the lazy worker supervisor to detect whether a queue has a\n * ready-to-process job before spawning a long-lived worker process for it.\n *\n * The probes MUST NOT:\n * - call `queue.process()` or otherwise install handlers\n * - import any module worker handler code\n * - create BullMQ `Worker` instances\n *\n * Probes are best-effort and fail-soft: a probe error returns \"no pending\"\n * so the supervisor can keep polling instead of crashing the runtime.\n */\n\nimport fs from 'node:fs'\nimport path from 'node:path'\nimport type { QueueStrategyType, RedisConnectionOptions } from './types'\n\nexport type QueuePendingProbeOptions = {\n /** Async strategy: Redis connection override (mirrors `AsyncQueueOptions['connection']`). */\n connection?: RedisConnectionOptions\n /** Local strategy: override `QUEUE_BASE_DIR` for test isolation. */\n baseDir?: string\n}\n\nexport type QueuePendingProbeResult = {\n queueName: string\n strategy: QueueStrategyType\n /** Number of jobs ready to process now (no delay or delay already elapsed). */\n ready: number\n /** Number of jobs scheduled for the future (still waiting on `availableAt`). */\n delayedFuture: number\n /** Number of jobs currently being processed. May be unavailable for some strategies. */\n active: number\n /**\n * True when the probe could not query the underlying storage at all\n * (filesystem error, Redis unreachable, optional dependency missing, etc.).\n * The supervisor treats `error: true` as \"do not start worker yet\".\n */\n error: boolean\n errorMessage?: string\n}\n\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\n\nconst fsp = fs.promises\n\nfunction emptyResult(queueName: string, strategy: QueueStrategyType): QueuePendingProbeResult {\n return { queueName, strategy, ready: 0, delayedFuture: 0, active: 0, error: false }\n}\n\nfunction errorResult(\n queueName: string,\n strategy: QueueStrategyType,\n err: unknown,\n): QueuePendingProbeResult {\n const message = err instanceof Error ? err.message : String(err)\n return {\n queueName,\n strategy,\n ready: 0,\n delayedFuture: 0,\n active: 0,\n error: true,\n errorMessage: message,\n }\n}\n\nasync function probeLocalQueue(\n queueName: string,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const envBaseDir = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(envBaseDir || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueFile = path.join(baseDir, queueName, 'queue.json')\n\n let raw: string\n try {\n raw = await fsp.readFile(queueFile, 'utf8')\n } catch (err) {\n const fsErr = err as NodeJS.ErrnoException\n if (fsErr?.code === 'ENOENT') {\n return emptyResult(queueName, 'local')\n }\n return errorResult(queueName, 'local', err)\n }\n\n let parsed: unknown\n try {\n parsed = JSON.parse(raw)\n } catch (err) {\n return errorResult(queueName, 'local', err)\n }\n\n if (!Array.isArray(parsed)) {\n return emptyResult(queueName, 'local')\n }\n\n const now = Date.now()\n let ready = 0\n let delayedFuture = 0\n\n for (const entry of parsed) {\n if (!entry || typeof entry !== 'object') continue\n const availableAt = (entry as { availableAt?: unknown }).availableAt\n if (typeof availableAt !== 'string' || availableAt.length === 0) {\n ready++\n continue\n }\n const ts = Date.parse(availableAt)\n if (!Number.isFinite(ts) || ts <= now) {\n ready++\n } else {\n delayedFuture++\n }\n }\n\n return { queueName, strategy: 'local', ready, delayedFuture, active: 0, error: false }\n}\n\ntype BullMQModuleShape = {\n Queue: new <T>(name: string, opts: { connection: RedisConnectionOptions }) => {\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n close: () => Promise<void>\n }\n}\n\nlet cachedBullMQ: BullMQModuleShape | null | undefined\n\nasync function loadBullMQ(): Promise<BullMQModuleShape | null> {\n if (cachedBullMQ !== undefined) return cachedBullMQ\n try {\n cachedBullMQ = (await import('bullmq')) as unknown as BullMQModuleShape\n } catch {\n cachedBullMQ = null\n }\n return cachedBullMQ\n}\n\nasync function probeAsyncQueue(\n queueName: string,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const bullmq = await loadBullMQ()\n if (!bullmq) {\n return errorResult(queueName, 'async', new Error('bullmq is not installed'))\n }\n\n const { getRedisUrl } = await import('@open-mercato/shared/lib/redis/connection')\n let connection = options?.connection\n if (!connection) {\n const url = getRedisUrl('QUEUE')\n if (!url) {\n return errorResult(queueName, 'async', new Error('QUEUE Redis URL is not configured'))\n }\n connection = { url }\n }\n\n let queue: InstanceType<BullMQModuleShape['Queue']> | null = null\n try {\n queue = new bullmq.Queue(queueName, { connection })\n const counts = await queue.getJobCounts('waiting', 'delayed', 'active')\n const waiting = counts.waiting ?? 0\n const delayed = counts.delayed ?? 0\n const active = counts.active ?? 0\n return {\n queueName,\n strategy: 'async',\n ready: waiting,\n delayedFuture: delayed,\n active,\n error: false,\n }\n } catch (err) {\n return errorResult(queueName, 'async', err)\n } finally {\n if (queue) {\n try {\n await queue.close()\n } catch {\n /* swallow shutdown errors \u2014 probe must not throw on cleanup */\n }\n }\n }\n}\n\n/**\n * Read-only pending-job probe for a queue.\n *\n * `strategy` defaults to the value of `QUEUE_STRATEGY` (via\n * `resolveQueueStrategy`). The probe never installs handlers and never\n * starts a BullMQ Worker, so it is safe to call from a lightweight\n * supervisor process that watches many queues.\n */\nexport async function getQueuePendingProbe(\n queueName: string,\n strategy?: QueueStrategyType,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const resolvedStrategy: QueueStrategyType = strategy\n ?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')\n\n if (resolvedStrategy === 'async') {\n return probeAsyncQueue(queueName, options)\n }\n return probeLocalQueue(queueName, options)\n}\n\n/** Reset the cached bullmq module reference. Test-only. */\nexport function __resetPendingProbeBullMQCache(): void {\n cachedBullMQ = undefined\n}\n"],
|
|
5
|
+
"mappings": "AAeA,OAAO,QAAQ;AACf,OAAO,UAAU;AA4BjB,MAAM,+BAA+B;AAErC,MAAM,MAAM,GAAG;AAEf,SAAS,YAAY,WAAmB,UAAsD;AAC5F,SAAO,EAAE,WAAW,UAAU,OAAO,GAAG,eAAe,GAAG,QAAQ,GAAG,OAAO,MAAM;AACpF;AAEA,SAAS,YACP,WACA,UACA,KACyB;AACzB,QAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,cAAc;AAAA,EAChB;AACF;AAEA,eAAe,gBACb,WACA,SACkC;AAClC,QAAM,cAAe,WAAgE;AACrF,QAAM,aAAa,aAAa,KAAK;AACrC,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,cAAc,4BAA4B;AAC5D,QAAM,YAAY,KAAK,KAAK,SAAS,WAAW,YAAY;AAE5D,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,EAC5C,SAAS,KAAK;AACZ,UAAM,QAAQ;AACd,QAAI,OAAO,SAAS,UAAU;AAC5B,aAAO,YAAY,WAAW,OAAO;AAAA,IACvC;AACA,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;AAAA,EACzB,SAAS,KAAK;AACZ,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C;AAEA,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,WAAO,YAAY,WAAW,OAAO;AAAA,EACvC;AAEA,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,QAAQ;AACZ,MAAI,gBAAgB;AAEpB,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,SAAS,OAAO,UAAU,SAAU;AACzC,UAAM,cAAe,MAAoC;AACzD,QAAI,OAAO,gBAAgB,YAAY,YAAY,WAAW,GAAG;AAC/D;AACA;AAAA,IACF;AACA,UAAM,KAAK,KAAK,MAAM,WAAW;AACjC,QAAI,CAAC,OAAO,SAAS,EAAE,KAAK,MAAM,KAAK;AACrC;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,UAAU,SAAS,OAAO,eAAe,QAAQ,GAAG,OAAO,MAAM;AACvF;AASA,IAAI;AAEJ,eAAe,aAAgD;AAC7D,MAAI,iBAAiB,OAAW,QAAO;AACvC,MAAI;AACF,mBAAgB,MAAM,OAAO,QAAQ;AAAA,EACvC,QAAQ;AACN,mBAAe;AAAA,EACjB;AACA,SAAO;AACT;AAEA,eAAe,gBACb,WACA,SACkC;AAClC,QAAM,SAAS,MAAM,WAAW;AAChC,MAAI,CAAC,QAAQ;AACX,WAAO,YAAY,WAAW,SAAS,IAAI,MAAM,yBAAyB,CAAC;AAAA,EAC7E;AAEA,QAAM,EAAE,YAAY,IAAI,MAAM,OAAO,2CAA2C;AAChF,MAAI,aAAa,SAAS;AAC1B,MAAI,CAAC,YAAY;AACf,UAAM,MAAM,YAAY,OAAO;AAC/B,QAAI,CAAC,KAAK;AACR,aAAO,YAAY,WAAW,SAAS,IAAI,MAAM,mCAAmC,CAAC;AAAA,IACvF;AACA,iBAAa,EAAE,IAAI;AAAA,EACrB;AAEA,MAAI,QAAyD;AAC7D,MAAI;AACF,YAAQ,IAAI,OAAO,MAAM,WAAW,EAAE,WAAW,CAAC;AAClD,UAAM,SAAS,MAAM,MAAM,aAAa,WAAW,WAAW,QAAQ;AACtE,UAAM,UAAU,OAAO,WAAW;AAClC,UAAM,UAAU,OAAO,WAAW;AAClC,UAAM,SAAS,OAAO,UAAU;AAChC,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,OAAO;AAAA,MACP,eAAe;AAAA,MACf;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF,SAAS,KAAK;AACZ,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C,UAAE;AACA,QAAI,OAAO;AACT,UAAI;AACF,cAAM,MAAM,MAAM;AAAA,MACpB,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF;AACF;AAUA,eAAsB,qBACpB,WACA,UACA,SACkC;AAClC,QAAM,mBAAsC,aACtC,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAEzD,MAAI,qBAAqB,SAAS;AAChC,WAAO,gBAAgB,WAAW,OAAO;AAAA,EAC3C;AACA,SAAO,gBAAgB,WAAW,OAAO;AAC3C;AAGO,SAAS,iCAAuC;AACrD,iBAAe;AACjB;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/strategies/async.js
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import { getRedisUrlOrThrow
|
|
2
|
-
import { getTelemetryRuntime } from "@open-mercato/shared/lib/telemetry/runtime";
|
|
3
|
-
import { attachTraceMetadata, runJobInTrace } from "../tracing.js";
|
|
1
|
+
import { getRedisUrlOrThrow } from "@open-mercato/shared/lib/redis/connection";
|
|
4
2
|
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
5
3
|
const packageLogger = createLogger("queue");
|
|
6
4
|
const REMOVABLE_JOB_STATES = ["waiting", "delayed", "prioritized", "paused", "waiting-children"];
|
|
@@ -18,7 +16,7 @@ function payloadMatchesScope(payload, scope) {
|
|
|
18
16
|
}
|
|
19
17
|
function resolveConnection(options) {
|
|
20
18
|
if (options?.url) {
|
|
21
|
-
return
|
|
19
|
+
return { url: options.url };
|
|
22
20
|
}
|
|
23
21
|
if (options?.host) {
|
|
24
22
|
return {
|
|
@@ -27,23 +25,18 @@ function resolveConnection(options) {
|
|
|
27
25
|
username: options.username,
|
|
28
26
|
password: options.password,
|
|
29
27
|
db: options.db,
|
|
30
|
-
tls: options.tls
|
|
31
|
-
family: options.family
|
|
28
|
+
tls: options.tls
|
|
32
29
|
};
|
|
33
30
|
}
|
|
34
|
-
return
|
|
31
|
+
return { url: getRedisUrlOrThrow("QUEUE") };
|
|
35
32
|
}
|
|
36
33
|
function createAsyncQueue(name, options) {
|
|
37
34
|
const connection = resolveConnection(options?.connection);
|
|
38
35
|
const concurrency = options?.concurrency ?? 1;
|
|
39
|
-
const attempts = options?.attempts ?? 3;
|
|
40
|
-
const lockDuration = options?.lockDuration;
|
|
41
|
-
const maxStalledCount = options?.maxStalledCount;
|
|
42
36
|
const logger = packageLogger.child({ queue: name });
|
|
43
37
|
let bullQueue = null;
|
|
44
38
|
let bullWorker = null;
|
|
45
39
|
let bullmqModule = null;
|
|
46
|
-
let telemetryPromise = null;
|
|
47
40
|
async function getBullMQ() {
|
|
48
41
|
if (!bullmqModule) {
|
|
49
42
|
try {
|
|
@@ -56,72 +49,44 @@ function createAsyncQueue(name, options) {
|
|
|
56
49
|
}
|
|
57
50
|
return bullmqModule;
|
|
58
51
|
}
|
|
59
|
-
async function getQueueTelemetry() {
|
|
60
|
-
if (!telemetryPromise) {
|
|
61
|
-
telemetryPromise = (async () => {
|
|
62
|
-
if (!getTelemetryRuntime()?.canUseGlobalTracePropagation()) return void 0;
|
|
63
|
-
try {
|
|
64
|
-
const mod = await import("bullmq-otel");
|
|
65
|
-
return new mod.BullMQOtel("open-mercato");
|
|
66
|
-
} catch {
|
|
67
|
-
packageLogger.warn("bullmq-otel not available; using built-in trace carrier", { queue: name });
|
|
68
|
-
return void 0;
|
|
69
|
-
}
|
|
70
|
-
})();
|
|
71
|
-
}
|
|
72
|
-
return telemetryPromise;
|
|
73
|
-
}
|
|
74
52
|
async function getQueue() {
|
|
75
53
|
if (!bullQueue) {
|
|
76
54
|
const { Queue: BullQueueClass } = await getBullMQ();
|
|
77
|
-
|
|
78
|
-
bullQueue = new BullQueueClass(name, { connection, ...telemetry ? { telemetry } : {} });
|
|
55
|
+
bullQueue = new BullQueueClass(name, { connection });
|
|
79
56
|
}
|
|
80
57
|
return bullQueue;
|
|
81
58
|
}
|
|
82
59
|
async function enqueue(data, options2) {
|
|
83
60
|
const queue = await getQueue();
|
|
84
|
-
const telemetry = await getQueueTelemetry();
|
|
85
|
-
const metadata = telemetry ? void 0 : attachTraceMetadata(void 0);
|
|
86
61
|
const jobData = {
|
|
87
62
|
id: crypto.randomUUID(),
|
|
88
63
|
payload: data,
|
|
89
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
90
|
-
...metadata ? { metadata } : {}
|
|
64
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
91
65
|
};
|
|
92
66
|
const job = await queue.add(jobData.id, jobData, {
|
|
93
67
|
delay: options2?.delayMs && options2.delayMs > 0 ? options2.delayMs : void 0,
|
|
94
68
|
removeOnComplete: true,
|
|
95
69
|
removeOnFail: 1e3,
|
|
96
|
-
attempts,
|
|
70
|
+
attempts: 3,
|
|
97
71
|
backoff: { type: "exponential", delay: 1e3 }
|
|
98
72
|
});
|
|
99
73
|
return job.id ?? jobData.id;
|
|
100
74
|
}
|
|
101
75
|
async function process(handler) {
|
|
102
76
|
const { Worker } = await getBullMQ();
|
|
103
|
-
const telemetry = await getQueueTelemetry();
|
|
104
77
|
bullWorker = new Worker(
|
|
105
78
|
name,
|
|
106
79
|
async (job) => {
|
|
107
80
|
const jobData = job.data;
|
|
108
|
-
|
|
81
|
+
await handler(jobData, {
|
|
109
82
|
jobId: job.id ?? jobData.id,
|
|
110
83
|
attemptNumber: job.attemptsMade + 1,
|
|
111
84
|
queueName: name
|
|
112
|
-
};
|
|
113
|
-
if (telemetry) {
|
|
114
|
-
await handler(jobData, ctx);
|
|
115
|
-
} else {
|
|
116
|
-
await runJobInTrace(name, jobData.metadata, () => handler(jobData, ctx));
|
|
117
|
-
}
|
|
85
|
+
});
|
|
118
86
|
},
|
|
119
87
|
{
|
|
120
88
|
connection,
|
|
121
|
-
concurrency
|
|
122
|
-
...telemetry ? { telemetry } : {},
|
|
123
|
-
...lockDuration !== void 0 ? { lockDuration } : {},
|
|
124
|
-
...maxStalledCount !== void 0 ? { maxStalledCount } : {}
|
|
89
|
+
concurrency
|
|
125
90
|
}
|
|
126
91
|
);
|
|
127
92
|
bullWorker.on("completed", (job) => {
|
|
@@ -133,11 +98,6 @@ function createAsyncQueue(name, options) {
|
|
|
133
98
|
const error = err;
|
|
134
99
|
logger.error("Job failed", { jobId: jobWithId?.id, err: error });
|
|
135
100
|
});
|
|
136
|
-
bullWorker.on("stalled", (jobId) => {
|
|
137
|
-
logger.warn("Job stalled and will be redelivered \u2014 the handler may run concurrently with a previous delivery", {
|
|
138
|
-
jobId: typeof jobId === "string" ? jobId : null
|
|
139
|
-
});
|
|
140
|
-
});
|
|
141
101
|
bullWorker.on("error", (err) => {
|
|
142
102
|
const error = err;
|
|
143
103
|
logger.error("Worker error", { err: error });
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/async.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow, parseRedisUrl } from '@open-mercato/shared/lib/redis/connection'\nimport { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst packageLogger = createLogger('queue')\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n family?: number\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n data?: T\n remove: () => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions; telemetry?: unknown }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: {\n connection: ConnectionOptions\n concurrency: number\n telemetry?: unknown\n lockDuration?: number\n maxStalledCount?: number\n }\n ) => BullWorkerInterface\n}\n\n/** The `bullmq-otel` package (optional). Loaded only when an OTLP backend is active. */\ntype BullMQOtelModule = { BullMQOtel: new (tracerName: string) => object }\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects ioredis connection fields rather than a nested URL string.\n * Parse URL-based configuration at this boundary while keeping the public\n * queue API compatible with existing `{ url }` callers.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return parseRedisUrl(options.url)\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n family: options.family,\n }\n }\n\n return parseRedisUrl(getRedisUrlOrThrow('QUEUE'))\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n const attempts = options?.attempts ?? 3\n const lockDuration = options?.lockDuration\n const maxStalledCount = options?.maxStalledCount\n const logger = packageLogger.child({ queue: name })\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n // Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or\n // undefined (use our own metadata._trace carrier instead). Memoized as the\n // in-flight promise so concurrent first-time callers share one resolution.\n let telemetryPromise: Promise<object | undefined> | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n /**\n * When an OTLP backend is active, delegate async-queue tracing to `bullmq-otel`\n * (richer BullMQ-internal spans: add / process / wait / attempts). Returns\n * `undefined` \u2014 meaning \"use our own `metadata._trace` carrier\" \u2014 when telemetry\n * is off, a non-OTEL backend is selected, or `bullmq-otel` isn't installed. (The\n * `local` strategy always uses our carrier; it isn't BullMQ, so `bullmq-otel`\n * cannot instrument it.)\n */\n async function getQueueTelemetry(): Promise<object | undefined> {\n if (!telemetryPromise) {\n telemetryPromise = (async () => {\n if (!getTelemetryRuntime()?.canUseGlobalTracePropagation()) return undefined\n try {\n const mod = (await import('bullmq-otel')) as unknown as BullMQOtelModule\n return new mod.BullMQOtel('open-mercato')\n } catch {\n packageLogger.warn('bullmq-otel not available; using built-in trace carrier', { queue: name })\n return undefined\n }\n })()\n }\n return telemetryPromise\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n const telemetry = await getQueueTelemetry()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection, ...(telemetry ? { telemetry } : {}) })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n // When bullmq-otel handles propagation, don't also attach our carrier.\n const telemetry = await getQueueTelemetry()\n const metadata = telemetry ? undefined : attachTraceMetadata(undefined)\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(metadata ? { metadata } : {}),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n const telemetry = await getQueueTelemetry()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n const ctx = {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n }\n // With bullmq-otel active, BullMQ owns the process span and active\n // context (the handler's pg/undici spans nest under it). Otherwise\n // continue the trace from our own carrier.\n if (telemetry) {\n await handler(jobData, ctx)\n } else {\n await runJobInTrace(name, jobData.metadata, () => handler(jobData, ctx))\n }\n },\n {\n connection,\n concurrency,\n ...(telemetry ? { telemetry } : {}),\n ...(lockDuration !== undefined ? { lockDuration } : {}),\n ...(maxStalledCount !== undefined ? { maxStalledCount } : {}),\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n logger.info('Job completed', { jobId: jobWithId.id })\n })\n\n bullWorker.on('failed', (job, err) => {\n const jobWithId = job as { id?: string } | undefined\n const error = err as Error\n logger.error('Job failed', { jobId: jobWithId?.id, err: error })\n })\n\n // A stalled job is redelivered under the same id while the previous worker\n // may still be running it, so this is the signal that a handler is about to\n // be executed twice. BullMQ's docs require surfacing it: without this line\n // duplicate processing is invisible.\n bullWorker.on('stalled', (jobId) => {\n logger.warn('Job stalled and will be redelivered \u2014 the handler may run concurrently with a previous delivery', {\n jobId: typeof jobId === 'string' ? jobId : null,\n })\n })\n\n bullWorker.on('error', (err) => {\n const error = err as Error\n logger.error('Worker error', { err: error })\n })\n\n logger.info('Worker started', { concurrency })\n\n // For async strategy, return a sentinel result indicating worker mode\n // processed=-1 signals that this is a continuous worker, not a batch process\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const queue = await getQueue()\n\n // Obliterate removes all jobs from the queue\n await queue.obliterate({ force: true })\n\n return { removed: -1 } // BullMQ obliterate doesn't return count\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n const queue = await getQueue()\n const jobs = await queue.getJobs(REMOVABLE_JOB_STATES, 0, -1)\n let removed = 0\n\n for (const job of jobs) {\n if (!payloadMatchesScope(job.data?.payload, scope)) continue\n try {\n await job.remove()\n removed++\n } catch {\n // The job may have started between enumeration and removal. In-flight\n // cancellation is handled by the caller's lock/heartbeat contract.\n }\n }\n\n return { removed }\n }\n\n async function close(): Promise<void> {\n if (bullWorker) {\n await bullWorker.close()\n bullWorker = null\n }\n if (bullQueue) {\n await bullQueue.close()\n bullQueue = null\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const queue = await getQueue()\n const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')\n return {\n waiting: counts.waiting ?? 0,\n active: counts.active ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'async',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,
|
|
4
|
+
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst packageLogger = createLogger('queue')\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n url?: string\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n data?: T\n remove: () => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: { connection: ConnectionOptions; concurrency: number }\n ) => BullWorkerInterface\n}\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects an ioredis-compatible connection object. Preserve the full\n * Redis URL under the `url` key so rediss://, username, database, and query\n * params are not lost in translation.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return { url: options.url }\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n }\n }\n\n return { url: getRedisUrlOrThrow('QUEUE') }\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n const logger = packageLogger.child({ queue: name })\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts: 3,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n await handler(jobData, {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n })\n },\n {\n connection,\n concurrency,\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n logger.info('Job completed', { jobId: jobWithId.id })\n })\n\n bullWorker.on('failed', (job, err) => {\n const jobWithId = job as { id?: string } | undefined\n const error = err as Error\n logger.error('Job failed', { jobId: jobWithId?.id, err: error })\n })\n\n bullWorker.on('error', (err) => {\n const error = err as Error\n logger.error('Worker error', { err: error })\n })\n\n logger.info('Worker started', { concurrency })\n\n // For async strategy, return a sentinel result indicating worker mode\n // processed=-1 signals that this is a continuous worker, not a batch process\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const queue = await getQueue()\n\n // Obliterate removes all jobs from the queue\n await queue.obliterate({ force: true })\n\n return { removed: -1 } // BullMQ obliterate doesn't return count\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n const queue = await getQueue()\n const jobs = await queue.getJobs(REMOVABLE_JOB_STATES, 0, -1)\n let removed = 0\n\n for (const job of jobs) {\n if (!payloadMatchesScope(job.data?.payload, scope)) continue\n try {\n await job.remove()\n removed++\n } catch {\n // The job may have started between enumeration and removal. In-flight\n // cancellation is handled by the caller's lock/heartbeat contract.\n }\n }\n\n return { removed }\n }\n\n async function close(): Promise<void> {\n if (bullWorker) {\n await bullWorker.close()\n bullWorker = null\n }\n if (bullQueue) {\n await bullQueue.close()\n bullQueue = null\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const queue = await getQueue()\n const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')\n return {\n waiting: counts.waiting ?? 0,\n active: counts.active ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'async',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,0BAA0B;AACnC,SAAS,oBAAoB;AAE7B,MAAM,gBAAgB,aAAa,OAAO;AAiD1C,MAAM,uBAAuB,CAAC,WAAW,WAAW,eAAe,UAAU,kBAAkB;AAE/F,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AASA,SAAS,kBAAkB,SAA8D;AACvF,MAAI,SAAS,KAAK;AAChB,WAAO,EAAE,KAAK,QAAQ,IAAI;AAAA,EAC5B;AAEA,MAAI,SAAS,MAAM;AACjB,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ;AAAA,MACtB,UAAU,QAAQ;AAAA,MAClB,UAAU,QAAQ;AAAA,MAClB,IAAI,QAAQ;AAAA,MACZ,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAEA,SAAO,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAC5C;AAeO,SAAS,iBACd,MACA,SACU;AACV,QAAM,aAAa,kBAAkB,SAAS,UAAU;AACxD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,MAAI,YAAqD;AACzD,MAAI,aAAyC;AAC7C,MAAI,eAAoC;AAMxC,iBAAe,YAAmC;AAChD,QAAI,CAAC,cAAc;AACjB,UAAI;AACF,uBAAe,MAAM,OAAO,QAAQ;AAAA,MACtC,QAAQ;AACN,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,WAAsD;AACnE,QAAI,CAAC,WAAW;AACd,YAAM,EAAE,OAAO,eAAe,IAAI,MAAM,UAAU;AAClD,kBAAY,IAAI,eAA6B,MAAM,EAAE,WAAW,CAAC;AAAA,IACnE;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,UAAwB;AAAA,MAC5B,IAAI,OAAO,WAAW;AAAA,MACtB,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IACpC;AAEA,UAAM,MAAM,MAAM,MAAM,IAAI,QAAQ,IAAI,SAAS;AAAA,MAC/C,OAAOA,UAAS,WAAWA,SAAQ,UAAU,IAAIA,SAAQ,UAAU;AAAA,MACnE,kBAAkB;AAAA,MAClB,cAAc;AAAA,MACd,UAAU;AAAA,MACV,SAAS,EAAE,MAAM,eAAe,OAAO,IAAK;AAAA,IAC9C,CAAC;AAED,WAAO,IAAI,MAAM,QAAQ;AAAA,EAC3B;AAEA,iBAAe,QAAQ,SAAgD;AACrE,UAAM,EAAE,OAAO,IAAI,MAAM,UAAU;AAGnC,iBAAa,IAAI;AAAA,MACf;AAAA,MACA,OAAO,QAAQ;AACb,cAAM,UAAU,IAAI;AACpB,cAAM,QAAQ,SAAS;AAAA,UACrB,OAAO,IAAI,MAAM,QAAQ;AAAA,UACzB,eAAe,IAAI,eAAe;AAAA,UAClC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAGA,eAAW,GAAG,aAAa,CAAC,QAAQ;AAClC,YAAM,YAAY;AAClB,aAAO,KAAK,iBAAiB,EAAE,OAAO,UAAU,GAAG,CAAC;AAAA,IACtD,CAAC;AAED,eAAW,GAAG,UAAU,CAAC,KAAK,QAAQ;AACpC,YAAM,YAAY;AAClB,YAAM,QAAQ;AACd,aAAO,MAAM,cAAc,EAAE,OAAO,WAAW,IAAI,KAAK,MAAM,CAAC;AAAA,IACjE,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,QAAQ;AAC9B,YAAM,QAAQ;AACd,aAAO,MAAM,gBAAgB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC7C,CAAC;AAED,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAI7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,UAAM,QAAQ,MAAM,SAAS;AAG7B,UAAM,MAAM,WAAW,EAAE,OAAO,KAAK,CAAC;AAEtC,WAAO,EAAE,SAAS,GAAG;AAAA,EACvB;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,OAAO,MAAM,MAAM,QAAQ,sBAAsB,GAAG,EAAE;AAC5D,QAAI,UAAU;AAEd,eAAW,OAAO,MAAM;AACtB,UAAI,CAAC,oBAAoB,IAAI,MAAM,SAAS,KAAK,EAAG;AACpD,UAAI;AACF,cAAM,IAAI,OAAO;AACjB;AAAA,MACF,QAAQ;AAAA,MAGR;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ;AAAA,EACnB;AAEA,iBAAe,QAAuB;AACpC,QAAI,YAAY;AACd,YAAM,WAAW,MAAM;AACvB,mBAAa;AAAA,IACf;AACA,QAAI,WAAW;AACb,YAAM,UAAU,MAAM;AACtB,kBAAY;AAAA,IACd;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,SAAS,MAAM,MAAM,aAAa,WAAW,UAAU,aAAa,QAAQ;AAClF,WAAO;AAAA,MACL,SAAS,OAAO,WAAW;AAAA,MAC3B,QAAQ,OAAO,UAAU;AAAA,MACzB,WAAW,OAAO,aAAa;AAAA,MAC/B,QAAQ,OAAO,UAAU;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/dist/strategies/local.js
CHANGED
|
@@ -2,7 +2,6 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import crypto from "node:crypto";
|
|
4
4
|
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
5
|
-
import { attachTraceMetadata, runJobInTrace } from "../tracing.js";
|
|
6
5
|
const packageLogger = createLogger("queue");
|
|
7
6
|
function payloadMatchesScope(payload, scope) {
|
|
8
7
|
if (!payload || typeof payload !== "object") return false;
|
|
@@ -119,13 +118,11 @@ function createLocalQueue(name, options) {
|
|
|
119
118
|
}
|
|
120
119
|
async function enqueue(data, options2) {
|
|
121
120
|
const availableAt = options2?.delayMs && options2.delayMs > 0 ? new Date(Date.now() + options2.delayMs).toISOString() : void 0;
|
|
122
|
-
const metadata = attachTraceMetadata(void 0);
|
|
123
121
|
const job = {
|
|
124
122
|
id: generateId(),
|
|
125
123
|
payload: data,
|
|
126
124
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
127
|
-
...availableAt ? { availableAt } : {}
|
|
128
|
-
...metadata ? { metadata } : {}
|
|
125
|
+
...availableAt ? { availableAt } : {}
|
|
129
126
|
};
|
|
130
127
|
await withFileLock(async () => {
|
|
131
128
|
const jobs = await readQueue();
|
|
@@ -158,16 +155,12 @@ function createLocalQueue(name, options) {
|
|
|
158
155
|
for (const job of jobsToProcess) {
|
|
159
156
|
const attemptNumber = (job.attemptCount ?? 0) + 1;
|
|
160
157
|
try {
|
|
161
|
-
await
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
attemptNumber,
|
|
168
|
-
queueName: name
|
|
169
|
-
})
|
|
170
|
-
)
|
|
158
|
+
await Promise.resolve(
|
|
159
|
+
handler(job, {
|
|
160
|
+
jobId: job.id,
|
|
161
|
+
attemptNumber,
|
|
162
|
+
queueName: name
|
|
163
|
+
})
|
|
171
164
|
);
|
|
172
165
|
processed++;
|
|
173
166
|
lastJobId = job.id;
|
|
@@ -178,7 +171,7 @@ function createLocalQueue(name, options) {
|
|
|
178
171
|
failed++;
|
|
179
172
|
lastJobId = job.id;
|
|
180
173
|
if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {
|
|
181
|
-
logger.error("Job exhausted all attempts
|
|
174
|
+
logger.error("Job exhausted all attempts, moving to dead letter", { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS });
|
|
182
175
|
deadJobIds.add(job.id);
|
|
183
176
|
} else {
|
|
184
177
|
const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/local.ts"],
|
|
4
|
-
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.\n * **This strategy keeps no failed-job store**: once attempts are exhausted the job is\n * removed from `queue.json` and only counted in `state.failedCount`, so the payload is\n * lost and the failure survives solely as an error log line. The `async` strategy keeps\n * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000\n * failures and removes older ones as later failures arrive. Workflows that require\n * no-loss persistence must write their own durable record before enqueueing, regardless\n * of strategy.\n *\n * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option \u2014 callers cannot\n * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)\n * See the retry handling in `process()` below.\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences to preserve the atomicity guarantees the\n * previous synchronous implementation relied on.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(() => fn(), () => fn())\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n async function backupCorruptedQueueFile(content: string): Promise<string> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n await fsp.writeFile(backupFile, content, 'utf8')\n await fsp.writeFile(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await backupCorruptedQueueFile(content)\n logger.error('Backed up corrupted queue file and recreated queue.json', { backupFile })\n return []\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const metadata = attachTraceMetadata(undefined)\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n ...(metadata ? { metadata } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await runJobInTrace(name, job.metadata, () =>\n Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, pollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;
|
|
4
|
+
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential\n * backoff and moved to a dead-letter store once attempts are exhausted (see the\n * retry handling in `process()` below).\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences to preserve the atomicity guarantees the\n * previous synchronous implementation relied on.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(() => fn(), () => fn())\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n async function backupCorruptedQueueFile(content: string): Promise<string> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n await fsp.writeFile(backupFile, content, 'utf8')\n await fsp.writeFile(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await backupCorruptedQueueFile(content)\n logger.error('Backed up corrupted queue file and recreated queue.json', { backupFile })\n return []\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts, moving to dead letter', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, pollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAG7B,MAAM,gBAAgB,aAAa,OAAO;AAa1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAE9B,MAAM,MAAM,GAAG;AA0BR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAG9C,MAAI,eAAsD;AAC1D,MAAI,eAAe;AACnB,MAAI,gBAAsC;AAC1C,QAAM,iBAAiB,oBAAI,IAAY;AAIvC,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY,KAAK,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC;AACnD,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAEA,iBAAe,yBAAyB,SAAkC;AACxE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,OAAO;AAC3E,UAAM,IAAI,UAAU,YAAY,SAAS,MAAM;AAC/C,UAAM,IAAI,UAAU,WAAW,MAAM,MAAM;AAC3C,WAAO;AAAA,EACT;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,aAAO,MAAM,6BAA6B,EAAE,KAAK,UAAU,CAAC;AAC5D,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAW,CAAC;AAC9D,YAAM,aAAa,MAAM,yBAAyB,OAAO;AACzD,aAAO,MAAM,2DAA2D,EAAE,WAAW,CAAC;AACtF,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,MAAM;AAAA,EACtE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,MAAM;AAAA,EACvE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,IACvC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AAED,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,eAAW,OAAO,eAAe;AAC/B,qBAAe,IAAI,IAAI,EAAE;AAAA,IAC3B;AAEA,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,QAAI;AACF,iBAAW,OAAO,eAAe;AAC/B,cAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,YAAI;AACF,gBAAM,QAAQ;AAAA,YACZ,QAAQ,KAAK;AAAA,cACX,OAAO,IAAI;AAAA,cACX;AAAA,cACA,WAAW;AAAA,YACb,CAAC;AAAA,UACH;AACA;AACA,sBAAY,IAAI;AAChB,0BAAgB,IAAI,IAAI,EAAE;AAC1B,iBAAO,KAAK,iBAAiB,EAAE,OAAO,IAAI,GAAG,CAAC;AAAA,QAChD,SAAS,OAAO;AACd,iBAAO,MAAM,cAAc,EAAE,OAAO,IAAI,IAAI,eAAe,aAAa,sBAAsB,KAAK,MAAM,CAAC;AAC1G;AACA,sBAAY,IAAI;AAChB,cAAI,iBAAiB,sBAAsB;AACzC,mBAAO,MAAM,qDAAqD,EAAE,OAAO,IAAI,IAAI,aAAa,qBAAqB,CAAC;AACtH,uBAAW,IAAI,IAAI,EAAE;AAAA,UACvB,OAAO;AACL,kBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,yBAAa,IAAI,IAAI,IAAI;AAAA,cACvB,GAAG;AAAA,cACH,cAAc;AAAA,cACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,YAC5D,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,UAAI,YAAY;AACd,cAAM,aAAa,YAAY;AAE7B,gBAAM,cAAc,MAAM,UAAU;AACpC,gBAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,gBAAM,WAAW,WAAW;AAE5B,gBAAM,WAAuB;AAAA,YAC3B,iBAAiB;AAAA,YACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,YAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,UACrD;AACA,gBAAM,WAAW,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAEA,aAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,IACxC,UAAE;AACA,iBAAW,OAAO,eAAe;AAC/B,uBAAe,OAAO,IAAI,EAAE;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAKA,iBAAe,iBAAgC;AAE7C,QAAI,gBAAgB,CAAC,cAAe;AAEpC,mBAAe;AACf,QAAI;AACF,YAAM,aAAa,aAAa;AAAA,IAClC,SAAS,OAAO;AACd,aAAO,MAAM,iBAAiB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC9C,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAGA,oBAAgB;AAGhB,UAAM,aAAa,OAAO;AAG1B,mBAAe,YAAY,MAAM;AAC/B,qBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,eAAO,MAAM,oBAAoB,EAAE,IAAI,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH,GAAG,YAAY;AAEf,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAG7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,eAAe,KAAK,OAAO,CAAC,QAAQ,eAAe,IAAI,IAAI,EAAE,KAAK,CAAC,oBAAoB,IAAI,SAAS,KAAK,CAAC;AAChH,YAAM,UAAU,KAAK,SAAS,aAAa;AAC3C,UAAI,UAAU,GAAG;AACf,cAAM,WAAW,YAAY;AAAA,MAC/B;AACA,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AAEpC,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AACA,oBAAgB;AAGhB,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,eAAO,KAAK,wCAAwC,EAAE,WAAW,iBAAiB,CAAC;AACnF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/dist/worker/runner.js
CHANGED
|
@@ -1,9 +1,5 @@
|
|
|
1
1
|
import { createQueue } from "../factory.js";
|
|
2
2
|
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
3
|
-
import {
|
|
4
|
-
getTelemetryRuntime,
|
|
5
|
-
isTelemetryBackendEnabled
|
|
6
|
-
} from "@open-mercato/shared/lib/telemetry/runtime";
|
|
7
3
|
const logger = createLogger("queue").child({ component: "worker" });
|
|
8
4
|
const managedQueues = /* @__PURE__ */ new Set();
|
|
9
5
|
const managedShutdownHooks = /* @__PURE__ */ new Set();
|
|
@@ -41,11 +37,6 @@ function registerShutdownHandlers() {
|
|
|
41
37
|
managedShutdownHooks.clear();
|
|
42
38
|
unregisterShutdownHandlers(sigtermHandler, sigintHandler);
|
|
43
39
|
shutdownInProgress = false;
|
|
44
|
-
try {
|
|
45
|
-
await getTelemetryRuntime()?.shutdown();
|
|
46
|
-
} catch (error) {
|
|
47
|
-
logger.error("Error flushing telemetry during shutdown", { err: error });
|
|
48
|
-
}
|
|
49
40
|
if (!hasError) {
|
|
50
41
|
logger.info("Worker closed successfully");
|
|
51
42
|
}
|
|
@@ -71,23 +62,15 @@ async function runWorker(options) {
|
|
|
71
62
|
handler,
|
|
72
63
|
connection,
|
|
73
64
|
concurrency = 1,
|
|
74
|
-
lockDuration,
|
|
75
|
-
maxStalledCount,
|
|
76
65
|
gracefulShutdown = true,
|
|
77
66
|
background = false,
|
|
78
67
|
strategy: strategyOption
|
|
79
68
|
} = options;
|
|
80
|
-
if (!getTelemetryRuntime() && isTelemetryBackendEnabled()) {
|
|
81
|
-
const { initTelemetry } = await import("@open-mercato/telemetry");
|
|
82
|
-
await initTelemetry();
|
|
83
|
-
}
|
|
84
69
|
const strategy = strategyOption ?? (process.env.QUEUE_STRATEGY === "async" ? "async" : "local");
|
|
85
70
|
logger.info("Starting worker for queue", { queueName, strategy });
|
|
86
71
|
const queue = createQueue(queueName, strategy, {
|
|
87
72
|
connection,
|
|
88
|
-
concurrency
|
|
89
|
-
lockDuration,
|
|
90
|
-
maxStalledCount
|
|
73
|
+
concurrency
|
|
91
74
|
});
|
|
92
75
|
if (gracefulShutdown) {
|
|
93
76
|
managedQueues.add(queue);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/worker/runner.ts"],
|
|
4
|
-
"sourcesContent": ["import { createQueue } from '../factory'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport
|
|
5
|
-
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;
|
|
4
|
+
"sourcesContent": ["import { createQueue } from '../factory'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, JobHandler, AsyncQueueOptions, QueueStrategyType } from '../types'\n\nconst logger = createLogger('queue').child({ component: 'worker' })\n\n/**\n * Options for running a queue worker.\n */\nexport type WorkerRunnerOptions<T = unknown> = {\n /** Name of the queue to process */\n queueName: string\n /** Handler function to process each job */\n handler: JobHandler<T>\n /** Redis connection options (only used for async strategy) */\n connection?: AsyncQueueOptions['connection']\n /** Number of concurrent jobs to process */\n concurrency?: number\n /** Whether to set up graceful shutdown handlers */\n gracefulShutdown?: boolean\n /** If true, don't block - return immediately after starting processing (for multi-queue mode) */\n background?: boolean\n /** Queue strategy to use. Defaults to QUEUE_STRATEGY env var or 'local' */\n strategy?: QueueStrategyType\n}\n\nconst managedQueues = new Set<Queue<unknown>>()\nconst managedShutdownHooks = new Set<() => Promise<void> | void>()\nlet shutdownHandlersRegistered = false\nlet shutdownInProgress = false\n\nfunction unregisterShutdownHandlers(sigtermHandler: () => void, sigintHandler: () => void): void {\n process.off('SIGTERM', sigtermHandler)\n process.off('SIGINT', sigintHandler)\n shutdownHandlersRegistered = false\n}\n\nfunction registerShutdownHandlers(): void {\n if (shutdownHandlersRegistered) return\n\n const shutdown = async (signal: string) => {\n if (shutdownInProgress) return\n shutdownInProgress = true\n\n logger.info('Received signal, shutting down gracefully', { signal })\n\n let hasError = false\n for (const queue of managedQueues) {\n try {\n await queue.close()\n } catch (error) {\n hasError = true\n logger.error('Error during shutdown', { err: error })\n }\n }\n\n managedQueues.clear()\n for (const hook of managedShutdownHooks) {\n try {\n await hook()\n } catch (error) {\n hasError = true\n logger.error('Error during shutdown hook', { err: error })\n }\n }\n managedShutdownHooks.clear()\n unregisterShutdownHandlers(sigtermHandler, sigintHandler)\n shutdownInProgress = false\n\n if (!hasError) {\n logger.info('Worker closed successfully')\n }\n\n process.exit(hasError ? 1 : 0)\n }\n\n const sigtermHandler = () => {\n void shutdown('SIGTERM')\n }\n\n const sigintHandler = () => {\n void shutdown('SIGINT')\n }\n\n process.on('SIGTERM', sigtermHandler)\n process.on('SIGINT', sigintHandler)\n shutdownHandlersRegistered = true\n}\n\n/**\n * Register a process-local service that must stop before a worker exits.\n * The returned callback removes the hook when the service is stopped early.\n */\nexport function registerWorkerShutdownHook(hook: () => Promise<void> | void): () => void {\n managedShutdownHooks.add(hook)\n return () => managedShutdownHooks.delete(hook)\n}\n\n/**\n * Runs a queue worker that processes jobs continuously.\n *\n * This function:\n * 1. Creates an async queue instance\n * 2. Starts a BullMQ worker\n * 3. Sets up graceful shutdown on SIGTERM/SIGINT\n * 4. Keeps the process running until shutdown\n *\n * @template T - The job payload type\n * @param options - Worker configuration\n *\n * @example\n * ```typescript\n * import { runWorker } from '@open-mercato/queue/worker'\n *\n * await runWorker({\n * queueName: 'events',\n * handler: async (job, ctx) => {\n * console.log(`Processing ${ctx.jobId}:`, job.payload)\n * },\n * connection: { url: process.env.REDIS_URL },\n * concurrency: 5,\n * })\n * ```\n */\nexport async function runWorker<T = unknown>(\n options: WorkerRunnerOptions<T>\n): Promise<void> {\n const {\n queueName,\n handler,\n connection,\n concurrency = 1,\n gracefulShutdown = true,\n background = false,\n strategy: strategyOption,\n } = options\n\n // Determine queue strategy from option, env var, or default to 'local'\n const strategy: QueueStrategyType = strategyOption\n ?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')\n\n logger.info('Starting worker for queue', { queueName, strategy })\n\n const queue = createQueue<T>(queueName, strategy, {\n connection,\n concurrency,\n })\n\n // Set up graceful shutdown\n if (gracefulShutdown) {\n managedQueues.add(queue as Queue<unknown>)\n registerShutdownHandlers()\n }\n\n // Start processing\n await queue.process(handler)\n\n logger.info('Worker running', { concurrency })\n\n if (background) {\n // Return immediately for multi-queue mode\n return\n }\n\n logger.info('Press Ctrl+C to stop')\n\n // Keep the process alive (single-queue mode)\n await new Promise(() => {\n // This promise never resolves, keeping the worker running\n })\n}\n\n/**\n * Creates a worker handler that routes jobs to specific handlers based on job type.\n *\n * @template T - Base job payload type (must include a 'type' field)\n * @param handlers - Map of job types to their handlers\n *\n * @example\n * ```typescript\n * const handler = createRoutedHandler({\n * 'user.created': async (job) => { ... },\n * 'order.placed': async (job) => { ... },\n * })\n *\n * await runWorker({ queueName: 'events', handler })\n * ```\n */\nexport function createRoutedHandler<T extends { type: string }>(\n handlers: Record<string, JobHandler<T>>\n): JobHandler<T> {\n return async (job, ctx) => {\n const type = job.payload.type\n const handler = handlers[type]\n\n if (!handler) {\n logger.warn('No handler registered for job type', { type })\n return\n }\n\n await handler(job, ctx)\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAG7B,MAAM,SAAS,aAAa,OAAO,EAAE,MAAM,EAAE,WAAW,SAAS,CAAC;AAsBlE,MAAM,gBAAgB,oBAAI,IAAoB;AAC9C,MAAM,uBAAuB,oBAAI,IAAgC;AACjE,IAAI,6BAA6B;AACjC,IAAI,qBAAqB;AAEzB,SAAS,2BAA2B,gBAA4B,eAAiC;AAC/F,UAAQ,IAAI,WAAW,cAAc;AACrC,UAAQ,IAAI,UAAU,aAAa;AACnC,+BAA6B;AAC/B;AAEA,SAAS,2BAAiC;AACxC,MAAI,2BAA4B;AAEhC,QAAM,WAAW,OAAO,WAAmB;AACzC,QAAI,mBAAoB;AACxB,yBAAqB;AAErB,WAAO,KAAK,6CAA6C,EAAE,OAAO,CAAC;AAEnE,QAAI,WAAW;AACf,eAAW,SAAS,eAAe;AACjC,UAAI;AACF,cAAM,MAAM,MAAM;AAAA,MACpB,SAAS,OAAO;AACd,mBAAW;AACX,eAAO,MAAM,yBAAyB,EAAE,KAAK,MAAM,CAAC;AAAA,MACtD;AAAA,IACF;AAEA,kBAAc,MAAM;AACpB,eAAW,QAAQ,sBAAsB;AACvC,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAO;AACd,mBAAW;AACX,eAAO,MAAM,8BAA8B,EAAE,KAAK,MAAM,CAAC;AAAA,MAC3D;AAAA,IACF;AACA,yBAAqB,MAAM;AAC3B,+BAA2B,gBAAgB,aAAa;AACxD,yBAAqB;AAErB,QAAI,CAAC,UAAU;AACb,aAAO,KAAK,4BAA4B;AAAA,IAC1C;AAEA,YAAQ,KAAK,WAAW,IAAI,CAAC;AAAA,EAC/B;AAEA,QAAM,iBAAiB,MAAM;AAC3B,SAAK,SAAS,SAAS;AAAA,EACzB;AAEA,QAAM,gBAAgB,MAAM;AAC1B,SAAK,SAAS,QAAQ;AAAA,EACxB;AAEA,UAAQ,GAAG,WAAW,cAAc;AACpC,UAAQ,GAAG,UAAU,aAAa;AAClC,+BAA6B;AAC/B;AAMO,SAAS,2BAA2B,MAA8C;AACvF,uBAAqB,IAAI,IAAI;AAC7B,SAAO,MAAM,qBAAqB,OAAO,IAAI;AAC/C;AA4BA,eAAsB,UACpB,SACe;AACf,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd,mBAAmB;AAAA,IACnB,aAAa;AAAA,IACb,UAAU;AAAA,EACZ,IAAI;AAGJ,QAAM,WAA8B,mBAC9B,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAEzD,SAAO,KAAK,6BAA6B,EAAE,WAAW,SAAS,CAAC;AAEhE,QAAM,QAAQ,YAAe,WAAW,UAAU;AAAA,IAChD;AAAA,IACA;AAAA,EACF,CAAC;AAGD,MAAI,kBAAkB;AACpB,kBAAc,IAAI,KAAuB;AACzC,6BAAyB;AAAA,EAC3B;AAGA,QAAM,MAAM,QAAQ,OAAO;AAE3B,SAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAE7C,MAAI,YAAY;AAEd;AAAA,EACF;AAEA,SAAO,KAAK,sBAAsB;AAGlC,QAAM,IAAI,QAAQ,MAAM;AAAA,EAExB,CAAC;AACH;AAkBO,SAAS,oBACd,UACe;AACf,SAAO,OAAO,KAAK,QAAQ;AACzB,UAAM,OAAO,IAAI,QAAQ;AACzB,UAAM,UAAU,SAAS,IAAI;AAE7B,QAAI,CAAC,SAAS;AACZ,aAAO,KAAK,sCAAsC,EAAE,KAAK,CAAC;AAC1D;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,GAAG;AAAA,EACxB;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|