@open-mercato/queue 0.6.6-develop.6465.1.019f0fb26f → 0.6.6-develop.6471.1.9ab5bdd79a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/strategies/async.js +7 -4
- package/dist/strategies/async.js.map +2 -2
- package/dist/strategies/local.js +13 -10
- package/dist/strategies/local.js.map +2 -2
- package/dist/worker/registry.js +3 -1
- package/dist/worker/registry.js.map +2 -2
- package/dist/worker/runner.js +9 -7
- package/dist/worker/runner.js.map +2 -2
- package/package.json +2 -2
- package/src/__tests__/local.strategy.test.ts +22 -15
- package/src/strategies/async.ts +8 -4
- package/src/strategies/local.ts +14 -10
- package/src/worker/registry.ts +4 -1
- package/src/worker/runner.ts +10 -7
package/dist/strategies/async.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { getRedisUrlOrThrow } from "@open-mercato/shared/lib/redis/connection";
|
|
2
|
+
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
3
|
+
const packageLogger = createLogger("queue");
|
|
2
4
|
const REMOVABLE_JOB_STATES = ["waiting", "delayed", "prioritized", "paused", "waiting-children"];
|
|
3
5
|
function payloadMatchesScope(payload, scope) {
|
|
4
6
|
if (!payload || typeof payload !== "object") return false;
|
|
@@ -31,6 +33,7 @@ function resolveConnection(options) {
|
|
|
31
33
|
function createAsyncQueue(name, options) {
|
|
32
34
|
const connection = resolveConnection(options?.connection);
|
|
33
35
|
const concurrency = options?.concurrency ?? 1;
|
|
36
|
+
const logger = packageLogger.child({ queue: name });
|
|
34
37
|
let bullQueue = null;
|
|
35
38
|
let bullWorker = null;
|
|
36
39
|
let bullmqModule = null;
|
|
@@ -88,18 +91,18 @@ function createAsyncQueue(name, options) {
|
|
|
88
91
|
);
|
|
89
92
|
bullWorker.on("completed", (job) => {
|
|
90
93
|
const jobWithId = job;
|
|
91
|
-
|
|
94
|
+
logger.info("Job completed", { jobId: jobWithId.id });
|
|
92
95
|
});
|
|
93
96
|
bullWorker.on("failed", (job, err) => {
|
|
94
97
|
const jobWithId = job;
|
|
95
98
|
const error = err;
|
|
96
|
-
|
|
99
|
+
logger.error("Job failed", { jobId: jobWithId?.id, err: error });
|
|
97
100
|
});
|
|
98
101
|
bullWorker.on("error", (err) => {
|
|
99
102
|
const error = err;
|
|
100
|
-
|
|
103
|
+
logger.error("Worker error", { err: error });
|
|
101
104
|
});
|
|
102
|
-
|
|
105
|
+
logger.info("Worker started", { concurrency });
|
|
103
106
|
return { processed: -1, failed: -1, lastJobId: void 0 };
|
|
104
107
|
}
|
|
105
108
|
async function clear() {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/async.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n url?: string\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n data?: T\n remove: () => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: { connection: ConnectionOptions; concurrency: number }\n ) => BullWorkerInterface\n}\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects an ioredis-compatible connection object. Preserve the full\n * Redis URL under the `url` key so rediss://, username, database, and query\n * params are not lost in translation.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return { url: options.url }\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n }\n }\n\n return { url: getRedisUrlOrThrow('QUEUE') }\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts: 3,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n await handler(jobData, {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n })\n },\n {\n connection,\n concurrency,\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n
|
|
5
|
-
"mappings": "AACA,SAAS,0BAA0B;
|
|
4
|
+
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst packageLogger = createLogger('queue')\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n url?: string\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n data?: T\n remove: () => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: { connection: ConnectionOptions; concurrency: number }\n ) => BullWorkerInterface\n}\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects an ioredis-compatible connection object. Preserve the full\n * Redis URL under the `url` key so rediss://, username, database, and query\n * params are not lost in translation.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return { url: options.url }\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n }\n }\n\n return { url: getRedisUrlOrThrow('QUEUE') }\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n const logger = packageLogger.child({ queue: name })\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts: 3,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n await handler(jobData, {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n })\n },\n {\n connection,\n concurrency,\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n logger.info('Job completed', { jobId: jobWithId.id })\n })\n\n bullWorker.on('failed', (job, err) => {\n const jobWithId = job as { id?: string } | undefined\n const error = err as Error\n logger.error('Job failed', { jobId: jobWithId?.id, err: error })\n })\n\n bullWorker.on('error', (err) => {\n const error = err as Error\n logger.error('Worker error', { err: error })\n })\n\n logger.info('Worker started', { concurrency })\n\n // For async strategy, return a sentinel result indicating worker mode\n // processed=-1 signals that this is a continuous worker, not a batch process\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const queue = await getQueue()\n\n // Obliterate removes all jobs from the queue\n await queue.obliterate({ force: true })\n\n return { removed: -1 } // BullMQ obliterate doesn't return count\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n const queue = await getQueue()\n const jobs = await queue.getJobs(REMOVABLE_JOB_STATES, 0, -1)\n let removed = 0\n\n for (const job of jobs) {\n if (!payloadMatchesScope(job.data?.payload, scope)) continue\n try {\n await job.remove()\n removed++\n } catch {\n // The job may have started between enumeration and removal. In-flight\n // cancellation is handled by the caller's lock/heartbeat contract.\n }\n }\n\n return { removed }\n }\n\n async function close(): Promise<void> {\n if (bullWorker) {\n await bullWorker.close()\n bullWorker = null\n }\n if (bullQueue) {\n await bullQueue.close()\n bullQueue = null\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const queue = await getQueue()\n const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')\n return {\n waiting: counts.waiting ?? 0,\n active: counts.active ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'async',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,0BAA0B;AACnC,SAAS,oBAAoB;AAE7B,MAAM,gBAAgB,aAAa,OAAO;AAiD1C,MAAM,uBAAuB,CAAC,WAAW,WAAW,eAAe,UAAU,kBAAkB;AAE/F,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AASA,SAAS,kBAAkB,SAA8D;AACvF,MAAI,SAAS,KAAK;AAChB,WAAO,EAAE,KAAK,QAAQ,IAAI;AAAA,EAC5B;AAEA,MAAI,SAAS,MAAM;AACjB,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ;AAAA,MACtB,UAAU,QAAQ;AAAA,MAClB,UAAU,QAAQ;AAAA,MAClB,IAAI,QAAQ;AAAA,MACZ,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAEA,SAAO,EAAE,KAAK,mBAAmB,OAAO,EAAE;AAC5C;AAeO,SAAS,iBACd,MACA,SACU;AACV,QAAM,aAAa,kBAAkB,SAAS,UAAU;AACxD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,MAAI,YAAqD;AACzD,MAAI,aAAyC;AAC7C,MAAI,eAAoC;AAMxC,iBAAe,YAAmC;AAChD,QAAI,CAAC,cAAc;AACjB,UAAI;AACF,uBAAe,MAAM,OAAO,QAAQ;AAAA,MACtC,QAAQ;AACN,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,WAAsD;AACnE,QAAI,CAAC,WAAW;AACd,YAAM,EAAE,OAAO,eAAe,IAAI,MAAM,UAAU;AAClD,kBAAY,IAAI,eAA6B,MAAM,EAAE,WAAW,CAAC;AAAA,IACnE;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,UAAwB;AAAA,MAC5B,IAAI,OAAO,WAAW;AAAA,MACtB,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IACpC;AAEA,UAAM,MAAM,MAAM,MAAM,IAAI,QAAQ,IAAI,SAAS;AAAA,MAC/C,OAAOA,UAAS,WAAWA,SAAQ,UAAU,IAAIA,SAAQ,UAAU;AAAA,MACnE,kBAAkB;AAAA,MAClB,cAAc;AAAA,MACd,UAAU;AAAA,MACV,SAAS,EAAE,MAAM,eAAe,OAAO,IAAK;AAAA,IAC9C,CAAC;AAED,WAAO,IAAI,MAAM,QAAQ;AAAA,EAC3B;AAEA,iBAAe,QAAQ,SAAgD;AACrE,UAAM,EAAE,OAAO,IAAI,MAAM,UAAU;AAGnC,iBAAa,IAAI;AAAA,MACf;AAAA,MACA,OAAO,QAAQ;AACb,cAAM,UAAU,IAAI;AACpB,cAAM,QAAQ,SAAS;AAAA,UACrB,OAAO,IAAI,MAAM,QAAQ;AAAA,UACzB,eAAe,IAAI,eAAe;AAAA,UAClC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAGA,eAAW,GAAG,aAAa,CAAC,QAAQ;AAClC,YAAM,YAAY;AAClB,aAAO,KAAK,iBAAiB,EAAE,OAAO,UAAU,GAAG,CAAC;AAAA,IACtD,CAAC;AAED,eAAW,GAAG,UAAU,CAAC,KAAK,QAAQ;AACpC,YAAM,YAAY;AAClB,YAAM,QAAQ;AACd,aAAO,MAAM,cAAc,EAAE,OAAO,WAAW,IAAI,KAAK,MAAM,CAAC;AAAA,IACjE,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,QAAQ;AAC9B,YAAM,QAAQ;AACd,aAAO,MAAM,gBAAgB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC7C,CAAC;AAED,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAI7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,UAAM,QAAQ,MAAM,SAAS;AAG7B,UAAM,MAAM,WAAW,EAAE,OAAO,KAAK,CAAC;AAEtC,WAAO,EAAE,SAAS,GAAG;AAAA,EACvB;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,OAAO,MAAM,MAAM,QAAQ,sBAAsB,GAAG,EAAE;AAC5D,QAAI,UAAU;AAEd,eAAW,OAAO,MAAM;AACtB,UAAI,CAAC,oBAAoB,IAAI,MAAM,SAAS,KAAK,EAAG;AACpD,UAAI;AACF,cAAM,IAAI,OAAO;AACjB;AAAA,MACF,QAAQ;AAAA,MAGR;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ;AAAA,EACnB;AAEA,iBAAe,QAAuB;AACpC,QAAI,YAAY;AACd,YAAM,WAAW,MAAM;AACvB,mBAAa;AAAA,IACf;AACA,QAAI,WAAW;AACb,YAAM,UAAU,MAAM;AACtB,kBAAY;AAAA,IACd;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,SAAS,MAAM,MAAM,aAAa,WAAW,UAAU,aAAa,QAAQ;AAClF,WAAO;AAAA,MACL,SAAS,OAAO,WAAW;AAAA,MAC3B,QAAQ,OAAO,UAAU;AAAA,MACzB,WAAW,OAAO,aAAa;AAAA,MAC/B,QAAQ,OAAO,UAAU;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/dist/strategies/local.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import crypto from "node:crypto";
|
|
4
|
+
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
5
|
+
const packageLogger = createLogger("queue");
|
|
4
6
|
function payloadMatchesScope(payload, scope) {
|
|
5
7
|
if (!payload || typeof payload !== "object") return false;
|
|
6
8
|
const scopedPayload = payload;
|
|
@@ -25,6 +27,7 @@ function createLocalQueue(name, options) {
|
|
|
25
27
|
const queueDir = path.join(baseDir, name);
|
|
26
28
|
const queueFile = path.join(queueDir, "queue.json");
|
|
27
29
|
const stateFile = path.join(queueDir, "state.json");
|
|
30
|
+
const logger = packageLogger.child({ queue: name });
|
|
28
31
|
const concurrency = options?.concurrency ?? 1;
|
|
29
32
|
const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL;
|
|
30
33
|
let pollingTimer = null;
|
|
@@ -76,7 +79,7 @@ function createLocalQueue(name, options) {
|
|
|
76
79
|
if (readError.code === "ENOENT") {
|
|
77
80
|
return [];
|
|
78
81
|
}
|
|
79
|
-
|
|
82
|
+
logger.error("Failed to read queue file", { err: readError });
|
|
80
83
|
throw new Error(`Queue file unreadable: ${readError.message}`);
|
|
81
84
|
}
|
|
82
85
|
try {
|
|
@@ -87,9 +90,9 @@ function createLocalQueue(name, options) {
|
|
|
87
90
|
return parsed;
|
|
88
91
|
} catch (error) {
|
|
89
92
|
const parseError = error;
|
|
90
|
-
|
|
93
|
+
logger.error("Failed to parse queue file", { err: parseError });
|
|
91
94
|
const backupFile = await backupCorruptedQueueFile(content);
|
|
92
|
-
|
|
95
|
+
logger.error("Backed up corrupted queue file and recreated queue.json", { backupFile });
|
|
93
96
|
return [];
|
|
94
97
|
}
|
|
95
98
|
}
|
|
@@ -162,13 +165,13 @@ function createLocalQueue(name, options) {
|
|
|
162
165
|
processed++;
|
|
163
166
|
lastJobId = job.id;
|
|
164
167
|
completedJobIds.add(job.id);
|
|
165
|
-
|
|
168
|
+
logger.info("Job completed", { jobId: job.id });
|
|
166
169
|
} catch (error) {
|
|
167
|
-
|
|
170
|
+
logger.error("Job failed", { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error });
|
|
168
171
|
failed++;
|
|
169
172
|
lastJobId = job.id;
|
|
170
173
|
if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {
|
|
171
|
-
|
|
174
|
+
logger.error("Job exhausted all attempts, moving to dead letter", { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS });
|
|
172
175
|
deadJobIds.add(job.id);
|
|
173
176
|
} else {
|
|
174
177
|
const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1);
|
|
@@ -207,7 +210,7 @@ function createLocalQueue(name, options) {
|
|
|
207
210
|
try {
|
|
208
211
|
await processBatch(activeHandler);
|
|
209
212
|
} catch (error) {
|
|
210
|
-
|
|
213
|
+
logger.error("Polling error", { err: error });
|
|
211
214
|
} finally {
|
|
212
215
|
isProcessing = false;
|
|
213
216
|
}
|
|
@@ -220,10 +223,10 @@ function createLocalQueue(name, options) {
|
|
|
220
223
|
await processBatch(handler);
|
|
221
224
|
pollingTimer = setInterval(() => {
|
|
222
225
|
pollAndProcess().catch((err) => {
|
|
223
|
-
|
|
226
|
+
logger.error("Poll cycle error", { err });
|
|
224
227
|
});
|
|
225
228
|
}, pollInterval);
|
|
226
|
-
|
|
229
|
+
logger.info("Worker started", { concurrency });
|
|
227
230
|
return { processed: -1, failed: -1, lastJobId: void 0 };
|
|
228
231
|
}
|
|
229
232
|
async function clear() {
|
|
@@ -260,7 +263,7 @@ function createLocalQueue(name, options) {
|
|
|
260
263
|
const startTime = Date.now();
|
|
261
264
|
while (isProcessing) {
|
|
262
265
|
if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {
|
|
263
|
-
|
|
266
|
+
logger.warn("Force closing after shutdown timeout", { timeoutMs: SHUTDOWN_TIMEOUT });
|
|
264
267
|
break;
|
|
265
268
|
}
|
|
266
269
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/local.ts"],
|
|
4
|
-
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential\n * backoff and moved to a dead-letter store once attempts are exhausted (see the\n * retry handling in `process()` below).\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences to preserve the atomicity guarantees the\n * previous synchronous implementation relied on.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(() => fn(), () => fn())\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n async function backupCorruptedQueueFile(content: string): Promise<string> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n await fsp.writeFile(backupFile, content, 'utf8')\n await fsp.writeFile(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n console.error(`[queue:${name}] Failed to read queue file:`, readError.message)\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n console.error(`[queue:${name}] Failed to read queue file:`, parseError.message)\n const backupFile = await backupCorruptedQueueFile(content)\n console.error(`[queue:${name}] Backed up corrupted queue file to ${backupFile} and recreated queue.json`)\n return []\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n console.log(`[queue:${name}] Job ${job.id} completed`)\n } catch (error) {\n console.error(`[queue:${name}] Job ${job.id} failed (attempt ${attemptNumber}/${DEFAULT_MAX_ATTEMPTS}):`, error)\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n console.error(`[queue:${name}] Job ${job.id} exhausted all ${DEFAULT_MAX_ATTEMPTS} attempts, moving to dead letter`)\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n console.error(`[queue:${name}] Polling error:`, error)\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n console.error(`[queue:${name}] Poll cycle error:`, err)\n })\n }, pollInterval)\n\n console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n console.warn(`[queue:${name}] Force closing after ${SHUTDOWN_TIMEOUT}ms timeout`)\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;
|
|
4
|
+
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential\n * backoff and moved to a dead-letter store once attempts are exhausted (see the\n * retry handling in `process()` below).\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences to preserve the atomicity guarantees the\n * previous synchronous implementation relied on.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(() => fn(), () => fn())\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n async function backupCorruptedQueueFile(content: string): Promise<string> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n await fsp.writeFile(backupFile, content, 'utf8')\n await fsp.writeFile(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await backupCorruptedQueueFile(content)\n logger.error('Backed up corrupted queue file and recreated queue.json', { backupFile })\n return []\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts, moving to dead letter', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, pollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAG7B,MAAM,gBAAgB,aAAa,OAAO;AAa1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAE9B,MAAM,MAAM,GAAG;AA0BR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAG9C,MAAI,eAAsD;AAC1D,MAAI,eAAe;AACnB,MAAI,gBAAsC;AAC1C,QAAM,iBAAiB,oBAAI,IAAY;AAIvC,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY,KAAK,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC;AACnD,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAEA,iBAAe,yBAAyB,SAAkC;AACxE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,OAAO;AAC3E,UAAM,IAAI,UAAU,YAAY,SAAS,MAAM;AAC/C,UAAM,IAAI,UAAU,WAAW,MAAM,MAAM;AAC3C,WAAO;AAAA,EACT;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,aAAO,MAAM,6BAA6B,EAAE,KAAK,UAAU,CAAC;AAC5D,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAW,CAAC;AAC9D,YAAM,aAAa,MAAM,yBAAyB,OAAO;AACzD,aAAO,MAAM,2DAA2D,EAAE,WAAW,CAAC;AACtF,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,MAAM;AAAA,EACtE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,MAAM;AAAA,EACvE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,IACvC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AAED,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,eAAW,OAAO,eAAe;AAC/B,qBAAe,IAAI,IAAI,EAAE;AAAA,IAC3B;AAEA,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,QAAI;AACF,iBAAW,OAAO,eAAe;AAC/B,cAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,YAAI;AACF,gBAAM,QAAQ;AAAA,YACZ,QAAQ,KAAK;AAAA,cACX,OAAO,IAAI;AAAA,cACX;AAAA,cACA,WAAW;AAAA,YACb,CAAC;AAAA,UACH;AACA;AACA,sBAAY,IAAI;AAChB,0BAAgB,IAAI,IAAI,EAAE;AAC1B,iBAAO,KAAK,iBAAiB,EAAE,OAAO,IAAI,GAAG,CAAC;AAAA,QAChD,SAAS,OAAO;AACd,iBAAO,MAAM,cAAc,EAAE,OAAO,IAAI,IAAI,eAAe,aAAa,sBAAsB,KAAK,MAAM,CAAC;AAC1G;AACA,sBAAY,IAAI;AAChB,cAAI,iBAAiB,sBAAsB;AACzC,mBAAO,MAAM,qDAAqD,EAAE,OAAO,IAAI,IAAI,aAAa,qBAAqB,CAAC;AACtH,uBAAW,IAAI,IAAI,EAAE;AAAA,UACvB,OAAO;AACL,kBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,yBAAa,IAAI,IAAI,IAAI;AAAA,cACvB,GAAG;AAAA,cACH,cAAc;AAAA,cACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,YAC5D,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,UAAI,YAAY;AACd,cAAM,aAAa,YAAY;AAE7B,gBAAM,cAAc,MAAM,UAAU;AACpC,gBAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,gBAAM,WAAW,WAAW;AAE5B,gBAAM,WAAuB;AAAA,YAC3B,iBAAiB;AAAA,YACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,YAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,UACrD;AACA,gBAAM,WAAW,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAEA,aAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,IACxC,UAAE;AACA,iBAAW,OAAO,eAAe;AAC/B,uBAAe,OAAO,IAAI,EAAE;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAKA,iBAAe,iBAAgC;AAE7C,QAAI,gBAAgB,CAAC,cAAe;AAEpC,mBAAe;AACf,QAAI;AACF,YAAM,aAAa,aAAa;AAAA,IAClC,SAAS,OAAO;AACd,aAAO,MAAM,iBAAiB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC9C,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAGA,oBAAgB;AAGhB,UAAM,aAAa,OAAO;AAG1B,mBAAe,YAAY,MAAM;AAC/B,qBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,eAAO,MAAM,oBAAoB,EAAE,IAAI,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH,GAAG,YAAY;AAEf,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAG7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,eAAe,KAAK,OAAO,CAAC,QAAQ,eAAe,IAAI,IAAI,EAAE,KAAK,CAAC,oBAAoB,IAAI,SAAS,KAAK,CAAC;AAChH,YAAM,UAAU,KAAK,SAAS,aAAa;AAC3C,UAAI,UAAU,GAAG;AACf,cAAM,WAAW,YAAY;AAAA,MAC/B;AACA,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AAEpC,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AACA,oBAAgB;AAGhB,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,eAAO,KAAK,wCAAwC,EAAE,WAAW,iBAAiB,CAAC;AACnF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/dist/worker/registry.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { applyWorkerOverridesToDescriptors } from "@open-mercato/shared/modules/overrides";
|
|
2
|
+
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
3
|
+
const logger = createLogger("queue").child({ component: "worker-registry" });
|
|
2
4
|
const workers = /* @__PURE__ */ new Map();
|
|
3
5
|
function registerWorker(worker) {
|
|
4
6
|
if (workers.has(worker.id)) {
|
|
5
|
-
|
|
7
|
+
logger.warn("Worker already registered, overwriting", { workerId: worker.id });
|
|
6
8
|
}
|
|
7
9
|
workers.set(worker.id, worker);
|
|
8
10
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/worker/registry.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * Worker Registry\n *\n * Provides registration and lookup for auto-discovered queue workers.\n * Workers are registered during bootstrap and accessed by the CLI worker command.\n */\n\nimport type { WorkerDescriptor } from '../types'\nimport { applyWorkerOverridesToDescriptors } from '@open-mercato/shared/modules/overrides'\n\nconst workers: Map<string, WorkerDescriptor> = new Map()\n\n/**\n * Register a single worker.\n * @param worker - The worker descriptor to register\n */\nexport function registerWorker(worker: WorkerDescriptor): void {\n if (workers.has(worker.id)) {\n
|
|
5
|
-
"mappings": "AAQA,SAAS,yCAAyC;
|
|
4
|
+
"sourcesContent": ["/**\n * Worker Registry\n *\n * Provides registration and lookup for auto-discovered queue workers.\n * Workers are registered during bootstrap and accessed by the CLI worker command.\n */\n\nimport type { WorkerDescriptor } from '../types'\nimport { applyWorkerOverridesToDescriptors } from '@open-mercato/shared/modules/overrides'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst logger = createLogger('queue').child({ component: 'worker-registry' })\n\nconst workers: Map<string, WorkerDescriptor> = new Map()\n\n/**\n * Register a single worker.\n * @param worker - The worker descriptor to register\n */\nexport function registerWorker(worker: WorkerDescriptor): void {\n if (workers.has(worker.id)) {\n logger.warn('Worker already registered, overwriting', { workerId: worker.id })\n }\n workers.set(worker.id, worker)\n}\n\n/**\n * Register multiple workers at once (typically from module discovery).\n * @param list - Array of worker descriptors to register\n */\nexport function registerModuleWorkers(list: WorkerDescriptor[]): void {\n for (const worker of applyWorkerOverridesToDescriptors(list)) {\n registerWorker(worker)\n }\n}\n\n/**\n * Get all registered workers.\n * @returns Array of all worker descriptors\n */\nexport function getWorkers(): WorkerDescriptor[] {\n return Array.from(workers.values())\n}\n\n/**\n * Get workers registered for a specific queue.\n * @param queue - The queue name to filter by\n * @returns Array of workers for the specified queue\n */\nexport function getWorkersByQueue(queue: string): WorkerDescriptor[] {\n return Array.from(workers.values()).filter((w) => w.queue === queue)\n}\n\n/**\n * Get a specific worker by ID.\n * @param id - The worker ID to look up\n * @returns The worker descriptor if found, undefined otherwise\n */\nexport function getWorker(id: string): WorkerDescriptor | undefined {\n return workers.get(id)\n}\n\n/**\n * Get all unique queue names that have registered workers.\n * @returns Array of queue names\n */\nexport function getRegisteredQueues(): string[] {\n const queues = new Set<string>()\n for (const worker of workers.values()) {\n queues.add(worker.queue)\n }\n return Array.from(queues)\n}\n\n/**\n * Clear all registered workers (useful for testing).\n */\nexport function clearWorkers(): void {\n workers.clear()\n}\n"],
|
|
5
|
+
"mappings": "AAQA,SAAS,yCAAyC;AAClD,SAAS,oBAAoB;AAE7B,MAAM,SAAS,aAAa,OAAO,EAAE,MAAM,EAAE,WAAW,kBAAkB,CAAC;AAE3E,MAAM,UAAyC,oBAAI,IAAI;AAMhD,SAAS,eAAe,QAAgC;AAC7D,MAAI,QAAQ,IAAI,OAAO,EAAE,GAAG;AAC1B,WAAO,KAAK,0CAA0C,EAAE,UAAU,OAAO,GAAG,CAAC;AAAA,EAC/E;AACA,UAAQ,IAAI,OAAO,IAAI,MAAM;AAC/B;AAMO,SAAS,sBAAsB,MAAgC;AACpE,aAAW,UAAU,kCAAkC,IAAI,GAAG;AAC5D,mBAAe,MAAM;AAAA,EACvB;AACF;AAMO,SAAS,aAAiC;AAC/C,SAAO,MAAM,KAAK,QAAQ,OAAO,CAAC;AACpC;AAOO,SAAS,kBAAkB,OAAmC;AACnE,SAAO,MAAM,KAAK,QAAQ,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK;AACrE;AAOO,SAAS,UAAU,IAA0C;AAClE,SAAO,QAAQ,IAAI,EAAE;AACvB;AAMO,SAAS,sBAAgC;AAC9C,QAAM,SAAS,oBAAI,IAAY;AAC/B,aAAW,UAAU,QAAQ,OAAO,GAAG;AACrC,WAAO,IAAI,OAAO,KAAK;AAAA,EACzB;AACA,SAAO,MAAM,KAAK,MAAM;AAC1B;AAKO,SAAS,eAAqB;AACnC,UAAQ,MAAM;AAChB;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/worker/runner.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { createQueue } from "../factory.js";
|
|
2
|
+
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
3
|
+
const logger = createLogger("queue").child({ component: "worker" });
|
|
2
4
|
const managedQueues = /* @__PURE__ */ new Set();
|
|
3
5
|
let shutdownHandlersRegistered = false;
|
|
4
6
|
let shutdownInProgress = false;
|
|
@@ -12,21 +14,21 @@ function registerShutdownHandlers() {
|
|
|
12
14
|
const shutdown = async (signal) => {
|
|
13
15
|
if (shutdownInProgress) return;
|
|
14
16
|
shutdownInProgress = true;
|
|
15
|
-
|
|
17
|
+
logger.info("Received signal, shutting down gracefully", { signal });
|
|
16
18
|
let hasError = false;
|
|
17
19
|
for (const queue of managedQueues) {
|
|
18
20
|
try {
|
|
19
21
|
await queue.close();
|
|
20
22
|
} catch (error) {
|
|
21
23
|
hasError = true;
|
|
22
|
-
|
|
24
|
+
logger.error("Error during shutdown", { err: error });
|
|
23
25
|
}
|
|
24
26
|
}
|
|
25
27
|
managedQueues.clear();
|
|
26
28
|
unregisterShutdownHandlers(sigtermHandler, sigintHandler);
|
|
27
29
|
shutdownInProgress = false;
|
|
28
30
|
if (!hasError) {
|
|
29
|
-
|
|
31
|
+
logger.info("Worker closed successfully");
|
|
30
32
|
}
|
|
31
33
|
process.exit(hasError ? 1 : 0);
|
|
32
34
|
};
|
|
@@ -51,7 +53,7 @@ async function runWorker(options) {
|
|
|
51
53
|
strategy: strategyOption
|
|
52
54
|
} = options;
|
|
53
55
|
const strategy = strategyOption ?? (process.env.QUEUE_STRATEGY === "async" ? "async" : "local");
|
|
54
|
-
|
|
56
|
+
logger.info("Starting worker for queue", { queueName, strategy });
|
|
55
57
|
const queue = createQueue(queueName, strategy, {
|
|
56
58
|
connection,
|
|
57
59
|
concurrency
|
|
@@ -61,11 +63,11 @@ async function runWorker(options) {
|
|
|
61
63
|
registerShutdownHandlers();
|
|
62
64
|
}
|
|
63
65
|
await queue.process(handler);
|
|
64
|
-
|
|
66
|
+
logger.info("Worker running", { concurrency });
|
|
65
67
|
if (background) {
|
|
66
68
|
return;
|
|
67
69
|
}
|
|
68
|
-
|
|
70
|
+
logger.info("Press Ctrl+C to stop");
|
|
69
71
|
await new Promise(() => {
|
|
70
72
|
});
|
|
71
73
|
}
|
|
@@ -74,7 +76,7 @@ function createRoutedHandler(handlers) {
|
|
|
74
76
|
const type = job.payload.type;
|
|
75
77
|
const handler = handlers[type];
|
|
76
78
|
if (!handler) {
|
|
77
|
-
|
|
79
|
+
logger.warn("No handler registered for job type", { type });
|
|
78
80
|
return;
|
|
79
81
|
}
|
|
80
82
|
await handler(job, ctx);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/worker/runner.ts"],
|
|
4
|
-
"sourcesContent": ["import { createQueue } from '../factory'\nimport type { Queue, JobHandler, AsyncQueueOptions, QueueStrategyType } from '../types'\n\n/**\n * Options for running a queue worker.\n */\nexport type WorkerRunnerOptions<T = unknown> = {\n /** Name of the queue to process */\n queueName: string\n /** Handler function to process each job */\n handler: JobHandler<T>\n /** Redis connection options (only used for async strategy) */\n connection?: AsyncQueueOptions['connection']\n /** Number of concurrent jobs to process */\n concurrency?: number\n /** Whether to set up graceful shutdown handlers */\n gracefulShutdown?: boolean\n /** If true, don't block - return immediately after starting processing (for multi-queue mode) */\n background?: boolean\n /** Queue strategy to use. Defaults to QUEUE_STRATEGY env var or 'local' */\n strategy?: QueueStrategyType\n}\n\nconst managedQueues = new Set<Queue<unknown>>()\nlet shutdownHandlersRegistered = false\nlet shutdownInProgress = false\n\nfunction unregisterShutdownHandlers(sigtermHandler: () => void, sigintHandler: () => void): void {\n process.off('SIGTERM', sigtermHandler)\n process.off('SIGINT', sigintHandler)\n shutdownHandlersRegistered = false\n}\n\nfunction registerShutdownHandlers(): void {\n if (shutdownHandlersRegistered) return\n\n const shutdown = async (signal: string) => {\n if (shutdownInProgress) return\n shutdownInProgress = true\n\n
|
|
5
|
-
"mappings": "AAAA,SAAS,mBAAmB;
|
|
4
|
+
"sourcesContent": ["import { createQueue } from '../factory'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, JobHandler, AsyncQueueOptions, QueueStrategyType } from '../types'\n\nconst logger = createLogger('queue').child({ component: 'worker' })\n\n/**\n * Options for running a queue worker.\n */\nexport type WorkerRunnerOptions<T = unknown> = {\n /** Name of the queue to process */\n queueName: string\n /** Handler function to process each job */\n handler: JobHandler<T>\n /** Redis connection options (only used for async strategy) */\n connection?: AsyncQueueOptions['connection']\n /** Number of concurrent jobs to process */\n concurrency?: number\n /** Whether to set up graceful shutdown handlers */\n gracefulShutdown?: boolean\n /** If true, don't block - return immediately after starting processing (for multi-queue mode) */\n background?: boolean\n /** Queue strategy to use. Defaults to QUEUE_STRATEGY env var or 'local' */\n strategy?: QueueStrategyType\n}\n\nconst managedQueues = new Set<Queue<unknown>>()\nlet shutdownHandlersRegistered = false\nlet shutdownInProgress = false\n\nfunction unregisterShutdownHandlers(sigtermHandler: () => void, sigintHandler: () => void): void {\n process.off('SIGTERM', sigtermHandler)\n process.off('SIGINT', sigintHandler)\n shutdownHandlersRegistered = false\n}\n\nfunction registerShutdownHandlers(): void {\n if (shutdownHandlersRegistered) return\n\n const shutdown = async (signal: string) => {\n if (shutdownInProgress) return\n shutdownInProgress = true\n\n logger.info('Received signal, shutting down gracefully', { signal })\n\n let hasError = false\n for (const queue of managedQueues) {\n try {\n await queue.close()\n } catch (error) {\n hasError = true\n logger.error('Error during shutdown', { err: error })\n }\n }\n\n managedQueues.clear()\n unregisterShutdownHandlers(sigtermHandler, sigintHandler)\n shutdownInProgress = false\n\n if (!hasError) {\n logger.info('Worker closed successfully')\n }\n\n process.exit(hasError ? 1 : 0)\n }\n\n const sigtermHandler = () => {\n void shutdown('SIGTERM')\n }\n\n const sigintHandler = () => {\n void shutdown('SIGINT')\n }\n\n process.on('SIGTERM', sigtermHandler)\n process.on('SIGINT', sigintHandler)\n shutdownHandlersRegistered = true\n}\n\n/**\n * Runs a queue worker that processes jobs continuously.\n *\n * This function:\n * 1. Creates an async queue instance\n * 2. Starts a BullMQ worker\n * 3. Sets up graceful shutdown on SIGTERM/SIGINT\n * 4. Keeps the process running until shutdown\n *\n * @template T - The job payload type\n * @param options - Worker configuration\n *\n * @example\n * ```typescript\n * import { runWorker } from '@open-mercato/queue/worker'\n *\n * await runWorker({\n * queueName: 'events',\n * handler: async (job, ctx) => {\n * console.log(`Processing ${ctx.jobId}:`, job.payload)\n * },\n * connection: { url: process.env.REDIS_URL },\n * concurrency: 5,\n * })\n * ```\n */\nexport async function runWorker<T = unknown>(\n options: WorkerRunnerOptions<T>\n): Promise<void> {\n const {\n queueName,\n handler,\n connection,\n concurrency = 1,\n gracefulShutdown = true,\n background = false,\n strategy: strategyOption,\n } = options\n\n // Determine queue strategy from option, env var, or default to 'local'\n const strategy: QueueStrategyType = strategyOption\n ?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')\n\n logger.info('Starting worker for queue', { queueName, strategy })\n\n const queue = createQueue<T>(queueName, strategy, {\n connection,\n concurrency,\n })\n\n // Set up graceful shutdown\n if (gracefulShutdown) {\n managedQueues.add(queue as Queue<unknown>)\n registerShutdownHandlers()\n }\n\n // Start processing\n await queue.process(handler)\n\n logger.info('Worker running', { concurrency })\n\n if (background) {\n // Return immediately for multi-queue mode\n return\n }\n\n logger.info('Press Ctrl+C to stop')\n\n // Keep the process alive (single-queue mode)\n await new Promise(() => {\n // This promise never resolves, keeping the worker running\n })\n}\n\n/**\n * Creates a worker handler that routes jobs to specific handlers based on job type.\n *\n * @template T - Base job payload type (must include a 'type' field)\n * @param handlers - Map of job types to their handlers\n *\n * @example\n * ```typescript\n * const handler = createRoutedHandler({\n * 'user.created': async (job) => { ... },\n * 'order.placed': async (job) => { ... },\n * })\n *\n * await runWorker({ queueName: 'events', handler })\n * ```\n */\nexport function createRoutedHandler<T extends { type: string }>(\n handlers: Record<string, JobHandler<T>>\n): JobHandler<T> {\n return async (job, ctx) => {\n const type = job.payload.type\n const handler = handlers[type]\n\n if (!handler) {\n logger.warn('No handler registered for job type', { type })\n return\n }\n\n await handler(job, ctx)\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAG7B,MAAM,SAAS,aAAa,OAAO,EAAE,MAAM,EAAE,WAAW,SAAS,CAAC;AAsBlE,MAAM,gBAAgB,oBAAI,IAAoB;AAC9C,IAAI,6BAA6B;AACjC,IAAI,qBAAqB;AAEzB,SAAS,2BAA2B,gBAA4B,eAAiC;AAC/F,UAAQ,IAAI,WAAW,cAAc;AACrC,UAAQ,IAAI,UAAU,aAAa;AACnC,+BAA6B;AAC/B;AAEA,SAAS,2BAAiC;AACxC,MAAI,2BAA4B;AAEhC,QAAM,WAAW,OAAO,WAAmB;AACzC,QAAI,mBAAoB;AACxB,yBAAqB;AAErB,WAAO,KAAK,6CAA6C,EAAE,OAAO,CAAC;AAEnE,QAAI,WAAW;AACf,eAAW,SAAS,eAAe;AACjC,UAAI;AACF,cAAM,MAAM,MAAM;AAAA,MACpB,SAAS,OAAO;AACd,mBAAW;AACX,eAAO,MAAM,yBAAyB,EAAE,KAAK,MAAM,CAAC;AAAA,MACtD;AAAA,IACF;AAEA,kBAAc,MAAM;AACpB,+BAA2B,gBAAgB,aAAa;AACxD,yBAAqB;AAErB,QAAI,CAAC,UAAU;AACb,aAAO,KAAK,4BAA4B;AAAA,IAC1C;AAEA,YAAQ,KAAK,WAAW,IAAI,CAAC;AAAA,EAC/B;AAEA,QAAM,iBAAiB,MAAM;AAC3B,SAAK,SAAS,SAAS;AAAA,EACzB;AAEA,QAAM,gBAAgB,MAAM;AAC1B,SAAK,SAAS,QAAQ;AAAA,EACxB;AAEA,UAAQ,GAAG,WAAW,cAAc;AACpC,UAAQ,GAAG,UAAU,aAAa;AAClC,+BAA6B;AAC/B;AA4BA,eAAsB,UACpB,SACe;AACf,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd,mBAAmB;AAAA,IACnB,aAAa;AAAA,IACb,UAAU;AAAA,EACZ,IAAI;AAGJ,QAAM,WAA8B,mBAC9B,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAEzD,SAAO,KAAK,6BAA6B,EAAE,WAAW,SAAS,CAAC;AAEhE,QAAM,QAAQ,YAAe,WAAW,UAAU;AAAA,IAChD;AAAA,IACA;AAAA,EACF,CAAC;AAGD,MAAI,kBAAkB;AACpB,kBAAc,IAAI,KAAuB;AACzC,6BAAyB;AAAA,EAC3B;AAGA,QAAM,MAAM,QAAQ,OAAO;AAE3B,SAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAE7C,MAAI,YAAY;AAEd;AAAA,EACF;AAEA,SAAO,KAAK,sBAAsB;AAGlC,QAAM,IAAI,QAAQ,MAAM;AAAA,EAExB,CAAC;AACH;AAkBO,SAAS,oBACd,UACe;AACf,SAAO,OAAO,KAAK,QAAQ;AACzB,UAAM,OAAO,IAAI,QAAQ;AACzB,UAAM,UAAU,SAAS,IAAI;AAE7B,QAAI,CAAC,SAAS;AACZ,aAAO,KAAK,sCAAsC,EAAE,KAAK,CAAC;AAC1D;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,GAAG;AAAA,EACxB;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/queue",
|
|
3
|
-
"version": "0.6.6-develop.
|
|
3
|
+
"version": "0.6.6-develop.6471.1.9ab5bdd79a",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Multi-strategy job queue with local and BullMQ support",
|
|
6
6
|
"type": "module",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"access": "public"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@open-mercato/shared": "0.6.6-develop.
|
|
55
|
+
"@open-mercato/shared": "0.6.6-develop.6471.1.9ab5bdd79a"
|
|
56
56
|
},
|
|
57
57
|
"repository": {
|
|
58
58
|
"type": "git",
|
|
@@ -2,9 +2,24 @@ import fs from 'node:fs'
|
|
|
2
2
|
import os from 'node:os'
|
|
3
3
|
import path from 'node:path'
|
|
4
4
|
|
|
5
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
5
6
|
import { createQueue } from '../factory'
|
|
6
7
|
import type { QueuedJob } from '../types'
|
|
7
8
|
|
|
9
|
+
jest.mock('@open-mercato/shared/lib/logger', () => {
|
|
10
|
+
const mocked = {
|
|
11
|
+
debug: jest.fn(),
|
|
12
|
+
info: jest.fn(),
|
|
13
|
+
warn: jest.fn(),
|
|
14
|
+
error: jest.fn(),
|
|
15
|
+
child: jest.fn(),
|
|
16
|
+
}
|
|
17
|
+
mocked.child.mockImplementation(() => mocked)
|
|
18
|
+
return { createLogger: jest.fn(() => mocked) }
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
const queueLoggerError = createLogger('queue').error as jest.Mock
|
|
22
|
+
|
|
8
23
|
function readJson(p: string) { return JSON.parse(fs.readFileSync(p, 'utf8')) }
|
|
9
24
|
|
|
10
25
|
describe('Queue - local strategy', () => {
|
|
@@ -253,7 +268,7 @@ describe('Queue - local strategy', () => {
|
|
|
253
268
|
const queueDir = path.join('.mercato', 'queue', 'test-queue')
|
|
254
269
|
const queuePath = path.join(queueDir, 'queue.json')
|
|
255
270
|
const brokenContent = '{"nope"'
|
|
256
|
-
|
|
271
|
+
queueLoggerError.mockClear()
|
|
257
272
|
|
|
258
273
|
fs.mkdirSync(queueDir, { recursive: true })
|
|
259
274
|
fs.writeFileSync(queuePath, brokenContent, 'utf8')
|
|
@@ -270,15 +285,15 @@ describe('Queue - local strategy', () => {
|
|
|
270
285
|
|
|
271
286
|
expect(backupFiles).toHaveLength(1)
|
|
272
287
|
expect(fs.readFileSync(path.join(queueDir, backupFiles[0]), 'utf8')).toBe(brokenContent)
|
|
273
|
-
expect(
|
|
274
|
-
'
|
|
275
|
-
expect.any(
|
|
288
|
+
expect(queueLoggerError).toHaveBeenCalledWith(
|
|
289
|
+
'Failed to parse queue file',
|
|
290
|
+
{ err: expect.any(Error) },
|
|
276
291
|
)
|
|
277
|
-
expect(
|
|
278
|
-
|
|
292
|
+
expect(queueLoggerError).toHaveBeenCalledWith(
|
|
293
|
+
'Backed up corrupted queue file and recreated queue.json',
|
|
294
|
+
{ backupFile: expect.stringContaining('queue.corrupted.') },
|
|
279
295
|
)
|
|
280
296
|
|
|
281
|
-
errorSpy.mockRestore()
|
|
282
297
|
await queue.close()
|
|
283
298
|
})
|
|
284
299
|
|
|
@@ -288,8 +303,6 @@ describe('Queue - local strategy', () => {
|
|
|
288
303
|
|
|
289
304
|
await queue.enqueue({ shouldFail: true })
|
|
290
305
|
|
|
291
|
-
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
292
|
-
|
|
293
306
|
await queue.process((job) => {
|
|
294
307
|
if (job.payload.shouldFail) throw new Error('transient')
|
|
295
308
|
}, { limit: 10 })
|
|
@@ -299,7 +312,6 @@ describe('Queue - local strategy', () => {
|
|
|
299
312
|
expect(remaining[0].attemptCount).toBe(1)
|
|
300
313
|
expect(remaining[0].availableAt).toBeDefined()
|
|
301
314
|
|
|
302
|
-
errorSpy.mockRestore()
|
|
303
315
|
await queue.close()
|
|
304
316
|
})
|
|
305
317
|
|
|
@@ -309,8 +321,6 @@ describe('Queue - local strategy', () => {
|
|
|
309
321
|
|
|
310
322
|
await queue.enqueue({ value: 1 })
|
|
311
323
|
|
|
312
|
-
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
313
|
-
|
|
314
324
|
// Manually set attemptCount to simulate prior failures
|
|
315
325
|
const jobs = readJson(queuePath)
|
|
316
326
|
jobs[0].attemptCount = 2
|
|
@@ -322,7 +332,6 @@ describe('Queue - local strategy', () => {
|
|
|
322
332
|
const remaining = readJson(queuePath)
|
|
323
333
|
expect(remaining).toHaveLength(0)
|
|
324
334
|
|
|
325
|
-
errorSpy.mockRestore()
|
|
326
335
|
await queue.close()
|
|
327
336
|
})
|
|
328
337
|
|
|
@@ -332,7 +341,6 @@ describe('Queue - local strategy', () => {
|
|
|
332
341
|
|
|
333
342
|
await queue.enqueue({ value: 1 })
|
|
334
343
|
|
|
335
|
-
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
336
344
|
const beforeProcess = Date.now()
|
|
337
345
|
|
|
338
346
|
await queue.process(() => { throw new Error('fail') }, { limit: 10 })
|
|
@@ -342,7 +350,6 @@ describe('Queue - local strategy', () => {
|
|
|
342
350
|
const availableAt = new Date(remaining[0].availableAt).getTime()
|
|
343
351
|
expect(availableAt).toBeGreaterThanOrEqual(beforeProcess + 1000)
|
|
344
352
|
|
|
345
|
-
errorSpy.mockRestore()
|
|
346
353
|
await queue.close()
|
|
347
354
|
})
|
|
348
355
|
|
package/src/strategies/async.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'
|
|
2
2
|
import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
|
|
3
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
4
|
+
|
|
5
|
+
const packageLogger = createLogger('queue')
|
|
3
6
|
|
|
4
7
|
// BullMQ interface types - we define the shape we use to maintain type safety
|
|
5
8
|
// while keeping bullmq as an optional peer dependency
|
|
@@ -108,6 +111,7 @@ export function createAsyncQueue<T = unknown>(
|
|
|
108
111
|
): Queue<T> {
|
|
109
112
|
const connection = resolveConnection(options?.connection)
|
|
110
113
|
const concurrency = options?.concurrency ?? 1
|
|
114
|
+
const logger = packageLogger.child({ queue: name })
|
|
111
115
|
|
|
112
116
|
let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
|
|
113
117
|
let bullWorker: BullWorkerInterface | null = null
|
|
@@ -184,21 +188,21 @@ export function createAsyncQueue<T = unknown>(
|
|
|
184
188
|
// Set up event handlers
|
|
185
189
|
bullWorker.on('completed', (job) => {
|
|
186
190
|
const jobWithId = job as { id?: string }
|
|
187
|
-
|
|
191
|
+
logger.info('Job completed', { jobId: jobWithId.id })
|
|
188
192
|
})
|
|
189
193
|
|
|
190
194
|
bullWorker.on('failed', (job, err) => {
|
|
191
195
|
const jobWithId = job as { id?: string } | undefined
|
|
192
196
|
const error = err as Error
|
|
193
|
-
|
|
197
|
+
logger.error('Job failed', { jobId: jobWithId?.id, err: error })
|
|
194
198
|
})
|
|
195
199
|
|
|
196
200
|
bullWorker.on('error', (err) => {
|
|
197
201
|
const error = err as Error
|
|
198
|
-
|
|
202
|
+
logger.error('Worker error', { err: error })
|
|
199
203
|
})
|
|
200
204
|
|
|
201
|
-
|
|
205
|
+
logger.info('Worker started', { concurrency })
|
|
202
206
|
|
|
203
207
|
// For async strategy, return a sentinel result indicating worker mode
|
|
204
208
|
// processed=-1 signals that this is a continuous worker, not a batch process
|
package/src/strategies/local.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import crypto from 'node:crypto'
|
|
4
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
4
5
|
import type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'
|
|
5
6
|
|
|
7
|
+
const packageLogger = createLogger('queue')
|
|
8
|
+
|
|
6
9
|
type LocalState = {
|
|
7
10
|
lastProcessedId?: string
|
|
8
11
|
completedCount?: number
|
|
@@ -70,6 +73,7 @@ export function createLocalQueue<T = unknown>(
|
|
|
70
73
|
const queueDir = path.join(baseDir, name)
|
|
71
74
|
const queueFile = path.join(queueDir, 'queue.json')
|
|
72
75
|
const stateFile = path.join(queueDir, 'state.json')
|
|
76
|
+
const logger = packageLogger.child({ queue: name })
|
|
73
77
|
// Note: concurrency is stored for logging/compatibility but jobs are processed sequentially
|
|
74
78
|
const concurrency = options?.concurrency ?? 1
|
|
75
79
|
const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL
|
|
@@ -139,7 +143,7 @@ export function createLocalQueue<T = unknown>(
|
|
|
139
143
|
if (readError.code === 'ENOENT') {
|
|
140
144
|
return []
|
|
141
145
|
}
|
|
142
|
-
|
|
146
|
+
logger.error('Failed to read queue file', { err: readError })
|
|
143
147
|
throw new Error(`Queue file unreadable: ${readError.message}`)
|
|
144
148
|
}
|
|
145
149
|
|
|
@@ -153,9 +157,9 @@ export function createLocalQueue<T = unknown>(
|
|
|
153
157
|
return parsed as StoredJob<T>[]
|
|
154
158
|
} catch (error: unknown) {
|
|
155
159
|
const parseError = error as Error
|
|
156
|
-
|
|
160
|
+
logger.error('Failed to parse queue file', { err: parseError })
|
|
157
161
|
const backupFile = await backupCorruptedQueueFile(content)
|
|
158
|
-
|
|
162
|
+
logger.error('Backed up corrupted queue file and recreated queue.json', { backupFile })
|
|
159
163
|
return []
|
|
160
164
|
}
|
|
161
165
|
}
|
|
@@ -252,13 +256,13 @@ export function createLocalQueue<T = unknown>(
|
|
|
252
256
|
processed++
|
|
253
257
|
lastJobId = job.id
|
|
254
258
|
completedJobIds.add(job.id)
|
|
255
|
-
|
|
259
|
+
logger.info('Job completed', { jobId: job.id })
|
|
256
260
|
} catch (error) {
|
|
257
|
-
|
|
261
|
+
logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })
|
|
258
262
|
failed++
|
|
259
263
|
lastJobId = job.id
|
|
260
264
|
if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {
|
|
261
|
-
|
|
265
|
+
logger.error('Job exhausted all attempts, moving to dead letter', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })
|
|
262
266
|
deadJobIds.add(job.id)
|
|
263
267
|
} else {
|
|
264
268
|
const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)
|
|
@@ -309,7 +313,7 @@ export function createLocalQueue<T = unknown>(
|
|
|
309
313
|
try {
|
|
310
314
|
await processBatch(activeHandler)
|
|
311
315
|
} catch (error) {
|
|
312
|
-
|
|
316
|
+
logger.error('Polling error', { err: error })
|
|
313
317
|
} finally {
|
|
314
318
|
isProcessing = false
|
|
315
319
|
}
|
|
@@ -333,11 +337,11 @@ export function createLocalQueue<T = unknown>(
|
|
|
333
337
|
// Start polling interval for new jobs
|
|
334
338
|
pollingTimer = setInterval(() => {
|
|
335
339
|
pollAndProcess().catch((err) => {
|
|
336
|
-
|
|
340
|
+
logger.error('Poll cycle error', { err })
|
|
337
341
|
})
|
|
338
342
|
}, pollInterval)
|
|
339
343
|
|
|
340
|
-
|
|
344
|
+
logger.info('Worker started', { concurrency })
|
|
341
345
|
|
|
342
346
|
// Return sentinel value indicating continuous worker mode (like async strategy)
|
|
343
347
|
return { processed: -1, failed: -1, lastJobId: undefined }
|
|
@@ -384,7 +388,7 @@ export function createLocalQueue<T = unknown>(
|
|
|
384
388
|
|
|
385
389
|
while (isProcessing) {
|
|
386
390
|
if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {
|
|
387
|
-
|
|
391
|
+
logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })
|
|
388
392
|
break
|
|
389
393
|
}
|
|
390
394
|
await new Promise((resolve) => setTimeout(resolve, 50))
|
package/src/worker/registry.ts
CHANGED
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
|
|
8
8
|
import type { WorkerDescriptor } from '../types'
|
|
9
9
|
import { applyWorkerOverridesToDescriptors } from '@open-mercato/shared/modules/overrides'
|
|
10
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
11
|
+
|
|
12
|
+
const logger = createLogger('queue').child({ component: 'worker-registry' })
|
|
10
13
|
|
|
11
14
|
const workers: Map<string, WorkerDescriptor> = new Map()
|
|
12
15
|
|
|
@@ -16,7 +19,7 @@ const workers: Map<string, WorkerDescriptor> = new Map()
|
|
|
16
19
|
*/
|
|
17
20
|
export function registerWorker(worker: WorkerDescriptor): void {
|
|
18
21
|
if (workers.has(worker.id)) {
|
|
19
|
-
|
|
22
|
+
logger.warn('Worker already registered, overwriting', { workerId: worker.id })
|
|
20
23
|
}
|
|
21
24
|
workers.set(worker.id, worker)
|
|
22
25
|
}
|
package/src/worker/runner.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { createQueue } from '../factory'
|
|
2
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
2
3
|
import type { Queue, JobHandler, AsyncQueueOptions, QueueStrategyType } from '../types'
|
|
3
4
|
|
|
5
|
+
const logger = createLogger('queue').child({ component: 'worker' })
|
|
6
|
+
|
|
4
7
|
/**
|
|
5
8
|
* Options for running a queue worker.
|
|
6
9
|
*/
|
|
@@ -38,7 +41,7 @@ function registerShutdownHandlers(): void {
|
|
|
38
41
|
if (shutdownInProgress) return
|
|
39
42
|
shutdownInProgress = true
|
|
40
43
|
|
|
41
|
-
|
|
44
|
+
logger.info('Received signal, shutting down gracefully', { signal })
|
|
42
45
|
|
|
43
46
|
let hasError = false
|
|
44
47
|
for (const queue of managedQueues) {
|
|
@@ -46,7 +49,7 @@ function registerShutdownHandlers(): void {
|
|
|
46
49
|
await queue.close()
|
|
47
50
|
} catch (error) {
|
|
48
51
|
hasError = true
|
|
49
|
-
|
|
52
|
+
logger.error('Error during shutdown', { err: error })
|
|
50
53
|
}
|
|
51
54
|
}
|
|
52
55
|
|
|
@@ -55,7 +58,7 @@ function registerShutdownHandlers(): void {
|
|
|
55
58
|
shutdownInProgress = false
|
|
56
59
|
|
|
57
60
|
if (!hasError) {
|
|
58
|
-
|
|
61
|
+
logger.info('Worker closed successfully')
|
|
59
62
|
}
|
|
60
63
|
|
|
61
64
|
process.exit(hasError ? 1 : 0)
|
|
@@ -117,7 +120,7 @@ export async function runWorker<T = unknown>(
|
|
|
117
120
|
const strategy: QueueStrategyType = strategyOption
|
|
118
121
|
?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')
|
|
119
122
|
|
|
120
|
-
|
|
123
|
+
logger.info('Starting worker for queue', { queueName, strategy })
|
|
121
124
|
|
|
122
125
|
const queue = createQueue<T>(queueName, strategy, {
|
|
123
126
|
connection,
|
|
@@ -133,14 +136,14 @@ export async function runWorker<T = unknown>(
|
|
|
133
136
|
// Start processing
|
|
134
137
|
await queue.process(handler)
|
|
135
138
|
|
|
136
|
-
|
|
139
|
+
logger.info('Worker running', { concurrency })
|
|
137
140
|
|
|
138
141
|
if (background) {
|
|
139
142
|
// Return immediately for multi-queue mode
|
|
140
143
|
return
|
|
141
144
|
}
|
|
142
145
|
|
|
143
|
-
|
|
146
|
+
logger.info('Press Ctrl+C to stop')
|
|
144
147
|
|
|
145
148
|
// Keep the process alive (single-queue mode)
|
|
146
149
|
await new Promise(() => {
|
|
@@ -172,7 +175,7 @@ export function createRoutedHandler<T extends { type: string }>(
|
|
|
172
175
|
const handler = handlers[type]
|
|
173
176
|
|
|
174
177
|
if (!handler) {
|
|
175
|
-
|
|
178
|
+
logger.warn('No handler registered for job type', { type })
|
|
176
179
|
return
|
|
177
180
|
}
|
|
178
181
|
|