@open-mercato/queue 0.4.11-develop.2209.62f4737bb7 → 0.4.11-develop.2213.e24130dd79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/strategies/async.js +3 -2
- package/dist/strategies/async.js.map +2 -2
- package/dist/strategies/local.js +25 -10
- package/dist/strategies/local.js.map +2 -2
- package/package.json +2 -2
- package/src/__tests__/async.strategy.test.ts +19 -0
- package/src/__tests__/local.strategy.test.ts +86 -0
- package/src/strategies/async.ts +10 -2
- package/src/strategies/local.ts +31 -21
package/dist/strategies/async.js
CHANGED
|
@@ -50,8 +50,9 @@ function createAsyncQueue(name, options) {
|
|
|
50
50
|
const job = await queue.add(jobData.id, jobData, {
|
|
51
51
|
delay: options2?.delayMs && options2.delayMs > 0 ? options2.delayMs : void 0,
|
|
52
52
|
removeOnComplete: true,
|
|
53
|
-
removeOnFail: 1e3
|
|
54
|
-
|
|
53
|
+
removeOnFail: 1e3,
|
|
54
|
+
attempts: 3,
|
|
55
|
+
backoff: { type: "exponential", delay: 1e3 }
|
|
55
56
|
});
|
|
56
57
|
return job.id ?? jobData.id;
|
|
57
58
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/async.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions } from '../types'\nimport { getRedisUrl } from '@open-mercato/shared/lib/redis/connection'\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n url?: string\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {
|
|
5
|
-
"mappings": "AACA,SAAS,mBAAmB;
|
|
4
|
+
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions } from '../types'\nimport { getRedisUrl } from '@open-mercato/shared/lib/redis/connection'\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n url?: string\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: { connection: ConnectionOptions; concurrency: number }\n ) => BullWorkerInterface\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects an ioredis-compatible connection object. Preserve the full\n * Redis URL under the `url` key so rediss://, username, database, and query\n * params are not lost in translation.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return { url: options.url }\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n }\n }\n\n return { url: getRedisUrl('QUEUE') }\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts: 3,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n await handler(jobData, {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n })\n },\n {\n connection,\n concurrency,\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n console.log(`[queue:${name}] Job ${jobWithId.id} completed`)\n })\n\n bullWorker.on('failed', (job, err) => {\n const jobWithId = job as { id?: string } | undefined\n const error = err as Error\n console.error(`[queue:${name}] Job ${jobWithId?.id} failed:`, error.message)\n })\n\n bullWorker.on('error', (err) => {\n const error = err as Error\n console.error(`[queue:${name}] Worker error:`, error.message)\n })\n\n console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)\n\n // For async strategy, return a sentinel result indicating worker mode\n // processed=-1 signals that this is a continuous worker, not a batch process\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const queue = await getQueue()\n\n // Obliterate removes all jobs from the queue\n await queue.obliterate({ force: true })\n\n return { removed: -1 } // BullMQ obliterate doesn't return count\n }\n\n async function close(): Promise<void> {\n if (bullWorker) {\n await bullWorker.close()\n bullWorker = null\n }\n if (bullQueue) {\n await bullQueue.close()\n bullQueue = null\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const queue = await getQueue()\n const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')\n return {\n waiting: counts.waiting ?? 0,\n active: counts.active ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'async',\n enqueue,\n process,\n clear,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,mBAAmB;AAoD5B,SAAS,kBAAkB,SAA8D;AACvF,MAAI,SAAS,KAAK;AAChB,WAAO,EAAE,KAAK,QAAQ,IAAI;AAAA,EAC5B;AAEA,MAAI,SAAS,MAAM;AACjB,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ;AAAA,MACtB,UAAU,QAAQ;AAAA,MAClB,UAAU,QAAQ;AAAA,MAClB,IAAI,QAAQ;AAAA,MACZ,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAEA,SAAO,EAAE,KAAK,YAAY,OAAO,EAAE;AACrC;AAeO,SAAS,iBACd,MACA,SACU;AACV,QAAM,aAAa,kBAAkB,SAAS,UAAU;AACxD,QAAM,cAAc,SAAS,eAAe;AAE5C,MAAI,YAAqD;AACzD,MAAI,aAAyC;AAC7C,MAAI,eAAoC;AAMxC,iBAAe,YAAmC;AAChD,QAAI,CAAC,cAAc;AACjB,UAAI;AACF,uBAAe,MAAM,OAAO,QAAQ;AAAA,MACtC,QAAQ;AACN,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,WAAsD;AACnE,QAAI,CAAC,WAAW;AACd,YAAM,EAAE,OAAO,eAAe,IAAI,MAAM,UAAU;AAClD,kBAAY,IAAI,eAA6B,MAAM,EAAE,WAAW,CAAC;AAAA,IACnE;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,UAAwB;AAAA,MAC5B,IAAI,OAAO,WAAW;AAAA,MACtB,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IACpC;AAEA,UAAM,MAAM,MAAM,MAAM,IAAI,QAAQ,IAAI,SAAS;AAAA,MAC/C,OAAOA,UAAS,WAAWA,SAAQ,UAAU,IAAIA,SAAQ,UAAU;AAAA,MACnE,kBAAkB;AAAA,MAClB,cAAc;AAAA,MACd,UAAU;AAAA,MACV,SAAS,EAAE,MAAM,eAAe,OAAO,IAAK;AAAA,IAC9C,CAAC;AAED,WAAO,IAAI,MAAM,QAAQ;AAAA,EAC3B;AAEA,iBAAe,QAAQ,SAAgD;AACrE,UAAM,EAAE,OAAO,IAAI,MAAM,UAAU;AAGnC,iBAAa,IAAI;AAAA,MACf;AAAA,MACA,OAAO,QAAQ;AACb,cAAM,UAAU,IAAI;AACpB,cAAM,QAAQ,SAAS;AAAA,UACrB,OAAO,IAAI,MAAM,QAAQ;AAAA,UACzB,eAAe,IAAI,eAAe;AAAA,UAClC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAGA,eAAW,GAAG,aAAa,CAAC,QAAQ;AAClC,YAAM,YAAY;AAClB,cAAQ,IAAI,UAAU,IAAI,SAAS,UAAU,EAAE,YAAY;AAAA,IAC7D,CAAC;AAED,eAAW,GAAG,UAAU,CAAC,KAAK,QAAQ;AACpC,YAAM,YAAY;AAClB,YAAM,QAAQ;AACd,cAAQ,MAAM,UAAU,IAAI,SAAS,WAAW,EAAE,YAAY,MAAM,OAAO;AAAA,IAC7E,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,QAAQ;AAC9B,YAAM,QAAQ;AACd,cAAQ,MAAM,UAAU,IAAI,mBAAmB,MAAM,OAAO;AAAA,IAC9D,CAAC;AAED,YAAQ,IAAI,UAAU,IAAI,qCAAqC,WAAW,EAAE;AAI5E,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,UAAM,QAAQ,MAAM,SAAS;AAG7B,UAAM,MAAM,WAAW,EAAE,OAAO,KAAK,CAAC;AAEtC,WAAO,EAAE,SAAS,GAAG;AAAA,EACvB;AAEA,iBAAe,QAAuB;AACpC,QAAI,YAAY;AACd,YAAM,WAAW,MAAM;AACvB,mBAAa;AAAA,IACf;AACA,QAAI,WAAW;AACb,YAAM,UAAU,MAAM;AACtB,kBAAY;AAAA,IACd;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,SAAS,MAAM,MAAM,aAAa,WAAW,UAAU,aAAa,QAAQ;AAClF,WAAO;AAAA,MACL,SAAS,OAAO,WAAW;AAAA,MAC3B,QAAQ,OAAO,UAAU;AAAA,MACzB,WAAW,OAAO,aAAa;AAAA,MAC/B,QAAQ,OAAO,UAAU;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/dist/strategies/local.js
CHANGED
|
@@ -3,6 +3,8 @@ import path from "node:path";
|
|
|
3
3
|
import crypto from "node:crypto";
|
|
4
4
|
const DEFAULT_POLL_INTERVAL = 1e3;
|
|
5
5
|
const DEFAULT_LOCAL_QUEUE_BASE_DIR = ".mercato/queue";
|
|
6
|
+
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
7
|
+
const RETRY_BACKOFF_BASE_MS = 1e3;
|
|
6
8
|
function createLocalQueue(name, options) {
|
|
7
9
|
const nodeProcess = globalThis.process;
|
|
8
10
|
const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR;
|
|
@@ -104,8 +106,7 @@ function createLocalQueue(name, options) {
|
|
|
104
106
|
async function processBatch(handler, options2) {
|
|
105
107
|
const state = readState();
|
|
106
108
|
const jobs = readQueue();
|
|
107
|
-
const
|
|
108
|
-
const pendingJobs = jobs.slice(lastProcessedIndex + 1).filter((job) => {
|
|
109
|
+
const pendingJobs = jobs.filter((job) => {
|
|
109
110
|
if (!job.availableAt) return true;
|
|
110
111
|
return new Date(job.availableAt).getTime() <= Date.now();
|
|
111
112
|
});
|
|
@@ -113,34 +114,48 @@ function createLocalQueue(name, options) {
|
|
|
113
114
|
let processed = 0;
|
|
114
115
|
let failed = 0;
|
|
115
116
|
let lastJobId;
|
|
116
|
-
const
|
|
117
|
+
const completedJobIds = /* @__PURE__ */ new Set();
|
|
118
|
+
const deadJobIds = /* @__PURE__ */ new Set();
|
|
119
|
+
const retryUpdates = /* @__PURE__ */ new Map();
|
|
117
120
|
for (const job of jobsToProcess) {
|
|
121
|
+
const attemptNumber = (job.attemptCount ?? 0) + 1;
|
|
118
122
|
try {
|
|
119
123
|
await Promise.resolve(
|
|
120
124
|
handler(job, {
|
|
121
125
|
jobId: job.id,
|
|
122
|
-
attemptNumber
|
|
126
|
+
attemptNumber,
|
|
123
127
|
queueName: name
|
|
124
128
|
})
|
|
125
129
|
);
|
|
126
130
|
processed++;
|
|
127
131
|
lastJobId = job.id;
|
|
128
|
-
|
|
132
|
+
completedJobIds.add(job.id);
|
|
129
133
|
console.log(`[queue:${name}] Job ${job.id} completed`);
|
|
130
134
|
} catch (error) {
|
|
131
|
-
console.error(`[queue:${name}] Job ${job.id} failed:`, error);
|
|
135
|
+
console.error(`[queue:${name}] Job ${job.id} failed (attempt ${attemptNumber}/${DEFAULT_MAX_ATTEMPTS}):`, error);
|
|
132
136
|
failed++;
|
|
133
137
|
lastJobId = job.id;
|
|
134
|
-
|
|
138
|
+
if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {
|
|
139
|
+
console.error(`[queue:${name}] Job ${job.id} exhausted all ${DEFAULT_MAX_ATTEMPTS} attempts, moving to dead letter`);
|
|
140
|
+
deadJobIds.add(job.id);
|
|
141
|
+
} else {
|
|
142
|
+
const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1);
|
|
143
|
+
retryUpdates.set(job.id, {
|
|
144
|
+
...job,
|
|
145
|
+
attemptCount: attemptNumber,
|
|
146
|
+
availableAt: new Date(Date.now() + backoffMs).toISOString()
|
|
147
|
+
});
|
|
148
|
+
}
|
|
135
149
|
}
|
|
136
150
|
}
|
|
137
|
-
|
|
138
|
-
|
|
151
|
+
const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0;
|
|
152
|
+
if (hasChanges) {
|
|
153
|
+
const updatedJobs = jobs.filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id)).map((j) => retryUpdates.get(j.id) ?? j);
|
|
139
154
|
writeQueue(updatedJobs);
|
|
140
155
|
const newState = {
|
|
141
156
|
lastProcessedId: lastJobId,
|
|
142
157
|
completedCount: (state.completedCount ?? 0) + processed,
|
|
143
|
-
failedCount: (state.failedCount ?? 0) +
|
|
158
|
+
failedCount: (state.failedCount ?? 0) + deadJobIds.size
|
|
144
159
|
};
|
|
145
160
|
writeState(newState);
|
|
146
161
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/local.ts"],
|
|
4
|
-
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions } from '../types'\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n * - No retry mechanism for failed jobs\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n function ensureDir(): void {\n // Use atomic operations to handle race conditions\n try {\n fs.mkdirSync(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n fs.writeFileSync(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n fs.writeFileSync(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n function backupCorruptedQueueFile(content: string): string {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n fs.writeFileSync(backupFile, content, 'utf8')\n fs.writeFileSync(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n function readQueue(): StoredJob<T>[] {\n ensureDir()\n let content: string\n\n try {\n content = fs.readFileSync(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n console.error(`[queue:${name}] Failed to read queue file:`, readError.message)\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n console.error(`[queue:${name}] Failed to read queue file:`, parseError.message)\n const backupFile = backupCorruptedQueueFile(content)\n console.error(`[queue:${name}] Backed up corrupted queue file to ${backupFile} and recreated queue.json`)\n return []\n }\n }\n\n function writeQueue(jobs: StoredJob<T>[]): void {\n ensureDir()\n fs.writeFileSync(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n function readState(): LocalState {\n ensureDir()\n try {\n const content = fs.readFileSync(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n function writeState(state: LocalState): void {\n ensureDir()\n fs.writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const jobs = readQueue()\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n }\n jobs.push(job)\n writeQueue(jobs)\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const state = readState()\n const jobs = readQueue()\n\n // Find jobs that haven't been processed yet\n const lastProcessedIndex = state.lastProcessedId\n ? jobs.findIndex((j) => j.id === state.lastProcessedId)\n : -1\n\n const pendingJobs = jobs\n .slice(lastProcessedIndex + 1)\n .filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const jobIdsToRemove = new Set<string>()\n\n for (const job of jobsToProcess) {\n try {\n await Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber: 1,\n queueName: name,\n })\n )\n processed++\n lastJobId = job.id\n jobIdsToRemove.add(job.id)\n console.log(`[queue:${name}] Job ${job.id} completed`)\n } catch (error) {\n console.error(`[queue:${name}] Job ${job.id} failed:`, error)\n failed++\n lastJobId = job.id\n jobIdsToRemove.add(job.id) // Remove failed jobs too (matching async strategy)\n }\n }\n\n // Remove processed jobs from queue (matching async removeOnComplete behavior)\n if (jobIdsToRemove.size > 0) {\n const updatedJobs = jobs.filter((j) => !jobIdsToRemove.has(j.id))\n writeQueue(updatedJobs)\n\n // Update state with running counts\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + failed,\n }\n writeState(newState)\n }\n\n return { processed, failed, lastJobId }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n console.error(`[queue:${name}] Polling error:`, error)\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n console.error(`[queue:${name}] Poll cycle error:`, err)\n })\n }, pollInterval)\n\n console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const jobs = readQueue()\n const removed = jobs.length\n writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = readState()\n writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n console.warn(`[queue:${name}] Force closing after ${SHUTDOWN_TIMEOUT}ms timeout`)\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const state = readState()\n const jobs = readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;
|
|
4
|
+
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions } from '../types'\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n * - No retry mechanism for failed jobs\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n function ensureDir(): void {\n // Use atomic operations to handle race conditions\n try {\n fs.mkdirSync(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n fs.writeFileSync(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n fs.writeFileSync(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n function backupCorruptedQueueFile(content: string): string {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n fs.writeFileSync(backupFile, content, 'utf8')\n fs.writeFileSync(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n function readQueue(): StoredJob<T>[] {\n ensureDir()\n let content: string\n\n try {\n content = fs.readFileSync(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n console.error(`[queue:${name}] Failed to read queue file:`, readError.message)\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n console.error(`[queue:${name}] Failed to read queue file:`, parseError.message)\n const backupFile = backupCorruptedQueueFile(content)\n console.error(`[queue:${name}] Backed up corrupted queue file to ${backupFile} and recreated queue.json`)\n return []\n }\n }\n\n function writeQueue(jobs: StoredJob<T>[]): void {\n ensureDir()\n fs.writeFileSync(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n function readState(): LocalState {\n ensureDir()\n try {\n const content = fs.readFileSync(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n function writeState(state: LocalState): void {\n ensureDir()\n fs.writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const jobs = readQueue()\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n }\n jobs.push(job)\n writeQueue(jobs)\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const state = readState()\n const jobs = readQueue()\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n console.log(`[queue:${name}] Job ${job.id} completed`)\n } catch (error) {\n console.error(`[queue:${name}] Job ${job.id} failed (attempt ${attemptNumber}/${DEFAULT_MAX_ATTEMPTS}):`, error)\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n console.error(`[queue:${name}] Job ${job.id} exhausted all ${DEFAULT_MAX_ATTEMPTS} attempts, moving to dead letter`)\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n const updatedJobs = jobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n writeState(newState)\n }\n\n return { processed, failed, lastJobId }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n console.error(`[queue:${name}] Polling error:`, error)\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n console.error(`[queue:${name}] Poll cycle error:`, err)\n })\n }, pollInterval)\n\n console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const jobs = readQueue()\n const removed = jobs.length\n writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = readState()\n writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n console.warn(`[queue:${name}] Force closing after ${SHUTDOWN_TIMEOUT}ms timeout`)\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const state = readState()\n const jobs = readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AAenB,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAkBvB,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAG9C,MAAI,eAAsD;AAC1D,MAAI,eAAe;AACnB,MAAI,gBAAsC;AAM1C,WAAS,YAAkB;AAEzB,QAAI;AACF,SAAG,UAAU,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC5C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,SAAG,cAAc,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACpE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,SAAG,cAAc,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACpE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAEA,WAAS,yBAAyB,SAAyB;AACzD,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,OAAO;AAC3E,OAAG,cAAc,YAAY,SAAS,MAAM;AAC5C,OAAG,cAAc,WAAW,MAAM,MAAM;AACxC,WAAO;AAAA,EACT;AAEA,WAAS,YAA4B;AACnC,cAAU;AACV,QAAI;AAEJ,QAAI;AACF,gBAAU,GAAG,aAAa,WAAW,MAAM;AAAA,IAC7C,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,cAAQ,MAAM,UAAU,IAAI,gCAAgC,UAAU,OAAO;AAC7E,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,cAAQ,MAAM,UAAU,IAAI,gCAAgC,WAAW,OAAO;AAC9E,YAAM,aAAa,yBAAyB,OAAO;AACnD,cAAQ,MAAM,UAAU,IAAI,uCAAuC,UAAU,2BAA2B;AACxG,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,WAAS,WAAW,MAA4B;AAC9C,cAAU;AACV,OAAG,cAAc,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,MAAM;AAAA,EACnE;AAEA,WAAS,YAAwB;AAC/B,cAAU;AACV,QAAI;AACF,YAAM,UAAU,GAAG,aAAa,WAAW,MAAM;AACjD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,WAAS,WAAW,OAAyB;AAC3C,cAAU;AACV,OAAG,cAAc,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,MAAM;AAAA,EACpE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,OAAO,UAAU;AACvB,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,IACvC;AACA,SAAK,KAAK,GAAG;AACb,eAAW,IAAI;AACf,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,QAAQ,UAAU;AACxB,UAAM,OAAO,UAAU;AAEvB,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,eAAW,OAAO,eAAe;AAC/B,YAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,UAAI;AACF,cAAM,QAAQ;AAAA,UACZ,QAAQ,KAAK;AAAA,YACX,OAAO,IAAI;AAAA,YACX;AAAA,YACA,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AACA;AACA,oBAAY,IAAI;AAChB,wBAAgB,IAAI,IAAI,EAAE;AAC1B,gBAAQ,IAAI,UAAU,IAAI,SAAS,IAAI,EAAE,YAAY;AAAA,MACvD,SAAS,OAAO;AACd,gBAAQ,MAAM,UAAU,IAAI,SAAS,IAAI,EAAE,oBAAoB,aAAa,IAAI,oBAAoB,MAAM,KAAK;AAC/G;AACA,oBAAY,IAAI;AAChB,YAAI,iBAAiB,sBAAsB;AACzC,kBAAQ,MAAM,UAAU,IAAI,SAAS,IAAI,EAAE,kBAAkB,oBAAoB,kCAAkC;AACnH,qBAAW,IAAI,IAAI,EAAE;AAAA,QACvB,OAAO;AACL,gBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,uBAAa,IAAI,IAAI,IAAI;AAAA,YACvB,GAAG;AAAA,YACH,cAAc;AAAA,YACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,UAC5D,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,UAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,QAAI,YAAY;AACd,YAAM,cAAc,KACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,iBAAW,WAAW;AAEtB,YAAM,WAAuB;AAAA,QAC3B,iBAAiB;AAAA,QACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,QAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,MACrD;AACA,iBAAW,QAAQ;AAAA,IACrB;AAEA,WAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,EACxC;AAKA,iBAAe,iBAAgC;AAE7C,QAAI,gBAAgB,CAAC,cAAe;AAEpC,mBAAe;AACf,QAAI;AACF,YAAM,aAAa,aAAa;AAAA,IAClC,SAAS,OAAO;AACd,cAAQ,MAAM,UAAU,IAAI,oBAAoB,KAAK;AAAA,IACvD,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAGA,oBAAgB;AAGhB,UAAM,aAAa,OAAO;AAG1B,mBAAe,YAAY,MAAM;AAC/B,qBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,gBAAQ,MAAM,UAAU,IAAI,uBAAuB,GAAG;AAAA,MACxD,CAAC;AAAA,IACH,GAAG,YAAY;AAEf,YAAQ,IAAI,UAAU,IAAI,qCAAqC,WAAW,EAAE;AAG5E,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,UAAM,OAAO,UAAU;AACvB,UAAM,UAAU,KAAK;AACrB,eAAW,CAAC,CAAC;AAEb,UAAM,QAAQ,UAAU;AACxB,eAAW;AAAA,MACT,gBAAgB,MAAM;AAAA,MACtB,aAAa,MAAM;AAAA,IACrB,CAAC;AACD,WAAO,EAAE,QAAQ;AAAA,EACnB;AAEA,iBAAe,QAAuB;AAEpC,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AACA,oBAAgB;AAGhB,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,gBAAQ,KAAK,UAAU,IAAI,yBAAyB,gBAAgB,YAAY;AAChF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,UAAM,QAAQ,UAAU;AACxB,UAAM,OAAO,UAAU;AAEvB,WAAO;AAAA,MACL,SAAS,KAAK;AAAA;AAAA,MACd,QAAQ;AAAA;AAAA,MACR,WAAW,MAAM,kBAAkB;AAAA,MACnC,QAAQ,MAAM,eAAe;AAAA,IAC/B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/queue",
|
|
3
|
-
"version": "0.4.11-develop.
|
|
3
|
+
"version": "0.4.11-develop.2213.e24130dd79",
|
|
4
4
|
"description": "Multi-strategy job queue with local and BullMQ support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"access": "public"
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
|
-
"@open-mercato/shared": "0.4.11-develop.
|
|
53
|
+
"@open-mercato/shared": "0.4.11-develop.2213.e24130dd79"
|
|
54
54
|
},
|
|
55
55
|
"stableVersion": "0.4.10"
|
|
56
56
|
}
|
|
@@ -93,6 +93,25 @@ describe('Queue - async strategy', () => {
|
|
|
93
93
|
})
|
|
94
94
|
})
|
|
95
95
|
|
|
96
|
+
it('enqueues jobs with retry attempts and exponential backoff', async () => {
|
|
97
|
+
const queue = createQueue<{ value: number }>('test-queue', 'async')
|
|
98
|
+
|
|
99
|
+
await queue.enqueue({ value: 42 })
|
|
100
|
+
|
|
101
|
+
expect(queueAdd).toHaveBeenCalledWith(
|
|
102
|
+
expect.any(String),
|
|
103
|
+
expect.objectContaining({ payload: { value: 42 } }),
|
|
104
|
+
expect.objectContaining({
|
|
105
|
+
attempts: 3,
|
|
106
|
+
backoff: { type: 'exponential', delay: 1000 },
|
|
107
|
+
removeOnComplete: true,
|
|
108
|
+
removeOnFail: 1000,
|
|
109
|
+
}),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
await queue.close()
|
|
113
|
+
})
|
|
114
|
+
|
|
96
115
|
it('keeps structured Redis options when host-based config is used', async () => {
|
|
97
116
|
const queue = createQueue<{ value: number }>('test-queue', 'async', {
|
|
98
117
|
connection: {
|
|
@@ -205,6 +205,92 @@ describe('Queue - local strategy', () => {
|
|
|
205
205
|
await queue.close()
|
|
206
206
|
})
|
|
207
207
|
|
|
208
|
+
test('failed jobs are retained in queue for retry', async () => {
|
|
209
|
+
const queue = createQueue<{ shouldFail: boolean }>('test-queue', 'local')
|
|
210
|
+
const queuePath = path.join('.mercato', 'queue', 'test-queue', 'queue.json')
|
|
211
|
+
|
|
212
|
+
await queue.enqueue({ shouldFail: true })
|
|
213
|
+
|
|
214
|
+
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
215
|
+
|
|
216
|
+
await queue.process((job) => {
|
|
217
|
+
if (job.payload.shouldFail) throw new Error('transient')
|
|
218
|
+
}, { limit: 10 })
|
|
219
|
+
|
|
220
|
+
const remaining = readJson(queuePath)
|
|
221
|
+
expect(remaining).toHaveLength(1)
|
|
222
|
+
expect(remaining[0].attemptCount).toBe(1)
|
|
223
|
+
expect(remaining[0].availableAt).toBeDefined()
|
|
224
|
+
|
|
225
|
+
errorSpy.mockRestore()
|
|
226
|
+
await queue.close()
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
test('failed jobs are removed after max attempts', async () => {
|
|
230
|
+
const queue = createQueue<{ value: number }>('test-queue', 'local')
|
|
231
|
+
const queuePath = path.join('.mercato', 'queue', 'test-queue', 'queue.json')
|
|
232
|
+
|
|
233
|
+
await queue.enqueue({ value: 1 })
|
|
234
|
+
|
|
235
|
+
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
236
|
+
|
|
237
|
+
// Manually set attemptCount to simulate prior failures
|
|
238
|
+
const jobs = readJson(queuePath)
|
|
239
|
+
jobs[0].attemptCount = 2
|
|
240
|
+
jobs[0].availableAt = undefined
|
|
241
|
+
fs.writeFileSync(queuePath, JSON.stringify(jobs, null, 2), 'utf8')
|
|
242
|
+
|
|
243
|
+
await queue.process(() => { throw new Error('permanent') }, { limit: 10 })
|
|
244
|
+
|
|
245
|
+
const remaining = readJson(queuePath)
|
|
246
|
+
expect(remaining).toHaveLength(0)
|
|
247
|
+
|
|
248
|
+
errorSpy.mockRestore()
|
|
249
|
+
await queue.close()
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
test('retry jobs include exponential backoff delay', async () => {
|
|
253
|
+
const queue = createQueue<{ value: number }>('test-queue', 'local')
|
|
254
|
+
const queuePath = path.join('.mercato', 'queue', 'test-queue', 'queue.json')
|
|
255
|
+
|
|
256
|
+
await queue.enqueue({ value: 1 })
|
|
257
|
+
|
|
258
|
+
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
259
|
+
const beforeProcess = Date.now()
|
|
260
|
+
|
|
261
|
+
await queue.process(() => { throw new Error('fail') }, { limit: 10 })
|
|
262
|
+
|
|
263
|
+
const remaining = readJson(queuePath)
|
|
264
|
+
expect(remaining).toHaveLength(1)
|
|
265
|
+
const availableAt = new Date(remaining[0].availableAt).getTime()
|
|
266
|
+
expect(availableAt).toBeGreaterThanOrEqual(beforeProcess + 1000)
|
|
267
|
+
|
|
268
|
+
errorSpy.mockRestore()
|
|
269
|
+
await queue.close()
|
|
270
|
+
})
|
|
271
|
+
|
|
272
|
+
test('attempt number is passed correctly in job context', async () => {
|
|
273
|
+
const queue = createQueue<{ value: number }>('test-queue', 'local')
|
|
274
|
+
const queuePath = path.join('.mercato', 'queue', 'test-queue', 'queue.json')
|
|
275
|
+
const attempts: number[] = []
|
|
276
|
+
|
|
277
|
+
await queue.enqueue({ value: 1 })
|
|
278
|
+
|
|
279
|
+
// Set attemptCount to 1 to simulate a retry
|
|
280
|
+
const jobs = readJson(queuePath)
|
|
281
|
+
jobs[0].attemptCount = 1
|
|
282
|
+
jobs[0].availableAt = undefined
|
|
283
|
+
fs.writeFileSync(queuePath, JSON.stringify(jobs, null, 2), 'utf8')
|
|
284
|
+
|
|
285
|
+
await queue.process((_job, ctx) => {
|
|
286
|
+
attempts.push(ctx.attemptNumber)
|
|
287
|
+
}, { limit: 10 })
|
|
288
|
+
|
|
289
|
+
expect(attempts).toEqual([2])
|
|
290
|
+
|
|
291
|
+
await queue.close()
|
|
292
|
+
})
|
|
293
|
+
|
|
208
294
|
test('job context contains correct information', async () => {
|
|
209
295
|
const queue = createQueue<{ value: number }>('context-test', 'local')
|
|
210
296
|
let capturedContext: any = null
|
package/src/strategies/async.ts
CHANGED
|
@@ -17,7 +17,13 @@ interface BullQueueInterface<T> {
|
|
|
17
17
|
add: (
|
|
18
18
|
name: string,
|
|
19
19
|
data: T,
|
|
20
|
-
opts?: {
|
|
20
|
+
opts?: {
|
|
21
|
+
removeOnComplete?: boolean
|
|
22
|
+
removeOnFail?: number
|
|
23
|
+
delay?: number
|
|
24
|
+
attempts?: number
|
|
25
|
+
backoff?: { type: string; delay: number }
|
|
26
|
+
},
|
|
21
27
|
) => Promise<{ id?: string }>
|
|
22
28
|
obliterate: (opts?: { force?: boolean }) => Promise<void>
|
|
23
29
|
close: () => Promise<void>
|
|
@@ -128,7 +134,9 @@ export function createAsyncQueue<T = unknown>(
|
|
|
128
134
|
const job = await queue.add(jobData.id, jobData, {
|
|
129
135
|
delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,
|
|
130
136
|
removeOnComplete: true,
|
|
131
|
-
removeOnFail: 1000,
|
|
137
|
+
removeOnFail: 1000,
|
|
138
|
+
attempts: 3,
|
|
139
|
+
backoff: { type: 'exponential', delay: 1000 },
|
|
132
140
|
})
|
|
133
141
|
|
|
134
142
|
return job.id ?? jobData.id
|
package/src/strategies/local.ts
CHANGED
|
@@ -11,11 +11,14 @@ type LocalState = {
|
|
|
11
11
|
|
|
12
12
|
type StoredJob<T> = QueuedJob<T> & {
|
|
13
13
|
availableAt?: string
|
|
14
|
+
attemptCount?: number
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
/** Default polling interval in milliseconds */
|
|
17
18
|
const DEFAULT_POLL_INTERVAL = 1000
|
|
18
19
|
const DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'
|
|
20
|
+
const DEFAULT_MAX_ATTEMPTS = 3
|
|
21
|
+
const RETRY_BACKOFF_BASE_MS = 1000
|
|
19
22
|
|
|
20
23
|
/**
|
|
21
24
|
* Creates a file-based local queue.
|
|
@@ -176,17 +179,10 @@ export function createLocalQueue<T = unknown>(
|
|
|
176
179
|
const state = readState()
|
|
177
180
|
const jobs = readQueue()
|
|
178
181
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
const pendingJobs = jobs
|
|
185
|
-
.slice(lastProcessedIndex + 1)
|
|
186
|
-
.filter((job) => {
|
|
187
|
-
if (!job.availableAt) return true
|
|
188
|
-
return new Date(job.availableAt).getTime() <= Date.now()
|
|
189
|
-
})
|
|
182
|
+
const pendingJobs = jobs.filter((job) => {
|
|
183
|
+
if (!job.availableAt) return true
|
|
184
|
+
return new Date(job.availableAt).getTime() <= Date.now()
|
|
185
|
+
})
|
|
190
186
|
const jobsToProcess = options?.limit
|
|
191
187
|
? pendingJobs.slice(0, options.limit)
|
|
192
188
|
: pendingJobs
|
|
@@ -194,39 +190,53 @@ export function createLocalQueue<T = unknown>(
|
|
|
194
190
|
let processed = 0
|
|
195
191
|
let failed = 0
|
|
196
192
|
let lastJobId: string | undefined
|
|
197
|
-
const
|
|
193
|
+
const completedJobIds = new Set<string>()
|
|
194
|
+
const deadJobIds = new Set<string>()
|
|
195
|
+
const retryUpdates = new Map<string, StoredJob<T>>()
|
|
198
196
|
|
|
199
197
|
for (const job of jobsToProcess) {
|
|
198
|
+
const attemptNumber = (job.attemptCount ?? 0) + 1
|
|
200
199
|
try {
|
|
201
200
|
await Promise.resolve(
|
|
202
201
|
handler(job, {
|
|
203
202
|
jobId: job.id,
|
|
204
|
-
attemptNumber
|
|
203
|
+
attemptNumber,
|
|
205
204
|
queueName: name,
|
|
206
205
|
})
|
|
207
206
|
)
|
|
208
207
|
processed++
|
|
209
208
|
lastJobId = job.id
|
|
210
|
-
|
|
209
|
+
completedJobIds.add(job.id)
|
|
211
210
|
console.log(`[queue:${name}] Job ${job.id} completed`)
|
|
212
211
|
} catch (error) {
|
|
213
|
-
console.error(`[queue:${name}] Job ${job.id} failed:`, error)
|
|
212
|
+
console.error(`[queue:${name}] Job ${job.id} failed (attempt ${attemptNumber}/${DEFAULT_MAX_ATTEMPTS}):`, error)
|
|
214
213
|
failed++
|
|
215
214
|
lastJobId = job.id
|
|
216
|
-
|
|
215
|
+
if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {
|
|
216
|
+
console.error(`[queue:${name}] Job ${job.id} exhausted all ${DEFAULT_MAX_ATTEMPTS} attempts, moving to dead letter`)
|
|
217
|
+
deadJobIds.add(job.id)
|
|
218
|
+
} else {
|
|
219
|
+
const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)
|
|
220
|
+
retryUpdates.set(job.id, {
|
|
221
|
+
...job,
|
|
222
|
+
attemptCount: attemptNumber,
|
|
223
|
+
availableAt: new Date(Date.now() + backoffMs).toISOString(),
|
|
224
|
+
})
|
|
225
|
+
}
|
|
217
226
|
}
|
|
218
227
|
}
|
|
219
228
|
|
|
220
|
-
|
|
221
|
-
if (
|
|
222
|
-
const updatedJobs = jobs
|
|
229
|
+
const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0
|
|
230
|
+
if (hasChanges) {
|
|
231
|
+
const updatedJobs = jobs
|
|
232
|
+
.filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))
|
|
233
|
+
.map((j) => retryUpdates.get(j.id) ?? j)
|
|
223
234
|
writeQueue(updatedJobs)
|
|
224
235
|
|
|
225
|
-
// Update state with running counts
|
|
226
236
|
const newState: LocalState = {
|
|
227
237
|
lastProcessedId: lastJobId,
|
|
228
238
|
completedCount: (state.completedCount ?? 0) + processed,
|
|
229
|
-
failedCount: (state.failedCount ?? 0) +
|
|
239
|
+
failedCount: (state.failedCount ?? 0) + deadJobIds.size,
|
|
230
240
|
}
|
|
231
241
|
writeState(newState)
|
|
232
242
|
}
|