@open-mercato/queue 0.4.11-develop.2363.d48093712b → 0.4.11-develop.2502.29119c6047
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/strategies/local.js
CHANGED
|
@@ -5,6 +5,7 @@ const DEFAULT_POLL_INTERVAL = 1e3;
|
|
|
5
5
|
const DEFAULT_LOCAL_QUEUE_BASE_DIR = ".mercato/queue";
|
|
6
6
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
7
7
|
const RETRY_BACKOFF_BASE_MS = 1e3;
|
|
8
|
+
const fsp = fs.promises;
|
|
8
9
|
function createLocalQueue(name, options) {
|
|
9
10
|
const nodeProcess = globalThis.process;
|
|
10
11
|
const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR;
|
|
@@ -17,37 +18,46 @@ function createLocalQueue(name, options) {
|
|
|
17
18
|
let pollingTimer = null;
|
|
18
19
|
let isProcessing = false;
|
|
19
20
|
let activeHandler = null;
|
|
20
|
-
|
|
21
|
+
let fileOpChain = Promise.resolve();
|
|
22
|
+
function withFileLock(fn) {
|
|
23
|
+
const run = fileOpChain.then(() => fn(), () => fn());
|
|
24
|
+
fileOpChain = run.then(
|
|
25
|
+
() => void 0,
|
|
26
|
+
() => void 0
|
|
27
|
+
);
|
|
28
|
+
return run;
|
|
29
|
+
}
|
|
30
|
+
async function ensureDir() {
|
|
21
31
|
try {
|
|
22
|
-
|
|
32
|
+
await fsp.mkdir(queueDir, { recursive: true });
|
|
23
33
|
} catch (e) {
|
|
24
34
|
const error = e;
|
|
25
35
|
if (error.code !== "EEXIST") throw error;
|
|
26
36
|
}
|
|
27
37
|
try {
|
|
28
|
-
|
|
38
|
+
await fsp.writeFile(queueFile, "[]", { encoding: "utf8", flag: "wx" });
|
|
29
39
|
} catch (e) {
|
|
30
40
|
const error = e;
|
|
31
41
|
if (error.code !== "EEXIST") throw error;
|
|
32
42
|
}
|
|
33
43
|
try {
|
|
34
|
-
|
|
44
|
+
await fsp.writeFile(stateFile, "{}", { encoding: "utf8", flag: "wx" });
|
|
35
45
|
} catch (e) {
|
|
36
46
|
const error = e;
|
|
37
47
|
if (error.code !== "EEXIST") throw error;
|
|
38
48
|
}
|
|
39
49
|
}
|
|
40
|
-
function backupCorruptedQueueFile(content) {
|
|
50
|
+
async function backupCorruptedQueueFile(content) {
|
|
41
51
|
const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`);
|
|
42
|
-
|
|
43
|
-
|
|
52
|
+
await fsp.writeFile(backupFile, content, "utf8");
|
|
53
|
+
await fsp.writeFile(queueFile, "[]", "utf8");
|
|
44
54
|
return backupFile;
|
|
45
55
|
}
|
|
46
|
-
function readQueue() {
|
|
47
|
-
ensureDir();
|
|
56
|
+
async function readQueue() {
|
|
57
|
+
await ensureDir();
|
|
48
58
|
let content;
|
|
49
59
|
try {
|
|
50
|
-
content =
|
|
60
|
+
content = await fsp.readFile(queueFile, "utf8");
|
|
51
61
|
} catch (error) {
|
|
52
62
|
const readError = error;
|
|
53
63
|
if (readError.code === "ENOENT") {
|
|
@@ -65,33 +75,32 @@ function createLocalQueue(name, options) {
|
|
|
65
75
|
} catch (error) {
|
|
66
76
|
const parseError = error;
|
|
67
77
|
console.error(`[queue:${name}] Failed to read queue file:`, parseError.message);
|
|
68
|
-
const backupFile = backupCorruptedQueueFile(content);
|
|
78
|
+
const backupFile = await backupCorruptedQueueFile(content);
|
|
69
79
|
console.error(`[queue:${name}] Backed up corrupted queue file to ${backupFile} and recreated queue.json`);
|
|
70
80
|
return [];
|
|
71
81
|
}
|
|
72
82
|
}
|
|
73
|
-
function writeQueue(jobs) {
|
|
74
|
-
ensureDir();
|
|
75
|
-
|
|
83
|
+
async function writeQueue(jobs) {
|
|
84
|
+
await ensureDir();
|
|
85
|
+
await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), "utf8");
|
|
76
86
|
}
|
|
77
|
-
function readState() {
|
|
78
|
-
ensureDir();
|
|
87
|
+
async function readState() {
|
|
88
|
+
await ensureDir();
|
|
79
89
|
try {
|
|
80
|
-
const content =
|
|
90
|
+
const content = await fsp.readFile(stateFile, "utf8");
|
|
81
91
|
return JSON.parse(content);
|
|
82
92
|
} catch {
|
|
83
93
|
return {};
|
|
84
94
|
}
|
|
85
95
|
}
|
|
86
|
-
function writeState(state) {
|
|
87
|
-
ensureDir();
|
|
88
|
-
|
|
96
|
+
async function writeState(state) {
|
|
97
|
+
await ensureDir();
|
|
98
|
+
await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), "utf8");
|
|
89
99
|
}
|
|
90
100
|
function generateId() {
|
|
91
101
|
return crypto.randomUUID();
|
|
92
102
|
}
|
|
93
103
|
async function enqueue(data, options2) {
|
|
94
|
-
const jobs = readQueue();
|
|
95
104
|
const availableAt = options2?.delayMs && options2.delayMs > 0 ? new Date(Date.now() + options2.delayMs).toISOString() : void 0;
|
|
96
105
|
const job = {
|
|
97
106
|
id: generateId(),
|
|
@@ -99,13 +108,19 @@ function createLocalQueue(name, options) {
|
|
|
99
108
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
100
109
|
...availableAt ? { availableAt } : {}
|
|
101
110
|
};
|
|
102
|
-
|
|
103
|
-
|
|
111
|
+
await withFileLock(async () => {
|
|
112
|
+
const jobs = await readQueue();
|
|
113
|
+
jobs.push(job);
|
|
114
|
+
await writeQueue(jobs);
|
|
115
|
+
});
|
|
104
116
|
return job.id;
|
|
105
117
|
}
|
|
106
118
|
async function processBatch(handler, options2) {
|
|
107
|
-
const state =
|
|
108
|
-
|
|
119
|
+
const { state, jobs } = await withFileLock(async () => {
|
|
120
|
+
const stateRead = await readState();
|
|
121
|
+
const jobsRead = await readQueue();
|
|
122
|
+
return { state: stateRead, jobs: jobsRead };
|
|
123
|
+
});
|
|
109
124
|
const pendingJobs = jobs.filter((job) => {
|
|
110
125
|
if (!job.availableAt) return true;
|
|
111
126
|
return new Date(job.availableAt).getTime() <= Date.now();
|
|
@@ -150,14 +165,17 @@ function createLocalQueue(name, options) {
|
|
|
150
165
|
}
|
|
151
166
|
const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0;
|
|
152
167
|
if (hasChanges) {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
168
|
+
await withFileLock(async () => {
|
|
169
|
+
const currentJobs = await readQueue();
|
|
170
|
+
const updatedJobs = currentJobs.filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id)).map((j) => retryUpdates.get(j.id) ?? j);
|
|
171
|
+
await writeQueue(updatedJobs);
|
|
172
|
+
const newState = {
|
|
173
|
+
lastProcessedId: lastJobId,
|
|
174
|
+
completedCount: (state.completedCount ?? 0) + processed,
|
|
175
|
+
failedCount: (state.failedCount ?? 0) + deadJobIds.size
|
|
176
|
+
};
|
|
177
|
+
await writeState(newState);
|
|
178
|
+
});
|
|
161
179
|
}
|
|
162
180
|
return { processed, failed, lastJobId };
|
|
163
181
|
}
|
|
@@ -187,15 +205,17 @@ function createLocalQueue(name, options) {
|
|
|
187
205
|
return { processed: -1, failed: -1, lastJobId: void 0 };
|
|
188
206
|
}
|
|
189
207
|
async function clear() {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
208
|
+
return withFileLock(async () => {
|
|
209
|
+
const jobs = await readQueue();
|
|
210
|
+
const removed = jobs.length;
|
|
211
|
+
await writeQueue([]);
|
|
212
|
+
const state = await readState();
|
|
213
|
+
await writeState({
|
|
214
|
+
completedCount: state.completedCount,
|
|
215
|
+
failedCount: state.failedCount
|
|
216
|
+
});
|
|
217
|
+
return { removed };
|
|
197
218
|
});
|
|
198
|
-
return { removed };
|
|
199
219
|
}
|
|
200
220
|
async function close() {
|
|
201
221
|
if (pollingTimer) {
|
|
@@ -214,16 +234,18 @@ function createLocalQueue(name, options) {
|
|
|
214
234
|
}
|
|
215
235
|
}
|
|
216
236
|
async function getJobCounts() {
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
237
|
+
return withFileLock(async () => {
|
|
238
|
+
const state = await readState();
|
|
239
|
+
const jobs = await readQueue();
|
|
240
|
+
return {
|
|
241
|
+
waiting: jobs.length,
|
|
242
|
+
// All jobs in queue are waiting (processed ones are removed)
|
|
243
|
+
active: 0,
|
|
244
|
+
// Local strategy doesn't track active jobs
|
|
245
|
+
completed: state.completedCount ?? 0,
|
|
246
|
+
failed: state.failedCount ?? 0
|
|
247
|
+
};
|
|
248
|
+
});
|
|
227
249
|
}
|
|
228
250
|
return {
|
|
229
251
|
name,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/local.ts"],
|
|
4
|
-
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions } from '../types'\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n * - No retry mechanism for failed jobs\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n function ensureDir(): void {\n // Use atomic operations to handle race conditions\n try {\n fs.mkdirSync(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n fs.writeFileSync(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n fs.writeFileSync(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n function backupCorruptedQueueFile(content: string): string {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n fs.writeFileSync(backupFile, content, 'utf8')\n fs.writeFileSync(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n function readQueue(): StoredJob<T>[] {\n ensureDir()\n let content: string\n\n try {\n content = fs.readFileSync(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n console.error(`[queue:${name}] Failed to read queue file:`, readError.message)\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n console.error(`[queue:${name}] Failed to read queue file:`, parseError.message)\n const backupFile = backupCorruptedQueueFile(content)\n console.error(`[queue:${name}] Backed up corrupted queue file to ${backupFile} and recreated queue.json`)\n return []\n }\n }\n\n function writeQueue(jobs: StoredJob<T>[]): void {\n ensureDir()\n fs.writeFileSync(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n function readState(): LocalState {\n ensureDir()\n try {\n const content = fs.readFileSync(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n function writeState(state: LocalState): void {\n ensureDir()\n fs.writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const jobs = readQueue()\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n }\n jobs.push(job)\n writeQueue(jobs)\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const state = readState()\n const jobs = readQueue()\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n console.log(`[queue:${name}] Job ${job.id} completed`)\n } catch (error) {\n console.error(`[queue:${name}] Job ${job.id} failed (attempt ${attemptNumber}/${DEFAULT_MAX_ATTEMPTS}):`, error)\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n console.error(`[queue:${name}] Job ${job.id} exhausted all ${DEFAULT_MAX_ATTEMPTS} attempts, moving to dead letter`)\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n const updatedJobs = jobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n writeState(newState)\n }\n\n return { processed, failed, lastJobId }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n console.error(`[queue:${name}] Polling error:`, error)\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n console.error(`[queue:${name}] Poll cycle error:`, err)\n })\n }, pollInterval)\n\n console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const jobs = readQueue()\n const removed = jobs.length\n writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = readState()\n writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n console.warn(`[queue:${name}] Force closing after ${SHUTDOWN_TIMEOUT}ms timeout`)\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const state = readState()\n const jobs = readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AAenB,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;
|
|
4
|
+
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions } from '../types'\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n * - No retry mechanism for failed jobs\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences to preserve the atomicity guarantees the\n * previous synchronous implementation relied on.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(() => fn(), () => fn())\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n async function backupCorruptedQueueFile(content: string): Promise<string> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n await fsp.writeFile(backupFile, content, 'utf8')\n await fsp.writeFile(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n console.error(`[queue:${name}] Failed to read queue file:`, readError.message)\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n console.error(`[queue:${name}] Failed to read queue file:`, parseError.message)\n const backupFile = await backupCorruptedQueueFile(content)\n console.error(`[queue:${name}] Backed up corrupted queue file to ${backupFile} and recreated queue.json`)\n return []\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n console.log(`[queue:${name}] Job ${job.id} completed`)\n } catch (error) {\n console.error(`[queue:${name}] Job ${job.id} failed (attempt ${attemptNumber}/${DEFAULT_MAX_ATTEMPTS}):`, error)\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n console.error(`[queue:${name}] Job ${job.id} exhausted all ${DEFAULT_MAX_ATTEMPTS} attempts, moving to dead letter`)\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n console.error(`[queue:${name}] Polling error:`, error)\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n console.error(`[queue:${name}] Poll cycle error:`, err)\n })\n }, pollInterval)\n\n console.log(`[queue:${name}] Worker started with concurrency ${concurrency}`)\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n console.warn(`[queue:${name}] Force closing after ${SHUTDOWN_TIMEOUT}ms timeout`)\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AAenB,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAE9B,MAAM,MAAM,GAAG;AAuBR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAG9C,MAAI,eAAsD;AAC1D,MAAI,eAAe;AACnB,MAAI,gBAAsC;AAI1C,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY,KAAK,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC;AACnD,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAEA,iBAAe,yBAAyB,SAAkC;AACxE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,OAAO;AAC3E,UAAM,IAAI,UAAU,YAAY,SAAS,MAAM;AAC/C,UAAM,IAAI,UAAU,WAAW,MAAM,MAAM;AAC3C,WAAO;AAAA,EACT;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,cAAQ,MAAM,UAAU,IAAI,gCAAgC,UAAU,OAAO;AAC7E,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,cAAQ,MAAM,UAAU,IAAI,gCAAgC,WAAW,OAAO;AAC9E,YAAM,aAAa,MAAM,yBAAyB,OAAO;AACzD,cAAQ,MAAM,UAAU,IAAI,uCAAuC,UAAU,2BAA2B;AACxG,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,MAAM;AAAA,EACtE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,MAAM;AAAA,EACvE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,IACvC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AAED,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,eAAW,OAAO,eAAe;AAC/B,YAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,UAAI;AACF,cAAM,QAAQ;AAAA,UACZ,QAAQ,KAAK;AAAA,YACX,OAAO,IAAI;AAAA,YACX;AAAA,YACA,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AACA;AACA,oBAAY,IAAI;AAChB,wBAAgB,IAAI,IAAI,EAAE;AAC1B,gBAAQ,IAAI,UAAU,IAAI,SAAS,IAAI,EAAE,YAAY;AAAA,MACvD,SAAS,OAAO;AACd,gBAAQ,MAAM,UAAU,IAAI,SAAS,IAAI,EAAE,oBAAoB,aAAa,IAAI,oBAAoB,MAAM,KAAK;AAC/G;AACA,oBAAY,IAAI;AAChB,YAAI,iBAAiB,sBAAsB;AACzC,kBAAQ,MAAM,UAAU,IAAI,SAAS,IAAI,EAAE,kBAAkB,oBAAoB,kCAAkC;AACnH,qBAAW,IAAI,IAAI,EAAE;AAAA,QACvB,OAAO;AACL,gBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,uBAAa,IAAI,IAAI,IAAI;AAAA,YACvB,GAAG;AAAA,YACH,cAAc;AAAA,YACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,UAC5D,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,UAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,QAAI,YAAY;AACd,YAAM,aAAa,YAAY;AAE7B,cAAM,cAAc,MAAM,UAAU;AACpC,cAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,cAAM,WAAW,WAAW;AAE5B,cAAM,WAAuB;AAAA,UAC3B,iBAAiB;AAAA,UACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,UAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,QACrD;AACA,cAAM,WAAW,QAAQ;AAAA,MAC3B,CAAC;AAAA,IACH;AAEA,WAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,EACxC;AAKA,iBAAe,iBAAgC;AAE7C,QAAI,gBAAgB,CAAC,cAAe;AAEpC,mBAAe;AACf,QAAI;AACF,YAAM,aAAa,aAAa;AAAA,IAClC,SAAS,OAAO;AACd,cAAQ,MAAM,UAAU,IAAI,oBAAoB,KAAK;AAAA,IACvD,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAGA,oBAAgB;AAGhB,UAAM,aAAa,OAAO;AAG1B,mBAAe,YAAY,MAAM;AAC/B,qBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,gBAAQ,MAAM,UAAU,IAAI,uBAAuB,GAAG;AAAA,MACxD,CAAC;AAAA,IACH,GAAG,YAAY;AAEf,YAAQ,IAAI,UAAU,IAAI,qCAAqC,WAAW,EAAE;AAG5E,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AAEpC,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AACA,oBAAgB;AAGhB,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,gBAAQ,KAAK,UAAU,IAAI,yBAAyB,gBAAgB,YAAY;AAChF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/queue",
|
|
3
|
-
"version": "0.4.11-develop.
|
|
3
|
+
"version": "0.4.11-develop.2502.29119c6047",
|
|
4
4
|
"description": "Multi-strategy job queue with local and BullMQ support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"access": "public"
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
|
-
"@open-mercato/shared": "0.4.11-develop.
|
|
53
|
+
"@open-mercato/shared": "0.4.11-develop.2502.29119c6047"
|
|
54
54
|
},
|
|
55
55
|
"stableVersion": "0.4.10"
|
|
56
56
|
}
|
|
@@ -309,4 +309,84 @@ describe('Queue - local strategy', () => {
|
|
|
309
309
|
|
|
310
310
|
await queue.close()
|
|
311
311
|
})
|
|
312
|
+
|
|
313
|
+
// Regression: queue operations MUST use async fs.promises.* so they do not
|
|
314
|
+
// block the Node.js event loop. See GitHub issue #1401.
|
|
315
|
+
test('queue operations do not call synchronous fs APIs on queue files', async () => {
|
|
316
|
+
const queueDir = path.join('.mercato', 'queue', 'sync-free')
|
|
317
|
+
const touchesQueue = (args: unknown[]) =>
|
|
318
|
+
args.some((arg) => typeof arg === 'string' && arg.includes(queueDir))
|
|
319
|
+
|
|
320
|
+
const syncCalls: string[] = []
|
|
321
|
+
const mkdirSpy = jest.spyOn(fs, 'mkdirSync').mockImplementation((...args: any[]) => {
|
|
322
|
+
if (touchesQueue(args)) syncCalls.push(`mkdirSync(${args[0]})`)
|
|
323
|
+
return undefined as any
|
|
324
|
+
})
|
|
325
|
+
const readSpy = jest.spyOn(fs, 'readFileSync').mockImplementation((...args: any[]) => {
|
|
326
|
+
if (touchesQueue(args)) syncCalls.push(`readFileSync(${args[0]})`)
|
|
327
|
+
return '' as any
|
|
328
|
+
})
|
|
329
|
+
const writeSpy = jest.spyOn(fs, 'writeFileSync').mockImplementation((...args: any[]) => {
|
|
330
|
+
if (touchesQueue(args)) syncCalls.push(`writeFileSync(${args[0]})`)
|
|
331
|
+
return undefined as any
|
|
332
|
+
})
|
|
333
|
+
|
|
334
|
+
try {
|
|
335
|
+
const queue = createQueue<{ value: number }>('sync-free', 'local')
|
|
336
|
+
|
|
337
|
+
await queue.enqueue({ value: 1 })
|
|
338
|
+
await queue.enqueue({ value: 2 })
|
|
339
|
+
await queue.getJobCounts()
|
|
340
|
+
await queue.process((_job) => {}, { limit: 10 })
|
|
341
|
+
await queue.clear()
|
|
342
|
+
await queue.close()
|
|
343
|
+
|
|
344
|
+
expect(syncCalls).toEqual([])
|
|
345
|
+
} finally {
|
|
346
|
+
mkdirSpy.mockRestore()
|
|
347
|
+
readSpy.mockRestore()
|
|
348
|
+
writeSpy.mockRestore()
|
|
349
|
+
}
|
|
350
|
+
})
|
|
351
|
+
|
|
352
|
+
// Regression: serialize enqueue calls so async fs writes cannot clobber
|
|
353
|
+
// each other. Before the async conversion this was trivially safe because
|
|
354
|
+
// sync I/O executed atomically. With async fs a mutex is required.
|
|
355
|
+
test('concurrent enqueues do not lose jobs', async () => {
|
|
356
|
+
const queue = createQueue<{ value: number }>('concurrent-queue', 'local')
|
|
357
|
+
const queuePath = path.join('.mercato', 'queue', 'concurrent-queue', 'queue.json')
|
|
358
|
+
|
|
359
|
+
const enqueueCount = 50
|
|
360
|
+
await Promise.all(
|
|
361
|
+
Array.from({ length: enqueueCount }, (_, idx) => queue.enqueue({ value: idx })),
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
const stored = readJson(queuePath)
|
|
365
|
+
expect(stored).toHaveLength(enqueueCount)
|
|
366
|
+
const storedValues = stored.map((job: any) => job.payload.value).sort((a: number, b: number) => a - b)
|
|
367
|
+
expect(storedValues).toEqual(Array.from({ length: enqueueCount }, (_, idx) => idx))
|
|
368
|
+
|
|
369
|
+
await queue.close()
|
|
370
|
+
})
|
|
371
|
+
|
|
372
|
+
// Regression: jobs enqueued while a batch is running must survive the
|
|
373
|
+
// subsequent write that removes completed jobs. The pre-fix snapshot-only
|
|
374
|
+
// write would clobber them.
|
|
375
|
+
test('jobs enqueued during batch handler are preserved on final write', async () => {
|
|
376
|
+
const queue = createQueue<{ value: number; latecomer?: boolean }>('race-queue', 'local')
|
|
377
|
+
const queuePath = path.join('.mercato', 'queue', 'race-queue', 'queue.json')
|
|
378
|
+
|
|
379
|
+
await queue.enqueue({ value: 1 })
|
|
380
|
+
|
|
381
|
+
await queue.process(async () => {
|
|
382
|
+
// Mid-handler, enqueue a second job. It should survive the final write.
|
|
383
|
+
await queue.enqueue({ value: 2, latecomer: true })
|
|
384
|
+
}, { limit: 10 })
|
|
385
|
+
|
|
386
|
+
const remaining = readJson(queuePath)
|
|
387
|
+
expect(remaining).toHaveLength(1)
|
|
388
|
+
expect(remaining[0].payload).toEqual({ value: 2, latecomer: true })
|
|
389
|
+
|
|
390
|
+
await queue.close()
|
|
391
|
+
})
|
|
312
392
|
})
|
package/src/strategies/local.ts
CHANGED
|
@@ -20,6 +20,8 @@ const DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'
|
|
|
20
20
|
const DEFAULT_MAX_ATTEMPTS = 3
|
|
21
21
|
const RETRY_BACKOFF_BASE_MS = 1000
|
|
22
22
|
|
|
23
|
+
const fsp = fs.promises
|
|
24
|
+
|
|
23
25
|
/**
|
|
24
26
|
* Creates a file-based local queue.
|
|
25
27
|
*
|
|
@@ -32,6 +34,11 @@ const RETRY_BACKOFF_BASE_MS = 1000
|
|
|
32
34
|
* - Not suitable for production or multi-process environments
|
|
33
35
|
* - No retry mechanism for failed jobs
|
|
34
36
|
*
|
|
37
|
+
* All file I/O is asynchronous (`fs.promises.*`) so queue operations do not
|
|
38
|
+
* block the Node.js event loop. A per-queue promise chain serializes
|
|
39
|
+
* read-modify-write sequences to preserve the atomicity guarantees the
|
|
40
|
+
* previous synchronous implementation relied on.
|
|
41
|
+
*
|
|
35
42
|
* @template T - The payload type for jobs
|
|
36
43
|
* @param name - Queue name (used for directory naming)
|
|
37
44
|
* @param options - Local queue options
|
|
@@ -56,14 +63,25 @@ export function createLocalQueue<T = unknown>(
|
|
|
56
63
|
let isProcessing = false
|
|
57
64
|
let activeHandler: JobHandler<T> | null = null
|
|
58
65
|
|
|
66
|
+
// Per-queue mutex. Serializes read-modify-write segments so async fs calls
|
|
67
|
+
// cannot interleave and clobber each other's writes.
|
|
68
|
+
let fileOpChain: Promise<unknown> = Promise.resolve()
|
|
69
|
+
function withFileLock<R>(fn: () => Promise<R>): Promise<R> {
|
|
70
|
+
const run = fileOpChain.then(() => fn(), () => fn())
|
|
71
|
+
fileOpChain = run.then(
|
|
72
|
+
() => undefined,
|
|
73
|
+
() => undefined,
|
|
74
|
+
)
|
|
75
|
+
return run
|
|
76
|
+
}
|
|
77
|
+
|
|
59
78
|
// -------------------------------------------------------------------------
|
|
60
79
|
// File Operations
|
|
61
80
|
// -------------------------------------------------------------------------
|
|
62
81
|
|
|
63
|
-
function ensureDir(): void {
|
|
64
|
-
// Use atomic operations to handle race conditions
|
|
82
|
+
async function ensureDir(): Promise<void> {
|
|
65
83
|
try {
|
|
66
|
-
|
|
84
|
+
await fsp.mkdir(queueDir, { recursive: true })
|
|
67
85
|
} catch (e: unknown) {
|
|
68
86
|
const error = e as NodeJS.ErrnoException
|
|
69
87
|
if (error.code !== 'EEXIST') throw error
|
|
@@ -71,7 +89,7 @@ export function createLocalQueue<T = unknown>(
|
|
|
71
89
|
|
|
72
90
|
// Initialize queue file with exclusive create flag
|
|
73
91
|
try {
|
|
74
|
-
|
|
92
|
+
await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })
|
|
75
93
|
} catch (e: unknown) {
|
|
76
94
|
const error = e as NodeJS.ErrnoException
|
|
77
95
|
if (error.code !== 'EEXIST') throw error
|
|
@@ -79,26 +97,26 @@ export function createLocalQueue<T = unknown>(
|
|
|
79
97
|
|
|
80
98
|
// Initialize state file with exclusive create flag
|
|
81
99
|
try {
|
|
82
|
-
|
|
100
|
+
await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })
|
|
83
101
|
} catch (e: unknown) {
|
|
84
102
|
const error = e as NodeJS.ErrnoException
|
|
85
103
|
if (error.code !== 'EEXIST') throw error
|
|
86
104
|
}
|
|
87
105
|
}
|
|
88
106
|
|
|
89
|
-
function backupCorruptedQueueFile(content: string): string {
|
|
107
|
+
async function backupCorruptedQueueFile(content: string): Promise<string> {
|
|
90
108
|
const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)
|
|
91
|
-
|
|
92
|
-
|
|
109
|
+
await fsp.writeFile(backupFile, content, 'utf8')
|
|
110
|
+
await fsp.writeFile(queueFile, '[]', 'utf8')
|
|
93
111
|
return backupFile
|
|
94
112
|
}
|
|
95
113
|
|
|
96
|
-
function readQueue(): StoredJob<T>[] {
|
|
97
|
-
ensureDir()
|
|
114
|
+
async function readQueue(): Promise<StoredJob<T>[]> {
|
|
115
|
+
await ensureDir()
|
|
98
116
|
let content: string
|
|
99
117
|
|
|
100
118
|
try {
|
|
101
|
-
content =
|
|
119
|
+
content = await fsp.readFile(queueFile, 'utf8')
|
|
102
120
|
} catch (error: unknown) {
|
|
103
121
|
const readError = error as NodeJS.ErrnoException
|
|
104
122
|
if (readError.code === 'ENOENT') {
|
|
@@ -119,30 +137,30 @@ export function createLocalQueue<T = unknown>(
|
|
|
119
137
|
} catch (error: unknown) {
|
|
120
138
|
const parseError = error as Error
|
|
121
139
|
console.error(`[queue:${name}] Failed to read queue file:`, parseError.message)
|
|
122
|
-
const backupFile = backupCorruptedQueueFile(content)
|
|
140
|
+
const backupFile = await backupCorruptedQueueFile(content)
|
|
123
141
|
console.error(`[queue:${name}] Backed up corrupted queue file to ${backupFile} and recreated queue.json`)
|
|
124
142
|
return []
|
|
125
143
|
}
|
|
126
144
|
}
|
|
127
145
|
|
|
128
|
-
function writeQueue(jobs: StoredJob<T>[]): void {
|
|
129
|
-
ensureDir()
|
|
130
|
-
|
|
146
|
+
async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {
|
|
147
|
+
await ensureDir()
|
|
148
|
+
await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')
|
|
131
149
|
}
|
|
132
150
|
|
|
133
|
-
function readState(): LocalState {
|
|
134
|
-
ensureDir()
|
|
151
|
+
async function readState(): Promise<LocalState> {
|
|
152
|
+
await ensureDir()
|
|
135
153
|
try {
|
|
136
|
-
const content =
|
|
154
|
+
const content = await fsp.readFile(stateFile, 'utf8')
|
|
137
155
|
return JSON.parse(content) as LocalState
|
|
138
156
|
} catch {
|
|
139
157
|
return {}
|
|
140
158
|
}
|
|
141
159
|
}
|
|
142
160
|
|
|
143
|
-
function writeState(state: LocalState): void {
|
|
144
|
-
ensureDir()
|
|
145
|
-
|
|
161
|
+
async function writeState(state: LocalState): Promise<void> {
|
|
162
|
+
await ensureDir()
|
|
163
|
+
await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')
|
|
146
164
|
}
|
|
147
165
|
|
|
148
166
|
function generateId(): string {
|
|
@@ -154,7 +172,6 @@ export function createLocalQueue<T = unknown>(
|
|
|
154
172
|
// -------------------------------------------------------------------------
|
|
155
173
|
|
|
156
174
|
async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {
|
|
157
|
-
const jobs = readQueue()
|
|
158
175
|
const availableAt = options?.delayMs && options.delayMs > 0
|
|
159
176
|
? new Date(Date.now() + options.delayMs).toISOString()
|
|
160
177
|
: undefined
|
|
@@ -164,8 +181,11 @@ export function createLocalQueue<T = unknown>(
|
|
|
164
181
|
createdAt: new Date().toISOString(),
|
|
165
182
|
...(availableAt ? { availableAt } : {}),
|
|
166
183
|
}
|
|
167
|
-
|
|
168
|
-
|
|
184
|
+
await withFileLock(async () => {
|
|
185
|
+
const jobs = await readQueue()
|
|
186
|
+
jobs.push(job)
|
|
187
|
+
await writeQueue(jobs)
|
|
188
|
+
})
|
|
169
189
|
return job.id
|
|
170
190
|
}
|
|
171
191
|
|
|
@@ -176,8 +196,11 @@ export function createLocalQueue<T = unknown>(
|
|
|
176
196
|
handler: JobHandler<T>,
|
|
177
197
|
options?: ProcessOptions
|
|
178
198
|
): Promise<ProcessResult> {
|
|
179
|
-
const state =
|
|
180
|
-
|
|
199
|
+
const { state, jobs } = await withFileLock(async () => {
|
|
200
|
+
const stateRead = await readState()
|
|
201
|
+
const jobsRead = await readQueue()
|
|
202
|
+
return { state: stateRead, jobs: jobsRead }
|
|
203
|
+
})
|
|
181
204
|
|
|
182
205
|
const pendingJobs = jobs.filter((job) => {
|
|
183
206
|
if (!job.availableAt) return true
|
|
@@ -228,17 +251,21 @@ export function createLocalQueue<T = unknown>(
|
|
|
228
251
|
|
|
229
252
|
const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0
|
|
230
253
|
if (hasChanges) {
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
254
|
+
await withFileLock(async () => {
|
|
255
|
+
// Re-read so jobs enqueued during handler execution are preserved.
|
|
256
|
+
const currentJobs = await readQueue()
|
|
257
|
+
const updatedJobs = currentJobs
|
|
258
|
+
.filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))
|
|
259
|
+
.map((j) => retryUpdates.get(j.id) ?? j)
|
|
260
|
+
await writeQueue(updatedJobs)
|
|
261
|
+
|
|
262
|
+
const newState: LocalState = {
|
|
263
|
+
lastProcessedId: lastJobId,
|
|
264
|
+
completedCount: (state.completedCount ?? 0) + processed,
|
|
265
|
+
failedCount: (state.failedCount ?? 0) + deadJobIds.size,
|
|
266
|
+
}
|
|
267
|
+
await writeState(newState)
|
|
268
|
+
})
|
|
242
269
|
}
|
|
243
270
|
|
|
244
271
|
return { processed, failed, lastJobId }
|
|
@@ -290,16 +317,18 @@ export function createLocalQueue<T = unknown>(
|
|
|
290
317
|
}
|
|
291
318
|
|
|
292
319
|
async function clear(): Promise<{ removed: number }> {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
320
|
+
return withFileLock(async () => {
|
|
321
|
+
const jobs = await readQueue()
|
|
322
|
+
const removed = jobs.length
|
|
323
|
+
await writeQueue([])
|
|
324
|
+
// Reset state but preserve counts for historical tracking
|
|
325
|
+
const state = await readState()
|
|
326
|
+
await writeState({
|
|
327
|
+
completedCount: state.completedCount,
|
|
328
|
+
failedCount: state.failedCount,
|
|
329
|
+
})
|
|
330
|
+
return { removed }
|
|
301
331
|
})
|
|
302
|
-
return { removed }
|
|
303
332
|
}
|
|
304
333
|
|
|
305
334
|
async function close(): Promise<void> {
|
|
@@ -329,15 +358,17 @@ export function createLocalQueue<T = unknown>(
|
|
|
329
358
|
completed: number
|
|
330
359
|
failed: number
|
|
331
360
|
}> {
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
361
|
+
return withFileLock(async () => {
|
|
362
|
+
const state = await readState()
|
|
363
|
+
const jobs = await readQueue()
|
|
364
|
+
|
|
365
|
+
return {
|
|
366
|
+
waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)
|
|
367
|
+
active: 0, // Local strategy doesn't track active jobs
|
|
368
|
+
completed: state.completedCount ?? 0,
|
|
369
|
+
failed: state.failedCount ?? 0,
|
|
370
|
+
}
|
|
371
|
+
})
|
|
341
372
|
}
|
|
342
373
|
|
|
343
374
|
return {
|