@open-mercato/queue 0.6.8-develop.6925.1.cefa200fa4 → 0.6.8-develop.6930.1.1e5976efc3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/strategies/local.js +136 -11
- package/dist/strategies/local.js.map +2 -2
- package/package.json +3 -3
- package/src/__tests__/local.strategy.test.ts +157 -8
- package/src/strategies/local.ts +220 -15
package/dist/strategies/local.js
CHANGED
|
@@ -20,6 +20,12 @@ const DEFAULT_POLL_INTERVAL = 1e3;
|
|
|
20
20
|
const DEFAULT_LOCAL_QUEUE_BASE_DIR = ".mercato/queue";
|
|
21
21
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
22
22
|
const RETRY_BACKOFF_BASE_MS = 1e3;
|
|
23
|
+
const LOCK_STALE_MS = 15e3;
|
|
24
|
+
const LOCK_ACQUIRE_TIMEOUT_MS = 3e4;
|
|
25
|
+
const LOCK_RETRY_MIN_MS = 2;
|
|
26
|
+
const LOCK_RETRY_MAX_MS = 20;
|
|
27
|
+
const RENAME_MAX_RETRIES = 5;
|
|
28
|
+
const RENAME_RETRY_BASE_MS = 10;
|
|
23
29
|
const fsp = fs.promises;
|
|
24
30
|
function createLocalQueue(name, options) {
|
|
25
31
|
const nodeProcess = globalThis.process;
|
|
@@ -28,6 +34,8 @@ function createLocalQueue(name, options) {
|
|
|
28
34
|
const queueDir = path.join(baseDir, name);
|
|
29
35
|
const queueFile = path.join(queueDir, "queue.json");
|
|
30
36
|
const stateFile = path.join(queueDir, "state.json");
|
|
37
|
+
const lockDir = path.join(queueDir, "queue.lock");
|
|
38
|
+
const lockOwnerFile = path.join(lockDir, "owner");
|
|
31
39
|
const logger = packageLogger.child({ queue: name });
|
|
32
40
|
const concurrency = options?.concurrency ?? 1;
|
|
33
41
|
const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL;
|
|
@@ -37,13 +45,121 @@ function createLocalQueue(name, options) {
|
|
|
37
45
|
const inFlightJobIds = /* @__PURE__ */ new Set();
|
|
38
46
|
let fileOpChain = Promise.resolve();
|
|
39
47
|
function withFileLock(fn) {
|
|
40
|
-
const run = fileOpChain.then(
|
|
48
|
+
const run = fileOpChain.then(
|
|
49
|
+
() => runExclusively(fn),
|
|
50
|
+
() => runExclusively(fn)
|
|
51
|
+
);
|
|
41
52
|
fileOpChain = run.then(
|
|
42
53
|
() => void 0,
|
|
43
54
|
() => void 0
|
|
44
55
|
);
|
|
45
56
|
return run;
|
|
46
57
|
}
|
|
58
|
+
async function runExclusively(fn) {
|
|
59
|
+
await ensureDir();
|
|
60
|
+
const release = await acquireDirectoryLock();
|
|
61
|
+
try {
|
|
62
|
+
return await fn();
|
|
63
|
+
} finally {
|
|
64
|
+
await release();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function sleep(ms) {
|
|
68
|
+
return new Promise((resolve) => {
|
|
69
|
+
setTimeout(resolve, ms);
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
async function lockHeldForMs() {
|
|
73
|
+
try {
|
|
74
|
+
const stats = await fsp.stat(lockDir);
|
|
75
|
+
return Date.now() - stats.mtimeMs;
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
async function reclaimStaleLock(heldForMs) {
|
|
81
|
+
const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`;
|
|
82
|
+
try {
|
|
83
|
+
await fsp.rename(lockDir, reclaimedPath);
|
|
84
|
+
} catch {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
logger.warn("Reclaimed a stale queue lock", { lockDir, heldForMs });
|
|
88
|
+
await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
async function readLockOwner() {
|
|
92
|
+
try {
|
|
93
|
+
return await fsp.readFile(lockOwnerFile, "utf8");
|
|
94
|
+
} catch {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async function releaseDirectoryLock(token) {
|
|
99
|
+
if (await readLockOwner() !== token) return;
|
|
100
|
+
await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
async function acquireDirectoryLock() {
|
|
104
|
+
const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS;
|
|
105
|
+
for (; ; ) {
|
|
106
|
+
let acquired = false;
|
|
107
|
+
try {
|
|
108
|
+
await fsp.mkdir(lockDir);
|
|
109
|
+
acquired = true;
|
|
110
|
+
} catch (e) {
|
|
111
|
+
const error = e;
|
|
112
|
+
if (error.code !== "EEXIST") throw error;
|
|
113
|
+
}
|
|
114
|
+
if (acquired) {
|
|
115
|
+
const token = crypto.randomUUID();
|
|
116
|
+
try {
|
|
117
|
+
await fsp.writeFile(lockOwnerFile, token, "utf8");
|
|
118
|
+
} catch (error) {
|
|
119
|
+
await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {
|
|
120
|
+
});
|
|
121
|
+
throw error;
|
|
122
|
+
}
|
|
123
|
+
return () => releaseDirectoryLock(token);
|
|
124
|
+
}
|
|
125
|
+
const heldForMs = await lockHeldForMs();
|
|
126
|
+
if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {
|
|
127
|
+
await reclaimStaleLock(heldForMs);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (Date.now() >= deadline) {
|
|
131
|
+
throw new Error(
|
|
132
|
+
`[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS);
|
|
136
|
+
await sleep(jitter);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
async function writeFileAtomic(targetFile, content) {
|
|
140
|
+
const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`;
|
|
141
|
+
try {
|
|
142
|
+
await fsp.writeFile(tempFile, content, "utf8");
|
|
143
|
+
await renameWithContentionRetry(tempFile, targetFile);
|
|
144
|
+
} catch (error) {
|
|
145
|
+
await fsp.rm(tempFile, { force: true }).catch(() => {
|
|
146
|
+
});
|
|
147
|
+
throw error;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
async function renameWithContentionRetry(fromFile, toFile) {
|
|
151
|
+
const contentionCodes = /* @__PURE__ */ new Set(["EPERM", "EBUSY", "EACCES"]);
|
|
152
|
+
for (let attempt = 0; ; attempt++) {
|
|
153
|
+
try {
|
|
154
|
+
await fsp.rename(fromFile, toFile);
|
|
155
|
+
return;
|
|
156
|
+
} catch (e) {
|
|
157
|
+
const error = e;
|
|
158
|
+
if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error;
|
|
159
|
+
await sleep(RENAME_RETRY_BASE_MS * (attempt + 1));
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
47
163
|
async function ensureDir() {
|
|
48
164
|
try {
|
|
49
165
|
await fsp.mkdir(queueDir, { recursive: true });
|
|
@@ -64,11 +180,15 @@ function createLocalQueue(name, options) {
|
|
|
64
180
|
if (error.code !== "EEXIST") throw error;
|
|
65
181
|
}
|
|
66
182
|
}
|
|
67
|
-
async function
|
|
68
|
-
const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`);
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
183
|
+
async function quarantineCorruptedQueueFile() {
|
|
184
|
+
const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`);
|
|
185
|
+
try {
|
|
186
|
+
await fsp.rename(queueFile, backupFile);
|
|
187
|
+
return backupFile;
|
|
188
|
+
} catch (e) {
|
|
189
|
+
logger.error("Failed to quarantine the corrupted queue file", { err: e });
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
72
192
|
}
|
|
73
193
|
async function readQueue() {
|
|
74
194
|
await ensureDir();
|
|
@@ -92,14 +212,19 @@ function createLocalQueue(name, options) {
|
|
|
92
212
|
} catch (error) {
|
|
93
213
|
const parseError = error;
|
|
94
214
|
logger.error("Failed to parse queue file", { err: parseError });
|
|
95
|
-
const backupFile = await
|
|
96
|
-
|
|
97
|
-
|
|
215
|
+
const backupFile = await quarantineCorruptedQueueFile();
|
|
216
|
+
if (backupFile) {
|
|
217
|
+
logger.error("Quarantined corrupted queue file; its jobs are recoverable from the backup", { backupFile });
|
|
218
|
+
}
|
|
219
|
+
const recoveryHint = backupFile ? `has been quarantined as ${backupFile}` : "could not be quarantined and was left in place";
|
|
220
|
+
throw new Error(
|
|
221
|
+
`[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`
|
|
222
|
+
);
|
|
98
223
|
}
|
|
99
224
|
}
|
|
100
225
|
async function writeQueue(jobs) {
|
|
101
226
|
await ensureDir();
|
|
102
|
-
await
|
|
227
|
+
await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2));
|
|
103
228
|
}
|
|
104
229
|
async function readState() {
|
|
105
230
|
await ensureDir();
|
|
@@ -112,7 +237,7 @@ function createLocalQueue(name, options) {
|
|
|
112
237
|
}
|
|
113
238
|
async function writeState(state) {
|
|
114
239
|
await ensureDir();
|
|
115
|
-
await
|
|
240
|
+
await writeFileAtomic(stateFile, JSON.stringify(state, null, 2));
|
|
116
241
|
}
|
|
117
242
|
function generateId() {
|
|
118
243
|
return crypto.randomUUID();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/local.ts"],
|
|
4
|
-
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.\n * **This strategy keeps no failed-job store**: once attempts are exhausted the job is\n * removed from `queue.json` and only counted in `state.failedCount`, so the payload is\n * lost and the failure survives solely as an error log line. The `async` strategy keeps\n * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000\n * failures and removes older ones as later failures arrive. Workflows that require\n * no-loss persistence must write their own durable record before enqueueing, regardless\n * of strategy.\n *\n * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option \u2014 callers cannot\n * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)\n * See the retry handling in `process()` below.\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences to preserve the atomicity guarantees the\n * previous synchronous implementation relied on.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(() => fn(), () => fn())\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n async function backupCorruptedQueueFile(content: string): Promise<string> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n await fsp.writeFile(backupFile, content, 'utf8')\n await fsp.writeFile(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await backupCorruptedQueueFile(content)\n logger.error('Backed up corrupted queue file and recreated queue.json', { backupFile })\n return []\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const metadata = attachTraceMetadata(undefined)\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n ...(metadata ? { metadata } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await runJobInTrace(name, job.metadata, () =>\n Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, pollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAE7B,SAAS,qBAAqB,qBAAqB;AAEnD,MAAM,gBAAgB,aAAa,OAAO;AAa1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;
|
|
4
|
+
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\n/**\n * Cross-process lock tuning. A held lock only ever spans local file I/O \u2014 job\n * handlers run outside it \u2014 so realistic hold times are milliseconds and the\n * stale threshold sits orders of magnitude above them. It exists solely so a\n * process that dies mid-segment cannot wedge the queue forever. A holder that\n * was merely suspended rather than dead can still be reclaimed, which is why\n * every acquisition carries an owner token and releases only its own lock.\n */\nconst LOCK_STALE_MS = 15_000\nconst LOCK_ACQUIRE_TIMEOUT_MS = 30_000\nconst LOCK_RETRY_MIN_MS = 2\nconst LOCK_RETRY_MAX_MS = 20\nconst RENAME_MAX_RETRIES = 5\nconst RENAME_RETRY_BASE_MS = 10\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production: there is no dead-letter store, no throughput\n * beyond one job at a time, and every operation rewrites the whole queue file\n *\n * Multiple processes MAY share a queue directory, which is the default\n * development topology: the dev worker runs in its own process alongside the\n * Next.js server. What that buys you, and what it does not:\n *\n * - **Safe** \u2014 concurrent producers. Every read-modify-write segment takes the\n * `queue.lock` directory lock and every persist swaps the file in with an\n * atomic rename, so the file cannot be torn, no enqueue is lost to a\n * concurrent one, and a reader always observes one complete document.\n * Writers contend, though, so throughput degrades as processes are added.\n * - **NOT safe** \u2014 concurrent consumers. `process()` deliberately runs job\n * handlers outside the lock, so two worker processes polling the same queue\n * would both claim the same pending jobs and execute them twice. There is no\n * per-job lease. Run exactly one worker process per queue; use the `async`\n * strategy when you need more than one.\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.\n * **This strategy keeps no failed-job store**: once attempts are exhausted the job is\n * removed from `queue.json` and only counted in `state.failedCount`, so the payload is\n * lost and the failure survives solely as an error log line. The `async` strategy keeps\n * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000\n * failures and removes older ones as later failures arrive. Workflows that require\n * no-loss persistence must write their own durable record before enqueueing, regardless\n * of strategy.\n *\n * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option \u2014 callers cannot\n * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)\n * See the retry handling in `process()` below.\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences within one instance, and the `queue.lock`\n * directory lock extends that serialization across instances and processes.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const lockDir = path.join(queueDir, 'queue.lock')\n const lockOwnerFile = path.join(lockDir, 'owner')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes. It only covers this\n // instance, so it also guarantees at most one outstanding `queue.lock`\n // acquisition per instance \u2014 the directory lock below is not reentrant.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(\n () => runExclusively(fn),\n () => runExclusively(fn),\n )\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n /**\n * Runs `fn` while holding the cross-process `queue.lock`, so read-modify-write\n * segments issued by other queue instances \u2014 in this process or another one \u2014\n * cannot interleave with it.\n */\n async function runExclusively<R>(fn: () => Promise<R>): Promise<R> {\n await ensureDir()\n const release = await acquireDirectoryLock()\n try {\n return await fn()\n } finally {\n await release()\n }\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => { setTimeout(resolve, ms) })\n }\n\n async function lockHeldForMs(): Promise<number | null> {\n try {\n const stats = await fsp.stat(lockDir)\n return Date.now() - stats.mtimeMs\n } catch {\n return null\n }\n }\n\n /**\n * Reclaims a lock whose holder died. The rename is the serialization point:\n * only one racer can move `queue.lock` aside, so two processes cannot both\n * decide a stale lock is theirs to clear and then both create a fresh one.\n */\n async function reclaimStaleLock(heldForMs: number): Promise<void> {\n const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`\n try {\n await fsp.rename(lockDir, reclaimedPath)\n } catch {\n return\n }\n logger.warn('Reclaimed a stale queue lock', { lockDir, heldForMs })\n await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {})\n }\n\n async function readLockOwner(): Promise<string | null> {\n try {\n return await fsp.readFile(lockOwnerFile, 'utf8')\n } catch {\n return null\n }\n }\n\n /**\n * Releases the lock only when this acquisition still owns it. A holder that\n * was suspended past `LOCK_STALE_MS` has had its lock reclaimed *and\n * replaced* by whoever reclaimed it, so an unconditional removal here would\n * delete the successor's lock and let a third caller into the critical\n * section alongside it. A missing or mismatched token means someone else owns\n * the path now, and the correct action is to leave it alone.\n */\n async function releaseDirectoryLock(token: string): Promise<void> {\n if (await readLockOwner() !== token) return\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n }\n\n /**\n * Acquires the cross-process advisory lock for this queue directory.\n * `mkdir` without `recursive` is an atomic exclusive create on every platform\n * Node.js supports, which makes it the portable primitive here \u2014 no runtime\n * dependency, and no reliance on advisory `flock` semantics. The owner token\n * written into the directory is what lets the release distinguish this\n * acquisition from a successor's.\n */\n async function acquireDirectoryLock(): Promise<() => Promise<void>> {\n const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS\n\n for (;;) {\n let acquired = false\n try {\n await fsp.mkdir(lockDir)\n acquired = true\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n if (acquired) {\n const token = crypto.randomUUID()\n try {\n await fsp.writeFile(lockOwnerFile, token, 'utf8')\n } catch (error: unknown) {\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n throw error\n }\n return () => releaseDirectoryLock(token)\n }\n\n const heldForMs = await lockHeldForMs()\n if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {\n await reclaimStaleLock(heldForMs)\n continue\n }\n\n if (Date.now() >= deadline) {\n throw new Error(\n `[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`,\n )\n }\n\n const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)\n await sleep(jitter)\n }\n }\n\n /**\n * Persists `content` by writing a unique sibling temp file and renaming it\n * onto `targetFile`. `rename` within a directory is atomic, so a concurrent\n * reader sees either the previous document or the new one in full \u2014 never the\n * torn result of a truncate-then-write.\n */\n async function writeFileAtomic(targetFile: string, content: string): Promise<void> {\n const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`\n try {\n await fsp.writeFile(tempFile, content, 'utf8')\n await renameWithContentionRetry(tempFile, targetFile)\n } catch (error: unknown) {\n await fsp.rm(tempFile, { force: true }).catch(() => {})\n throw error\n }\n }\n\n /**\n * Windows rejects a rename onto a file another process currently has open,\n * so retry briefly on the contention codes it raises. POSIX renames replace\n * the target unconditionally and take the first attempt.\n */\n async function renameWithContentionRetry(fromFile: string, toFile: string): Promise<void> {\n const contentionCodes = new Set(['EPERM', 'EBUSY', 'EACCES'])\n for (let attempt = 0; ; attempt++) {\n try {\n await fsp.rename(fromFile, toFile)\n return\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error\n await sleep(RENAME_RETRY_BASE_MS * (attempt + 1))\n }\n }\n }\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n /**\n * Moves an unparsable queue file aside so its jobs stay recoverable. The\n * caller is expected to surface the failure rather than continue on an empty\n * queue: silently recreating `queue.json` here is what turned an unreadable\n * file into permanent, unreported job loss.\n */\n async function quarantineCorruptedQueueFile(): Promise<string | null> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`)\n try {\n await fsp.rename(queueFile, backupFile)\n return backupFile\n } catch (e: unknown) {\n logger.error('Failed to quarantine the corrupted queue file', { err: e as Error })\n return null\n }\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await quarantineCorruptedQueueFile()\n if (backupFile) {\n logger.error('Quarantined corrupted queue file; its jobs are recoverable from the backup', { backupFile })\n }\n const recoveryHint = backupFile\n ? `has been quarantined as ${backupFile}`\n : 'could not be quarantined and was left in place'\n throw new Error(\n `[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`,\n )\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2))\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await writeFileAtomic(stateFile, JSON.stringify(state, null, 2))\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const metadata = attachTraceMetadata(undefined)\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n ...(metadata ? { metadata } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await runJobInTrace(name, job.metadata, () =>\n Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, pollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAE7B,SAAS,qBAAqB,qBAAqB;AAEnD,MAAM,gBAAgB,aAAa,OAAO;AAa1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAU9B,MAAM,gBAAgB;AACtB,MAAM,0BAA0B;AAChC,MAAM,oBAAoB;AAC1B,MAAM,oBAAoB;AAC1B,MAAM,qBAAqB;AAC3B,MAAM,uBAAuB;AAE7B,MAAM,MAAM,GAAG;AAmDR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,UAAU,KAAK,KAAK,UAAU,YAAY;AAChD,QAAM,gBAAgB,KAAK,KAAK,SAAS,OAAO;AAChD,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAG9C,MAAI,eAAsD;AAC1D,MAAI,eAAe;AACnB,MAAI,gBAAsC;AAC1C,QAAM,iBAAiB,oBAAI,IAAY;AAMvC,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY;AAAA,MACtB,MAAM,eAAe,EAAE;AAAA,MACvB,MAAM,eAAe,EAAE;AAAA,IACzB;AACA,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAOA,iBAAe,eAAkB,IAAkC;AACjE,UAAM,UAAU;AAChB,UAAM,UAAU,MAAM,qBAAqB;AAC3C,QAAI;AACF,aAAO,MAAM,GAAG;AAAA,IAClB,UAAE;AACA,YAAM,QAAQ;AAAA,IAChB;AAAA,EACF;AAMA,WAAS,MAAM,IAA2B;AACxC,WAAO,IAAI,QAAQ,CAAC,YAAY;AAAE,iBAAW,SAAS,EAAE;AAAA,IAAE,CAAC;AAAA,EAC7D;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,YAAM,QAAQ,MAAM,IAAI,KAAK,OAAO;AACpC,aAAO,KAAK,IAAI,IAAI,MAAM;AAAA,IAC5B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAOA,iBAAe,iBAAiB,WAAkC;AAChE,UAAM,gBAAgB,GAAG,OAAO,UAAU,OAAO,WAAW,CAAC;AAC7D,QAAI;AACF,YAAM,IAAI,OAAO,SAAS,aAAa;AAAA,IACzC,QAAQ;AACN;AAAA,IACF;AACA,WAAO,KAAK,gCAAgC,EAAE,SAAS,UAAU,CAAC;AAClE,UAAM,IAAI,GAAG,eAAe,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC9E;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,aAAO,MAAM,IAAI,SAAS,eAAe,MAAM;AAAA,IACjD,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAUA,iBAAe,qBAAqB,OAA8B;AAChE,QAAI,MAAM,cAAc,MAAM,MAAO;AACrC,UAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACxE;AAUA,iBAAe,uBAAqD;AAClE,UAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,eAAS;AACP,UAAI,WAAW;AACf,UAAI;AACF,cAAM,IAAI,MAAM,OAAO;AACvB,mBAAW;AAAA,MACb,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,MAAM,SAAS,SAAU,OAAM;AAAA,MACrC;AAEA,UAAI,UAAU;AACZ,cAAM,QAAQ,OAAO,WAAW;AAChC,YAAI;AACF,gBAAM,IAAI,UAAU,eAAe,OAAO,MAAM;AAAA,QAClD,SAAS,OAAgB;AACvB,gBAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACtE,gBAAM;AAAA,QACR;AACA,eAAO,MAAM,qBAAqB,KAAK;AAAA,MACzC;AAEA,YAAM,YAAY,MAAM,cAAc;AACtC,UAAI,cAAc,QAAQ,YAAY,eAAe;AACnD,cAAM,iBAAiB,SAAS;AAChC;AAAA,MACF;AAEA,UAAI,KAAK,IAAI,KAAK,UAAU;AAC1B,cAAM,IAAI;AAAA,UACR,8BAA8B,uBAAuB,oCAAoC,OAAO;AAAA,QAClG;AAAA,MACF;AAEA,YAAM,SAAS,oBAAoB,KAAK,OAAO,KAAK,oBAAoB;AACxE,YAAM,MAAM,MAAM;AAAA,IACpB;AAAA,EACF;AAQA,iBAAe,gBAAgB,YAAoB,SAAgC;AACjF,UAAM,WAAW,GAAG,UAAU,IAAI,OAAO,WAAW,CAAC;AACrD,QAAI;AACF,YAAM,IAAI,UAAU,UAAU,SAAS,MAAM;AAC7C,YAAM,0BAA0B,UAAU,UAAU;AAAA,IACtD,SAAS,OAAgB;AACvB,YAAM,IAAI,GAAG,UAAU,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACtD,YAAM;AAAA,IACR;AAAA,EACF;AAOA,iBAAe,0BAA0B,UAAkB,QAA+B;AACxF,UAAM,kBAAkB,oBAAI,IAAI,CAAC,SAAS,SAAS,QAAQ,CAAC;AAC5D,aAAS,UAAU,KAAK,WAAW;AACjC,UAAI;AACF,cAAM,IAAI,OAAO,UAAU,MAAM;AACjC;AAAA,MACF,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,WAAW,sBAAsB,CAAC,MAAM,QAAQ,CAAC,gBAAgB,IAAI,MAAM,IAAI,EAAG,OAAM;AAC5F,cAAM,MAAM,wBAAwB,UAAU,EAAE;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAQA,iBAAe,+BAAuD;AACpE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,IAAI,OAAO,WAAW,CAAC,OAAO;AAClG,QAAI;AACF,YAAM,IAAI,OAAO,WAAW,UAAU;AACtC,aAAO;AAAA,IACT,SAAS,GAAY;AACnB,aAAO,MAAM,iDAAiD,EAAE,KAAK,EAAW,CAAC;AACjF,aAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,aAAO,MAAM,6BAA6B,EAAE,KAAK,UAAU,CAAC;AAC5D,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAW,CAAC;AAC9D,YAAM,aAAa,MAAM,6BAA6B;AACtD,UAAI,YAAY;AACd,eAAO,MAAM,8EAA8E,EAAE,WAAW,CAAC;AAAA,MAC3G;AACA,YAAM,eAAe,aACjB,2BAA2B,UAAU,KACrC;AACJ,YAAM,IAAI;AAAA,QACR,yBAAyB,SAAS,uBAAuB,YAAY,KAAK,WAAW,OAAO;AAAA,MAC9F;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,EACjE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,WAAW,oBAAoB,MAAS;AAC9C,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,MACrC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AAED,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,eAAW,OAAO,eAAe;AAC/B,qBAAe,IAAI,IAAI,EAAE;AAAA,IAC3B;AAEA,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,QAAI;AACF,iBAAW,OAAO,eAAe;AAC/B,cAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,YAAI;AACF,gBAAM;AAAA,YAAc;AAAA,YAAM,IAAI;AAAA,YAAU,MACtC,QAAQ;AAAA,cACN,QAAQ,KAAK;AAAA,gBACX,OAAO,IAAI;AAAA,gBACX;AAAA,gBACA,WAAW;AAAA,cACb,CAAC;AAAA,YACH;AAAA,UACF;AACA;AACA,sBAAY,IAAI;AAChB,0BAAgB,IAAI,IAAI,EAAE;AAC1B,iBAAO,KAAK,iBAAiB,EAAE,OAAO,IAAI,GAAG,CAAC;AAAA,QAChD,SAAS,OAAO;AACd,iBAAO,MAAM,cAAc,EAAE,OAAO,IAAI,IAAI,eAAe,aAAa,sBAAsB,KAAK,MAAM,CAAC;AAC1G;AACA,sBAAY,IAAI;AAChB,cAAI,iBAAiB,sBAAsB;AACzC,mBAAO,MAAM,kEAAkE,EAAE,OAAO,IAAI,IAAI,aAAa,qBAAqB,CAAC;AACnI,uBAAW,IAAI,IAAI,EAAE;AAAA,UACvB,OAAO;AACL,kBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,yBAAa,IAAI,IAAI,IAAI;AAAA,cACvB,GAAG;AAAA,cACH,cAAc;AAAA,cACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,YAC5D,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,UAAI,YAAY;AACd,cAAM,aAAa,YAAY;AAE7B,gBAAM,cAAc,MAAM,UAAU;AACpC,gBAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,gBAAM,WAAW,WAAW;AAE5B,gBAAM,WAAuB;AAAA,YAC3B,iBAAiB;AAAA,YACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,YAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,UACrD;AACA,gBAAM,WAAW,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAEA,aAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,IACxC,UAAE;AACA,iBAAW,OAAO,eAAe;AAC/B,uBAAe,OAAO,IAAI,EAAE;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAKA,iBAAe,iBAAgC;AAE7C,QAAI,gBAAgB,CAAC,cAAe;AAEpC,mBAAe;AACf,QAAI;AACF,YAAM,aAAa,aAAa;AAAA,IAClC,SAAS,OAAO;AACd,aAAO,MAAM,iBAAiB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC9C,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAGA,oBAAgB;AAGhB,UAAM,aAAa,OAAO;AAG1B,mBAAe,YAAY,MAAM;AAC/B,qBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,eAAO,MAAM,oBAAoB,EAAE,IAAI,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH,GAAG,YAAY;AAEf,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAG7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,eAAe,KAAK,OAAO,CAAC,QAAQ,eAAe,IAAI,IAAI,EAAE,KAAK,CAAC,oBAAoB,IAAI,SAAS,KAAK,CAAC;AAChH,YAAM,UAAU,KAAK,SAAS,aAAa;AAC3C,UAAI,UAAU,GAAG;AACf,cAAM,WAAW,YAAY;AAAA,MAC/B;AACA,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AAEpC,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AACA,oBAAgB;AAGhB,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,eAAO,KAAK,wCAAwC,EAAE,WAAW,iBAAiB,CAAC;AACnF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/queue",
|
|
3
|
-
"version": "0.6.8-develop.
|
|
3
|
+
"version": "0.6.8-develop.6930.1.1e5976efc3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Multi-strategy job queue with local and BullMQ support",
|
|
6
6
|
"type": "module",
|
|
@@ -56,8 +56,8 @@
|
|
|
56
56
|
"access": "public"
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@open-mercato/shared": "0.6.8-develop.
|
|
60
|
-
"@open-mercato/telemetry": "0.6.8-develop.
|
|
59
|
+
"@open-mercato/shared": "0.6.8-develop.6930.1.1e5976efc3",
|
|
60
|
+
"@open-mercato/telemetry": "0.6.8-develop.6930.1.1e5976efc3"
|
|
61
61
|
},
|
|
62
62
|
"repository": {
|
|
63
63
|
"type": "git",
|
|
@@ -22,6 +22,14 @@ const queueLoggerError = createLogger('queue').error as jest.Mock
|
|
|
22
22
|
|
|
23
23
|
function readJson(p: string) { return JSON.parse(fs.readFileSync(p, 'utf8')) }
|
|
24
24
|
|
|
25
|
+
async function waitUntil(condition: () => boolean, timeoutMs = 5000): Promise<void> {
|
|
26
|
+
const deadline = Date.now() + timeoutMs
|
|
27
|
+
while (!condition()) {
|
|
28
|
+
if (Date.now() > deadline) throw new Error('[internal] Timed out waiting for the expected filesystem state')
|
|
29
|
+
await new Promise((resolve) => { setTimeout(resolve, 5) })
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
25
33
|
describe('Queue - local strategy', () => {
|
|
26
34
|
const origCwd = process.cwd()
|
|
27
35
|
let tmp: string
|
|
@@ -263,7 +271,10 @@ describe('Queue - local strategy', () => {
|
|
|
263
271
|
await queue.close()
|
|
264
272
|
})
|
|
265
273
|
|
|
266
|
-
|
|
274
|
+
// Regression (#5149): an unparsable queue file used to be replaced with `[]`
|
|
275
|
+
// and reported only in the log, so `enqueue` resolved against a queue that had
|
|
276
|
+
// just been emptied. The bytes must now be preserved and the caller told.
|
|
277
|
+
test('corrupted queue file is quarantined and the failure reaches the caller', async () => {
|
|
267
278
|
const queue = createQueue<{ value: number }>('test-queue', 'local')
|
|
268
279
|
const queueDir = path.join('.mercato', 'queue', 'test-queue')
|
|
269
280
|
const queuePath = path.join(queueDir, 'queue.json')
|
|
@@ -273,12 +284,7 @@ describe('Queue - local strategy', () => {
|
|
|
273
284
|
fs.mkdirSync(queueDir, { recursive: true })
|
|
274
285
|
fs.writeFileSync(queuePath, brokenContent, 'utf8')
|
|
275
286
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const queueContent = readJson(queuePath)
|
|
279
|
-
expect(queueContent).toHaveLength(1)
|
|
280
|
-
expect(queueContent[0].id).toBe(jobId)
|
|
281
|
-
expect(queueContent[0].payload).toEqual({ value: 42 })
|
|
287
|
+
await expect(queue.enqueue({ value: 42 })).rejects.toThrow(/quarantined/)
|
|
282
288
|
|
|
283
289
|
const backupFiles = fs.readdirSync(queueDir)
|
|
284
290
|
.filter((fileName) => fileName.startsWith('queue.corrupted.') && fileName.endsWith('.json'))
|
|
@@ -290,13 +296,156 @@ describe('Queue - local strategy', () => {
|
|
|
290
296
|
{ err: expect.any(Error) },
|
|
291
297
|
)
|
|
292
298
|
expect(queueLoggerError).toHaveBeenCalledWith(
|
|
293
|
-
'
|
|
299
|
+
'Quarantined corrupted queue file; its jobs are recoverable from the backup',
|
|
294
300
|
{ backupFile: expect.stringContaining('queue.corrupted.') },
|
|
295
301
|
)
|
|
296
302
|
|
|
303
|
+
// The failed segment must not strand the cross-process lock, and the queue
|
|
304
|
+
// has to be usable again on the very next call.
|
|
305
|
+
expect(fs.existsSync(path.join(queueDir, 'queue.lock'))).toBe(false)
|
|
306
|
+
|
|
307
|
+
const jobId = await queue.enqueue({ value: 43 })
|
|
308
|
+
const queueContent = readJson(queuePath)
|
|
309
|
+
expect(queueContent).toHaveLength(1)
|
|
310
|
+
expect(queueContent[0].id).toBe(jobId)
|
|
311
|
+
expect(queueContent[0].payload).toEqual({ value: 43 })
|
|
312
|
+
|
|
297
313
|
await queue.close()
|
|
298
314
|
})
|
|
299
315
|
|
|
316
|
+
// Regression (#5149): two queue instances sharing a directory — the default
|
|
317
|
+
// dev topology, where the Next server and the worker are separate processes —
|
|
318
|
+
// used to interleave truncate-then-write calls. Each instance had its own
|
|
319
|
+
// in-process mutex, so nothing serialized them: the file was left unparsable
|
|
320
|
+
// and the jobs written by the losing writer disappeared.
|
|
321
|
+
test('concurrent writers on separate instances neither corrupt the file nor lose jobs', async () => {
|
|
322
|
+
const queueDir = path.join('.mercato', 'queue', 'multi-writer-queue')
|
|
323
|
+
const queuePath = path.join(queueDir, 'queue.json')
|
|
324
|
+
const writerA = createQueue<{ writer: string; index: number; payload: string }>('multi-writer-queue', 'local')
|
|
325
|
+
const writerB = createQueue<{ writer: string; index: number; payload: string }>('multi-writer-queue', 'local')
|
|
326
|
+
const jobsPerWriter = 25
|
|
327
|
+
|
|
328
|
+
// Deliberately mismatched payload sizes: the reported corruption signature
|
|
329
|
+
// is a short complete array followed by the tail of a longer earlier write.
|
|
330
|
+
const enqueueAll = (queue: typeof writerA, label: string, payloadSize: number) =>
|
|
331
|
+
Array.from({ length: jobsPerWriter }, (_, index) =>
|
|
332
|
+
queue.enqueue({ writer: label, index, payload: 'x'.repeat(payloadSize) }))
|
|
333
|
+
|
|
334
|
+
await Promise.all([
|
|
335
|
+
...enqueueAll(writerA, 'A', 4000),
|
|
336
|
+
...enqueueAll(writerB, 'B', 40),
|
|
337
|
+
])
|
|
338
|
+
|
|
339
|
+
const stored = readJson(queuePath)
|
|
340
|
+
expect(stored).toHaveLength(jobsPerWriter * 2)
|
|
341
|
+
expect(stored.filter((job: any) => job.payload.writer === 'A')).toHaveLength(jobsPerWriter)
|
|
342
|
+
expect(stored.filter((job: any) => job.payload.writer === 'B')).toHaveLength(jobsPerWriter)
|
|
343
|
+
|
|
344
|
+
const strayFiles = fs.readdirSync(queueDir)
|
|
345
|
+
.filter((fileName) => fileName.startsWith('queue.corrupted.') || fileName.endsWith('.tmp'))
|
|
346
|
+
expect(strayFiles).toEqual([])
|
|
347
|
+
|
|
348
|
+
await writerA.close()
|
|
349
|
+
await writerB.close()
|
|
350
|
+
}, 60_000)
|
|
351
|
+
|
|
352
|
+
// Regression (#5149): `queue.json` was persisted with a plain `writeFile`,
|
|
353
|
+
// which truncates the existing inode and then streams the payload into it.
|
|
354
|
+
// Persisting through a temp file plus `rename` replaces the inode instead, so
|
|
355
|
+
// a concurrent reader can only ever see one complete document.
|
|
356
|
+
test('each persist swaps in a replacement file instead of truncating in place', async () => {
|
|
357
|
+
const queue = createQueue<{ value: number }>('atomic-write-queue', 'local')
|
|
358
|
+
const queueDir = path.join('.mercato', 'queue', 'atomic-write-queue')
|
|
359
|
+
const queuePath = path.join(queueDir, 'queue.json')
|
|
360
|
+
const writeFileSpy = jest.spyOn(fs.promises, 'writeFile')
|
|
361
|
+
|
|
362
|
+
try {
|
|
363
|
+
await queue.enqueue({ value: 1 })
|
|
364
|
+
await queue.enqueue({ value: 2 })
|
|
365
|
+
|
|
366
|
+
// The invariant, asserted portably: queue.json is only ever created with
|
|
367
|
+
// the exclusive `wx` flag by ensureDir, never written in place. Every
|
|
368
|
+
// persist goes to a temp file that is then renamed over it.
|
|
369
|
+
const inPlaceWrites = writeFileSpy.mock.calls.filter(([target, , options]) => {
|
|
370
|
+
if (typeof target !== 'string' || path.resolve(target) !== path.resolve(queuePath)) return false
|
|
371
|
+
return (options as { flag?: string } | undefined)?.flag !== 'wx'
|
|
372
|
+
})
|
|
373
|
+
expect(inPlaceWrites).toEqual([])
|
|
374
|
+
|
|
375
|
+
expect(readJson(queuePath)).toHaveLength(2)
|
|
376
|
+
expect(fs.readdirSync(queueDir).filter((fileName) => fileName.endsWith('.tmp'))).toEqual([])
|
|
377
|
+
} finally {
|
|
378
|
+
writeFileSpy.mockRestore()
|
|
379
|
+
await queue.close()
|
|
380
|
+
}
|
|
381
|
+
})
|
|
382
|
+
|
|
383
|
+
// Regression (#5149): the cross-process lock must not outlive the process
|
|
384
|
+
// that took it, or a crash mid-segment would wedge every queue consumer.
|
|
385
|
+
test('a stale lock left behind by a dead process is reclaimed', async () => {
|
|
386
|
+
const queue = createQueue<{ value: number }>('stale-lock-queue', 'local')
|
|
387
|
+
const queueDir = path.join('.mercato', 'queue', 'stale-lock-queue')
|
|
388
|
+
const lockPath = path.join(queueDir, 'queue.lock')
|
|
389
|
+
|
|
390
|
+
fs.mkdirSync(lockPath, { recursive: true })
|
|
391
|
+
const wellPastTheStaleThreshold = new Date(Date.now() - 60_000)
|
|
392
|
+
fs.utimesSync(lockPath, wellPastTheStaleThreshold, wellPastTheStaleThreshold)
|
|
393
|
+
|
|
394
|
+
const jobId = await queue.enqueue({ value: 7 })
|
|
395
|
+
|
|
396
|
+
expect(typeof jobId).toBe('string')
|
|
397
|
+
expect(readJson(path.join(queueDir, 'queue.json'))).toHaveLength(1)
|
|
398
|
+
expect(fs.existsSync(lockPath)).toBe(false)
|
|
399
|
+
|
|
400
|
+
await queue.close()
|
|
401
|
+
})
|
|
402
|
+
|
|
403
|
+
// Regression (#5149): reclaiming a stale lock replaces it, so the holder that
|
|
404
|
+
// was reclaimed must not remove the replacement on its way out. An
|
|
405
|
+
// unconditional release deleted the successor's lock and let a third caller
|
|
406
|
+
// into the critical section beside it — the lost-update window this fix
|
|
407
|
+
// exists to close.
|
|
408
|
+
test('a holder whose lock was reclaimed does not delete the lock that replaced it', async () => {
|
|
409
|
+
const queue = createQueue<{ value: number }>('reclaim-release-queue', 'local')
|
|
410
|
+
const queueDir = path.join('.mercato', 'queue', 'reclaim-release-queue')
|
|
411
|
+
const lockPath = path.join(queueDir, 'queue.lock')
|
|
412
|
+
const ownerPath = path.join(lockPath, 'owner')
|
|
413
|
+
const successorToken = 'successor-owner-token'
|
|
414
|
+
|
|
415
|
+
let releaseStall = () => {}
|
|
416
|
+
const stalled = new Promise<void>((resolve) => { releaseStall = resolve })
|
|
417
|
+
const realWriteFile = fs.promises.writeFile
|
|
418
|
+
const writeFileSpy = jest.spyOn(fs.promises, 'writeFile').mockImplementation(
|
|
419
|
+
async (target: any, content: any, options: any) => {
|
|
420
|
+
// Stall inside the critical section: the temp file is only ever written
|
|
421
|
+
// while this queue holds the lock.
|
|
422
|
+
if (typeof target === 'string' && target.endsWith('.tmp')) await stalled
|
|
423
|
+
return realWriteFile(target, content, options)
|
|
424
|
+
},
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
try {
|
|
428
|
+
const enqueued = queue.enqueue({ value: 1 })
|
|
429
|
+
await waitUntil(() => fs.existsSync(ownerPath))
|
|
430
|
+
|
|
431
|
+
// Stand in for the peer that found this lock stale: move it aside, drop
|
|
432
|
+
// it, and take a fresh one of its own.
|
|
433
|
+
fs.rmSync(lockPath, { recursive: true, force: true })
|
|
434
|
+
fs.mkdirSync(lockPath, { recursive: true })
|
|
435
|
+
fs.writeFileSync(ownerPath, successorToken, 'utf8')
|
|
436
|
+
|
|
437
|
+
releaseStall()
|
|
438
|
+
await enqueued
|
|
439
|
+
|
|
440
|
+
expect(fs.existsSync(lockPath)).toBe(true)
|
|
441
|
+
expect(fs.readFileSync(ownerPath, 'utf8')).toBe(successorToken)
|
|
442
|
+
} finally {
|
|
443
|
+
writeFileSpy.mockRestore()
|
|
444
|
+
fs.rmSync(lockPath, { recursive: true, force: true })
|
|
445
|
+
await queue.close()
|
|
446
|
+
}
|
|
447
|
+
})
|
|
448
|
+
|
|
300
449
|
test('failed jobs are retained in queue for retry', async () => {
|
|
301
450
|
const queue = createQueue<{ shouldFail: boolean }>('test-queue', 'local')
|
|
302
451
|
const queuePath = path.join('.mercato', 'queue', 'test-queue', 'queue.json')
|
package/src/strategies/local.ts
CHANGED
|
@@ -37,6 +37,21 @@ const DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'
|
|
|
37
37
|
const DEFAULT_MAX_ATTEMPTS = 3
|
|
38
38
|
const RETRY_BACKOFF_BASE_MS = 1000
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Cross-process lock tuning. A held lock only ever spans local file I/O — job
|
|
42
|
+
* handlers run outside it — so realistic hold times are milliseconds and the
|
|
43
|
+
* stale threshold sits orders of magnitude above them. It exists solely so a
|
|
44
|
+
* process that dies mid-segment cannot wedge the queue forever. A holder that
|
|
45
|
+
* was merely suspended rather than dead can still be reclaimed, which is why
|
|
46
|
+
* every acquisition carries an owner token and releases only its own lock.
|
|
47
|
+
*/
|
|
48
|
+
const LOCK_STALE_MS = 15_000
|
|
49
|
+
const LOCK_ACQUIRE_TIMEOUT_MS = 30_000
|
|
50
|
+
const LOCK_RETRY_MIN_MS = 2
|
|
51
|
+
const LOCK_RETRY_MAX_MS = 20
|
|
52
|
+
const RENAME_MAX_RETRIES = 5
|
|
53
|
+
const RENAME_RETRY_BASE_MS = 10
|
|
54
|
+
|
|
40
55
|
const fsp = fs.promises
|
|
41
56
|
|
|
42
57
|
/**
|
|
@@ -48,7 +63,23 @@ const fsp = fs.promises
|
|
|
48
63
|
*
|
|
49
64
|
* **Limitations:**
|
|
50
65
|
* - Jobs are processed sequentially (concurrency option is for logging/compatibility only)
|
|
51
|
-
* - Not suitable for production
|
|
66
|
+
* - Not suitable for production: there is no dead-letter store, no throughput
|
|
67
|
+
* beyond one job at a time, and every operation rewrites the whole queue file
|
|
68
|
+
*
|
|
69
|
+
* Multiple processes MAY share a queue directory, which is the default
|
|
70
|
+
* development topology: the dev worker runs in its own process alongside the
|
|
71
|
+
* Next.js server. What that buys you, and what it does not:
|
|
72
|
+
*
|
|
73
|
+
* - **Safe** — concurrent producers. Every read-modify-write segment takes the
|
|
74
|
+
* `queue.lock` directory lock and every persist swaps the file in with an
|
|
75
|
+
* atomic rename, so the file cannot be torn, no enqueue is lost to a
|
|
76
|
+
* concurrent one, and a reader always observes one complete document.
|
|
77
|
+
* Writers contend, though, so throughput degrades as processes are added.
|
|
78
|
+
* - **NOT safe** — concurrent consumers. `process()` deliberately runs job
|
|
79
|
+
* handlers outside the lock, so two worker processes polling the same queue
|
|
80
|
+
* would both claim the same pending jobs and execute them twice. There is no
|
|
81
|
+
* per-job lease. Run exactly one worker process per queue; use the `async`
|
|
82
|
+
* strategy when you need more than one.
|
|
52
83
|
*
|
|
53
84
|
* Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.
|
|
54
85
|
* **This strategy keeps no failed-job store**: once attempts are exhausted the job is
|
|
@@ -65,8 +96,8 @@ const fsp = fs.promises
|
|
|
65
96
|
*
|
|
66
97
|
* All file I/O is asynchronous (`fs.promises.*`) so queue operations do not
|
|
67
98
|
* block the Node.js event loop. A per-queue promise chain serializes
|
|
68
|
-
* read-modify-write sequences
|
|
69
|
-
*
|
|
99
|
+
* read-modify-write sequences within one instance, and the `queue.lock`
|
|
100
|
+
* directory lock extends that serialization across instances and processes.
|
|
70
101
|
*
|
|
71
102
|
* @template T - The payload type for jobs
|
|
72
103
|
* @param name - Queue name (used for directory naming)
|
|
@@ -83,6 +114,8 @@ export function createLocalQueue<T = unknown>(
|
|
|
83
114
|
const queueDir = path.join(baseDir, name)
|
|
84
115
|
const queueFile = path.join(queueDir, 'queue.json')
|
|
85
116
|
const stateFile = path.join(queueDir, 'state.json')
|
|
117
|
+
const lockDir = path.join(queueDir, 'queue.lock')
|
|
118
|
+
const lockOwnerFile = path.join(lockDir, 'owner')
|
|
86
119
|
const logger = packageLogger.child({ queue: name })
|
|
87
120
|
// Note: concurrency is stored for logging/compatibility but jobs are processed sequentially
|
|
88
121
|
const concurrency = options?.concurrency ?? 1
|
|
@@ -95,10 +128,15 @@ export function createLocalQueue<T = unknown>(
|
|
|
95
128
|
const inFlightJobIds = new Set<string>()
|
|
96
129
|
|
|
97
130
|
// Per-queue mutex. Serializes read-modify-write segments so async fs calls
|
|
98
|
-
// cannot interleave and clobber each other's writes.
|
|
131
|
+
// cannot interleave and clobber each other's writes. It only covers this
|
|
132
|
+
// instance, so it also guarantees at most one outstanding `queue.lock`
|
|
133
|
+
// acquisition per instance — the directory lock below is not reentrant.
|
|
99
134
|
let fileOpChain: Promise<unknown> = Promise.resolve()
|
|
100
135
|
function withFileLock<R>(fn: () => Promise<R>): Promise<R> {
|
|
101
|
-
const run = fileOpChain.then(
|
|
136
|
+
const run = fileOpChain.then(
|
|
137
|
+
() => runExclusively(fn),
|
|
138
|
+
() => runExclusively(fn),
|
|
139
|
+
)
|
|
102
140
|
fileOpChain = run.then(
|
|
103
141
|
() => undefined,
|
|
104
142
|
() => undefined,
|
|
@@ -106,10 +144,160 @@ export function createLocalQueue<T = unknown>(
|
|
|
106
144
|
return run
|
|
107
145
|
}
|
|
108
146
|
|
|
147
|
+
/**
|
|
148
|
+
* Runs `fn` while holding the cross-process `queue.lock`, so read-modify-write
|
|
149
|
+
* segments issued by other queue instances — in this process or another one —
|
|
150
|
+
* cannot interleave with it.
|
|
151
|
+
*/
|
|
152
|
+
async function runExclusively<R>(fn: () => Promise<R>): Promise<R> {
|
|
153
|
+
await ensureDir()
|
|
154
|
+
const release = await acquireDirectoryLock()
|
|
155
|
+
try {
|
|
156
|
+
return await fn()
|
|
157
|
+
} finally {
|
|
158
|
+
await release()
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
109
162
|
// -------------------------------------------------------------------------
|
|
110
163
|
// File Operations
|
|
111
164
|
// -------------------------------------------------------------------------
|
|
112
165
|
|
|
166
|
+
function sleep(ms: number): Promise<void> {
|
|
167
|
+
return new Promise((resolve) => { setTimeout(resolve, ms) })
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async function lockHeldForMs(): Promise<number | null> {
|
|
171
|
+
try {
|
|
172
|
+
const stats = await fsp.stat(lockDir)
|
|
173
|
+
return Date.now() - stats.mtimeMs
|
|
174
|
+
} catch {
|
|
175
|
+
return null
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Reclaims a lock whose holder died. The rename is the serialization point:
|
|
181
|
+
* only one racer can move `queue.lock` aside, so two processes cannot both
|
|
182
|
+
* decide a stale lock is theirs to clear and then both create a fresh one.
|
|
183
|
+
*/
|
|
184
|
+
async function reclaimStaleLock(heldForMs: number): Promise<void> {
|
|
185
|
+
const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`
|
|
186
|
+
try {
|
|
187
|
+
await fsp.rename(lockDir, reclaimedPath)
|
|
188
|
+
} catch {
|
|
189
|
+
return
|
|
190
|
+
}
|
|
191
|
+
logger.warn('Reclaimed a stale queue lock', { lockDir, heldForMs })
|
|
192
|
+
await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {})
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async function readLockOwner(): Promise<string | null> {
|
|
196
|
+
try {
|
|
197
|
+
return await fsp.readFile(lockOwnerFile, 'utf8')
|
|
198
|
+
} catch {
|
|
199
|
+
return null
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Releases the lock only when this acquisition still owns it. A holder that
|
|
205
|
+
* was suspended past `LOCK_STALE_MS` has had its lock reclaimed *and
|
|
206
|
+
* replaced* by whoever reclaimed it, so an unconditional removal here would
|
|
207
|
+
* delete the successor's lock and let a third caller into the critical
|
|
208
|
+
* section alongside it. A missing or mismatched token means someone else owns
|
|
209
|
+
* the path now, and the correct action is to leave it alone.
|
|
210
|
+
*/
|
|
211
|
+
async function releaseDirectoryLock(token: string): Promise<void> {
|
|
212
|
+
if (await readLockOwner() !== token) return
|
|
213
|
+
await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Acquires the cross-process advisory lock for this queue directory.
|
|
218
|
+
* `mkdir` without `recursive` is an atomic exclusive create on every platform
|
|
219
|
+
* Node.js supports, which makes it the portable primitive here — no runtime
|
|
220
|
+
* dependency, and no reliance on advisory `flock` semantics. The owner token
|
|
221
|
+
* written into the directory is what lets the release distinguish this
|
|
222
|
+
* acquisition from a successor's.
|
|
223
|
+
*/
|
|
224
|
+
async function acquireDirectoryLock(): Promise<() => Promise<void>> {
|
|
225
|
+
const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS
|
|
226
|
+
|
|
227
|
+
for (;;) {
|
|
228
|
+
let acquired = false
|
|
229
|
+
try {
|
|
230
|
+
await fsp.mkdir(lockDir)
|
|
231
|
+
acquired = true
|
|
232
|
+
} catch (e: unknown) {
|
|
233
|
+
const error = e as NodeJS.ErrnoException
|
|
234
|
+
if (error.code !== 'EEXIST') throw error
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (acquired) {
|
|
238
|
+
const token = crypto.randomUUID()
|
|
239
|
+
try {
|
|
240
|
+
await fsp.writeFile(lockOwnerFile, token, 'utf8')
|
|
241
|
+
} catch (error: unknown) {
|
|
242
|
+
await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})
|
|
243
|
+
throw error
|
|
244
|
+
}
|
|
245
|
+
return () => releaseDirectoryLock(token)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const heldForMs = await lockHeldForMs()
|
|
249
|
+
if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {
|
|
250
|
+
await reclaimStaleLock(heldForMs)
|
|
251
|
+
continue
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
if (Date.now() >= deadline) {
|
|
255
|
+
throw new Error(
|
|
256
|
+
`[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`,
|
|
257
|
+
)
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)
|
|
261
|
+
await sleep(jitter)
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Persists `content` by writing a unique sibling temp file and renaming it
|
|
267
|
+
* onto `targetFile`. `rename` within a directory is atomic, so a concurrent
|
|
268
|
+
* reader sees either the previous document or the new one in full — never the
|
|
269
|
+
* torn result of a truncate-then-write.
|
|
270
|
+
*/
|
|
271
|
+
async function writeFileAtomic(targetFile: string, content: string): Promise<void> {
|
|
272
|
+
const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`
|
|
273
|
+
try {
|
|
274
|
+
await fsp.writeFile(tempFile, content, 'utf8')
|
|
275
|
+
await renameWithContentionRetry(tempFile, targetFile)
|
|
276
|
+
} catch (error: unknown) {
|
|
277
|
+
await fsp.rm(tempFile, { force: true }).catch(() => {})
|
|
278
|
+
throw error
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Windows rejects a rename onto a file another process currently has open,
|
|
284
|
+
* so retry briefly on the contention codes it raises. POSIX renames replace
|
|
285
|
+
* the target unconditionally and take the first attempt.
|
|
286
|
+
*/
|
|
287
|
+
async function renameWithContentionRetry(fromFile: string, toFile: string): Promise<void> {
|
|
288
|
+
const contentionCodes = new Set(['EPERM', 'EBUSY', 'EACCES'])
|
|
289
|
+
for (let attempt = 0; ; attempt++) {
|
|
290
|
+
try {
|
|
291
|
+
await fsp.rename(fromFile, toFile)
|
|
292
|
+
return
|
|
293
|
+
} catch (e: unknown) {
|
|
294
|
+
const error = e as NodeJS.ErrnoException
|
|
295
|
+
if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error
|
|
296
|
+
await sleep(RENAME_RETRY_BASE_MS * (attempt + 1))
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
113
301
|
async function ensureDir(): Promise<void> {
|
|
114
302
|
try {
|
|
115
303
|
await fsp.mkdir(queueDir, { recursive: true })
|
|
@@ -135,11 +323,21 @@ export function createLocalQueue<T = unknown>(
|
|
|
135
323
|
}
|
|
136
324
|
}
|
|
137
325
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
326
|
+
/**
|
|
327
|
+
* Moves an unparsable queue file aside so its jobs stay recoverable. The
|
|
328
|
+
* caller is expected to surface the failure rather than continue on an empty
|
|
329
|
+
* queue: silently recreating `queue.json` here is what turned an unreadable
|
|
330
|
+
* file into permanent, unreported job loss.
|
|
331
|
+
*/
|
|
332
|
+
async function quarantineCorruptedQueueFile(): Promise<string | null> {
|
|
333
|
+
const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`)
|
|
334
|
+
try {
|
|
335
|
+
await fsp.rename(queueFile, backupFile)
|
|
336
|
+
return backupFile
|
|
337
|
+
} catch (e: unknown) {
|
|
338
|
+
logger.error('Failed to quarantine the corrupted queue file', { err: e as Error })
|
|
339
|
+
return null
|
|
340
|
+
}
|
|
143
341
|
}
|
|
144
342
|
|
|
145
343
|
async function readQueue(): Promise<StoredJob<T>[]> {
|
|
@@ -168,15 +366,22 @@ export function createLocalQueue<T = unknown>(
|
|
|
168
366
|
} catch (error: unknown) {
|
|
169
367
|
const parseError = error as Error
|
|
170
368
|
logger.error('Failed to parse queue file', { err: parseError })
|
|
171
|
-
const backupFile = await
|
|
172
|
-
|
|
173
|
-
|
|
369
|
+
const backupFile = await quarantineCorruptedQueueFile()
|
|
370
|
+
if (backupFile) {
|
|
371
|
+
logger.error('Quarantined corrupted queue file; its jobs are recoverable from the backup', { backupFile })
|
|
372
|
+
}
|
|
373
|
+
const recoveryHint = backupFile
|
|
374
|
+
? `has been quarantined as ${backupFile}`
|
|
375
|
+
: 'could not be quarantined and was left in place'
|
|
376
|
+
throw new Error(
|
|
377
|
+
`[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`,
|
|
378
|
+
)
|
|
174
379
|
}
|
|
175
380
|
}
|
|
176
381
|
|
|
177
382
|
async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {
|
|
178
383
|
await ensureDir()
|
|
179
|
-
await
|
|
384
|
+
await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2))
|
|
180
385
|
}
|
|
181
386
|
|
|
182
387
|
async function readState(): Promise<LocalState> {
|
|
@@ -191,7 +396,7 @@ export function createLocalQueue<T = unknown>(
|
|
|
191
396
|
|
|
192
397
|
async function writeState(state: LocalState): Promise<void> {
|
|
193
398
|
await ensureDir()
|
|
194
|
-
await
|
|
399
|
+
await writeFileAtomic(stateFile, JSON.stringify(state, null, 2))
|
|
195
400
|
}
|
|
196
401
|
|
|
197
402
|
function generateId(): string {
|