@open-mercato/queue 0.6.8-develop.6924.1.a8d208fcdc → 0.6.8-develop.6930.1.1e5976efc3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,6 +20,12 @@ const DEFAULT_POLL_INTERVAL = 1e3;
20
20
  const DEFAULT_LOCAL_QUEUE_BASE_DIR = ".mercato/queue";
21
21
  const DEFAULT_MAX_ATTEMPTS = 3;
22
22
  const RETRY_BACKOFF_BASE_MS = 1e3;
23
+ const LOCK_STALE_MS = 15e3;
24
+ const LOCK_ACQUIRE_TIMEOUT_MS = 3e4;
25
+ const LOCK_RETRY_MIN_MS = 2;
26
+ const LOCK_RETRY_MAX_MS = 20;
27
+ const RENAME_MAX_RETRIES = 5;
28
+ const RENAME_RETRY_BASE_MS = 10;
23
29
  const fsp = fs.promises;
24
30
  function createLocalQueue(name, options) {
25
31
  const nodeProcess = globalThis.process;
@@ -28,6 +34,8 @@ function createLocalQueue(name, options) {
28
34
  const queueDir = path.join(baseDir, name);
29
35
  const queueFile = path.join(queueDir, "queue.json");
30
36
  const stateFile = path.join(queueDir, "state.json");
37
+ const lockDir = path.join(queueDir, "queue.lock");
38
+ const lockOwnerFile = path.join(lockDir, "owner");
31
39
  const logger = packageLogger.child({ queue: name });
32
40
  const concurrency = options?.concurrency ?? 1;
33
41
  const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL;
@@ -37,13 +45,121 @@ function createLocalQueue(name, options) {
37
45
  const inFlightJobIds = /* @__PURE__ */ new Set();
38
46
  let fileOpChain = Promise.resolve();
39
47
  function withFileLock(fn) {
40
- const run = fileOpChain.then(() => fn(), () => fn());
48
+ const run = fileOpChain.then(
49
+ () => runExclusively(fn),
50
+ () => runExclusively(fn)
51
+ );
41
52
  fileOpChain = run.then(
42
53
  () => void 0,
43
54
  () => void 0
44
55
  );
45
56
  return run;
46
57
  }
58
+ async function runExclusively(fn) {
59
+ await ensureDir();
60
+ const release = await acquireDirectoryLock();
61
+ try {
62
+ return await fn();
63
+ } finally {
64
+ await release();
65
+ }
66
+ }
67
+ function sleep(ms) {
68
+ return new Promise((resolve) => {
69
+ setTimeout(resolve, ms);
70
+ });
71
+ }
72
+ async function lockHeldForMs() {
73
+ try {
74
+ const stats = await fsp.stat(lockDir);
75
+ return Date.now() - stats.mtimeMs;
76
+ } catch {
77
+ return null;
78
+ }
79
+ }
80
+ async function reclaimStaleLock(heldForMs) {
81
+ const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`;
82
+ try {
83
+ await fsp.rename(lockDir, reclaimedPath);
84
+ } catch {
85
+ return;
86
+ }
87
+ logger.warn("Reclaimed a stale queue lock", { lockDir, heldForMs });
88
+ await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {
89
+ });
90
+ }
91
+ async function readLockOwner() {
92
+ try {
93
+ return await fsp.readFile(lockOwnerFile, "utf8");
94
+ } catch {
95
+ return null;
96
+ }
97
+ }
98
+ async function releaseDirectoryLock(token) {
99
+ if (await readLockOwner() !== token) return;
100
+ await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {
101
+ });
102
+ }
103
+ async function acquireDirectoryLock() {
104
+ const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS;
105
+ for (; ; ) {
106
+ let acquired = false;
107
+ try {
108
+ await fsp.mkdir(lockDir);
109
+ acquired = true;
110
+ } catch (e) {
111
+ const error = e;
112
+ if (error.code !== "EEXIST") throw error;
113
+ }
114
+ if (acquired) {
115
+ const token = crypto.randomUUID();
116
+ try {
117
+ await fsp.writeFile(lockOwnerFile, token, "utf8");
118
+ } catch (error) {
119
+ await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {
120
+ });
121
+ throw error;
122
+ }
123
+ return () => releaseDirectoryLock(token);
124
+ }
125
+ const heldForMs = await lockHeldForMs();
126
+ if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {
127
+ await reclaimStaleLock(heldForMs);
128
+ continue;
129
+ }
130
+ if (Date.now() >= deadline) {
131
+ throw new Error(
132
+ `[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`
133
+ );
134
+ }
135
+ const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS);
136
+ await sleep(jitter);
137
+ }
138
+ }
139
+ async function writeFileAtomic(targetFile, content) {
140
+ const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`;
141
+ try {
142
+ await fsp.writeFile(tempFile, content, "utf8");
143
+ await renameWithContentionRetry(tempFile, targetFile);
144
+ } catch (error) {
145
+ await fsp.rm(tempFile, { force: true }).catch(() => {
146
+ });
147
+ throw error;
148
+ }
149
+ }
150
+ async function renameWithContentionRetry(fromFile, toFile) {
151
+ const contentionCodes = /* @__PURE__ */ new Set(["EPERM", "EBUSY", "EACCES"]);
152
+ for (let attempt = 0; ; attempt++) {
153
+ try {
154
+ await fsp.rename(fromFile, toFile);
155
+ return;
156
+ } catch (e) {
157
+ const error = e;
158
+ if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error;
159
+ await sleep(RENAME_RETRY_BASE_MS * (attempt + 1));
160
+ }
161
+ }
162
+ }
47
163
  async function ensureDir() {
48
164
  try {
49
165
  await fsp.mkdir(queueDir, { recursive: true });
@@ -64,11 +180,15 @@ function createLocalQueue(name, options) {
64
180
  if (error.code !== "EEXIST") throw error;
65
181
  }
66
182
  }
67
- async function backupCorruptedQueueFile(content) {
68
- const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`);
69
- await fsp.writeFile(backupFile, content, "utf8");
70
- await fsp.writeFile(queueFile, "[]", "utf8");
71
- return backupFile;
183
+ async function quarantineCorruptedQueueFile() {
184
+ const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`);
185
+ try {
186
+ await fsp.rename(queueFile, backupFile);
187
+ return backupFile;
188
+ } catch (e) {
189
+ logger.error("Failed to quarantine the corrupted queue file", { err: e });
190
+ return null;
191
+ }
72
192
  }
73
193
  async function readQueue() {
74
194
  await ensureDir();
@@ -92,14 +212,19 @@ function createLocalQueue(name, options) {
92
212
  } catch (error) {
93
213
  const parseError = error;
94
214
  logger.error("Failed to parse queue file", { err: parseError });
95
- const backupFile = await backupCorruptedQueueFile(content);
96
- logger.error("Backed up corrupted queue file and recreated queue.json", { backupFile });
97
- return [];
215
+ const backupFile = await quarantineCorruptedQueueFile();
216
+ if (backupFile) {
217
+ logger.error("Quarantined corrupted queue file; its jobs are recoverable from the backup", { backupFile });
218
+ }
219
+ const recoveryHint = backupFile ? `has been quarantined as ${backupFile}` : "could not be quarantined and was left in place";
220
+ throw new Error(
221
+ `[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`
222
+ );
98
223
  }
99
224
  }
100
225
  async function writeQueue(jobs) {
101
226
  await ensureDir();
102
- await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), "utf8");
227
+ await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2));
103
228
  }
104
229
  async function readState() {
105
230
  await ensureDir();
@@ -112,7 +237,7 @@ function createLocalQueue(name, options) {
112
237
  }
113
238
  async function writeState(state) {
114
239
  await ensureDir();
115
- await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), "utf8");
240
+ await writeFileAtomic(stateFile, JSON.stringify(state, null, 2));
116
241
  }
117
242
  function generateId() {
118
243
  return crypto.randomUUID();
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../src/strategies/local.ts"],
4
- "sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production or multi-process environments\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.\n * **This strategy keeps no failed-job store**: once attempts are exhausted the job is\n * removed from `queue.json` and only counted in `state.failedCount`, so the payload is\n * lost and the failure survives solely as an error log line. The `async` strategy keeps\n * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000\n * failures and removes older ones as later failures arrive. Workflows that require\n * no-loss persistence must write their own durable record before enqueueing, regardless\n * of strategy.\n *\n * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option \u2014 callers cannot\n * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)\n * See the retry handling in `process()` below.\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences to preserve the atomicity guarantees the\n * previous synchronous implementation relied on.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(() => fn(), () => fn())\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n async function backupCorruptedQueueFile(content: string): Promise<string> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)\n await fsp.writeFile(backupFile, content, 'utf8')\n await fsp.writeFile(queueFile, '[]', 'utf8')\n return backupFile\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await backupCorruptedQueueFile(content)\n logger.error('Backed up corrupted queue file and recreated queue.json', { backupFile })\n return []\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const metadata = attachTraceMetadata(undefined)\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n ...(metadata ? { metadata } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await runJobInTrace(name, job.metadata, () =>\n Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, pollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
5
- "mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAE7B,SAAS,qBAAqB,qBAAqB;AAEnD,MAAM,gBAAgB,aAAa,OAAO;AAa1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAE9B,MAAM,MAAM,GAAG;AAmCR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAG9C,MAAI,eAAsD;AAC1D,MAAI,eAAe;AACnB,MAAI,gBAAsC;AAC1C,QAAM,iBAAiB,oBAAI,IAAY;AAIvC,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY,KAAK,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC;AACnD,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAEA,iBAAe,yBAAyB,SAAkC;AACxE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,OAAO;AAC3E,UAAM,IAAI,UAAU,YAAY,SAAS,MAAM;AAC/C,UAAM,IAAI,UAAU,WAAW,MAAM,MAAM;AAC3C,WAAO;AAAA,EACT;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,aAAO,MAAM,6BAA6B,EAAE,KAAK,UAAU,CAAC;AAC5D,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAW,CAAC;AAC9D,YAAM,aAAa,MAAM,yBAAyB,OAAO;AACzD,aAAO,MAAM,2DAA2D,EAAE,WAAW,CAAC;AACtF,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,MAAM;AAAA,EACtE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,IAAI,UAAU,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,MAAM;AAAA,EACvE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,WAAW,oBAAoB,MAAS;AAC9C,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,MACrC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AAED,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,eAAW,OAAO,eAAe;AAC/B,qBAAe,IAAI,IAAI,EAAE;AAAA,IAC3B;AAEA,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,QAAI;AACF,iBAAW,OAAO,eAAe;AAC/B,cAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,YAAI;AACF,gBAAM;AAAA,YAAc;AAAA,YAAM,IAAI;AAAA,YAAU,MACtC,QAAQ;AAAA,cACN,QAAQ,KAAK;AAAA,gBACX,OAAO,IAAI;AAAA,gBACX;AAAA,gBACA,WAAW;AAAA,cACb,CAAC;AAAA,YACH;AAAA,UACF;AACA;AACA,sBAAY,IAAI;AAChB,0BAAgB,IAAI,IAAI,EAAE;AAC1B,iBAAO,KAAK,iBAAiB,EAAE,OAAO,IAAI,GAAG,CAAC;AAAA,QAChD,SAAS,OAAO;AACd,iBAAO,MAAM,cAAc,EAAE,OAAO,IAAI,IAAI,eAAe,aAAa,sBAAsB,KAAK,MAAM,CAAC;AAC1G;AACA,sBAAY,IAAI;AAChB,cAAI,iBAAiB,sBAAsB;AACzC,mBAAO,MAAM,kEAAkE,EAAE,OAAO,IAAI,IAAI,aAAa,qBAAqB,CAAC;AACnI,uBAAW,IAAI,IAAI,EAAE;AAAA,UACvB,OAAO;AACL,kBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,yBAAa,IAAI,IAAI,IAAI;AAAA,cACvB,GAAG;AAAA,cACH,cAAc;AAAA,cACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,YAC5D,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,UAAI,YAAY;AACd,cAAM,aAAa,YAAY;AAE7B,gBAAM,cAAc,MAAM,UAAU;AACpC,gBAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,gBAAM,WAAW,WAAW;AAE5B,gBAAM,WAAuB;AAAA,YAC3B,iBAAiB;AAAA,YACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,YAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,UACrD;AACA,gBAAM,WAAW,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAEA,aAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,IACxC,UAAE;AACA,iBAAW,OAAO,eAAe;AAC/B,uBAAe,OAAO,IAAI,EAAE;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAKA,iBAAe,iBAAgC;AAE7C,QAAI,gBAAgB,CAAC,cAAe;AAEpC,mBAAe;AACf,QAAI;AACF,YAAM,aAAa,aAAa;AAAA,IAClC,SAAS,OAAO;AACd,aAAO,MAAM,iBAAiB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC9C,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAGA,oBAAgB;AAGhB,UAAM,aAAa,OAAO;AAG1B,mBAAe,YAAY,MAAM;AAC/B,qBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,eAAO,MAAM,oBAAoB,EAAE,IAAI,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH,GAAG,YAAY;AAEf,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAG7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,eAAe,KAAK,OAAO,CAAC,QAAQ,eAAe,IAAI,IAAI,EAAE,KAAK,CAAC,oBAAoB,IAAI,SAAS,KAAK,CAAC;AAChH,YAAM,UAAU,KAAK,SAAS,aAAa;AAC3C,UAAI,UAAU,GAAG;AACf,cAAM,WAAW,YAAY;AAAA,MAC/B;AACA,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AAEpC,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AACA,oBAAgB;AAGhB,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,eAAO,KAAK,wCAAwC,EAAE,WAAW,iBAAiB,CAAC;AACnF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
4
+ "sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Default polling interval in milliseconds */\nconst DEFAULT_POLL_INTERVAL = 1000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\n/**\n * Cross-process lock tuning. A held lock only ever spans local file I/O \u2014 job\n * handlers run outside it \u2014 so realistic hold times are milliseconds and the\n * stale threshold sits orders of magnitude above them. It exists solely so a\n * process that dies mid-segment cannot wedge the queue forever. A holder that\n * was merely suspended rather than dead can still be reclaimed, which is why\n * every acquisition carries an owner token and releases only its own lock.\n */\nconst LOCK_STALE_MS = 15_000\nconst LOCK_ACQUIRE_TIMEOUT_MS = 30_000\nconst LOCK_RETRY_MIN_MS = 2\nconst LOCK_RETRY_MAX_MS = 20\nconst RENAME_MAX_RETRIES = 5\nconst RENAME_RETRY_BASE_MS = 10\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production: there is no dead-letter store, no throughput\n * beyond one job at a time, and every operation rewrites the whole queue file\n *\n * Multiple processes MAY share a queue directory, which is the default\n * development topology: the dev worker runs in its own process alongside the\n * Next.js server. What that buys you, and what it does not:\n *\n * - **Safe** \u2014 concurrent producers. Every read-modify-write segment takes the\n * `queue.lock` directory lock and every persist swaps the file in with an\n * atomic rename, so the file cannot be torn, no enqueue is lost to a\n * concurrent one, and a reader always observes one complete document.\n * Writers contend, though, so throughput degrades as processes are added.\n * - **NOT safe** \u2014 concurrent consumers. `process()` deliberately runs job\n * handlers outside the lock, so two worker processes polling the same queue\n * would both claim the same pending jobs and execute them twice. There is no\n * per-job lease. Run exactly one worker process per queue; use the `async`\n * strategy when you need more than one.\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.\n * **This strategy keeps no failed-job store**: once attempts are exhausted the job is\n * removed from `queue.json` and only counted in `state.failedCount`, so the payload is\n * lost and the failure survives solely as an error log line. The `async` strategy keeps\n * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000\n * failures and removes older ones as later failures arrive. Workflows that require\n * no-loss persistence must write their own durable record before enqueueing, regardless\n * of strategy.\n *\n * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option \u2014 callers cannot\n * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)\n * See the retry handling in `process()` below.\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences within one instance, and the `queue.lock`\n * directory lock extends that serialization across instances and processes.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const lockDir = path.join(queueDir, 'queue.lock')\n const lockOwnerFile = path.join(lockDir, 'owner')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let isProcessing = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes. It only covers this\n // instance, so it also guarantees at most one outstanding `queue.lock`\n // acquisition per instance \u2014 the directory lock below is not reentrant.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(\n () => runExclusively(fn),\n () => runExclusively(fn),\n )\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n /**\n * Runs `fn` while holding the cross-process `queue.lock`, so read-modify-write\n * segments issued by other queue instances \u2014 in this process or another one \u2014\n * cannot interleave with it.\n */\n async function runExclusively<R>(fn: () => Promise<R>): Promise<R> {\n await ensureDir()\n const release = await acquireDirectoryLock()\n try {\n return await fn()\n } finally {\n await release()\n }\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => { setTimeout(resolve, ms) })\n }\n\n async function lockHeldForMs(): Promise<number | null> {\n try {\n const stats = await fsp.stat(lockDir)\n return Date.now() - stats.mtimeMs\n } catch {\n return null\n }\n }\n\n /**\n * Reclaims a lock whose holder died. The rename is the serialization point:\n * only one racer can move `queue.lock` aside, so two processes cannot both\n * decide a stale lock is theirs to clear and then both create a fresh one.\n */\n async function reclaimStaleLock(heldForMs: number): Promise<void> {\n const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`\n try {\n await fsp.rename(lockDir, reclaimedPath)\n } catch {\n return\n }\n logger.warn('Reclaimed a stale queue lock', { lockDir, heldForMs })\n await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {})\n }\n\n async function readLockOwner(): Promise<string | null> {\n try {\n return await fsp.readFile(lockOwnerFile, 'utf8')\n } catch {\n return null\n }\n }\n\n /**\n * Releases the lock only when this acquisition still owns it. A holder that\n * was suspended past `LOCK_STALE_MS` has had its lock reclaimed *and\n * replaced* by whoever reclaimed it, so an unconditional removal here would\n * delete the successor's lock and let a third caller into the critical\n * section alongside it. A missing or mismatched token means someone else owns\n * the path now, and the correct action is to leave it alone.\n */\n async function releaseDirectoryLock(token: string): Promise<void> {\n if (await readLockOwner() !== token) return\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n }\n\n /**\n * Acquires the cross-process advisory lock for this queue directory.\n * `mkdir` without `recursive` is an atomic exclusive create on every platform\n * Node.js supports, which makes it the portable primitive here \u2014 no runtime\n * dependency, and no reliance on advisory `flock` semantics. The owner token\n * written into the directory is what lets the release distinguish this\n * acquisition from a successor's.\n */\n async function acquireDirectoryLock(): Promise<() => Promise<void>> {\n const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS\n\n for (;;) {\n let acquired = false\n try {\n await fsp.mkdir(lockDir)\n acquired = true\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n if (acquired) {\n const token = crypto.randomUUID()\n try {\n await fsp.writeFile(lockOwnerFile, token, 'utf8')\n } catch (error: unknown) {\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n throw error\n }\n return () => releaseDirectoryLock(token)\n }\n\n const heldForMs = await lockHeldForMs()\n if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {\n await reclaimStaleLock(heldForMs)\n continue\n }\n\n if (Date.now() >= deadline) {\n throw new Error(\n `[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`,\n )\n }\n\n const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)\n await sleep(jitter)\n }\n }\n\n /**\n * Persists `content` by writing a unique sibling temp file and renaming it\n * onto `targetFile`. `rename` within a directory is atomic, so a concurrent\n * reader sees either the previous document or the new one in full \u2014 never the\n * torn result of a truncate-then-write.\n */\n async function writeFileAtomic(targetFile: string, content: string): Promise<void> {\n const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`\n try {\n await fsp.writeFile(tempFile, content, 'utf8')\n await renameWithContentionRetry(tempFile, targetFile)\n } catch (error: unknown) {\n await fsp.rm(tempFile, { force: true }).catch(() => {})\n throw error\n }\n }\n\n /**\n * Windows rejects a rename onto a file another process currently has open,\n * so retry briefly on the contention codes it raises. POSIX renames replace\n * the target unconditionally and take the first attempt.\n */\n async function renameWithContentionRetry(fromFile: string, toFile: string): Promise<void> {\n const contentionCodes = new Set(['EPERM', 'EBUSY', 'EACCES'])\n for (let attempt = 0; ; attempt++) {\n try {\n await fsp.rename(fromFile, toFile)\n return\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error\n await sleep(RENAME_RETRY_BASE_MS * (attempt + 1))\n }\n }\n }\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n /**\n * Moves an unparsable queue file aside so its jobs stay recoverable. The\n * caller is expected to surface the failure rather than continue on an empty\n * queue: silently recreating `queue.json` here is what turned an unreadable\n * file into permanent, unreported job loss.\n */\n async function quarantineCorruptedQueueFile(): Promise<string | null> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`)\n try {\n await fsp.rename(queueFile, backupFile)\n return backupFile\n } catch (e: unknown) {\n logger.error('Failed to quarantine the corrupted queue file', { err: e as Error })\n return null\n }\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await quarantineCorruptedQueueFile()\n if (backupFile) {\n logger.error('Quarantined corrupted queue file; its jobs are recoverable from the backup', { backupFile })\n }\n const recoveryHint = backupFile\n ? `has been quarantined as ${backupFile}`\n : 'could not be quarantined and was left in place'\n throw new Error(\n `[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`,\n )\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2))\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await writeFileAtomic(stateFile, JSON.stringify(state, null, 2))\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const metadata = attachTraceMetadata(undefined)\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n ...(metadata ? { metadata } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await runJobInTrace(name, job.metadata, () =>\n Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(): Promise<void> {\n // Skip if already processing to avoid concurrent file access\n if (isProcessing || !activeHandler) return\n\n isProcessing = true\n try {\n await processBatch(activeHandler)\n } catch (error) {\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n }\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n // Process any pending jobs immediately\n await processBatch(handler)\n\n // Start polling interval for new jobs\n pollingTimer = setInterval(() => {\n pollAndProcess().catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, pollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n activeHandler = null\n\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
5
+ "mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAE7B,SAAS,qBAAqB,qBAAqB;AAEnD,MAAM,gBAAgB,aAAa,OAAO;AAa1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAC9B,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAU9B,MAAM,gBAAgB;AACtB,MAAM,0BAA0B;AAChC,MAAM,oBAAoB;AAC1B,MAAM,oBAAoB;AAC1B,MAAM,qBAAqB;AAC3B,MAAM,uBAAuB;AAE7B,MAAM,MAAM,GAAG;AAmDR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,UAAU,KAAK,KAAK,UAAU,YAAY;AAChD,QAAM,gBAAgB,KAAK,KAAK,SAAS,OAAO;AAChD,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAG9C,MAAI,eAAsD;AAC1D,MAAI,eAAe;AACnB,MAAI,gBAAsC;AAC1C,QAAM,iBAAiB,oBAAI,IAAY;AAMvC,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY;AAAA,MACtB,MAAM,eAAe,EAAE;AAAA,MACvB,MAAM,eAAe,EAAE;AAAA,IACzB;AACA,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAOA,iBAAe,eAAkB,IAAkC;AACjE,UAAM,UAAU;AAChB,UAAM,UAAU,MAAM,qBAAqB;AAC3C,QAAI;AACF,aAAO,MAAM,GAAG;AAAA,IAClB,UAAE;AACA,YAAM,QAAQ;AAAA,IAChB;AAAA,EACF;AAMA,WAAS,MAAM,IAA2B;AACxC,WAAO,IAAI,QAAQ,CAAC,YAAY;AAAE,iBAAW,SAAS,EAAE;AAAA,IAAE,CAAC;AAAA,EAC7D;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,YAAM,QAAQ,MAAM,IAAI,KAAK,OAAO;AACpC,aAAO,KAAK,IAAI,IAAI,MAAM;AAAA,IAC5B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAOA,iBAAe,iBAAiB,WAAkC;AAChE,UAAM,gBAAgB,GAAG,OAAO,UAAU,OAAO,WAAW,CAAC;AAC7D,QAAI;AACF,YAAM,IAAI,OAAO,SAAS,aAAa;AAAA,IACzC,QAAQ;AACN;AAAA,IACF;AACA,WAAO,KAAK,gCAAgC,EAAE,SAAS,UAAU,CAAC;AAClE,UAAM,IAAI,GAAG,eAAe,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC9E;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,aAAO,MAAM,IAAI,SAAS,eAAe,MAAM;AAAA,IACjD,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAUA,iBAAe,qBAAqB,OAA8B;AAChE,QAAI,MAAM,cAAc,MAAM,MAAO;AACrC,UAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACxE;AAUA,iBAAe,uBAAqD;AAClE,UAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,eAAS;AACP,UAAI,WAAW;AACf,UAAI;AACF,cAAM,IAAI,MAAM,OAAO;AACvB,mBAAW;AAAA,MACb,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,MAAM,SAAS,SAAU,OAAM;AAAA,MACrC;AAEA,UAAI,UAAU;AACZ,cAAM,QAAQ,OAAO,WAAW;AAChC,YAAI;AACF,gBAAM,IAAI,UAAU,eAAe,OAAO,MAAM;AAAA,QAClD,SAAS,OAAgB;AACvB,gBAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACtE,gBAAM;AAAA,QACR;AACA,eAAO,MAAM,qBAAqB,KAAK;AAAA,MACzC;AAEA,YAAM,YAAY,MAAM,cAAc;AACtC,UAAI,cAAc,QAAQ,YAAY,eAAe;AACnD,cAAM,iBAAiB,SAAS;AAChC;AAAA,MACF;AAEA,UAAI,KAAK,IAAI,KAAK,UAAU;AAC1B,cAAM,IAAI;AAAA,UACR,8BAA8B,uBAAuB,oCAAoC,OAAO;AAAA,QAClG;AAAA,MACF;AAEA,YAAM,SAAS,oBAAoB,KAAK,OAAO,KAAK,oBAAoB;AACxE,YAAM,MAAM,MAAM;AAAA,IACpB;AAAA,EACF;AAQA,iBAAe,gBAAgB,YAAoB,SAAgC;AACjF,UAAM,WAAW,GAAG,UAAU,IAAI,OAAO,WAAW,CAAC;AACrD,QAAI;AACF,YAAM,IAAI,UAAU,UAAU,SAAS,MAAM;AAC7C,YAAM,0BAA0B,UAAU,UAAU;AAAA,IACtD,SAAS,OAAgB;AACvB,YAAM,IAAI,GAAG,UAAU,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACtD,YAAM;AAAA,IACR;AAAA,EACF;AAOA,iBAAe,0BAA0B,UAAkB,QAA+B;AACxF,UAAM,kBAAkB,oBAAI,IAAI,CAAC,SAAS,SAAS,QAAQ,CAAC;AAC5D,aAAS,UAAU,KAAK,WAAW;AACjC,UAAI;AACF,cAAM,IAAI,OAAO,UAAU,MAAM;AACjC;AAAA,MACF,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,WAAW,sBAAsB,CAAC,MAAM,QAAQ,CAAC,gBAAgB,IAAI,MAAM,IAAI,EAAG,OAAM;AAC5F,cAAM,MAAM,wBAAwB,UAAU,EAAE;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAQA,iBAAe,+BAAuD;AACpE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,IAAI,OAAO,WAAW,CAAC,OAAO;AAClG,QAAI;AACF,YAAM,IAAI,OAAO,WAAW,UAAU;AACtC,aAAO;AAAA,IACT,SAAS,GAAY;AACnB,aAAO,MAAM,iDAAiD,EAAE,KAAK,EAAW,CAAC;AACjF,aAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,aAAO,MAAM,6BAA6B,EAAE,KAAK,UAAU,CAAC;AAC5D,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAW,CAAC;AAC9D,YAAM,aAAa,MAAM,6BAA6B;AACtD,UAAI,YAAY;AACd,eAAO,MAAM,8EAA8E,EAAE,WAAW,CAAC;AAAA,MAC3G;AACA,YAAM,eAAe,aACjB,2BAA2B,UAAU,KACrC;AACJ,YAAM,IAAI;AAAA,QACR,yBAAyB,SAAS,uBAAuB,YAAY,KAAK,WAAW,OAAO;AAAA,MAC9F;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,EACjE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,WAAW,oBAAoB,MAAS;AAC9C,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,MACrC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AAED,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,eAAW,OAAO,eAAe;AAC/B,qBAAe,IAAI,IAAI,EAAE;AAAA,IAC3B;AAEA,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,QAAI;AACF,iBAAW,OAAO,eAAe;AAC/B,cAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,YAAI;AACF,gBAAM;AAAA,YAAc;AAAA,YAAM,IAAI;AAAA,YAAU,MACtC,QAAQ;AAAA,cACN,QAAQ,KAAK;AAAA,gBACX,OAAO,IAAI;AAAA,gBACX;AAAA,gBACA,WAAW;AAAA,cACb,CAAC;AAAA,YACH;AAAA,UACF;AACA;AACA,sBAAY,IAAI;AAChB,0BAAgB,IAAI,IAAI,EAAE;AAC1B,iBAAO,KAAK,iBAAiB,EAAE,OAAO,IAAI,GAAG,CAAC;AAAA,QAChD,SAAS,OAAO;AACd,iBAAO,MAAM,cAAc,EAAE,OAAO,IAAI,IAAI,eAAe,aAAa,sBAAsB,KAAK,MAAM,CAAC;AAC1G;AACA,sBAAY,IAAI;AAChB,cAAI,iBAAiB,sBAAsB;AACzC,mBAAO,MAAM,kEAAkE,EAAE,OAAO,IAAI,IAAI,aAAa,qBAAqB,CAAC;AACnI,uBAAW,IAAI,IAAI,EAAE;AAAA,UACvB,OAAO;AACL,kBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,yBAAa,IAAI,IAAI,IAAI;AAAA,cACvB,GAAG;AAAA,cACH,cAAc;AAAA,cACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,YAC5D,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,UAAI,YAAY;AACd,cAAM,aAAa,YAAY;AAE7B,gBAAM,cAAc,MAAM,UAAU;AACpC,gBAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,gBAAM,WAAW,WAAW;AAE5B,gBAAM,WAAuB;AAAA,YAC3B,iBAAiB;AAAA,YACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,YAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,UACrD;AACA,gBAAM,WAAW,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAEA,aAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,IACxC,UAAE;AACA,iBAAW,OAAO,eAAe;AAC/B,uBAAe,OAAO,IAAI,EAAE;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAKA,iBAAe,iBAAgC;AAE7C,QAAI,gBAAgB,CAAC,cAAe;AAEpC,mBAAe;AACf,QAAI;AACF,YAAM,aAAa,aAAa;AAAA,IAClC,SAAS,OAAO;AACd,aAAO,MAAM,iBAAiB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC9C,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAGA,oBAAgB;AAGhB,UAAM,aAAa,OAAO;AAG1B,mBAAe,YAAY,MAAM;AAC/B,qBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,eAAO,MAAM,oBAAoB,EAAE,IAAI,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH,GAAG,YAAY;AAEf,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAG7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,eAAe,KAAK,OAAO,CAAC,QAAQ,eAAe,IAAI,IAAI,EAAE,KAAK,CAAC,oBAAoB,IAAI,SAAS,KAAK,CAAC;AAChH,YAAM,UAAU,KAAK,SAAS,aAAa;AAC3C,UAAI,UAAU,GAAG;AACf,cAAM,WAAW,YAAY;AAAA,MAC/B;AACA,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AAEpC,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AACA,oBAAgB;AAGhB,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,eAAO,KAAK,wCAAwC,EAAE,WAAW,iBAAiB,CAAC;AACnF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
6
6
  "names": ["options"]
7
7
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@open-mercato/queue",
3
- "version": "0.6.8-develop.6924.1.a8d208fcdc",
3
+ "version": "0.6.8-develop.6930.1.1e5976efc3",
4
4
  "license": "MIT",
5
5
  "description": "Multi-strategy job queue with local and BullMQ support",
6
6
  "type": "module",
@@ -56,8 +56,8 @@
56
56
  "access": "public"
57
57
  },
58
58
  "dependencies": {
59
- "@open-mercato/shared": "0.6.8-develop.6924.1.a8d208fcdc",
60
- "@open-mercato/telemetry": "0.6.8-develop.6924.1.a8d208fcdc"
59
+ "@open-mercato/shared": "0.6.8-develop.6930.1.1e5976efc3",
60
+ "@open-mercato/telemetry": "0.6.8-develop.6930.1.1e5976efc3"
61
61
  },
62
62
  "repository": {
63
63
  "type": "git",
@@ -22,6 +22,14 @@ const queueLoggerError = createLogger('queue').error as jest.Mock
22
22
 
23
23
  function readJson(p: string) { return JSON.parse(fs.readFileSync(p, 'utf8')) }
24
24
 
25
+ async function waitUntil(condition: () => boolean, timeoutMs = 5000): Promise<void> {
26
+ const deadline = Date.now() + timeoutMs
27
+ while (!condition()) {
28
+ if (Date.now() > deadline) throw new Error('[internal] Timed out waiting for the expected filesystem state')
29
+ await new Promise((resolve) => { setTimeout(resolve, 5) })
30
+ }
31
+ }
32
+
25
33
  describe('Queue - local strategy', () => {
26
34
  const origCwd = process.cwd()
27
35
  let tmp: string
@@ -263,7 +271,10 @@ describe('Queue - local strategy', () => {
263
271
  await queue.close()
264
272
  })
265
273
 
266
- test('corrupted queue file is backed up and recreated', async () => {
274
+ // Regression (#5149): an unparsable queue file used to be replaced with `[]`
275
+ // and reported only in the log, so `enqueue` resolved against a queue that had
276
+ // just been emptied. The bytes must now be preserved and the caller told.
277
+ test('corrupted queue file is quarantined and the failure reaches the caller', async () => {
267
278
  const queue = createQueue<{ value: number }>('test-queue', 'local')
268
279
  const queueDir = path.join('.mercato', 'queue', 'test-queue')
269
280
  const queuePath = path.join(queueDir, 'queue.json')
@@ -273,12 +284,7 @@ describe('Queue - local strategy', () => {
273
284
  fs.mkdirSync(queueDir, { recursive: true })
274
285
  fs.writeFileSync(queuePath, brokenContent, 'utf8')
275
286
 
276
- const jobId = await queue.enqueue({ value: 42 })
277
-
278
- const queueContent = readJson(queuePath)
279
- expect(queueContent).toHaveLength(1)
280
- expect(queueContent[0].id).toBe(jobId)
281
- expect(queueContent[0].payload).toEqual({ value: 42 })
287
+ await expect(queue.enqueue({ value: 42 })).rejects.toThrow(/quarantined/)
282
288
 
283
289
  const backupFiles = fs.readdirSync(queueDir)
284
290
  .filter((fileName) => fileName.startsWith('queue.corrupted.') && fileName.endsWith('.json'))
@@ -290,13 +296,156 @@ describe('Queue - local strategy', () => {
290
296
  { err: expect.any(Error) },
291
297
  )
292
298
  expect(queueLoggerError).toHaveBeenCalledWith(
293
- 'Backed up corrupted queue file and recreated queue.json',
299
+ 'Quarantined corrupted queue file; its jobs are recoverable from the backup',
294
300
  { backupFile: expect.stringContaining('queue.corrupted.') },
295
301
  )
296
302
 
303
+ // The failed segment must not strand the cross-process lock, and the queue
304
+ // has to be usable again on the very next call.
305
+ expect(fs.existsSync(path.join(queueDir, 'queue.lock'))).toBe(false)
306
+
307
+ const jobId = await queue.enqueue({ value: 43 })
308
+ const queueContent = readJson(queuePath)
309
+ expect(queueContent).toHaveLength(1)
310
+ expect(queueContent[0].id).toBe(jobId)
311
+ expect(queueContent[0].payload).toEqual({ value: 43 })
312
+
297
313
  await queue.close()
298
314
  })
299
315
 
316
+ // Regression (#5149): two queue instances sharing a directory — the default
317
+ // dev topology, where the Next server and the worker are separate processes —
318
+ // used to interleave truncate-then-write calls. Each instance had its own
319
+ // in-process mutex, so nothing serialized them: the file was left unparsable
320
+ // and the jobs written by the losing writer disappeared.
321
+ test('concurrent writers on separate instances neither corrupt the file nor lose jobs', async () => {
322
+ const queueDir = path.join('.mercato', 'queue', 'multi-writer-queue')
323
+ const queuePath = path.join(queueDir, 'queue.json')
324
+ const writerA = createQueue<{ writer: string; index: number; payload: string }>('multi-writer-queue', 'local')
325
+ const writerB = createQueue<{ writer: string; index: number; payload: string }>('multi-writer-queue', 'local')
326
+ const jobsPerWriter = 25
327
+
328
+ // Deliberately mismatched payload sizes: the reported corruption signature
329
+ // is a short complete array followed by the tail of a longer earlier write.
330
+ const enqueueAll = (queue: typeof writerA, label: string, payloadSize: number) =>
331
+ Array.from({ length: jobsPerWriter }, (_, index) =>
332
+ queue.enqueue({ writer: label, index, payload: 'x'.repeat(payloadSize) }))
333
+
334
+ await Promise.all([
335
+ ...enqueueAll(writerA, 'A', 4000),
336
+ ...enqueueAll(writerB, 'B', 40),
337
+ ])
338
+
339
+ const stored = readJson(queuePath)
340
+ expect(stored).toHaveLength(jobsPerWriter * 2)
341
+ expect(stored.filter((job: any) => job.payload.writer === 'A')).toHaveLength(jobsPerWriter)
342
+ expect(stored.filter((job: any) => job.payload.writer === 'B')).toHaveLength(jobsPerWriter)
343
+
344
+ const strayFiles = fs.readdirSync(queueDir)
345
+ .filter((fileName) => fileName.startsWith('queue.corrupted.') || fileName.endsWith('.tmp'))
346
+ expect(strayFiles).toEqual([])
347
+
348
+ await writerA.close()
349
+ await writerB.close()
350
+ }, 60_000)
351
+
352
+ // Regression (#5149): `queue.json` was persisted with a plain `writeFile`,
353
+ // which truncates the existing inode and then streams the payload into it.
354
+ // Persisting through a temp file plus `rename` replaces the inode instead, so
355
+ // a concurrent reader can only ever see one complete document.
356
+ test('each persist swaps in a replacement file instead of truncating in place', async () => {
357
+ const queue = createQueue<{ value: number }>('atomic-write-queue', 'local')
358
+ const queueDir = path.join('.mercato', 'queue', 'atomic-write-queue')
359
+ const queuePath = path.join(queueDir, 'queue.json')
360
+ const writeFileSpy = jest.spyOn(fs.promises, 'writeFile')
361
+
362
+ try {
363
+ await queue.enqueue({ value: 1 })
364
+ await queue.enqueue({ value: 2 })
365
+
366
+ // The invariant, asserted portably: queue.json is only ever created with
367
+ // the exclusive `wx` flag by ensureDir, never written in place. Every
368
+ // persist goes to a temp file that is then renamed over it.
369
+ const inPlaceWrites = writeFileSpy.mock.calls.filter(([target, , options]) => {
370
+ if (typeof target !== 'string' || path.resolve(target) !== path.resolve(queuePath)) return false
371
+ return (options as { flag?: string } | undefined)?.flag !== 'wx'
372
+ })
373
+ expect(inPlaceWrites).toEqual([])
374
+
375
+ expect(readJson(queuePath)).toHaveLength(2)
376
+ expect(fs.readdirSync(queueDir).filter((fileName) => fileName.endsWith('.tmp'))).toEqual([])
377
+ } finally {
378
+ writeFileSpy.mockRestore()
379
+ await queue.close()
380
+ }
381
+ })
382
+
383
+ // Regression (#5149): the cross-process lock must not outlive the process
384
+ // that took it, or a crash mid-segment would wedge every queue consumer.
385
+ test('a stale lock left behind by a dead process is reclaimed', async () => {
386
+ const queue = createQueue<{ value: number }>('stale-lock-queue', 'local')
387
+ const queueDir = path.join('.mercato', 'queue', 'stale-lock-queue')
388
+ const lockPath = path.join(queueDir, 'queue.lock')
389
+
390
+ fs.mkdirSync(lockPath, { recursive: true })
391
+ const wellPastTheStaleThreshold = new Date(Date.now() - 60_000)
392
+ fs.utimesSync(lockPath, wellPastTheStaleThreshold, wellPastTheStaleThreshold)
393
+
394
+ const jobId = await queue.enqueue({ value: 7 })
395
+
396
+ expect(typeof jobId).toBe('string')
397
+ expect(readJson(path.join(queueDir, 'queue.json'))).toHaveLength(1)
398
+ expect(fs.existsSync(lockPath)).toBe(false)
399
+
400
+ await queue.close()
401
+ })
402
+
403
+ // Regression (#5149): reclaiming a stale lock replaces it, so the holder that
404
+ // was reclaimed must not remove the replacement on its way out. An
405
+ // unconditional release deleted the successor's lock and let a third caller
406
+ // into the critical section beside it — the lost-update window this fix
407
+ // exists to close.
408
+ test('a holder whose lock was reclaimed does not delete the lock that replaced it', async () => {
409
+ const queue = createQueue<{ value: number }>('reclaim-release-queue', 'local')
410
+ const queueDir = path.join('.mercato', 'queue', 'reclaim-release-queue')
411
+ const lockPath = path.join(queueDir, 'queue.lock')
412
+ const ownerPath = path.join(lockPath, 'owner')
413
+ const successorToken = 'successor-owner-token'
414
+
415
+ let releaseStall = () => {}
416
+ const stalled = new Promise<void>((resolve) => { releaseStall = resolve })
417
+ const realWriteFile = fs.promises.writeFile
418
+ const writeFileSpy = jest.spyOn(fs.promises, 'writeFile').mockImplementation(
419
+ async (target: any, content: any, options: any) => {
420
+ // Stall inside the critical section: the temp file is only ever written
421
+ // while this queue holds the lock.
422
+ if (typeof target === 'string' && target.endsWith('.tmp')) await stalled
423
+ return realWriteFile(target, content, options)
424
+ },
425
+ )
426
+
427
+ try {
428
+ const enqueued = queue.enqueue({ value: 1 })
429
+ await waitUntil(() => fs.existsSync(ownerPath))
430
+
431
+ // Stand in for the peer that found this lock stale: move it aside, drop
432
+ // it, and take a fresh one of its own.
433
+ fs.rmSync(lockPath, { recursive: true, force: true })
434
+ fs.mkdirSync(lockPath, { recursive: true })
435
+ fs.writeFileSync(ownerPath, successorToken, 'utf8')
436
+
437
+ releaseStall()
438
+ await enqueued
439
+
440
+ expect(fs.existsSync(lockPath)).toBe(true)
441
+ expect(fs.readFileSync(ownerPath, 'utf8')).toBe(successorToken)
442
+ } finally {
443
+ writeFileSpy.mockRestore()
444
+ fs.rmSync(lockPath, { recursive: true, force: true })
445
+ await queue.close()
446
+ }
447
+ })
448
+
300
449
  test('failed jobs are retained in queue for retry', async () => {
301
450
  const queue = createQueue<{ shouldFail: boolean }>('test-queue', 'local')
302
451
  const queuePath = path.join('.mercato', 'queue', 'test-queue', 'queue.json')
@@ -37,6 +37,21 @@ const DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'
37
37
  const DEFAULT_MAX_ATTEMPTS = 3
38
38
  const RETRY_BACKOFF_BASE_MS = 1000
39
39
 
40
+ /**
41
+ * Cross-process lock tuning. A held lock only ever spans local file I/O — job
42
+ * handlers run outside it — so realistic hold times are milliseconds and the
43
+ * stale threshold sits orders of magnitude above them. It exists solely so a
44
+ * process that dies mid-segment cannot wedge the queue forever. A holder that
45
+ * was merely suspended rather than dead can still be reclaimed, which is why
46
+ * every acquisition carries an owner token and releases only its own lock.
47
+ */
48
+ const LOCK_STALE_MS = 15_000
49
+ const LOCK_ACQUIRE_TIMEOUT_MS = 30_000
50
+ const LOCK_RETRY_MIN_MS = 2
51
+ const LOCK_RETRY_MAX_MS = 20
52
+ const RENAME_MAX_RETRIES = 5
53
+ const RENAME_RETRY_BASE_MS = 10
54
+
40
55
  const fsp = fs.promises
41
56
 
42
57
  /**
@@ -48,7 +63,23 @@ const fsp = fs.promises
48
63
  *
49
64
  * **Limitations:**
50
65
  * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)
51
- * - Not suitable for production or multi-process environments
66
+ * - Not suitable for production: there is no dead-letter store, no throughput
67
+ * beyond one job at a time, and every operation rewrites the whole queue file
68
+ *
69
+ * Multiple processes MAY share a queue directory, which is the default
70
+ * development topology: the dev worker runs in its own process alongside the
71
+ * Next.js server. What that buys you, and what it does not:
72
+ *
73
+ * - **Safe** — concurrent producers. Every read-modify-write segment takes the
74
+ * `queue.lock` directory lock and every persist swaps the file in with an
75
+ * atomic rename, so the file cannot be torn, no enqueue is lost to a
76
+ * concurrent one, and a reader always observes one complete document.
77
+ * Writers contend, though, so throughput degrades as processes are added.
78
+ * - **NOT safe** — concurrent consumers. `process()` deliberately runs job
79
+ * handlers outside the lock, so two worker processes polling the same queue
80
+ * would both claim the same pending jobs and execute them twice. There is no
81
+ * per-job lease. Run exactly one worker process per queue; use the `async`
82
+ * strategy when you need more than one.
52
83
  *
53
84
  * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.
54
85
  * **This strategy keeps no failed-job store**: once attempts are exhausted the job is
@@ -65,8 +96,8 @@ const fsp = fs.promises
65
96
  *
66
97
  * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not
67
98
  * block the Node.js event loop. A per-queue promise chain serializes
68
- * read-modify-write sequences to preserve the atomicity guarantees the
69
- * previous synchronous implementation relied on.
99
+ * read-modify-write sequences within one instance, and the `queue.lock`
100
+ * directory lock extends that serialization across instances and processes.
70
101
  *
71
102
  * @template T - The payload type for jobs
72
103
  * @param name - Queue name (used for directory naming)
@@ -83,6 +114,8 @@ export function createLocalQueue<T = unknown>(
83
114
  const queueDir = path.join(baseDir, name)
84
115
  const queueFile = path.join(queueDir, 'queue.json')
85
116
  const stateFile = path.join(queueDir, 'state.json')
117
+ const lockDir = path.join(queueDir, 'queue.lock')
118
+ const lockOwnerFile = path.join(lockDir, 'owner')
86
119
  const logger = packageLogger.child({ queue: name })
87
120
  // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially
88
121
  const concurrency = options?.concurrency ?? 1
@@ -95,10 +128,15 @@ export function createLocalQueue<T = unknown>(
95
128
  const inFlightJobIds = new Set<string>()
96
129
 
97
130
  // Per-queue mutex. Serializes read-modify-write segments so async fs calls
98
- // cannot interleave and clobber each other's writes.
131
+ // cannot interleave and clobber each other's writes. It only covers this
132
+ // instance, so it also guarantees at most one outstanding `queue.lock`
133
+ // acquisition per instance — the directory lock below is not reentrant.
99
134
  let fileOpChain: Promise<unknown> = Promise.resolve()
100
135
  function withFileLock<R>(fn: () => Promise<R>): Promise<R> {
101
- const run = fileOpChain.then(() => fn(), () => fn())
136
+ const run = fileOpChain.then(
137
+ () => runExclusively(fn),
138
+ () => runExclusively(fn),
139
+ )
102
140
  fileOpChain = run.then(
103
141
  () => undefined,
104
142
  () => undefined,
@@ -106,10 +144,160 @@ export function createLocalQueue<T = unknown>(
106
144
  return run
107
145
  }
108
146
 
147
+ /**
148
+ * Runs `fn` while holding the cross-process `queue.lock`, so read-modify-write
149
+ * segments issued by other queue instances — in this process or another one —
150
+ * cannot interleave with it.
151
+ */
152
+ async function runExclusively<R>(fn: () => Promise<R>): Promise<R> {
153
+ await ensureDir()
154
+ const release = await acquireDirectoryLock()
155
+ try {
156
+ return await fn()
157
+ } finally {
158
+ await release()
159
+ }
160
+ }
161
+
109
162
  // -------------------------------------------------------------------------
110
163
  // File Operations
111
164
  // -------------------------------------------------------------------------
112
165
 
166
+ function sleep(ms: number): Promise<void> {
167
+ return new Promise((resolve) => { setTimeout(resolve, ms) })
168
+ }
169
+
170
+ async function lockHeldForMs(): Promise<number | null> {
171
+ try {
172
+ const stats = await fsp.stat(lockDir)
173
+ return Date.now() - stats.mtimeMs
174
+ } catch {
175
+ return null
176
+ }
177
+ }
178
+
179
+ /**
180
+ * Reclaims a lock whose holder died. The rename is the serialization point:
181
+ * only one racer can move `queue.lock` aside, so two processes cannot both
182
+ * decide a stale lock is theirs to clear and then both create a fresh one.
183
+ */
184
+ async function reclaimStaleLock(heldForMs: number): Promise<void> {
185
+ const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`
186
+ try {
187
+ await fsp.rename(lockDir, reclaimedPath)
188
+ } catch {
189
+ return
190
+ }
191
+ logger.warn('Reclaimed a stale queue lock', { lockDir, heldForMs })
192
+ await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {})
193
+ }
194
+
195
+ async function readLockOwner(): Promise<string | null> {
196
+ try {
197
+ return await fsp.readFile(lockOwnerFile, 'utf8')
198
+ } catch {
199
+ return null
200
+ }
201
+ }
202
+
203
+ /**
204
+ * Releases the lock only when this acquisition still owns it. A holder that
205
+ * was suspended past `LOCK_STALE_MS` has had its lock reclaimed *and
206
+ * replaced* by whoever reclaimed it, so an unconditional removal here would
207
+ * delete the successor's lock and let a third caller into the critical
208
+ * section alongside it. A missing or mismatched token means someone else owns
209
+ * the path now, and the correct action is to leave it alone.
210
+ */
211
+ async function releaseDirectoryLock(token: string): Promise<void> {
212
+ if (await readLockOwner() !== token) return
213
+ await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})
214
+ }
215
+
216
+ /**
217
+ * Acquires the cross-process advisory lock for this queue directory.
218
+ * `mkdir` without `recursive` is an atomic exclusive create on every platform
219
+ * Node.js supports, which makes it the portable primitive here — no runtime
220
+ * dependency, and no reliance on advisory `flock` semantics. The owner token
221
+ * written into the directory is what lets the release distinguish this
222
+ * acquisition from a successor's.
223
+ */
224
+ async function acquireDirectoryLock(): Promise<() => Promise<void>> {
225
+ const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS
226
+
227
+ for (;;) {
228
+ let acquired = false
229
+ try {
230
+ await fsp.mkdir(lockDir)
231
+ acquired = true
232
+ } catch (e: unknown) {
233
+ const error = e as NodeJS.ErrnoException
234
+ if (error.code !== 'EEXIST') throw error
235
+ }
236
+
237
+ if (acquired) {
238
+ const token = crypto.randomUUID()
239
+ try {
240
+ await fsp.writeFile(lockOwnerFile, token, 'utf8')
241
+ } catch (error: unknown) {
242
+ await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})
243
+ throw error
244
+ }
245
+ return () => releaseDirectoryLock(token)
246
+ }
247
+
248
+ const heldForMs = await lockHeldForMs()
249
+ if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {
250
+ await reclaimStaleLock(heldForMs)
251
+ continue
252
+ }
253
+
254
+ if (Date.now() >= deadline) {
255
+ throw new Error(
256
+ `[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`,
257
+ )
258
+ }
259
+
260
+ const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)
261
+ await sleep(jitter)
262
+ }
263
+ }
264
+
265
+ /**
266
+ * Persists `content` by writing a unique sibling temp file and renaming it
267
+ * onto `targetFile`. `rename` within a directory is atomic, so a concurrent
268
+ * reader sees either the previous document or the new one in full — never the
269
+ * torn result of a truncate-then-write.
270
+ */
271
+ async function writeFileAtomic(targetFile: string, content: string): Promise<void> {
272
+ const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`
273
+ try {
274
+ await fsp.writeFile(tempFile, content, 'utf8')
275
+ await renameWithContentionRetry(tempFile, targetFile)
276
+ } catch (error: unknown) {
277
+ await fsp.rm(tempFile, { force: true }).catch(() => {})
278
+ throw error
279
+ }
280
+ }
281
+
282
+ /**
283
+ * Windows rejects a rename onto a file another process currently has open,
284
+ * so retry briefly on the contention codes it raises. POSIX renames replace
285
+ * the target unconditionally and take the first attempt.
286
+ */
287
+ async function renameWithContentionRetry(fromFile: string, toFile: string): Promise<void> {
288
+ const contentionCodes = new Set(['EPERM', 'EBUSY', 'EACCES'])
289
+ for (let attempt = 0; ; attempt++) {
290
+ try {
291
+ await fsp.rename(fromFile, toFile)
292
+ return
293
+ } catch (e: unknown) {
294
+ const error = e as NodeJS.ErrnoException
295
+ if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error
296
+ await sleep(RENAME_RETRY_BASE_MS * (attempt + 1))
297
+ }
298
+ }
299
+ }
300
+
113
301
  async function ensureDir(): Promise<void> {
114
302
  try {
115
303
  await fsp.mkdir(queueDir, { recursive: true })
@@ -135,11 +323,21 @@ export function createLocalQueue<T = unknown>(
135
323
  }
136
324
  }
137
325
 
138
- async function backupCorruptedQueueFile(content: string): Promise<string> {
139
- const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.json`)
140
- await fsp.writeFile(backupFile, content, 'utf8')
141
- await fsp.writeFile(queueFile, '[]', 'utf8')
142
- return backupFile
326
+ /**
327
+ * Moves an unparsable queue file aside so its jobs stay recoverable. The
328
+ * caller is expected to surface the failure rather than continue on an empty
329
+ * queue: silently recreating `queue.json` here is what turned an unreadable
330
+ * file into permanent, unreported job loss.
331
+ */
332
+ async function quarantineCorruptedQueueFile(): Promise<string | null> {
333
+ const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`)
334
+ try {
335
+ await fsp.rename(queueFile, backupFile)
336
+ return backupFile
337
+ } catch (e: unknown) {
338
+ logger.error('Failed to quarantine the corrupted queue file', { err: e as Error })
339
+ return null
340
+ }
143
341
  }
144
342
 
145
343
  async function readQueue(): Promise<StoredJob<T>[]> {
@@ -168,15 +366,22 @@ export function createLocalQueue<T = unknown>(
168
366
  } catch (error: unknown) {
169
367
  const parseError = error as Error
170
368
  logger.error('Failed to parse queue file', { err: parseError })
171
- const backupFile = await backupCorruptedQueueFile(content)
172
- logger.error('Backed up corrupted queue file and recreated queue.json', { backupFile })
173
- return []
369
+ const backupFile = await quarantineCorruptedQueueFile()
370
+ if (backupFile) {
371
+ logger.error('Quarantined corrupted queue file; its jobs are recoverable from the backup', { backupFile })
372
+ }
373
+ const recoveryHint = backupFile
374
+ ? `has been quarantined as ${backupFile}`
375
+ : 'could not be quarantined and was left in place'
376
+ throw new Error(
377
+ `[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`,
378
+ )
174
379
  }
175
380
  }
176
381
 
177
382
  async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {
178
383
  await ensureDir()
179
- await fsp.writeFile(queueFile, JSON.stringify(jobs, null, 2), 'utf8')
384
+ await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2))
180
385
  }
181
386
 
182
387
  async function readState(): Promise<LocalState> {
@@ -191,7 +396,7 @@ export function createLocalQueue<T = unknown>(
191
396
 
192
397
  async function writeState(state: LocalState): Promise<void> {
193
398
  await ensureDir()
194
- await fsp.writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8')
399
+ await writeFileAtomic(stateFile, JSON.stringify(state, null, 2))
195
400
  }
196
401
 
197
402
  function generateId(): string {