@open-mercato/queue 0.7.1-develop.7183.1.db9678eeb8 → 0.7.1-develop.7186.1.6e080a5017
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/strategies/async.js +23 -1
- package/dist/strategies/async.js.map +2 -2
- package/dist/strategies/local.js +18 -1
- package/dist/strategies/local.js.map +2 -2
- package/package.json +3 -3
- package/src/__tests__/job-failure-reporting.test.ts +240 -0
- package/src/strategies/async.ts +56 -2
- package/src/strategies/local.ts +34 -1
package/dist/strategies/async.js
CHANGED
|
@@ -56,6 +56,17 @@ function createAsyncQueue(name, options) {
|
|
|
56
56
|
const maxStalledCount = options?.maxStalledCount;
|
|
57
57
|
const onJobAbandoned = options?.onJobAbandoned;
|
|
58
58
|
const logger = packageLogger.child({ queue: name });
|
|
59
|
+
function reportQueueError(error, code, attributes) {
|
|
60
|
+
try {
|
|
61
|
+
getTelemetryRuntime()?.reportError(error, {
|
|
62
|
+
module: "queue",
|
|
63
|
+
code,
|
|
64
|
+
attributes: { queue: name, ...attributes }
|
|
65
|
+
});
|
|
66
|
+
} catch (telemetryError) {
|
|
67
|
+
logger.warn("Failed to report a queue error to telemetry", { code, err: telemetryError });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
59
70
|
let bullQueue = null;
|
|
60
71
|
let bullWorker = null;
|
|
61
72
|
let bullmqModule = null;
|
|
@@ -87,6 +98,7 @@ function createAsyncQueue(name, options) {
|
|
|
87
98
|
jobId,
|
|
88
99
|
err: hookError
|
|
89
100
|
});
|
|
101
|
+
reportQueueError(hookError, "queue.abandon_report_failed", { jobId: jobId ?? void 0 });
|
|
90
102
|
return;
|
|
91
103
|
}
|
|
92
104
|
try {
|
|
@@ -96,6 +108,7 @@ function createAsyncQueue(name, options) {
|
|
|
96
108
|
jobId,
|
|
97
109
|
err: ackError
|
|
98
110
|
});
|
|
111
|
+
reportQueueError(ackError, "queue.abandon_ack_failed", { jobId: jobId ?? void 0 });
|
|
99
112
|
}
|
|
100
113
|
})().finally(() => {
|
|
101
114
|
inFlightAbandonedJobIds.delete(payload.id);
|
|
@@ -118,6 +131,7 @@ function createAsyncQueue(name, options) {
|
|
|
118
131
|
}
|
|
119
132
|
} catch (sweepError) {
|
|
120
133
|
logger.error("Abandoned-job sweep failed", { err: sweepError });
|
|
134
|
+
reportQueueError(sweepError, "queue.abandon_sweep_failed");
|
|
121
135
|
}
|
|
122
136
|
}
|
|
123
137
|
let telemetryPromise = null;
|
|
@@ -208,7 +222,14 @@ function createAsyncQueue(name, options) {
|
|
|
208
222
|
bullWorker.on("failed", (job, err) => {
|
|
209
223
|
const failedJob = job;
|
|
210
224
|
const error = err;
|
|
211
|
-
|
|
225
|
+
const attemptNumber = failedJob?.attemptsMade ?? 0;
|
|
226
|
+
const maxAttempts = failedJob?.opts?.attempts ?? attempts;
|
|
227
|
+
const exhausted = attemptNumber >= maxAttempts;
|
|
228
|
+
logger.error("Job failed", { jobId: failedJob?.id, attemptNumber, maxAttempts, err: error });
|
|
229
|
+
reportQueueError(error, exhausted ? "queue.job_exhausted" : "queue.job_failed", {
|
|
230
|
+
jobId: failedJob?.id,
|
|
231
|
+
attemptNumber
|
|
232
|
+
});
|
|
212
233
|
if (!onJobAbandoned) return;
|
|
213
234
|
if (!isAbandonedJobReason(error?.message ?? "")) return;
|
|
214
235
|
if (!failedJob?.data) return;
|
|
@@ -222,6 +243,7 @@ function createAsyncQueue(name, options) {
|
|
|
222
243
|
bullWorker.on("error", (err) => {
|
|
223
244
|
const error = err;
|
|
224
245
|
logger.error("Worker error", { err: error });
|
|
246
|
+
reportQueueError(error, "queue.worker_error");
|
|
225
247
|
});
|
|
226
248
|
if (onJobAbandoned) {
|
|
227
249
|
void sweepAbandonedJobs();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/async.ts"],
|
|
4
|
-
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow, parseRedisUrl, REDIS_WIRE_PROTOCOL } from '@open-mercato/shared/lib/redis/connection'\nimport type { RedisProtocolVersion } from '@open-mercato/shared/lib/redis/connection'\nimport { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst packageLogger = createLogger('queue')\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n family?: number\n protocol?: RedisProtocolVersion\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n id?: string\n data?: T\n failedReason?: string\n remove: () => Promise<void>\n updateData?: (data: T) => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions; telemetry?: unknown }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: {\n connection: ConnectionOptions\n concurrency: number\n telemetry?: unknown\n lockDuration?: number\n maxStalledCount?: number\n }\n ) => BullWorkerInterface\n}\n\n/** The `bullmq-otel` package (optional). Loaded only when an OTLP backend is active. */\ntype BullMQOtelModule = { BullMQOtel: new (tracerName: string) => object }\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\n/**\n * The failures BullMQ records when it gives up on a job *before* handing it to the processor.\n *\n * Both are written as a `defa` (deferred failure) marker on the job, after which the next worker\n * short-circuits in `Worker.processJob` via `getUnrecoverableErrorMessage` and fails the job without\n * calling the handler. The first comes from the stalled-job script once a job's cumulative stall\n * count passes `maxStalledCount`; the second from `maxStartedAttempts`.\n *\n * Matching the reason is what tells \"the queue abandoned this\" from \"the handler ran and threw\", and\n * it is deliberately stateless: the alternative \u2014 tracking which jobs this process has entered \u2014 can\n * only answer \"did the handler run *here*\", which is the wrong question the moment more than one\n * worker is running. `bullmq-abandoned-reasons.test.ts` asserts these strings still exist in the\n * installed BullMQ, so an upgrade that renames them fails loudly instead of silently disabling the\n * callback.\n */\nexport const ABANDONED_JOB_REASONS = [\n 'job stalled more than allowable limit',\n 'job started more than allowable limit',\n] as const\n\nfunction isAbandonedJobReason(message: string): boolean {\n return (ABANDONED_JOB_REASONS as readonly string[]).includes(message)\n}\n\n/**\n * Metadata key written onto the stored job once `onJobAbandoned` has completed for it.\n *\n * BullMQ's failed set is the durable record of abandoned jobs (`removeOnFail` keeps them), so it\n * doubles as the dead-letter queue for reports: the sweep re-delivers any abandoned job that does not\n * carry this marker. The marker \u2014 not `job.remove()` \u2014 is the acknowledgement, so the failed job\n * itself survives for diagnosis.\n */\nconst ABANDON_REPORT_ACK_KEY = 'abandonReportedAt'\n// NOTE for anyone adding a retry action: the marker lives inside the job's own payload envelope, so a\n// job retried from admin tooling carries it into its next life and a second abandonment of that job\n// would never be reported. A retry path must clear `metadata.abandonReportedAt` when it re-enqueues.\n\n/**\n * How often a worker re-sweeps the failed set for unacknowledged abandoned jobs.\n *\n * Override with `QUEUE_ABANDONED_SWEEP_INTERVAL_MS` to trade recovery latency against Redis chatter.\n */\nexport const ABANDONED_JOB_SWEEP_INTERVAL_MS = 5 * 60 * 1000\n\n/**\n * How long `close()` waits for in-flight reports before giving up on them.\n *\n * Bounded on purpose: the hook reaches a database, and a shutdown during an infrastructure incident\n * is exactly when that write can hang rather than fail. An unbounded wait would turn a graceful\n * shutdown into a SIGKILL and skip the telemetry flush that follows it. Abandoning the wait is safe\n * because delivery is at-least-once \u2014 an unacknowledged report is re-delivered by the next worker's\n * start-up sweep, the same path that covers a process which died mid-report.\n */\nexport const ABANDONED_JOB_DRAIN_TIMEOUT_MS = 5000\n\nfunction resolveSweepIntervalMs(): number {\n const configured = Number.parseInt(process.env.QUEUE_ABANDONED_SWEEP_INTERVAL_MS ?? '', 10)\n return Number.isFinite(configured) && configured > 0 ? configured : ABANDONED_JOB_SWEEP_INTERVAL_MS\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects ioredis connection fields rather than a nested URL string.\n * Parse URL-based configuration at this boundary while keeping the public\n * queue API compatible with existing `{ url }` callers.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return parseRedisUrl(options.url)\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n family: options.family,\n protocol: REDIS_WIRE_PROTOCOL,\n }\n }\n\n return parseRedisUrl(getRedisUrlOrThrow('QUEUE'))\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n const attempts = options?.attempts ?? 3\n const lockDuration = options?.lockDuration\n const maxStalledCount = options?.maxStalledCount\n const onJobAbandoned = options?.onJobAbandoned\n const logger = packageLogger.child({ queue: name })\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n let abandonedSweepTimer: ReturnType<typeof setInterval> | null = null\n let closing = false\n\n // In-flight `onJobAbandoned` calls. Detached from the caller that started them (the 'failed'\n // listener or the sweep), so `close()` drains them rather than letting a deploy truncate a repair\n // mid-write. The id set stops the two callers from double-reporting a job inside one process.\n const pendingAbandonedReports = new Set<Promise<void>>()\n const inFlightAbandonedJobIds = new Set<string>()\n\n type AbandonedJobRecord = {\n id?: string\n data?: QueuedJob<T>\n updateData?: (data: QueuedJob<T>) => Promise<void>\n }\n\n // The acknowledgement that makes delivery at-least-once: written only after the callback returns,\n // so a callback that threw or a process that died mid-report leaves the job unmarked and a later\n // sweep retries it. Requires the driver to expose `updateData`; without it the report simply stays\n // unacknowledged and repeats, which the idempotency contract permits.\n async function acknowledgeAbandonedReport(job: AbandonedJobRecord): Promise<void> {\n if (!job.data || typeof job.updateData !== 'function') return\n await job.updateData({\n ...job.data,\n metadata: { ...(job.data.metadata ?? {}), [ABANDON_REPORT_ACK_KEY]: new Date().toISOString() },\n })\n }\n\n function reportAbandonedJob(job: AbandonedJobRecord, reason: string): Promise<void> | null {\n if (!onJobAbandoned) return null\n const payload = job.data\n if (!payload) return null\n if (payload.metadata && payload.metadata[ABANDON_REPORT_ACK_KEY]) return null\n if (inFlightAbandonedJobIds.has(payload.id)) return null\n inFlightAbandonedJobIds.add(payload.id)\n\n const jobId = job.id ?? null\n logger.warn('Job abandoned by the queue without running its handler', { jobId, reason })\n const report = (async () => {\n try {\n await onJobAbandoned(payload, { jobId, reason })\n } catch (hookError) {\n logger.error('onJobAbandoned handler threw; the report stays unacknowledged and the sweep will retry it', {\n jobId,\n err: hookError as Error,\n })\n return\n }\n try {\n await acknowledgeAbandonedReport(job)\n } catch (ackError) {\n logger.error('Failed to acknowledge an abandoned-job report; the sweep may repeat it', {\n jobId,\n err: ackError as Error,\n })\n }\n })().finally(() => {\n inFlightAbandonedJobIds.delete(payload.id)\n })\n pendingAbandonedReports.add(report)\n void report.then(() => pendingAbandonedReports.delete(report))\n return report\n }\n\n // The failed set is this strategy's dead-letter queue for abandoned jobs. Enumerating it on worker\n // start and on an interval, and re-delivering anything unacknowledged, is what upgrades the\n // 'failed'-listener fast path from at-most-once to at-least-once: a report lost to a crash is\n // simply still unmarked when the next sweep looks. Residual loss: `removeOnFail` caps the set, so\n // a job evicted before any sweep sees it is gone for good.\n async function sweepAbandonedJobs(): Promise<void> {\n if (!onJobAbandoned || closing) return\n try {\n const queue = await getQueue()\n const failedJobs = await queue.getJobs(['failed'], 0, -1)\n // Re-checked after the awaits: a sweep already past its guard when `close()` ran would\n // otherwise start a report the drain has stopped waiting for. The next start-up sweep\n // re-delivers it, so stopping here loses nothing.\n if (closing) return\n for (const failedJob of failedJobs) {\n // Re-checked every iteration for the same reason: a long fan-out must not outlive the drain.\n if (closing) return\n const reason = failedJob.failedReason ?? ''\n if (!isAbandonedJobReason(reason)) continue\n // Awaited one at a time. Each report opens a request container and writes to the database, and\n // the worst case for this loop is the first worker start after the feature ships, on a\n // deployment that has been accumulating abandoned jobs \u2014 the largest backlog, on the process\n // least able to absorb it. The sweep is a recovery path with no latency requirement (five\n // minutes late is its normal mode), so pacing costs nothing worth having.\n await reportAbandonedJob(failedJob, reason)\n }\n } catch (sweepError) {\n logger.error('Abandoned-job sweep failed', { err: sweepError as Error })\n }\n }\n // Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or\n // undefined (use our own metadata._trace carrier instead). Memoized as the\n // in-flight promise so concurrent first-time callers share one resolution.\n let telemetryPromise: Promise<object | undefined> | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n /**\n * When an OTLP backend is active, delegate async-queue tracing to `bullmq-otel`\n * (richer BullMQ-internal spans: add / process / wait / attempts). Returns\n * `undefined` \u2014 meaning \"use our own `metadata._trace` carrier\" \u2014 when telemetry\n * is off, a non-OTEL backend is selected, or `bullmq-otel` isn't installed. (The\n * `local` strategy always uses our carrier; it isn't BullMQ, so `bullmq-otel`\n * cannot instrument it.)\n */\n async function getQueueTelemetry(): Promise<object | undefined> {\n if (!telemetryPromise) {\n telemetryPromise = (async () => {\n if (!getTelemetryRuntime()?.canUseGlobalTracePropagation()) return undefined\n try {\n const mod = (await import('bullmq-otel')) as unknown as BullMQOtelModule\n return new mod.BullMQOtel('open-mercato')\n } catch {\n packageLogger.warn('bullmq-otel not available; using built-in trace carrier', { queue: name })\n return undefined\n }\n })()\n }\n return telemetryPromise\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n const telemetry = await getQueueTelemetry()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection, ...(telemetry ? { telemetry } : {}) })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n // When bullmq-otel handles propagation, don't also attach our carrier.\n const telemetry = await getQueueTelemetry()\n const metadata = telemetry ? undefined : attachTraceMetadata(undefined)\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(metadata ? { metadata } : {}),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n const telemetry = await getQueueTelemetry()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n const ctx = {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n }\n // With bullmq-otel active, BullMQ owns the process span and active\n // context (the handler's pg/undici spans nest under it). Otherwise\n // continue the trace from our own carrier.\n if (telemetry) {\n await handler(jobData, ctx)\n } else {\n await runJobInTrace(name, jobData.metadata, () => handler(jobData, ctx))\n }\n },\n {\n connection,\n concurrency,\n ...(telemetry ? { telemetry } : {}),\n ...(lockDuration !== undefined ? { lockDuration } : {}),\n ...(maxStalledCount !== undefined ? { maxStalledCount } : {}),\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n logger.info('Job completed', { jobId: jobWithId.id })\n })\n\n bullWorker.on('failed', (job, err) => {\n const failedJob = job as AbandonedJobRecord | undefined\n const error = err as Error\n logger.error('Job failed', { jobId: failedJob?.id, err: error })\n\n if (!onJobAbandoned) return\n // Any other reason means a handler ran and threw. That failure is the handler's own and it has\n // already had its chance to record it.\n if (!isAbandonedJobReason(error?.message ?? '')) return\n // No payload means the queue could not give us the job at all. There is nothing to hand the\n // callback and nothing it could repair, so reporting could only ever be a false alarm \u2014 the\n // 'Job failed' line above still records it.\n if (!failedJob?.data) return\n\n // The fast path: report the moment the abandonment is observed. `reportAbandonedJob` runs the\n // callback detached with its own try/catch \u2014 this is an EventEmitter, where an unhandled\n // rejection is fatal to the process \u2014 and acknowledges the job only afterwards, so a report\n // lost here is retried by the sweep.\n reportAbandonedJob(failedJob, error.message)\n })\n\n // A stalled job is redelivered under the same id while the previous worker\n // may still be running it, so this is the signal that a handler is about to\n // be executed twice. BullMQ's docs require surfacing it: without this line\n // duplicate processing is invisible.\n bullWorker.on('stalled', (jobId) => {\n logger.warn('Job stalled and will be redelivered \u2014 the handler may run concurrently with a previous delivery', {\n jobId: typeof jobId === 'string' ? jobId : null,\n })\n })\n\n bullWorker.on('error', (err) => {\n const error = err as Error\n logger.error('Worker error', { err: error })\n })\n\n if (onJobAbandoned) {\n // Sweep immediately so reports lost to a previous process's crash are re-delivered as soon as\n // a worker is back, then keep re-sweeping for anything the fast path loses while running.\n // The timer is unref'd so it never holds the process open.\n void sweepAbandonedJobs()\n abandonedSweepTimer = setInterval(() => {\n void sweepAbandonedJobs()\n }, resolveSweepIntervalMs())\n ;(abandonedSweepTimer as unknown as { unref?: () => void }).unref?.()\n }\n\n logger.info('Worker started', { concurrency })\n\n // For async strategy, return a sentinel result indicating worker mode\n // processed=-1 signals that this is a continuous worker, not a batch process\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const queue = await getQueue()\n\n // Obliterate removes all jobs from the queue\n await queue.obliterate({ force: true })\n\n return { removed: -1 } // BullMQ obliterate doesn't return count\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n const queue = await getQueue()\n const jobs = await queue.getJobs(REMOVABLE_JOB_STATES, 0, -1)\n let removed = 0\n\n for (const job of jobs) {\n if (!payloadMatchesScope(job.data?.payload, scope)) continue\n try {\n await job.remove()\n removed++\n } catch {\n // The job may have started between enumeration and removal. In-flight\n // cancellation is handled by the caller's lock/heartbeat contract.\n }\n }\n\n return { removed }\n }\n\n async function close(): Promise<void> {\n closing = true\n if (abandonedSweepTimer) {\n clearInterval(abandonedSweepTimer)\n abandonedSweepTimer = null\n }\n if (bullWorker) {\n await bullWorker.close()\n bullWorker = null\n }\n // Drain any abandonment report still in flight, so a deploy-time shutdown cannot cut off the very\n // repair the callback exists to perform. Bounded: the hook writes to a database, and a shutdown\n // during an incident is exactly when that write can hang instead of failing. Giving up costs\n // nothing permanent \u2014 an unacknowledged report is re-delivered by the next start-up sweep.\n if (pendingAbandonedReports.size) {\n const drained = Promise.all([...pendingAbandonedReports]).then(() => true)\n const expired = new Promise<boolean>((resolve) => {\n const timer = setTimeout(() => resolve(false), ABANDONED_JOB_DRAIN_TIMEOUT_MS)\n ;(timer as unknown as { unref?: () => void }).unref?.()\n })\n if (!(await Promise.race([drained, expired]))) {\n logger.warn('Abandoned-job reports still in flight at shutdown; the sweep will retry them', {\n pending: pendingAbandonedReports.size,\n })\n }\n }\n if (bullQueue) {\n await bullQueue.close()\n bullQueue = null\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const queue = await getQueue()\n const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')\n return {\n waiting: counts.waiting ?? 0,\n active: counts.active ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'async',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,oBAAoB,eAAe,2BAA2B;AAEvE,SAAS,2BAA2B;AACpC,SAAS,qBAAqB,qBAAqB;AACnD,SAAS,oBAAoB;AAE7B,MAAM,gBAAgB,aAAa,OAAO;AA8D1C,MAAM,uBAAuB,CAAC,WAAW,WAAW,eAAe,UAAU,kBAAkB;AAiBxF,MAAM,wBAAwB;AAAA,EACnC;AAAA,EACA;AACF;AAEA,SAAS,qBAAqB,SAA0B;AACtD,SAAQ,sBAA4C,SAAS,OAAO;AACtE;AAUA,MAAM,yBAAyB;AAUxB,MAAM,kCAAkC,IAAI,KAAK;AAWjD,MAAM,iCAAiC;AAE9C,SAAS,yBAAiC;AACxC,QAAM,aAAa,OAAO,SAAS,QAAQ,IAAI,qCAAqC,IAAI,EAAE;AAC1F,SAAO,OAAO,SAAS,UAAU,KAAK,aAAa,IAAI,aAAa;AACtE;AAEA,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AASA,SAAS,kBAAkB,SAA8D;AACvF,MAAI,SAAS,KAAK;AAChB,WAAO,cAAc,QAAQ,GAAG;AAAA,EAClC;AAEA,MAAI,SAAS,MAAM;AACjB,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ;AAAA,MACtB,UAAU,QAAQ;AAAA,MAClB,UAAU,QAAQ;AAAA,MAClB,IAAI,QAAQ;AAAA,MACZ,KAAK,QAAQ;AAAA,MACb,QAAQ,QAAQ;AAAA,MAChB,UAAU;AAAA,IACZ;AAAA,EACF;AAEA,SAAO,cAAc,mBAAmB,OAAO,CAAC;AAClD;AAeO,SAAS,iBACd,MACA,SACU;AACV,QAAM,aAAa,kBAAkB,SAAS,UAAU;AACxD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,eAAe,SAAS;AAC9B,QAAM,kBAAkB,SAAS;AACjC,QAAM,iBAAiB,SAAS;AAChC,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;
|
|
4
|
+
"sourcesContent": ["import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { getRedisUrlOrThrow, parseRedisUrl, REDIS_WIRE_PROTOCOL } from '@open-mercato/shared/lib/redis/connection'\nimport type { RedisProtocolVersion } from '@open-mercato/shared/lib/redis/connection'\nimport { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst packageLogger = createLogger('queue')\n\n// BullMQ interface types - we define the shape we use to maintain type safety\n// while keeping bullmq as an optional peer dependency\ntype ConnectionOptions = {\n host?: string\n port?: number\n username?: string\n password?: string\n db?: number\n tls?: Record<string, unknown>\n family?: number\n protocol?: RedisProtocolVersion\n}\n\ninterface BullQueueInterface<T> {\n add: (\n name: string,\n data: T,\n opts?: {\n removeOnComplete?: boolean\n removeOnFail?: number\n delay?: number\n attempts?: number\n backoff?: { type: string; delay: number }\n },\n ) => Promise<{ id?: string }>\n obliterate: (opts?: { force?: boolean }) => Promise<void>\n close: () => Promise<void>\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n getJobs: (types: string[], start?: number, end?: number) => Promise<Array<{\n id?: string\n data?: T\n failedReason?: string\n remove: () => Promise<void>\n updateData?: (data: T) => Promise<void>\n }>>\n}\n\ninterface BullWorkerInterface {\n on: (event: string, handler: (...args: unknown[]) => void) => void\n close: () => Promise<void>\n}\n\ninterface BullMQModule {\n Queue: new <T>(name: string, opts: { connection: ConnectionOptions; telemetry?: unknown }) => BullQueueInterface<T>\n Worker: new <T>(\n name: string,\n processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,\n opts: {\n connection: ConnectionOptions\n concurrency: number\n telemetry?: unknown\n lockDuration?: number\n maxStalledCount?: number\n }\n ) => BullWorkerInterface\n}\n\n/** The `bullmq-otel` package (optional). Loaded only when an OTLP backend is active. */\ntype BullMQOtelModule = { BullMQOtel: new (tracerName: string) => object }\n\nconst REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']\n\n/**\n * The failures BullMQ records when it gives up on a job *before* handing it to the processor.\n *\n * Both are written as a `defa` (deferred failure) marker on the job, after which the next worker\n * short-circuits in `Worker.processJob` via `getUnrecoverableErrorMessage` and fails the job without\n * calling the handler. The first comes from the stalled-job script once a job's cumulative stall\n * count passes `maxStalledCount`; the second from `maxStartedAttempts`.\n *\n * Matching the reason is what tells \"the queue abandoned this\" from \"the handler ran and threw\", and\n * it is deliberately stateless: the alternative \u2014 tracking which jobs this process has entered \u2014 can\n * only answer \"did the handler run *here*\", which is the wrong question the moment more than one\n * worker is running. `bullmq-abandoned-reasons.test.ts` asserts these strings still exist in the\n * installed BullMQ, so an upgrade that renames them fails loudly instead of silently disabling the\n * callback.\n */\nexport const ABANDONED_JOB_REASONS = [\n 'job stalled more than allowable limit',\n 'job started more than allowable limit',\n] as const\n\nfunction isAbandonedJobReason(message: string): boolean {\n return (ABANDONED_JOB_REASONS as readonly string[]).includes(message)\n}\n\n/**\n * Metadata key written onto the stored job once `onJobAbandoned` has completed for it.\n *\n * BullMQ's failed set is the durable record of abandoned jobs (`removeOnFail` keeps them), so it\n * doubles as the dead-letter queue for reports: the sweep re-delivers any abandoned job that does not\n * carry this marker. The marker \u2014 not `job.remove()` \u2014 is the acknowledgement, so the failed job\n * itself survives for diagnosis.\n */\nconst ABANDON_REPORT_ACK_KEY = 'abandonReportedAt'\n// NOTE for anyone adding a retry action: the marker lives inside the job's own payload envelope, so a\n// job retried from admin tooling carries it into its next life and a second abandonment of that job\n// would never be reported. A retry path must clear `metadata.abandonReportedAt` when it re-enqueues.\n\n/**\n * How often a worker re-sweeps the failed set for unacknowledged abandoned jobs.\n *\n * Override with `QUEUE_ABANDONED_SWEEP_INTERVAL_MS` to trade recovery latency against Redis chatter.\n */\nexport const ABANDONED_JOB_SWEEP_INTERVAL_MS = 5 * 60 * 1000\n\n/**\n * How long `close()` waits for in-flight reports before giving up on them.\n *\n * Bounded on purpose: the hook reaches a database, and a shutdown during an infrastructure incident\n * is exactly when that write can hang rather than fail. An unbounded wait would turn a graceful\n * shutdown into a SIGKILL and skip the telemetry flush that follows it. Abandoning the wait is safe\n * because delivery is at-least-once \u2014 an unacknowledged report is re-delivered by the next worker's\n * start-up sweep, the same path that covers a process which died mid-report.\n */\nexport const ABANDONED_JOB_DRAIN_TIMEOUT_MS = 5000\n\nfunction resolveSweepIntervalMs(): number {\n const configured = Number.parseInt(process.env.QUEUE_ABANDONED_SWEEP_INTERVAL_MS ?? '', 10)\n return Number.isFinite(configured) && configured > 0 ? configured : ABANDONED_JOB_SWEEP_INTERVAL_MS\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/**\n * Resolves Redis connection options from various sources.\n *\n * BullMQ expects ioredis connection fields rather than a nested URL string.\n * Parse URL-based configuration at this boundary while keeping the public\n * queue API compatible with existing `{ url }` callers.\n */\nfunction resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {\n if (options?.url) {\n return parseRedisUrl(options.url)\n }\n\n if (options?.host) {\n return {\n host: options.host,\n port: options.port ?? 6379,\n username: options.username,\n password: options.password,\n db: options.db,\n tls: options.tls,\n family: options.family,\n protocol: REDIS_WIRE_PROTOCOL,\n }\n }\n\n return parseRedisUrl(getRedisUrlOrThrow('QUEUE'))\n}\n\n/**\n * Creates a BullMQ-based async queue.\n *\n * This strategy provides:\n * - Persistent job storage in Redis\n * - Automatic retries with exponential backoff\n * - Concurrent job processing\n * - Job prioritization and scheduling\n *\n * @template T - The payload type for jobs\n * @param name - Queue name\n * @param options - Async queue options\n */\nexport function createAsyncQueue<T = unknown>(\n name: string,\n options?: AsyncQueueOptions\n): Queue<T> {\n const connection = resolveConnection(options?.connection)\n const concurrency = options?.concurrency ?? 1\n const attempts = options?.attempts ?? 3\n const lockDuration = options?.lockDuration\n const maxStalledCount = options?.maxStalledCount\n const onJobAbandoned = options?.onJobAbandoned\n const logger = packageLogger.child({ queue: name })\n\n /**\n * Report a queue-level failure outward. Wrapped so a telemetry fault can never\n * escape into an EventEmitter handler, where an unhandled rejection is fatal to\n * the process.\n */\n function reportQueueError(\n error: unknown,\n code: string,\n attributes?: Record<string, string | number | undefined>,\n ): void {\n try {\n getTelemetryRuntime()?.reportError(error, {\n module: 'queue',\n code,\n attributes: { queue: name, ...attributes },\n })\n } catch (telemetryError) {\n // Reporting is never worth a worker \u2014 but a systematically broken bridge\n // must not be silent either, or a worker stops reporting and nothing says so.\n logger.warn('Failed to report a queue error to telemetry', { code, err: telemetryError as Error })\n }\n }\n\n let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null\n let bullWorker: BullWorkerInterface | null = null\n let bullmqModule: BullMQModule | null = null\n let abandonedSweepTimer: ReturnType<typeof setInterval> | null = null\n let closing = false\n\n // In-flight `onJobAbandoned` calls. Detached from the caller that started them (the 'failed'\n // listener or the sweep), so `close()` drains them rather than letting a deploy truncate a repair\n // mid-write. The id set stops the two callers from double-reporting a job inside one process.\n const pendingAbandonedReports = new Set<Promise<void>>()\n const inFlightAbandonedJobIds = new Set<string>()\n\n type AbandonedJobRecord = {\n id?: string\n data?: QueuedJob<T>\n updateData?: (data: QueuedJob<T>) => Promise<void>\n }\n\n /** What BullMQ hands the `failed` handler, beyond the abandonment fields. */\n type FailedJobRecord = AbandonedJobRecord & {\n attemptsMade?: number\n opts?: { attempts?: number }\n }\n\n // The acknowledgement that makes delivery at-least-once: written only after the callback returns,\n // so a callback that threw or a process that died mid-report leaves the job unmarked and a later\n // sweep retries it. Requires the driver to expose `updateData`; without it the report simply stays\n // unacknowledged and repeats, which the idempotency contract permits.\n async function acknowledgeAbandonedReport(job: AbandonedJobRecord): Promise<void> {\n if (!job.data || typeof job.updateData !== 'function') return\n await job.updateData({\n ...job.data,\n metadata: { ...(job.data.metadata ?? {}), [ABANDON_REPORT_ACK_KEY]: new Date().toISOString() },\n })\n }\n\n function reportAbandonedJob(job: AbandonedJobRecord, reason: string): Promise<void> | null {\n if (!onJobAbandoned) return null\n const payload = job.data\n if (!payload) return null\n if (payload.metadata && payload.metadata[ABANDON_REPORT_ACK_KEY]) return null\n if (inFlightAbandonedJobIds.has(payload.id)) return null\n inFlightAbandonedJobIds.add(payload.id)\n\n const jobId = job.id ?? null\n logger.warn('Job abandoned by the queue without running its handler', { jobId, reason })\n const report = (async () => {\n try {\n await onJobAbandoned(payload, { jobId, reason })\n } catch (hookError) {\n logger.error('onJobAbandoned handler threw; the report stays unacknowledged and the sweep will retry it', {\n jobId,\n err: hookError as Error,\n })\n reportQueueError(hookError as Error, 'queue.abandon_report_failed', { jobId: jobId ?? undefined })\n return\n }\n try {\n await acknowledgeAbandonedReport(job)\n } catch (ackError) {\n logger.error('Failed to acknowledge an abandoned-job report; the sweep may repeat it', {\n jobId,\n err: ackError as Error,\n })\n reportQueueError(ackError as Error, 'queue.abandon_ack_failed', { jobId: jobId ?? undefined })\n }\n })().finally(() => {\n inFlightAbandonedJobIds.delete(payload.id)\n })\n pendingAbandonedReports.add(report)\n void report.then(() => pendingAbandonedReports.delete(report))\n return report\n }\n\n // The failed set is this strategy's dead-letter queue for abandoned jobs. Enumerating it on worker\n // start and on an interval, and re-delivering anything unacknowledged, is what upgrades the\n // 'failed'-listener fast path from at-most-once to at-least-once: a report lost to a crash is\n // simply still unmarked when the next sweep looks. Residual loss: `removeOnFail` caps the set, so\n // a job evicted before any sweep sees it is gone for good.\n async function sweepAbandonedJobs(): Promise<void> {\n if (!onJobAbandoned || closing) return\n try {\n const queue = await getQueue()\n const failedJobs = await queue.getJobs(['failed'], 0, -1)\n // Re-checked after the awaits: a sweep already past its guard when `close()` ran would\n // otherwise start a report the drain has stopped waiting for. The next start-up sweep\n // re-delivers it, so stopping here loses nothing.\n if (closing) return\n for (const failedJob of failedJobs) {\n // Re-checked every iteration for the same reason: a long fan-out must not outlive the drain.\n if (closing) return\n const reason = failedJob.failedReason ?? ''\n if (!isAbandonedJobReason(reason)) continue\n // Awaited one at a time. Each report opens a request container and writes to the database, and\n // the worst case for this loop is the first worker start after the feature ships, on a\n // deployment that has been accumulating abandoned jobs \u2014 the largest backlog, on the process\n // least able to absorb it. The sweep is a recovery path with no latency requirement (five\n // minutes late is its normal mode), so pacing costs nothing worth having.\n await reportAbandonedJob(failedJob, reason)\n }\n } catch (sweepError) {\n logger.error('Abandoned-job sweep failed', { err: sweepError as Error })\n reportQueueError(sweepError as Error, 'queue.abandon_sweep_failed')\n }\n }\n // Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or\n // undefined (use our own metadata._trace carrier instead). Memoized as the\n // in-flight promise so concurrent first-time callers share one resolution.\n let telemetryPromise: Promise<object | undefined> | null = null\n\n // -------------------------------------------------------------------------\n // Lazy BullMQ initialization\n // -------------------------------------------------------------------------\n\n async function getBullMQ(): Promise<BullMQModule> {\n if (!bullmqModule) {\n try {\n bullmqModule = await import('bullmq') as unknown as BullMQModule\n } catch {\n throw new Error(\n 'BullMQ is required for async queue strategy. Install it with: npm install bullmq'\n )\n }\n }\n return bullmqModule\n }\n\n /**\n * When an OTLP backend is active, delegate async-queue tracing to `bullmq-otel`\n * (richer BullMQ-internal spans: add / process / wait / attempts). Returns\n * `undefined` \u2014 meaning \"use our own `metadata._trace` carrier\" \u2014 when telemetry\n * is off, a non-OTEL backend is selected, or `bullmq-otel` isn't installed. (The\n * `local` strategy always uses our carrier; it isn't BullMQ, so `bullmq-otel`\n * cannot instrument it.)\n */\n async function getQueueTelemetry(): Promise<object | undefined> {\n if (!telemetryPromise) {\n telemetryPromise = (async () => {\n if (!getTelemetryRuntime()?.canUseGlobalTracePropagation()) return undefined\n try {\n const mod = (await import('bullmq-otel')) as unknown as BullMQOtelModule\n return new mod.BullMQOtel('open-mercato')\n } catch {\n packageLogger.warn('bullmq-otel not available; using built-in trace carrier', { queue: name })\n return undefined\n }\n })()\n }\n return telemetryPromise\n }\n\n async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {\n if (!bullQueue) {\n const { Queue: BullQueueClass } = await getBullMQ()\n const telemetry = await getQueueTelemetry()\n bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection, ...(telemetry ? { telemetry } : {}) })\n }\n return bullQueue\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const queue = await getQueue()\n // When bullmq-otel handles propagation, don't also attach our carrier.\n const telemetry = await getQueueTelemetry()\n const metadata = telemetry ? undefined : attachTraceMetadata(undefined)\n const jobData: QueuedJob<T> = {\n id: crypto.randomUUID(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(metadata ? { metadata } : {}),\n }\n\n const job = await queue.add(jobData.id, jobData, {\n delay: options?.delayMs && options.delayMs > 0 ? options.delayMs : undefined,\n removeOnComplete: true,\n removeOnFail: 1000,\n attempts,\n backoff: { type: 'exponential', delay: 1000 },\n })\n\n return job.id ?? jobData.id\n }\n\n async function process(handler: JobHandler<T>): Promise<ProcessResult> {\n const { Worker } = await getBullMQ()\n const telemetry = await getQueueTelemetry()\n\n // Create worker that processes jobs\n bullWorker = new Worker<QueuedJob<T>>(\n name,\n async (job) => {\n const jobData = job.data\n const ctx = {\n jobId: job.id ?? jobData.id,\n attemptNumber: job.attemptsMade + 1,\n queueName: name,\n }\n // With bullmq-otel active, BullMQ owns the process span and active\n // context (the handler's pg/undici spans nest under it). Otherwise\n // continue the trace from our own carrier.\n if (telemetry) {\n await handler(jobData, ctx)\n } else {\n await runJobInTrace(name, jobData.metadata, () => handler(jobData, ctx))\n }\n },\n {\n connection,\n concurrency,\n ...(telemetry ? { telemetry } : {}),\n ...(lockDuration !== undefined ? { lockDuration } : {}),\n ...(maxStalledCount !== undefined ? { maxStalledCount } : {}),\n }\n )\n\n // Set up event handlers\n bullWorker.on('completed', (job) => {\n const jobWithId = job as { id?: string }\n logger.info('Job completed', { jobId: jobWithId.id })\n })\n\n bullWorker.on('failed', (job, err) => {\n const failedJob = job as FailedJobRecord | undefined\n const error = err as Error\n // BullMQ counts `attemptsMade` on the job itself, so the last delivery is\n // identifiable here without any bookkeeping of our own. Its per-job\n // `opts.attempts` wins over the strategy default, because a caller may have\n // overridden it.\n const attemptNumber = failedJob?.attemptsMade ?? 0\n const maxAttempts = failedJob?.opts?.attempts ?? attempts\n const exhausted = attemptNumber >= maxAttempts\n logger.error('Job failed', { jobId: failedJob?.id, attemptNumber, maxAttempts, err: error })\n // A log line reaches the backend's LOGS signal; a job that died is an error\n // and belongs in the error signal too, with a code the backend can group on.\n // This is the only outward record for a handler that rethrows after doing its\n // own bookkeeping \u2014 a sync worker marking its run `failed`, for instance.\n //\n // One report per failure, coded by what the failure means: a job that has\n // burned every retry is dead-lettered, which is the condition an operator\n // pages on, and it must be distinguishable from a first attempt that BullMQ\n // will simply retry. Matches the local strategy exactly, so an alert written\n // against one holds for the other.\n reportQueueError(error, exhausted ? 'queue.job_exhausted' : 'queue.job_failed', {\n jobId: failedJob?.id,\n attemptNumber,\n })\n\n if (!onJobAbandoned) return\n // Any other reason means a handler ran and threw. That failure is the handler's own and it has\n // already had its chance to record it.\n if (!isAbandonedJobReason(error?.message ?? '')) return\n // No payload means the queue could not give us the job at all. There is nothing to hand the\n // callback and nothing it could repair, so reporting could only ever be a false alarm \u2014 the\n // 'Job failed' line above still records it.\n if (!failedJob?.data) return\n\n // The fast path: report the moment the abandonment is observed. `reportAbandonedJob` runs the\n // callback detached with its own try/catch \u2014 this is an EventEmitter, where an unhandled\n // rejection is fatal to the process \u2014 and acknowledges the job only afterwards, so a report\n // lost here is retried by the sweep.\n reportAbandonedJob(failedJob, error.message)\n })\n\n // A stalled job is redelivered under the same id while the previous worker\n // may still be running it, so this is the signal that a handler is about to\n // be executed twice. BullMQ's docs require surfacing it: without this line\n // duplicate processing is invisible.\n bullWorker.on('stalled', (jobId) => {\n logger.warn('Job stalled and will be redelivered \u2014 the handler may run concurrently with a previous delivery', {\n jobId: typeof jobId === 'string' ? jobId : null,\n })\n })\n\n bullWorker.on('error', (err) => {\n const error = err as Error\n logger.error('Worker error', { err: error })\n reportQueueError(error, 'queue.worker_error')\n })\n\n if (onJobAbandoned) {\n // Sweep immediately so reports lost to a previous process's crash are re-delivered as soon as\n // a worker is back, then keep re-sweeping for anything the fast path loses while running.\n // The timer is unref'd so it never holds the process open.\n void sweepAbandonedJobs()\n abandonedSweepTimer = setInterval(() => {\n void sweepAbandonedJobs()\n }, resolveSweepIntervalMs())\n ;(abandonedSweepTimer as unknown as { unref?: () => void }).unref?.()\n }\n\n logger.info('Worker started', { concurrency })\n\n // For async strategy, return a sentinel result indicating worker mode\n // processed=-1 signals that this is a continuous worker, not a batch process\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n const queue = await getQueue()\n\n // Obliterate removes all jobs from the queue\n await queue.obliterate({ force: true })\n\n return { removed: -1 } // BullMQ obliterate doesn't return count\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n const queue = await getQueue()\n const jobs = await queue.getJobs(REMOVABLE_JOB_STATES, 0, -1)\n let removed = 0\n\n for (const job of jobs) {\n if (!payloadMatchesScope(job.data?.payload, scope)) continue\n try {\n await job.remove()\n removed++\n } catch {\n // The job may have started between enumeration and removal. In-flight\n // cancellation is handled by the caller's lock/heartbeat contract.\n }\n }\n\n return { removed }\n }\n\n async function close(): Promise<void> {\n closing = true\n if (abandonedSweepTimer) {\n clearInterval(abandonedSweepTimer)\n abandonedSweepTimer = null\n }\n if (bullWorker) {\n await bullWorker.close()\n bullWorker = null\n }\n // Drain any abandonment report still in flight, so a deploy-time shutdown cannot cut off the very\n // repair the callback exists to perform. Bounded: the hook writes to a database, and a shutdown\n // during an incident is exactly when that write can hang instead of failing. Giving up costs\n // nothing permanent \u2014 an unacknowledged report is re-delivered by the next start-up sweep.\n if (pendingAbandonedReports.size) {\n const drained = Promise.all([...pendingAbandonedReports]).then(() => true)\n const expired = new Promise<boolean>((resolve) => {\n const timer = setTimeout(() => resolve(false), ABANDONED_JOB_DRAIN_TIMEOUT_MS)\n ;(timer as unknown as { unref?: () => void }).unref?.()\n })\n if (!(await Promise.race([drained, expired]))) {\n logger.warn('Abandoned-job reports still in flight at shutdown; the sweep will retry them', {\n pending: pendingAbandonedReports.size,\n })\n }\n }\n if (bullQueue) {\n await bullQueue.close()\n bullQueue = null\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n const queue = await getQueue()\n const counts = await queue.getJobCounts('waiting', 'active', 'completed', 'failed')\n return {\n waiting: counts.waiting ?? 0,\n active: counts.active ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n }\n }\n\n return {\n name,\n strategy: 'async',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,oBAAoB,eAAe,2BAA2B;AAEvE,SAAS,2BAA2B;AACpC,SAAS,qBAAqB,qBAAqB;AACnD,SAAS,oBAAoB;AAE7B,MAAM,gBAAgB,aAAa,OAAO;AA8D1C,MAAM,uBAAuB,CAAC,WAAW,WAAW,eAAe,UAAU,kBAAkB;AAiBxF,MAAM,wBAAwB;AAAA,EACnC;AAAA,EACA;AACF;AAEA,SAAS,qBAAqB,SAA0B;AACtD,SAAQ,sBAA4C,SAAS,OAAO;AACtE;AAUA,MAAM,yBAAyB;AAUxB,MAAM,kCAAkC,IAAI,KAAK;AAWjD,MAAM,iCAAiC;AAE9C,SAAS,yBAAiC;AACxC,QAAM,aAAa,OAAO,SAAS,QAAQ,IAAI,qCAAqC,IAAI,EAAE;AAC1F,SAAO,OAAO,SAAS,UAAU,KAAK,aAAa,IAAI,aAAa;AACtE;AAEA,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AASA,SAAS,kBAAkB,SAA8D;AACvF,MAAI,SAAS,KAAK;AAChB,WAAO,cAAc,QAAQ,GAAG;AAAA,EAClC;AAEA,MAAI,SAAS,MAAM;AACjB,WAAO;AAAA,MACL,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ,QAAQ;AAAA,MACtB,UAAU,QAAQ;AAAA,MAClB,UAAU,QAAQ;AAAA,MAClB,IAAI,QAAQ;AAAA,MACZ,KAAK,QAAQ;AAAA,MACb,QAAQ,QAAQ;AAAA,MAChB,UAAU;AAAA,IACZ;AAAA,EACF;AAEA,SAAO,cAAc,mBAAmB,OAAO,CAAC;AAClD;AAeO,SAAS,iBACd,MACA,SACU;AACV,QAAM,aAAa,kBAAkB,SAAS,UAAU;AACxD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,eAAe,SAAS;AAC9B,QAAM,kBAAkB,SAAS;AACjC,QAAM,iBAAiB,SAAS;AAChC,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAOlD,WAAS,iBACP,OACA,MACA,YACM;AACN,QAAI;AACF,0BAAoB,GAAG,YAAY,OAAO;AAAA,QACxC,QAAQ;AAAA,QACR;AAAA,QACA,YAAY,EAAE,OAAO,MAAM,GAAG,WAAW;AAAA,MAC3C,CAAC;AAAA,IACH,SAAS,gBAAgB;AAGvB,aAAO,KAAK,+CAA+C,EAAE,MAAM,KAAK,eAAwB,CAAC;AAAA,IACnG;AAAA,EACF;AAEA,MAAI,YAAqD;AACzD,MAAI,aAAyC;AAC7C,MAAI,eAAoC;AACxC,MAAI,sBAA6D;AACjE,MAAI,UAAU;AAKd,QAAM,0BAA0B,oBAAI,IAAmB;AACvD,QAAM,0BAA0B,oBAAI,IAAY;AAkBhD,iBAAe,2BAA2B,KAAwC;AAChF,QAAI,CAAC,IAAI,QAAQ,OAAO,IAAI,eAAe,WAAY;AACvD,UAAM,IAAI,WAAW;AAAA,MACnB,GAAG,IAAI;AAAA,MACP,UAAU,EAAE,GAAI,IAAI,KAAK,YAAY,CAAC,GAAI,CAAC,sBAAsB,IAAG,oBAAI,KAAK,GAAE,YAAY,EAAE;AAAA,IAC/F,CAAC;AAAA,EACH;AAEA,WAAS,mBAAmB,KAAyB,QAAsC;AACzF,QAAI,CAAC,eAAgB,QAAO;AAC5B,UAAM,UAAU,IAAI;AACpB,QAAI,CAAC,QAAS,QAAO;AACrB,QAAI,QAAQ,YAAY,QAAQ,SAAS,sBAAsB,EAAG,QAAO;AACzE,QAAI,wBAAwB,IAAI,QAAQ,EAAE,EAAG,QAAO;AACpD,4BAAwB,IAAI,QAAQ,EAAE;AAEtC,UAAM,QAAQ,IAAI,MAAM;AACxB,WAAO,KAAK,0DAA0D,EAAE,OAAO,OAAO,CAAC;AACvF,UAAM,UAAU,YAAY;AAC1B,UAAI;AACF,cAAM,eAAe,SAAS,EAAE,OAAO,OAAO,CAAC;AAAA,MACjD,SAAS,WAAW;AAClB,eAAO,MAAM,6FAA6F;AAAA,UACxG;AAAA,UACA,KAAK;AAAA,QACP,CAAC;AACD,yBAAiB,WAAoB,+BAA+B,EAAE,OAAO,SAAS,OAAU,CAAC;AACjG;AAAA,MACF;AACA,UAAI;AACF,cAAM,2BAA2B,GAAG;AAAA,MACtC,SAAS,UAAU;AACjB,eAAO,MAAM,0EAA0E;AAAA,UACrF;AAAA,UACA,KAAK;AAAA,QACP,CAAC;AACD,yBAAiB,UAAmB,4BAA4B,EAAE,OAAO,SAAS,OAAU,CAAC;AAAA,MAC/F;AAAA,IACF,GAAG,EAAE,QAAQ,MAAM;AACjB,8BAAwB,OAAO,QAAQ,EAAE;AAAA,IAC3C,CAAC;AACD,4BAAwB,IAAI,MAAM;AAClC,SAAK,OAAO,KAAK,MAAM,wBAAwB,OAAO,MAAM,CAAC;AAC7D,WAAO;AAAA,EACT;AAOA,iBAAe,qBAAoC;AACjD,QAAI,CAAC,kBAAkB,QAAS;AAChC,QAAI;AACF,YAAM,QAAQ,MAAM,SAAS;AAC7B,YAAM,aAAa,MAAM,MAAM,QAAQ,CAAC,QAAQ,GAAG,GAAG,EAAE;AAIxD,UAAI,QAAS;AACb,iBAAW,aAAa,YAAY;AAElC,YAAI,QAAS;AACb,cAAM,SAAS,UAAU,gBAAgB;AACzC,YAAI,CAAC,qBAAqB,MAAM,EAAG;AAMnC,cAAM,mBAAmB,WAAW,MAAM;AAAA,MAC5C;AAAA,IACF,SAAS,YAAY;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAoB,CAAC;AACvE,uBAAiB,YAAqB,4BAA4B;AAAA,IACpE;AAAA,EACF;AAIA,MAAI,mBAAuD;AAM3D,iBAAe,YAAmC;AAChD,QAAI,CAAC,cAAc;AACjB,UAAI;AACF,uBAAe,MAAM,OAAO,QAAQ;AAAA,MACtC,QAAQ;AACN,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAUA,iBAAe,oBAAiD;AAC9D,QAAI,CAAC,kBAAkB;AACrB,0BAAoB,YAAY;AAC9B,YAAI,CAAC,oBAAoB,GAAG,6BAA6B,EAAG,QAAO;AACnE,YAAI;AACF,gBAAM,MAAO,MAAM,OAAO,aAAa;AACvC,iBAAO,IAAI,IAAI,WAAW,cAAc;AAAA,QAC1C,QAAQ;AACN,wBAAc,KAAK,2DAA2D,EAAE,OAAO,KAAK,CAAC;AAC7F,iBAAO;AAAA,QACT;AAAA,MACF,GAAG;AAAA,IACL;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,WAAsD;AACnE,QAAI,CAAC,WAAW;AACd,YAAM,EAAE,OAAO,eAAe,IAAI,MAAM,UAAU;AAClD,YAAM,YAAY,MAAM,kBAAkB;AAC1C,kBAAY,IAAI,eAA6B,MAAM,EAAE,YAAY,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC,EAAG,CAAC;AAAA,IACxG;AACA,WAAO;AAAA,EACT;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,QAAQ,MAAM,SAAS;AAE7B,UAAM,YAAY,MAAM,kBAAkB;AAC1C,UAAM,WAAW,YAAY,SAAY,oBAAoB,MAAS;AACtE,UAAM,UAAwB;AAAA,MAC5B,IAAI,OAAO,WAAW;AAAA,MACtB,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAEA,UAAM,MAAM,MAAM,MAAM,IAAI,QAAQ,IAAI,SAAS;AAAA,MAC/C,OAAOA,UAAS,WAAWA,SAAQ,UAAU,IAAIA,SAAQ,UAAU;AAAA,MACnE,kBAAkB;AAAA,MAClB,cAAc;AAAA,MACd;AAAA,MACA,SAAS,EAAE,MAAM,eAAe,OAAO,IAAK;AAAA,IAC9C,CAAC;AAED,WAAO,IAAI,MAAM,QAAQ;AAAA,EAC3B;AAEA,iBAAeC,SAAQ,SAAgD;AACrE,UAAM,EAAE,OAAO,IAAI,MAAM,UAAU;AACnC,UAAM,YAAY,MAAM,kBAAkB;AAG1C,iBAAa,IAAI;AAAA,MACf;AAAA,MACA,OAAO,QAAQ;AACb,cAAM,UAAU,IAAI;AACpB,cAAM,MAAM;AAAA,UACV,OAAO,IAAI,MAAM,QAAQ;AAAA,UACzB,eAAe,IAAI,eAAe;AAAA,UAClC,WAAW;AAAA,QACb;AAIA,YAAI,WAAW;AACb,gBAAM,QAAQ,SAAS,GAAG;AAAA,QAC5B,OAAO;AACL,gBAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,QAAQ,SAAS,GAAG,CAAC;AAAA,QACzE;AAAA,MACF;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC;AAAA,QACjC,GAAI,iBAAiB,SAAY,EAAE,aAAa,IAAI,CAAC;AAAA,QACrD,GAAI,oBAAoB,SAAY,EAAE,gBAAgB,IAAI,CAAC;AAAA,MAC7D;AAAA,IACF;AAGA,eAAW,GAAG,aAAa,CAAC,QAAQ;AAClC,YAAM,YAAY;AAClB,aAAO,KAAK,iBAAiB,EAAE,OAAO,UAAU,GAAG,CAAC;AAAA,IACtD,CAAC;AAED,eAAW,GAAG,UAAU,CAAC,KAAK,QAAQ;AACpC,YAAM,YAAY;AAClB,YAAM,QAAQ;AAKd,YAAM,gBAAgB,WAAW,gBAAgB;AACjD,YAAM,cAAc,WAAW,MAAM,YAAY;AACjD,YAAM,YAAY,iBAAiB;AACnC,aAAO,MAAM,cAAc,EAAE,OAAO,WAAW,IAAI,eAAe,aAAa,KAAK,MAAM,CAAC;AAW3F,uBAAiB,OAAO,YAAY,wBAAwB,oBAAoB;AAAA,QAC9E,OAAO,WAAW;AAAA,QAClB;AAAA,MACF,CAAC;AAED,UAAI,CAAC,eAAgB;AAGrB,UAAI,CAAC,qBAAqB,OAAO,WAAW,EAAE,EAAG;AAIjD,UAAI,CAAC,WAAW,KAAM;AAMtB,yBAAmB,WAAW,MAAM,OAAO;AAAA,IAC7C,CAAC;AAMD,eAAW,GAAG,WAAW,CAAC,UAAU;AAClC,aAAO,KAAK,wGAAmG;AAAA,QAC7G,OAAO,OAAO,UAAU,WAAW,QAAQ;AAAA,MAC7C,CAAC;AAAA,IACH,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,QAAQ;AAC9B,YAAM,QAAQ;AACd,aAAO,MAAM,gBAAgB,EAAE,KAAK,MAAM,CAAC;AAC3C,uBAAiB,OAAO,oBAAoB;AAAA,IAC9C,CAAC;AAED,QAAI,gBAAgB;AAIlB,WAAK,mBAAmB;AACxB,4BAAsB,YAAY,MAAM;AACtC,aAAK,mBAAmB;AAAA,MAC1B,GAAG,uBAAuB,CAAC;AAC1B,MAAC,oBAA0D,QAAQ;AAAA,IACtE;AAEA,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAI7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,UAAM,QAAQ,MAAM,SAAS;AAG7B,UAAM,MAAM,WAAW,EAAE,OAAO,KAAK,CAAC;AAEtC,WAAO,EAAE,SAAS,GAAG;AAAA,EACvB;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,OAAO,MAAM,MAAM,QAAQ,sBAAsB,GAAG,EAAE;AAC5D,QAAI,UAAU;AAEd,eAAW,OAAO,MAAM;AACtB,UAAI,CAAC,oBAAoB,IAAI,MAAM,SAAS,KAAK,EAAG;AACpD,UAAI;AACF,cAAM,IAAI,OAAO;AACjB;AAAA,MACF,QAAQ;AAAA,MAGR;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ;AAAA,EACnB;AAEA,iBAAe,QAAuB;AACpC,cAAU;AACV,QAAI,qBAAqB;AACvB,oBAAc,mBAAmB;AACjC,4BAAsB;AAAA,IACxB;AACA,QAAI,YAAY;AACd,YAAM,WAAW,MAAM;AACvB,mBAAa;AAAA,IACf;AAKA,QAAI,wBAAwB,MAAM;AAChC,YAAM,UAAU,QAAQ,IAAI,CAAC,GAAG,uBAAuB,CAAC,EAAE,KAAK,MAAM,IAAI;AACzE,YAAM,UAAU,IAAI,QAAiB,CAAC,YAAY;AAChD,cAAM,QAAQ,WAAW,MAAM,QAAQ,KAAK,GAAG,8BAA8B;AAC5E,QAAC,MAA4C,QAAQ;AAAA,MACxD,CAAC;AACD,UAAI,CAAE,MAAM,QAAQ,KAAK,CAAC,SAAS,OAAO,CAAC,GAAI;AAC7C,eAAO,KAAK,gFAAgF;AAAA,UAC1F,SAAS,wBAAwB;AAAA,QACnC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,WAAW;AACb,YAAM,UAAU,MAAM;AACtB,kBAAY;AAAA,IACd;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,SAAS,MAAM,MAAM,aAAa,WAAW,UAAU,aAAa,QAAQ;AAClF,WAAO;AAAA,MACL,SAAS,OAAO,WAAW;AAAA,MAC3B,QAAQ,OAAO,UAAU;AAAA,MACzB,WAAW,OAAO,aAAa;AAAA,MAC/B,QAAQ,OAAO,UAAU;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA,SAAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options", "process"]
|
|
7
7
|
}
|
package/dist/strategies/local.js
CHANGED
|
@@ -2,6 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import crypto from "node:crypto";
|
|
4
4
|
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
5
|
+
import { getTelemetryRuntime } from "@open-mercato/shared/lib/telemetry/runtime";
|
|
5
6
|
import { attachTraceMetadata, runJobInTrace } from "../tracing.js";
|
|
6
7
|
const packageLogger = createLogger("queue");
|
|
7
8
|
function payloadMatchesScope(payload, scope) {
|
|
@@ -38,6 +39,17 @@ function createLocalQueue(name, options) {
|
|
|
38
39
|
const lockDir = path.join(queueDir, "queue.lock");
|
|
39
40
|
const lockOwnerFile = path.join(lockDir, "owner");
|
|
40
41
|
const logger = packageLogger.child({ queue: name });
|
|
42
|
+
function reportQueueError(error, code, attributes) {
|
|
43
|
+
try {
|
|
44
|
+
getTelemetryRuntime()?.reportError(error, {
|
|
45
|
+
module: "queue",
|
|
46
|
+
code,
|
|
47
|
+
attributes: { queue: name, ...attributes }
|
|
48
|
+
});
|
|
49
|
+
} catch (telemetryError) {
|
|
50
|
+
logger.warn("Failed to report a queue error to telemetry", { code, err: telemetryError });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
41
53
|
const concurrency = options?.concurrency ?? 1;
|
|
42
54
|
const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL;
|
|
43
55
|
const fallbackPollInterval = Math.max(pollInterval, DEFAULT_FALLBACK_POLL_INTERVAL);
|
|
@@ -309,9 +321,14 @@ function createLocalQueue(name, options) {
|
|
|
309
321
|
logger.info("Job completed", { jobId: job.id });
|
|
310
322
|
} catch (error) {
|
|
311
323
|
logger.error("Job failed", { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error });
|
|
324
|
+
const exhausted = attemptNumber >= DEFAULT_MAX_ATTEMPTS;
|
|
325
|
+
reportQueueError(error, exhausted ? "queue.job_exhausted" : "queue.job_failed", {
|
|
326
|
+
jobId: job.id,
|
|
327
|
+
attemptNumber
|
|
328
|
+
});
|
|
312
329
|
failed++;
|
|
313
330
|
lastJobId = job.id;
|
|
314
|
-
if (
|
|
331
|
+
if (exhausted) {
|
|
315
332
|
logger.error("Job exhausted all attempts; dropping it (no dead-letter store)", { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS });
|
|
316
333
|
deadJobIds.add(job.id);
|
|
317
334
|
} else {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/strategies/local.ts"],
|
|
4
|
-
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\ntype QueueFileIdentity = {\n device: number\n inode: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Polling interval while delayed or retrying work remains queued. */\nconst DEFAULT_POLL_INTERVAL = 1000\n/** Idle safety interval for missed filesystem watcher events. */\nconst DEFAULT_FALLBACK_POLL_INTERVAL = 5000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\n/**\n * Cross-process lock tuning. A held lock only ever spans local file I/O \u2014 job\n * handlers run outside it \u2014 so realistic hold times are milliseconds and the\n * stale threshold sits orders of magnitude above them. It exists solely so a\n * process that dies mid-segment cannot wedge the queue forever. A holder that\n * was merely suspended rather than dead can still be reclaimed, which is why\n * every acquisition carries an owner token and releases only its own lock.\n */\nconst LOCK_STALE_MS = 15_000\nconst LOCK_ACQUIRE_TIMEOUT_MS = 30_000\nconst LOCK_RETRY_MIN_MS = 2\nconst LOCK_RETRY_MAX_MS = 20\nconst RENAME_MAX_RETRIES = 5\nconst RENAME_RETRY_BASE_MS = 10\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production: there is no dead-letter store, no throughput\n * beyond one job at a time, and every operation rewrites the whole queue file\n *\n * Multiple processes MAY share a queue directory, which is the default\n * development topology: the dev worker runs in its own process alongside the\n * Next.js server. What that buys you, and what it does not:\n *\n * - **Safe** \u2014 concurrent producers. Every read-modify-write segment takes the\n * `queue.lock` directory lock and every persist swaps the file in with an\n * atomic rename, so the file cannot be torn, no enqueue is lost to a\n * concurrent one, and a reader always observes one complete document.\n * Writers contend, though, so throughput degrades as processes are added.\n * - **NOT safe** \u2014 concurrent consumers. `process()` deliberately runs job\n * handlers outside the lock, so two worker processes polling the same queue\n * would both claim the same pending jobs and execute them twice. There is no\n * per-job lease. Run exactly one worker process per queue; use the `async`\n * strategy when you need more than one.\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.\n * **This strategy keeps no failed-job store**: once attempts are exhausted the job is\n * removed from `queue.json` and only counted in `state.failedCount`, so the payload is\n * lost and the failure survives solely as an error log line. The `async` strategy keeps\n * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000\n * failures and removes older ones as later failures arrive. Workflows that require\n * no-loss persistence must write their own durable record before enqueueing, regardless\n * of strategy.\n *\n * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option \u2014 callers cannot\n * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)\n * See the retry handling in `process()` below.\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences within one instance, and the `queue.lock`\n * directory lock extends that serialization across instances and processes.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const lockDir = path.join(queueDir, 'queue.lock')\n const lockOwnerFile = path.join(lockDir, 'owner')\n const logger = packageLogger.child({ queue: name })\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n const fallbackPollInterval = Math.max(pollInterval, DEFAULT_FALLBACK_POLL_INTERVAL)\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let queuedPollTimer: ReturnType<typeof setTimeout> | null = null\n let queueWatcher: fs.FSWatcher | null = null\n let queueWatcherIdentity: QueueFileIdentity | null = null\n let watcherRefreshChain: Promise<void> = Promise.resolve()\n let hasQueuedJobs = false\n let isProcessing = false\n let pollRequested = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes. It only covers this\n // instance, so it also guarantees at most one outstanding `queue.lock`\n // acquisition per instance \u2014 the directory lock below is not reentrant.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(\n () => runExclusively(fn),\n () => runExclusively(fn),\n )\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n /**\n * Runs `fn` while holding the cross-process `queue.lock`, so read-modify-write\n * segments issued by other queue instances \u2014 in this process or another one \u2014\n * cannot interleave with it.\n */\n async function runExclusively<R>(fn: () => Promise<R>): Promise<R> {\n await ensureDir()\n const release = await acquireDirectoryLock()\n try {\n return await fn()\n } finally {\n await release()\n }\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => { setTimeout(resolve, ms) })\n }\n\n async function lockHeldForMs(): Promise<number | null> {\n try {\n const stats = await fsp.stat(lockDir)\n return Date.now() - stats.mtimeMs\n } catch {\n return null\n }\n }\n\n /**\n * Reclaims a lock whose holder died. The rename is the serialization point:\n * only one racer can move `queue.lock` aside, so two processes cannot both\n * decide a stale lock is theirs to clear and then both create a fresh one.\n */\n async function reclaimStaleLock(heldForMs: number): Promise<void> {\n const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`\n try {\n await fsp.rename(lockDir, reclaimedPath)\n } catch {\n return\n }\n logger.warn('Reclaimed a stale queue lock', { lockDir, heldForMs })\n await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {})\n }\n\n async function readLockOwner(): Promise<string | null> {\n try {\n return await fsp.readFile(lockOwnerFile, 'utf8')\n } catch {\n return null\n }\n }\n\n /**\n * Releases the lock only when this acquisition still owns it. A holder that\n * was suspended past `LOCK_STALE_MS` has had its lock reclaimed *and\n * replaced* by whoever reclaimed it, so an unconditional removal here would\n * delete the successor's lock and let a third caller into the critical\n * section alongside it. A missing or mismatched token means someone else owns\n * the path now, and the correct action is to leave it alone.\n */\n async function releaseDirectoryLock(token: string): Promise<void> {\n if (await readLockOwner() !== token) return\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n }\n\n /**\n * Acquires the cross-process advisory lock for this queue directory.\n * `mkdir` without `recursive` is an atomic exclusive create on every platform\n * Node.js supports, which makes it the portable primitive here \u2014 no runtime\n * dependency, and no reliance on advisory `flock` semantics. The owner token\n * written into the directory is what lets the release distinguish this\n * acquisition from a successor's.\n */\n async function acquireDirectoryLock(): Promise<() => Promise<void>> {\n const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS\n\n for (;;) {\n let acquired = false\n try {\n await fsp.mkdir(lockDir)\n acquired = true\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n if (acquired) {\n const token = crypto.randomUUID()\n try {\n await fsp.writeFile(lockOwnerFile, token, 'utf8')\n } catch (error: unknown) {\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n throw error\n }\n return () => releaseDirectoryLock(token)\n }\n\n const heldForMs = await lockHeldForMs()\n if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {\n await reclaimStaleLock(heldForMs)\n continue\n }\n\n if (Date.now() >= deadline) {\n throw new Error(\n `[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`,\n )\n }\n\n const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)\n await sleep(jitter)\n }\n }\n\n /**\n * Persists `content` by writing a unique sibling temp file and renaming it\n * onto `targetFile`. `rename` within a directory is atomic, so a concurrent\n * reader sees either the previous document or the new one in full \u2014 never the\n * torn result of a truncate-then-write.\n */\n async function writeFileAtomic(targetFile: string, content: string): Promise<void> {\n const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`\n try {\n await fsp.writeFile(tempFile, content, 'utf8')\n await renameWithContentionRetry(tempFile, targetFile)\n } catch (error: unknown) {\n await fsp.rm(tempFile, { force: true }).catch(() => {})\n throw error\n }\n }\n\n /**\n * Windows rejects a rename onto a file another process currently has open,\n * so retry briefly on the contention codes it raises. POSIX renames replace\n * the target unconditionally and take the first attempt.\n */\n async function renameWithContentionRetry(fromFile: string, toFile: string): Promise<void> {\n const contentionCodes = new Set(['EPERM', 'EBUSY', 'EACCES'])\n for (let attempt = 0; ; attempt++) {\n try {\n await fsp.rename(fromFile, toFile)\n return\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error\n await sleep(RENAME_RETRY_BASE_MS * (attempt + 1))\n }\n }\n }\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n /**\n * Moves an unparsable queue file aside so its jobs stay recoverable. The\n * caller is expected to surface the failure rather than continue on an empty\n * queue: silently recreating `queue.json` here is what turned an unreadable\n * file into permanent, unreported job loss.\n */\n async function quarantineCorruptedQueueFile(): Promise<string | null> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`)\n try {\n await fsp.rename(queueFile, backupFile)\n return backupFile\n } catch (e: unknown) {\n logger.error('Failed to quarantine the corrupted queue file', { err: e as Error })\n return null\n }\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await quarantineCorruptedQueueFile()\n if (backupFile) {\n logger.error('Quarantined corrupted queue file; its jobs are recoverable from the backup', { backupFile })\n }\n const recoveryHint = backupFile\n ? `has been quarantined as ${backupFile}`\n : 'could not be quarantined and was left in place'\n throw new Error(\n `[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`,\n )\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2))\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await writeFileAtomic(stateFile, JSON.stringify(state, null, 2))\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const metadata = attachTraceMetadata(undefined)\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n ...(metadata ? { metadata } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n hasQueuedJobs = jobs.length > 0\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await runJobInTrace(name, job.metadata, () =>\n Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n failed++\n lastJobId = job.id\n if (attemptNumber >= DEFAULT_MAX_ATTEMPTS) {\n logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n hasQueuedJobs = updatedJobs.length > 0\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(rethrow = false): Promise<void> {\n if (!activeHandler) return\n if (isProcessing) {\n pollRequested = true\n return\n }\n\n isProcessing = true\n try {\n do {\n pollRequested = false\n const handler = activeHandler\n if (!handler) break\n await processBatch(handler)\n } while (pollRequested)\n } catch (error) {\n if (rethrow) throw error\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n scheduleQueuedPoll()\n }\n }\n\n function scheduleQueuedPoll(): void {\n if (!activeHandler || !hasQueuedJobs) {\n if (queuedPollTimer) {\n clearTimeout(queuedPollTimer)\n queuedPollTimer = null\n }\n return\n }\n if (queuedPollTimer) return\n queuedPollTimer = setTimeout(() => {\n queuedPollTimer = null\n void pollAndProcess()\n }, pollInterval)\n }\n\n function closeQueueWatcher(): void {\n if (queueWatcher) {\n queueWatcher.close()\n queueWatcher = null\n }\n queueWatcherIdentity = null\n }\n\n function refreshQueueWatcher(): Promise<void> {\n const refresh = watcherRefreshChain.then(async () => {\n if (!activeHandler) return\n try {\n await ensureDir()\n const stats = await fsp.stat(queueFile)\n const nextIdentity = { device: stats.dev, inode: stats.ino }\n if (\n queueWatcher\n && queueWatcherIdentity?.device === nextIdentity.device\n && queueWatcherIdentity.inode === nextIdentity.inode\n ) {\n return\n }\n\n closeQueueWatcher()\n const watcher = fs.watch(queueFile, (eventType) => {\n if (eventType === 'rename') {\n queueWatcherIdentity = null\n void refreshQueueWatcher()\n }\n void pollAndProcess()\n })\n watcher.on('error', (err) => {\n logger.error('Queue watch error; fallback polling remains active', { err })\n if (queueWatcher === watcher) {\n closeQueueWatcher()\n }\n })\n if (!activeHandler) {\n watcher.close()\n return\n }\n queueWatcher = watcher\n queueWatcherIdentity = nextIdentity\n } catch (err) {\n logger.error('Failed to watch queue file; fallback polling remains active', { err })\n }\n })\n watcherRefreshChain = refresh.catch(() => undefined)\n return refresh\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n if (activeHandler) {\n await close()\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n try {\n await refreshQueueWatcher()\n await pollAndProcess(true)\n } catch (error) {\n await close()\n throw error\n }\n\n pollingTimer = setInterval(() => {\n refreshQueueWatcher().then(() => pollAndProcess()).catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, fallbackPollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n hasQueuedJobs = false\n scheduleQueuedPoll()\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n hasQueuedJobs = retainedJobs.length > 0\n scheduleQueuedPoll()\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n activeHandler = null\n if (queuedPollTimer) {\n clearTimeout(queuedPollTimer)\n queuedPollTimer = null\n }\n closeQueueWatcher()\n\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAE7B,SAAS,qBAAqB,qBAAqB;AAEnD,MAAM,gBAAgB,aAAa,OAAO;AAkB1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAE9B,MAAM,iCAAiC;AACvC,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAU9B,MAAM,gBAAgB;AACtB,MAAM,0BAA0B;AAChC,MAAM,oBAAoB;AAC1B,MAAM,oBAAoB;AAC1B,MAAM,qBAAqB;AAC3B,MAAM,uBAAuB;AAE7B,MAAM,MAAM,GAAG;AAmDR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,UAAU,KAAK,KAAK,UAAU,YAAY;AAChD,QAAM,gBAAgB,KAAK,KAAK,SAAS,OAAO;AAChD,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAElD,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAC9C,QAAM,uBAAuB,KAAK,IAAI,cAAc,8BAA8B;AAGlF,MAAI,eAAsD;AAC1D,MAAI,kBAAwD;AAC5D,MAAI,eAAoC;AACxC,MAAI,uBAAiD;AACrD,MAAI,sBAAqC,QAAQ,QAAQ;AACzD,MAAI,gBAAgB;AACpB,MAAI,eAAe;AACnB,MAAI,gBAAgB;AACpB,MAAI,gBAAsC;AAC1C,QAAM,iBAAiB,oBAAI,IAAY;AAMvC,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY;AAAA,MACtB,MAAM,eAAe,EAAE;AAAA,MACvB,MAAM,eAAe,EAAE;AAAA,IACzB;AACA,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAOA,iBAAe,eAAkB,IAAkC;AACjE,UAAM,UAAU;AAChB,UAAM,UAAU,MAAM,qBAAqB;AAC3C,QAAI;AACF,aAAO,MAAM,GAAG;AAAA,IAClB,UAAE;AACA,YAAM,QAAQ;AAAA,IAChB;AAAA,EACF;AAMA,WAAS,MAAM,IAA2B;AACxC,WAAO,IAAI,QAAQ,CAAC,YAAY;AAAE,iBAAW,SAAS,EAAE;AAAA,IAAE,CAAC;AAAA,EAC7D;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,YAAM,QAAQ,MAAM,IAAI,KAAK,OAAO;AACpC,aAAO,KAAK,IAAI,IAAI,MAAM;AAAA,IAC5B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAOA,iBAAe,iBAAiB,WAAkC;AAChE,UAAM,gBAAgB,GAAG,OAAO,UAAU,OAAO,WAAW,CAAC;AAC7D,QAAI;AACF,YAAM,IAAI,OAAO,SAAS,aAAa;AAAA,IACzC,QAAQ;AACN;AAAA,IACF;AACA,WAAO,KAAK,gCAAgC,EAAE,SAAS,UAAU,CAAC;AAClE,UAAM,IAAI,GAAG,eAAe,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC9E;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,aAAO,MAAM,IAAI,SAAS,eAAe,MAAM;AAAA,IACjD,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAUA,iBAAe,qBAAqB,OAA8B;AAChE,QAAI,MAAM,cAAc,MAAM,MAAO;AACrC,UAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACxE;AAUA,iBAAe,uBAAqD;AAClE,UAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,eAAS;AACP,UAAI,WAAW;AACf,UAAI;AACF,cAAM,IAAI,MAAM,OAAO;AACvB,mBAAW;AAAA,MACb,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,MAAM,SAAS,SAAU,OAAM;AAAA,MACrC;AAEA,UAAI,UAAU;AACZ,cAAM,QAAQ,OAAO,WAAW;AAChC,YAAI;AACF,gBAAM,IAAI,UAAU,eAAe,OAAO,MAAM;AAAA,QAClD,SAAS,OAAgB;AACvB,gBAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACtE,gBAAM;AAAA,QACR;AACA,eAAO,MAAM,qBAAqB,KAAK;AAAA,MACzC;AAEA,YAAM,YAAY,MAAM,cAAc;AACtC,UAAI,cAAc,QAAQ,YAAY,eAAe;AACnD,cAAM,iBAAiB,SAAS;AAChC;AAAA,MACF;AAEA,UAAI,KAAK,IAAI,KAAK,UAAU;AAC1B,cAAM,IAAI;AAAA,UACR,8BAA8B,uBAAuB,oCAAoC,OAAO;AAAA,QAClG;AAAA,MACF;AAEA,YAAM,SAAS,oBAAoB,KAAK,OAAO,KAAK,oBAAoB;AACxE,YAAM,MAAM,MAAM;AAAA,IACpB;AAAA,EACF;AAQA,iBAAe,gBAAgB,YAAoB,SAAgC;AACjF,UAAM,WAAW,GAAG,UAAU,IAAI,OAAO,WAAW,CAAC;AACrD,QAAI;AACF,YAAM,IAAI,UAAU,UAAU,SAAS,MAAM;AAC7C,YAAM,0BAA0B,UAAU,UAAU;AAAA,IACtD,SAAS,OAAgB;AACvB,YAAM,IAAI,GAAG,UAAU,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACtD,YAAM;AAAA,IACR;AAAA,EACF;AAOA,iBAAe,0BAA0B,UAAkB,QAA+B;AACxF,UAAM,kBAAkB,oBAAI,IAAI,CAAC,SAAS,SAAS,QAAQ,CAAC;AAC5D,aAAS,UAAU,KAAK,WAAW;AACjC,UAAI;AACF,cAAM,IAAI,OAAO,UAAU,MAAM;AACjC;AAAA,MACF,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,WAAW,sBAAsB,CAAC,MAAM,QAAQ,CAAC,gBAAgB,IAAI,MAAM,IAAI,EAAG,OAAM;AAC5F,cAAM,MAAM,wBAAwB,UAAU,EAAE;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAQA,iBAAe,+BAAuD;AACpE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,IAAI,OAAO,WAAW,CAAC,OAAO;AAClG,QAAI;AACF,YAAM,IAAI,OAAO,WAAW,UAAU;AACtC,aAAO;AAAA,IACT,SAAS,GAAY;AACnB,aAAO,MAAM,iDAAiD,EAAE,KAAK,EAAW,CAAC;AACjF,aAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,aAAO,MAAM,6BAA6B,EAAE,KAAK,UAAU,CAAC;AAC5D,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAW,CAAC;AAC9D,YAAM,aAAa,MAAM,6BAA6B;AACtD,UAAI,YAAY;AACd,eAAO,MAAM,8EAA8E,EAAE,WAAW,CAAC;AAAA,MAC3G;AACA,YAAM,eAAe,aACjB,2BAA2B,UAAU,KACrC;AACJ,YAAM,IAAI;AAAA,QACR,yBAAyB,SAAS,uBAAuB,YAAY,KAAK,WAAW,OAAO;AAAA,MAC9F;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,EACjE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,WAAW,oBAAoB,MAAS;AAC9C,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,MACrC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AACD,oBAAgB,KAAK,SAAS;AAE9B,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,eAAW,OAAO,eAAe;AAC/B,qBAAe,IAAI,IAAI,EAAE;AAAA,IAC3B;AAEA,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,QAAI;AACF,iBAAW,OAAO,eAAe;AAC/B,cAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,YAAI;AACF,gBAAM;AAAA,YAAc;AAAA,YAAM,IAAI;AAAA,YAAU,MACtC,QAAQ;AAAA,cACN,QAAQ,KAAK;AAAA,gBACX,OAAO,IAAI;AAAA,gBACX;AAAA,gBACA,WAAW;AAAA,cACb,CAAC;AAAA,YACH;AAAA,UACF;AACA;AACA,sBAAY,IAAI;AAChB,0BAAgB,IAAI,IAAI,EAAE;AAC1B,iBAAO,KAAK,iBAAiB,EAAE,OAAO,IAAI,GAAG,CAAC;AAAA,QAChD,SAAS,OAAO;AACd,iBAAO,MAAM,cAAc,EAAE,OAAO,IAAI,IAAI,eAAe,aAAa,sBAAsB,KAAK,MAAM,CAAC;AAC1G;AACA,sBAAY,IAAI;AAChB,cAAI,iBAAiB,sBAAsB;AACzC,mBAAO,MAAM,kEAAkE,EAAE,OAAO,IAAI,IAAI,aAAa,qBAAqB,CAAC;AACnI,uBAAW,IAAI,IAAI,EAAE;AAAA,UACvB,OAAO;AACL,kBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,yBAAa,IAAI,IAAI,IAAI;AAAA,cACvB,GAAG;AAAA,cACH,cAAc;AAAA,cACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,YAC5D,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,UAAI,YAAY;AACd,cAAM,aAAa,YAAY;AAE7B,gBAAM,cAAc,MAAM,UAAU;AACpC,gBAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,gBAAM,WAAW,WAAW;AAC5B,0BAAgB,YAAY,SAAS;AAErC,gBAAM,WAAuB;AAAA,YAC3B,iBAAiB;AAAA,YACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,YAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,UACrD;AACA,gBAAM,WAAW,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAEA,aAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,IACxC,UAAE;AACA,iBAAW,OAAO,eAAe;AAC/B,uBAAe,OAAO,IAAI,EAAE;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAKA,iBAAe,eAAe,UAAU,OAAsB;AAC5D,QAAI,CAAC,cAAe;AACpB,QAAI,cAAc;AAChB,sBAAgB;AAChB;AAAA,IACF;AAEA,mBAAe;AACf,QAAI;AACF,SAAG;AACD,wBAAgB;AAChB,cAAM,UAAU;AAChB,YAAI,CAAC,QAAS;AACd,cAAM,aAAa,OAAO;AAAA,MAC5B,SAAS;AAAA,IACX,SAAS,OAAO;AACd,UAAI,QAAS,OAAM;AACnB,aAAO,MAAM,iBAAiB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC9C,UAAE;AACA,qBAAe;AACf,yBAAmB;AAAA,IACrB;AAAA,EACF;AAEA,WAAS,qBAA2B;AAClC,QAAI,CAAC,iBAAiB,CAAC,eAAe;AACpC,UAAI,iBAAiB;AACnB,qBAAa,eAAe;AAC5B,0BAAkB;AAAA,MACpB;AACA;AAAA,IACF;AACA,QAAI,gBAAiB;AACrB,sBAAkB,WAAW,MAAM;AACjC,wBAAkB;AAClB,WAAK,eAAe;AAAA,IACtB,GAAG,YAAY;AAAA,EACjB;AAEA,WAAS,oBAA0B;AACjC,QAAI,cAAc;AAChB,mBAAa,MAAM;AACnB,qBAAe;AAAA,IACjB;AACA,2BAAuB;AAAA,EACzB;AAEA,WAAS,sBAAqC;AAC5C,UAAM,UAAU,oBAAoB,KAAK,YAAY;AACnD,UAAI,CAAC,cAAe;AACpB,UAAI;AACF,cAAM,UAAU;AAChB,cAAM,QAAQ,MAAM,IAAI,KAAK,SAAS;AACtC,cAAM,eAAe,EAAE,QAAQ,MAAM,KAAK,OAAO,MAAM,IAAI;AAC3D,YACE,gBACG,sBAAsB,WAAW,aAAa,UAC9C,qBAAqB,UAAU,aAAa,OAC/C;AACA;AAAA,QACF;AAEA,0BAAkB;AAClB,cAAM,UAAU,GAAG,MAAM,WAAW,CAAC,cAAc;AACjD,cAAI,cAAc,UAAU;AAC1B,mCAAuB;AACvB,iBAAK,oBAAoB;AAAA,UAC3B;AACA,eAAK,eAAe;AAAA,QACtB,CAAC;AACD,gBAAQ,GAAG,SAAS,CAAC,QAAQ;AAC3B,iBAAO,MAAM,sDAAsD,EAAE,IAAI,CAAC;AAC1E,cAAI,iBAAiB,SAAS;AAC5B,8BAAkB;AAAA,UACpB;AAAA,QACF,CAAC;AACD,YAAI,CAAC,eAAe;AAClB,kBAAQ,MAAM;AACd;AAAA,QACF;AACA,uBAAe;AACf,+BAAuB;AAAA,MACzB,SAAS,KAAK;AACZ,eAAO,MAAM,+DAA+D,EAAE,IAAI,CAAC;AAAA,MACrF;AAAA,IACF,CAAC;AACD,0BAAsB,QAAQ,MAAM,MAAM,MAAS;AACnD,WAAO;AAAA,EACT;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAEA,QAAI,eAAe;AACjB,YAAM,MAAM;AAAA,IACd;AAGA,oBAAgB;AAEhB,QAAI;AACF,YAAM,oBAAoB;AAC1B,YAAM,eAAe,IAAI;AAAA,IAC3B,SAAS,OAAO;AACd,YAAM,MAAM;AACZ,YAAM;AAAA,IACR;AAEA,mBAAe,YAAY,MAAM;AAC/B,0BAAoB,EAAE,KAAK,MAAM,eAAe,CAAC,EAAE,MAAM,CAAC,QAAQ;AAChE,eAAO,MAAM,oBAAoB,EAAE,IAAI,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH,GAAG,oBAAoB;AAEvB,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAG7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AACnB,sBAAgB;AAChB,yBAAmB;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,eAAe,KAAK,OAAO,CAAC,QAAQ,eAAe,IAAI,IAAI,EAAE,KAAK,CAAC,oBAAoB,IAAI,SAAS,KAAK,CAAC;AAChH,YAAM,UAAU,KAAK,SAAS,aAAa;AAC3C,UAAI,UAAU,GAAG;AACf,cAAM,WAAW,YAAY;AAAA,MAC/B;AACA,sBAAgB,aAAa,SAAS;AACtC,yBAAmB;AACnB,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AACpC,oBAAgB;AAChB,QAAI,iBAAiB;AACnB,mBAAa,eAAe;AAC5B,wBAAkB;AAAA,IACpB;AACA,sBAAkB;AAGlB,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AAEA,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,eAAO,KAAK,wCAAwC,EAAE,WAAW,iBAAiB,CAAC;AACnF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
4
|
+
"sourcesContent": ["import fs from 'node:fs'\nimport path from 'node:path'\nimport crypto from 'node:crypto'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'\nimport type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'\nimport { attachTraceMetadata, runJobInTrace } from '../tracing'\n\nconst packageLogger = createLogger('queue')\n\ntype LocalState = {\n lastProcessedId?: string\n completedCount?: number\n failedCount?: number\n}\n\ntype StoredJob<T> = QueuedJob<T> & {\n availableAt?: string\n attemptCount?: number\n}\n\ntype QueueFileIdentity = {\n device: number\n inode: number\n}\n\nfunction payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {\n if (!payload || typeof payload !== 'object') return false\n const scopedPayload = payload as { tenantId?: unknown; organizationId?: unknown; jobType?: unknown }\n if (scopedPayload.tenantId !== scope.tenantId) return false\n if (scope.organizationId !== undefined) {\n if ((scopedPayload.organizationId ?? null) !== scope.organizationId) return false\n }\n if (scope.jobTypes?.length) {\n return typeof scopedPayload.jobType === 'string' && scope.jobTypes.includes(scopedPayload.jobType)\n }\n return true\n}\n\n/** Polling interval while delayed or retrying work remains queued. */\nconst DEFAULT_POLL_INTERVAL = 1000\n/** Idle safety interval for missed filesystem watcher events. */\nconst DEFAULT_FALLBACK_POLL_INTERVAL = 5000\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\nconst DEFAULT_MAX_ATTEMPTS = 3\nconst RETRY_BACKOFF_BASE_MS = 1000\n\n/**\n * Cross-process lock tuning. A held lock only ever spans local file I/O \u2014 job\n * handlers run outside it \u2014 so realistic hold times are milliseconds and the\n * stale threshold sits orders of magnitude above them. It exists solely so a\n * process that dies mid-segment cannot wedge the queue forever. A holder that\n * was merely suspended rather than dead can still be reclaimed, which is why\n * every acquisition carries an owner token and releases only its own lock.\n */\nconst LOCK_STALE_MS = 15_000\nconst LOCK_ACQUIRE_TIMEOUT_MS = 30_000\nconst LOCK_RETRY_MIN_MS = 2\nconst LOCK_RETRY_MAX_MS = 20\nconst RENAME_MAX_RETRIES = 5\nconst RENAME_RETRY_BASE_MS = 10\n\nconst fsp = fs.promises\n\n/**\n * Creates a file-based local queue.\n *\n * Jobs are stored in JSON files within a directory structure:\n * - `.mercato/queue/<name>/queue.json` - Array of queued jobs\n * - `.mercato/queue/<name>/state.json` - Processing state (last processed ID)\n *\n * **Limitations:**\n * - Jobs are processed sequentially (concurrency option is for logging/compatibility only)\n * - Not suitable for production: there is no dead-letter store, no throughput\n * beyond one job at a time, and every operation rewrites the whole queue file\n *\n * Multiple processes MAY share a queue directory, which is the default\n * development topology: the dev worker runs in its own process alongside the\n * Next.js server. What that buys you, and what it does not:\n *\n * - **Safe** \u2014 concurrent producers. Every read-modify-write segment takes the\n * `queue.lock` directory lock and every persist swaps the file in with an\n * atomic rename, so the file cannot be torn, no enqueue is lost to a\n * concurrent one, and a reader always observes one complete document.\n * Writers contend, though, so throughput degrades as processes are added.\n * - **NOT safe** \u2014 concurrent consumers. `process()` deliberately runs job\n * handlers outside the lock, so two worker processes polling the same queue\n * would both claim the same pending jobs and execute them twice. There is no\n * per-job lease. Run exactly one worker process per queue; use the `async`\n * strategy when you need more than one.\n *\n * Failed jobs are retried up to `DEFAULT_MAX_ATTEMPTS` times with exponential backoff.\n * **This strategy keeps no failed-job store**: once attempts are exhausted the job is\n * removed from `queue.json` and only counted in `state.failedCount`, so the payload is\n * lost and the failure survives solely as an error log line. The `async` strategy keeps\n * only a bounded inspection window: `removeOnFail: 1000` retains the most recent 1000\n * failures and removes older ones as later failures arrive. Workflows that require\n * no-loss persistence must write their own durable record before enqueueing, regardless\n * of strategy.\n *\n * `DEFAULT_MAX_ATTEMPTS` is a module constant, not a per-job option \u2014 callers cannot\n * request a different attempt count. (`async` likewise hard-codes `attempts: 3`.)\n * See the retry handling in `process()` below.\n *\n * All file I/O is asynchronous (`fs.promises.*`) so queue operations do not\n * block the Node.js event loop. A per-queue promise chain serializes\n * read-modify-write sequences within one instance, and the `queue.lock`\n * directory lock extends that serialization across instances and processes.\n *\n * @template T - The payload type for jobs\n * @param name - Queue name (used for directory naming)\n * @param options - Local queue options\n */\nexport function createLocalQueue<T = unknown>(\n name: string,\n options?: LocalQueueOptions\n): Queue<T> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const queueBaseDirFromEnv = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(queueBaseDirFromEnv || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueDir = path.join(baseDir, name)\n const queueFile = path.join(queueDir, 'queue.json')\n const stateFile = path.join(queueDir, 'state.json')\n const lockDir = path.join(queueDir, 'queue.lock')\n const lockOwnerFile = path.join(lockDir, 'owner')\n const logger = packageLogger.child({ queue: name })\n\n /**\n * Report a job failure outward, so the log line is not the only record of it.\n * Wrapped: reporting is never worth a poll cycle.\n */\n function reportQueueError(\n error: unknown,\n code: string,\n attributes?: Record<string, string | number | undefined>,\n ): void {\n try {\n getTelemetryRuntime()?.reportError(error, {\n module: 'queue',\n code,\n attributes: { queue: name, ...attributes },\n })\n } catch (telemetryError) {\n // Reporting is never worth a worker \u2014 but a systematically broken bridge\n // must not be silent either, or a worker stops reporting and nothing says so.\n logger.warn('Failed to report a queue error to telemetry', { code, err: telemetryError as Error })\n }\n }\n // Note: concurrency is stored for logging/compatibility but jobs are processed sequentially\n const concurrency = options?.concurrency ?? 1\n const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL\n const fallbackPollInterval = Math.max(pollInterval, DEFAULT_FALLBACK_POLL_INTERVAL)\n\n // Worker state for continuous polling\n let pollingTimer: ReturnType<typeof setInterval> | null = null\n let queuedPollTimer: ReturnType<typeof setTimeout> | null = null\n let queueWatcher: fs.FSWatcher | null = null\n let queueWatcherIdentity: QueueFileIdentity | null = null\n let watcherRefreshChain: Promise<void> = Promise.resolve()\n let hasQueuedJobs = false\n let isProcessing = false\n let pollRequested = false\n let activeHandler: JobHandler<T> | null = null\n const inFlightJobIds = new Set<string>()\n\n // Per-queue mutex. Serializes read-modify-write segments so async fs calls\n // cannot interleave and clobber each other's writes. It only covers this\n // instance, so it also guarantees at most one outstanding `queue.lock`\n // acquisition per instance \u2014 the directory lock below is not reentrant.\n let fileOpChain: Promise<unknown> = Promise.resolve()\n function withFileLock<R>(fn: () => Promise<R>): Promise<R> {\n const run = fileOpChain.then(\n () => runExclusively(fn),\n () => runExclusively(fn),\n )\n fileOpChain = run.then(\n () => undefined,\n () => undefined,\n )\n return run\n }\n\n /**\n * Runs `fn` while holding the cross-process `queue.lock`, so read-modify-write\n * segments issued by other queue instances \u2014 in this process or another one \u2014\n * cannot interleave with it.\n */\n async function runExclusively<R>(fn: () => Promise<R>): Promise<R> {\n await ensureDir()\n const release = await acquireDirectoryLock()\n try {\n return await fn()\n } finally {\n await release()\n }\n }\n\n // -------------------------------------------------------------------------\n // File Operations\n // -------------------------------------------------------------------------\n\n function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => { setTimeout(resolve, ms) })\n }\n\n async function lockHeldForMs(): Promise<number | null> {\n try {\n const stats = await fsp.stat(lockDir)\n return Date.now() - stats.mtimeMs\n } catch {\n return null\n }\n }\n\n /**\n * Reclaims a lock whose holder died. The rename is the serialization point:\n * only one racer can move `queue.lock` aside, so two processes cannot both\n * decide a stale lock is theirs to clear and then both create a fresh one.\n */\n async function reclaimStaleLock(heldForMs: number): Promise<void> {\n const reclaimedPath = `${lockDir}.stale.${crypto.randomUUID()}`\n try {\n await fsp.rename(lockDir, reclaimedPath)\n } catch {\n return\n }\n logger.warn('Reclaimed a stale queue lock', { lockDir, heldForMs })\n await fsp.rm(reclaimedPath, { recursive: true, force: true }).catch(() => {})\n }\n\n async function readLockOwner(): Promise<string | null> {\n try {\n return await fsp.readFile(lockOwnerFile, 'utf8')\n } catch {\n return null\n }\n }\n\n /**\n * Releases the lock only when this acquisition still owns it. A holder that\n * was suspended past `LOCK_STALE_MS` has had its lock reclaimed *and\n * replaced* by whoever reclaimed it, so an unconditional removal here would\n * delete the successor's lock and let a third caller into the critical\n * section alongside it. A missing or mismatched token means someone else owns\n * the path now, and the correct action is to leave it alone.\n */\n async function releaseDirectoryLock(token: string): Promise<void> {\n if (await readLockOwner() !== token) return\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n }\n\n /**\n * Acquires the cross-process advisory lock for this queue directory.\n * `mkdir` without `recursive` is an atomic exclusive create on every platform\n * Node.js supports, which makes it the portable primitive here \u2014 no runtime\n * dependency, and no reliance on advisory `flock` semantics. The owner token\n * written into the directory is what lets the release distinguish this\n * acquisition from a successor's.\n */\n async function acquireDirectoryLock(): Promise<() => Promise<void>> {\n const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS\n\n for (;;) {\n let acquired = false\n try {\n await fsp.mkdir(lockDir)\n acquired = true\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n if (acquired) {\n const token = crypto.randomUUID()\n try {\n await fsp.writeFile(lockOwnerFile, token, 'utf8')\n } catch (error: unknown) {\n await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => {})\n throw error\n }\n return () => releaseDirectoryLock(token)\n }\n\n const heldForMs = await lockHeldForMs()\n if (heldForMs !== null && heldForMs > LOCK_STALE_MS) {\n await reclaimStaleLock(heldForMs)\n continue\n }\n\n if (Date.now() >= deadline) {\n throw new Error(\n `[internal] Timed out after ${LOCK_ACQUIRE_TIMEOUT_MS}ms waiting for the queue lock at ${lockDir}`,\n )\n }\n\n const jitter = LOCK_RETRY_MIN_MS + Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)\n await sleep(jitter)\n }\n }\n\n /**\n * Persists `content` by writing a unique sibling temp file and renaming it\n * onto `targetFile`. `rename` within a directory is atomic, so a concurrent\n * reader sees either the previous document or the new one in full \u2014 never the\n * torn result of a truncate-then-write.\n */\n async function writeFileAtomic(targetFile: string, content: string): Promise<void> {\n const tempFile = `${targetFile}.${crypto.randomUUID()}.tmp`\n try {\n await fsp.writeFile(tempFile, content, 'utf8')\n await renameWithContentionRetry(tempFile, targetFile)\n } catch (error: unknown) {\n await fsp.rm(tempFile, { force: true }).catch(() => {})\n throw error\n }\n }\n\n /**\n * Windows rejects a rename onto a file another process currently has open,\n * so retry briefly on the contention codes it raises. POSIX renames replace\n * the target unconditionally and take the first attempt.\n */\n async function renameWithContentionRetry(fromFile: string, toFile: string): Promise<void> {\n const contentionCodes = new Set(['EPERM', 'EBUSY', 'EACCES'])\n for (let attempt = 0; ; attempt++) {\n try {\n await fsp.rename(fromFile, toFile)\n return\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (attempt >= RENAME_MAX_RETRIES || !error.code || !contentionCodes.has(error.code)) throw error\n await sleep(RENAME_RETRY_BASE_MS * (attempt + 1))\n }\n }\n }\n\n async function ensureDir(): Promise<void> {\n try {\n await fsp.mkdir(queueDir, { recursive: true })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize queue file with exclusive create flag\n try {\n await fsp.writeFile(queueFile, '[]', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n\n // Initialize state file with exclusive create flag\n try {\n await fsp.writeFile(stateFile, '{}', { encoding: 'utf8', flag: 'wx' })\n } catch (e: unknown) {\n const error = e as NodeJS.ErrnoException\n if (error.code !== 'EEXIST') throw error\n }\n }\n\n /**\n * Moves an unparsable queue file aside so its jobs stay recoverable. The\n * caller is expected to surface the failure rather than continue on an empty\n * queue: silently recreating `queue.json` here is what turned an unreadable\n * file into permanent, unreported job loss.\n */\n async function quarantineCorruptedQueueFile(): Promise<string | null> {\n const backupFile = path.join(queueDir, `queue.corrupted.${Date.now()}.${crypto.randomUUID()}.json`)\n try {\n await fsp.rename(queueFile, backupFile)\n return backupFile\n } catch (e: unknown) {\n logger.error('Failed to quarantine the corrupted queue file', { err: e as Error })\n return null\n }\n }\n\n async function readQueue(): Promise<StoredJob<T>[]> {\n await ensureDir()\n let content: string\n\n try {\n content = await fsp.readFile(queueFile, 'utf8')\n } catch (error: unknown) {\n const readError = error as NodeJS.ErrnoException\n if (readError.code === 'ENOENT') {\n return []\n }\n logger.error('Failed to read queue file', { err: readError })\n throw new Error(`Queue file unreadable: ${readError.message}`)\n }\n\n try {\n const parsed = JSON.parse(content) as unknown\n\n if (!Array.isArray(parsed)) {\n throw new Error('Queue file must contain a JSON array')\n }\n\n return parsed as StoredJob<T>[]\n } catch (error: unknown) {\n const parseError = error as Error\n logger.error('Failed to parse queue file', { err: parseError })\n const backupFile = await quarantineCorruptedQueueFile()\n if (backupFile) {\n logger.error('Quarantined corrupted queue file; its jobs are recoverable from the backup', { backupFile })\n }\n const recoveryHint = backupFile\n ? `has been quarantined as ${backupFile}`\n : 'could not be quarantined and was left in place'\n throw new Error(\n `[internal] Queue file ${queueFile} was unparsable and ${recoveryHint}: ${parseError.message}`,\n )\n }\n }\n\n async function writeQueue(jobs: StoredJob<T>[]): Promise<void> {\n await ensureDir()\n await writeFileAtomic(queueFile, JSON.stringify(jobs, null, 2))\n }\n\n async function readState(): Promise<LocalState> {\n await ensureDir()\n try {\n const content = await fsp.readFile(stateFile, 'utf8')\n return JSON.parse(content) as LocalState\n } catch {\n return {}\n }\n }\n\n async function writeState(state: LocalState): Promise<void> {\n await ensureDir()\n await writeFileAtomic(stateFile, JSON.stringify(state, null, 2))\n }\n\n function generateId(): string {\n return crypto.randomUUID()\n }\n\n // -------------------------------------------------------------------------\n // Queue Implementation\n // -------------------------------------------------------------------------\n\n async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {\n const availableAt = options?.delayMs && options.delayMs > 0\n ? new Date(Date.now() + options.delayMs).toISOString()\n : undefined\n const metadata = attachTraceMetadata(undefined)\n const job: StoredJob<T> = {\n id: generateId(),\n payload: data,\n createdAt: new Date().toISOString(),\n ...(availableAt ? { availableAt } : {}),\n ...(metadata ? { metadata } : {}),\n }\n await withFileLock(async () => {\n const jobs = await readQueue()\n jobs.push(job)\n await writeQueue(jobs)\n })\n return job.id\n }\n\n /**\n * Process pending jobs in a single batch (internal helper).\n */\n async function processBatch(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n const { state, jobs } = await withFileLock(async () => {\n const stateRead = await readState()\n const jobsRead = await readQueue()\n return { state: stateRead, jobs: jobsRead }\n })\n hasQueuedJobs = jobs.length > 0\n\n const pendingJobs = jobs.filter((job) => {\n if (!job.availableAt) return true\n return new Date(job.availableAt).getTime() <= Date.now()\n })\n const jobsToProcess = options?.limit\n ? pendingJobs.slice(0, options.limit)\n : pendingJobs\n\n for (const job of jobsToProcess) {\n inFlightJobIds.add(job.id)\n }\n\n let processed = 0\n let failed = 0\n let lastJobId: string | undefined\n const completedJobIds = new Set<string>()\n const deadJobIds = new Set<string>()\n const retryUpdates = new Map<string, StoredJob<T>>()\n\n try {\n for (const job of jobsToProcess) {\n const attemptNumber = (job.attemptCount ?? 0) + 1\n try {\n await runJobInTrace(name, job.metadata, () =>\n Promise.resolve(\n handler(job, {\n jobId: job.id,\n attemptNumber,\n queueName: name,\n })\n )\n )\n processed++\n lastJobId = job.id\n completedJobIds.add(job.id)\n logger.info('Job completed', { jobId: job.id })\n } catch (error) {\n logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })\n const exhausted = attemptNumber >= DEFAULT_MAX_ATTEMPTS\n // One report per failure, coded by what the failure means: a job that\n // has burned every retry is dead, which is the condition an operator\n // pages on, and it must be distinguishable from a first attempt that\n // will simply be retried. Reporting both would double-count `om.errors`\n // on the final attempt.\n reportQueueError(error, exhausted ? 'queue.job_exhausted' : 'queue.job_failed', {\n jobId: job.id,\n attemptNumber,\n })\n failed++\n lastJobId = job.id\n if (exhausted) {\n logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })\n deadJobIds.add(job.id)\n } else {\n const backoffMs = RETRY_BACKOFF_BASE_MS * Math.pow(2, attemptNumber - 1)\n retryUpdates.set(job.id, {\n ...job,\n attemptCount: attemptNumber,\n availableAt: new Date(Date.now() + backoffMs).toISOString(),\n })\n }\n }\n }\n\n const hasChanges = completedJobIds.size > 0 || deadJobIds.size > 0 || retryUpdates.size > 0\n if (hasChanges) {\n await withFileLock(async () => {\n // Re-read so jobs enqueued during handler execution are preserved.\n const currentJobs = await readQueue()\n const updatedJobs = currentJobs\n .filter((j) => !completedJobIds.has(j.id) && !deadJobIds.has(j.id))\n .map((j) => retryUpdates.get(j.id) ?? j)\n await writeQueue(updatedJobs)\n hasQueuedJobs = updatedJobs.length > 0\n\n const newState: LocalState = {\n lastProcessedId: lastJobId,\n completedCount: (state.completedCount ?? 0) + processed,\n failedCount: (state.failedCount ?? 0) + deadJobIds.size,\n }\n await writeState(newState)\n })\n }\n\n return { processed, failed, lastJobId }\n } finally {\n for (const job of jobsToProcess) {\n inFlightJobIds.delete(job.id)\n }\n }\n }\n\n /**\n * Poll for and process new jobs.\n */\n async function pollAndProcess(rethrow = false): Promise<void> {\n if (!activeHandler) return\n if (isProcessing) {\n pollRequested = true\n return\n }\n\n isProcessing = true\n try {\n do {\n pollRequested = false\n const handler = activeHandler\n if (!handler) break\n await processBatch(handler)\n } while (pollRequested)\n } catch (error) {\n if (rethrow) throw error\n logger.error('Polling error', { err: error })\n } finally {\n isProcessing = false\n scheduleQueuedPoll()\n }\n }\n\n function scheduleQueuedPoll(): void {\n if (!activeHandler || !hasQueuedJobs) {\n if (queuedPollTimer) {\n clearTimeout(queuedPollTimer)\n queuedPollTimer = null\n }\n return\n }\n if (queuedPollTimer) return\n queuedPollTimer = setTimeout(() => {\n queuedPollTimer = null\n void pollAndProcess()\n }, pollInterval)\n }\n\n function closeQueueWatcher(): void {\n if (queueWatcher) {\n queueWatcher.close()\n queueWatcher = null\n }\n queueWatcherIdentity = null\n }\n\n function refreshQueueWatcher(): Promise<void> {\n const refresh = watcherRefreshChain.then(async () => {\n if (!activeHandler) return\n try {\n await ensureDir()\n const stats = await fsp.stat(queueFile)\n const nextIdentity = { device: stats.dev, inode: stats.ino }\n if (\n queueWatcher\n && queueWatcherIdentity?.device === nextIdentity.device\n && queueWatcherIdentity.inode === nextIdentity.inode\n ) {\n return\n }\n\n closeQueueWatcher()\n const watcher = fs.watch(queueFile, (eventType) => {\n if (eventType === 'rename') {\n queueWatcherIdentity = null\n void refreshQueueWatcher()\n }\n void pollAndProcess()\n })\n watcher.on('error', (err) => {\n logger.error('Queue watch error; fallback polling remains active', { err })\n if (queueWatcher === watcher) {\n closeQueueWatcher()\n }\n })\n if (!activeHandler) {\n watcher.close()\n return\n }\n queueWatcher = watcher\n queueWatcherIdentity = nextIdentity\n } catch (err) {\n logger.error('Failed to watch queue file; fallback polling remains active', { err })\n }\n })\n watcherRefreshChain = refresh.catch(() => undefined)\n return refresh\n }\n\n async function process(\n handler: JobHandler<T>,\n options?: ProcessOptions\n ): Promise<ProcessResult> {\n // If limit is specified, do a single batch (backward compatibility)\n if (options?.limit) {\n return processBatch(handler, options)\n }\n\n if (activeHandler) {\n await close()\n }\n\n // Start continuous polling mode (like BullMQ Worker)\n activeHandler = handler\n\n try {\n await refreshQueueWatcher()\n await pollAndProcess(true)\n } catch (error) {\n await close()\n throw error\n }\n\n pollingTimer = setInterval(() => {\n refreshQueueWatcher().then(() => pollAndProcess()).catch((err) => {\n logger.error('Poll cycle error', { err })\n })\n }, fallbackPollInterval)\n\n logger.info('Worker started', { concurrency })\n\n // Return sentinel value indicating continuous worker mode (like async strategy)\n return { processed: -1, failed: -1, lastJobId: undefined }\n }\n\n async function clear(): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const removed = jobs.length\n await writeQueue([])\n hasQueuedJobs = false\n scheduleQueuedPoll()\n // Reset state but preserve counts for historical tracking\n const state = await readState()\n await writeState({\n completedCount: state.completedCount,\n failedCount: state.failedCount,\n })\n return { removed }\n })\n }\n\n async function removeQueuedJobsByScope(scope: QueueJobScope): Promise<{ removed: number }> {\n return withFileLock(async () => {\n const jobs = await readQueue()\n const retainedJobs = jobs.filter((job) => inFlightJobIds.has(job.id) || !payloadMatchesScope(job.payload, scope))\n const removed = jobs.length - retainedJobs.length\n if (removed > 0) {\n await writeQueue(retainedJobs)\n }\n hasQueuedJobs = retainedJobs.length > 0\n scheduleQueuedPoll()\n return { removed }\n })\n }\n\n async function close(): Promise<void> {\n activeHandler = null\n if (queuedPollTimer) {\n clearTimeout(queuedPollTimer)\n queuedPollTimer = null\n }\n closeQueueWatcher()\n\n // Stop polling timer\n if (pollingTimer) {\n clearInterval(pollingTimer)\n pollingTimer = null\n }\n // Wait for any in-progress processing to complete (with timeout)\n const SHUTDOWN_TIMEOUT = 5000\n const startTime = Date.now()\n\n while (isProcessing) {\n if (Date.now() - startTime > SHUTDOWN_TIMEOUT) {\n logger.warn('Force closing after shutdown timeout', { timeoutMs: SHUTDOWN_TIMEOUT })\n break\n }\n await new Promise((resolve) => setTimeout(resolve, 50))\n }\n }\n\n async function getJobCounts(): Promise<{\n waiting: number\n active: number\n completed: number\n failed: number\n }> {\n return withFileLock(async () => {\n const state = await readState()\n const jobs = await readQueue()\n\n return {\n waiting: jobs.length, // All jobs in queue are waiting (processed ones are removed)\n active: 0, // Local strategy doesn't track active jobs\n completed: state.completedCount ?? 0,\n failed: state.failedCount ?? 0,\n }\n })\n }\n\n return {\n name,\n strategy: 'local',\n enqueue,\n process,\n clear,\n removeQueuedJobsByScope,\n close,\n getJobCounts,\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,QAAQ;AACf,OAAO,UAAU;AACjB,OAAO,YAAY;AACnB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AAEpC,SAAS,qBAAqB,qBAAqB;AAEnD,MAAM,gBAAgB,aAAa,OAAO;AAkB1C,SAAS,oBAAoB,SAAkB,OAA+B;AAC5E,MAAI,CAAC,WAAW,OAAO,YAAY,SAAU,QAAO;AACpD,QAAM,gBAAgB;AACtB,MAAI,cAAc,aAAa,MAAM,SAAU,QAAO;AACtD,MAAI,MAAM,mBAAmB,QAAW;AACtC,SAAK,cAAc,kBAAkB,UAAU,MAAM,eAAgB,QAAO;AAAA,EAC9E;AACA,MAAI,MAAM,UAAU,QAAQ;AAC1B,WAAO,OAAO,cAAc,YAAY,YAAY,MAAM,SAAS,SAAS,cAAc,OAAO;AAAA,EACnG;AACA,SAAO;AACT;AAGA,MAAM,wBAAwB;AAE9B,MAAM,iCAAiC;AACvC,MAAM,+BAA+B;AACrC,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAU9B,MAAM,gBAAgB;AACtB,MAAM,0BAA0B;AAChC,MAAM,oBAAoB;AAC1B,MAAM,oBAAoB;AAC1B,MAAM,qBAAqB;AAC3B,MAAM,uBAAuB;AAE7B,MAAM,MAAM,GAAG;AAmDR,SAAS,iBACd,MACA,SACU;AACV,QAAM,cAAe,WAAgE;AACrF,QAAM,sBAAsB,aAAa,KAAK;AAC9C,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,uBAAuB,4BAA4B;AACrE,QAAM,WAAW,KAAK,KAAK,SAAS,IAAI;AACxC,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,YAAY,KAAK,KAAK,UAAU,YAAY;AAClD,QAAM,UAAU,KAAK,KAAK,UAAU,YAAY;AAChD,QAAM,gBAAgB,KAAK,KAAK,SAAS,OAAO;AAChD,QAAM,SAAS,cAAc,MAAM,EAAE,OAAO,KAAK,CAAC;AAMlD,WAAS,iBACP,OACA,MACA,YACM;AACN,QAAI;AACF,0BAAoB,GAAG,YAAY,OAAO;AAAA,QACxC,QAAQ;AAAA,QACR;AAAA,QACA,YAAY,EAAE,OAAO,MAAM,GAAG,WAAW;AAAA,MAC3C,CAAC;AAAA,IACH,SAAS,gBAAgB;AAGvB,aAAO,KAAK,+CAA+C,EAAE,MAAM,KAAK,eAAwB,CAAC;AAAA,IACnG;AAAA,EACF;AAEA,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,eAAe,SAAS,gBAAgB;AAC9C,QAAM,uBAAuB,KAAK,IAAI,cAAc,8BAA8B;AAGlF,MAAI,eAAsD;AAC1D,MAAI,kBAAwD;AAC5D,MAAI,eAAoC;AACxC,MAAI,uBAAiD;AACrD,MAAI,sBAAqC,QAAQ,QAAQ;AACzD,MAAI,gBAAgB;AACpB,MAAI,eAAe;AACnB,MAAI,gBAAgB;AACpB,MAAI,gBAAsC;AAC1C,QAAM,iBAAiB,oBAAI,IAAY;AAMvC,MAAI,cAAgC,QAAQ,QAAQ;AACpD,WAAS,aAAgB,IAAkC;AACzD,UAAM,MAAM,YAAY;AAAA,MACtB,MAAM,eAAe,EAAE;AAAA,MACvB,MAAM,eAAe,EAAE;AAAA,IACzB;AACA,kBAAc,IAAI;AAAA,MAChB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAOA,iBAAe,eAAkB,IAAkC;AACjE,UAAM,UAAU;AAChB,UAAM,UAAU,MAAM,qBAAqB;AAC3C,QAAI;AACF,aAAO,MAAM,GAAG;AAAA,IAClB,UAAE;AACA,YAAM,QAAQ;AAAA,IAChB;AAAA,EACF;AAMA,WAAS,MAAM,IAA2B;AACxC,WAAO,IAAI,QAAQ,CAAC,YAAY;AAAE,iBAAW,SAAS,EAAE;AAAA,IAAE,CAAC;AAAA,EAC7D;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,YAAM,QAAQ,MAAM,IAAI,KAAK,OAAO;AACpC,aAAO,KAAK,IAAI,IAAI,MAAM;AAAA,IAC5B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAOA,iBAAe,iBAAiB,WAAkC;AAChE,UAAM,gBAAgB,GAAG,OAAO,UAAU,OAAO,WAAW,CAAC;AAC7D,QAAI;AACF,YAAM,IAAI,OAAO,SAAS,aAAa;AAAA,IACzC,QAAQ;AACN;AAAA,IACF;AACA,WAAO,KAAK,gCAAgC,EAAE,SAAS,UAAU,CAAC;AAClE,UAAM,IAAI,GAAG,eAAe,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC9E;AAEA,iBAAe,gBAAwC;AACrD,QAAI;AACF,aAAO,MAAM,IAAI,SAAS,eAAe,MAAM;AAAA,IACjD,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAUA,iBAAe,qBAAqB,OAA8B;AAChE,QAAI,MAAM,cAAc,MAAM,MAAO;AACrC,UAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACxE;AAUA,iBAAe,uBAAqD;AAClE,UAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,eAAS;AACP,UAAI,WAAW;AACf,UAAI;AACF,cAAM,IAAI,MAAM,OAAO;AACvB,mBAAW;AAAA,MACb,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,MAAM,SAAS,SAAU,OAAM;AAAA,MACrC;AAEA,UAAI,UAAU;AACZ,cAAM,QAAQ,OAAO,WAAW;AAChC,YAAI;AACF,gBAAM,IAAI,UAAU,eAAe,OAAO,MAAM;AAAA,QAClD,SAAS,OAAgB;AACvB,gBAAM,IAAI,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACtE,gBAAM;AAAA,QACR;AACA,eAAO,MAAM,qBAAqB,KAAK;AAAA,MACzC;AAEA,YAAM,YAAY,MAAM,cAAc;AACtC,UAAI,cAAc,QAAQ,YAAY,eAAe;AACnD,cAAM,iBAAiB,SAAS;AAChC;AAAA,MACF;AAEA,UAAI,KAAK,IAAI,KAAK,UAAU;AAC1B,cAAM,IAAI;AAAA,UACR,8BAA8B,uBAAuB,oCAAoC,OAAO;AAAA,QAClG;AAAA,MACF;AAEA,YAAM,SAAS,oBAAoB,KAAK,OAAO,KAAK,oBAAoB;AACxE,YAAM,MAAM,MAAM;AAAA,IACpB;AAAA,EACF;AAQA,iBAAe,gBAAgB,YAAoB,SAAgC;AACjF,UAAM,WAAW,GAAG,UAAU,IAAI,OAAO,WAAW,CAAC;AACrD,QAAI;AACF,YAAM,IAAI,UAAU,UAAU,SAAS,MAAM;AAC7C,YAAM,0BAA0B,UAAU,UAAU;AAAA,IACtD,SAAS,OAAgB;AACvB,YAAM,IAAI,GAAG,UAAU,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACtD,YAAM;AAAA,IACR;AAAA,EACF;AAOA,iBAAe,0BAA0B,UAAkB,QAA+B;AACxF,UAAM,kBAAkB,oBAAI,IAAI,CAAC,SAAS,SAAS,QAAQ,CAAC;AAC5D,aAAS,UAAU,KAAK,WAAW;AACjC,UAAI;AACF,cAAM,IAAI,OAAO,UAAU,MAAM;AACjC;AAAA,MACF,SAAS,GAAY;AACnB,cAAM,QAAQ;AACd,YAAI,WAAW,sBAAsB,CAAC,MAAM,QAAQ,CAAC,gBAAgB,IAAI,MAAM,IAAI,EAAG,OAAM;AAC5F,cAAM,MAAM,wBAAwB,UAAU,EAAE;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,YAA2B;AACxC,QAAI;AACF,YAAM,IAAI,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,IAC/C,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAGA,QAAI;AACF,YAAM,IAAI,UAAU,WAAW,MAAM,EAAE,UAAU,QAAQ,MAAM,KAAK,CAAC;AAAA,IACvE,SAAS,GAAY;AACnB,YAAM,QAAQ;AACd,UAAI,MAAM,SAAS,SAAU,OAAM;AAAA,IACrC;AAAA,EACF;AAQA,iBAAe,+BAAuD;AACpE,UAAM,aAAa,KAAK,KAAK,UAAU,mBAAmB,KAAK,IAAI,CAAC,IAAI,OAAO,WAAW,CAAC,OAAO;AAClG,QAAI;AACF,YAAM,IAAI,OAAO,WAAW,UAAU;AACtC,aAAO;AAAA,IACT,SAAS,GAAY;AACnB,aAAO,MAAM,iDAAiD,EAAE,KAAK,EAAW,CAAC;AACjF,aAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YAAqC;AAClD,UAAM,UAAU;AAChB,QAAI;AAEJ,QAAI;AACF,gBAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,IAChD,SAAS,OAAgB;AACvB,YAAM,YAAY;AAClB,UAAI,UAAU,SAAS,UAAU;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,aAAO,MAAM,6BAA6B,EAAE,KAAK,UAAU,CAAC;AAC5D,YAAM,IAAI,MAAM,0BAA0B,UAAU,OAAO,EAAE;AAAA,IAC/D;AAEA,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,UAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,cAAM,IAAI,MAAM,sCAAsC;AAAA,MACxD;AAEA,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,aAAa;AACnB,aAAO,MAAM,8BAA8B,EAAE,KAAK,WAAW,CAAC;AAC9D,YAAM,aAAa,MAAM,6BAA6B;AACtD,UAAI,YAAY;AACd,eAAO,MAAM,8EAA8E,EAAE,WAAW,CAAC;AAAA,MAC3G;AACA,YAAM,eAAe,aACjB,2BAA2B,UAAU,KACrC;AACJ,YAAM,IAAI;AAAA,QACR,yBAAyB,SAAS,uBAAuB,YAAY,KAAK,WAAW,OAAO;AAAA,MAC9F;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,WAAW,MAAqC;AAC7D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAEA,iBAAe,YAAiC;AAC9C,UAAM,UAAU;AAChB,QAAI;AACF,YAAM,UAAU,MAAM,IAAI,SAAS,WAAW,MAAM;AACpD,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAEA,iBAAe,WAAW,OAAkC;AAC1D,UAAM,UAAU;AAChB,UAAM,gBAAgB,WAAW,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,EACjE;AAEA,WAAS,aAAqB;AAC5B,WAAO,OAAO,WAAW;AAAA,EAC3B;AAMA,iBAAe,QAAQ,MAASA,UAA2C;AACzE,UAAM,cAAcA,UAAS,WAAWA,SAAQ,UAAU,IACtD,IAAI,KAAK,KAAK,IAAI,IAAIA,SAAQ,OAAO,EAAE,YAAY,IACnD;AACJ,UAAM,WAAW,oBAAoB,MAAS;AAC9C,UAAM,MAAoB;AAAA,MACxB,IAAI,WAAW;AAAA,MACf,SAAS;AAAA,MACT,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,MACrC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AACA,UAAM,aAAa,YAAY;AAC7B,YAAM,OAAO,MAAM,UAAU;AAC7B,WAAK,KAAK,GAAG;AACb,YAAM,WAAW,IAAI;AAAA,IACvB,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAKA,iBAAe,aACb,SACAA,UACwB;AACxB,UAAM,EAAE,OAAO,KAAK,IAAI,MAAM,aAAa,YAAY;AACrD,YAAM,YAAY,MAAM,UAAU;AAClC,YAAM,WAAW,MAAM,UAAU;AACjC,aAAO,EAAE,OAAO,WAAW,MAAM,SAAS;AAAA,IAC5C,CAAC;AACD,oBAAgB,KAAK,SAAS;AAE9B,UAAM,cAAc,KAAK,OAAO,CAAC,QAAQ;AACvC,UAAI,CAAC,IAAI,YAAa,QAAO;AAC7B,aAAO,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,KAAK,KAAK,IAAI;AAAA,IACzD,CAAC;AACD,UAAM,gBAAgBA,UAAS,QAC3B,YAAY,MAAM,GAAGA,SAAQ,KAAK,IAClC;AAEJ,eAAW,OAAO,eAAe;AAC/B,qBAAe,IAAI,IAAI,EAAE;AAAA,IAC3B;AAEA,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,QAAI;AACJ,UAAM,kBAAkB,oBAAI,IAAY;AACxC,UAAM,aAAa,oBAAI,IAAY;AACnC,UAAM,eAAe,oBAAI,IAA0B;AAEnD,QAAI;AACF,iBAAW,OAAO,eAAe;AAC/B,cAAM,iBAAiB,IAAI,gBAAgB,KAAK;AAChD,YAAI;AACF,gBAAM;AAAA,YAAc;AAAA,YAAM,IAAI;AAAA,YAAU,MACtC,QAAQ;AAAA,cACN,QAAQ,KAAK;AAAA,gBACX,OAAO,IAAI;AAAA,gBACX;AAAA,gBACA,WAAW;AAAA,cACb,CAAC;AAAA,YACH;AAAA,UACF;AACA;AACA,sBAAY,IAAI;AAChB,0BAAgB,IAAI,IAAI,EAAE;AAC1B,iBAAO,KAAK,iBAAiB,EAAE,OAAO,IAAI,GAAG,CAAC;AAAA,QAChD,SAAS,OAAO;AACd,iBAAO,MAAM,cAAc,EAAE,OAAO,IAAI,IAAI,eAAe,aAAa,sBAAsB,KAAK,MAAM,CAAC;AAC1G,gBAAM,YAAY,iBAAiB;AAMnC,2BAAiB,OAAO,YAAY,wBAAwB,oBAAoB;AAAA,YAC9E,OAAO,IAAI;AAAA,YACX;AAAA,UACF,CAAC;AACD;AACA,sBAAY,IAAI;AAChB,cAAI,WAAW;AACb,mBAAO,MAAM,kEAAkE,EAAE,OAAO,IAAI,IAAI,aAAa,qBAAqB,CAAC;AACnI,uBAAW,IAAI,IAAI,EAAE;AAAA,UACvB,OAAO;AACL,kBAAM,YAAY,wBAAwB,KAAK,IAAI,GAAG,gBAAgB,CAAC;AACvE,yBAAa,IAAI,IAAI,IAAI;AAAA,cACvB,GAAG;AAAA,cACH,cAAc;AAAA,cACd,aAAa,IAAI,KAAK,KAAK,IAAI,IAAI,SAAS,EAAE,YAAY;AAAA,YAC5D,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,gBAAgB,OAAO,KAAK,WAAW,OAAO,KAAK,aAAa,OAAO;AAC1F,UAAI,YAAY;AACd,cAAM,aAAa,YAAY;AAE7B,gBAAM,cAAc,MAAM,UAAU;AACpC,gBAAM,cAAc,YACjB,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,EAAE,KAAK,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,EACjE,IAAI,CAAC,MAAM,aAAa,IAAI,EAAE,EAAE,KAAK,CAAC;AACzC,gBAAM,WAAW,WAAW;AAC5B,0BAAgB,YAAY,SAAS;AAErC,gBAAM,WAAuB;AAAA,YAC3B,iBAAiB;AAAA,YACjB,iBAAiB,MAAM,kBAAkB,KAAK;AAAA,YAC9C,cAAc,MAAM,eAAe,KAAK,WAAW;AAAA,UACrD;AACA,gBAAM,WAAW,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAEA,aAAO,EAAE,WAAW,QAAQ,UAAU;AAAA,IACxC,UAAE;AACA,iBAAW,OAAO,eAAe;AAC/B,uBAAe,OAAO,IAAI,EAAE;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAKA,iBAAe,eAAe,UAAU,OAAsB;AAC5D,QAAI,CAAC,cAAe;AACpB,QAAI,cAAc;AAChB,sBAAgB;AAChB;AAAA,IACF;AAEA,mBAAe;AACf,QAAI;AACF,SAAG;AACD,wBAAgB;AAChB,cAAM,UAAU;AAChB,YAAI,CAAC,QAAS;AACd,cAAM,aAAa,OAAO;AAAA,MAC5B,SAAS;AAAA,IACX,SAAS,OAAO;AACd,UAAI,QAAS,OAAM;AACnB,aAAO,MAAM,iBAAiB,EAAE,KAAK,MAAM,CAAC;AAAA,IAC9C,UAAE;AACA,qBAAe;AACf,yBAAmB;AAAA,IACrB;AAAA,EACF;AAEA,WAAS,qBAA2B;AAClC,QAAI,CAAC,iBAAiB,CAAC,eAAe;AACpC,UAAI,iBAAiB;AACnB,qBAAa,eAAe;AAC5B,0BAAkB;AAAA,MACpB;AACA;AAAA,IACF;AACA,QAAI,gBAAiB;AACrB,sBAAkB,WAAW,MAAM;AACjC,wBAAkB;AAClB,WAAK,eAAe;AAAA,IACtB,GAAG,YAAY;AAAA,EACjB;AAEA,WAAS,oBAA0B;AACjC,QAAI,cAAc;AAChB,mBAAa,MAAM;AACnB,qBAAe;AAAA,IACjB;AACA,2BAAuB;AAAA,EACzB;AAEA,WAAS,sBAAqC;AAC5C,UAAM,UAAU,oBAAoB,KAAK,YAAY;AACnD,UAAI,CAAC,cAAe;AACpB,UAAI;AACF,cAAM,UAAU;AAChB,cAAM,QAAQ,MAAM,IAAI,KAAK,SAAS;AACtC,cAAM,eAAe,EAAE,QAAQ,MAAM,KAAK,OAAO,MAAM,IAAI;AAC3D,YACE,gBACG,sBAAsB,WAAW,aAAa,UAC9C,qBAAqB,UAAU,aAAa,OAC/C;AACA;AAAA,QACF;AAEA,0BAAkB;AAClB,cAAM,UAAU,GAAG,MAAM,WAAW,CAAC,cAAc;AACjD,cAAI,cAAc,UAAU;AAC1B,mCAAuB;AACvB,iBAAK,oBAAoB;AAAA,UAC3B;AACA,eAAK,eAAe;AAAA,QACtB,CAAC;AACD,gBAAQ,GAAG,SAAS,CAAC,QAAQ;AAC3B,iBAAO,MAAM,sDAAsD,EAAE,IAAI,CAAC;AAC1E,cAAI,iBAAiB,SAAS;AAC5B,8BAAkB;AAAA,UACpB;AAAA,QACF,CAAC;AACD,YAAI,CAAC,eAAe;AAClB,kBAAQ,MAAM;AACd;AAAA,QACF;AACA,uBAAe;AACf,+BAAuB;AAAA,MACzB,SAAS,KAAK;AACZ,eAAO,MAAM,+DAA+D,EAAE,IAAI,CAAC;AAAA,MACrF;AAAA,IACF,CAAC;AACD,0BAAsB,QAAQ,MAAM,MAAM,MAAS;AACnD,WAAO;AAAA,EACT;AAEA,iBAAe,QACb,SACAA,UACwB;AAExB,QAAIA,UAAS,OAAO;AAClB,aAAO,aAAa,SAASA,QAAO;AAAA,IACtC;AAEA,QAAI,eAAe;AACjB,YAAM,MAAM;AAAA,IACd;AAGA,oBAAgB;AAEhB,QAAI;AACF,YAAM,oBAAoB;AAC1B,YAAM,eAAe,IAAI;AAAA,IAC3B,SAAS,OAAO;AACd,YAAM,MAAM;AACZ,YAAM;AAAA,IACR;AAEA,mBAAe,YAAY,MAAM;AAC/B,0BAAoB,EAAE,KAAK,MAAM,eAAe,CAAC,EAAE,MAAM,CAAC,QAAQ;AAChE,eAAO,MAAM,oBAAoB,EAAE,IAAI,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH,GAAG,oBAAoB;AAEvB,WAAO,KAAK,kBAAkB,EAAE,YAAY,CAAC;AAG7C,WAAO,EAAE,WAAW,IAAI,QAAQ,IAAI,WAAW,OAAU;AAAA,EAC3D;AAEA,iBAAe,QAAsC;AACnD,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,UAAU,KAAK;AACrB,YAAM,WAAW,CAAC,CAAC;AACnB,sBAAgB;AAChB,yBAAmB;AAEnB,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,WAAW;AAAA,QACf,gBAAgB,MAAM;AAAA,QACtB,aAAa,MAAM;AAAA,MACrB,CAAC;AACD,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,wBAAwB,OAAoD;AACzF,WAAO,aAAa,YAAY;AAC9B,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM,eAAe,KAAK,OAAO,CAAC,QAAQ,eAAe,IAAI,IAAI,EAAE,KAAK,CAAC,oBAAoB,IAAI,SAAS,KAAK,CAAC;AAChH,YAAM,UAAU,KAAK,SAAS,aAAa;AAC3C,UAAI,UAAU,GAAG;AACf,cAAM,WAAW,YAAY;AAAA,MAC/B;AACA,sBAAgB,aAAa,SAAS;AACtC,yBAAmB;AACnB,aAAO,EAAE,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,iBAAe,QAAuB;AACpC,oBAAgB;AAChB,QAAI,iBAAiB;AACnB,mBAAa,eAAe;AAC5B,wBAAkB;AAAA,IACpB;AACA,sBAAkB;AAGlB,QAAI,cAAc;AAChB,oBAAc,YAAY;AAC1B,qBAAe;AAAA,IACjB;AAEA,UAAM,mBAAmB;AACzB,UAAM,YAAY,KAAK,IAAI;AAE3B,WAAO,cAAc;AACnB,UAAI,KAAK,IAAI,IAAI,YAAY,kBAAkB;AAC7C,eAAO,KAAK,wCAAwC,EAAE,WAAW,iBAAiB,CAAC;AACnF;AAAA,MACF;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAAA,IACxD;AAAA,EACF;AAEA,iBAAe,eAKZ;AACD,WAAO,aAAa,YAAY;AAC9B,YAAM,QAAQ,MAAM,UAAU;AAC9B,YAAM,OAAO,MAAM,UAAU;AAE7B,aAAO;AAAA,QACL,SAAS,KAAK;AAAA;AAAA,QACd,QAAQ;AAAA;AAAA,QACR,WAAW,MAAM,kBAAkB;AAAA,QACnC,QAAQ,MAAM,eAAe;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["options"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/queue",
|
|
3
|
-
"version": "0.7.1-develop.
|
|
3
|
+
"version": "0.7.1-develop.7186.1.6e080a5017",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Multi-strategy job queue with local and BullMQ support",
|
|
6
6
|
"type": "module",
|
|
@@ -56,8 +56,8 @@
|
|
|
56
56
|
"access": "public"
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@open-mercato/shared": "0.7.1-develop.
|
|
60
|
-
"@open-mercato/telemetry": "0.7.1-develop.
|
|
59
|
+
"@open-mercato/shared": "0.7.1-develop.7186.1.6e080a5017",
|
|
60
|
+
"@open-mercato/telemetry": "0.7.1-develop.7186.1.6e080a5017"
|
|
61
61
|
},
|
|
62
62
|
"repository": {
|
|
63
63
|
"type": "git",
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import os from 'node:os'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
|
|
5
|
+
import { createQueue } from '../factory'
|
|
6
|
+
import {
|
|
7
|
+
registerTelemetryRuntime,
|
|
8
|
+
resetTelemetryRuntime,
|
|
9
|
+
type TelemetryRuntime,
|
|
10
|
+
} from '@open-mercato/shared/lib/telemetry/runtime'
|
|
11
|
+
import type { QueuedJob } from '../types'
|
|
12
|
+
|
|
13
|
+
type Reported = {
|
|
14
|
+
code?: string
|
|
15
|
+
attributes?: Record<string, string | number | boolean | undefined>
|
|
16
|
+
message: string
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
type WorkerListener = (...args: unknown[]) => void
|
|
20
|
+
|
|
21
|
+
const capturedListeners = new Map<string, WorkerListener[]>()
|
|
22
|
+
|
|
23
|
+
jest.mock('@open-mercato/shared/lib/redis/connection', () => ({
|
|
24
|
+
getRedisUrlOrThrow: jest.fn(() => 'redis://127.0.0.1:6379'),
|
|
25
|
+
parseRedisUrl: jest.requireActual('@open-mercato/shared/lib/redis/connection').parseRedisUrl,
|
|
26
|
+
REDIS_WIRE_PROTOCOL: 3,
|
|
27
|
+
}))
|
|
28
|
+
|
|
29
|
+
jest.mock('bullmq', () => {
|
|
30
|
+
class MockQueue<T> {
|
|
31
|
+
add = jest.fn(async () => ({ id: 'bull-job-id' }))
|
|
32
|
+
close = jest.fn(async () => {})
|
|
33
|
+
obliterate = jest.fn(async () => {})
|
|
34
|
+
getJobCounts = jest.fn(async () => ({ waiting: 0, active: 0, completed: 0, failed: 0 }))
|
|
35
|
+
getJobs = jest.fn(async () => [] as Array<{ id?: string; data?: T }>)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
class MockWorker<T> {
|
|
39
|
+
constructor(
|
|
40
|
+
_name: string,
|
|
41
|
+
_processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,
|
|
42
|
+
_opts: unknown,
|
|
43
|
+
) {}
|
|
44
|
+
|
|
45
|
+
on = (event: string, listener: WorkerListener) => {
|
|
46
|
+
const existing = capturedListeners.get(event) ?? []
|
|
47
|
+
existing.push(listener)
|
|
48
|
+
capturedListeners.set(event, existing)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
close = jest.fn(async () => {})
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return { Queue: MockQueue, Worker: MockWorker }
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
function runtimeStub() {
|
|
58
|
+
const reported: Reported[] = []
|
|
59
|
+
const runtime = {
|
|
60
|
+
canUseGlobalTracePropagation: () => false,
|
|
61
|
+
captureTraceContext: () => ({}),
|
|
62
|
+
continueTrace: <T>(_carrier: unknown, _name: string, fn: () => T) => fn(),
|
|
63
|
+
withSpan: <T>(_name: string, fn: (span: { setAttributes: () => void }) => T) => fn({ setAttributes: () => {} }),
|
|
64
|
+
recordHttpDuration: () => {},
|
|
65
|
+
reportError: (
|
|
66
|
+
error: unknown,
|
|
67
|
+
context?: { code?: string; attributes?: Record<string, string | number | boolean | undefined> },
|
|
68
|
+
) => {
|
|
69
|
+
reported.push({
|
|
70
|
+
code: context?.code,
|
|
71
|
+
attributes: context?.attributes,
|
|
72
|
+
message: error instanceof Error ? error.message : String(error),
|
|
73
|
+
})
|
|
74
|
+
},
|
|
75
|
+
shutdown: async () => {},
|
|
76
|
+
} as unknown as TelemetryRuntime
|
|
77
|
+
return { runtime, reported }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** What BullMQ hands its `failed` listener: `attemptsMade` already counts the attempt that just failed. */
|
|
81
|
+
function failedJob(overrides: { attemptsMade: number; opts?: { attempts?: number } }) {
|
|
82
|
+
return {
|
|
83
|
+
id: 'job-1',
|
|
84
|
+
data: { id: 'job-1', payload: { runId: 'run-1' }, createdAt: new Date(0).toISOString() } satisfies QueuedJob<{ runId: string }>,
|
|
85
|
+
...overrides,
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function emit(event: string, ...args: unknown[]): void {
|
|
90
|
+
for (const listener of capturedListeners.get(event) ?? []) listener(...args)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
describe('queue job failures are reported, not only logged', () => {
|
|
94
|
+
afterEach(() => {
|
|
95
|
+
resetTelemetryRuntime()
|
|
96
|
+
capturedListeners.clear()
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
describe('local strategy', () => {
|
|
100
|
+
const origCwd = process.cwd()
|
|
101
|
+
let tmp: string
|
|
102
|
+
|
|
103
|
+
beforeEach(() => {
|
|
104
|
+
tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'queue-report-'))
|
|
105
|
+
process.chdir(tmp)
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
afterEach(() => {
|
|
109
|
+
process.chdir(origCwd)
|
|
110
|
+
try { fs.rmSync(tmp, { recursive: true, force: true }) } catch {}
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
it('reports a failed job with the queue and attempt', async () => {
|
|
114
|
+
const { runtime, reported } = runtimeStub()
|
|
115
|
+
registerTelemetryRuntime(runtime)
|
|
116
|
+
const queue = createQueue<{ value: number }>('data-sync', 'local')
|
|
117
|
+
await queue.enqueue({ value: 1 })
|
|
118
|
+
|
|
119
|
+
await queue.process(() => { throw new Error('import batch blew up') }, { limit: 10 })
|
|
120
|
+
|
|
121
|
+
expect(reported).toEqual([
|
|
122
|
+
expect.objectContaining({
|
|
123
|
+
code: 'queue.job_failed',
|
|
124
|
+
message: 'import batch blew up',
|
|
125
|
+
attributes: expect.objectContaining({ queue: 'data-sync', attemptNumber: 1 }),
|
|
126
|
+
}),
|
|
127
|
+
])
|
|
128
|
+
|
|
129
|
+
await queue.close()
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
it('reports the final attempt as exhausted, exactly once', async () => {
|
|
133
|
+
const { runtime, reported } = runtimeStub()
|
|
134
|
+
registerTelemetryRuntime(runtime)
|
|
135
|
+
const queue = createQueue<{ value: number }>('data-sync', 'local')
|
|
136
|
+
await queue.enqueue({ value: 1 })
|
|
137
|
+
|
|
138
|
+
const queuePath = path.join('.mercato', 'queue', 'data-sync', 'queue.json')
|
|
139
|
+
const jobs = JSON.parse(fs.readFileSync(queuePath, 'utf8')) as Array<Record<string, unknown>>
|
|
140
|
+
jobs[0].attemptCount = 2
|
|
141
|
+
jobs[0].availableAt = undefined
|
|
142
|
+
fs.writeFileSync(queuePath, JSON.stringify(jobs, null, 2), 'utf8')
|
|
143
|
+
|
|
144
|
+
await queue.process(() => { throw new Error('permanent') }, { limit: 10 })
|
|
145
|
+
|
|
146
|
+
// One report, not two: a dead-lettered job would otherwise count twice in
|
|
147
|
+
// `om.errors` and raise two issues in a Sentry-shaped backend.
|
|
148
|
+
expect(reported).toEqual([
|
|
149
|
+
expect.objectContaining({
|
|
150
|
+
code: 'queue.job_exhausted',
|
|
151
|
+
message: 'permanent',
|
|
152
|
+
attributes: expect.objectContaining({ queue: 'data-sync', attemptNumber: 3 }),
|
|
153
|
+
}),
|
|
154
|
+
])
|
|
155
|
+
|
|
156
|
+
await queue.close()
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
it('is a no-op with telemetry off', async () => {
|
|
160
|
+
const queue = createQueue<{ value: number }>('data-sync', 'local')
|
|
161
|
+
await queue.enqueue({ value: 1 })
|
|
162
|
+
|
|
163
|
+
const result = await queue.process(() => { throw new Error('boom') }, { limit: 10 })
|
|
164
|
+
|
|
165
|
+
expect(result?.failed).toBe(1)
|
|
166
|
+
|
|
167
|
+
await queue.close()
|
|
168
|
+
})
|
|
169
|
+
})
|
|
170
|
+
|
|
171
|
+
describe('async strategy', () => {
|
|
172
|
+
it('reports a job the queue failed, alongside the log line', async () => {
|
|
173
|
+
const { runtime, reported } = runtimeStub()
|
|
174
|
+
registerTelemetryRuntime(runtime)
|
|
175
|
+
const queue = createQueue<{ runId: string }>('data-sync', 'async')
|
|
176
|
+
await queue.process(async () => {})
|
|
177
|
+
|
|
178
|
+
emit('failed', failedJob({ attemptsMade: 1 }), new Error('handler rethrew after marking the run failed'))
|
|
179
|
+
|
|
180
|
+
expect(reported).toEqual([
|
|
181
|
+
expect.objectContaining({
|
|
182
|
+
code: 'queue.job_failed',
|
|
183
|
+
message: 'handler rethrew after marking the run failed',
|
|
184
|
+
attributes: expect.objectContaining({ queue: 'data-sync', jobId: 'job-1', attemptNumber: 1 }),
|
|
185
|
+
}),
|
|
186
|
+
])
|
|
187
|
+
|
|
188
|
+
await queue.close()
|
|
189
|
+
})
|
|
190
|
+
|
|
191
|
+
// The dead-letter signal is what an operator pages on, so the two strategies
|
|
192
|
+
// must agree on it: an alert written against one has to hold for the other.
|
|
193
|
+
it('reports the final attempt as exhausted, exactly once', async () => {
|
|
194
|
+
const { runtime, reported } = runtimeStub()
|
|
195
|
+
registerTelemetryRuntime(runtime)
|
|
196
|
+
const queue = createQueue<{ runId: string }>('data-sync', 'async')
|
|
197
|
+
await queue.process(async () => {})
|
|
198
|
+
|
|
199
|
+
emit('failed', failedJob({ attemptsMade: 3 }), new Error('permanent'))
|
|
200
|
+
|
|
201
|
+
expect(reported).toEqual([
|
|
202
|
+
expect.objectContaining({
|
|
203
|
+
code: 'queue.job_exhausted',
|
|
204
|
+
message: 'permanent',
|
|
205
|
+
attributes: expect.objectContaining({ queue: 'data-sync', jobId: 'job-1', attemptNumber: 3 }),
|
|
206
|
+
}),
|
|
207
|
+
])
|
|
208
|
+
|
|
209
|
+
await queue.close()
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
it('honours a per-job attempts override rather than the strategy default', async () => {
|
|
213
|
+
const { runtime, reported } = runtimeStub()
|
|
214
|
+
registerTelemetryRuntime(runtime)
|
|
215
|
+
const queue = createQueue<{ runId: string }>('data-sync', 'async')
|
|
216
|
+
await queue.process(async () => {})
|
|
217
|
+
|
|
218
|
+
emit('failed', failedJob({ attemptsMade: 1, opts: { attempts: 1 } }), new Error('no retries wanted'))
|
|
219
|
+
|
|
220
|
+
expect(reported.map((entry) => entry.code)).toEqual(['queue.job_exhausted'])
|
|
221
|
+
|
|
222
|
+
await queue.close()
|
|
223
|
+
})
|
|
224
|
+
|
|
225
|
+
it('reports a worker-level error', async () => {
|
|
226
|
+
const { runtime, reported } = runtimeStub()
|
|
227
|
+
registerTelemetryRuntime(runtime)
|
|
228
|
+
const queue = createQueue<{ runId: string }>('data-sync', 'async')
|
|
229
|
+
await queue.process(async () => {})
|
|
230
|
+
|
|
231
|
+
emit('error', new Error('redis connection lost'))
|
|
232
|
+
|
|
233
|
+
expect(reported).toEqual([
|
|
234
|
+
expect.objectContaining({ code: 'queue.worker_error', message: 'redis connection lost' }),
|
|
235
|
+
])
|
|
236
|
+
|
|
237
|
+
await queue.close()
|
|
238
|
+
})
|
|
239
|
+
})
|
|
240
|
+
})
|
package/src/strategies/async.ts
CHANGED
|
@@ -195,6 +195,29 @@ export function createAsyncQueue<T = unknown>(
|
|
|
195
195
|
const onJobAbandoned = options?.onJobAbandoned
|
|
196
196
|
const logger = packageLogger.child({ queue: name })
|
|
197
197
|
|
|
198
|
+
/**
|
|
199
|
+
* Report a queue-level failure outward. Wrapped so a telemetry fault can never
|
|
200
|
+
* escape into an EventEmitter handler, where an unhandled rejection is fatal to
|
|
201
|
+
* the process.
|
|
202
|
+
*/
|
|
203
|
+
function reportQueueError(
|
|
204
|
+
error: unknown,
|
|
205
|
+
code: string,
|
|
206
|
+
attributes?: Record<string, string | number | undefined>,
|
|
207
|
+
): void {
|
|
208
|
+
try {
|
|
209
|
+
getTelemetryRuntime()?.reportError(error, {
|
|
210
|
+
module: 'queue',
|
|
211
|
+
code,
|
|
212
|
+
attributes: { queue: name, ...attributes },
|
|
213
|
+
})
|
|
214
|
+
} catch (telemetryError) {
|
|
215
|
+
// Reporting is never worth a worker — but a systematically broken bridge
|
|
216
|
+
// must not be silent either, or a worker stops reporting and nothing says so.
|
|
217
|
+
logger.warn('Failed to report a queue error to telemetry', { code, err: telemetryError as Error })
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
198
221
|
let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
|
|
199
222
|
let bullWorker: BullWorkerInterface | null = null
|
|
200
223
|
let bullmqModule: BullMQModule | null = null
|
|
@@ -213,6 +236,12 @@ export function createAsyncQueue<T = unknown>(
|
|
|
213
236
|
updateData?: (data: QueuedJob<T>) => Promise<void>
|
|
214
237
|
}
|
|
215
238
|
|
|
239
|
+
/** What BullMQ hands the `failed` handler, beyond the abandonment fields. */
|
|
240
|
+
type FailedJobRecord = AbandonedJobRecord & {
|
|
241
|
+
attemptsMade?: number
|
|
242
|
+
opts?: { attempts?: number }
|
|
243
|
+
}
|
|
244
|
+
|
|
216
245
|
// The acknowledgement that makes delivery at-least-once: written only after the callback returns,
|
|
217
246
|
// so a callback that threw or a process that died mid-report leaves the job unmarked and a later
|
|
218
247
|
// sweep retries it. Requires the driver to expose `updateData`; without it the report simply stays
|
|
@@ -243,6 +272,7 @@ export function createAsyncQueue<T = unknown>(
|
|
|
243
272
|
jobId,
|
|
244
273
|
err: hookError as Error,
|
|
245
274
|
})
|
|
275
|
+
reportQueueError(hookError as Error, 'queue.abandon_report_failed', { jobId: jobId ?? undefined })
|
|
246
276
|
return
|
|
247
277
|
}
|
|
248
278
|
try {
|
|
@@ -252,6 +282,7 @@ export function createAsyncQueue<T = unknown>(
|
|
|
252
282
|
jobId,
|
|
253
283
|
err: ackError as Error,
|
|
254
284
|
})
|
|
285
|
+
reportQueueError(ackError as Error, 'queue.abandon_ack_failed', { jobId: jobId ?? undefined })
|
|
255
286
|
}
|
|
256
287
|
})().finally(() => {
|
|
257
288
|
inFlightAbandonedJobIds.delete(payload.id)
|
|
@@ -289,6 +320,7 @@ export function createAsyncQueue<T = unknown>(
|
|
|
289
320
|
}
|
|
290
321
|
} catch (sweepError) {
|
|
291
322
|
logger.error('Abandoned-job sweep failed', { err: sweepError as Error })
|
|
323
|
+
reportQueueError(sweepError as Error, 'queue.abandon_sweep_failed')
|
|
292
324
|
}
|
|
293
325
|
}
|
|
294
326
|
// Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or
|
|
@@ -412,9 +444,30 @@ export function createAsyncQueue<T = unknown>(
|
|
|
412
444
|
})
|
|
413
445
|
|
|
414
446
|
bullWorker.on('failed', (job, err) => {
|
|
415
|
-
const failedJob = job as
|
|
447
|
+
const failedJob = job as FailedJobRecord | undefined
|
|
416
448
|
const error = err as Error
|
|
417
|
-
|
|
449
|
+
// BullMQ counts `attemptsMade` on the job itself, so the last delivery is
|
|
450
|
+
// identifiable here without any bookkeeping of our own. Its per-job
|
|
451
|
+
// `opts.attempts` wins over the strategy default, because a caller may have
|
|
452
|
+
// overridden it.
|
|
453
|
+
const attemptNumber = failedJob?.attemptsMade ?? 0
|
|
454
|
+
const maxAttempts = failedJob?.opts?.attempts ?? attempts
|
|
455
|
+
const exhausted = attemptNumber >= maxAttempts
|
|
456
|
+
logger.error('Job failed', { jobId: failedJob?.id, attemptNumber, maxAttempts, err: error })
|
|
457
|
+
// A log line reaches the backend's LOGS signal; a job that died is an error
|
|
458
|
+
// and belongs in the error signal too, with a code the backend can group on.
|
|
459
|
+
// This is the only outward record for a handler that rethrows after doing its
|
|
460
|
+
// own bookkeeping — a sync worker marking its run `failed`, for instance.
|
|
461
|
+
//
|
|
462
|
+
// One report per failure, coded by what the failure means: a job that has
|
|
463
|
+
// burned every retry is dead-lettered, which is the condition an operator
|
|
464
|
+
// pages on, and it must be distinguishable from a first attempt that BullMQ
|
|
465
|
+
// will simply retry. Matches the local strategy exactly, so an alert written
|
|
466
|
+
// against one holds for the other.
|
|
467
|
+
reportQueueError(error, exhausted ? 'queue.job_exhausted' : 'queue.job_failed', {
|
|
468
|
+
jobId: failedJob?.id,
|
|
469
|
+
attemptNumber,
|
|
470
|
+
})
|
|
418
471
|
|
|
419
472
|
if (!onJobAbandoned) return
|
|
420
473
|
// Any other reason means a handler ran and threw. That failure is the handler's own and it has
|
|
@@ -445,6 +498,7 @@ export function createAsyncQueue<T = unknown>(
|
|
|
445
498
|
bullWorker.on('error', (err) => {
|
|
446
499
|
const error = err as Error
|
|
447
500
|
logger.error('Worker error', { err: error })
|
|
501
|
+
reportQueueError(error, 'queue.worker_error')
|
|
448
502
|
})
|
|
449
503
|
|
|
450
504
|
if (onJobAbandoned) {
|
package/src/strategies/local.ts
CHANGED
|
@@ -2,6 +2,7 @@ import fs from 'node:fs'
|
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import crypto from 'node:crypto'
|
|
4
4
|
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
5
|
+
import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
|
|
5
6
|
import type { Queue, QueuedJob, JobHandler, LocalQueueOptions, ProcessOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'
|
|
6
7
|
import { attachTraceMetadata, runJobInTrace } from '../tracing'
|
|
7
8
|
|
|
@@ -124,6 +125,28 @@ export function createLocalQueue<T = unknown>(
|
|
|
124
125
|
const lockDir = path.join(queueDir, 'queue.lock')
|
|
125
126
|
const lockOwnerFile = path.join(lockDir, 'owner')
|
|
126
127
|
const logger = packageLogger.child({ queue: name })
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Report a job failure outward, so the log line is not the only record of it.
|
|
131
|
+
* Wrapped: reporting is never worth a poll cycle.
|
|
132
|
+
*/
|
|
133
|
+
function reportQueueError(
|
|
134
|
+
error: unknown,
|
|
135
|
+
code: string,
|
|
136
|
+
attributes?: Record<string, string | number | undefined>,
|
|
137
|
+
): void {
|
|
138
|
+
try {
|
|
139
|
+
getTelemetryRuntime()?.reportError(error, {
|
|
140
|
+
module: 'queue',
|
|
141
|
+
code,
|
|
142
|
+
attributes: { queue: name, ...attributes },
|
|
143
|
+
})
|
|
144
|
+
} catch (telemetryError) {
|
|
145
|
+
// Reporting is never worth a worker — but a systematically broken bridge
|
|
146
|
+
// must not be silent either, or a worker stops reporting and nothing says so.
|
|
147
|
+
logger.warn('Failed to report a queue error to telemetry', { code, err: telemetryError as Error })
|
|
148
|
+
}
|
|
149
|
+
}
|
|
127
150
|
// Note: concurrency is stored for logging/compatibility but jobs are processed sequentially
|
|
128
151
|
const concurrency = options?.concurrency ?? 1
|
|
129
152
|
const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL
|
|
@@ -493,9 +516,19 @@ export function createLocalQueue<T = unknown>(
|
|
|
493
516
|
logger.info('Job completed', { jobId: job.id })
|
|
494
517
|
} catch (error) {
|
|
495
518
|
logger.error('Job failed', { jobId: job.id, attemptNumber, maxAttempts: DEFAULT_MAX_ATTEMPTS, err: error })
|
|
519
|
+
const exhausted = attemptNumber >= DEFAULT_MAX_ATTEMPTS
|
|
520
|
+
// One report per failure, coded by what the failure means: a job that
|
|
521
|
+
// has burned every retry is dead, which is the condition an operator
|
|
522
|
+
// pages on, and it must be distinguishable from a first attempt that
|
|
523
|
+
// will simply be retried. Reporting both would double-count `om.errors`
|
|
524
|
+
// on the final attempt.
|
|
525
|
+
reportQueueError(error, exhausted ? 'queue.job_exhausted' : 'queue.job_failed', {
|
|
526
|
+
jobId: job.id,
|
|
527
|
+
attemptNumber,
|
|
528
|
+
})
|
|
496
529
|
failed++
|
|
497
530
|
lastJobId = job.id
|
|
498
|
-
if (
|
|
531
|
+
if (exhausted) {
|
|
499
532
|
logger.error('Job exhausted all attempts; dropping it (no dead-letter store)', { jobId: job.id, maxAttempts: DEFAULT_MAX_ATTEMPTS })
|
|
500
533
|
deadJobIds.add(job.id)
|
|
501
534
|
} else {
|