bullmq 6.3.10 → 6.3.12

package/dist/cjs/classes/worker.js

@@ -46,7 +46,11 @@ class Worker extends queue_base_1.QueueBase {
         this.drained = false;
         this.limitUntil = 0;
         this.processorAcceptsSignal = false;
-        this.stalledCheckerRunning = false;
+        // Bumped whenever the currently running stalled-checker loop should be
+        // abandoned (e.g. by `pause()`). The loop compares its own captured value
+        // against the live counter on every iteration and exits as soon as they
+        // diverge, without anyone having to await it.
+        this.stalledCheckerGeneration = 0;
         this.waiting = null;
         this.running = false;
         this.mainLoopRunning = null;
@@ -671,6 +675,12 @@ class Worker extends queue_base_1.QueueBase {
      * Pauses the processing of this queue only for this worker.
      */
     async pause(doNotWaitActive) {
+        // Kept in the caller's active context (not detached): a correctly
+        // implemented AsyncLocalStorage-based context manager isolates concurrent
+        // async chains, so a background stalled-checker tick (which detaches its
+        // own context; see `stalledChecker()`) cannot overwrite this span's
+        // context. Staying attached lets applications correlate `pause` with the
+        // request or operation that initiated it.
         await this.trace(enums_1.SpanKind.INTERNAL, 'pause', this.name, async (span) => {
             var _a;
             span === null || span === void 0 ? void 0 : span.setAttributes({
@@ -683,6 +693,14 @@ class Worker extends queue_base_1.QueueBase {
                 if (!doNotWaitActive) {
                     await this.whenCurrentJobsFinished();
                 }
+                // Invalidate the currently running stalled-checker loop and cancel
+                // its pending timer. This is intentionally fire-and-forget: the
+                // loop may be in the middle of an in-flight Redis call (which, on
+                // a connection error, retries with its own delay), so awaiting it
+                // here could block `pause()` for an unbounded amount of time. The
+                // loop itself checks the generation on every iteration and will
+                // exit on its own as soon as it next has a chance to.
+                this.stalledCheckerGeneration++;
                 (_a = this.stalledCheckStopper) === null || _a === void 0 ? void 0 : _a.call(this);
                 this.emit('paused');
             }
@@ -695,6 +713,10 @@ class Worker extends queue_base_1.QueueBase {
     async resume() {
         try {
             if (!this.running || this.paused) {
+                let restartStalledChecker = false;
+                // Kept in the caller's active context for the same reason as
+                // `pause()`: only the background stalled-checker loop needs to
+                // detach its own context, so it cannot leak into this span.
                 await this.trace(enums_1.SpanKind.INTERNAL, 'resume', this.name, async (span) => {
                     span === null || span === void 0 ? void 0 : span.setAttributes({
                         [enums_1.TelemetryAttributes.WorkerId]: this.id,
@@ -709,10 +731,15 @@ class Worker extends queue_base_1.QueueBase {
                     else {
                         // Main loop is still running (pause was called with doNotWaitActive=true).
                         // Restart the stalled checker since pause() stopped it.
-                        await this.startStalledCheckTimer();
+                        restartStalledChecker = true;
                     }
                     this.emit('resumed');
                 });
+                // Started outside of the trace above so that the stalled checker loop
+                // does not inherit the (already ended) `resume` span as its parent.
+                if (restartStalledChecker) {
+                    await this.startStalledCheckTimer();
+                }
             }
         }
         catch (error) {
@@ -801,28 +828,88 @@ class Worker extends queue_base_1.QueueBase {
      * @see {@link https://docs.bullmq.io/patterns/manually-fetching-jobs}
      */
     async startStalledCheckTimer() {
-        if (!this.opts.skipStalledCheck) {
-            if (!this.closing && !this.stalledCheckerRunning) {
-                await this.trace(enums_1.SpanKind.INTERNAL, 'startStalledCheckTimer', this.name, async (span) => {
-                    span === null || span === void 0 ? void 0 : span.setAttributes({
-                        [enums_1.TelemetryAttributes.WorkerId]: this.id,
-                        [enums_1.TelemetryAttributes.WorkerName]: this.opts.name,
-                    });
-                    this.stalledCheckerRunning = true;
-                    this.stalledChecker()
-                        .catch(err => {
-                        this.emit('error', err);
-                    })
-                        .finally(() => {
-                        this.stalledCheckerRunning = false;
-                    });
-                });
-            }
+        var _a, _b;
+        if (this.opts.skipStalledCheck) {
+            return;
         }
+        if (this.closing) {
+            return;
+        }
+        // Claim a new generation for this loop synchronously (before the first
+        // await), so that two concurrent callers (e.g. the automatic `run()`
+        // startup racing a manual `startStalledCheckTimer()` call, or `resume()`
+        // racing a not-yet-finished old loop that `pause()` just invalidated)
+        // cannot both end up owning a checker loop: only the call that captures
+        // the latest generation value proceeds past the check below.
+        const generation = ++this.stalledCheckerGeneration;
+        // Capture whatever loop (if any) is still winding down. Bumping the
+        // generation above only makes it stop at its next checkpoint; it does
+        // not cancel an in-flight `moveStalledJobsToWait` call, so we must let it
+        // fully finish before starting a new loop, otherwise both loops could run
+        // concurrently and scan for stalled jobs at the same time. This is
+        // chained below rather than awaited here in the public call path: an
+        // in-flight Redis call can be stuck behind `checkConnectionError`'s retry
+        // delay (or a slow reconnect) well after the worker has been paused, and
+        // callers such as `resume()` must return promptly instead of blocking on
+        // it, exactly like the previous "reuse the still-running checker"
+        // behavior used to.
+        const previousStalledCheckerPromise = this.stalledCheckerPromise;
+        // Wake up the previous loop's pending interval timer (if any) so it can
+        // observe the new generation and exit promptly, instead of leaving it to
+        // wait out the remainder of its `stalledInterval` wait.
+        (_a = this.stalledCheckStopper) === null || _a === void 0 ? void 0 : _a.call(this);
+        await this.trace(enums_1.SpanKind.INTERNAL, 'startStalledCheckTimer', this.name, async (span) => {
+            span === null || span === void 0 ? void 0 : span.setAttributes({
+                [enums_1.TelemetryAttributes.WorkerId]: this.id,
+                [enums_1.TelemetryAttributes.WorkerName]: this.opts.name,
+            });
+        });
+        // The trace above is asynchronous, so the worker may have been closed, or
+        // another call may have claimed a newer generation, while it was in
+        // flight.
+        if (this.closing || generation !== this.stalledCheckerGeneration) {
+            return;
+        }
+        // Chain the new loop to start only once the previous one has fully
+        // exited (its errors are already handled by its own `.catch` below, so
+        // swallow them here), guaranteeing the two never scan concurrently -
+        // without awaiting that settlement from this public method, so
+        // `startStalledCheckTimer()`/`resume()` return promptly regardless of how
+        // long the previous loop's in-flight Redis call takes to settle.
+        this.stalledCheckerPromise = ((_b = previousStalledCheckerPromise === null || previousStalledCheckerPromise === void 0 ? void 0 : previousStalledCheckerPromise.catch(() => undefined)) !== null && _b !== void 0 ? _b : Promise.resolve())
+            .then(() => {
+            // The wait above can take a while (e.g. a connection error retry
+            // delay), during which the worker may have closed or yet another
+            // caller may have claimed a newer generation.
+            if (this.closing || generation !== this.stalledCheckerGeneration) {
+                return;
+            }
+            return (0, utils_1.withDetachedContext)(this.opts.telemetry, () => this.stalledChecker(generation));
+        })
+            .catch(err => {
+            this.emit('error', err);
+        });
     }
-    async stalledChecker() {
-        while (!(this.closing || this.paused)) {
+    async stalledChecker(generation) {
+        // The entire loop (including its recurring interval timer) is started
+        // under a detached root context by the caller above. Context managers
+        // backed by `AsyncLocalStorage` associate every continuation created
+        // inside a callback with whatever context was active when that
+        // continuation was created; detaching only the individual Redis call on
+        // each tick would still leave the long-lived `setTimeout` chain (and
+        // therefore every subsequent tick) parented to the caller's ambient span
+        // for the worker's whole lifetime, since a later no-op detached callback
+        // cannot retroactively replace a store that AsyncLocalStorage already
+        // restored to the outer (caller) context once the initial detached call
+        // returned.
+        while (!(this.closing || this.paused) &&
+            generation === this.stalledCheckerGeneration) {
             await this.checkConnectionError(() => this.moveStalledJobsToWait());
+            if (this.closing ||
+                this.paused ||
+                generation !== this.stalledCheckerGeneration) {
+                break;
+            }
             await new Promise(resolve => {
                 const timeout = setTimeout(resolve, this.opts.stalledInterval);
                 this.stalledCheckStopper = () => {

package/dist/cjs/postgres/sql-loader.js

@@ -2,65 +2,51 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadMigrationSql = loadMigrationSql;
 exports.loadCommandSql = loadCommandSql;
-const fs_1 = require("fs");
-const path_1 = require("path");
-const url_1 = require("url");
-function getDirname() {
-    if (typeof __dirname !== 'undefined' && __dirname) {
-        return __dirname;
-    }
-    const stack = new Error().stack || '';
-    for (const line of stack.split('\n')) {
-        const match = line.match(/(file:\/\/\/[^\s)]+)/);
-        if (match) {
-            try {
-                return (0, path_1.dirname)((0, url_1.fileURLToPath)(match[1]));
-            }
-            catch (_a) {
-                // continue searching
-            }
-        }
-    }
-    throw new Error('Could not determine sql-loader directory path');
-}
-const currentDir = getDirname();
+const sql_scripts_1 = require("./sql-scripts");
 /**
- * Loads a migration's SQL from its `.sql` file — the portable source of truth
- * shared with the Elixir/Python ports. Results are cached after the first read.
+ * Loads a migration's SQL — the portable source of truth shared with the
+ * Elixir/Python ports. Results are cached after the first lookup.
  *
- * The `.sql` files live next to this module under `migrations/`. The published
- * build copies them alongside the compiled output (a `copy:sql` build step,
- * analogous to how the Lua scripts are bundled), so the same relative lookup
- * works at runtime.
- */
-const MIGRATIONS_DIR = (0, path_1.join)(currentDir, 'migrations');
-/**
- * Runtime queries live under `commands/`. Each `.sql` file is one parameterized
- * statement (a `SELECT fn(...)` for the PL/pgSQL operations, or a direct
- * query). They contain NO schema/namespace references — the connection's
- * `search_path` selects the schema — so they are portable verbatim to the
- * Python/Elixir/PHP/Rust ports (mirroring how the Redis backend's `.lua`
- * scripts never hardcode the key prefix).
+ * The SQL is inlined into the published JavaScript at build time (see
+ * scripts/generateSqlScripts.js), exactly like the Redis backend's `.lua`
+ * scripts, so no filesystem access is needed at runtime. This keeps the
+ * PostgreSQL backend usable from single-file bundles (bun build --compile,
+ * esbuild --bundle, pkg, Node SEA, deno compile) where the `.sql` files are
+ * not shipped alongside the code:
+ *
+ * @see https://github.com/taskforcesh/bullmq/issues/4603
+ *
+ * @param file - Migration filename, e.g. `0001_schema.sql`.
  */
-const COMMANDS_DIR = (0, path_1.join)(currentDir, 'commands');
 const migrationCache = new Map();
 const commandCache = new Map();
 function loadMigrationSql(file) {
     let sql = migrationCache.get(file);
     if (sql === undefined) {
-        sql = (0, fs_1.readFileSync)((0, path_1.join)(MIGRATIONS_DIR, file), 'utf8');
+        sql = sql_scripts_1.SQL_MIGRATIONS[file];
+        if (sql === undefined) {
+            throw new Error(`Could not find migration SQL for '${file}'`);
+        }
         migrationCache.set(file, sql);
     }
     return sql;
 }
 /**
  * Loads a runtime command's SQL by name (without the `.sql` extension), cached
- * after the first read.
+ * after the first lookup. Runtime queries contain NO schema/namespace
+ * references — the connection's `search_path` selects the schema — so they are
+ * portable verbatim to the Python/Elixir/PHP/Rust ports (mirroring how the
+ * Redis backend's `.lua` scripts never hardcode the key prefix).
+ *
+ * @param name - Command name, e.g. `add_job`.
  */
 function loadCommandSql(name) {
     let sql = commandCache.get(name);
     if (sql === undefined) {
-        sql = (0, fs_1.readFileSync)((0, path_1.join)(COMMANDS_DIR, `${name}.sql`), 'utf8');
+        sql = sql_scripts_1.SQL_COMMANDS[`${name}.sql`];
+        if (sql === undefined) {
+            throw new Error(`Could not find command SQL for '${name}'`);
+        }
         commandCache.set(name, sql);
     }
     return sql;