@pauldeng/node-red-contrib-bullmq 1.0.2 → 2.0.0

package/lib/connections.js

@@ -1,8 +1,17 @@
 "use strict";
 
-const DEFAULT_REDIS_HOST = "localhost";
+const { isIP } = require("node:net");
+
+// Shared by both backends; "localhost" is the right default either way.
+const DEFAULT_HOST = "localhost";
 const DEFAULT_REDIS_PORT = 6379;
 const CLUSTER_PREFIX = "{bull}";
+const DEFAULT_POSTGRES_PORT = 5432;
+const DEFAULT_POSTGRES_POOL_MAX = 2;
+// The only fail-fast lever on the PostgreSQL path (skipWaitingForReady is
+// Redis-only) -- matches the Redis path's connectTimeout.
+const POSTGRES_CONNECTION_TIMEOUT_MS = 10000;
+const VALID_BACKENDS = ["redis", "postgres"];
 
 function isPresent(value) {
   return value !== undefined && value !== null && value !== "";
@@ -21,18 +30,50 @@ function toBoolean(value, defaultValue = false) {
   return ["true", "1", "yes", "on"].includes(String(value).toLowerCase());
 }
 
-function toPort(value, defaultValue = DEFAULT_REDIS_PORT) {
+function toPort(value, defaultValue = DEFAULT_REDIS_PORT, label = "Redis") {
   if (!isPresent(value)) {
     return defaultValue;
   }
   const port = Number(value);
   if (!Number.isInteger(port) || port < 1 || port > 65535) {
-    throw new Error(`Invalid Redis port: ${value}`);
+    throw new Error(`Invalid ${label} port: ${value}`);
   }
   return port;
 }
 
-function parseEndpoint(endpoint, defaultPort = DEFAULT_REDIS_PORT) {
+// A saved 2.0.0 flow has no backend property at all, so absent/blank MUST
+// mean "redis" -- that is the only way an existing flow keeps working
+// untouched.
+function normalizeBackend(config) {
+  if (!isPresent(config.backend)) {
+    return "redis";
+  }
+  const backend = String(config.backend).trim().toLowerCase();
+  if (!VALID_BACKENDS.includes(backend)) {
+    throw new Error(`Unsupported backend: ${config.backend}`);
+  }
+  return backend;
+}
+
+function toPoolMax(value) {
+  if (!isPresent(value)) {
+    return DEFAULT_POSTGRES_POOL_MAX;
+  }
+  const max = Number(value);
+  if (!Number.isInteger(max) || max < 1) {
+    throw new Error(
+      "PostgreSQL pool max must be a positive whole number of connections",
+    );
+  }
+  return max;
+}
+
+function parseEndpoint(
+  endpoint,
+  defaultPort = DEFAULT_REDIS_PORT,
+  tls,
+  tlsName = "TLS",
+) {
   if (typeof endpoint === "object" && endpoint !== null) {
     const host = endpoint.host || endpoint.address;
     if (!isPresent(host)) {
@@ -48,36 +89,54 @@ function parseEndpoint(endpoint, defaultPort = DEFAULT_REDIS_PORT) {
   if (!text) {
     throw new Error("Redis endpoint cannot be empty");
   }
+  if (isIP(text)) {
+    return { host: text, port: defaultPort };
+  }
 
-  let host = text;
-  let port = defaultPort;
-  const urlMatch = text.match(
-    /^(?:redis|rediss):\/\/(?:[^@]+@)?([^/:]+)(?::(\d+))?/i,
-  );
-  if (urlMatch) {
-    host = urlMatch[1];
-    port = toPort(urlMatch[2], defaultPort);
-  } else if (text.includes(":")) {
-    const parts = text.split(":");
-    port = toPort(parts.pop(), defaultPort);
-    host = parts.join(":");
+  const hasScheme = /^[a-z][a-z\d+.-]*:\/\//i.test(text);
+  let url;
+  try {
+    url = new URL(hasScheme ? text : `redis://${text}`);
+  } catch {
+    throw new Error(`Invalid Redis endpoint: ${text}`);
+  }
+  if (url.protocol !== "redis:" && url.protocol !== "rediss:") {
+    throw new Error(`Unsupported Redis endpoint protocol: ${url.protocol}`);
+  }
+  if (url.username || url.password) {
+    throw new Error(
+      "Redis endpoint URLs cannot include credentials; use the config node credential fields",
+    );
+  }
+  if (hasScheme && tls !== undefined) {
+    const secure = url.protocol === "rediss:";
+    if (secure !== tls) {
+      throw new Error(
+        `${url.protocol}// endpoint requires ${tlsName} to be ${secure ? "enabled" : "disabled"}`,
+      );
+    }
   }
 
-  if (!host.trim()) {
+  const host = url.hostname.replace(/^\[|\]$/g, "");
+  if (!host) {
     throw new Error(`Redis endpoint requires a host: ${text}`);
   }
-
-  return { host: host.trim(), port };
+  return { host, port: toPort(url.port, defaultPort) };
 }
 
-function parseEndpointList(value, defaultPort = DEFAULT_REDIS_PORT) {
+function parseEndpointList(
+  value,
+  defaultPort = DEFAULT_REDIS_PORT,
+  tls,
+  tlsName = "TLS",
+) {
   if (Array.isArray(value)) {
     return value
       .flatMap((item) =>
         typeof item === "string" ? item.split(/[\n,]+/) : [item],
       )
       .filter((item) => isPresent(item))
-      .map((item) => parseEndpoint(item, defaultPort));
+      .map((item) => parseEndpoint(item, defaultPort, tls, tlsName));
   }
 
   if (!isPresent(value)) {
@@ -88,33 +147,95 @@ function parseEndpointList(value, defaultPort = DEFAULT_REDIS_PORT) {
     .split(/[\n,]+/)
     .map((item) => item.trim())
     .filter(Boolean)
-    .map((item) => parseEndpoint(item, defaultPort));
+    .map((item) => parseEndpoint(item, defaultPort, tls, tlsName));
+}
+
+function readSecret(credentials, name) {
+  return credentials && isPresent(credentials[name])
+    ? credentials[name]
+    : undefined;
 }
 
-function readSecret(config, credentials, name) {
-  if (credentials && isPresent(credentials[name])) {
-    return credentials[name];
+// Blank means "keep every job", which is BullMQ's own default and grows
+// without bound. A count keeps the newest N and is what production needs.
+function toKeepCount(value, name) {
+  if (!isPresent(value)) {
+    return undefined;
   }
-  return config[name];
+  const count = Number(value);
+  if (!Number.isInteger(count) || count < 0) {
+    throw new Error(
+      `${name} must be a non-negative whole number of jobs to keep`,
+    );
+  }
+  return count;
+}
+
+function buildDefaultJobOptions(config) {
+  const defaults = {};
+  const removeOnComplete = toKeepCount(
+    config.removeOnComplete,
+    "removeOnComplete",
+  );
+  const removeOnFail = toKeepCount(config.removeOnFail, "removeOnFail");
+  if (removeOnComplete !== undefined) {
+    defaults.removeOnComplete = removeOnComplete;
+  }
+  if (removeOnFail !== undefined) {
+    defaults.removeOnFail = removeOnFail;
+  }
+  return Object.keys(defaults).length > 0 ? defaults : undefined;
 }
 
 function normalizeQueueConfig(config = {}, credentials = {}) {
-  const deploymentInput = String(
-    config.deployment || config.mode || config.redisMode || "single",
-  ).toLowerCase();
-  const deployment =
-    deploymentInput === "memorydb" ? "cluster" : deploymentInput;
+  const removedAlias = [
+    "mode",
+    "redisMode",
+    "host",
+    "queueName",
+    "startupNodes",
+    "masterName",
+    "nameOfMaster",
+  ].find((name) => isPresent(config[name]));
+  if (removedAlias) {
+    throw new Error(`Unsupported config field: ${removedAlias}`);
+  }
+  const plaintextSecret = [
+    "password",
+    "sentinelPassword",
+    "tlsCa",
+    "tlsCert",
+    "tlsKey",
+  ].find((name) => isPresent(config[name]));
+  if (plaintextSecret) {
+    throw new Error(
+      `${plaintextSecret} must be stored in Node-RED credentials`,
+    );
+  }
+
+  // Validate only the fields belonging to the selected backend. Hidden
+  // editor rows keep their values, so a flow switched to postgres still
+  // carries deployment/clusterNodes/prefix in its saved JSON -- rejecting
+  // those would make switching backends impossible without hand-editing
+  // flows.
+  if (normalizeBackend(config) === "postgres") {
+    return normalizePostgresConfig(config, credentials);
+  }
+
+  const deployment = String(config.deployment || "single").toLowerCase();
 
   if (!["single", "cluster", "sentinel"].includes(deployment)) {
-    throw new Error(`Unsupported Redis deployment mode: ${deploymentInput}`);
+    throw new Error(`Unsupported Redis deployment mode: ${deployment}`);
   }
 
-  const host = config.host || config.address || DEFAULT_REDIS_HOST;
+  const host = config.address || DEFAULT_HOST;
   const port = toPort(config.port, DEFAULT_REDIS_PORT);
-  const queueName = String(config.queueName || config.name || "").trim();
+  const queueName = String(config.name || "").trim();
   if (!queueName) {
     throw new Error("BullMQ queue name is required");
   }
+  const tls = toBoolean(config.tls, false);
+  const sentinelTls = toBoolean(config.sentinelTls, false);
 
   const normalized = {
     queueName,
@@ -123,22 +244,34 @@ function normalizeQueueConfig(config = {}, credentials = {}) {
     port,
     db: isPresent(config.db) ? Number(config.db) : undefined,
     username: config.username || undefined,
-    password: readSecret(config, credentials, "password") || undefined,
-    tls: toBoolean(config.tls, false),
+    password: readSecret(credentials, "password"),
+    tls,
     tlsRejectUnauthorized: toBoolean(config.tlsRejectUnauthorized, true),
-    tlsCa: readSecret(config, credentials, "tlsCa") || undefined,
-    tlsCert: readSecret(config, credentials, "tlsCert") || undefined,
-    tlsKey: readSecret(config, credentials, "tlsKey") || undefined,
+    tlsCa: readSecret(credentials, "tlsCa"),
+    tlsCert: readSecret(credentials, "tlsCert"),
+    tlsKey: readSecret(credentials, "tlsKey"),
     tlsServerName: config.tlsServerName || undefined,
     prefix: config.prefix || undefined,
-    clusterNodes: parseEndpointList(config.clusterNodes || config.startupNodes),
-    sentinelMasterName:
-      config.sentinelMasterName || config.masterName || config.nameOfMaster,
-    sentinels: parseEndpointList(config.sentinels, 26379),
+    defaultJobOptions: buildDefaultJobOptions(config),
+    clusterNodes: parseEndpointList(
+      config.clusterNodes,
+      DEFAULT_REDIS_PORT,
+      tls,
+      "TLS",
+    ),
+    sentinelMasterName: config.sentinelMasterName,
+    sentinels: parseEndpointList(
+      config.sentinels,
+      26379,
+      sentinelTls,
+      "Sentinel TLS",
+    ),
     sentinelUsername: config.sentinelUsername || undefined,
-    sentinelPassword:
-      readSecret(config, credentials, "sentinelPassword") || undefined,
-    sentinelTls: toBoolean(config.sentinelTls, false),
+    sentinelPassword: readSecret(credentials, "sentinelPassword"),
+    sentinelTls,
+    telemetry: toBoolean(config.telemetry, false),
+    telemetryServiceName: config.telemetryServiceName || undefined,
+    telemetryMetrics: toBoolean(config.telemetryMetrics, false),
   };
 
   if (deployment === "cluster") {
@@ -192,8 +325,73 @@ function buildTlsOptions(config, enabled) {
   return tls;
 }
 
+// Produces the pg pool config BullMQ's createPostgresBackend will receive as
+// `connection`. Only the fields PostgreSQL needs are read; every Redis-only
+// field a switched flow still carries in its saved JSON (deployment,
+// clusterNodes, sentinels, prefix, ...) is simply never touched here, so it
+// is ignored rather than rejected.
+function normalizePostgresConfig(config, credentials) {
+  const queueName = String(config.name || "").trim();
+  if (!queueName) {
+    throw new Error("BullMQ queue name is required");
+  }
+  const tls = toBoolean(config.tls, false);
+  const ssl = buildTlsOptions(
+    {
+      tlsRejectUnauthorized: toBoolean(config.tlsRejectUnauthorized, true),
+      tlsServerName: config.tlsServerName || undefined,
+      tlsCa: readSecret(credentials, "tlsCa"),
+      tlsCert: readSecret(credentials, "tlsCert"),
+      tlsKey: readSecret(credentials, "tlsKey"),
+    },
+    tls,
+  );
+
+  const postgres = {
+    host: String(config.address || DEFAULT_HOST).trim(),
+    port: toPort(config.port, DEFAULT_POSTGRES_PORT, "PostgreSQL"),
+    database: config.database || undefined,
+    user: config.username || undefined,
+    password: readSecret(credentials, "password"),
+    schema: config.schema || undefined,
+    max: toPoolMax(config.max),
+    connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
+    migrate: toBoolean(config.migrate, true),
+  };
+  if (ssl) {
+    postgres.ssl = ssl;
+  }
+
+  return {
+    backend: "postgres",
+    queueName,
+    postgres,
+    defaultJobOptions: buildDefaultJobOptions(config),
+    telemetry: toBoolean(config.telemetry, false),
+    telemetryServiceName: config.telemetryServiceName || undefined,
+    telemetryMetrics: toBoolean(config.telemetryMetrics, false),
+  };
+}
+
+// BullMQ's production guidance: exponential reconnect backoff with a 1s floor
+// and a 20s ceiling, rather than ioredis's default 50ms-2s, which hammers a
+// down Redis once per connection role per runtime node.
+const RECONNECT_FLOOR_MS = 1000;
+const RECONNECT_CEILING_MS = 20000;
+
+function reconnectBackoff(attempt) {
+  return Math.min(
+    RECONNECT_FLOOR_MS * 2 ** (attempt - 1),
+    RECONNECT_CEILING_MS,
+  );
+}
+
+function isConsumerRole(role) {
+  return role === "worker" || role === "events";
+}
+
 function retryValueForRole(role) {
-  return role === "worker" || role === "events" ? null : 1;
+  return isConsumerRole(role) ? null : 1;
 }
 
 function buildStandaloneOptions(config, role) {
@@ -203,8 +401,16 @@ function buildStandaloneOptions(config, role) {
     maxRetriesPerRequest: retryValueForRole(role),
     enableReadyCheck: true,
     connectTimeout: 10000,
+    retryStrategy: reconnectBackoff,
   };
 
+  // Deliberately NOT enableOfflineQueue:false, which BullMQ's production guide
+  // suggests for producers. Measured against a healthy Redis, it rejects a
+  // command issued during the connect window -- which in Node-RED is every
+  // deploy-time message. Keeping the offline queue accepts those, while
+  // maxRetriesPerRequest above still fails a genuinely-down Redis in about a
+  // second instead of hanging.
+
   if (config.db !== undefined && !Number.isNaN(config.db)) {
     options.db = config.db;
   }
@@ -228,6 +434,7 @@ function buildRedisDescriptor(config, role = "producer") {
       maxRetriesPerRequest: retryValueForRole(role),
       enableReadyCheck: true,
       connectTimeout: 10000,
+      retryStrategy: reconnectBackoff,
     };
     if (config.username) {
       redisOptions.username = config.username;
@@ -248,6 +455,7 @@ function buildRedisDescriptor(config, role = "producer") {
         slotsRefreshTimeout: 2000,
         maxRedirections: 16,
         dnsLookup: (address, callback) => callback(null, address),
+        clusterRetryStrategy: reconnectBackoff,
       },
     };
   }
@@ -261,6 +469,7 @@ function buildRedisDescriptor(config, role = "producer") {
     }
     options.sentinels = config.sentinels;
     options.name = config.sentinelMasterName;
+    options.sentinelRetryStrategy = reconnectBackoff;
     if (config.sentinelUsername) {
       options.sentinelUsername = config.sentinelUsername;
     }
@@ -285,7 +494,7 @@ function createRedisConnection(descriptor, IORedis) {
   return new IORedis(descriptor.options);
 }
 
-function buildBullMQOptions(config, connection) {
+function buildBullMQOptions(config, connection, telemetry, role = "producer") {
   const options = {};
   if (connection) {
     options.connection = connection;
@@ -293,13 +502,27 @@ function buildBullMQOptions(config, connection) {
   if (config.prefix) {
     options.prefix = config.prefix;
   }
+  if (telemetry) {
+    options.telemetry = telemetry;
+  }
+  // Without this, BullMQ awaits a connection-ready promise that never settles
+  // while Redis is unreachable, so a producer command hangs forever instead of
+  // erroring. Consumers must keep waiting for the connection to come back.
+  // Only createRedisBackend forwards skipWaitingForReady (verified in
+  // node_modules/bullmq/dist/cjs/utils/create-backend.js) -- createPostgresBackend
+  // never reads it, so setting it there would be a misleading no-op.
+  if (!isConsumerRole(role) && config.backend !== "postgres") {
+    options.skipWaitingForReady = true;
+  }
   return options;
 }
 
 module.exports = {
+  POSTGRES_CONNECTION_TIMEOUT_MS,
   buildBullMQOptions,
   buildRedisDescriptor,
   createRedisConnection,
   normalizeQueueConfig,
+  normalizePostgresConfig,
   parseEndpointList,
 };

package/lib/scheduler.js

@@ -4,78 +4,8 @@ function hasOwn(object, key) {
   return Object.prototype.hasOwnProperty.call(object, key);
 }
 
-function clonePlain(value) {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    return value;
-  }
-  return { ...value };
-}
-
-function normalizeRepeatOptions(repeat) {
-  const normalized = clonePlain(repeat) || {};
-  if (
-    normalized.cron &&
-    normalized.pattern &&
-    normalized.cron !== normalized.pattern
-  ) {
-    throw new Error(
-      "repeat.cron and repeat.pattern must match when both are supplied",
-    );
-  }
-  if (normalized.cron) {
-    normalized.pattern = normalized.cron;
-    delete normalized.cron;
-  }
-  return normalized;
-}
-
-function getJobData(msg) {
-  if (hasOwn(msg, "jobData")) {
-    return msg.jobData;
-  }
-  return { payload: msg.payload };
-}
-
-function normalizeAddRequest(msg = {}) {
-  const jobopts = clonePlain(msg.jobopts) || {};
-  const name = msg.jobName || "default";
-
-  if (jobopts.repeat) {
-    const schedulerId = String(msg.schedulerId || jobopts.jobId || "").trim();
-    if (!schedulerId) {
-      throw new Error(
-        "scheduled jobs require msg.schedulerId or msg.jobopts.jobId",
-      );
-    }
-
-    const repeat = normalizeRepeatOptions(jobopts.repeat);
-    delete jobopts.repeat;
-    delete jobopts.jobId;
-
-    return {
-      kind: "scheduler",
-      schedulerId,
-      repeat,
-      template: {
-        name,
-        data: getJobData(msg),
-        opts: jobopts,
-      },
-    };
-  }
-
-  return {
-    kind: "job",
-    name,
-    data: getJobData(msg),
-    opts: jobopts,
-  };
-}
-
-function getLegacySchedulerId(msg = {}) {
-  const schedulerId = String(
-    msg.schedulerId || msg.jobid || msg.jobId || "",
-  ).trim();
+function getSchedulerId(msg = {}) {
+  const schedulerId = String(msg.schedulerId || "").trim();
   if (!schedulerId) {
     throw new Error("A scheduler id is required");
   }
@@ -99,6 +29,8 @@ function serializeScheduler(scheduler) {
     "offset",
     "tz",
     "endDate",
+    "startDate",
+    "iterationCount",
   ]) {
     if (hasOwn(scheduler, key)) {
       result[key] = scheduler[key];
@@ -108,7 +40,6 @@ function serializeScheduler(scheduler) {
 }
 
 module.exports = {
-  getLegacySchedulerId,
-  normalizeAddRequest,
+  getSchedulerId,
   serializeScheduler,
 };

package/lib/serialization.js

@@ -1,9 +1,5 @@
 "use strict";
 
-function hasOwn(object, key) {
-  return Object.prototype.hasOwnProperty.call(object, key);
-}
-
 function copyKnownFields(source, fields) {
   if (!source) {
     return source;
@@ -31,6 +27,7 @@ function serializeJob(job) {
     "opts",
     "progress",
     "attemptsMade",
+    "attemptsStarted",
     "failedReason",
     "stacktrace",
     "returnvalue",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@pauldeng/node-red-contrib-bullmq",
-  "version": "1.0.2",
-  "description": "BullMQ-backed Redis job queue nodes for Node-RED",
+  "version": "2.0.0",
+  "description": "BullMQ job queue nodes for Node-RED, backed by Redis or PostgreSQL",
   "main": "bull-queue.js",
   "files": [
     "bull-queue.js",
@@ -12,7 +12,7 @@
     "docs/*.md"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=22.9"
   },
   "scripts": {
     "test": "node --test test/*.test.js",
@@ -25,8 +25,20 @@
     "validate": "npx --yes node-red-dev validate"
   },
   "dependencies": {
-    "bullmq": "5.80.2",
-    "ioredis": "5.10.1"
+    "bullmq": "6.3.1",
+    "ioredis": "5.11.1"
+  },
+  "peerDependencies": {
+    "bullmq-otel": ">=2.0.0",
+    "pg": ">=8.0.0"
+  },
+  "peerDependenciesMeta": {
+    "bullmq-otel": {
+      "optional": true
+    },
+    "pg": {
+      "optional": true
+    }
   },
   "license": "MIT",
   "keywords": [
@@ -36,11 +48,13 @@
     "queue",
     "bull",
     "redis",
+    "postgres",
+    "postgresql",
     "worker",
     "jobs"
   ],
   "node-red": {
-    "version": ">=4.1.0 <6",
+    "version": ">=5.0.0 <6",
     "nodes": {
       "bull-queue": "bull-queue.js"
     }
@@ -62,9 +76,11 @@
     "email": "dengpeng.cn@gmail.com"
   },
   "devDependencies": {
-    "@playwright/test": "1.61.1",
-    "node-red": "5.0.1",
+    "@playwright/test": "1.62.1",
+    "bullmq-otel": "2.0.1",
+    "node-red": "5.0.4",
     "node-red-node-test-helper": "^0.3.6",
-    "prettier": "3.9.4"
+    "pg": "8.23.0",
+    "prettier": "3.9.6"
   }
 }