@avada-falcon/worker-sdk 0.5.0

package/src/worker/tierManager.js

@@ -0,0 +1,55 @@
+import {Worker} from 'bullmq';
+import {queueName} from '../queue/queues.js';
+
+export class TierManager {
+  constructor(concurrency, redisOpts, processor) {
+    this._workers = [];
+
+    let createdCount = 0;
+    for (const [tier, limit] of Object.entries(concurrency)) {
+      if (!Number.isInteger(limit)) {
+        console.warn(
+          `[worker-sdk] TierManager: invalid concurrency for tier "${tier}": ${JSON.stringify(limit)} (expected integer >= 0). Skipping this tier.`
+        );
+        continue;
+      }
+      if (limit <= 0) {
+        console.warn(`[worker-sdk] Skipping tier: ${tier} (concurrency: ${limit})`);
+        continue;
+      }
+      const queueName = TierManager.queueName(tier);
+      const worker = new Worker(queueName, processor, {
+        connection: redisOpts,
+        concurrency: limit
+      });
+      worker.on('error', err => {
+        console.error(`[worker-sdk] Worker error on ${queueName}:`, err.message);
+      });
+      this._workers.push(worker);
+      createdCount++;
+    }
+
+    if (createdCount === 0) {
+      const tierCount = Object.keys(concurrency).length;
+      if (tierCount === 0) {
+        console.warn(
+          '[worker-sdk] TierManager: concurrency object is empty — no tier workers created. ' +
+          'Set at least one tier (heavy/medium/light) to a positive integer.'
+        );
+      } else {
+        console.warn(
+          '[worker-sdk] TierManager: all tiers have concurrency: 0 — this worker will not process any jobs. ' +
+          'This is almost certainly a config mistake.'
+        );
+      }
+    }
+  }
+
+  static queueName(tier) {
+    return queueName(tier);
+  }
+
+  async closeAll() {
+    await Promise.all(this._workers.map(w => w.close()));
+  }
+}

package/src/worker/worker.js

@@ -0,0 +1,204 @@
+import os from 'os';
+import {DelayedError} from 'bullmq';
+
+import Redis from 'ioredis';
+import {loadConfig} from '../config/loader.js';
+import {HandlerRegistry} from './handlerRegistry.js';
+import {JobExecutor, setFileLogger, setLokiShipper} from './jobExecutor.js';
+import {TierManager} from './tierManager.js';
+import {CronManager} from '../cron/cronManager.js';
+import {ShutdownManager} from '../shutdown/shutdownManager.js';
+import {FileLogger} from '../logging/fileLogger.js';
+import {LokiShipper} from '../logging/lokiShipper.js';
+import {Heartbeat} from './heartbeat.js';
+
+/** Key under job.data holding how many times the job has been declined by a worker. */
+const REQUEUE_COUNT_KEY = '__requeues';
+const DEFAULT_MAX_REQUEUES = 50;
+const DEFAULT_REQUEUE_DELAY_MS = 200;
+
+function requeueCount(job) {
+  const count = job.data?.[REQUEUE_COUNT_KEY];
+  return Number.isInteger(count) && count > 0 ? count : 0;
+}
+
+/**
+ * Create a worker instance.
+ * @param {string} configPath - Path to YAML config file
+ * @returns {{register: Function, start: Function, stop: Function}}
+ */
+export function createWorker(configPath) {
+  const config = loadConfig(configPath);
+  const workerId = config.worker?.id || `${os.hostname()}-${process.pid}`;
+  console.info(`[worker-sdk] Worker ID: ${workerId}`);
+  const registry = new HandlerRegistry();
+  const executor = new JobExecutor();
+  const shutdown = new ShutdownManager();
+
+  let tierManager;
+  let cronManager;
+
+  function register(name, handler) {
+    registry.register(name, handler);
+  }
+
+  async function start() {
+    // Validate all registered handlers have config
+    const unconfigured = registry.names().filter(n => !config.jobs[n]);
+    if (unconfigured.length > 0) {
+      throw new Error(`Registered handlers not defined in config: ${unconfigured.join(', ')}`);
+    }
+
+    // Set up file logging if configured
+    if (config.logging?.dir) {
+      const retentionDays = config.logging.retentionDays || 30;
+      const fl = new FileLogger({dir: config.logging.dir, retentionDays});
+      setFileLogger(fl);
+
+      // Clean up old log files on startup
+      fl.cleanup();
+
+      console.info(`[worker-sdk] File logging enabled: ${config.logging.dir} (retention: ${retentionDays} days)`);
+    }
+
+    // Set up Loki log shipping if configured
+    let lokiShipper = null;
+    if (config.logging?.loki?.url) {
+      lokiShipper = new LokiShipper({
+        url: config.logging.loki.url,
+        batchSize: config.logging.loki.batchSize,
+        flushInterval: config.logging.loki.flushInterval,
+        labels: config.logging.loki.labels,
+        workerId
+      });
+      setLokiShipper(lokiShipper);
+      console.info(`[worker-sdk] Loki shipping enabled: ${config.logging.loki.url}`);
+    }
+
+    // Shared Redis opts for all BullMQ instances (workers, queues, cron)
+    const redisOpts = {...config.redis, maxRetriesPerRequest: null};
+
+    // Job processor — shared across all tier workers
+    const processor = async (job, token) => {
+      const handler = registry.get(job.name);
+      const jobConfig = config.jobs[job.name] || {};
+      const timeout = jobConfig.timeout || 30000;
+      const maxRequeues = jobConfig.maxRequeues ?? DEFAULT_MAX_REQUEUES;
+
+      // Let a handler DECLINE to run on this worker and hand the job back to the queue,
+      // delayed briefly, instead of blocking — a job blocked inside the handler stays
+      // "active" and pinned to this worker where no idle peer can steal it. App-level
+      // admission gates (e.g. a per-worker memory budget) call ctx.requeue().
+      //
+      // moveToDelayed delays only THIS job (not a queue-wide rate limit, so other workers
+      // keep pulling other jobs), then DelayedError tells BullMQ the job was already
+      // re-scheduled — handleFailed treats it as non-failure: no attempt burned, no
+      // 'failed' event. `token` is BullMQ's job-lock token, the 2nd processor arg.
+      //
+      // CONTRACT: requeue() always throws. A handler that wraps it in try/catch and
+      // then returns normally makes BullMQ complete a job that is also sitting in the
+      // delayed set — it runs twice. Call it as `return ctx.requeue()` or let it throw.
+      const requeue = async (retryMs = DEFAULT_REQUEUE_DELAY_MS) => {
+        const count = requeueCount(job) + 1;
+
+        // No attempt is burned on a requeue, so an admission gate that never opens
+        // (leaked memory budget, gate bug) would ping-pong this job forever, silently.
+        // Cap it and fail loudly instead.
+        if (count > maxRequeues) {
+          throw new Error(
+            `Job ${job.name}:${job.id} was declined ${count - 1} times ` +
+            `(maxRequeues: ${maxRequeues}) — admission gate never opened.`
+          );
+        }
+
+        // Persist the counter: the in-memory job.data copy is dropped once the job
+        // goes back to Redis, so it has to be written before re-scheduling.
+        await job.updateData({...job.data, [REQUEUE_COUNT_KEY]: count});
+
+        // Jitter the delay. Without it, every job this worker declines wakes at the
+        // same instant and gets declined again in lockstep (thundering herd).
+        const delay = retryMs + Math.floor(Math.random() * retryMs);
+
+        console.warn(
+          `[worker-sdk] Declined ${job.name}:${job.id} — requeued in ${delay}ms ` +
+          `(${count}/${maxRequeues})`
+        );
+
+        await job.moveToDelayed(Date.now() + delay, token);
+        throw new DelayedError();
+      };
+
+      return executor.run(handler, job, timeout, {requeue});
+    };
+
+    // Create tier workers
+    tierManager = new TierManager(config.concurrency, redisOpts, processor);
+
+    // Register cron jobs (same redisOpts)
+    cronManager = new CronManager(redisOpts);
+    await cronManager.register(config.jobs, {leader: config.cron.leader});
+
+    // Dashboard is no longer started by createWorker — run createDashboard separately
+    if (config.dashboard.port) {
+      console.warn(
+        '[worker-sdk] dashboard.port is set but createWorker no longer runs the dashboard. ' +
+        'Use createDashboard("./worker.config.yml") in a separate process/container instead.'
+      );
+    }
+
+    // Heartbeat — registered first so it shuts down first (signals "dead" within one TTL)
+    let heartbeat = null;
+    let heartbeatRedis = null;
+    if (config.worker.heartbeat.enabled) {
+      heartbeatRedis = new Redis({
+        ...redisOpts,
+        lazyConnect: false,
+        maxRetriesPerRequest: 3
+      });
+      try {
+        heartbeat = new Heartbeat({
+          redis: heartbeatRedis,
+          workerId,
+          tiers: config.concurrency,
+          intervalMs: config.worker.heartbeat.intervalMs,
+          ttlMs: config.worker.heartbeat.ttlMs
+        });
+        await heartbeat.start();
+      } catch (err) {
+        // Heartbeat constructor or start() failed — clean up the dedicated connection
+        // before rethrowing so we don't leak a TCP socket.
+        await heartbeatRedis.quit().catch(() => {});
+        throw err;
+      }
+      shutdown.register('heartbeat', async () => {
+        await heartbeat.stop();
+        await heartbeatRedis.quit();
+      });
+    }
+
+    // Register shutdown handlers (executed in FIFO registration order).
+    // Loki shipper must be registered LAST so it flushes AFTER workers drain —
+    // in-flight jobs can still push logs while BullMQ workers are closing.
+    shutdown.register('tierManager', () => tierManager.closeAll());
+    shutdown.register('cronManager', () => cronManager.closeAll());
+    if (lokiShipper) {
+      shutdown.register('lokiShipper', () => lokiShipper.stop());
+    }
+    shutdown.installSignalHandlers(() => process.exit(0));
+
+    console.info('[worker-sdk] Worker started');
+    console.info(`[worker-sdk] Jobs: ${registry.names().join(', ')}`);
+    console.info(`[worker-sdk] Tiers: ${JSON.stringify(config.concurrency)}`);
+  }
+
+  async function stop() {
+    await shutdown.shutdown();
+  }
+
+  return {
+    start,
+    stop,
+    register,
+    workerId
+  };
+}