@avada-falcon/worker-sdk 0.5.0

package/src/logging/lokiShipper.js

@@ -0,0 +1,219 @@
+/**
+ * Buffers log entries and ships them to a Loki server via HTTP push API.
+ *
+ * Batches by size (batchSize) or time (flushInterval), whichever hits first.
+ * Groups entries by stream labels (level + job) to minimize payload size.
+ * Retries with exponential backoff, capped at maxRetryAttempts.
+ * Drops entries silently if buffer exceeds maxBufferSize — never crashes the worker.
+ */
+const MAX_RETRY_ATTEMPTS = 5;
+
+export class LokiShipper {
+  /**
+   * @param {object} options
+   * @param {string} options.url - Loki base URL (e.g., http://loki:3100)
+   * @param {number} [options.batchSize=100]
+   * @param {number} [options.flushInterval=5000]
+   * @param {object} [options.labels={}] - static labels added to every stream
+   * @param {string} [options.workerId] - Optional per-worker identifier added to every stream label. Overrides any workerId in options.labels.
+   * @param {number} [options.maxBufferSize=10000]
+   */
+  constructor({url, batchSize = 100, flushInterval = 5000, labels = {}, workerId, maxBufferSize = 10000}) {
+    if (!url) throw new Error('LokiShipper requires a url');
+
+    this._url = url.replace(/\/$/, '') + '/loki/api/v1/push';
+    this._batchSize = batchSize;
+    this._maxBufferSize = maxBufferSize;
+    this._staticLabels = workerId ? {...labels, workerId} : {...labels};
+    this._buffer = [];
+    this._flushing = false;
+    this._flushPromise = null;
+    this._stopped = false;
+    this._droppedCount = 0;
+    this._retryDelayMs = 1000;
+    this._maxRetryDelayMs = 30000;
+    this._stopController = new AbortController();
+
+    this._timer = setInterval(() => this._flush().catch(() => {}), flushInterval);
+    this._timer.unref?.();
+  }
+
+  /**
+   * Add a log entry to the buffer.
+   * @param {object} entry - {job, id, level, msg, data}
+   */
+  push(entry) {
+    if (this._stopped) return;
+
+    if (this._buffer.length >= this._maxBufferSize) {
+      this._droppedCount++;
+      return;
+    }
+
+    this._buffer.push({
+      ts: Date.now(),
+      entry
+    });
+
+    if (this._buffer.length >= this._batchSize) {
+      this._flush().catch(() => {});
+    }
+  }
+
+  bufferedCount() {
+    return this._buffer.length;
+  }
+
+  droppedCount() {
+    return this._droppedCount;
+  }
+
+  /**
+   * Flush current buffer to Loki. Caps retries to avoid blocking other flushes.
+   * Tracks the in-flight promise so stop() can wait for it before its final flush.
+   */
+  _flush(maxAttempts = MAX_RETRY_ATTEMPTS, {final = false} = {}) {
+    if (this._flushing || this._buffer.length === 0) return Promise.resolve();
+    this._flushing = true;
+    this._flushPromise = this._doFlush(maxAttempts, final).finally(() => {
+      this._flushing = false;
+      this._flushPromise = null;
+      // Entries pushed while we were flushing may already fill a batch;
+      // kick another flush so they don't wait for the next interval tick.
+      if (!this._stopped && this._buffer.length >= this._batchSize) {
+        this._flush().catch(() => {});
+      }
+    });
+    return this._flushPromise;
+  }
+
+  // `final` lets stop()'s last flush run even though _stopped is already set
+  async _doFlush(maxAttempts, final) {
+    const batch = this._buffer.splice(0);
+    const payload = this._buildPayload(batch);
+
+    let delayMs = this._retryDelayMs;
+    let sent = false;
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      if (this._stopped && !final) break;
+      try {
+        const res = await fetch(this._url, {
+          method: 'POST',
+          headers: {'Content-Type': 'application/json'},
+          body: JSON.stringify(payload)
+        });
+        if (res.ok) {
+          sent = true;
+          break;
+        }
+        // 4xx won't succeed on retry — drop with warning
+        if (res.status >= 400 && res.status < 500) {
+          process.stderr.write(
+            `[worker-sdk] LokiShipper: dropping batch of ${batch.length} — Loki returned ${res.status}\n`
+          );
+          sent = true; // don't re-queue, don't retry
+          break;
+        }
+        // 5xx — retry
+        throw new Error(`Loki returned HTTP ${res.status}`);
+      } catch {
+        if (this._stopped && !final) break;
+        if (attempt === maxAttempts - 1) break;
+        await this._sleep(delayMs);
+        delayMs = Math.min(delayMs * 2, this._maxRetryDelayMs);
+      }
+    }
+
+    if (!sent) {
+      if (this._stopped && !final) {
+        // Stopped mid-retry — re-prepend so stop()'s final flush has one more shot
+        this._buffer = batch.concat(this._buffer);
+      } else {
+        // Retries exhausted — log a warning and drop
+        process.stderr.write(
+          `[worker-sdk] LokiShipper: dropping batch of ${batch.length} after ${maxAttempts} retries\n`
+        );
+        this._droppedCount += batch.length;
+      }
+    }
+  }
+
+  /**
+   * Abortable sleep — wakes immediately if stop() is called.
+   */
+  _sleep(ms) {
+    const signal = this._stopController.signal;
+    if (signal.aborted) return Promise.resolve();
+    return new Promise(resolve => {
+      const onAbort = () => {
+        clearTimeout(timer);
+        resolve();
+      };
+      const timer = setTimeout(() => {
+        // Detach the abort listener so repeated sleeps don't accumulate
+        // listeners on the shared signal for the shipper's lifetime.
+        signal.removeEventListener('abort', onAbort);
+        resolve();
+      }, ms);
+      signal.addEventListener('abort', onAbort, {once: true});
+    });
+  }
+
+  /**
+   * Build Loki push payload from buffered entries.
+   * Groups entries by stream labels (level + job) to compress payload.
+   */
+  _buildPayload(batch) {
+    const streams = new Map();
+
+    for (const {ts, entry} of batch) {
+      // Static labels never vary — only level + job determine the stream
+      const key = `${entry.level}\u0000${entry.job}`;
+
+      if (!streams.has(key)) {
+        streams.set(key, {
+          stream: {...this._staticLabels, level: entry.level, job: entry.job},
+          values: []
+        });
+      }
+
+      let line;
+      try {
+        line = JSON.stringify({
+          id: entry.id,
+          msg: entry.msg,
+          ...(entry.data ? {data: entry.data} : {})
+        });
+      } catch {
+        // entry.data isn't JSON-serializable (circular, BigInt, ...) — ship without it
+        line = JSON.stringify({id: entry.id, msg: entry.msg, data: '[unserializable]'});
+      }
+      const tsNs = `${ts}000000`; // ms → ns
+      streams.get(key).values.push([tsNs, line]);
+    }
+
+    return {streams: Array.from(streams.values())};
+  }
+
+  /**
+   * Stop the shipper — flush remaining buffer, cancel timer, abort pending sleeps.
+   */
+  async stop() {
+    this._stopped = true;
+    this._stopController.abort();
+    if (this._timer) clearInterval(this._timer);
+    // Wait for any in-flight flush to settle first — if it was mid-retry it
+    // re-prepends its batch to the buffer, so the final flush below can still
+    // send it instead of the batch being lost on shutdown.
+    const inFlight = this._flushPromise;
+    if (inFlight) {
+      await inFlight.catch(() => {});
+    }
+    // Final flush — a single attempt (no retry backoff) so shutdown stays
+    // bounded even when Loki is down.
+    if (this._buffer.length > 0) {
+      await this._flush(1, {final: true});
+    }
+  }
+}

package/src/queue/queues.js

@@ -0,0 +1,62 @@
+import {Queue} from 'bullmq';
+
+/**
+ * Single source of truth for how the SDK talks to tier queues:
+ * naming, default job options, and Queue construction/caching.
+ * Consumed by the client, cron manager, worker, dashboard, and admin probes.
+ */
+export const QUEUE_PREFIX = 'worker';
+export const DEFAULT_TIER = 'medium';
+
+export function queueName(tier) {
+  return `${QUEUE_PREFIX}-${tier}`;
+}
+
+/**
+ * Build BullMQ job options from a job's YAML config.
+ * Shared by createClient.add and CronManager.register so retry mapping and
+ * retention policy live in exactly one place.
+ * @param {object} jobConfig - {retry?: {maxAttempts?, baseDelay?}, ...}
+ */
+export function buildJobOpts(jobConfig) {
+  const opts = {
+    attempts: jobConfig.retry?.maxAttempts || 1,
+    removeOnComplete: {count: 1000},
+    removeOnFail: {count: 5000}
+  };
+
+  if (jobConfig.retry?.baseDelay) {
+    opts.backoff = {
+      type: 'exponential',
+      delay: jobConfig.retry.baseDelay
+    };
+  }
+
+  return opts;
+}
+
+/**
+ * Lazy per-tier Queue cache. Queues are created on first use and closed
+ * together via closeAll().
+ */
+export class QueuePool {
+  /**
+   * @param {object} connection - ioredis options (or instance) passed to BullMQ
+   */
+  constructor(connection) {
+    this._connection = connection;
+    this._queues = new Map();
+  }
+
+  get(tier) {
+    const name = queueName(tier);
+    if (!this._queues.has(name)) {
+      this._queues.set(name, new Queue(name, {connection: this._connection}));
+    }
+    return this._queues.get(name);
+  }
+
+  async closeAll() {
+    await Promise.all(Array.from(this._queues.values()).map(q => q.close()));
+  }
+}

package/src/shutdown/shutdownManager.js

@@ -0,0 +1,50 @@
+const FORCE_EXIT_TIMEOUT_MS = 30000;
+
+export class ShutdownManager {
+  constructor() {
+    this._handlers = [];
+    this._shutdownPromise = null;
+  }
+
+  register(name, handler) {
+    this._handlers.push({name, handler});
+  }
+
+  /**
+   * Execute all shutdown handlers in registration order.
+   * Returns the same promise if called multiple times (idempotent).
+   */
+  async shutdown() {
+    if (this._shutdownPromise) return this._shutdownPromise;
+    this._shutdownPromise = this._runHandlers();
+    return this._shutdownPromise;
+  }
+
+  async _runHandlers() {
+    for (const {name, handler} of this._handlers) {
+      try {
+        await handler();
+      } catch (err) {
+        console.error(`[shutdown] Error in "${name}":`, err.message);
+      }
+    }
+  }
+
+  installSignalHandlers(onComplete) {
+    const handle = async (signal) => {
+      console.info(`[shutdown] Received ${signal}, shutting down...`);
+
+      const forceTimer = setTimeout(() => {
+        console.error(`[shutdown] Force exit after ${FORCE_EXIT_TIMEOUT_MS}ms timeout`);
+        process.exit(1);
+      }, FORCE_EXIT_TIMEOUT_MS);
+      forceTimer.unref();
+
+      await this.shutdown();
+      if (onComplete) onComplete();
+    };
+
+    process.on('SIGTERM', () => handle('SIGTERM'));
+    process.on('SIGINT', () => handle('SIGINT'));
+  }
+}

package/src/worker/handlerRegistry.js

@@ -0,0 +1,47 @@
+export class HandlerRegistry {
+  constructor() {
+    this._handlers = new Map();
+  }
+
+  /**
+   * Register a handler function for a job name.
+   * @param {string} name - Job name
+   * @param {Function} handler - async function(payload, context)
+   */
+  register(name, handler) {
+    if (typeof handler !== 'function') {
+      throw new Error('Handler must be a function');
+    }
+    this._handlers.set(name, handler);
+  }
+
+  /**
+   * Get handler by job name. Throws if not found.
+   * @param {string} name
+   * @returns {Function}
+   */
+  get(name) {
+    const handler = this._handlers.get(name);
+    if (!handler) {
+      throw new Error(`No handler registered for job: ${name}`);
+    }
+    return handler;
+  }
+
+  /**
+   * Check if a handler is registered.
+   * @param {string} name
+   * @returns {boolean}
+   */
+  has(name) {
+    return this._handlers.has(name);
+  }
+
+  /**
+   * List all registered job names.
+   * @returns {string[]}
+   */
+  names() {
+    return Array.from(this._handlers.keys());
+  }
+}

package/src/worker/heartbeat.js

@@ -0,0 +1,106 @@
+import os from 'os';
+import {HEARTBEAT_KEY_PREFIX} from './heartbeatKey.js';
+
+/**
+ * Heartbeat — writes a TTL'd key to Redis on a fixed interval.
+ *
+ * Key format: worker:heartbeat:<workerId>
+ * Value: JSON-encoded WorkerInfo ({workerId, hostname, pid, tiers, startedAt, lastBeat})
+ * TTL: ttlMs (auto-expires if worker dies between beats)
+ *
+ * Failure mode: if _beat() throws (Redis blip), log a warning and continue.
+ * The next interval tick retries; no state is lost. If Redis is unreachable
+ * past ttlMs, the key expires — observers correctly conclude the worker is dead.
+ */
+export class Heartbeat {
+  /**
+   * @param {object} options
+   * @param {object} options.redis - ioredis client with set/del
+   * @param {string} options.workerId - unique worker identity
+   * @param {object} options.tiers - {heavy, medium, light} concurrency
+   * @param {number} options.intervalMs - beat cadence
+   * @param {number} options.ttlMs - Redis key TTL
+   */
+  constructor({redis, workerId, tiers, intervalMs, ttlMs}) {
+    this._redis = redis;
+    this._workerId = workerId;
+    this._tiers = tiers;
+    this._intervalMs = intervalMs;
+    this._ttlMs = ttlMs;
+    this._key = `${HEARTBEAT_KEY_PREFIX}${workerId}`;
+    this._hostname = os.hostname();
+    this._pid = process.pid;
+    this._startedAt = Date.now();
+    this._timer = null;
+    this._started = false;
+    this._stopped = false;
+  }
+
+  async start() {
+    if (this._stopped) {
+      throw new Error('Heartbeat: cannot restart after stop() — create a new instance');
+    }
+    if (this._started) {
+      process.stderr.write(
+        '[worker-sdk] Heartbeat: start() called twice — ignoring second call\n'
+      );
+      return;
+    }
+    this._started = true;
+    await this._beat().catch(err => {
+      process.stderr.write(
+        `[worker-sdk] Heartbeat: initial beat failed (${err.message}); will retry on interval\n`
+      );
+    });
+    this._timer = setInterval(() => {
+      this._beat().catch(err => {
+        process.stderr.write(
+          `[worker-sdk] Heartbeat: beat failed (${err.message}); retrying next interval\n`
+        );
+      });
+    }, this._intervalMs);
+    this._timer.unref?.();
+  }
+
+  async _beat() {
+    const payload = JSON.stringify({
+      workerId: this._workerId,
+      hostname: this._hostname,
+      pid: this._pid,
+      tiers: this._tiers,
+      startedAt: this._startedAt,
+      lastBeat: Date.now()
+    });
+    this._inFlightBeat = this._redis.set(this._key, payload, 'PX', this._ttlMs);
+    try {
+      await this._inFlightBeat;
+    } finally {
+      this._inFlightBeat = null;
+    }
+  }
+
+  async stop() {
+    if (this._stopped) return;
+    this._stopped = true;
+    if (this._timer) {
+      clearInterval(this._timer);
+      this._timer = null;
+    }
+    // Wait for any in-flight beat to settle before deleting the key
+    if (this._inFlightBeat) {
+      try {
+        await this._inFlightBeat;
+      } catch {
+        // beat errored; carry on
+      }
+    }
+    if (!this._started) return; // never wrote a key; nothing to delete
+    if (this._redis && typeof this._redis.del === 'function') {
+      try {
+        await this._redis.del(this._key);
+      } catch {
+        // ignore — stopping is best-effort; key will expire via TTL anyway
+      }
+    }
+  }
+}

package/src/worker/heartbeatKey.js

@@ -0,0 +1,6 @@
+/**
+ * Prefix for worker heartbeat keys in Redis.
+ * Used by both the Heartbeat module (writer) and listWorkers (reader).
+ * Keys take the form `${HEARTBEAT_KEY_PREFIX}<workerId>`.
+ */
+export const HEARTBEAT_KEY_PREFIX = 'worker:heartbeat:';

package/src/worker/jobExecutor.js

@@ -0,0 +1,169 @@
+import {AsyncLocalStorage} from 'node:async_hooks';
+import {inspect} from 'node:util';
+
+const jobStorage = new AsyncLocalStorage();
+
+/**
+ * Stringify a value for log capture without ever throwing — circular refs,
+ * BigInt, and throwing toJSON() must not crash the job that logged them.
+ */
+function safeStringify(value) {
+  if (typeof value === 'string') return value;
+  try {
+    const json = JSON.stringify(value);
+    return json === undefined ? String(value) : json;
+  } catch {
+    return inspect(value, {depth: 3, breakLength: Infinity});
+  }
+}
+let consolePatched = false;
+let fileLogger = null;
+
+/**
+ * Set the file logger instance. Called by createWorker after loading config.
+ * @param {import('../logging/fileLogger.js').FileLogger|null} logger
+ */
+export function setFileLogger(logger) {
+  fileLogger = logger;
+}
+
+let lokiShipper = null;
+
+/**
+ * Set the Loki shipper instance. Called by createWorker after loading config.
+ * @param {import('../logging/lokiShipper.js').LokiShipper|null} shipper
+ */
+export function setLokiShipper(shipper) {
+  lokiShipper = shipper;
+}
+
+/**
+ * Write to the file logger and Loki shipper if configured.
+ */
+function writeLogEntry(entry) {
+  if (fileLogger) {
+    fileLogger.write(entry);
+  }
+  if (lokiShipper) {
+    lokiShipper.push(entry);
+  }
+}
+
+/**
+ * Patch console once at module load to route logs to the active job (if any).
+ * Uses AsyncLocalStorage so each concurrent job gets its own context — no race conditions.
+ */
+function patchConsoleOnce() {
+  if (consolePatched) return;
+  consolePatched = true;
+
+  const originalLog = console.log;
+  const originalInfo = console.info;
+  const originalWarn = console.warn;
+  const originalError = console.error;
+
+  const capture = (originalFn, level) => {
+    return (...args) => {
+      originalFn.apply(console, args);
+
+      const store = jobStorage.getStore();
+      if (store) {
+        const message = args.map(safeStringify).join(' ');
+        store.job.log(`[${level}] ${message}`).catch(() => {});
+        writeLogEntry({job: store.jobName, id: store.jobId, level, msg: message});
+      }
+    };
+  };
+
+  console.log = capture(originalLog, 'LOG');
+  console.info = capture(originalInfo, 'INFO');
+  console.warn = capture(originalWarn, 'WARN');
+  console.error = capture(originalError, 'ERROR');
+}
+
+export class JobExecutor {
+  constructor() {
+    patchConsoleOnce();
+  }
+
+  /**
+   * Execute a handler with timeout and structured context.
+   * Console output is automatically captured to the job's BullMQ logs
+   * and file logs via AsyncLocalStorage (concurrency-safe).
+   *
+   * @param {Function} handler - async function(payload, context)
+   * @param {import('bullmq').Job} job - BullMQ job instance
+   * @param {number} timeoutMs - Timeout in milliseconds
+   * @param {object} [extraContext] - Extra fields merged into the handler context
+   *   by the caller (e.g. `requeue` for admission control). Keys here override
+   *   nothing — they are added alongside jobId/attempt/signal/logger.
+   * @returns {Promise<*>} Handler result
+   */
+  async run(handler, job, timeoutMs, extraContext = {}) {
+    const ac = new AbortController();
+    let timer;
+
+    if (timeoutMs > 0) {
+      timer = setTimeout(() => ac.abort(), timeoutMs);
+    }
+
+    const logger = createJobLogger(job);
+
+    const context = {
+      // Extra fields injected by the processor (e.g. `requeue` for admission control).
+      // Spread first so the core fields below can never be shadowed by a caller.
+      ...extraContext,
+      jobId: job.id,
+      attempt: job.attemptsMade + 1,
+      signal: ac.signal,
+      logger
+    };
+
+    // Store both the BullMQ job (for job.log) and metadata (for file logging)
+    const store = {job, jobName: job.name, jobId: job.id};
+
+    try {
+      return await jobStorage.run(store, () => handler(job.data, context));
+    } finally {
+      if (timer) clearTimeout(timer);
+    }
+  }
+}
+
+/**
+ * Create a structured logger that writes to BullMQ job logs + terminal + file.
+ */
+function createJobLogger(job) {
+  const prefix = `[${job.name}:${job.id}]`;
+
+  const write = (level, message, data) => {
+    const entry = data
+      ? `[${level.toUpperCase()}] ${message} ${safeStringify(data)}`
+      : `[${level.toUpperCase()}] ${message}`;
+
+    // Write to Redis (visible in Bull Board)
+    job.log(entry).catch(() => {});
+
+    // Write to terminal
+    const output = `${prefix} ${entry}\n`;
+    if (level === 'error') {
+      process.stderr.write(output);
+    } else {
+      process.stdout.write(output);
+    }
+
+    writeLogEntry({
+      job: job.name,
+      id: job.id,
+      level: level.toUpperCase(),
+      msg: message,
+      data
+    });
+  };
+
+  return {
+    info: (message, data) => write('info', message, data),
+    warn: (message, data) => write('warn', message, data),
+    error: (message, data) => write('error', message, data)
+  };
+}