@avada-falcon/worker-sdk 0.5.0

package/README.md

@@ -0,0 +1,442 @@
+# @avada-falcon/worker-sdk
+
+Self-hosted background job runner powered by BullMQ + Redis. Replace Firebase Cloud Functions with a simple, local job queue.
+
+## Features
+
+- **Tier-based concurrency** — heavy/medium/light queues with configurable limits
+- **Cron jobs** — first-class scheduled job support via BullMQ repeatable jobs
+- **Bull Board dashboard** — built-in web UI for inspecting jobs, retries, errors
+- **TLS Redis support** — connect to remote Redis over TLS (production-ready)
+- **Env-driven config** — single YAML works across local dev and production
+- **Console capture** — `console.log/warn/error` inside handlers automatically appears in Bull Board
+- **Readable job IDs** — `{jobName}-{uuid}` format
+- **Graceful shutdown** — drains in-flight jobs on SIGTERM/SIGINT
+
+## Install
+
+```bash
+npm install @avada-falcon/worker-sdk
+```
+
+Requires Node.js >= 20 and a Redis instance you can reach.
+
+## Quick Start
+
+### 1. Create config
+
+`worker.config.yml`:
+
+```yaml
+redis:
+  host: ${REDIS_HOST:-127.0.0.1}
+  port: ${REDIS_PORT:-6379}
+  password: ${REDIS_PASSWORD:-}
+  tls: ${REDIS_TLS:-}
+
+logging:
+  dir: ./logs                        # local file buffer (short retention)
+  retentionDays: 7
+  loki:                              # optional — ship to Loki for long-term search
+    url: ${LOKI_URL:-}
+    batchSize: 100
+    flushInterval: 5000
+    labels:
+      app: my-app
+      env: production
+
+dashboard:
+  port: 3800
+  auth:
+    username: admin
+    password: ${WORKER_DASHBOARD_PASSWORD}
+
+concurrency:
+  heavy: 2
+  medium: 5
+  light: 10
+
+jobs:
+  processOrder:
+    tier: heavy
+    timeout: 60000
+    retry:
+      maxAttempts: 3
+      baseDelay: 2000
+
+  sendNotification:
+    tier: light
+    timeout: 10000
+
+  dailyReport:
+    tier: medium
+    timeout: 300000
+    cron: "0 0 * * *"
+```
+
+YAML supports `${VAR}` and `${VAR:-default}` env interpolation. The same file can run locally (defaults to `127.0.0.1`) or in production (override via env vars).
+
+### 2. Define handlers
+
+```js
+// jobs/processOrder.js
+export async function execute(payload, context) {
+  const {logger, signal, jobId, attempt} = context;
+
+  logger.info('Processing order', {orderId: payload.orderId});
+
+  // console.log/warn/error inside this function (and any code it calls)
+  // is automatically captured to Bull Board logs
+  console.log('Doing some work...');
+
+  return {success: true};
+}
+```
+
+### 3. Start worker
+
+```js
+// worker.mjs
+import {createWorker} from '@minhdevtree/worker-sdk';
+import {execute as processOrder} from './jobs/processOrder.js';
+import {execute as sendNotification} from './jobs/sendNotification.js';
+import {execute as dailyReport} from './jobs/dailyReport.js';
+
+const worker = createWorker('./worker.config.yml');
+
+worker.register('processOrder', processOrder);
+worker.register('sendNotification', sendNotification);
+worker.register('dailyReport', dailyReport);
+
+await worker.start();
+```
+
+Run it: `node worker.mjs`
+
+### 4. Push jobs from your app
+
+```js
+import {createClient} from '@minhdevtree/worker-sdk';
+
+const client = createClient('./worker.config.yml');
+
+await client.add('processOrder', {orderId: 42});
+// → returns {id: "processOrder-e7a3b5c2-3a4f-..."}
+```
+
+The client is lightweight — only creates BullMQ Queue instances on demand. Safe to import anywhere in your app backend.
+
+### 5. Run dashboard as a separate service (recommended)
+
+```js
+// dashboard.mjs
+import {createDashboard} from '@minhdevtree/worker-sdk';
+
+const dashboard = createDashboard('./worker.config.yml');
+await dashboard.start();
+```
+
+Run it: `node dashboard.mjs`
+
+Running the dashboard as its own process means it stays up even when workers restart, and a single dashboard can serve any number of workers (they all share Redis).
+
+## Tiers
+
+Jobs are grouped by resource weight. Each tier maps to a separate BullMQ Worker with its own concurrency limit:
+
+| Tier | Default Concurrency | Use case |
+|------|-------------------|----------|
+| heavy | 2 | CPU/memory intensive work — bulk API calls, image processing |
+| medium | 5 | Moderate processing — page scanning, batch operations |
+| light | 10 | Quick tasks — status updates, notifications |
+
+Override defaults in `worker.config.yml`:
+
+```yaml
+concurrency:
+  heavy: 3
+  medium: 10
+  light: 20
+```
+
+## Dashboard
+
+Bull Board UI mounted at `http://localhost:3800` (port configurable). Features:
+
+- View all queues with job counts
+- Filter jobs by status (completed/failed/waiting/active/delayed)
+- Inspect payload, result, logs, error/stack trace
+- Retry/delete/promote individual jobs
+- View repeatable cron schedules
+
+Basic auth is required — set `username` and `password` in config.
+
+## Handler Context
+
+```js
+export async function execute(payload, context) {
+  context.jobId    // unique job ID — format: jobName-uuid
+  context.attempt  // current attempt number (1-based)
+  context.logger   // {info, warn, error} — writes to Bull Board + stdout + log file
+  context.signal   // AbortSignal — fires when timeout expires
+  context.requeue  // async (retryMs?) — decline this job, hand it back to the queue
+}
+```
+
+### Declining a job with `context.requeue()`
+
+Use it when this worker is *temporarily* unable to run the job — an app-level admission
+gate such as a per-worker memory budget. Blocking inside the handler instead would keep
+the job `active` and pinned to this worker, where no idle peer can pick it up.
+
+```js
+export async function execute(payload, context) {
+  if (!memoryBudget.tryAcquire(payload.size)) {
+    return context.requeue(500);   // back to the queue, retried in ~500-1000ms
+  }
+  // ...
+}
+```
+
+- The job is moved to `delayed` and re-dispatched to whichever worker is free. **No attempt
+  is burned** and no `failed` event fires — a decline is not a failure.
+- `retryMs` defaults to `200` and is jittered (actual delay is `retryMs`–`2×retryMs`) so
+  a batch of declined jobs doesn't wake in lockstep.
+- **`requeue()` always throws.** Do not wrap it in a `try/catch` that swallows the error —
+  BullMQ would then complete a job that is also sitting in the delayed set, and it would
+  run twice. Call it as `return context.requeue()`, or let the throw propagate.
+- Declines are capped per job at `jobs.<name>.maxRequeues` (default `50`). Past the cap the
+  job fails with a real error instead of ping-ponging forever on a gate that never opens.
+
+```yaml
+jobs:
+  heavyImport:
+    tier: heavy
+    timeout: 60000
+    maxRequeues: 100     # optional — default 50
+```
+
+## File Logging
+
+When `logging.dir` is configured, all job logs are written to daily JSON line files on disk:
+
+```
+logs/
+  2026-04-14.log
+  2026-04-13.log
+  ...
+```
+
+Each line is a JSON object:
+
+```json
+{"ts":"2026-04-14T10:30:00.123Z","job":"processOrder","id":"processOrder-abc123","level":"INFO","msg":"Processing","data":{"orderId":42}}
+```
+
+Both `context.logger.info/warn/error` and captured `console.log/warn/error` are written.
+
+Local files act as a short-term buffer — set `retentionDays` to how many days you want to keep on disk (e.g. 7). Old files are auto-deleted on worker startup. For long-term archive, configure `logging.loki` to ship logs to Grafana Loki (see below).
+
+Search with grep:
+
+```bash
+# All errors from a specific day
+grep '"ERROR"' logs/2026-04-14.log
+
+# Everything for a specific job ID
+grep 'processOrder-abc123' logs/2026-04-14.log
+
+# Search across multiple days
+grep 'shopId.*xyz' logs/2026-04-*.log
+```
+
+For Docker deployments, mount the logs directory as a volume so files persist on the host:
+
+```yaml
+volumes:
+  - ./logs:/app/functions/logs
+```
+
+## Cron Jobs
+
+Add a `cron` field to any job in the config. The SDK registers it as a BullMQ repeatable job — schedule survives restarts.
+
+```yaml
+jobs:
+  dailyReport:
+    tier: medium
+    timeout: 300000
+    cron: "0 0 * * *"        # every day at midnight
+```
+
+The handler is registered like any other job — same `execute(payload, context)` signature.
+
+## Env Variables
+
+| Variable | Required | Description |
+|---|---|---|
+| `WORKER_DASHBOARD_PASSWORD` | Yes | Bull Board dashboard password |
+| `REDIS_HOST` | No | Defaults to YAML config |
+| `REDIS_PORT` | No | Defaults to YAML config |
+| `REDIS_PASSWORD` | No | Defaults to YAML config |
+| `REDIS_TLS` | No | Set to `true` to enable TLS |
+
+Any field in `worker.config.yml` can be made env-driven via `${VAR_NAME:-default}` syntax.
+
+## Long-term log search with Loki
+
+This SDK configures BullMQ to retain only the last ~1000 completed jobs in Redis (`removeOnComplete`). For long-term historical search (weeks or months), configure Loki:
+
+```yaml
+logging:
+  loki:
+    url: http://loki:3100
+    batchSize: 100
+    flushInterval: 5000
+    labels:
+      app: my-app
+```
+
+The SDK will push every log entry (both `context.logger.*` and captured `console.*`) to Loki in batches. Use Grafana to search by job name, level, shop ID, date range.
+
+If `loki.url` is empty or missing, Loki shipping is disabled — the SDK falls back to file-only logging.
+
+Log retention in Loki is controlled by the Loki server's own `retention_period` config, not by the SDK.
+
+**Setup your Loki stack** — see [SETUP.md](./SETUP.md) for a Docker Compose example that runs Loki + Grafana alongside the worker.
+
+## Multi-worker deployments
+
+A single Redis + Loki + Bull Board can serve any number of workers. Scale horizontally on one machine (`docker compose up --scale`) or across hosts.
+
+### Identity and specialization
+
+```yaml
+# worker.config.yml
+worker:
+  id: ${WORKER_ID:-}              # empty → auto-generated ${hostname}-${pid}
+
+concurrency:
+  heavy: 2
+  medium: 5
+  light: 0                        # 0 = opt out — this worker skips the light tier
+```
+
+Each worker gets its own ID. In Docker Compose `--scale N` mode the auto-generated default (`${hostname}-${pid}`) gives meaningful IDs because Docker assigns unique hostnames per replica. For multi-host deployments set `WORKER_ID=mac-mini` / `WORKER_ID=vps-hanoi` per host so Grafana labels stay readable.
+
+Tier opt-out via `concurrency: 0` enables heterogeneous pools: a beefy box can be heavy-only, a small box can be light-only. BullMQ distributes jobs atomically — a worker that doesn't subscribe to a tier never pulls from it.
+
+### Liveness: heartbeats
+
+```yaml
+worker:
+  heartbeat:
+    enabled: true                 # default
+    intervalMs: 10000             # beat every 10s
+    ttlMs: 30000                  # key expires after 30s of silence
+```
+
+Every worker writes a TTL'd key `worker:heartbeat:<workerId>` to Redis on the interval. If a worker dies or loses Redis connectivity, the key expires automatically. Enumerate live workers:
+
+```js
+import {listWorkers} from '@minhdevtree/worker-sdk';
+import Redis from 'ioredis';
+
+const redis = new Redis({host, port, password});
+const workers = await listWorkers(redis);
+// → [{workerId, hostname, pid, tiers, startedAt, lastBeat}, ...]
+```
+
+### Health checks
+
+Four admin helpers for monitoring cluster state:
+
+```js
+import {
+  pingRedis,
+  getQueueDepths,
+  checkDashboard,
+  getClusterHealth
+} from '@minhdevtree/worker-sdk';
+
+// Redis PING + latency
+await pingRedis(redis);
+// → {ok: true, latencyMs: 3}
+
+// Per-tier queue depth (waiting/active/delayed/completed/failed/paused + total)
+// Each tier reports its own ok flag — a per-tier failure doesn't abort the others.
+await getQueueDepths(redis, ['heavy', 'medium', 'light']);
+// → {heavy: {ok: true, waiting: 2, active: 1, ..., total: 3}, ...}
+
+// Dashboard /health endpoint probe (no auth)
+await checkDashboard('http://host:3800');
+// → {ok: true, latencyMs: 12, status: 200, uptime: 1234, timestamp: '...'}
+
+// One-shot aggregate: runs all probes in parallel
+await getClusterHealth({
+  redis,
+  tiers: ['heavy', 'medium', 'light'],   // optional
+  dashboardUrl: 'http://host:3800',      // optional
+  timeouts: {                            // optional — defaults shown
+    redisMs: 2000,
+    workersMs: 5000,
+    queuesMs: 5000,
+    dashboardMs: 3000
+  }
+});
+// → {ok, status: 'healthy'|'degraded'|'unhealthy', checkedAt, redis,
+//    workers: {count, items}, queues: {byTier}, dashboard}
+```
+
+`getClusterHealth` doesn't short-circuit — a failure in one probe reports `ok: false` for that section but still returns the others. Top-level `ok` is true only if every probed section is ok. The `status` field classifies the rollup: `healthy` (everything ok), `unhealthy` (Redis itself is down — the critical dependency), or `degraded` (Redis ok but at least one other section failed).
+
+`checkDashboard` validates that `baseUrl` parses and uses `http:` or `https:`, sets `redirect: 'manual'`, and rejects responses larger than 64 KiB. These are basic SSRF guards — if you wire `baseUrl` to user input, add your own allow-list on top.
+
+### Scheduled jobs: the cron leader
+
+```yaml
+cron:
+  leader: ${CRON_LEADER:-false}   # EXACTLY ONE worker should set this to true
+```
+
+In a multi-worker pool exactly one worker should be designated the cron leader. Only that worker registers scheduled jobs — others skip registration. If no worker has `cron.leader: true` and your `jobs` config includes cron entries, the SDK warns on startup and scheduled jobs do not fire.
+
+For a pool of three, one compose-file pattern:
+
+```yaml
+services:
+  worker-leader:
+    environment: {CRON_LEADER: "true"}
+    # one replica, always on
+  worker:
+    environment: {CRON_LEADER: "false"}
+    # scale this service: docker compose up -d --scale worker=N
+```
+
+### Filtering per worker in Grafana
+
+Every log pushed to Loki carries the `workerId` label automatically:
+
+```logql
+# just worker-2's logs
+{app="my-app", workerId="mac-mini-2"}
+
+# errors from any worker
+{app="my-app", level="ERROR"}
+
+# cross-worker comparison by job name
+{app="my-app", job="generateAnchor"} | json
+```
+
+## Migration from Firebase Pub/Sub
+
+| Before (Firebase) | After (Worker SDK) |
+|---|---|
+| `functions.runWith({memory, timeout})` | `worker.config.yml` job entry |
+| `.pubsub.topic('name').onPublish(fn)` | `worker.register('name', execute)` |
+| `JSON.parse(Buffer.from(message.data))` | `payload` (already parsed) |
+| `console.log()` | Works as-is — captured to Bull Board |
+| `publishTopic('next', data)` | `client.add('next', data)` |
+| Runs on Google Cloud | Runs on your machine |
+
+See `SETUP.md` for the full integration guide with handler structure, file layout, and migration steps.

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@avada-falcon/worker-sdk",
+  "version": "0.5.0",
+  "description": "Self-hosted background job runner powered by BullMQ + Redis",
+  "type": "module",
+  "main": "src/index.cjs",
+  "files": [
+    "src"
+  ],
+  "exports": {
+    ".": {
+      "import": "./src/index.js",
+      "require": "./src/index.cjs"
+    }
+  },
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://gitlab.com/avada/seoon-team/worker-sdk.git"
+  },
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "worker",
+    "job-queue",
+    "bullmq",
+    "redis",
+    "background-jobs",
+    "self-hosted"
+  ],
+  "dependencies": {
+    "bullmq": "^5.0.0",
+    "@bull-board/api": "^6.0.0",
+    "@bull-board/express": "^6.0.0",
+    "express": "^5.0.0",
+    "ioredis": "^5.0.0",
+    "js-yaml": "^4.1.0"
+  },
+  "devDependencies": {
+    "vitest": "^4.0.0"
+  }
+}

package/src/admin/checkDashboard.js

@@ -0,0 +1,77 @@
+const MAX_BODY_BYTES = 64 * 1024;
+
+/**
+ * Probe the Bull Board dashboard `/health` endpoint.
+ *
+ * The `/health` route is unauthenticated — no credentials needed.
+ *
+ * Hardening:
+ *  - `baseUrl` must parse as a URL with `http:` or `https:` scheme. Other
+ *    schemes (file:, gopher:, ...) are rejected to limit SSRF surface.
+ *  - Redirects are not followed (`redirect: 'manual'`) so a misbehaving
+ *    dashboard cannot bounce the probe to an internal metadata endpoint.
+ *  - Response bodies larger than 64 KiB (by Content-Length) are rejected.
+ *  - The abort timer covers both the request and the JSON body parse.
+ *
+ * Caller is still responsible for not exposing `baseUrl` to untrusted input
+ * without additional allow-listing.
+ *
+ * @param {string} baseUrl - Dashboard URL (e.g. 'http://192.168.1.162:3800')
+ * @param {object} [options]
+ * @param {number} [options.timeoutMs=3000]
+ * @returns {Promise<{ok: boolean, latencyMs: number, status?: number, uptime?: number, timestamp?: string, error?: string}>}
+ */
+export async function checkDashboard(baseUrl, {timeoutMs = 3000} = {}) {
+  if (!baseUrl || typeof baseUrl !== 'string') {
+    return {ok: false, latencyMs: 0, error: 'baseUrl is required'};
+  }
+
+  let parsed;
+  try {
+    parsed = new URL(baseUrl);
+  } catch {
+    return {ok: false, latencyMs: 0, error: 'invalid baseUrl'};
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    return {ok: false, latencyMs: 0, error: `unsupported protocol: ${parsed.protocol}`};
+  }
+
+  const url = baseUrl.replace(/\/+$/, '') + '/health';
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  timer.unref?.();
+
+  const start = Date.now();
+  const fail = (error, status) => ({
+    ok: false,
+    latencyMs: Date.now() - start,
+    ...(status !== undefined && {status}),
+    error
+  });
+
+  try {
+    const res = await fetch(url, {signal: controller.signal, redirect: 'manual'});
+    if (!res.ok) {
+      return fail(`HTTP ${res.status}`, res.status);
+    }
+    const contentLength = Number(res.headers?.get?.('content-length') ?? 0);
+    if (contentLength > MAX_BODY_BYTES) {
+      return fail(`response too large (${contentLength} bytes)`, res.status);
+    }
+    const body = await res.json().catch(() => ({}));
+    if (body && typeof body.status === 'string' && body.status !== 'ok') {
+      return fail(`dashboard status: ${body.status}`, res.status);
+    }
+    return {
+      ok: true,
+      latencyMs: Date.now() - start,
+      status: res.status,
+      uptime: body.uptime,
+      timestamp: body.timestamp
+    };
+  } catch (err) {
+    return fail(err.name === 'AbortError' ? `timeout after ${timeoutMs}ms` : err.message);
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/admin/getClusterHealth.js

@@ -0,0 +1,114 @@
+import {listWorkers} from './listWorkers.js';
+import {pingRedis} from './pingRedis.js';
+import {getQueueDepths} from './getQueueDepths.js';
+import {checkDashboard} from './checkDashboard.js';
+import {withTimeout} from './withTimeout.js';
+
+/**
+ * Aggregate cluster health: Redis, workers, queue depths, (optional) dashboard.
+ *
+ * All probes run in parallel. A failure in one probe does not short-circuit
+ * the others — each section reports its own `ok` flag. The top-level `ok` is
+ * true only when every probed section is ok.
+ *
+ * `status` classifies the rollup:
+ *   - `'healthy'`   — every probed section ok
+ *   - `'unhealthy'` — Redis is down (the critical dependency)
+ *   - `'degraded'`  — Redis ok, but at least one other section failed
+ *
+ * Throws only on programmer error (missing `redis`); never on transient failures.
+ *
+ * @param {object} params
+ * @param {import('ioredis').Redis} params.redis - ioredis client (used for PING + listWorkers)
+ * @param {object|import('ioredis').Redis} [params.connection] - Passed to BullMQ Queue. Defaults to `redis`.
+ * @param {string[]} [params.tiers] - Tier names to probe (e.g. ['heavy','medium','light']). Omit to skip queue depths.
+ * @param {string} [params.dashboardUrl] - Base URL (e.g. 'http://host:3800'). Omit to skip dashboard probe.
+ * @param {object} [params.timeouts] - Per-probe timeouts in ms
+ * @param {number} [params.timeouts.redisMs=2000]
+ * @param {number} [params.timeouts.dashboardMs=3000]
+ * @param {number} [params.timeouts.queuesMs=5000]
+ * @param {number} [params.timeouts.workersMs=5000]
+ * @returns {Promise<{
+ *   ok: boolean,
+ *   status: 'healthy' | 'degraded' | 'unhealthy',
+ *   checkedAt: string,
+ *   redis: object,
+ *   workers: {ok: boolean, count: number, items: Array, error?: string},
+ *   queues?: {ok: boolean, byTier?: object, error?: string},
+ *   dashboard?: object
+ * }>}
+ */
+export async function getClusterHealth({
+  redis,
+  connection,
+  tiers,
+  dashboardUrl,
+  timeouts = {}
+}) {
+  if (!redis) {
+    throw new Error('getClusterHealth: `redis` is required');
+  }
+
+  const {
+    redisMs = 2000,
+    dashboardMs = 3000,
+    queuesMs = 5000,
+    workersMs = 5000
+  } = timeouts;
+  const queueConnection = connection || redis;
+
+  const [redisResult, workersResult, queuesResult, dashboardResult] = await Promise.all([
+    pingRedis(redis, {timeoutMs: redisMs}),
+    safeRunWithTimeout(() => listWorkers(redis), workersMs),
+    tiers && tiers.length > 0
+      ? safeRunWithTimeout(() => getQueueDepths(queueConnection, tiers, {timeoutMs: queuesMs}), queuesMs)
+      : null,
+    dashboardUrl ? checkDashboard(dashboardUrl, {timeoutMs: dashboardMs}) : null
+  ]);
+
+  const workers = workersResult.ok
+    ? {ok: true, count: workersResult.value.length, items: workersResult.value}
+    : {ok: false, count: 0, items: [], error: workersResult.error};
+
+  const sections = {
+    checkedAt: new Date().toISOString(),
+    redis: redisResult,
+    workers
+  };
+
+  if (queuesResult) {
+    if (!queuesResult.ok) {
+      sections.queues = {ok: false, error: queuesResult.error};
+    } else {
+      const byTier = queuesResult.value;
+      const allTiersOk = Object.values(byTier).every(t => t.ok !== false);
+      sections.queues = {ok: allTiersOk, byTier};
+    }
+  }
+
+  if (dashboardResult) {
+    sections.dashboard = dashboardResult;
+  }
+
+  sections.ok =
+    sections.redis.ok &&
+    sections.workers.ok &&
+    (sections.queues?.ok ?? true) &&
+    (sections.dashboard?.ok ?? true);
+
+  sections.status = sections.ok
+    ? 'healthy'
+    : !sections.redis.ok
+      ? 'unhealthy'
+      : 'degraded';
+
+  return sections;
+}
+
+async function safeRunWithTimeout(fn, timeoutMs) {
+  try {
+    return {ok: true, value: await withTimeout(fn(), timeoutMs)};
+  } catch (err) {
+    return {ok: false, error: err.message};
+  }
+}