@avada-falcon/worker-sdk 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +442 -0
- package/package.json +50 -0
- package/src/admin/checkDashboard.js +77 -0
- package/src/admin/getClusterHealth.js +114 -0
- package/src/admin/getQueueDepths.js +66 -0
- package/src/admin/listWorkers.js +44 -0
- package/src/admin/pingRedis.js +29 -0
- package/src/admin/withTimeout.js +21 -0
- package/src/client/client.js +39 -0
- package/src/config/loader.js +178 -0
- package/src/cron/cronManager.js +35 -0
- package/src/dashboard/server.js +79 -0
- package/src/dashboard/standalone.js +67 -0
- package/src/index.cjs +18 -0
- package/src/index.js +8 -0
- package/src/logging/fileLogger.js +77 -0
- package/src/logging/lokiShipper.js +219 -0
- package/src/queue/queues.js +62 -0
- package/src/shutdown/shutdownManager.js +50 -0
- package/src/worker/handlerRegistry.js +47 -0
- package/src/worker/heartbeat.js +106 -0
- package/src/worker/heartbeatKey.js +6 -0
- package/src/worker/jobExecutor.js +169 -0
- package/src/worker/tierManager.js +55 -0
- package/src/worker/worker.js +204 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import {Queue} from 'bullmq';
|
|
2
|
+
import {queueName} from '../queue/queues.js';
|
|
3
|
+
import {withTimeout} from './withTimeout.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Get BullMQ queue depths for the given tiers.
|
|
7
|
+
*
|
|
8
|
+
* Returns one entry per tier with `{ok, ...counts, total}`. A failure on one
|
|
9
|
+
* tier does not abort the others — that tier's entry becomes
|
|
10
|
+
* `{ok: false, error}` and the rest still report their counts.
|
|
11
|
+
*
|
|
12
|
+
* Creates short-lived Queue instances and closes them before returning.
|
|
13
|
+
* If `connection` is an ioredis instance, BullMQ reuses it; the instance is
|
|
14
|
+
* not closed by this helper.
|
|
15
|
+
*
|
|
16
|
+
* @param {object|import('ioredis').Redis} connection - ioredis client OR connection options
|
|
17
|
+
* @param {string[]} tiers - tier names (e.g. ['heavy', 'medium', 'light'])
|
|
18
|
+
* @param {object} [options]
|
|
19
|
+
* @param {number} [options.timeoutMs=5000] - Per-tier timeout for getJobCounts
|
|
20
|
+
* @returns {Promise<Record<string, {ok: boolean, waiting?, active?, delayed?, completed?, failed?, paused?, total?, error?: string}>>}
|
|
21
|
+
*/
|
|
22
|
+
export async function getQueueDepths(connection, tiers, {timeoutMs = 5000} = {}) {
|
|
23
|
+
if (!Array.isArray(tiers) || tiers.length === 0) {
|
|
24
|
+
return {};
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const queues = tiers.map(tier => ({
|
|
28
|
+
tier,
|
|
29
|
+
queue: new Queue(queueName(tier), {connection})
|
|
30
|
+
}));
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
const settled = await Promise.allSettled(
|
|
34
|
+
queues.map(async ({queue}) => {
|
|
35
|
+
const counts = await withTimeout(
|
|
36
|
+
queue.getJobCounts(
|
|
37
|
+
'waiting',
|
|
38
|
+
'active',
|
|
39
|
+
'delayed',
|
|
40
|
+
'completed',
|
|
41
|
+
'failed',
|
|
42
|
+
'paused'
|
|
43
|
+
),
|
|
44
|
+
timeoutMs
|
|
45
|
+
);
|
|
46
|
+
const total =
|
|
47
|
+
(counts.waiting || 0) +
|
|
48
|
+
(counts.active || 0) +
|
|
49
|
+
(counts.delayed || 0) +
|
|
50
|
+
(counts.paused || 0);
|
|
51
|
+
return {ok: true, ...counts, total};
|
|
52
|
+
})
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
const out = {};
|
|
56
|
+
settled.forEach((r, i) => {
|
|
57
|
+
const tier = tiers[i];
|
|
58
|
+
out[tier] = r.status === 'fulfilled'
|
|
59
|
+
? r.value
|
|
60
|
+
: {ok: false, error: r.reason?.message || String(r.reason)};
|
|
61
|
+
});
|
|
62
|
+
return out;
|
|
63
|
+
} finally {
|
|
64
|
+
await Promise.all(queues.map(({queue}) => queue.close().catch(() => {})));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import {HEARTBEAT_KEY_PREFIX} from '../worker/heartbeatKey.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* List all currently-alive workers by scanning heartbeat keys in Redis.
|
|
5
|
+
* Keys with missing or malformed values are skipped silently.
|
|
6
|
+
*
|
|
7
|
+
* @param {object} redis - ioredis client (needs scan, mget)
|
|
8
|
+
* @returns {Promise<Array<{workerId, hostname, pid, tiers, startedAt, lastBeat}>>}
|
|
9
|
+
*/
|
|
10
|
+
export async function listWorkers(redis) {
|
|
11
|
+
const keys = await scanAll(redis, `${HEARTBEAT_KEY_PREFIX}*`);
|
|
12
|
+
if (keys.length === 0) return [];
|
|
13
|
+
|
|
14
|
+
const values = await redis.mget(...keys);
|
|
15
|
+
const workers = [];
|
|
16
|
+
for (const v of values) {
|
|
17
|
+
if (v === null || v === undefined) continue;
|
|
18
|
+
try {
|
|
19
|
+
const parsed = JSON.parse(v);
|
|
20
|
+
if (parsed && typeof parsed === 'object' && typeof parsed.workerId === 'string') {
|
|
21
|
+
workers.push(parsed);
|
|
22
|
+
}
|
|
23
|
+
// else: malformed shape — skip silently like malformed JSON
|
|
24
|
+
} catch {
|
|
25
|
+
// malformed entry — skip
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return workers;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Iterate SCAN cursor until complete, collecting all matched keys.
|
|
33
|
+
* Uses SCAN (not KEYS) so this is safe to call on large Redis instances.
|
|
34
|
+
*/
|
|
35
|
+
async function scanAll(redis, pattern) {
|
|
36
|
+
const keys = [];
|
|
37
|
+
let cursor = '0';
|
|
38
|
+
do {
|
|
39
|
+
const [next, batch] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100);
|
|
40
|
+
for (const k of batch) keys.push(k);
|
|
41
|
+
cursor = next;
|
|
42
|
+
} while (cursor !== '0');
|
|
43
|
+
return keys;
|
|
44
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import {withTimeout} from './withTimeout.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Ping Redis and measure round-trip latency.
|
|
5
|
+
*
|
|
6
|
+
* Never throws on transient failures (timeout, connection error, unexpected
|
|
7
|
+
* reply) — returns a structured `{ok: false, error}` instead.
|
|
8
|
+
*
|
|
9
|
+
* @param {object} redis - ioredis client
|
|
10
|
+
* @param {object} [options]
|
|
11
|
+
* @param {number} [options.timeoutMs=2000] - Reject if PING doesn't return in time
|
|
12
|
+
* @returns {Promise<{ok: boolean, latencyMs: number, error?: string}>}
|
|
13
|
+
*/
|
|
14
|
+
export async function pingRedis(redis, {timeoutMs = 2000} = {}) {
|
|
15
|
+
if (!redis || typeof redis.ping !== 'function') {
|
|
16
|
+
return {ok: false, latencyMs: 0, error: 'redis client is required'};
|
|
17
|
+
}
|
|
18
|
+
const start = Date.now();
|
|
19
|
+
try {
|
|
20
|
+
const result = await withTimeout(redis.ping(), timeoutMs);
|
|
21
|
+
const latencyMs = Date.now() - start;
|
|
22
|
+
if (result !== 'PONG') {
|
|
23
|
+
return {ok: false, latencyMs, error: `unexpected response: ${result}`};
|
|
24
|
+
}
|
|
25
|
+
return {ok: true, latencyMs};
|
|
26
|
+
} catch (err) {
|
|
27
|
+
return {ok: false, latencyMs: Date.now() - start, error: err.message};
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Race a promise against a timeout. The timer never keeps the process alive
|
|
3
|
+
* (unref) and is always cleared once the race settles.
|
|
4
|
+
* Shared by the admin probes (pingRedis, getQueueDepths, getClusterHealth).
|
|
5
|
+
*
|
|
6
|
+
* Note: this does NOT cancel the underlying operation — checkDashboard uses
|
|
7
|
+
* AbortController instead because fetch supports real cancellation.
|
|
8
|
+
*
|
|
9
|
+
* @param {Promise} promise
|
|
10
|
+
* @param {number} ms
|
|
11
|
+
* @returns {Promise<*>} Resolves/rejects with the promise, or rejects with
|
|
12
|
+
* `timeout after <ms>ms` if the timeout wins.
|
|
13
|
+
*/
|
|
14
|
+
export function withTimeout(promise, ms) {
|
|
15
|
+
let timer;
|
|
16
|
+
const timeout = new Promise((_, reject) => {
|
|
17
|
+
timer = setTimeout(() => reject(new Error(`timeout after ${ms}ms`)), ms);
|
|
18
|
+
timer.unref?.();
|
|
19
|
+
});
|
|
20
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
21
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import {randomUUID} from 'crypto';
|
|
2
|
+
import {loadConfig} from '../config/loader.js';
|
|
3
|
+
import {QueuePool, buildJobOpts, DEFAULT_TIER} from '../queue/queues.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Create a lightweight job-pushing client.
|
|
7
|
+
* Reads YAML config for job-to-tier mapping. Lazily creates BullMQ queues.
|
|
8
|
+
*
|
|
9
|
+
* @param {string} configPath - Path to worker YAML config
|
|
10
|
+
* @returns {{add: Function, close: Function}}
|
|
11
|
+
*/
|
|
12
|
+
export function createClient(configPath) {
|
|
13
|
+
const config = loadConfig(configPath);
|
|
14
|
+
const pool = new QueuePool(config.redis);
|
|
15
|
+
const optsCache = new Map(); // job opts are static per job name — build once
|
|
16
|
+
|
|
17
|
+
return {
|
|
18
|
+
async add(jobName, payload) {
|
|
19
|
+
const jobConfig = config.jobs[jobName];
|
|
20
|
+
if (!jobConfig) {
|
|
21
|
+
throw new Error(`Unknown job: ${jobName}. Define it in your worker config.`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const queue = pool.get(jobConfig.tier || DEFAULT_TIER);
|
|
25
|
+
if (!optsCache.has(jobName)) {
|
|
26
|
+
optsCache.set(jobName, buildJobOpts(jobConfig));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
return queue.add(jobName, payload, {
|
|
30
|
+
...optsCache.get(jobName),
|
|
31
|
+
jobId: `${jobName}-${randomUUID()}`
|
|
32
|
+
});
|
|
33
|
+
},
|
|
34
|
+
|
|
35
|
+
async close() {
|
|
36
|
+
await pool.closeAll();
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import {readFileSync} from 'fs';
|
|
2
|
+
import yaml from 'js-yaml';
|
|
3
|
+
|
|
4
|
+
const DEFAULT_CONCURRENCY = {heavy: 2, medium: 5, light: 10};
|
|
5
|
+
const DEFAULT_HEARTBEAT_INTERVAL_MS = 10000;
|
|
6
|
+
const DEFAULT_HEARTBEAT_TTL_MS = 30000;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Load and parse YAML config with environment variable interpolation.
|
|
10
|
+
* @param {string} configPath - Path to the YAML config file
|
|
11
|
+
* @returns {object} Parsed config
|
|
12
|
+
*/
|
|
13
|
+
export function loadConfig(configPath) {
|
|
14
|
+
const raw = readFileSync(configPath, 'utf-8');
|
|
15
|
+
const interpolated = interpolateEnv(raw);
|
|
16
|
+
const loaded = yaml.load(interpolated);
|
|
17
|
+
// Empty/comments-only YAML parses to undefined; a scalar isn't a usable config.
|
|
18
|
+
// Fall through to the redis validation below for a helpful error.
|
|
19
|
+
const config = loaded && typeof loaded === 'object' ? loaded : {};
|
|
20
|
+
|
|
21
|
+
config.concurrency = {...DEFAULT_CONCURRENCY, ...config.concurrency};
|
|
22
|
+
config.dashboard = config.dashboard || {};
|
|
23
|
+
config.jobs = config.jobs || {};
|
|
24
|
+
|
|
25
|
+
config.worker = config.worker || {};
|
|
26
|
+
config.cron = config.cron || {};
|
|
27
|
+
|
|
28
|
+
normalizeRedisOptions(config.redis);
|
|
29
|
+
normalizeLokiOptions(config.logging);
|
|
30
|
+
normalizeWorkerOptions(config.worker);
|
|
31
|
+
normalizeCronOptions(config.cron);
|
|
32
|
+
|
|
33
|
+
// Validate Redis config exists after normalization
|
|
34
|
+
if (!config.redis || (!config.redis.host && !config.redis.port)) {
|
|
35
|
+
throw new Error(
|
|
36
|
+
'worker.config.yml: redis config is required. Set redis.host/port in YAML or via REDIS_HOST/REDIS_PORT env vars.'
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return config;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Coerce obj[key] from string to integer in place (env vars are always
|
|
45
|
+
* strings). Throws on non-numeric strings; leaves non-string values
|
|
46
|
+
* (already numbers, or absent) untouched.
|
|
47
|
+
*/
|
|
48
|
+
function coerceInt(obj, key, label) {
|
|
49
|
+
if (typeof obj[key] !== 'string') return;
|
|
50
|
+
const parsed = parseInt(obj[key], 10);
|
|
51
|
+
if (Number.isNaN(parsed)) throw new Error(`Invalid ${label}: "${obj[key]}"`);
|
|
52
|
+
obj[key] = parsed;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Coerce a boolean config value that may arrive as a string env var.
|
|
57
|
+
* Strict: only 'true'/true and 'false'/false are recognized; anything else
|
|
58
|
+
* (including typos like "yes"/"1") gets the default — be explicit in config.
|
|
59
|
+
*/
|
|
60
|
+
function coerceBool(value, defaultValue) {
|
|
61
|
+
if (value === 'true' || value === true) return true;
|
|
62
|
+
if (value === 'false' || value === false) return false;
|
|
63
|
+
return defaultValue;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Normalize Redis connection options after env interpolation.
|
|
68
|
+
* - Strip empty/null fields (when env var is unset, YAML parses ${VAR:-} as null)
|
|
69
|
+
* - Coerce port from string to number (env vars are always strings)
|
|
70
|
+
* - Convert tls: "true" to {} (enables TLS); empty/null/false removes the key
|
|
71
|
+
*/
|
|
72
|
+
function normalizeRedisOptions(redis) {
|
|
73
|
+
if (!redis) return;
|
|
74
|
+
|
|
75
|
+
// Drop empty/null fields so they don't override ioredis defaults
|
|
76
|
+
for (const key of Object.keys(redis)) {
|
|
77
|
+
if (redis[key] === null || redis[key] === '' || redis[key] === undefined) {
|
|
78
|
+
delete redis[key];
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
coerceInt(redis, 'port', 'redis.port');
|
|
83
|
+
|
|
84
|
+
// 'false'/false → no TLS; 'true'/true → {} (enables TLS)
|
|
85
|
+
if (redis.tls === 'false' || redis.tls === false) {
|
|
86
|
+
delete redis.tls;
|
|
87
|
+
} else if (redis.tls === 'true' || redis.tls === true) {
|
|
88
|
+
redis.tls = {};
|
|
89
|
+
}
|
|
90
|
+
// If redis.tls is already an object, leave it alone (advanced TLS config)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Normalize logging.loki options after env interpolation.
|
|
95
|
+
* - Remove loki block entirely if url is empty (keeps SDK in file-only mode).
|
|
96
|
+
* - Coerce batchSize and flushInterval from string to number (env vars are always strings).
|
|
97
|
+
* - Apply defaults: batchSize=100, flushInterval=5000.
|
|
98
|
+
*/
|
|
99
|
+
function normalizeLokiOptions(logging) {
|
|
100
|
+
if (!logging || !logging.loki) return;
|
|
101
|
+
|
|
102
|
+
const loki = logging.loki;
|
|
103
|
+
|
|
104
|
+
// Empty url = Loki disabled
|
|
105
|
+
if (!loki.url) {
|
|
106
|
+
delete logging.loki;
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
coerceInt(loki, 'batchSize', 'loki.batchSize');
|
|
111
|
+
coerceInt(loki, 'flushInterval', 'loki.flushInterval');
|
|
112
|
+
|
|
113
|
+
// Defaults (use ?? to preserve valid 0 / false values)
|
|
114
|
+
loki.batchSize = loki.batchSize ?? 100;
|
|
115
|
+
loki.flushInterval = loki.flushInterval ?? 5000;
|
|
116
|
+
loki.labels = loki.labels ?? {};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Normalize worker options after env interpolation.
|
|
121
|
+
* - Apply heartbeat defaults (enabled=true, intervalMs=10000, ttlMs=30000)
|
|
122
|
+
* - Coerce intervalMs/ttlMs from string to number
|
|
123
|
+
* - Coerce enabled from string to boolean
|
|
124
|
+
* - Validate intervalMs < ttlMs (else key expires between beats)
|
|
125
|
+
*/
|
|
126
|
+
function normalizeWorkerOptions(worker) {
|
|
127
|
+
worker.heartbeat = worker.heartbeat || {};
|
|
128
|
+
const hb = worker.heartbeat;
|
|
129
|
+
|
|
130
|
+
hb.enabled = coerceBool(hb.enabled, true);
|
|
131
|
+
|
|
132
|
+
coerceInt(hb, 'intervalMs', 'worker.heartbeat.intervalMs');
|
|
133
|
+
hb.intervalMs = hb.intervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
|
|
134
|
+
|
|
135
|
+
coerceInt(hb, 'ttlMs', 'worker.heartbeat.ttlMs');
|
|
136
|
+
hb.ttlMs = hb.ttlMs ?? DEFAULT_HEARTBEAT_TTL_MS;
|
|
137
|
+
|
|
138
|
+
if (hb.intervalMs <= 0) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
`worker.heartbeat.intervalMs must be > 0 (got ${hb.intervalMs}); ` +
|
|
141
|
+
'a 0 or negative interval would tight-loop or never fire'
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
if (hb.ttlMs <= 0) {
|
|
145
|
+
throw new Error(
|
|
146
|
+
`worker.heartbeat.ttlMs must be > 0 (got ${hb.ttlMs}); ` +
|
|
147
|
+
'a 0 or negative TTL would expire the key immediately'
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (hb.intervalMs >= hb.ttlMs) {
|
|
152
|
+
throw new Error(
|
|
153
|
+
`worker.heartbeat.intervalMs (${hb.intervalMs}) must be less than ttlMs (${hb.ttlMs}); ` +
|
|
154
|
+
'otherwise the heartbeat key expires between beats'
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Normalize cron options after env interpolation.
|
|
161
|
+
* - Coerce leader from string to boolean; default false
|
|
162
|
+
*/
|
|
163
|
+
function normalizeCronOptions(cron) {
|
|
164
|
+
cron.leader = coerceBool(cron.leader, false);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Replace ${VAR_NAME} with process.env.VAR_NAME.
|
|
169
|
+
* Supports default values: ${VAR_NAME:-default_value}
|
|
170
|
+
*/
|
|
171
|
+
function interpolateEnv(str) {
|
|
172
|
+
return str.replace(/\$\{(\w+)(?::-([^}]*))?\}/g, (match, varName, defaultValue) => {
|
|
173
|
+
const value = process.env[varName];
|
|
174
|
+
if (value !== undefined) return value;
|
|
175
|
+
if (defaultValue !== undefined) return defaultValue;
|
|
176
|
+
return match;
|
|
177
|
+
});
|
|
178
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import {QueuePool, buildJobOpts, DEFAULT_TIER} from '../queue/queues.js';
|
|
2
|
+
|
|
3
|
+
export class CronManager {
|
|
4
|
+
constructor(redisOpts) {
|
|
5
|
+
this._pool = new QueuePool(redisOpts);
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
async register(jobs, {leader = false} = {}) {
|
|
9
|
+
// cfg may be null for empty YAML mapping values (`jobs: {myJob:}`)
|
|
10
|
+
const cronJobs = Object.entries(jobs).filter(([, cfg]) => cfg?.cron);
|
|
11
|
+
|
|
12
|
+
if (cronJobs.length === 0) return;
|
|
13
|
+
|
|
14
|
+
if (!leader) {
|
|
15
|
+
console.warn(
|
|
16
|
+
`[worker-sdk] cron.leader=false — skipping registration of ${cronJobs.length} cron job(s). ` +
|
|
17
|
+
`One worker in the pool must set cron.leader=true to register schedules.`
|
|
18
|
+
);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Upserts are independent (distinct scheduler ids) — run them concurrently
|
|
23
|
+
await Promise.all(cronJobs.map(([jobName, jobConfig]) =>
|
|
24
|
+
this._pool.get(jobConfig.tier || DEFAULT_TIER).upsertJobScheduler(
|
|
25
|
+
jobName,
|
|
26
|
+
{pattern: jobConfig.cron},
|
|
27
|
+
{name: jobName, opts: buildJobOpts(jobConfig)}
|
|
28
|
+
)
|
|
29
|
+
));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async closeAll() {
|
|
33
|
+
await this._pool.closeAll();
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import express from 'express';
|
|
2
|
+
import {timingSafeEqual} from 'node:crypto';
|
|
3
|
+
import {createBullBoard} from '@bull-board/api';
|
|
4
|
+
import {BullMQAdapter} from '@bull-board/api/bullMQAdapter';
|
|
5
|
+
import {ExpressAdapter} from '@bull-board/express';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Constant-time buffer comparison (length differences still short-circuit,
|
|
9
|
+
* which is standard practice for basic-auth checks).
|
|
10
|
+
*/
|
|
11
|
+
function safeEqual(bufA, bufB) {
|
|
12
|
+
if (bufA.length !== bufB.length) return false;
|
|
13
|
+
return timingSafeEqual(bufA, bufB);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Create Express app with Bull Board and health endpoint.
|
|
18
|
+
* @param {object} options
|
|
19
|
+
* @param {import('bullmq').Queue[]} options.queues - BullMQ queues to display
|
|
20
|
+
* @param {object} options.auth - {username, password} for basic auth
|
|
21
|
+
* @returns {import('express').Express}
|
|
22
|
+
*/
|
|
23
|
+
export function createDashboardApp({queues, auth}) {
|
|
24
|
+
const app = express();
|
|
25
|
+
|
|
26
|
+
// Health endpoint (no auth)
|
|
27
|
+
const startTime = Date.now();
|
|
28
|
+
app.get('/health', (req, res) => {
|
|
29
|
+
res.json({
|
|
30
|
+
status: 'ok',
|
|
31
|
+
uptime: Math.floor((Date.now() - startTime) / 1000),
|
|
32
|
+
timestamp: new Date().toISOString()
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// Basic auth for all other routes
|
|
37
|
+
if (auth?.username && auth?.password) {
|
|
38
|
+
const expectedUser = Buffer.from(String(auth.username));
|
|
39
|
+
const expectedPass = Buffer.from(String(auth.password));
|
|
40
|
+
|
|
41
|
+
app.use((req, res, next) => {
|
|
42
|
+
if (req.path === '/health') return next();
|
|
43
|
+
|
|
44
|
+
const header = req.headers.authorization;
|
|
45
|
+
if (!header || !header.startsWith('Basic ')) {
|
|
46
|
+
res.set('WWW-Authenticate', 'Basic realm="Worker Dashboard"');
|
|
47
|
+
return res.status(401).send('Authentication required');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const credentials = Buffer.from(header.slice(6), 'base64').toString();
|
|
51
|
+
// Split on the FIRST colon only — RFC 7617 allows colons in the password
|
|
52
|
+
const sep = credentials.indexOf(':');
|
|
53
|
+
const username = sep === -1 ? credentials : credentials.slice(0, sep);
|
|
54
|
+
const password = sep === -1 ? '' : credentials.slice(sep + 1);
|
|
55
|
+
|
|
56
|
+
const userOk = safeEqual(Buffer.from(username), expectedUser);
|
|
57
|
+
const passOk = safeEqual(Buffer.from(password), expectedPass);
|
|
58
|
+
if (userOk && passOk) {
|
|
59
|
+
return next();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
res.set('WWW-Authenticate', 'Basic realm="Worker Dashboard"');
|
|
63
|
+
return res.status(401).send('Invalid credentials');
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Bull Board
|
|
68
|
+
const serverAdapter = new ExpressAdapter();
|
|
69
|
+
serverAdapter.setBasePath('/');
|
|
70
|
+
|
|
71
|
+
createBullBoard({
|
|
72
|
+
queues: queues.map(q => new BullMQAdapter(q)),
|
|
73
|
+
serverAdapter
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
app.use('/', serverAdapter.getRouter());
|
|
77
|
+
|
|
78
|
+
return app;
|
|
79
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import {Queue} from 'bullmq';
|
|
2
|
+
import {loadConfig} from '../config/loader.js';
|
|
3
|
+
import {queueName} from '../queue/queues.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Create a standalone Bull Board dashboard server.
|
|
7
|
+
* Reads Redis + dashboard config from YAML, serves Bull Board on its own port.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* const dashboard = createDashboard('./worker.config.yml');
|
|
11
|
+
* await dashboard.start();
|
|
12
|
+
*
|
|
13
|
+
* @param {string} configPath - Path to worker YAML config
|
|
14
|
+
* @returns {{start: Function, stop: Function, server: import('http').Server|null}}
|
|
15
|
+
*/
|
|
16
|
+
export function createDashboard(configPath) {
|
|
17
|
+
const config = loadConfig(configPath);
|
|
18
|
+
|
|
19
|
+
if (!config.dashboard || config.dashboard.port === undefined) {
|
|
20
|
+
throw new Error('worker.config.yml: dashboard config is required for standalone dashboard.');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const redisOpts = {...config.redis, maxRetriesPerRequest: null};
|
|
24
|
+
|
|
25
|
+
const queues = Object.keys(config.concurrency).map(tier =>
|
|
26
|
+
new Queue(queueName(tier), {connection: redisOpts})
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
if (!config.dashboard.auth?.username || !config.dashboard.auth?.password) {
|
|
30
|
+
console.warn('[worker-sdk] WARNING: Standalone dashboard running without authentication!');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
let app = null;
|
|
34
|
+
let server = null;
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
get server() {
|
|
38
|
+
return server;
|
|
39
|
+
},
|
|
40
|
+
async start() {
|
|
41
|
+
if (!app) {
|
|
42
|
+
// Lazy-load so importing the SDK doesn't pull in express/bull-board
|
|
43
|
+
// for consumers that never run the dashboard
|
|
44
|
+
const {createDashboardApp} = await import('./server.js');
|
|
45
|
+
app = createDashboardApp({queues, auth: config.dashboard.auth});
|
|
46
|
+
}
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
server = app.listen(config.dashboard.port);
|
|
49
|
+
server.once('error', async (err) => {
|
|
50
|
+
await Promise.all(queues.map(q => q.close().catch(() => {})));
|
|
51
|
+
reject(err);
|
|
52
|
+
});
|
|
53
|
+
server.once('listening', () => {
|
|
54
|
+
console.info(`[worker-sdk] Standalone dashboard running on port ${server.address().port}`);
|
|
55
|
+
resolve();
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
},
|
|
59
|
+
async stop() {
|
|
60
|
+
if (server) {
|
|
61
|
+
await new Promise(resolve => server.close(resolve));
|
|
62
|
+
server = null;
|
|
63
|
+
}
|
|
64
|
+
await Promise.all(queues.map(q => q.close()));
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
}
|
package/src/index.cjs
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// CJS wrapper for environments that can't use ESM (e.g. Babel-compiled code).
|
|
2
|
+
// Each export lazily imports only its own module and forwards all arguments,
|
|
3
|
+
// so signatures never go stale and unused subsystems are never loaded.
|
|
4
|
+
const wrap = (path, name) => async (...args) => {
|
|
5
|
+
const mod = await import(path);
|
|
6
|
+
return mod[name](...args);
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
module.exports = {
|
|
10
|
+
createWorker: wrap('./worker/worker.js', 'createWorker'),
|
|
11
|
+
createClient: wrap('./client/client.js', 'createClient'),
|
|
12
|
+
createDashboard: wrap('./dashboard/standalone.js', 'createDashboard'),
|
|
13
|
+
listWorkers: wrap('./admin/listWorkers.js', 'listWorkers'),
|
|
14
|
+
pingRedis: wrap('./admin/pingRedis.js', 'pingRedis'),
|
|
15
|
+
getQueueDepths: wrap('./admin/getQueueDepths.js', 'getQueueDepths'),
|
|
16
|
+
checkDashboard: wrap('./admin/checkDashboard.js', 'checkDashboard'),
|
|
17
|
+
getClusterHealth: wrap('./admin/getClusterHealth.js', 'getClusterHealth')
|
|
18
|
+
};
|
package/src/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export {createWorker} from './worker/worker.js';
|
|
2
|
+
export {createClient} from './client/client.js';
|
|
3
|
+
export {createDashboard} from './dashboard/standalone.js';
|
|
4
|
+
export {listWorkers} from './admin/listWorkers.js';
|
|
5
|
+
export {pingRedis} from './admin/pingRedis.js';
|
|
6
|
+
export {getQueueDepths} from './admin/getQueueDepths.js';
|
|
7
|
+
export {checkDashboard} from './admin/checkDashboard.js';
|
|
8
|
+
export {getClusterHealth} from './admin/getClusterHealth.js';
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import {mkdirSync, appendFileSync, readdirSync, unlinkSync, statSync} from 'fs';
|
|
2
|
+
import {join} from 'path';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* File logger that writes structured JSON lines to daily log files.
|
|
6
|
+
* Thread-safe via appendFileSync (atomic append on most OS).
|
|
7
|
+
*/
|
|
8
|
+
export class FileLogger {
|
|
9
|
+
/**
|
|
10
|
+
* @param {object} options
|
|
11
|
+
* @param {string} options.dir - Directory to write log files
|
|
12
|
+
* @param {number} [options.retentionDays=30] - Delete log files older than this
|
|
13
|
+
*/
|
|
14
|
+
constructor({dir, retentionDays = 30}) {
|
|
15
|
+
this._dir = dir;
|
|
16
|
+
this._retentionDays = retentionDays;
|
|
17
|
+
|
|
18
|
+
mkdirSync(dir, {recursive: true});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Write a log entry to today's file.
|
|
23
|
+
* @param {object} entry - {job, id, level, msg, data}
|
|
24
|
+
*/
|
|
25
|
+
write(entry) {
|
|
26
|
+
const now = new Date();
|
|
27
|
+
const filename = `${formatDate(now)}.log`;
|
|
28
|
+
const filepath = join(this._dir, filename);
|
|
29
|
+
|
|
30
|
+
try {
|
|
31
|
+
// Stringify inside the try — unserializable entry.data (circular, BigInt)
|
|
32
|
+
// must not crash the worker either
|
|
33
|
+
const line = JSON.stringify({
|
|
34
|
+
ts: now.toISOString(),
|
|
35
|
+
job: entry.job,
|
|
36
|
+
id: entry.id,
|
|
37
|
+
level: entry.level,
|
|
38
|
+
msg: entry.msg,
|
|
39
|
+
...(entry.data ? {data: entry.data} : {})
|
|
40
|
+
}) + '\n';
|
|
41
|
+
appendFileSync(filepath, line);
|
|
42
|
+
} catch (err) {
|
|
43
|
+
// Don't crash the worker if log write fails
|
|
44
|
+
process.stderr.write(`[worker-sdk] Failed to write log file: ${err.message}\n`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Delete log files older than retentionDays.
|
|
50
|
+
*/
|
|
51
|
+
cleanup() {
|
|
52
|
+
if (!this._retentionDays) return;
|
|
53
|
+
|
|
54
|
+
const cutoff = Date.now() - this._retentionDays * 24 * 60 * 60 * 1000;
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
const files = readdirSync(this._dir).filter(f => f.endsWith('.log'));
|
|
58
|
+
|
|
59
|
+
for (const file of files) {
|
|
60
|
+
const filepath = join(this._dir, file);
|
|
61
|
+
const stat = statSync(filepath);
|
|
62
|
+
if (stat.mtimeMs < cutoff) {
|
|
63
|
+
unlinkSync(filepath);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
} catch (err) {
|
|
67
|
+
process.stderr.write(`[worker-sdk] Log cleanup failed: ${err.message}\n`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function formatDate(date) {
|
|
73
|
+
const y = date.getFullYear();
|
|
74
|
+
const m = String(date.getMonth() + 1).padStart(2, '0');
|
|
75
|
+
const d = String(date.getDate()).padStart(2, '0');
|
|
76
|
+
return `${y}-${m}-${d}`;
|
|
77
|
+
}
|