bunqueue 2.8.42 → 2.8.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/application/latencyTracker.js +3 -3
- package/dist/application/metricsExporter.d.ts +3 -1
- package/dist/application/metricsExporter.js +63 -16
- package/dist/application/prometheusOperationalMetrics.d.ts +31 -0
- package/dist/application/prometheusOperationalMetrics.js +53 -0
- package/dist/application/queueManager.d.ts +6 -0
- package/dist/application/queueManager.js +24 -1
- package/dist/application/types.d.ts +3 -0
- package/dist/application/types.js +1 -0
- package/dist/application/workerManager.d.ts +1 -0
- package/dist/application/workerManager.js +4 -1
- package/dist/cli/output.js +2 -1
- package/dist/config/resolve.d.ts +1 -0
- package/dist/config/resolve.js +10 -0
- package/dist/config/types.d.ts +6 -0
- package/dist/infrastructure/backup/backupTelemetry.d.ts +30 -0
- package/dist/infrastructure/backup/backupTelemetry.js +63 -0
- package/dist/infrastructure/backup/index.d.ts +1 -1
- package/dist/infrastructure/backup/index.js +1 -1
- package/dist/infrastructure/backup/s3Backup.d.ts +6 -2
- package/dist/infrastructure/backup/s3Backup.js +22 -9
- package/dist/infrastructure/backup/s3BackupConfig.d.ts +7 -0
- package/dist/infrastructure/backup/s3BackupConfig.js +5 -0
- package/dist/infrastructure/backup/s3BackupIo.d.ts +23 -0
- package/dist/infrastructure/backup/s3BackupIo.js +106 -0
- package/dist/infrastructure/backup/s3BackupOperations.d.ts +6 -15
- package/dist/infrastructure/backup/s3BackupOperations.js +139 -259
- package/dist/infrastructure/backup/sqliteBackupFiles.d.ts +22 -0
- package/dist/infrastructure/backup/sqliteBackupFiles.js +130 -0
- package/dist/infrastructure/cloud/statsRefresh.d.ts +1 -0
- package/dist/infrastructure/persistence/sqlite.js +6 -1
- package/dist/infrastructure/server/bootstrap.d.ts +2 -0
- package/dist/infrastructure/server/bootstrap.js +30 -1
- package/dist/infrastructure/server/http.d.ts +2 -0
- package/dist/infrastructure/server/http.js +9 -58
- package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +10 -0
- package/dist/infrastructure/server/httpDashboardEndpoints.js +146 -0
- package/dist/infrastructure/server/httpEndpoints.d.ts +5 -9
- package/dist/infrastructure/server/httpEndpoints.js +16 -160
- package/dist/infrastructure/server/httpResponse.d.ts +2 -0
- package/dist/infrastructure/server/httpResponse.js +12 -0
- package/dist/infrastructure/server/httpRouter.d.ts +6 -0
- package/dist/infrastructure/server/httpRouter.js +50 -0
- package/dist/shared/histogram.d.ts +2 -2
- package/dist/shared/histogram.js +4 -4
- package/package.json +2 -2
|
@@ -14,6 +14,14 @@ import { S3BackupManager } from '../backup';
|
|
|
14
14
|
import { CloudAgent } from '../cloud';
|
|
15
15
|
import { SHARD_COUNT } from '../../shared/hash';
|
|
16
16
|
import { resolveCloudConfig, resolveBackupConfig, resolveTlsServerOptions, } from '../../config';
|
|
17
|
+
/** Return a startup error when an enabled file backup has no file to snapshot. */
|
|
18
|
+
export function backupStartupError(config) {
|
|
19
|
+
if (config.s3BackupEnabled && !config.dataPath) {
|
|
20
|
+
return ('S3 backup requires persistent SQLite storage. Set BUNQUEUE_DATA_PATH ' +
|
|
21
|
+
'or storage.dataPath before enabling S3_BACKUP_ENABLED.');
|
|
22
|
+
}
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
17
25
|
/** Print startup banner */
|
|
18
26
|
function printBanner(config, cloudUrl) {
|
|
19
27
|
const dim = '\x1b[2m';
|
|
@@ -68,6 +76,12 @@ export function bootServer(fileConfig, config) {
|
|
|
68
76
|
if (validLevels.includes(logLevel))
|
|
69
77
|
Logger.setLevel(logLevel);
|
|
70
78
|
}
|
|
79
|
+
const backupError = backupStartupError(config);
|
|
80
|
+
if (backupError) {
|
|
81
|
+
serverLog.error(backupError);
|
|
82
|
+
process.exitCode = 1;
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
71
85
|
// Resolve cloud config
|
|
72
86
|
const cloudConfig = resolveCloudConfig(fileConfig, config.dataPath);
|
|
73
87
|
// Resolve TLS config — fail fast on partial cert/key before binding anything
|
|
@@ -83,6 +97,7 @@ export function bootServer(fileConfig, config) {
|
|
|
83
97
|
// Create queue manager
|
|
84
98
|
const queueManager = new QueueManager({
|
|
85
99
|
dataPath: config.dataPath,
|
|
100
|
+
maxPrometheusQueues: config.maxPrometheusQueues,
|
|
86
101
|
});
|
|
87
102
|
// Start TCP + HTTP servers; a bind failure must not leave a half-started process
|
|
88
103
|
let tcpServer;
|
|
@@ -101,6 +116,7 @@ export function bootServer(fileConfig, config) {
|
|
|
101
116
|
authTokens: config.authTokens,
|
|
102
117
|
corsOrigins: config.corsOrigins,
|
|
103
118
|
requireAuthForMetrics: config.requireAuthForMetrics,
|
|
119
|
+
getTcpConnectionCount: () => tcpServer.getConnectionCount(),
|
|
104
120
|
...(tlsConfig && { tls: tlsConfig }),
|
|
105
121
|
});
|
|
106
122
|
}
|
|
@@ -114,10 +130,23 @@ export function bootServer(fileConfig, config) {
|
|
|
114
130
|
let backupManager = null;
|
|
115
131
|
if (config.dataPath) {
|
|
116
132
|
const backupConfig = resolveBackupConfig(fileConfig, config.dataPath);
|
|
117
|
-
backupManager = new S3BackupManager(
|
|
133
|
+
backupManager = new S3BackupManager({
|
|
134
|
+
...backupConfig,
|
|
135
|
+
flushBeforeBackup: () => {
|
|
136
|
+
queueManager.flushPersistence();
|
|
137
|
+
},
|
|
138
|
+
});
|
|
118
139
|
backupManager.setDashboardEmit(queueManager.emitDashboardEvent.bind(queueManager));
|
|
119
140
|
backupManager.start();
|
|
120
141
|
}
|
|
142
|
+
queueManager.setOperationalMetricsProvider(() => ({
|
|
143
|
+
...(backupManager && { backup: backupManager.getMetrics() }),
|
|
144
|
+
connections: {
|
|
145
|
+
tcp: tcpServer.getConnectionCount(),
|
|
146
|
+
websocket: httpServer.getWsClientCount(),
|
|
147
|
+
sse: httpServer.getSseClientCount(),
|
|
148
|
+
},
|
|
149
|
+
}));
|
|
121
150
|
// Initialize bunqueue Cloud agent (remote dashboard telemetry)
|
|
122
151
|
const cloudAgent = cloudConfig ? CloudAgent.createFromConfig(queueManager, cloudConfig) : null;
|
|
123
152
|
if (cloudAgent) {
|
|
@@ -14,6 +14,8 @@ export interface HttpServerConfig {
|
|
|
14
14
|
authTokens?: string[];
|
|
15
15
|
corsOrigins?: string[];
|
|
16
16
|
requireAuthForMetrics?: boolean;
|
|
17
|
+
/** Current TCP client count, supplied by the TCP server when available. */
|
|
18
|
+
getTcpConnectionCount?: () => number;
|
|
17
19
|
/** Native TLS termination (https/wss). Protocol and routes are unchanged. */
|
|
18
20
|
tls?: TlsServerOptions;
|
|
19
21
|
}
|
|
@@ -3,19 +3,13 @@
|
|
|
3
3
|
* REST API and WebSocket support
|
|
4
4
|
*/
|
|
5
5
|
import { constantTimeEqual, uuid } from '../../shared/hash';
|
|
6
|
-
import { validateQueueName } from './protocol';
|
|
7
6
|
import { httpLog } from '../../shared/logger';
|
|
8
7
|
import { getRateLimiter } from './rateLimiter';
|
|
9
8
|
import { SseHandler } from './sseHandler';
|
|
10
9
|
import { WsHandler } from './wsHandler';
|
|
11
|
-
import { jsonResponse, corsResponse, healthEndpoint, gcEndpoint, heapStatsEndpoint,
|
|
12
|
-
import { routeJobRoutes } from './httpRouteJobs';
|
|
10
|
+
import { jsonResponse, corsResponse, healthEndpoint, readinessEndpoint, gcEndpoint, heapStatsEndpoint, } from './httpEndpoints';
|
|
13
11
|
import { loadTlsOptions } from './tls';
|
|
14
|
-
import {
|
|
15
|
-
import { routeQueueConfigRoutes } from './httpRouteQueueConfig';
|
|
16
|
-
import { routeResourceRoutes } from './httpRouteResources';
|
|
17
|
-
// Pre-compiled regex patterns for URL matching
|
|
18
|
-
const RE_DASHBOARD_QUEUE_DETAIL = /^\/dashboard\/queues\/([^/]+)$/;
|
|
12
|
+
import { routeHttpRequest } from './httpRouter';
|
|
19
13
|
/**
|
|
20
14
|
* Validate auth token against valid tokens set
|
|
21
15
|
*/
|
|
@@ -81,13 +75,13 @@ export function createHttpServer(queueManager, config) {
|
|
|
81
75
|
}
|
|
82
76
|
// Health endpoints (no auth, no rate limit)
|
|
83
77
|
if (path === '/health') {
|
|
84
|
-
return withCors(healthEndpoint(queueManager, wsHandler.size, sseHandler.size));
|
|
78
|
+
return withCors(healthEndpoint(queueManager, wsHandler.size, sseHandler.size, config.getTcpConnectionCount?.() ?? 0));
|
|
85
79
|
}
|
|
86
80
|
if (path === '/healthz' || path === '/live') {
|
|
87
81
|
return withCors(new Response('OK', { status: 200 }));
|
|
88
82
|
}
|
|
89
83
|
if (path === '/ready') {
|
|
90
|
-
return
|
|
84
|
+
return readinessEndpoint(queueManager, corsOrigins);
|
|
91
85
|
}
|
|
92
86
|
// Debug endpoints (require auth)
|
|
93
87
|
if (path === '/gc' && req.method === 'POST') {
|
|
@@ -138,6 +132,9 @@ export function createHttpServer(queueManager, config) {
|
|
|
138
132
|
// Prometheus metrics
|
|
139
133
|
if (path === '/prometheus' && req.method === 'GET') {
|
|
140
134
|
if (config.requireAuthForMetrics) {
|
|
135
|
+
if (authTokens.size === 0) {
|
|
136
|
+
return jsonResponse({ ok: false, error: 'Metrics authentication is enabled but no tokens are configured' }, 503, corsOrigins);
|
|
137
|
+
}
|
|
141
138
|
const denied = checkAuth(req, authTokens);
|
|
142
139
|
if (denied)
|
|
143
140
|
return denied;
|
|
@@ -161,7 +158,7 @@ export function createHttpServer(queueManager, config) {
|
|
|
161
158
|
// connections (TCP/WebSocket). Orphaned HTTP jobs are handled by stall detection.
|
|
162
159
|
const ctx = { queueManager, authTokens, authenticated: true };
|
|
163
160
|
try {
|
|
164
|
-
return await
|
|
161
|
+
return await routeHttpRequest(req, path, ctx, corsOrigins);
|
|
165
162
|
}
|
|
166
163
|
catch (err) {
|
|
167
164
|
const message = err instanceof Error ? err.message : 'Internal error';
|
|
@@ -194,7 +191,7 @@ export function createHttpServer(queueManager, config) {
|
|
|
194
191
|
queueManager.unregisterWorkersByClientId(clientId);
|
|
195
192
|
queueManager
|
|
196
193
|
.releaseClientJobs(clientId)
|
|
197
|
-
.then(() =>
|
|
194
|
+
.then(() => undefined)
|
|
198
195
|
.catch((err) => {
|
|
199
196
|
httpLog.error('Failed to release WebSocket client jobs', {
|
|
200
197
|
clientId,
|
|
@@ -237,49 +234,3 @@ export function createHttpServer(queueManager, config) {
|
|
|
237
234
|
},
|
|
238
235
|
};
|
|
239
236
|
}
|
|
240
|
-
/** Route HTTP request to handler */
|
|
241
|
-
async function routeRequest(req, path, ctx, corsOrigins) {
|
|
242
|
-
const method = req.method;
|
|
243
|
-
// Stats endpoint
|
|
244
|
-
if (path === '/stats' && method === 'GET') {
|
|
245
|
-
return statsEndpoint(ctx.queueManager, corsOrigins);
|
|
246
|
-
}
|
|
247
|
-
if (path === '/metrics' && method === 'GET') {
|
|
248
|
-
return metricsEndpoint(ctx.queueManager, corsOrigins);
|
|
249
|
-
}
|
|
250
|
-
// Dashboard endpoints
|
|
251
|
-
if (path === '/dashboard' && method === 'GET') {
|
|
252
|
-
return dashboardOverviewEndpoint(ctx.queueManager, corsOrigins);
|
|
253
|
-
}
|
|
254
|
-
if (path === '/dashboard/queues' && method === 'GET') {
|
|
255
|
-
const url = new URL(req.url);
|
|
256
|
-
const limit = Math.min(Math.max(parseInt(url.searchParams.get('limit') ?? '100', 10) || 100, 1), 500);
|
|
257
|
-
const offset = Math.max(parseInt(url.searchParams.get('offset') ?? '0', 10) || 0, 0);
|
|
258
|
-
return dashboardQueuesEndpoint(ctx.queueManager, limit, offset, corsOrigins);
|
|
259
|
-
}
|
|
260
|
-
const dashQueueMatch = path.match(RE_DASHBOARD_QUEUE_DETAIL);
|
|
261
|
-
if (dashQueueMatch && method === 'GET') {
|
|
262
|
-
const queue = decodeURIComponent(dashQueueMatch[1]);
|
|
263
|
-
const queueError = validateQueueName(queue);
|
|
264
|
-
if (queueError)
|
|
265
|
-
return jsonResponse({ ok: false, error: queueError }, 400, corsOrigins);
|
|
266
|
-
const url = new URL(req.url);
|
|
267
|
-
const includeJobs = url.searchParams.get('includeJobs') === 'true';
|
|
268
|
-
return dashboardQueueDetailEndpoint(ctx.queueManager, queue, includeJobs, corsOrigins);
|
|
269
|
-
}
|
|
270
|
-
// Route through sub-routers
|
|
271
|
-
let result;
|
|
272
|
-
result = await routeJobRoutes(req, path, method, ctx, corsOrigins);
|
|
273
|
-
if (result)
|
|
274
|
-
return result;
|
|
275
|
-
result = await routeQueueRoutes(req, path, method, ctx, corsOrigins);
|
|
276
|
-
if (result)
|
|
277
|
-
return result;
|
|
278
|
-
result = await routeQueueConfigRoutes(req, path, method, ctx, corsOrigins);
|
|
279
|
-
if (result)
|
|
280
|
-
return result;
|
|
281
|
-
result = await routeResourceRoutes(req, path, method, ctx, corsOrigins);
|
|
282
|
-
if (result)
|
|
283
|
-
return result;
|
|
284
|
-
return jsonResponse({ ok: false, error: 'Not found' }, 404, corsOrigins);
|
|
285
|
-
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP dashboard endpoints
|
|
3
|
+
*/
|
|
4
|
+
import type { QueueManager } from '../../application/queueManager';
|
|
5
|
+
/** Dashboard overview endpoint - aggregates all dashboard data in a single call */
|
|
6
|
+
export declare function dashboardOverviewEndpoint(queueManager: QueueManager, corsOrigins?: Set<string>): Response;
|
|
7
|
+
/** Dashboard queues endpoint - paginated queues with per-queue stats */
|
|
8
|
+
export declare function dashboardQueuesEndpoint(queueManager: QueueManager, limit: number, offset: number, corsOrigins?: Set<string>): Response;
|
|
9
|
+
/** Dashboard single queue detail endpoint */
|
|
10
|
+
export declare function dashboardQueueDetailEndpoint(queueManager: QueueManager, queue: string, includeJobs: boolean, corsOrigins?: Set<string>): Response;
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP dashboard endpoints
|
|
3
|
+
*/
|
|
4
|
+
import { latencyTracker } from '../../application/latencyTracker';
|
|
5
|
+
import { throughputTracker } from '../../application/throughputTracker';
|
|
6
|
+
import { pausedView } from '../../shared/pausedView';
|
|
7
|
+
import { jsonResponse } from './httpResponse';
|
|
8
|
+
/** Dashboard overview endpoint - aggregates all dashboard data in a single call */
|
|
9
|
+
export function dashboardOverviewEndpoint(queueManager, corsOrigins) {
|
|
10
|
+
const stats = queueManager.getStats();
|
|
11
|
+
const rates = throughputTracker.getRates();
|
|
12
|
+
const latencies = latencyTracker.getPercentiles();
|
|
13
|
+
const avgLatencies = latencyTracker.getAverages();
|
|
14
|
+
const memStats = queueManager.getMemoryStats();
|
|
15
|
+
const workers = queueManager.workerManager.list();
|
|
16
|
+
const workerStats = queueManager.workerManager.getStats();
|
|
17
|
+
const crons = queueManager.listCrons();
|
|
18
|
+
const storage = queueManager.getStorageStatus();
|
|
19
|
+
const mem = process.memoryUsage();
|
|
20
|
+
return jsonResponse({
|
|
21
|
+
ok: true,
|
|
22
|
+
stats: {
|
|
23
|
+
waiting: stats.waiting,
|
|
24
|
+
active: stats.active,
|
|
25
|
+
delayed: stats.delayed,
|
|
26
|
+
completed: stats.completed,
|
|
27
|
+
dlq: stats.dlq,
|
|
28
|
+
totalPushed: Number(stats.totalPushed),
|
|
29
|
+
totalPulled: Number(stats.totalPulled),
|
|
30
|
+
totalCompleted: Number(stats.totalCompleted),
|
|
31
|
+
totalFailed: Number(stats.totalFailed),
|
|
32
|
+
uptime: stats.uptime,
|
|
33
|
+
},
|
|
34
|
+
throughput: rates,
|
|
35
|
+
latency: {
|
|
36
|
+
averages: avgLatencies,
|
|
37
|
+
percentiles: latencies,
|
|
38
|
+
},
|
|
39
|
+
memory: {
|
|
40
|
+
heapUsed: Math.round(mem.heapUsed / 1024 / 1024),
|
|
41
|
+
heapTotal: Math.round(mem.heapTotal / 1024 / 1024),
|
|
42
|
+
rss: Math.round(mem.rss / 1024 / 1024),
|
|
43
|
+
},
|
|
44
|
+
collections: memStats,
|
|
45
|
+
workers: {
|
|
46
|
+
total: workerStats.total,
|
|
47
|
+
active: workerStats.active,
|
|
48
|
+
list: workers
|
|
49
|
+
.slice(0, 100)
|
|
50
|
+
.map((w) => ({
|
|
51
|
+
id: w.id,
|
|
52
|
+
name: w.name,
|
|
53
|
+
queues: w.queues,
|
|
54
|
+
lastSeen: w.lastSeen,
|
|
55
|
+
activeJobs: w.activeJobs,
|
|
56
|
+
processedJobs: w.processedJobs,
|
|
57
|
+
failedJobs: w.failedJobs,
|
|
58
|
+
})),
|
|
59
|
+
truncated: workers.length > 100,
|
|
60
|
+
},
|
|
61
|
+
crons: {
|
|
62
|
+
total: crons.length,
|
|
63
|
+
list: crons
|
|
64
|
+
.slice(0, 100)
|
|
65
|
+
.map((c) => ({
|
|
66
|
+
name: c.name,
|
|
67
|
+
queue: c.queue,
|
|
68
|
+
schedule: c.schedule ?? null,
|
|
69
|
+
repeatEvery: c.repeatEvery ?? null,
|
|
70
|
+
nextRun: c.nextRun,
|
|
71
|
+
executions: c.executions,
|
|
72
|
+
})),
|
|
73
|
+
truncated: crons.length > 100,
|
|
74
|
+
},
|
|
75
|
+
storage,
|
|
76
|
+
timestamp: Date.now(),
|
|
77
|
+
}, 200, corsOrigins);
|
|
78
|
+
}
|
|
79
|
+
/** Dashboard queues endpoint - paginated queues with per-queue stats */
|
|
80
|
+
export function dashboardQueuesEndpoint(queueManager, limit, offset, corsOrigins) {
|
|
81
|
+
const allQueues = queueManager.listQueues();
|
|
82
|
+
const total = allQueues.length;
|
|
83
|
+
const queueNames = allQueues.slice(offset, offset + limit);
|
|
84
|
+
const perQueueStats = queueManager.getPerQueueStats();
|
|
85
|
+
const queues = queueNames.map((name) => {
|
|
86
|
+
const stats = perQueueStats.get(name);
|
|
87
|
+
return {
|
|
88
|
+
name,
|
|
89
|
+
waiting: stats?.waiting ?? 0,
|
|
90
|
+
delayed: stats?.delayed ?? 0,
|
|
91
|
+
active: stats?.active ?? 0,
|
|
92
|
+
dlq: stats?.dlq ?? 0,
|
|
93
|
+
paused: queueManager.isPaused(name),
|
|
94
|
+
};
|
|
95
|
+
});
|
|
96
|
+
return jsonResponse({ ok: true, queues, total, limit, offset, timestamp: Date.now() }, 200, corsOrigins);
|
|
97
|
+
}
|
|
98
|
+
/** Dashboard single queue detail endpoint */
|
|
99
|
+
export function dashboardQueueDetailEndpoint(queueManager, queue, includeJobs, corsOrigins) {
|
|
100
|
+
const rawCounts = queueManager.getQueueJobCounts(queue);
|
|
101
|
+
const paused = queueManager.isPaused(queue);
|
|
102
|
+
const pv = pausedView(rawCounts.waiting, rawCounts.prioritized, paused);
|
|
103
|
+
const counts = {
|
|
104
|
+
...rawCounts,
|
|
105
|
+
waiting: pv.waiting,
|
|
106
|
+
prioritized: pv.prioritized,
|
|
107
|
+
paused: pv.paused,
|
|
108
|
+
};
|
|
109
|
+
const dlqJobs = queueManager.getDlq(queue, 10);
|
|
110
|
+
const priorityCounts = queueManager.getCountsPerPriority(queue);
|
|
111
|
+
const result = {
|
|
112
|
+
ok: true,
|
|
113
|
+
name: queue,
|
|
114
|
+
counts,
|
|
115
|
+
paused,
|
|
116
|
+
priorityCounts,
|
|
117
|
+
dlqPreview: dlqJobs.map((j) => ({
|
|
118
|
+
id: j.id,
|
|
119
|
+
data: j.data,
|
|
120
|
+
attempts: j.attempts,
|
|
121
|
+
createdAt: j.createdAt,
|
|
122
|
+
})),
|
|
123
|
+
timestamp: Date.now(),
|
|
124
|
+
};
|
|
125
|
+
if (includeJobs) {
|
|
126
|
+
const waiting = queueManager.getJobs(queue, { state: 'waiting', end: 10 });
|
|
127
|
+
const active = queueManager.getJobs(queue, { state: 'active', end: 10 });
|
|
128
|
+
const delayed = queueManager.getJobs(queue, { state: 'delayed', end: 10 });
|
|
129
|
+
const pausedJobs = paused ? queueManager.getJobs(queue, { state: 'paused', end: 10 }) : [];
|
|
130
|
+
const toSummary = (j) => ({
|
|
131
|
+
id: j.id,
|
|
132
|
+
priority: j.priority,
|
|
133
|
+
createdAt: j.createdAt,
|
|
134
|
+
runAt: j.runAt,
|
|
135
|
+
attempts: j.attempts,
|
|
136
|
+
progress: j.progress,
|
|
137
|
+
});
|
|
138
|
+
result.jobs = {
|
|
139
|
+
waiting: waiting.map(toSummary),
|
|
140
|
+
active: active.map(toSummary),
|
|
141
|
+
delayed: delayed.map(toSummary),
|
|
142
|
+
paused: pausedJobs.map(toSummary),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
return jsonResponse(result, 200, corsOrigins);
|
|
146
|
+
}
|
|
@@ -2,26 +2,22 @@
|
|
|
2
2
|
* HTTP Endpoints - Health, diagnostics, and stats endpoints
|
|
3
3
|
*/
|
|
4
4
|
import type { QueueManager } from '../../application/queueManager';
|
|
5
|
-
|
|
6
|
-
export
|
|
5
|
+
export { dashboardOverviewEndpoint, dashboardQueueDetailEndpoint, dashboardQueuesEndpoint, } from './httpDashboardEndpoints';
|
|
6
|
+
export { jsonResponse } from './httpResponse';
|
|
7
7
|
/** Parse JSON body from request. Returns parsed object or 400 Response on invalid JSON.
|
|
8
8
|
* Empty/missing body returns {} for backward compatibility with optional-body routes. */
|
|
9
9
|
export declare function parseJsonBody(req: Request, cors: Set<string>): Promise<Record<string, unknown> | Response>;
|
|
10
10
|
/** CORS preflight response */
|
|
11
11
|
export declare function corsResponse(corsOrigins: Set<string>): Response;
|
|
12
12
|
/** Health check endpoint */
|
|
13
|
-
export declare function healthEndpoint(queueManager: QueueManager, wsCount: number, sseCount: number): Response;
|
|
13
|
+
export declare function healthEndpoint(queueManager: QueueManager, wsCount: number, sseCount: number, tcpCount?: number): Response;
|
|
14
|
+
/** Readiness check - persistence failures must stop new traffic. */
|
|
15
|
+
export declare function readinessEndpoint(queueManager: QueueManager, corsOrigins?: Set<string>): Response;
|
|
14
16
|
/** GC endpoint - force garbage collection */
|
|
15
17
|
export declare function gcEndpoint(queueManager: QueueManager): Response;
|
|
16
18
|
/** Heap stats endpoint - for debugging memory leaks */
|
|
17
19
|
export declare function heapStatsEndpoint(queueManager: QueueManager): Promise<Response>;
|
|
18
20
|
/** Stats endpoint */
|
|
19
21
|
export declare function statsEndpoint(queueManager: QueueManager, corsOrigins?: Set<string>): Response;
|
|
20
|
-
/** Dashboard overview endpoint - aggregates all dashboard data in a single call */
|
|
21
|
-
export declare function dashboardOverviewEndpoint(queueManager: QueueManager, corsOrigins?: Set<string>): Response;
|
|
22
|
-
/** Dashboard queues endpoint - paginated queues with per-queue stats */
|
|
23
|
-
export declare function dashboardQueuesEndpoint(queueManager: QueueManager, limit: number, offset: number, corsOrigins?: Set<string>): Response;
|
|
24
|
-
/** Dashboard single queue detail endpoint */
|
|
25
|
-
export declare function dashboardQueueDetailEndpoint(queueManager: QueueManager, queue: string, includeJobs: boolean, corsOrigins?: Set<string>): Response;
|
|
26
22
|
/** Metrics endpoint */
|
|
27
23
|
export declare function metricsEndpoint(queueManager: QueueManager, corsOrigins?: Set<string>): Response;
|
|
@@ -3,20 +3,9 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { VERSION } from '../../shared/version';
|
|
5
5
|
import { throughputTracker } from '../../application/throughputTracker';
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
export function jsonResponse(data, status = 200, corsOrigins) {
|
|
10
|
-
const headers = {
|
|
11
|
-
'Content-Type': 'application/json',
|
|
12
|
-
};
|
|
13
|
-
if (corsOrigins) {
|
|
14
|
-
headers['Access-Control-Allow-Origin'] = corsOrigins.has('*')
|
|
15
|
-
? '*'
|
|
16
|
-
: Array.from(corsOrigins).join(', ');
|
|
17
|
-
}
|
|
18
|
-
return new Response(JSON.stringify(data), { status, headers });
|
|
19
|
-
}
|
|
6
|
+
import { jsonResponse } from './httpResponse';
|
|
7
|
+
export { dashboardOverviewEndpoint, dashboardQueueDetailEndpoint, dashboardQueuesEndpoint, } from './httpDashboardEndpoints';
|
|
8
|
+
export { jsonResponse } from './httpResponse';
|
|
20
9
|
/** Parse JSON body from request. Returns parsed object or 400 Response on invalid JSON.
|
|
21
10
|
* Empty/missing body returns {} for backward compatibility with optional-body routes. */
|
|
22
11
|
export async function parseJsonBody(req, cors) {
|
|
@@ -45,7 +34,7 @@ export function corsResponse(corsOrigins) {
|
|
|
45
34
|
});
|
|
46
35
|
}
|
|
47
36
|
/** Health check endpoint */
|
|
48
|
-
export function healthEndpoint(queueManager, wsCount, sseCount) {
|
|
37
|
+
export function healthEndpoint(queueManager, wsCount, sseCount, tcpCount = 0) {
|
|
49
38
|
const stats = queueManager.getStats();
|
|
50
39
|
const uptime = process.uptime();
|
|
51
40
|
const memoryUsage = process.memoryUsage();
|
|
@@ -64,7 +53,7 @@ export function healthEndpoint(queueManager, wsCount, sseCount) {
|
|
|
64
53
|
dlq: stats.dlq,
|
|
65
54
|
},
|
|
66
55
|
connections: {
|
|
67
|
-
tcp:
|
|
56
|
+
tcp: tcpCount,
|
|
68
57
|
ws: wsCount,
|
|
69
58
|
sse: sseCount,
|
|
70
59
|
},
|
|
@@ -80,7 +69,17 @@ export function healthEndpoint(queueManager, wsCount, sseCount) {
|
|
|
80
69
|
since: storageStatus.since,
|
|
81
70
|
},
|
|
82
71
|
}),
|
|
83
|
-
});
|
|
72
|
+
}, isHealthy ? 200 : 503);
|
|
73
|
+
}
|
|
74
|
+
/** Readiness check - persistence failures must stop new traffic. */
|
|
75
|
+
export function readinessEndpoint(queueManager, corsOrigins) {
|
|
76
|
+
const storage = queueManager.getStorageStatus();
|
|
77
|
+
const ready = !storage.diskFull;
|
|
78
|
+
return jsonResponse({
|
|
79
|
+
ok: ready,
|
|
80
|
+
ready,
|
|
81
|
+
...(ready ? {} : { storage: { diskFull: true, error: storage.error, since: storage.since } }),
|
|
82
|
+
}, ready ? 200 : 503, corsOrigins);
|
|
84
83
|
}
|
|
85
84
|
/** GC endpoint - force garbage collection */
|
|
86
85
|
export function gcEndpoint(queueManager) {
|
|
@@ -165,149 +164,6 @@ export function statsEndpoint(queueManager, corsOrigins) {
|
|
|
165
164
|
collections: memStats,
|
|
166
165
|
}, 200, corsOrigins);
|
|
167
166
|
}
|
|
168
|
-
/** Dashboard overview endpoint - aggregates all dashboard data in a single call */
|
|
169
|
-
export function dashboardOverviewEndpoint(queueManager, corsOrigins) {
|
|
170
|
-
const stats = queueManager.getStats();
|
|
171
|
-
const rates = throughputTracker.getRates();
|
|
172
|
-
const latencies = latencyTracker.getPercentiles();
|
|
173
|
-
const avgLatencies = latencyTracker.getAverages();
|
|
174
|
-
const memStats = queueManager.getMemoryStats();
|
|
175
|
-
const workers = queueManager.workerManager.list();
|
|
176
|
-
const workerStats = queueManager.workerManager.getStats();
|
|
177
|
-
const crons = queueManager.listCrons();
|
|
178
|
-
const storage = queueManager.getStorageStatus();
|
|
179
|
-
const mem = process.memoryUsage();
|
|
180
|
-
return jsonResponse({
|
|
181
|
-
ok: true,
|
|
182
|
-
stats: {
|
|
183
|
-
waiting: stats.waiting,
|
|
184
|
-
active: stats.active,
|
|
185
|
-
delayed: stats.delayed,
|
|
186
|
-
completed: stats.completed,
|
|
187
|
-
dlq: stats.dlq,
|
|
188
|
-
totalPushed: Number(stats.totalPushed),
|
|
189
|
-
totalPulled: Number(stats.totalPulled),
|
|
190
|
-
totalCompleted: Number(stats.totalCompleted),
|
|
191
|
-
totalFailed: Number(stats.totalFailed),
|
|
192
|
-
uptime: stats.uptime,
|
|
193
|
-
},
|
|
194
|
-
throughput: rates,
|
|
195
|
-
latency: {
|
|
196
|
-
averages: avgLatencies,
|
|
197
|
-
percentiles: latencies,
|
|
198
|
-
},
|
|
199
|
-
memory: {
|
|
200
|
-
heapUsed: Math.round(mem.heapUsed / 1024 / 1024),
|
|
201
|
-
heapTotal: Math.round(mem.heapTotal / 1024 / 1024),
|
|
202
|
-
rss: Math.round(mem.rss / 1024 / 1024),
|
|
203
|
-
},
|
|
204
|
-
collections: memStats,
|
|
205
|
-
workers: {
|
|
206
|
-
total: workerStats.total,
|
|
207
|
-
active: workerStats.active,
|
|
208
|
-
list: workers
|
|
209
|
-
.slice(0, 100)
|
|
210
|
-
.map((w) => ({
|
|
211
|
-
id: w.id,
|
|
212
|
-
name: w.name,
|
|
213
|
-
queues: w.queues,
|
|
214
|
-
lastSeen: w.lastSeen,
|
|
215
|
-
activeJobs: w.activeJobs,
|
|
216
|
-
processedJobs: w.processedJobs,
|
|
217
|
-
failedJobs: w.failedJobs,
|
|
218
|
-
})),
|
|
219
|
-
truncated: workers.length > 100,
|
|
220
|
-
},
|
|
221
|
-
crons: {
|
|
222
|
-
total: crons.length,
|
|
223
|
-
list: crons
|
|
224
|
-
.slice(0, 100)
|
|
225
|
-
.map((c) => ({
|
|
226
|
-
name: c.name,
|
|
227
|
-
queue: c.queue,
|
|
228
|
-
schedule: c.schedule ?? null,
|
|
229
|
-
repeatEvery: c.repeatEvery ?? null,
|
|
230
|
-
nextRun: c.nextRun,
|
|
231
|
-
executions: c.executions,
|
|
232
|
-
})),
|
|
233
|
-
truncated: crons.length > 100,
|
|
234
|
-
},
|
|
235
|
-
storage,
|
|
236
|
-
timestamp: Date.now(),
|
|
237
|
-
}, 200, corsOrigins);
|
|
238
|
-
}
|
|
239
|
-
/** Dashboard queues endpoint - paginated queues with per-queue stats */
|
|
240
|
-
export function dashboardQueuesEndpoint(queueManager, limit, offset, corsOrigins) {
|
|
241
|
-
const allQueues = queueManager.listQueues();
|
|
242
|
-
const total = allQueues.length;
|
|
243
|
-
const queueNames = allQueues.slice(offset, offset + limit);
|
|
244
|
-
const perQueueStats = queueManager.getPerQueueStats();
|
|
245
|
-
const queues = queueNames.map((name) => {
|
|
246
|
-
const stats = perQueueStats.get(name);
|
|
247
|
-
return {
|
|
248
|
-
name,
|
|
249
|
-
waiting: stats?.waiting ?? 0,
|
|
250
|
-
delayed: stats?.delayed ?? 0,
|
|
251
|
-
active: stats?.active ?? 0,
|
|
252
|
-
dlq: stats?.dlq ?? 0,
|
|
253
|
-
paused: queueManager.isPaused(name),
|
|
254
|
-
};
|
|
255
|
-
});
|
|
256
|
-
return jsonResponse({ ok: true, queues, total, limit, offset, timestamp: Date.now() }, 200, corsOrigins);
|
|
257
|
-
}
|
|
258
|
-
/** Dashboard single queue detail endpoint */
|
|
259
|
-
export function dashboardQueueDetailEndpoint(queueManager, queue, includeJobs, corsOrigins) {
|
|
260
|
-
const rawCounts = queueManager.getQueueJobCounts(queue);
|
|
261
|
-
const paused = queueManager.isPaused(queue);
|
|
262
|
-
// Keep counts consistent with the per-state lists below (#92): a paused queue
|
|
263
|
-
// reports ready jobs under `paused`, not `waiting`/`prioritized`.
|
|
264
|
-
const pv = pausedView(rawCounts.waiting, rawCounts.prioritized, paused);
|
|
265
|
-
const counts = {
|
|
266
|
-
...rawCounts,
|
|
267
|
-
waiting: pv.waiting,
|
|
268
|
-
prioritized: pv.prioritized,
|
|
269
|
-
paused: pv.paused,
|
|
270
|
-
};
|
|
271
|
-
const dlqJobs = queueManager.getDlq(queue, 10);
|
|
272
|
-
const priorityCounts = queueManager.getCountsPerPriority(queue);
|
|
273
|
-
const result = {
|
|
274
|
-
ok: true,
|
|
275
|
-
name: queue,
|
|
276
|
-
counts,
|
|
277
|
-
paused,
|
|
278
|
-
priorityCounts,
|
|
279
|
-
dlqPreview: dlqJobs.map((j) => ({
|
|
280
|
-
id: j.id,
|
|
281
|
-
data: j.data,
|
|
282
|
-
attempts: j.attempts,
|
|
283
|
-
createdAt: j.createdAt,
|
|
284
|
-
})),
|
|
285
|
-
timestamp: Date.now(),
|
|
286
|
-
};
|
|
287
|
-
if (includeJobs) {
|
|
288
|
-
// When paused, ready jobs surface under `paused` (waiting is empty) — match
|
|
289
|
-
// the counts above (#92).
|
|
290
|
-
const waiting = queueManager.getJobs(queue, { state: 'waiting', end: 10 });
|
|
291
|
-
const active = queueManager.getJobs(queue, { state: 'active', end: 10 });
|
|
292
|
-
const delayed = queueManager.getJobs(queue, { state: 'delayed', end: 10 });
|
|
293
|
-
const pausedJobs = paused ? queueManager.getJobs(queue, { state: 'paused', end: 10 }) : [];
|
|
294
|
-
const toSummary = (j) => ({
|
|
295
|
-
id: j.id,
|
|
296
|
-
priority: j.priority,
|
|
297
|
-
createdAt: j.createdAt,
|
|
298
|
-
runAt: j.runAt,
|
|
299
|
-
attempts: j.attempts,
|
|
300
|
-
progress: j.progress,
|
|
301
|
-
});
|
|
302
|
-
result.jobs = {
|
|
303
|
-
waiting: waiting.map(toSummary),
|
|
304
|
-
active: active.map(toSummary),
|
|
305
|
-
delayed: delayed.map(toSummary),
|
|
306
|
-
paused: pausedJobs.map(toSummary),
|
|
307
|
-
};
|
|
308
|
-
}
|
|
309
|
-
return jsonResponse(result, 200, corsOrigins);
|
|
310
|
-
}
|
|
311
167
|
/** Metrics endpoint */
|
|
312
168
|
export function metricsEndpoint(queueManager, corsOrigins) {
|
|
313
169
|
const stats = queueManager.getStats();
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Build a JSON response with optional CORS headers. */
|
|
2
|
+
export function jsonResponse(data, status = 200, corsOrigins) {
|
|
3
|
+
const headers = {
|
|
4
|
+
'Content-Type': 'application/json',
|
|
5
|
+
};
|
|
6
|
+
if (corsOrigins) {
|
|
7
|
+
headers['Access-Control-Allow-Origin'] = corsOrigins.has('*')
|
|
8
|
+
? '*'
|
|
9
|
+
: Array.from(corsOrigins).join(', ');
|
|
10
|
+
}
|
|
11
|
+
return new Response(JSON.stringify(data), { status, headers });
|
|
12
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Authenticated HTTP route dispatcher.
|
|
3
|
+
*/
|
|
4
|
+
import type { HandlerContext } from './types';
|
|
5
|
+
/** Route an authenticated HTTP request to its handler. */
|
|
6
|
+
export declare function routeHttpRequest(req: Request, path: string, ctx: HandlerContext, corsOrigins: Set<string>): Promise<Response>;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Authenticated HTTP route dispatcher.
|
|
3
|
+
*/
|
|
4
|
+
import { validateQueueName } from './protocol';
|
|
5
|
+
import { dashboardOverviewEndpoint, dashboardQueueDetailEndpoint, dashboardQueuesEndpoint, jsonResponse, metricsEndpoint, statsEndpoint, } from './httpEndpoints';
|
|
6
|
+
import { routeJobRoutes } from './httpRouteJobs';
|
|
7
|
+
import { routeQueueConfigRoutes } from './httpRouteQueueConfig';
|
|
8
|
+
import { routeQueueRoutes } from './httpRouteQueues';
|
|
9
|
+
import { routeResourceRoutes } from './httpRouteResources';
|
|
10
|
+
const RE_DASHBOARD_QUEUE_DETAIL = /^\/dashboard\/queues\/([^/]+)$/;
|
|
11
|
+
/** Route an authenticated HTTP request to its handler. */
|
|
12
|
+
export async function routeHttpRequest(req, path, ctx, corsOrigins) {
|
|
13
|
+
const method = req.method;
|
|
14
|
+
if (path === '/stats' && method === 'GET') {
|
|
15
|
+
return statsEndpoint(ctx.queueManager, corsOrigins);
|
|
16
|
+
}
|
|
17
|
+
if (path === '/metrics' && method === 'GET') {
|
|
18
|
+
return metricsEndpoint(ctx.queueManager, corsOrigins);
|
|
19
|
+
}
|
|
20
|
+
if (path === '/dashboard' && method === 'GET') {
|
|
21
|
+
return dashboardOverviewEndpoint(ctx.queueManager, corsOrigins);
|
|
22
|
+
}
|
|
23
|
+
if (path === '/dashboard/queues' && method === 'GET') {
|
|
24
|
+
const url = new URL(req.url);
|
|
25
|
+
const limit = Math.min(Math.max(parseInt(url.searchParams.get('limit') ?? '100', 10) || 100, 1), 500);
|
|
26
|
+
const offset = Math.max(parseInt(url.searchParams.get('offset') ?? '0', 10) || 0, 0);
|
|
27
|
+
return dashboardQueuesEndpoint(ctx.queueManager, limit, offset, corsOrigins);
|
|
28
|
+
}
|
|
29
|
+
const queueMatch = path.match(RE_DASHBOARD_QUEUE_DETAIL);
|
|
30
|
+
if (queueMatch && method === 'GET') {
|
|
31
|
+
const queue = decodeURIComponent(queueMatch[1]);
|
|
32
|
+
const queueError = validateQueueName(queue);
|
|
33
|
+
if (queueError)
|
|
34
|
+
return jsonResponse({ ok: false, error: queueError }, 400, corsOrigins);
|
|
35
|
+
const includeJobs = new URL(req.url).searchParams.get('includeJobs') === 'true';
|
|
36
|
+
return dashboardQueueDetailEndpoint(ctx.queueManager, queue, includeJobs, corsOrigins);
|
|
37
|
+
}
|
|
38
|
+
const routers = [
|
|
39
|
+
routeJobRoutes,
|
|
40
|
+
routeQueueRoutes,
|
|
41
|
+
routeQueueConfigRoutes,
|
|
42
|
+
routeResourceRoutes,
|
|
43
|
+
];
|
|
44
|
+
for (const router of routers) {
|
|
45
|
+
const result = await router(req, path, method, ctx, corsOrigins);
|
|
46
|
+
if (result)
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
return jsonResponse({ ok: false, error: 'Not found' }, 404, corsOrigins);
|
|
50
|
+
}
|