bunqueue 2.8.42 → 2.8.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +3 -3
  2. package/dist/application/latencyTracker.js +3 -3
  3. package/dist/application/metricsExporter.d.ts +3 -1
  4. package/dist/application/metricsExporter.js +63 -16
  5. package/dist/application/prometheusOperationalMetrics.d.ts +31 -0
  6. package/dist/application/prometheusOperationalMetrics.js +53 -0
  7. package/dist/application/queueManager.d.ts +6 -0
  8. package/dist/application/queueManager.js +24 -1
  9. package/dist/application/types.d.ts +3 -0
  10. package/dist/application/types.js +1 -0
  11. package/dist/application/workerManager.d.ts +1 -0
  12. package/dist/application/workerManager.js +4 -1
  13. package/dist/cli/output.js +2 -1
  14. package/dist/config/resolve.d.ts +1 -0
  15. package/dist/config/resolve.js +10 -0
  16. package/dist/config/types.d.ts +6 -0
  17. package/dist/infrastructure/backup/backupTelemetry.d.ts +30 -0
  18. package/dist/infrastructure/backup/backupTelemetry.js +63 -0
  19. package/dist/infrastructure/backup/index.d.ts +1 -1
  20. package/dist/infrastructure/backup/index.js +1 -1
  21. package/dist/infrastructure/backup/s3Backup.d.ts +6 -2
  22. package/dist/infrastructure/backup/s3Backup.js +22 -9
  23. package/dist/infrastructure/backup/s3BackupConfig.d.ts +7 -0
  24. package/dist/infrastructure/backup/s3BackupConfig.js +5 -0
  25. package/dist/infrastructure/backup/s3BackupIo.d.ts +23 -0
  26. package/dist/infrastructure/backup/s3BackupIo.js +106 -0
  27. package/dist/infrastructure/backup/s3BackupOperations.d.ts +6 -15
  28. package/dist/infrastructure/backup/s3BackupOperations.js +139 -259
  29. package/dist/infrastructure/backup/sqliteBackupFiles.d.ts +22 -0
  30. package/dist/infrastructure/backup/sqliteBackupFiles.js +130 -0
  31. package/dist/infrastructure/cloud/statsRefresh.d.ts +1 -0
  32. package/dist/infrastructure/persistence/sqlite.js +6 -1
  33. package/dist/infrastructure/server/bootstrap.d.ts +2 -0
  34. package/dist/infrastructure/server/bootstrap.js +30 -1
  35. package/dist/infrastructure/server/http.d.ts +2 -0
  36. package/dist/infrastructure/server/http.js +9 -58
  37. package/dist/infrastructure/server/httpDashboardEndpoints.d.ts +10 -0
  38. package/dist/infrastructure/server/httpDashboardEndpoints.js +146 -0
  39. package/dist/infrastructure/server/httpEndpoints.d.ts +5 -9
  40. package/dist/infrastructure/server/httpEndpoints.js +16 -160
  41. package/dist/infrastructure/server/httpResponse.d.ts +2 -0
  42. package/dist/infrastructure/server/httpResponse.js +12 -0
  43. package/dist/infrastructure/server/httpRouter.d.ts +6 -0
  44. package/dist/infrastructure/server/httpRouter.js +50 -0
  45. package/dist/shared/histogram.d.ts +2 -2
  46. package/dist/shared/histogram.js +4 -4
  47. package/package.json +2 -2
package/README.md CHANGED
@@ -181,10 +181,10 @@ anywhere in your stack — add a job from TypeScript, process it from Python:
181
181
  | Where your code runs | Install |
182
182
  | -------------------- | ------- |
183
183
  | **Node.js ≥ 20, Deno ≥ 2, Bun, Cloudflare Workers** | [`npm install bunqueue-client`](https://www.npmjs.com/package/bunqueue-client) |
184
- | **Python ≥ 3.9** | PyPI coming soon — today: `pip install "git+https://github.com/egeominotti/bunqueue.git#subdirectory=sdk/python"` |
185
- | **PHP ≥ 8.1** | Packagist coming soon — today: use [`sdk/php`](./sdk/php) |
184
+ | **Python ≥ 3.9** | [`pip install bunqueue-client`](https://pypi.org/project/bunqueue-client/) |
185
+ | **PHP ≥ 8.1** | [`composer require bunqueue/client`](https://packagist.org/packages/bunqueue/client) |
186
186
  | **Go ≥ 1.26.5** | `go get github.com/egeominotti/bunqueue/sdk/go` |
187
- | **Rust ≥ 1.85** | crates.io coming soon — today: use [`sdk/rust`](./sdk/rust) as a path dependency |
187
+ | **Rust ≥ 1.85** | [`cargo add bunqueue-client`](https://crates.io/crates/bunqueue-client) |
188
188
  | **Elixir ≥ 1.15** | Hex coming soon — today: use [`sdk/elixir`](./sdk/elixir) as a path dependency |
189
189
 
190
190
  ```typescript
@@ -11,11 +11,11 @@ export class LatencyTracker {
11
11
  /** Generate Prometheus output for all histograms */
12
12
  toPrometheus() {
13
13
  return [
14
- this.push.toPrometheus('bunqueue_push_duration_ms', 'Push operation latency in milliseconds'),
14
+ this.push.toPrometheus('bunqueue_push_duration_seconds', 'Push operation latency in seconds', 0.001),
15
15
  '',
16
- this.pull.toPrometheus('bunqueue_pull_duration_ms', 'Pull operation latency in milliseconds'),
16
+ this.pull.toPrometheus('bunqueue_pull_duration_seconds', 'Pull operation latency in seconds', 0.001),
17
17
  '',
18
- this.ack.toPrometheus('bunqueue_ack_duration_ms', 'Ack operation latency in milliseconds'),
18
+ this.ack.toPrometheus('bunqueue_ack_duration_seconds', 'Ack operation latency in seconds', 0.001),
19
19
  ].join('\n');
20
20
  }
21
21
  /** Get average latencies */
@@ -5,6 +5,8 @@
5
5
  import type { WorkerManager } from './workerManager';
6
6
  import type { WebhookManager } from './webhookManager';
7
7
  import type { PerQueueStats } from './statsManager';
8
+ import { type SystemMetrics } from './prometheusOperationalMetrics';
9
+ export type { OperationalMetrics, SystemMetrics } from './prometheusOperationalMetrics';
8
10
  /** Stats data structure */
9
11
  export interface QueueStats {
10
12
  waiting: number;
@@ -22,4 +24,4 @@ export interface QueueStats {
22
24
  cronPending: number;
23
25
  }
24
26
  /** Generate Prometheus metrics */
25
- export declare function generatePrometheusMetrics(stats: QueueStats, workerManager: WorkerManager, webhookManager: WebhookManager, perQueueStats?: Map<string, PerQueueStats>): string;
27
+ export declare function generatePrometheusMetrics(stats: QueueStats, workerManager: WorkerManager, webhookManager: WebhookManager, perQueueStats?: Map<string, PerQueueStats>, systemMetrics?: SystemMetrics): string;
@@ -3,10 +3,22 @@
3
3
  * Prometheus metrics generation
4
4
  */
5
5
  import { latencyTracker } from './latencyTracker';
6
+ import { appendOperationalMetrics } from './prometheusOperationalMetrics';
7
+ function escapeLabel(value) {
8
+ return value.replaceAll('\\', '\\\\').replaceAll('\n', '\\n').replaceAll('"', '\\"');
9
+ }
6
10
  /** Generate Prometheus metrics */
7
- export function generatePrometheusMetrics(stats, workerManager, webhookManager, perQueueStats) {
11
+ export function generatePrometheusMetrics(stats, workerManager, webhookManager, perQueueStats, systemMetrics) {
8
12
  const workerStats = workerManager.getStats();
9
13
  const webhookStats = webhookManager.getStats();
14
+ const memory = process.memoryUsage();
15
+ const system = systemMetrics ?? {
16
+ storageDegraded: false,
17
+ storageDiskFull: false,
18
+ processHeapUsedBytes: memory.heapUsed,
19
+ processHeapTotalBytes: memory.heapTotal,
20
+ processResidentMemoryBytes: memory.rss,
21
+ };
10
22
  const lines = [
11
23
  '# HELP bunqueue_jobs_waiting Number of jobs waiting in queue',
12
24
  '# TYPE bunqueue_jobs_waiting gauge',
@@ -52,18 +64,26 @@ export function generatePrometheusMetrics(stats, workerManager, webhookManager,
52
64
  '# TYPE bunqueue_uptime_seconds gauge',
53
65
  `bunqueue_uptime_seconds ${Math.floor(stats.uptime / 1000)}`,
54
66
  '',
55
- '# HELP bunqueue_cron_jobs_total Total number of cron jobs',
56
- '# TYPE bunqueue_cron_jobs_total gauge',
57
- `bunqueue_cron_jobs_total ${stats.cronJobs}`,
67
+ '# HELP bunqueue_cron_jobs_registered Number of registered cron jobs',
68
+ '# TYPE bunqueue_cron_jobs_registered gauge',
69
+ `bunqueue_cron_jobs_registered ${stats.cronJobs}`,
58
70
  '',
59
- '# HELP bunqueue_workers_total Total number of registered workers',
60
- '# TYPE bunqueue_workers_total gauge',
61
- `bunqueue_workers_total ${workerStats.total}`,
71
+ '# HELP bunqueue_workers_registered Number of registered workers',
72
+ '# TYPE bunqueue_workers_registered gauge',
73
+ `bunqueue_workers_registered ${workerStats.total}`,
62
74
  '',
63
75
  '# HELP bunqueue_workers_active Number of active workers',
64
76
  '# TYPE bunqueue_workers_active gauge',
65
77
  `bunqueue_workers_active ${workerStats.active}`,
66
78
  '',
79
+ '# HELP bunqueue_worker_active_jobs Number of jobs currently processed by workers',
80
+ '# TYPE bunqueue_worker_active_jobs gauge',
81
+ `bunqueue_worker_active_jobs ${workerStats.activeJobs}`,
82
+ '',
83
+ '# HELP bunqueue_worker_concurrency_slots Configured worker concurrency capacity',
84
+ '# TYPE bunqueue_worker_concurrency_slots gauge',
85
+ `bunqueue_worker_concurrency_slots ${workerStats.concurrencySlots}`,
86
+ '',
67
87
  '# HELP bunqueue_workers_processed_total Total jobs processed by workers',
68
88
  '# TYPE bunqueue_workers_processed_total counter',
69
89
  `bunqueue_workers_processed_total ${workerStats.totalProcessed}`,
@@ -72,52 +92,79 @@ export function generatePrometheusMetrics(stats, workerManager, webhookManager,
72
92
  '# TYPE bunqueue_workers_failed_total counter',
73
93
  `bunqueue_workers_failed_total ${workerStats.totalFailed}`,
74
94
  '',
75
- '# HELP bunqueue_webhooks_total Total number of webhooks',
76
- '# TYPE bunqueue_webhooks_total gauge',
77
- `bunqueue_webhooks_total ${webhookStats.total}`,
95
+ '# HELP bunqueue_webhooks_registered Number of registered webhooks',
96
+ '# TYPE bunqueue_webhooks_registered gauge',
97
+ `bunqueue_webhooks_registered ${webhookStats.total}`,
78
98
  '',
79
99
  '# HELP bunqueue_webhooks_enabled Number of enabled webhooks',
80
100
  '# TYPE bunqueue_webhooks_enabled gauge',
81
101
  `bunqueue_webhooks_enabled ${webhookStats.enabled}`,
102
+ '',
103
+ '# HELP bunqueue_storage_degraded Whether persistent storage is degraded',
104
+ '# TYPE bunqueue_storage_degraded gauge',
105
+ `bunqueue_storage_degraded ${system.storageDegraded ? 1 : 0}`,
106
+ '',
107
+ '# HELP bunqueue_storage_disk_full Whether persistent storage reported a full disk',
108
+ '# TYPE bunqueue_storage_disk_full gauge',
109
+ `bunqueue_storage_disk_full ${system.storageDiskFull ? 1 : 0}`,
110
+ '',
111
+ '# HELP bunqueue_process_heap_used_bytes Process heap bytes currently used',
112
+ '# TYPE bunqueue_process_heap_used_bytes gauge',
113
+ `bunqueue_process_heap_used_bytes ${system.processHeapUsedBytes}`,
114
+ '',
115
+ '# HELP bunqueue_process_heap_total_bytes Total process heap bytes',
116
+ '# TYPE bunqueue_process_heap_total_bytes gauge',
117
+ `bunqueue_process_heap_total_bytes ${system.processHeapTotalBytes}`,
118
+ '',
119
+ '# HELP bunqueue_process_resident_memory_bytes Process resident memory size in bytes',
120
+ '# TYPE bunqueue_process_resident_memory_bytes gauge',
121
+ `bunqueue_process_resident_memory_bytes ${system.processResidentMemoryBytes}`,
82
122
  ];
123
+ if (system.sqliteDatabaseSizeBytes !== undefined) {
124
+ lines.push('', '# HELP bunqueue_sqlite_database_size_bytes SQLite database file size in bytes', '# TYPE bunqueue_sqlite_database_size_bytes gauge', `bunqueue_sqlite_database_size_bytes ${system.sqliteDatabaseSizeBytes}`);
125
+ }
83
126
  // Per-queue metrics
84
127
  if (perQueueStats && perQueueStats.size > 0) {
85
128
  lines.push('');
86
129
  lines.push('# HELP bunqueue_queue_jobs_waiting Number of waiting jobs per queue');
87
130
  lines.push('# TYPE bunqueue_queue_jobs_waiting gauge');
88
131
  for (const [queue, qs] of perQueueStats) {
89
- lines.push(`bunqueue_queue_jobs_waiting{queue="${queue}"} ${qs.waiting}`);
132
+ lines.push(`bunqueue_queue_jobs_waiting{queue="${escapeLabel(queue)}"} ${qs.waiting}`);
90
133
  }
91
134
  lines.push('');
92
135
  lines.push('# HELP bunqueue_queue_jobs_prioritized Number of prioritized jobs per queue');
93
136
  lines.push('# TYPE bunqueue_queue_jobs_prioritized gauge');
94
137
  for (const [queue, qs] of perQueueStats) {
95
- lines.push(`bunqueue_queue_jobs_prioritized{queue="${queue}"} ${qs.prioritized}`);
138
+ lines.push(`bunqueue_queue_jobs_prioritized{queue="${escapeLabel(queue)}"} ${qs.prioritized}`);
96
139
  }
97
140
  lines.push('');
98
141
  lines.push('# HELP bunqueue_queue_jobs_delayed Number of delayed jobs per queue');
99
142
  lines.push('# TYPE bunqueue_queue_jobs_delayed gauge');
100
143
  for (const [queue, qs] of perQueueStats) {
101
- lines.push(`bunqueue_queue_jobs_delayed{queue="${queue}"} ${qs.delayed}`);
144
+ lines.push(`bunqueue_queue_jobs_delayed{queue="${escapeLabel(queue)}"} ${qs.delayed}`);
102
145
  }
103
146
  lines.push('');
104
147
  lines.push('# HELP bunqueue_queue_jobs_active Number of active jobs per queue');
105
148
  lines.push('# TYPE bunqueue_queue_jobs_active gauge');
106
149
  for (const [queue, qs] of perQueueStats) {
107
- lines.push(`bunqueue_queue_jobs_active{queue="${queue}"} ${qs.active}`);
150
+ lines.push(`bunqueue_queue_jobs_active{queue="${escapeLabel(queue)}"} ${qs.active}`);
108
151
  }
109
152
  lines.push('');
110
153
  lines.push('# HELP bunqueue_queue_jobs_dlq Number of DLQ jobs per queue');
111
154
  lines.push('# TYPE bunqueue_queue_jobs_dlq gauge');
112
155
  for (const [queue, qs] of perQueueStats) {
113
- lines.push(`bunqueue_queue_jobs_dlq{queue="${queue}"} ${qs.dlq}`);
156
+ lines.push(`bunqueue_queue_jobs_dlq{queue="${escapeLabel(queue)}"} ${qs.dlq}`);
114
157
  }
115
158
  }
159
+ appendOperationalMetrics(lines, {
160
+ ...system,
161
+ perQueueMetricsExported: system.perQueueMetricsExported ?? perQueueStats?.size ?? 0,
162
+ });
116
163
  // Append latency histograms
117
164
  const histogramOutput = latencyTracker.toPrometheus();
118
165
  if (histogramOutput) {
119
166
  lines.push('');
120
167
  lines.push(histogramOutput);
121
168
  }
122
- return lines.join('\n');
169
+ return `${lines.join('\n')}\n`;
123
170
  }
@@ -0,0 +1,31 @@
1
+ import { type BackupMetrics } from '../infrastructure/backup/backupTelemetry';
2
+ /** Runtime/storage gauges which are not part of queue state. */
3
+ export interface SystemMetrics {
4
+ storageDegraded: boolean;
5
+ storageDiskFull: boolean;
6
+ sqliteDatabaseSizeBytes?: number;
7
+ processHeapUsedBytes: number;
8
+ processHeapTotalBytes: number;
9
+ processResidentMemoryBytes: number;
10
+ processCpuSeconds?: number;
11
+ processStartTimeSeconds?: number;
12
+ perQueueMetricsExported?: number;
13
+ perQueueMetricsOmitted?: number;
14
+ operational?: OperationalMetrics;
15
+ }
16
+ /** Server components that are owned outside QueueManager. */
17
+ export interface OperationalMetrics {
18
+ backup?: BackupMetrics;
19
+ connections?: {
20
+ tcp: number;
21
+ websocket: number;
22
+ sse: number;
23
+ };
24
+ }
25
+ /** Select a deterministic bounded prefix and report exact conservation. */
26
+ export declare function selectPrometheusQueues(queueNames: Iterable<string>, requestedLimit: number): {
27
+ selected: Set<string>;
28
+ omitted: number;
29
+ };
30
+ /** Append bounded operational and standard process collectors. */
31
+ export declare function appendOperationalMetrics(lines: string[], system: SystemMetrics): void;
@@ -0,0 +1,53 @@
1
+ import { VERSION } from '../shared/version';
2
+ import { EMPTY_BACKUP_METRICS } from '../infrastructure/backup/backupTelemetry';
3
+ const PROCESS_START_TIME_SECONDS = Date.now() / 1000 - process.uptime();
4
+ /** Select a deterministic bounded prefix and report exact conservation. */
5
+ export function selectPrometheusQueues(queueNames, requestedLimit) {
6
+ const names = [...queueNames];
7
+ const limit = Number.isFinite(requestedLimit) ? Math.max(0, Math.floor(requestedLimit)) : 0;
8
+ const selected = new Set(names.slice(0, limit));
9
+ return { selected, omitted: names.length - selected.size };
10
+ }
11
+ function escapeLabel(value) {
12
+ return value.replaceAll('\\', '\\\\').replaceAll('\n', '\\n').replaceAll('"', '\\"');
13
+ }
14
+ function metric(lines, name, help, type, value) {
15
+ lines.push('', `# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, `${name} ${value}`);
16
+ }
17
+ /** Append bounded operational and standard process collectors. */
18
+ export function appendOperationalMetrics(lines, system) {
19
+ const cpu = system.processCpuSeconds ?? cpuSeconds();
20
+ const processStart = system.processStartTimeSeconds ?? PROCESS_START_TIME_SECONDS;
21
+ const backup = system.operational?.backup ?? EMPTY_BACKUP_METRICS;
22
+ const connections = system.operational?.connections ?? { tcp: 0, websocket: 0, sse: 0 };
23
+ const bunVersion = typeof Bun === 'undefined' ? 'unknown' : Bun.version;
24
+ lines.push('', '# HELP bunqueue_build_info Build and runtime identity', '# TYPE bunqueue_build_info gauge', `bunqueue_build_info{bun_version="${escapeLabel(bunVersion)}",version="${escapeLabel(VERSION)}"} 1`);
25
+ metric(lines, 'process_cpu_seconds_total', 'Total process CPU time in seconds', 'counter', cpu);
26
+ metric(lines, 'process_start_time_seconds', 'Start time of the process since the Unix epoch in seconds', 'gauge', processStart);
27
+ metric(lines, 'process_resident_memory_bytes', 'Resident memory size in bytes', 'gauge', system.processResidentMemoryBytes);
28
+ metric(lines, 'process_heap_bytes', 'Process heap size in bytes', 'gauge', system.processHeapTotalBytes);
29
+ lines.push('', '# HELP bunqueue_connections Current server connections by bounded transport', '# TYPE bunqueue_connections gauge', `bunqueue_connections{transport="tcp"} ${connections.tcp}`, `bunqueue_connections{transport="websocket"} ${connections.websocket}`, `bunqueue_connections{transport="sse"} ${connections.sse}`);
30
+ metric(lines, 'bunqueue_queue_metrics_exported', 'Queue names included in labelled Prometheus series', 'gauge', system.perQueueMetricsExported ?? 0);
31
+ metric(lines, 'bunqueue_queue_metrics_omitted', 'Queue names omitted by the Prometheus cardinality limit', 'gauge', system.perQueueMetricsOmitted ?? 0);
32
+ appendBackupMetrics(lines, backup);
33
+ }
34
+ function appendBackupMetrics(lines, backup) {
35
+ metric(lines, 'bunqueue_backup_enabled', 'Whether scheduled S3 backup is enabled', 'gauge', +backup.enabled);
36
+ metric(lines, 'bunqueue_backup_scheduler_running', 'Whether the scheduled S3 backup timer is running', 'gauge', +backup.schedulerRunning);
37
+ metric(lines, 'bunqueue_backup_in_progress', 'Whether an S3 backup is in progress', 'gauge', +backup.inProgress);
38
+ metric(lines, 'bunqueue_backup_interval_seconds', 'Configured S3 backup interval in seconds', 'gauge', backup.intervalSeconds);
39
+ metric(lines, 'bunqueue_backup_retention', 'Configured number of S3 backups to retain', 'gauge', backup.retention);
40
+ metric(lines, 'bunqueue_backup_attempts_total', 'S3 backup attempts started', 'counter', backup.attemptsTotal);
41
+ metric(lines, 'bunqueue_backup_successes_total', 'Successful S3 backup attempts', 'counter', backup.successesTotal);
42
+ metric(lines, 'bunqueue_backup_failures_total', 'Failed S3 backup attempts', 'counter', backup.failuresTotal);
43
+ metric(lines, 'bunqueue_backup_overlap_rejections_total', 'S3 backup requests rejected because another backup was active', 'counter', backup.overlapRejectionsTotal);
44
+ metric(lines, 'bunqueue_backup_consecutive_failures', 'Consecutive failed S3 backup attempts', 'gauge', backup.consecutiveFailures);
45
+ metric(lines, 'bunqueue_backup_last_success_timestamp_seconds', 'Unix timestamp of the last successful S3 backup, or zero', 'gauge', backup.lastSuccessTimestampSeconds);
46
+ metric(lines, 'bunqueue_backup_last_failure_timestamp_seconds', 'Unix timestamp of the last failed S3 backup, or zero', 'gauge', backup.lastFailureTimestampSeconds);
47
+ metric(lines, 'bunqueue_backup_last_duration_seconds', 'Duration of the last S3 backup attempt', 'gauge', backup.lastDurationSeconds);
48
+ metric(lines, 'bunqueue_backup_last_size_bytes', 'Compressed size of the last successful S3 backup', 'gauge', backup.lastSizeBytes);
49
+ }
50
+ function cpuSeconds() {
51
+ const usage = process.cpuUsage();
52
+ return (usage.user + usage.system) / 1_000_000;
53
+ }
@@ -11,6 +11,7 @@ import type { DlqConfig, DlqEntry, DlqFilter, DlqStats } from '../domain/types/d
11
11
  import { Shard } from '../domain/queue/shard';
12
12
  import { WebhookManager } from './webhookManager';
13
13
  import { WorkerManager } from './workerManager';
14
+ import { type OperationalMetrics } from './metricsExporter';
14
15
  import { type SetLike } from '../shared/lru';
15
16
  import type { QueueManagerConfig } from './types';
16
17
  import * as statsMgr from './statsManager';
@@ -53,6 +54,7 @@ export declare class QueueManager {
53
54
  private recoveryStats;
54
55
  private readonly maxLogsPerJob;
55
56
  private readonly metrics;
57
+ private operationalMetricsProvider;
56
58
  private readonly perQueueMetrics;
57
59
  private readonly startTime;
58
60
  private readonly backgroundTaskHandles;
@@ -251,6 +253,8 @@ export declare class QueueManager {
251
253
  clearLogs(jobId: JobId, keepLogs?: number): void;
252
254
  getPerQueueStats(): Map<string, statsMgr.PerQueueStats>;
253
255
  getPrometheusMetrics(): string;
256
+ /** Attach metrics owned by server components outside the queue state machine. */
257
+ setOperationalMetricsProvider(provider: () => OperationalMetrics): void;
254
258
  addCron(input: CronJobInput): CronJob;
255
259
  /** Remove an orphaned cron job from the queue by its uniqueKey */
256
260
  private removeOrphanedCronJob;
@@ -363,6 +367,8 @@ export declare class QueueManager {
363
367
  error: string | null;
364
368
  since: number | null;
365
369
  };
370
+ /** Persist pending buffered writes before an external SQLite snapshot. */
371
+ flushPersistence(): number;
366
372
  compactMemory(): void;
367
373
  shutdown(): void;
368
374
  }
@@ -25,6 +25,7 @@ import * as queryOps from './operations/queryOperations';
25
25
  import * as dlqOps from './dlqManager';
26
26
  import * as logsOps from './jobLogsManager';
27
27
  import { generatePrometheusMetrics } from './metricsExporter';
28
+ import { selectPrometheusQueues } from './prometheusOperationalMetrics';
28
29
  import { LRUMap, BoundedSet, BoundedMap } from '../shared/lru';
29
30
  import { DEFAULT_CONFIG } from './types';
30
31
  import * as lockMgr from './lockManager';
@@ -98,6 +99,7 @@ export class QueueManager {
98
99
  totalCompleted: { value: 0n },
99
100
  totalFailed: { value: 0n },
100
101
  };
102
+ operationalMetricsProvider = null;
101
103
  // LRU-bounded so high-cardinality / dynamically-named queues cannot grow it
102
104
  // without bound. Live queues stay resident (recently accessed on every
103
105
  // ack/fail); only long-idle ephemeral names are evicted. obliterate() also
@@ -1081,7 +1083,24 @@ export class QueueManager {
1081
1083
  return statsMgr.getPerQueueStats(this.contextFactory.getStatsContext(), this.queueNamesCache);
1082
1084
  }
1083
1085
  getPrometheusMetrics() {
1084
- return generatePrometheusMetrics(this.getStats(), this.workerManager, this.webhookManager, this.getPerQueueStats());
1086
+ const storageStatus = this.getStorageStatus();
1087
+ const memory = process.memoryUsage();
1088
+ const queueSelection = selectPrometheusQueues(this.queueNamesCache, this.config.maxPrometheusQueues);
1089
+ return generatePrometheusMetrics(this.getStats(), this.workerManager, this.webhookManager, statsMgr.getPerQueueStats(this.contextFactory.getStatsContext(), queueSelection.selected), {
1090
+ storageDegraded: storageStatus.diskFull,
1091
+ storageDiskFull: storageStatus.diskFull,
1092
+ ...(this.storage && { sqliteDatabaseSizeBytes: this.storage.getSize() }),
1093
+ processHeapUsedBytes: memory.heapUsed,
1094
+ processHeapTotalBytes: memory.heapTotal,
1095
+ processResidentMemoryBytes: memory.rss,
1096
+ perQueueMetricsExported: queueSelection.selected.size,
1097
+ perQueueMetricsOmitted: queueSelection.omitted,
1098
+ operational: this.operationalMetricsProvider?.(),
1099
+ });
1100
+ }
1101
+ /** Attach metrics owned by server components outside the queue state machine. */
1102
+ setOperationalMetricsProvider(provider) {
1103
+ this.operationalMetricsProvider = provider;
1085
1104
  }
1086
1105
  // ============ Cron Operations ============
1087
1106
  addCron(input) {
@@ -1620,6 +1639,10 @@ export class QueueManager {
1620
1639
  return { diskFull: false, error: null, since: null };
1621
1640
  return this.storage.getDiskFullStatus();
1622
1641
  }
1642
+ /** Persist pending buffered writes before an external SQLite snapshot. */
1643
+ flushPersistence() {
1644
+ return this.storage?.flushWriteBuffer() ?? 0;
1645
+ }
1623
1646
  compactMemory() {
1624
1647
  statsMgr.compactMemory(this.contextFactory.getStatsContext());
1625
1648
  this.dashboardEmit?.('memory:compacted', {});
@@ -21,6 +21,8 @@ export interface QueueManagerConfig {
21
21
  maxJobLogs?: number;
22
22
  maxCustomIds?: number;
23
23
  maxWaitingDeps?: number;
24
+ /** Maximum queue label values emitted in one Prometheus scrape; zero disables them. */
25
+ maxPrometheusQueues?: number;
24
26
  cleanupIntervalMs?: number;
25
27
  jobTimeoutCheckMs?: number;
26
28
  dependencyCheckMs?: number;
@@ -34,6 +36,7 @@ export declare const DEFAULT_CONFIG: {
34
36
  maxJobLogs: number;
35
37
  maxCustomIds: number;
36
38
  maxWaitingDeps: number;
39
+ maxPrometheusQueues: number;
37
40
  cleanupIntervalMs: number;
38
41
  jobTimeoutCheckMs: number;
39
42
  dependencyCheckMs: number;
@@ -7,6 +7,7 @@ export const DEFAULT_CONFIG = {
7
7
  maxJobLogs: 10_000,
8
8
  maxCustomIds: 50_000,
9
9
  maxWaitingDeps: 10_000,
10
+ maxPrometheusQueues: 100,
10
11
  cleanupIntervalMs: 10_000,
11
12
  jobTimeoutCheckMs: 5_000,
12
13
  dependencyCheckMs: 30_000, // Safety fallback only; event-driven handles fast path
@@ -56,5 +56,6 @@ export declare class WorkerManager {
56
56
  totalProcessed: number;
57
57
  totalFailed: number;
58
58
  activeJobs: number;
59
+ concurrencySlots: number;
59
60
  };
60
61
  }
@@ -202,10 +202,12 @@ export class WorkerManager {
202
202
  getStats() {
203
203
  const now = Date.now();
204
204
  let activeWorkers = 0;
205
- // Only iterate once for active count (time-based, can't use counter)
205
+ let concurrencySlots = 0;
206
+ // Only iterate once for time-based active count and configured capacity.
206
207
  for (const worker of this.workers.values()) {
207
208
  if (now - worker.lastSeen < WORKER_TIMEOUT_MS) {
208
209
  activeWorkers++;
210
+ concurrencySlots += worker.concurrency;
209
211
  }
210
212
  }
211
213
  return {
@@ -214,6 +216,7 @@ export class WorkerManager {
214
216
  totalProcessed: this.totalProcessedCounter,
215
217
  totalFailed: this.totalFailedCounter,
216
218
  activeJobs: this.totalActiveJobsCounter,
219
+ concurrencySlots,
217
220
  };
218
221
  }
219
222
  }
@@ -144,7 +144,8 @@ function formatStats(stats) {
144
144
  lines.push(` Total Failed: ${str(stats.totalFailed)}`);
145
145
  }
146
146
  if (stats.uptime !== undefined) {
147
- lines.push('', ` ${color('Uptime:', colors.cyan)} ${str(stats.uptime)}s`);
147
+ const uptimeSeconds = typeof stats.uptime === 'number' ? Math.floor(stats.uptime / 1000) : stats.uptime;
148
+ lines.push('', ` ${color('Uptime:', colors.cyan)} ${str(uptimeSeconds)}s`);
148
149
  }
149
150
  if (stats.pushPerSec !== undefined) {
150
151
  lines.push(` Push/sec: ${str(stats.pushPerSec)}`);
@@ -18,6 +18,7 @@ export interface ResolvedConfig {
18
18
  dataPath: string | undefined;
19
19
  corsOrigins: string[];
20
20
  requireAuthForMetrics: boolean;
21
+ maxPrometheusQueues: number;
21
22
  s3BackupEnabled: boolean;
22
23
  shutdownTimeoutMs: number;
23
24
  statsIntervalMs: number;
@@ -24,12 +24,16 @@ export function resolveServerConfig(fileConfig) {
24
24
  Bun.env.SQLITE_PATH,
25
25
  corsOrigins: fc?.cors?.origins ?? Bun.env.CORS_ALLOW_ORIGIN?.split(',').filter(Boolean) ?? [],
26
26
  requireAuthForMetrics: fc?.auth?.requireAuthForMetrics ?? Bun.env.METRICS_AUTH === 'true',
27
+ maxPrometheusQueues: nonNegativeInteger(fc?.telemetry?.maxPrometheusQueues ?? parseInt(Bun.env.METRICS_MAX_QUEUES ?? '100', 10), 100),
27
28
  s3BackupEnabled: fc?.backup?.enabled ??
28
29
  (Bun.env.S3_BACKUP_ENABLED === '1' || Bun.env.S3_BACKUP_ENABLED === 'true'),
29
30
  shutdownTimeoutMs: fc?.timeouts?.shutdown ?? parseInt(Bun.env.SHUTDOWN_TIMEOUT_MS ?? '30000', 10),
30
31
  statsIntervalMs: fc?.timeouts?.stats ?? parseInt(Bun.env.STATS_INTERVAL_MS ?? '300000', 10),
31
32
  };
32
33
  }
34
+ function nonNegativeInteger(value, fallback) {
35
+ return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
36
+ }
33
37
  /**
34
38
  * Resolve server TLS options from resolved config. Returns null when TLS is
35
39
  * not configured; throws when only one of cert/key is set (fail fast at
@@ -81,12 +85,18 @@ export function resolveCloudConfig(fileConfig, dataPath) {
81
85
  /** Resolve S3 backup config: config file > env vars */
82
86
  export function resolveBackupConfig(fileConfig, databasePath) {
83
87
  const fc = fileConfig?.backup;
88
+ const virtualHostedStyle = Bun.env.S3_VIRTUAL_HOSTED_STYLE;
84
89
  return {
85
90
  enabled: fc?.enabled ?? (Bun.env.S3_BACKUP_ENABLED === '1' || Bun.env.S3_BACKUP_ENABLED === 'true'),
86
91
  accessKeyId: fc?.accessKeyId ?? Bun.env.S3_ACCESS_KEY_ID ?? Bun.env.AWS_ACCESS_KEY_ID ?? '',
87
92
  secretAccessKey: fc?.secretAccessKey ?? Bun.env.S3_SECRET_ACCESS_KEY ?? Bun.env.AWS_SECRET_ACCESS_KEY ?? '',
93
+ sessionToken: fc?.sessionToken ?? Bun.env.S3_SESSION_TOKEN ?? Bun.env.AWS_SESSION_TOKEN,
88
94
  bucket: fc?.bucket ?? Bun.env.S3_BUCKET ?? Bun.env.AWS_BUCKET ?? '',
89
95
  endpoint: fc?.endpoint ?? Bun.env.S3_ENDPOINT ?? Bun.env.AWS_ENDPOINT,
96
+ virtualHostedStyle: fc?.virtualHostedStyle ??
97
+ (virtualHostedStyle === undefined
98
+ ? undefined
99
+ : virtualHostedStyle === '1' || virtualHostedStyle === 'true'),
90
100
  region: fc?.region ?? Bun.env.S3_REGION ?? Bun.env.AWS_REGION ?? S3_DEFAULTS.region,
91
101
  intervalMs: fc?.interval ?? (parseInt(Bun.env.S3_BACKUP_INTERVAL ?? '', 10) || S3_DEFAULTS.intervalMs),
92
102
  retention: fc?.retention ?? (parseInt(Bun.env.S3_BACKUP_RETENTION ?? '', 10) || S3_DEFAULTS.retention),
@@ -22,6 +22,10 @@ export interface BunqueueConfig {
22
22
  storage?: {
23
23
  dataPath?: string;
24
24
  };
25
+ telemetry?: {
26
+ /** Maximum queue label values exposed to Prometheus; zero disables per-queue series. */
27
+ maxPrometheusQueues?: number;
28
+ };
25
29
  cors?: {
26
30
  origins?: string[];
27
31
  };
@@ -35,8 +39,10 @@ export interface BunqueueConfig {
35
39
  bucket?: string;
36
40
  accessKeyId?: string;
37
41
  secretAccessKey?: string;
42
+ sessionToken?: string;
38
43
  region?: string;
39
44
  endpoint?: string;
45
+ virtualHostedStyle?: boolean;
40
46
  interval?: number;
41
47
  retention?: number;
42
48
  prefix?: string;
@@ -0,0 +1,30 @@
1
+ import type { BackupResult } from './s3BackupConfig';
2
+ /** Bounded, label-free backup metrics exposed to Prometheus. */
3
+ export interface BackupMetrics {
4
+ enabled: boolean;
5
+ schedulerRunning: boolean;
6
+ inProgress: boolean;
7
+ intervalSeconds: number;
8
+ retention: number;
9
+ attemptsTotal: number;
10
+ successesTotal: number;
11
+ failuresTotal: number;
12
+ overlapRejectionsTotal: number;
13
+ consecutiveFailures: number;
14
+ lastSuccessTimestampSeconds: number;
15
+ lastFailureTimestampSeconds: number;
16
+ lastDurationSeconds: number;
17
+ lastSizeBytes: number;
18
+ }
19
+ export declare const EMPTY_BACKUP_METRICS: BackupMetrics;
20
+ /** State machine for backup attempts; all transitions preserve counter conservation. */
21
+ export declare class BackupTelemetry {
22
+ private readonly metrics;
23
+ constructor(enabled: boolean, intervalMs: number, retention: number);
24
+ setSchedulerRunning(running: boolean): void;
25
+ tryStart(): boolean;
26
+ succeed(result: BackupResult, durationMs: number, nowMs?: number): void;
27
+ fail(durationMs: number, nowMs?: number): void;
28
+ snapshot(): BackupMetrics;
29
+ private finish;
30
+ }
@@ -0,0 +1,63 @@
1
+ export const EMPTY_BACKUP_METRICS = {
2
+ enabled: false,
3
+ schedulerRunning: false,
4
+ inProgress: false,
5
+ intervalSeconds: 0,
6
+ retention: 0,
7
+ attemptsTotal: 0,
8
+ successesTotal: 0,
9
+ failuresTotal: 0,
10
+ overlapRejectionsTotal: 0,
11
+ consecutiveFailures: 0,
12
+ lastSuccessTimestampSeconds: 0,
13
+ lastFailureTimestampSeconds: 0,
14
+ lastDurationSeconds: 0,
15
+ lastSizeBytes: 0,
16
+ };
17
+ /** State machine for backup attempts; all transitions preserve counter conservation. */
18
+ export class BackupTelemetry {
19
+ metrics;
20
+ constructor(enabled, intervalMs, retention) {
21
+ this.metrics = {
22
+ ...EMPTY_BACKUP_METRICS,
23
+ enabled,
24
+ intervalSeconds: intervalMs / 1000,
25
+ retention,
26
+ };
27
+ }
28
+ setSchedulerRunning(running) {
29
+ this.metrics.schedulerRunning = running;
30
+ }
31
+ tryStart() {
32
+ if (this.metrics.inProgress) {
33
+ this.metrics.overlapRejectionsTotal++;
34
+ return false;
35
+ }
36
+ this.metrics.inProgress = true;
37
+ this.metrics.attemptsTotal++;
38
+ return true;
39
+ }
40
+ succeed(result, durationMs, nowMs = Date.now()) {
41
+ this.finish(durationMs);
42
+ this.metrics.successesTotal++;
43
+ this.metrics.consecutiveFailures = 0;
44
+ this.metrics.lastSuccessTimestampSeconds = nowMs / 1000;
45
+ this.metrics.lastSizeBytes = result.compressedSize ?? result.size ?? 0;
46
+ }
47
+ fail(durationMs, nowMs = Date.now()) {
48
+ this.finish(durationMs);
49
+ this.metrics.failuresTotal++;
50
+ this.metrics.consecutiveFailures++;
51
+ this.metrics.lastFailureTimestampSeconds = nowMs / 1000;
52
+ }
53
+ snapshot() {
54
+ return { ...this.metrics };
55
+ }
56
+ finish(durationMs) {
57
+ if (!this.metrics.inProgress) {
58
+ throw new Error('Cannot finish backup telemetry without an active attempt');
59
+ }
60
+ this.metrics.inProgress = false;
61
+ this.metrics.lastDurationSeconds = Math.max(durationMs, 0) / 1000;
62
+ }
63
+ }
@@ -1,4 +1,4 @@
1
1
  /**
2
2
  * Backup Module Exports
3
3
  */
4
- export { S3BackupManager, type S3BackupConfig, type BackupResult } from './s3Backup';
4
+ export { S3BackupManager, type S3BackupConfig, type BackupResult, type BackupMetrics, } from './s3Backup';
@@ -1,4 +1,4 @@
1
1
  /**
2
2
  * Backup Module Exports
3
3
  */
4
- export { S3BackupManager } from './s3Backup';
4
+ export { S3BackupManager, } from './s3Backup';