bullpane 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/bin/bullpane.mjs +28 -1
- package/dist/server.mjs +1502 -101
- package/migrations/mysql/0010_connection_kind.sql +4 -0
- package/migrations/sqlite/0003_connection_kind.sql +3 -0
- package/package.json +5 -2
- package/web/assets/{FlowsPage-CxMz_9Et.js → FlowsPage-CRkXK3KR.js} +1 -1
- package/web/assets/{JobTreePage-BT79W1zX.js → JobTreePage-CcNNhc6M.js} +1 -1
- package/web/assets/{McpConsentPage-B2YnL6fN.js → McpConsentPage-BXb_LIdx.js} +1 -1
- package/web/assets/{flowLayout-DoC2dCG0.js → flowLayout-CPGuWm9t.js} +1 -1
- package/web/assets/index-xn9W6Th4.js +548 -0
- package/web/index.html +1 -1
- package/web/assets/index-DuAopWKm.js +0 -543
package/dist/server.mjs
CHANGED
|
@@ -4,9 +4,16 @@ var __export = (target, all) => {
|
|
|
4
4
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
5
5
|
};
|
|
6
6
|
|
|
7
|
-
// packages/
|
|
8
|
-
import {
|
|
9
|
-
|
|
7
|
+
// packages/pg-inspector/src/inspector.ts
|
|
8
|
+
import {
|
|
9
|
+
assertSchemaCompatibility,
|
|
10
|
+
createPostgresBackend,
|
|
11
|
+
Job,
|
|
12
|
+
Queue,
|
|
13
|
+
SchemaMigrationRequiredError,
|
|
14
|
+
UnrecoverableError
|
|
15
|
+
} from "bullmq";
|
|
16
|
+
import pg from "pg";
|
|
10
17
|
|
|
11
18
|
// packages/shared/src/index.ts
|
|
12
19
|
import { z } from "zod";
|
|
@@ -142,21 +149,53 @@ var ssoSettingsSchema = z.object({
|
|
|
142
149
|
autoProvisionDomains: z.array(ssoDomainSchema).max(50).optional()
|
|
143
150
|
});
|
|
144
151
|
var SSO_AUTO_PROVISION_ROLE = "viewer";
|
|
145
|
-
var
|
|
152
|
+
var CONNECTION_KINDS = ["redis", "postgres"];
|
|
153
|
+
var DEFAULT_NAMESPACE = {
|
|
154
|
+
redis: "bull",
|
|
155
|
+
postgres: "bullmq"
|
|
156
|
+
};
|
|
157
|
+
var URL_SCHEMES = {
|
|
158
|
+
redis: { test: /^rediss?:\/\//, message: "Must start with redis:// or rediss://" },
|
|
159
|
+
postgres: { test: /^postgres(ql)?:\/\//, message: "Must start with postgres:// or postgresql://" }
|
|
160
|
+
};
|
|
161
|
+
var PG_SCHEMA_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
|
162
|
+
function connectionIssues(v) {
|
|
163
|
+
const issues = [];
|
|
164
|
+
const kind = v.kind ?? "redis";
|
|
165
|
+
if (v.url !== void 0 && !URL_SCHEMES[kind].test.test(v.url)) {
|
|
166
|
+
issues.push({ path: "url", message: URL_SCHEMES[kind].message });
|
|
167
|
+
}
|
|
168
|
+
if (kind === "postgres") {
|
|
169
|
+
if (v.prefix !== void 0 && !PG_SCHEMA_RE.test(v.prefix)) {
|
|
170
|
+
issues.push({ path: "prefix", message: "A Postgres schema: letters, digits and _, not starting with a digit" });
|
|
171
|
+
}
|
|
172
|
+
if (v.cluster === true) issues.push({ path: "cluster", message: "Cluster mode is Redis only" });
|
|
173
|
+
}
|
|
174
|
+
return issues;
|
|
175
|
+
}
|
|
176
|
+
var connectionFields = z.object({
|
|
146
177
|
name: z.string().min(1).max(80),
|
|
147
|
-
|
|
148
|
-
|
|
178
|
+
kind: z.enum(CONNECTION_KINDS).default("redis"),
|
|
179
|
+
url: z.string().min(1),
|
|
180
|
+
/** omitted → DEFAULT_NAMESPACE of the kind */
|
|
181
|
+
prefix: z.string().min(1).max(64).optional(),
|
|
149
182
|
cluster: z.boolean().default(false),
|
|
150
183
|
queueFilter: z.string().max(200).nullable().optional()
|
|
151
184
|
});
|
|
152
|
-
var
|
|
185
|
+
var createConnectionSchema = connectionFields.superRefine((v, ctx) => {
|
|
186
|
+
for (const issue of connectionIssues(v)) ctx.addIssue({ code: z.ZodIssueCode.custom, path: [issue.path], message: issue.message });
|
|
187
|
+
}).transform((v) => ({ ...v, prefix: v.prefix ?? DEFAULT_NAMESPACE[v.kind] }));
|
|
188
|
+
var updateConnectionSchema = connectionFields.omit({ kind: true }).partial();
|
|
153
189
|
var reorderSchema = z.object({
|
|
154
190
|
ids: z.array(z.string().min(1)).min(1)
|
|
155
191
|
});
|
|
156
192
|
var testConnectionSchema = z.object({
|
|
193
|
+
kind: z.enum(CONNECTION_KINDS).default("redis"),
|
|
157
194
|
url: z.string().min(1),
|
|
158
195
|
prefix: z.string().optional(),
|
|
159
196
|
cluster: z.boolean().optional()
|
|
197
|
+
}).superRefine((v, ctx) => {
|
|
198
|
+
for (const issue of connectionIssues(v)) ctx.addIssue({ code: z.ZodIssueCode.custom, path: [issue.path], message: issue.message });
|
|
160
199
|
});
|
|
161
200
|
var JOB_STATES = [
|
|
162
201
|
"waiting",
|
|
@@ -427,34 +466,1317 @@ var EMPTY_COUNTS = {
|
|
|
427
466
|
paused: 0,
|
|
428
467
|
"waiting-children": 0
|
|
429
468
|
};
|
|
430
|
-
var ROLE_RANK = { admin: 3, operator: 2, viewer: 1 };
|
|
431
|
-
function hasRole(userRole, required) {
|
|
432
|
-
return ROLE_RANK[userRole] >= ROLE_RANK[required];
|
|
433
|
-
}
|
|
434
|
-
function redactRedisUrl(url) {
|
|
435
|
-
try {
|
|
436
|
-
const u = new URL(url);
|
|
437
|
-
if (u.password) u.password = "****";
|
|
438
|
-
return u.toString();
|
|
439
|
-
} catch {
|
|
440
|
-
return url.replace(/:\/\/([^:@/]*):([^@/]*)@/, "://$1:****@");
|
|
469
|
+
var ROLE_RANK = { admin: 3, operator: 2, viewer: 1 };
|
|
470
|
+
function hasRole(userRole, required) {
|
|
471
|
+
return ROLE_RANK[userRole] >= ROLE_RANK[required];
|
|
472
|
+
}
|
|
473
|
+
function redactRedisUrl(url) {
|
|
474
|
+
try {
|
|
475
|
+
const u = new URL(url);
|
|
476
|
+
if (u.password) u.password = "****";
|
|
477
|
+
return u.toString();
|
|
478
|
+
} catch {
|
|
479
|
+
return url.replace(/:\/\/([^:@/]*):([^@/]*)@/, "://$1:****@");
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
var MCP_ACCESS_LEVELS = ["off", "read", "write"];
|
|
483
|
+
var MCP_ACCESS_RANK = { off: 0, read: 1, write: 2 };
|
|
484
|
+
function mcpRoleCeiling(role) {
|
|
485
|
+
return hasRole(role, "operator") ? "write" : "read";
|
|
486
|
+
}
|
|
487
|
+
function mcpEffectiveAccess(ceiling, granted, role) {
|
|
488
|
+
const levels = [ceiling, granted, mcpRoleCeiling(role)];
|
|
489
|
+
return levels.reduce((min, l) => MCP_ACCESS_RANK[l] < MCP_ACCESS_RANK[min] ? l : min);
|
|
490
|
+
}
|
|
491
|
+
var updateMcpSettingsSchema = z.object({ maxAccess: z.enum(MCP_ACCESS_LEVELS) });
|
|
492
|
+
var mcpConsentDecisionSchema = z.object({
|
|
493
|
+
request: z.string().min(1).max(4096),
|
|
494
|
+
approve: z.boolean(),
|
|
495
|
+
access: z.enum(["read", "write"]).default("read")
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
// packages/inspector/src/glob.ts
|
|
499
|
+
function globToRegExp(glob) {
|
|
500
|
+
const escaped = glob.split("").map((ch) => {
|
|
501
|
+
if (ch === "*") return ".*";
|
|
502
|
+
if (ch === "?") return ".";
|
|
503
|
+
return ch.replace(/[.+^${}()|[\]\\/]/g, "\\$&");
|
|
504
|
+
}).join("");
|
|
505
|
+
return new RegExp(`^${escaped}$`);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// packages/pg-inspector/src/rows.ts
|
|
509
|
+
function num(v, fallback = 0) {
|
|
510
|
+
if (typeof v === "number") return Number.isFinite(v) ? v : fallback;
|
|
511
|
+
if (typeof v === "string" && v !== "") {
|
|
512
|
+
const n = Number(v);
|
|
513
|
+
return Number.isFinite(n) ? n : fallback;
|
|
514
|
+
}
|
|
515
|
+
return fallback;
|
|
516
|
+
}
|
|
517
|
+
function numOrNull(v) {
|
|
518
|
+
if (v === null || v === void 0 || v === "") return null;
|
|
519
|
+
const n = num(v, Number.NaN);
|
|
520
|
+
return Number.isNaN(n) ? null : n;
|
|
521
|
+
}
|
|
522
|
+
function queueKeyOf(schema, queue) {
|
|
523
|
+
return `${schema}:${queue}`;
|
|
524
|
+
}
|
|
525
|
+
function jobKeyOf(schema, queue, id3) {
|
|
526
|
+
return `${schema}:${queue}:${id3}`;
|
|
527
|
+
}
|
|
528
|
+
function parentRef(schema, parentQueue, parentId) {
|
|
529
|
+
if (typeof parentQueue !== "string" || parentQueue === "" || parentId === null || parentId === void 0) return null;
|
|
530
|
+
return { id: String(parentId), queueKey: queueKeyOf(schema, parentQueue), queue: parentQueue };
|
|
531
|
+
}
|
|
532
|
+
function bullpaneState(physical, priority) {
|
|
533
|
+
switch (physical) {
|
|
534
|
+
case "waiting":
|
|
535
|
+
return num(priority) > 0 ? "prioritized" : "waiting";
|
|
536
|
+
case "active":
|
|
537
|
+
case "completed":
|
|
538
|
+
case "failed":
|
|
539
|
+
case "delayed":
|
|
540
|
+
case "waiting-children":
|
|
541
|
+
return physical;
|
|
542
|
+
default:
|
|
543
|
+
return "unknown";
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
function progressOf(v) {
|
|
547
|
+
if (v === null || v === void 0) return 0;
|
|
548
|
+
if (typeof v === "number" || typeof v === "string") return v;
|
|
549
|
+
if (typeof v === "object" && !Array.isArray(v)) return v;
|
|
550
|
+
return JSON.stringify(v);
|
|
551
|
+
}
|
|
552
|
+
function attemptsOf(v) {
|
|
553
|
+
return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : null;
|
|
554
|
+
}
|
|
555
|
+
function rowToSummary(schema, row, previewBytes) {
|
|
556
|
+
const state = bullpaneState(row.state, row.priority);
|
|
557
|
+
const read = row.data_preview !== null && row.data_preview !== void 0;
|
|
558
|
+
const bytes = read ? num(row.data_bytes) : num(row.data_stored);
|
|
559
|
+
return {
|
|
560
|
+
id: String(row.id),
|
|
561
|
+
name: row.name ?? "",
|
|
562
|
+
timestamp: num(row.added_at_ms),
|
|
563
|
+
processedOn: numOrNull(row.processed_at_ms),
|
|
564
|
+
finishedOn: numOrNull(row.finished_at_ms),
|
|
565
|
+
attemptsMade: num(row.attempts_made),
|
|
566
|
+
attempts: attemptsOf(row.opt_attempts),
|
|
567
|
+
failedReason: row.failed_reason ?? null,
|
|
568
|
+
progress: progressOf(row.progress),
|
|
569
|
+
delay: num(row.delay_ms),
|
|
570
|
+
// Postgres stores the due time itself (process_at_ms): no score decoding.
|
|
571
|
+
delayedUntil: state === "delayed" ? numOrNull(row.process_at_ms) : null,
|
|
572
|
+
priority: num(row.priority),
|
|
573
|
+
dataPreview: read ? row.data_preview : "",
|
|
574
|
+
// Above the list cap the payload was not read at all; `dataBytes` is then the
|
|
575
|
+
// stored (possibly compressed) size, which is what pg_column_size can tell
|
|
576
|
+
// without detoasting it.
|
|
577
|
+
dataTruncated: !read || bytes > previewBytes,
|
|
578
|
+
dataBytes: bytes,
|
|
579
|
+
parent: parentRef(schema, row.parent_queue, row.parent_id),
|
|
580
|
+
groupId: row.group_id ?? null,
|
|
581
|
+
repeatJobKey: row.scheduler_id ?? null,
|
|
582
|
+
stalledCounter: num(row.stalled_count),
|
|
583
|
+
state
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
function rowToDetail(schema, row) {
|
|
587
|
+
const state = bullpaneState(row.state, row.priority);
|
|
588
|
+
const opts = row.opts && typeof row.opts === "object" ? row.opts : {};
|
|
589
|
+
const group = opts.group;
|
|
590
|
+
const processed = num(row.processed);
|
|
591
|
+
const unprocessed = num(row.unprocessed);
|
|
592
|
+
return {
|
|
593
|
+
id: String(row.id),
|
|
594
|
+
name: row.name ?? "",
|
|
595
|
+
timestamp: num(row.added_at_ms),
|
|
596
|
+
processedOn: numOrNull(row.processed_at_ms),
|
|
597
|
+
finishedOn: numOrNull(row.finished_at_ms),
|
|
598
|
+
attemptsMade: num(row.attempts_made),
|
|
599
|
+
attempts: attemptsOf(opts.attempts),
|
|
600
|
+
failedReason: row.failed_reason ?? null,
|
|
601
|
+
progress: progressOf(row.progress),
|
|
602
|
+
delay: num(row.delay_ms),
|
|
603
|
+
delayedUntil: state === "delayed" ? numOrNull(row.process_at_ms) : null,
|
|
604
|
+
priority: num(row.priority),
|
|
605
|
+
dataBytes: numOrNull(row.data_bytes),
|
|
606
|
+
parent: parentRef(schema, row.parent_queue, row.parent_id),
|
|
607
|
+
groupId: group && (typeof group.id === "string" || typeof group.id === "number") ? String(group.id) : null,
|
|
608
|
+
repeatJobKey: row.scheduler_id ?? null,
|
|
609
|
+
stalledCounter: num(row.stalled_count),
|
|
610
|
+
state,
|
|
611
|
+
data: row.data ?? null,
|
|
612
|
+
opts,
|
|
613
|
+
returnvalue: row.return_value ?? null,
|
|
614
|
+
stacktrace: stacktraceOf(row.stacktrace),
|
|
615
|
+
logs: row.logs ?? [],
|
|
616
|
+
logsCount: num(row.logs_count),
|
|
617
|
+
dependencies: processed + unprocessed > 0 ? { processed, unprocessed } : null
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
function stacktraceOf(v) {
|
|
621
|
+
if (v === null || v === void 0) return [];
|
|
622
|
+
if (Array.isArray(v)) return v.map((s) => typeof s === "string" ? s : JSON.stringify(s));
|
|
623
|
+
return [typeof v === "string" ? v : JSON.stringify(v)];
|
|
624
|
+
}
|
|
625
|
+
function rowToScheduler(row) {
|
|
626
|
+
const key = String(row.scheduler_id);
|
|
627
|
+
const name = row.name ?? key;
|
|
628
|
+
return {
|
|
629
|
+
key,
|
|
630
|
+
name,
|
|
631
|
+
next: numOrNull(row.next_run_ms),
|
|
632
|
+
pattern: row.pattern ?? null,
|
|
633
|
+
every: numOrNull(row.every_ms),
|
|
634
|
+
tz: row.tz ?? null,
|
|
635
|
+
offset: numOrNull(row.offset_ms),
|
|
636
|
+
limit: numOrNull(row.limit_count),
|
|
637
|
+
iterationCount: numOrNull(row.iteration_count),
|
|
638
|
+
startDate: numOrNull(row.start_date_ms),
|
|
639
|
+
endDate: numOrNull(row.end_date_ms),
|
|
640
|
+
template: row.template_data !== null || row.template_opts !== null ? { name, data: row.template_data ?? null, opts: row.template_opts ?? null } : null
|
|
641
|
+
};
|
|
642
|
+
}
|
|
643
|
+
function prunesCompleted(v) {
|
|
644
|
+
if (v === void 0 || v === null || v === false) return false;
|
|
645
|
+
if (v === true || typeof v === "number") return true;
|
|
646
|
+
if (typeof v === "object") {
|
|
647
|
+
const o = v;
|
|
648
|
+
return typeof o.count === "number" || typeof o.age === "number";
|
|
649
|
+
}
|
|
650
|
+
return false;
|
|
651
|
+
}
|
|
652
|
+
function metricPoints(newestFirst) {
|
|
653
|
+
if (!Array.isArray(newestFirst)) return [];
|
|
654
|
+
return newestFirst.map((v) => num(v)).reverse();
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
// packages/pg-inspector/src/metrics.ts
|
|
658
|
+
function side(row, nowMin) {
|
|
659
|
+
if (!row) {
|
|
660
|
+
return { has: false, read: (w) => ({ total: 0, covered: w }) };
|
|
661
|
+
}
|
|
662
|
+
const count2 = num(row.count);
|
|
663
|
+
const prevTs = numOrNull(row.prevTs);
|
|
664
|
+
if (prevTs === null) return { has: true, read: (w) => ({ total: count2, covered: w }) };
|
|
665
|
+
const m0 = Math.floor(prevTs / 6e4);
|
|
666
|
+
const ref = Math.max(nowMin, m0);
|
|
667
|
+
const pending = count2 - num(row.prevCount);
|
|
668
|
+
const points = (row.data ?? []).map((v) => num(v));
|
|
669
|
+
return {
|
|
670
|
+
has: true,
|
|
671
|
+
read: (w) => {
|
|
672
|
+
const startMin = ref - w + 1;
|
|
673
|
+
if (m0 < startMin) return { total: 0, covered: w };
|
|
674
|
+
const n = Math.min(m0 - startMin, points.length);
|
|
675
|
+
let total = pending;
|
|
676
|
+
for (let i = 0; i < n; i += 1) total += points[i] ?? 0;
|
|
677
|
+
return { total, covered: Math.min(ref - m0 + 1 + n, w) };
|
|
678
|
+
}
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
function percentile(sorted, q) {
|
|
682
|
+
const idx = Math.max(1, Math.ceil(q / 100 * sorted.length));
|
|
683
|
+
return sorted[idx - 1] ?? -1;
|
|
684
|
+
}
|
|
685
|
+
function windowMetricsFrom(request, rows, durations, now) {
|
|
686
|
+
const nowMin = Math.floor(now / 6e4);
|
|
687
|
+
const completed = side(rows?.find((r) => r.kind === "completed"), nowMin);
|
|
688
|
+
const failed = side(rows?.find((r) => r.kind === "failed"), nowMin);
|
|
689
|
+
const sampled = (durations ?? []).map(([f, d]) => [num(f), num(d)]);
|
|
690
|
+
return {
|
|
691
|
+
hasMetrics: completed.has || failed.has,
|
|
692
|
+
rates: request.rateWindows.map((windowMinutes) => {
|
|
693
|
+
const c = completed.read(windowMinutes);
|
|
694
|
+
const f = failed.read(windowMinutes);
|
|
695
|
+
return { windowMinutes, completed: c.total, failed: f.total, coveredMinutes: Math.min(c.covered, f.covered) };
|
|
696
|
+
}),
|
|
697
|
+
durations: request.durationWindows.map((windowMinutes) => {
|
|
698
|
+
const since = now - windowMinutes * 6e4;
|
|
699
|
+
const ds = sampled.filter(([f]) => f >= since).map(([, d]) => d).sort((a, b) => a - b);
|
|
700
|
+
return {
|
|
701
|
+
windowMinutes,
|
|
702
|
+
sampled: ds.length,
|
|
703
|
+
p50Ms: ds.length > 0 ? percentile(ds, 50) : null,
|
|
704
|
+
p95Ms: ds.length > 0 ? percentile(ds, 95) : null
|
|
705
|
+
};
|
|
706
|
+
}),
|
|
707
|
+
collectedAt: now
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
function ratesFrom(input) {
|
|
711
|
+
const pct = (c, f) => c + f === 0 ? null : Math.round(c / (c + f) * 1e3) / 10;
|
|
712
|
+
if (input.completed || input.failed) {
|
|
713
|
+
const nowMin = Math.floor(input.now / 6e4);
|
|
714
|
+
const c = side(input.completed ?? void 0, nowMin).read(input.windowMinutes);
|
|
715
|
+
const f = side(input.failed ?? void 0, nowMin).read(input.windowMinutes);
|
|
716
|
+
return {
|
|
717
|
+
windowMinutes: Math.min(c.covered, f.covered),
|
|
718
|
+
completed: c.total,
|
|
719
|
+
failed: f.total,
|
|
720
|
+
successPct: pct(c.total, f.total),
|
|
721
|
+
source: "metrics",
|
|
722
|
+
retentionSkewed: false
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
return {
|
|
726
|
+
windowMinutes: input.windowMinutes,
|
|
727
|
+
completed: input.storedCompleted,
|
|
728
|
+
failed: input.storedFailed,
|
|
729
|
+
successPct: pct(input.storedCompleted, input.storedFailed),
|
|
730
|
+
// QueueRates calls this source "zset" (the Redis name); here it is a COUNT of stored rows.
|
|
731
|
+
source: "zset",
|
|
732
|
+
retentionSkewed: input.prunesCompleted && input.storedFailed > 0
|
|
733
|
+
};
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// packages/pg-inspector/src/sql.ts
|
|
737
|
+
var STATE_SQL = {
|
|
738
|
+
waiting: { where: "state = 'waiting' AND priority = 0", order: ["seq"] },
|
|
739
|
+
// job_ready_idx
|
|
740
|
+
prioritized: { where: "state = 'waiting' AND priority > 0", order: ["priority", "seq"] },
|
|
741
|
+
// job_ready_idx
|
|
742
|
+
active: { where: "state = 'active'", order: ["seq"] },
|
|
743
|
+
// small by nature (bounded by worker concurrency)
|
|
744
|
+
completed: { where: "state = 'completed'", order: ["finished_at_ms"] },
|
|
745
|
+
// job_finished_idx
|
|
746
|
+
failed: { where: "state = 'failed'", order: ["finished_at_ms"] },
|
|
747
|
+
// job_finished_idx
|
|
748
|
+
delayed: { where: "state = 'delayed'", order: ["process_at_ms"] },
|
|
749
|
+
// job_delayed_idx
|
|
750
|
+
"waiting-children": { where: "state = 'waiting-children'", order: ["seq"] },
|
|
751
|
+
// job_waiting_children_idx
|
|
752
|
+
paused: null
|
|
753
|
+
};
|
|
754
|
+
function orderBy(state, order) {
|
|
755
|
+
const s = STATE_SQL[state];
|
|
756
|
+
if (!s) throw new Error(`no physical state for ${state}`);
|
|
757
|
+
const dir = order === "asc" ? "ASC" : "DESC";
|
|
758
|
+
return s.order.map((c) => `${c} ${dir}`).join(", ");
|
|
759
|
+
}
|
|
760
|
+
function summaryColumns(alias, previewParam, capParam) {
|
|
761
|
+
const j = alias;
|
|
762
|
+
return `${j}.id, ${j}.name, ${j}.state::text AS state, ${j}.priority, ${j}.delay_ms, ${j}.attempts_made,
|
|
763
|
+
${j}.added_at_ms, ${j}.processed_at_ms, ${j}.finished_at_ms, ${j}.process_at_ms, ${j}.failed_reason,
|
|
764
|
+
${j}.progress, ${j}.opts->'attempts' AS opt_attempts, ${j}.opts #>> '{group,id}' AS group_id,
|
|
765
|
+
${j}.scheduler_id, ${j}.stalled_count, ${j}.parent_queue, ${j}.parent_id,
|
|
766
|
+
pg_column_size(${j}.data) AS data_stored,
|
|
767
|
+
CASE WHEN pg_column_size(${j}.data) <= $${capParam} THEN left(${j}.data::text, $${previewParam}) END AS data_preview,
|
|
768
|
+
CASE WHEN pg_column_size(${j}.data) <= $${capParam} THEN octet_length(${j}.data::text) END AS data_bytes`;
|
|
769
|
+
}
|
|
770
|
+
var count = (state, where) => `CASE WHEN '${state}' = ANY(string_to_array(q.skip, ',')) THEN NULL ELSE (SELECT count(*) FROM job WHERE queue = q.queue AND ${where}) END`;
|
|
771
|
+
var QUEUE_STATS = `
|
|
772
|
+
SELECT q.queue,
|
|
773
|
+
${count("waiting", "state = 'waiting' AND priority = 0")} AS waiting,
|
|
774
|
+
${count("prioritized", "state = 'waiting' AND priority > 0")} AS prioritized,
|
|
775
|
+
${count("active", "state = 'active'")} AS active,
|
|
776
|
+
${count("completed", "state = 'completed'")} AS completed,
|
|
777
|
+
${count("failed", "state = 'failed'")} AS failed,
|
|
778
|
+
${count("delayed", "state = 'delayed'")} AS delayed,
|
|
779
|
+
${count("waiting-children", "state = 'waiting-children'")} AS waiting_children,
|
|
780
|
+
-- An active job whose lock expired: the worker stopped renewing it. This is
|
|
781
|
+
-- what Redis' stalled SET means, read straight off job_active_idx.
|
|
782
|
+
(SELECT count(*) FROM job WHERE queue = q.queue AND state = 'active' AND locked_until_ms < $2) AS stalled,
|
|
783
|
+
EXISTS (SELECT 1 FROM meta WHERE queue = q.queue AND field = 'paused') AS paused,
|
|
784
|
+
(SELECT value FROM meta WHERE queue = q.queue AND field = 'version') AS version,
|
|
785
|
+
(SELECT count(*) FROM scheduler WHERE queue = q.queue) AS schedulers,
|
|
786
|
+
-- the whole metrics row per side: count, prev_ts and prev_count carry the
|
|
787
|
+
-- current minute, which is not in data until the minute rolls over
|
|
788
|
+
(SELECT json_build_object('kind', kind, 'count', count, 'prevTs', prev_ts, 'prevCount', prev_count, 'data', data[1:$3])
|
|
789
|
+
FROM metrics WHERE queue = q.queue AND kind = 'completed') AS m_completed,
|
|
790
|
+
(SELECT json_build_object('kind', kind, 'count', count, 'prevTs', prev_ts, 'prevCount', prev_count, 'data', data[1:$3])
|
|
791
|
+
FROM metrics WHERE queue = q.queue AND kind = 'failed') AS m_failed,
|
|
792
|
+
(SELECT data[1:$3] FROM metrics WHERE queue = q.queue AND kind = 'completed') AS m_completed_data,
|
|
793
|
+
(SELECT data[1:$3] FROM metrics WHERE queue = q.queue AND kind = 'failed') AS m_failed_data,
|
|
794
|
+
-- Only read when the queue has no metrics (the rate then falls back to stored
|
|
795
|
+
-- rows); a busy queue with metrics would otherwise count an hour of jobs per read.
|
|
796
|
+
CASE WHEN EXISTS (SELECT 1 FROM metrics WHERE queue = q.queue) THEN NULL
|
|
797
|
+
ELSE (SELECT count(*) FROM job WHERE queue = q.queue AND state = 'completed' AND finished_at_ms >= $4) END AS w_completed,
|
|
798
|
+
CASE WHEN EXISTS (SELECT 1 FROM metrics WHERE queue = q.queue) THEN NULL
|
|
799
|
+
ELSE (SELECT count(*) FROM job WHERE queue = q.queue AND state = 'failed' AND finished_at_ms >= $4) END AS w_failed,
|
|
800
|
+
-- removeOnComplete lives in each job's opts; one job tells whether the queue prunes.
|
|
801
|
+
(SELECT opts->'removeOnComplete' FROM job WHERE queue = q.queue AND state = 'completed'
|
|
802
|
+
ORDER BY finished_at_ms DESC LIMIT 1) AS remove_on_complete
|
|
803
|
+
FROM unnest($1::text[], $5::text[]) AS q(queue, skip)`;
|
|
804
|
+
var DISCOVER_QUEUES = `
|
|
805
|
+
WITH RECURSIVE jq AS (
|
|
806
|
+
(SELECT queue FROM job ORDER BY queue LIMIT 1)
|
|
807
|
+
UNION ALL
|
|
808
|
+
SELECT (SELECT j.queue FROM job j WHERE j.queue > jq.queue ORDER BY j.queue LIMIT 1)
|
|
809
|
+
FROM jq WHERE jq.queue IS NOT NULL
|
|
810
|
+
)
|
|
811
|
+
SELECT queue FROM jq WHERE queue IS NOT NULL
|
|
812
|
+
UNION
|
|
813
|
+
SELECT DISTINCT queue FROM meta
|
|
814
|
+
ORDER BY queue`;
|
|
815
|
+
var PING = `SELECT current_setting('server_version') AS version`;
|
|
816
|
+
var SERVER_INFO = `
|
|
817
|
+
SELECT current_setting('server_version') AS version,
|
|
818
|
+
extract(epoch FROM now() - pg_postmaster_start_time())::bigint AS uptime,
|
|
819
|
+
(SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()) AS clients,
|
|
820
|
+
current_setting('max_connections')::int AS max_connections,
|
|
821
|
+
pg_database_size(current_database()) AS db_bytes,
|
|
822
|
+
COALESCE(pg_total_relation_size(to_regclass('job')), 0) AS job_bytes,
|
|
823
|
+
COALESCE(pg_total_relation_size(to_regclass('event')), 0) AS event_bytes,
|
|
824
|
+
d.xact_commit + d.xact_rollback AS transactions,
|
|
825
|
+
d.blks_hit, d.blks_read, d.deadlocks
|
|
826
|
+
FROM pg_stat_database d
|
|
827
|
+
WHERE d.datname = current_database()`;
|
|
828
|
+
var WINDOW_COUNTS = `
|
|
829
|
+
SELECT
|
|
830
|
+
(SELECT count(*) FROM job WHERE queue = $1 AND state = 'completed' AND finished_at_ms >= $2) AS completed,
|
|
831
|
+
(SELECT count(*) FROM job WHERE queue = $1 AND state = 'failed' AND finished_at_ms >= $2) AS failed`;
|
|
832
|
+
var METRICS_COUNTERS = `SELECT kind, count FROM metrics WHERE queue = $1`;
|
|
833
|
+
var METRICS = `SELECT kind, data[1:$2] AS data FROM metrics WHERE queue = $1`;
|
|
834
|
+
var WINDOW_METRICS = `
|
|
835
|
+
SELECT q.queue,
|
|
836
|
+
(SELECT json_agg(json_build_object('kind', m.kind, 'count', m.count, 'prevTs', m.prev_ts,
|
|
837
|
+
'prevCount', m.prev_count, 'data', m.data[1:$3]))
|
|
838
|
+
FROM metrics m WHERE m.queue = q.queue) AS metrics,
|
|
839
|
+
(SELECT json_agg(json_build_array(s.finished_at_ms, s.finished_at_ms - s.processed_at_ms))
|
|
840
|
+
FROM (SELECT finished_at_ms, processed_at_ms FROM job
|
|
841
|
+
WHERE queue = q.queue AND state = 'completed' AND finished_at_ms >= q.since
|
|
842
|
+
ORDER BY finished_at_ms DESC LIMIT $4) s
|
|
843
|
+
WHERE s.processed_at_ms IS NOT NULL AND s.finished_at_ms >= s.processed_at_ms) AS durations
|
|
844
|
+
FROM unnest($1::text[], $2::bigint[]) AS q(queue, since)`;
|
|
845
|
+
var JOB_DETAIL = `
|
|
846
|
+
SELECT j.id, j.name, j.state::text AS state, j.priority, j.delay_ms, j.attempts_made,
|
|
847
|
+
j.added_at_ms, j.processed_at_ms, j.finished_at_ms, j.process_at_ms, j.failed_reason,
|
|
848
|
+
j.progress, j.data, j.opts, j.return_value, j.stacktrace, j.scheduler_id, j.stalled_count,
|
|
849
|
+
j.parent_queue, j.parent_id, octet_length(j.data::text) AS data_bytes,
|
|
850
|
+
(SELECT array_agg(t.row ORDER BY t.idx)
|
|
851
|
+
FROM (SELECT idx, row FROM job_log WHERE queue = $1 AND job_id = $2 ORDER BY idx DESC LIMIT $3) t) AS logs,
|
|
852
|
+
(SELECT count(*) FROM job_log WHERE queue = $1 AND job_id = $2) AS logs_count,
|
|
853
|
+
(SELECT count(*) FILTER (WHERE status = 'pending') FROM job_dependency
|
|
854
|
+
WHERE parent_queue = $1 AND parent_id = $2) AS unprocessed,
|
|
855
|
+
(SELECT count(*) FILTER (WHERE status <> 'pending') FROM job_dependency
|
|
856
|
+
WHERE parent_queue = $1 AND parent_id = $2) AS processed,
|
|
857
|
+
EXISTS (SELECT 1 FROM meta WHERE queue = $1 AND field = 'paused') AS queue_paused
|
|
858
|
+
FROM job j
|
|
859
|
+
WHERE j.queue = $1 AND j.id = $2`;
|
|
860
|
+
var JOB_LOGS = `
|
|
861
|
+
SELECT
|
|
862
|
+
ARRAY(SELECT row FROM job_log WHERE queue = $1 AND job_id = $2 ORDER BY idx OFFSET $3 LIMIT $4) AS logs,
|
|
863
|
+
(SELECT count(*) FROM job_log WHERE queue = $1 AND job_id = $2) AS count`;
|
|
864
|
+
var QUEUE_SETUP = `
|
|
865
|
+
SELECT
|
|
866
|
+
(SELECT json_object_agg(field, value) FROM meta WHERE queue = $1) AS meta,
|
|
867
|
+
(SELECT expire_at_ms FROM rate_limit WHERE queue = $1) AS limiter_expire_at,
|
|
868
|
+
EXISTS (SELECT 1 FROM metrics WHERE queue = $1 AND kind = 'completed') AS metrics_enabled`;
|
|
869
|
+
var WORKERS = `
|
|
870
|
+
SELECT application_name FROM pg_stat_activity
|
|
871
|
+
WHERE datname = current_database() AND (application_name = $1 OR application_name LIKE $2)`;
|
|
872
|
+
var SCHEDULERS = `
|
|
873
|
+
SELECT
|
|
874
|
+
(SELECT count(*) FROM scheduler WHERE queue = $1) AS total,
|
|
875
|
+
(SELECT json_agg(row_to_json(s)) FROM (
|
|
876
|
+
SELECT scheduler_id, name, next_run_ms, pattern, every_ms, tz, offset_ms, limit_count,
|
|
877
|
+
iteration_count, start_date_ms, end_date_ms,
|
|
878
|
+
left(template_data::text, $4) AS template_data, left(template_opts::text, $4) AS template_opts
|
|
879
|
+
FROM scheduler WHERE queue = $1
|
|
880
|
+
ORDER BY next_run_ms ASC NULLS LAST, scheduler_id
|
|
881
|
+
OFFSET $2 LIMIT $3) s) AS rows`;
|
|
882
|
+
var SAMPLE_PARENTS = `
|
|
883
|
+
WITH sampled AS (
|
|
884
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'completed' ORDER BY finished_at_ms DESC LIMIT $2)
|
|
885
|
+
UNION ALL
|
|
886
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'failed' ORDER BY finished_at_ms DESC LIMIT $2)
|
|
887
|
+
UNION ALL
|
|
888
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'waiting-children' ORDER BY seq DESC LIMIT $2)
|
|
889
|
+
UNION ALL
|
|
890
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'waiting' AND priority = 0 ORDER BY seq DESC LIMIT $2)
|
|
891
|
+
UNION ALL
|
|
892
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'active' ORDER BY seq DESC LIMIT $2)
|
|
893
|
+
UNION ALL
|
|
894
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'delayed' ORDER BY process_at_ms DESC LIMIT $2)
|
|
895
|
+
)
|
|
896
|
+
SELECT
|
|
897
|
+
(SELECT count(*) FROM sampled) AS sampled,
|
|
898
|
+
(SELECT json_agg(json_build_array(parent_queue, n)) FROM (
|
|
899
|
+
SELECT parent_queue, count(*) AS n FROM sampled WHERE parent_queue IS NOT NULL GROUP BY parent_queue
|
|
900
|
+
) g) AS edges`;
|
|
901
|
+
var TREE_NODES = `
|
|
902
|
+
SELECT r.ord, j.id IS NOT NULL AS found, j.name, j.state::text AS state, j.priority, j.added_at_ms,
|
|
903
|
+
j.finished_at_ms, j.attempts_made, j.failed_reason, j.progress, j.parent_queue, j.parent_id,
|
|
904
|
+
(SELECT count(*) FILTER (WHERE status = 'pending') FROM job_dependency
|
|
905
|
+
WHERE parent_queue = r.queue AND parent_id = r.id) AS unprocessed,
|
|
906
|
+
(SELECT count(*) FILTER (WHERE status <> 'pending') FROM job_dependency
|
|
907
|
+
WHERE parent_queue = r.queue AND parent_id = r.id) AS processed,
|
|
908
|
+
(SELECT json_agg(json_build_array(c.child_queue, c.child_id)) FROM (
|
|
909
|
+
SELECT child_queue, child_id FROM job_dependency
|
|
910
|
+
WHERE parent_queue = r.queue AND parent_id = r.id
|
|
911
|
+
ORDER BY child_key LIMIT $3 + 1) c) AS children
|
|
912
|
+
FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS r(queue, id, ord)
|
|
913
|
+
LEFT JOIN job j ON j.queue = r.queue AND j.id = r.id
|
|
914
|
+
ORDER BY r.ord`;
|
|
915
|
+
|
|
916
|
+
// packages/pg-inspector/src/connection.ts
|
|
917
|
+
var LIBPQ_ALIASED_MODES = /* @__PURE__ */ new Set(["prefer", "require", "verify-ca"]);
|
|
918
|
+
function nodePgConnectionString(url) {
|
|
919
|
+
let parsed;
|
|
920
|
+
try {
|
|
921
|
+
parsed = new URL(url);
|
|
922
|
+
} catch {
|
|
923
|
+
return url;
|
|
924
|
+
}
|
|
925
|
+
const mode = parsed.searchParams.get("sslmode");
|
|
926
|
+
if (!mode || !LIBPQ_ALIASED_MODES.has(mode) || parsed.searchParams.has("uselibpqcompat")) return url;
|
|
927
|
+
parsed.searchParams.set("uselibpqcompat", "true");
|
|
928
|
+
return parsed.toString();
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
// packages/pg-inspector/src/inspector.ts
|
|
932
|
+
var DEFAULTS = {
|
|
933
|
+
discoveryTtlMs: 3e4,
|
|
934
|
+
previewBytes: 2048,
|
|
935
|
+
maxScanPerCall: 1e3,
|
|
936
|
+
connectTimeoutMs: 5e3,
|
|
937
|
+
listFieldCapBytes: 32 * 1024,
|
|
938
|
+
searchFieldCapBytes: 256 * 1024,
|
|
939
|
+
searchByteBudget: 8 * 1024 * 1024
|
|
940
|
+
};
|
|
941
|
+
var READ_POOL_MAX = 4;
|
|
942
|
+
var STATEMENT_TIMEOUT_MS = 1e4;
|
|
943
|
+
var SESSION_SETTINGS = [
|
|
944
|
+
["statement_timeout", STATEMENT_TIMEOUT_MS],
|
|
945
|
+
["max_parallel_workers_per_gather", 0]
|
|
946
|
+
];
|
|
947
|
+
var APPLICATION_NAME = "bullpane-dashboard";
|
|
948
|
+
var STATS_METRIC_POINTS = 60;
|
|
949
|
+
var DEFAULT_RATE_WINDOW_MINUTES = 60;
|
|
950
|
+
var SETUP_CACHE_MS = 1e4;
|
|
951
|
+
var STATS_CACHE_MS = 2e3;
|
|
952
|
+
var LARGE_COUNT = 1e5;
|
|
953
|
+
var LARGE_COUNT_MAX_TTL_MS = 6e4;
|
|
954
|
+
var DETAIL_LOG_TAIL = 100;
|
|
955
|
+
var BULK_CONCURRENCY = 8;
|
|
956
|
+
var FORCE_DISCOVERY_MIN_MS = 5e3;
|
|
957
|
+
var MAX_TREE_CLIMB = 50;
|
|
958
|
+
function quoteIdent(name) {
|
|
959
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
960
|
+
}
|
|
961
|
+
function likeSubstring(query) {
|
|
962
|
+
return `%${query.replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
|
|
963
|
+
}
|
|
964
|
+
function errorMessage(err) {
|
|
965
|
+
return err instanceof Error ? err.message : String(err);
|
|
966
|
+
}
|
|
967
|
+
var PgInspector = class {
|
|
968
|
+
config;
|
|
969
|
+
opts;
|
|
970
|
+
pool;
|
|
971
|
+
/**
|
|
972
|
+
* How reads carry their settings (schema, statement timeout, no parallel query):
|
|
973
|
+
* - "session": as startup options on each connection. One round trip per
|
|
974
|
+
* read. Used on a direct connection.
|
|
975
|
+
* - "transaction": behind a transaction pooler (PgBouncer, Supavisor, RDS
|
|
976
|
+
* Proxy...), which refuses or silently drops startup options and hands each
|
|
977
|
+
* transaction a different server connection. Each read is then its own
|
|
978
|
+
* READ ONLY transaction with SET LOCAL, so nothing depends on session state.
|
|
979
|
+
* Decided once in ensureReady, by trying the first and checking it took effect.
|
|
980
|
+
*/
|
|
981
|
+
mode = "session";
|
|
982
|
+
filter;
|
|
983
|
+
ready = null;
|
|
984
|
+
discovered = null;
|
|
985
|
+
discovering = null;
|
|
986
|
+
setupCache = /* @__PURE__ */ new Map();
|
|
987
|
+
/** per queue + rate window; see STATS_CACHE_MS */
|
|
988
|
+
statsCache = /* @__PURE__ */ new Map();
|
|
989
|
+
/** per queue + state, filled by stats reads and by job pages; see STATS_CACHE_MS */
|
|
990
|
+
countCache = /* @__PURE__ */ new Map();
|
|
991
|
+
/** bullmq Queue per queue name, created lazily for writes only. */
|
|
992
|
+
queues = /* @__PURE__ */ new Map();
|
|
993
|
+
closed = false;
|
|
994
|
+
constructor(config, options = {}) {
|
|
995
|
+
if (config.kind !== "postgres") throw new Error("PgInspector only opens postgres connections");
|
|
996
|
+
this.config = { ...config, kind: "postgres", prefix: config.prefix ?? "bullmq", cluster: false };
|
|
997
|
+
this.opts = { ...DEFAULTS };
|
|
998
|
+
for (const [k, v] of Object.entries(options)) {
|
|
999
|
+
if (v !== void 0 && k in DEFAULTS) this.opts[k] = v;
|
|
1000
|
+
}
|
|
1001
|
+
this.filter = config.queueFilter ? globToRegExp(config.queueFilter) : null;
|
|
1002
|
+
this.pool = this.createPool(true);
|
|
1003
|
+
}
|
|
1004
|
+
/**
|
|
1005
|
+
* The read pool. The settings every read runs with:
|
|
1006
|
+
* - search_path = the BullMQ schema, so its unqualified table names resolve;
|
|
1007
|
+
* - statement_timeout, so no read can hold the database for long;
|
|
1008
|
+
* - no parallel query: a count over a big state would otherwise run as a
|
|
1009
|
+
* parallel index scan on 3 cores at once, and the smoke test measured
|
|
1010
|
+
* what that does to the workers sharing the database (throughput down to
|
|
1011
|
+
* a third under a read storm). A dashboard gets one core per query, ever.
|
|
1012
|
+
* `withOptions: false` is the pooler fallback: no startup options at all.
|
|
1013
|
+
*/
|
|
1014
|
+
createPool(withOptions) {
|
|
1015
|
+
const pool = new pg.Pool({
|
|
1016
|
+
connectionString: nodePgConnectionString(this.config.url),
|
|
1017
|
+
max: READ_POOL_MAX,
|
|
1018
|
+
idleTimeoutMillis: 3e4,
|
|
1019
|
+
connectionTimeoutMillis: this.opts.connectTimeoutMs,
|
|
1020
|
+
application_name: APPLICATION_NAME,
|
|
1021
|
+
...withOptions ? { options: `-c search_path=${quoteIdent(this.config.prefix)} -c ${SESSION_SETTINGS.map(([k, v]) => `${k}=${v}`).join(" -c ")}` } : {}
|
|
1022
|
+
});
|
|
1023
|
+
pool.on("error", () => void 0);
|
|
1024
|
+
return pool;
|
|
1025
|
+
}
|
|
1026
|
+
// ---------------------------------------------------------------------------
|
|
1027
|
+
// connection
|
|
1028
|
+
// ---------------------------------------------------------------------------
|
|
1029
|
+
/**
|
|
1030
|
+
* Once per inspector: the schema exists and was migrated by a BullMQ this
|
|
1031
|
+
* version can read (BullMQ's own assertSchemaCompatibility, read only — the
|
|
1032
|
+
* dashboard never runs migrations on a customer database). Retried on the
|
|
1033
|
+
* next call after a failure.
|
|
1034
|
+
*/
|
|
1035
|
+
async ensureReady() {
|
|
1036
|
+
if (this.closed) throw new Error("inspector_closed");
|
|
1037
|
+
if (!this.ready) {
|
|
1038
|
+
this.ready = (async () => {
|
|
1039
|
+
await this.pickMode();
|
|
1040
|
+
const client = await this.pool.connect();
|
|
1041
|
+
try {
|
|
1042
|
+
await assertSchemaCompatibility(client, this.config.prefix);
|
|
1043
|
+
} catch (err) {
|
|
1044
|
+
if (err instanceof SchemaMigrationRequiredError) {
|
|
1045
|
+
throw new Error(
|
|
1046
|
+
`postgres_schema_missing: no BullMQ schema "${this.config.prefix}" in this database. Check the schema name, or start a BullMQ worker with \`migrate: true\` once.`
|
|
1047
|
+
);
|
|
1048
|
+
}
|
|
1049
|
+
throw err;
|
|
1050
|
+
} finally {
|
|
1051
|
+
client.release();
|
|
1052
|
+
}
|
|
1053
|
+
})().catch((err) => {
|
|
1054
|
+
this.ready = null;
|
|
1055
|
+
throw err;
|
|
1056
|
+
});
|
|
1057
|
+
}
|
|
1058
|
+
return this.ready;
|
|
1059
|
+
}
|
|
1060
|
+
/**
|
|
1061
|
+
* Session mode if the server takes the startup options AND they took effect;
|
|
1062
|
+
* transaction mode otherwise. A pooler can refuse them ("unsupported startup
|
|
1063
|
+
* parameter", PgBouncer ≥ 1.20 accepts search_path only) or drop them silently
|
|
1064
|
+
* (`ignore_startup_parameters`), so success alone proves nothing: the
|
|
1065
|
+
* settings are read back.
|
|
1066
|
+
*/
|
|
1067
|
+
async pickMode() {
|
|
1068
|
+
if (this.mode === "transaction") return;
|
|
1069
|
+
try {
|
|
1070
|
+
const { rows } = await this.pool.query(
|
|
1071
|
+
"SELECT current_setting('statement_timeout') AS timeout, current_setting('max_parallel_workers_per_gather') AS parallel"
|
|
1072
|
+
);
|
|
1073
|
+
const applied = rows[0]?.parallel === "0" && rows[0]?.timeout !== "0";
|
|
1074
|
+
if (!applied) this.mode = "transaction";
|
|
1075
|
+
} catch (err) {
|
|
1076
|
+
if (!/unsupported startup parameter/i.test(errorMessage(err))) throw err;
|
|
1077
|
+
const old = this.pool;
|
|
1078
|
+
this.pool = this.createPool(false);
|
|
1079
|
+
void old.end().catch(() => void 0);
|
|
1080
|
+
this.mode = "transaction";
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
async query(text3, values = []) {
|
|
1084
|
+
await this.ensureReady();
|
|
1085
|
+
if (this.mode === "session") return (await this.pool.query(text3, values)).rows;
|
|
1086
|
+
const client = await this.pool.connect();
|
|
1087
|
+
try {
|
|
1088
|
+
await client.query("BEGIN READ ONLY");
|
|
1089
|
+
await client.query(
|
|
1090
|
+
`SELECT set_config('search_path', $1, true), ${SESSION_SETTINGS.map(([k], i) => `set_config('${k}', $${i + 2}, true)`).join(", ")}`,
|
|
1091
|
+
[quoteIdent(this.config.prefix), ...SESSION_SETTINGS.map(([, v]) => String(v))]
|
|
1092
|
+
);
|
|
1093
|
+
const res = await client.query(text3, values);
|
|
1094
|
+
await client.query("COMMIT");
|
|
1095
|
+
return res.rows;
|
|
1096
|
+
} catch (err) {
|
|
1097
|
+
await client.query("ROLLBACK").catch(() => void 0);
|
|
1098
|
+
throw err;
|
|
1099
|
+
} finally {
|
|
1100
|
+
client.release();
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
/** "session" or "transaction" (behind a pooler); see `mode`. For diagnostics and tests. */
|
|
1104
|
+
async connectionMode() {
|
|
1105
|
+
await this.ensureReady();
|
|
1106
|
+
return this.mode;
|
|
1107
|
+
}
|
|
1108
|
+
async ping() {
|
|
1109
|
+
const t0 = performance.now();
|
|
1110
|
+
try {
|
|
1111
|
+
const [row] = await this.query(PING);
|
|
1112
|
+
return { ok: true, latencyMs: Math.round((performance.now() - t0) * 10) / 10, redisVersion: row?.version ?? null, error: null };
|
|
1113
|
+
} catch (err) {
|
|
1114
|
+
return { ok: false, latencyMs: Math.round((performance.now() - t0) * 10) / 10, redisVersion: null, error: errorMessage(err) };
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
async serverInfo() {
|
|
1118
|
+
const startedAt = Date.now();
|
|
1119
|
+
const [r] = await this.query(SERVER_INFO);
|
|
1120
|
+
const latencyMs = Date.now() - startedAt;
|
|
1121
|
+
const hit = numOrNull(r?.blks_hit);
|
|
1122
|
+
const read = numOrNull(r?.blks_read);
|
|
1123
|
+
const lookups = (hit ?? 0) + (read ?? 0);
|
|
1124
|
+
return {
|
|
1125
|
+
backend: "postgres",
|
|
1126
|
+
serverVersion: String(r?.version ?? "unknown"),
|
|
1127
|
+
schema: this.config.prefix,
|
|
1128
|
+
uptimeSeconds: num(r?.uptime),
|
|
1129
|
+
connectedClients: num(r?.clients),
|
|
1130
|
+
maxConnections: num(r?.max_connections),
|
|
1131
|
+
databaseSizeBytes: num(r?.db_bytes),
|
|
1132
|
+
jobTableBytes: num(r?.job_bytes),
|
|
1133
|
+
eventTableBytes: num(r?.event_bytes),
|
|
1134
|
+
totalTransactions: numOrNull(r?.transactions),
|
|
1135
|
+
cacheHitRatePct: lookups > 0 ? Math.round((hit ?? 0) / lookups * 1e3) / 10 : null,
|
|
1136
|
+
deadlocks: numOrNull(r?.deadlocks),
|
|
1137
|
+
latencyMs,
|
|
1138
|
+
sampledAt: new Date(startedAt).toISOString()
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
async close() {
|
|
1142
|
+
if (this.closed) return;
|
|
1143
|
+
this.closed = true;
|
|
1144
|
+
await Promise.allSettled([...this.queues.values()].map((q) => q.close()));
|
|
1145
|
+
this.queues.clear();
|
|
1146
|
+
await this.pool.end().catch(() => void 0);
|
|
1147
|
+
}
|
|
1148
|
+
// ---------------------------------------------------------------------------
|
|
1149
|
+
// discovery
|
|
1150
|
+
// ---------------------------------------------------------------------------
|
|
1151
|
+
/** `SELECT DISTINCT queue FROM meta`, cached for discoveryTtlMs. See sql.ts. */
|
|
1152
|
+
async discoverQueues(opts = {}) {
|
|
1153
|
+
const age = this.discovered ? Date.now() - this.discovered.at : Number.POSITIVE_INFINITY;
|
|
1154
|
+
if (this.discovered && age < (opts.force ? FORCE_DISCOVERY_MIN_MS : this.opts.discoveryTtlMs)) {
|
|
1155
|
+
return this.discovered.names;
|
|
1156
|
+
}
|
|
1157
|
+
if (!this.discovering) {
|
|
1158
|
+
this.discovering = this.query(DISCOVER_QUEUES).then((rows) => {
|
|
1159
|
+
const names = rows.map((r) => r.queue).filter((n) => !this.filter || this.filter.test(n));
|
|
1160
|
+
names.sort((a, b) => a.localeCompare(b));
|
|
1161
|
+
this.discovered = { at: Date.now(), names };
|
|
1162
|
+
return names;
|
|
1163
|
+
}).finally(() => {
|
|
1164
|
+
this.discovering = null;
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
return this.discovering;
|
|
1168
|
+
}
|
|
1169
|
+
/** One query lists every queue: discovery is complete from the first call. */
|
|
1170
|
+
async discoveryStatus() {
|
|
1171
|
+
return { complete: true, scannedIterations: 0, totalKeys: null };
|
|
1172
|
+
}
|
|
1173
|
+
// ---------------------------------------------------------------------------
|
|
1174
|
+
// reads
|
|
1175
|
+
// ---------------------------------------------------------------------------
|
|
1176
|
+
async getQueueStats(queueNames, opts = {}) {
|
|
1177
|
+
if (queueNames.length === 0) return {};
|
|
1178
|
+
const windowMinutes = opts.rateWindowMinutes ?? DEFAULT_RATE_WINDOW_MINUTES;
|
|
1179
|
+
const now = Date.now();
|
|
1180
|
+
for (const [k, v] of this.statsCache) if (now - v.at >= STATS_CACHE_MS) this.statsCache.delete(k);
|
|
1181
|
+
const key = (q) => `${windowMinutes}\0${q}`;
|
|
1182
|
+
const stale = queueNames.filter((q) => !this.statsCache.has(key(q)));
|
|
1183
|
+
if (stale.length > 0) {
|
|
1184
|
+
const read = this.readQueueStats(stale, windowMinutes);
|
|
1185
|
+
for (const q of stale) {
|
|
1186
|
+
const value = read.then((all) => all[q]);
|
|
1187
|
+
this.statsCache.set(key(q), { at: now, value });
|
|
1188
|
+
value.catch(() => this.statsCache.delete(key(q)));
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
const out = {};
|
|
1192
|
+
for (const q of queueNames) {
|
|
1193
|
+
const stats = await this.statsCache.get(key(q))?.value;
|
|
1194
|
+
if (!stats) continue;
|
|
1195
|
+
const { metrics: _metrics, ...withoutMetrics } = stats;
|
|
1196
|
+
out[q] = opts.withMetrics ? stats : withoutMetrics;
|
|
1197
|
+
}
|
|
1198
|
+
return out;
|
|
1199
|
+
}
|
|
1200
|
+
/** A count still fresh for its size (see LARGE_COUNT), from stats or from a job page. */
|
|
1201
|
+
cachedCount(queue, state) {
|
|
1202
|
+
const c = this.countCache.get(`${queue}\0${state}`);
|
|
1203
|
+
return c && Date.now() - c.at < c.ttl ? c.n : null;
|
|
1204
|
+
}
|
|
1205
|
+
rememberCount(queue, state, n) {
|
|
1206
|
+
const ttl = n < LARGE_COUNT ? STATS_CACHE_MS : Math.min(LARGE_COUNT_MAX_TTL_MS, Math.round(STATS_CACHE_MS * n / LARGE_COUNT));
|
|
1207
|
+
this.countCache.set(`${queue}\0${state}`, { at: Date.now(), n, ttl });
|
|
1208
|
+
if (this.countCache.size > 1e4) this.countCache.clear();
|
|
1209
|
+
}
|
|
1210
|
+
async readQueueStats(queueNames, windowMinutes) {
|
|
1211
|
+
const out = {};
|
|
1212
|
+
const now = Date.now();
|
|
1213
|
+
const points = Math.max(STATS_METRIC_POINTS, windowMinutes);
|
|
1214
|
+
const countable = ["waiting", "prioritized", "active", "completed", "failed", "delayed", "waiting-children"];
|
|
1215
|
+
const reuse = queueNames.map(
|
|
1216
|
+
(q) => countable.filter((state) => {
|
|
1217
|
+
const n = this.cachedCount(q, state);
|
|
1218
|
+
return n !== null && n >= LARGE_COUNT;
|
|
1219
|
+
})
|
|
1220
|
+
);
|
|
1221
|
+
const rows = await this.query(QUEUE_STATS, [
|
|
1222
|
+
queueNames,
|
|
1223
|
+
now,
|
|
1224
|
+
points,
|
|
1225
|
+
now - windowMinutes * 6e4,
|
|
1226
|
+
reuse.map((states) => states.join(","))
|
|
1227
|
+
]);
|
|
1228
|
+
const counted = (queue, state, value) => {
|
|
1229
|
+
if (value === null || value === void 0) return this.cachedCount(queue, state) ?? 0;
|
|
1230
|
+
const n = num(value);
|
|
1231
|
+
this.rememberCount(queue, state, n);
|
|
1232
|
+
return n;
|
|
1233
|
+
};
|
|
1234
|
+
for (const r of rows) {
|
|
1235
|
+
const queue = String(r.queue);
|
|
1236
|
+
const metricsCompleted = metricPoints(r.m_completed_data);
|
|
1237
|
+
const metricsFailed = metricPoints(r.m_failed_data);
|
|
1238
|
+
const version = typeof r.version === "string" && r.version !== "" ? r.version : null;
|
|
1239
|
+
const stats = {
|
|
1240
|
+
counts: {
|
|
1241
|
+
...EMPTY_COUNTS,
|
|
1242
|
+
waiting: counted(queue, "waiting", r.waiting),
|
|
1243
|
+
prioritized: counted(queue, "prioritized", r.prioritized),
|
|
1244
|
+
active: counted(queue, "active", r.active),
|
|
1245
|
+
completed: counted(queue, "completed", r.completed),
|
|
1246
|
+
failed: counted(queue, "failed", r.failed),
|
|
1247
|
+
delayed: counted(queue, "delayed", r.delayed),
|
|
1248
|
+
"waiting-children": counted(queue, "waiting-children", r.waiting_children)
|
|
1249
|
+
},
|
|
1250
|
+
isPaused: r.paused === true,
|
|
1251
|
+
isPro: version !== null && version.startsWith("bullmq-pro"),
|
|
1252
|
+
groupsCount: 0,
|
|
1253
|
+
rates: ratesFrom({
|
|
1254
|
+
windowMinutes,
|
|
1255
|
+
now,
|
|
1256
|
+
completed: r.m_completed ?? null,
|
|
1257
|
+
failed: r.m_failed ?? null,
|
|
1258
|
+
storedCompleted: num(r.w_completed),
|
|
1259
|
+
storedFailed: num(r.w_failed),
|
|
1260
|
+
prunesCompleted: prunesCompleted(r.remove_on_complete)
|
|
1261
|
+
}),
|
|
1262
|
+
library: version,
|
|
1263
|
+
schedulersCount: num(r.schedulers),
|
|
1264
|
+
stalledCount: num(r.stalled)
|
|
1265
|
+
};
|
|
1266
|
+
stats.metrics = { completed: metricsCompleted.slice(-STATS_METRIC_POINTS), failed: metricsFailed.slice(-STATS_METRIC_POINTS) };
|
|
1267
|
+
out[queue] = stats;
|
|
1268
|
+
}
|
|
1269
|
+
return out;
|
|
1270
|
+
}
|
|
1271
|
+
async getWindowCounts(queueName, since) {
|
|
1272
|
+
const [r] = await this.query(WINDOW_COUNTS, [queueName, since]);
|
|
1273
|
+
return { completed: num(r?.completed), failed: num(r?.failed) };
|
|
1274
|
+
}
|
|
1275
|
+
/** The cumulative `count` of the metrics rows; null per side when the row is absent (no metrics). */
|
|
1276
|
+
async getMetricsCounters(queueName) {
|
|
1277
|
+
const collectedAt = Date.now();
|
|
1278
|
+
const rows = await this.query(METRICS_COUNTERS, [queueName]);
|
|
1279
|
+
const read = (kind) => numOrNull(rows.find((r) => r.kind === kind)?.count);
|
|
1280
|
+
return { completed: read("completed"), failed: read("failed"), collectedAt };
|
|
1281
|
+
}
|
|
1282
|
+
async getWindowMetrics(requests, opts = {}) {
|
|
1283
|
+
const out = {};
|
|
1284
|
+
if (requests.length === 0) return out;
|
|
1285
|
+
const now = opts.now ?? Date.now();
|
|
1286
|
+
const sample = Math.max(1, Math.min(opts.durationSample ?? 100, 100));
|
|
1287
|
+
const maxRate = Math.max(1, ...requests.flatMap((r) => r.rateWindows));
|
|
1288
|
+
const sinces = requests.map((r) => now - Math.max(0, ...r.durationWindows) * 6e4);
|
|
1289
|
+
const rows = await this.query(
|
|
1290
|
+
WINDOW_METRICS,
|
|
1291
|
+
[requests.map((r) => r.queue), sinces, maxRate + 1, sample]
|
|
1292
|
+
);
|
|
1293
|
+
requests.forEach((request, i) => {
|
|
1294
|
+
const row = rows[i];
|
|
1295
|
+
if (!row) return;
|
|
1296
|
+
out[request.queue] = windowMetricsFrom(request, row.metrics, request.durationWindows.length > 0 ? row.durations : null, now);
|
|
1297
|
+
});
|
|
1298
|
+
return out;
|
|
1299
|
+
}
|
|
1300
|
+
/**
|
|
1301
|
+
* A page of one state: the ids come off the state's partial index (OFFSET over
|
|
1302
|
+
* a narrow index, never over full rows), then only that page is joined to
|
|
1303
|
+
* `job` for its columns. The total comes from the count cache when a stats read
|
|
1304
|
+
* just counted the state (the usual case: the queue page polls both), and is
|
|
1305
|
+
* counted in the same statement otherwise.
|
|
1306
|
+
*/
|
|
1307
|
+
async getJobs(queueName, state, opts) {
|
|
1308
|
+
const s = STATE_SQL[state];
|
|
1309
|
+
if (!s) return { jobs: [], total: 0, start: opts.start, end: opts.end };
|
|
1310
|
+
const ob = orderBy(state, opts.order);
|
|
1311
|
+
const limit = Math.max(0, opts.end - opts.start + 1);
|
|
1312
|
+
const known = this.cachedCount(queueName, state);
|
|
1313
|
+
const sql2 = `
|
|
1314
|
+
WITH page AS (
|
|
1315
|
+
SELECT id, row_number() OVER (ORDER BY ${ob}) AS ord
|
|
1316
|
+
FROM job WHERE queue = $1 AND ${s.where} ORDER BY ${ob} OFFSET $2 LIMIT $3
|
|
1317
|
+
)
|
|
1318
|
+
SELECT ${known === null ? `(SELECT count(*) FROM job WHERE queue = $1 AND ${s.where})` : "NULL::bigint"} AS total,
|
|
1319
|
+
(SELECT json_agg(r ORDER BY r.ord) FROM (
|
|
1320
|
+
SELECT page.ord, ${summaryColumns("j", 4, 5)}
|
|
1321
|
+
FROM page JOIN job j ON j.queue = $1 AND j.id = page.id) r) AS rows`;
|
|
1322
|
+
const [r] = await this.query(sql2, [
|
|
1323
|
+
queueName,
|
|
1324
|
+
Math.max(0, opts.start),
|
|
1325
|
+
limit,
|
|
1326
|
+
this.opts.previewBytes,
|
|
1327
|
+
this.opts.listFieldCapBytes
|
|
1328
|
+
]);
|
|
1329
|
+
const total = known ?? num(r?.total);
|
|
1330
|
+
if (known === null) this.rememberCount(queueName, state, total);
|
|
1331
|
+
return {
|
|
1332
|
+
jobs: (r?.rows ?? []).map((row) => rowToSummary(this.config.prefix, row, this.opts.previewBytes)),
|
|
1333
|
+
total,
|
|
1334
|
+
start: opts.start,
|
|
1335
|
+
end: opts.end
|
|
1336
|
+
};
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1339
|
+
* Bounded, resumable search, with the same bounds as Redis: at most
|
|
1340
|
+
* maxScanPerCall jobs and at most searchByteBudget payload bytes per call, and
|
|
1341
|
+
* payloads over searchFieldCapBytes are matched on id / name / error only.
|
|
1342
|
+
* Case-insensitive substring (ILIKE with the wildcards escaped). The cursor is
|
|
1343
|
+
* the index of the next job to inspect (newest = 0).
|
|
1344
|
+
*/
|
|
1345
|
+
async searchJobs(queueName, state, query, opts) {
|
|
1346
|
+
const s = STATE_SQL[state];
|
|
1347
|
+
if (!s) return { jobs: [], nextCursor: null, scanned: 0, total: 0, skippedLargePayloads: 0 };
|
|
1348
|
+
const cursor = Math.max(0, Math.trunc(Number(opts.cursor ?? 0)) || 0);
|
|
1349
|
+
const limit = Math.max(1, opts.limit);
|
|
1350
|
+
const ob = orderBy(state, "desc");
|
|
1351
|
+
const known = this.cachedCount(queueName, state);
|
|
1352
|
+
const sql2 = `
|
|
1353
|
+
WITH scan AS (
|
|
1354
|
+
SELECT id, row_number() OVER (ORDER BY ${ob}) AS rn
|
|
1355
|
+
FROM job WHERE queue = $1 AND ${s.where} ORDER BY ${ob} OFFSET $2 LIMIT $3
|
|
1356
|
+
), sized AS (
|
|
1357
|
+
SELECT s.rn, s.id, pg_column_size(j.data) AS sz
|
|
1358
|
+
FROM scan s JOIN job j ON j.queue = $1 AND j.id = s.id
|
|
1359
|
+
), budgeted AS (
|
|
1360
|
+
-- a job is inspected when the payload bytes spent BEFORE it are inside the budget
|
|
1361
|
+
SELECT rn, id, sz > $6 AS too_big FROM (
|
|
1362
|
+
SELECT rn, id, sz, sum(CASE WHEN sz > $6 THEN 0 ELSE sz END) OVER (ORDER BY rn) AS spent FROM sized
|
|
1363
|
+
) b
|
|
1364
|
+
WHERE spent - CASE WHEN sz > $6 THEN 0 ELSE sz END < $7
|
|
1365
|
+
), hits AS (
|
|
1366
|
+
SELECT b.rn, ${summaryColumns("j", 8, 9)}
|
|
1367
|
+
FROM budgeted b JOIN job j ON j.queue = $1 AND j.id = b.id
|
|
1368
|
+
WHERE j.id ILIKE $4 ESCAPE '\\' OR j.name ILIKE $4 ESCAPE '\\' OR j.failed_reason ILIKE $4 ESCAPE '\\'
|
|
1369
|
+
OR (NOT b.too_big AND j.data::text ILIKE $4 ESCAPE '\\')
|
|
1370
|
+
ORDER BY b.rn LIMIT $5
|
|
1371
|
+
)
|
|
1372
|
+
SELECT ${known === null ? `(SELECT count(*) FROM job WHERE queue = $1 AND ${s.where})` : "NULL::bigint"} AS total,
|
|
1373
|
+
(SELECT count(*) FROM budgeted) AS scanned,
|
|
1374
|
+
(SELECT json_agg(rn) FROM budgeted WHERE too_big) AS big,
|
|
1375
|
+
(SELECT json_agg(h ORDER BY h.rn) FROM hits h) AS rows`;
|
|
1376
|
+
const [r] = await this.query(sql2, [
|
|
1377
|
+
queueName,
|
|
1378
|
+
cursor,
|
|
1379
|
+
this.opts.maxScanPerCall,
|
|
1380
|
+
likeSubstring(query),
|
|
1381
|
+
limit,
|
|
1382
|
+
this.opts.searchFieldCapBytes,
|
|
1383
|
+
this.opts.searchByteBudget,
|
|
1384
|
+
this.opts.previewBytes,
|
|
1385
|
+
this.opts.listFieldCapBytes
|
|
1386
|
+
]);
|
|
1387
|
+
const total = known ?? num(r?.total);
|
|
1388
|
+
if (known === null) this.rememberCount(queueName, state, total);
|
|
1389
|
+
const rows = r?.rows ?? [];
|
|
1390
|
+
const stoppedAt = rows.length >= limit ? num(rows[rows.length - 1]?.rn) : cursor + num(r?.scanned);
|
|
1391
|
+
const big = (r?.big ?? []).filter((rn) => rn <= stoppedAt).length;
|
|
1392
|
+
return {
|
|
1393
|
+
jobs: rows.map((row) => rowToSummary(this.config.prefix, row, this.opts.previewBytes)),
|
|
1394
|
+
nextCursor: stoppedAt >= total ? null : String(stoppedAt),
|
|
1395
|
+
scanned: stoppedAt - cursor,
|
|
1396
|
+
total,
|
|
1397
|
+
skippedLargePayloads: big
|
|
1398
|
+
};
|
|
1399
|
+
}
|
|
1400
|
+
async getJob(queueName, jobId) {
|
|
1401
|
+
const [row] = await this.query(JOB_DETAIL, [queueName, jobId, DETAIL_LOG_TAIL]);
|
|
1402
|
+
return row ? rowToDetail(this.config.prefix, row) : null;
|
|
1403
|
+
}
|
|
1404
|
+
async getJobLogs(queueName, jobId, opts) {
|
|
1405
|
+
const start = Math.max(0, opts.start);
|
|
1406
|
+
const limit = opts.end < 0 ? null : Math.max(0, opts.end - start + 1);
|
|
1407
|
+
const [r] = await this.query(JOB_LOGS, [queueName, jobId, start, limit]);
|
|
1408
|
+
return { logs: r?.logs ?? [], count: num(r?.count) };
|
|
1409
|
+
}
|
|
1410
|
+
async getMetrics(queueName, points) {
|
|
1411
|
+
const rows = await this.query(METRICS, [queueName, Math.max(1, points)]);
|
|
1412
|
+
const read = (kind) => metricPoints(rows.find((r) => r.kind === kind)?.data);
|
|
1413
|
+
return { completed: read("completed"), failed: read("failed") };
|
|
1414
|
+
}
|
|
1415
|
+
/**
|
|
1416
|
+
* meta + the limiter window + whether metrics are collected, then the workers
|
|
1417
|
+
* from pg_stat_activity. Cached SETUP_CACHE_MS, like Redis (pg_stat_activity
|
|
1418
|
+
* is cheap, but there is no reason to read it on every poll).
|
|
1419
|
+
*/
|
|
1420
|
+
async getQueueSetup(queueName) {
|
|
1421
|
+
const cached = this.setupCache.get(queueName);
|
|
1422
|
+
if (cached && Date.now() - cached.at < SETUP_CACHE_MS) return cached.value;
|
|
1423
|
+
const [[r], workers] = await Promise.all([
|
|
1424
|
+
this.query(QUEUE_SETUP, [queueName]),
|
|
1425
|
+
this.listWorkers(queueName).catch(() => null)
|
|
1426
|
+
]);
|
|
1427
|
+
const meta = r?.meta ?? {};
|
|
1428
|
+
const version = meta.version ?? null;
|
|
1429
|
+
const max = numOrNull(meta.max);
|
|
1430
|
+
const duration = numOrNull(meta.duration);
|
|
1431
|
+
const expireAt = numOrNull(r?.limiter_expire_at);
|
|
1432
|
+
const ttl = expireAt === null ? -1 : expireAt - Date.now();
|
|
1433
|
+
const { version: _v, paused: _p, concurrency: _c, max: _m, duration: _d, ...rest } = meta;
|
|
1434
|
+
const rawMeta = {};
|
|
1435
|
+
for (const [k, v] of Object.entries(rest)) if (v !== null && v !== void 0) rawMeta[k] = v;
|
|
1436
|
+
const value = {
|
|
1437
|
+
library: version,
|
|
1438
|
+
isPro: version !== null && version.startsWith("bullmq-pro"),
|
|
1439
|
+
isPaused: meta.paused !== void 0,
|
|
1440
|
+
globalConcurrency: numOrNull(meta.concurrency),
|
|
1441
|
+
globalRateLimit: max !== null && duration !== null ? { max, durationMs: duration } : null,
|
|
1442
|
+
rateLimitedNow: ttl >= 0 ? { ttlMs: ttl } : null,
|
|
1443
|
+
workers,
|
|
1444
|
+
groups: null,
|
|
1445
|
+
batch: "unknown",
|
|
1446
|
+
metricsEnabled: r?.metrics_enabled === true,
|
|
1447
|
+
maxLenEvents: numOrNull(meta["opts.maxLenEvents"]),
|
|
1448
|
+
rawMeta
|
|
1449
|
+
};
|
|
1450
|
+
this.setupCache.set(queueName, { at: Date.now(), value });
|
|
1451
|
+
return value;
|
|
1452
|
+
}
|
|
1453
|
+
async listWorkers(queueName) {
|
|
1454
|
+
const rows = await this.query(WORKERS, [
|
|
1455
|
+
queueName,
|
|
1456
|
+
`${queueName.replace(/[\\%_]/g, (c) => `\\${c}`)}:w:%`
|
|
1457
|
+
]);
|
|
1458
|
+
const names = rows.map((r) => {
|
|
1459
|
+
const idx = r.application_name.indexOf(":w:");
|
|
1460
|
+
return idx === -1 ? "worker" : r.application_name.slice(idx + 3);
|
|
1461
|
+
});
|
|
1462
|
+
return { count: names.length, names };
|
|
1463
|
+
}
|
|
1464
|
+
// --- BullMQ Pro groups: the open-source Postgres backend has no groups ------
|
|
1465
|
+
async getGroups() {
|
|
1466
|
+
return { groups: [], total: 0, byStatus: { waiting: 0, limited: 0, maxed: 0, paused: 0 } };
|
|
1467
|
+
}
|
|
1468
|
+
async getGroupJobs(_queueName, _groupId, opts) {
|
|
1469
|
+
return { jobs: [], total: 0, start: opts.start, end: opts.end };
|
|
1470
|
+
}
|
|
1471
|
+
// ---------------------------------------------------------------------------
|
|
1472
|
+
// job schedulers
|
|
1473
|
+
// ---------------------------------------------------------------------------
|
|
1474
|
+
async getSchedulers(queueName, opts) {
|
|
1475
|
+
const limit = opts.end < 0 ? null : Math.max(0, opts.end - opts.start + 1);
|
|
1476
|
+
const [r] = await this.query(SCHEDULERS, [
|
|
1477
|
+
queueName,
|
|
1478
|
+
Math.max(0, opts.start),
|
|
1479
|
+
limit,
|
|
1480
|
+
this.opts.previewBytes
|
|
1481
|
+
]);
|
|
1482
|
+
return { schedulers: (r?.rows ?? []).map(rowToScheduler), total: num(r?.total) };
|
|
1483
|
+
}
|
|
1484
|
+
/** Official API: removes the scheduler AND the delayed job it has queued. */
|
|
1485
|
+
async removeScheduler(queueName, key) {
|
|
1486
|
+
const queue = await this.getQueue(queueName);
|
|
1487
|
+
return { removed: await queue.removeJobScheduler(key) === true };
|
|
1488
|
+
}
|
|
1489
|
+
// ---------------------------------------------------------------------------
|
|
1490
|
+
// flows
|
|
1491
|
+
// ---------------------------------------------------------------------------
|
|
1492
|
+
async sampleFlowEdges(queueName, opts = {}) {
|
|
1493
|
+
const sample = Math.max(1, opts.sample ?? 50);
|
|
1494
|
+
const [r] = await this.query(SAMPLE_PARENTS, [queueName, sample]);
|
|
1495
|
+
const edges = (r?.edges ?? []).map(([parentQueue, count2]) => ({ parentQueue, childQueue: queueName, count: num(count2) }));
|
|
1496
|
+
edges.sort((a, b) => b.count - a.count);
|
|
1497
|
+
return { edges, sampled: num(r?.sampled) };
|
|
1498
|
+
}
|
|
1499
|
+
/**
|
|
1500
|
+
* Walk ONE flow instance, breadth first, at most `maxNodes` jobs read. Same
|
|
1501
|
+
* shape and budget as the Redis walk, but each wave is ONE statement across
|
|
1502
|
+
* every queue it touches (job_dependency holds both ends of every edge).
|
|
1503
|
+
*/
|
|
1504
|
+
async getJobTree(queueName, jobId, opts = {}) {
|
|
1505
|
+
const maxNodes = Math.max(1, Math.min(opts.maxNodes ?? 200, 500));
|
|
1506
|
+
const schema = this.config.prefix;
|
|
1507
|
+
let visited = 0;
|
|
1508
|
+
const read = async (refs) => {
|
|
1509
|
+
if (refs.length === 0) return [];
|
|
1510
|
+
visited += refs.length;
|
|
1511
|
+
const rows = await this.query(TREE_NODES, [refs.map((r) => r.queue), refs.map((r) => r.id), maxNodes]);
|
|
1512
|
+
return refs.map((_, i) => {
|
|
1513
|
+
const row = rows[i];
|
|
1514
|
+
return row && row.found ? row : null;
|
|
1515
|
+
});
|
|
1516
|
+
};
|
|
1517
|
+
const [focus] = await read([{ queue: queueName, id: jobId }]);
|
|
1518
|
+
if (!focus) return null;
|
|
1519
|
+
const focusKey = jobKeyOf(schema, queueName, jobId);
|
|
1520
|
+
let root = { queue: queueName, id: jobId };
|
|
1521
|
+
let climbedLevels = 0;
|
|
1522
|
+
if (opts.fromRoot !== false) {
|
|
1523
|
+
const seen = /* @__PURE__ */ new Set([focusKey]);
|
|
1524
|
+
let current2 = focus;
|
|
1525
|
+
while (current2.parent_queue && current2.parent_id && climbedLevels < MAX_TREE_CLIMB && visited < maxNodes) {
|
|
1526
|
+
const next = { queue: current2.parent_queue, id: current2.parent_id };
|
|
1527
|
+
const key = jobKeyOf(schema, next.queue, next.id);
|
|
1528
|
+
if (seen.has(key)) break;
|
|
1529
|
+
const [parent] = await read([next]);
|
|
1530
|
+
if (!parent) break;
|
|
1531
|
+
seen.add(key);
|
|
1532
|
+
root = next;
|
|
1533
|
+
climbedLevels += 1;
|
|
1534
|
+
current2 = parent;
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1538
|
+
let frontier = [{ ...root, parentKey: null }];
|
|
1539
|
+
let truncated = false;
|
|
1540
|
+
while (frontier.length > 0 && !truncated) {
|
|
1541
|
+
const wave = frontier.filter((f) => !nodes.has(jobKeyOf(schema, f.queue, f.id)));
|
|
1542
|
+
const room = maxNodes - nodes.size;
|
|
1543
|
+
if (room <= 0) {
|
|
1544
|
+
truncated = wave.length > 0;
|
|
1545
|
+
break;
|
|
1546
|
+
}
|
|
1547
|
+
const slice = wave.length > room ? (truncated = true, wave.slice(0, room)) : wave;
|
|
1548
|
+
const rows = await read(slice);
|
|
1549
|
+
const next = [];
|
|
1550
|
+
slice.forEach((entry, i) => {
|
|
1551
|
+
const key = jobKeyOf(schema, entry.queue, entry.id);
|
|
1552
|
+
const row = rows[i] ?? null;
|
|
1553
|
+
nodes.set(key, treeNode(schema, entry, row, maxNodes));
|
|
1554
|
+
for (const [childQueue, childId] of (row?.children ?? []).slice(0, maxNodes)) {
|
|
1555
|
+
const childKey = jobKeyOf(schema, childQueue, childId);
|
|
1556
|
+
if (!nodes.has(childKey)) next.push({ queue: childQueue, id: childId, parentKey: key });
|
|
1557
|
+
}
|
|
1558
|
+
});
|
|
1559
|
+
frontier = next;
|
|
1560
|
+
}
|
|
1561
|
+
return {
|
|
1562
|
+
rootKey: jobKeyOf(schema, root.queue, root.id),
|
|
1563
|
+
focusKey,
|
|
1564
|
+
nodes: [...nodes.values()],
|
|
1565
|
+
truncated: truncated || nodes.size >= maxNodes,
|
|
1566
|
+
visited,
|
|
1567
|
+
climbedLevels
|
|
1568
|
+
};
|
|
1569
|
+
}
|
|
1570
|
+
// ---------------------------------------------------------------------------
|
|
1571
|
+
// writes (official bullmq API; we never write BullMQ's tables ourselves)
|
|
1572
|
+
// ---------------------------------------------------------------------------
|
|
1573
|
+
/**
|
|
1574
|
+
* One bullmq Queue per queue name, created on the first write. BullMQ builds
|
|
1575
|
+
* its own pool from the config object (that is the only way it honours a
|
|
1576
|
+
* non-default schema); `max: 1` and a short idle timeout keep a dashboard
|
|
1577
|
+
* that touched 50 queues from holding 50 connections. A Queue never opens the
|
|
1578
|
+
* LISTEN connection — only Workers and QueueEvents do.
|
|
1579
|
+
*/
|
|
1580
|
+
async getQueue(queueName) {
|
|
1581
|
+
await this.ensureReady();
|
|
1582
|
+
this.invalidateStats();
|
|
1583
|
+
let q = this.queues.get(queueName);
|
|
1584
|
+
if (q) return q;
|
|
1585
|
+
q = new Queue(
|
|
1586
|
+
queueName,
|
|
1587
|
+
{
|
|
1588
|
+
connection: {
|
|
1589
|
+
connectionString: nodePgConnectionString(this.config.url),
|
|
1590
|
+
schema: this.config.prefix,
|
|
1591
|
+
max: 1,
|
|
1592
|
+
idleTimeoutMillis: 1e4,
|
|
1593
|
+
application_name: APPLICATION_NAME
|
|
1594
|
+
},
|
|
1595
|
+
skipMetasUpdate: true
|
|
1596
|
+
},
|
|
1597
|
+
createPostgresBackend
|
|
1598
|
+
);
|
|
1599
|
+
q.on("error", () => void 0);
|
|
1600
|
+
this.queues.set(queueName, q);
|
|
1601
|
+
return q;
|
|
1602
|
+
}
|
|
1603
|
+
/**
|
|
1604
|
+
* Every write goes through getQueue, so dropping the stats window here keeps
|
|
1605
|
+
* an action from being followed by a count from before it.
|
|
1606
|
+
*/
|
|
1607
|
+
invalidateStats() {
|
|
1608
|
+
this.statsCache.clear();
|
|
1609
|
+
this.countCache.clear();
|
|
1610
|
+
}
|
|
1611
|
+
async getBullJob(queueName, jobId) {
|
|
1612
|
+
const queue = await this.getQueue(queueName);
|
|
1613
|
+
const job = await Job.fromId(queue, jobId);
|
|
1614
|
+
if (!job) throw new Error("job_not_found");
|
|
1615
|
+
return job;
|
|
1616
|
+
}
|
|
1617
|
+
async addJob(queueName, name, data, opts = {}) {
|
|
1618
|
+
const queue = await this.getQueue(queueName);
|
|
1619
|
+
const job = await queue.add(name, data, opts);
|
|
1620
|
+
return { id: String(job.id) };
|
|
1621
|
+
}
|
|
1622
|
+
async retryJob(queueName, jobId) {
|
|
1623
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1624
|
+
const state = await job.getState();
|
|
1625
|
+
if (state !== "failed" && state !== "completed") throw new Error(`cannot_retry_job_in_state_${state}`);
|
|
1626
|
+
await job.retry(state);
|
|
1627
|
+
}
|
|
1628
|
+
async removeJob(queueName, jobId) {
|
|
1629
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1630
|
+
await job.remove();
|
|
1631
|
+
}
|
|
1632
|
+
/** Same rules as the Redis inspector: a scheduler's job asks the operator first. */
|
|
1633
|
+
async promoteJob(queueName, jobId, scheduler = "run_copy") {
|
|
1634
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1635
|
+
if (!job.repeatJobKey) {
|
|
1636
|
+
await job.promote();
|
|
1637
|
+
return { mode: "promoted" };
|
|
1638
|
+
}
|
|
1639
|
+
if (scheduler === "skip_next") {
|
|
1640
|
+
await job.promote();
|
|
1641
|
+
return { mode: "skipped_next", schedulerId: job.repeatJobKey };
|
|
1642
|
+
}
|
|
1643
|
+
const state = await job.getState();
|
|
1644
|
+
if (state !== "delayed") throw new Error(`cannot_promote_job_in_state_${state}`);
|
|
1645
|
+
const { repeat: _r, jobId: _j, repeatJobKey: _k, prevMillis: _p, delay: _d, timestamp: _t, ...opts } = job.opts;
|
|
1646
|
+
const copy = await this.addJob(queueName, job.name, job.data, opts);
|
|
1647
|
+
return { mode: "ran_copy", jobId: copy.id, schedulerId: job.repeatJobKey };
|
|
1648
|
+
}
|
|
1649
|
+
/** Partial result, bounded concurrency: see Inspector.bulkJobAction. */
|
|
1650
|
+
async bulkJobAction(queueName, action, jobIds) {
|
|
1651
|
+
const ok = [];
|
|
1652
|
+
const failed = [];
|
|
1653
|
+
const ids = [...new Set(jobIds)];
|
|
1654
|
+
const run = async (jobId) => {
|
|
1655
|
+
try {
|
|
1656
|
+
if (action === "retry") await this.retryJob(queueName, jobId);
|
|
1657
|
+
else if (action === "remove") await this.removeJob(queueName, jobId);
|
|
1658
|
+
else await this.promoteJob(queueName, jobId);
|
|
1659
|
+
ok.push(jobId);
|
|
1660
|
+
} catch (err) {
|
|
1661
|
+
failed.push({ jobId, reason: errorMessage(err) });
|
|
1662
|
+
}
|
|
1663
|
+
};
|
|
1664
|
+
for (let i = 0; i < ids.length; i += BULK_CONCURRENCY) {
|
|
1665
|
+
await Promise.all(ids.slice(i, i + BULK_CONCURRENCY).map(run));
|
|
1666
|
+
}
|
|
1667
|
+
return { action, ok, failed, requested: ids.length };
|
|
1668
|
+
}
|
|
1669
|
+
/** Operator "discard": an active job straight to failed, no retry (see RedisInspector.discardJob). */
|
|
1670
|
+
async discardJob(queueName, jobId) {
|
|
1671
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1672
|
+
const state = await job.getState();
|
|
1673
|
+
if (state !== "active") throw new Error(`cannot_discard_job_in_state_${state}`);
|
|
1674
|
+
try {
|
|
1675
|
+
await job.moveToFailed(new UnrecoverableError("Discarded from Bullpane"), "0");
|
|
1676
|
+
} catch (err) {
|
|
1677
|
+
throw new Error(`cannot_discard_active_job: ${errorMessage(err)}`);
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
async pauseQueue(queueName) {
|
|
1681
|
+
await (await this.getQueue(queueName)).pause();
|
|
1682
|
+
this.setupCache.delete(queueName);
|
|
1683
|
+
this.invalidateStats();
|
|
1684
|
+
}
|
|
1685
|
+
async resumeQueue(queueName) {
|
|
1686
|
+
await (await this.getQueue(queueName)).resume();
|
|
1687
|
+
this.setupCache.delete(queueName);
|
|
1688
|
+
this.invalidateStats();
|
|
1689
|
+
}
|
|
1690
|
+
async cleanQueue(queueName, state, graceMs, limit) {
|
|
1691
|
+
const ids = await (await this.getQueue(queueName)).clean(graceMs, limit, state);
|
|
1692
|
+
return { removed: ids.length };
|
|
1693
|
+
}
|
|
1694
|
+
async retryAll(queueName, state) {
|
|
1695
|
+
await (await this.getQueue(queueName)).retryJobs({ state });
|
|
1696
|
+
}
|
|
1697
|
+
async drainQueue(queueName, includeDelayed) {
|
|
1698
|
+
await (await this.getQueue(queueName)).drain(includeDelayed);
|
|
1699
|
+
}
|
|
1700
|
+
async obliterateQueue(queueName) {
|
|
1701
|
+
const queue = await this.getQueue(queueName);
|
|
1702
|
+
await queue.obliterate({ force: true });
|
|
1703
|
+
this.queues.delete(queueName);
|
|
1704
|
+
await queue.close().catch(() => void 0);
|
|
1705
|
+
this.discovered = null;
|
|
1706
|
+
}
|
|
1707
|
+
};
|
|
1708
|
+
function treeNode(schema, entry, row, maxNodes) {
|
|
1709
|
+
const key = jobKeyOf(schema, entry.queue, entry.id);
|
|
1710
|
+
const base = { key, id: entry.id, queueName: entry.queue, queueKey: queueKeyOf(schema, entry.queue) };
|
|
1711
|
+
if (!row) {
|
|
1712
|
+
return {
|
|
1713
|
+
...base,
|
|
1714
|
+
name: "",
|
|
1715
|
+
state: "unknown",
|
|
1716
|
+
timestamp: 0,
|
|
1717
|
+
finishedOn: null,
|
|
1718
|
+
attemptsMade: 0,
|
|
1719
|
+
failedReason: null,
|
|
1720
|
+
progress: null,
|
|
1721
|
+
parentKey: entry.parentKey,
|
|
1722
|
+
dependencies: null,
|
|
1723
|
+
childrenTruncated: false,
|
|
1724
|
+
missing: true
|
|
1725
|
+
};
|
|
441
1726
|
}
|
|
1727
|
+
const processed = num(row.processed);
|
|
1728
|
+
const unprocessed = num(row.unprocessed);
|
|
1729
|
+
return {
|
|
1730
|
+
...base,
|
|
1731
|
+
name: row.name ?? "",
|
|
1732
|
+
state: bullpaneState(row.state, row.priority),
|
|
1733
|
+
timestamp: num(row.added_at_ms),
|
|
1734
|
+
finishedOn: numOrNull(row.finished_at_ms),
|
|
1735
|
+
attemptsMade: num(row.attempts_made),
|
|
1736
|
+
failedReason: row.failed_reason ?? null,
|
|
1737
|
+
progress: progressOf(row.progress),
|
|
1738
|
+
// the walk's own edge wins over the stored parent: it is the edge being drawn
|
|
1739
|
+
parentKey: entry.parentKey ?? (row.parent_queue && row.parent_id ? jobKeyOf(schema, row.parent_queue, row.parent_id) : null),
|
|
1740
|
+
dependencies: processed + unprocessed > 0 ? { processed, unprocessed } : null,
|
|
1741
|
+
// TREE_NODES reads maxNodes + 1 children: one more than the walk could hold means there are more
|
|
1742
|
+
childrenTruncated: (row.children?.length ?? 0) > maxNodes,
|
|
1743
|
+
missing: false
|
|
1744
|
+
};
|
|
442
1745
|
}
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
1746
|
+
|
|
1747
|
+
// packages/pg-inspector/src/pool.ts
|
|
1748
|
+
var PgInspectorPool = class {
|
|
1749
|
+
constructor(options = {}) {
|
|
1750
|
+
this.options = options;
|
|
1751
|
+
}
|
|
1752
|
+
inspectors = /* @__PURE__ */ new Map();
|
|
1753
|
+
get(config) {
|
|
1754
|
+
const existing = this.inspectors.get(config.id);
|
|
1755
|
+
if (existing && sameTarget(existing.config, config)) return existing;
|
|
1756
|
+
if (existing) void existing.close().catch(() => void 0);
|
|
1757
|
+
const created = new PgInspector(config, this.options);
|
|
1758
|
+
this.inspectors.set(config.id, created);
|
|
1759
|
+
return created;
|
|
1760
|
+
}
|
|
1761
|
+
async evict(id3) {
|
|
1762
|
+
const existing = this.inspectors.get(id3);
|
|
1763
|
+
if (!existing) return;
|
|
1764
|
+
this.inspectors.delete(id3);
|
|
1765
|
+
await existing.close();
|
|
1766
|
+
}
|
|
1767
|
+
async closeAll() {
|
|
1768
|
+
const all = [...this.inspectors.values()];
|
|
1769
|
+
this.inspectors.clear();
|
|
1770
|
+
await Promise.allSettled(all.map((i) => i.close()));
|
|
1771
|
+
}
|
|
1772
|
+
};
|
|
1773
|
+
function sameTarget(a, b) {
|
|
1774
|
+
return a.url === b.url && a.prefix === (b.prefix ?? "bullmq") && (a.queueFilter ?? null) === (b.queueFilter ?? null);
|
|
451
1775
|
}
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
access: z.enum(["read", "write"]).default("read")
|
|
457
|
-
});
|
|
1776
|
+
|
|
1777
|
+
// packages/redis-inspector/src/inspector.ts
|
|
1778
|
+
import { Cluster as Cluster2 } from "ioredis";
|
|
1779
|
+
import { Job as Job2, Queue as Queue2, UnrecoverableError as UnrecoverableError2 } from "bullmq";
|
|
458
1780
|
|
|
459
1781
|
// packages/redis-inspector/src/connection.ts
|
|
460
1782
|
import Redis, { Cluster } from "ioredis";
|
|
@@ -664,14 +1986,6 @@ function queueNameFromQueueKey(prefix, qualified) {
|
|
|
664
1986
|
}
|
|
665
1987
|
|
|
666
1988
|
// packages/redis-inspector/src/util.ts
|
|
667
|
-
function globToRegExp(glob) {
|
|
668
|
-
const escaped = glob.split("").map((ch) => {
|
|
669
|
-
if (ch === "*") return ".*";
|
|
670
|
-
if (ch === "?") return ".";
|
|
671
|
-
return ch.replace(/[.+^${}()|[\]\\/]/g, "\\$&");
|
|
672
|
-
}).join("");
|
|
673
|
-
return new RegExp(`^${escaped}$`);
|
|
674
|
-
}
|
|
675
1989
|
function safeJsonParse(raw, fallback) {
|
|
676
1990
|
if (raw === null || raw === void 0) return fallback === void 0 ? null : fallback;
|
|
677
1991
|
try {
|
|
@@ -721,7 +2035,7 @@ function totalKeysFromInfo(info) {
|
|
|
721
2035
|
}
|
|
722
2036
|
return seen ? total : null;
|
|
723
2037
|
}
|
|
724
|
-
function
|
|
2038
|
+
function errorMessage2(err) {
|
|
725
2039
|
if (err instanceof Error) return err.message;
|
|
726
2040
|
return String(err);
|
|
727
2041
|
}
|
|
@@ -870,13 +2184,13 @@ function asNumber(reply, fallback = 0) {
|
|
|
870
2184
|
function asStringArray(reply) {
|
|
871
2185
|
return asArray(reply).map((v) => typeof v === "string" ? v : String(v ?? ""));
|
|
872
2186
|
}
|
|
873
|
-
function
|
|
2187
|
+
function metricPoints2(reply) {
|
|
874
2188
|
return asStringArray(reply).map((s) => Number(s) || 0).reverse();
|
|
875
2189
|
}
|
|
876
2190
|
function str(v) {
|
|
877
2191
|
return typeof v === "string" && v !== "" ? v : typeof v === "number" ? String(v) : null;
|
|
878
2192
|
}
|
|
879
|
-
function
|
|
2193
|
+
function rowToScheduler2(row) {
|
|
880
2194
|
const key = String(row[0] ?? "");
|
|
881
2195
|
const name = str(row[2]);
|
|
882
2196
|
const data = str(row[11]);
|
|
@@ -982,7 +2296,7 @@ function pipelineScript(pipeline, name, args) {
|
|
|
982
2296
|
}
|
|
983
2297
|
|
|
984
2298
|
// packages/redis-inspector/src/inspector.ts
|
|
985
|
-
var
|
|
2299
|
+
var DEFAULTS2 = {
|
|
986
2300
|
discoveryTtlMs: 3e4,
|
|
987
2301
|
previewBytes: 2048,
|
|
988
2302
|
maxScanPerCall: 1e3,
|
|
@@ -995,12 +2309,12 @@ var DEFAULTS = {
|
|
|
995
2309
|
searchByteBudget: 8 * 1024 * 1024
|
|
996
2310
|
};
|
|
997
2311
|
var SCAN_COUNT = 1e3;
|
|
998
|
-
var
|
|
999
|
-
var
|
|
1000
|
-
var
|
|
1001
|
-
var
|
|
1002
|
-
var
|
|
1003
|
-
var
|
|
2312
|
+
var STATS_METRIC_POINTS2 = 60;
|
|
2313
|
+
var BULK_CONCURRENCY2 = 8;
|
|
2314
|
+
var DEFAULT_RATE_WINDOW_MINUTES2 = 60;
|
|
2315
|
+
var FORCE_DISCOVERY_MIN_MS2 = 5e3;
|
|
2316
|
+
var SETUP_CACHE_MS2 = 1e4;
|
|
2317
|
+
var DETAIL_LOG_TAIL2 = 100;
|
|
1004
2318
|
var FLOW_SAMPLE_STATES = ["completed", "failed", "waiting-children", "waiting", "active", "delayed"];
|
|
1005
2319
|
var RedisInspector = class {
|
|
1006
2320
|
config;
|
|
@@ -1025,12 +2339,14 @@ var RedisInspector = class {
|
|
|
1025
2339
|
bullmqCluster = null;
|
|
1026
2340
|
closed = false;
|
|
1027
2341
|
constructor(config, options = {}) {
|
|
2342
|
+
if (config.kind && config.kind !== "redis") throw new Error(`RedisInspector cannot open a ${config.kind} connection`);
|
|
1028
2343
|
this.config = {
|
|
1029
2344
|
...config,
|
|
2345
|
+
kind: "redis",
|
|
1030
2346
|
prefix: config.prefix ?? "bull",
|
|
1031
2347
|
cluster: config.cluster ?? false
|
|
1032
2348
|
};
|
|
1033
|
-
this.opts = { ...
|
|
2349
|
+
this.opts = { ...DEFAULTS2, ...stripUndefined(options) };
|
|
1034
2350
|
this.filter = config.queueFilter ? globToRegExp(config.queueFilter) : null;
|
|
1035
2351
|
this.client = createReadClient(this.config, this.opts.connectTimeoutMs);
|
|
1036
2352
|
this.client.on("error", () => void 0);
|
|
@@ -1091,7 +2407,7 @@ var RedisInspector = class {
|
|
|
1091
2407
|
ok: false,
|
|
1092
2408
|
latencyMs: Math.round((performance.now() - t0) * 10) / 10,
|
|
1093
2409
|
redisVersion: null,
|
|
1094
|
-
error:
|
|
2410
|
+
error: errorMessage2(err)
|
|
1095
2411
|
};
|
|
1096
2412
|
}
|
|
1097
2413
|
}
|
|
@@ -1174,7 +2490,7 @@ var RedisInspector = class {
|
|
|
1174
2490
|
async discoverQueues(opts = {}) {
|
|
1175
2491
|
const now = Date.now();
|
|
1176
2492
|
const age = this.discovered ? now - this.discovered.at : Number.POSITIVE_INFINITY;
|
|
1177
|
-
if (this.discovered && age < (opts.force ?
|
|
2493
|
+
if (this.discovered && age < (opts.force ? FORCE_DISCOVERY_MIN_MS2 : this.opts.discoveryTtlMs)) {
|
|
1178
2494
|
return this.discovered.names;
|
|
1179
2495
|
}
|
|
1180
2496
|
if (!this.discovering) {
|
|
@@ -1328,19 +2644,19 @@ var RedisInspector = class {
|
|
|
1328
2644
|
if (queueNames.length === 0) return out;
|
|
1329
2645
|
const c = await this.ensureConnected();
|
|
1330
2646
|
const withMetrics = opts.withMetrics ? 1 : 0;
|
|
1331
|
-
const windowMinutes = opts.rateWindowMinutes ??
|
|
2647
|
+
const windowMinutes = opts.rateWindowMinutes ?? DEFAULT_RATE_WINDOW_MINUTES2;
|
|
1332
2648
|
const since = Date.now() - windowMinutes * 6e4;
|
|
1333
2649
|
let replies;
|
|
1334
2650
|
if (c instanceof Cluster2) {
|
|
1335
2651
|
replies = await Promise.all(
|
|
1336
2652
|
queueNames.map(
|
|
1337
|
-
(q) => callScript(c, "queueStats", [...this.statsKeys(q), withMetrics,
|
|
2653
|
+
(q) => callScript(c, "queueStats", [...this.statsKeys(q), withMetrics, STATS_METRIC_POINTS2, since, queueKeyPrefix(this.config.prefix, q)]).catch(() => null)
|
|
1338
2654
|
)
|
|
1339
2655
|
);
|
|
1340
2656
|
} else {
|
|
1341
2657
|
const pipeline = c.pipeline();
|
|
1342
2658
|
for (const q of queueNames) {
|
|
1343
|
-
pipelineScript(pipeline, "queueStats", [...this.statsKeys(q), withMetrics,
|
|
2659
|
+
pipelineScript(pipeline, "queueStats", [...this.statsKeys(q), withMetrics, STATS_METRIC_POINTS2, since, queueKeyPrefix(this.config.prefix, q)]);
|
|
1344
2660
|
}
|
|
1345
2661
|
const results = await pipeline.exec() ?? [];
|
|
1346
2662
|
replies = results.map(([err, reply]) => err ? null : reply);
|
|
@@ -1497,7 +2813,7 @@ var RedisInspector = class {
|
|
|
1497
2813
|
p + JOB_KEY.processed(jobId),
|
|
1498
2814
|
...allStateKeys(this.config.prefix, queueName),
|
|
1499
2815
|
jobId,
|
|
1500
|
-
|
|
2816
|
+
DETAIL_LOG_TAIL2
|
|
1501
2817
|
]);
|
|
1502
2818
|
if (!Array.isArray(reply)) return null;
|
|
1503
2819
|
const hash = flatToHash(asArray(reply[0]));
|
|
@@ -1515,8 +2831,8 @@ var RedisInspector = class {
|
|
|
1515
2831
|
async getJobLogs(queueName, jobId, opts) {
|
|
1516
2832
|
const c = await this.ensureConnected();
|
|
1517
2833
|
const key = queueKeyPrefix(this.config.prefix, queueName) + JOB_KEY.logs(jobId);
|
|
1518
|
-
const [logs,
|
|
1519
|
-
return { logs, count };
|
|
2834
|
+
const [logs, count2] = await Promise.all([c.lrange(key, opts.start, opts.end), c.llen(key)]);
|
|
2835
|
+
return { logs, count: count2 };
|
|
1520
2836
|
}
|
|
1521
2837
|
/** Metric lists are LPUSHed (newest at index 0); we return oldest first. */
|
|
1522
2838
|
async getMetrics(queueName, points) {
|
|
@@ -1527,7 +2843,7 @@ var RedisInspector = class {
|
|
|
1527
2843
|
c.lrange(p + QUEUE_KEY.metricsCompletedData, 0, n - 1),
|
|
1528
2844
|
c.lrange(p + QUEUE_KEY.metricsFailedData, 0, n - 1)
|
|
1529
2845
|
]);
|
|
1530
|
-
return { completed:
|
|
2846
|
+
return { completed: metricPoints2(completed), failed: metricPoints2(failed) };
|
|
1531
2847
|
}
|
|
1532
2848
|
// ---------------------------------------------------------------------------
|
|
1533
2849
|
// BullMQ Pro groups (read only)
|
|
@@ -1543,7 +2859,7 @@ var RedisInspector = class {
|
|
|
1543
2859
|
*/
|
|
1544
2860
|
async getQueueSetup(queueName) {
|
|
1545
2861
|
const cached = this.setupCache.get(queueName);
|
|
1546
|
-
if (cached && Date.now() - cached.at <
|
|
2862
|
+
if (cached && Date.now() - cached.at < SETUP_CACHE_MS2) return cached.value;
|
|
1547
2863
|
const c = await this.ensureConnected();
|
|
1548
2864
|
const p = queueKeyPrefix(this.config.prefix, queueName);
|
|
1549
2865
|
const [reply, workers] = await Promise.all([
|
|
@@ -1682,7 +2998,7 @@ var RedisInspector = class {
|
|
|
1682
2998
|
const reply = asArray(
|
|
1683
2999
|
await callScript(c, "getSchedulers", [p + SCHEDULER_KEY.repeat, opts.start, opts.end, p, this.opts.previewBytes])
|
|
1684
3000
|
);
|
|
1685
|
-
const schedulers = asArray(reply[1]).map((row) =>
|
|
3001
|
+
const schedulers = asArray(reply[1]).map((row) => rowToScheduler2(asArray(row)));
|
|
1686
3002
|
return { schedulers, total: asNumber(reply[0]) };
|
|
1687
3003
|
}
|
|
1688
3004
|
/**
|
|
@@ -1756,7 +3072,7 @@ var RedisInspector = class {
|
|
|
1756
3072
|
if (opts.fromRoot !== false) {
|
|
1757
3073
|
const seen = /* @__PURE__ */ new Set([focusKey]);
|
|
1758
3074
|
let current2 = focus;
|
|
1759
|
-
while (current2.parentKey && climbedLevels <
|
|
3075
|
+
while (current2.parentKey && climbedLevels < MAX_TREE_CLIMB2) {
|
|
1760
3076
|
const next = current2.parentKey;
|
|
1761
3077
|
if (seen.has(next)) break;
|
|
1762
3078
|
const parent = await readOne(next);
|
|
@@ -1870,7 +3186,7 @@ var RedisInspector = class {
|
|
|
1870
3186
|
this.bullmqCluster = ownedCluster;
|
|
1871
3187
|
}
|
|
1872
3188
|
}
|
|
1873
|
-
q = new
|
|
3189
|
+
q = new Queue2(queueName, {
|
|
1874
3190
|
connection: this.bullmqCluster ?? connection,
|
|
1875
3191
|
prefix: this.config.prefix,
|
|
1876
3192
|
skipMetasUpdate: true
|
|
@@ -1881,7 +3197,7 @@ var RedisInspector = class {
|
|
|
1881
3197
|
}
|
|
1882
3198
|
async getBullJob(queueName, jobId) {
|
|
1883
3199
|
const queue = await this.getQueue(queueName);
|
|
1884
|
-
const job = await
|
|
3200
|
+
const job = await Job2.fromId(queue, jobId);
|
|
1885
3201
|
if (!job) throw new Error("job_not_found");
|
|
1886
3202
|
return job;
|
|
1887
3203
|
}
|
|
@@ -1940,11 +3256,11 @@ var RedisInspector = class {
|
|
|
1940
3256
|
else await this.promoteJob(queueName, jobId);
|
|
1941
3257
|
ok.push(jobId);
|
|
1942
3258
|
} catch (err) {
|
|
1943
|
-
failed.push({ jobId, reason:
|
|
3259
|
+
failed.push({ jobId, reason: errorMessage2(err) });
|
|
1944
3260
|
}
|
|
1945
3261
|
};
|
|
1946
|
-
for (let i = 0; i < ids.length; i +=
|
|
1947
|
-
await Promise.all(ids.slice(i, i +
|
|
3262
|
+
for (let i = 0; i < ids.length; i += BULK_CONCURRENCY2) {
|
|
3263
|
+
await Promise.all(ids.slice(i, i + BULK_CONCURRENCY2).map(run));
|
|
1948
3264
|
}
|
|
1949
3265
|
return { action, ok, failed, requested: ids.length };
|
|
1950
3266
|
}
|
|
@@ -1963,9 +3279,9 @@ var RedisInspector = class {
|
|
|
1963
3279
|
throw new Error(`cannot_discard_job_in_state_${state}`);
|
|
1964
3280
|
}
|
|
1965
3281
|
try {
|
|
1966
|
-
await job.moveToFailed(new
|
|
3282
|
+
await job.moveToFailed(new UnrecoverableError2("Discarded from Bullpane"), "0");
|
|
1967
3283
|
} catch (err) {
|
|
1968
|
-
throw new Error(`cannot_discard_active_job: ${
|
|
3284
|
+
throw new Error(`cannot_discard_active_job: ${errorMessage2(err)}`);
|
|
1969
3285
|
}
|
|
1970
3286
|
}
|
|
1971
3287
|
async pauseQueue(queueName) {
|
|
@@ -2003,9 +3319,9 @@ function parseStats(reply, withMetrics, windowMinutes) {
|
|
|
2003
3319
|
STATE_ORDER.forEach((state, i) => {
|
|
2004
3320
|
counts[state] = asNumber(r[i]);
|
|
2005
3321
|
});
|
|
2006
|
-
const metricsCompleted =
|
|
2007
|
-
const metricsFailed =
|
|
2008
|
-
const
|
|
3322
|
+
const metricsCompleted = metricPoints2(r[11] ?? []);
|
|
3323
|
+
const metricsFailed = metricPoints2(r[12] ?? []);
|
|
3324
|
+
const prunesCompleted2 = optsPrunesCompleted(typeof r[16] === "string" ? r[16] : null);
|
|
2009
3325
|
const totalCompleted = toIntOrNull(typeof r[17] === "string" ? r[17] : null);
|
|
2010
3326
|
const totalFailed = toIntOrNull(typeof r[18] === "string" ? r[18] : null);
|
|
2011
3327
|
const hasMetrics = totalCompleted !== null || totalFailed !== null;
|
|
@@ -2037,7 +3353,7 @@ function parseStats(reply, withMetrics, windowMinutes) {
|
|
|
2037
3353
|
source: "zset",
|
|
2038
3354
|
// only skewed if the queue really does prune completed jobs AND there are
|
|
2039
3355
|
// failures to unbalance the ratio; with no failures, 100% is still 100%.
|
|
2040
|
-
retentionSkewed:
|
|
3356
|
+
retentionSkewed: prunesCompleted2 && failed > 0
|
|
2041
3357
|
};
|
|
2042
3358
|
}
|
|
2043
3359
|
const stats = {
|
|
@@ -2080,7 +3396,7 @@ var JOB_STATE_SET = new Set(STATE_ORDER);
|
|
|
2080
3396
|
function isJobState(s) {
|
|
2081
3397
|
return JOB_STATE_SET.has(s);
|
|
2082
3398
|
}
|
|
2083
|
-
var
|
|
3399
|
+
var MAX_TREE_CLIMB2 = 50;
|
|
2084
3400
|
function splitJobKey(prefix, fullKey) {
|
|
2085
3401
|
const cut = fullKey.lastIndexOf(":");
|
|
2086
3402
|
if (cut <= 0) return null;
|
|
@@ -2178,7 +3494,7 @@ var RedisInspectorPool = class {
|
|
|
2178
3494
|
inspectors = /* @__PURE__ */ new Map();
|
|
2179
3495
|
get(config) {
|
|
2180
3496
|
const existing = this.inspectors.get(config.id);
|
|
2181
|
-
if (existing &&
|
|
3497
|
+
if (existing && sameTarget2(existing.config, config)) return existing;
|
|
2182
3498
|
if (existing) {
|
|
2183
3499
|
void existing.close().catch(() => void 0);
|
|
2184
3500
|
}
|
|
@@ -2202,11 +3518,26 @@ var RedisInspectorPool = class {
|
|
|
2202
3518
|
return [...this.inspectors.keys()];
|
|
2203
3519
|
}
|
|
2204
3520
|
};
|
|
2205
|
-
function
|
|
3521
|
+
function sameTarget2(a, b) {
|
|
2206
3522
|
return a.url === b.url && a.prefix === (b.prefix ?? "bull") && a.cluster === (b.cluster ?? false) && (a.queueFilter ?? null) === (b.queueFilter ?? null);
|
|
2207
3523
|
}
|
|
2208
|
-
|
|
2209
|
-
|
|
3524
|
+
|
|
3525
|
+
// apps/server/src/services/inspectorPool.ts
|
|
3526
|
+
function createInspectorPool2(options = {}) {
|
|
3527
|
+
const redis = new RedisInspectorPool(options);
|
|
3528
|
+
const postgres = new PgInspectorPool(options);
|
|
3529
|
+
const pick = (config) => config.kind === "postgres" ? postgres : redis;
|
|
3530
|
+
return {
|
|
3531
|
+
get: (config) => pick(config).get(config),
|
|
3532
|
+
// A connection's kind never changes (updateConnectionSchema omits it), but
|
|
3533
|
+
// evicting from both costs nothing and cannot leave a stale inspector behind.
|
|
3534
|
+
evict: async (id3) => {
|
|
3535
|
+
await Promise.all([redis.evict(id3), postgres.evict(id3)]);
|
|
3536
|
+
},
|
|
3537
|
+
closeAll: async () => {
|
|
3538
|
+
await Promise.all([redis.closeAll(), postgres.closeAll()]);
|
|
3539
|
+
}
|
|
3540
|
+
};
|
|
2210
3541
|
}
|
|
2211
3542
|
|
|
2212
3543
|
// apps/server/src/app.ts
|
|
@@ -2487,6 +3818,8 @@ var ssoProviders = mysqlTable(
|
|
|
2487
3818
|
var connections = mysqlTable("connections", {
|
|
2488
3819
|
id: id(),
|
|
2489
3820
|
name: varchar("name", { length: 80 }).notNull(),
|
|
3821
|
+
/** "redis" | "postgres" (BullMQ 6's Postgres backend). `prefix` holds the Postgres schema. */
|
|
3822
|
+
kind: varchar("kind", { length: 16 }).$type().notNull().default("redis"),
|
|
2490
3823
|
url: text("url").notNull(),
|
|
2491
3824
|
prefix: varchar("prefix", { length: 64 }).notNull().default("bull"),
|
|
2492
3825
|
cluster: boolean("cluster").notNull().default(false),
|
|
@@ -2713,6 +4046,7 @@ var ssoProviders2 = sqliteTable(
|
|
|
2713
4046
|
var connections2 = sqliteTable("connections", {
|
|
2714
4047
|
id: id2(),
|
|
2715
4048
|
name: text2("name").notNull(),
|
|
4049
|
+
kind: text2("kind").$type().notNull().default("redis"),
|
|
2716
4050
|
url: text2("url").notNull(),
|
|
2717
4051
|
prefix: text2("prefix").notNull().default("bull"),
|
|
2718
4052
|
cluster: bool("cluster").notNull().default(false),
|
|
@@ -2941,8 +4275,8 @@ var readOnlyLocked = () => new HttpError(423, "read_only", "This instance runs i
|
|
|
2941
4275
|
var invalidLicense = (reason) => new HttpError(400, "invalid_license", `Invalid license: ${reason}`);
|
|
2942
4276
|
var licenseAlreadyActivated = (message) => new HttpError(409, "license_already_activated", message);
|
|
2943
4277
|
var licenseServerUnavailable = (reason) => new HttpError(502, "license_server_unavailable", `Could not reach the license server: ${reason}`);
|
|
2944
|
-
var redisUnavailable = (err) => new HttpError(502, "redis_unavailable", `Redis connection failed: ${
|
|
2945
|
-
function
|
|
4278
|
+
var redisUnavailable = (err) => new HttpError(502, "redis_unavailable", `Redis connection failed: ${errorMessage3(err)}`);
|
|
4279
|
+
function errorMessage3(err) {
|
|
2946
4280
|
if (err instanceof Error) return err.message;
|
|
2947
4281
|
return typeof err === "string" ? err : JSON.stringify(err);
|
|
2948
4282
|
}
|
|
@@ -2986,7 +4320,7 @@ function registerErrorHandling(app) {
|
|
|
2986
4320
|
if (mapped.unexpected) {
|
|
2987
4321
|
request.log.error({ err }, "unhandled error");
|
|
2988
4322
|
} else if (mapped.status >= 500) {
|
|
2989
|
-
request.log.warn({ err:
|
|
4323
|
+
request.log.warn({ err: errorMessage3(err), url: request.url }, mapped.body.error);
|
|
2990
4324
|
}
|
|
2991
4325
|
void reply.status(mapped.status).send(mapped.body);
|
|
2992
4326
|
});
|
|
@@ -4890,7 +6224,17 @@ var CONNECTION_ERROR_PATTERNS = [
|
|
|
4890
6224
|
/CLUSTERDOWN/i,
|
|
4891
6225
|
/Failed to refresh slots cache/i,
|
|
4892
6226
|
/All sentinels are unreachable/i,
|
|
4893
|
-
/socket closed unexpectedly/i
|
|
6227
|
+
/socket closed unexpectedly/i,
|
|
6228
|
+
// Postgres (node-postgres / BullMQ's Postgres backend)
|
|
6229
|
+
/password authentication failed/i,
|
|
6230
|
+
/no pg_hba\.conf entry/i,
|
|
6231
|
+
/Connection terminated/i,
|
|
6232
|
+
/timeout exceeded when trying to connect/i,
|
|
6233
|
+
/the database system is (starting up|shutting down)/i,
|
|
6234
|
+
/too many clients already/i,
|
|
6235
|
+
/postgres_schema_missing/,
|
|
6236
|
+
/the PostgreSQL schema requires BullMQ/i,
|
|
6237
|
+
/the PostgreSQL backend requires server version/i
|
|
4894
6238
|
];
|
|
4895
6239
|
var NOT_FOUND_PATTERNS = [/missing key for job/i, /job .* not found/i, /could not be found/i, /does not exist/i];
|
|
4896
6240
|
function isRedisConnectionError(err) {
|
|
@@ -4899,14 +6243,26 @@ function isRedisConnectionError(err) {
|
|
|
4899
6243
|
if (name === "MaxRetriesPerRequestError" || name === "ClusterAllFailedError") return true;
|
|
4900
6244
|
const code = err.code;
|
|
4901
6245
|
if (code && CONNECTION_ERROR_PATTERNS.some((p) => p.test(code))) return true;
|
|
6246
|
+
if (code && /^(08...|28...|3D000|57P0\d)$/.test(code)) return true;
|
|
4902
6247
|
}
|
|
4903
|
-
const message =
|
|
6248
|
+
const message = errorMessage3(err);
|
|
4904
6249
|
return CONNECTION_ERROR_PATTERNS.some((p) => p.test(message));
|
|
4905
6250
|
}
|
|
6251
|
+
function isPermissionDenied(err) {
|
|
6252
|
+
const code = typeof err === "object" && err !== null ? err.code : void 0;
|
|
6253
|
+
return code === "42501" || /permission denied for (table|schema|function|sequence|relation)/i.test(errorMessage3(err));
|
|
6254
|
+
}
|
|
4906
6255
|
function mapInspectorError(err) {
|
|
4907
6256
|
if (err instanceof HttpError) return err;
|
|
4908
6257
|
if (isRedisConnectionError(err)) return redisUnavailable(err);
|
|
4909
|
-
|
|
6258
|
+
if (isPermissionDenied(err)) {
|
|
6259
|
+
return new HttpError(
|
|
6260
|
+
403,
|
|
6261
|
+
"database_permission_denied",
|
|
6262
|
+
`The database refused the action: ${errorMessage3(err)}. This connection's role can read but not write; give it the same rights your workers have to use actions from Bullpane.`
|
|
6263
|
+
);
|
|
6264
|
+
}
|
|
6265
|
+
const message = errorMessage3(err);
|
|
4910
6266
|
if (NOT_FOUND_PATTERNS.some((p) => p.test(message))) return notFound("Job");
|
|
4911
6267
|
return conflict(message);
|
|
4912
6268
|
}
|
|
@@ -4930,14 +6286,14 @@ async function connectionRoutes(app) {
|
|
|
4930
6286
|
const input = createConnectionSchema.parse(request.body);
|
|
4931
6287
|
const created = await app.ctx.connections.create(input);
|
|
4932
6288
|
request.auditTarget({ connectionId: created.id, connectionName: created.name });
|
|
4933
|
-
request.auditDetail({ prefix: created.prefix, cluster: created.cluster, queueFilter: created.queueFilter });
|
|
6289
|
+
request.auditDetail({ kind: created.kind, prefix: created.prefix, cluster: created.cluster, queueFilter: created.queueFilter });
|
|
4934
6290
|
reply.status(201);
|
|
4935
6291
|
return created;
|
|
4936
6292
|
});
|
|
4937
6293
|
app.post("/connections/test", { preHandler: [admin] }, async (request) => {
|
|
4938
6294
|
const input = testConnectionSchema.parse(request.body);
|
|
4939
6295
|
const id3 = `test:${Date.now()}:${Math.random().toString(36).slice(2)}`;
|
|
4940
|
-
const inspector = app.ctx.pool.get({ id: id3, url: input.url, prefix: input.prefix
|
|
6296
|
+
const inspector = app.ctx.pool.get({ id: id3, kind: input.kind, url: input.url, prefix: input.prefix, cluster: input.cluster ?? false });
|
|
4941
6297
|
try {
|
|
4942
6298
|
return await inspector.ping();
|
|
4943
6299
|
} catch (err) {
|
|
@@ -6197,6 +7553,7 @@ function toConnectionDto(row, status) {
|
|
|
6197
7553
|
const dto = {
|
|
6198
7554
|
id: row.id,
|
|
6199
7555
|
name: row.name,
|
|
7556
|
+
kind: row.kind,
|
|
6200
7557
|
url: redactRedisUrl(row.url),
|
|
6201
7558
|
prefix: row.prefix,
|
|
6202
7559
|
cluster: row.cluster,
|
|
@@ -6258,6 +7615,7 @@ var ConnectionsService = class {
|
|
|
6258
7615
|
await this.db.insert(connections3).values({
|
|
6259
7616
|
id: id3,
|
|
6260
7617
|
name: input.name,
|
|
7618
|
+
kind: input.kind,
|
|
6261
7619
|
url: input.url,
|
|
6262
7620
|
prefix: input.prefix,
|
|
6263
7621
|
cluster: input.cluster,
|
|
@@ -6295,7 +7653,9 @@ var ConnectionsService = class {
|
|
|
6295
7653
|
return this.list();
|
|
6296
7654
|
}
|
|
6297
7655
|
async update(id3, input) {
|
|
6298
|
-
await this.getRow(id3);
|
|
7656
|
+
const current2 = await this.getRow(id3);
|
|
7657
|
+
const issue = connectionIssues({ ...input, kind: current2.kind })[0];
|
|
7658
|
+
if (issue) throw validation(issue.message, [{ path: [issue.path], message: issue.message }]);
|
|
6299
7659
|
const patch = {};
|
|
6300
7660
|
if (input.name !== void 0) patch.name = input.name;
|
|
6301
7661
|
if (input.url !== void 0) patch.url = input.url;
|
|
@@ -6325,6 +7685,7 @@ var ConnectionsService = class {
|
|
|
6325
7685
|
inspectorFor(row) {
|
|
6326
7686
|
return this.pool.get({
|
|
6327
7687
|
id: row.id,
|
|
7688
|
+
kind: row.kind,
|
|
6328
7689
|
url: row.url,
|
|
6329
7690
|
prefix: row.prefix,
|
|
6330
7691
|
cluster: row.cluster,
|
|
@@ -6881,7 +8242,7 @@ var EditionService = class {
|
|
|
6881
8242
|
if (!this.needsRefresh(boot)) return;
|
|
6882
8243
|
await this.refresh();
|
|
6883
8244
|
} catch (err) {
|
|
6884
|
-
this.log.warn({ err:
|
|
8245
|
+
this.log.warn({ err: errorMessage3(err) }, "license refresh failed");
|
|
6885
8246
|
}
|
|
6886
8247
|
}
|
|
6887
8248
|
async doRefresh() {
|
|
@@ -6947,7 +8308,7 @@ var EditionService = class {
|
|
|
6947
8308
|
await this.client.deactivate({ key, activationId: state.activationId });
|
|
6948
8309
|
this.log.info({ activationId: state.activationId }, "license activation released");
|
|
6949
8310
|
} catch (err) {
|
|
6950
|
-
this.log.warn({ err:
|
|
8311
|
+
this.log.warn({ err: errorMessage3(err) }, "could not release the license activation; free it from the customer portal if needed");
|
|
6951
8312
|
}
|
|
6952
8313
|
}
|
|
6953
8314
|
rebuild() {
|
|
@@ -7002,7 +8363,7 @@ function toCheckError(err) {
|
|
|
7002
8363
|
};
|
|
7003
8364
|
}
|
|
7004
8365
|
if (err instanceof HttpError) return { code: err.error, message: err.message, definitive: false };
|
|
7005
|
-
return { code: "unknown", message:
|
|
8366
|
+
return { code: "unknown", message: errorMessage3(err), definitive: false };
|
|
7006
8367
|
}
|
|
7007
8368
|
function toHttpError(err) {
|
|
7008
8369
|
if (err instanceof HttpError) return err;
|
|
@@ -7020,7 +8381,7 @@ function toHttpError(err) {
|
|
|
7020
8381
|
return licenseServerUnavailable(err.message);
|
|
7021
8382
|
}
|
|
7022
8383
|
}
|
|
7023
|
-
return licenseServerUnavailable(
|
|
8384
|
+
return licenseServerUnavailable(errorMessage3(err));
|
|
7024
8385
|
}
|
|
7025
8386
|
|
|
7026
8387
|
// apps/server/src/services/attention.ts
|
|
@@ -7311,6 +8672,7 @@ function toFolderDto(row, queues) {
|
|
|
7311
8672
|
// apps/server/src/services/health.ts
|
|
7312
8673
|
var MIN_SAMPLE_INTERVAL_MS = 2e3;
|
|
7313
8674
|
var HISTORY_POINTS = 200;
|
|
8675
|
+
var EVENT_TABLE_WARN_BYTES = 1024 * 1024 * 1024;
|
|
7314
8676
|
var HealthService = class {
|
|
7315
8677
|
constructor(connections5) {
|
|
7316
8678
|
this.connections = connections5;
|
|
@@ -7328,10 +8690,11 @@ var HealthService = class {
|
|
|
7328
8690
|
const st = this.stateFor(row.id);
|
|
7329
8691
|
const snap = await this.sample(row, st);
|
|
7330
8692
|
const info = snap.info;
|
|
7331
|
-
const memoryUsedPct = info && info.maxMemoryBytes && info.maxMemoryBytes > 0 ? Math.round(info.usedMemoryBytes / info.maxMemoryBytes * 1e3) / 10 : null;
|
|
8693
|
+
const memoryUsedPct = info && info.backend !== "postgres" && info.maxMemoryBytes && info.maxMemoryBytes > 0 ? Math.round(info.usedMemoryBytes / info.maxMemoryBytes * 1e3) / 10 : null;
|
|
7332
8694
|
return {
|
|
7333
8695
|
connectionId: row.id,
|
|
7334
8696
|
connectionName: row.name,
|
|
8697
|
+
kind: row.kind,
|
|
7335
8698
|
ok: snap.error === null,
|
|
7336
8699
|
error: snap.error,
|
|
7337
8700
|
info,
|
|
@@ -7339,7 +8702,7 @@ var HealthService = class {
|
|
|
7339
8702
|
cpuCores: snap.cpuCores,
|
|
7340
8703
|
memoryUsedPct,
|
|
7341
8704
|
history: [...st.history],
|
|
7342
|
-
warnings: buildWarnings(info, snap.error, memoryUsedPct)
|
|
8705
|
+
warnings: row.kind === "postgres" ? buildPostgresWarnings(info?.backend === "postgres" ? info : null, snap.error) : buildWarnings(info?.backend === "postgres" ? null : info, snap.error, memoryUsedPct)
|
|
7343
8706
|
};
|
|
7344
8707
|
}
|
|
7345
8708
|
/** Drop cached state when a connection is edited or deleted. */
|
|
@@ -7373,8 +8736,9 @@ var HealthService = class {
|
|
|
7373
8736
|
at,
|
|
7374
8737
|
info,
|
|
7375
8738
|
error: null,
|
|
7376
|
-
|
|
7377
|
-
|
|
8739
|
+
// Postgres: transactions/sec from xact_commit + xact_rollback. No CPU counter there.
|
|
8740
|
+
commandsPerSec: rate(throughputCounter(previous?.info), throughputCounter(info), elapsedSec),
|
|
8741
|
+
cpuCores: info.backend === "postgres" ? null : rate(redisInfo(previous?.info)?.cpuSecondsTotal, info.cpuSecondsTotal, elapsedSec)
|
|
7378
8742
|
};
|
|
7379
8743
|
pushHistory(st, snap2);
|
|
7380
8744
|
return snap2;
|
|
@@ -7402,12 +8766,20 @@ function rate(before, after, elapsedSec) {
|
|
|
7402
8766
|
if (delta < 0) return null;
|
|
7403
8767
|
return Math.round(delta / elapsedSec * 100) / 100;
|
|
7404
8768
|
}
|
|
8769
|
+
function redisInfo(info) {
|
|
8770
|
+
return info && info.backend !== "postgres" ? info : null;
|
|
8771
|
+
}
|
|
8772
|
+
function throughputCounter(info) {
|
|
8773
|
+
if (!info) return null;
|
|
8774
|
+
return info.backend === "postgres" ? info.totalTransactions : info.totalCommandsProcessed;
|
|
8775
|
+
}
|
|
7405
8776
|
function pushHistory(st, snap) {
|
|
7406
8777
|
if (!snap.info) return;
|
|
7407
8778
|
st.history.push({
|
|
7408
8779
|
t: snap.at,
|
|
7409
8780
|
latencyMs: snap.info.latencyMs,
|
|
7410
|
-
|
|
8781
|
+
// Postgres has no memory ceiling to plot; the database size is what grows.
|
|
8782
|
+
memoryBytes: snap.info.backend === "postgres" ? snap.info.databaseSizeBytes : snap.info.usedMemoryBytes,
|
|
7411
8783
|
commandsPerSec: snap.commandsPerSec,
|
|
7412
8784
|
cpuCores: snap.cpuCores,
|
|
7413
8785
|
connectedClients: snap.info.connectedClients
|
|
@@ -7463,6 +8835,34 @@ function buildWarnings(info, error, memoryUsedPct) {
|
|
|
7463
8835
|
}
|
|
7464
8836
|
return out;
|
|
7465
8837
|
}
|
|
8838
|
+
function buildPostgresWarnings(info, error) {
|
|
8839
|
+
if (error !== null || info === null) {
|
|
8840
|
+
const detail = error?.trim();
|
|
8841
|
+
return [{ level: "critical", code: "unreachable", message: detail ? `Postgres is unreachable: ${detail}` : "Postgres is unreachable" }];
|
|
8842
|
+
}
|
|
8843
|
+
const out = [];
|
|
8844
|
+
if (info.maxConnections > 0 && info.connectedClients / info.maxConnections >= 0.9) {
|
|
8845
|
+
out.push({
|
|
8846
|
+
level: "critical",
|
|
8847
|
+
code: "connections_high",
|
|
8848
|
+
message: `${info.connectedClients} of ${info.maxConnections} connections in use: new workers will be refused`
|
|
8849
|
+
});
|
|
8850
|
+
} else if (info.maxConnections > 0 && info.connectedClients / info.maxConnections >= 0.75) {
|
|
8851
|
+
out.push({ level: "warn", code: "connections_high", message: `${info.connectedClients} of ${info.maxConnections} connections in use` });
|
|
8852
|
+
}
|
|
8853
|
+
if (info.eventTableBytes >= EVENT_TABLE_WARN_BYTES) {
|
|
8854
|
+
const gib = Math.round(info.eventTableBytes / 1024 ** 3 * 10) / 10;
|
|
8855
|
+
out.push({
|
|
8856
|
+
level: "warn",
|
|
8857
|
+
code: "event_table_large",
|
|
8858
|
+
message: `BullMQ's event table is ${gib} GiB and is never trimmed by BullMQ 6: delete old rows on a schedule`
|
|
8859
|
+
});
|
|
8860
|
+
}
|
|
8861
|
+
if (info.latencyMs >= 250) {
|
|
8862
|
+
out.push({ level: "warn", code: "latency_high", message: `Health query round trip took ${info.latencyMs} ms` });
|
|
8863
|
+
}
|
|
8864
|
+
return out;
|
|
8865
|
+
}
|
|
7466
8866
|
|
|
7467
8867
|
// apps/server/src/ee/services/sso.ts
|
|
7468
8868
|
import { asc as asc2, eq as eq8 } from "drizzle-orm";
|
|
@@ -8493,13 +9893,13 @@ async function waitForDatabase(database2, log, opts = {}) {
|
|
|
8493
9893
|
lastError = err;
|
|
8494
9894
|
if (database2.dialect === "sqlite") break;
|
|
8495
9895
|
const elapsed = Math.round((Date.now() - startedAt) / 1e3);
|
|
8496
|
-
log.info(`${name} not reachable yet (${elapsed}s): ${
|
|
9896
|
+
log.info(`${name} not reachable yet (${elapsed}s): ${errorMessage4(err)}. Retrying in ${intervalMs / 1e3}s...`);
|
|
8497
9897
|
await new Promise((r) => setTimeout(r, intervalMs));
|
|
8498
9898
|
}
|
|
8499
9899
|
}
|
|
8500
|
-
throw new Error(`${name} did not become reachable within ${timeoutMs / 1e3}s: ${
|
|
9900
|
+
throw new Error(`${name} did not become reachable within ${timeoutMs / 1e3}s: ${errorMessage4(lastError)}`);
|
|
8501
9901
|
}
|
|
8502
|
-
function
|
|
9902
|
+
function errorMessage4(err) {
|
|
8503
9903
|
return err instanceof Error ? err.message : String(err);
|
|
8504
9904
|
}
|
|
8505
9905
|
|
|
@@ -8572,6 +9972,7 @@ async function seedDemo(ctx, log) {
|
|
|
8572
9972
|
if (!connection && await ctx.connections.count() === 0) {
|
|
8573
9973
|
const created = await ctx.connections.create({
|
|
8574
9974
|
name: DEMO_CONNECTION_NAME,
|
|
9975
|
+
kind: "redis",
|
|
8575
9976
|
url: config.demoRedisUrl,
|
|
8576
9977
|
prefix: "bull",
|
|
8577
9978
|
cluster: false,
|
|
@@ -8637,7 +10038,7 @@ async function seedConnections(connections5, wanted, log) {
|
|
|
8637
10038
|
async function main() {
|
|
8638
10039
|
const config = loadConfig(process.env, { warn: (m) => console.warn(`[config] ${m}`) });
|
|
8639
10040
|
const database2 = createDatabase(config.database);
|
|
8640
|
-
const pool =
|
|
10041
|
+
const pool = createInspectorPool2({
|
|
8641
10042
|
discoveryTtlMs: config.queueDiscoveryTtl * 1e3,
|
|
8642
10043
|
previewBytes: config.jobPreviewBytes
|
|
8643
10044
|
});
|