bullpane 0.5.2 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/bin/bullpane.mjs +28 -1
- package/dist/lua/searchJobs.lua +37 -7
- package/dist/server.mjs +1535 -111
- package/migrations/mysql/0010_connection_kind.sql +4 -0
- package/migrations/sqlite/0003_connection_kind.sql +3 -0
- package/package.json +5 -2
- package/web/assets/{FlowsPage-CxMz_9Et.js → FlowsPage-BAChK8uT.js} +1 -1
- package/web/assets/{JobTreePage-BT79W1zX.js → JobTreePage-Vvq6aoQU.js} +1 -1
- package/web/assets/{McpConsentPage-B2YnL6fN.js → McpConsentPage-BHmbIbrR.js} +1 -1
- package/web/assets/{flowLayout-DoC2dCG0.js → flowLayout-g8b3XZHS.js} +1 -1
- package/web/assets/index-BuZ0fAkt.js +548 -0
- package/web/index.html +1 -1
- package/web/assets/index-DuAopWKm.js +0 -543
package/dist/server.mjs
CHANGED
|
@@ -4,9 +4,16 @@ var __export = (target, all) => {
|
|
|
4
4
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
5
5
|
};
|
|
6
6
|
|
|
7
|
-
// packages/
|
|
8
|
-
import {
|
|
9
|
-
|
|
7
|
+
// packages/pg-inspector/src/inspector.ts
|
|
8
|
+
import {
|
|
9
|
+
assertSchemaCompatibility,
|
|
10
|
+
createPostgresBackend,
|
|
11
|
+
Job,
|
|
12
|
+
Queue,
|
|
13
|
+
SchemaMigrationRequiredError,
|
|
14
|
+
UnrecoverableError
|
|
15
|
+
} from "bullmq";
|
|
16
|
+
import pg from "pg";
|
|
10
17
|
|
|
11
18
|
// packages/shared/src/index.ts
|
|
12
19
|
import { z } from "zod";
|
|
@@ -142,21 +149,53 @@ var ssoSettingsSchema = z.object({
|
|
|
142
149
|
autoProvisionDomains: z.array(ssoDomainSchema).max(50).optional()
|
|
143
150
|
});
|
|
144
151
|
var SSO_AUTO_PROVISION_ROLE = "viewer";
|
|
145
|
-
var
|
|
152
|
+
var CONNECTION_KINDS = ["redis", "postgres"];
|
|
153
|
+
var DEFAULT_NAMESPACE = {
|
|
154
|
+
redis: "bull",
|
|
155
|
+
postgres: "bullmq"
|
|
156
|
+
};
|
|
157
|
+
var URL_SCHEMES = {
|
|
158
|
+
redis: { test: /^rediss?:\/\//, message: "Must start with redis:// or rediss://" },
|
|
159
|
+
postgres: { test: /^postgres(ql)?:\/\//, message: "Must start with postgres:// or postgresql://" }
|
|
160
|
+
};
|
|
161
|
+
var PG_SCHEMA_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
|
162
|
+
function connectionIssues(v) {
|
|
163
|
+
const issues = [];
|
|
164
|
+
const kind = v.kind ?? "redis";
|
|
165
|
+
if (v.url !== void 0 && !URL_SCHEMES[kind].test.test(v.url)) {
|
|
166
|
+
issues.push({ path: "url", message: URL_SCHEMES[kind].message });
|
|
167
|
+
}
|
|
168
|
+
if (kind === "postgres") {
|
|
169
|
+
if (v.prefix !== void 0 && !PG_SCHEMA_RE.test(v.prefix)) {
|
|
170
|
+
issues.push({ path: "prefix", message: "A Postgres schema: letters, digits and _, not starting with a digit" });
|
|
171
|
+
}
|
|
172
|
+
if (v.cluster === true) issues.push({ path: "cluster", message: "Cluster mode is Redis only" });
|
|
173
|
+
}
|
|
174
|
+
return issues;
|
|
175
|
+
}
|
|
176
|
+
var connectionFields = z.object({
|
|
146
177
|
name: z.string().min(1).max(80),
|
|
147
|
-
|
|
148
|
-
|
|
178
|
+
kind: z.enum(CONNECTION_KINDS).default("redis"),
|
|
179
|
+
url: z.string().min(1),
|
|
180
|
+
/** omitted → DEFAULT_NAMESPACE of the kind */
|
|
181
|
+
prefix: z.string().min(1).max(64).optional(),
|
|
149
182
|
cluster: z.boolean().default(false),
|
|
150
183
|
queueFilter: z.string().max(200).nullable().optional()
|
|
151
184
|
});
|
|
152
|
-
var
|
|
185
|
+
var createConnectionSchema = connectionFields.superRefine((v, ctx) => {
|
|
186
|
+
for (const issue of connectionIssues(v)) ctx.addIssue({ code: z.ZodIssueCode.custom, path: [issue.path], message: issue.message });
|
|
187
|
+
}).transform((v) => ({ ...v, prefix: v.prefix ?? DEFAULT_NAMESPACE[v.kind] }));
|
|
188
|
+
var updateConnectionSchema = connectionFields.omit({ kind: true }).partial();
|
|
153
189
|
var reorderSchema = z.object({
|
|
154
190
|
ids: z.array(z.string().min(1)).min(1)
|
|
155
191
|
});
|
|
156
192
|
var testConnectionSchema = z.object({
|
|
193
|
+
kind: z.enum(CONNECTION_KINDS).default("redis"),
|
|
157
194
|
url: z.string().min(1),
|
|
158
195
|
prefix: z.string().optional(),
|
|
159
196
|
cluster: z.boolean().optional()
|
|
197
|
+
}).superRefine((v, ctx) => {
|
|
198
|
+
for (const issue of connectionIssues(v)) ctx.addIssue({ code: z.ZodIssueCode.custom, path: [issue.path], message: issue.message });
|
|
160
199
|
});
|
|
161
200
|
var JOB_STATES = [
|
|
162
201
|
"waiting",
|
|
@@ -215,10 +254,16 @@ var listJobsQuerySchema = z.object({
|
|
|
215
254
|
});
|
|
216
255
|
var searchJobsQuerySchema = z.object({
|
|
217
256
|
state: jobStateSchema.default("failed"),
|
|
218
|
-
q: z.string().
|
|
257
|
+
q: z.string().max(500).default(""),
|
|
258
|
+
/**
|
|
259
|
+
* BullMQ Pro: only jobs of this group (exact id). A delayed, failed or completed
|
|
260
|
+
* job sits in the queue-wide state key, not under its group, so this is the only
|
|
261
|
+
* way to list a group's jobs outside `waiting`. With it, `q` may be empty.
|
|
262
|
+
*/
|
|
263
|
+
groupId: z.string().min(1).max(200).optional(),
|
|
219
264
|
cursor: z.string().optional(),
|
|
220
265
|
limit: z.coerce.number().int().min(1).max(200).default(50)
|
|
221
|
-
});
|
|
266
|
+
}).refine((v) => v.q.length > 0 || v.groupId !== void 0, { message: "q or groupId is required", path: ["q"] });
|
|
222
267
|
var hideQueueSchema = z.object({
|
|
223
268
|
queueName: z.string().min(1).max(255)
|
|
224
269
|
});
|
|
@@ -427,34 +472,1317 @@ var EMPTY_COUNTS = {
|
|
|
427
472
|
paused: 0,
|
|
428
473
|
"waiting-children": 0
|
|
429
474
|
};
|
|
430
|
-
var ROLE_RANK = { admin: 3, operator: 2, viewer: 1 };
|
|
431
|
-
function hasRole(userRole, required) {
|
|
432
|
-
return ROLE_RANK[userRole] >= ROLE_RANK[required];
|
|
433
|
-
}
|
|
434
|
-
function redactRedisUrl(url) {
|
|
435
|
-
try {
|
|
436
|
-
const u = new URL(url);
|
|
437
|
-
if (u.password) u.password = "****";
|
|
438
|
-
return u.toString();
|
|
439
|
-
} catch {
|
|
440
|
-
return url.replace(/:\/\/([^:@/]*):([^@/]*)@/, "://$1:****@");
|
|
475
|
+
var ROLE_RANK = { admin: 3, operator: 2, viewer: 1 };
|
|
476
|
+
function hasRole(userRole, required) {
|
|
477
|
+
return ROLE_RANK[userRole] >= ROLE_RANK[required];
|
|
478
|
+
}
|
|
479
|
+
function redactRedisUrl(url) {
|
|
480
|
+
try {
|
|
481
|
+
const u = new URL(url);
|
|
482
|
+
if (u.password) u.password = "****";
|
|
483
|
+
return u.toString();
|
|
484
|
+
} catch {
|
|
485
|
+
return url.replace(/:\/\/([^:@/]*):([^@/]*)@/, "://$1:****@");
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
var MCP_ACCESS_LEVELS = ["off", "read", "write"];
|
|
489
|
+
var MCP_ACCESS_RANK = { off: 0, read: 1, write: 2 };
|
|
490
|
+
function mcpRoleCeiling(role) {
|
|
491
|
+
return hasRole(role, "operator") ? "write" : "read";
|
|
492
|
+
}
|
|
493
|
+
function mcpEffectiveAccess(ceiling, granted, role) {
|
|
494
|
+
const levels = [ceiling, granted, mcpRoleCeiling(role)];
|
|
495
|
+
return levels.reduce((min, l) => MCP_ACCESS_RANK[l] < MCP_ACCESS_RANK[min] ? l : min);
|
|
496
|
+
}
|
|
497
|
+
var updateMcpSettingsSchema = z.object({ maxAccess: z.enum(MCP_ACCESS_LEVELS) });
|
|
498
|
+
var mcpConsentDecisionSchema = z.object({
|
|
499
|
+
request: z.string().min(1).max(4096),
|
|
500
|
+
approve: z.boolean(),
|
|
501
|
+
access: z.enum(["read", "write"]).default("read")
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
// packages/inspector/src/glob.ts
|
|
505
|
+
function globToRegExp(glob) {
|
|
506
|
+
const escaped = glob.split("").map((ch) => {
|
|
507
|
+
if (ch === "*") return ".*";
|
|
508
|
+
if (ch === "?") return ".";
|
|
509
|
+
return ch.replace(/[.+^${}()|[\]\\/]/g, "\\$&");
|
|
510
|
+
}).join("");
|
|
511
|
+
return new RegExp(`^${escaped}$`);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// packages/pg-inspector/src/rows.ts
|
|
515
|
+
function num(v, fallback = 0) {
|
|
516
|
+
if (typeof v === "number") return Number.isFinite(v) ? v : fallback;
|
|
517
|
+
if (typeof v === "string" && v !== "") {
|
|
518
|
+
const n = Number(v);
|
|
519
|
+
return Number.isFinite(n) ? n : fallback;
|
|
520
|
+
}
|
|
521
|
+
return fallback;
|
|
522
|
+
}
|
|
523
|
+
function numOrNull(v) {
|
|
524
|
+
if (v === null || v === void 0 || v === "") return null;
|
|
525
|
+
const n = num(v, Number.NaN);
|
|
526
|
+
return Number.isNaN(n) ? null : n;
|
|
527
|
+
}
|
|
528
|
+
function queueKeyOf(schema, queue) {
|
|
529
|
+
return `${schema}:${queue}`;
|
|
530
|
+
}
|
|
531
|
+
function jobKeyOf(schema, queue, id3) {
|
|
532
|
+
return `${schema}:${queue}:${id3}`;
|
|
533
|
+
}
|
|
534
|
+
function parentRef(schema, parentQueue, parentId) {
|
|
535
|
+
if (typeof parentQueue !== "string" || parentQueue === "" || parentId === null || parentId === void 0) return null;
|
|
536
|
+
return { id: String(parentId), queueKey: queueKeyOf(schema, parentQueue), queue: parentQueue };
|
|
537
|
+
}
|
|
538
|
+
function bullpaneState(physical, priority) {
|
|
539
|
+
switch (physical) {
|
|
540
|
+
case "waiting":
|
|
541
|
+
return num(priority) > 0 ? "prioritized" : "waiting";
|
|
542
|
+
case "active":
|
|
543
|
+
case "completed":
|
|
544
|
+
case "failed":
|
|
545
|
+
case "delayed":
|
|
546
|
+
case "waiting-children":
|
|
547
|
+
return physical;
|
|
548
|
+
default:
|
|
549
|
+
return "unknown";
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
function progressOf(v) {
|
|
553
|
+
if (v === null || v === void 0) return 0;
|
|
554
|
+
if (typeof v === "number" || typeof v === "string") return v;
|
|
555
|
+
if (typeof v === "object" && !Array.isArray(v)) return v;
|
|
556
|
+
return JSON.stringify(v);
|
|
557
|
+
}
|
|
558
|
+
function attemptsOf(v) {
|
|
559
|
+
return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : null;
|
|
560
|
+
}
|
|
561
|
+
function rowToSummary(schema, row, previewBytes) {
|
|
562
|
+
const state = bullpaneState(row.state, row.priority);
|
|
563
|
+
const read = row.data_preview !== null && row.data_preview !== void 0;
|
|
564
|
+
const bytes = read ? num(row.data_bytes) : num(row.data_stored);
|
|
565
|
+
return {
|
|
566
|
+
id: String(row.id),
|
|
567
|
+
name: row.name ?? "",
|
|
568
|
+
timestamp: num(row.added_at_ms),
|
|
569
|
+
processedOn: numOrNull(row.processed_at_ms),
|
|
570
|
+
finishedOn: numOrNull(row.finished_at_ms),
|
|
571
|
+
attemptsMade: num(row.attempts_made),
|
|
572
|
+
attempts: attemptsOf(row.opt_attempts),
|
|
573
|
+
failedReason: row.failed_reason ?? null,
|
|
574
|
+
progress: progressOf(row.progress),
|
|
575
|
+
delay: num(row.delay_ms),
|
|
576
|
+
// Postgres stores the due time itself (process_at_ms): no score decoding.
|
|
577
|
+
delayedUntil: state === "delayed" ? numOrNull(row.process_at_ms) : null,
|
|
578
|
+
priority: num(row.priority),
|
|
579
|
+
dataPreview: read ? row.data_preview : "",
|
|
580
|
+
// Above the list cap the payload was not read at all; `dataBytes` is then the
|
|
581
|
+
// stored (possibly compressed) size, which is what pg_column_size can tell
|
|
582
|
+
// without detoasting it.
|
|
583
|
+
dataTruncated: !read || bytes > previewBytes,
|
|
584
|
+
dataBytes: bytes,
|
|
585
|
+
parent: parentRef(schema, row.parent_queue, row.parent_id),
|
|
586
|
+
groupId: row.group_id ?? null,
|
|
587
|
+
repeatJobKey: row.scheduler_id ?? null,
|
|
588
|
+
stalledCounter: num(row.stalled_count),
|
|
589
|
+
state
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
function rowToDetail(schema, row) {
|
|
593
|
+
const state = bullpaneState(row.state, row.priority);
|
|
594
|
+
const opts = row.opts && typeof row.opts === "object" ? row.opts : {};
|
|
595
|
+
const group = opts.group;
|
|
596
|
+
const processed = num(row.processed);
|
|
597
|
+
const unprocessed = num(row.unprocessed);
|
|
598
|
+
return {
|
|
599
|
+
id: String(row.id),
|
|
600
|
+
name: row.name ?? "",
|
|
601
|
+
timestamp: num(row.added_at_ms),
|
|
602
|
+
processedOn: numOrNull(row.processed_at_ms),
|
|
603
|
+
finishedOn: numOrNull(row.finished_at_ms),
|
|
604
|
+
attemptsMade: num(row.attempts_made),
|
|
605
|
+
attempts: attemptsOf(opts.attempts),
|
|
606
|
+
failedReason: row.failed_reason ?? null,
|
|
607
|
+
progress: progressOf(row.progress),
|
|
608
|
+
delay: num(row.delay_ms),
|
|
609
|
+
delayedUntil: state === "delayed" ? numOrNull(row.process_at_ms) : null,
|
|
610
|
+
priority: num(row.priority),
|
|
611
|
+
dataBytes: numOrNull(row.data_bytes),
|
|
612
|
+
parent: parentRef(schema, row.parent_queue, row.parent_id),
|
|
613
|
+
groupId: group && (typeof group.id === "string" || typeof group.id === "number") ? String(group.id) : null,
|
|
614
|
+
repeatJobKey: row.scheduler_id ?? null,
|
|
615
|
+
stalledCounter: num(row.stalled_count),
|
|
616
|
+
state,
|
|
617
|
+
data: row.data ?? null,
|
|
618
|
+
opts,
|
|
619
|
+
returnvalue: row.return_value ?? null,
|
|
620
|
+
stacktrace: stacktraceOf(row.stacktrace),
|
|
621
|
+
logs: row.logs ?? [],
|
|
622
|
+
logsCount: num(row.logs_count),
|
|
623
|
+
dependencies: processed + unprocessed > 0 ? { processed, unprocessed } : null
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
function stacktraceOf(v) {
|
|
627
|
+
if (v === null || v === void 0) return [];
|
|
628
|
+
if (Array.isArray(v)) return v.map((s) => typeof s === "string" ? s : JSON.stringify(s));
|
|
629
|
+
return [typeof v === "string" ? v : JSON.stringify(v)];
|
|
630
|
+
}
|
|
631
|
+
function rowToScheduler(row) {
|
|
632
|
+
const key = String(row.scheduler_id);
|
|
633
|
+
const name = row.name ?? key;
|
|
634
|
+
return {
|
|
635
|
+
key,
|
|
636
|
+
name,
|
|
637
|
+
next: numOrNull(row.next_run_ms),
|
|
638
|
+
pattern: row.pattern ?? null,
|
|
639
|
+
every: numOrNull(row.every_ms),
|
|
640
|
+
tz: row.tz ?? null,
|
|
641
|
+
offset: numOrNull(row.offset_ms),
|
|
642
|
+
limit: numOrNull(row.limit_count),
|
|
643
|
+
iterationCount: numOrNull(row.iteration_count),
|
|
644
|
+
startDate: numOrNull(row.start_date_ms),
|
|
645
|
+
endDate: numOrNull(row.end_date_ms),
|
|
646
|
+
template: row.template_data !== null || row.template_opts !== null ? { name, data: row.template_data ?? null, opts: row.template_opts ?? null } : null
|
|
647
|
+
};
|
|
648
|
+
}
|
|
649
|
+
function prunesCompleted(v) {
|
|
650
|
+
if (v === void 0 || v === null || v === false) return false;
|
|
651
|
+
if (v === true || typeof v === "number") return true;
|
|
652
|
+
if (typeof v === "object") {
|
|
653
|
+
const o = v;
|
|
654
|
+
return typeof o.count === "number" || typeof o.age === "number";
|
|
655
|
+
}
|
|
656
|
+
return false;
|
|
657
|
+
}
|
|
658
|
+
function metricPoints(newestFirst) {
|
|
659
|
+
if (!Array.isArray(newestFirst)) return [];
|
|
660
|
+
return newestFirst.map((v) => num(v)).reverse();
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// packages/pg-inspector/src/metrics.ts
|
|
664
|
+
function side(row, nowMin) {
|
|
665
|
+
if (!row) {
|
|
666
|
+
return { has: false, read: (w) => ({ total: 0, covered: w }) };
|
|
667
|
+
}
|
|
668
|
+
const count2 = num(row.count);
|
|
669
|
+
const prevTs = numOrNull(row.prevTs);
|
|
670
|
+
if (prevTs === null) return { has: true, read: (w) => ({ total: count2, covered: w }) };
|
|
671
|
+
const m0 = Math.floor(prevTs / 6e4);
|
|
672
|
+
const ref = Math.max(nowMin, m0);
|
|
673
|
+
const pending = count2 - num(row.prevCount);
|
|
674
|
+
const points = (row.data ?? []).map((v) => num(v));
|
|
675
|
+
return {
|
|
676
|
+
has: true,
|
|
677
|
+
read: (w) => {
|
|
678
|
+
const startMin = ref - w + 1;
|
|
679
|
+
if (m0 < startMin) return { total: 0, covered: w };
|
|
680
|
+
const n = Math.min(m0 - startMin, points.length);
|
|
681
|
+
let total = pending;
|
|
682
|
+
for (let i = 0; i < n; i += 1) total += points[i] ?? 0;
|
|
683
|
+
return { total, covered: Math.min(ref - m0 + 1 + n, w) };
|
|
684
|
+
}
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
function percentile(sorted, q) {
|
|
688
|
+
const idx = Math.max(1, Math.ceil(q / 100 * sorted.length));
|
|
689
|
+
return sorted[idx - 1] ?? -1;
|
|
690
|
+
}
|
|
691
|
+
function windowMetricsFrom(request, rows, durations, now) {
|
|
692
|
+
const nowMin = Math.floor(now / 6e4);
|
|
693
|
+
const completed = side(rows?.find((r) => r.kind === "completed"), nowMin);
|
|
694
|
+
const failed = side(rows?.find((r) => r.kind === "failed"), nowMin);
|
|
695
|
+
const sampled = (durations ?? []).map(([f, d]) => [num(f), num(d)]);
|
|
696
|
+
return {
|
|
697
|
+
hasMetrics: completed.has || failed.has,
|
|
698
|
+
rates: request.rateWindows.map((windowMinutes) => {
|
|
699
|
+
const c = completed.read(windowMinutes);
|
|
700
|
+
const f = failed.read(windowMinutes);
|
|
701
|
+
return { windowMinutes, completed: c.total, failed: f.total, coveredMinutes: Math.min(c.covered, f.covered) };
|
|
702
|
+
}),
|
|
703
|
+
durations: request.durationWindows.map((windowMinutes) => {
|
|
704
|
+
const since = now - windowMinutes * 6e4;
|
|
705
|
+
const ds = sampled.filter(([f]) => f >= since).map(([, d]) => d).sort((a, b) => a - b);
|
|
706
|
+
return {
|
|
707
|
+
windowMinutes,
|
|
708
|
+
sampled: ds.length,
|
|
709
|
+
p50Ms: ds.length > 0 ? percentile(ds, 50) : null,
|
|
710
|
+
p95Ms: ds.length > 0 ? percentile(ds, 95) : null
|
|
711
|
+
};
|
|
712
|
+
}),
|
|
713
|
+
collectedAt: now
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
function ratesFrom(input) {
|
|
717
|
+
const pct = (c, f) => c + f === 0 ? null : Math.round(c / (c + f) * 1e3) / 10;
|
|
718
|
+
if (input.completed || input.failed) {
|
|
719
|
+
const nowMin = Math.floor(input.now / 6e4);
|
|
720
|
+
const c = side(input.completed ?? void 0, nowMin).read(input.windowMinutes);
|
|
721
|
+
const f = side(input.failed ?? void 0, nowMin).read(input.windowMinutes);
|
|
722
|
+
return {
|
|
723
|
+
windowMinutes: Math.min(c.covered, f.covered),
|
|
724
|
+
completed: c.total,
|
|
725
|
+
failed: f.total,
|
|
726
|
+
successPct: pct(c.total, f.total),
|
|
727
|
+
source: "metrics",
|
|
728
|
+
retentionSkewed: false
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
return {
|
|
732
|
+
windowMinutes: input.windowMinutes,
|
|
733
|
+
completed: input.storedCompleted,
|
|
734
|
+
failed: input.storedFailed,
|
|
735
|
+
successPct: pct(input.storedCompleted, input.storedFailed),
|
|
736
|
+
// QueueRates calls this source "zset" (the Redis name); here it is a COUNT of stored rows.
|
|
737
|
+
source: "zset",
|
|
738
|
+
retentionSkewed: input.prunesCompleted && input.storedFailed > 0
|
|
739
|
+
};
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// packages/pg-inspector/src/sql.ts
|
|
743
|
+
var STATE_SQL = {
|
|
744
|
+
waiting: { where: "state = 'waiting' AND priority = 0", order: ["seq"] },
|
|
745
|
+
// job_ready_idx
|
|
746
|
+
prioritized: { where: "state = 'waiting' AND priority > 0", order: ["priority", "seq"] },
|
|
747
|
+
// job_ready_idx
|
|
748
|
+
active: { where: "state = 'active'", order: ["seq"] },
|
|
749
|
+
// small by nature (bounded by worker concurrency)
|
|
750
|
+
completed: { where: "state = 'completed'", order: ["finished_at_ms"] },
|
|
751
|
+
// job_finished_idx
|
|
752
|
+
failed: { where: "state = 'failed'", order: ["finished_at_ms"] },
|
|
753
|
+
// job_finished_idx
|
|
754
|
+
delayed: { where: "state = 'delayed'", order: ["process_at_ms"] },
|
|
755
|
+
// job_delayed_idx
|
|
756
|
+
"waiting-children": { where: "state = 'waiting-children'", order: ["seq"] },
|
|
757
|
+
// job_waiting_children_idx
|
|
758
|
+
paused: null
|
|
759
|
+
};
|
|
760
|
+
function orderBy(state, order) {
|
|
761
|
+
const s = STATE_SQL[state];
|
|
762
|
+
if (!s) throw new Error(`no physical state for ${state}`);
|
|
763
|
+
const dir = order === "asc" ? "ASC" : "DESC";
|
|
764
|
+
return s.order.map((c) => `${c} ${dir}`).join(", ");
|
|
765
|
+
}
|
|
766
|
+
function summaryColumns(alias, previewParam, capParam) {
|
|
767
|
+
const j = alias;
|
|
768
|
+
return `${j}.id, ${j}.name, ${j}.state::text AS state, ${j}.priority, ${j}.delay_ms, ${j}.attempts_made,
|
|
769
|
+
${j}.added_at_ms, ${j}.processed_at_ms, ${j}.finished_at_ms, ${j}.process_at_ms, ${j}.failed_reason,
|
|
770
|
+
${j}.progress, ${j}.opts->'attempts' AS opt_attempts, ${j}.opts #>> '{group,id}' AS group_id,
|
|
771
|
+
${j}.scheduler_id, ${j}.stalled_count, ${j}.parent_queue, ${j}.parent_id,
|
|
772
|
+
pg_column_size(${j}.data) AS data_stored,
|
|
773
|
+
CASE WHEN pg_column_size(${j}.data) <= $${capParam} THEN left(${j}.data::text, $${previewParam}) END AS data_preview,
|
|
774
|
+
CASE WHEN pg_column_size(${j}.data) <= $${capParam} THEN octet_length(${j}.data::text) END AS data_bytes`;
|
|
775
|
+
}
|
|
776
|
+
var count = (state, where) => `CASE WHEN '${state}' = ANY(string_to_array(q.skip, ',')) THEN NULL ELSE (SELECT count(*) FROM job WHERE queue = q.queue AND ${where}) END`;
|
|
777
|
+
var QUEUE_STATS = `
|
|
778
|
+
SELECT q.queue,
|
|
779
|
+
${count("waiting", "state = 'waiting' AND priority = 0")} AS waiting,
|
|
780
|
+
${count("prioritized", "state = 'waiting' AND priority > 0")} AS prioritized,
|
|
781
|
+
${count("active", "state = 'active'")} AS active,
|
|
782
|
+
${count("completed", "state = 'completed'")} AS completed,
|
|
783
|
+
${count("failed", "state = 'failed'")} AS failed,
|
|
784
|
+
${count("delayed", "state = 'delayed'")} AS delayed,
|
|
785
|
+
${count("waiting-children", "state = 'waiting-children'")} AS waiting_children,
|
|
786
|
+
-- An active job whose lock expired: the worker stopped renewing it. This is
|
|
787
|
+
-- what Redis' stalled SET means, read straight off job_active_idx.
|
|
788
|
+
(SELECT count(*) FROM job WHERE queue = q.queue AND state = 'active' AND locked_until_ms < $2) AS stalled,
|
|
789
|
+
EXISTS (SELECT 1 FROM meta WHERE queue = q.queue AND field = 'paused') AS paused,
|
|
790
|
+
(SELECT value FROM meta WHERE queue = q.queue AND field = 'version') AS version,
|
|
791
|
+
(SELECT count(*) FROM scheduler WHERE queue = q.queue) AS schedulers,
|
|
792
|
+
-- the whole metrics row per side: count, prev_ts and prev_count carry the
|
|
793
|
+
-- current minute, which is not in data until the minute rolls over
|
|
794
|
+
(SELECT json_build_object('kind', kind, 'count', count, 'prevTs', prev_ts, 'prevCount', prev_count, 'data', data[1:$3])
|
|
795
|
+
FROM metrics WHERE queue = q.queue AND kind = 'completed') AS m_completed,
|
|
796
|
+
(SELECT json_build_object('kind', kind, 'count', count, 'prevTs', prev_ts, 'prevCount', prev_count, 'data', data[1:$3])
|
|
797
|
+
FROM metrics WHERE queue = q.queue AND kind = 'failed') AS m_failed,
|
|
798
|
+
(SELECT data[1:$3] FROM metrics WHERE queue = q.queue AND kind = 'completed') AS m_completed_data,
|
|
799
|
+
(SELECT data[1:$3] FROM metrics WHERE queue = q.queue AND kind = 'failed') AS m_failed_data,
|
|
800
|
+
-- Only read when the queue has no metrics (the rate then falls back to stored
|
|
801
|
+
-- rows); a busy queue with metrics would otherwise count an hour of jobs per read.
|
|
802
|
+
CASE WHEN EXISTS (SELECT 1 FROM metrics WHERE queue = q.queue) THEN NULL
|
|
803
|
+
ELSE (SELECT count(*) FROM job WHERE queue = q.queue AND state = 'completed' AND finished_at_ms >= $4) END AS w_completed,
|
|
804
|
+
CASE WHEN EXISTS (SELECT 1 FROM metrics WHERE queue = q.queue) THEN NULL
|
|
805
|
+
ELSE (SELECT count(*) FROM job WHERE queue = q.queue AND state = 'failed' AND finished_at_ms >= $4) END AS w_failed,
|
|
806
|
+
-- removeOnComplete lives in each job's opts; one job tells whether the queue prunes.
|
|
807
|
+
(SELECT opts->'removeOnComplete' FROM job WHERE queue = q.queue AND state = 'completed'
|
|
808
|
+
ORDER BY finished_at_ms DESC LIMIT 1) AS remove_on_complete
|
|
809
|
+
FROM unnest($1::text[], $5::text[]) AS q(queue, skip)`;
|
|
810
|
+
var DISCOVER_QUEUES = `
|
|
811
|
+
WITH RECURSIVE jq AS (
|
|
812
|
+
(SELECT queue FROM job ORDER BY queue LIMIT 1)
|
|
813
|
+
UNION ALL
|
|
814
|
+
SELECT (SELECT j.queue FROM job j WHERE j.queue > jq.queue ORDER BY j.queue LIMIT 1)
|
|
815
|
+
FROM jq WHERE jq.queue IS NOT NULL
|
|
816
|
+
)
|
|
817
|
+
SELECT queue FROM jq WHERE queue IS NOT NULL
|
|
818
|
+
UNION
|
|
819
|
+
SELECT DISTINCT queue FROM meta
|
|
820
|
+
ORDER BY queue`;
|
|
821
|
+
var PING = `SELECT current_setting('server_version') AS version`;
|
|
822
|
+
var SERVER_INFO = `
|
|
823
|
+
SELECT current_setting('server_version') AS version,
|
|
824
|
+
extract(epoch FROM now() - pg_postmaster_start_time())::bigint AS uptime,
|
|
825
|
+
(SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()) AS clients,
|
|
826
|
+
current_setting('max_connections')::int AS max_connections,
|
|
827
|
+
pg_database_size(current_database()) AS db_bytes,
|
|
828
|
+
COALESCE(pg_total_relation_size(to_regclass('job')), 0) AS job_bytes,
|
|
829
|
+
COALESCE(pg_total_relation_size(to_regclass('event')), 0) AS event_bytes,
|
|
830
|
+
d.xact_commit + d.xact_rollback AS transactions,
|
|
831
|
+
d.blks_hit, d.blks_read, d.deadlocks
|
|
832
|
+
FROM pg_stat_database d
|
|
833
|
+
WHERE d.datname = current_database()`;
|
|
834
|
+
var WINDOW_COUNTS = `
|
|
835
|
+
SELECT
|
|
836
|
+
(SELECT count(*) FROM job WHERE queue = $1 AND state = 'completed' AND finished_at_ms >= $2) AS completed,
|
|
837
|
+
(SELECT count(*) FROM job WHERE queue = $1 AND state = 'failed' AND finished_at_ms >= $2) AS failed`;
|
|
838
|
+
var METRICS_COUNTERS = `SELECT kind, count FROM metrics WHERE queue = $1`;
|
|
839
|
+
var METRICS = `SELECT kind, data[1:$2] AS data FROM metrics WHERE queue = $1`;
|
|
840
|
+
var WINDOW_METRICS = `
|
|
841
|
+
SELECT q.queue,
|
|
842
|
+
(SELECT json_agg(json_build_object('kind', m.kind, 'count', m.count, 'prevTs', m.prev_ts,
|
|
843
|
+
'prevCount', m.prev_count, 'data', m.data[1:$3]))
|
|
844
|
+
FROM metrics m WHERE m.queue = q.queue) AS metrics,
|
|
845
|
+
(SELECT json_agg(json_build_array(s.finished_at_ms, s.finished_at_ms - s.processed_at_ms))
|
|
846
|
+
FROM (SELECT finished_at_ms, processed_at_ms FROM job
|
|
847
|
+
WHERE queue = q.queue AND state = 'completed' AND finished_at_ms >= q.since
|
|
848
|
+
ORDER BY finished_at_ms DESC LIMIT $4) s
|
|
849
|
+
WHERE s.processed_at_ms IS NOT NULL AND s.finished_at_ms >= s.processed_at_ms) AS durations
|
|
850
|
+
FROM unnest($1::text[], $2::bigint[]) AS q(queue, since)`;
|
|
851
|
+
var JOB_DETAIL = `
|
|
852
|
+
SELECT j.id, j.name, j.state::text AS state, j.priority, j.delay_ms, j.attempts_made,
|
|
853
|
+
j.added_at_ms, j.processed_at_ms, j.finished_at_ms, j.process_at_ms, j.failed_reason,
|
|
854
|
+
j.progress, j.data, j.opts, j.return_value, j.stacktrace, j.scheduler_id, j.stalled_count,
|
|
855
|
+
j.parent_queue, j.parent_id, octet_length(j.data::text) AS data_bytes,
|
|
856
|
+
(SELECT array_agg(t.row ORDER BY t.idx)
|
|
857
|
+
FROM (SELECT idx, row FROM job_log WHERE queue = $1 AND job_id = $2 ORDER BY idx DESC LIMIT $3) t) AS logs,
|
|
858
|
+
(SELECT count(*) FROM job_log WHERE queue = $1 AND job_id = $2) AS logs_count,
|
|
859
|
+
(SELECT count(*) FILTER (WHERE status = 'pending') FROM job_dependency
|
|
860
|
+
WHERE parent_queue = $1 AND parent_id = $2) AS unprocessed,
|
|
861
|
+
(SELECT count(*) FILTER (WHERE status <> 'pending') FROM job_dependency
|
|
862
|
+
WHERE parent_queue = $1 AND parent_id = $2) AS processed,
|
|
863
|
+
EXISTS (SELECT 1 FROM meta WHERE queue = $1 AND field = 'paused') AS queue_paused
|
|
864
|
+
FROM job j
|
|
865
|
+
WHERE j.queue = $1 AND j.id = $2`;
|
|
866
|
+
var JOB_LOGS = `
|
|
867
|
+
SELECT
|
|
868
|
+
ARRAY(SELECT row FROM job_log WHERE queue = $1 AND job_id = $2 ORDER BY idx OFFSET $3 LIMIT $4) AS logs,
|
|
869
|
+
(SELECT count(*) FROM job_log WHERE queue = $1 AND job_id = $2) AS count`;
|
|
870
|
+
var QUEUE_SETUP = `
|
|
871
|
+
SELECT
|
|
872
|
+
(SELECT json_object_agg(field, value) FROM meta WHERE queue = $1) AS meta,
|
|
873
|
+
(SELECT expire_at_ms FROM rate_limit WHERE queue = $1) AS limiter_expire_at,
|
|
874
|
+
EXISTS (SELECT 1 FROM metrics WHERE queue = $1 AND kind = 'completed') AS metrics_enabled`;
|
|
875
|
+
var WORKERS = `
|
|
876
|
+
SELECT application_name FROM pg_stat_activity
|
|
877
|
+
WHERE datname = current_database() AND (application_name = $1 OR application_name LIKE $2)`;
|
|
878
|
+
var SCHEDULERS = `
|
|
879
|
+
SELECT
|
|
880
|
+
(SELECT count(*) FROM scheduler WHERE queue = $1) AS total,
|
|
881
|
+
(SELECT json_agg(row_to_json(s)) FROM (
|
|
882
|
+
SELECT scheduler_id, name, next_run_ms, pattern, every_ms, tz, offset_ms, limit_count,
|
|
883
|
+
iteration_count, start_date_ms, end_date_ms,
|
|
884
|
+
left(template_data::text, $4) AS template_data, left(template_opts::text, $4) AS template_opts
|
|
885
|
+
FROM scheduler WHERE queue = $1
|
|
886
|
+
ORDER BY next_run_ms ASC NULLS LAST, scheduler_id
|
|
887
|
+
OFFSET $2 LIMIT $3) s) AS rows`;
|
|
888
|
+
var SAMPLE_PARENTS = `
|
|
889
|
+
WITH sampled AS (
|
|
890
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'completed' ORDER BY finished_at_ms DESC LIMIT $2)
|
|
891
|
+
UNION ALL
|
|
892
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'failed' ORDER BY finished_at_ms DESC LIMIT $2)
|
|
893
|
+
UNION ALL
|
|
894
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'waiting-children' ORDER BY seq DESC LIMIT $2)
|
|
895
|
+
UNION ALL
|
|
896
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'waiting' AND priority = 0 ORDER BY seq DESC LIMIT $2)
|
|
897
|
+
UNION ALL
|
|
898
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'active' ORDER BY seq DESC LIMIT $2)
|
|
899
|
+
UNION ALL
|
|
900
|
+
(SELECT parent_queue FROM job WHERE queue = $1 AND state = 'delayed' ORDER BY process_at_ms DESC LIMIT $2)
|
|
901
|
+
)
|
|
902
|
+
SELECT
|
|
903
|
+
(SELECT count(*) FROM sampled) AS sampled,
|
|
904
|
+
(SELECT json_agg(json_build_array(parent_queue, n)) FROM (
|
|
905
|
+
SELECT parent_queue, count(*) AS n FROM sampled WHERE parent_queue IS NOT NULL GROUP BY parent_queue
|
|
906
|
+
) g) AS edges`;
|
|
907
|
+
var TREE_NODES = `
|
|
908
|
+
SELECT r.ord, j.id IS NOT NULL AS found, j.name, j.state::text AS state, j.priority, j.added_at_ms,
|
|
909
|
+
j.finished_at_ms, j.attempts_made, j.failed_reason, j.progress, j.parent_queue, j.parent_id,
|
|
910
|
+
(SELECT count(*) FILTER (WHERE status = 'pending') FROM job_dependency
|
|
911
|
+
WHERE parent_queue = r.queue AND parent_id = r.id) AS unprocessed,
|
|
912
|
+
(SELECT count(*) FILTER (WHERE status <> 'pending') FROM job_dependency
|
|
913
|
+
WHERE parent_queue = r.queue AND parent_id = r.id) AS processed,
|
|
914
|
+
(SELECT json_agg(json_build_array(c.child_queue, c.child_id)) FROM (
|
|
915
|
+
SELECT child_queue, child_id FROM job_dependency
|
|
916
|
+
WHERE parent_queue = r.queue AND parent_id = r.id
|
|
917
|
+
ORDER BY child_key LIMIT $3 + 1) c) AS children
|
|
918
|
+
FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS r(queue, id, ord)
|
|
919
|
+
LEFT JOIN job j ON j.queue = r.queue AND j.id = r.id
|
|
920
|
+
ORDER BY r.ord`;
|
|
921
|
+
|
|
922
|
+
// packages/pg-inspector/src/connection.ts
|
|
923
|
+
var LIBPQ_ALIASED_MODES = /* @__PURE__ */ new Set(["prefer", "require", "verify-ca"]);
|
|
924
|
+
function nodePgConnectionString(url) {
|
|
925
|
+
let parsed;
|
|
926
|
+
try {
|
|
927
|
+
parsed = new URL(url);
|
|
928
|
+
} catch {
|
|
929
|
+
return url;
|
|
930
|
+
}
|
|
931
|
+
const mode = parsed.searchParams.get("sslmode");
|
|
932
|
+
if (!mode || !LIBPQ_ALIASED_MODES.has(mode) || parsed.searchParams.has("uselibpqcompat")) return url;
|
|
933
|
+
parsed.searchParams.set("uselibpqcompat", "true");
|
|
934
|
+
return parsed.toString();
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
// packages/pg-inspector/src/inspector.ts
|
|
938
|
+
var DEFAULTS = {
|
|
939
|
+
discoveryTtlMs: 3e4,
|
|
940
|
+
previewBytes: 2048,
|
|
941
|
+
maxScanPerCall: 1e3,
|
|
942
|
+
connectTimeoutMs: 5e3,
|
|
943
|
+
listFieldCapBytes: 32 * 1024,
|
|
944
|
+
searchFieldCapBytes: 256 * 1024,
|
|
945
|
+
searchByteBudget: 8 * 1024 * 1024
|
|
946
|
+
};
|
|
947
|
+
var READ_POOL_MAX = 4;
|
|
948
|
+
var STATEMENT_TIMEOUT_MS = 1e4;
|
|
949
|
+
var SESSION_SETTINGS = [
|
|
950
|
+
["statement_timeout", STATEMENT_TIMEOUT_MS],
|
|
951
|
+
["max_parallel_workers_per_gather", 0]
|
|
952
|
+
];
|
|
953
|
+
var APPLICATION_NAME = "bullpane-dashboard";
|
|
954
|
+
var STATS_METRIC_POINTS = 60;
|
|
955
|
+
var DEFAULT_RATE_WINDOW_MINUTES = 60;
|
|
956
|
+
var SETUP_CACHE_MS = 1e4;
|
|
957
|
+
var STATS_CACHE_MS = 2e3;
|
|
958
|
+
var LARGE_COUNT = 1e5;
|
|
959
|
+
var LARGE_COUNT_MAX_TTL_MS = 6e4;
|
|
960
|
+
var DETAIL_LOG_TAIL = 100;
|
|
961
|
+
var BULK_CONCURRENCY = 8;
|
|
962
|
+
var FORCE_DISCOVERY_MIN_MS = 5e3;
|
|
963
|
+
var MAX_TREE_CLIMB = 50;
|
|
964
|
+
function quoteIdent(name) {
|
|
965
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
966
|
+
}
|
|
967
|
+
function likeSubstring(query) {
|
|
968
|
+
return `%${query.replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
|
|
969
|
+
}
|
|
970
|
+
function errorMessage(err) {
|
|
971
|
+
return err instanceof Error ? err.message : String(err);
|
|
972
|
+
}
|
|
973
|
+
var PgInspector = class {
|
|
974
|
+
config;
|
|
975
|
+
opts;
|
|
976
|
+
pool;
|
|
977
|
+
/**
|
|
978
|
+
* How reads carry their settings (schema, statement timeout, no parallel query):
|
|
979
|
+
* - "session": as startup options on each connection. One round trip per
|
|
980
|
+
* read. Used on a direct connection.
|
|
981
|
+
* - "transaction": behind a transaction pooler (PgBouncer, Supavisor, RDS
|
|
982
|
+
* Proxy...), which refuses or silently drops startup options and hands each
|
|
983
|
+
* transaction a different server connection. Each read is then its own
|
|
984
|
+
* READ ONLY transaction with SET LOCAL, so nothing depends on session state.
|
|
985
|
+
* Decided once in ensureReady, by trying the first and checking it took effect.
|
|
986
|
+
*/
|
|
987
|
+
mode = "session";
|
|
988
|
+
filter;
|
|
989
|
+
ready = null;
|
|
990
|
+
discovered = null;
|
|
991
|
+
discovering = null;
|
|
992
|
+
setupCache = /* @__PURE__ */ new Map();
|
|
993
|
+
/** per queue + rate window; see STATS_CACHE_MS */
|
|
994
|
+
statsCache = /* @__PURE__ */ new Map();
|
|
995
|
+
/** per queue + state, filled by stats reads and by job pages; see STATS_CACHE_MS */
|
|
996
|
+
countCache = /* @__PURE__ */ new Map();
|
|
997
|
+
/** bullmq Queue per queue name, created lazily for writes only. */
|
|
998
|
+
queues = /* @__PURE__ */ new Map();
|
|
999
|
+
closed = false;
|
|
1000
|
+
constructor(config, options = {}) {
|
|
1001
|
+
if (config.kind !== "postgres") throw new Error("PgInspector only opens postgres connections");
|
|
1002
|
+
this.config = { ...config, kind: "postgres", prefix: config.prefix ?? "bullmq", cluster: false };
|
|
1003
|
+
this.opts = { ...DEFAULTS };
|
|
1004
|
+
for (const [k, v] of Object.entries(options)) {
|
|
1005
|
+
if (v !== void 0 && k in DEFAULTS) this.opts[k] = v;
|
|
1006
|
+
}
|
|
1007
|
+
this.filter = config.queueFilter ? globToRegExp(config.queueFilter) : null;
|
|
1008
|
+
this.pool = this.createPool(true);
|
|
1009
|
+
}
|
|
1010
|
+
/**
|
|
1011
|
+
* The read pool. The settings every read runs with:
|
|
1012
|
+
* - search_path = the BullMQ schema, so its unqualified table names resolve;
|
|
1013
|
+
* - statement_timeout, so no read can hold the database for long;
|
|
1014
|
+
* - no parallel query: a count over a big state would otherwise run as a
|
|
1015
|
+
* parallel index scan on 3 cores at once, and the smoke test measured
|
|
1016
|
+
* what that does to the workers sharing the database (throughput down to
|
|
1017
|
+
* a third under a read storm). A dashboard gets one core per query, ever.
|
|
1018
|
+
* `withOptions: false` is the pooler fallback: no startup options at all.
|
|
1019
|
+
*/
|
|
1020
|
+
createPool(withOptions) {
|
|
1021
|
+
const pool = new pg.Pool({
|
|
1022
|
+
connectionString: nodePgConnectionString(this.config.url),
|
|
1023
|
+
max: READ_POOL_MAX,
|
|
1024
|
+
idleTimeoutMillis: 3e4,
|
|
1025
|
+
connectionTimeoutMillis: this.opts.connectTimeoutMs,
|
|
1026
|
+
application_name: APPLICATION_NAME,
|
|
1027
|
+
...withOptions ? { options: `-c search_path=${quoteIdent(this.config.prefix)} -c ${SESSION_SETTINGS.map(([k, v]) => `${k}=${v}`).join(" -c ")}` } : {}
|
|
1028
|
+
});
|
|
1029
|
+
pool.on("error", () => void 0);
|
|
1030
|
+
return pool;
|
|
1031
|
+
}
|
|
1032
|
+
// ---------------------------------------------------------------------------
|
|
1033
|
+
// connection
|
|
1034
|
+
// ---------------------------------------------------------------------------
|
|
1035
|
+
/**
|
|
1036
|
+
* Once per inspector: the schema exists and was migrated by a BullMQ this
|
|
1037
|
+
* version can read (BullMQ's own assertSchemaCompatibility, read only — the
|
|
1038
|
+
* dashboard never runs migrations on a customer database). Retried on the
|
|
1039
|
+
* next call after a failure.
|
|
1040
|
+
*/
|
|
1041
|
+
async ensureReady() {
|
|
1042
|
+
if (this.closed) throw new Error("inspector_closed");
|
|
1043
|
+
if (!this.ready) {
|
|
1044
|
+
this.ready = (async () => {
|
|
1045
|
+
await this.pickMode();
|
|
1046
|
+
const client = await this.pool.connect();
|
|
1047
|
+
try {
|
|
1048
|
+
await assertSchemaCompatibility(client, this.config.prefix);
|
|
1049
|
+
} catch (err) {
|
|
1050
|
+
if (err instanceof SchemaMigrationRequiredError) {
|
|
1051
|
+
throw new Error(
|
|
1052
|
+
`postgres_schema_missing: no BullMQ schema "${this.config.prefix}" in this database. Check the schema name, or start a BullMQ worker with \`migrate: true\` once.`
|
|
1053
|
+
);
|
|
1054
|
+
}
|
|
1055
|
+
throw err;
|
|
1056
|
+
} finally {
|
|
1057
|
+
client.release();
|
|
1058
|
+
}
|
|
1059
|
+
})().catch((err) => {
|
|
1060
|
+
this.ready = null;
|
|
1061
|
+
throw err;
|
|
1062
|
+
});
|
|
1063
|
+
}
|
|
1064
|
+
return this.ready;
|
|
1065
|
+
}
|
|
1066
|
+
/**
|
|
1067
|
+
* Session mode if the server takes the startup options AND they took effect;
|
|
1068
|
+
* transaction mode otherwise. A pooler can refuse them ("unsupported startup
|
|
1069
|
+
* parameter", PgBouncer ≥ 1.20 accepts search_path only) or drop them silently
|
|
1070
|
+
* (`ignore_startup_parameters`), so success alone proves nothing: the
|
|
1071
|
+
* settings are read back.
|
|
1072
|
+
*/
|
|
1073
|
+
async pickMode() {
|
|
1074
|
+
if (this.mode === "transaction") return;
|
|
1075
|
+
try {
|
|
1076
|
+
const { rows } = await this.pool.query(
|
|
1077
|
+
"SELECT current_setting('statement_timeout') AS timeout, current_setting('max_parallel_workers_per_gather') AS parallel"
|
|
1078
|
+
);
|
|
1079
|
+
const applied = rows[0]?.parallel === "0" && rows[0]?.timeout !== "0";
|
|
1080
|
+
if (!applied) this.mode = "transaction";
|
|
1081
|
+
} catch (err) {
|
|
1082
|
+
if (!/unsupported startup parameter/i.test(errorMessage(err))) throw err;
|
|
1083
|
+
const old = this.pool;
|
|
1084
|
+
this.pool = this.createPool(false);
|
|
1085
|
+
void old.end().catch(() => void 0);
|
|
1086
|
+
this.mode = "transaction";
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
async query(text3, values = []) {
|
|
1090
|
+
await this.ensureReady();
|
|
1091
|
+
if (this.mode === "session") return (await this.pool.query(text3, values)).rows;
|
|
1092
|
+
const client = await this.pool.connect();
|
|
1093
|
+
try {
|
|
1094
|
+
await client.query("BEGIN READ ONLY");
|
|
1095
|
+
await client.query(
|
|
1096
|
+
`SELECT set_config('search_path', $1, true), ${SESSION_SETTINGS.map(([k], i) => `set_config('${k}', $${i + 2}, true)`).join(", ")}`,
|
|
1097
|
+
[quoteIdent(this.config.prefix), ...SESSION_SETTINGS.map(([, v]) => String(v))]
|
|
1098
|
+
);
|
|
1099
|
+
const res = await client.query(text3, values);
|
|
1100
|
+
await client.query("COMMIT");
|
|
1101
|
+
return res.rows;
|
|
1102
|
+
} catch (err) {
|
|
1103
|
+
await client.query("ROLLBACK").catch(() => void 0);
|
|
1104
|
+
throw err;
|
|
1105
|
+
} finally {
|
|
1106
|
+
client.release();
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
/** "session" or "transaction" (behind a pooler); see `mode`. For diagnostics and tests. */
|
|
1110
|
+
async connectionMode() {
|
|
1111
|
+
await this.ensureReady();
|
|
1112
|
+
return this.mode;
|
|
1113
|
+
}
|
|
1114
|
+
async ping() {
|
|
1115
|
+
const t0 = performance.now();
|
|
1116
|
+
try {
|
|
1117
|
+
const [row] = await this.query(PING);
|
|
1118
|
+
return { ok: true, latencyMs: Math.round((performance.now() - t0) * 10) / 10, redisVersion: row?.version ?? null, error: null };
|
|
1119
|
+
} catch (err) {
|
|
1120
|
+
return { ok: false, latencyMs: Math.round((performance.now() - t0) * 10) / 10, redisVersion: null, error: errorMessage(err) };
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
async serverInfo() {
|
|
1124
|
+
const startedAt = Date.now();
|
|
1125
|
+
const [r] = await this.query(SERVER_INFO);
|
|
1126
|
+
const latencyMs = Date.now() - startedAt;
|
|
1127
|
+
const hit = numOrNull(r?.blks_hit);
|
|
1128
|
+
const read = numOrNull(r?.blks_read);
|
|
1129
|
+
const lookups = (hit ?? 0) + (read ?? 0);
|
|
1130
|
+
return {
|
|
1131
|
+
backend: "postgres",
|
|
1132
|
+
serverVersion: String(r?.version ?? "unknown"),
|
|
1133
|
+
schema: this.config.prefix,
|
|
1134
|
+
uptimeSeconds: num(r?.uptime),
|
|
1135
|
+
connectedClients: num(r?.clients),
|
|
1136
|
+
maxConnections: num(r?.max_connections),
|
|
1137
|
+
databaseSizeBytes: num(r?.db_bytes),
|
|
1138
|
+
jobTableBytes: num(r?.job_bytes),
|
|
1139
|
+
eventTableBytes: num(r?.event_bytes),
|
|
1140
|
+
totalTransactions: numOrNull(r?.transactions),
|
|
1141
|
+
cacheHitRatePct: lookups > 0 ? Math.round((hit ?? 0) / lookups * 1e3) / 10 : null,
|
|
1142
|
+
deadlocks: numOrNull(r?.deadlocks),
|
|
1143
|
+
latencyMs,
|
|
1144
|
+
sampledAt: new Date(startedAt).toISOString()
|
|
1145
|
+
};
|
|
1146
|
+
}
|
|
1147
|
+
async close() {
|
|
1148
|
+
if (this.closed) return;
|
|
1149
|
+
this.closed = true;
|
|
1150
|
+
await Promise.allSettled([...this.queues.values()].map((q) => q.close()));
|
|
1151
|
+
this.queues.clear();
|
|
1152
|
+
await this.pool.end().catch(() => void 0);
|
|
1153
|
+
}
|
|
1154
|
+
// ---------------------------------------------------------------------------
|
|
1155
|
+
// discovery
|
|
1156
|
+
// ---------------------------------------------------------------------------
|
|
1157
|
+
/** `SELECT DISTINCT queue FROM meta`, cached for discoveryTtlMs. See sql.ts. */
|
|
1158
|
+
async discoverQueues(opts = {}) {
|
|
1159
|
+
const age = this.discovered ? Date.now() - this.discovered.at : Number.POSITIVE_INFINITY;
|
|
1160
|
+
if (this.discovered && age < (opts.force ? FORCE_DISCOVERY_MIN_MS : this.opts.discoveryTtlMs)) {
|
|
1161
|
+
return this.discovered.names;
|
|
1162
|
+
}
|
|
1163
|
+
if (!this.discovering) {
|
|
1164
|
+
this.discovering = this.query(DISCOVER_QUEUES).then((rows) => {
|
|
1165
|
+
const names = rows.map((r) => r.queue).filter((n) => !this.filter || this.filter.test(n));
|
|
1166
|
+
names.sort((a, b) => a.localeCompare(b));
|
|
1167
|
+
this.discovered = { at: Date.now(), names };
|
|
1168
|
+
return names;
|
|
1169
|
+
}).finally(() => {
|
|
1170
|
+
this.discovering = null;
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
return this.discovering;
|
|
1174
|
+
}
|
|
1175
|
+
/** One query lists every queue: discovery is complete from the first call. */
|
|
1176
|
+
async discoveryStatus() {
|
|
1177
|
+
return { complete: true, scannedIterations: 0, totalKeys: null };
|
|
1178
|
+
}
|
|
1179
|
+
// ---------------------------------------------------------------------------
|
|
1180
|
+
// reads
|
|
1181
|
+
// ---------------------------------------------------------------------------
|
|
1182
|
+
async getQueueStats(queueNames, opts = {}) {
|
|
1183
|
+
if (queueNames.length === 0) return {};
|
|
1184
|
+
const windowMinutes = opts.rateWindowMinutes ?? DEFAULT_RATE_WINDOW_MINUTES;
|
|
1185
|
+
const now = Date.now();
|
|
1186
|
+
for (const [k, v] of this.statsCache) if (now - v.at >= STATS_CACHE_MS) this.statsCache.delete(k);
|
|
1187
|
+
const key = (q) => `${windowMinutes}\0${q}`;
|
|
1188
|
+
const stale = queueNames.filter((q) => !this.statsCache.has(key(q)));
|
|
1189
|
+
if (stale.length > 0) {
|
|
1190
|
+
const read = this.readQueueStats(stale, windowMinutes);
|
|
1191
|
+
for (const q of stale) {
|
|
1192
|
+
const value = read.then((all) => all[q]);
|
|
1193
|
+
this.statsCache.set(key(q), { at: now, value });
|
|
1194
|
+
value.catch(() => this.statsCache.delete(key(q)));
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
const out = {};
|
|
1198
|
+
for (const q of queueNames) {
|
|
1199
|
+
const stats = await this.statsCache.get(key(q))?.value;
|
|
1200
|
+
if (!stats) continue;
|
|
1201
|
+
const { metrics: _metrics, ...withoutMetrics } = stats;
|
|
1202
|
+
out[q] = opts.withMetrics ? stats : withoutMetrics;
|
|
1203
|
+
}
|
|
1204
|
+
return out;
|
|
1205
|
+
}
|
|
1206
|
+
/** A count still fresh for its size (see LARGE_COUNT), from stats or from a job page. */
|
|
1207
|
+
cachedCount(queue, state) {
|
|
1208
|
+
const c = this.countCache.get(`${queue}\0${state}`);
|
|
1209
|
+
return c && Date.now() - c.at < c.ttl ? c.n : null;
|
|
1210
|
+
}
|
|
1211
|
+
rememberCount(queue, state, n) {
|
|
1212
|
+
const ttl = n < LARGE_COUNT ? STATS_CACHE_MS : Math.min(LARGE_COUNT_MAX_TTL_MS, Math.round(STATS_CACHE_MS * n / LARGE_COUNT));
|
|
1213
|
+
this.countCache.set(`${queue}\0${state}`, { at: Date.now(), n, ttl });
|
|
1214
|
+
if (this.countCache.size > 1e4) this.countCache.clear();
|
|
1215
|
+
}
|
|
1216
|
+
async readQueueStats(queueNames, windowMinutes) {
|
|
1217
|
+
const out = {};
|
|
1218
|
+
const now = Date.now();
|
|
1219
|
+
const points = Math.max(STATS_METRIC_POINTS, windowMinutes);
|
|
1220
|
+
const countable = ["waiting", "prioritized", "active", "completed", "failed", "delayed", "waiting-children"];
|
|
1221
|
+
const reuse = queueNames.map(
|
|
1222
|
+
(q) => countable.filter((state) => {
|
|
1223
|
+
const n = this.cachedCount(q, state);
|
|
1224
|
+
return n !== null && n >= LARGE_COUNT;
|
|
1225
|
+
})
|
|
1226
|
+
);
|
|
1227
|
+
const rows = await this.query(QUEUE_STATS, [
|
|
1228
|
+
queueNames,
|
|
1229
|
+
now,
|
|
1230
|
+
points,
|
|
1231
|
+
now - windowMinutes * 6e4,
|
|
1232
|
+
reuse.map((states) => states.join(","))
|
|
1233
|
+
]);
|
|
1234
|
+
const counted = (queue, state, value) => {
|
|
1235
|
+
if (value === null || value === void 0) return this.cachedCount(queue, state) ?? 0;
|
|
1236
|
+
const n = num(value);
|
|
1237
|
+
this.rememberCount(queue, state, n);
|
|
1238
|
+
return n;
|
|
1239
|
+
};
|
|
1240
|
+
for (const r of rows) {
|
|
1241
|
+
const queue = String(r.queue);
|
|
1242
|
+
const metricsCompleted = metricPoints(r.m_completed_data);
|
|
1243
|
+
const metricsFailed = metricPoints(r.m_failed_data);
|
|
1244
|
+
const version = typeof r.version === "string" && r.version !== "" ? r.version : null;
|
|
1245
|
+
const stats = {
|
|
1246
|
+
counts: {
|
|
1247
|
+
...EMPTY_COUNTS,
|
|
1248
|
+
waiting: counted(queue, "waiting", r.waiting),
|
|
1249
|
+
prioritized: counted(queue, "prioritized", r.prioritized),
|
|
1250
|
+
active: counted(queue, "active", r.active),
|
|
1251
|
+
completed: counted(queue, "completed", r.completed),
|
|
1252
|
+
failed: counted(queue, "failed", r.failed),
|
|
1253
|
+
delayed: counted(queue, "delayed", r.delayed),
|
|
1254
|
+
"waiting-children": counted(queue, "waiting-children", r.waiting_children)
|
|
1255
|
+
},
|
|
1256
|
+
isPaused: r.paused === true,
|
|
1257
|
+
isPro: version !== null && version.startsWith("bullmq-pro"),
|
|
1258
|
+
groupsCount: 0,
|
|
1259
|
+
rates: ratesFrom({
|
|
1260
|
+
windowMinutes,
|
|
1261
|
+
now,
|
|
1262
|
+
completed: r.m_completed ?? null,
|
|
1263
|
+
failed: r.m_failed ?? null,
|
|
1264
|
+
storedCompleted: num(r.w_completed),
|
|
1265
|
+
storedFailed: num(r.w_failed),
|
|
1266
|
+
prunesCompleted: prunesCompleted(r.remove_on_complete)
|
|
1267
|
+
}),
|
|
1268
|
+
library: version,
|
|
1269
|
+
schedulersCount: num(r.schedulers),
|
|
1270
|
+
stalledCount: num(r.stalled)
|
|
1271
|
+
};
|
|
1272
|
+
stats.metrics = { completed: metricsCompleted.slice(-STATS_METRIC_POINTS), failed: metricsFailed.slice(-STATS_METRIC_POINTS) };
|
|
1273
|
+
out[queue] = stats;
|
|
1274
|
+
}
|
|
1275
|
+
return out;
|
|
1276
|
+
}
|
|
1277
|
+
async getWindowCounts(queueName, since) {
|
|
1278
|
+
const [r] = await this.query(WINDOW_COUNTS, [queueName, since]);
|
|
1279
|
+
return { completed: num(r?.completed), failed: num(r?.failed) };
|
|
1280
|
+
}
|
|
1281
|
+
/** The cumulative `count` of the metrics rows; null per side when the row is absent (no metrics). */
|
|
1282
|
+
async getMetricsCounters(queueName) {
|
|
1283
|
+
const collectedAt = Date.now();
|
|
1284
|
+
const rows = await this.query(METRICS_COUNTERS, [queueName]);
|
|
1285
|
+
const read = (kind) => numOrNull(rows.find((r) => r.kind === kind)?.count);
|
|
1286
|
+
return { completed: read("completed"), failed: read("failed"), collectedAt };
|
|
1287
|
+
}
|
|
1288
|
+
async getWindowMetrics(requests, opts = {}) {
|
|
1289
|
+
const out = {};
|
|
1290
|
+
if (requests.length === 0) return out;
|
|
1291
|
+
const now = opts.now ?? Date.now();
|
|
1292
|
+
const sample = Math.max(1, Math.min(opts.durationSample ?? 100, 100));
|
|
1293
|
+
const maxRate = Math.max(1, ...requests.flatMap((r) => r.rateWindows));
|
|
1294
|
+
const sinces = requests.map((r) => now - Math.max(0, ...r.durationWindows) * 6e4);
|
|
1295
|
+
const rows = await this.query(
|
|
1296
|
+
WINDOW_METRICS,
|
|
1297
|
+
[requests.map((r) => r.queue), sinces, maxRate + 1, sample]
|
|
1298
|
+
);
|
|
1299
|
+
requests.forEach((request, i) => {
|
|
1300
|
+
const row = rows[i];
|
|
1301
|
+
if (!row) return;
|
|
1302
|
+
out[request.queue] = windowMetricsFrom(request, row.metrics, request.durationWindows.length > 0 ? row.durations : null, now);
|
|
1303
|
+
});
|
|
1304
|
+
return out;
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* A page of one state: the ids come off the state's partial index (OFFSET over
|
|
1308
|
+
* a narrow index, never over full rows), then only that page is joined to
|
|
1309
|
+
* `job` for its columns. The total comes from the count cache when a stats read
|
|
1310
|
+
* just counted the state (the usual case: the queue page polls both), and is
|
|
1311
|
+
* counted in the same statement otherwise.
|
|
1312
|
+
*/
|
|
1313
|
+
async getJobs(queueName, state, opts) {
|
|
1314
|
+
const s = STATE_SQL[state];
|
|
1315
|
+
if (!s) return { jobs: [], total: 0, start: opts.start, end: opts.end };
|
|
1316
|
+
const ob = orderBy(state, opts.order);
|
|
1317
|
+
const limit = Math.max(0, opts.end - opts.start + 1);
|
|
1318
|
+
const known = this.cachedCount(queueName, state);
|
|
1319
|
+
const sql2 = `
|
|
1320
|
+
WITH page AS (
|
|
1321
|
+
SELECT id, row_number() OVER (ORDER BY ${ob}) AS ord
|
|
1322
|
+
FROM job WHERE queue = $1 AND ${s.where} ORDER BY ${ob} OFFSET $2 LIMIT $3
|
|
1323
|
+
)
|
|
1324
|
+
SELECT ${known === null ? `(SELECT count(*) FROM job WHERE queue = $1 AND ${s.where})` : "NULL::bigint"} AS total,
|
|
1325
|
+
(SELECT json_agg(r ORDER BY r.ord) FROM (
|
|
1326
|
+
SELECT page.ord, ${summaryColumns("j", 4, 5)}
|
|
1327
|
+
FROM page JOIN job j ON j.queue = $1 AND j.id = page.id) r) AS rows`;
|
|
1328
|
+
const [r] = await this.query(sql2, [
|
|
1329
|
+
queueName,
|
|
1330
|
+
Math.max(0, opts.start),
|
|
1331
|
+
limit,
|
|
1332
|
+
this.opts.previewBytes,
|
|
1333
|
+
this.opts.listFieldCapBytes
|
|
1334
|
+
]);
|
|
1335
|
+
const total = known ?? num(r?.total);
|
|
1336
|
+
if (known === null) this.rememberCount(queueName, state, total);
|
|
1337
|
+
return {
|
|
1338
|
+
jobs: (r?.rows ?? []).map((row) => rowToSummary(this.config.prefix, row, this.opts.previewBytes)),
|
|
1339
|
+
total,
|
|
1340
|
+
start: opts.start,
|
|
1341
|
+
end: opts.end
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
/**
|
|
1345
|
+
* Bounded, resumable search, with the same bounds as Redis: at most
|
|
1346
|
+
* maxScanPerCall jobs and at most searchByteBudget payload bytes per call, and
|
|
1347
|
+
* payloads over searchFieldCapBytes are matched on id / name / error only.
|
|
1348
|
+
* Case-insensitive substring (ILIKE with the wildcards escaped). The cursor is
|
|
1349
|
+
* the index of the next job to inspect (newest = 0).
|
|
1350
|
+
*/
|
|
1351
|
+
async searchJobs(queueName, state, query, opts) {
|
|
1352
|
+
const s = STATE_SQL[state];
|
|
1353
|
+
if (!s || opts.groupId !== void 0) return { jobs: [], nextCursor: null, scanned: 0, total: 0, skippedLargePayloads: 0 };
|
|
1354
|
+
const cursor = Math.max(0, Math.trunc(Number(opts.cursor ?? 0)) || 0);
|
|
1355
|
+
const limit = Math.max(1, opts.limit);
|
|
1356
|
+
const ob = orderBy(state, "desc");
|
|
1357
|
+
const known = this.cachedCount(queueName, state);
|
|
1358
|
+
const sql2 = `
|
|
1359
|
+
WITH scan AS (
|
|
1360
|
+
SELECT id, row_number() OVER (ORDER BY ${ob}) AS rn
|
|
1361
|
+
FROM job WHERE queue = $1 AND ${s.where} ORDER BY ${ob} OFFSET $2 LIMIT $3
|
|
1362
|
+
), sized AS (
|
|
1363
|
+
SELECT s.rn, s.id, pg_column_size(j.data) AS sz
|
|
1364
|
+
FROM scan s JOIN job j ON j.queue = $1 AND j.id = s.id
|
|
1365
|
+
), budgeted AS (
|
|
1366
|
+
-- a job is inspected when the payload bytes spent BEFORE it are inside the budget
|
|
1367
|
+
SELECT rn, id, sz > $6 AS too_big FROM (
|
|
1368
|
+
SELECT rn, id, sz, sum(CASE WHEN sz > $6 THEN 0 ELSE sz END) OVER (ORDER BY rn) AS spent FROM sized
|
|
1369
|
+
) b
|
|
1370
|
+
WHERE spent - CASE WHEN sz > $6 THEN 0 ELSE sz END < $7
|
|
1371
|
+
), hits AS (
|
|
1372
|
+
SELECT b.rn, ${summaryColumns("j", 8, 9)}
|
|
1373
|
+
FROM budgeted b JOIN job j ON j.queue = $1 AND j.id = b.id
|
|
1374
|
+
WHERE j.id ILIKE $4 ESCAPE '\\' OR j.name ILIKE $4 ESCAPE '\\' OR j.failed_reason ILIKE $4 ESCAPE '\\'
|
|
1375
|
+
OR (NOT b.too_big AND j.data::text ILIKE $4 ESCAPE '\\')
|
|
1376
|
+
ORDER BY b.rn LIMIT $5
|
|
1377
|
+
)
|
|
1378
|
+
SELECT ${known === null ? `(SELECT count(*) FROM job WHERE queue = $1 AND ${s.where})` : "NULL::bigint"} AS total,
|
|
1379
|
+
(SELECT count(*) FROM budgeted) AS scanned,
|
|
1380
|
+
(SELECT json_agg(rn) FROM budgeted WHERE too_big) AS big,
|
|
1381
|
+
(SELECT json_agg(h ORDER BY h.rn) FROM hits h) AS rows`;
|
|
1382
|
+
const [r] = await this.query(sql2, [
|
|
1383
|
+
queueName,
|
|
1384
|
+
cursor,
|
|
1385
|
+
this.opts.maxScanPerCall,
|
|
1386
|
+
likeSubstring(query),
|
|
1387
|
+
limit,
|
|
1388
|
+
this.opts.searchFieldCapBytes,
|
|
1389
|
+
this.opts.searchByteBudget,
|
|
1390
|
+
this.opts.previewBytes,
|
|
1391
|
+
this.opts.listFieldCapBytes
|
|
1392
|
+
]);
|
|
1393
|
+
const total = known ?? num(r?.total);
|
|
1394
|
+
if (known === null) this.rememberCount(queueName, state, total);
|
|
1395
|
+
const rows = r?.rows ?? [];
|
|
1396
|
+
const stoppedAt = rows.length >= limit ? num(rows[rows.length - 1]?.rn) : cursor + num(r?.scanned);
|
|
1397
|
+
const big = (r?.big ?? []).filter((rn) => rn <= stoppedAt).length;
|
|
1398
|
+
return {
|
|
1399
|
+
jobs: rows.map((row) => rowToSummary(this.config.prefix, row, this.opts.previewBytes)),
|
|
1400
|
+
nextCursor: stoppedAt >= total ? null : String(stoppedAt),
|
|
1401
|
+
scanned: stoppedAt - cursor,
|
|
1402
|
+
total,
|
|
1403
|
+
skippedLargePayloads: big
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
async getJob(queueName, jobId) {
|
|
1407
|
+
const [row] = await this.query(JOB_DETAIL, [queueName, jobId, DETAIL_LOG_TAIL]);
|
|
1408
|
+
return row ? rowToDetail(this.config.prefix, row) : null;
|
|
1409
|
+
}
|
|
1410
|
+
async getJobLogs(queueName, jobId, opts) {
|
|
1411
|
+
const start = Math.max(0, opts.start);
|
|
1412
|
+
const limit = opts.end < 0 ? null : Math.max(0, opts.end - start + 1);
|
|
1413
|
+
const [r] = await this.query(JOB_LOGS, [queueName, jobId, start, limit]);
|
|
1414
|
+
return { logs: r?.logs ?? [], count: num(r?.count) };
|
|
1415
|
+
}
|
|
1416
|
+
async getMetrics(queueName, points) {
|
|
1417
|
+
const rows = await this.query(METRICS, [queueName, Math.max(1, points)]);
|
|
1418
|
+
const read = (kind) => metricPoints(rows.find((r) => r.kind === kind)?.data);
|
|
1419
|
+
return { completed: read("completed"), failed: read("failed") };
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
* meta + the limiter window + whether metrics are collected, then the workers
|
|
1423
|
+
* from pg_stat_activity. Cached SETUP_CACHE_MS, like Redis (pg_stat_activity
|
|
1424
|
+
* is cheap, but there is no reason to read it on every poll).
|
|
1425
|
+
*/
|
|
1426
|
+
async getQueueSetup(queueName) {
|
|
1427
|
+
const cached = this.setupCache.get(queueName);
|
|
1428
|
+
if (cached && Date.now() - cached.at < SETUP_CACHE_MS) return cached.value;
|
|
1429
|
+
const [[r], workers] = await Promise.all([
|
|
1430
|
+
this.query(QUEUE_SETUP, [queueName]),
|
|
1431
|
+
this.listWorkers(queueName).catch(() => null)
|
|
1432
|
+
]);
|
|
1433
|
+
const meta = r?.meta ?? {};
|
|
1434
|
+
const version = meta.version ?? null;
|
|
1435
|
+
const max = numOrNull(meta.max);
|
|
1436
|
+
const duration = numOrNull(meta.duration);
|
|
1437
|
+
const expireAt = numOrNull(r?.limiter_expire_at);
|
|
1438
|
+
const ttl = expireAt === null ? -1 : expireAt - Date.now();
|
|
1439
|
+
const { version: _v, paused: _p, concurrency: _c, max: _m, duration: _d, ...rest } = meta;
|
|
1440
|
+
const rawMeta = {};
|
|
1441
|
+
for (const [k, v] of Object.entries(rest)) if (v !== null && v !== void 0) rawMeta[k] = v;
|
|
1442
|
+
const value = {
|
|
1443
|
+
library: version,
|
|
1444
|
+
isPro: version !== null && version.startsWith("bullmq-pro"),
|
|
1445
|
+
isPaused: meta.paused !== void 0,
|
|
1446
|
+
globalConcurrency: numOrNull(meta.concurrency),
|
|
1447
|
+
globalRateLimit: max !== null && duration !== null ? { max, durationMs: duration } : null,
|
|
1448
|
+
rateLimitedNow: ttl >= 0 ? { ttlMs: ttl } : null,
|
|
1449
|
+
workers,
|
|
1450
|
+
groups: null,
|
|
1451
|
+
batch: "unknown",
|
|
1452
|
+
metricsEnabled: r?.metrics_enabled === true,
|
|
1453
|
+
maxLenEvents: numOrNull(meta["opts.maxLenEvents"]),
|
|
1454
|
+
rawMeta
|
|
1455
|
+
};
|
|
1456
|
+
this.setupCache.set(queueName, { at: Date.now(), value });
|
|
1457
|
+
return value;
|
|
1458
|
+
}
|
|
1459
|
+
async listWorkers(queueName) {
|
|
1460
|
+
const rows = await this.query(WORKERS, [
|
|
1461
|
+
queueName,
|
|
1462
|
+
`${queueName.replace(/[\\%_]/g, (c) => `\\${c}`)}:w:%`
|
|
1463
|
+
]);
|
|
1464
|
+
const names = rows.map((r) => {
|
|
1465
|
+
const idx = r.application_name.indexOf(":w:");
|
|
1466
|
+
return idx === -1 ? "worker" : r.application_name.slice(idx + 3);
|
|
1467
|
+
});
|
|
1468
|
+
return { count: names.length, names };
|
|
1469
|
+
}
|
|
1470
|
+
// --- BullMQ Pro groups: the open-source Postgres backend has no groups ------
|
|
1471
|
+
async getGroups() {
|
|
1472
|
+
return { groups: [], total: 0, byStatus: { waiting: 0, limited: 0, maxed: 0, paused: 0 } };
|
|
1473
|
+
}
|
|
1474
|
+
async getGroupJobs(_queueName, _groupId, opts) {
|
|
1475
|
+
return { jobs: [], total: 0, start: opts.start, end: opts.end };
|
|
1476
|
+
}
|
|
1477
|
+
// ---------------------------------------------------------------------------
|
|
1478
|
+
// job schedulers
|
|
1479
|
+
// ---------------------------------------------------------------------------
|
|
1480
|
+
async getSchedulers(queueName, opts) {
|
|
1481
|
+
const limit = opts.end < 0 ? null : Math.max(0, opts.end - opts.start + 1);
|
|
1482
|
+
const [r] = await this.query(SCHEDULERS, [
|
|
1483
|
+
queueName,
|
|
1484
|
+
Math.max(0, opts.start),
|
|
1485
|
+
limit,
|
|
1486
|
+
this.opts.previewBytes
|
|
1487
|
+
]);
|
|
1488
|
+
return { schedulers: (r?.rows ?? []).map(rowToScheduler), total: num(r?.total) };
|
|
1489
|
+
}
|
|
1490
|
+
/** Official API: removes the scheduler AND the delayed job it has queued. */
|
|
1491
|
+
async removeScheduler(queueName, key) {
|
|
1492
|
+
const queue = await this.getQueue(queueName);
|
|
1493
|
+
return { removed: await queue.removeJobScheduler(key) === true };
|
|
1494
|
+
}
|
|
1495
|
+
// ---------------------------------------------------------------------------
|
|
1496
|
+
// flows
|
|
1497
|
+
// ---------------------------------------------------------------------------
|
|
1498
|
+
async sampleFlowEdges(queueName, opts = {}) {
|
|
1499
|
+
const sample = Math.max(1, opts.sample ?? 50);
|
|
1500
|
+
const [r] = await this.query(SAMPLE_PARENTS, [queueName, sample]);
|
|
1501
|
+
const edges = (r?.edges ?? []).map(([parentQueue, count2]) => ({ parentQueue, childQueue: queueName, count: num(count2) }));
|
|
1502
|
+
edges.sort((a, b) => b.count - a.count);
|
|
1503
|
+
return { edges, sampled: num(r?.sampled) };
|
|
1504
|
+
}
|
|
1505
|
+
/**
|
|
1506
|
+
* Walk ONE flow instance, breadth first, at most `maxNodes` jobs read. Same
|
|
1507
|
+
* shape and budget as the Redis walk, but each wave is ONE statement across
|
|
1508
|
+
* every queue it touches (job_dependency holds both ends of every edge).
|
|
1509
|
+
*/
|
|
1510
|
+
async getJobTree(queueName, jobId, opts = {}) {
|
|
1511
|
+
const maxNodes = Math.max(1, Math.min(opts.maxNodes ?? 200, 500));
|
|
1512
|
+
const schema = this.config.prefix;
|
|
1513
|
+
let visited = 0;
|
|
1514
|
+
const read = async (refs) => {
|
|
1515
|
+
if (refs.length === 0) return [];
|
|
1516
|
+
visited += refs.length;
|
|
1517
|
+
const rows = await this.query(TREE_NODES, [refs.map((r) => r.queue), refs.map((r) => r.id), maxNodes]);
|
|
1518
|
+
return refs.map((_, i) => {
|
|
1519
|
+
const row = rows[i];
|
|
1520
|
+
return row && row.found ? row : null;
|
|
1521
|
+
});
|
|
1522
|
+
};
|
|
1523
|
+
const [focus] = await read([{ queue: queueName, id: jobId }]);
|
|
1524
|
+
if (!focus) return null;
|
|
1525
|
+
const focusKey = jobKeyOf(schema, queueName, jobId);
|
|
1526
|
+
let root = { queue: queueName, id: jobId };
|
|
1527
|
+
let climbedLevels = 0;
|
|
1528
|
+
if (opts.fromRoot !== false) {
|
|
1529
|
+
const seen = /* @__PURE__ */ new Set([focusKey]);
|
|
1530
|
+
let current2 = focus;
|
|
1531
|
+
while (current2.parent_queue && current2.parent_id && climbedLevels < MAX_TREE_CLIMB && visited < maxNodes) {
|
|
1532
|
+
const next = { queue: current2.parent_queue, id: current2.parent_id };
|
|
1533
|
+
const key = jobKeyOf(schema, next.queue, next.id);
|
|
1534
|
+
if (seen.has(key)) break;
|
|
1535
|
+
const [parent] = await read([next]);
|
|
1536
|
+
if (!parent) break;
|
|
1537
|
+
seen.add(key);
|
|
1538
|
+
root = next;
|
|
1539
|
+
climbedLevels += 1;
|
|
1540
|
+
current2 = parent;
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1544
|
+
let frontier = [{ ...root, parentKey: null }];
|
|
1545
|
+
let truncated = false;
|
|
1546
|
+
while (frontier.length > 0 && !truncated) {
|
|
1547
|
+
const wave = frontier.filter((f) => !nodes.has(jobKeyOf(schema, f.queue, f.id)));
|
|
1548
|
+
const room = maxNodes - nodes.size;
|
|
1549
|
+
if (room <= 0) {
|
|
1550
|
+
truncated = wave.length > 0;
|
|
1551
|
+
break;
|
|
1552
|
+
}
|
|
1553
|
+
const slice = wave.length > room ? (truncated = true, wave.slice(0, room)) : wave;
|
|
1554
|
+
const rows = await read(slice);
|
|
1555
|
+
const next = [];
|
|
1556
|
+
slice.forEach((entry, i) => {
|
|
1557
|
+
const key = jobKeyOf(schema, entry.queue, entry.id);
|
|
1558
|
+
const row = rows[i] ?? null;
|
|
1559
|
+
nodes.set(key, treeNode(schema, entry, row, maxNodes));
|
|
1560
|
+
for (const [childQueue, childId] of (row?.children ?? []).slice(0, maxNodes)) {
|
|
1561
|
+
const childKey = jobKeyOf(schema, childQueue, childId);
|
|
1562
|
+
if (!nodes.has(childKey)) next.push({ queue: childQueue, id: childId, parentKey: key });
|
|
1563
|
+
}
|
|
1564
|
+
});
|
|
1565
|
+
frontier = next;
|
|
1566
|
+
}
|
|
1567
|
+
return {
|
|
1568
|
+
rootKey: jobKeyOf(schema, root.queue, root.id),
|
|
1569
|
+
focusKey,
|
|
1570
|
+
nodes: [...nodes.values()],
|
|
1571
|
+
truncated: truncated || nodes.size >= maxNodes,
|
|
1572
|
+
visited,
|
|
1573
|
+
climbedLevels
|
|
1574
|
+
};
|
|
1575
|
+
}
|
|
1576
|
+
// ---------------------------------------------------------------------------
|
|
1577
|
+
// writes (official bullmq API; we never write BullMQ's tables ourselves)
|
|
1578
|
+
// ---------------------------------------------------------------------------
|
|
1579
|
+
/**
|
|
1580
|
+
* One bullmq Queue per queue name, created on the first write. BullMQ builds
|
|
1581
|
+
* its own pool from the config object (that is the only way it honours a
|
|
1582
|
+
* non-default schema); `max: 1` and a short idle timeout keep a dashboard
|
|
1583
|
+
* that touched 50 queues from holding 50 connections. A Queue never opens the
|
|
1584
|
+
* LISTEN connection — only Workers and QueueEvents do.
|
|
1585
|
+
*/
|
|
1586
|
+
async getQueue(queueName) {
|
|
1587
|
+
await this.ensureReady();
|
|
1588
|
+
this.invalidateStats();
|
|
1589
|
+
let q = this.queues.get(queueName);
|
|
1590
|
+
if (q) return q;
|
|
1591
|
+
q = new Queue(
|
|
1592
|
+
queueName,
|
|
1593
|
+
{
|
|
1594
|
+
connection: {
|
|
1595
|
+
connectionString: nodePgConnectionString(this.config.url),
|
|
1596
|
+
schema: this.config.prefix,
|
|
1597
|
+
max: 1,
|
|
1598
|
+
idleTimeoutMillis: 1e4,
|
|
1599
|
+
application_name: APPLICATION_NAME
|
|
1600
|
+
},
|
|
1601
|
+
skipMetasUpdate: true
|
|
1602
|
+
},
|
|
1603
|
+
createPostgresBackend
|
|
1604
|
+
);
|
|
1605
|
+
q.on("error", () => void 0);
|
|
1606
|
+
this.queues.set(queueName, q);
|
|
1607
|
+
return q;
|
|
1608
|
+
}
|
|
1609
|
+
/**
|
|
1610
|
+
* Every write goes through getQueue, so dropping the stats window here keeps
|
|
1611
|
+
* an action from being followed by a count from before it.
|
|
1612
|
+
*/
|
|
1613
|
+
invalidateStats() {
|
|
1614
|
+
this.statsCache.clear();
|
|
1615
|
+
this.countCache.clear();
|
|
1616
|
+
}
|
|
1617
|
+
async getBullJob(queueName, jobId) {
|
|
1618
|
+
const queue = await this.getQueue(queueName);
|
|
1619
|
+
const job = await Job.fromId(queue, jobId);
|
|
1620
|
+
if (!job) throw new Error("job_not_found");
|
|
1621
|
+
return job;
|
|
1622
|
+
}
|
|
1623
|
+
async addJob(queueName, name, data, opts = {}) {
|
|
1624
|
+
const queue = await this.getQueue(queueName);
|
|
1625
|
+
const job = await queue.add(name, data, opts);
|
|
1626
|
+
return { id: String(job.id) };
|
|
1627
|
+
}
|
|
1628
|
+
async retryJob(queueName, jobId) {
|
|
1629
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1630
|
+
const state = await job.getState();
|
|
1631
|
+
if (state !== "failed" && state !== "completed") throw new Error(`cannot_retry_job_in_state_${state}`);
|
|
1632
|
+
await job.retry(state);
|
|
1633
|
+
}
|
|
1634
|
+
async removeJob(queueName, jobId) {
|
|
1635
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1636
|
+
await job.remove();
|
|
1637
|
+
}
|
|
1638
|
+
/** Same rules as the Redis inspector: a scheduler's job asks the operator first. */
|
|
1639
|
+
async promoteJob(queueName, jobId, scheduler = "run_copy") {
|
|
1640
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1641
|
+
if (!job.repeatJobKey) {
|
|
1642
|
+
await job.promote();
|
|
1643
|
+
return { mode: "promoted" };
|
|
1644
|
+
}
|
|
1645
|
+
if (scheduler === "skip_next") {
|
|
1646
|
+
await job.promote();
|
|
1647
|
+
return { mode: "skipped_next", schedulerId: job.repeatJobKey };
|
|
1648
|
+
}
|
|
1649
|
+
const state = await job.getState();
|
|
1650
|
+
if (state !== "delayed") throw new Error(`cannot_promote_job_in_state_${state}`);
|
|
1651
|
+
const { repeat: _r, jobId: _j, repeatJobKey: _k, prevMillis: _p, delay: _d, timestamp: _t, ...opts } = job.opts;
|
|
1652
|
+
const copy = await this.addJob(queueName, job.name, job.data, opts);
|
|
1653
|
+
return { mode: "ran_copy", jobId: copy.id, schedulerId: job.repeatJobKey };
|
|
1654
|
+
}
|
|
1655
|
+
/** Partial result, bounded concurrency: see Inspector.bulkJobAction. */
|
|
1656
|
+
async bulkJobAction(queueName, action, jobIds) {
|
|
1657
|
+
const ok = [];
|
|
1658
|
+
const failed = [];
|
|
1659
|
+
const ids = [...new Set(jobIds)];
|
|
1660
|
+
const run = async (jobId) => {
|
|
1661
|
+
try {
|
|
1662
|
+
if (action === "retry") await this.retryJob(queueName, jobId);
|
|
1663
|
+
else if (action === "remove") await this.removeJob(queueName, jobId);
|
|
1664
|
+
else await this.promoteJob(queueName, jobId);
|
|
1665
|
+
ok.push(jobId);
|
|
1666
|
+
} catch (err) {
|
|
1667
|
+
failed.push({ jobId, reason: errorMessage(err) });
|
|
1668
|
+
}
|
|
1669
|
+
};
|
|
1670
|
+
for (let i = 0; i < ids.length; i += BULK_CONCURRENCY) {
|
|
1671
|
+
await Promise.all(ids.slice(i, i + BULK_CONCURRENCY).map(run));
|
|
1672
|
+
}
|
|
1673
|
+
return { action, ok, failed, requested: ids.length };
|
|
1674
|
+
}
|
|
1675
|
+
/** Operator "discard": an active job straight to failed, no retry (see RedisInspector.discardJob). */
|
|
1676
|
+
async discardJob(queueName, jobId) {
|
|
1677
|
+
const job = await this.getBullJob(queueName, jobId);
|
|
1678
|
+
const state = await job.getState();
|
|
1679
|
+
if (state !== "active") throw new Error(`cannot_discard_job_in_state_${state}`);
|
|
1680
|
+
try {
|
|
1681
|
+
await job.moveToFailed(new UnrecoverableError("Discarded from Bullpane"), "0");
|
|
1682
|
+
} catch (err) {
|
|
1683
|
+
throw new Error(`cannot_discard_active_job: ${errorMessage(err)}`);
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
async pauseQueue(queueName) {
|
|
1687
|
+
await (await this.getQueue(queueName)).pause();
|
|
1688
|
+
this.setupCache.delete(queueName);
|
|
1689
|
+
this.invalidateStats();
|
|
1690
|
+
}
|
|
1691
|
+
async resumeQueue(queueName) {
|
|
1692
|
+
await (await this.getQueue(queueName)).resume();
|
|
1693
|
+
this.setupCache.delete(queueName);
|
|
1694
|
+
this.invalidateStats();
|
|
1695
|
+
}
|
|
1696
|
+
async cleanQueue(queueName, state, graceMs, limit) {
|
|
1697
|
+
const ids = await (await this.getQueue(queueName)).clean(graceMs, limit, state);
|
|
1698
|
+
return { removed: ids.length };
|
|
1699
|
+
}
|
|
1700
|
+
async retryAll(queueName, state) {
|
|
1701
|
+
await (await this.getQueue(queueName)).retryJobs({ state });
|
|
1702
|
+
}
|
|
1703
|
+
async drainQueue(queueName, includeDelayed) {
|
|
1704
|
+
await (await this.getQueue(queueName)).drain(includeDelayed);
|
|
1705
|
+
}
|
|
1706
|
+
async obliterateQueue(queueName) {
|
|
1707
|
+
const queue = await this.getQueue(queueName);
|
|
1708
|
+
await queue.obliterate({ force: true });
|
|
1709
|
+
this.queues.delete(queueName);
|
|
1710
|
+
await queue.close().catch(() => void 0);
|
|
1711
|
+
this.discovered = null;
|
|
1712
|
+
}
|
|
1713
|
+
};
|
|
1714
|
+
function treeNode(schema, entry, row, maxNodes) {
|
|
1715
|
+
const key = jobKeyOf(schema, entry.queue, entry.id);
|
|
1716
|
+
const base = { key, id: entry.id, queueName: entry.queue, queueKey: queueKeyOf(schema, entry.queue) };
|
|
1717
|
+
if (!row) {
|
|
1718
|
+
return {
|
|
1719
|
+
...base,
|
|
1720
|
+
name: "",
|
|
1721
|
+
state: "unknown",
|
|
1722
|
+
timestamp: 0,
|
|
1723
|
+
finishedOn: null,
|
|
1724
|
+
attemptsMade: 0,
|
|
1725
|
+
failedReason: null,
|
|
1726
|
+
progress: null,
|
|
1727
|
+
parentKey: entry.parentKey,
|
|
1728
|
+
dependencies: null,
|
|
1729
|
+
childrenTruncated: false,
|
|
1730
|
+
missing: true
|
|
1731
|
+
};
|
|
441
1732
|
}
|
|
1733
|
+
const processed = num(row.processed);
|
|
1734
|
+
const unprocessed = num(row.unprocessed);
|
|
1735
|
+
return {
|
|
1736
|
+
...base,
|
|
1737
|
+
name: row.name ?? "",
|
|
1738
|
+
state: bullpaneState(row.state, row.priority),
|
|
1739
|
+
timestamp: num(row.added_at_ms),
|
|
1740
|
+
finishedOn: numOrNull(row.finished_at_ms),
|
|
1741
|
+
attemptsMade: num(row.attempts_made),
|
|
1742
|
+
failedReason: row.failed_reason ?? null,
|
|
1743
|
+
progress: progressOf(row.progress),
|
|
1744
|
+
// the walk's own edge wins over the stored parent: it is the edge being drawn
|
|
1745
|
+
parentKey: entry.parentKey ?? (row.parent_queue && row.parent_id ? jobKeyOf(schema, row.parent_queue, row.parent_id) : null),
|
|
1746
|
+
dependencies: processed + unprocessed > 0 ? { processed, unprocessed } : null,
|
|
1747
|
+
// TREE_NODES reads maxNodes + 1 children: one more than the walk could hold means there are more
|
|
1748
|
+
childrenTruncated: (row.children?.length ?? 0) > maxNodes,
|
|
1749
|
+
missing: false
|
|
1750
|
+
};
|
|
442
1751
|
}
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
1752
|
+
|
|
1753
|
+
// packages/pg-inspector/src/pool.ts
|
|
1754
|
+
var PgInspectorPool = class {
|
|
1755
|
+
constructor(options = {}) {
|
|
1756
|
+
this.options = options;
|
|
1757
|
+
}
|
|
1758
|
+
inspectors = /* @__PURE__ */ new Map();
|
|
1759
|
+
get(config) {
|
|
1760
|
+
const existing = this.inspectors.get(config.id);
|
|
1761
|
+
if (existing && sameTarget(existing.config, config)) return existing;
|
|
1762
|
+
if (existing) void existing.close().catch(() => void 0);
|
|
1763
|
+
const created = new PgInspector(config, this.options);
|
|
1764
|
+
this.inspectors.set(config.id, created);
|
|
1765
|
+
return created;
|
|
1766
|
+
}
|
|
1767
|
+
async evict(id3) {
|
|
1768
|
+
const existing = this.inspectors.get(id3);
|
|
1769
|
+
if (!existing) return;
|
|
1770
|
+
this.inspectors.delete(id3);
|
|
1771
|
+
await existing.close();
|
|
1772
|
+
}
|
|
1773
|
+
async closeAll() {
|
|
1774
|
+
const all = [...this.inspectors.values()];
|
|
1775
|
+
this.inspectors.clear();
|
|
1776
|
+
await Promise.allSettled(all.map((i) => i.close()));
|
|
1777
|
+
}
|
|
1778
|
+
};
|
|
1779
|
+
function sameTarget(a, b) {
|
|
1780
|
+
return a.url === b.url && a.prefix === (b.prefix ?? "bullmq") && (a.queueFilter ?? null) === (b.queueFilter ?? null);
|
|
451
1781
|
}
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
access: z.enum(["read", "write"]).default("read")
|
|
457
|
-
});
|
|
1782
|
+
|
|
1783
|
+
// packages/redis-inspector/src/inspector.ts
|
|
1784
|
+
import { Cluster as Cluster2 } from "ioredis";
|
|
1785
|
+
import { Job as Job2, Queue as Queue2, UnrecoverableError as UnrecoverableError2 } from "bullmq";
|
|
458
1786
|
|
|
459
1787
|
// packages/redis-inspector/src/connection.ts
|
|
460
1788
|
import Redis, { Cluster } from "ioredis";
|
|
@@ -664,14 +1992,6 @@ function queueNameFromQueueKey(prefix, qualified) {
|
|
|
664
1992
|
}
|
|
665
1993
|
|
|
666
1994
|
// packages/redis-inspector/src/util.ts
|
|
667
|
-
function globToRegExp(glob) {
|
|
668
|
-
const escaped = glob.split("").map((ch) => {
|
|
669
|
-
if (ch === "*") return ".*";
|
|
670
|
-
if (ch === "?") return ".";
|
|
671
|
-
return ch.replace(/[.+^${}()|[\]\\/]/g, "\\$&");
|
|
672
|
-
}).join("");
|
|
673
|
-
return new RegExp(`^${escaped}$`);
|
|
674
|
-
}
|
|
675
1995
|
function safeJsonParse(raw, fallback) {
|
|
676
1996
|
if (raw === null || raw === void 0) return fallback === void 0 ? null : fallback;
|
|
677
1997
|
try {
|
|
@@ -721,7 +2041,7 @@ function totalKeysFromInfo(info) {
|
|
|
721
2041
|
}
|
|
722
2042
|
return seen ? total : null;
|
|
723
2043
|
}
|
|
724
|
-
function
|
|
2044
|
+
function errorMessage2(err) {
|
|
725
2045
|
if (err instanceof Error) return err.message;
|
|
726
2046
|
return String(err);
|
|
727
2047
|
}
|
|
@@ -870,13 +2190,13 @@ function asNumber(reply, fallback = 0) {
|
|
|
870
2190
|
function asStringArray(reply) {
|
|
871
2191
|
return asArray(reply).map((v) => typeof v === "string" ? v : String(v ?? ""));
|
|
872
2192
|
}
|
|
873
|
-
function
|
|
2193
|
+
function metricPoints2(reply) {
|
|
874
2194
|
return asStringArray(reply).map((s) => Number(s) || 0).reverse();
|
|
875
2195
|
}
|
|
876
2196
|
function str(v) {
|
|
877
2197
|
return typeof v === "string" && v !== "" ? v : typeof v === "number" ? String(v) : null;
|
|
878
2198
|
}
|
|
879
|
-
function
|
|
2199
|
+
function rowToScheduler2(row) {
|
|
880
2200
|
const key = String(row[0] ?? "");
|
|
881
2201
|
const name = str(row[2]);
|
|
882
2202
|
const data = str(row[11]);
|
|
@@ -982,10 +2302,11 @@ function pipelineScript(pipeline, name, args) {
|
|
|
982
2302
|
}
|
|
983
2303
|
|
|
984
2304
|
// packages/redis-inspector/src/inspector.ts
|
|
985
|
-
var
|
|
2305
|
+
var DEFAULTS2 = {
|
|
986
2306
|
discoveryTtlMs: 3e4,
|
|
987
2307
|
previewBytes: 2048,
|
|
988
2308
|
maxScanPerCall: 1e3,
|
|
2309
|
+
groupScanPerCall: 1e4,
|
|
989
2310
|
connectTimeoutMs: 5e3,
|
|
990
2311
|
maxScanIterations: 2e3,
|
|
991
2312
|
discoveryScanBudgetMs: 1500,
|
|
@@ -995,12 +2316,12 @@ var DEFAULTS = {
|
|
|
995
2316
|
searchByteBudget: 8 * 1024 * 1024
|
|
996
2317
|
};
|
|
997
2318
|
var SCAN_COUNT = 1e3;
|
|
998
|
-
var
|
|
999
|
-
var
|
|
1000
|
-
var
|
|
1001
|
-
var
|
|
1002
|
-
var
|
|
1003
|
-
var
|
|
2319
|
+
var STATS_METRIC_POINTS2 = 60;
|
|
2320
|
+
var BULK_CONCURRENCY2 = 8;
|
|
2321
|
+
var DEFAULT_RATE_WINDOW_MINUTES2 = 60;
|
|
2322
|
+
var FORCE_DISCOVERY_MIN_MS2 = 5e3;
|
|
2323
|
+
var SETUP_CACHE_MS2 = 1e4;
|
|
2324
|
+
var DETAIL_LOG_TAIL2 = 100;
|
|
1004
2325
|
var FLOW_SAMPLE_STATES = ["completed", "failed", "waiting-children", "waiting", "active", "delayed"];
|
|
1005
2326
|
var RedisInspector = class {
|
|
1006
2327
|
config;
|
|
@@ -1025,12 +2346,14 @@ var RedisInspector = class {
|
|
|
1025
2346
|
bullmqCluster = null;
|
|
1026
2347
|
closed = false;
|
|
1027
2348
|
constructor(config, options = {}) {
|
|
2349
|
+
if (config.kind && config.kind !== "redis") throw new Error(`RedisInspector cannot open a ${config.kind} connection`);
|
|
1028
2350
|
this.config = {
|
|
1029
2351
|
...config,
|
|
2352
|
+
kind: "redis",
|
|
1030
2353
|
prefix: config.prefix ?? "bull",
|
|
1031
2354
|
cluster: config.cluster ?? false
|
|
1032
2355
|
};
|
|
1033
|
-
this.opts = { ...
|
|
2356
|
+
this.opts = { ...DEFAULTS2, ...stripUndefined(options) };
|
|
1034
2357
|
this.filter = config.queueFilter ? globToRegExp(config.queueFilter) : null;
|
|
1035
2358
|
this.client = createReadClient(this.config, this.opts.connectTimeoutMs);
|
|
1036
2359
|
this.client.on("error", () => void 0);
|
|
@@ -1091,7 +2414,7 @@ var RedisInspector = class {
|
|
|
1091
2414
|
ok: false,
|
|
1092
2415
|
latencyMs: Math.round((performance.now() - t0) * 10) / 10,
|
|
1093
2416
|
redisVersion: null,
|
|
1094
|
-
error:
|
|
2417
|
+
error: errorMessage2(err)
|
|
1095
2418
|
};
|
|
1096
2419
|
}
|
|
1097
2420
|
}
|
|
@@ -1174,7 +2497,7 @@ var RedisInspector = class {
|
|
|
1174
2497
|
async discoverQueues(opts = {}) {
|
|
1175
2498
|
const now = Date.now();
|
|
1176
2499
|
const age = this.discovered ? now - this.discovered.at : Number.POSITIVE_INFINITY;
|
|
1177
|
-
if (this.discovered && age < (opts.force ?
|
|
2500
|
+
if (this.discovered && age < (opts.force ? FORCE_DISCOVERY_MIN_MS2 : this.opts.discoveryTtlMs)) {
|
|
1178
2501
|
return this.discovered.names;
|
|
1179
2502
|
}
|
|
1180
2503
|
if (!this.discovering) {
|
|
@@ -1328,19 +2651,19 @@ var RedisInspector = class {
|
|
|
1328
2651
|
if (queueNames.length === 0) return out;
|
|
1329
2652
|
const c = await this.ensureConnected();
|
|
1330
2653
|
const withMetrics = opts.withMetrics ? 1 : 0;
|
|
1331
|
-
const windowMinutes = opts.rateWindowMinutes ??
|
|
2654
|
+
const windowMinutes = opts.rateWindowMinutes ?? DEFAULT_RATE_WINDOW_MINUTES2;
|
|
1332
2655
|
const since = Date.now() - windowMinutes * 6e4;
|
|
1333
2656
|
let replies;
|
|
1334
2657
|
if (c instanceof Cluster2) {
|
|
1335
2658
|
replies = await Promise.all(
|
|
1336
2659
|
queueNames.map(
|
|
1337
|
-
(q) => callScript(c, "queueStats", [...this.statsKeys(q), withMetrics,
|
|
2660
|
+
(q) => callScript(c, "queueStats", [...this.statsKeys(q), withMetrics, STATS_METRIC_POINTS2, since, queueKeyPrefix(this.config.prefix, q)]).catch(() => null)
|
|
1338
2661
|
)
|
|
1339
2662
|
);
|
|
1340
2663
|
} else {
|
|
1341
2664
|
const pipeline = c.pipeline();
|
|
1342
2665
|
for (const q of queueNames) {
|
|
1343
|
-
pipelineScript(pipeline, "queueStats", [...this.statsKeys(q), withMetrics,
|
|
2666
|
+
pipelineScript(pipeline, "queueStats", [...this.statsKeys(q), withMetrics, STATS_METRIC_POINTS2, since, queueKeyPrefix(this.config.prefix, q)]);
|
|
1344
2667
|
}
|
|
1345
2668
|
const results = await pipeline.exec() ?? [];
|
|
1346
2669
|
replies = results.map(([err, reply]) => err ? null : reply);
|
|
@@ -1459,22 +2782,28 @@ var RedisInspector = class {
|
|
|
1459
2782
|
* are not read at all), plain substring match done in Lua. The cursor is the
|
|
1460
2783
|
* index of the next job (newest = 0). Measured: 1000 × 1 MB jobs used to hold
|
|
1461
2784
|
* Redis for 13 s per call; both bounds keep a call in the milliseconds.
|
|
2785
|
+
*
|
|
2786
|
+
* `groupId` filters before any payload is read (one HMGET of the group fields
|
|
2787
|
+
* and opts per job), so a group scan covers groupScanPerCall jobs per call.
|
|
1462
2788
|
*/
|
|
1463
2789
|
async searchJobs(queueName, state, query, opts) {
|
|
1464
2790
|
const c = await this.ensureConnected();
|
|
1465
2791
|
const cursor = Math.max(0, toInt(opts.cursor ?? "0", 0));
|
|
2792
|
+
const group = opts.groupId ?? "";
|
|
1466
2793
|
const reply = asArray(
|
|
1467
2794
|
await callScript(c, "getJobsSearch", [
|
|
1468
2795
|
stateKey(this.config.prefix, queueName, state),
|
|
1469
2796
|
STATE_KEY[state].type,
|
|
1470
2797
|
cursor,
|
|
1471
|
-
this.opts.maxScanPerCall,
|
|
2798
|
+
group ? this.opts.groupScanPerCall : this.opts.maxScanPerCall,
|
|
1472
2799
|
query.toLowerCase(),
|
|
1473
2800
|
Math.max(1, opts.limit),
|
|
1474
2801
|
queueKeyPrefix(this.config.prefix, queueName),
|
|
1475
2802
|
this.opts.previewBytes,
|
|
1476
2803
|
this.opts.searchFieldCapBytes,
|
|
1477
2804
|
this.opts.searchByteBudget,
|
|
2805
|
+
group,
|
|
2806
|
+
GROUP_ID_FIELDS.join(","),
|
|
1478
2807
|
...JOB_SUMMARY_FIELDS
|
|
1479
2808
|
])
|
|
1480
2809
|
);
|
|
@@ -1497,7 +2826,7 @@ var RedisInspector = class {
|
|
|
1497
2826
|
p + JOB_KEY.processed(jobId),
|
|
1498
2827
|
...allStateKeys(this.config.prefix, queueName),
|
|
1499
2828
|
jobId,
|
|
1500
|
-
|
|
2829
|
+
DETAIL_LOG_TAIL2
|
|
1501
2830
|
]);
|
|
1502
2831
|
if (!Array.isArray(reply)) return null;
|
|
1503
2832
|
const hash = flatToHash(asArray(reply[0]));
|
|
@@ -1515,8 +2844,8 @@ var RedisInspector = class {
|
|
|
1515
2844
|
async getJobLogs(queueName, jobId, opts) {
|
|
1516
2845
|
const c = await this.ensureConnected();
|
|
1517
2846
|
const key = queueKeyPrefix(this.config.prefix, queueName) + JOB_KEY.logs(jobId);
|
|
1518
|
-
const [logs,
|
|
1519
|
-
return { logs, count };
|
|
2847
|
+
const [logs, count2] = await Promise.all([c.lrange(key, opts.start, opts.end), c.llen(key)]);
|
|
2848
|
+
return { logs, count: count2 };
|
|
1520
2849
|
}
|
|
1521
2850
|
/** Metric lists are LPUSHed (newest at index 0); we return oldest first. */
|
|
1522
2851
|
async getMetrics(queueName, points) {
|
|
@@ -1527,7 +2856,7 @@ var RedisInspector = class {
|
|
|
1527
2856
|
c.lrange(p + QUEUE_KEY.metricsCompletedData, 0, n - 1),
|
|
1528
2857
|
c.lrange(p + QUEUE_KEY.metricsFailedData, 0, n - 1)
|
|
1529
2858
|
]);
|
|
1530
|
-
return { completed:
|
|
2859
|
+
return { completed: metricPoints2(completed), failed: metricPoints2(failed) };
|
|
1531
2860
|
}
|
|
1532
2861
|
// ---------------------------------------------------------------------------
|
|
1533
2862
|
// BullMQ Pro groups (read only)
|
|
@@ -1543,7 +2872,7 @@ var RedisInspector = class {
|
|
|
1543
2872
|
*/
|
|
1544
2873
|
async getQueueSetup(queueName) {
|
|
1545
2874
|
const cached = this.setupCache.get(queueName);
|
|
1546
|
-
if (cached && Date.now() - cached.at <
|
|
2875
|
+
if (cached && Date.now() - cached.at < SETUP_CACHE_MS2) return cached.value;
|
|
1547
2876
|
const c = await this.ensureConnected();
|
|
1548
2877
|
const p = queueKeyPrefix(this.config.prefix, queueName);
|
|
1549
2878
|
const [reply, workers] = await Promise.all([
|
|
@@ -1682,7 +3011,7 @@ var RedisInspector = class {
|
|
|
1682
3011
|
const reply = asArray(
|
|
1683
3012
|
await callScript(c, "getSchedulers", [p + SCHEDULER_KEY.repeat, opts.start, opts.end, p, this.opts.previewBytes])
|
|
1684
3013
|
);
|
|
1685
|
-
const schedulers = asArray(reply[1]).map((row) =>
|
|
3014
|
+
const schedulers = asArray(reply[1]).map((row) => rowToScheduler2(asArray(row)));
|
|
1686
3015
|
return { schedulers, total: asNumber(reply[0]) };
|
|
1687
3016
|
}
|
|
1688
3017
|
/**
|
|
@@ -1756,7 +3085,7 @@ var RedisInspector = class {
|
|
|
1756
3085
|
if (opts.fromRoot !== false) {
|
|
1757
3086
|
const seen = /* @__PURE__ */ new Set([focusKey]);
|
|
1758
3087
|
let current2 = focus;
|
|
1759
|
-
while (current2.parentKey && climbedLevels <
|
|
3088
|
+
while (current2.parentKey && climbedLevels < MAX_TREE_CLIMB2) {
|
|
1760
3089
|
const next = current2.parentKey;
|
|
1761
3090
|
if (seen.has(next)) break;
|
|
1762
3091
|
const parent = await readOne(next);
|
|
@@ -1870,7 +3199,7 @@ var RedisInspector = class {
|
|
|
1870
3199
|
this.bullmqCluster = ownedCluster;
|
|
1871
3200
|
}
|
|
1872
3201
|
}
|
|
1873
|
-
q = new
|
|
3202
|
+
q = new Queue2(queueName, {
|
|
1874
3203
|
connection: this.bullmqCluster ?? connection,
|
|
1875
3204
|
prefix: this.config.prefix,
|
|
1876
3205
|
skipMetasUpdate: true
|
|
@@ -1881,7 +3210,7 @@ var RedisInspector = class {
|
|
|
1881
3210
|
}
|
|
1882
3211
|
async getBullJob(queueName, jobId) {
|
|
1883
3212
|
const queue = await this.getQueue(queueName);
|
|
1884
|
-
const job = await
|
|
3213
|
+
const job = await Job2.fromId(queue, jobId);
|
|
1885
3214
|
if (!job) throw new Error("job_not_found");
|
|
1886
3215
|
return job;
|
|
1887
3216
|
}
|
|
@@ -1940,11 +3269,11 @@ var RedisInspector = class {
|
|
|
1940
3269
|
else await this.promoteJob(queueName, jobId);
|
|
1941
3270
|
ok.push(jobId);
|
|
1942
3271
|
} catch (err) {
|
|
1943
|
-
failed.push({ jobId, reason:
|
|
3272
|
+
failed.push({ jobId, reason: errorMessage2(err) });
|
|
1944
3273
|
}
|
|
1945
3274
|
};
|
|
1946
|
-
for (let i = 0; i < ids.length; i +=
|
|
1947
|
-
await Promise.all(ids.slice(i, i +
|
|
3275
|
+
for (let i = 0; i < ids.length; i += BULK_CONCURRENCY2) {
|
|
3276
|
+
await Promise.all(ids.slice(i, i + BULK_CONCURRENCY2).map(run));
|
|
1948
3277
|
}
|
|
1949
3278
|
return { action, ok, failed, requested: ids.length };
|
|
1950
3279
|
}
|
|
@@ -1963,9 +3292,9 @@ var RedisInspector = class {
|
|
|
1963
3292
|
throw new Error(`cannot_discard_job_in_state_${state}`);
|
|
1964
3293
|
}
|
|
1965
3294
|
try {
|
|
1966
|
-
await job.moveToFailed(new
|
|
3295
|
+
await job.moveToFailed(new UnrecoverableError2("Discarded from Bullpane"), "0");
|
|
1967
3296
|
} catch (err) {
|
|
1968
|
-
throw new Error(`cannot_discard_active_job: ${
|
|
3297
|
+
throw new Error(`cannot_discard_active_job: ${errorMessage2(err)}`);
|
|
1969
3298
|
}
|
|
1970
3299
|
}
|
|
1971
3300
|
async pauseQueue(queueName) {
|
|
@@ -2003,9 +3332,9 @@ function parseStats(reply, withMetrics, windowMinutes) {
|
|
|
2003
3332
|
STATE_ORDER.forEach((state, i) => {
|
|
2004
3333
|
counts[state] = asNumber(r[i]);
|
|
2005
3334
|
});
|
|
2006
|
-
const metricsCompleted =
|
|
2007
|
-
const metricsFailed =
|
|
2008
|
-
const
|
|
3335
|
+
const metricsCompleted = metricPoints2(r[11] ?? []);
|
|
3336
|
+
const metricsFailed = metricPoints2(r[12] ?? []);
|
|
3337
|
+
const prunesCompleted2 = optsPrunesCompleted(typeof r[16] === "string" ? r[16] : null);
|
|
2009
3338
|
const totalCompleted = toIntOrNull(typeof r[17] === "string" ? r[17] : null);
|
|
2010
3339
|
const totalFailed = toIntOrNull(typeof r[18] === "string" ? r[18] : null);
|
|
2011
3340
|
const hasMetrics = totalCompleted !== null || totalFailed !== null;
|
|
@@ -2037,7 +3366,7 @@ function parseStats(reply, withMetrics, windowMinutes) {
|
|
|
2037
3366
|
source: "zset",
|
|
2038
3367
|
// only skewed if the queue really does prune completed jobs AND there are
|
|
2039
3368
|
// failures to unbalance the ratio; with no failures, 100% is still 100%.
|
|
2040
|
-
retentionSkewed:
|
|
3369
|
+
retentionSkewed: prunesCompleted2 && failed > 0
|
|
2041
3370
|
};
|
|
2042
3371
|
}
|
|
2043
3372
|
const stats = {
|
|
@@ -2080,7 +3409,7 @@ var JOB_STATE_SET = new Set(STATE_ORDER);
|
|
|
2080
3409
|
function isJobState(s) {
|
|
2081
3410
|
return JOB_STATE_SET.has(s);
|
|
2082
3411
|
}
|
|
2083
|
-
var
|
|
3412
|
+
var MAX_TREE_CLIMB2 = 50;
|
|
2084
3413
|
function splitJobKey(prefix, fullKey) {
|
|
2085
3414
|
const cut = fullKey.lastIndexOf(":");
|
|
2086
3415
|
if (cut <= 0) return null;
|
|
@@ -2178,7 +3507,7 @@ var RedisInspectorPool = class {
|
|
|
2178
3507
|
inspectors = /* @__PURE__ */ new Map();
|
|
2179
3508
|
get(config) {
|
|
2180
3509
|
const existing = this.inspectors.get(config.id);
|
|
2181
|
-
if (existing &&
|
|
3510
|
+
if (existing && sameTarget2(existing.config, config)) return existing;
|
|
2182
3511
|
if (existing) {
|
|
2183
3512
|
void existing.close().catch(() => void 0);
|
|
2184
3513
|
}
|
|
@@ -2202,11 +3531,26 @@ var RedisInspectorPool = class {
|
|
|
2202
3531
|
return [...this.inspectors.keys()];
|
|
2203
3532
|
}
|
|
2204
3533
|
};
|
|
2205
|
-
function
|
|
3534
|
+
function sameTarget2(a, b) {
|
|
2206
3535
|
return a.url === b.url && a.prefix === (b.prefix ?? "bull") && a.cluster === (b.cluster ?? false) && (a.queueFilter ?? null) === (b.queueFilter ?? null);
|
|
2207
3536
|
}
|
|
2208
|
-
|
|
2209
|
-
|
|
3537
|
+
|
|
3538
|
+
// apps/server/src/services/inspectorPool.ts
|
|
3539
|
+
function createInspectorPool2(options = {}) {
|
|
3540
|
+
const redis = new RedisInspectorPool(options);
|
|
3541
|
+
const postgres = new PgInspectorPool(options);
|
|
3542
|
+
const pick = (config) => config.kind === "postgres" ? postgres : redis;
|
|
3543
|
+
return {
|
|
3544
|
+
get: (config) => pick(config).get(config),
|
|
3545
|
+
// A connection's kind never changes (updateConnectionSchema omits it), but
|
|
3546
|
+
// evicting from both costs nothing and cannot leave a stale inspector behind.
|
|
3547
|
+
evict: async (id3) => {
|
|
3548
|
+
await Promise.all([redis.evict(id3), postgres.evict(id3)]);
|
|
3549
|
+
},
|
|
3550
|
+
closeAll: async () => {
|
|
3551
|
+
await Promise.all([redis.closeAll(), postgres.closeAll()]);
|
|
3552
|
+
}
|
|
3553
|
+
};
|
|
2210
3554
|
}
|
|
2211
3555
|
|
|
2212
3556
|
// apps/server/src/app.ts
|
|
@@ -2487,6 +3831,8 @@ var ssoProviders = mysqlTable(
|
|
|
2487
3831
|
var connections = mysqlTable("connections", {
|
|
2488
3832
|
id: id(),
|
|
2489
3833
|
name: varchar("name", { length: 80 }).notNull(),
|
|
3834
|
+
/** "redis" | "postgres" (BullMQ 6's Postgres backend). `prefix` holds the Postgres schema. */
|
|
3835
|
+
kind: varchar("kind", { length: 16 }).$type().notNull().default("redis"),
|
|
2490
3836
|
url: text("url").notNull(),
|
|
2491
3837
|
prefix: varchar("prefix", { length: 64 }).notNull().default("bull"),
|
|
2492
3838
|
cluster: boolean("cluster").notNull().default(false),
|
|
@@ -2713,6 +4059,7 @@ var ssoProviders2 = sqliteTable(
|
|
|
2713
4059
|
var connections2 = sqliteTable("connections", {
|
|
2714
4060
|
id: id2(),
|
|
2715
4061
|
name: text2("name").notNull(),
|
|
4062
|
+
kind: text2("kind").$type().notNull().default("redis"),
|
|
2716
4063
|
url: text2("url").notNull(),
|
|
2717
4064
|
prefix: text2("prefix").notNull().default("bull"),
|
|
2718
4065
|
cluster: bool("cluster").notNull().default(false),
|
|
@@ -2941,8 +4288,8 @@ var readOnlyLocked = () => new HttpError(423, "read_only", "This instance runs i
|
|
|
2941
4288
|
var invalidLicense = (reason) => new HttpError(400, "invalid_license", `Invalid license: ${reason}`);
|
|
2942
4289
|
var licenseAlreadyActivated = (message) => new HttpError(409, "license_already_activated", message);
|
|
2943
4290
|
var licenseServerUnavailable = (reason) => new HttpError(502, "license_server_unavailable", `Could not reach the license server: ${reason}`);
|
|
2944
|
-
var redisUnavailable = (err) => new HttpError(502, "redis_unavailable", `Redis connection failed: ${
|
|
2945
|
-
function
|
|
4291
|
+
var redisUnavailable = (err) => new HttpError(502, "redis_unavailable", `Redis connection failed: ${errorMessage3(err)}`);
|
|
4292
|
+
function errorMessage3(err) {
|
|
2946
4293
|
if (err instanceof Error) return err.message;
|
|
2947
4294
|
return typeof err === "string" ? err : JSON.stringify(err);
|
|
2948
4295
|
}
|
|
@@ -2986,7 +4333,7 @@ function registerErrorHandling(app) {
|
|
|
2986
4333
|
if (mapped.unexpected) {
|
|
2987
4334
|
request.log.error({ err }, "unhandled error");
|
|
2988
4335
|
} else if (mapped.status >= 500) {
|
|
2989
|
-
request.log.warn({ err:
|
|
4336
|
+
request.log.warn({ err: errorMessage3(err), url: request.url }, mapped.body.error);
|
|
2990
4337
|
}
|
|
2991
4338
|
void reply.status(mapped.status).send(mapped.body);
|
|
2992
4339
|
});
|
|
@@ -4890,7 +6237,17 @@ var CONNECTION_ERROR_PATTERNS = [
|
|
|
4890
6237
|
/CLUSTERDOWN/i,
|
|
4891
6238
|
/Failed to refresh slots cache/i,
|
|
4892
6239
|
/All sentinels are unreachable/i,
|
|
4893
|
-
/socket closed unexpectedly/i
|
|
6240
|
+
/socket closed unexpectedly/i,
|
|
6241
|
+
// Postgres (node-postgres / BullMQ's Postgres backend)
|
|
6242
|
+
/password authentication failed/i,
|
|
6243
|
+
/no pg_hba\.conf entry/i,
|
|
6244
|
+
/Connection terminated/i,
|
|
6245
|
+
/timeout exceeded when trying to connect/i,
|
|
6246
|
+
/the database system is (starting up|shutting down)/i,
|
|
6247
|
+
/too many clients already/i,
|
|
6248
|
+
/postgres_schema_missing/,
|
|
6249
|
+
/the PostgreSQL schema requires BullMQ/i,
|
|
6250
|
+
/the PostgreSQL backend requires server version/i
|
|
4894
6251
|
];
|
|
4895
6252
|
var NOT_FOUND_PATTERNS = [/missing key for job/i, /job .* not found/i, /could not be found/i, /does not exist/i];
|
|
4896
6253
|
function isRedisConnectionError(err) {
|
|
@@ -4899,14 +6256,26 @@ function isRedisConnectionError(err) {
|
|
|
4899
6256
|
if (name === "MaxRetriesPerRequestError" || name === "ClusterAllFailedError") return true;
|
|
4900
6257
|
const code = err.code;
|
|
4901
6258
|
if (code && CONNECTION_ERROR_PATTERNS.some((p) => p.test(code))) return true;
|
|
6259
|
+
if (code && /^(08...|28...|3D000|57P0\d)$/.test(code)) return true;
|
|
4902
6260
|
}
|
|
4903
|
-
const message =
|
|
6261
|
+
const message = errorMessage3(err);
|
|
4904
6262
|
return CONNECTION_ERROR_PATTERNS.some((p) => p.test(message));
|
|
4905
6263
|
}
|
|
6264
|
+
function isPermissionDenied(err) {
|
|
6265
|
+
const code = typeof err === "object" && err !== null ? err.code : void 0;
|
|
6266
|
+
return code === "42501" || /permission denied for (table|schema|function|sequence|relation)/i.test(errorMessage3(err));
|
|
6267
|
+
}
|
|
4906
6268
|
function mapInspectorError(err) {
|
|
4907
6269
|
if (err instanceof HttpError) return err;
|
|
4908
6270
|
if (isRedisConnectionError(err)) return redisUnavailable(err);
|
|
4909
|
-
|
|
6271
|
+
if (isPermissionDenied(err)) {
|
|
6272
|
+
return new HttpError(
|
|
6273
|
+
403,
|
|
6274
|
+
"database_permission_denied",
|
|
6275
|
+
`The database refused the action: ${errorMessage3(err)}. This connection's role can read but not write; give it the same rights your workers have to use actions from Bullpane.`
|
|
6276
|
+
);
|
|
6277
|
+
}
|
|
6278
|
+
const message = errorMessage3(err);
|
|
4910
6279
|
if (NOT_FOUND_PATTERNS.some((p) => p.test(message))) return notFound("Job");
|
|
4911
6280
|
return conflict(message);
|
|
4912
6281
|
}
|
|
@@ -4930,14 +6299,14 @@ async function connectionRoutes(app) {
|
|
|
4930
6299
|
const input = createConnectionSchema.parse(request.body);
|
|
4931
6300
|
const created = await app.ctx.connections.create(input);
|
|
4932
6301
|
request.auditTarget({ connectionId: created.id, connectionName: created.name });
|
|
4933
|
-
request.auditDetail({ prefix: created.prefix, cluster: created.cluster, queueFilter: created.queueFilter });
|
|
6302
|
+
request.auditDetail({ kind: created.kind, prefix: created.prefix, cluster: created.cluster, queueFilter: created.queueFilter });
|
|
4934
6303
|
reply.status(201);
|
|
4935
6304
|
return created;
|
|
4936
6305
|
});
|
|
4937
6306
|
app.post("/connections/test", { preHandler: [admin] }, async (request) => {
|
|
4938
6307
|
const input = testConnectionSchema.parse(request.body);
|
|
4939
6308
|
const id3 = `test:${Date.now()}:${Math.random().toString(36).slice(2)}`;
|
|
4940
|
-
const inspector = app.ctx.pool.get({ id: id3, url: input.url, prefix: input.prefix
|
|
6309
|
+
const inspector = app.ctx.pool.get({ id: id3, kind: input.kind, url: input.url, prefix: input.prefix, cluster: input.cluster ?? false });
|
|
4941
6310
|
try {
|
|
4942
6311
|
return await inspector.ping();
|
|
4943
6312
|
} catch (err) {
|
|
@@ -5114,7 +6483,7 @@ async function jobRoutes(app) {
|
|
|
5114
6483
|
const query = searchJobsQuerySchema.parse(request.query);
|
|
5115
6484
|
const inspector = await app.ctx.connections.getInspector(request.params.id);
|
|
5116
6485
|
return withRedis(
|
|
5117
|
-
() => inspector.searchJobs(request.params.queue, query.state, query.q, { cursor: query.cursor ?? null, limit: query.limit })
|
|
6486
|
+
() => inspector.searchJobs(request.params.queue, query.state, query.q, { cursor: query.cursor ?? null, limit: query.limit, groupId: query.groupId })
|
|
5118
6487
|
);
|
|
5119
6488
|
});
|
|
5120
6489
|
app.post(base, { preHandler: [operator] }, async (request, reply) => {
|
|
@@ -6197,6 +7566,7 @@ function toConnectionDto(row, status) {
|
|
|
6197
7566
|
const dto = {
|
|
6198
7567
|
id: row.id,
|
|
6199
7568
|
name: row.name,
|
|
7569
|
+
kind: row.kind,
|
|
6200
7570
|
url: redactRedisUrl(row.url),
|
|
6201
7571
|
prefix: row.prefix,
|
|
6202
7572
|
cluster: row.cluster,
|
|
@@ -6258,6 +7628,7 @@ var ConnectionsService = class {
|
|
|
6258
7628
|
await this.db.insert(connections3).values({
|
|
6259
7629
|
id: id3,
|
|
6260
7630
|
name: input.name,
|
|
7631
|
+
kind: input.kind,
|
|
6261
7632
|
url: input.url,
|
|
6262
7633
|
prefix: input.prefix,
|
|
6263
7634
|
cluster: input.cluster,
|
|
@@ -6295,7 +7666,9 @@ var ConnectionsService = class {
|
|
|
6295
7666
|
return this.list();
|
|
6296
7667
|
}
|
|
6297
7668
|
async update(id3, input) {
|
|
6298
|
-
await this.getRow(id3);
|
|
7669
|
+
const current2 = await this.getRow(id3);
|
|
7670
|
+
const issue = connectionIssues({ ...input, kind: current2.kind })[0];
|
|
7671
|
+
if (issue) throw validation(issue.message, [{ path: [issue.path], message: issue.message }]);
|
|
6299
7672
|
const patch = {};
|
|
6300
7673
|
if (input.name !== void 0) patch.name = input.name;
|
|
6301
7674
|
if (input.url !== void 0) patch.url = input.url;
|
|
@@ -6325,6 +7698,7 @@ var ConnectionsService = class {
|
|
|
6325
7698
|
inspectorFor(row) {
|
|
6326
7699
|
return this.pool.get({
|
|
6327
7700
|
id: row.id,
|
|
7701
|
+
kind: row.kind,
|
|
6328
7702
|
url: row.url,
|
|
6329
7703
|
prefix: row.prefix,
|
|
6330
7704
|
cluster: row.cluster,
|
|
@@ -6881,7 +8255,7 @@ var EditionService = class {
|
|
|
6881
8255
|
if (!this.needsRefresh(boot)) return;
|
|
6882
8256
|
await this.refresh();
|
|
6883
8257
|
} catch (err) {
|
|
6884
|
-
this.log.warn({ err:
|
|
8258
|
+
this.log.warn({ err: errorMessage3(err) }, "license refresh failed");
|
|
6885
8259
|
}
|
|
6886
8260
|
}
|
|
6887
8261
|
async doRefresh() {
|
|
@@ -6947,7 +8321,7 @@ var EditionService = class {
|
|
|
6947
8321
|
await this.client.deactivate({ key, activationId: state.activationId });
|
|
6948
8322
|
this.log.info({ activationId: state.activationId }, "license activation released");
|
|
6949
8323
|
} catch (err) {
|
|
6950
|
-
this.log.warn({ err:
|
|
8324
|
+
this.log.warn({ err: errorMessage3(err) }, "could not release the license activation; free it from the customer portal if needed");
|
|
6951
8325
|
}
|
|
6952
8326
|
}
|
|
6953
8327
|
rebuild() {
|
|
@@ -7002,7 +8376,7 @@ function toCheckError(err) {
|
|
|
7002
8376
|
};
|
|
7003
8377
|
}
|
|
7004
8378
|
if (err instanceof HttpError) return { code: err.error, message: err.message, definitive: false };
|
|
7005
|
-
return { code: "unknown", message:
|
|
8379
|
+
return { code: "unknown", message: errorMessage3(err), definitive: false };
|
|
7006
8380
|
}
|
|
7007
8381
|
function toHttpError(err) {
|
|
7008
8382
|
if (err instanceof HttpError) return err;
|
|
@@ -7020,7 +8394,7 @@ function toHttpError(err) {
|
|
|
7020
8394
|
return licenseServerUnavailable(err.message);
|
|
7021
8395
|
}
|
|
7022
8396
|
}
|
|
7023
|
-
return licenseServerUnavailable(
|
|
8397
|
+
return licenseServerUnavailable(errorMessage3(err));
|
|
7024
8398
|
}
|
|
7025
8399
|
|
|
7026
8400
|
// apps/server/src/services/attention.ts
|
|
@@ -7311,6 +8685,7 @@ function toFolderDto(row, queues) {
|
|
|
7311
8685
|
// apps/server/src/services/health.ts
|
|
7312
8686
|
var MIN_SAMPLE_INTERVAL_MS = 2e3;
|
|
7313
8687
|
var HISTORY_POINTS = 200;
|
|
8688
|
+
var EVENT_TABLE_WARN_BYTES = 1024 * 1024 * 1024;
|
|
7314
8689
|
var HealthService = class {
|
|
7315
8690
|
constructor(connections5) {
|
|
7316
8691
|
this.connections = connections5;
|
|
@@ -7328,10 +8703,11 @@ var HealthService = class {
|
|
|
7328
8703
|
const st = this.stateFor(row.id);
|
|
7329
8704
|
const snap = await this.sample(row, st);
|
|
7330
8705
|
const info = snap.info;
|
|
7331
|
-
const memoryUsedPct = info && info.maxMemoryBytes && info.maxMemoryBytes > 0 ? Math.round(info.usedMemoryBytes / info.maxMemoryBytes * 1e3) / 10 : null;
|
|
8706
|
+
const memoryUsedPct = info && info.backend !== "postgres" && info.maxMemoryBytes && info.maxMemoryBytes > 0 ? Math.round(info.usedMemoryBytes / info.maxMemoryBytes * 1e3) / 10 : null;
|
|
7332
8707
|
return {
|
|
7333
8708
|
connectionId: row.id,
|
|
7334
8709
|
connectionName: row.name,
|
|
8710
|
+
kind: row.kind,
|
|
7335
8711
|
ok: snap.error === null,
|
|
7336
8712
|
error: snap.error,
|
|
7337
8713
|
info,
|
|
@@ -7339,7 +8715,7 @@ var HealthService = class {
|
|
|
7339
8715
|
cpuCores: snap.cpuCores,
|
|
7340
8716
|
memoryUsedPct,
|
|
7341
8717
|
history: [...st.history],
|
|
7342
|
-
warnings: buildWarnings(info, snap.error, memoryUsedPct)
|
|
8718
|
+
warnings: row.kind === "postgres" ? buildPostgresWarnings(info?.backend === "postgres" ? info : null, snap.error) : buildWarnings(info?.backend === "postgres" ? null : info, snap.error, memoryUsedPct)
|
|
7343
8719
|
};
|
|
7344
8720
|
}
|
|
7345
8721
|
/** Drop cached state when a connection is edited or deleted. */
|
|
@@ -7373,8 +8749,9 @@ var HealthService = class {
|
|
|
7373
8749
|
at,
|
|
7374
8750
|
info,
|
|
7375
8751
|
error: null,
|
|
7376
|
-
|
|
7377
|
-
|
|
8752
|
+
// Postgres: transactions/sec from xact_commit + xact_rollback. No CPU counter there.
|
|
8753
|
+
commandsPerSec: rate(throughputCounter(previous?.info), throughputCounter(info), elapsedSec),
|
|
8754
|
+
cpuCores: info.backend === "postgres" ? null : rate(redisInfo(previous?.info)?.cpuSecondsTotal, info.cpuSecondsTotal, elapsedSec)
|
|
7378
8755
|
};
|
|
7379
8756
|
pushHistory(st, snap2);
|
|
7380
8757
|
return snap2;
|
|
@@ -7402,12 +8779,20 @@ function rate(before, after, elapsedSec) {
|
|
|
7402
8779
|
if (delta < 0) return null;
|
|
7403
8780
|
return Math.round(delta / elapsedSec * 100) / 100;
|
|
7404
8781
|
}
|
|
8782
|
+
function redisInfo(info) {
|
|
8783
|
+
return info && info.backend !== "postgres" ? info : null;
|
|
8784
|
+
}
|
|
8785
|
+
function throughputCounter(info) {
|
|
8786
|
+
if (!info) return null;
|
|
8787
|
+
return info.backend === "postgres" ? info.totalTransactions : info.totalCommandsProcessed;
|
|
8788
|
+
}
|
|
7405
8789
|
function pushHistory(st, snap) {
|
|
7406
8790
|
if (!snap.info) return;
|
|
7407
8791
|
st.history.push({
|
|
7408
8792
|
t: snap.at,
|
|
7409
8793
|
latencyMs: snap.info.latencyMs,
|
|
7410
|
-
|
|
8794
|
+
// Postgres has no memory ceiling to plot; the database size is what grows.
|
|
8795
|
+
memoryBytes: snap.info.backend === "postgres" ? snap.info.databaseSizeBytes : snap.info.usedMemoryBytes,
|
|
7411
8796
|
commandsPerSec: snap.commandsPerSec,
|
|
7412
8797
|
cpuCores: snap.cpuCores,
|
|
7413
8798
|
connectedClients: snap.info.connectedClients
|
|
@@ -7463,6 +8848,34 @@ function buildWarnings(info, error, memoryUsedPct) {
|
|
|
7463
8848
|
}
|
|
7464
8849
|
return out;
|
|
7465
8850
|
}
|
|
8851
|
+
function buildPostgresWarnings(info, error) {
|
|
8852
|
+
if (error !== null || info === null) {
|
|
8853
|
+
const detail = error?.trim();
|
|
8854
|
+
return [{ level: "critical", code: "unreachable", message: detail ? `Postgres is unreachable: ${detail}` : "Postgres is unreachable" }];
|
|
8855
|
+
}
|
|
8856
|
+
const out = [];
|
|
8857
|
+
if (info.maxConnections > 0 && info.connectedClients / info.maxConnections >= 0.9) {
|
|
8858
|
+
out.push({
|
|
8859
|
+
level: "critical",
|
|
8860
|
+
code: "connections_high",
|
|
8861
|
+
message: `${info.connectedClients} of ${info.maxConnections} connections in use: new workers will be refused`
|
|
8862
|
+
});
|
|
8863
|
+
} else if (info.maxConnections > 0 && info.connectedClients / info.maxConnections >= 0.75) {
|
|
8864
|
+
out.push({ level: "warn", code: "connections_high", message: `${info.connectedClients} of ${info.maxConnections} connections in use` });
|
|
8865
|
+
}
|
|
8866
|
+
if (info.eventTableBytes >= EVENT_TABLE_WARN_BYTES) {
|
|
8867
|
+
const gib = Math.round(info.eventTableBytes / 1024 ** 3 * 10) / 10;
|
|
8868
|
+
out.push({
|
|
8869
|
+
level: "warn",
|
|
8870
|
+
code: "event_table_large",
|
|
8871
|
+
message: `BullMQ's event table is ${gib} GiB and is never trimmed by BullMQ 6: delete old rows on a schedule`
|
|
8872
|
+
});
|
|
8873
|
+
}
|
|
8874
|
+
if (info.latencyMs >= 250) {
|
|
8875
|
+
out.push({ level: "warn", code: "latency_high", message: `Health query round trip took ${info.latencyMs} ms` });
|
|
8876
|
+
}
|
|
8877
|
+
return out;
|
|
8878
|
+
}
|
|
7466
8879
|
|
|
7467
8880
|
// apps/server/src/ee/services/sso.ts
|
|
7468
8881
|
import { asc as asc2, eq as eq8 } from "drizzle-orm";
|
|
@@ -7799,21 +9212,31 @@ var MCP_TOOLS = [
|
|
|
7799
9212
|
tool({
|
|
7800
9213
|
name: "search_jobs",
|
|
7801
9214
|
title: "Search jobs",
|
|
7802
|
-
description: "Searches the jobs of one state for a text in the job id, name, data or failure reason. Scans in pages: pass next_cursor back as cursor to continue.",
|
|
9215
|
+
description: "Searches the jobs of one state for a text in the job id, name, data or failure reason. With group_id (BullMQ Pro) only that group's jobs are returned and query may be omitted: this is how to list a group's delayed, failed or completed jobs, which list_groups does not count. Scans in pages: pass next_cursor back as cursor to continue.",
|
|
7803
9216
|
access: "read",
|
|
7804
9217
|
inputSchema: object(
|
|
7805
9218
|
{
|
|
7806
9219
|
...QUEUE,
|
|
7807
|
-
query: str4("Text to look for"),
|
|
9220
|
+
query: str4("Text to look for (optional with group_id)"),
|
|
9221
|
+
group_id: str4("BullMQ Pro group id (exact): only jobs of this group"),
|
|
7808
9222
|
state: str4("Job state to search", { enum: [...JOB_STATES], default: "failed" }),
|
|
7809
9223
|
cursor: str4("Cursor from a previous call's nextCursor"),
|
|
7810
9224
|
limit: int3("Max matches", 1, 200)
|
|
7811
9225
|
},
|
|
7812
|
-
["connection_id", "queue"
|
|
9226
|
+
["connection_id", "queue"]
|
|
7813
9227
|
),
|
|
7814
9228
|
annotations: ro,
|
|
7815
|
-
schema: zQueue.extend({
|
|
7816
|
-
|
|
9229
|
+
schema: zQueue.extend({
|
|
9230
|
+
query: z10.string().max(500).optional(),
|
|
9231
|
+
group_id: z10.string().min(1).max(200).optional(),
|
|
9232
|
+
state: zState.optional(),
|
|
9233
|
+
cursor: z10.string().optional(),
|
|
9234
|
+
limit: z10.number().int().min(1).max(200).optional()
|
|
9235
|
+
}).refine((a) => !!a.query || !!a.group_id, { message: "query or group_id is required", path: ["query"] }),
|
|
9236
|
+
run: (a, ctx) => forward(ctx, {
|
|
9237
|
+
method: "GET",
|
|
9238
|
+
url: `${queuePath(a)}/jobs/search${qs({ q: a.query || void 0, groupId: a.group_id, state: a.state, cursor: a.cursor, limit: a.limit })}`
|
|
9239
|
+
})
|
|
7817
9240
|
}),
|
|
7818
9241
|
tool({
|
|
7819
9242
|
name: "get_job",
|
|
@@ -7848,7 +9271,7 @@ var MCP_TOOLS = [
|
|
|
7848
9271
|
tool({
|
|
7849
9272
|
name: "list_groups",
|
|
7850
9273
|
title: "List groups (BullMQ Pro)",
|
|
7851
|
-
description: "BullMQ Pro groups of a queue: status (rate limited, paused, maxed), waiting and active per group. Empty on plain BullMQ.",
|
|
9274
|
+
description: "BullMQ Pro groups of a queue: status (rate limited, paused, maxed), waiting and active per group. Empty on plain BullMQ. Delayed jobs are not under their group in Pro, so a group with only delayed jobs is not listed: use search_jobs with group_id and state delayed.",
|
|
7852
9275
|
access: "read",
|
|
7853
9276
|
inputSchema: object({ ...QUEUE, page: int3("Page, from 1", 1, 1e4), page_size: int3("Per page", 1, 200) }, ["connection_id", "queue"]),
|
|
7854
9277
|
annotations: ro,
|
|
@@ -8493,13 +9916,13 @@ async function waitForDatabase(database2, log, opts = {}) {
|
|
|
8493
9916
|
lastError = err;
|
|
8494
9917
|
if (database2.dialect === "sqlite") break;
|
|
8495
9918
|
const elapsed = Math.round((Date.now() - startedAt) / 1e3);
|
|
8496
|
-
log.info(`${name} not reachable yet (${elapsed}s): ${
|
|
9919
|
+
log.info(`${name} not reachable yet (${elapsed}s): ${errorMessage4(err)}. Retrying in ${intervalMs / 1e3}s...`);
|
|
8497
9920
|
await new Promise((r) => setTimeout(r, intervalMs));
|
|
8498
9921
|
}
|
|
8499
9922
|
}
|
|
8500
|
-
throw new Error(`${name} did not become reachable within ${timeoutMs / 1e3}s: ${
|
|
9923
|
+
throw new Error(`${name} did not become reachable within ${timeoutMs / 1e3}s: ${errorMessage4(lastError)}`);
|
|
8501
9924
|
}
|
|
8502
|
-
function
|
|
9925
|
+
function errorMessage4(err) {
|
|
8503
9926
|
return err instanceof Error ? err.message : String(err);
|
|
8504
9927
|
}
|
|
8505
9928
|
|
|
@@ -8572,6 +9995,7 @@ async function seedDemo(ctx, log) {
|
|
|
8572
9995
|
if (!connection && await ctx.connections.count() === 0) {
|
|
8573
9996
|
const created = await ctx.connections.create({
|
|
8574
9997
|
name: DEMO_CONNECTION_NAME,
|
|
9998
|
+
kind: "redis",
|
|
8575
9999
|
url: config.demoRedisUrl,
|
|
8576
10000
|
prefix: "bull",
|
|
8577
10001
|
cluster: false,
|
|
@@ -8637,7 +10061,7 @@ async function seedConnections(connections5, wanted, log) {
|
|
|
8637
10061
|
async function main() {
|
|
8638
10062
|
const config = loadConfig(process.env, { warn: (m) => console.warn(`[config] ${m}`) });
|
|
8639
10063
|
const database2 = createDatabase(config.database);
|
|
8640
|
-
const pool =
|
|
10064
|
+
const pool = createInspectorPool2({
|
|
8641
10065
|
discoveryTtlMs: config.queueDiscoveryTtl * 1e3,
|
|
8642
10066
|
previewBytes: config.jobPreviewBytes
|
|
8643
10067
|
});
|