bullpane 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +35 -0
- package/LICENSE-ee +46 -0
- package/README.md +60 -0
- package/bin/bullpane.mjs +99 -0
- package/dist/lua/getGroups.lua +65 -0
- package/dist/lua/getJob.lua +61 -0
- package/dist/lua/getJobs.lua +156 -0
- package/dist/lua/getSchedulers.lua +76 -0
- package/dist/lua/getTreeNode.lua +105 -0
- package/dist/lua/queueSetup.lua +47 -0
- package/dist/lua/queueStats.lua +140 -0
- package/dist/lua/sampleParents.lua +65 -0
- package/dist/lua/searchJobs.lua +146 -0
- package/dist/lua/windowMetrics.lua +167 -0
- package/dist/server.mjs +8703 -0
- package/migrations/mysql/0001_init.sql +102 -0
- package/migrations/mysql/0002_alert_scopes.sql +9 -0
- package/migrations/mysql/0003_hidden_queues.sql +21 -0
- package/migrations/mysql/0004_audit_log.sql +57 -0
- package/migrations/mysql/0005_sso.sql +48 -0
- package/migrations/mysql/0006_connection_position.sql +29 -0
- package/migrations/mysql/0007_user_disabled.sql +12 -0
- package/migrations/mysql/0008_alert_wide_scopes.sql +18 -0
- package/migrations/mysql/0009_mcp.sql +62 -0
- package/migrations/sqlite/0001_init.sql +158 -0
- package/migrations/sqlite/0002_mcp.sql +38 -0
- package/package.json +56 -0
- package/web/assets/FlowsPage-CxMz_9Et.js +6 -0
- package/web/assets/JobTreePage-BT79W1zX.js +6 -0
- package/web/assets/McpConsentPage-B2YnL6fN.js +1 -0
- package/web/assets/flowLayout-BnuhLJ6X.css +1 -0
- package/web/assets/flowLayout-DoC2dCG0.js +23 -0
- package/web/assets/index-C2gL_lbF.css +1 -0
- package/web/assets/index-DuAopWKm.js +543 -0
- package/web/index.html +18 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
Copyright (c) 2026 Matheus Morett
|
|
2
|
+
|
|
3
|
+
Bullpane is open core. Its source is licensed in two parts:
|
|
4
|
+
|
|
5
|
+
* Every file inside a directory named `ee` (today `apps/server/src/ee/` and
|
|
6
|
+
`apps/web/src/ee/`) is licensed under the Bullpane Commercial License, in the
|
|
7
|
+
`LICENSE` file inside that directory. That is the code of the Pro features.
|
|
8
|
+
* Everything else in this repository is licensed under the MIT license below.
|
|
9
|
+
|
|
10
|
+
Releases up to and including 0.3.0 were published entirely under the MIT
|
|
11
|
+
license, and those copies keep it.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
MIT License
|
|
16
|
+
|
|
17
|
+
Copyright (c) 2026 Matheus Morett
|
|
18
|
+
|
|
19
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
20
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
21
|
+
in the Software without restriction, including without limitation the rights
|
|
22
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
23
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
24
|
+
furnished to do so, subject to the following conditions:
|
|
25
|
+
|
|
26
|
+
The above copyright notice and this permission notice shall be included in all
|
|
27
|
+
copies or substantial portions of the Software.
|
|
28
|
+
|
|
29
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
30
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
31
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
32
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
33
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
34
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
35
|
+
SOFTWARE.
|
package/LICENSE-ee
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
Bullpane Commercial License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Matheus Morett
|
|
4
|
+
|
|
5
|
+
This license covers every file inside a directory named `ee` in the Bullpane
|
|
6
|
+
source code (the "Pro Software"). Everything outside those directories is
|
|
7
|
+
licensed under the MIT license in the root `LICENSE` file.
|
|
8
|
+
|
|
9
|
+
1. Production use requires a subscription. You may use the Pro Software in
|
|
10
|
+
production only while you hold a valid Bullpane Pro subscription or license
|
|
11
|
+
key, and only on the number of installations it covers.
|
|
12
|
+
|
|
13
|
+
2. What you may do without a subscription. You may read, copy and modify the
|
|
14
|
+
Pro Software, and run it for development and testing. You may also run it,
|
|
15
|
+
unmodified, as part of the free edition of Bullpane, where the Pro features
|
|
16
|
+
stay locked until a valid license key is activated.
|
|
17
|
+
|
|
18
|
+
3. What you may not do:
|
|
19
|
+
(a) remove, disable, bypass or alter the license key verification, the
|
|
20
|
+
edition and feature checks, or any other mechanism that limits the Pro
|
|
21
|
+
features to licensed installations, or use a build in which any of them
|
|
22
|
+
was removed, disabled, bypassed or altered;
|
|
23
|
+
(b) sell, rent, sublicense, or offer as a hosted service the Pro Software or
|
|
24
|
+
any work derived from it;
|
|
25
|
+
(c) distribute the Pro Software or any work derived from it, except as part
|
|
26
|
+
of a copy of the Bullpane source code that keeps every mechanism listed
|
|
27
|
+
in (a) intact and keeps this file;
|
|
28
|
+
(d) remove or alter this notice or any copyright notice.
|
|
29
|
+
|
|
30
|
+
4. Modifications. You may make modifications only for the uses allowed above.
|
|
31
|
+
The licensor keeps all rights to the Pro Software and to any modifications
|
|
32
|
+
of it you choose to submit back, which are licensed to the licensor as
|
|
33
|
+
described in `CONTRIBUTING.md`.
|
|
34
|
+
|
|
35
|
+
5. Third-party components keep their own licenses.
|
|
36
|
+
|
|
37
|
+
6. Termination. If you break any of these terms, your rights under this license
|
|
38
|
+
end immediately.
|
|
39
|
+
|
|
40
|
+
THE PRO SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
41
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
42
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
43
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
44
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
45
|
+
CONNECTION WITH THE PRO SOFTWARE OR THE USE OR OTHER DEALINGS IN THE PRO
|
|
46
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Bullpane
|
|
2
|
+
|
|
3
|
+
**A fast, self-hosted dashboard for [BullMQ](https://bullmq.io) and BullMQ Pro.**
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
npx bullpane --redis redis://localhost:6379
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Open <http://localhost:3000>. That is the whole install: no database to set up, no account, no login.
|
|
10
|
+
|
|
11
|
+
<img src="https://bullpane.com/shots/overview.jpg" alt="Bullpane overview: every queue of a connection with counts, rates and the ones that need attention" width="880">
|
|
12
|
+
|
|
13
|
+
[bullpane.com](https://bullpane.com) · [Live demo](https://demo.bullpane.com) · [GitHub](https://github.com/madmorett/bullpane) · [Changelog](https://bullpane.com/changelog)
|
|
14
|
+
|
|
15
|
+
## What you get
|
|
16
|
+
|
|
17
|
+
- Every queue on a Redis, with counts, success rate and what needs attention.
|
|
18
|
+
- Jobs by state, search inside job data, job detail with data, return value, stack trace and logs.
|
|
19
|
+
- Retry, promote, remove and bulk actions through the official `bullmq` API. Pause, resume, clean, drain.
|
|
20
|
+
- Job schedulers, flows (parent/child trees), stalled jobs, Redis health.
|
|
21
|
+
- BullMQ Pro groups: per-group concurrency, rate limits and paused groups.
|
|
22
|
+
- Safe on a busy production Redis: no `KEYS`, one round trip per read, payloads truncated inside Redis.
|
|
23
|
+
|
|
24
|
+
Works with BullMQ 4, 5 and 6 on Redis, Redis Cluster and Valkey, and with BullMQ Pro.
|
|
25
|
+
|
|
26
|
+
**Pro** (USD 39/month, one installation, unlimited users) adds login with roles, SSO, alerts to Slack or webhooks, folders, a flow graph, an audit log and an MCP server for Claude and other AI clients.
|
|
27
|
+
|
|
28
|
+
## Options
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
npx bullpane [--redis <url>] [options]
|
|
32
|
+
|
|
33
|
+
--redis <url> Redis your BullMQ workers use, added as a connection
|
|
34
|
+
--prefix <prefix> BullMQ prefix of that connection (default: bull)
|
|
35
|
+
--port <port> HTTP port (default: 3000)
|
|
36
|
+
--host <host> Interface to listen on (default: 127.0.0.1)
|
|
37
|
+
--data-dir <dir> Where the SQLite database lives (default: ~/.bullpane)
|
|
38
|
+
--database-url <url> mysql://user:pass@host:3306/db to use MySQL instead
|
|
39
|
+
--read-only Refuse every write with 423
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The free edition has no login, so it listens on `127.0.0.1` by default. Use `--host 0.0.0.0` only on a private network, or unlock Pro for login and roles.
|
|
43
|
+
|
|
44
|
+
Every environment variable of the Docker image works here too.
|
|
45
|
+
|
|
46
|
+
## Running it for a team
|
|
47
|
+
|
|
48
|
+
Use the Docker image, next to your stack:
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
docker run -d -p 3000:3000 -v bullpane-data:/data bullpane/bullpane
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Versus bull-board
|
|
55
|
+
|
|
56
|
+
bull-board is a middleware you mount in your app: a viewer, with no search inside job data, no roles and no alerts. Bullpane runs on its own, so it never ships with your app and can be put in front of production. A full comparison is at [bullpane.com/vs/bull-board](https://bullpane.com/vs/bull-board).
|
|
57
|
+
|
|
58
|
+
## License
|
|
59
|
+
|
|
60
|
+
Open core. The bundle contains the free core (MIT) and the Pro features, which are licensed under the Bullpane Commercial License (`LICENSE-ee`): readable, free for development and testing, a subscription for production. See `LICENSE`.
|
package/bin/bullpane.mjs
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* `npx bullpane` — the dashboard without Docker. Same server as the image,
|
|
4
|
+
* bundled; SQLite in ~/.bullpane unless DATABASE_URL says otherwise.
|
|
5
|
+
*
|
|
6
|
+
* Flags win over environment variables, which win over the defaults below.
|
|
7
|
+
* Every variable the image honours (docs: apps/server/README.md) works here too.
|
|
8
|
+
*/
|
|
9
|
+
import { homedir } from "node:os";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { readFileSync } from "node:fs";
|
|
13
|
+
import { parseArgs } from "node:util";
|
|
14
|
+
|
|
15
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
16
|
+
const { version } = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
|
|
17
|
+
|
|
18
|
+
const HELP = `Bullpane ${version} — self-hosted dashboard for BullMQ and BullMQ Pro
|
|
19
|
+
|
|
20
|
+
Usage
|
|
21
|
+
npx bullpane [--redis <url>] [options]
|
|
22
|
+
|
|
23
|
+
Options
|
|
24
|
+
--redis <url> Redis your BullMQ workers use, added as a connection
|
|
25
|
+
(e.g. redis://localhost:6379). More can be added in the UI.
|
|
26
|
+
--prefix <prefix> BullMQ prefix of that connection (default: bull)
|
|
27
|
+
--port <port> HTTP port (default: 3000, or $PORT)
|
|
28
|
+
--host <host> Interface to listen on (default: 127.0.0.1). The free
|
|
29
|
+
edition has no login: only use 0.0.0.0 on a private network.
|
|
30
|
+
--data-dir <dir> Where the SQLite database lives (default: ~/.bullpane)
|
|
31
|
+
--database-url <url> mysql://user:pass@host:3306/db to use MySQL instead
|
|
32
|
+
--read-only Refuse every write (retry, remove, pause...) with 423
|
|
33
|
+
-v, --version Print the version
|
|
34
|
+
-h, --help Print this help
|
|
35
|
+
|
|
36
|
+
Docs: https://bullpane.com · Docker: ghcr.io/madmorett/bullpane`;
|
|
37
|
+
|
|
38
|
+
let args;
|
|
39
|
+
try {
|
|
40
|
+
args = parseArgs({
|
|
41
|
+
options: {
|
|
42
|
+
redis: { type: "string" },
|
|
43
|
+
prefix: { type: "string" },
|
|
44
|
+
port: { type: "string" },
|
|
45
|
+
host: { type: "string" },
|
|
46
|
+
"data-dir": { type: "string" },
|
|
47
|
+
"database-url": { type: "string" },
|
|
48
|
+
"read-only": { type: "boolean" },
|
|
49
|
+
version: { type: "boolean", short: "v" },
|
|
50
|
+
help: { type: "boolean", short: "h" },
|
|
51
|
+
},
|
|
52
|
+
strict: true,
|
|
53
|
+
}).values;
|
|
54
|
+
} catch (err) {
|
|
55
|
+
console.error(`${err.message}\n\n${HELP}`);
|
|
56
|
+
process.exit(2);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (args.help) {
|
|
60
|
+
console.log(HELP);
|
|
61
|
+
process.exit(0);
|
|
62
|
+
}
|
|
63
|
+
if (args.version) {
|
|
64
|
+
console.log(version);
|
|
65
|
+
process.exit(0);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const env = process.env;
|
|
69
|
+
const set = (key, flag, fallback) => {
|
|
70
|
+
if (flag !== undefined) env[key] = String(flag);
|
|
71
|
+
else if (env[key] === undefined || env[key] === "") {
|
|
72
|
+
if (fallback !== undefined) env[key] = String(fallback);
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
set("PORT", args.port, 3000);
|
|
77
|
+
set("HOST", args.host, "127.0.0.1");
|
|
78
|
+
set("BULLPANE_DATA_DIR", args["data-dir"], path.join(homedir(), ".bullpane"));
|
|
79
|
+
set("DATABASE_URL", args["database-url"]);
|
|
80
|
+
set("BULLPANE_READ_ONLY", args["read-only"] ? "true" : undefined);
|
|
81
|
+
set("WEB_DIST", undefined, path.join(root, "web"));
|
|
82
|
+
set("PUBLIC_URL", undefined, `http://localhost:${env.PORT}`);
|
|
83
|
+
set("NODE_ENV", undefined, "production");
|
|
84
|
+
|
|
85
|
+
if (args.redis) {
|
|
86
|
+
let name = "Redis";
|
|
87
|
+
try {
|
|
88
|
+
const u = new URL(args.redis);
|
|
89
|
+
name = `${u.hostname}${u.port ? `:${u.port}` : ""}`; // never the password
|
|
90
|
+
} catch {
|
|
91
|
+
console.error(`--redis must be a URL like redis://localhost:6379`);
|
|
92
|
+
process.exit(2);
|
|
93
|
+
}
|
|
94
|
+
// Created at boot only when no connection of that name exists yet, so a
|
|
95
|
+
// second run (or edits made in the UI) never duplicates or overwrites it.
|
|
96
|
+
env.BULLPANE_CONNECTIONS = JSON.stringify([{ name, url: args.redis, prefix: args.prefix ?? "bull" }]);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
await import("../dist/server.mjs");
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
BullMQ Pro groups, read only. Layout: keys.ts (verified against bullmq-pro 7.48.0).
|
|
3
|
+
|
|
4
|
+
KEYS[1] groups zset, status "waiting" (score = round-robin order)
|
|
5
|
+
KEYS[2] groups:limit zset, status "limited" (score = unix ms the limit lifts)
|
|
6
|
+
KEYS[3] groups:max zset, status "maxed" (score = unix ms it hit the cap)
|
|
7
|
+
KEYS[4] groups:paused zset, status "paused" (score = unix ms it was paused)
|
|
8
|
+
KEYS[5] groups:active:count hash gid -> jobs being processed
|
|
9
|
+
KEYS[6] groups:concurrency hash gid -> legacy per-group concurrency
|
|
10
|
+
|
|
11
|
+
ARGV[1] start (0-based, inclusive)
|
|
12
|
+
ARGV[2] end (0-based, inclusive; -1 = to the end)
|
|
13
|
+
ARGV[3] queue key prefix `${prefix}:${queue}:`
|
|
14
|
+
|
|
15
|
+
A group sits in exactly one of the four status zsets, so "all groups" is their
|
|
16
|
+
concatenation in that order — the same order QueuePro.getGroups pages through.
|
|
17
|
+
The per-group keys (`groups:${gid}`, `:p`, `:meta`) mirror GROUP_KEY in keys.ts
|
|
18
|
+
and hang off ARGV[3]: same queue, same hash tag, cluster safe. Everything here
|
|
19
|
+
is O(1) per group on the page (LLEN / ZCARD / HGET / HMGET).
|
|
20
|
+
|
|
21
|
+
Returns { total, { waiting, limited, maxed, paused }, rows } with
|
|
22
|
+
rows = { { id, status, waiting, prioritized, active, conc|false, lm|false, ld|false, score }, ... }
|
|
23
|
+
where waiting already includes prioritized.
|
|
24
|
+
]]
|
|
25
|
+
local rcall = redis.call
|
|
26
|
+
local qprefix = ARGV[3]
|
|
27
|
+
local rangeStart = tonumber(ARGV[1])
|
|
28
|
+
local rangeEnd = tonumber(ARGV[2])
|
|
29
|
+
|
|
30
|
+
local statuses = { "waiting", "limited", "maxed", "paused" }
|
|
31
|
+
local counts = {}
|
|
32
|
+
local total = 0
|
|
33
|
+
for i = 1, 4 do
|
|
34
|
+
counts[i] = rcall("ZCARD", KEYS[i])
|
|
35
|
+
total = total + counts[i]
|
|
36
|
+
end
|
|
37
|
+
if rangeEnd < 0 or rangeEnd >= total then rangeEnd = total - 1 end
|
|
38
|
+
|
|
39
|
+
-- Page across the four zsets as if they were one list: zset i owns the virtual
|
|
40
|
+
-- indices [offset, offset + counts[i] - 1].
|
|
41
|
+
local rows = {}
|
|
42
|
+
local offset = 0
|
|
43
|
+
for i = 1, 4 do
|
|
44
|
+
local n = counts[i]
|
|
45
|
+
local last = offset + n - 1
|
|
46
|
+
if n > 0 and rangeStart <= last and rangeEnd >= offset then
|
|
47
|
+
local from = math.max(rangeStart, offset) - offset
|
|
48
|
+
local to = math.min(rangeEnd, last) - offset
|
|
49
|
+
local raw = rcall("ZRANGE", KEYS[i], from, to, "WITHSCORES")
|
|
50
|
+
for j = 1, #raw, 2 do
|
|
51
|
+
local gid = raw[j]
|
|
52
|
+
local groupKey = qprefix .. "groups:" .. gid
|
|
53
|
+
local waiting = rcall("LLEN", groupKey)
|
|
54
|
+
local prioritized = rcall("ZCARD", groupKey .. ":p")
|
|
55
|
+
local active = tonumber(rcall("HGET", KEYS[5], gid)) or 0
|
|
56
|
+
local meta = rcall("HMGET", groupKey .. ":meta", "conc", "lm", "ld")
|
|
57
|
+
-- Pro reads the legacy hash first, then the meta hash (increaseGroupConcurrency.lua).
|
|
58
|
+
local conc = rcall("HGET", KEYS[6], gid) or meta[1] or false
|
|
59
|
+
rows[#rows + 1] = { gid, statuses[i], waiting + prioritized, prioritized, active, conc, meta[2] or false, meta[3] or false, raw[j + 1] }
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
offset = offset + n
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
return { total, counts, rows }
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Full detail of one job in one round trip.
|
|
3
|
+
|
|
4
|
+
KEYS[1] job hash
|
|
5
|
+
KEYS[2] `${id}:logs` list
|
|
6
|
+
KEYS[3] `${id}:dependencies` set (unprocessed children)
|
|
7
|
+
KEYS[4] `${id}:processed` hash (processed children)
|
|
8
|
+
KEYS[5..12] state keys in STATE_ORDER:
|
|
9
|
+
wait, active, completed, failed, delayed, prioritized, paused, waiting-children
|
|
10
|
+
|
|
11
|
+
ARGV[1] job id
|
|
12
|
+
ARGV[2] how many log lines (from the tail) to return
|
|
13
|
+
|
|
14
|
+
Returns nil when the hash does not exist, otherwise
|
|
15
|
+
{ hgetallFlat, logs, logsCount, dependenciesCount, processedCount, state, delayedScore }
|
|
16
|
+
`delayedScore` is the job's score in the `delayed` zset (false otherwise):
|
|
17
|
+
timestamp * 0x1000 + jobId % 0x1000, i.e. when the job becomes runnable.
|
|
18
|
+
|
|
19
|
+
State detection: ZSCORE for zsets (O(1)), LPOS for lists (Redis >= 6.0.6,
|
|
20
|
+
O(N) but in C and without shipping the list). LPOS is wrapped in pcall so an
|
|
21
|
+
older Redis degrades to "unknown" instead of erroring.
|
|
22
|
+
]]
|
|
23
|
+
local rcall = redis.call
|
|
24
|
+
local jobKey = KEYS[1]
|
|
25
|
+
|
|
26
|
+
if rcall("EXISTS", jobKey) == 0 then
|
|
27
|
+
return nil
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
local hash = rcall("HGETALL", jobKey)
|
|
31
|
+
|
|
32
|
+
local tail = tonumber(ARGV[2]) or 100
|
|
33
|
+
local logs = rcall("LRANGE", KEYS[2], -tail, -1)
|
|
34
|
+
local logsCount = rcall("LLEN", KEYS[2])
|
|
35
|
+
local depsCount = rcall("SCARD", KEYS[3])
|
|
36
|
+
local processedCount = rcall("HLEN", KEYS[4])
|
|
37
|
+
|
|
38
|
+
local id = ARGV[1]
|
|
39
|
+
local state = "unknown"
|
|
40
|
+
local delayedScore = false
|
|
41
|
+
|
|
42
|
+
-- zsets first: cheap and the most common resting states
|
|
43
|
+
local inDelayed = rcall("ZSCORE", KEYS[9], id)
|
|
44
|
+
if rcall("ZSCORE", KEYS[7], id) then state = "completed"
|
|
45
|
+
elseif rcall("ZSCORE", KEYS[8], id) then state = "failed"
|
|
46
|
+
elseif inDelayed then state = "delayed" delayedScore = inDelayed
|
|
47
|
+
elseif rcall("ZSCORE", KEYS[10], id) then state = "prioritized"
|
|
48
|
+
elseif rcall("ZSCORE", KEYS[12], id) then state = "waiting-children"
|
|
49
|
+
else
|
|
50
|
+
local function inList(key)
|
|
51
|
+
local ok, pos = pcall(rcall, "LPOS", key, id)
|
|
52
|
+
if ok and pos then return true end
|
|
53
|
+
return false
|
|
54
|
+
end
|
|
55
|
+
if inList(KEYS[6]) then state = "active"
|
|
56
|
+
elseif inList(KEYS[5]) then state = "waiting"
|
|
57
|
+
elseif inList(KEYS[11]) then state = "paused"
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
return { hash, logs, logsCount, depsCount, processedCount, state, delayedScore }
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
One page of job summaries for a state (or a Pro group list) in one round trip.
|
|
3
|
+
|
|
4
|
+
KEYS[1] the state key (list or zset)
|
|
5
|
+
KEYS[2] optional, list kind only: a zset whose members come AFTER the list —
|
|
6
|
+
a Pro group's prioritized jobs (`groups:${gid}:p`), which Pro serves
|
|
7
|
+
once the group's list is empty (getGroup.lua in bullmq-pro).
|
|
8
|
+
|
|
9
|
+
ARGV[1] "list" | "zset"
|
|
10
|
+
ARGV[2] start (0-based, inclusive)
|
|
11
|
+
ARGV[3] end (0-based, inclusive)
|
|
12
|
+
ARGV[4] "asc" | "desc"
|
|
13
|
+
ARGV[5] queue key prefix `${prefix}:${queue}:` (job hashes live at prefix .. id)
|
|
14
|
+
ARGV[6] previewBytes: `data` / `returnvalue` are cut to this many bytes IN REDIS
|
|
15
|
+
ARGV[7] maxFieldBytes: a `data` / `returnvalue` field LONGER than this is not
|
|
16
|
+
read at all (HSTRLEN first, O(1)). HMGET copies the whole field into
|
|
17
|
+
Lua before we can truncate it, so a page of 200 × 1 MB payloads would
|
|
18
|
+
block Redis for ~300 ms; with the cap a page costs at most
|
|
19
|
+
pageSize × maxFieldBytes. The row still carries the size so the UI
|
|
20
|
+
can say "payload 1.0 MB — open the job".
|
|
21
|
+
ARGV[8..] hash fields to HMGET, in the order the caller expects them back
|
|
22
|
+
|
|
23
|
+
Returns { total, jobs } where each job is
|
|
24
|
+
{ id, field1, field2, ..., dataTruncated (0|1), dataBytes, score }
|
|
25
|
+
`score` is the member's zset score (false for lists). For `delayed` it encodes
|
|
26
|
+
when the job becomes runnable — timestamp * 0x1000 + jobId % 0x1000 — which the
|
|
27
|
+
hash alone cannot tell after a backoff retry.
|
|
28
|
+
Hashes that vanished between the range read and the HMGET are skipped.
|
|
29
|
+
|
|
30
|
+
Ordering mirrors bullmq's getRanges (queue-getters.js):
|
|
31
|
+
lists are LPUSHed, so head = newest. desc = LRANGE start end; asc reads from the tail.
|
|
32
|
+
zsets: desc = ZREVRANGE (highest score = newest), asc = ZRANGE.
|
|
33
|
+
]]
|
|
34
|
+
local rcall = redis.call
|
|
35
|
+
local key = KEYS[1]
|
|
36
|
+
local extra = KEYS[2]
|
|
37
|
+
local kind = ARGV[1]
|
|
38
|
+
local rangeStart = tonumber(ARGV[2])
|
|
39
|
+
local rangeEnd = tonumber(ARGV[3])
|
|
40
|
+
local desc = ARGV[4] ~= "asc"
|
|
41
|
+
local qprefix = ARGV[5]
|
|
42
|
+
local previewBytes = tonumber(ARGV[6])
|
|
43
|
+
local maxFieldBytes = tonumber(ARGV[7])
|
|
44
|
+
|
|
45
|
+
local fields = {}
|
|
46
|
+
for i = 8, #ARGV do
|
|
47
|
+
fields[#fields + 1] = ARGV[i]
|
|
48
|
+
end
|
|
49
|
+
-- A field name no job hash has: swapped in for `data` / `returnvalue` when the
|
|
50
|
+
-- real field is over the cap, so the HMGET keeps its shape and returns nil there.
|
|
51
|
+
local SKIP = "\0skip"
|
|
52
|
+
|
|
53
|
+
-- Positions of the two payload fields we truncate.
|
|
54
|
+
local dataIdx, retIdx = nil, nil
|
|
55
|
+
for i, f in ipairs(fields) do
|
|
56
|
+
if f == "data" then dataIdx = i end
|
|
57
|
+
if f == "returnvalue" then retIdx = i end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
local function isMarker(id)
|
|
61
|
+
return string.sub(id, 1, 2) == "0:"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
local total
|
|
65
|
+
local ids
|
|
66
|
+
-- id -> raw score string, filled for zsets only
|
|
67
|
+
local scores = {}
|
|
68
|
+
if kind == "list" then
|
|
69
|
+
total = rcall("LLEN", key)
|
|
70
|
+
-- legacy marker at the tail is not a job
|
|
71
|
+
if total > 0 then
|
|
72
|
+
local last = rcall("LINDEX", key, -1)
|
|
73
|
+
if last and isMarker(last) then total = total - 1 end
|
|
74
|
+
end
|
|
75
|
+
if desc then
|
|
76
|
+
ids = rcall("LRANGE", key, rangeStart, rangeEnd)
|
|
77
|
+
else
|
|
78
|
+
-- asc = oldest first = from the tail. Reverse afterwards.
|
|
79
|
+
-- Explicit branches: -( -1 + 1 ) is -0 in Lua, which Redis rejects as a non-integer.
|
|
80
|
+
local fromTail = 0
|
|
81
|
+
if rangeEnd ~= -1 then fromTail = -(rangeEnd + 1) end
|
|
82
|
+
local toTail = -1
|
|
83
|
+
if rangeStart ~= -1 then toTail = -(rangeStart + 1) end
|
|
84
|
+
local raw = rcall("LRANGE", key, fromTail, toTail)
|
|
85
|
+
ids = {}
|
|
86
|
+
for i = #raw, 1, -1 do ids[#ids + 1] = raw[i] end
|
|
87
|
+
end
|
|
88
|
+
-- The page runs past the list: continue into the appended zset (ascending score
|
|
89
|
+
-- = highest priority first, the order Pro will serve them in).
|
|
90
|
+
if extra then
|
|
91
|
+
local listTotal = total
|
|
92
|
+
local extraTotal = rcall("ZCARD", extra)
|
|
93
|
+
total = listTotal + extraTotal
|
|
94
|
+
if extraTotal > 0 and (rangeEnd == -1 or rangeEnd >= listTotal) then
|
|
95
|
+
local from = math.max(0, rangeStart - listTotal)
|
|
96
|
+
local to = -1
|
|
97
|
+
if rangeEnd ~= -1 then to = rangeEnd - listTotal end
|
|
98
|
+
local more = rcall("ZRANGE", extra, from, to)
|
|
99
|
+
for _, id in ipairs(more) do ids[#ids + 1] = id end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
else
|
|
103
|
+
total = rcall("ZCARD", key)
|
|
104
|
+
local flat
|
|
105
|
+
if desc then
|
|
106
|
+
flat = rcall("ZREVRANGE", key, rangeStart, rangeEnd, "WITHSCORES")
|
|
107
|
+
else
|
|
108
|
+
flat = rcall("ZRANGE", key, rangeStart, rangeEnd, "WITHSCORES")
|
|
109
|
+
end
|
|
110
|
+
ids = {}
|
|
111
|
+
for i = 1, #flat, 2 do
|
|
112
|
+
ids[#ids + 1] = flat[i]
|
|
113
|
+
scores[flat[i]] = flat[i + 1]
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
local jobs = {}
|
|
118
|
+
for _, id in ipairs(ids) do
|
|
119
|
+
if not isMarker(id) then
|
|
120
|
+
local hkey = qprefix .. id
|
|
121
|
+
-- Size first (O(1)); only then decide whether the payload is worth copying.
|
|
122
|
+
local dataBytes = dataIdx and rcall("HSTRLEN", hkey, "data") or 0
|
|
123
|
+
local retBytes = retIdx and rcall("HSTRLEN", hkey, "returnvalue") or 0
|
|
124
|
+
local want = fields
|
|
125
|
+
if (dataIdx and dataBytes > maxFieldBytes) or (retIdx and retBytes > maxFieldBytes) then
|
|
126
|
+
want = {}
|
|
127
|
+
for i = 1, #fields do want[i] = fields[i] end
|
|
128
|
+
if dataIdx and dataBytes > maxFieldBytes then want[dataIdx] = SKIP end
|
|
129
|
+
if retIdx and retBytes > maxFieldBytes then want[retIdx] = SKIP end
|
|
130
|
+
end
|
|
131
|
+
local vals = rcall("HMGET", hkey, unpack(want))
|
|
132
|
+
local alive = false
|
|
133
|
+
for i = 1, #vals do
|
|
134
|
+
if vals[i] then alive = true break end
|
|
135
|
+
end
|
|
136
|
+
if alive then
|
|
137
|
+
local truncated = 0
|
|
138
|
+
-- Truncate INSIDE Redis: a 500 KB payload costs the same to render as a tiny one.
|
|
139
|
+
if dataIdx and dataBytes > previewBytes then
|
|
140
|
+
if vals[dataIdx] then vals[dataIdx] = string.sub(vals[dataIdx], 1, previewBytes) end
|
|
141
|
+
truncated = 1
|
|
142
|
+
end
|
|
143
|
+
if retIdx and vals[retIdx] and #vals[retIdx] > previewBytes then
|
|
144
|
+
vals[retIdx] = string.sub(vals[retIdx], 1, previewBytes)
|
|
145
|
+
end
|
|
146
|
+
local row = { id }
|
|
147
|
+
for i = 1, #vals do row[#row + 1] = vals[i] end
|
|
148
|
+
row[#row + 1] = truncated
|
|
149
|
+
row[#row + 1] = dataBytes
|
|
150
|
+
row[#row + 1] = scores[id] or false
|
|
151
|
+
jobs[#jobs + 1] = row
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
return { total, jobs }
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Job schedulers ("repeatable jobs") of ONE queue, read only, one round trip.
|
|
3
|
+
|
|
4
|
+
Job schedulers do not live in any of the 8 job states. BullMQ keeps them in:
|
|
5
|
+
`repeat` zset schedulerId -> next run (unix ms)
|
|
6
|
+
`repeat:${id}` hash name, pattern | every, tz, offset, limit, ic,
|
|
7
|
+
startDate, endDate, data, opts
|
|
8
|
+
(verified against bullmq 5.81.4 `classes/job-scheduler.js` + a live Redis).
|
|
9
|
+
|
|
10
|
+
KEYS[1] `repeat` zset
|
|
11
|
+
|
|
12
|
+
ARGV[1] start (0-based, inclusive)
|
|
13
|
+
ARGV[2] end (0-based, inclusive)
|
|
14
|
+
ARGV[3] queue key prefix `${prefix}:${queue}:` — used to build `repeat:${id}`
|
|
15
|
+
ARGV[4] previewBytes: max bytes of `data`/`opts` returned per scheduler
|
|
16
|
+
|
|
17
|
+
Returns { total, rows } with
|
|
18
|
+
rows = { { id, next, name, pattern, every, tz, offset, limit, ic,
|
|
19
|
+
startDate, endDate, data, opts, truncated }, ... }
|
|
20
|
+
|
|
21
|
+
Redis accesses, each justified:
|
|
22
|
+
1. ZCARD KEYS[1] -> total, for pagination. O(1).
|
|
23
|
+
2. ZRANGE KEYS[1] start end WITHSCORES
|
|
24
|
+
-> the page, ordered by next run. O(log N + page).
|
|
25
|
+
3. one HMGET per row on `repeat:${id}`
|
|
26
|
+
-> the scheduler definition. O(1) each, and the
|
|
27
|
+
page size is bounded by the caller (<= 200),
|
|
28
|
+
so this is a bounded fan-out inside a single
|
|
29
|
+
script rather than N round trips.
|
|
30
|
+
Every key is `${prefix}:${queue}:...`, i.e. the same hash tag: cluster safe.
|
|
31
|
+
|
|
32
|
+
`data`/`opts` are truncated HERE (string.sub) so a scheduler stamping out a
|
|
33
|
+
500 KB template costs the same to list as a tiny one.
|
|
34
|
+
]]
|
|
35
|
+
local rcall = redis.call
|
|
36
|
+
local qprefix = ARGV[3]
|
|
37
|
+
local preview = tonumber(ARGV[4]) or 2048
|
|
38
|
+
|
|
39
|
+
local total = rcall("ZCARD", KEYS[1])
|
|
40
|
+
local raw = rcall("ZRANGE", KEYS[1], tonumber(ARGV[1]), tonumber(ARGV[2]), "WITHSCORES")
|
|
41
|
+
|
|
42
|
+
local rows = {}
|
|
43
|
+
for i = 1, #raw, 2 do
|
|
44
|
+
local id = raw[i]
|
|
45
|
+
local next_run = raw[i + 1]
|
|
46
|
+
|
|
47
|
+
-- HMGET keeps the field order fixed so the caller does not have to parse pairs.
|
|
48
|
+
-- Field list mirrors SCHEDULER_FIELDS in keys.ts — keep both in sync.
|
|
49
|
+
local h = rcall(
|
|
50
|
+
"HMGET", qprefix .. "repeat:" .. id,
|
|
51
|
+
"name", "pattern", "every", "tz", "offset", "limit", "ic",
|
|
52
|
+
"startDate", "endDate", "data", "opts"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
local truncated = 0
|
|
56
|
+
local data = h[10]
|
|
57
|
+
if data and #data > preview then
|
|
58
|
+
data = string.sub(data, 1, preview)
|
|
59
|
+
truncated = 1
|
|
60
|
+
end
|
|
61
|
+
local opts = h[11]
|
|
62
|
+
if opts and #opts > preview then
|
|
63
|
+
opts = string.sub(opts, 1, preview)
|
|
64
|
+
truncated = 1
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
rows[#rows + 1] = {
|
|
68
|
+
id, next_run,
|
|
69
|
+
h[1] or false, h[2] or false, h[3] or false, h[4] or false, h[5] or false,
|
|
70
|
+
h[6] or false, h[7] or false, h[8] or false, h[9] or false,
|
|
71
|
+
data or false, opts or false,
|
|
72
|
+
truncated,
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
return { total, rows }
|