cogworker 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cogworker/attempts.rb +47 -0
- data/lib/cogworker/basic_fetch.rb +25 -2
- data/lib/cogworker/heartbeat.rb +4 -2
- data/lib/cogworker/manager.rb +4 -0
- data/lib/cogworker/periodic/ticker.rb +16 -1
- data/lib/cogworker/process.rb +13 -5
- data/lib/cogworker/processor.rb +4 -0
- data/lib/cogworker/queue.rb +17 -0
- data/lib/cogworker/redis_keys.rb +4 -0
- data/lib/cogworker/throughput.rb +51 -0
- data/lib/cogworker/version.rb +1 -1
- data/lib/cogworker/web/assets/nocturne/fonts/inter-latin.woff2 +0 -0
- data/lib/cogworker/web/assets/nocturne/styles.css +582 -0
- data/lib/cogworker/web/assets/phosphor/Phosphor.woff2 +0 -0
- data/lib/cogworker/web/assets/phosphor/style.css +4627 -0
- data/lib/cogworker/web/layout.rb +232 -141
- data/lib/cogworker/web/routes/history.rb +86 -32
- data/lib/cogworker/web/routes/jobs.rb +469 -0
- data/lib/cogworker/web/routes/overview.rb +746 -0
- data/lib/cogworker/web/routes/schedules.rb +197 -0
- data/lib/cogworker/web/routes/workers.rb +227 -0
- data/lib/cogworker/web.rb +4 -3
- metadata +11 -9
- data/lib/cogworker/web/assets/tailwind.css +0 -1
- data/lib/cogworker/web/routes/busy.rb +0 -99
- data/lib/cogworker/web/routes/dead.rb +0 -93
- data/lib/cogworker/web/routes/periodic.rb +0 -67
- data/lib/cogworker/web/routes/queues.rb +0 -101
- data/lib/cogworker/web/routes/retries.rb +0 -89
- data/lib/cogworker/web/routes/scheduled.rb +0 -49
- data/lib/cogworker/web/routes/stats.rb +0 -298
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'cgi'
|
|
3
4
|
require 'json'
|
|
4
5
|
|
|
5
6
|
module Cogworker
|
|
@@ -22,7 +23,8 @@ module Cogworker
|
|
|
22
23
|
def registered(app)
|
|
23
24
|
app.get('/history') do
|
|
24
25
|
status = History::STATUSES.include?(params['status']) ? params['status'] : 'all'
|
|
25
|
-
|
|
26
|
+
jid = params['jid'].to_s
|
|
27
|
+
content = Routes::History.render_content(request.script_name, status, jid)
|
|
26
28
|
if hx_request?
|
|
27
29
|
content
|
|
28
30
|
else
|
|
@@ -37,7 +39,9 @@ module Cogworker
|
|
|
37
39
|
# its sort/filter/scroll state) on every tick.
|
|
38
40
|
app.get('/history/data') do
|
|
39
41
|
status = History::STATUSES.include?(params['status']) ? params['status'] : 'all'
|
|
42
|
+
jid = params['jid'].to_s
|
|
40
43
|
entries, = Cogworker::History::Storage.page(status, 1, Cogworker::History.max_entries)
|
|
44
|
+
entries = entries.select { |e| e['jid'] == jid } unless jid.empty?
|
|
41
45
|
[200, { 'content-type' => 'application/json' }, [JSON.generate(entries)]]
|
|
42
46
|
end
|
|
43
47
|
end
|
|
@@ -57,47 +61,76 @@ module Cogworker
|
|
|
57
61
|
# from there — retention (`max_entries`) already bounds this to a
|
|
58
62
|
# size AG Grid handles comfortably, so there's no need for the
|
|
59
63
|
# gem's own server-side paging on top of it.
|
|
60
|
-
|
|
64
|
+
#
|
|
65
|
+
# `jid` (optional — empty string means "not filtering") narrows this
|
|
66
|
+
# down to one job's own run history across every retry attempt (a
|
|
67
|
+
# retried job keeps its original jid throughout, so this is exactly
|
|
68
|
+
# "this job's full timeline", unlike the Jobs tab's own Retry
|
|
69
|
+
# history, which only shows its *failures*, capped to the last few —
|
|
70
|
+
# see `Routes::Jobs#attempts_section`, which links here).
|
|
71
|
+
def render_content(script_name, status, jid = '')
|
|
61
72
|
entries, = Cogworker::History::Storage.page(status, 1, Cogworker::History.max_entries)
|
|
62
|
-
|
|
73
|
+
entries = entries.select { |e| e['jid'] == jid } unless jid.empty?
|
|
74
|
+
<<~HTML
|
|
75
|
+
<div style="display: flex; flex-direction: column; gap: 16px;">
|
|
76
|
+
#{page_header(script_name, status, jid)}
|
|
77
|
+
#{grid(entries, script_name, status, jid)}
|
|
78
|
+
</div>
|
|
79
|
+
HTML
|
|
63
80
|
end
|
|
64
81
|
|
|
65
|
-
def
|
|
66
|
-
links = STATUSES.map { |status| filter_link(script_name, status, active: status == current_status) }.join
|
|
67
|
-
|
|
82
|
+
def page_header(script_name, current_status, jid)
|
|
83
|
+
links = STATUSES.map { |status| filter_link(script_name, status, jid, active: status == current_status) }.join
|
|
84
|
+
<<~HTML
|
|
85
|
+
<div style="display: flex; align-items: flex-end; justify-content: space-between; gap: 16px; flex-wrap: wrap;">
|
|
86
|
+
<h2 style="margin: 0;">History</h2>
|
|
87
|
+
<div class="seg">#{links}</div>
|
|
88
|
+
</div>
|
|
89
|
+
#{jid.empty? ? '' : jid_filter_banner(script_name, current_status, jid)}
|
|
90
|
+
HTML
|
|
68
91
|
end
|
|
69
92
|
|
|
70
|
-
def filter_link(script_name, status, active:)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
end
|
|
76
|
-
href = Layout.path(script_name, "history?status=#{status}")
|
|
77
|
-
%(<a href="#{href}" class="px-3 py-1 rounded-md text-sm font-medium #{classes}">#{Layout.h(status.capitalize)}</a>)
|
|
93
|
+
def filter_link(script_name, status, jid, active:)
|
|
94
|
+
query = "status=#{status}"
|
|
95
|
+
query += "&jid=#{CGI.escape(jid)}" unless jid.empty?
|
|
96
|
+
href = Layout.path(script_name, "history?#{query}")
|
|
97
|
+
%(<label class="seg-opt"><input type="radio" name="status" #{'checked' if active} onchange="location.href='#{href}'">#{Layout.h(status.capitalize)}</label>)
|
|
78
98
|
end
|
|
79
99
|
|
|
80
|
-
|
|
100
|
+
# Shown alongside the status filter whenever a `?jid=` narrowed the
|
|
101
|
+
# grid down to one job — otherwise there'd be no indication *why*
|
|
102
|
+
# the grid suddenly has far fewer rows, and no way back to the
|
|
103
|
+
# unfiltered view short of hand-editing the URL.
|
|
104
|
+
def jid_filter_banner(script_name, status, jid)
|
|
105
|
+
clear_href = Layout.path(script_name, "history?status=#{status}")
|
|
106
|
+
<<~HTML
|
|
107
|
+
<div style="display: flex; align-items: center; gap: 8px; font-size: 13px; color: var(--color-neutral-400);">
|
|
108
|
+
<span>Filtered to job <span class="mono">#{Layout.h(jid)}</span></span>
|
|
109
|
+
<a href="#{clear_href}" class="btn btn-secondary" style="font-size: 12px; padding: 3px 8px;">clear</a>
|
|
110
|
+
</div>
|
|
111
|
+
HTML
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def grid(entries, script_name, status, jid)
|
|
81
115
|
<<~HTML
|
|
82
116
|
<div id="history-grid" class="ag-theme-alpine" style="height: 70vh; width: 100%;"></div>
|
|
83
117
|
|
|
84
|
-
<dialog id="history-backtrace-dialog" class="
|
|
85
|
-
<div
|
|
86
|
-
<
|
|
87
|
-
|
|
88
|
-
<button type="button" onclick="this.closest('dialog').close()"
|
|
89
|
-
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200">✕</button>
|
|
90
|
-
</div>
|
|
91
|
-
<pre id="history-backtrace-content" class="text-xs whitespace-pre-wrap max-h-[60vh] overflow-y-auto bg-gray-50 dark:bg-gray-950 p-3 rounded border border-gray-200 dark:border-gray-800"></pre>
|
|
118
|
+
<dialog id="history-backtrace-dialog" class="dialog" style="width: min(720px, 90vw);">
|
|
119
|
+
<div style="display: flex; justify-content: space-between; align-items: center;">
|
|
120
|
+
<span class="dialog-title">Backtrace</span>
|
|
121
|
+
<button type="button" onclick="this.closest('dialog').close()" class="btn btn-icon btn-ghost" aria-label="Close">✕</button>
|
|
92
122
|
</div>
|
|
123
|
+
<pre id="history-backtrace-content" class="mono" style="margin: 0; font-size: 12px; line-height: 1.6; white-space: pre-wrap; max-height: 60vh; overflow-y: auto; background: var(--color-bg); border: 1px solid var(--color-divider); border-radius: var(--radius-sm); padding: var(--space-3);"></pre>
|
|
93
124
|
</dialog>
|
|
94
125
|
|
|
95
|
-
#{grid_script(entries, script_name, status)}
|
|
126
|
+
#{grid_script(entries, script_name, status, jid)}
|
|
96
127
|
HTML
|
|
97
128
|
end
|
|
98
129
|
|
|
99
|
-
def grid_script(entries, script_name, status)
|
|
100
|
-
|
|
130
|
+
def grid_script(entries, script_name, status, jid)
|
|
131
|
+
query = "status=#{status}"
|
|
132
|
+
query += "&jid=#{CGI.escape(jid)}" unless jid.empty?
|
|
133
|
+
data_url = Layout.path(script_name, "history/data?#{query}")
|
|
101
134
|
<<~HTML
|
|
102
135
|
<script>
|
|
103
136
|
(function () {
|
|
@@ -124,12 +157,31 @@ module Cogworker
|
|
|
124
157
|
document.getElementById('history-backtrace-dialog').showModal();
|
|
125
158
|
};
|
|
126
159
|
|
|
160
|
+
// Reads the actual nocturne tokens at render time (not a
|
|
161
|
+
// hardcoded hex) so this stays in sync with a retuned ramp,
|
|
162
|
+
// and resolves correctly whichever of the dark/light
|
|
163
|
+
// `@media` blocks in styles.css is currently active —
|
|
164
|
+
// matching the same `.tag`/`.tag-success`/`.tag-danger`
|
|
165
|
+
// look used everywhere else, since AG Grid's own
|
|
166
|
+
// cellRenderer can't just apply those CSS classes to cells
|
|
167
|
+
// it builds from a plain string/DOM node.
|
|
168
|
+
var cssVar = function (name) { return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); };
|
|
169
|
+
|
|
127
170
|
function statusCellRenderer(p) {
|
|
128
171
|
var ok = p.value === 'success';
|
|
129
|
-
var
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
172
|
+
var bg = cssVar(ok ? '--color-success-800' : '--color-danger-800');
|
|
173
|
+
var fg = cssVar(ok ? '--color-success-100' : '--color-danger-100');
|
|
174
|
+
// `line-height: 1` resets AG Grid's own row-height-driven
|
|
175
|
+
// `.ag-cell { line-height: <rowHeight>px; }` (inherited
|
|
176
|
+
// here since this span never set its own) — without it,
|
|
177
|
+
// the plain text content's line box takes on the full
|
|
178
|
+
// row height (e.g. 41px) before padding is even added,
|
|
179
|
+
// ballooning the pill well past the cell's own height and
|
|
180
|
+
// getting top/bottom-clipped by the cell's `overflow:
|
|
181
|
+
// hidden`, which also clips away its rounded corners.
|
|
182
|
+
return '<span style="display:inline-flex;align-items:center;font-size:11px;letter-spacing:0.02em;' +
|
|
183
|
+
'line-height:1;padding:3px 10px;border-radius:6px;background:' + bg + ';color:' + fg + ';">' +
|
|
184
|
+
p.value + '</span>';
|
|
133
185
|
}
|
|
134
186
|
|
|
135
187
|
function errorValueGetter(p) {
|
|
@@ -140,7 +192,9 @@ module Cogworker
|
|
|
140
192
|
if (!p.data.backtrace) return p.value || '';
|
|
141
193
|
var span = document.createElement('span');
|
|
142
194
|
span.textContent = p.value;
|
|
143
|
-
span.
|
|
195
|
+
span.style.color = cssVar('--color-danger-300');
|
|
196
|
+
span.style.textDecoration = 'underline';
|
|
197
|
+
span.style.cursor = 'pointer';
|
|
144
198
|
span.title = 'Click to view backtrace';
|
|
145
199
|
span.addEventListener('click', function () { window.cogworkerShowHistoryBacktrace(p.data.jid); });
|
|
146
200
|
return span;
|
|
@@ -197,7 +251,7 @@ module Cogworker
|
|
|
197
251
|
document.getElementById('history-grid').classList.add('ag-theme-alpine-dark');
|
|
198
252
|
}
|
|
199
253
|
|
|
200
|
-
// AG Grid isn't htmx-swapped (unlike
|
|
254
|
+
// AG Grid isn't htmx-swapped (unlike Workers/Stats/Overview), so
|
|
201
255
|
// it needs its own poll — gated by the same global toggle —
|
|
202
256
|
// that replaces just `rowData` in place rather than
|
|
203
257
|
// reloading the fragment and tearing the grid instance down.
|
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'cgi'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
module Cogworker
|
|
7
|
+
class Web
|
|
8
|
+
module Routes
|
|
9
|
+
# One filterable, searchable table across every individual job
|
|
10
|
+
# instance — enqueued, running, scheduled (`perform_in`/`perform_at`),
|
|
11
|
+
# retrying, and dead — replacing the four separate `Routes::Queues`*/
|
|
12
|
+
# `Retries`/`Scheduled`/`Dead` tabs' own per-status tables with one
|
|
13
|
+
# list plus a status filter, and a click-through detail panel (args,
|
|
14
|
+
# retry timeline, last error, actions) in place of `Routes::Dead`'s
|
|
15
|
+
# flat rows. (*`Routes::Queues`' own queue-level browsing survives
|
|
16
|
+
# separately, as `Routes::Overview` layout B — this tab is the
|
|
17
|
+
# cross-queue, cross-status view; the two overlap a little on purpose,
|
|
18
|
+
# the same way a real Sidekiq-style admin wants both "what's in this
|
|
19
|
+
# queue" and "find this one job wherever it is".)
|
|
20
|
+
#
|
|
21
|
+
# `?status=`/`?q=`/`?selected=` are plain query params, read fresh on
|
|
22
|
+
# every request and carried through the self-poll and every action's
|
|
23
|
+
# redirect (`query_string`) — the same full-page-link pattern
|
|
24
|
+
# `Routes::Overview`'s layout switcher and `Routes::History`'s status
|
|
25
|
+
# filter already use, not client-side state.
|
|
26
|
+
module Jobs
|
|
27
|
+
CONTENT_ID = 'jobs-content'
|
|
28
|
+
STATUSES = %w[All Enqueued Running Scheduled Retrying Dead].freeze
|
|
29
|
+
# `Cogworker::Attempts` itself already caps storage at `MAX_ENTRIES`
|
|
30
|
+
# (25) — this is a *second*, smaller cap on top of that, just for
|
|
31
|
+
# this compact sidebar panel: showing all 25 here would make the
|
|
32
|
+
# panel far taller than the row list next to it. The full trail
|
|
33
|
+
# (still capped at 25, but the same 25 either way) is one click
|
|
34
|
+
# away via the History link below, not lost.
|
|
35
|
+
ATTEMPTS_DISPLAY_LIMIT = 5
|
|
36
|
+
TAG_CLASS = { 'Enqueued' => 'tag-neutral', 'Running' => 'tag-accent', 'Scheduled' => 'tag-accent-2',
|
|
37
|
+
'Retrying' => 'tag-warning', 'Dead' => 'tag-danger' }.freeze
|
|
38
|
+
|
|
39
|
+
module_function
|
|
40
|
+
|
|
41
|
+
def registered(app)
|
|
42
|
+
app.get('/jobs') do
|
|
43
|
+
content = Jobs.render_content(request.script_name, params)
|
|
44
|
+
if hx_request?
|
|
45
|
+
content
|
|
46
|
+
else
|
|
47
|
+
Layout.wrap('Jobs',
|
|
48
|
+
Layout.poll_div(CONTENT_ID, request.script_name, "jobs#{Jobs.query_string(params)}",
|
|
49
|
+
content), script_name: request.script_name)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
app.post('/jobs/enqueued/delete') do
|
|
54
|
+
Cogworker::Queue.new(params['queue']).delete(params['raw'])
|
|
55
|
+
Jobs.respond(self, params)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
app.post('/jobs/scheduled/delete') do
|
|
59
|
+
Cogworker.config.redis { |c| c.zrem(RedisKeys::SCHEDULE, params['raw']) }
|
|
60
|
+
Jobs.respond(self, params)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
app.post('/jobs/retrying/delete') do
|
|
64
|
+
raw = params['raw']
|
|
65
|
+
Cogworker.config.redis { |c| c.zrem(RedisKeys::RETRY, raw) }
|
|
66
|
+
Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
|
|
67
|
+
Jobs.respond(self, params)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Same "graduate back onto its queue" move `Routes::Dead#retry`/
|
|
71
|
+
# `Cogworker::Scheduled#graduate` make — `zrem` winning (not
|
|
72
|
+
# losing) gates it so two tabs retrying the same entry at once
|
|
73
|
+
# can't both requeue it; the `redis` gem's `#zrem` returns a
|
|
74
|
+
# **Boolean** for a single member, so this checks truthiness, not
|
|
75
|
+
# `== 1` (`true == 1` is `false` in Ruby).
|
|
76
|
+
app.post('/jobs/retrying/retry_now') do
|
|
77
|
+
raw = params['raw']
|
|
78
|
+
Cogworker.config.redis do |c|
|
|
79
|
+
if c.zrem(RedisKeys::RETRY, raw)
|
|
80
|
+
job = JSON.parse(raw)
|
|
81
|
+
c.sadd(RedisKeys::QUEUES, job['queue'])
|
|
82
|
+
c.lpush(RedisKeys.queue(job['queue']), raw)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
Jobs.respond(self, params)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
app.post('/jobs/dead/delete') do
|
|
89
|
+
raw = params['raw']
|
|
90
|
+
Cogworker.config.redis { |c| c.zrem(RedisKeys::DEAD, raw) }
|
|
91
|
+
Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
|
|
92
|
+
Jobs.respond(self, params)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
app.post('/jobs/dead/delete_all') do
|
|
96
|
+
jids = Cogworker.config.redis { |c| c.zrange(RedisKeys::DEAD, 0, -1) }.map { |raw| JSON.parse(raw)['jid'] }
|
|
97
|
+
Cogworker.config.redis { |c| c.del(RedisKeys::DEAD) }
|
|
98
|
+
jids.each { |jid| Cogworker::Attempts.clear(jid) }
|
|
99
|
+
Jobs.respond(self, params)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
app.post('/jobs/dead/retry') do
|
|
103
|
+
raw = params['raw']
|
|
104
|
+
Cogworker.config.redis do |c|
|
|
105
|
+
if c.zrem(RedisKeys::DEAD, raw)
|
|
106
|
+
job = JSON.parse(raw)
|
|
107
|
+
c.sadd(RedisKeys::QUEUES, job['queue'])
|
|
108
|
+
c.lpush(RedisKeys.queue(job['queue']), raw)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
Jobs.respond(self, params)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Moves this one entry's score on its own ZSET — "run sooner" or
|
|
115
|
+
# "push back a flaky retry" — without touching anything else about
|
|
116
|
+
# it (attempt count, error, args all carry over unchanged, same
|
|
117
|
+
# raw payload). `zrem` winning first (not losing) guards this the
|
|
118
|
+
# same way every other per-entry action here does: if the entry
|
|
119
|
+
# was deleted/retried by another tab in between, there's nothing
|
|
120
|
+
# left to reschedule, so it's silently skipped rather than
|
|
121
|
+
# resurrecting a stale copy.
|
|
122
|
+
app.post('/jobs/retrying/reschedule') do
|
|
123
|
+
Jobs.reschedule(RedisKeys::RETRY, params['raw'], params['minutes'])
|
|
124
|
+
Jobs.respond(self, params)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
app.post('/jobs/scheduled/reschedule') do
|
|
128
|
+
Jobs.reschedule(RedisKeys::SCHEDULE, params['raw'], params['minutes'])
|
|
129
|
+
Jobs.respond(self, params)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# A relative offset ("run in N minutes"), not an absolute date/time
|
|
134
|
+
# picker: a `datetime-local` input reports the *browser's* local
|
|
135
|
+
# wall-clock with no timezone attached, and correctly converting
|
|
136
|
+
# that back to the server's UTC epoch needs its own bit of
|
|
137
|
+
# client-side JS (`new Date(value).getTime()`) — a real, tested
|
|
138
|
+
# moving part for what this is mostly used for (nudge a flaky retry
|
|
139
|
+
# sooner or later by a few minutes). A plain relative number needs
|
|
140
|
+
# none of that: the server computes the absolute score itself, with
|
|
141
|
+
# no timezone to get wrong.
|
|
142
|
+
RESCHEDULE_MAX_MINUTES = 10_080 # 1 week — a sane upper bound on the input, not a real limit elsewhere
|
|
143
|
+
|
|
144
|
+
def reschedule(zset_key, raw, minutes_param)
|
|
145
|
+
minutes = minutes_param.to_i.clamp(1, RESCHEDULE_MAX_MINUTES)
|
|
146
|
+
Cogworker.config.redis do |c|
|
|
147
|
+
c.zadd(zset_key, Time.now.to_f + (minutes * 60), raw) if c.zrem(zset_key, raw)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# After an action, htmx gets the refreshed fragment swapped into
|
|
152
|
+
# #jobs-content in place, keeping the same filter/search/selection;
|
|
153
|
+
# a plain form submission (no JS) falls back to a normal redirect
|
|
154
|
+
# to that same URL.
|
|
155
|
+
def respond(action, params)
|
|
156
|
+
if action.hx_request?
|
|
157
|
+
render_content(action.request.script_name, params)
|
|
158
|
+
else
|
|
159
|
+
action.redirect(Layout.path(action.request.script_name, "jobs#{query_string(params)}"))
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def query_string(params)
|
|
164
|
+
status = STATUSES.include?(params['status']) ? params['status'] : 'All'
|
|
165
|
+
q = params['q'].to_s
|
|
166
|
+
selected = params['selected'].to_s
|
|
167
|
+
parts = []
|
|
168
|
+
parts << "status=#{CGI.escape(status)}" unless status == 'All'
|
|
169
|
+
parts << "q=#{CGI.escape(q)}" unless q.empty?
|
|
170
|
+
parts << "selected=#{CGI.escape(selected)}" unless selected.empty?
|
|
171
|
+
parts.empty? ? '' : "?#{parts.join('&')}"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def render_content(script_name, params)
|
|
175
|
+
status = STATUSES.include?(params['status']) ? params['status'] : 'All'
|
|
176
|
+
query = params['q'].to_s
|
|
177
|
+
rows = all_rows
|
|
178
|
+
filtered = rows.select { |r| status == 'All' || r[:status] == status }
|
|
179
|
+
filtered = filtered.select { |r| matches_query?(r, query) } unless query.strip.empty?
|
|
180
|
+
filtered.sort_by! { |r| -(r[:at] || 0) }
|
|
181
|
+
|
|
182
|
+
selected = rows.find { |r| r[:jid] == params['selected'] }
|
|
183
|
+
|
|
184
|
+
<<~HTML
|
|
185
|
+
<div style="display: flex; flex-direction: column; gap: 16px;">
|
|
186
|
+
#{page_header(script_name, status, query, filtered.size, rows.size)}
|
|
187
|
+
<div style="display: grid; grid-template-columns: minmax(0, 1fr) #{selected ? 'minmax(320px, 400px)' : '0px'}; gap: 16px; align-items: start;">
|
|
188
|
+
#{jobs_table(filtered, script_name, params)}
|
|
189
|
+
#{selected ? detail_panel(selected, script_name, params) : ''}
|
|
190
|
+
</div>
|
|
191
|
+
</div>
|
|
192
|
+
HTML
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def matches_query?(row, query)
|
|
196
|
+
haystack = "#{row[:klass]}#{row[:jid]}#{row[:args]}".downcase
|
|
197
|
+
haystack.include?(query.strip.downcase)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def page_header(script_name, status, query, shown_count, total_count)
|
|
201
|
+
filters = STATUSES.map do |s|
|
|
202
|
+
checked = s == status ? ' checked' : ''
|
|
203
|
+
href = Layout.path(script_name, "jobs#{query_string('status' => s, 'q' => query)}")
|
|
204
|
+
%(<label class="seg-opt"><input type="radio" name="status"#{checked} onchange="location.href='#{href}'">#{Layout.h(s)}</label>)
|
|
205
|
+
end.join
|
|
206
|
+
search_action = Layout.path(script_name, 'jobs')
|
|
207
|
+
<<~HTML
|
|
208
|
+
<div style="display: flex; align-items: flex-end; justify-content: space-between; gap: 16px; flex-wrap: wrap;">
|
|
209
|
+
<div>
|
|
210
|
+
<h2 style="margin: 0 0 4px;">Jobs</h2>
|
|
211
|
+
<p class="text-muted" style="margin: 0; font-size: 13px;">#{shown_count} of #{total_count} shown</p>
|
|
212
|
+
</div>
|
|
213
|
+
<div style="display: flex; align-items: center; gap: 10px; flex-wrap: wrap;">
|
|
214
|
+
<form method="get" action="#{search_action}">
|
|
215
|
+
<input type="hidden" name="status" value="#{Layout.h(status)}">
|
|
216
|
+
<input class="input" style="width: 240px;" type="search" name="q" placeholder="Search class, jid or args" value="#{Layout.h(query)}">
|
|
217
|
+
</form>
|
|
218
|
+
<div class="seg">#{filters}</div>
|
|
219
|
+
</div>
|
|
220
|
+
</div>
|
|
221
|
+
HTML
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def jobs_table(rows, script_name, params)
|
|
225
|
+
table_rows = rows.map do |r|
|
|
226
|
+
href = Layout.path(script_name, "jobs#{query_string(params.merge('selected' => r[:jid]))}")
|
|
227
|
+
job_cell = <<~HTML
|
|
228
|
+
<a href="#{href}" style="text-decoration: none; color: inherit; display: block;">
|
|
229
|
+
<div class="mono" style="font-size: 14px;">#{Layout.h(r[:klass])}</div>
|
|
230
|
+
<div class="mono" style="font-size: 11px; color: var(--color-neutral-500);">#{Layout.h(r[:jid])}</div>
|
|
231
|
+
</a>
|
|
232
|
+
HTML
|
|
233
|
+
[job_cell, Layout.h(r[:queue]), status_tag(r[:status]),
|
|
234
|
+
Layout.h(r[:attempt]), r[:next_action] ? Layout.time_tag(r[:next_action]) : '—',
|
|
235
|
+
r[:at] ? Layout.time_tag(r[:at]) : '—', row_actions(r, script_name, params)]
|
|
236
|
+
end
|
|
237
|
+
Layout.table(%w[Job Queue Status Attempt NextRun At Actions], table_rows,
|
|
238
|
+
empty_message: 'No jobs match this view.')
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Finer-grained than `Layout.badge`'s 4 semantic variants (success/
|
|
242
|
+
# warning/danger/default): Running/Scheduled get their own accent/
|
|
243
|
+
# accent-2 tint rather than falling back to neutral, so a glance at
|
|
244
|
+
# the Status column tells "in flight" and "not due yet" apart from
|
|
245
|
+
# a plain "waiting" enqueued row.
|
|
246
|
+
def status_tag(status)
|
|
247
|
+
%(<span class="tag #{TAG_CLASS.fetch(status, 'tag-neutral')}">#{Layout.h(status)}</span>)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def row_actions(row, script_name, params)
|
|
251
|
+
case row[:source]
|
|
252
|
+
when :enqueued
|
|
253
|
+
delete_button('enqueued', script_name, row, params, extra: { 'queue' => row[:queue] })
|
|
254
|
+
when :scheduled
|
|
255
|
+
delete_button('scheduled', script_name, row, params)
|
|
256
|
+
when :retrying
|
|
257
|
+
retry_now_button('retrying', script_name, row, params) + delete_button('retrying', script_name, row,
|
|
258
|
+
params)
|
|
259
|
+
when :dead
|
|
260
|
+
retry_button('dead', script_name, row, params) + delete_button('dead', script_name, row, params)
|
|
261
|
+
else
|
|
262
|
+
''
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def delete_button(bucket, script_name, row, params, extra: {})
|
|
267
|
+
action = Layout.path(script_name, "jobs/#{bucket}/delete")
|
|
268
|
+
form_with_extra(action, 'raw', row[:raw], 'delete', params, variant: :danger, icon: 'trash', extra: extra)
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def retry_now_button(bucket, script_name, row, params)
|
|
272
|
+
action = Layout.path(script_name, "jobs/#{bucket}/retry_now")
|
|
273
|
+
form_with_extra(action, 'raw', row[:raw], 'retry now', params, variant: :primary, icon: 'arrow-clockwise',
|
|
274
|
+
extra: {})
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def retry_button(bucket, script_name, row, params)
|
|
278
|
+
action = Layout.path(script_name, "jobs/#{bucket}/retry")
|
|
279
|
+
form_with_extra(action, 'raw', row[:raw], 'retry', params, variant: :primary, icon: 'arrow-clockwise',
|
|
280
|
+
extra: {})
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# `Layout.form_button` only carries one hidden field. Every action
|
|
284
|
+
# here needs more: `status`/`q`/`selected` (so the fragment/redirect
|
|
285
|
+
# `respond` sends back preserves the view the user was looking at —
|
|
286
|
+
# these forms are the only way that state reaches the POST at all,
|
|
287
|
+
# since it isn't in the URL a plain form submits to) plus, for an
|
|
288
|
+
# enqueued-job delete, `queue` (`Queue#delete` isn't keyed off the
|
|
289
|
+
# raw payload alone the way the ZSETs are).
|
|
290
|
+
def form_with_extra(action, hidden_name, hidden_value, label, params, variant:, icon: nil, extra:)
|
|
291
|
+
hidden = extra.merge('status' => params['status'].to_s, 'q' => params['q'].to_s,
|
|
292
|
+
'selected' => params['selected'].to_s)
|
|
293
|
+
hidden_inputs = hidden.map { |k, v| %(<input type="hidden" name="#{k}" value="#{Layout.h(v)}">) }.join
|
|
294
|
+
classes = "btn #{Layout::BUTTON_VARIANTS.fetch(variant)}"
|
|
295
|
+
<<~HTML
|
|
296
|
+
<form style="display: inline;" hx-post="#{action}" hx-target="##{CONTENT_ID}" hx-swap="innerHTML" method="post" action="#{action}">
|
|
297
|
+
<input type="hidden" name="#{hidden_name}" value="#{Layout.h(hidden_value)}">
|
|
298
|
+
#{hidden_inputs}
|
|
299
|
+
<button type="submit" class="#{classes}" style="font-size: 13px; padding: 4px 10px;">#{Layout.icon_tag(icon)}#{Layout.h(label)}</button>
|
|
300
|
+
</form>
|
|
301
|
+
HTML
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def detail_panel(row, script_name, params)
|
|
305
|
+
close_href = Layout.path(script_name, "jobs#{query_string(params.merge('selected' => nil))}")
|
|
306
|
+
<<~HTML
|
|
307
|
+
<aside style="background: var(--color-surface); border-radius: var(--radius-md); box-shadow: var(--shadow-md); padding: 16px 18px; display: flex; flex-direction: column; gap: 14px; position: sticky; top: 84px;">
|
|
308
|
+
<div style="display: flex; align-items: flex-start; justify-content: space-between; gap: 10px;">
|
|
309
|
+
<div>
|
|
310
|
+
<div class="mono" style="font-size: 16px;">#{Layout.h(row[:klass])}</div>
|
|
311
|
+
<div class="mono" style="font-size: 11px; color: var(--color-neutral-500); margin-top: 2px;">#{Layout.h(row[:jid])}</div>
|
|
312
|
+
</div>
|
|
313
|
+
<a href="#{close_href}" class="btn btn-icon btn-secondary" aria-label="Close"><i class="ph ph-x"></i></a>
|
|
314
|
+
</div>
|
|
315
|
+
<div style="display: flex; flex-wrap: wrap; gap: 6px;">
|
|
316
|
+
#{status_tag(row[:status])}
|
|
317
|
+
<span class="tag tag-neutral">#{Layout.h(row[:queue])}</span>
|
|
318
|
+
<span class="tag tag-outline">attempt #{Layout.h(row[:attempt])}</span>
|
|
319
|
+
</div>
|
|
320
|
+
<div>
|
|
321
|
+
<div style="font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--color-neutral-500); margin-bottom: 6px;">Arguments</div>
|
|
322
|
+
<pre class="mono" style="margin: 0; font-size: 12px; line-height: 1.6; background: var(--color-bg); border: 1px solid var(--color-divider); border-radius: var(--radius-sm); padding: 10px; overflow-x: auto;">#{Layout.h(JSON.pretty_generate(row[:args]))}</pre>
|
|
323
|
+
</div>
|
|
324
|
+
#{attempts_section(row, script_name)}
|
|
325
|
+
#{last_error_section(row)}
|
|
326
|
+
#{reschedule_section(row, script_name, params)}
|
|
327
|
+
<div style="display: flex; gap: 8px; flex-wrap: wrap;">
|
|
328
|
+
#{row_actions(row, script_name, params)}
|
|
329
|
+
</div>
|
|
330
|
+
</aside>
|
|
331
|
+
HTML
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
def reschedule_section(row, script_name, params)
|
|
335
|
+
return '' unless %w[Retrying Scheduled].include?(row[:status])
|
|
336
|
+
|
|
337
|
+
action = Layout.path(script_name, "jobs/#{row[:source]}/reschedule")
|
|
338
|
+
hidden = { 'status' => params['status'].to_s, 'q' => params['q'].to_s,
|
|
339
|
+
'selected' => params['selected'].to_s }
|
|
340
|
+
hidden_inputs = hidden.map { |k, v| %(<input type="hidden" name="#{k}" value="#{Layout.h(v)}">) }.join
|
|
341
|
+
<<~HTML
|
|
342
|
+
<div>
|
|
343
|
+
<div style="font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--color-neutral-500); margin-bottom: 6px;">Reschedule</div>
|
|
344
|
+
<form style="display: flex; align-items: center; gap: 8px; flex-wrap: wrap;" hx-post="#{action}" hx-target="##{CONTENT_ID}" hx-swap="innerHTML" method="post" action="#{action}">
|
|
345
|
+
<input type="hidden" name="raw" value="#{Layout.h(row[:raw])}">
|
|
346
|
+
#{hidden_inputs}
|
|
347
|
+
<span style="font-size: 13px; color: var(--color-neutral-400);">run in</span>
|
|
348
|
+
<input class="input" type="number" name="minutes" min="1" max="#{RESCHEDULE_MAX_MINUTES}" value="5" style="width: 72px;">
|
|
349
|
+
<span style="font-size: 13px; color: var(--color-neutral-400);">minutes</span>
|
|
350
|
+
<button type="submit" class="btn btn-secondary" style="font-size: 13px; padding: 4px 10px;">#{Layout.icon_tag('clock-countdown')}reschedule</button>
|
|
351
|
+
</form>
|
|
352
|
+
</div>
|
|
353
|
+
HTML
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
def attempts_section(row, script_name)
|
|
357
|
+
return '' unless %w[Retrying Dead].include?(row[:status])
|
|
358
|
+
|
|
359
|
+
attempts = Cogworker::Attempts.for(row[:jid]).reverse
|
|
360
|
+
return '' if attempts.empty?
|
|
361
|
+
|
|
362
|
+
shown = attempts.first(ATTEMPTS_DISPLAY_LIMIT)
|
|
363
|
+
items = shown.map do |a|
|
|
364
|
+
dot = a['outcome'] == 'dead' ? 'var(--color-danger)' : 'var(--color-warning)'
|
|
365
|
+
<<~HTML
|
|
366
|
+
<div style="display: grid; grid-template-columns: 18px minmax(0, 1fr); gap: 10px;">
|
|
367
|
+
<div style="display: flex; flex-direction: column; align-items: center;">
|
|
368
|
+
<span style="width: 9px; height: 9px; border-radius: 50%; background: #{dot}; margin-top: 5px;"></span>
|
|
369
|
+
</div>
|
|
370
|
+
<div style="padding-bottom: 12px;">
|
|
371
|
+
<div style="font-size: 13px;">Attempt #{a['attempt']} · #{a['outcome'] == 'dead' ? 'moved to dead set' : 'failed'}</div>
|
|
372
|
+
<div style="font-size: 12px; color: var(--color-neutral-500);">#{Layout.h(Time.at(a['failed_at']).utc.strftime(Web.time_format))} · #{Layout.h(a['error_class'])}: #{Layout.h(a['error_message'])}</div>
|
|
373
|
+
</div>
|
|
374
|
+
</div>
|
|
375
|
+
HTML
|
|
376
|
+
end.join
|
|
377
|
+
<<~HTML
|
|
378
|
+
<div>
|
|
379
|
+
<div style="display: flex; align-items: center; justify-content: space-between; gap: 8px; margin-bottom: 8px;">
|
|
380
|
+
<div style="font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--color-neutral-500);">
|
|
381
|
+
Retry history#{" (last #{shown.size} of #{attempts.size})" if attempts.size > shown.size}
|
|
382
|
+
</div>
|
|
383
|
+
#{history_link(row, script_name)}
|
|
384
|
+
</div>
|
|
385
|
+
<div style="display: flex; flex-direction: column;">#{items}</div>
|
|
386
|
+
</div>
|
|
387
|
+
HTML
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
# Full timeline for this job — not just its failures (`Cogworker::
|
|
391
|
+
# Attempts` above, capped to `ATTEMPTS_DISPLAY_LIMIT` here on top of
|
|
392
|
+
# its own storage cap) but every completed run, success included,
|
|
393
|
+
# since a retried job keeps its original jid across every attempt.
|
|
394
|
+
def history_link(row, script_name)
|
|
395
|
+
href = Layout.path(script_name, "history?jid=#{CGI.escape(row[:jid])}")
|
|
396
|
+
%(<a href="#{href}" style="font-size: 12px; white-space: nowrap;">view in History</a>)
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
def last_error_section(row)
|
|
400
|
+
return '' unless row[:error_class]
|
|
401
|
+
|
|
402
|
+
<<~HTML
|
|
403
|
+
<div>
|
|
404
|
+
<div style="font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--color-neutral-500); margin-bottom: 6px;">Last error</div>
|
|
405
|
+
<pre class="mono" style="margin: 0; font-size: 12px; line-height: 1.6; background: var(--color-bg); border: 1px solid color-mix(in srgb, var(--color-danger) 35%, transparent); border-radius: var(--radius-sm); padding: 10px; overflow-x: auto; color: var(--color-danger-300);">#{Layout.h(row[:error_class])}: #{Layout.h(row[:error_message])}</pre>
|
|
406
|
+
</div>
|
|
407
|
+
HTML
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
def all_rows
|
|
411
|
+
enqueued_rows + running_rows + scheduled_rows + retrying_rows + dead_rows
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
def enqueued_rows
|
|
415
|
+
Cogworker.config.redis { |c| c.smembers(RedisKeys::QUEUES) }.sort.flat_map do |queue_name|
|
|
416
|
+
Cogworker::Queue.new(queue_name).map do |r|
|
|
417
|
+
{ jid: r.jid, klass: r.klass, queue: r.queue, status: 'Enqueued', attempt: '—', next_action: nil,
|
|
418
|
+
at: r.item['created_at'] || r.item['enqueued_at'], args: r.args, error_class: nil,
|
|
419
|
+
error_message: nil, raw: r.value, source: :enqueued }
|
|
420
|
+
end
|
|
421
|
+
end
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
def running_rows
|
|
425
|
+
Cogworker::WorkSet.new.map do |_identity, _tid, work|
|
|
426
|
+
job = work.job
|
|
427
|
+
{ jid: job['jid'], klass: job['class'], queue: work.queue, status: 'Running',
|
|
428
|
+
attempt: (job['retry_count'].to_i + 1).to_s, next_action: nil, at: work.run_at, args: job['args'],
|
|
429
|
+
error_class: nil, error_message: nil, raw: nil, source: :running }
|
|
430
|
+
end
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
def scheduled_rows
|
|
434
|
+
entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::SCHEDULE, 0, -1, withscores: true) }
|
|
435
|
+
entries.map do |raw, score|
|
|
436
|
+
job = JSON.parse(raw)
|
|
437
|
+
{ jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Scheduled', attempt: '—',
|
|
438
|
+
next_action: score, at: job['created_at'], args: job['args'], error_class: nil, error_message: nil,
|
|
439
|
+
raw: raw, source: :scheduled }
|
|
440
|
+
end
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
def retrying_rows
|
|
444
|
+
entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::RETRY, 0, -1, withscores: true) }
|
|
445
|
+
entries.map do |raw, score|
|
|
446
|
+
job = JSON.parse(raw)
|
|
447
|
+
{ jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Retrying',
|
|
448
|
+
attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: score,
|
|
449
|
+
at: job['failed_at'] || job['created_at'], args: job['args'], error_class: job['error_class'],
|
|
450
|
+
error_message: job['error_message'], raw: raw, source: :retrying }
|
|
451
|
+
end
|
|
452
|
+
end
|
|
453
|
+
|
|
454
|
+
def dead_rows
|
|
455
|
+
entries = Cogworker.config.redis { |c| c.zrevrange(RedisKeys::DEAD, 0, -1, withscores: true) }
|
|
456
|
+
entries.map do |raw, score|
|
|
457
|
+
job = JSON.parse(raw)
|
|
458
|
+
{ jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Dead',
|
|
459
|
+
attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: nil, at: score,
|
|
460
|
+
args: job['args'], error_class: job['error_class'], error_message: job['error_message'], raw: raw,
|
|
461
|
+
source: :dead }
|
|
462
|
+
end
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
Cogworker::Web.register(Cogworker::Web::Routes::Jobs)
|