wurk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +43 -0
- data/CONTRIBUTING.md +73 -0
- data/LICENSE +21 -0
- data/README.md +137 -0
- data/SECURITY.md +39 -0
- data/app/controllers/wurk/api/pagination.rb +67 -0
- data/app/controllers/wurk/api/serializers.rb +131 -0
- data/app/controllers/wurk/api_controller.rb +248 -0
- data/app/controllers/wurk/application_controller.rb +7 -0
- data/app/controllers/wurk/dashboard_controller.rb +48 -0
- data/config/locales/en.yml +15 -0
- data/config/routes.rb +34 -0
- data/exe/wurk +22 -0
- data/lib/active_job/queue_adapters/wurk_adapter.rb +96 -0
- data/lib/generators/wurk/install/install_generator.rb +22 -0
- data/lib/generators/wurk/install/templates/wurk.rb +16 -0
- data/lib/wurk/active_job/wrapper.rb +32 -0
- data/lib/wurk/api/fast.rb +78 -0
- data/lib/wurk/batch/buffer.rb +26 -0
- data/lib/wurk/batch/callback_job.rb +37 -0
- data/lib/wurk/batch/callbacks.rb +176 -0
- data/lib/wurk/batch/client_middleware.rb +27 -0
- data/lib/wurk/batch/death_handler.rb +39 -0
- data/lib/wurk/batch/empty.rb +21 -0
- data/lib/wurk/batch/server_middleware.rb +62 -0
- data/lib/wurk/batch/status.rb +140 -0
- data/lib/wurk/batch.rb +351 -0
- data/lib/wurk/batch_set.rb +67 -0
- data/lib/wurk/capsule.rb +176 -0
- data/lib/wurk/cli.rb +349 -0
- data/lib/wurk/client/buffered.rb +372 -0
- data/lib/wurk/client.rb +330 -0
- data/lib/wurk/compat.rb +136 -0
- data/lib/wurk/component.rb +136 -0
- data/lib/wurk/configuration.rb +373 -0
- data/lib/wurk/context.rb +35 -0
- data/lib/wurk/cron.rb +636 -0
- data/lib/wurk/dashboard_manifest.rb +39 -0
- data/lib/wurk/dead_set.rb +78 -0
- data/lib/wurk/deploy.rb +91 -0
- data/lib/wurk/embedded.rb +94 -0
- data/lib/wurk/encryption.rb +276 -0
- data/lib/wurk/engine.rb +81 -0
- data/lib/wurk/fetcher/reaper.rb +264 -0
- data/lib/wurk/fetcher/reliable.rb +138 -0
- data/lib/wurk/fetcher.rb +11 -0
- data/lib/wurk/health.rb +193 -0
- data/lib/wurk/heartbeat.rb +211 -0
- data/lib/wurk/iterable_job.rb +292 -0
- data/lib/wurk/job/options.rb +70 -0
- data/lib/wurk/job.rb +33 -0
- data/lib/wurk/job_logger.rb +68 -0
- data/lib/wurk/job_record.rb +156 -0
- data/lib/wurk/job_retry.rb +320 -0
- data/lib/wurk/job_set.rb +212 -0
- data/lib/wurk/job_util.rb +162 -0
- data/lib/wurk/keys.rb +52 -0
- data/lib/wurk/launcher.rb +289 -0
- data/lib/wurk/leader.rb +221 -0
- data/lib/wurk/limiter/base.rb +138 -0
- data/lib/wurk/limiter/bucket.rb +80 -0
- data/lib/wurk/limiter/concurrent.rb +132 -0
- data/lib/wurk/limiter/leaky.rb +91 -0
- data/lib/wurk/limiter/points.rb +89 -0
- data/lib/wurk/limiter/server_middleware.rb +77 -0
- data/lib/wurk/limiter/unlimited.rb +48 -0
- data/lib/wurk/limiter/window.rb +80 -0
- data/lib/wurk/limiter.rb +255 -0
- data/lib/wurk/logger.rb +81 -0
- data/lib/wurk/lua/loader.rb +53 -0
- data/lib/wurk/lua.rb +187 -0
- data/lib/wurk/manager.rb +132 -0
- data/lib/wurk/metrics/history.rb +151 -0
- data/lib/wurk/metrics/query.rb +173 -0
- data/lib/wurk/metrics/rollup.rb +169 -0
- data/lib/wurk/metrics/statsd.rb +197 -0
- data/lib/wurk/metrics.rb +7 -0
- data/lib/wurk/middleware/chain.rb +128 -0
- data/lib/wurk/middleware/current_attributes.rb +87 -0
- data/lib/wurk/middleware/expiry.rb +50 -0
- data/lib/wurk/middleware/i18n.rb +63 -0
- data/lib/wurk/middleware/interrupt_handler.rb +45 -0
- data/lib/wurk/middleware/poison_pill.rb +149 -0
- data/lib/wurk/middleware.rb +34 -0
- data/lib/wurk/process_set.rb +243 -0
- data/lib/wurk/processor.rb +247 -0
- data/lib/wurk/queue.rb +108 -0
- data/lib/wurk/queues.rb +80 -0
- data/lib/wurk/rails.rb +9 -0
- data/lib/wurk/railtie.rb +28 -0
- data/lib/wurk/redis_pool.rb +79 -0
- data/lib/wurk/retry_set.rb +17 -0
- data/lib/wurk/scheduled.rb +189 -0
- data/lib/wurk/scheduled_set.rb +18 -0
- data/lib/wurk/sorted_entry.rb +95 -0
- data/lib/wurk/stats.rb +190 -0
- data/lib/wurk/swarm/child_boot.rb +105 -0
- data/lib/wurk/swarm.rb +260 -0
- data/lib/wurk/testing.rb +102 -0
- data/lib/wurk/topology.rb +74 -0
- data/lib/wurk/unique.rb +240 -0
- data/lib/wurk/version.rb +5 -0
- data/lib/wurk/web/config.rb +180 -0
- data/lib/wurk/web/enterprise.rb +138 -0
- data/lib/wurk/web/search.rb +139 -0
- data/lib/wurk/web.rb +25 -0
- data/lib/wurk/work_set.rb +116 -0
- data/lib/wurk/worker/setter.rb +93 -0
- data/lib/wurk/worker.rb +216 -0
- data/lib/wurk.rb +238 -0
- data/vendor/assets/dashboard/assets/index-8P3N_m1X.js +152 -0
- data/vendor/assets/dashboard/assets/index-Bqz4_SOQ.css +1 -0
- data/vendor/assets/dashboard/index.html +13 -0
- data/vendor/assets/dashboard/wurk-manifest.json +4 -0
- metadata +232 -0
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'api/serializers'
|
|
4
|
+
require_relative 'api/pagination'
|
|
5
|
+
require 'wurk/web'
|
|
6
|
+
|
|
7
|
+
module Wurk
|
|
8
|
+
# JSON APIs consumed by the React SPA. Action methods stay thin; mapping to
|
|
9
|
+
# the wire shape lives in `Wurk::Api::Serializers`, and pagination lives in
|
|
10
|
+
# `Wurk::Api::Pagination`. SSE lives in #stream.
|
|
11
|
+
#
|
|
12
|
+
# Wire-compat: every payload field reads from the canonical Wurk inspector
|
|
13
|
+
# objects (Stats, Queue, RetrySet, ScheduledSet, DeadSet, ProcessSet,
|
|
14
|
+
# BatchSet, Cron::LoopSet) so dashboards stay aligned with the Redis schema
|
|
15
|
+
# in `docs/target/sidekiq-{free,pro,ent}.md`.
|
|
16
|
+
class ApiController < ApplicationController
|
|
17
|
+
include ActionController::Live
|
|
18
|
+
|
|
19
|
+
STREAM_TICK_SECONDS = 2.0
|
|
20
|
+
STREAM_MAX_DURATION = 600.0
|
|
21
|
+
|
|
22
|
+
HISTORY_WINDOW_UNITS = { 's' => 1, 'm' => 60, 'h' => 3600, 'd' => 86_400 }.freeze
|
|
23
|
+
DEFAULT_HISTORY_WINDOW = 24 * 3600
|
|
24
|
+
|
|
25
|
+
skip_forgery_protection only: %i[
|
|
26
|
+
stream reset_limiter pause_cron unpause_cron enqueue_cron
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
# Boot-time flags the SPA reads once to shape the UI (e.g. hide destructive
|
|
30
|
+
# actions and show the read-only banner). Always a GET, so it stays
|
|
31
|
+
# reachable while read-only mode blocks mutations.
|
|
32
|
+
def meta
|
|
33
|
+
render json: { read_only: ::Wurk::Web.config.read_only? }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def stats
|
|
37
|
+
render json: ::Wurk::Api::Serializers.stats_payload(::Wurk::Stats.new)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def queues
|
|
41
|
+
render json: ::Wurk::Stats.new.queue_summaries.map { |q| ::Wurk::Api::Serializers.queue_summary(q) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def queue
|
|
45
|
+
q = ::Wurk::Queue.new(params[:name].to_s)
|
|
46
|
+
page = ::Wurk::Api::Pagination.window(params)
|
|
47
|
+
jobs = ::Wurk::Api::Pagination.slice(q, page) { |rec| ::Wurk::Api::Serializers.job_record(rec) }
|
|
48
|
+
render json: {
|
|
49
|
+
name: q.name, size: q.size, latency: q.latency, paused: q.paused?,
|
|
50
|
+
page: page[:page], count: page[:count], jobs: jobs
|
|
51
|
+
}
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def retries = render_sorted_set(::Wurk::RetrySet.new)
|
|
55
|
+
def scheduled = render_sorted_set(::Wurk::ScheduledSet.new)
|
|
56
|
+
def dead = render_sorted_set(::Wurk::DeadSet.new)
|
|
57
|
+
|
|
58
|
+
def processes
|
|
59
|
+
render json: ::Wurk::ProcessSet.new.map { |p| ::Wurk::Api::Serializers.process_row(p) }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def batches
|
|
63
|
+
set = ::Wurk::BatchSet.new
|
|
64
|
+
page = ::Wurk::Api::Pagination.window(params)
|
|
65
|
+
rows = ::Wurk::Api::Pagination.slice(set, page) { |status| status.data.transform_keys(&:to_sym) }
|
|
66
|
+
render json: { total: set.size, page: page[:page], count: page[:count], batches: rows }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def batch
|
|
70
|
+
status = ::Wurk::Batch::Status.new(params[:bid].to_s)
|
|
71
|
+
return render(json: { error: 'unknown batch' }, status: :not_found) unless status.exists?
|
|
72
|
+
|
|
73
|
+
render json: status.data.transform_keys(&:to_sym)
|
|
74
|
+
rescue ::ArgumentError
|
|
75
|
+
render json: { error: 'unknown batch' }, status: :not_found
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def limiters
|
|
79
|
+
names = ::Wurk::Web::Enterprise::Limits.list(filter: params[:substr])
|
|
80
|
+
page = ::Wurk::Api::Pagination.window(params)
|
|
81
|
+
render json: { total: names.size, page: page[:page], count: page[:count], limiters: limiter_rows(names, page) }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def reset_limiter
|
|
85
|
+
::Wurk::Web::Enterprise::Limits.reset(params[:name].to_s)
|
|
86
|
+
render json: { ok: true }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def cron
|
|
90
|
+
now = ::Time.now.to_i
|
|
91
|
+
render json: ::Wurk::Cron::LoopSet.new.map { |lp| ::Wurk::Api::Serializers.cron_row(lp, now) }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def pause_cron = render_cron_action(::Wurk::Web::Enterprise::Periodic.pause(params[:lid].to_s))
|
|
95
|
+
def unpause_cron = render_cron_action(::Wurk::Web::Enterprise::Periodic.unpause(params[:lid].to_s))
|
|
96
|
+
|
|
97
|
+
def enqueue_cron
|
|
98
|
+
jid = ::Wurk::Web::Enterprise::Periodic.enqueue_now(params[:lid].to_s)
|
|
99
|
+
return render(json: { error: 'unknown loop' }, status: :not_found) if jid.nil?
|
|
100
|
+
|
|
101
|
+
render json: { ok: true, jid: jid }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def cron_history
|
|
105
|
+
render json: { lid: params[:lid].to_s, history: ::Wurk::Web::Enterprise::Periodic.history(params[:lid].to_s) }
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def metrics
|
|
109
|
+
minutes = ::Wurk::Api::Pagination.clamp_int(params[:minutes], 1, ::Wurk::Metrics::Query::MAX_MINUTES, 60)
|
|
110
|
+
rows = ::Wurk::Web::Enterprise::Historical.top(minutes: minutes, class_filter: params[:substr])
|
|
111
|
+
render json: { minutes: minutes, top_jobs: rows.map { |(klass, totals)| ::Wurk::Api::Serializers.metric_row(klass, totals) } }
|
|
112
|
+
rescue ::Wurk::Metrics::Query::WindowTooWide => e
|
|
113
|
+
render json: { error: e.message }, status: :bad_request
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def metrics_for_job
|
|
117
|
+
klass = params[:klass].to_s
|
|
118
|
+
minutes, hours = metrics_window(params)
|
|
119
|
+
rows = ::Wurk::Web::Enterprise::Historical.for_job(klass, minutes: minutes, hours: hours)
|
|
120
|
+
series = rows.map { |row| row.merge(at: row[:at].to_f) }
|
|
121
|
+
render json: { klass: klass, minutes: minutes, hours: hours, series: series }
|
|
122
|
+
rescue ::ArgumentError, ::Wurk::Metrics::Query::WindowTooWide => e
|
|
123
|
+
render json: { error: e.message }, status: :bad_request
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Cluster-total throughput/failures time-series for the dashboard charts.
|
|
127
|
+
# `:bucket` is 1m/5m/1h; `?window=24h` (s/m/h/d suffix) is clamped to the
|
|
128
|
+
# bucket's retention. Recharts-ready array under `series`.
|
|
129
|
+
def history
|
|
130
|
+
window = parse_window(params[:window])
|
|
131
|
+
series = ::Wurk::Web::Enterprise::Historical.history(params[:bucket].to_s, window: window)
|
|
132
|
+
render json: { bucket: params[:bucket].to_s, window: window, series: series.map { |row| ::Wurk::Api::Serializers.history_point(row) } }
|
|
133
|
+
rescue ::ArgumentError => e
|
|
134
|
+
render json: { error: e.message }, status: :bad_request
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def search
|
|
138
|
+
substr = params[:substr].to_s
|
|
139
|
+
return render(json: { substr: substr, total: 0, hits: [] }) if substr.empty?
|
|
140
|
+
|
|
141
|
+
hits = ::Wurk::Web::Search.new(substr, kinds: parse_search_kinds(params), limit: parse_search_limit(params)).to_a
|
|
142
|
+
render json: { substr: substr, total: hits.size, hits: hits }
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# SSE: one `event: stats` per tick with a fresh Stats snapshot. Caps at
|
|
146
|
+
# `STREAM_MAX_DURATION` so a stale browser tab can't tie a Rails worker
|
|
147
|
+
# forever — the client reconnects automatically when the stream closes.
|
|
148
|
+
#
|
|
149
|
+
# `?max_duration=` and `?tick=` are test/debug knobs; the SPA never sets
|
|
150
|
+
# them. `?max_duration=0` emits one tick and closes.
|
|
151
|
+
def stream
|
|
152
|
+
stream_headers!
|
|
153
|
+
clamp = ::Wurk::Api::Pagination.method(:clamp_float)
|
|
154
|
+
tick = clamp.call(params[:tick], 0.0, STREAM_TICK_SECONDS, STREAM_TICK_SECONDS)
|
|
155
|
+
max_dur = clamp.call(params[:max_duration], 0.0, STREAM_MAX_DURATION, STREAM_MAX_DURATION)
|
|
156
|
+
sse = ::ActionController::Live::SSE.new(response.stream, retry: (STREAM_TICK_SECONDS * 1000).to_i)
|
|
157
|
+
drive_stream(sse, tick, max_dur)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
def render_sorted_set(set)
|
|
163
|
+
page = ::Wurk::Api::Pagination.window(params)
|
|
164
|
+
total = set.size
|
|
165
|
+
entries = ::Wurk::Api::Pagination.slice(set, page) { |entry| ::Wurk::Api::Serializers.sorted_entry(entry) }
|
|
166
|
+
render json: { total: total, page: page[:page], count: page[:count], entries: entries }
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# substr is already applied by Limits.list (matches on name), so slice the
|
|
170
|
+
# filtered names directly — only the page's rows pay the per-limiter Redis
|
|
171
|
+
# reads in limiter_row (which folds in live status).
|
|
172
|
+
def limiter_rows(names, page)
|
|
173
|
+
(names.slice(page[:page] * page[:count], page[:count]) || []).map do |name|
|
|
174
|
+
::Wurk::Api::Serializers.limiter_row(name, limiter_meta(name))
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def limiter_meta(name)
|
|
179
|
+
raw = ::Wurk.redis { |c| c.call('HGETALL', "lmtr:#{name}") }
|
|
180
|
+
raw.is_a?(Array) ? raw.each_slice(2).to_h : raw
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def stream_headers!
|
|
184
|
+
response.headers['Content-Type'] = 'text/event-stream'
|
|
185
|
+
response.headers['Cache-Control'] = 'no-cache'
|
|
186
|
+
response.headers['X-Accel-Buffering'] = 'no'
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def drive_stream(sse, tick, max_dur)
|
|
190
|
+
deadline = monotime + max_dur
|
|
191
|
+
loop do
|
|
192
|
+
sse.write(stream_tick_payload, event: 'stats')
|
|
193
|
+
break if monotime >= deadline
|
|
194
|
+
|
|
195
|
+
sleep tick
|
|
196
|
+
end
|
|
197
|
+
rescue ::IOError, ::ActionController::Live::ClientDisconnected
|
|
198
|
+
# client closed; nothing to clean up.
|
|
199
|
+
ensure
|
|
200
|
+
sse.close
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def monotime
|
|
204
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def stream_tick_payload
|
|
208
|
+
::Wurk::Api::Serializers.stats_payload(::Wurk::Stats.new).merge(at: ::Time.now.to_f)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def render_cron_action(success)
|
|
212
|
+
return render(json: { error: 'unknown loop' }, status: :not_found) unless success
|
|
213
|
+
|
|
214
|
+
render json: { ok: true }
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def parse_search_kinds(params)
|
|
218
|
+
params[:kinds].is_a?(::Array) ? params[:kinds] : params[:kinds].to_s.split(',')
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def parse_search_limit(params)
|
|
222
|
+
::Wurk::Api::Pagination.clamp_int(
|
|
223
|
+
params[:limit], 1, ::Wurk::Web::Search::MAX_LIMIT, ::Wurk::Web::Search::DEFAULT_LIMIT
|
|
224
|
+
)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Resolves the per-class metrics window. `minutes:` wins when present;
|
|
228
|
+
# `hours:` is used otherwise; default falls back to 60 minutes so callers
|
|
229
|
+
# that pass neither still get a useful series.
|
|
230
|
+
# `?window=24h` → seconds. Accepts an s/m/h/d suffix (bare number = seconds).
|
|
231
|
+
# Falls back to 24h on a missing or unparseable value; the Query layer
|
|
232
|
+
# clamps the result to the bucket's retention.
|
|
233
|
+
def parse_window(raw)
|
|
234
|
+
match = raw.to_s.strip.downcase.match(/\A(\d+)([smhd]?)\z/)
|
|
235
|
+
return DEFAULT_HISTORY_WINDOW unless match
|
|
236
|
+
|
|
237
|
+
Integer(match[1]) * HISTORY_WINDOW_UNITS.fetch(match[2].empty? ? 's' : match[2])
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def metrics_window(params)
|
|
241
|
+
pagination = ::Wurk::Api::Pagination
|
|
242
|
+
minutes = pagination.clamp_int(params[:minutes], 1, ::Wurk::Metrics::Query::MAX_MINUTES, 60) if params[:minutes]
|
|
243
|
+
hours = pagination.clamp_int(params[:hours], 1, ::Wurk::Metrics::Query::MAX_HOURS, 24) if params[:hours]
|
|
244
|
+
minutes ||= 60 if hours.nil?
|
|
245
|
+
[minutes, hours]
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'net/http'
|
|
4
|
+
require 'uri'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# Serves the SPA shell. Everything else is JSON from ApiController.
|
|
8
|
+
#
|
|
9
|
+
# Production: returns the precompiled index.html shipped in
|
|
10
|
+
# vendor/assets/dashboard. The release pipeline (`frontend:build` →
|
|
11
|
+
# `gem build`) writes both that file and the wurk-manifest.json validated
|
|
12
|
+
# at engine boot.
|
|
13
|
+
#
|
|
14
|
+
# Development: when WURK_VITE_DEV=1 is set, fetches the shell from the
|
|
15
|
+
# Vite dev server (default :5173) so contributors get HMR without
|
|
16
|
+
# rebuilding the bundle on every change.
|
|
17
|
+
class DashboardController < ApplicationController
|
|
18
|
+
VITE_DEV_URL = 'http://localhost:5173/'
|
|
19
|
+
INDEX_REL_PATH = ['vendor', 'assets', 'dashboard', 'index.html'].freeze
|
|
20
|
+
|
|
21
|
+
def index
|
|
22
|
+
render layout: false, html: spa_html.html_safe
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
def spa_html
|
|
28
|
+
ENV['WURK_VITE_DEV'] == '1' ? fetch_vite_dev_shell : read_built_index
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def fetch_vite_dev_shell
|
|
32
|
+
uri = ::URI.parse(VITE_DEV_URL)
|
|
33
|
+
::Net::HTTP.get(uri)
|
|
34
|
+
rescue ::StandardError => e
|
|
35
|
+
raise "Wurk dashboard: cannot reach Vite dev server at #{VITE_DEV_URL} " \
|
|
36
|
+
"(#{e.class}: #{e.message}). Run `bin/rake frontend:dev` from the gem root."
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def read_built_index
|
|
40
|
+
path = ::Wurk::Engine.root.join(*INDEX_REL_PATH)
|
|
41
|
+
unless path.exist?
|
|
42
|
+
raise "Wurk dashboard: precompiled SPA index.html missing at #{path}. " \
|
|
43
|
+
'Run `bin/rake frontend:build`.'
|
|
44
|
+
end
|
|
45
|
+
path.read
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
en:
|
|
2
|
+
wurk:
|
|
3
|
+
dashboard:
|
|
4
|
+
title: "Wurk"
|
|
5
|
+
nav:
|
|
6
|
+
overview: "Overview"
|
|
7
|
+
queues: "Queues"
|
|
8
|
+
retries: "Retries"
|
|
9
|
+
scheduled: "Scheduled"
|
|
10
|
+
dead: "Dead"
|
|
11
|
+
processes: "Processes"
|
|
12
|
+
batches: "Batches"
|
|
13
|
+
limiters: "Limiters"
|
|
14
|
+
cron: "Cron"
|
|
15
|
+
metrics: "Metrics"
|
data/config/routes.rb
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Wurk::Engine.routes.draw do
|
|
4
|
+
root to: 'dashboard#index'
|
|
5
|
+
|
|
6
|
+
# JSON APIs consumed by the SPA. Nested under whatever mount the host chose.
|
|
7
|
+
scope :api, defaults: { format: :json } do
|
|
8
|
+
get 'meta', to: 'api#meta'
|
|
9
|
+
get 'stats', to: 'api#stats'
|
|
10
|
+
get 'queues', to: 'api#queues'
|
|
11
|
+
get 'queues/:name', to: 'api#queue', as: :api_queue
|
|
12
|
+
get 'retries', to: 'api#retries'
|
|
13
|
+
get 'scheduled', to: 'api#scheduled'
|
|
14
|
+
get 'dead', to: 'api#dead'
|
|
15
|
+
get 'processes', to: 'api#processes'
|
|
16
|
+
get 'batches', to: 'api#batches'
|
|
17
|
+
get 'batches/:bid', to: 'api#batch', as: :api_batch
|
|
18
|
+
get 'limiters', to: 'api#limiters'
|
|
19
|
+
post 'limiters/:name/reset', to: 'api#reset_limiter', as: :api_reset_limiter
|
|
20
|
+
get 'cron', to: 'api#cron'
|
|
21
|
+
post 'cron/:lid/pause', to: 'api#pause_cron', as: :api_pause_cron
|
|
22
|
+
post 'cron/:lid/unpause', to: 'api#unpause_cron', as: :api_unpause_cron
|
|
23
|
+
post 'cron/:lid/enqueue', to: 'api#enqueue_cron', as: :api_enqueue_cron
|
|
24
|
+
get 'cron/:lid/history', to: 'api#cron_history', as: :api_cron_history
|
|
25
|
+
get 'metrics', to: 'api#metrics'
|
|
26
|
+
get 'metrics/:klass', to: 'api#metrics_for_job', as: :api_metrics_for_job, constraints: { klass: %r{[^/]+} }
|
|
27
|
+
get 'history/:bucket', to: 'api#history', as: :api_history
|
|
28
|
+
get 'search', to: 'api#search'
|
|
29
|
+
get 'stream', to: 'api#stream' # SSE
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# SPA catch-all — let React Router handle the rest.
|
|
33
|
+
get '*path', to: 'dashboard#index', constraints: ->(req) { req.format == :html }
|
|
34
|
+
end
|
data/exe/wurk
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# Standalone runner. Does NOT load the Rails engine.
|
|
5
|
+
# Usage: `exe/wurk -C config/wurk.yml`
|
|
6
|
+
|
|
7
|
+
$stdout.sync = true
|
|
8
|
+
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
|
9
|
+
|
|
10
|
+
require 'wurk'
|
|
11
|
+
|
|
12
|
+
begin
|
|
13
|
+
cli = Wurk::CLI.instance
|
|
14
|
+
cli.parse
|
|
15
|
+
cli.run
|
|
16
|
+
rescue StandardError => e
|
|
17
|
+
raise e if $DEBUG
|
|
18
|
+
|
|
19
|
+
warn "wurk: #{e.message}"
|
|
20
|
+
warn e.backtrace.join("\n")
|
|
21
|
+
exit 1
|
|
22
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# ActiveJob adapter. Rails resolves `:wurk` to this class via the
|
|
4
|
+
# QueueAdapters lookup convention (`const_get("WurkAdapter")`). The file is
|
|
5
|
+
# loaded eagerly by lib/wurk/engine.rb so the constant is defined before any
|
|
6
|
+
# host app sets `config.active_job.queue_adapter = :wurk`.
|
|
7
|
+
#
|
|
8
|
+
# Spec: docs/target/sidekiq-free.md §28 (ActiveJob integration).
|
|
9
|
+
|
|
10
|
+
begin
|
|
11
|
+
gem 'activejob', '>= 7.0'
|
|
12
|
+
require 'active_job'
|
|
13
|
+
require_relative '../../wurk/active_job/wrapper'
|
|
14
|
+
|
|
15
|
+
ActiveSupport.on_load(:active_job) do
|
|
16
|
+
# Lets native AJs configure Wurk options directly — `sidekiq_options retry: 3`
|
|
17
|
+
# etc. Skip when already included (e.g. Sidekiq loaded alongside in a mixed
|
|
18
|
+
# codebase mid-migration).
|
|
19
|
+
include Wurk::Job::Options unless respond_to?(:sidekiq_options)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
module ActiveJob
|
|
23
|
+
module QueueAdapters
|
|
24
|
+
# Older Rails ships a built-in placeholder; remove before defining.
|
|
25
|
+
remove_const(:WurkAdapter) if const_defined?(:WurkAdapter, false)
|
|
26
|
+
|
|
27
|
+
parent = const_defined?(:AbstractAdapter) ? AbstractAdapter : Object
|
|
28
|
+
class WurkAdapter < parent
|
|
29
|
+
# Class var (not class ivar) so subclasses share the same stop flag —
|
|
30
|
+
# matches Sidekiq exactly, which third-party gems read directly.
|
|
31
|
+
@@stopping = false # rubocop:disable Style/ClassVars
|
|
32
|
+
|
|
33
|
+
callback = -> { @@stopping = true } # rubocop:disable Style/ClassVars
|
|
34
|
+
|
|
35
|
+
Wurk.configure_client { |config| config.on(:quiet, &callback) }
|
|
36
|
+
Wurk.configure_server { |config| config.on(:quiet, &callback) }
|
|
37
|
+
|
|
38
|
+
# Defer enqueue until the surrounding DB transaction commits so we
|
|
39
|
+
# never push a job whose row hasn't landed. Matches Sidekiq.
|
|
40
|
+
def enqueue_after_transaction_commit?
|
|
41
|
+
true
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def enqueue(job)
|
|
45
|
+
wrapper = Sidekiq::ActiveJob::Wrapper.set(
|
|
46
|
+
wrapped: job.class,
|
|
47
|
+
queue: job.queue_name
|
|
48
|
+
)
|
|
49
|
+
job.provider_job_id = wrapper.perform_async(job.serialize)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def enqueue_at(job, timestamp)
|
|
53
|
+
job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
|
|
54
|
+
wrapped: job.class,
|
|
55
|
+
queue: job.queue_name
|
|
56
|
+
).perform_at(timestamp, job.serialize)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def enqueue_all(jobs)
|
|
60
|
+
jobs.group_by(&:class).sum { |job_class, group| push_grouped_by_queue(job_class, group) }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def stopping? = @@stopping
|
|
64
|
+
|
|
65
|
+
# Backwards-compat alias. Sidekiq exposes this name for jobs enqueued
|
|
66
|
+
# by very old Rails versions that wrote the older constant in payloads.
|
|
67
|
+
JobWrapper = Sidekiq::ActiveJob::Wrapper
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def push_grouped_by_queue(job_class, group)
|
|
72
|
+
group.group_by(&:queue_name).sum do |queue, same_queue|
|
|
73
|
+
immediate, scheduled = same_queue.partition { |j| j.scheduled_at.nil? }
|
|
74
|
+
count = 0
|
|
75
|
+
count += bulk_push(job_class, queue, immediate, scheduled: false) if immediate.any?
|
|
76
|
+
count += bulk_push(job_class, queue, scheduled, scheduled: true) if scheduled.any?
|
|
77
|
+
count
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def bulk_push(job_class, queue, jobs, scheduled:)
|
|
82
|
+
items = {
|
|
83
|
+
'class' => Sidekiq::ActiveJob::Wrapper,
|
|
84
|
+
'wrapped' => job_class,
|
|
85
|
+
'queue' => queue,
|
|
86
|
+
'args' => jobs.map { |j| [j.serialize] }
|
|
87
|
+
}
|
|
88
|
+
items['at'] = jobs.map { |j| j.scheduled_at&.to_f } if scheduled
|
|
89
|
+
Sidekiq::Client.push_bulk(items).compact.size
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
rescue Gem::LoadError
|
|
95
|
+
# ActiveJob not present — adapter unavailable, which matches Sidekiq behavior.
|
|
96
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
module Generators
|
|
7
|
+
# `bin/rails g wurk:install`
|
|
8
|
+
# Writes a config initializer and adds a commented-out mount line to routes.
|
|
9
|
+
# The host app picks the mount path; the generator suggests /wurk.
|
|
10
|
+
class InstallGenerator < ::Rails::Generators::Base
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
def copy_initializer
|
|
14
|
+
template "wurk.rb", "config/initializers/wurk.rb"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def insert_mount_line
|
|
18
|
+
route(%(# mount Wurk::Engine => "/wurk" # uncomment to expose the Wurk dashboard))
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Wurk configuration. Generated by `bin/rails g wurk:install`.
|
|
4
|
+
# See docs/target/sidekiq-free.md for the full surface area.
|
|
5
|
+
|
|
6
|
+
Wurk.configure_server do |config|
|
|
7
|
+
# config.redis = { url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0") }
|
|
8
|
+
# config.workers = 2
|
|
9
|
+
# config.concurrency = 10
|
|
10
|
+
# config.queues = %w[critical default low]
|
|
11
|
+
# config.shutdown_timeout = 25
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
Wurk.configure_client do |config|
|
|
15
|
+
# config.redis = { url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0") }
|
|
16
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../job'
|
|
4
|
+
|
|
5
|
+
# Defined under the `Sidekiq::ActiveJob` namespace (not `Wurk::ActiveJob`) so
|
|
6
|
+
# the canonical `class` string written to Redis stays `"Sidekiq::ActiveJob::Wrapper"`
|
|
7
|
+
# — the exact wire shape Sidekiq emits. Mixed Sidekiq/Wurk worker pools can
|
|
8
|
+
# read the same payloads either way. Wire-compat is sacred.
|
|
9
|
+
#
|
|
10
|
+
# `Wurk::ActiveJob::Wrapper` and `ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper`
|
|
11
|
+
# both resolve here so legacy enqueued payloads load on the gem swap.
|
|
12
|
+
#
|
|
13
|
+
# Spec: docs/target/sidekiq-free.md §28.
|
|
14
|
+
module Sidekiq
|
|
15
|
+
module ActiveJob
|
|
16
|
+
class Wrapper
|
|
17
|
+
include Wurk::Job
|
|
18
|
+
|
|
19
|
+
def perform(job_data)
|
|
20
|
+
::ActiveJob::Base.execute(job_data.merge('provider_job_id' => jid))
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Wurk-namespaced alias kept tight against the Sidekiq definition above; the
|
|
27
|
+
# extra module is a pure constant rebind, not a second class definition.
|
|
28
|
+
module Wurk # rubocop:disable Style/OneClassPerFile
|
|
29
|
+
module ActiveJob
|
|
30
|
+
Wrapper = ::Sidekiq::ActiveJob::Wrapper
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../lua'
|
|
4
|
+
require_relative '../job_record'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
module API
|
|
8
|
+
# Pro parity (§11): Lua-backed O(1)-round-trip replacements for the
|
|
9
|
+
# ruby-side LRANGE / ZSCAN loops in Queue and SortedSet. Mixed into the
|
|
10
|
+
# existing data API classes so the surface is `Sidekiq::Queue#delete_job(jid)`,
|
|
11
|
+
# `Sidekiq::DeadSet#scan(pattern) { |JobRecord| … }` etc. — wire-compat
|
|
12
|
+
# with Pro consumer code that drops in on a one-line require swap.
|
|
13
|
+
#
|
|
14
|
+
# We don't reimplement `Queue#size` (already LLEN, unchanged per spec).
|
|
15
|
+
module Fast
|
|
16
|
+
# Extension to Wurk::Queue. Pure server-side delete by jid / class — no
|
|
17
|
+
# network round-trips per match. Returns the count of payloads removed.
|
|
18
|
+
module QueueExt
|
|
19
|
+
# @return [Integer] payloads removed (0 when jid absent; >0 only in
|
|
20
|
+
# the corner case of duplicate-jid corruption).
|
|
21
|
+
def delete_job(jid)
|
|
22
|
+
raise ArgumentError, 'jid required' if jid.nil? || jid.to_s.empty?
|
|
23
|
+
|
|
24
|
+
Wurk.redis do |conn|
|
|
25
|
+
Wurk::Lua::Loader.eval_cached(
|
|
26
|
+
conn,
|
|
27
|
+
:fast_delete_job,
|
|
28
|
+
keys: [Keys.queue(name)],
|
|
29
|
+
argv: [jid.to_s]
|
|
30
|
+
)
|
|
31
|
+
end.to_i
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# @param klass [Class, String, Symbol]
|
|
35
|
+
# @return [Integer] payloads removed.
|
|
36
|
+
def delete_by_class(klass)
|
|
37
|
+
klass_name = klass.is_a?(Class) ? klass.name : klass.to_s
|
|
38
|
+
raise ArgumentError, 'class name required' if klass_name.empty?
|
|
39
|
+
|
|
40
|
+
Wurk.redis do |conn|
|
|
41
|
+
Wurk::Lua::Loader.eval_cached(
|
|
42
|
+
conn,
|
|
43
|
+
:fast_delete_by_class,
|
|
44
|
+
keys: [Keys.queue(name)],
|
|
45
|
+
argv: [klass_name]
|
|
46
|
+
)
|
|
47
|
+
end.to_i
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Extension to Wurk::SortedSet / JobSet. The base `scan(match, count)`
|
|
52
|
+
# already yields `(value, score)` pairs via ZSCAN — Pro promotes the
|
|
53
|
+
# surface to yield `JobRecord` directly so callers can `entry.delete`
|
|
54
|
+
# / `entry.retry` without re-parsing JSON.
|
|
55
|
+
#
|
|
56
|
+
# The new behavior is enabled by *block arity*: a 1-arg block (or
|
|
57
|
+
# block.lambda? + a 1-param signature) receives `JobRecord`; legacy
|
|
58
|
+
# 2-arg blocks (`|value, score|`) keep the raw shape. This preserves
|
|
59
|
+
# the existing two-arg contract used by JobSet#find_job internally.
|
|
60
|
+
module SortedSetExt
|
|
61
|
+
def scan(match, count = 100, &block)
|
|
62
|
+
return enum_for(:scan, match, count) unless block
|
|
63
|
+
|
|
64
|
+
if block.arity == 1
|
|
65
|
+
super do |value, score|
|
|
66
|
+
block.call(Wurk::SortedEntry.new(self, score, value))
|
|
67
|
+
end
|
|
68
|
+
else
|
|
69
|
+
super
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
Wurk::Queue.include(Wurk::API::Fast::QueueExt)
|
|
78
|
+
Wurk::SortedSet.prepend(Wurk::API::Fast::SortedSetExt)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
class Batch
|
|
5
|
+
# Accumulates batched payloads inside an autoflush `Batch#jobs` block so
|
|
6
|
+
# the whole block flushes in one pipeline (`autoflush == true`) or every
|
|
7
|
+
# N jobs (`autoflush == Integer`). Client#raw_push fills it; Batch#jobs
|
|
8
|
+
# drains the remainder at block exit.
|
|
9
|
+
Buffer = Struct.new(:payloads, :threshold) do
|
|
10
|
+
def add(items)
|
|
11
|
+
payloads.concat(items)
|
|
12
|
+
self
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def ready?
|
|
16
|
+
!threshold.nil? && payloads.length >= threshold
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def drain
|
|
20
|
+
out = payloads.dup
|
|
21
|
+
payloads.clear
|
|
22
|
+
out
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|