wurk 1.12.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/config/routes.rb +8 -0
- data/exe/sidekiq +8 -0
- data/exe/wurk +1 -1
- data/lib/generators/wurk/install/install_generator.rb +56 -0
- data/lib/wurk/boot_checks.rb +80 -0
- data/lib/wurk/cli.rb +55 -17
- data/lib/wurk/configuration.rb +27 -6
- data/lib/wurk/fetcher/orphan_reclaim.rb +48 -11
- data/lib/wurk/{middleware → fetcher}/poison_pill.rb +3 -1
- data/lib/wurk/fetcher/reaper.rb +28 -9
- data/lib/wurk/fetcher/reliable.rb +4 -5
- data/lib/wurk/fetcher/unit_of_work.rb +1 -1
- data/lib/wurk/job_logger.rb +4 -2
- data/lib/wurk/job_util.rb +14 -0
- data/lib/wurk/launcher.rb +30 -3
- data/lib/wurk/leader.rb +1 -8
- data/lib/wurk/manager.rb +27 -1
- data/lib/wurk/middleware/timeout.rb +6 -6
- data/lib/wurk/preflight/app_scan.rb +182 -0
- data/lib/wurk/preflight/command.rb +21 -5
- data/lib/wurk/preflight/redis_guard.rb +98 -0
- data/lib/wurk/preflight.rb +9 -5
- data/lib/wurk/rails_boot.rb +7 -2
- data/lib/wurk/redis_options.rb +5 -0
- data/lib/wurk/redis_pool.rb +24 -1
- data/lib/wurk/sentry.rb +11 -0
- data/lib/wurk/swarm.rb +63 -14
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/rack_app.rb +8 -6
- data/lib/wurk/web/stats_routes.rb +93 -0
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +11 -2
data/lib/wurk/sentry.rb
CHANGED
|
@@ -28,9 +28,20 @@ module Wurk
|
|
|
28
28
|
#
|
|
29
29
|
# See docs/sentry.md.
|
|
30
30
|
module Sentry
|
|
31
|
+
# Sentry matches an excluded name by ancestry, so Handled also covers
|
|
32
|
+
# `JobRetry::Skip`, its subclass.
|
|
31
33
|
CONTROL_FLOW_ERRORS = %w[Wurk::JobRetry::Handled Wurk::Shutdown].freeze
|
|
32
34
|
|
|
33
35
|
class << self
|
|
36
|
+
# The retry machinery's own signals, for a Sentry setup that does not run
|
|
37
|
+
# {install!} (the web process, a `Sentry.init` that runs first):
|
|
38
|
+
#
|
|
39
|
+
# config.excluded_exceptions += Wurk::Sentry.excluded_exceptions
|
|
40
|
+
#
|
|
41
|
+
# A `Rails.error` subscriber sees these cross the reloader on every failed
|
|
42
|
+
# attempt, as it does under Sidekiq.
|
|
43
|
+
def excluded_exceptions = CONTROL_FLOW_ERRORS
|
|
44
|
+
|
|
34
45
|
# Registers the server middleware and the error handler. Idempotent:
|
|
35
46
|
# `Chain#add` dedupes by class, and a previously-installed
|
|
36
47
|
# {ErrorHandler} is replaced rather than stacked, so calling this twice
|
data/lib/wurk/swarm.rb
CHANGED
|
@@ -6,6 +6,8 @@ require_relative 'component'
|
|
|
6
6
|
require_relative 'heartbeat'
|
|
7
7
|
require_relative 'launcher'
|
|
8
8
|
require_relative 'fetcher/reliable'
|
|
9
|
+
require_relative 'fetcher/reaper'
|
|
10
|
+
require_relative 'capsule'
|
|
9
11
|
require_relative 'keys'
|
|
10
12
|
require_relative 'swarm/child_boot'
|
|
11
13
|
require_relative 'swarm/backoff'
|
|
@@ -152,8 +154,10 @@ module Wurk
|
|
|
152
154
|
@crash_watch = CrashWatch.new(exit_after: crash_exit_after, young_within: Backoff::RESET_AFTER)
|
|
153
155
|
@exit_on_crash_loop = exit_on_crash_loop
|
|
154
156
|
@crash_looped = false
|
|
157
|
+
@title_program = nil
|
|
155
158
|
@restart = build_restart
|
|
156
159
|
@liveness = build_liveness(heartbeat_timeout)
|
|
160
|
+
@orphan_reaper = Fetcher::Reaper.new(config)
|
|
157
161
|
init_fork_unsafe_handles
|
|
158
162
|
end
|
|
159
163
|
|
|
@@ -188,10 +192,12 @@ module Wurk
|
|
|
188
192
|
|
|
189
193
|
@assignments = @topology.assignments.freeze
|
|
190
194
|
@owner_pid = ::Process.pid
|
|
195
|
+
@title_program = ::File.basename($PROGRAM_NAME) if install_signals
|
|
191
196
|
install_signal_handlers if install_signals
|
|
192
197
|
close_parent_sockets
|
|
193
198
|
@config.prepare_for_fork!
|
|
194
199
|
fork_children
|
|
200
|
+
update_proctitle
|
|
195
201
|
child_pids
|
|
196
202
|
end
|
|
197
203
|
|
|
@@ -295,10 +301,25 @@ module Wurk
|
|
|
295
301
|
check_crash_loop unless @stopping
|
|
296
302
|
requiet_children
|
|
297
303
|
check_memory_pressure
|
|
304
|
+
update_proctitle
|
|
298
305
|
rescue StandardError => e
|
|
299
306
|
handle_exception(e, { context: 'swarm-supervise' })
|
|
300
307
|
end
|
|
301
308
|
|
|
309
|
+
# A swarm that owns its signals owns its process, so it titles it after the
|
|
310
|
+
# binary that started it: `wurkswarm myapp [3 of 3 processes]`, or
|
|
311
|
+
# `sidekiqswarm …`. docs/target/ does not give Enterprise's parent title, so
|
|
312
|
+
# this one is Wurk's own. One co-hosted in a web server leaves the host's
|
|
313
|
+
# title alone.
|
|
314
|
+
# Children replace the inherited title on their first beat.
|
|
315
|
+
def update_proctitle
|
|
316
|
+
return unless @title_program
|
|
317
|
+
|
|
318
|
+
title = "#{@title_program} #{@config[:tag] || default_tag} " \
|
|
319
|
+
"[#{child_pids.size} of #{@assignments.size} processes]"
|
|
320
|
+
$PROGRAM_NAME = title unless title == $PROGRAM_NAME
|
|
321
|
+
end
|
|
322
|
+
|
|
302
323
|
def advance_restart
|
|
303
324
|
@lock.synchronize { @restart.advance }
|
|
304
325
|
end
|
|
@@ -543,19 +564,45 @@ module Wurk
|
|
|
543
564
|
end
|
|
544
565
|
|
|
545
566
|
def on_child_exit(pid, status)
|
|
546
|
-
@lock.synchronize
|
|
547
|
-
|
|
548
|
-
|
|
567
|
+
meta = @lock.synchronize { retire_child(pid, status) }
|
|
568
|
+
reclaim_child_lists(pid, meta[:slot]) if meta && !redis_outage?(status)
|
|
569
|
+
end
|
|
549
570
|
|
|
550
|
-
|
|
551
|
-
|
|
571
|
+
def retire_child(pid, status)
|
|
572
|
+
meta = @children.delete(pid)
|
|
573
|
+
return unless meta
|
|
552
574
|
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
575
|
+
@liveness&.forget(pid)
|
|
576
|
+
return meta if @restart.claim_exit(pid)
|
|
577
|
+
|
|
578
|
+
if @stopping
|
|
579
|
+
logger.info { "swarm: child #{pid} exited (status=#{status&.exitstatus})" }
|
|
580
|
+
else
|
|
581
|
+
schedule_respawn(pid, status, meta)
|
|
558
582
|
end
|
|
583
|
+
meta
|
|
584
|
+
end
|
|
585
|
+
|
|
586
|
+
# A reaped child holds nothing any more: a drained one emptied its
|
|
587
|
+
# private lists itself and costs one pipelined LLEN here, a SIGKILLed or
|
|
588
|
+
# crashed one has its in-flight jobs back on their queues now rather than
|
|
589
|
+
# when a sibling's sweep comes round. Outside the lock, since it talks to
|
|
590
|
+
# Redis. The sockets it opens close before the next fork (#fork_child).
|
|
591
|
+
# A child that exited for want of Redis never fetched, and is skipped. A
|
|
592
|
+
# failure here leaves the lists to the periodic sweep, as before.
|
|
593
|
+
def reclaim_child_lists(pid, slot)
|
|
594
|
+
moved = @orphan_reaper.reclaim_exited(pid, child_queue_names(slot))
|
|
595
|
+
logger.info { "swarm: returned #{moved} job(s) held by child #{pid} to their queues" } if moved.positive?
|
|
596
|
+
rescue StandardError => e
|
|
597
|
+
handle_exception(e, { context: 'swarm-reclaim' })
|
|
598
|
+
end
|
|
599
|
+
|
|
600
|
+
# What ChildBoot#apply_slot_to_config makes of the default capsule, parsed
|
|
601
|
+
# the same way, plus every other capsule as configured.
|
|
602
|
+
def child_queue_names(slot)
|
|
603
|
+
slot_capsule = Capsule.new('default', @config).tap { |c| c.queues = slot.queues }
|
|
604
|
+
others = @config.capsules.except('default').values
|
|
605
|
+
[slot_capsule, *others].flat_map(&:queues).uniq
|
|
559
606
|
end
|
|
560
607
|
|
|
561
608
|
# Arm the slot's backoff instead of sleeping the supervise thread; the next
|
|
@@ -705,18 +752,19 @@ module Wurk
|
|
|
705
752
|
# child table is never held across a sleep.
|
|
706
753
|
def hard_kill_stragglers
|
|
707
754
|
killed = @lock.synchronize do
|
|
708
|
-
|
|
709
|
-
|
|
755
|
+
stragglers = @children.dup
|
|
756
|
+
stragglers.each_key { |pid| safe_kill(pid, 'KILL') }
|
|
710
757
|
@children.clear
|
|
711
|
-
|
|
758
|
+
stragglers
|
|
712
759
|
end
|
|
713
|
-
reap_killed(killed)
|
|
760
|
+
reap_killed(killed.keys).each { |pid| reclaim_child_lists(pid, killed[pid][:slot]) }
|
|
714
761
|
end
|
|
715
762
|
|
|
716
763
|
# Bounded and best-effort: SIGKILL is asynchronous, so poll rather than
|
|
717
764
|
# block on a pid that may be wedged in uninterruptible sleep. A straggler
|
|
718
765
|
# still unreaped when the budget runs out is dropped exactly as before —
|
|
719
766
|
# the drain must not stall on it.
|
|
767
|
+
# Returns the pids it reaped.
|
|
720
768
|
def reap_killed(pids)
|
|
721
769
|
pending = pids
|
|
722
770
|
HARD_KILL_REAP_ATTEMPTS.times do
|
|
@@ -725,6 +773,7 @@ module Wurk
|
|
|
725
773
|
|
|
726
774
|
sleep HARD_KILL_REAP_INTERVAL
|
|
727
775
|
end
|
|
776
|
+
pids - pending
|
|
728
777
|
end
|
|
729
778
|
|
|
730
779
|
def reaped?(pid)
|
data/lib/wurk/version.rb
CHANGED
data/lib/wurk/web/rack_app.rb
CHANGED
|
@@ -2,17 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative 'extension'
|
|
4
4
|
require_relative 'pool_scope'
|
|
5
|
+
require_relative 'stats_routes'
|
|
5
6
|
|
|
6
7
|
module Wurk
|
|
7
8
|
class Web
|
|
8
9
|
# Upstream-compatible class-level Rack surface (#204): Sidekiq apps do
|
|
9
10
|
# `run Sidekiq::Web`, `mount Sidekiq::Web => "/sidekiq"`, and rack-test
|
|
10
11
|
# ecosystem suites call `Sidekiq::Web.call(env)` directly. This entry serves
|
|
11
|
-
# the
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
12
|
+
# the machine-readable routes (StatsRoutes) and the registered third-party
|
|
13
|
+
# Web extensions (#187's renderer) under their own route paths. Everything
|
|
14
|
+
# else is the dashboard, which is the engine's: in a booted Rails app the
|
|
15
|
+
# request goes there, through the engine's own copy of the gates below, so
|
|
16
|
+
# the stock install line shows the dashboard. Without an engine those paths
|
|
17
|
+
# 404 rather than half-serve the SPA.
|
|
16
18
|
#
|
|
17
19
|
# Same CSRF model as upstream Sidekiq 8 and ExtensionsController (spec
|
|
18
20
|
# §25.1): unsafe methods must carry `Sec-Fetch-Site: same-origin`.
|
|
@@ -54,7 +56,7 @@ module Wurk
|
|
|
54
56
|
def dispatch(env)
|
|
55
57
|
return [403, NOT_FOUND_HEADERS.dup, ['Forbidden']] unless safe_request?(env)
|
|
56
58
|
|
|
57
|
-
dispatch_to_extensions(env)
|
|
59
|
+
StatsRoutes.serve(env) || dispatch_to_extensions(env)
|
|
58
60
|
end
|
|
59
61
|
|
|
60
62
|
def safe_request?(env)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'rack/request'
|
|
5
|
+
require_relative 'pool_scope'
|
|
6
|
+
|
|
7
|
+
module Wurk
|
|
8
|
+
class Web
|
|
9
|
+
# Sidekiq Web's machine-readable routes (spec §25.4): `HEAD /` (a Redis
|
|
10
|
+
# round trip, for uptime checks), `GET /stats`, `GET /stats/queues`, and
|
|
11
|
+
# `GET /dashboard/stats` redirecting to `/stats`. Uptime probes and scripts
|
|
12
|
+
# written against `/sidekiq/stats` read these shapes, so they are
|
|
13
|
+
# upstream's, not the SPA's `/api/stats`.
|
|
14
|
+
#
|
|
15
|
+
# Plain Rack, so it serves both the engine (config/routes.rb points these
|
|
16
|
+
# paths here, behind the engine's gates) and the Rails-free
|
|
17
|
+
# `run Sidekiq::Web` (Web.call dispatches here, behind the same gates).
|
|
18
|
+
module StatsRoutes
|
|
19
|
+
REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human].freeze
|
|
20
|
+
READ_METHODS = %w[GET HEAD].freeze
|
|
21
|
+
JSON_HEADERS = { 'content-type' => 'application/json', 'cache-control' => 'private, no-store' }.freeze
|
|
22
|
+
|
|
23
|
+
class << self
|
|
24
|
+
def call(env)
|
|
25
|
+
serve(env) || [404, { 'content-type' => 'text/plain' }, ['Not Found']]
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# The response for a path this module owns, or nil.
|
|
29
|
+
def serve(env)
|
|
30
|
+
method = env['REQUEST_METHOD']
|
|
31
|
+
return unless READ_METHODS.include?(method)
|
|
32
|
+
|
|
33
|
+
path = env['PATH_INFO'].to_s
|
|
34
|
+
return head_root if method == 'HEAD' && ['', '/'].include?(path)
|
|
35
|
+
|
|
36
|
+
case path
|
|
37
|
+
when '/stats' then json(stats)
|
|
38
|
+
when '/stats/queues' then json(PoolScope.scope { Wurk::Stats.new.queues })
|
|
39
|
+
when '/dashboard/stats' then redirect(env)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
# Errors propagate: a probe must see a failure when Redis is down.
|
|
46
|
+
def head_root
|
|
47
|
+
PoolScope.scope { Wurk.redis(idempotent: true) { |conn| conn.call('LLEN', 'queue:default') } }
|
|
48
|
+
[200, { 'content-type' => 'text/html' }, []]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def stats
|
|
52
|
+
PoolScope.scope do
|
|
53
|
+
s = Wurk::Stats.new
|
|
54
|
+
{
|
|
55
|
+
sidekiq: {
|
|
56
|
+
processed: s.processed, failed: s.failed, busy: s.workers_size,
|
|
57
|
+
processes: s.processes_size, enqueued: s.enqueued, scheduled: s.scheduled_size,
|
|
58
|
+
retries: s.retry_size, dead: s.dead_size, default_latency: s.default_queue_latency
|
|
59
|
+
},
|
|
60
|
+
redis: redis_stats,
|
|
61
|
+
server_utc_time: ::Time.now.utc.strftime('%H:%M:%S UTC')
|
|
62
|
+
}
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def redis_stats
|
|
67
|
+
info = Wurk.redis_pool.info
|
|
68
|
+
info.slice(*REDIS_KEYS).merge('store_name' => store_name(info), 'store_version' => store_version(info))
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def store_name(info)
|
|
72
|
+
return 'Dragonfly' if info.key?('dragonfly_version')
|
|
73
|
+
return 'Valkey' if info.key?('valkey_version')
|
|
74
|
+
|
|
75
|
+
'Redis'
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def store_version(info)
|
|
79
|
+
info['dragonfly_version'] || info['valkey_version'] || info['redis_version']
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def json(payload)
|
|
83
|
+
[200, JSON_HEADERS.dup, [::JSON.generate(payload)]]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def redirect(env)
|
|
87
|
+
request = ::Rack::Request.new(env)
|
|
88
|
+
[302, { 'location' => "#{request.base_url}#{env['SCRIPT_NAME']}/stats" }, []]
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
data/lib/wurk.rb
CHANGED
|
@@ -315,7 +315,7 @@ require_relative 'wurk/health'
|
|
|
315
315
|
|
|
316
316
|
# Poison-pill detection — orphan recovery counter + dead set on threshold.
|
|
317
317
|
# Spec: docs/target/sidekiq-pro.md §3.2.
|
|
318
|
-
require_relative 'wurk/
|
|
318
|
+
require_relative 'wurk/fetcher/poison_pill'
|
|
319
319
|
|
|
320
320
|
# InterruptHandler self-prepends to the head of the server chain so a
|
|
321
321
|
# cooperatively-cancelled job (IterableJob) is re-pushed + cleanly skipped
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wurk
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.13.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- developerz.ai
|
|
@@ -130,6 +130,7 @@ description: 'Wire-compatible Ruby background job processor: same Redis schema,
|
|
|
130
130
|
email:
|
|
131
131
|
- admin@developerz.ai
|
|
132
132
|
executables:
|
|
133
|
+
- sidekiq
|
|
133
134
|
- sidekiqswarm
|
|
134
135
|
- wurk
|
|
135
136
|
- wurkswarm
|
|
@@ -149,6 +150,7 @@ files:
|
|
|
149
150
|
- app/controllers/wurk/extensions_controller.rb
|
|
150
151
|
- app/controllers/wurk/profiles_controller.rb
|
|
151
152
|
- config/routes.rb
|
|
153
|
+
- exe/sidekiq
|
|
152
154
|
- exe/sidekiqswarm
|
|
153
155
|
- exe/wurk
|
|
154
156
|
- exe/wurkswarm
|
|
@@ -263,6 +265,7 @@ files:
|
|
|
263
265
|
- lib/wurk/fetcher/oom_drain.rb
|
|
264
266
|
- lib/wurk/fetcher/orphan_grace.rb
|
|
265
267
|
- lib/wurk/fetcher/orphan_reclaim.rb
|
|
268
|
+
- lib/wurk/fetcher/poison_pill.rb
|
|
266
269
|
- lib/wurk/fetcher/private_list_key.rb
|
|
267
270
|
- lib/wurk/fetcher/reaper.rb
|
|
268
271
|
- lib/wurk/fetcher/reliable.rb
|
|
@@ -362,18 +365,19 @@ files:
|
|
|
362
365
|
- lib/wurk/middleware/expiry.rb
|
|
363
366
|
- lib/wurk/middleware/i18n.rb
|
|
364
367
|
- lib/wurk/middleware/interrupt_handler.rb
|
|
365
|
-
- lib/wurk/middleware/poison_pill.rb
|
|
366
368
|
- lib/wurk/middleware/status.rb
|
|
367
369
|
- lib/wurk/middleware/timeout.rb
|
|
368
370
|
- lib/wurk/pid_cache.rb
|
|
369
371
|
- lib/wurk/pool_checkout.rb
|
|
370
372
|
- lib/wurk/preflight.rb
|
|
373
|
+
- lib/wurk/preflight/app_scan.rb
|
|
371
374
|
- lib/wurk/preflight/checks.rb
|
|
372
375
|
- lib/wurk/preflight/command.rb
|
|
373
376
|
- lib/wurk/preflight/keyspace.rb
|
|
374
377
|
- lib/wurk/preflight/owners.rb
|
|
375
378
|
- lib/wurk/preflight/payload_sample.rb
|
|
376
379
|
- lib/wurk/preflight/reader.rb
|
|
380
|
+
- lib/wurk/preflight/redis_guard.rb
|
|
377
381
|
- lib/wurk/preflight/report.rb
|
|
378
382
|
- lib/wurk/preflight/verify.rb
|
|
379
383
|
- lib/wurk/process_set.rb
|
|
@@ -439,6 +443,7 @@ files:
|
|
|
439
443
|
- lib/wurk/web/pool_scope.rb
|
|
440
444
|
- lib/wurk/web/rack_app.rb
|
|
441
445
|
- lib/wurk/web/search.rb
|
|
446
|
+
- lib/wurk/web/stats_routes.rb
|
|
442
447
|
- lib/wurk/web/views/_job_info.erb
|
|
443
448
|
- lib/wurk/web/views/_paging.erb
|
|
444
449
|
- lib/wurk/work_set.rb
|
|
@@ -512,6 +517,10 @@ metadata:
|
|
|
512
517
|
changelog_uri: https://github.com/developerz-ai/wurk/blob/main/CHANGELOG.md
|
|
513
518
|
bug_tracker_uri: https://github.com/developerz-ai/wurk/issues
|
|
514
519
|
rubygems_mfa_required: 'true'
|
|
520
|
+
post_install_message: |
|
|
521
|
+
wurk: `bundle exec sidekiq` / `sidekiqswarm` keep working; `wurk` / `wurkswarm` are the same commands.
|
|
522
|
+
Keeping a sidekiq-* add-on gem? Bundle the pinned shim: docs/migrate-from-sidekiq.md in the wurk repository.
|
|
523
|
+
Before a cutover from Sidekiq, run `bundle exec wurk preflight` against production Redis (read-only).
|
|
515
524
|
rdoc_options: []
|
|
516
525
|
require_paths:
|
|
517
526
|
- lib
|