wurk 1.9.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/app/controllers/wurk/api_controller.rb +4 -1
  3. data/lib/wurk/boot_checks.rb +87 -0
  4. data/lib/wurk/capsule.rb +33 -8
  5. data/lib/wurk/cli.rb +73 -8
  6. data/lib/wurk/client/buffered.rb +240 -26
  7. data/lib/wurk/compat.rb +11 -0
  8. data/lib/wurk/configuration.rb +114 -16
  9. data/lib/wurk/cron.rb +45 -0
  10. data/lib/wurk/debounce.rb +2 -2
  11. data/lib/wurk/error_coalescer.rb +35 -0
  12. data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
  13. data/lib/wurk/fetcher/capped.rb +4 -0
  14. data/lib/wurk/fetcher/many_queues.rb +17 -1
  15. data/lib/wurk/fetcher/oom_drain.rb +102 -0
  16. data/lib/wurk/fetcher/orphan_grace.rb +60 -44
  17. data/lib/wurk/fetcher/private_list_key.rb +1 -16
  18. data/lib/wurk/fetcher/reaper.rb +169 -127
  19. data/lib/wurk/fetcher/reliable.rb +26 -4
  20. data/lib/wurk/fetcher/self_reconcile.rb +175 -0
  21. data/lib/wurk/fetcher/unit_of_work.rb +32 -8
  22. data/lib/wurk/health.rb +23 -4
  23. data/lib/wurk/health_probe.rb +64 -0
  24. data/lib/wurk/heartbeat.rb +55 -3
  25. data/lib/wurk/job/options.rb +9 -1
  26. data/lib/wurk/job.rb +2 -24
  27. data/lib/wurk/job_record.rb +17 -9
  28. data/lib/wurk/job_retry.rb +38 -10
  29. data/lib/wurk/job_set.rb +13 -5
  30. data/lib/wurk/launcher.rb +134 -52
  31. data/lib/wurk/leader.rb +9 -3
  32. data/lib/wurk/limiter/base.rb +17 -1
  33. data/lib/wurk/lua/debounce.lua +11 -6
  34. data/lib/wurk/lua/loader.rb +13 -1
  35. data/lib/wurk/lua.rb +71 -8
  36. data/lib/wurk/manager.rb +1 -1
  37. data/lib/wurk/metrics/process_counters.rb +27 -0
  38. data/lib/wurk/metrics/prometheus.rb +35 -14
  39. data/lib/wurk/middleware/current_attributes.rb +26 -10
  40. data/lib/wurk/middleware/poison_pill.rb +1 -2
  41. data/lib/wurk/process_set.rb +3 -1
  42. data/lib/wurk/processor.rb +82 -18
  43. data/lib/wurk/queue_consumers.rb +134 -0
  44. data/lib/wurk/rails_boot.rb +40 -2
  45. data/lib/wurk/rails_reloader.rb +51 -0
  46. data/lib/wurk/railtie.rb +20 -1
  47. data/lib/wurk/redact.rb +12 -0
  48. data/lib/wurk/redis_pool.rb +34 -9
  49. data/lib/wurk/scheduled.rb +142 -67
  50. data/lib/wurk/sentry/death_handler.rb +29 -0
  51. data/lib/wurk/sentry/error_handler.rb +52 -10
  52. data/lib/wurk/sentry/middleware.rb +8 -1
  53. data/lib/wurk/sentry/retry_policy.rb +12 -3
  54. data/lib/wurk/sentry.rb +42 -5
  55. data/lib/wurk/sorted_entry.rb +38 -13
  56. data/lib/wurk/swarm/child_boot.rb +42 -31
  57. data/lib/wurk/swarm/crash_watch.rb +65 -0
  58. data/lib/wurk/swarm.rb +79 -11
  59. data/lib/wurk/testing.rb +66 -0
  60. data/lib/wurk/version.rb +1 -1
  61. data/lib/wurk/web/extension.rb +6 -1
  62. data/lib/wurk/work_set.rb +6 -0
  63. data/lib/wurk/worker/setter.rb +11 -6
  64. data/lib/wurk/worker.rb +17 -1
  65. data/lib/wurk.rb +1 -1
  66. data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
  67. data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
  68. data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
  69. data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
  70. data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
  71. data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
  72. data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
  73. data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
  74. data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
  75. data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
  76. data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
  77. data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
  78. data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
  79. data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
  80. data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
  81. data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
  82. data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
  83. data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
  84. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
  85. data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
  86. data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
  87. data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
  88. data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
  89. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
  90. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
  91. data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
  92. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
  93. data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
  94. data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
  95. data/vendor/assets/dashboard/index.html +3 -3
  96. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  97. metadata +41 -30
  98. data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
  99. data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
  100. data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
@@ -0,0 +1,134 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require_relative 'component'
5
+ require_relative 'keys'
6
+ require_relative 'timer_loop'
7
+
8
+ module Wurk
9
+ # Who listens to which queue, read from what every process already publishes:
10
+ # the `queues` list in its heartbeat `info`. That list is the union across a
11
+ # process's capsules, so a queue served by one capsule counts. Expired
12
+ # identities have no `info` and are not counted. No new Redis keys.
13
+ module QueueConsumers
14
+ module_function
15
+
16
+ # `{ queue => live subscribers }`. A quiet process still counts: it is a
17
+ # deliberate pause, not a queue nobody thought to list.
18
+ def counts(conn)
19
+ ids = conn.call('SMEMBERS', Keys::PROCESSES)
20
+ return {} if ids.empty?
21
+
22
+ tally(conn.pipelined { |pipe| ids.each { |id| pipe.call('HGET', id, 'info') } })
23
+ end
24
+
25
+ def tally(infos)
26
+ infos.each_with_object(Hash.new(0)) do |info, out|
27
+ subscribed(info).each { |queue| out[queue] += 1 }
28
+ end
29
+ end
30
+
31
+ def subscribed(info)
32
+ return [] if info.nil?
33
+
34
+ Array(::JSON.parse(info)['queues']).map(&:to_s).uniq
35
+ rescue ::JSON::ParserError, ::TypeError
36
+ []
37
+ end
38
+
39
+ # Warns about non-empty queues no live process subscribes to: such jobs wait
40
+ # forever with no error, retry or report. Leader only, one tick a minute.
41
+ #
42
+ # A queue is reported only after `grace` seconds unconsumed on every tick. A
43
+ # rolling restart or a deploy that adds a queue passes through that state
44
+ # briefly, and one leader's in-memory clock is enough to ride it out. A
45
+ # leader change restarts the clock, which only delays a warning.
46
+ class Watch
47
+ include Component
48
+
49
+ TICK = 60
50
+ GRACE = 300
51
+ THROTTLE = 3600
52
+
53
+ attr_reader :config
54
+
55
+ def initialize(config, tick: TICK, grace: GRACE, throttle: THROTTLE,
56
+ clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
57
+ @config = config
58
+ @grace = grace
59
+ @throttle = throttle
60
+ @clock = clock
61
+ @timer = TimerLoop.new(tick)
62
+ @unconsumed_since = {}
63
+ @warned_at = {}
64
+ @thread = nil
65
+ end
66
+
67
+ def start
68
+ return @thread if @thread
69
+
70
+ @timer.reset
71
+ @thread = safe_thread('queue-consumers') { @timer.run { tick } }
72
+ end
73
+
74
+ def terminate
75
+ @timer.terminate
76
+ @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
77
+ end
78
+
79
+ def tick
80
+ return unless leader?
81
+
82
+ check
83
+ rescue StandardError => e
84
+ handle_exception(e, { context: 'queue-consumers' })
85
+ end
86
+
87
+ # One pass, bypassing the leader gate. Returns the queues warned about.
88
+ def check
89
+ now = @clock.call
90
+ unconsumed = read_unconsumed
91
+ @unconsumed_since.select! { |queue, _| unconsumed.key?(queue) }
92
+ unconsumed.filter_map do |queue, size|
93
+ @unconsumed_since[queue] ||= now
94
+ next unless warn_due?(queue, now)
95
+
96
+ @warned_at[queue] = now
97
+ logger.warn { warning(queue, size) }
98
+ queue
99
+ end
100
+ end
101
+
102
+ private
103
+
104
+ def warn_due?(queue, now)
105
+ return false if now - @unconsumed_since[queue] < @grace
106
+
107
+ at = @warned_at[queue]
108
+ at.nil? || now - at >= @throttle
109
+ end
110
+
111
+ def warning(queue, size)
112
+ "Queue #{queue.inspect} has #{size} jobs and no live process subscribes to it, so they will wait " \
113
+ 'forever. Add it to the worker\'s queues (or drain it if it is obsolete).'
114
+ end
115
+
116
+ def read_unconsumed
117
+ @config.redis(idempotent: true) do |conn|
118
+ counts = QueueConsumers.counts(conn)
119
+ # An empty fleet is a different alarm, and one every monitor already has.
120
+ next {} if counts.empty?
121
+
122
+ non_empty(conn, conn.call('SMEMBERS', Keys::QUEUES_SET).reject { |queue| counts.key?(queue) })
123
+ end
124
+ end
125
+
126
+ def non_empty(conn, names)
127
+ return {} if names.empty?
128
+
129
+ sizes = conn.pipelined { |pipe| names.each { |queue| pipe.call('LLEN', Keys.queue(queue)) } }
130
+ names.zip(sizes).select { |_, size| size.positive? }.to_h
131
+ end
132
+ end
133
+ end
134
+ end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'rails_reloader'
4
+
3
5
  module Wurk
4
6
  # Coordinates how Wurk boots inside a Rails host: whether this process should
5
7
  # boot at all, whether it may fork the swarm, and the boot itself. The Railtie
@@ -13,6 +15,8 @@ module Wurk
13
15
  # the final reap.
14
16
  DRAIN_JOIN_SLACK = 1
15
17
 
18
+ BOOT_INITIALIZER = :wurk_boot
19
+
16
20
  module_function
17
21
 
18
22
  # Invoked from the `wurk.server_mode` initializer, before config/initializers
@@ -34,7 +38,27 @@ module Wurk
34
38
  Wurk.configuration[:client_in_server] = true
35
39
  end
36
40
 
37
- # Invoked from after_initialize, once the host app has fully initialized.
41
+ # A host that already replaced the reloader keeps its own. A config already
42
+ # frozen, by workers embedded from an initializer, can't take it: their
43
+ # Processors resolved the default themselves (Processor#resolve_reloader).
44
+ def install_reloader(app = ::Rails.application, config = Wurk.configuration)
45
+ return if config.frozen?
46
+ return unless config[:reloader].equal?(Configuration::DEFAULTS[:reloader])
47
+
48
+ config[:reloader] = RailsReloader.new(app)
49
+ end
50
+
51
+ # Set by the railtie's first initializer. Rails builds the initializer list
52
+ # once, so if that one ran the Finisher entry is in the list too. A railtie
53
+ # required from a host's own initializer missed it, and the after_initialize
54
+ # hook boots instead.
55
+ def boot_scheduled? = @boot_scheduled == true
56
+
57
+ def boot_scheduled=(value)
58
+ @boot_scheduled = value
59
+ end
60
+
61
+ # Invoked last of all initializers, once the host app has fully initialized.
38
62
  def boot(app = ::Rails.application)
39
63
  return if skip_boot?
40
64
 
@@ -156,7 +180,9 @@ module Wurk
156
180
  false
157
181
  end
158
182
 
159
- def boot_swarm
183
+ def boot_swarm # rubocop:disable Metrics/AbcSize
184
+ warn_on_production_fork
185
+ Health.options_from_env
160
186
  timeout = Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
161
187
  swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology, shutdown_timeout: timeout)
162
188
  supervisor = nil
@@ -181,6 +207,18 @@ module Wurk
181
207
  end
182
208
  end
183
209
 
210
+ # A deliberate default, but a surprise after a gem swap: stock Sidekiq never
211
+ # runs jobs in the web process, and a separate worker on top of this runs
212
+ # everything against two pools. WURK_EMBED=1 says it is intended.
213
+ def warn_on_production_fork
214
+ return unless ::Rails.env.production? && ENV['WURK_EMBED'] != '1'
215
+
216
+ logger.warn do
217
+ 'wurk: this web process is forking a worker swarm. If workers run elsewhere, set WURK_DISABLED=1 here; ' \
218
+ 'if this is intended, set WURK_EMBED=1 to silence this.'
219
+ end
220
+ end
221
+
184
222
  # at_exit fires on the host's main thread, but the supervise thread owns the
185
223
  # child table and two threads inside `shutdown` race on it — so request the
186
224
  # drain and wait for the supervisor to run it. Draining here is the fallback
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ # Runs each job inside the host's `Rails.application.reloader`, whose executor
5
+ # hooks enable the query cache, reset CurrentAttributes, release the thread's
6
+ # Active Record connection and, in development, reload changed code.
7
+ #
8
+ # Installed as `config[:reloader]` by RailsBoot when the railtie is loaded.
9
+ # `wurk` / `wurkswarm` load the host app with no railtie (see the note at the
10
+ # end of lib/wurk.rb), so Processor also asks `.default` for itself.
11
+ class RailsReloader
12
+ # The source Rails' error reporter tags these failures with. Subscribers
13
+ # filter on the name, so it stays the one a Sidekiq app already matches.
14
+ SOURCE = 'job.sidekiq'
15
+
16
+ # @return [RailsReloader, nil] nil unless this process has a Rails
17
+ # application, which is what keeps a plain Ruby worker free of any per-job
18
+ # wrapping.
19
+ def self.default
20
+ return unless defined?(::Rails) && ::Rails.respond_to?(:application)
21
+
22
+ app = ::Rails.application
23
+ new(app) if app
24
+ end
25
+
26
+ def initialize(app)
27
+ @app = app
28
+ # Rails 7.0's reloader takes no keywords.
29
+ takes_source = app.reloader.method(:wrap).parameters.any? { |kind, _| %i[key keyreq keyrest].include?(kind) }
30
+ @wrap_options = takes_source ? { source: SOURCE }.freeze : {}.freeze
31
+ # A host can start embedded workers from an initializer, while the app is
32
+ # still booting and its reloader hooks aren't installed. Those jobs run bare
33
+ # until boot reaches after_initialize. Not `app.initialized?`: a swarm child
34
+ # forked during boot never returns to where Rails would set it.
35
+ @ready = false
36
+ ::ActiveSupport.on_load(:after_initialize, yield: true) { @ready = true }
37
+ end
38
+
39
+ def call(&)
40
+ return yield unless @ready
41
+
42
+ @app.reloader.wrap(**@wrap_options, &)
43
+ end
44
+
45
+ # The default `as_json` walks instance variables, and the Application's never
46
+ # end. Configuration holds this object, so serializing that would overflow too.
47
+ def as_json(*) = {}
48
+
49
+ def inspect = "#<#{self.class.name} app=#{@app.class.name}>"
50
+ end
51
+ end
data/lib/wurk/railtie.rb CHANGED
@@ -28,11 +28,30 @@ module Wurk
28
28
  # (`configure_server` / `server? is false`). RailsBoot decides whether this
29
29
  # process serves.
30
30
  initializer 'wurk.server_mode', before: :load_config_initializers do |app|
31
+ Wurk::RailsBoot.boot_scheduled = true
31
32
  Wurk::RailsBoot.enter_server_mode_if_serving(app)
32
33
  end
33
34
 
35
+ # Registered at gem load, so it runs ahead of the host's own hooks: one of
36
+ # them can still replace or wrap the reloader.
34
37
  config.after_initialize do |app|
35
- Wurk::RailsBoot.boot(app)
38
+ Wurk::RailsBoot.install_reloader(app)
39
+ Wurk::RailsBoot.boot(app) unless Wurk::RailsBoot.boot_scheduled?
40
+ end
41
+
42
+ # Appended to Rails' own Finisher so it runs dead last. A boot here sees what
43
+ # a host's `after_initialize` configured (a Redis URL, say), and a forked
44
+ # child has the routes and dev-reload hooks Rails installs after those hooks
45
+ # fire, which it could never pick up later. `after:` can't express this: it
46
+ # drags the whole Finisher chain ahead of initializers not yet run.
47
+ #
48
+ # Deferred to before_configuration because naming Rails::Application at load
49
+ # would pull ActionDispatch into a host loading only rails/railtie.
50
+ config.before_configuration do
51
+ finisher = ::Rails::Application::Finisher
52
+ unless finisher.initializers.any? { |i| i.name == Wurk::RailsBoot::BOOT_INITIALIZER }
53
+ finisher.initializer(Wurk::RailsBoot::BOOT_INITIALIZER) { |app| Wurk::RailsBoot.boot(app) }
54
+ end
36
55
  end
37
56
  end
38
57
  end
data/lib/wurk/redact.rb CHANGED
@@ -21,6 +21,7 @@ module Wurk
21
21
  # Fail closed: a hook that raises must not fall back to showing the args it
22
22
  # was installed to hide.
23
23
  FAILED = ['[REDACTED: redact_args raised]'].freeze
24
+ UNPARSEABLE = '[REDACTED: unparseable job]'
24
25
 
25
26
  module_function
26
27
 
@@ -28,6 +29,17 @@ module Wurk
28
29
  config.redact_args
29
30
  end
30
31
 
32
+ # A raw payload string, with its args replaced. Anything that does not parse to
33
+ # a job, or to JSON at all, could be holding the args in any shape: it is dropped.
34
+ def jobstr(raw, hook)
35
+ job = ::JSON.parse(raw)
36
+ return UNPARSEABLE unless job.is_a?(::Hash)
37
+
38
+ ::JSON.generate(job.merge('args' => args(job, hook)))
39
+ rescue ::JSON::ParserError, ::EncodingError, ::TypeError
40
+ UNPARSEABLE
41
+ end
42
+
31
43
  # The args to display for `job` (a parsed payload Hash) under `hook`.
32
44
  def args(job, hook)
33
45
  result = hook.call(::JSON.parse(::JSON.generate(job)))
@@ -58,10 +58,9 @@ module Wurk
58
58
  # which is what lets every pooled connection survive a Redis restart, a
59
59
  # failover or an idle-socket reap without surfacing an error. The price is
60
60
  # that a reply lost mid-command (a ReadTimeout included) can apply that
61
- # command twice: a duplicate LPUSH, a double INCR, or a ZPOPBYSCORE whose
62
- # first pop is lost with its reply (Sidekiq #3303 — same exposure upstream).
63
- # 0 would close that window but turn every stale socket into an error the
64
- # pool cannot replay for a non-idempotent block, so it stays at parity.
61
+ # command twice: a duplicate LPUSH or a double INCR. 0 would close that
62
+ # window but turn every stale socket into an error the pool cannot replay
63
+ # for a non-idempotent block, so it stays at parity.
65
64
  DEFAULT_CONNECT_TIMEOUT = 1.0
66
65
  DEFAULT_READ_TIMEOUT = 2.5
67
66
  DEFAULT_WRITE_TIMEOUT = 2.5
@@ -182,6 +181,12 @@ module Wurk
182
181
  @pool.shutdown { |conn| safe_close(conn) }
183
182
  end
184
183
 
184
+ # Non-terminal: closes idle sockets before a fork; the pool stays usable, so
185
+ # host threads holding it never see PoolShuttingDownError.
186
+ def reload!
187
+ @pool.reload { |conn| safe_close(conn) }
188
+ end
189
+
185
190
  # Redis INFO parsed to a Hash, with this pool's own `size` / `available`
186
191
  # slot counts merged in — one call gives a heartbeat both Redis health and
187
192
  # local pool saturation. (Real Redis INFO has no `size`/`available` field.)
@@ -259,8 +264,10 @@ module Wurk
259
264
  @reaper_lock.synchronize do
260
265
  return if @reaper_closed || @reaper&.alive?
261
266
 
262
- @reaper_stop = Thread::Queue.new
263
- @reaper = Thread.new(@reaper_stop) { |stop| reap_loop(stop) }
267
+ @reaper_mutex = Thread::Mutex.new
268
+ @reaper_wake = Thread::ConditionVariable.new
269
+ @reaper_stopped = false
270
+ @reaper = Thread.new { reap_loop }
264
271
  @reaper.name = "wurk-redis-reaper-#{@name}"
265
272
  end
266
273
  end
@@ -270,7 +277,10 @@ module Wurk
270
277
  @reaper_closed = true
271
278
  next unless @reaper&.alive?
272
279
 
273
- @reaper_stop << true
280
+ @reaper_mutex.synchronize do
281
+ @reaper_stopped = true
282
+ @reaper_wake.broadcast
283
+ end
274
284
  @reaper
275
285
  end
276
286
  reaper&.join
@@ -281,9 +291,24 @@ module Wurk
281
291
  # a checked-out connection is not on that stack to be found. #disconnect!
282
292
  # joins this thread before shutting the pool down, so a sweep never meets a
283
293
  # pool mid-shutdown.
284
- def reap_loop(stop)
294
+ def reap_loop
285
295
  interval = [@redis_idle_timeout / 2.0, REAP_MAX_INTERVAL].min
286
- @pool.reap(idle_seconds: @redis_idle_timeout) { |conn| safe_close(conn) } until stop.pop(timeout: interval)
296
+ loop do
297
+ break if wait_for_stop(interval)
298
+
299
+ @pool.reap(idle_seconds: @redis_idle_timeout) { |conn| safe_close(conn) }
300
+ end
301
+ end
302
+
303
+ # The sweep's wait: a Mutex+ConditionVariable, the timed wait the tree's
304
+ # periodic, stop-signalled loops use (Wurk::Client::Buffered's drainer,
305
+ # Wurk::TimerLoop). `stop` wakes the sweep at once instead of leaving it
306
+ # parked until the interval is up.
307
+ def wait_for_stop(interval)
308
+ @reaper_mutex.synchronize do
309
+ @reaper_wake.wait(@reaper_mutex, interval) unless @reaper_stopped
310
+ @reaper_stopped
311
+ end
287
312
  end
288
313
 
289
314
  # Runs the block on the checked-out `conn`, retrying transient RedisClient