wurk 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/wurk/api_controller.rb +4 -1
- data/lib/wurk/boot_checks.rb +87 -0
- data/lib/wurk/capsule.rb +33 -8
- data/lib/wurk/cli.rb +73 -8
- data/lib/wurk/client/buffered.rb +240 -26
- data/lib/wurk/compat.rb +11 -0
- data/lib/wurk/configuration.rb +114 -16
- data/lib/wurk/cron.rb +45 -0
- data/lib/wurk/debounce.rb +2 -2
- data/lib/wurk/error_coalescer.rb +35 -0
- data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
- data/lib/wurk/fetcher/capped.rb +4 -0
- data/lib/wurk/fetcher/many_queues.rb +17 -1
- data/lib/wurk/fetcher/oom_drain.rb +102 -0
- data/lib/wurk/fetcher/orphan_grace.rb +60 -44
- data/lib/wurk/fetcher/private_list_key.rb +1 -16
- data/lib/wurk/fetcher/reaper.rb +169 -127
- data/lib/wurk/fetcher/reliable.rb +26 -4
- data/lib/wurk/fetcher/self_reconcile.rb +175 -0
- data/lib/wurk/fetcher/unit_of_work.rb +32 -8
- data/lib/wurk/health.rb +23 -4
- data/lib/wurk/health_probe.rb +64 -0
- data/lib/wurk/heartbeat.rb +55 -3
- data/lib/wurk/job/options.rb +9 -1
- data/lib/wurk/job.rb +2 -24
- data/lib/wurk/job_record.rb +17 -9
- data/lib/wurk/job_retry.rb +38 -10
- data/lib/wurk/job_set.rb +13 -5
- data/lib/wurk/launcher.rb +134 -52
- data/lib/wurk/leader.rb +9 -3
- data/lib/wurk/limiter/base.rb +17 -1
- data/lib/wurk/lua/debounce.lua +11 -6
- data/lib/wurk/lua/loader.rb +13 -1
- data/lib/wurk/lua.rb +71 -8
- data/lib/wurk/manager.rb +1 -1
- data/lib/wurk/metrics/process_counters.rb +27 -0
- data/lib/wurk/metrics/prometheus.rb +35 -14
- data/lib/wurk/middleware/current_attributes.rb +26 -10
- data/lib/wurk/middleware/poison_pill.rb +1 -2
- data/lib/wurk/process_set.rb +3 -1
- data/lib/wurk/processor.rb +82 -18
- data/lib/wurk/queue_consumers.rb +134 -0
- data/lib/wurk/rails_boot.rb +40 -2
- data/lib/wurk/rails_reloader.rb +51 -0
- data/lib/wurk/railtie.rb +20 -1
- data/lib/wurk/redact.rb +12 -0
- data/lib/wurk/redis_pool.rb +34 -9
- data/lib/wurk/scheduled.rb +142 -67
- data/lib/wurk/sentry/death_handler.rb +29 -0
- data/lib/wurk/sentry/error_handler.rb +52 -10
- data/lib/wurk/sentry/middleware.rb +8 -1
- data/lib/wurk/sentry/retry_policy.rb +12 -3
- data/lib/wurk/sentry.rb +42 -5
- data/lib/wurk/sorted_entry.rb +38 -13
- data/lib/wurk/swarm/child_boot.rb +42 -31
- data/lib/wurk/swarm/crash_watch.rb +65 -0
- data/lib/wurk/swarm.rb +79 -11
- data/lib/wurk/testing.rb +66 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/extension.rb +6 -1
- data/lib/wurk/work_set.rb +6 -0
- data/lib/wurk/worker/setter.rb +11 -6
- data/lib/wurk/worker.rb +17 -1
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
- data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
- data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +41 -30
- data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
- data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
- data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require_relative 'component'
|
|
5
|
+
require_relative 'keys'
|
|
6
|
+
require_relative 'timer_loop'
|
|
7
|
+
|
|
8
|
+
module Wurk
|
|
9
|
+
# Who listens to which queue, read from what every process already publishes:
|
|
10
|
+
# the `queues` list in its heartbeat `info`. That list is the union across a
|
|
11
|
+
# process's capsules, so a queue served by one capsule counts. Expired
|
|
12
|
+
# identities have no `info` and are not counted. No new Redis keys.
|
|
13
|
+
module QueueConsumers
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
# `{ queue => live subscribers }`. A quiet process still counts: it is a
|
|
17
|
+
# deliberate pause, not a queue nobody thought to list.
|
|
18
|
+
def counts(conn)
|
|
19
|
+
ids = conn.call('SMEMBERS', Keys::PROCESSES)
|
|
20
|
+
return {} if ids.empty?
|
|
21
|
+
|
|
22
|
+
tally(conn.pipelined { |pipe| ids.each { |id| pipe.call('HGET', id, 'info') } })
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def tally(infos)
|
|
26
|
+
infos.each_with_object(Hash.new(0)) do |info, out|
|
|
27
|
+
subscribed(info).each { |queue| out[queue] += 1 }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def subscribed(info)
|
|
32
|
+
return [] if info.nil?
|
|
33
|
+
|
|
34
|
+
Array(::JSON.parse(info)['queues']).map(&:to_s).uniq
|
|
35
|
+
rescue ::JSON::ParserError, ::TypeError
|
|
36
|
+
[]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Warns about non-empty queues no live process subscribes to: such jobs wait
|
|
40
|
+
# forever with no error, retry or report. Leader only, one tick a minute.
|
|
41
|
+
#
|
|
42
|
+
# A queue is reported only after `grace` seconds unconsumed on every tick. A
|
|
43
|
+
# rolling restart or a deploy that adds a queue passes through that state
|
|
44
|
+
# briefly, and one leader's in-memory clock is enough to ride it out. A
|
|
45
|
+
# leader change restarts the clock, which only delays a warning.
|
|
46
|
+
class Watch
|
|
47
|
+
include Component
|
|
48
|
+
|
|
49
|
+
TICK = 60
|
|
50
|
+
GRACE = 300
|
|
51
|
+
THROTTLE = 3600
|
|
52
|
+
|
|
53
|
+
attr_reader :config
|
|
54
|
+
|
|
55
|
+
def initialize(config, tick: TICK, grace: GRACE, throttle: THROTTLE,
|
|
56
|
+
clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
|
|
57
|
+
@config = config
|
|
58
|
+
@grace = grace
|
|
59
|
+
@throttle = throttle
|
|
60
|
+
@clock = clock
|
|
61
|
+
@timer = TimerLoop.new(tick)
|
|
62
|
+
@unconsumed_since = {}
|
|
63
|
+
@warned_at = {}
|
|
64
|
+
@thread = nil
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def start
|
|
68
|
+
return @thread if @thread
|
|
69
|
+
|
|
70
|
+
@timer.reset
|
|
71
|
+
@thread = safe_thread('queue-consumers') { @timer.run { tick } }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def terminate
|
|
75
|
+
@timer.terminate
|
|
76
|
+
@thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def tick
|
|
80
|
+
return unless leader?
|
|
81
|
+
|
|
82
|
+
check
|
|
83
|
+
rescue StandardError => e
|
|
84
|
+
handle_exception(e, { context: 'queue-consumers' })
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# One pass, bypassing the leader gate. Returns the queues warned about.
|
|
88
|
+
def check
|
|
89
|
+
now = @clock.call
|
|
90
|
+
unconsumed = read_unconsumed
|
|
91
|
+
@unconsumed_since.select! { |queue, _| unconsumed.key?(queue) }
|
|
92
|
+
unconsumed.filter_map do |queue, size|
|
|
93
|
+
@unconsumed_since[queue] ||= now
|
|
94
|
+
next unless warn_due?(queue, now)
|
|
95
|
+
|
|
96
|
+
@warned_at[queue] = now
|
|
97
|
+
logger.warn { warning(queue, size) }
|
|
98
|
+
queue
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
private
|
|
103
|
+
|
|
104
|
+
def warn_due?(queue, now)
|
|
105
|
+
return false if now - @unconsumed_since[queue] < @grace
|
|
106
|
+
|
|
107
|
+
at = @warned_at[queue]
|
|
108
|
+
at.nil? || now - at >= @throttle
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def warning(queue, size)
|
|
112
|
+
"Queue #{queue.inspect} has #{size} jobs and no live process subscribes to it, so they will wait " \
|
|
113
|
+
'forever. Add it to the worker\'s queues (or drain it if it is obsolete).'
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def read_unconsumed
|
|
117
|
+
@config.redis(idempotent: true) do |conn|
|
|
118
|
+
counts = QueueConsumers.counts(conn)
|
|
119
|
+
# An empty fleet is a different alarm, and one every monitor already has.
|
|
120
|
+
next {} if counts.empty?
|
|
121
|
+
|
|
122
|
+
non_empty(conn, conn.call('SMEMBERS', Keys::QUEUES_SET).reject { |queue| counts.key?(queue) })
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def non_empty(conn, names)
|
|
127
|
+
return {} if names.empty?
|
|
128
|
+
|
|
129
|
+
sizes = conn.pipelined { |pipe| names.each { |queue| pipe.call('LLEN', Keys.queue(queue)) } }
|
|
130
|
+
names.zip(sizes).select { |_, size| size.positive? }.to_h
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
data/lib/wurk/rails_boot.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'rails_reloader'
|
|
4
|
+
|
|
3
5
|
module Wurk
|
|
4
6
|
# Coordinates how Wurk boots inside a Rails host: whether this process should
|
|
5
7
|
# boot at all, whether it may fork the swarm, and the boot itself. The Railtie
|
|
@@ -13,6 +15,8 @@ module Wurk
|
|
|
13
15
|
# the final reap.
|
|
14
16
|
DRAIN_JOIN_SLACK = 1
|
|
15
17
|
|
|
18
|
+
BOOT_INITIALIZER = :wurk_boot
|
|
19
|
+
|
|
16
20
|
module_function
|
|
17
21
|
|
|
18
22
|
# Invoked from the `wurk.server_mode` initializer, before config/initializers
|
|
@@ -34,7 +38,27 @@ module Wurk
|
|
|
34
38
|
Wurk.configuration[:client_in_server] = true
|
|
35
39
|
end
|
|
36
40
|
|
|
37
|
-
#
|
|
41
|
+
# A host that already replaced the reloader keeps its own. A config already
|
|
42
|
+
# frozen, by workers embedded from an initializer, can't take it: their
|
|
43
|
+
# Processors resolved the default themselves (Processor#resolve_reloader).
|
|
44
|
+
def install_reloader(app = ::Rails.application, config = Wurk.configuration)
|
|
45
|
+
return if config.frozen?
|
|
46
|
+
return unless config[:reloader].equal?(Configuration::DEFAULTS[:reloader])
|
|
47
|
+
|
|
48
|
+
config[:reloader] = RailsReloader.new(app)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Set by the railtie's first initializer. Rails builds the initializer list
|
|
52
|
+
# once, so if that one ran the Finisher entry is in the list too. A railtie
|
|
53
|
+
# required from a host's own initializer missed it, and the after_initialize
|
|
54
|
+
# hook boots instead.
|
|
55
|
+
def boot_scheduled? = @boot_scheduled == true
|
|
56
|
+
|
|
57
|
+
def boot_scheduled=(value)
|
|
58
|
+
@boot_scheduled = value
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Invoked last of all initializers, once the host app has fully initialized.
|
|
38
62
|
def boot(app = ::Rails.application)
|
|
39
63
|
return if skip_boot?
|
|
40
64
|
|
|
@@ -156,7 +180,9 @@ module Wurk
|
|
|
156
180
|
false
|
|
157
181
|
end
|
|
158
182
|
|
|
159
|
-
def boot_swarm
|
|
183
|
+
def boot_swarm # rubocop:disable Metrics/AbcSize
|
|
184
|
+
warn_on_production_fork
|
|
185
|
+
Health.options_from_env
|
|
160
186
|
timeout = Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
|
|
161
187
|
swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology, shutdown_timeout: timeout)
|
|
162
188
|
supervisor = nil
|
|
@@ -181,6 +207,18 @@ module Wurk
|
|
|
181
207
|
end
|
|
182
208
|
end
|
|
183
209
|
|
|
210
|
+
# A deliberate default, but a surprise after a gem swap: stock Sidekiq never
|
|
211
|
+
# runs jobs in the web process, and a separate worker on top of this runs
|
|
212
|
+
# everything against two pools. WURK_EMBED=1 says it is intended.
|
|
213
|
+
def warn_on_production_fork
|
|
214
|
+
return unless ::Rails.env.production? && ENV['WURK_EMBED'] != '1'
|
|
215
|
+
|
|
216
|
+
logger.warn do
|
|
217
|
+
'wurk: this web process is forking a worker swarm. If workers run elsewhere, set WURK_DISABLED=1 here; ' \
|
|
218
|
+
'if this is intended, set WURK_EMBED=1 to silence this.'
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
|
|
184
222
|
# at_exit fires on the host's main thread, but the supervise thread owns the
|
|
185
223
|
# child table and two threads inside `shutdown` race on it — so request the
|
|
186
224
|
# drain and wait for the supervisor to run it. Draining here is the fallback
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
# Runs each job inside the host's `Rails.application.reloader`, whose executor
|
|
5
|
+
# hooks enable the query cache, reset CurrentAttributes, release the thread's
|
|
6
|
+
# Active Record connection and, in development, reload changed code.
|
|
7
|
+
#
|
|
8
|
+
# Installed as `config[:reloader]` by RailsBoot when the railtie is loaded.
|
|
9
|
+
# `wurk` / `wurkswarm` load the host app with no railtie (see the note at the
|
|
10
|
+
# end of lib/wurk.rb), so Processor also asks `.default` for itself.
|
|
11
|
+
class RailsReloader
|
|
12
|
+
# The source Rails' error reporter tags these failures with. Subscribers
|
|
13
|
+
# filter on the name, so it stays the one a Sidekiq app already matches.
|
|
14
|
+
SOURCE = 'job.sidekiq'
|
|
15
|
+
|
|
16
|
+
# @return [RailsReloader, nil] nil unless this process has a Rails
|
|
17
|
+
# application, which is what keeps a plain Ruby worker free of any per-job
|
|
18
|
+
# wrapping.
|
|
19
|
+
def self.default
|
|
20
|
+
return unless defined?(::Rails) && ::Rails.respond_to?(:application)
|
|
21
|
+
|
|
22
|
+
app = ::Rails.application
|
|
23
|
+
new(app) if app
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(app)
|
|
27
|
+
@app = app
|
|
28
|
+
# Rails 7.0's reloader takes no keywords.
|
|
29
|
+
takes_source = app.reloader.method(:wrap).parameters.any? { |kind, _| %i[key keyreq keyrest].include?(kind) }
|
|
30
|
+
@wrap_options = takes_source ? { source: SOURCE }.freeze : {}.freeze
|
|
31
|
+
# A host can start embedded workers from an initializer, while the app is
|
|
32
|
+
# still booting and its reloader hooks aren't installed. Those jobs run bare
|
|
33
|
+
# until boot reaches after_initialize. Not `app.initialized?`: a swarm child
|
|
34
|
+
# forked during boot never returns to where Rails would set it.
|
|
35
|
+
@ready = false
|
|
36
|
+
::ActiveSupport.on_load(:after_initialize, yield: true) { @ready = true }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def call(&)
|
|
40
|
+
return yield unless @ready
|
|
41
|
+
|
|
42
|
+
@app.reloader.wrap(**@wrap_options, &)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# The default `as_json` walks instance variables, and the Application's never
|
|
46
|
+
# end. Configuration holds this object, so serializing that would overflow too.
|
|
47
|
+
def as_json(*) = {}
|
|
48
|
+
|
|
49
|
+
def inspect = "#<#{self.class.name} app=#{@app.class.name}>"
|
|
50
|
+
end
|
|
51
|
+
end
|
data/lib/wurk/railtie.rb
CHANGED
|
@@ -28,11 +28,30 @@ module Wurk
|
|
|
28
28
|
# (`configure_server` / `server? is false`). RailsBoot decides whether this
|
|
29
29
|
# process serves.
|
|
30
30
|
initializer 'wurk.server_mode', before: :load_config_initializers do |app|
|
|
31
|
+
Wurk::RailsBoot.boot_scheduled = true
|
|
31
32
|
Wurk::RailsBoot.enter_server_mode_if_serving(app)
|
|
32
33
|
end
|
|
33
34
|
|
|
35
|
+
# Registered at gem load, so it runs ahead of the host's own hooks: one of
|
|
36
|
+
# them can still replace or wrap the reloader.
|
|
34
37
|
config.after_initialize do |app|
|
|
35
|
-
Wurk::RailsBoot.
|
|
38
|
+
Wurk::RailsBoot.install_reloader(app)
|
|
39
|
+
Wurk::RailsBoot.boot(app) unless Wurk::RailsBoot.boot_scheduled?
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Appended to Rails' own Finisher so it runs dead last. A boot here sees what
|
|
43
|
+
# a host's `after_initialize` configured (a Redis URL, say), and a forked
|
|
44
|
+
# child has the routes and dev-reload hooks Rails installs after those hooks
|
|
45
|
+
# fire, which it could never pick up later. `after:` can't express this: it
|
|
46
|
+
# drags the whole Finisher chain ahead of initializers not yet run.
|
|
47
|
+
#
|
|
48
|
+
# Deferred to before_configuration because naming Rails::Application at load
|
|
49
|
+
# would pull ActionDispatch into a host loading only rails/railtie.
|
|
50
|
+
config.before_configuration do
|
|
51
|
+
finisher = ::Rails::Application::Finisher
|
|
52
|
+
unless finisher.initializers.any? { |i| i.name == Wurk::RailsBoot::BOOT_INITIALIZER }
|
|
53
|
+
finisher.initializer(Wurk::RailsBoot::BOOT_INITIALIZER) { |app| Wurk::RailsBoot.boot(app) }
|
|
54
|
+
end
|
|
36
55
|
end
|
|
37
56
|
end
|
|
38
57
|
end
|
data/lib/wurk/redact.rb
CHANGED
|
@@ -21,6 +21,7 @@ module Wurk
|
|
|
21
21
|
# Fail closed: a hook that raises must not fall back to showing the args it
|
|
22
22
|
# was installed to hide.
|
|
23
23
|
FAILED = ['[REDACTED: redact_args raised]'].freeze
|
|
24
|
+
UNPARSEABLE = '[REDACTED: unparseable job]'
|
|
24
25
|
|
|
25
26
|
module_function
|
|
26
27
|
|
|
@@ -28,6 +29,17 @@ module Wurk
|
|
|
28
29
|
config.redact_args
|
|
29
30
|
end
|
|
30
31
|
|
|
32
|
+
# A raw payload string, with its args replaced. Anything that does not parse to
|
|
33
|
+
# a job, or to JSON at all, could be holding the args in any shape: it is dropped.
|
|
34
|
+
def jobstr(raw, hook)
|
|
35
|
+
job = ::JSON.parse(raw)
|
|
36
|
+
return UNPARSEABLE unless job.is_a?(::Hash)
|
|
37
|
+
|
|
38
|
+
::JSON.generate(job.merge('args' => args(job, hook)))
|
|
39
|
+
rescue ::JSON::ParserError, ::EncodingError, ::TypeError
|
|
40
|
+
UNPARSEABLE
|
|
41
|
+
end
|
|
42
|
+
|
|
31
43
|
# The args to display for `job` (a parsed payload Hash) under `hook`.
|
|
32
44
|
def args(job, hook)
|
|
33
45
|
result = hook.call(::JSON.parse(::JSON.generate(job)))
|
data/lib/wurk/redis_pool.rb
CHANGED
|
@@ -58,10 +58,9 @@ module Wurk
|
|
|
58
58
|
# which is what lets every pooled connection survive a Redis restart, a
|
|
59
59
|
# failover or an idle-socket reap without surfacing an error. The price is
|
|
60
60
|
# that a reply lost mid-command (a ReadTimeout included) can apply that
|
|
61
|
-
# command twice: a duplicate LPUSH
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
# pool cannot replay for a non-idempotent block, so it stays at parity.
|
|
61
|
+
# command twice: a duplicate LPUSH or a double INCR. 0 would close that
|
|
62
|
+
# window but turn every stale socket into an error the pool cannot replay
|
|
63
|
+
# for a non-idempotent block, so it stays at parity.
|
|
65
64
|
DEFAULT_CONNECT_TIMEOUT = 1.0
|
|
66
65
|
DEFAULT_READ_TIMEOUT = 2.5
|
|
67
66
|
DEFAULT_WRITE_TIMEOUT = 2.5
|
|
@@ -182,6 +181,12 @@ module Wurk
|
|
|
182
181
|
@pool.shutdown { |conn| safe_close(conn) }
|
|
183
182
|
end
|
|
184
183
|
|
|
184
|
+
# Non-terminal: closes idle sockets before a fork; the pool stays usable, so
|
|
185
|
+
# host threads holding it never see PoolShuttingDownError.
|
|
186
|
+
def reload!
|
|
187
|
+
@pool.reload { |conn| safe_close(conn) }
|
|
188
|
+
end
|
|
189
|
+
|
|
185
190
|
# Redis INFO parsed to a Hash, with this pool's own `size` / `available`
|
|
186
191
|
# slot counts merged in — one call gives a heartbeat both Redis health and
|
|
187
192
|
# local pool saturation. (Real Redis INFO has no `size`/`available` field.)
|
|
@@ -259,8 +264,10 @@ module Wurk
|
|
|
259
264
|
@reaper_lock.synchronize do
|
|
260
265
|
return if @reaper_closed || @reaper&.alive?
|
|
261
266
|
|
|
262
|
-
@
|
|
263
|
-
@
|
|
267
|
+
@reaper_mutex = Thread::Mutex.new
|
|
268
|
+
@reaper_wake = Thread::ConditionVariable.new
|
|
269
|
+
@reaper_stopped = false
|
|
270
|
+
@reaper = Thread.new { reap_loop }
|
|
264
271
|
@reaper.name = "wurk-redis-reaper-#{@name}"
|
|
265
272
|
end
|
|
266
273
|
end
|
|
@@ -270,7 +277,10 @@ module Wurk
|
|
|
270
277
|
@reaper_closed = true
|
|
271
278
|
next unless @reaper&.alive?
|
|
272
279
|
|
|
273
|
-
@
|
|
280
|
+
@reaper_mutex.synchronize do
|
|
281
|
+
@reaper_stopped = true
|
|
282
|
+
@reaper_wake.broadcast
|
|
283
|
+
end
|
|
274
284
|
@reaper
|
|
275
285
|
end
|
|
276
286
|
reaper&.join
|
|
@@ -281,9 +291,24 @@ module Wurk
|
|
|
281
291
|
# a checked-out connection is not on that stack to be found. #disconnect!
|
|
282
292
|
# joins this thread before shutting the pool down, so a sweep never meets a
|
|
283
293
|
# pool mid-shutdown.
|
|
284
|
-
def reap_loop
|
|
294
|
+
def reap_loop
|
|
285
295
|
interval = [@redis_idle_timeout / 2.0, REAP_MAX_INTERVAL].min
|
|
286
|
-
|
|
296
|
+
loop do
|
|
297
|
+
break if wait_for_stop(interval)
|
|
298
|
+
|
|
299
|
+
@pool.reap(idle_seconds: @redis_idle_timeout) { |conn| safe_close(conn) }
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# The sweep's wait: a Mutex+ConditionVariable, the timed wait the tree's
|
|
304
|
+
# periodic, stop-signalled loops use (Wurk::Client::Buffered's drainer,
|
|
305
|
+
# Wurk::TimerLoop). `stop` wakes the sweep at once instead of leaving it
|
|
306
|
+
# parked until the interval is up.
|
|
307
|
+
def wait_for_stop(interval)
|
|
308
|
+
@reaper_mutex.synchronize do
|
|
309
|
+
@reaper_wake.wait(@reaper_mutex, interval) unless @reaper_stopped
|
|
310
|
+
@reaper_stopped
|
|
311
|
+
end
|
|
287
312
|
end
|
|
288
313
|
|
|
289
314
|
# Runs the block on the checked-out `conn`, retrying transient RedisClient
|