wurk 1.12.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/config/routes.rb +8 -0
- data/exe/sidekiq +8 -0
- data/exe/wurk +1 -1
- data/lib/generators/wurk/install/install_generator.rb +56 -0
- data/lib/wurk/boot_checks.rb +80 -0
- data/lib/wurk/cli.rb +55 -17
- data/lib/wurk/configuration.rb +27 -6
- data/lib/wurk/fetcher/orphan_reclaim.rb +48 -11
- data/lib/wurk/{middleware → fetcher}/poison_pill.rb +3 -1
- data/lib/wurk/fetcher/reaper.rb +28 -9
- data/lib/wurk/fetcher/reliable.rb +4 -5
- data/lib/wurk/fetcher/unit_of_work.rb +1 -1
- data/lib/wurk/job_logger.rb +4 -2
- data/lib/wurk/job_util.rb +14 -0
- data/lib/wurk/launcher.rb +30 -3
- data/lib/wurk/leader.rb +1 -8
- data/lib/wurk/manager.rb +27 -1
- data/lib/wurk/middleware/timeout.rb +6 -6
- data/lib/wurk/preflight/app_scan.rb +182 -0
- data/lib/wurk/preflight/command.rb +21 -5
- data/lib/wurk/preflight/redis_guard.rb +98 -0
- data/lib/wurk/preflight.rb +9 -5
- data/lib/wurk/rails_boot.rb +7 -2
- data/lib/wurk/redis_options.rb +5 -0
- data/lib/wurk/redis_pool.rb +24 -1
- data/lib/wurk/sentry.rb +11 -0
- data/lib/wurk/swarm.rb +63 -14
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/rack_app.rb +8 -6
- data/lib/wurk/web/stats_routes.rb +93 -0
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +11 -2
data/lib/wurk/job_logger.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Wurk
|
|
|
12
12
|
# 1. prepare(job_hash) { ... } → sets thread-local context, applies
|
|
13
13
|
# per-job log_level, yields to the rest of dispatch.
|
|
14
14
|
# 2. call(item, queue) { ... } → wraps the actual perform with the
|
|
15
|
-
# start/done/fail log line trio and the elapsed-
|
|
15
|
+
# start/done/fail log line trio and the elapsed-seconds measurement.
|
|
16
16
|
#
|
|
17
17
|
# Skipping default logging is controlled by config[:skip_default_job_logging];
|
|
18
18
|
# the prepare step still runs so context/log_level still apply.
|
|
@@ -68,8 +68,10 @@ module Wurk
|
|
|
68
68
|
|
|
69
69
|
private
|
|
70
70
|
|
|
71
|
+
# Seconds, as upstream logs it (`elapsed=0.007`): log scrapers and
|
|
72
|
+
# dashboards built on Sidekiq's line read the field that way.
|
|
71
73
|
def elapsed(start)
|
|
72
|
-
(
|
|
74
|
+
(::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
|
|
73
75
|
end
|
|
74
76
|
|
|
75
77
|
# The attribute list is a host setting (defaults to %w[bid tags]), so the
|
data/lib/wurk/job_util.rb
CHANGED
|
@@ -73,6 +73,20 @@ module Wurk
|
|
|
73
73
|
seconds.finite? && seconds.positive?
|
|
74
74
|
end
|
|
75
75
|
|
|
76
|
+
# A Duration's `to_s`: "5" for `5.seconds`, "1.5" for `1.5.seconds`. Same
|
|
77
|
+
# shape {Wurk::Unique.coerce_ttl} reads back from a `unique_for`.
|
|
78
|
+
NUMERIC_SECONDS = /\A\d+(?:\.\d+)?\z/
|
|
79
|
+
|
|
80
|
+
# The read side of a bound, as a Float or nil. Wider than
|
|
81
|
+
# {positive_seconds?} by one shape only: wurk < 1.12 wrote a Duration
|
|
82
|
+
# `timeout:` into the payload as its String ("5"), and those jobs are still
|
|
83
|
+
# queued. Never used to decide what is written; {validate_bounds!} keeps
|
|
84
|
+
# rejecting a String at declaration.
|
|
85
|
+
def self.payload_seconds(value)
|
|
86
|
+
value = value.to_f if value.is_a?(::String) && NUMERIC_SECONDS.match?(value)
|
|
87
|
+
value.to_f if positive_seconds?(value)
|
|
88
|
+
end
|
|
89
|
+
|
|
76
90
|
# The exact bytes a job is stored as inside the `schedule` ZSET. `at` is
|
|
77
91
|
# dropped because it *is* the score, and `enqueued_at` because it marks
|
|
78
92
|
# arrival on an immediate queue — the promoter restamps it on the way out
|
data/lib/wurk/launcher.rb
CHANGED
|
@@ -47,6 +47,20 @@ module Wurk
|
|
|
47
47
|
# (Sidekiq's drop-in surface). The single source of truth is Heartbeat.
|
|
48
48
|
BEAT_PAUSE = Heartbeat::BEAT_PAUSE
|
|
49
49
|
|
|
50
|
+
# The process title, upstream's format and program name word for word:
|
|
51
|
+
# `sidekiq 8.1.5 myapp [3 of 10 busy]`, then `stopping` once quieted. Probes
|
|
52
|
+
# and runbooks that `ps | grep '^sidekiq'` or read the busy count keep
|
|
53
|
+
# working across the gem swap. Mutable like upstream's, so code that appends
|
|
54
|
+
# its own segment still can. docs/target/ is silent; the oracle is Sidekiq
|
|
55
|
+
# 8.1's Launcher (docs/deployment.md, "Process titles").
|
|
56
|
+
PROCTITLES = [ # rubocop:disable Style/MutableConstant
|
|
57
|
+
proc { 'sidekiq' },
|
|
58
|
+
proc { ::Sidekiq::VERSION },
|
|
59
|
+
proc { |_me, data| data['tag'] },
|
|
60
|
+
proc { |me, _data| "[#{Processor::WORK_STATE.size} of #{me.config.total_concurrency} busy]" },
|
|
61
|
+
proc { |me, _data| 'stopping' if me.stopping? }
|
|
62
|
+
]
|
|
63
|
+
|
|
50
64
|
# Bound on how long #stop waits for the boot-time reclaim sweep before
|
|
51
65
|
# moving on — it can still be scanning a large keyspace when a fast
|
|
52
66
|
# shutdown lands right after boot; teardown must not hang on it.
|
|
@@ -96,9 +110,12 @@ module Wurk
|
|
|
96
110
|
# 3. start the managers. Everything below is periodic background work
|
|
97
111
|
# whose first tick is seconds away, so it has nothing to gain from
|
|
98
112
|
# going ahead of them.
|
|
99
|
-
# 4. start the reaper and the boot-time reclaim sweep
|
|
100
|
-
#
|
|
101
|
-
#
|
|
113
|
+
# 4. start the reaper and the boot-time reclaim sweep, ahead of the
|
|
114
|
+
# periodic loops: a process restarted outside a swarm (exe/wurk under
|
|
115
|
+
# systemd, a pod) has its previous incarnation's jobs recovered from
|
|
116
|
+
# here. The sweep takes the cluster reaper lock, so a fleet booting
|
|
117
|
+
# together makes one keyspace pass; a swarm child's SIGKILLed sibling
|
|
118
|
+
# is recovered by the parent, not by this sweep.
|
|
102
119
|
# 5. write the `config.periodic` registrations (Ent registers loops on
|
|
103
120
|
# startup), then start the periodic loops. None of them ticks at
|
|
104
121
|
# zero — TimerLoop#run waits an interval before its first yield,
|
|
@@ -404,12 +421,22 @@ module Wurk
|
|
|
404
421
|
# the dashboard queued at `<identity>-signals`. Lazily builds the
|
|
405
422
|
# Heartbeat the first time we beat so callers that bypass `run`
|
|
406
423
|
# (embedded mode, tests) still work.
|
|
424
|
+
# An embedded launcher shares its process with the host, whose title it is.
|
|
407
425
|
def beat
|
|
426
|
+
$PROGRAM_NAME = proctitle unless @embedded
|
|
408
427
|
ensure_heartbeat
|
|
409
428
|
sigs = @heartbeat.beat!
|
|
410
429
|
sigs&.each { |sig| dispatch_signal(sig) }
|
|
411
430
|
end
|
|
412
431
|
|
|
432
|
+
# Upstream hands each segment the process's heartbeat data; these are the
|
|
433
|
+
# fields of it a title can use without a Redis read.
|
|
434
|
+
def proctitle
|
|
435
|
+
data = { 'tag' => @config[:tag] || default_tag, 'identity' => identity, 'hostname' => hostname,
|
|
436
|
+
'pid' => ::Process.pid, 'concurrency' => @config.total_concurrency }
|
|
437
|
+
PROCTITLES.map { |segment| segment.call(self, data) }.compact.join(' ')
|
|
438
|
+
end
|
|
439
|
+
|
|
413
440
|
def ensure_heartbeat
|
|
414
441
|
return if @heartbeat
|
|
415
442
|
|
data/lib/wurk/leader.rb
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'securerandom'
|
|
4
|
-
require 'socket'
|
|
5
4
|
require_relative 'component'
|
|
6
5
|
require_relative 'fetcher/oom_drain'
|
|
7
6
|
require_relative 'lua'
|
|
@@ -282,13 +281,7 @@ module Wurk
|
|
|
282
281
|
# Matches `Wurk::Component#identity` so `ProcessSet::Process#leader?`
|
|
283
282
|
# (which compares `dear-leader == process.identity`) sees a match.
|
|
284
283
|
def cluster_identity
|
|
285
|
-
|
|
286
|
-
end
|
|
287
|
-
|
|
288
|
-
def hostname
|
|
289
|
-
ENV['DYNO'] || ::Socket.gethostname
|
|
290
|
-
rescue StandardError
|
|
291
|
-
'localhost'
|
|
284
|
+
Component.identity
|
|
292
285
|
end
|
|
293
286
|
|
|
294
287
|
def spawn_loop_thread
|
data/lib/wurk/manager.rb
CHANGED
|
@@ -167,12 +167,38 @@ module Wurk
|
|
|
167
167
|
|
|
168
168
|
jobs = cleanup.map(&:job).compact
|
|
169
169
|
logger.warn { "Terminating #{cleanup.size} busy threads" }
|
|
170
|
-
|
|
170
|
+
log_interrupted(jobs)
|
|
171
171
|
# `&.` like #quiet: a TERM in the pre-prepare! window (traps install
|
|
172
172
|
# before launcher.run) reaches here with no fetcher built yet.
|
|
173
173
|
capsule.fetcher&.bulk_requeue(jobs)
|
|
174
174
|
end
|
|
175
175
|
|
|
176
|
+
# One WARN per job a deploy cut short, because each one runs again from the
|
|
177
|
+
# top (docs/idempotency.md) and the operator chasing a double side effect
|
|
178
|
+
# needs its jid. Capped: a capsule can hold hundreds of threads.
|
|
179
|
+
REQUEUE_LOG_LIMIT = 20
|
|
180
|
+
private_constant :REQUEUE_LOG_LIMIT
|
|
181
|
+
|
|
182
|
+
def log_interrupted(jobs)
|
|
183
|
+
jobs.first(REQUEUE_LOG_LIMIT).each do |uow|
|
|
184
|
+
logger.warn { "Interrupting #{describe_job(uow)}: past the shutdown deadline, it is requeued to run again" }
|
|
185
|
+
end
|
|
186
|
+
extra = jobs.size - REQUEUE_LOG_LIMIT
|
|
187
|
+
logger.warn { "Interrupting #{extra} more in-flight jobs, which run again too" } if extra.positive?
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# The payload is parsed here, off the hot path, because the fetcher never
|
|
191
|
+
# parses. A plug-in fetcher's unit of work may carry something that is not
|
|
192
|
+
# job JSON; it is named by its queue alone.
|
|
193
|
+
def describe_job(uow)
|
|
194
|
+
job = Wurk.load_json(uow.job)
|
|
195
|
+
job = {} unless job.is_a?(::Hash)
|
|
196
|
+
klass = job['wrapped'] || job['class']
|
|
197
|
+
"#{klass} jid=#{job['jid']} queue=#{uow.queue_name}"
|
|
198
|
+
rescue ::JSON::ParserError, ::TypeError
|
|
199
|
+
"job queue=#{uow.queue_name}"
|
|
200
|
+
end
|
|
201
|
+
|
|
176
202
|
# The @workers Set is mutated from Processor threads (processor_result)
|
|
177
203
|
# while the lifecycle methods read/iterate it from the manager thread.
|
|
178
204
|
# Snapshot under @plock, then act on the copy outside the lock.
|
|
@@ -86,7 +86,7 @@ module Wurk
|
|
|
86
86
|
def within_attempt(timer, job, seconds, &)
|
|
87
87
|
return yield unless seconds
|
|
88
88
|
|
|
89
|
-
timer.watch(seconds, ::Wurk::Job::TimedOut, timeout_message(job
|
|
89
|
+
timer.watch(seconds, ::Wurk::Job::TimedOut, timeout_message(job), &)
|
|
90
90
|
end
|
|
91
91
|
|
|
92
92
|
# A bound written through Wurk is type-checked at declaration, where the
|
|
@@ -95,8 +95,7 @@ module Wurk
|
|
|
95
95
|
# passed that check, and a bound we can't use on one of those has to leave
|
|
96
96
|
# the job unbounded rather than break it.
|
|
97
97
|
def bound_for(job)
|
|
98
|
-
|
|
99
|
-
seconds if ::Wurk::JobUtil.positive_seconds?(seconds)
|
|
98
|
+
::Wurk::JobUtil.payload_seconds(job['timeout'])
|
|
100
99
|
end
|
|
101
100
|
|
|
102
101
|
# What is left of the absolute cutoff, in seconds. Negative is not an
|
|
@@ -122,9 +121,10 @@ module Wurk
|
|
|
122
121
|
|
|
123
122
|
# What an operator reads in the retry set, the dead set, a tracked job's
|
|
124
123
|
# status row, and their error tracker — so each names the class and the
|
|
125
|
-
# bound it broke
|
|
126
|
-
|
|
127
|
-
|
|
124
|
+
# bound it broke, as the payload carries it: `timeout: 5` reads "5s", not
|
|
125
|
+
# the Float it was armed as.
|
|
126
|
+
def timeout_message(job)
|
|
127
|
+
"#{job['class']} timed out after #{job['timeout']}s"
|
|
128
128
|
end
|
|
129
129
|
|
|
130
130
|
def deadline_message(job)
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
module Preflight
|
|
5
|
+
# `wurk preflight --app` rows read from the app's source and bundle, not from
|
|
6
|
+
# Redis: the patterns that carried over from stock Sidekiq and behave
|
|
7
|
+
# differently on Wurk. A text scan, so a row is a place to look, not a verdict.
|
|
8
|
+
class AppScan
|
|
9
|
+
SOURCE_DIRS = %w[app lib config].freeze
|
|
10
|
+
MAX_SITES = 20
|
|
11
|
+
|
|
12
|
+
# Sidekiq 8 and Wurk store these as epoch milliseconds; Sidekiq 7 stored
|
|
13
|
+
# float seconds, and code written against it reads them that way.
|
|
14
|
+
TIME_FIELD_READ = /\[['"](?:failed_at|created_at|enqueued_at|retried_at)['"]\]/
|
|
15
|
+
FLUSH = /\b(?:flushdb|flushall)\b/i
|
|
16
|
+
PRO_ENT_GUARD = /defined\?\(\s*(?:::)?Sidekiq::(?:Pro|Enterprise|Ent)\b|Sidekiq\.(?:pro|ent)\?/
|
|
17
|
+
RUNNER_COMMAND = /\b(?:wurk|wurkswarm|sidekiq|sidekiqswarm)\b/
|
|
18
|
+
WEB_SERVER_COMMAND = /\brails\s+(?:s|server)\b|\bpuma\b/
|
|
19
|
+
PROCFILES = %w[Procfile Procfile.dev].freeze
|
|
20
|
+
|
|
21
|
+
# The upstream versions each plugin's own suite runs green against Wurk
|
|
22
|
+
# (`ref:` in test/ecosystem/<gem>/PIN, which preflight_app_scan_test.rb
|
|
23
|
+
# holds this to).
|
|
24
|
+
TESTED_PLUGINS = {
|
|
25
|
+
'sentry-sidekiq' => '6.5.0',
|
|
26
|
+
'sidekiq-cron' => '2.4.0',
|
|
27
|
+
'sidekiq-failures' => '1.1.0',
|
|
28
|
+
'sidekiq-unique-jobs' => '8.1.0'
|
|
29
|
+
}.freeze
|
|
30
|
+
|
|
31
|
+
def initialize(root:, report:, specs: ::Gem.loaded_specs)
|
|
32
|
+
@root = root
|
|
33
|
+
@report = report
|
|
34
|
+
@specs = specs
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def run
|
|
38
|
+
time_fields
|
|
39
|
+
connection_pool_pin
|
|
40
|
+
rails_error_subscribers
|
|
41
|
+
web_also_runs_workers
|
|
42
|
+
flush_at_boot
|
|
43
|
+
plugin_versions
|
|
44
|
+
pro_ent_guards
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def time_fields
|
|
50
|
+
sites = grep(SOURCE_DIRS, TIME_FIELD_READ)
|
|
51
|
+
return if sites.empty?
|
|
52
|
+
|
|
53
|
+
@report.info('time_field_read',
|
|
54
|
+
"#{sites.size} place(s) read failed_at, created_at, enqueued_at or retried_at off a job. Wurk " \
|
|
55
|
+
'stores them as epoch milliseconds, as Sidekiq 8 does; code that reads seconds is off by 1000x: ' \
|
|
56
|
+
"#{listing(sites)}.")
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def connection_pool_pin
|
|
60
|
+
needed = wurk_requirement('connection_pool')
|
|
61
|
+
return unless needed
|
|
62
|
+
|
|
63
|
+
locked = locked_version('connection_pool')
|
|
64
|
+
return pin_unreadable(needed) unless locked
|
|
65
|
+
return if needed.satisfied_by?(locked)
|
|
66
|
+
|
|
67
|
+
@report.warning('connection_pool_pin', "Gemfile.lock holds connection_pool #{locked}; Wurk needs #{needed}" \
|
|
68
|
+
"#{gemfile_pin('connection_pool')}. Relax the pin before the swap.")
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def pin_unreadable(needed)
|
|
72
|
+
@report.info('connection_pool_pin',
|
|
73
|
+
"No Gemfile.lock under #{@root}. Wurk needs connection_pool #{needed}; check the bundle's with " \
|
|
74
|
+
"`bundle exec ruby -e 'puts Gem.loaded_specs[%(connection_pool)].version'`.")
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Jobs run inside the Rails reloader, and what raises there crosses
|
|
78
|
+
# Rails.error, so a subscriber may see an error on every attempt.
|
|
79
|
+
def rails_error_subscribers
|
|
80
|
+
reporter = defined?(::Rails) && ::Rails.respond_to?(:error) && ::Rails.error
|
|
81
|
+
subscribers = reporter ? Array(reporter.instance_variable_get(:@subscribers)) : []
|
|
82
|
+
return if subscribers.empty?
|
|
83
|
+
|
|
84
|
+
names = subscribers.map { |sub| sub.is_a?(::Module) ? sub.name : sub.class.name }.uniq
|
|
85
|
+
@report.info('rails_error_subscriber',
|
|
86
|
+
"Rails.error has #{subscribers.size} subscriber(s): #{listing(names)}. Jobs run inside the " \
|
|
87
|
+
'Rails reloader, so a subscriber may see a job error on every attempt, not only the last. ' \
|
|
88
|
+
'See docs/sentry.md.')
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Under a Rails web server Wurk boots workers in the web process too, so a
|
|
92
|
+
# Procfile that also starts a worker runs two, and boot code that flushes
|
|
93
|
+
# Redis runs in each.
|
|
94
|
+
def web_also_runs_workers
|
|
95
|
+
PROCFILES.each do |name|
|
|
96
|
+
entries = procfile(name)
|
|
97
|
+
web = entries['web']
|
|
98
|
+
next unless web && !web.include?('WURK_DISABLED=1') && WEB_SERVER_COMMAND.match?(web)
|
|
99
|
+
next unless entries.any? { |process, command| process != 'web' && RUNNER_COMMAND.match?(command) }
|
|
100
|
+
|
|
101
|
+
@report.warning('web_would_embed', "#{name} starts a worker, and its web process would boot workers too. " \
|
|
102
|
+
'Set WURK_DISABLED=1 on the web entry (or wherever its env is set).')
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def flush_at_boot
|
|
107
|
+
sites = grep(%w[config], FLUSH)
|
|
108
|
+
return if sites.empty?
|
|
109
|
+
|
|
110
|
+
@report.warning('flush_at_boot', 'Boot code flushes Redis, and a Wurk worker runs it in every process it ' \
|
|
111
|
+
"boots (every swarm child): #{listing(sites)}. Remove it, or scope it to " \
|
|
112
|
+
'one place that runs once.')
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def plugin_versions
|
|
116
|
+
TESTED_PLUGINS.each do |name, tested|
|
|
117
|
+
spec = @specs[name]
|
|
118
|
+
next if spec.nil? || spec.version.to_s == tested
|
|
119
|
+
|
|
120
|
+
@report.info('plugin_version', "#{name} #{spec.version} is bundled; its suite runs green against Wurk at " \
|
|
121
|
+
"#{tested}. Pin #{tested}, or run that version's suite before the swap.")
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def pro_ent_guards
|
|
126
|
+
sites = grep(SOURCE_DIRS, PRO_ENT_GUARD)
|
|
127
|
+
return if sites.empty?
|
|
128
|
+
|
|
129
|
+
@report.info('pro_ent_guard', "#{sites.size} branch(es) test for Sidekiq Pro or Enterprise. Under Wurk the " \
|
|
130
|
+
'`Sidekiq::Pro` and `Sidekiq::Enterprise` namespaces exist but hold only what ' \
|
|
131
|
+
'Wurk aliases, and `Sidekiq.pro?` and `.ent?` return false: check which ' \
|
|
132
|
+
"branch each one now takes: #{listing(sites)}.")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def grep(dirs, pattern)
|
|
136
|
+
dirs.flat_map { |dir| ::Dir.glob(::File.join(@root, dir, '**', '*.rb')).sort }.flat_map do |path|
|
|
137
|
+
::File.foreach(path).with_index(1).filter_map do |line, number|
|
|
138
|
+
"#{relative(path)}:#{number}" if pattern.match?(line.scrub)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def procfile(name)
|
|
144
|
+
path = ::File.join(@root, name)
|
|
145
|
+
return {} unless ::File.file?(path)
|
|
146
|
+
|
|
147
|
+
::File.foreach(path).each_with_object({}) do |line, out|
|
|
148
|
+
process, command = line.split(':', 2)
|
|
149
|
+
out[process.strip] = command.strip if command && !process.strip.start_with?('#')
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def wurk_requirement(name)
|
|
154
|
+
dependency = @specs['wurk']&.dependencies&.find { |dep| dep.name == name }
|
|
155
|
+
dependency&.requirement
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def locked_version(name)
|
|
159
|
+
lockfile = ::File.join(@root, 'Gemfile.lock')
|
|
160
|
+
return unless ::File.file?(lockfile)
|
|
161
|
+
|
|
162
|
+
version = ::File.read(lockfile)[/^ {4}#{::Regexp.escape(name)} \(([^)]+)\)$/, 1]
|
|
163
|
+
::Gem::Version.new(version) if version && ::Gem::Version.correct?(version)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def gemfile_pin(name)
|
|
167
|
+
gemfile = ::File.join(@root, 'Gemfile')
|
|
168
|
+
return '' unless ::File.file?(gemfile)
|
|
169
|
+
|
|
170
|
+
number = ::File.foreach(gemfile).with_index(1).find { |line, _| line.match?(/^\s*gem\s+['"]#{name}['"]/) }&.last
|
|
171
|
+
number ? ", and Gemfile:#{number} pins it" : ''
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def relative(path) = path.delete_prefix("#{@root}/")
|
|
175
|
+
|
|
176
|
+
def listing(sites)
|
|
177
|
+
extra = sites.size - MAX_SITES
|
|
178
|
+
sites.first(MAX_SITES).join(', ') + (extra.positive? ? " and #{extra} more" : '')
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
@@ -4,6 +4,7 @@ require 'logger'
|
|
|
4
4
|
require 'optparse'
|
|
5
5
|
require_relative '../preflight'
|
|
6
6
|
require_relative 'verify'
|
|
7
|
+
require_relative 'redis_guard'
|
|
7
8
|
|
|
8
9
|
module Wurk
|
|
9
10
|
module Preflight
|
|
@@ -81,7 +82,7 @@ module Wurk
|
|
|
81
82
|
|
|
82
83
|
def run_preflight(config)
|
|
83
84
|
Preflight.run(config, limit: @opts[:limit], sample: @opts[:sample], pause: pause, progress: progress,
|
|
84
|
-
app: @
|
|
85
|
+
app: @app_root)
|
|
85
86
|
end
|
|
86
87
|
|
|
87
88
|
def run_verify(config)
|
|
@@ -93,15 +94,30 @@ module Wurk
|
|
|
93
94
|
|
|
94
95
|
# WURK_DISABLED keeps the railtie from forking a swarm in this process. Server
|
|
95
96
|
# mode makes `configure_server` blocks run, so the Redis read is the one this
|
|
96
|
-
# app's workers would use.
|
|
97
|
+
# app's workers would use. Wurk's pools refuse writes while the app boots
|
|
98
|
+
# (RedisGuard), so an initializer that writes fails this run instead.
|
|
97
99
|
def boot_app
|
|
98
100
|
path = ::File.expand_path(@opts[:app])
|
|
99
|
-
@err.puts("wurk #{@name}: --app boots #{path} and runs its initializers.
|
|
100
|
-
|
|
101
|
+
@err.puts("wurk #{@name}: --app boots #{path} and runs its initializers. Wurk's Redis clients refuse writes " \
|
|
102
|
+
'while it boots; a client the app opens itself is not covered.')
|
|
101
103
|
::ENV['WURK_DISABLED'] = '1'
|
|
104
|
+
RedisGuard.install(:read_only, label: "wurk #{@name} --app")
|
|
102
105
|
@config.enter_server_mode
|
|
103
106
|
::Kernel.require(::File.directory?(path) ? ::File.join(path, 'config/environment') : path)
|
|
104
|
-
::Rails.application.eager_load! if
|
|
107
|
+
::Rails.application.eager_load! if rails_app
|
|
108
|
+
@app_root = app_root(path)
|
|
109
|
+
ensure
|
|
110
|
+
RedisGuard.uninstall
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def rails_app = defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
|
|
114
|
+
|
|
115
|
+
# Where the source rows read: the directory named, else the booted Rails
|
|
116
|
+
# app's root, else the named file's directory.
|
|
117
|
+
def app_root(path)
|
|
118
|
+
return path if ::File.directory?(path)
|
|
119
|
+
|
|
120
|
+
rails_app ? ::Rails.root.to_s : ::File.dirname(path)
|
|
105
121
|
end
|
|
106
122
|
|
|
107
123
|
def parser
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rbconfig'
|
|
4
|
+
require_relative 'reader'
|
|
5
|
+
require_relative '../redis_pool'
|
|
6
|
+
|
|
7
|
+
module Wurk
|
|
8
|
+
module Preflight
|
|
9
|
+
# What keeps app code off Redis while `wurk boot-check` or `wurk preflight
|
|
10
|
+
# --app` boots the app. Their initializers run, `configure_server` blocks
|
|
11
|
+
# included, and some of those write Redis at boot (a dev-only `flushdb`, a
|
|
12
|
+
# seed). Installed before the app loads, it rides every pool Wurk builds
|
|
13
|
+
# (RedisPool.guard) as a redis-client middleware, so `Sidekiq.redis`,
|
|
14
|
+
# `Wurk.redis` and every Wurk data view are covered whichever method or
|
|
15
|
+
# pipeline sends the command. A client the app opens itself (`Redis.new`,
|
|
16
|
+
# `Rails.cache`) is not Wurk's and is not covered.
|
|
17
|
+
module RedisGuard
|
|
18
|
+
# Not a RedisClient::Error, so no pool or caller retries it as a blip.
|
|
19
|
+
class Refused < StandardError; end
|
|
20
|
+
|
|
21
|
+
# Sent by redis-client itself while it dials; none of them changes data.
|
|
22
|
+
HANDSHAKE = %w[HELLO AUTH SELECT ROLE PING].freeze
|
|
23
|
+
CLIENT_HANDSHAKE = %w[SETINFO SETNAME].freeze
|
|
24
|
+
LIB = ::File.expand_path('../..', __dir__)
|
|
25
|
+
GEM_FRAMES = %r{/(?:redis-client|hiredis-client|connection_pool)-[^/]+/}
|
|
26
|
+
STDLIB = ::RbConfig::CONFIG['rubylibdir']
|
|
27
|
+
|
|
28
|
+
# Opens no socket at all, so no command of any kind reaches the server.
|
|
29
|
+
module Closed
|
|
30
|
+
def connect(_config)
|
|
31
|
+
raise Refused, RedisGuard.message('opened a Redis connection', 'runs no Redis command')
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Lets through the reads a preflight may send, and the handshake.
|
|
36
|
+
module ReadOnly
|
|
37
|
+
def call(command, _config)
|
|
38
|
+
RedisGuard.check!(command)
|
|
39
|
+
super
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def call_pipelined(commands, _config)
|
|
43
|
+
commands.each { |command| RedisGuard.check!(command) }
|
|
44
|
+
super
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
class << self
|
|
49
|
+
attr_reader :label
|
|
50
|
+
|
|
51
|
+
# `mode` is :closed or :read_only. Process-wide, as the boot it guards is.
|
|
52
|
+
def install(mode, label:)
|
|
53
|
+
@label = label
|
|
54
|
+
RedisPool.guard = mode == :closed ? Closed : ReadOnly
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def uninstall
|
|
58
|
+
RedisPool.guard = nil
|
|
59
|
+
@label = nil
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def check!(command)
|
|
63
|
+
return if allowed?(command)
|
|
64
|
+
|
|
65
|
+
raise Refused, message("sent #{command.first.to_s.upcase}", 'sends only reads')
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def allowed?(command)
|
|
69
|
+
name = command.first.to_s.upcase
|
|
70
|
+
return CLIENT_HANDSHAKE.include?(command[1].to_s.upcase) if name == 'CLIENT'
|
|
71
|
+
|
|
72
|
+
Reader::ALLOWED.include?(name) || HANDSHAKE.include?(name)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def message(what, policy)
|
|
76
|
+
"#{label}: #{culprit} #{what} while the app booted, and #{label} #{policy}. App code that touches Redis " \
|
|
77
|
+
'at boot (a flushdb, a seed) would reach a real server here. Move it into `config.on(:startup)`, which ' \
|
|
78
|
+
"only a running worker fires#{', or pass --redis to allow reads' if policy.start_with?('runs')}."
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# The first frame outside Wurk, the Redis client and the pool: the app
|
|
82
|
+
# file (an initializer, a gem's railtie) that asked for Redis.
|
|
83
|
+
def culprit(locations = caller_locations)
|
|
84
|
+
frame = locations.find { |loc| app_frame?(loc.absolute_path || loc.path) }
|
|
85
|
+
frame ? "#{frame.path}:#{frame.lineno}" : 'app code'
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
def app_frame?(path)
|
|
91
|
+
return false if path.nil? || path.start_with?('<internal:', LIB, STDLIB)
|
|
92
|
+
|
|
93
|
+
!GEM_FRAMES.match?(path)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
data/lib/wurk/preflight.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative 'preflight/report'
|
|
|
5
5
|
require_relative 'preflight/keyspace'
|
|
6
6
|
require_relative 'preflight/owners'
|
|
7
7
|
require_relative 'preflight/checks'
|
|
8
|
+
require_relative 'preflight/app_scan'
|
|
8
9
|
|
|
9
10
|
module Wurk
|
|
10
11
|
# `wurk preflight`: what a live Redis holds, and what will and will not carry
|
|
@@ -16,8 +17,9 @@ module Wurk
|
|
|
16
17
|
|
|
17
18
|
module_function
|
|
18
19
|
|
|
20
|
+
# `app` is the root of the booted app (`--app`), or nil.
|
|
19
21
|
# @return [Report]
|
|
20
|
-
def run(config, limit: DEFAULT_LIMIT, sample: DEFAULT_SAMPLE, pause: 0.005, progress: nil, app:
|
|
22
|
+
def run(config, limit: DEFAULT_LIMIT, sample: DEFAULT_SAMPLE, pause: 0.005, progress: nil, app: nil)
|
|
21
23
|
reader = Reader.new(config)
|
|
22
24
|
report = Report.new('preflight')
|
|
23
25
|
keyspace = Keyspace.new(reader, limit: limit, pause: pause, progress: progress).walk
|
|
@@ -25,16 +27,18 @@ module Wurk
|
|
|
25
27
|
checks = Checks.new(reader: reader, report: report, keyspace: keyspace, owners: Owners.new(reader),
|
|
26
28
|
sample: sample)
|
|
27
29
|
checks.run
|
|
28
|
-
app ? app_findings(checks, report) : report.info('app_checks_skipped', 'No --app:
|
|
30
|
+
app ? app_findings(checks, report, app) : report.info('app_checks_skipped', 'No --app: app code not inspected.')
|
|
29
31
|
report
|
|
30
32
|
ensure
|
|
31
33
|
reader&.close
|
|
32
34
|
end
|
|
33
35
|
|
|
34
|
-
def app_findings(checks, report)
|
|
36
|
+
def app_findings(checks, report, root)
|
|
35
37
|
checks.app
|
|
36
|
-
|
|
37
|
-
|
|
38
|
+
AppScan.new(root: root, report: report).run
|
|
39
|
+
report.info(Report::APP_CODE_RAN, "The app was booted, so its initializers ran. Wurk's own Redis clients sent " \
|
|
40
|
+
'reads only while it booted, but a client the app opens itself ' \
|
|
41
|
+
'(Redis.new, Rails.cache) is not covered by that.')
|
|
38
42
|
end
|
|
39
43
|
end
|
|
40
44
|
end
|
data/lib/wurk/rails_boot.rb
CHANGED
|
@@ -263,6 +263,12 @@ module Wurk
|
|
|
263
263
|
swarm.shutdown unless supervisor&.alive?
|
|
264
264
|
end
|
|
265
265
|
|
|
266
|
+
def check_embedded_boot(config)
|
|
267
|
+
BootChecks.log_process_role(config, "embedded in #{server_name}, threads only")
|
|
268
|
+
BootChecks.run(config, nil)
|
|
269
|
+
BootChecks.warn_on_ignored_topology(config, "a worker embedded in #{server_name}")
|
|
270
|
+
end
|
|
271
|
+
|
|
266
272
|
# Sidekiq-embedded parity: a threads-only worker inside the web process, no
|
|
267
273
|
# fork. Redis validation failure keeps the host serving HTTP (log + carry
|
|
268
274
|
# on). at_exit drains on the graceful shutdown the web server runs on TERM.
|
|
@@ -274,8 +280,7 @@ module Wurk
|
|
|
274
280
|
# already running with nothing to drain them on host exit.
|
|
275
281
|
at_exit { instance.stop }
|
|
276
282
|
instance.run
|
|
277
|
-
|
|
278
|
-
BootChecks.run(Wurk.configuration, nil)
|
|
283
|
+
check_embedded_boot(Wurk.configuration)
|
|
279
284
|
instance
|
|
280
285
|
rescue StandardError => e
|
|
281
286
|
logger.error { "wurk: embedded boot failed: #{e.class}: #{e.message}" }
|
data/lib/wurk/redis_options.rb
CHANGED
|
@@ -109,6 +109,11 @@ module Wurk
|
|
|
109
109
|
"#{unknown.map(&:inspect).join(', ')}. Supported keys: #{known_keys.sort.join(', ')}."
|
|
110
110
|
end
|
|
111
111
|
|
|
112
|
+
# `driver: :hiredis`, the C driver a fiber scheduler cannot switch away from.
|
|
113
|
+
def fiber_blocking_driver?(driver)
|
|
114
|
+
driver.to_s.include?('hiredis')
|
|
115
|
+
end
|
|
116
|
+
|
|
112
117
|
# Sentinel sets go through RedisClient.sentinel — RedisClient.config
|
|
113
118
|
# rejects `sentinels:` outright. Same routing Sidekiq does.
|
|
114
119
|
def sentinel?(client_config)
|
data/lib/wurk/redis_pool.rb
CHANGED
|
@@ -114,6 +114,13 @@ module Wurk
|
|
|
114
114
|
|
|
115
115
|
attr_reader :size, :url, :name, :pool_timeout, :client_config, :redis_idle_timeout
|
|
116
116
|
|
|
117
|
+
class << self
|
|
118
|
+
# A redis-client middleware every pool built from now on carries, after
|
|
119
|
+
# any the host configured. Set only by the boot of a command that must not
|
|
120
|
+
# write Redis (Preflight::RedisGuard); nil everywhere else.
|
|
121
|
+
attr_accessor :guard
|
|
122
|
+
end
|
|
123
|
+
|
|
117
124
|
# Takes the standard Sidekiq `config.redis` hash: `pool_timeout` tunes the
|
|
118
125
|
# ConnectionPool checkout; `connect_timeout`/`read_timeout`/`write_timeout`/
|
|
119
126
|
# `reconnect_attempts` plus any other redis-client key (driver, ssl_params,
|
|
@@ -226,16 +233,32 @@ module Wurk
|
|
|
226
233
|
# …) so this class stays about pooling; host-supplied keys win over the
|
|
227
234
|
# defaults.
|
|
228
235
|
def build_client_config(options)
|
|
229
|
-
RedisOptions.normalize(options, defaults: DEFAULT_CLIENT_CONFIG)
|
|
236
|
+
config = RedisOptions.normalize(options, defaults: DEFAULT_CLIENT_CONFIG)
|
|
237
|
+
config[:middlewares] = [*(config[:middlewares] || []), self.class.guard] if self.class.guard
|
|
238
|
+
config.freeze
|
|
230
239
|
end
|
|
231
240
|
|
|
232
241
|
# Wrapped in the CompatClient decorator so `Sidekiq.redis { |c| c.smembers }`
|
|
233
242
|
# method-style commands work like Sidekiq 7+ (#204). Wurk's own code paths
|
|
234
243
|
# use #call, which the decorator forwards.
|
|
235
244
|
def build_client
|
|
245
|
+
warn_on_fiber_blocking_driver
|
|
236
246
|
RedisClientAdapter::CompatClient.new(redis_client_config.new_client)
|
|
237
247
|
end
|
|
238
248
|
|
|
249
|
+
# hiredis waits on the socket in C, where a fiber scheduler (Falcon, Async)
|
|
250
|
+
# cannot switch away, so one slow reply stalls every fiber on the thread.
|
|
251
|
+
# Checked on the connection build, the first moment a scheduler is visible.
|
|
252
|
+
def warn_on_fiber_blocking_driver
|
|
253
|
+
return if @fiber_warned || ::Fiber.scheduler.nil? || !RedisOptions.fiber_blocking_driver?(@client_config[:driver])
|
|
254
|
+
|
|
255
|
+
@fiber_warned = true
|
|
256
|
+
Wurk.logger.warn do
|
|
257
|
+
"Redis pool #{@name} uses the hiredis driver under a fiber scheduler, which blocks every fiber on the " \
|
|
258
|
+
'thread while it waits on Redis. Use `driver: :ruby` in the config.redis of this process.'
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
239
262
|
# A Sentinel set is a different constructor, not a different keyword:
|
|
240
263
|
# `RedisClient.config(sentinels: [...])` raises. Sidekiq routes the same way.
|
|
241
264
|
def redis_client_config
|