wurk 1.12.0 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,7 @@ module Wurk
12
12
  # 1. prepare(job_hash) { ... } → sets thread-local context, applies
13
13
  # per-job log_level, yields to the rest of dispatch.
14
14
  # 2. call(item, queue) { ... } → wraps the actual perform with the
15
- # start/done/fail log line trio and the elapsed-ms measurement.
15
+ # start/done/fail log line trio and the elapsed-seconds measurement.
16
16
  #
17
17
  # Skipping default logging is controlled by config[:skip_default_job_logging];
18
18
  # the prepare step still runs so context/log_level still apply.
@@ -68,8 +68,10 @@ module Wurk
68
68
 
69
69
  private
70
70
 
71
+ # Seconds, as upstream logs it (`elapsed=0.007`): log scrapers and
72
+ # dashboards built on Sidekiq's line read the field that way.
71
73
  def elapsed(start)
72
- ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000).round(3)
74
+ (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
73
75
  end
74
76
 
75
77
  # The attribute list is a host setting (defaults to %w[bid tags]), so the
data/lib/wurk/job_util.rb CHANGED
@@ -73,6 +73,20 @@ module Wurk
73
73
  seconds.finite? && seconds.positive?
74
74
  end
75
75
 
76
+ # A Duration's `to_s`: "5" for `5.seconds`, "1.5" for `1.5.seconds`. Same
77
+ # shape {Wurk::Unique.coerce_ttl} reads back from a `unique_for`.
78
+ NUMERIC_SECONDS = /\A\d+(?:\.\d+)?\z/
79
+
80
+ # The read side of a bound, as a Float or nil. Wider than
81
+ # {positive_seconds?} by one shape only: wurk < 1.12 wrote a Duration
82
+ # `timeout:` into the payload as its String ("5"), and those jobs are still
83
+ # queued. Never used to decide what is written; {validate_bounds!} keeps
84
+ # rejecting a String at declaration.
85
+ def self.payload_seconds(value)
86
+ value = value.to_f if value.is_a?(::String) && NUMERIC_SECONDS.match?(value)
87
+ value.to_f if positive_seconds?(value)
88
+ end
89
+
76
90
  # The exact bytes a job is stored as inside the `schedule` ZSET. `at` is
77
91
  # dropped because it *is* the score, and `enqueued_at` because it marks
78
92
  # arrival on an immediate queue — the promoter restamps it on the way out
data/lib/wurk/launcher.rb CHANGED
@@ -47,6 +47,20 @@ module Wurk
47
47
  # (Sidekiq's drop-in surface). The single source of truth is Heartbeat.
48
48
  BEAT_PAUSE = Heartbeat::BEAT_PAUSE
49
49
 
50
+ # The process title, upstream's format and program name word for word:
51
+ # `sidekiq 8.1.5 myapp [3 of 10 busy]`, then `stopping` once quieted. Probes
52
+ # and runbooks that `ps | grep '^sidekiq'` or read the busy count keep
53
+ # working across the gem swap. Mutable like upstream's, so code that appends
54
+ # its own segment still can. docs/target/ is silent; the oracle is Sidekiq
55
+ # 8.1's Launcher (docs/deployment.md, "Process titles").
56
+ PROCTITLES = [ # rubocop:disable Style/MutableConstant
57
+ proc { 'sidekiq' },
58
+ proc { ::Sidekiq::VERSION },
59
+ proc { |_me, data| data['tag'] },
60
+ proc { |me, _data| "[#{Processor::WORK_STATE.size} of #{me.config.total_concurrency} busy]" },
61
+ proc { |me, _data| 'stopping' if me.stopping? }
62
+ ]
63
+
50
64
  # Bound on how long #stop waits for the boot-time reclaim sweep before
51
65
  # moving on — it can still be scanning a large keyspace when a fast
52
66
  # shutdown lands right after boot; teardown must not hang on it.
@@ -96,9 +110,12 @@ module Wurk
96
110
  # 3. start the managers. Everything below is periodic background work
97
111
  # whose first tick is seconds away, so it has nothing to gain from
98
112
  # going ahead of them.
99
- # 4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
100
- # not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
101
- # so it stays ahead of the periodic loops.
113
+ # 4. start the reaper and the boot-time reclaim sweep, ahead of the
114
+ # periodic loops: a process restarted outside a swarm (exe/wurk under
115
+ # systemd, a pod) has its previous incarnation's jobs recovered from
116
+ # here. The sweep takes the cluster reaper lock, so a fleet booting
117
+ # together makes one keyspace pass; a swarm child's SIGKILLed sibling
118
+ # is recovered by the parent, not by this sweep.
102
119
  # 5. write the `config.periodic` registrations (Ent registers loops on
103
120
  # startup), then start the periodic loops. None of them ticks at
104
121
  # zero — TimerLoop#run waits an interval before its first yield,
@@ -404,12 +421,22 @@ module Wurk
404
421
  # the dashboard queued at `<identity>-signals`. Lazily builds the
405
422
  # Heartbeat the first time we beat so callers that bypass `run`
406
423
  # (embedded mode, tests) still work.
424
+ # An embedded launcher shares its process with the host, whose title it is.
407
425
  def beat
426
+ $PROGRAM_NAME = proctitle unless @embedded
408
427
  ensure_heartbeat
409
428
  sigs = @heartbeat.beat!
410
429
  sigs&.each { |sig| dispatch_signal(sig) }
411
430
  end
412
431
 
432
+ # Upstream hands each segment the process's heartbeat data; these are the
433
+ # fields of it a title can use without a Redis read.
434
+ def proctitle
435
+ data = { 'tag' => @config[:tag] || default_tag, 'identity' => identity, 'hostname' => hostname,
436
+ 'pid' => ::Process.pid, 'concurrency' => @config.total_concurrency }
437
+ PROCTITLES.map { |segment| segment.call(self, data) }.compact.join(' ')
438
+ end
439
+
413
440
  def ensure_heartbeat
414
441
  return if @heartbeat
415
442
 
data/lib/wurk/leader.rb CHANGED
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'securerandom'
4
- require 'socket'
5
4
  require_relative 'component'
6
5
  require_relative 'fetcher/oom_drain'
7
6
  require_relative 'lua'
@@ -282,13 +281,7 @@ module Wurk
282
281
  # Matches `Wurk::Component#identity` so `ProcessSet::Process#leader?`
283
282
  # (which compares `dear-leader == process.identity`) sees a match.
284
283
  def cluster_identity
285
- "#{hostname}:#{::Process.pid}:#{Component::PROCESS_NONCE}"
286
- end
287
-
288
- def hostname
289
- ENV['DYNO'] || ::Socket.gethostname
290
- rescue StandardError
291
- 'localhost'
284
+ Component.identity
292
285
  end
293
286
 
294
287
  def spawn_loop_thread
data/lib/wurk/manager.rb CHANGED
@@ -167,12 +167,38 @@ module Wurk
167
167
 
168
168
  jobs = cleanup.map(&:job).compact
169
169
  logger.warn { "Terminating #{cleanup.size} busy threads" }
170
- logger.debug { "Jobs still in progress #{jobs.inspect}" }
170
+ log_interrupted(jobs)
171
171
  # `&.` like #quiet: a TERM in the pre-prepare! window (traps install
172
172
  # before launcher.run) reaches here with no fetcher built yet.
173
173
  capsule.fetcher&.bulk_requeue(jobs)
174
174
  end
175
175
 
176
+ # One WARN per job a deploy cut short, because each one runs again from the
177
+ # top (docs/idempotency.md) and the operator chasing a double side effect
178
+ # needs its jid. Capped: a capsule can hold hundreds of threads.
179
+ REQUEUE_LOG_LIMIT = 20
180
+ private_constant :REQUEUE_LOG_LIMIT
181
+
182
+ def log_interrupted(jobs)
183
+ jobs.first(REQUEUE_LOG_LIMIT).each do |uow|
184
+ logger.warn { "Interrupting #{describe_job(uow)}: past the shutdown deadline, it is requeued to run again" }
185
+ end
186
+ extra = jobs.size - REQUEUE_LOG_LIMIT
187
+ logger.warn { "Interrupting #{extra} more in-flight jobs, which run again too" } if extra.positive?
188
+ end
189
+
190
+ # The payload is parsed here, off the hot path, because the fetcher never
191
+ # parses. A plug-in fetcher's unit of work may carry something that is not
192
+ # job JSON; it is named by its queue alone.
193
+ def describe_job(uow)
194
+ job = Wurk.load_json(uow.job)
195
+ job = {} unless job.is_a?(::Hash)
196
+ klass = job['wrapped'] || job['class']
197
+ "#{klass} jid=#{job['jid']} queue=#{uow.queue_name}"
198
+ rescue ::JSON::ParserError, ::TypeError
199
+ "job queue=#{uow.queue_name}"
200
+ end
201
+
176
202
  # The @workers Set is mutated from Processor threads (processor_result)
177
203
  # while the lifecycle methods read/iterate it from the manager thread.
178
204
  # Snapshot under @plock, then act on the copy outside the lock.
@@ -86,7 +86,7 @@ module Wurk
86
86
  def within_attempt(timer, job, seconds, &)
87
87
  return yield unless seconds
88
88
 
89
- timer.watch(seconds, ::Wurk::Job::TimedOut, timeout_message(job, seconds), &)
89
+ timer.watch(seconds, ::Wurk::Job::TimedOut, timeout_message(job), &)
90
90
  end
91
91
 
92
92
  # A bound written through Wurk is type-checked at declaration, where the
@@ -95,8 +95,7 @@ module Wurk
95
95
  # passed that check, and a bound we can't use on one of those has to leave
96
96
  # the job unbounded rather than break it.
97
97
  def bound_for(job)
98
- seconds = job['timeout']
99
- seconds if ::Wurk::JobUtil.positive_seconds?(seconds)
98
+ ::Wurk::JobUtil.payload_seconds(job['timeout'])
100
99
  end
101
100
 
102
101
  # What is left of the absolute cutoff, in seconds. Negative is not an
@@ -122,9 +121,10 @@ module Wurk
122
121
 
123
122
  # What an operator reads in the retry set, the dead set, a tracked job's
124
123
  # status row, and their error tracker — so each names the class and the
125
- # bound it broke.
126
- def timeout_message(job, seconds)
127
- "#{job['class']} timed out after #{seconds}s"
124
+ # bound it broke, as the payload carries it: `timeout: 5` reads "5s", not
125
+ # the Float it was armed as.
126
+ def timeout_message(job)
127
+ "#{job['class']} timed out after #{job['timeout']}s"
128
128
  end
129
129
 
130
130
  def deadline_message(job)
@@ -0,0 +1,182 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ module Preflight
5
+ # `wurk preflight --app` rows read from the app's source and bundle, not from
6
+ # Redis: the patterns that carried over from stock Sidekiq and behave
7
+ # differently on Wurk. A text scan, so a row is a place to look, not a verdict.
8
+ class AppScan
9
+ SOURCE_DIRS = %w[app lib config].freeze
10
+ MAX_SITES = 20
11
+
12
+ # Sidekiq 8 and Wurk store these as epoch milliseconds; Sidekiq 7 stored
13
+ # float seconds, and code written against it reads them that way.
14
+ TIME_FIELD_READ = /\[['"](?:failed_at|created_at|enqueued_at|retried_at)['"]\]/
15
+ FLUSH = /\b(?:flushdb|flushall)\b/i
16
+ PRO_ENT_GUARD = /defined\?\(\s*(?:::)?Sidekiq::(?:Pro|Enterprise|Ent)\b|Sidekiq\.(?:pro|ent)\?/
17
+ RUNNER_COMMAND = /\b(?:wurk|wurkswarm|sidekiq|sidekiqswarm)\b/
18
+ WEB_SERVER_COMMAND = /\brails\s+(?:s|server)\b|\bpuma\b/
19
+ PROCFILES = %w[Procfile Procfile.dev].freeze
20
+
21
+ # The upstream versions each plugin's own suite runs green against Wurk
22
+ # (`ref:` in test/ecosystem/<gem>/PIN, which preflight_app_scan_test.rb
23
+ # holds this to).
24
+ TESTED_PLUGINS = {
25
+ 'sentry-sidekiq' => '6.5.0',
26
+ 'sidekiq-cron' => '2.4.0',
27
+ 'sidekiq-failures' => '1.1.0',
28
+ 'sidekiq-unique-jobs' => '8.1.0'
29
+ }.freeze
30
+
31
+ def initialize(root:, report:, specs: ::Gem.loaded_specs)
32
+ @root = root
33
+ @report = report
34
+ @specs = specs
35
+ end
36
+
37
+ def run
38
+ time_fields
39
+ connection_pool_pin
40
+ rails_error_subscribers
41
+ web_also_runs_workers
42
+ flush_at_boot
43
+ plugin_versions
44
+ pro_ent_guards
45
+ end
46
+
47
+ private
48
+
49
+ def time_fields
50
+ sites = grep(SOURCE_DIRS, TIME_FIELD_READ)
51
+ return if sites.empty?
52
+
53
+ @report.info('time_field_read',
54
+ "#{sites.size} place(s) read failed_at, created_at, enqueued_at or retried_at off a job. Wurk " \
55
+ 'stores them as epoch milliseconds, as Sidekiq 8 does; code that reads seconds is off by 1000x: ' \
56
+ "#{listing(sites)}.")
57
+ end
58
+
59
+ def connection_pool_pin
60
+ needed = wurk_requirement('connection_pool')
61
+ return unless needed
62
+
63
+ locked = locked_version('connection_pool')
64
+ return pin_unreadable(needed) unless locked
65
+ return if needed.satisfied_by?(locked)
66
+
67
+ @report.warning('connection_pool_pin', "Gemfile.lock holds connection_pool #{locked}; Wurk needs #{needed}" \
68
+ "#{gemfile_pin('connection_pool')}. Relax the pin before the swap.")
69
+ end
70
+
71
+ def pin_unreadable(needed)
72
+ @report.info('connection_pool_pin',
73
+ "No Gemfile.lock under #{@root}. Wurk needs connection_pool #{needed}; check the bundle's with " \
74
+ "`bundle exec ruby -e 'puts Gem.loaded_specs[%(connection_pool)].version'`.")
75
+ end
76
+
77
+ # Jobs run inside the Rails reloader, and what raises there crosses
78
+ # Rails.error, so a subscriber may see an error on every attempt.
79
+ def rails_error_subscribers
80
+ reporter = defined?(::Rails) && ::Rails.respond_to?(:error) && ::Rails.error
81
+ subscribers = reporter ? Array(reporter.instance_variable_get(:@subscribers)) : []
82
+ return if subscribers.empty?
83
+
84
+ names = subscribers.map { |sub| sub.is_a?(::Module) ? sub.name : sub.class.name }.uniq
85
+ @report.info('rails_error_subscriber',
86
+ "Rails.error has #{subscribers.size} subscriber(s): #{listing(names)}. Jobs run inside the " \
87
+ 'Rails reloader, so a subscriber may see a job error on every attempt, not only the last. ' \
88
+ 'See docs/sentry.md.')
89
+ end
90
+
91
+ # Under a Rails web server Wurk boots workers in the web process too, so a
92
+ # Procfile that also starts a worker runs two, and boot code that flushes
93
+ # Redis runs in each.
94
+ def web_also_runs_workers
95
+ PROCFILES.each do |name|
96
+ entries = procfile(name)
97
+ web = entries['web']
98
+ next unless web && !web.include?('WURK_DISABLED=1') && WEB_SERVER_COMMAND.match?(web)
99
+ next unless entries.any? { |process, command| process != 'web' && RUNNER_COMMAND.match?(command) }
100
+
101
+ @report.warning('web_would_embed', "#{name} starts a worker, and its web process would boot workers too. " \
102
+ 'Set WURK_DISABLED=1 on the web entry (or wherever its env is set).')
103
+ end
104
+ end
105
+
106
+ def flush_at_boot
107
+ sites = grep(%w[config], FLUSH)
108
+ return if sites.empty?
109
+
110
+ @report.warning('flush_at_boot', 'Boot code flushes Redis, and a Wurk worker runs it in every process it ' \
111
+ "boots (every swarm child): #{listing(sites)}. Remove it, or scope it to " \
112
+ 'one place that runs once.')
113
+ end
114
+
115
+ def plugin_versions
116
+ TESTED_PLUGINS.each do |name, tested|
117
+ spec = @specs[name]
118
+ next if spec.nil? || spec.version.to_s == tested
119
+
120
+ @report.info('plugin_version', "#{name} #{spec.version} is bundled; its suite runs green against Wurk at " \
121
+ "#{tested}. Pin #{tested}, or run that version's suite before the swap.")
122
+ end
123
+ end
124
+
125
+ def pro_ent_guards
126
+ sites = grep(SOURCE_DIRS, PRO_ENT_GUARD)
127
+ return if sites.empty?
128
+
129
+ @report.info('pro_ent_guard', "#{sites.size} branch(es) test for Sidekiq Pro or Enterprise. Under Wurk the " \
130
+ '`Sidekiq::Pro` and `Sidekiq::Enterprise` namespaces exist but hold only what ' \
131
+ 'Wurk aliases, and `Sidekiq.pro?` and `.ent?` return false: check which ' \
132
+ "branch each one now takes: #{listing(sites)}.")
133
+ end
134
+
135
+ def grep(dirs, pattern)
136
+ dirs.flat_map { |dir| ::Dir.glob(::File.join(@root, dir, '**', '*.rb')).sort }.flat_map do |path|
137
+ ::File.foreach(path).with_index(1).filter_map do |line, number|
138
+ "#{relative(path)}:#{number}" if pattern.match?(line.scrub)
139
+ end
140
+ end
141
+ end
142
+
143
+ def procfile(name)
144
+ path = ::File.join(@root, name)
145
+ return {} unless ::File.file?(path)
146
+
147
+ ::File.foreach(path).each_with_object({}) do |line, out|
148
+ process, command = line.split(':', 2)
149
+ out[process.strip] = command.strip if command && !process.strip.start_with?('#')
150
+ end
151
+ end
152
+
153
+ def wurk_requirement(name)
154
+ dependency = @specs['wurk']&.dependencies&.find { |dep| dep.name == name }
155
+ dependency&.requirement
156
+ end
157
+
158
+ def locked_version(name)
159
+ lockfile = ::File.join(@root, 'Gemfile.lock')
160
+ return unless ::File.file?(lockfile)
161
+
162
+ version = ::File.read(lockfile)[/^ {4}#{::Regexp.escape(name)} \(([^)]+)\)$/, 1]
163
+ ::Gem::Version.new(version) if version && ::Gem::Version.correct?(version)
164
+ end
165
+
166
+ def gemfile_pin(name)
167
+ gemfile = ::File.join(@root, 'Gemfile')
168
+ return '' unless ::File.file?(gemfile)
169
+
170
+ number = ::File.foreach(gemfile).with_index(1).find { |line, _| line.match?(/^\s*gem\s+['"]#{name}['"]/) }&.last
171
+ number ? ", and Gemfile:#{number} pins it" : ''
172
+ end
173
+
174
+ def relative(path) = path.delete_prefix("#{@root}/")
175
+
176
+ def listing(sites)
177
+ extra = sites.size - MAX_SITES
178
+ sites.first(MAX_SITES).join(', ') + (extra.positive? ? " and #{extra} more" : '')
179
+ end
180
+ end
181
+ end
182
+ end
@@ -4,6 +4,7 @@ require 'logger'
4
4
  require 'optparse'
5
5
  require_relative '../preflight'
6
6
  require_relative 'verify'
7
+ require_relative 'redis_guard'
7
8
 
8
9
  module Wurk
9
10
  module Preflight
@@ -81,7 +82,7 @@ module Wurk
81
82
 
82
83
  def run_preflight(config)
83
84
  Preflight.run(config, limit: @opts[:limit], sample: @opts[:sample], pause: pause, progress: progress,
84
- app: @opts[:app] ? true : false)
85
+ app: @app_root)
85
86
  end
86
87
 
87
88
  def run_verify(config)
@@ -93,15 +94,30 @@ module Wurk
93
94
 
94
95
  # WURK_DISABLED keeps the railtie from forking a swarm in this process. Server
95
96
  # mode makes `configure_server` blocks run, so the Redis read is the one this
96
- # app's workers would use.
97
+ # app's workers would use. Wurk's pools refuse writes while the app boots
98
+ # (RedisGuard), so an initializer that writes fails this run instead.
97
99
  def boot_app
98
100
  path = ::File.expand_path(@opts[:app])
99
- @err.puts("wurk #{@name}: --app boots #{path} and runs its initializers. Anything they do is not read-only, " \
100
- "and an enqueue goes to the app's Redis, whatever --redis-url says.")
101
+ @err.puts("wurk #{@name}: --app boots #{path} and runs its initializers. Wurk's Redis clients refuse writes " \
102
+ 'while it boots; a client the app opens itself is not covered.')
101
103
  ::ENV['WURK_DISABLED'] = '1'
104
+ RedisGuard.install(:read_only, label: "wurk #{@name} --app")
102
105
  @config.enter_server_mode
103
106
  ::Kernel.require(::File.directory?(path) ? ::File.join(path, 'config/environment') : path)
104
- ::Rails.application.eager_load! if defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
107
+ ::Rails.application.eager_load! if rails_app
108
+ @app_root = app_root(path)
109
+ ensure
110
+ RedisGuard.uninstall
111
+ end
112
+
113
+ def rails_app = defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
114
+
115
+ # Where the source rows read: the directory named, else the booted Rails
116
+ # app's root, else the named file's directory.
117
+ def app_root(path)
118
+ return path if ::File.directory?(path)
119
+
120
+ rails_app ? ::Rails.root.to_s : ::File.dirname(path)
105
121
  end
106
122
 
107
123
  def parser
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rbconfig'
4
+ require_relative 'reader'
5
+ require_relative '../redis_pool'
6
+
7
+ module Wurk
8
+ module Preflight
9
+ # What keeps app code off Redis while `wurk boot-check` or `wurk preflight
10
+ # --app` boots the app. Their initializers run, `configure_server` blocks
11
+ # included, and some of those write Redis at boot (a dev-only `flushdb`, a
12
+ # seed). Installed before the app loads, it rides every pool Wurk builds
13
+ # (RedisPool.guard) as a redis-client middleware, so `Sidekiq.redis`,
14
+ # `Wurk.redis` and every Wurk data view are covered whichever method or
15
+ # pipeline sends the command. A client the app opens itself (`Redis.new`,
16
+ # `Rails.cache`) is not Wurk's and is not covered.
17
+ module RedisGuard
18
+ # Not a RedisClient::Error, so no pool or caller retries it as a blip.
19
+ class Refused < StandardError; end
20
+
21
+ # Sent by redis-client itself while it dials; none of them changes data.
22
+ HANDSHAKE = %w[HELLO AUTH SELECT ROLE PING].freeze
23
+ CLIENT_HANDSHAKE = %w[SETINFO SETNAME].freeze
24
+ LIB = ::File.expand_path('../..', __dir__)
25
+ GEM_FRAMES = %r{/(?:redis-client|hiredis-client|connection_pool)-[^/]+/}
26
+ STDLIB = ::RbConfig::CONFIG['rubylibdir']
27
+
28
+ # Opens no socket at all, so no command of any kind reaches the server.
29
+ module Closed
30
+ def connect(_config)
31
+ raise Refused, RedisGuard.message('opened a Redis connection', 'runs no Redis command')
32
+ end
33
+ end
34
+
35
+ # Lets through the reads a preflight may send, and the handshake.
36
+ module ReadOnly
37
+ def call(command, _config)
38
+ RedisGuard.check!(command)
39
+ super
40
+ end
41
+
42
+ def call_pipelined(commands, _config)
43
+ commands.each { |command| RedisGuard.check!(command) }
44
+ super
45
+ end
46
+ end
47
+
48
+ class << self
49
+ attr_reader :label
50
+
51
+ # `mode` is :closed or :read_only. Process-wide, as the boot it guards is.
52
+ def install(mode, label:)
53
+ @label = label
54
+ RedisPool.guard = mode == :closed ? Closed : ReadOnly
55
+ end
56
+
57
+ def uninstall
58
+ RedisPool.guard = nil
59
+ @label = nil
60
+ end
61
+
62
+ def check!(command)
63
+ return if allowed?(command)
64
+
65
+ raise Refused, message("sent #{command.first.to_s.upcase}", 'sends only reads')
66
+ end
67
+
68
+ def allowed?(command)
69
+ name = command.first.to_s.upcase
70
+ return CLIENT_HANDSHAKE.include?(command[1].to_s.upcase) if name == 'CLIENT'
71
+
72
+ Reader::ALLOWED.include?(name) || HANDSHAKE.include?(name)
73
+ end
74
+
75
+ def message(what, policy)
76
+ "#{label}: #{culprit} #{what} while the app booted, and #{label} #{policy}. App code that touches Redis " \
77
+ 'at boot (a flushdb, a seed) would reach a real server here. Move it into `config.on(:startup)`, which ' \
78
+ "only a running worker fires#{', or pass --redis to allow reads' if policy.start_with?('runs')}."
79
+ end
80
+
81
+ # The first frame outside Wurk, the Redis client and the pool: the app
82
+ # file (an initializer, a gem's railtie) that asked for Redis.
83
+ def culprit(locations = caller_locations)
84
+ frame = locations.find { |loc| app_frame?(loc.absolute_path || loc.path) }
85
+ frame ? "#{frame.path}:#{frame.lineno}" : 'app code'
86
+ end
87
+
88
+ private
89
+
90
+ def app_frame?(path)
91
+ return false if path.nil? || path.start_with?('<internal:', LIB, STDLIB)
92
+
93
+ !GEM_FRAMES.match?(path)
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -5,6 +5,7 @@ require_relative 'preflight/report'
5
5
  require_relative 'preflight/keyspace'
6
6
  require_relative 'preflight/owners'
7
7
  require_relative 'preflight/checks'
8
+ require_relative 'preflight/app_scan'
8
9
 
9
10
  module Wurk
10
11
  # `wurk preflight`: what a live Redis holds, and what will and will not carry
@@ -16,8 +17,9 @@ module Wurk
16
17
 
17
18
  module_function
18
19
 
20
+ # `app` is the root of the booted app (`--app`), or nil.
19
21
  # @return [Report]
20
- def run(config, limit: DEFAULT_LIMIT, sample: DEFAULT_SAMPLE, pause: 0.005, progress: nil, app: false)
22
+ def run(config, limit: DEFAULT_LIMIT, sample: DEFAULT_SAMPLE, pause: 0.005, progress: nil, app: nil)
21
23
  reader = Reader.new(config)
22
24
  report = Report.new('preflight')
23
25
  keyspace = Keyspace.new(reader, limit: limit, pause: pause, progress: progress).walk
@@ -25,16 +27,18 @@ module Wurk
25
27
  checks = Checks.new(reader: reader, report: report, keyspace: keyspace, owners: Owners.new(reader),
26
28
  sample: sample)
27
29
  checks.run
28
- app ? app_findings(checks, report) : report.info('app_checks_skipped', 'No --app: `lock:` classes not inspected.')
30
+ app ? app_findings(checks, report, app) : report.info('app_checks_skipped', 'No --app: app code not inspected.')
29
31
  report
30
32
  ensure
31
33
  reader&.close
32
34
  end
33
35
 
34
- def app_findings(checks, report)
36
+ def app_findings(checks, report, root)
35
37
  checks.app
36
- report.info(Report::APP_CODE_RAN, 'The app was booted, so its initializers ran. This run read Redis only, but ' \
37
- 'anything they wrote, on whichever Redis they use, is not covered by that.')
38
+ AppScan.new(root: root, report: report).run
39
+ report.info(Report::APP_CODE_RAN, "The app was booted, so its initializers ran. Wurk's own Redis clients sent " \
40
+ 'reads only while it booted, but a client the app opens itself ' \
41
+ '(Redis.new, Rails.cache) is not covered by that.')
38
42
  end
39
43
  end
40
44
  end
@@ -263,6 +263,12 @@ module Wurk
263
263
  swarm.shutdown unless supervisor&.alive?
264
264
  end
265
265
 
266
+ def check_embedded_boot(config)
267
+ BootChecks.log_process_role(config, "embedded in #{server_name}, threads only")
268
+ BootChecks.run(config, nil)
269
+ BootChecks.warn_on_ignored_topology(config, "a worker embedded in #{server_name}")
270
+ end
271
+
266
272
  # Sidekiq-embedded parity: a threads-only worker inside the web process, no
267
273
  # fork. Redis validation failure keeps the host serving HTTP (log + carry
268
274
  # on). at_exit drains on the graceful shutdown the web server runs on TERM.
@@ -274,8 +280,7 @@ module Wurk
274
280
  # already running with nothing to drain them on host exit.
275
281
  at_exit { instance.stop }
276
282
  instance.run
277
- BootChecks.log_process_role(Wurk.configuration, "embedded in #{server_name}, threads only")
278
- BootChecks.run(Wurk.configuration, nil)
283
+ check_embedded_boot(Wurk.configuration)
279
284
  instance
280
285
  rescue StandardError => e
281
286
  logger.error { "wurk: embedded boot failed: #{e.class}: #{e.message}" }
@@ -109,6 +109,11 @@ module Wurk
109
109
  "#{unknown.map(&:inspect).join(', ')}. Supported keys: #{known_keys.sort.join(', ')}."
110
110
  end
111
111
 
112
+ # `driver: :hiredis`, the C driver a fiber scheduler cannot switch away from.
113
+ def fiber_blocking_driver?(driver)
114
+ driver.to_s.include?('hiredis')
115
+ end
116
+
112
117
  # Sentinel sets go through RedisClient.sentinel — RedisClient.config
113
118
  # rejects `sentinels:` outright. Same routing Sidekiq does.
114
119
  def sentinel?(client_config)
@@ -114,6 +114,13 @@ module Wurk
114
114
 
115
115
  attr_reader :size, :url, :name, :pool_timeout, :client_config, :redis_idle_timeout
116
116
 
117
+ class << self
118
+ # A redis-client middleware every pool built from now on carries, after
119
+ # any the host configured. Set only by the boot of a command that must not
120
+ # write Redis (Preflight::RedisGuard); nil everywhere else.
121
+ attr_accessor :guard
122
+ end
123
+
117
124
  # Takes the standard Sidekiq `config.redis` hash: `pool_timeout` tunes the
118
125
  # ConnectionPool checkout; `connect_timeout`/`read_timeout`/`write_timeout`/
119
126
  # `reconnect_attempts` plus any other redis-client key (driver, ssl_params,
@@ -226,16 +233,32 @@ module Wurk
226
233
  # …) so this class stays about pooling; host-supplied keys win over the
227
234
  # defaults.
228
235
  def build_client_config(options)
229
- RedisOptions.normalize(options, defaults: DEFAULT_CLIENT_CONFIG).freeze
236
+ config = RedisOptions.normalize(options, defaults: DEFAULT_CLIENT_CONFIG)
237
+ config[:middlewares] = [*(config[:middlewares] || []), self.class.guard] if self.class.guard
238
+ config.freeze
230
239
  end
231
240
 
232
241
  # Wrapped in the CompatClient decorator so `Sidekiq.redis { |c| c.smembers }`
233
242
  # method-style commands work like Sidekiq 7+ (#204). Wurk's own code paths
234
243
  # use #call, which the decorator forwards.
235
244
  def build_client
245
+ warn_on_fiber_blocking_driver
236
246
  RedisClientAdapter::CompatClient.new(redis_client_config.new_client)
237
247
  end
238
248
 
249
+ # hiredis waits on the socket in C, where a fiber scheduler (Falcon, Async)
250
+ # cannot switch away, so one slow reply stalls every fiber on the thread.
251
+ # Checked on the connection build, the first moment a scheduler is visible.
252
+ def warn_on_fiber_blocking_driver
253
+ return if @fiber_warned || ::Fiber.scheduler.nil? || !RedisOptions.fiber_blocking_driver?(@client_config[:driver])
254
+
255
+ @fiber_warned = true
256
+ Wurk.logger.warn do
257
+ "Redis pool #{@name} uses the hiredis driver under a fiber scheduler, which blocks every fiber on the " \
258
+ 'thread while it waits on Redis. Use `driver: :ruby` in the config.redis of this process.'
259
+ end
260
+ end
261
+
239
262
  # A Sentinel set is a different constructor, not a different keyword:
240
263
  # `RedisClient.config(sentinels: [...])` raises. Sidekiq routes the same way.
241
264
  def redis_client_config