wurk 1.11.1 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -3
  3. data/config/routes.rb +8 -0
  4. data/exe/sidekiq +8 -0
  5. data/exe/wurk +4 -2
  6. data/lib/generators/wurk/install/install_generator.rb +56 -0
  7. data/lib/generators/wurk/install/templates/wurk.rb +5 -3
  8. data/lib/wurk/boot_checks.rb +210 -5
  9. data/lib/wurk/cli.rb +110 -10
  10. data/lib/wurk/configuration.rb +28 -7
  11. data/lib/wurk/fetcher/oom_drain.rb +0 -5
  12. data/lib/wurk/fetcher/orphan_reclaim.rb +108 -0
  13. data/lib/wurk/{middleware → fetcher}/poison_pill.rb +33 -47
  14. data/lib/wurk/fetcher/reaper.rb +46 -65
  15. data/lib/wurk/fetcher/reliable.rb +4 -5
  16. data/lib/wurk/fetcher/self_reconcile.rb +8 -14
  17. data/lib/wurk/fetcher/unit_of_work.rb +1 -1
  18. data/lib/wurk/job_logger.rb +4 -2
  19. data/lib/wurk/job_util.rb +32 -2
  20. data/lib/wurk/launcher.rb +30 -3
  21. data/lib/wurk/leader.rb +1 -8
  22. data/lib/wurk/lua/reclaim_orphan.lua +31 -0
  23. data/lib/wurk/lua.rb +5 -20
  24. data/lib/wurk/manager.rb +27 -1
  25. data/lib/wurk/middleware/timeout.rb +6 -6
  26. data/lib/wurk/preflight/app_scan.rb +182 -0
  27. data/lib/wurk/preflight/command.rb +21 -5
  28. data/lib/wurk/preflight/redis_guard.rb +98 -0
  29. data/lib/wurk/preflight.rb +9 -5
  30. data/lib/wurk/rails_boot.rb +45 -11
  31. data/lib/wurk/redis_options.rb +5 -0
  32. data/lib/wurk/redis_pool.rb +24 -1
  33. data/lib/wurk/sentry.rb +11 -0
  34. data/lib/wurk/swarm.rb +63 -14
  35. data/lib/wurk/unique.rb +12 -2
  36. data/lib/wurk/version.rb +1 -1
  37. data/lib/wurk/web/rack_app.rb +8 -6
  38. data/lib/wurk/web/stats_routes.rb +93 -0
  39. data/lib/wurk.rb +5 -1
  40. data/vendor/assets/dashboard/assets/{ArgsValue-CmaXArv8.js → ArgsValue-BJol9S0s.js} +1 -1
  41. data/vendor/assets/dashboard/assets/{BatchDetail-nc5yePZW.js → BatchDetail-D_zGQ9Sd.js} +1 -1
  42. data/vendor/assets/dashboard/assets/{Batches-DgEcyOZ_.js → Batches-Dt_xC660.js} +1 -1
  43. data/vendor/assets/dashboard/assets/{Busy-CUZncvCy.js → Busy-CWRthsmc.js} +1 -1
  44. data/vendor/assets/dashboard/assets/{Cron-CIZbaaZs.js → Cron-qeGr2YAm.js} +1 -1
  45. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +1 -0
  46. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +1 -0
  47. data/vendor/assets/dashboard/assets/{Extension-Doy4bJoA.js → Extension-DONXVLdA.js} +1 -1
  48. data/vendor/assets/dashboard/assets/{FilterBox-C4bO26_d.js → FilterBox-BbB6gXHp.js} +1 -1
  49. data/vendor/assets/dashboard/assets/{FlowDetail-Dh3SJdFh.js → FlowDetail-CrmnVRp0.js} +1 -1
  50. data/vendor/assets/dashboard/assets/{Flows-C5SMRuhq.js → Flows-g4GC8KmE.js} +1 -1
  51. data/vendor/assets/dashboard/assets/{JobDetailModal-Cg6sAsJB.js → JobDetailModal-C86k_1wj.js} +1 -1
  52. data/vendor/assets/dashboard/assets/{Limiters-BIxwDOJM.js → Limiters-zyO-iMlK.js} +1 -1
  53. data/vendor/assets/dashboard/assets/{Metrics-Qp9q0nF8.js → Metrics-Ckq5fNBv.js} +1 -1
  54. data/vendor/assets/dashboard/assets/{NotFound-DwjPxFjU.js → NotFound-C8NnyjlY.js} +1 -1
  55. data/vendor/assets/dashboard/assets/{Profiles-7dabYVe3.js → Profiles-DRl-FXPA.js} +1 -1
  56. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +1 -0
  57. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +1 -0
  58. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +1 -0
  59. data/vendor/assets/dashboard/assets/{Search-BAsVE2qw.js → Search-LaEpj2fM.js} +1 -1
  60. data/vendor/assets/dashboard/assets/{charts-DZs138BV.js → charts-D_PQLoyc.js} +1 -1
  61. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +141 -0
  62. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-DxkLZCIm.js → useResetPageOnEmpty-BiwhHqgh.js} +1 -1
  63. data/vendor/assets/dashboard/assets/{utils-B0kQFNE8.js → utils-CnWZzW2L.js} +1 -1
  64. data/vendor/assets/dashboard/index.html +1 -1
  65. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  66. metadata +37 -26
  67. data/vendor/assets/dashboard/assets/Dashboard-BWmfAook.js +0 -1
  68. data/vendor/assets/dashboard/assets/Dead-CGKsTZHq.js +0 -1
  69. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +0 -1
  70. data/vendor/assets/dashboard/assets/Retries-CNX8ZtTH.js +0 -1
  71. data/vendor/assets/dashboard/assets/Scheduled-ClVBj0nc.js +0 -1
  72. data/vendor/assets/dashboard/assets/index-CMzPFxNE.js +0 -141
@@ -452,7 +452,7 @@ module Wurk
452
452
  # The optional block is Pro's recovery callback: `|jobstr, pill|`, fired
453
453
  # once per orphan recovery (`pill` nil) and once on a poison kill (`pill`
454
454
  # responds to .jid/.klass/.count/.queue). The reaper drives it via
455
- # Wurk::Middleware::PoisonPill.track!. Spec: docs/target/sidekiq-pro.md §3.1.
455
+ # Wurk::Middleware::PoisonPill.recovered!. Spec: docs/target/sidekiq-pro.md §3.1.
456
456
  def super_fetch!(*, &block)
457
457
  @super_fetch_callback = block if block
458
458
  nil
@@ -877,7 +877,7 @@ module Wurk
877
877
  # and the like silently absent from a process that does run workers.
878
878
  def configure_server(&block)
879
879
  return unless block
880
- return yield self if server?
880
+ return run_server_block(block) if server?
881
881
 
882
882
  @deferred_server_blocks << block
883
883
  return if @configure_server_skip_warned
@@ -889,23 +889,34 @@ module Wurk
889
889
 
890
890
  def enter_server_mode
891
891
  self[:server] = true
892
- @deferred_server_blocks.shift.call(self) until @deferred_server_blocks.empty?
892
+ run_server_block(@deferred_server_blocks.shift) until @deferred_server_blocks.empty?
893
893
  end
894
894
 
895
895
  # A web process that also boots the swarm (Wurk::RailsBoot) is flagged
896
896
  # server, yet every enqueue it makes from a request is client work, so
897
- # `client_in_server` lets its configure_client blocks run too. Same
898
- # one-warning-per-gate contract as configure_server.
897
+ # `client_in_server` lets its configure_client blocks run too.
898
+ #
899
+ # In a server, skipping the client block is the Sidekiq contract and the
900
+ # usual case: an app configures both roles. So the skip itself is DEBUG,
901
+ # and BootChecks warns at boot only for the case worth one, a server
902
+ # process in which no configure_server block ran at all (#816).
899
903
  def configure_client
900
904
  return unless block_given?
901
905
  return yield self if !server? || @options[:client_in_server]
906
+
902
907
  return if @configure_client_skip_warned
903
908
 
904
909
  @configure_client_skip_warned = true
905
- logger.warn { 'configure_client block skipped: server? is true and client_in_server is false or nil' }
910
+ logger.debug { 'configure_client block skipped: server? is true and client_in_server is false or nil' }
906
911
  nil
907
912
  end
908
913
 
914
+ # True once a configure_client block was skipped and no configure_server
915
+ # block ran: the server got none of the settings the app made for a role.
916
+ def client_config_only?
917
+ @configure_client_skip_warned == true && !@configure_server_ran
918
+ end
919
+
909
920
  def server?
910
921
  @options[:server] == true
911
922
  end
@@ -920,9 +931,14 @@ module Wurk
920
931
 
921
932
  def topology=(value)
922
933
  guard_frozen!
934
+ @topology_assigned = true
923
935
  @topology = value
924
936
  end
925
937
 
938
+ # Whether the app declared a topology, as opposed to the default one
939
+ # #topology builds. Only a swarm reads it.
940
+ def topology_assigned? = @topology_assigned == true
941
+
926
942
  # Memory-based child recycling (Sidekiq Ent §7.5): the swarm parent TERMs
927
943
  # (and respawns) any child whose RSS exceeds this many MB. Set in code or
928
944
  # via SIDEKIQ_MAXMEM_MB (WURK_MAXMEM_MB is the native alias); an explicit
@@ -1001,6 +1017,11 @@ module Wurk
1001
1017
 
1002
1018
  private
1003
1019
 
1020
+ def run_server_block(block)
1021
+ @configure_server_ran = true
1022
+ block.call(self)
1023
+ end
1024
+
1004
1025
  # Pools read the hash once, when built, so the later assignment silently does
1005
1026
  # nothing to this process's connections until something rebuilds them.
1006
1027
  def warn_on_late_redis_assignment
@@ -1079,7 +1100,7 @@ module Wurk
1079
1100
  # as throttling or memory pressure.
1080
1101
  def default_child_count
1081
1102
  count, source = child_count_and_source
1082
- logger.info { "Wurk swarm: forking #{count} #{count == 1 ? 'child' : 'children'} (#{source})" }
1103
+ logger.info { "Wurk swarm size: #{count} #{count == 1 ? 'child' : 'children'} (#{source})" }
1083
1104
  count
1084
1105
  end
1085
1106
 
@@ -50,7 +50,6 @@ module Wurk
50
50
  return nil
51
51
  LUA
52
52
 
53
- RECLAIM = (HEADER + Lua::RELIABLE_RECLAIM).freeze
54
53
  REQUEUE = (HEADER + Lua::RELIABLE_REQUEUE).freeze
55
54
 
56
55
  # KEYS = [lock], ARGV = [ttl_ms]. The reaper's sweep lock: refused, no sweep
@@ -72,10 +71,6 @@ module Wurk
72
71
  run(conn, error, CLAIM, [Keys::PROCESSES, Component.identity, *keys], [info, beat, HEARTBEAT_TTL])
73
72
  end
74
73
 
75
- def reclaim(conn, error, source, destination)
76
- run(conn, error, RECLAIM, [source, destination], [])
77
- end
78
-
79
74
  # Returns 1 when moved, 0 when the job has left the private list since.
80
75
  def requeue(conn, error, private_list, public_q, job)
81
76
  run(conn, error, REQUEUE, [private_list, public_q], [job])
@@ -0,0 +1,108 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../keys'
4
+ require_relative '../lua'
5
+ require_relative 'poison_pill'
6
+ require_relative '../metrics/process_counters'
7
+ require_relative 'oom_drain'
8
+
9
+ module Wurk
10
+ class Fetcher
11
+ # Puts one payload from a private list back where it belongs, for both ways
12
+ # an orphan is found: the Reaper draining a dead owner's list, and
13
+ # SelfReconcile returning a payload its own live process lost track of.
14
+ #
15
+ # The poison-pill verdict is taken in the same script that moves the job
16
+ # (`lua/reclaim_orphan.lua`), so a job past the cap goes straight from the
17
+ # private list to the dead set and is never visible on its public queue. The
18
+ # notifications (death handlers, `super_fetch!`, `on_poison`) follow once
19
+ # the script has returned, for a job no worker can be running.
20
+ class OrphanReclaim
21
+ THRESHOLD = Middleware::PoisonPill::RECOVERY_THRESHOLD
22
+ TTL = Middleware::PoisonPill::RECOVERY_TTL
23
+
24
+ def initialize(config)
25
+ @config = config
26
+ end
27
+
28
+ # Returns true when this call took the payload off the private list, false
29
+ # when it had already left (another reaper, an ACK).
30
+ #
31
+ # Sent once, never re-sent on a fresh socket: a replay of a script that
32
+ # already ran finds the payload gone and answers "someone else moved it",
33
+ # which skipped the death handlers of a kill (#908), and a list holding two
34
+ # identical copies would move the second. A reply lost after the script ran
35
+ # is settled by reading back what it wrote instead (#settle_lost_reply).
36
+ def call(private_list, public_q, payload)
37
+ job = Middleware::PoisonPill.parse(payload)
38
+ keys = move_keys(private_list, public_q, job)
39
+ count = moved_count(keys, payload, now)
40
+ return false unless count
41
+
42
+ Metrics::ProcessCounters.increment(:reaper_reclaimed)
43
+ unless count == :uncounted
44
+ Middleware::PoisonPill.recovered!(payload, job, count: count, queue: queue_name(public_q),
45
+ config: @config)
46
+ end
47
+ true
48
+ end
49
+
50
+ private
51
+
52
+ def moved_count(keys, payload, score)
53
+ @config.redis { |conn| move(conn, keys, payload, score) }
54
+ rescue RedisClient::ConnectionError => e
55
+ settle_lost_reply(keys, payload, score, e)
56
+ end
57
+
58
+ def move_keys(private_list, public_q, job)
59
+ keys = [private_list, public_q, Keys::DEAD]
60
+ jid = job && job['jid']
61
+ keys << Middleware::PoisonPill.counter_key(jid) if jid.is_a?(String) && !jid.empty?
62
+ keys
63
+ end
64
+
65
+ # Past maxmemory the counter cannot be written, nor the job killed: it goes
66
+ # back uncounted rather than waiting on the memory it would help free.
67
+ def move(conn, keys, payload, score)
68
+ Wurk::Lua::Loader.eval_once(conn, :reclaim_orphan, keys: keys, argv: [payload, THRESHOLD, TTL, score])
69
+ rescue RedisClient::OutOfMemoryError => e
70
+ :uncounted if OomDrain.requeue(conn, e, keys[0], keys[1], payload) == 1
71
+ end
72
+
73
+ # The connection dropped around the script, so it ran or it did not. Only a
74
+ # kill needs settling: the job is gone from the private list, so no later
75
+ # sweep would find it, and the death handlers (a batch's `:death`, a unique
76
+ # lock's release) would never fire. The script wrote the dead-set member
77
+ # at the score this call chose, so that exact score proves the kill was
78
+ # this call's. Returns the recovery count for a kill this call made; nil
79
+ # for a requeue or a move someone else made, which need nothing more;
80
+ # raises the original error when the payload is still in the private list,
81
+ # which is the next pass's to move.
82
+ def settle_lost_reply(keys, payload, score, error)
83
+ private_list, _public_q, dead, counter = keys
84
+ still_there, dead_score, count = @config.redis(idempotent: true) do |conn|
85
+ conn.pipelined do |pipe|
86
+ pipe.call('LPOS', private_list, payload, 'RANK', -1)
87
+ pipe.call('ZSCORE', dead, payload)
88
+ pipe.call('GET', counter) if counter
89
+ end
90
+ end
91
+ # Exact on purpose: the same double, written by the script and read back.
92
+ return count.to_i if dead_score && Float(dead_score) == Float(score) # rubocop:disable Lint/FloatComparison
93
+ raise error if still_there
94
+
95
+ nil
96
+ end
97
+
98
+ # DeadSet#kill's score format, so the entry sorts with every other kill.
99
+ def now
100
+ ::Process.clock_gettime(::Process::CLOCK_REALTIME).to_s
101
+ end
102
+
103
+ def queue_name(public_q)
104
+ public_q.delete_prefix(Keys::QUEUE_PREFIX)
105
+ end
106
+ end
107
+ end
108
+ end
@@ -9,19 +9,22 @@ require_relative '../dead_set'
9
9
  module Wurk
10
10
  module Middleware
11
11
  # Pro parity (§3.2): poison-pill detection for reliable-fetch orphans.
12
- # When a job is recovered out of a dead process's private list, we
13
- # INCR a per-jid counter at `super_fetch:recovered:<jid>` with a 72h
14
- # TTL. Recoveries up to RECOVERY_THRESHOLD (3) requeue; the one that takes
15
- # the counter past it (the 4th orphaning) is the poison pill: the payload is moved to the dead set,
16
- # `jobs.poison` is emitted to statsd, and recovery callbacks fire so
12
+ # When a job is recovered out of a private list, a per-jid counter at
13
+ # `super_fetch:recovered:<jid>` goes up by one with a 72h TTL. Recoveries up
14
+ # to RECOVERY_THRESHOLD (3) requeue; the one that takes the counter past it
15
+ # (the 4th orphaning) is the poison pill: the payload is moved to the dead
16
+ # set, `jobs.poison` is emitted to statsd, and recovery callbacks fire so
17
17
  # operators can be paged.
18
18
  #
19
19
  # Counter key TTL is wire-compat with Sidekiq Pro — third-party tooling
20
20
  # that watches `super_fetch:recovered:*` expects 72h.
21
21
  #
22
- # No server-middleware registration: the producer is Reaper#drain, which
23
- # drives the lifecycle directly via `track!(payload, queue:)` on each
24
- # orphan it moves back to the public queue. The consumer is the ACK —
22
+ # Not a chain middleware, so the file lives with the fetcher; the constant
23
+ # keeps its Middleware path because Pro initializers name it. No
24
+ # server-middleware registration: the producer is Fetcher::OrphanReclaim,
25
+ # whose script bumps the counter and takes the requeue-or-kill decision in
26
+ # the same step as the move, then hands the outcome to {recovered!} for the
27
+ # notifications. The consumer is the ACK —
25
28
  # `Fetcher::Reliable::UnitOfWork#acknowledge` pipelines {clear_in} next to
26
29
  # its LREM, so an attempt that finished starts the next one at zero
27
30
  # without costing a round trip of its own.
@@ -44,37 +47,34 @@ module Wurk
44
47
 
45
48
  module_function
46
49
 
47
- # Called per recovered orphan job. Returns `:poison` when the threshold
48
- # was crossed and the job was killed; `:recovered` when the job is
49
- # being re-pushed (caller's responsibility — we don't touch the queue
50
- # here). Emits `jobs.recovered.fetch` on every call, `jobs.poison`
51
- # only on the kill path.
50
+ # Called per recovered orphan job, once Fetcher::OrphanReclaim has moved
51
+ # it: `count` is the recovery counter its script wrote (0 for a payload with
52
+ # no jid). Returns `:poison` when the count is past the threshold — the
53
+ # script already put the job in the dead set, never on its queue — and
54
+ # `:recovered` otherwise. Emits `jobs.recovered.fetch` for every parseable
55
+ # job, `jobs.poison` only on the kill path.
52
56
  #
53
57
  # Fires the Pro `super_fetch!` recovery callback (config.super_fetch_callback)
54
58
  # exactly once per call: `(jobstr, nil)` on plain recovery, `(jobstr, pill)`
55
59
  # on the kill path. The poison-only `on_poison` Hash callbacks fire
56
60
  # independently inside #mark_poison.
57
61
  #
58
- # @param payload [String, Hash] the job JSON or pre-parsed hash.
62
+ # @param payload [String] the job JSON as it sat in the private list.
63
+ # @param job [Hash, nil] the payload parsed by {parse}.
64
+ # @param count [Integer] the recovery count after this recovery.
59
65
  # @param queue [String, nil] the public queue name (without `queue:`).
60
- # @param config [Configuration] config that owns the super_fetch! callback;
61
- # defaults to the global so non-reaper callers (tests) need not pass it.
66
+ # @param config [Configuration] config that owns the super_fetch! callback.
62
67
  # @return [Symbol] :recovered | :poison
63
- def track!(payload, queue: nil, config: Wurk.configuration)
64
- job = parse(payload)
68
+ def recovered!(payload, job, count:, queue: nil, config: Wurk.configuration)
65
69
  unless job
66
70
  fire_super_fetch(config, payload, nil)
67
71
  return :recovered
68
72
  end
69
73
 
70
- jid = job['jid']
71
- klass = job['class']
72
- emit('jobs.recovered.fetch', klass, queue)
73
-
74
- count = bump_counter(jid) if jid && !jid.empty?
75
- if count && count > RECOVERY_THRESHOLD
74
+ emit('jobs.recovered.fetch', job['class'], queue)
75
+ if count > RECOVERY_THRESHOLD
76
76
  mark_poison(payload, job, queue: queue, count: count)
77
- fire_super_fetch(config, payload, Pill.new(jid: jid, klass: klass, count: count, queue: queue))
77
+ fire_super_fetch(config, payload, Pill.new(jid: job['jid'], klass: job['class'], count: count, queue: queue))
78
78
  :poison
79
79
  else
80
80
  fire_super_fetch(config, payload, nil)
@@ -168,29 +168,14 @@ module Wurk
168
168
 
169
169
  # ---- internals --------------------------------------------------
170
170
 
171
+ # The job Hash, or nil for anything that is not a JSON object.
171
172
  def parse(payload)
172
- return payload if payload.is_a?(::Hash)
173
- return nil unless payload.is_a?(::String)
174
-
175
- Wurk.load_json(payload)
173
+ job = Wurk.load_json(payload)
174
+ job if job.is_a?(::Hash)
176
175
  rescue ::JSON::ParserError
177
176
  nil
178
177
  end
179
178
 
180
- # No apply-safety claim: INCR is additive, so a block replayed after a
181
- # lost reply bumps twice and can carry a healthy job past
182
- # RECOVERY_THRESHOLD into the dead set. Raising instead only leaves the
183
- # count short — Reaper#drain rescues, and the job is already back on its
184
- # public queue, so it just misses one poison check.
185
- def bump_counter(jid)
186
- key = counter_key(jid)
187
- Wurk.redis do |conn|
188
- count = conn.call('INCR', key).to_i
189
- conn.call('EXPIRE', key, RECOVERY_TTL)
190
- count
191
- end
192
- end
193
-
194
179
  def emit(metric, klass, queue)
195
180
  tags = []
196
181
  tags << "class:#{klass}" if klass
@@ -198,6 +183,9 @@ module Wurk
198
183
  Wurk::Metrics::Statsd.increment(metric, tags: tags.empty? ? nil : tags)
199
184
  end
200
185
 
186
+ # The script already put `payload` in the dead set; DeadSet#kill writes the
187
+ # same member again, so the trim and the death handlers stay DeadSet's.
188
+ #
201
189
  # Death handlers fire (`notify_failure` defaults to true). A poison kill
202
190
  # is a death like any other exhaustion: suppressing it strands every
203
191
  # batch that owns the job — Batch::DeathHandler is a death handler, so a
@@ -206,8 +194,7 @@ module Wurk
206
194
  # (docs/target/sidekiq-pro.md §12); recorded in docs/idea/parity-divergences.md.
207
195
  def mark_poison(payload, job, queue:, count:)
208
196
  emit('jobs.poison', job['class'], queue)
209
- json = payload.is_a?(String) ? payload : Wurk.dump_json(job)
210
- Wurk::DeadSet.new.kill(json, ex: poisoned_error(job['class'], count))
197
+ Wurk::DeadSet.new.kill(payload, ex: poisoned_error(job['class'], count))
211
198
  fire_callbacks(jid: job['jid'], klass: job['class'], count: count, queue: queue)
212
199
  end
213
200
 
@@ -231,8 +218,7 @@ module Wurk
231
218
  cb = config.super_fetch_callback
232
219
  return unless cb
233
220
 
234
- jobstr = payload.is_a?(::String) ? payload : Wurk.dump_json(payload)
235
- cb.call(jobstr, pill)
221
+ cb.call(payload, pill)
236
222
  rescue StandardError => e
237
223
  config.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
238
224
  end
@@ -2,14 +2,14 @@
2
2
 
3
3
  require_relative '../component'
4
4
  require_relative '../keys'
5
- require_relative '../middleware/poison_pill'
6
- require_relative '../metrics/process_counters'
7
5
  require_relative '../timer_loop'
8
6
  require_relative 'private_list_key'
9
7
  require_relative 'unparseable_keys'
10
8
  require_relative 'orphan_grace'
11
9
  require_relative 'oom_drain'
10
+ require_relative 'orphan_reclaim'
12
11
  require_relative 'self_reconcile'
12
+ require_relative 'reliable'
13
13
 
14
14
  module Wurk
15
15
  class Fetcher
@@ -29,9 +29,10 @@ module Wurk
29
29
  # ignores a stale `processes` SET member — the SET carries no TTL, so
30
30
  # the member lingers after its heartbeat hash expires (60s) until
31
31
  # `ProcessSet#cleanup` prunes it. A `kill -9`ed sibling is reclaimed
32
- # the moment the supervisor reaps it, not 60s later. (Pid reuse by an
33
- # unrelated process in the same tree is the one blind spot — the
34
- # supervisor respawns with a fresh pid, so it does not arise in practice.)
32
+ # the moment the supervisor reaps it, not 60s later. Pid reuse by an
33
+ # unrelated process reads a dead owner as live; a swarm child does not
34
+ # depend on this check, since its parent drains its lists by exact key
35
+ # the moment it reaps it (#reclaim_exited), before the pid can be reused.
35
36
  # * any other incarnation — its pid means nothing in our namespace, so we
36
37
  # trust the heartbeat: the owner is alive iff its identity's `info`
37
38
  # hash still exists. Such reclaim waits out the 60s heartbeat TTL, the
@@ -45,10 +46,10 @@ module Wurk
45
46
  # process's own clock (OrphanGrace), and its identity must still have no
46
47
  # `info` hash. Our own incarnation skips both: kill(0) is not a snapshot.
47
48
  #
48
- # Re-pushed jobs run through Wurk::Middleware::PoisonPill, which caps a
49
- # job at RECOVERY_THRESHOLD recoveries within 72h: past the cap the job
50
- # is killed into the dead set instead of re-queued, so a job that crashes
51
- # its worker every time can't loop forever.
49
+ # Every move goes through OrphanReclaim, which caps a job at
50
+ # Middleware::PoisonPill::RECOVERY_THRESHOLD recoveries within 72h: past the
51
+ # cap the job goes from the private list straight to the dead set instead of
52
+ # its queue, so a job that crashes its worker every time can't loop forever.
52
53
  #
53
54
  # One sweep an interval: one SCAN of `queue:*|*` acting on every list, gated
54
55
  # by a cluster lock so about one process sweeps per interval. Every list, and
@@ -62,7 +63,7 @@ module Wurk
62
63
  # of a job.
63
64
  #
64
65
  # Spec: docs/target/sidekiq-pro.md §3.2.
65
- class Reaper # rubocop:disable Metrics/ClassLength
66
+ class Reaper
66
67
  include Component
67
68
 
68
69
  # Sweep cadence in seconds. 60s matches the heartbeat TTL — the floor below
@@ -103,8 +104,8 @@ module Wurk
103
104
  @sleeper = ::ConditionVariable.new
104
105
  @unparseable = UnparseableKeys.new(config)
105
106
  @next_sweep_at = 0.0
106
- @self_reconcile = SelfReconcile.new(config, confirm: reconcile_confirm, stop: -> { done? },
107
- on_moved: method(:recovered))
107
+ @orphan_reclaim = OrphanReclaim.new(config)
108
+ @self_reconcile = SelfReconcile.new(config, confirm: reconcile_confirm, stop: -> { done? })
108
109
  end
109
110
 
110
111
  # Spawns the sweep loop. Idempotent. The loop waits one interval before
@@ -173,13 +174,30 @@ module Wurk
173
174
  sweep.tap { wake_for_settling }
174
175
  end
175
176
 
176
- # #reclaim! for a booting process. Deliberately under no lock, not even a
177
- # short one: a replacement's sweep is what recovers the sibling it replaces,
178
- # and the sibling may have died a moment after another process booted (a
179
- # 2s lock turned a child killed right after boot from 1s into 61s). One
180
- # pass per process costs keyspace / SCAN_COUNT calls.
177
+ # The sweep a booting process runs, under the same lock as the periodic one:
178
+ # a rolling restart or a deploy of N processes is one keyspace pass, not N.
179
+ # A replacement's sweep is no longer what recovers the swarm child it
180
+ # replaces; the parent did that by exact key when it reaped the child
181
+ # (#reclaim_exited). What this pass still finds is other incarnations'
182
+ # lists, which wait on the heartbeat TTL and the grace either way.
181
183
  def boot_reclaim!
182
- reclaim!
184
+ reap
185
+ end
186
+
187
+ # Drains the private lists of a process known to be dead: a swarm child its
188
+ # parent has just reaped. The parent knows the pid, the queues the child
189
+ # served and, since the nonce is inherited across fork, the rest of each
190
+ # list's name, so neither a SCAN nor a liveness check is needed. One
191
+ # pipelined LLEN finds the lists holding anything (none, after a drained
192
+ # exit) and raises rather than paying the pool's backoff once per list
193
+ # when Redis is away. Returns the jobs moved.
194
+ def reclaim_exited(pid, queue_names)
195
+ lists = queue_names.to_h do |name|
196
+ public_q = Keys.queue(name)
197
+ [Reliable.private_queue_name(public_q, pid: pid), public_q]
198
+ end
199
+ lengths = redis(idempotent: true) { |c| c.pipelined { |p| lists.each_key { |l| p.call('LLEN', l) } } }
200
+ lists.zip(lengths).sum { |(list, public_q), len| len.positive? ? drain(list, public_q) : 0 }
183
201
  end
184
202
 
185
203
  # Lists whose grace has run out, whichever sweep started it. Runs on every
@@ -313,27 +331,19 @@ module Wurk
313
331
  identity.split(':')[0..1].join(':')
314
332
  end
315
333
 
316
- # Drain one orphaned private list back to its public queue. Each job is
317
- # moved with an atomic LMOVE (private tail → public tail) BEFORE the
318
- # poison check, so a crash mid-drain leaves the job safely in the public
319
- # queue (at-least-once), never lost. Poison jobs are killed to the dead
320
- # set by PoisonPill.track! and then LREM'd out of the public queue.
334
+ # Drain one orphaned private list back to its public queue, a job at a
335
+ # time from the tail: peek it, then OrphanReclaim moves it, to the public
336
+ # tail or, past the poison cap, to the dead set. The decision is taken
337
+ # before the job is visible, so a worker parked in BLMOVE can never take a
338
+ # job that is being killed. A peeked job that has left by the time it is
339
+ # moved (another reaper drained it) costs one more peek.
321
340
  #
322
- # Unlike the fetcher's LMOVE, this one does *not* claim apply-safety: a
323
- # replay after a lost reply moves the next job and never poison-checks the
324
- # one already on the public tail, so a job that kills its worker every time
325
- # would get a free recovery past the cap. A raise instead lands in the
326
- # rescue below, and the next sweep re-drains what's left.
341
+ # A raise lands in the rescue below; the job stays in the private list
342
+ # and the next sweep re-drains what's left.
327
343
  def drain(private_list, public_q)
328
- queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
329
344
  count = 0
330
- loop do
331
- job = redis { |c| reclaim_one(c, private_list, public_q) }
332
- break unless job
333
-
334
- count += 1
335
- Metrics::ProcessCounters.increment(:reaper_reclaimed)
336
- poison_off(public_q, job, queue_name)
345
+ while (job = redis(idempotent: true) { |c| c.call('LINDEX', private_list, -1) })
346
+ count += 1 if @orphan_reclaim.call(private_list, public_q, job)
337
347
  end
338
348
  count
339
349
  rescue StandardError => e
@@ -341,35 +351,6 @@ module Wurk
341
351
  count
342
352
  end
343
353
 
344
- # A full Redis must not hold a dead worker's jobs back from the workers
345
- # still able to drain the queue.
346
- def reclaim_one(conn, private_list, public_q)
347
- Wurk::Lua::Loader.eval_cached(conn, :reliable_reclaim, keys: [private_list, public_q], argv: [])
348
- rescue RedisClient::OutOfMemoryError => e
349
- OomDrain.reclaim(conn, e, private_list, public_q)
350
- end
351
-
352
- # SelfReconcile put a payload back: the same accounting as a reclaim, so a
353
- # job that kills its processor thread every time still hits the poison cap.
354
- def recovered(public_q, job)
355
- Metrics::ProcessCounters.increment(:reaper_reclaimed)
356
- poison_off(public_q, job, public_q.delete_prefix(Keys::QUEUE_PREFIX))
357
- end
358
-
359
- # The job is back on its queue whatever happens here. Past maxmemory the
360
- # count can't be taken (nor the job killed, leaving it where it is): raising
361
- # would end the drain at one job per pass, with the rest of the list waiting
362
- # on memory they would help free.
363
- def poison_off(public_q, job, queue_name)
364
- return unless Middleware::PoisonPill.track!(job, queue: queue_name, config: @config) == :poison
365
-
366
- # track! already ZADDed the payload to the dead set; pull the copy we
367
- # just moved onto the public tail so it isn't also re-run.
368
- redis { |c| c.call('LREM', public_q, -1, job) }
369
- rescue RedisClient::OutOfMemoryError => e
370
- report(e)
371
- end
372
-
373
354
  def acquire_lock?
374
355
  ttl = (@interval * LOCK_TTL_RATIO * 1000).ceil
375
356
  redis do |c|
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'socket'
4
3
  require_relative '../component'
5
4
  require_relative '../pid_cache'
6
5
  require_relative '../keys'
@@ -80,10 +79,10 @@ module Wurk
80
79
  # reaper's `kill(0)` liveness check would read a dead owner's list as
81
80
  # live (jobs stranded) or a live owner's as dead (job run twice). Keys
82
81
  # written before the nonce existed stay reclaimable — PrivateListKey
83
- # accepts both shapes.
84
- def self.private_queue_name(public_queue, index = 0)
85
- host = ENV['DYNO'] || Socket.gethostname
86
- "#{public_queue}|#{host}|#{::Process.pid}|#{Component::PROCESS_NONCE}|#{index}"
82
+ # accepts both shapes. `pid:` lets a swarm parent name a reaped child's
83
+ # lists (Reaper#reclaim_exited).
84
+ def self.private_queue_name(public_queue, index = 0, pid: ::Process.pid)
85
+ "#{public_queue}|#{Component.hostname}|#{pid}|#{Component::PROCESS_NONCE}|#{index}"
87
86
  end
88
87
 
89
88
  # Guards the generation bump only. Fetchers read the counter without it —
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../keys'
4
- require_relative '../lua'
4
+ require_relative 'orphan_reclaim'
5
5
 
6
6
  module Wurk
7
7
  class Fetcher
@@ -34,13 +34,14 @@ module Wurk
34
34
 
35
35
  Suspect = Struct.new(:copies, :since)
36
36
 
37
- # `on_moved` receives [public_queue, payload] for each payload put back, so
38
- # the Reaper can run it through the same poison accounting as a reclaim.
39
- def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false }, on_moved: nil)
37
+ # A payload put back goes through the same poison accounting as a reclaim
38
+ # (OrphanReclaim), so a job that kills its processor thread every time
39
+ # still hits the poison cap.
40
+ def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false })
40
41
  @config = config
41
42
  @confirm = confirm
42
43
  @stop = stop
43
- @on_moved = on_moved
44
+ @orphan_reclaim = OrphanReclaim.new(config)
44
45
  @suspects = {}
45
46
  end
46
47
 
@@ -154,16 +155,9 @@ module Wurk
154
155
  moved
155
156
  end
156
157
 
157
- # LPOS-guarded, so a replay after a lost reply moves nothing the second
158
- # time, and a payload settled since the last read stays settled.
158
+ # LPOS-guarded, so a payload settled since the last read stays settled.
159
159
  def requeue(list, public_q, payload)
160
- applied = @config.redis(idempotent: true) do |conn|
161
- Wurk::Lua::Loader.eval_cached(conn, :reliable_requeue, keys: [list, public_q], argv: [payload])
162
- rescue RedisClient::OutOfMemoryError => e
163
- OomDrain.requeue(conn, e, list, public_q, payload)
164
- end.to_i
165
- @on_moved&.call(public_q, payload) if applied == 1
166
- applied
160
+ @orphan_reclaim.call(list, public_q, payload) ? 1 : 0
167
161
  end
168
162
 
169
163
  def forget_all
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../queue_slot'
4
- require_relative '../middleware/poison_pill'
4
+ require_relative 'poison_pill'
5
5
 
6
6
  module Wurk
7
7
  class Fetcher
@@ -12,7 +12,7 @@ module Wurk
12
12
  # 1. prepare(job_hash) { ... } → sets thread-local context, applies
13
13
  # per-job log_level, yields to the rest of dispatch.
14
14
  # 2. call(item, queue) { ... } → wraps the actual perform with the
15
- # start/done/fail log line trio and the elapsed-ms measurement.
15
+ # start/done/fail log line trio and the elapsed-seconds measurement.
16
16
  #
17
17
  # Skipping default logging is controlled by config[:skip_default_job_logging];
18
18
  # the prepare step still runs so context/log_level still apply.
@@ -68,8 +68,10 @@ module Wurk
68
68
 
69
69
  private
70
70
 
71
+ # Seconds, as upstream logs it (`elapsed=0.007`): log scrapers and
72
+ # dashboards built on Sidekiq's line read the field that way.
71
73
  def elapsed(start)
72
- ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000).round(3)
74
+ (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
73
75
  end
74
76
 
75
77
  # The attribute list is a host setting (defaults to %w[bid tags]), so the