wurk 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +9 -9
  3. data/app/controllers/wurk/api_controller.rb +4 -1
  4. data/app/controllers/wurk/dashboard_controller.rb +11 -6
  5. data/exe/wurk +7 -2
  6. data/exe/wurkswarm +1 -2
  7. data/lib/generators/wurk/install/install_generator.rb +12 -0
  8. data/lib/sidekiq/web.rb +5 -0
  9. data/lib/wurk/boot_checks.rb +93 -0
  10. data/lib/wurk/capsule.rb +33 -8
  11. data/lib/wurk/cli.rb +204 -30
  12. data/lib/wurk/client/buffered.rb +240 -26
  13. data/lib/wurk/client.rb +16 -5
  14. data/lib/wurk/compat.rb +15 -0
  15. data/lib/wurk/configuration.rb +114 -16
  16. data/lib/wurk/cron.rb +141 -16
  17. data/lib/wurk/debounce.rb +2 -2
  18. data/lib/wurk/embedded.rb +8 -2
  19. data/lib/wurk/encryption.rb +20 -5
  20. data/lib/wurk/error_coalescer.rb +35 -0
  21. data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
  22. data/lib/wurk/fetcher/capped.rb +4 -0
  23. data/lib/wurk/fetcher/many_queues.rb +17 -1
  24. data/lib/wurk/fetcher/oom_drain.rb +102 -0
  25. data/lib/wurk/fetcher/orphan_grace.rb +60 -44
  26. data/lib/wurk/fetcher/private_list_key.rb +1 -16
  27. data/lib/wurk/fetcher/reaper.rb +169 -127
  28. data/lib/wurk/fetcher/reliable.rb +26 -4
  29. data/lib/wurk/fetcher/self_reconcile.rb +175 -0
  30. data/lib/wurk/fetcher/unit_of_work.rb +32 -8
  31. data/lib/wurk/health.rb +23 -4
  32. data/lib/wurk/health_probe.rb +64 -0
  33. data/lib/wurk/heartbeat.rb +55 -3
  34. data/lib/wurk/job/options.rb +9 -1
  35. data/lib/wurk/job.rb +2 -24
  36. data/lib/wurk/job_record.rb +17 -9
  37. data/lib/wurk/job_retry.rb +38 -10
  38. data/lib/wurk/job_set.rb +13 -5
  39. data/lib/wurk/launcher.rb +134 -52
  40. data/lib/wurk/leader.rb +71 -11
  41. data/lib/wurk/limiter/base.rb +17 -1
  42. data/lib/wurk/lua/debounce.lua +11 -6
  43. data/lib/wurk/lua/loader.rb +13 -1
  44. data/lib/wurk/lua.rb +71 -8
  45. data/lib/wurk/manager.rb +1 -1
  46. data/lib/wurk/metrics/process_counters.rb +27 -0
  47. data/lib/wurk/metrics/prometheus.rb +35 -14
  48. data/lib/wurk/middleware/current_attributes.rb +26 -10
  49. data/lib/wurk/middleware/poison_pill.rb +1 -2
  50. data/lib/wurk/preflight/checks.rb +203 -0
  51. data/lib/wurk/preflight/command.rb +135 -0
  52. data/lib/wurk/preflight/keyspace.rb +109 -0
  53. data/lib/wurk/preflight/owners.rb +74 -0
  54. data/lib/wurk/preflight/payload_sample.rb +108 -0
  55. data/lib/wurk/preflight/reader.rb +60 -0
  56. data/lib/wurk/preflight/report.rb +82 -0
  57. data/lib/wurk/preflight/verify.rb +161 -0
  58. data/lib/wurk/preflight.rb +40 -0
  59. data/lib/wurk/process_set.rb +3 -1
  60. data/lib/wurk/processor.rb +82 -18
  61. data/lib/wurk/queue_consumers.rb +134 -0
  62. data/lib/wurk/rails_boot.rb +40 -2
  63. data/lib/wurk/rails_reloader.rb +51 -0
  64. data/lib/wurk/railtie.rb +20 -1
  65. data/lib/wurk/redact.rb +12 -0
  66. data/lib/wurk/redis_pool.rb +34 -9
  67. data/lib/wurk/scheduled.rb +142 -67
  68. data/lib/wurk/sentry/death_handler.rb +29 -0
  69. data/lib/wurk/sentry/error_handler.rb +52 -10
  70. data/lib/wurk/sentry/middleware.rb +8 -1
  71. data/lib/wurk/sentry/retry_policy.rb +12 -3
  72. data/lib/wurk/sentry.rb +42 -5
  73. data/lib/wurk/sorted_entry.rb +38 -13
  74. data/lib/wurk/swarm/child_boot.rb +44 -31
  75. data/lib/wurk/swarm/crash_watch.rb +65 -0
  76. data/lib/wurk/swarm/test_mode_guard.rb +32 -0
  77. data/lib/wurk/swarm.rb +79 -11
  78. data/lib/wurk/testing.rb +74 -0
  79. data/lib/wurk/version.rb +1 -1
  80. data/lib/wurk/web/config.rb +8 -4
  81. data/lib/wurk/web/extension.rb +41 -62
  82. data/lib/wurk/web/helpers.rb +133 -0
  83. data/lib/wurk/web/paginator.rb +49 -0
  84. data/lib/wurk/web/rack_app.rb +30 -6
  85. data/lib/wurk/work_set.rb +6 -0
  86. data/lib/wurk/worker/setter.rb +11 -6
  87. data/lib/wurk/worker.rb +17 -1
  88. data/lib/wurk.rb +1 -1
  89. data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
  90. data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
  91. data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
  92. data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
  93. data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
  94. data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
  95. data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
  96. data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
  97. data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
  98. data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
  99. data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
  100. data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
  101. data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
  102. data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
  103. data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
  104. data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
  105. data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
  106. data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
  107. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
  108. data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
  109. data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
  110. data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
  111. data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
  112. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
  113. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
  114. data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
  115. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
  116. data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
  117. data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
  118. data/vendor/assets/dashboard/index.html +3 -3
  119. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  120. metadata +53 -30
  121. data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
  122. data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
  123. data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
@@ -0,0 +1,175 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../keys'
4
+ require_relative '../lua'
5
+
6
+ module Wurk
7
+ class Fetcher
8
+ # A live worker's check of its own private lists.
9
+ #
10
+ # The Reaper leaves a live owner's list alone, and a live owner only ever
11
+ # settles the jobs its threads know about. So a payload no thread knows about
12
+ # would wait there until the process exits. Two ways in:
13
+ # * a claim applied but its reply was lost (Redis restart, a stall past the
14
+ # read timeout, a dropped socket): the client re-sends and claims another
15
+ # job, so one job per thread per incident;
16
+ # * a job that failed while Redis was down, where the retry booking and the
17
+ # requeue both raised.
18
+ #
19
+ # A payload counts as stranded only if it is in the list and held by nothing:
20
+ # no thread's claim (Fetcher::Reliable::CLAIM_KEY, which covers the whole of
21
+ # a running job however long it runs) and no pending ACK. The list is read
22
+ # first and the holders second, so a claim landing between the two reads as
23
+ # held. What the reads cannot see is a unit in transit: claimed but not yet
24
+ # stamped, or an ACK on the wire. Those last microseconds to seconds, which
25
+ # is why one sighting proves nothing: it has to be seen stranded again at
26
+ # least `confirm` seconds later before it moves.
27
+ #
28
+ # Counted rather than compared as sets, because a client re-send can leave two
29
+ # byte-identical payloads, and a running job must not vouch for its twin.
30
+ class SelfReconcile
31
+ # Above anything a unit can spend in transit: the longest a held ACK sits on
32
+ # the wire is the pool's blip backoff plus its socket timeouts, ~18s.
33
+ DEFAULT_CONFIRM = 30
34
+
35
+ Suspect = Struct.new(:copies, :since)
36
+
37
+ # `on_moved` receives [public_queue, payload] for each payload put back, so
38
+ # the Reaper can run it through the same poison accounting as a reclaim.
39
+ def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false }, on_moved: nil)
40
+ @config = config
41
+ @confirm = confirm
42
+ @stop = stop
43
+ @on_moved = on_moved
44
+ @suspects = {}
45
+ end
46
+
47
+ # One pass. Returns how many payloads went back to their public queues.
48
+ def sweep
49
+ lists = own_lists
50
+ return 0 if lists.empty?
51
+
52
+ crowded = crowded_lists(lists)
53
+ return forget_all if crowded.empty?
54
+
55
+ contents = read(crowded.keys)
56
+ settle(crowded, contents, held_by_list)
57
+ end
58
+
59
+ private
60
+
61
+ # { private_list => public_queue } for every queue a Reliable fetcher here
62
+ # serves. Two capsules on one queue share a list, hence the Hash. A custom
63
+ # `config[:fetch_class]` keeps its own books, so its queues are left alone.
64
+ def own_lists
65
+ @config.capsules.each_value.with_object({}) do |capsule, lists|
66
+ next unless capsule.fetcher.is_a?(Reliable)
67
+
68
+ capsule.queues.uniq.each do |name|
69
+ lists[Reliable.private_queue_name(Keys.queue(name))] = Keys.queue(name)
70
+ end
71
+ end
72
+ end
73
+
74
+ # A list no longer than what its holders account for has nothing stranded
75
+ # in it, which is every list on every healthy sweep, and LLEN is what keeps
76
+ # a healthy sweep from reading every in-flight payload back over the wire.
77
+ def crowded_lists(lists)
78
+ lengths = read_lengths(lists.keys)
79
+ held = held_by_list
80
+ lists.select { |list, _| lengths[list] > held_count(held, list) }
81
+ end
82
+
83
+ def read_lengths(names)
84
+ replies = @config.redis(idempotent: true) do |conn|
85
+ conn.pipelined { |pipe| names.each { |list| pipe.call('LLEN', list) } }
86
+ end
87
+ names.zip(replies).to_h
88
+ end
89
+
90
+ def held_count(held, list)
91
+ held.fetch(list, {}).each_value.sum
92
+ end
93
+
94
+ def read(names)
95
+ replies = @config.redis(idempotent: true) do |conn|
96
+ conn.pipelined { |pipe| names.each { |list| pipe.call('LRANGE', list, 0, -1) } }
97
+ end
98
+ names.zip(replies).to_h
99
+ end
100
+
101
+ # { private_list => { payload => count } } across every thread and fetcher
102
+ # in this process. Thread.list, not a registry: nothing is written per job,
103
+ # and a thread that died mid-job stops vouching for its payload by itself.
104
+ def held_by_list
105
+ held_units.each_with_object({}) do |unit, held|
106
+ tally = (held[unit.private_queue] ||= Hash.new(0))
107
+ tally[unit.job.b] += 1
108
+ end
109
+ end
110
+
111
+ def held_units
112
+ claims = ::Thread.list.filter_map { |thread| thread.thread_variable_get(Reliable::CLAIM_KEY) }
113
+ fetchers = @config.capsules.each_value.map(&:fetcher).grep(Reliable)
114
+ claims + fetchers.flat_map(&:held_acks)
115
+ end
116
+
117
+ def settle(crowded, contents, held)
118
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
119
+ following = {}
120
+ moved = contents.sum do |list, payloads|
121
+ stranded(payloads, held[list]).sum do |payload, count|
122
+ judge(list, crowded[list], payload, count, now, following)
123
+ end
124
+ end
125
+ @suspects = following
126
+ moved
127
+ end
128
+
129
+ # Moves what has been seen stranded for long enough, remembers the rest.
130
+ def judge(list, public_q, payload, count, now, following) # rubocop:disable Metrics/ParameterLists
131
+ suspect = @suspects.dig(list, payload)
132
+ confirmed = suspect && now - suspect.since >= @confirm
133
+ return put_back(list, public_q, payload, [count, suspect.copies].min) if confirmed
134
+
135
+ (following[list] ||= {})[payload] = Suspect.new(count, suspect&.since || now)
136
+ 0
137
+ end
138
+
139
+ def stranded(payloads, held)
140
+ held ||= {}
141
+ payloads.map(&:b).tally.filter_map do |payload, count|
142
+ excess = count - held.fetch(payload, 0)
143
+ [payload, excess] if excess.positive?
144
+ end
145
+ end
146
+
147
+ def put_back(list, public_q, payload, count)
148
+ moved = 0
149
+ count.times do
150
+ break if @stop.call
151
+
152
+ moved += requeue(list, public_q, payload)
153
+ end
154
+ moved
155
+ end
156
+
157
+ # LPOS-guarded, so a replay after a lost reply moves nothing the second
158
+ # time, and a payload settled since the last read stays settled.
159
+ def requeue(list, public_q, payload)
160
+ applied = @config.redis(idempotent: true) do |conn|
161
+ Wurk::Lua::Loader.eval_cached(conn, :reliable_requeue, keys: [list, public_q], argv: [payload])
162
+ rescue RedisClient::OutOfMemoryError => e
163
+ OomDrain.requeue(conn, e, list, public_q, payload)
164
+ end.to_i
165
+ @on_moved&.call(public_q, payload) if applied == 1
166
+ applied
167
+ end
168
+
169
+ def forget_all
170
+ @suspects = {}
171
+ 0
172
+ end
173
+ end
174
+ end
175
+ end
@@ -41,6 +41,19 @@ module Wurk
41
41
 
42
42
  UnitOfWork = Struct.new(:queue, :queue_name, :private_queue, :job, :config, :jid, :fetcher,
43
43
  :slot_key, :slot_token, keyword_init: true) do
44
+ # The positional form is upstream's documented `UnitOfWork.new(queue,
45
+ # job, config)`, which gem test suites build directly. No fetcher means
46
+ # no private list, so #acknowledge has nothing to do. Anonymous `**` is
47
+ # the cheap spelling for the fetch path, which always passes keywords:
48
+ # one allocation and ~230ns per job (a named **fields costs two).
49
+ def initialize(queue = nil, job = nil, config = nil, **)
50
+ if queue
51
+ super(queue: queue, queue_name: queue.delete_prefix('queue:'), job: job, config: config, **)
52
+ else
53
+ super(**)
54
+ end
55
+ end
56
+
44
57
  # Deferred, never skipped: the LREM goes back to the fetcher, which
45
58
  # pipelines it in front of the next fetch's LMOVE instead of spending a
46
59
  # round trip of its own. Ordering against the job is unchanged — the
@@ -48,7 +61,7 @@ module Wurk
48
61
  # so all that moves is the wall clock. Every path that stops fetching
49
62
  # flushes first; see #flush_pending_acks.
50
63
  def acknowledge
51
- fetcher.defer_ack(self)
64
+ fetcher&.defer_ack(self)
52
65
  end
53
66
 
54
67
  # Queue this unit's ACK into an already-open pipeline. The counter DEL
@@ -107,18 +120,29 @@ module Wurk
107
120
  # both lists and runs twice — once from the public queue, once when the
108
121
  # reaper reclaims this process's private list.
109
122
  #
110
- # LREM then RPUSH in one MULTI, unconditionally: unlike bulk_requeue
123
+ # RPUSH then LREM in one script, unconditionally: unlike bulk_requeue
111
124
  # (which races a Processor ACKing the same UoW and so guards the push on
112
- # the LREM), the caller here owns the unit, and a job whose ACK already
113
- # went out must still be re-queued rather than dropped.
125
+ # the private copy), the caller here owns the unit, and a job whose ACK
126
+ # already went out must still be re-queued rather than dropped. A script
127
+ # because a MULTI would apply the LREM after a refused RPUSH, losing the
128
+ # job. Unconditional is also why it goes by call_once: redis-client
129
+ # re-sends on a dropped socket, and the replay would push the job a second
130
+ # time. A raise leaves the outcome unknown, and both outcomes are safe:
131
+ # moved, or still in the private list, where Fetcher::SelfReconcile finds
132
+ # it.
114
133
  def requeue
115
134
  config.redis do |conn|
116
- conn.multi do |tx|
117
- tx.call('LREM', private_queue, LREM_COUNT, job)
118
- tx.call('RPUSH', queue, job)
119
- end
135
+ next conn.call_once('RPUSH', queue, job) unless private_queue
136
+
137
+ Wurk::Lua::Loader.eval_once(conn, :requeue_unit, keys: [private_queue, queue], argv: [job])
120
138
  end
121
139
  end
140
+
141
+ # The Processor has settled this unit, one way or the other: stop
142
+ # vouching for its payload. See Fetcher::SelfReconcile.
143
+ def release_claim
144
+ ::Thread.current.thread_variable_set(CLAIM_KEY, nil)
145
+ end
122
146
  end
123
147
  end
124
148
  end
data/lib/wurk/health.rb CHANGED
@@ -6,14 +6,14 @@ require_relative 'metrics/prometheus'
6
6
 
7
7
  module Wurk
8
8
  # Thin HTTP listener for k8s liveness/readiness probes. Optional, off by
9
- # default — opt in with `config.health_check(port: 7433)`, then scrape
10
- # GET /metrics on that configured port (7433 in this example).
9
+ # default — opt in with `config.health_check(port: 7433)` or
10
+ # WURK_HEALTH_PORT=7433, then scrape GET /metrics on that port.
11
11
  # Mounting Wurk::Engine does not expose Prometheus metrics: this deliberate
12
12
  # separation keeps internal metrics off the host application's HTTP surface
13
13
  # so access can be restricted independently on the dedicated listener port.
14
14
  #
15
15
  # Endpoints:
16
- # * GET /live → 200 while the Launcher is running (not in quiet/stop).
16
+ # * GET /live → 200 until the Launcher is shutting down; quiet stays 200.
17
17
  # * GET /ready → 200 only when Redis is reachable AND the heartbeat has
18
18
  # fired within `ready_window` seconds AND, inside a swarm
19
19
  # child, enough of the swarm's children have (see
@@ -43,6 +43,22 @@ module Wurk
43
43
  # fleet of one and /ready judges them on their own heartbeat alone.
44
44
  attr_accessor :fleet_size
45
45
 
46
+ # WURK_HEALTH_PORT turns the listener on from the environment, so whoever
47
+ # owns the deployment manifest can add a probe without an application
48
+ # change. `config.health_check` wins when both are set. A bad value raises:
49
+ # a probe silently not listening would restart the pod forever instead.
50
+ def options_from_env(env = ENV)
51
+ raw = env['WURK_HEALTH_PORT'].to_s.strip
52
+ return nil if raw.empty?
53
+
54
+ port = Integer(raw, 10)
55
+ raise ArgumentError unless port.between?(0, 65_535)
56
+
57
+ { port: port }
58
+ rescue ArgumentError
59
+ raise ArgumentError, "WURK_HEALTH_PORT must be a port number (got #{raw.inspect})"
60
+ end
61
+
46
62
  def json(status, **extra)
47
63
  ::JSON.generate({ status: status }.merge(extra))
48
64
  end
@@ -278,8 +294,11 @@ module Wurk
278
294
  @fleet ||= Fleet.new(@config, @ready_window)
279
295
  end
280
296
 
297
+ # A quiet process is alive and is finishing its jobs. Failing here would have
298
+ # the orchestrator restart a pod an operator just quieted, and the
299
+ # replacement would fetch again. /ready is the probe that speaks to traffic.
281
300
  def live_response
282
- if @launcher.stopping?
301
+ if @launcher.shutting_down?
283
302
  [json('down', check: 'live', reason: 'stopping'), 503]
284
303
  else
285
304
  [json('ok', check: 'live'), 200]
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'component'
4
+ require_relative 'keys'
5
+
6
+ module Wurk
7
+ # `wurk health`: an exec-style liveness probe, for a pod with no HTTP port. It
8
+ # boots no application, so it is fast and only needs to reach Redis (REDIS_URL
9
+ # or --redis-url). Passes when enough processes on this host have a recent
10
+ # heartbeat.
11
+ #
12
+ # A Redis it cannot reach is not this pod's fault, and restarting workers does
13
+ # not bring Redis back, so that passes too unless `--strict`.
14
+ class HealthProbe
15
+ HEALTHY = 0
16
+ UNHEALTHY = 1
17
+ DEFAULT_WINDOW = 60
18
+
19
+ # At most connect + two round trips: about 2s in all against a hung server.
20
+ CLIENT_LIMITS = { connect_timeout: 0.5, read_timeout: 0.6, write_timeout: 0.6, reconnect_attempts: 0 }.freeze
21
+
22
+ def initialize(config, window: DEFAULT_WINDOW, min: 1, strict: false, hostname: Component.hostname, out: $stdout)
23
+ @config = config
24
+ @window = window
25
+ @min = min
26
+ @strict = strict
27
+ @prefix = "#{hostname}:"
28
+ @out = out
29
+ end
30
+
31
+ def call
32
+ fresh = fresh_processes
33
+ @out.puts "wurk health: #{fresh} process(es) on this host beat within #{@window}s (need #{@min})"
34
+ fresh >= @min ? HEALTHY : UNHEALTHY
35
+ rescue RedisClient::Error, ConnectionPool::TimeoutError => e
36
+ @out.puts "wurk health: cannot read Redis (#{e.class}: #{e.message})"
37
+ @strict ? UNHEALTHY : HEALTHY
38
+ end
39
+
40
+ private
41
+
42
+ def fresh_processes
43
+ now = ::Time.now.to_f
44
+ read_local_beats.count { |beat| beat && now - beat.to_f < @window }
45
+ end
46
+
47
+ def read_local_beats
48
+ client = build_client
49
+ ids = client.call('SMEMBERS', Keys::PROCESSES).select { |id| id.start_with?(@prefix) }
50
+ ids.empty? ? [] : client.pipelined { |pipe| ids.each { |id| pipe.call('HGET', id, 'beat') } }
51
+ ensure
52
+ client&.close
53
+ end
54
+
55
+ # Directly on redis-client rather than through RedisPool: the pool's retries
56
+ # and backoff are right for a worker and fatal here. They took 4s on a refused
57
+ # connection and 19s on a hung server, past a kubelet's probe timeout, so a
58
+ # store outage restarted every pod.
59
+ def build_client
60
+ opts = RedisOptions.normalize(@config.redis_config).except(*RedisOptions::POOL_KEYS).merge(CLIENT_LIMITS)
61
+ (RedisOptions.sentinel?(opts) ? RedisClient.sentinel(**opts) : RedisClient.config(**opts)).new_client
62
+ end
63
+ end
64
+ end
@@ -3,6 +3,7 @@
3
3
  require 'etc'
4
4
 
5
5
  require_relative 'component'
6
+ require_relative 'fetcher/oom_drain'
6
7
  require_relative 'keys'
7
8
  require_relative 'lua'
8
9
  require_relative 'processor'
@@ -38,12 +39,30 @@ module Wurk
38
39
  # network partition (rearmed when beat! rescues). `:beat` fires every tick.
39
40
  #
40
41
  # Spec: docs/target/sidekiq-free.md §12 (Launcher#❤️).
41
- class Heartbeat
42
+ class Heartbeat # rubocop:disable Metrics/ClassLength
42
43
  include Component
43
44
 
44
45
  # Cadence in seconds. Key TTL is 60s — a process is dead after ~6 misses.
45
46
  BEAT_PAUSE = 10
46
47
  TTL_SECONDS = 60
48
+
49
+ OOM_REFRESH_SLOTS = "#{Fetcher::OomDrain::HEADER}#{Lua::SCRIPTS.fetch(:refresh_slots)}".freeze
50
+
51
+ # KEYS = [processes, identity, work], ARGV = [info_json, beat, ttl_seconds, tid, work_json, ...]
52
+ OOM_BEAT = <<~LUA.freeze
53
+ #{Fetcher::OomDrain::HEADER}if redis.call('EXISTS', KEYS[2]) == 0 then
54
+ redis.call('HSET', KEYS[2], 'info', ARGV[1])
55
+ redis.call('SADD', KEYS[1], KEYS[2])
56
+ end
57
+ redis.call('HSET', KEYS[2], 'beat', ARGV[2])
58
+ redis.call('EXPIRE', KEYS[2], ARGV[3])
59
+ redis.call('UNLINK', KEYS[3])
60
+ for i = 4, #ARGV, 2 do
61
+ redis.call('HSET', KEYS[3], ARGV[i], ARGV[i + 1])
62
+ end
63
+ redis.call('EXPIRE', KEYS[3], ARGV[3])
64
+ return 1
65
+ LUA
47
66
  FALLBACK_PAGE_SIZE = 4096
48
67
 
49
68
  # statm counts pages, and a page is not 4 KB everywhere: arm64 kernels
@@ -131,11 +150,38 @@ module Wurk
131
150
  t0 = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
132
151
  redis(idempotent: true) do |conn|
133
152
  Wurk::Lua::Loader.pipelined_eval(conn) { |pipe, eval_method| write_beat(pipe, work_snapshot, eval_method) }
153
+ rescue RedisClient::OutOfMemoryError => e
154
+ beat_at_maxmemory(conn, e, work_snapshot)
134
155
  end
135
156
  rtt = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) - t0
136
157
  [drain_signals, rtt]
137
158
  end
138
159
 
160
+ # At maxmemory the beat's HSET and SADD are refused. A worker with every thread
161
+ # in a long job claims nothing, so nothing else proves it alive: its `beat`
162
+ # would stand still and Swarm::Liveness would TERM it, just when the fleet has
163
+ # to stay up. Its global-concurrency holds would lapse the same way, and the
164
+ # cap then over-admit. Shebang scripts, so by EVAL for the reason
165
+ # Fetcher::OomDrain gives. A server refusing one raises the original OOM, the
166
+ # failure without this.
167
+ def beat_at_maxmemory(conn, error, work_snapshot = {})
168
+ work = work_snapshot.flat_map { |tid, hash| [tid.to_s, Wurk.dump_json(hash)] }
169
+ conn.call('EVAL', OOM_BEAT, 3, Keys::PROCESSES, @identity, "#{@identity}:work",
170
+ Wurk.dump_json(info_hash), Time.now.to_f.to_s, TTL_SECONDS, *work)
171
+ refresh_holds_at_maxmemory(conn)
172
+ rescue RedisClient::ConnectionError
173
+ raise
174
+ rescue RedisClient::CommandError
175
+ raise error
176
+ end
177
+
178
+ def refresh_holds_at_maxmemory(conn)
179
+ held = QueueSlot::HELD.snapshot
180
+ return if held.empty?
181
+
182
+ conn.call('EVAL', OOM_REFRESH_SLOTS, held.size, *held.values, QueueSlot::TTL_SECONDS, *held.keys)
183
+ end
184
+
139
185
  def write_beat(pipe, work_snapshot, eval_method = :eval_cached)
140
186
  pipe.call('SADD', Keys::PROCESSES, @identity)
141
187
  pipe.call('HSET', @identity, *beat_hash_args(work_snapshot.size))
@@ -163,12 +209,18 @@ module Wurk
163
209
 
164
210
  def write_work_hash(pipe, work_snapshot)
165
211
  work_key = "#{@identity}:work"
166
- pipe.call('UNLINK', work_key)
167
- return if work_snapshot.empty?
212
+ return pipe.call('UNLINK', work_key) if work_snapshot.empty?
168
213
 
169
214
  args = work_snapshot.flat_map { |tid, hash| [tid.to_s, Wurk.dump_json(hash)] }
215
+ # In a transaction because UNLINK is allowed at maxmemory and HSET is not: on
216
+ # its own that erased the mirror of jobs still running. On Redis and Valkey a
217
+ # refused queued command aborts the EXEC, so the old mirror stays. Dragonfly
218
+ # does not abort, so there the mirror is still erased, as it always was.
219
+ pipe.call('MULTI')
220
+ pipe.call('UNLINK', work_key)
170
221
  pipe.call('HSET', work_key, *args)
171
222
  pipe.call('EXPIRE', work_key, TTL_SECONDS)
223
+ pipe.call('EXEC')
172
224
  end
173
225
 
174
226
  # Its own checkout, and never an idempotent one: RPOP is destructive and
@@ -40,7 +40,7 @@ module Wurk
40
40
  # table for the full set)
41
41
  # @return [Hash] the merged, string-keyed options hash
42
42
  def sidekiq_options(opts = {})
43
- stringified = opts.transform_keys(&:to_s)
43
+ stringified = opts.to_h { |key, val| [key.to_s, val.is_a?(::Hash) ? val.transform_keys(&:to_s) : val] }
44
44
  Wurk::JobUtil.validate_track!(stringified['track'], stringified) if stringified.key?('track')
45
45
  Wurk::JobUtil.validate_bounds!(stringified)
46
46
  merged = get_sidekiq_options.merge(stringified)
@@ -61,6 +61,8 @@ module Wurk
61
61
  get_sidekiq_options
62
62
  end
63
63
 
64
+ attr_writer :sidekiq_options_hash
65
+
64
66
  attr_reader :sidekiq_retry_in_block, :sidekiq_retries_exhausted_block
65
67
 
66
68
  def sidekiq_retry_in(&block)
@@ -96,4 +98,10 @@ module Wurk
96
98
  end
97
99
  end
98
100
  end
101
+
102
+ # One module under both names, as upstream's Sidekiq::Worker and Sidekiq::Job
103
+ # are, so a prepend, `method_defined?` or constant lookup on either sees the
104
+ # job methods. Assigned here because worker.rb and worker/setter.rb reopen it
105
+ # as `module Worker`, and everything that loads them loads this file first.
106
+ Worker = Job
99
107
  end
data/lib/wurk/job.rb CHANGED
@@ -4,13 +4,8 @@ require_relative 'worker'
4
4
  require_relative 'worker/setter'
5
5
 
6
6
  module Wurk
7
- # Sidekiq 7+ alias for Wurk::Worker. `include Wurk::Job` and
8
- # `include Sidekiq::Job` are the same surface.
9
- #
10
- # Instance-level `jid`, `_context`, `interrupted?`, and `logger` come
11
- # from Wurk::Worker. Class-level DSL (`sidekiq_options`, `perform_*`,
12
- # `set`, retry blocks) does too — Job is a pure alias module that
13
- # re-exposes Worker under the modern name.
7
+ # The job mixin's control-flow errors. Wurk::Worker and Wurk::Job are one
8
+ # module (see job/options.rb), whose DSL and instance methods are in worker.rb.
14
9
  module Job
15
10
  # Raised mid-iteration when the run loop must yield — either a swarm
16
11
  # shutdown signal or a cooperative cancellation. The interrupt-handler
@@ -39,22 +34,5 @@ module Wurk
39
34
  # same terminal state as one whose deadline had already passed before it
40
35
  # started. A job may still rescue it to clean up after itself.
41
36
  class DeadlineExceeded < RuntimeError; end
42
-
43
- # Per-call option carrier returned by `set(...)`. Sidekiq 7+ documents it
44
- # under the modern mixin name `Sidekiq::Job::Setter`; since
45
- # `Sidekiq::Job = Wurk::Job`, this rebind is what makes that constant
46
- # resolve (without it `Sidekiq::Job::Setter` raises NameError). Same class
47
- # as `Sidekiq::Worker::Setter`. Spec: docs/target/sidekiq-free.md §6.3.
48
- Setter = Wurk::Worker::Setter
49
-
50
- def self.included(base)
51
- base.include(Wurk::Worker)
52
- end
53
-
54
- # Mirror the module-level test helpers so `Sidekiq::Job.jobs /
55
- # clear_all / drain_all` work the same as `Sidekiq::Worker.*`.
56
- def self.jobs = Wurk::Worker.jobs
57
- def self.clear_all = Wurk::Worker.clear_all
58
- def self.drain_all = Wurk::Worker.drain_all
59
37
  end
60
38
  end
@@ -41,16 +41,13 @@ module Wurk
41
41
 
42
42
  def queue = @queue || item['queue']
43
43
 
44
- # Lazily parsed payload. Memoized; never re-parses. Invalid JSON reads as
45
- # an empty hash whose raw bytes become `args`, as upstream does, so one
46
- # corrupt payload can't break a dashboard page or a Queue#each walk.
44
+ # Lazily parsed payload. Memoized; never re-parses. Anything that is not a
45
+ # JSON object reads as an empty hash whose raw bytes become `args`, as
46
+ # upstream does for invalid JSON, so one corrupt payload can't break a
47
+ # dashboard page or a Queue#each walk. Valid JSON counts: `[1,2]` and `123`
48
+ # would otherwise raise from every reader that indexes it by field name.
47
49
  def item
48
- @item ||= begin
49
- Wurk.load_json(@value)
50
- rescue ::JSON::ParserError
51
- @invalid_args = [@value]
52
- {}
53
- end
50
+ @item ||= parse_payload
54
51
  end
55
52
 
56
53
  # Lazily serialized payload. When constructed from a Hash, we
@@ -155,6 +152,17 @@ module Wurk
155
152
 
156
153
  private
157
154
 
155
+ def parse_payload
156
+ parsed = Wurk.load_json(@value)
157
+ return parsed if parsed.is_a?(::Hash)
158
+
159
+ @invalid_args = [@value]
160
+ {}
161
+ rescue ::JSON::ParserError, ::EncodingError
162
+ @invalid_args = [@value]
163
+ {}
164
+ end
165
+
158
166
  def active_job_wrapper?
159
167
  klass.is_a?(String) && (ACTIVE_JOB_WRAPPER.match?(klass) || klass == 'Sidekiq::ActiveJob::Wrapper')
160
168
  end
@@ -3,6 +3,7 @@
3
3
  require 'zlib'
4
4
  require_relative 'component'
5
5
  require_relative 'dead_set'
6
+ require_relative 'fetcher/oom_drain'
6
7
 
7
8
  module Wurk
8
9
  # Owns the retry pipeline. When perform raises, JobRetry decides whether to
@@ -88,8 +89,13 @@ module Wurk
88
89
  # Component's `handle_exception` delegates to `config.handle_exception`.
89
90
  # When initialized with a Capsule, that's not defined directly; route
90
91
  # through the underlying Configuration.
92
+ #
93
+ # Reporting a hook's failure from inside the failure path, through a logger or
94
+ # handler that is itself host code: it can't be allowed to raise from here.
91
95
  def handle_exception(ex, ctx = {})
92
96
  inner_config.handle_exception(ex, ctx)
97
+ rescue ::StandardError => e
98
+ warn "wurk-retry: error reporter raised #{e.class}: #{e.message}"
93
99
  end
94
100
 
95
101
  private
@@ -167,11 +173,21 @@ module Wurk
167
173
  return unless msg['backtrace']
168
174
  return if exception.backtrace.nil?
169
175
 
170
- cleaned = (@backtrace_cleaner || ->(bt) { bt }).call(exception.backtrace)
176
+ cleaned = clean_backtrace(exception.backtrace)
171
177
  lines = msg['backtrace'] == true ? cleaned : cleaned[0...msg['backtrace'].to_i]
172
178
  msg['error_backtrace'] = compress_backtrace(lines)
173
179
  end
174
180
 
181
+ # Host code, like the other hooks here: a raise out of this path is a job
182
+ # the Processor could not book, and it kills those.
183
+ def clean_backtrace(backtrace)
184
+ cleaned = @backtrace_cleaner ? @backtrace_cleaner.call(backtrace) : backtrace
185
+ cleaned.is_a?(::Array) ? cleaned : backtrace
186
+ rescue ::Exception => e # rubocop:disable Lint/RescueException
187
+ handle_exception(e, context: 'Error calling backtrace_cleaner')
188
+ backtrace
189
+ end
190
+
175
191
  def retry_for_exceeded?(failed_at, retry_for)
176
192
  return false unless failed_at
177
193
 
@@ -182,9 +198,7 @@ module Wurk
182
198
  jitter = rand(10 * (count + 1))
183
199
  retry_at = ::Time.now.to_f + delay + jitter
184
200
  payload = Wurk.dump_json(msg)
185
- redis do |conn|
186
- conn.call('ZADD', Keys::RETRY, retry_at.to_s, payload)
187
- end
201
+ zadd_past_maxmemory(Keys::RETRY, retry_at.to_s, payload)
188
202
  Wurk::Metrics::Statsd.increment(
189
203
  'jobs.retried',
190
204
  tags: ["worker:#{msg['class']}", "queue:#{msg['queue']}"]
@@ -196,6 +210,17 @@ module Wurk
196
210
  Wurk::Middleware::Status.retrying(msg)
197
211
  end
198
212
 
213
+ # A failing job has to be booked even at maxmemory: the fetcher can still
214
+ # drain a full Redis, and a failure it could not book would sit in the private
215
+ # list until the next reconcile, then run and fail again.
216
+ def zadd_past_maxmemory(key, score, member)
217
+ redis do |conn|
218
+ conn.call('ZADD', key, score, member)
219
+ rescue RedisClient::OutOfMemoryError => e
220
+ Wurk::Fetcher::OomDrain.zadd(conn, e, key, score, member)
221
+ end
222
+ end
223
+
199
224
  def time_for(item)
200
225
  if item.is_a?(::Float)
201
226
  ::Time.at(item)
@@ -208,9 +233,6 @@ module Wurk
208
233
  # Caller branches on strategy; seconds is meaningful only for :default.
209
234
  def delay_for(jobinst, count, exception, msg)
210
235
  rv = run_retry_in_block(jobinst, count, exception, msg)
211
- # `10.minutes * count` (a Duration), `"60"` and `30.0` all mean seconds;
212
- # Symbols have no #to_i, so `:kill` / `:discard` pass through intact.
213
- rv = rv.to_i if rv.respond_to?(:to_i)
214
236
  default_delay = (count**4) + 15
215
237
 
216
238
  case rv
@@ -227,7 +249,10 @@ module Wurk
227
249
 
228
250
  def run_retry_in_block(jobinst, count, exception, msg)
229
251
  block = msg['wrapped'] ? wrapped_block(msg, :sidekiq_retry_in_block) : jobinst&.class&.sidekiq_retry_in_block
230
- block&.call(count, exception, msg)
252
+ rv = block&.call(count, exception, msg)
253
+ # `10.minutes * count` (a Duration), `"60"` and `30.0` all mean seconds;
254
+ # Symbols have no #to_i, so `:kill` / `:discard` pass through intact.
255
+ rv.respond_to?(:to_i) ? rv.to_i : rv
231
256
  rescue ::Exception => e # rubocop:disable Lint/RescueException
232
257
  handle_exception(e, context: "Failure scheduling retry via `sidekiq_retry_in` on #{jobinst&.class&.name}")
233
258
  nil
@@ -274,13 +299,16 @@ module Wurk
274
299
 
275
300
  def send_to_morgue(msg)
276
301
  logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
277
- DeadSet.new.kill_raw(Wurk.dump_json(msg))
302
+ payload = Wurk.dump_json(msg)
303
+ DeadSet.new.kill_raw(payload)
304
+ rescue RedisClient::OutOfMemoryError
305
+ zadd_past_maxmemory(Keys::DEAD, ::Process.clock_gettime(::Process::CLOCK_REALTIME).to_s, payload)
278
306
  end
279
307
 
280
308
  def run_death_handlers(job, exception)
281
309
  inner_config.death_handlers.each do |handler|
282
310
  handler.call(job, exception)
283
- rescue ::StandardError => e
311
+ rescue ::Exception => e # rubocop:disable Lint/RescueException
284
312
  handle_exception(e, context: 'Error calling death handler', job: job)
285
313
  end
286
314
  end