wurk 1.7.6 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +7 -4
  3. data/app/controllers/concerns/wurk/sse_streaming.rb +2 -1
  4. data/app/controllers/concerns/wurk/stream_concurrency_guard.rb +33 -9
  5. data/app/controllers/wurk/api/pagination.rb +25 -7
  6. data/app/controllers/wurk/api_controller.rb +49 -25
  7. data/app/controllers/wurk/extensions_controller.rb +8 -2
  8. data/app/controllers/wurk/profiles_controller.rb +45 -9
  9. data/config/routes.rb +31 -10
  10. data/lib/generators/wurk/install/templates/wurk.rb +1 -1
  11. data/lib/sidekiq/capsule.rb +4 -0
  12. data/lib/sidekiq/component.rb +4 -0
  13. data/lib/sidekiq/config.rb +4 -0
  14. data/lib/sidekiq/deploy.rb +4 -0
  15. data/lib/sidekiq/embedded.rb +4 -0
  16. data/lib/sidekiq/fetch.rb +4 -0
  17. data/lib/sidekiq/iterable_job.rb +4 -0
  18. data/lib/sidekiq/job/interrupt_handler.rb +4 -0
  19. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +4 -0
  20. data/lib/sidekiq/job/iterable/csv_enumerator.rb +4 -0
  21. data/lib/sidekiq/job/iterable.rb +4 -0
  22. data/lib/sidekiq/job_logger.rb +4 -0
  23. data/lib/sidekiq/job_util.rb +4 -0
  24. data/lib/sidekiq/loader.rb +4 -0
  25. data/lib/sidekiq/logger.rb +4 -0
  26. data/lib/sidekiq/metrics/query.rb +4 -0
  27. data/lib/sidekiq/middleware/current_attributes.rb +7 -0
  28. data/lib/sidekiq/middleware/i18n.rb +6 -0
  29. data/lib/sidekiq/middleware/modules.rb +4 -0
  30. data/lib/sidekiq/pro/web.rb +5 -0
  31. data/lib/sidekiq/profiler.rb +4 -0
  32. data/lib/sidekiq/redis_client_adapter.rb +4 -0
  33. data/lib/sidekiq/test_api.rb +5 -0
  34. data/lib/sidekiq/testing/inline.rb +8 -0
  35. data/lib/sidekiq/transaction_aware_client.rb +4 -0
  36. data/lib/sidekiq/worker_compatibility_alias.rb +4 -0
  37. data/lib/sidekiq-ent/periodic/testing.rb +5 -0
  38. data/lib/sidekiq-ent/web.rb +5 -0
  39. data/lib/sidekiq-ent.rb +7 -0
  40. data/lib/sidekiq-pro.rb +7 -0
  41. data/lib/wurk/api/fast.rb +8 -33
  42. data/lib/wurk/api/idempotency.rb +28 -5
  43. data/lib/wurk/api/swarm.rb +1 -1
  44. data/lib/wurk/batch/callbacks.rb +143 -92
  45. data/lib/wurk/batch/death_handler.rb +19 -8
  46. data/lib/wurk/batch/server_middleware.rb +37 -24
  47. data/lib/wurk/batch/status.rb +49 -15
  48. data/lib/wurk/batch.rb +164 -65
  49. data/lib/wurk/capsule.rb +0 -2
  50. data/lib/wurk/client/buffered.rb +227 -106
  51. data/lib/wurk/client.rb +33 -13
  52. data/lib/wurk/compat.rb +13 -0
  53. data/lib/wurk/configuration.rb +155 -24
  54. data/lib/wurk/cpu_count.rb +97 -0
  55. data/lib/wurk/cron.rb +97 -34
  56. data/lib/wurk/dead_set.rb +6 -7
  57. data/lib/wurk/encryption.rb +46 -13
  58. data/lib/wurk/fetcher/many_queues.rb +65 -0
  59. data/lib/wurk/fetcher/orphan_grace.rb +67 -0
  60. data/lib/wurk/fetcher/private_list_key.rb +76 -0
  61. data/lib/wurk/fetcher/reaper.rb +55 -74
  62. data/lib/wurk/fetcher/reliable.rb +23 -17
  63. data/lib/wurk/fetcher/unit_of_work.rb +19 -1
  64. data/lib/wurk/fetcher/unparseable_keys.rb +56 -0
  65. data/lib/wurk/flow/completion.rb +28 -3
  66. data/lib/wurk/flow/creation.rb +24 -4
  67. data/lib/wurk/flow.rb +24 -2
  68. data/lib/wurk/health.rb +182 -40
  69. data/lib/wurk/heartbeat.rb +37 -14
  70. data/lib/wurk/history.rb +7 -1
  71. data/lib/wurk/import/sidekiq_cron.rb +248 -0
  72. data/lib/wurk/job/options.rb +22 -3
  73. data/lib/wurk/job_logger.rb +13 -3
  74. data/lib/wurk/job_record.rb +68 -19
  75. data/lib/wurk/job_retry.rb +12 -6
  76. data/lib/wurk/job_set.rb +99 -51
  77. data/lib/wurk/job_util.rb +27 -8
  78. data/lib/wurk/launcher.rb +84 -40
  79. data/lib/wurk/leader.rb +53 -28
  80. data/lib/wurk/limiter/base.rb +11 -30
  81. data/lib/wurk/limiter/bucket.rb +14 -5
  82. data/lib/wurk/limiter/server_middleware.rb +2 -1
  83. data/lib/wurk/limiter.rb +1 -1
  84. data/lib/wurk/loader.rb +48 -0
  85. data/lib/wurk/lua/batch_ack_complete.lua +28 -0
  86. data/lib/wurk/lua/batch_ack_failed.lua +18 -0
  87. data/lib/wurk/lua/batch_ack_success.lua +35 -0
  88. data/lib/wurk/lua/batch_append_callback.lua +66 -0
  89. data/lib/wurk/lua/batch_delete.lua +31 -0
  90. data/lib/wurk/lua/batch_hold.lua +17 -0
  91. data/lib/wurk/lua/batch_invalidate.lua +14 -0
  92. data/lib/wurk/lua/batch_kid_done.lua +14 -0
  93. data/lib/wurk/lua/batch_push.lua +47 -0
  94. data/lib/wurk/lua/batch_remove_jobs.lua +29 -0
  95. data/lib/wurk/lua/batch_schedule.lua +27 -0
  96. data/lib/wurk/lua/fetch_first.lua +27 -0
  97. data/lib/wurk/lua/flow_abandon.lua +16 -21
  98. data/lib/wurk/lua/flow_advance.lua +41 -26
  99. data/lib/wurk/lua/flow_create.lua +24 -9
  100. data/lib/wurk/lua/flow_fail.lua +3 -3
  101. data/lib/wurk/lua/leader_campaign.lua +19 -0
  102. data/lib/wurk/lua/limiter_bucket_acquire.lua +3 -2
  103. data/lib/wurk/lua/limiter_list_sweep.lua +8 -9
  104. data/lib/wurk/lua/loader.rb +36 -23
  105. data/lib/wurk/lua/throttle_slot.lua +13 -26
  106. data/lib/wurk/lua.rb +31 -244
  107. data/lib/wurk/manager.rb +25 -18
  108. data/lib/wurk/metrics/accumulator.rb +31 -10
  109. data/lib/wurk/metrics/dashboard_series.rb +163 -0
  110. data/lib/wurk/metrics/histogram.rb +46 -0
  111. data/lib/wurk/metrics/history.rb +49 -57
  112. data/lib/wurk/metrics/prometheus.rb +232 -0
  113. data/lib/wurk/metrics/query.rb +127 -155
  114. data/lib/wurk/metrics/rollup.rb +2 -2
  115. data/lib/wurk/metrics/statsd.rb +1 -1
  116. data/lib/wurk/middleware/interrupt_handler.rb +6 -2
  117. data/lib/wurk/middleware/poison_pill.rb +5 -4
  118. data/lib/wurk/middleware/timeout.rb +8 -8
  119. data/lib/wurk/process_set.rb +5 -5
  120. data/lib/wurk/processor.rb +69 -31
  121. data/lib/wurk/profile_set.rb +19 -24
  122. data/lib/wurk/profiler.rb +48 -50
  123. data/lib/wurk/queue.rb +36 -19
  124. data/lib/wurk/rails_boot.rb +33 -1
  125. data/lib/wurk/railtie.rb +2 -0
  126. data/lib/wurk/rake_tasks.rb +18 -0
  127. data/lib/wurk/redact.rb +39 -0
  128. data/lib/wurk/redis_connection.rb +6 -3
  129. data/lib/wurk/redis_options.rb +21 -4
  130. data/lib/wurk/redis_pool.rb +119 -21
  131. data/lib/wurk/scheduled.rb +44 -11
  132. data/lib/wurk/sentry/error_handler.rb +53 -7
  133. data/lib/wurk/sentry/middleware.rb +7 -8
  134. data/lib/wurk/shutdown_gate.rb +24 -1
  135. data/lib/wurk/sorted_entry.rb +16 -5
  136. data/lib/wurk/stats.rb +9 -4
  137. data/lib/wurk/status.rb +2 -2
  138. data/lib/wurk/swarm/child_boot.rb +48 -38
  139. data/lib/wurk/swarm/liveness.rb +153 -0
  140. data/lib/wurk/swarm/restart.rb +24 -3
  141. data/lib/wurk/swarm.rb +115 -24
  142. data/lib/wurk/telemetry/server_middleware.rb +2 -5
  143. data/lib/wurk/testing.rb +15 -5
  144. data/lib/wurk/throttle.rb +16 -1
  145. data/lib/wurk/topology.rb +1 -5
  146. data/lib/wurk/unique.rb +1 -1
  147. data/lib/wurk/version.rb +1 -1
  148. data/lib/wurk/watchdog.rb +2 -2
  149. data/lib/wurk/web/config.rb +56 -20
  150. data/lib/wurk/web/enterprise.rb +21 -18
  151. data/lib/wurk/web/extension.rb +98 -22
  152. data/lib/wurk/web/rack_app.rb +12 -11
  153. data/lib/wurk/web/search.rb +18 -8
  154. data/lib/wurk/worker/setter.rb +17 -14
  155. data/lib/wurk/worker.rb +6 -67
  156. data/lib/wurk.rb +2 -3
  157. data/vendor/assets/dashboard/assets/ArgsValue-DsPb_KZ-.js +1 -0
  158. data/vendor/assets/dashboard/assets/BatchDetail-BCIXJOpZ.js +1 -0
  159. data/vendor/assets/dashboard/assets/Batches-IjaQfXW6.js +1 -0
  160. data/vendor/assets/dashboard/assets/Busy-CaQrf1Ve.js +1 -0
  161. data/vendor/assets/dashboard/assets/Cron-B18EHn_Y.js +1 -0
  162. data/vendor/assets/dashboard/assets/Dashboard-DdT5ZaUg.js +1 -0
  163. data/vendor/assets/dashboard/assets/Dead-DWIy-Mu8.js +1 -0
  164. data/vendor/assets/dashboard/assets/Extension-BjBUWhHf.js +1 -0
  165. data/vendor/assets/dashboard/assets/FilterBox-DwH3PXXt.js +1 -0
  166. data/vendor/assets/dashboard/assets/FlowDetail-DnexL72n.js +1 -0
  167. data/vendor/assets/dashboard/assets/FlowState-Cr4aX6-d.js +1 -0
  168. data/vendor/assets/dashboard/assets/Flows-DO6EimbZ.js +1 -0
  169. data/vendor/assets/dashboard/assets/JobDetailModal-aPjgq-U9.js +2 -0
  170. data/vendor/assets/dashboard/assets/Limiters-DxpwWCNe.js +1 -0
  171. data/vendor/assets/dashboard/assets/Metrics-C3JL1GIH.js +1 -0
  172. data/vendor/assets/dashboard/assets/NotFound-BfP9uAV9.js +1 -0
  173. data/vendor/assets/dashboard/assets/PageHeader-DbC97afG.js +1 -0
  174. data/vendor/assets/dashboard/assets/Profiles-BcaHreMe.js +1 -0
  175. data/vendor/assets/dashboard/assets/Queues-ZlBc5tyD.js +1 -0
  176. data/vendor/assets/dashboard/assets/Retries-Bg3nkeiL.js +1 -0
  177. data/vendor/assets/dashboard/assets/Scheduled-BrMZPZGl.js +1 -0
  178. data/vendor/assets/dashboard/assets/Search-B5X5pGht.js +1 -0
  179. data/vendor/assets/dashboard/assets/charts-uDyf4PN0.js +1 -0
  180. data/vendor/assets/dashboard/assets/i18n-CJeYfeVt.js +1 -0
  181. data/vendor/assets/dashboard/assets/index-B36QCAf-.css +1 -0
  182. data/vendor/assets/dashboard/assets/index-DQujgZgH.js +141 -0
  183. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-O3-5rMk3.js +1 -0
  184. data/vendor/assets/dashboard/assets/useSort-xfnwjQAu.js +1 -0
  185. data/vendor/assets/dashboard/assets/utils-BIKHwNKB.js +1 -0
  186. data/vendor/assets/dashboard/index.html +3 -3
  187. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  188. metadata +90 -34
  189. data/config/locales/en.yml +0 -15
  190. data/vendor/assets/dashboard/assets/ArgsValue-D-x_ifLY.js +0 -1
  191. data/vendor/assets/dashboard/assets/BatchDetail-C39NJuew.js +0 -1
  192. data/vendor/assets/dashboard/assets/Batches-CSwo7Asa.js +0 -1
  193. data/vendor/assets/dashboard/assets/Busy-BOFMu-sq.js +0 -1
  194. data/vendor/assets/dashboard/assets/Cron-Dy8RQzDI.js +0 -1
  195. data/vendor/assets/dashboard/assets/Dashboard-BuTHI-O1.js +0 -1
  196. data/vendor/assets/dashboard/assets/Dead-B9KRvQ0N.js +0 -1
  197. data/vendor/assets/dashboard/assets/Extension-BnBVHfux.js +0 -1
  198. data/vendor/assets/dashboard/assets/FilterBox-DC24zite.js +0 -1
  199. data/vendor/assets/dashboard/assets/FlowDetail-DyLuzUvt.js +0 -1
  200. data/vendor/assets/dashboard/assets/FlowState-DAPKUahm.js +0 -1
  201. data/vendor/assets/dashboard/assets/Flows-Fr3rjZM_.js +0 -1
  202. data/vendor/assets/dashboard/assets/JobDetailModal-N6kiJXq3.js +0 -2
  203. data/vendor/assets/dashboard/assets/Limiters-kbFA7uS1.js +0 -1
  204. data/vendor/assets/dashboard/assets/Metrics-Dj2uoZ3o.js +0 -1
  205. data/vendor/assets/dashboard/assets/PageHeader-B_F94azl.js +0 -1
  206. data/vendor/assets/dashboard/assets/Profiles-D_DjEezN.js +0 -1
  207. data/vendor/assets/dashboard/assets/Queues-CO4V9hAz.js +0 -1
  208. data/vendor/assets/dashboard/assets/Retries-DCWnzeLa.js +0 -1
  209. data/vendor/assets/dashboard/assets/Scheduled-BebDUjLU.js +0 -1
  210. data/vendor/assets/dashboard/assets/Search-Cvr5fy4Y.js +0 -1
  211. data/vendor/assets/dashboard/assets/Skeleton-Bu3Ke6rV.js +0 -1
  212. data/vendor/assets/dashboard/assets/charts-BCs9bQKz.js +0 -1
  213. data/vendor/assets/dashboard/assets/index-BIwyOC5Q.js +0 -141
  214. data/vendor/assets/dashboard/assets/index-DBQN6Jk8.css +0 -1
  215. data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Bzh-BJyL.js +0 -1
  216. data/vendor/assets/dashboard/assets/useSort-COA3fVJ5.js +0 -1
  217. data/vendor/assets/dashboard/assets/utils-BIrvZ1hi.js +0 -1
@@ -1,12 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'etc'
4
3
  require 'logger'
5
4
  require_relative 'middleware/chain'
6
5
  require_relative 'capsule'
7
6
  require_relative 'pool_checkout'
8
7
  require_relative 'context'
9
8
  require_relative 'topology'
9
+ require_relative 'cpu_count'
10
10
  require_relative 'redis_options'
11
11
  require_relative 'keys'
12
12
 
@@ -46,7 +46,7 @@ module Wurk
46
46
  reloader: proc { |&b| b.call },
47
47
  backtrace_cleaner: ->(bt) { bt },
48
48
  logged_job_attributes: %w[bid tags],
49
- redis_idle_timeout: nil,
49
+ redis_idle_timeout: nil, # seconds; RedisPool closes connections idle this long
50
50
  redis_error_handlers: [],
51
51
  # Wurk extras, appended after the mirrored keys so the Sidekiq prefix
52
52
  # above stays byte-for-byte what a gem reading @options expects.
@@ -121,6 +121,7 @@ module Wurk
121
121
  # Spec: docs/target/sidekiq-free.md §4.3.
122
122
  ERROR_HANDLER = lambda do |ex, ctx, cfg = Wurk.configuration|
123
123
  safe_ctx = ctx || {}
124
+ safe_ctx = cfg.redacted_context(safe_ctx) if cfg.respond_to?(:redacted_context)
124
125
  Wurk::Context.with(safe_ctx) do
125
126
  dev = $DEBUG || ENV['WURK_DEBUG'] || cfg.logger.debug?
126
127
  msg = dev ? ex.full_message : ex.detailed_message
@@ -157,7 +158,7 @@ module Wurk
157
158
  # Hash arrives frozen: the fetch path resolves caps once at boot, so a
158
159
  # Hash still mutable here is one a caller can add a queue to and have
159
160
  # nothing read it. FrozenError is the honest answer to that.
160
- @options[:global_concurrency] = normalize_global_concurrency(@options[:global_concurrency])
161
+ validate_constructor_options!
161
162
  @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
162
163
  @capsules = {}
163
164
  @directory = {}
@@ -177,6 +178,7 @@ module Wurk
177
178
 
178
179
  def []=(key, val)
179
180
  guard_frozen!
181
+ RedisPool.validate_idle_timeout(val) if key == :redis_idle_timeout
180
182
  @options[key] = val
181
183
  end
182
184
 
@@ -185,6 +187,7 @@ module Wurk
185
187
  alias has_key? key?
186
188
  def merge!(other)
187
189
  guard_frozen!
190
+ RedisPool.validate_idle_timeout(other[:redis_idle_timeout]) if other.key?(:redis_idle_timeout)
188
191
  @options.merge!(other)
189
192
  end
190
193
 
@@ -250,6 +253,12 @@ module Wurk
250
253
  @redis_config = @redis_config.merge(hash.transform_keys(&:to_sym))
251
254
  end
252
255
 
256
+ # Sidekiq's setter for `:redis_idle_timeout`; read when a pool is built, so
257
+ # call it in the initializer.
258
+ def reap_idle_redis_connections(timeout = 60)
259
+ self[:redis_idle_timeout] = timeout
260
+ end
261
+
253
262
  def redis_pool
254
263
  default_capsule.redis_pool
255
264
  end
@@ -360,6 +369,40 @@ module Wurk
360
369
  @options[:fetch_poll_interval]
361
370
  end
362
371
 
372
+ # Display-side argument redaction (Wurk::Redact): a callable taking a deep
373
+ # copy of the job Hash and returning the args to show in logs, the
374
+ # dashboard and the fallback error log. nil (default) shows args as-is.
375
+ def redact_args=(hook)
376
+ raise ArgumentError, 'redact_args must be nil or respond to #call' unless hook.nil? || hook.respond_to?(:call)
377
+
378
+ @options[:redact_args] = hook
379
+ end
380
+
381
+ def redact_args
382
+ @options[:redact_args]
383
+ end
384
+
385
+ # Seconds a swarm child may go without a heartbeat before the parent
386
+ # replaces it (Swarm::Liveness). false or 0 turns the check off. The floor
387
+ # sits above the heartbeat's own cadence so a child that is merely busy is
388
+ # never read as hung.
389
+ SWARM_HEARTBEAT_TIMEOUT_FLOOR = 20
390
+
391
+ def swarm_heartbeat_timeout=(seconds)
392
+ unless seconds == false || seconds == 0 || # rubocop:disable Style/NumericPredicate
393
+ ((seconds.is_a?(Integer) || seconds.is_a?(Float)) && seconds >= SWARM_HEARTBEAT_TIMEOUT_FLOOR)
394
+ raise ArgumentError,
395
+ "swarm_heartbeat_timeout must be false, 0, or a number >= #{SWARM_HEARTBEAT_TIMEOUT_FLOOR} " \
396
+ "(got #{seconds.inspect})"
397
+ end
398
+
399
+ @options[:swarm_heartbeat_timeout] = seconds
400
+ end
401
+
402
+ def swarm_heartbeat_timeout
403
+ @options[:swarm_heartbeat_timeout]
404
+ end
405
+
363
406
  # --- Reliability (Sidekiq Pro drop-in toggles) -----------------------
364
407
 
365
408
  # Sidekiq Pro's opt-in toggle for reliable fetch. Already the default in
@@ -664,7 +707,11 @@ module Wurk
664
707
  #
665
708
  # Off by default — call this in a `configure_server` block to enable.
666
709
  # Spec: docs/target/sidekiq-ent.md §7.1.2.
667
- def health_check(port:, bind: '0.0.0.0', ready_window: 30)
710
+ #
711
+ # `metrics:` serves the Prometheus scrape on the same listener (on by
712
+ # default). `min_ready:` — children with a fresh heartbeat required for
713
+ # /ready inside a swarm; nil means half the swarm, rounded up.
714
+ def health_check(port:, bind: '0.0.0.0', ready_window: 30, metrics: true, min_ready: nil)
668
715
  guard_frozen!
669
716
  p = Integer(port)
670
717
  rw = Integer(ready_window)
@@ -674,7 +721,9 @@ module Wurk
674
721
  b = bind.to_s
675
722
  raise ArgumentError, 'bind must be a non-empty string' if b.empty?
676
723
 
677
- @options[:health_check_options] = { port: p, bind: b, ready_window: rw }
724
+ @options[:health_check_options] = { port: p, bind: b, ready_window: rw,
725
+ metrics: health_check_metrics(metrics),
726
+ min_ready: health_check_min_ready(min_ready) }
678
727
  end
679
728
 
680
729
  # --- Lifecycle hooks --------------------------------------------------
@@ -698,7 +747,7 @@ module Wurk
698
747
 
699
748
  def handle_exception(ex, ctx = {})
700
749
  if error_handlers.empty?
701
- logger.error("#{ctx} #{ex.class}: #{ex.message}")
750
+ logger.error("#{redacted_context(ctx)} #{ex.class}: #{ex.message}")
702
751
  else
703
752
  error_handlers.each do |handler|
704
753
  handler.call(ex, ctx, self)
@@ -708,14 +757,31 @@ module Wurk
708
757
  end
709
758
  end
710
759
 
760
+ # The error context with the job's args passed through the `redact_args`
761
+ # hook, for the two reporters that print or tag the whole context: the
762
+ # fallback log line and the default ERROR_HANDLER (whose Wurk::Context a
763
+ # JSON log formatter emits). Host-registered error handlers still receive
764
+ # the real ctx — they are the host's own code.
765
+ def redacted_context(ctx)
766
+ hook = @options[:redact_args]
767
+ job = ctx.is_a?(Hash) ? ctx[:job] : nil
768
+ return ctx unless hook && job.is_a?(Hash) && job.key?('args')
769
+
770
+ require_relative 'redact'
771
+ ctx.merge(job: job.merge('args' => Wurk::Redact.args(job, hook)))
772
+ end
773
+
711
774
  # --- Configure blocks (Sidekiq.configure_server / _client) -----------
712
775
 
713
776
  def configure_server(&block)
714
777
  yield self if block && server?
715
778
  end
716
779
 
780
+ # A web process that also boots the swarm (Wurk::RailsBoot) is flagged
781
+ # server, yet every enqueue it makes from a request is client work, so
782
+ # `client_in_server` lets its configure_client blocks run too.
717
783
  def configure_client(&block)
718
- yield self if block && !server?
784
+ yield self if block && (!server? || @options[:client_in_server])
719
785
  end
720
786
 
721
787
  def server?
@@ -788,6 +854,7 @@ module Wurk
788
854
  prepare_for_fork!
789
855
  @capsules.each_value(&:freeze)
790
856
  @capsules.freeze
857
+ warn_on_small_db_pool
791
858
  self
792
859
  end
793
860
 
@@ -801,6 +868,11 @@ module Wurk
801
868
 
802
869
  private
803
870
 
871
+ def validate_constructor_options!
872
+ @options[:global_concurrency] = normalize_global_concurrency(@options[:global_concurrency])
873
+ RedisPool.validate_idle_timeout(@options[:redis_idle_timeout])
874
+ end
875
+
804
876
  # One flat fork running the default capsule's queues + concurrency. The
805
877
  # railtie boots this when a Rails host mounts the engine without declaring
806
878
  # a topology. queue_specs (not queues) so weights survive the round-trip.
@@ -810,20 +882,78 @@ module Wurk
810
882
  end
811
883
 
812
884
  # Number of swarm children when the host hasn't declared a topology
813
- # (Sidekiq Ent §7.2). Defaults to the CPU count. A whole number is an
814
- # absolute count; a fractional value is a CPU multiplier (e.g. `0.5` → half
815
- # the cores, rounded), supported since Sidekiq 8.0.2. WURK_COUNT is the
816
- # native name, SIDEKIQ_COUNT the drop-in alias. Unparseable input falls back
817
- # to the CPU count; the result is floored at 1 so the swarm always forks.
885
+ # (Sidekiq Ent §7.2). Defaults to the usable CPU count — the cgroup quota
886
+ # when one is tighter than the host's cores (Wurk::CpuCount). A whole number
887
+ # is an absolute count; a fractional value is a CPU multiplier (e.g. `0.5`
888
+ # → half the usable CPUs, rounded), supported since Sidekiq 8.0.2.
889
+ # WURK_COUNT is the native name, SIDEKIQ_COUNT the drop-in alias.
890
+ # Unparseable input falls back to the CPU count; the result is floored at 1
891
+ # so the swarm always forks. Logged once, with where the number came from:
892
+ # a pod that forks the wrong number of children is otherwise only visible
893
+ # as throttling or memory pressure.
818
894
  def default_child_count
819
- raw = ENV['WURK_COUNT'] || ENV.fetch('SIDEKIQ_COUNT', nil)
820
- return Etc.nprocessors if raw.nil? || raw.strip.empty?
895
+ count, source = child_count_and_source
896
+ logger.info { "Wurk swarm: forking #{count} #{count == 1 ? 'child' : 'children'} (#{source})" }
897
+ count
898
+ end
899
+
900
+ def child_count_and_source
901
+ name = ENV['WURK_COUNT'] ? 'WURK_COUNT' : 'SIDEKIQ_COUNT'
902
+ raw = ENV.fetch(name, nil)
903
+ cpus = Wurk::CpuCount.detect
904
+ return [cpus.count, cpus.source] if raw.nil? || raw.strip.empty?
905
+
906
+ env_child_count(name, raw, cpus)
907
+ end
821
908
 
909
+ def env_child_count(name, raw, cpus)
822
910
  value = Float(raw)
823
- count = (value % 1).zero? ? value.to_i : (value * Etc.nprocessors).round
824
- [count, 1].max
911
+ return [[value.to_i, 1].max, "#{name}=#{raw}"] if (value % 1).zero?
912
+
913
+ [[(value * cpus.count).round, 1].max, "#{name}=#{raw} x #{cpus.count} CPUs from #{cpus.source}"]
825
914
  rescue ArgumentError, TypeError
826
- Etc.nprocessors
915
+ [cpus.count, "#{cpus.source}; #{name}=#{raw.inspect} is not a number"]
916
+ end
917
+
918
+ def health_check_metrics(value)
919
+ raise ArgumentError, "metrics must be true or false (got #{value.inspect})" unless [true, false].include?(value)
920
+
921
+ value
922
+ end
923
+
924
+ def health_check_min_ready(value)
925
+ return nil if value.nil?
926
+ unless value.is_a?(Integer) && value.positive?
927
+ raise ArgumentError, "min_ready must be nil or an Integer >= 1 (got #{value.inspect})"
928
+ end
929
+
930
+ value
931
+ end
932
+
933
+ # Runs where every worker process settles its capsules — a swarm child
934
+ # after its slot is applied, `wurk`, embedded — so the concurrency compared
935
+ # is the one that process will actually run. Every job thread that touches
936
+ # the database checks out its own connection; a pool smaller than the
937
+ # thread count turns into ActiveRecord::ConnectionTimeoutError under load,
938
+ # long after boot. A warning rather than a raise: a job set that never
939
+ # touches the database is a legitimate reason to run a small pool.
940
+ def warn_on_small_db_pool(record_class = (::ActiveRecord::Base if defined?(::ActiveRecord::Base)))
941
+ return unless record_class
942
+
943
+ size = record_class.connection_pool.size
944
+ threads = total_concurrency
945
+ return if size >= threads
946
+
947
+ logger.warn do
948
+ "ActiveRecord connection pool size (#{size}) is smaller than this process's job concurrency " \
949
+ "(#{threads}); jobs that use the database will wait for connections and may raise " \
950
+ 'ActiveRecord::ConnectionTimeoutError. Raise `pool:` in config/database.yml to at least ' \
951
+ "#{threads} (plus any other threads that use the database)."
952
+ end
953
+ rescue StandardError
954
+ # No database configured, or an adapter that cannot report its pool:
955
+ # nothing to compare.
956
+ nil
827
957
  end
828
958
 
829
959
  # SIDEKIQ_MAXMEM_MB is the drop-in name; WURK_MAXMEM_MB the native alias.
@@ -897,15 +1027,16 @@ module Wurk
897
1027
  logger
898
1028
  end
899
1029
 
900
- # `size`/`name` are pool-structural (the caller owns them); every other
901
- # key the host set via `config.redis = {...}` — url, the split timeouts,
902
- # reconnect_attempts, driver, … — flows through to RedisPool verbatim.
903
- # `overrides` win over the host config (the web pool pins its own
904
- # pool_timeout this way). Every pool built here is wired to the redis-error
905
- # telemetry dispatcher.
1030
+ # `size` and the pool label are structural (the caller owns them); every
1031
+ # other key the host set via `config.redis = {...}` — url, the split
1032
+ # timeouts, reconnect_attempts, driver, a Sentinel `name:` … — flows through
1033
+ # to RedisPool (see RedisOptions.pool_kwargs). `overrides` win over the
1034
+ # host config (the web pool pins its own pool_timeout this way). Every pool
1035
+ # built here is wired to the redis-error telemetry dispatcher.
906
1036
  def build_redis_pool(size:, name:, **overrides)
907
1037
  RedisPool.new(size: size, name: name, on_error: method(:dispatch_redis_error),
908
- **@redis_config.except(:size, :name), **overrides)
1038
+ redis_idle_timeout: @options[:redis_idle_timeout],
1039
+ **RedisOptions.pool_kwargs(@redis_config), **overrides)
909
1040
  end
910
1041
 
911
1042
  # Fan a RedisPool retry/give-up event out to the registered handlers. A
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'etc'
4
+
5
+ module Wurk
6
+ # How many CPUs this process may actually use — the default swarm size.
7
+ #
8
+ # `Etc.nprocessors` reports the host's cores (or the affinity mask), not the
9
+ # container's share of them: a pod limited to 2 CPUs on a 64-core node reads
10
+ # 64, and a swarm that forks one child per core then runs 64 processes on 2
11
+ # CPUs' worth of CFS quota — throttled, with 32x the memory. The quota lives
12
+ # in the cgroup, so it is read from there and the smaller of the two wins.
13
+ #
14
+ # cgroup v2: `cpu.max` is `<quota> <period>` or `max <period>`, read in this
15
+ # process's own cgroup and every ancestor up to the mount root (a limit set on
16
+ # a parent slice binds the child too), smallest wins. cgroup v1:
17
+ # `cpu.cfs_quota_us` (-1 = unlimited) over `cpu.cfs_period_us`, in the cpu
18
+ # controller's mount. Either way `ceil(quota / period)`, floored at 1: a
19
+ # 1.5-CPU quota can keep two processes busy part of the time, and rounding a
20
+ # 0.5-CPU quota down to zero would fork nothing.
21
+ module CpuCount
22
+ Result = Data.define(:count, :source)
23
+
24
+ ROOT = '/sys/fs/cgroup'
25
+ PROC_SELF_CGROUP = '/proc/self/cgroup'
26
+ V1_CONTROLLER_DIRS = %w[cpu cpu,cpuacct cpuacct,cpu].freeze
27
+
28
+ module_function
29
+
30
+ def detect(root: ROOT, proc_cgroup: PROC_SELF_CGROUP, nprocessors: Etc.nprocessors)
31
+ quota = cgroup_v2_quota(root, proc_cgroup)
32
+ source = 'cgroup v2 cpu.max'
33
+ unless quota
34
+ quota = cgroup_v1_quota(root)
35
+ source = 'cgroup v1 cpu.cfs_quota_us'
36
+ end
37
+ return Result.new(nprocessors, 'Etc.nprocessors') if quota.nil? || quota >= nprocessors
38
+
39
+ Result.new(quota, source)
40
+ end
41
+
42
+ def cgroup_v2_quota(root, proc_cgroup)
43
+ v2_dirs(root, proc_cgroup).filter_map { |dir| v2_limit(File.join(dir, 'cpu.max')) }.min
44
+ end
45
+
46
+ # `0::<path>` is the v2 entry. Inside a container with its own cgroup
47
+ # namespace the path is `/` and the limit sits at the mount root; on a host
48
+ # (or without the namespace) it sits under the full path.
49
+ def v2_dirs(root, proc_cgroup)
50
+ line = read(proc_cgroup)&.lines&.find { |l| l.start_with?('0::') }
51
+ path = line ? line.strip.delete_prefix('0::') : '/'
52
+ dirs = []
53
+ loop do
54
+ dirs << File.join(root, path)
55
+ break if path == '/' || path.empty?
56
+
57
+ path = File.dirname(path)
58
+ end
59
+ dirs.uniq
60
+ end
61
+
62
+ def v2_limit(file)
63
+ quota, period = read(file)&.split
64
+ return nil if quota.nil? || quota == 'max'
65
+
66
+ ceil_ratio(quota, period)
67
+ end
68
+
69
+ def cgroup_v1_quota(root)
70
+ V1_CONTROLLER_DIRS.each do |dir|
71
+ quota = read(File.join(root, dir, 'cpu.cfs_quota_us'))
72
+ next unless quota
73
+
74
+ return nil if quota.strip.start_with?('-')
75
+
76
+ return ceil_ratio(quota, read(File.join(root, dir, 'cpu.cfs_period_us')))
77
+ end
78
+ nil
79
+ end
80
+
81
+ def ceil_ratio(quota, period)
82
+ quota = Integer(quota.to_s.strip, 10)
83
+ period = Integer(period.to_s.strip, 10)
84
+ return nil unless quota.positive? && period.positive?
85
+
86
+ (quota + period - 1) / period
87
+ rescue ArgumentError
88
+ nil
89
+ end
90
+
91
+ def read(path)
92
+ File.read(path)
93
+ rescue SystemCallError, IOError
94
+ nil
95
+ end
96
+ end
97
+ end
data/lib/wurk/cron.rb CHANGED
@@ -271,7 +271,11 @@ module Wurk
271
271
  class Loop
272
272
  attr_reader :lid, :schedule, :klass, :options, :tz
273
273
 
274
- def initialize(schedule:, klass:, options: {}, tz: nil, lid: nil)
274
+ # `paused:` is the loop HASH's `paused` field when read back from Redis —
275
+ # the runtime owns it (Web UI pause/unpause, ent §2.4). A code-declared
276
+ # `paused:` option only seeds that field at first registration, so it
277
+ # must not keep the loop paused once the field says otherwise.
278
+ def initialize(schedule:, klass:, options: {}, tz: nil, lid: nil, paused: nil)
275
279
  raise ArgumentError, 'klass must be a String' unless klass.is_a?(String) && !klass.empty?
276
280
 
277
281
  @schedule = schedule
@@ -280,6 +284,7 @@ module Wurk
280
284
  @tz = tz
281
285
  @parser = Parser.new(schedule)
282
286
  @lid = lid || Cron.lid(schedule, klass, @options)
287
+ @paused = paused.nil? ? [true, '1', 1].include?(@options['paused']) : paused
283
288
  end
284
289
 
285
290
  def parser
@@ -287,19 +292,51 @@ module Wurk
287
292
  end
288
293
 
289
294
  def paused?
290
- @options['paused'].to_s == '1' || @options['paused'] == true
295
+ @paused
291
296
  end
292
297
 
298
+ # The queue a tick lands in: the loop's own `queue:`, else the worker's
299
+ # `sidekiq_options` queue, as `perform_async` would pick (ent §2.2).
293
300
  def queue
294
- @options['queue'] || 'default'
301
+ @options['queue'] || worker_option('queue') || 'default'
295
302
  end
296
303
 
304
+ # `args:` is the static argument list splatted into perform. A bare value
305
+ # (a Hash most often) is one argument — `Array({ 'a' => 1 })` would turn
306
+ # it into `[['a', 1]]`.
297
307
  def args
298
- Array(@options['args'])
308
+ value = @options['args']
309
+ return [] if value.nil?
310
+
311
+ value.is_a?(::Array) ? value : [value]
299
312
  end
300
313
 
314
+ # The loop's `retry:`, else the worker's, else Sidekiq's default `true`.
301
315
  def retry_value
302
- @options.fetch('retry', true)
316
+ return @options['retry'] if @options.key?('retry')
317
+
318
+ worker_retry = worker_option('retry')
319
+ worker_retry.nil? || worker_retry
320
+ end
321
+
322
+ # The payload one tick pushes. Ent §2.2: a tick is `klass.perform_async(*args)`
323
+ # with the loop's `retry` / `queue` laid over the worker's own options, so
324
+ # the resolved class goes to the client — JobUtil#normalize_item merges its
325
+ # `get_sidekiq_options` (unique_for, encrypt, expires_in, …) only for a
326
+ # Class. Only the overrides the loop actually declares are set.
327
+ def job_item
328
+ worker = worker_class
329
+ item = { 'class' => worker.respond_to?(:get_sidekiq_options) ? worker : @klass, 'args' => args }
330
+ item['queue'] = @options['queue'] if @options.key?('queue')
331
+ item['retry'] = @options['retry'] if @options.key?('retry')
332
+ item
333
+ end
334
+
335
+ # The worker constant, or nil when it is not loadable in this process.
336
+ def worker_class
337
+ @klass.split('::').inject(::Object) { |mod, name| mod.const_get(name) }
338
+ rescue ::NameError
339
+ nil
303
340
  end
304
341
 
305
342
  def history
@@ -379,13 +416,18 @@ module Wurk
379
416
  def self.from_redis(lid, hash)
380
417
  h = hash.is_a?(Array) ? hash.each_slice(2).to_h : hash
381
418
  opts = h['options'] ? JSON.parse(h['options']) : {}
382
- opts['paused'] = '1' if h['paused'] == '1'
383
419
  tz = h['tz'].to_s.empty? ? nil : h['tz']
384
- new(lid: lid, schedule: h['schedule'], klass: h['klass'], options: opts, tz: tz)
420
+ new(lid: lid, schedule: h['schedule'], klass: h['klass'], options: opts, tz: tz,
421
+ paused: h['paused'] == '1')
385
422
  end
386
423
 
387
424
  private
388
425
 
426
+ def worker_option(key)
427
+ worker = worker_class
428
+ worker.get_sidekiq_options[key] if worker.respond_to?(:get_sidekiq_options)
429
+ end
430
+
389
431
  def stringify_options(opts)
390
432
  opts.transform_keys(&:to_s)
391
433
  end
@@ -448,18 +490,23 @@ module Wurk
448
490
  @config = config
449
491
  end
450
492
 
493
+ # Loops are read up front and yielded with no connection held, so the
494
+ # caller's own Redis work (the poller's claim + push) never nests a second
495
+ # checkout. A loop whose stored hash no longer parses is skipped with a
496
+ # warning: one corrupt record must not hide every loop after it.
451
497
  def each
452
498
  return enum_for(:each) unless block_given?
453
499
 
454
- redis do |c|
455
- lids = c.call('SMEMBERS', PERIODIC_KEY)
456
- lids.each do |lid|
500
+ rows = redis do |c|
501
+ c.call('SMEMBERS', PERIODIC_KEY).filter_map do |lid|
457
502
  h = c.call('HGETALL', "#{LOOP_PREFIX}#{lid}")
458
- next if h.nil? || h.empty?
459
-
460
- yield Loop.from_redis(lid, h)
503
+ [lid, h] unless h.nil? || h.empty?
461
504
  end
462
505
  end
506
+ rows.each do |lid, h|
507
+ loop_obj = parse(lid, h)
508
+ yield loop_obj if loop_obj
509
+ end
463
510
  end
464
511
 
465
512
  def size
@@ -478,6 +525,13 @@ module Wurk
478
525
 
479
526
  private
480
527
 
528
+ def parse(lid, hash)
529
+ Loop.from_redis(lid, hash)
530
+ rescue JSON::ParserError, ArgumentError => e
531
+ Wurk.logger&.warn("[cron] skipping unreadable loop lid=#{lid}: #{e.class}: #{e.message}")
532
+ nil
533
+ end
534
+
481
535
  def redis(idempotent: false, &)
482
536
  @config ? @config.redis(idempotent:, &) : Wurk.redis(idempotent:, &)
483
537
  end
@@ -513,9 +567,9 @@ module Wurk
513
567
  end
514
568
 
515
569
  # Once-per-minute tick driver. Only the cluster leader iterates the LoopSet
516
- # and enqueues; non-leaders return early. Missed-tick warning when
517
- # wall-clock has drifted more than `MISSED_TICK_THRESHOLD` seconds past the
518
- # expected fire.
570
+ # and enqueues; non-leaders return early. A slot that wall-clock has passed
571
+ # by more than `MISSED_TICK_THRESHOLD` seconds is skipped with a warning,
572
+ # never backfilled.
519
573
  class Poller
520
574
  include Component
521
575
 
@@ -556,10 +610,18 @@ module Wurk
556
610
  # `dear-leader`): non-leaders return early and never iterate the LoopSet.
557
611
  # The Launcher owns the lock's renewal — the poller no longer runs (or
558
612
  # expires) its own.
613
+ #
614
+ # Rescued per loop: the LoopSet order is stable, so one loop that raises
615
+ # on every tick (a push its middleware refuses, a class that no longer
616
+ # loads) would otherwise starve every loop after it, forever.
559
617
  def tick
560
618
  return unless leader?
561
619
 
562
- LoopSet.new(@config).each { |lp| enqueue_if_due(lp) }
620
+ LoopSet.new(@config).each do |lp|
621
+ enqueue_if_due(lp)
622
+ rescue StandardError => e
623
+ handle_exception(e, { context: 'cron-poller', lid: lp.lid, klass: lp.klass })
624
+ end
563
625
  rescue StandardError => e
564
626
  handle_exception(e, { context: 'cron-poller' })
565
627
  end
@@ -568,6 +630,10 @@ module Wurk
568
630
  # cached read, so a second poller can reach the same due loop for a few
569
631
  # seconds after a handover. Losing the CAS means another tick owns this
570
632
  # slot — enqueue nothing.
633
+ #
634
+ # A slot older than the missed-tick threshold is not backfilled (ent §2.6:
635
+ # no backfill on restart, missed ticks across a leader gap are dropped):
636
+ # the marks still advance past it, but nothing is pushed.
571
637
  def enqueue_if_due(loop_obj)
572
638
  return if loop_obj.paused?
573
639
 
@@ -577,8 +643,8 @@ module Wurk
577
643
 
578
644
  future = loop_obj.next_fire_after(slot, now)
579
645
  return unless claim_fire?(loop_obj, mark, now, future)
646
+ return skip_missed(loop_obj, slot, now) if now - slot > missed_threshold
580
647
 
581
- warn_missed_tick(loop_obj, slot, now)
582
648
  jid = enqueue_claimed!(loop_obj)
583
649
  record_history(loop_obj, jid, now)
584
650
  jid
@@ -609,13 +675,18 @@ module Wurk
609
675
  [next_fire, mark || next_fire.to_s]
610
676
  end
611
677
 
612
- def warn_missed_tick(loop_obj, expected, now)
613
- return if now - expected <= MISSED_TICK_THRESHOLD
678
+ # Never tighter than a tick and a half: a raised tick interval must not
679
+ # turn every on-time fire into a "missed" one.
680
+ def missed_threshold
681
+ [MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
682
+ end
614
683
 
684
+ def skip_missed(loop_obj, expected, now)
615
685
  logger.warn(
616
686
  "[cron] missed tick lid=#{loop_obj.lid} klass=#{loop_obj.klass} " \
617
- "expected_at=#{expected} fired_at=#{now} drift=#{now - expected}s"
687
+ "expected_at=#{expected} now=#{now} drift=#{now - expected}s — skipped, not backfilled"
618
688
  )
689
+ nil
619
690
  end
620
691
 
621
692
  # Compare-and-swap the fire marks: only the tick still looking at the
@@ -647,30 +718,22 @@ module Wurk
647
718
  end
648
719
 
649
720
  def enqueue!(loop_obj)
650
- aj = active_job_class(loop_obj.klass)
721
+ aj = active_job_class(loop_obj.worker_class)
651
722
  return enqueue_active_job(aj, loop_obj) if aj
652
723
 
653
- @client.push(
654
- 'class' => loop_obj.klass,
655
- 'args' => loop_obj.args,
656
- 'queue' => loop_obj.queue,
657
- 'retry' => loop_obj.retry_value
658
- )
724
+ @client.push(loop_obj.job_item)
659
725
  end
660
726
 
661
- # Resolve a loop's class name to an ActiveJob::Base subclass, or nil when
727
+ # The loop's class when it is an ActiveJob::Base subclass, or nil when
662
728
  # ActiveJob isn't loaded (standalone wurk) or the class isn't one.
663
729
  # sidekiq-cron parity: a cron loop targeting an ActiveJob must enqueue
664
730
  # through the AJ adapter so the job runs via Sidekiq::ActiveJob::Wrapper
665
731
  # with full callbacks/serialization — a bare `client.push('class' => …)`
666
732
  # would make the processor call `Klass.new.perform`, skipping all of AJ.
667
- def active_job_class(name)
733
+ def active_job_class(const)
668
734
  return nil unless defined?(::ActiveJob::Base)
669
735
 
670
- const = name.split('::').inject(::Object) { |mod, c| mod.const_get(c) }
671
736
  const if const.is_a?(::Class) && const < ::ActiveJob::Base
672
- rescue ::NameError
673
- nil
674
737
  end
675
738
 
676
739
  # Enqueue via the AJ adapter (→ wurk). Only override the queue when the
@@ -679,7 +742,7 @@ module Wurk
679
742
  # doesn't apply here. Returns the wurk jid (provider_job_id) so the fire
680
743
  # history records it; nil if a before_enqueue callback halted the push.
681
744
  def enqueue_active_job(klass, loop_obj)
682
- target = loop_obj.queue == 'default' ? klass : klass.set(queue: loop_obj.queue)
745
+ target = loop_obj.options.key?('queue') ? klass.set(queue: loop_obj.options['queue']) : klass
683
746
  job = target.perform_later(*loop_obj.args)
684
747
  job.provider_job_id if job.respond_to?(:provider_job_id)
685
748
  end