wurk 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +9 -9
  3. data/app/controllers/wurk/api_controller.rb +4 -1
  4. data/app/controllers/wurk/dashboard_controller.rb +11 -6
  5. data/exe/wurk +7 -2
  6. data/exe/wurkswarm +1 -2
  7. data/lib/generators/wurk/install/install_generator.rb +12 -0
  8. data/lib/sidekiq/web.rb +5 -0
  9. data/lib/wurk/boot_checks.rb +93 -0
  10. data/lib/wurk/capsule.rb +33 -8
  11. data/lib/wurk/cli.rb +204 -30
  12. data/lib/wurk/client/buffered.rb +240 -26
  13. data/lib/wurk/client.rb +16 -5
  14. data/lib/wurk/compat.rb +15 -0
  15. data/lib/wurk/configuration.rb +114 -16
  16. data/lib/wurk/cron.rb +141 -16
  17. data/lib/wurk/debounce.rb +2 -2
  18. data/lib/wurk/embedded.rb +8 -2
  19. data/lib/wurk/encryption.rb +20 -5
  20. data/lib/wurk/error_coalescer.rb +35 -0
  21. data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
  22. data/lib/wurk/fetcher/capped.rb +4 -0
  23. data/lib/wurk/fetcher/many_queues.rb +17 -1
  24. data/lib/wurk/fetcher/oom_drain.rb +102 -0
  25. data/lib/wurk/fetcher/orphan_grace.rb +60 -44
  26. data/lib/wurk/fetcher/private_list_key.rb +1 -16
  27. data/lib/wurk/fetcher/reaper.rb +169 -127
  28. data/lib/wurk/fetcher/reliable.rb +26 -4
  29. data/lib/wurk/fetcher/self_reconcile.rb +175 -0
  30. data/lib/wurk/fetcher/unit_of_work.rb +32 -8
  31. data/lib/wurk/health.rb +23 -4
  32. data/lib/wurk/health_probe.rb +64 -0
  33. data/lib/wurk/heartbeat.rb +55 -3
  34. data/lib/wurk/job/options.rb +9 -1
  35. data/lib/wurk/job.rb +2 -24
  36. data/lib/wurk/job_record.rb +17 -9
  37. data/lib/wurk/job_retry.rb +38 -10
  38. data/lib/wurk/job_set.rb +13 -5
  39. data/lib/wurk/launcher.rb +134 -52
  40. data/lib/wurk/leader.rb +71 -11
  41. data/lib/wurk/limiter/base.rb +17 -1
  42. data/lib/wurk/lua/debounce.lua +11 -6
  43. data/lib/wurk/lua/loader.rb +13 -1
  44. data/lib/wurk/lua.rb +71 -8
  45. data/lib/wurk/manager.rb +1 -1
  46. data/lib/wurk/metrics/process_counters.rb +27 -0
  47. data/lib/wurk/metrics/prometheus.rb +35 -14
  48. data/lib/wurk/middleware/current_attributes.rb +26 -10
  49. data/lib/wurk/middleware/poison_pill.rb +1 -2
  50. data/lib/wurk/preflight/checks.rb +203 -0
  51. data/lib/wurk/preflight/command.rb +135 -0
  52. data/lib/wurk/preflight/keyspace.rb +109 -0
  53. data/lib/wurk/preflight/owners.rb +74 -0
  54. data/lib/wurk/preflight/payload_sample.rb +108 -0
  55. data/lib/wurk/preflight/reader.rb +60 -0
  56. data/lib/wurk/preflight/report.rb +82 -0
  57. data/lib/wurk/preflight/verify.rb +161 -0
  58. data/lib/wurk/preflight.rb +40 -0
  59. data/lib/wurk/process_set.rb +3 -1
  60. data/lib/wurk/processor.rb +82 -18
  61. data/lib/wurk/queue_consumers.rb +134 -0
  62. data/lib/wurk/rails_boot.rb +40 -2
  63. data/lib/wurk/rails_reloader.rb +51 -0
  64. data/lib/wurk/railtie.rb +20 -1
  65. data/lib/wurk/redact.rb +12 -0
  66. data/lib/wurk/redis_pool.rb +34 -9
  67. data/lib/wurk/scheduled.rb +142 -67
  68. data/lib/wurk/sentry/death_handler.rb +29 -0
  69. data/lib/wurk/sentry/error_handler.rb +52 -10
  70. data/lib/wurk/sentry/middleware.rb +8 -1
  71. data/lib/wurk/sentry/retry_policy.rb +12 -3
  72. data/lib/wurk/sentry.rb +42 -5
  73. data/lib/wurk/sorted_entry.rb +38 -13
  74. data/lib/wurk/swarm/child_boot.rb +44 -31
  75. data/lib/wurk/swarm/crash_watch.rb +65 -0
  76. data/lib/wurk/swarm/test_mode_guard.rb +32 -0
  77. data/lib/wurk/swarm.rb +79 -11
  78. data/lib/wurk/testing.rb +74 -0
  79. data/lib/wurk/version.rb +1 -1
  80. data/lib/wurk/web/config.rb +8 -4
  81. data/lib/wurk/web/extension.rb +41 -62
  82. data/lib/wurk/web/helpers.rb +133 -0
  83. data/lib/wurk/web/paginator.rb +49 -0
  84. data/lib/wurk/web/rack_app.rb +30 -6
  85. data/lib/wurk/work_set.rb +6 -0
  86. data/lib/wurk/worker/setter.rb +11 -6
  87. data/lib/wurk/worker.rb +17 -1
  88. data/lib/wurk.rb +1 -1
  89. data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
  90. data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
  91. data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
  92. data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
  93. data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
  94. data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
  95. data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
  96. data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
  97. data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
  98. data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
  99. data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
  100. data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
  101. data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
  102. data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
  103. data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
  104. data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
  105. data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
  106. data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
  107. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
  108. data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
  109. data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
  110. data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
  111. data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
  112. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
  113. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
  114. data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
  115. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
  116. data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
  117. data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
  118. data/vendor/assets/dashboard/index.html +3 -3
  119. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  120. metadata +53 -30
  121. data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
  122. data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
  123. data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
@@ -9,6 +9,8 @@ require_relative 'topology'
9
9
  require_relative 'cpu_count'
10
10
  require_relative 'redis_options'
11
11
  require_relative 'keys'
12
+ require_relative 'error_coalescer'
13
+ require_relative 'metrics/process_counters'
12
14
 
13
15
  module Wurk
14
16
  # Owns runtime knobs (concurrency, queues, timeouts, lifecycle events,
@@ -98,7 +100,11 @@ module Wurk
98
100
  # the fetch path reads this once at boot, so an app that caps nothing
99
101
  # resolves to "no capped queues" before the first fetch and never asks
100
102
  # again.
101
- global_concurrency: {}
103
+ global_concurrency: {},
104
+ # See #swarm_crash_exit_after=. Long enough to outlast a slow rollout or a
105
+ # dependency coming back, short enough that a bad release does not sit
106
+ # looking Running for an afternoon.
107
+ swarm_crash_exit_after: 600
102
108
  }.freeze
103
109
 
104
110
  # :fork fires only inside swarm children, after fork + internal AR/Redis
@@ -151,7 +157,7 @@ module Wurk
151
157
  Wurk::Metrics::Statsd.reset!
152
158
  end
153
159
 
154
- def initialize(options = {})
160
+ def initialize(options = {}) # rubocop:disable Metrics/AbcSize
155
161
  @options = deep_dup_defaults.merge(options)
156
162
  # Through the same door `global_concurrency=` uses, so a cap passed to
157
163
  # `Configuration.new` is validated rather than trusted, and the default
@@ -162,14 +168,16 @@ module Wurk
162
168
  @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
163
169
  @capsules = {}
164
170
  @directory = {}
165
- @client_chain = Middleware::Chain.new
166
- @server_chain = Middleware::Chain.new(on_registration: method(:warn_on_late_server_registration))
171
+ @client_chain = Middleware::Chain.new(self)
172
+ @server_chain = Middleware::Chain.new(self, on_registration: method(:warn_on_late_server_registration))
167
173
  @redis_config = { url: ENV.fetch('REDIS_URL', 'redis://localhost:6379/0') }
168
174
  @web_redis_pool = nil
169
175
  @logger = nil
170
176
  @thread_priority = DEFAULT_THREAD_PRIORITY
171
177
  @telemetry_installed = false
172
178
  @frozen = false
179
+ @deferred_server_blocks = []
180
+ @fetch_error_coalescer = ErrorCoalescer.new
173
181
  end
174
182
 
175
183
  # --- Hash-like options access -----------------------------------------
@@ -192,6 +200,7 @@ module Wurk
192
200
  end
193
201
 
194
202
  def dig(*keys) = @options.dig(*keys)
203
+ def to_json(*) = Wurk.dump_json(@options)
195
204
 
196
205
  # --- Default capsule shortcuts ---------------------------------------
197
206
 
@@ -250,6 +259,7 @@ module Wurk
250
259
  def redis=(hash)
251
260
  guard_frozen!
252
261
  RedisOptions.validate!(hash)
262
+ warn_on_late_redis_assignment
253
263
  @redis_config = @redis_config.merge(hash.transform_keys(&:to_sym))
254
264
  end
255
265
 
@@ -263,6 +273,11 @@ module Wurk
263
273
  default_capsule.redis_pool
264
274
  end
265
275
 
276
+ # Parsed INFO, without the pool's own counters RedisPool#info merges in.
277
+ def redis_info
278
+ redis_pool.info.except('size', 'available')
279
+ end
280
+
266
281
  # Disconnect and drop every capsule's cached pools (main + fetch) plus the
267
282
  # web pool. Used by Wurk::Swarm so the parent never leaks sockets into forks
268
283
  # and each child can build fresh ones.
@@ -272,6 +287,12 @@ module Wurk
272
287
  @web_redis_pool = nil
273
288
  end
274
289
 
290
+ # Non-terminal counterpart, for a fork in a parent whose threads are live.
291
+ def reload_redis_pools!
292
+ @capsules.each_value(&:reload_redis_pools!)
293
+ @web_redis_pool&.reload!
294
+ end
295
+
275
296
  def new_redis_pool(size, name = 'custom')
276
297
  build_redis_pool(size: size, name: name)
277
298
  end
@@ -403,6 +424,20 @@ module Wurk
403
424
  @options[:swarm_heartbeat_timeout]
404
425
  end
405
426
 
427
+ # Seconds every swarm slot may spend crash-looping before the standalone
428
+ # runner reports it and exits non-zero. nil or false keeps respawning forever.
429
+ def swarm_crash_exit_after=(seconds)
430
+ unless seconds.nil? || seconds == false || (seconds.is_a?(Numeric) && seconds.positive?)
431
+ raise ArgumentError, "swarm_crash_exit_after must be nil, false, or a positive number (got #{seconds.inspect})"
432
+ end
433
+
434
+ @options[:swarm_crash_exit_after] = seconds || nil
435
+ end
436
+
437
+ def swarm_crash_exit_after
438
+ @options[:swarm_crash_exit_after]
439
+ end
440
+
406
441
  # --- Reliability (Sidekiq Pro drop-in toggles) -----------------------
407
442
 
408
443
  # Sidekiq Pro's opt-in toggle for reliable fetch. Already the default in
@@ -411,9 +446,8 @@ module Wurk
411
446
  # callback. It exists so a Pro initializer drops in unchanged instead of
412
447
  # raising NoMethodError.
413
448
  #
414
- # NOTE: `reliable_scheduler!` below is NOT a no-op. The default
415
- # `scheduled_enq` pops then pushes and has a job-loss window
416
- # (Wurk::Scheduled::Enq); only the toggle swaps in the atomic promoter.
449
+ # NOTE: `reliable_scheduler!` below is NOT a no-op: it swaps in the atomic
450
+ # promoter, much faster but skipping client middleware on promotion.
417
451
  #
418
452
  # The optional block is Pro's recovery callback: `|jobstr, pill|`, fired
419
453
  # once per orphan recovery (`pill` nil) and once on a poison kill (`pill`
@@ -426,8 +460,9 @@ module Wurk
426
460
 
427
461
  # Pro reliable scheduler (§4): promote due jobs from retry/schedule onto
428
462
  # their target queue in a single atomic Lua (ZRANGEBYSCORE+ZREM+LPUSH),
429
- # closing the pop→push job-loss window of the default poller. Swaps the
430
- # pluggable `scheduled_enq` for the atomic promoter; idempotent.
463
+ # much faster than the default poller but skipping client middleware on
464
+ # promotion. Swaps the pluggable `scheduled_enq` for the atomic promoter;
465
+ # idempotent.
431
466
  def reliable_scheduler!(*)
432
467
  self[:scheduled_enq] = Wurk::Scheduled::ReliableEnq
433
468
  nil
@@ -765,6 +800,7 @@ module Wurk
765
800
  attr_writer :logger
766
801
 
767
802
  def handle_exception(ex, ctx = {})
803
+ ctx = redact_jobstr(ctx)
768
804
  if error_handlers.empty?
769
805
  logger.error("#{redacted_context(ctx)} #{ex.class}: #{ex.message}")
770
806
  else
@@ -776,11 +812,26 @@ module Wurk
776
812
  end
777
813
  end
778
814
 
815
+ # What the fetch loop reports through. It sleeps 1s and retries on every
816
+ # thread of every worker, so an outage would otherwise call each handler once
817
+ # per thread per second. The count lands in the metric, the handlers see the
818
+ # first of each exception class and one per window after that.
819
+ def report_fetch_error(exception)
820
+ Metrics::ProcessCounters.increment(:fetch_errors)
821
+ suppressed = @fetch_error_coalescer.admit(exception.class)
822
+ return unless suppressed
823
+
824
+ context = { context: 'Error fetching job' }
825
+ context[:suppressed] = suppressed if suppressed.positive?
826
+ handle_exception(exception, context)
827
+ end
828
+
779
829
  # The error context with the job's args passed through the `redact_args`
780
830
  # hook, for the two reporters that print or tag the whole context: the
781
831
  # fallback log line and the default ERROR_HANDLER (whose Wurk::Context a
782
832
  # JSON log formatter emits). Host-registered error handlers still receive
783
- # the real ctx — they are the host's own code.
833
+ # the real `job` — they are the host's own code (`jobstr` is another matter:
834
+ # see #redact_jobstr).
784
835
  def redacted_context(ctx)
785
836
  hook = @options[:redact_args]
786
837
  job = ctx.is_a?(Hash) ? ctx[:job] : nil
@@ -790,16 +841,36 @@ module Wurk
790
841
  ctx.merge(job: job.merge('args' => Wurk::Redact.args(job, hook)))
791
842
  end
792
843
 
844
+ # The raw payload a job that failed to parse or dispatch is reported with. Unlike
845
+ # `job`, this reaches host handlers scrubbed too: it is a string nobody expects
846
+ # to carry the args, so handlers forward it to log aggregators and trackers as
847
+ # is. What it cannot be parsed from cannot be scrubbed, so that is withheld.
848
+ def redact_jobstr(ctx)
849
+ hook = @options[:redact_args]
850
+ raw = ctx.is_a?(Hash) ? ctx[:jobstr] : nil
851
+ return ctx unless hook && raw
852
+
853
+ require_relative 'redact'
854
+ ctx.merge(jobstr: Wurk::Redact.jobstr(raw, hook))
855
+ end
856
+ private :redact_jobstr
857
+
793
858
  # --- Configure blocks (Sidekiq.configure_server / _client) -----------
794
859
 
795
- # The gate is evaluated per call: a skipped block is never yielded and
796
- # never replayed, and an allowed one yields the configuration and returns
797
- # the block's value. A skip reports itself once per gate per instance —
860
+ # The gate is evaluated per call: an allowed block yields the configuration
861
+ # and returns the block's value. A skip reports itself once per gate per instance —
798
862
  # the flags below decide only whether the skip is reported again, and
799
863
  # they live outside @options so a frozen configuration can still warn.
800
- def configure_server
801
- return unless block_given?
864
+ #
865
+ # A skipped block is kept for #enter_server_mode. Gems call this at load
866
+ # time, and under the railtie that is Bundler.require, before the
867
+ # `wurk.server_mode` initializer: dropping it left sidekiq-cron, sentry-sidekiq
868
+ # and the like silently absent from a process that does run workers.
869
+ def configure_server(&block)
870
+ return unless block
802
871
  return yield self if server?
872
+
873
+ @deferred_server_blocks << block
803
874
  return if @configure_server_skip_warned
804
875
 
805
876
  @configure_server_skip_warned = true
@@ -807,6 +878,11 @@ module Wurk
807
878
  nil
808
879
  end
809
880
 
881
+ def enter_server_mode
882
+ self[:server] = true
883
+ @deferred_server_blocks.shift.call(self) until @deferred_server_blocks.empty?
884
+ end
885
+
810
886
  # A web process that also boots the swarm (Wurk::RailsBoot) is flagged
811
887
  # server, yet every enqueue it makes from a request is client work, so
812
888
  # `client_in_server` lets its configure_client blocks run too. Same
@@ -896,6 +972,16 @@ module Wurk
896
972
  self
897
973
  end
898
974
 
975
+ # A swarm child inherits the parent's prepare_for_fork! freeze, but its copy
976
+ # is its own, and Launcher#run freezes it again. In between it is what a
977
+ # non-swarm process is: open to `:fork` / `:startup` hooks and to gems that
978
+ # prepend Launcher#initialize and write an option, as sidekiq-cron does.
979
+ def reopen_in_forked_child
980
+ @options = @options.dup
981
+ @frozen = false
982
+ self
983
+ end
984
+
899
985
  def frozen?
900
986
  @frozen
901
987
  end
@@ -906,6 +992,18 @@ module Wurk
906
992
 
907
993
  private
908
994
 
995
+ # Pools read the hash once, when built, so the later assignment silently does
996
+ # nothing to this process's connections until something rebuilds them.
997
+ def warn_on_late_redis_assignment
998
+ return unless @capsules.each_value.any?(&:redis_pools_built?)
999
+
1000
+ logger.warn do
1001
+ 'config.redis= called after a Redis pool was already built; existing connections keep the old ' \
1002
+ 'settings until the pools are rebuilt (a swarm child rebuilds them, a single process does not). ' \
1003
+ 'Set config.redis before anything uses Redis.'
1004
+ end
1005
+ end
1006
+
909
1007
  def warn_on_late_server_registration(klass)
910
1008
  affected = @capsules.each_value.filter_map do |capsule|
911
1009
  capsule.name if capsule.prepared? && capsule.server_middleware_missing?(klass)
@@ -1023,7 +1121,7 @@ module Wurk
1023
1121
  threads = total_concurrency
1024
1122
  return if size >= threads
1025
1123
 
1026
- logger.warn do
1124
+ logger.error do
1027
1125
  "ActiveRecord connection pool size (#{size}) is smaller than this process's job concurrency " \
1028
1126
  "(#{threads}); jobs that use the database will wait for connections and may raise " \
1029
1127
  'ActiveRecord::ConnectionTimeoutError. Raise `pool:` in config/database.yml to at least ' \
data/lib/wurk/cron.rb CHANGED
@@ -450,6 +450,21 @@ module Wurk
450
450
  @pruned_klasses = {}
451
451
  end
452
452
 
453
+ # How late a slot may fire, in seconds, instead of being skipped, for every
454
+ # loop of this cluster's pollers. Default MISSED_TICK_THRESHOLD. A process-wide
455
+ # setting and not a loop option, which would enter the loop's identity.
456
+ #
457
+ # Coerced here, because the poller rescues per loop: a value it could not
458
+ # compare would stop every loop on every tick with nothing failing at boot.
459
+ def grace=(seconds)
460
+ grace = Float(seconds, exception: false)
461
+ unless grace&.finite? && grace.positive?
462
+ raise ArgumentError, "periodic grace must be a positive, finite number of seconds: #{seconds.inspect}"
463
+ end
464
+
465
+ (@config || Wurk.configuration)[:cron_grace] = grace
466
+ end
467
+
453
468
  def register(cron, klass, **opts)
454
469
  klass_name = klass.is_a?(String) ? klass : klass.to_s
455
470
  tz = opts.delete(:tz) || @tz
@@ -461,8 +476,53 @@ module Wurk
461
476
  loop_obj
462
477
  end
463
478
 
479
+ # Registers a sidekiq-cron shaped schedule: `{ name => { cron:, class:,
480
+ # queue:, args:, retry:, tz:, status: } }`, or a list of those carrying
481
+ # `name:`. Other keys (`description`, ...) are ignored. Raises ArgumentError
482
+ # on an entry with no cron or class, and on anything but a 5-field crontab.
483
+ def load_hash(schedule)
484
+ entries = schedule
485
+ entries = schedule.map { |name, entry| { 'name' => name }.merge(entry.to_h) } if schedule.is_a?(::Hash)
486
+ entries.map { |entry| register_entry(entry.to_h.transform_keys(&:to_s)) }
487
+ end
488
+
489
+ # The same shape from a YAML file (ERB allowed), like a `config/schedule.yml`.
490
+ def load_file(path)
491
+ require 'erb'
492
+ require 'yaml'
493
+ load_hash(::YAML.safe_load(::ERB.new(::File.read(path)).result, aliases: true) || {})
494
+ end
495
+
496
+ # Call after the last registration to make this block the whole truth:
497
+ # every other loop in the cluster is deleted. Off unless asked because a
498
+ # process registering only part of the schedule would delete the rest, and
499
+ # because during a rolling deploy old and new code then disagree.
500
+ def authoritative!
501
+ return self unless @prune
502
+
503
+ keep = @loops.map(&:lid)
504
+ LoopSet.new(@config).each { |existing| Cron.unregister(existing.lid) unless keep.include?(existing.lid) }
505
+ self
506
+ end
507
+
464
508
  private
465
509
 
510
+ def register_entry(entry)
511
+ cron = entry['cron'].to_s.strip
512
+ klass = (entry['class'] || entry['klass']).to_s
513
+ if cron.empty? || klass.empty?
514
+ raise ArgumentError, "schedule entry #{entry['name'].inspect} needs `cron` and `class`"
515
+ end
516
+
517
+ register(cron, klass, **entry_options(entry))
518
+ end
519
+
520
+ def entry_options(entry)
521
+ opts = { label: entry['name']&.to_s, queue: entry['queue'], args: entry['args'], retry: entry['retry'],
522
+ tz: entry['tz'], paused: entry['status'] == 'disabled' ? true : nil }
523
+ opts.compact
524
+ end
525
+
466
526
  # Once per class per boot: everything else registered for that class is
467
527
  # a leftover from a previous deploy (the lid hashes schedule+options, so
468
528
  # an edited `register` line yields a new loop and orphans the old one).
@@ -566,10 +626,12 @@ module Wurk
566
626
  end
567
627
  end
568
628
 
569
- # Once-per-minute tick driver. Only the cluster leader iterates the LoopSet
570
- # and enqueues; non-leaders return early. A slot that wall-clock has passed
571
- # by more than `MISSED_TICK_THRESHOLD` seconds is skipped with a warning,
572
- # never backfilled.
629
+ # Once-per-minute tick driver, plus one tick the moment this process wins the
630
+ # lock: at a tick's phase, a leader that lives under a minute would never tick.
631
+ # Only the cluster leader iterates the LoopSet and enqueues; non-leaders return
632
+ # early. A slot that wall-clock has passed by more than `MISSED_TICK_THRESHOLD`
633
+ # seconds is skipped with a warning, never backfilled, and the occurrence due
634
+ # now, if any, fires in its place.
573
635
  class Poller
574
636
  include Component
575
637
 
@@ -580,17 +642,22 @@ module Wurk
580
642
  # Operators never need to touch this; integration tests shrink it so a
581
643
  # due loop fires within the test window instead of waiting a full minute.
582
644
  @tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
583
- @timer = TimerLoop.new(@tick_interval)
645
+ @done = false
646
+ @woken = false
647
+ @mutex = ::Mutex.new
648
+ @sleeper = ::ConditionVariable.new
649
+ @on_leader = method(:wake_for_leadership)
584
650
  end
585
651
 
586
- # TimerLoop waits one interval before the first tick: don't fire a
587
- # catch-up burst the instant we boot (the leader is barely settled), and
588
- # let a short-lived process exit without ticking at all.
652
+ # The loop waits one interval before its first tick unless this process
653
+ # wins the lock first: let a process that never leads, or exits at once,
654
+ # exit without ticking at all.
589
655
  def start
590
656
  return @thread if @thread
591
657
 
592
- @timer.reset
593
- @thread = safe_thread('cron-poller') { @timer.run { tick } }
658
+ @mutex.synchronize { @done = false }
659
+ leader_hooks << @on_leader
660
+ @thread = safe_thread('cron-poller') { run_loop }
594
661
  end
595
662
 
596
663
  # Blocks until the thread is really gone: the launcher releases the
@@ -599,10 +666,14 @@ module Wurk
599
666
  #
600
667
  # Cleared only on a confirmed join (Thread#join returns nil on timeout):
601
668
  # a wedged thread must stay tracked so #start's guard returns it instead
602
- # of calling @timer.reset, which would un-terminate the loop it is still
603
- # inside and leave two tick threads double-enqueuing the same loops.
669
+ # of clearing @done, which would un-terminate the loop it is still inside
670
+ # and leave two tick threads double-enqueuing the same loops.
604
671
  def terminate
605
- @timer.terminate
672
+ leader_hooks.delete(@on_leader)
673
+ @mutex.synchronize do
674
+ @done = true
675
+ @sleeper.signal
676
+ end
606
677
  @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
607
678
  end
608
679
 
@@ -634,6 +705,10 @@ module Wurk
634
705
  # A slot older than the missed-tick threshold is not backfilled (ent §2.6:
635
706
  # no backfill on restart, missed ticks across a leader gap are dropped):
636
707
  # the marks still advance past it, but nothing is pushed.
708
+ #
709
+ # One claim, one push at most, whichever slot it pushes for: the occurrence
710
+ # due now stands in for a stale one under the stale one's mark, so a poller
711
+ # that read the same mark loses the CAS for both.
637
712
  def enqueue_if_due(loop_obj)
638
713
  return if loop_obj.paused?
639
714
 
@@ -641,9 +716,12 @@ module Wurk
641
716
  slot, mark = due_slot(loop_obj, now)
642
717
  return if slot.nil?
643
718
 
644
- future = loop_obj.next_fire_after(slot, now)
719
+ firing = slot_to_fire(loop_obj, slot, now)
720
+ future = loop_obj.next_fire_after(firing || slot, now)
645
721
  return unless claim_fire?(loop_obj, mark, now, future)
646
- return skip_missed(loop_obj, slot, now) if now - slot > missed_threshold
722
+
723
+ skip_missed(loop_obj, slot, now) unless firing == slot
724
+ return if firing.nil?
647
725
 
648
726
  jid = enqueue_claimed!(loop_obj)
649
727
  record_history(loop_obj, jid, now)
@@ -663,6 +741,53 @@ module Wurk
663
741
 
664
742
  private
665
743
 
744
+ def run_loop
745
+ wait_for_tick
746
+ until done?
747
+ tick
748
+ wait_for_tick
749
+ end
750
+ end
751
+
752
+ def wait_for_tick
753
+ @mutex.synchronize do
754
+ @sleeper.wait(@mutex, @tick_interval) unless @done || @woken
755
+ @woken = false
756
+ end
757
+ end
758
+
759
+ def done?
760
+ @mutex.synchronize { @done }
761
+ end
762
+
763
+ # Runs on the campaign thread. The gate's cached answer predates the win.
764
+ def wake_for_leadership
765
+ @leader_checked_at = nil
766
+ @mutex.synchronize do
767
+ @woken = true
768
+ @sleeper.signal
769
+ end
770
+ end
771
+
772
+ def leader_hooks
773
+ @config[:lifecycle_events].fetch(:leader)
774
+ end
775
+
776
+ def slot_to_fire(loop_obj, slot, now)
777
+ now - slot > missed_threshold ? latest_due(loop_obj, now) : slot
778
+ end
779
+
780
+ # The newest occurrence still inside the window a late slot may fire in.
781
+ def latest_due(loop_obj, now)
782
+ latest = nil
783
+ occurrence = loop_obj.next_fire_at(now - missed_threshold - 1)
784
+ while occurrence && occurrence <= now
785
+ latest = occurrence
786
+ occurrence = loop_obj.next_fire_at(occurrence)
787
+ end
788
+ latest
789
+ end
790
+
666
791
  # The slot this tick would fire and the token that claims it, or nil when
667
792
  # the loop has no resolvable occurrence or none is due yet. The token is
668
793
  # the stored `nf` when there is one, otherwise the slot we derived — see
@@ -678,7 +803,7 @@ module Wurk
678
803
  # Never tighter than a tick and a half: a raised tick interval must not
679
804
  # turn every on-time fire into a "missed" one.
680
805
  def missed_threshold
681
- [MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
806
+ [@config[:cron_grace] || MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
682
807
  end
683
808
 
684
809
  def skip_missed(loop_obj, expected, now)
data/lib/wurk/debounce.rb CHANGED
@@ -81,8 +81,8 @@ module Wurk
81
81
  reject_shorter_cap!(seconds, cap)
82
82
 
83
83
  argv = [seconds, cap, JobUtil.scheduled_member(job), GRACE]
84
- # Replay-safe: a re-run finds the member it just wrote, ZREMs it and
85
- # ZADDs it back, so a pool retry after a lost reply converges on one
84
+ # Replay-safe: a re-run finds the member it just wrote and ZADDs it
85
+ # over itself, so a pool retry after a lost reply converges on one
86
86
  # entry rather than a duplicate — the whole point of doing this in a
87
87
  # script rather than a pipeline.
88
88
  #
data/lib/wurk/embedded.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require_relative 'component'
4
4
  require_relative 'launcher'
5
+ require_relative 'swarm/test_mode_guard'
5
6
 
6
7
  module Wurk
7
8
  # Embeds Wurk inside an arbitrary Ruby process (Puma, rake task, custom
@@ -34,13 +35,13 @@ module Wurk
34
35
  # the worker threads have time to spin up before the host goes back to
35
36
  # serving traffic. Mirrors Sidekiq::Embedded#run.
36
37
  def run
38
+ Swarm::TestModeGuard.enforce!(logger)
37
39
  housekeeping
38
40
  fire_event(:startup, reverse: false, reraise: true)
39
41
  @launcher = build_launcher
40
42
  @launcher.run
41
43
  sleep 0.2
42
- logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
43
- logger.debug { Thread.list.map(&:name).to_s }
44
+ log_running
44
45
  rescue StandardError
45
46
  # The host is left believing Wurk never started, so anything the launcher
46
47
  # did get up would keep fetching, beating and campaigning for the leader
@@ -62,6 +63,11 @@ module Wurk
62
63
 
63
64
  private
64
65
 
66
+ def log_running
67
+ logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
68
+ logger.debug { Thread.list.map(&:name).to_s }
69
+ end
70
+
65
71
  # Guarded so the caller sees why the boot failed, not why the rollback did.
66
72
  def roll_back_partial_boot
67
73
  stop
@@ -83,6 +83,12 @@ module Wurk
83
83
  # is what callers see as the dead record's `error_class`.
84
84
  class DecryptionError < Error; end
85
85
 
86
+ # The job says `encrypt: true` but its last argument is not a Wurk envelope:
87
+ # another implementation's ciphertext, or a producer that never enabled
88
+ # crypto. Neither can be told from a plain value, and either way `perform`
89
+ # must not take it for the decrypted argument.
90
+ class NotSealedError < Error; end
91
+
86
92
  class << self
87
93
  attr_reader :active_version
88
94
 
@@ -228,7 +234,10 @@ module Wurk
228
234
  'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
229
235
  )
230
236
  Wurk::Metrics::Statsd.increment('jobs.encryption_error', tags: ["worker:#{job['class']}"])
231
- Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause)
237
+ Wurk.logger.error { "#{DEAD_REASON}: #{job['class']} jid=#{job['jid']} sent to the dead set: #{cause.message}" }
238
+ # No trim: a misconfigured producer or a lost key can fail a whole backlog at
239
+ # once, and the cap would destroy jobs that are recoverable once it is fixed.
240
+ Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause, trim: false)
232
241
  nil
233
242
  end
234
243
 
@@ -318,14 +327,19 @@ module Wurk
318
327
  # with every outer middleware and the processor; leaving plaintext in it
319
328
  # let any of them write the secret back to Redis (InterruptHandler's
320
329
  # re-push, limiter dead-routing) or hand it to an error reporter. A decrypt
321
- # failure (missing/rotated key, bad tag) is terminal and non-retryable,
322
- # so rather than let it bubble into the 25× retry pipeline we route the
330
+ # failure (missing/rotated key, bad tag, a last argument that is no envelope)
331
+ # is terminal and non-retryable, so rather than let it bubble into the 25× retry pipeline we route the
323
332
  # job straight to the dead set tagged `encryption_error` and ACK it via
324
333
  # JobRetry::Skip — see Wurk::Encryption.route_to_dead. Plaintext args
325
334
  # remain visible for triage per §4.6.
326
335
  class ServerMiddleware
327
336
  include Wurk::Middleware::ServerMiddleware
328
337
 
338
+ # Never quotes the argument: it may be a secret in the clear.
339
+ NOT_SEALED_MESSAGE = 'job has encrypt: true but its last argument is not a Wurk envelope, so it was not run. ' \
340
+ 'A producer without crypto enabled pushed it, or another implementation encrypted it. ' \
341
+ 'A retry re-seals the argument as it stands: retry only a job known to hold plaintext'
342
+
329
343
  def call(_worker, job, _queue)
330
344
  return yield unless Wurk::Encryption.enabled? && job['encrypt']
331
345
 
@@ -342,7 +356,8 @@ module Wurk
342
356
 
343
357
  def decrypt_last_arg!(job)
344
358
  args = job['args']
345
- return unless args.is_a?(::Array) && !args.empty? && Wurk::Encryption.envelope?(args.last)
359
+ return unless args.is_a?(::Array) && !args.empty?
360
+ raise NotSealedError, NOT_SEALED_MESSAGE unless Wurk::Encryption.envelope?(args.last)
346
361
 
347
362
  job['args'] = args[0..-2] + [Wurk::Encryption.decrypt(args.last)]
348
363
  rescue Wurk::Encryption::Error,
@@ -350,7 +365,7 @@ module Wurk
350
365
  ::ArgumentError,
351
366
  ::TypeError,
352
367
  ::JSON::ParserError => e
353
- # Wurk::Encryption::Error covers KeyMissingError / DecryptionError.
368
+ # Wurk::Encryption::Error covers KeyMissingError / DecryptionError / NotSealedError.
354
369
  # OpenSSL::Cipher::CipherError → tampered ciphertext / bad key (tag mismatch).
355
370
  # ArgumentError → Base64.strict_decode64 / Integer() on a malformed envelope.
356
371
  # TypeError → Integer(nil) on a missing 'v' field.
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ # Lets the first report of a key through, then one per window, and says how
5
+ # many it swallowed in between. Per process: every forked child speaks for
6
+ # itself, so a fleet still reports at worst children x keys per window.
7
+ class ErrorCoalescer
8
+ DEFAULT_WINDOW = 60
9
+
10
+ def initialize(window: DEFAULT_WINDOW, clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
11
+ @window = window
12
+ @clock = clock
13
+ @lock = ::Mutex.new
14
+ @entries = {}
15
+ end
16
+
17
+ # The number of reports suppressed since the last one when this one should
18
+ # go out (0 for a first report); nil when this one should be swallowed.
19
+ def admit(key)
20
+ @lock.synchronize do
21
+ now = @clock.call
22
+ entry = (@entries[key] ||= [nil, 0])
23
+ if entry[0] && now - entry[0] < @window
24
+ entry[1] += 1
25
+ next nil
26
+ end
27
+
28
+ suppressed = entry[1]
29
+ entry[0] = now
30
+ entry[1] = 0
31
+ suppressed
32
+ end
33
+ end
34
+ end
35
+ end