wurk 1.7.6 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -4
- data/app/controllers/concerns/wurk/sse_streaming.rb +2 -1
- data/app/controllers/concerns/wurk/stream_concurrency_guard.rb +33 -9
- data/app/controllers/wurk/api/pagination.rb +25 -7
- data/app/controllers/wurk/api_controller.rb +49 -25
- data/app/controllers/wurk/extensions_controller.rb +8 -2
- data/app/controllers/wurk/profiles_controller.rb +45 -9
- data/config/routes.rb +31 -10
- data/lib/generators/wurk/install/templates/wurk.rb +1 -1
- data/lib/sidekiq/capsule.rb +4 -0
- data/lib/sidekiq/component.rb +4 -0
- data/lib/sidekiq/config.rb +4 -0
- data/lib/sidekiq/deploy.rb +4 -0
- data/lib/sidekiq/embedded.rb +4 -0
- data/lib/sidekiq/fetch.rb +4 -0
- data/lib/sidekiq/iterable_job.rb +4 -0
- data/lib/sidekiq/job/interrupt_handler.rb +4 -0
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +4 -0
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +4 -0
- data/lib/sidekiq/job/iterable.rb +4 -0
- data/lib/sidekiq/job_logger.rb +4 -0
- data/lib/sidekiq/job_util.rb +4 -0
- data/lib/sidekiq/loader.rb +4 -0
- data/lib/sidekiq/logger.rb +4 -0
- data/lib/sidekiq/metrics/query.rb +4 -0
- data/lib/sidekiq/middleware/current_attributes.rb +7 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -0
- data/lib/sidekiq/middleware/modules.rb +4 -0
- data/lib/sidekiq/pro/web.rb +5 -0
- data/lib/sidekiq/profiler.rb +4 -0
- data/lib/sidekiq/redis_client_adapter.rb +4 -0
- data/lib/sidekiq/test_api.rb +5 -0
- data/lib/sidekiq/testing/inline.rb +8 -0
- data/lib/sidekiq/transaction_aware_client.rb +4 -0
- data/lib/sidekiq/worker_compatibility_alias.rb +4 -0
- data/lib/sidekiq-ent/periodic/testing.rb +5 -0
- data/lib/sidekiq-ent/web.rb +5 -0
- data/lib/sidekiq-ent.rb +7 -0
- data/lib/sidekiq-pro.rb +7 -0
- data/lib/wurk/api/fast.rb +8 -33
- data/lib/wurk/api/idempotency.rb +28 -5
- data/lib/wurk/api/swarm.rb +1 -1
- data/lib/wurk/batch/callbacks.rb +143 -92
- data/lib/wurk/batch/death_handler.rb +19 -8
- data/lib/wurk/batch/server_middleware.rb +37 -24
- data/lib/wurk/batch/status.rb +49 -15
- data/lib/wurk/batch.rb +164 -65
- data/lib/wurk/capsule.rb +0 -2
- data/lib/wurk/client/buffered.rb +227 -106
- data/lib/wurk/client.rb +33 -13
- data/lib/wurk/compat.rb +13 -0
- data/lib/wurk/configuration.rb +155 -24
- data/lib/wurk/cpu_count.rb +97 -0
- data/lib/wurk/cron.rb +97 -34
- data/lib/wurk/dead_set.rb +6 -7
- data/lib/wurk/encryption.rb +46 -13
- data/lib/wurk/fetcher/many_queues.rb +65 -0
- data/lib/wurk/fetcher/orphan_grace.rb +67 -0
- data/lib/wurk/fetcher/private_list_key.rb +76 -0
- data/lib/wurk/fetcher/reaper.rb +55 -74
- data/lib/wurk/fetcher/reliable.rb +23 -17
- data/lib/wurk/fetcher/unit_of_work.rb +19 -1
- data/lib/wurk/fetcher/unparseable_keys.rb +56 -0
- data/lib/wurk/flow/completion.rb +28 -3
- data/lib/wurk/flow/creation.rb +24 -4
- data/lib/wurk/flow.rb +24 -2
- data/lib/wurk/health.rb +182 -40
- data/lib/wurk/heartbeat.rb +37 -14
- data/lib/wurk/history.rb +7 -1
- data/lib/wurk/import/sidekiq_cron.rb +248 -0
- data/lib/wurk/job/options.rb +22 -3
- data/lib/wurk/job_logger.rb +13 -3
- data/lib/wurk/job_record.rb +68 -19
- data/lib/wurk/job_retry.rb +12 -6
- data/lib/wurk/job_set.rb +99 -51
- data/lib/wurk/job_util.rb +27 -8
- data/lib/wurk/launcher.rb +84 -40
- data/lib/wurk/leader.rb +53 -28
- data/lib/wurk/limiter/base.rb +11 -30
- data/lib/wurk/limiter/bucket.rb +14 -5
- data/lib/wurk/limiter/server_middleware.rb +2 -1
- data/lib/wurk/limiter.rb +1 -1
- data/lib/wurk/loader.rb +48 -0
- data/lib/wurk/lua/batch_ack_complete.lua +28 -0
- data/lib/wurk/lua/batch_ack_failed.lua +18 -0
- data/lib/wurk/lua/batch_ack_success.lua +35 -0
- data/lib/wurk/lua/batch_append_callback.lua +66 -0
- data/lib/wurk/lua/batch_delete.lua +31 -0
- data/lib/wurk/lua/batch_hold.lua +17 -0
- data/lib/wurk/lua/batch_invalidate.lua +14 -0
- data/lib/wurk/lua/batch_kid_done.lua +14 -0
- data/lib/wurk/lua/batch_push.lua +47 -0
- data/lib/wurk/lua/batch_remove_jobs.lua +29 -0
- data/lib/wurk/lua/batch_schedule.lua +27 -0
- data/lib/wurk/lua/fetch_first.lua +27 -0
- data/lib/wurk/lua/flow_abandon.lua +16 -21
- data/lib/wurk/lua/flow_advance.lua +41 -26
- data/lib/wurk/lua/flow_create.lua +24 -9
- data/lib/wurk/lua/flow_fail.lua +3 -3
- data/lib/wurk/lua/leader_campaign.lua +19 -0
- data/lib/wurk/lua/limiter_bucket_acquire.lua +3 -2
- data/lib/wurk/lua/limiter_list_sweep.lua +8 -9
- data/lib/wurk/lua/loader.rb +36 -23
- data/lib/wurk/lua/throttle_slot.lua +13 -26
- data/lib/wurk/lua.rb +31 -244
- data/lib/wurk/manager.rb +25 -18
- data/lib/wurk/metrics/accumulator.rb +31 -10
- data/lib/wurk/metrics/dashboard_series.rb +163 -0
- data/lib/wurk/metrics/histogram.rb +46 -0
- data/lib/wurk/metrics/history.rb +49 -57
- data/lib/wurk/metrics/prometheus.rb +232 -0
- data/lib/wurk/metrics/query.rb +127 -155
- data/lib/wurk/metrics/rollup.rb +2 -2
- data/lib/wurk/metrics/statsd.rb +1 -1
- data/lib/wurk/middleware/interrupt_handler.rb +6 -2
- data/lib/wurk/middleware/poison_pill.rb +5 -4
- data/lib/wurk/middleware/timeout.rb +8 -8
- data/lib/wurk/process_set.rb +5 -5
- data/lib/wurk/processor.rb +69 -31
- data/lib/wurk/profile_set.rb +19 -24
- data/lib/wurk/profiler.rb +48 -50
- data/lib/wurk/queue.rb +36 -19
- data/lib/wurk/rails_boot.rb +33 -1
- data/lib/wurk/railtie.rb +2 -0
- data/lib/wurk/rake_tasks.rb +18 -0
- data/lib/wurk/redact.rb +39 -0
- data/lib/wurk/redis_connection.rb +6 -3
- data/lib/wurk/redis_options.rb +21 -4
- data/lib/wurk/redis_pool.rb +119 -21
- data/lib/wurk/scheduled.rb +44 -11
- data/lib/wurk/sentry/error_handler.rb +53 -7
- data/lib/wurk/sentry/middleware.rb +7 -8
- data/lib/wurk/shutdown_gate.rb +24 -1
- data/lib/wurk/sorted_entry.rb +16 -5
- data/lib/wurk/stats.rb +9 -4
- data/lib/wurk/status.rb +2 -2
- data/lib/wurk/swarm/child_boot.rb +48 -38
- data/lib/wurk/swarm/liveness.rb +153 -0
- data/lib/wurk/swarm/restart.rb +24 -3
- data/lib/wurk/swarm.rb +115 -24
- data/lib/wurk/telemetry/server_middleware.rb +2 -5
- data/lib/wurk/testing.rb +15 -5
- data/lib/wurk/throttle.rb +16 -1
- data/lib/wurk/topology.rb +1 -5
- data/lib/wurk/unique.rb +1 -1
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/watchdog.rb +2 -2
- data/lib/wurk/web/config.rb +56 -20
- data/lib/wurk/web/enterprise.rb +21 -18
- data/lib/wurk/web/extension.rb +98 -22
- data/lib/wurk/web/rack_app.rb +12 -11
- data/lib/wurk/web/search.rb +18 -8
- data/lib/wurk/worker/setter.rb +17 -14
- data/lib/wurk/worker.rb +6 -67
- data/lib/wurk.rb +2 -3
- data/vendor/assets/dashboard/assets/ArgsValue-DsPb_KZ-.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BCIXJOpZ.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-IjaQfXW6.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-CaQrf1Ve.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-B18EHn_Y.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-DdT5ZaUg.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-DWIy-Mu8.js +1 -0
- data/vendor/assets/dashboard/assets/Extension-BjBUWhHf.js +1 -0
- data/vendor/assets/dashboard/assets/FilterBox-DwH3PXXt.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-DnexL72n.js +1 -0
- data/vendor/assets/dashboard/assets/FlowState-Cr4aX6-d.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-DO6EimbZ.js +1 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-aPjgq-U9.js +2 -0
- data/vendor/assets/dashboard/assets/Limiters-DxpwWCNe.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-C3JL1GIH.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-BfP9uAV9.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-DbC97afG.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-BcaHreMe.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-ZlBc5tyD.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-Bg3nkeiL.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-BrMZPZGl.js +1 -0
- data/vendor/assets/dashboard/assets/Search-B5X5pGht.js +1 -0
- data/vendor/assets/dashboard/assets/charts-uDyf4PN0.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-CJeYfeVt.js +1 -0
- data/vendor/assets/dashboard/assets/index-B36QCAf-.css +1 -0
- data/vendor/assets/dashboard/assets/index-DQujgZgH.js +141 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-O3-5rMk3.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-xfnwjQAu.js +1 -0
- data/vendor/assets/dashboard/assets/utils-BIKHwNKB.js +1 -0
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +90 -34
- data/config/locales/en.yml +0 -15
- data/vendor/assets/dashboard/assets/ArgsValue-D-x_ifLY.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-C39NJuew.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-CSwo7Asa.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-BOFMu-sq.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-Dy8RQzDI.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-BuTHI-O1.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-B9KRvQ0N.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-BnBVHfux.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-DC24zite.js +0 -1
- data/vendor/assets/dashboard/assets/FlowDetail-DyLuzUvt.js +0 -1
- data/vendor/assets/dashboard/assets/FlowState-DAPKUahm.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-Fr3rjZM_.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-N6kiJXq3.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-kbFA7uS1.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Dj2uoZ3o.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-B_F94azl.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-D_DjEezN.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-CO4V9hAz.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-DCWnzeLa.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-BebDUjLU.js +0 -1
- data/vendor/assets/dashboard/assets/Search-Cvr5fy4Y.js +0 -1
- data/vendor/assets/dashboard/assets/Skeleton-Bu3Ke6rV.js +0 -1
- data/vendor/assets/dashboard/assets/charts-BCs9bQKz.js +0 -1
- data/vendor/assets/dashboard/assets/index-BIwyOC5Q.js +0 -141
- data/vendor/assets/dashboard/assets/index-DBQN6Jk8.css +0 -1
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Bzh-BJyL.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-COA3fVJ5.js +0 -1
- data/vendor/assets/dashboard/assets/utils-BIrvZ1hi.js +0 -1
data/lib/wurk/configuration.rb
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'etc'
|
|
4
3
|
require 'logger'
|
|
5
4
|
require_relative 'middleware/chain'
|
|
6
5
|
require_relative 'capsule'
|
|
7
6
|
require_relative 'pool_checkout'
|
|
8
7
|
require_relative 'context'
|
|
9
8
|
require_relative 'topology'
|
|
9
|
+
require_relative 'cpu_count'
|
|
10
10
|
require_relative 'redis_options'
|
|
11
11
|
require_relative 'keys'
|
|
12
12
|
|
|
@@ -46,7 +46,7 @@ module Wurk
|
|
|
46
46
|
reloader: proc { |&b| b.call },
|
|
47
47
|
backtrace_cleaner: ->(bt) { bt },
|
|
48
48
|
logged_job_attributes: %w[bid tags],
|
|
49
|
-
redis_idle_timeout: nil,
|
|
49
|
+
redis_idle_timeout: nil, # seconds; RedisPool closes connections idle this long
|
|
50
50
|
redis_error_handlers: [],
|
|
51
51
|
# Wurk extras, appended after the mirrored keys so the Sidekiq prefix
|
|
52
52
|
# above stays byte-for-byte what a gem reading @options expects.
|
|
@@ -121,6 +121,7 @@ module Wurk
|
|
|
121
121
|
# Spec: docs/target/sidekiq-free.md §4.3.
|
|
122
122
|
ERROR_HANDLER = lambda do |ex, ctx, cfg = Wurk.configuration|
|
|
123
123
|
safe_ctx = ctx || {}
|
|
124
|
+
safe_ctx = cfg.redacted_context(safe_ctx) if cfg.respond_to?(:redacted_context)
|
|
124
125
|
Wurk::Context.with(safe_ctx) do
|
|
125
126
|
dev = $DEBUG || ENV['WURK_DEBUG'] || cfg.logger.debug?
|
|
126
127
|
msg = dev ? ex.full_message : ex.detailed_message
|
|
@@ -157,7 +158,7 @@ module Wurk
|
|
|
157
158
|
# Hash arrives frozen: the fetch path resolves caps once at boot, so a
|
|
158
159
|
# Hash still mutable here is one a caller can add a queue to and have
|
|
159
160
|
# nothing read it. FrozenError is the honest answer to that.
|
|
160
|
-
|
|
161
|
+
validate_constructor_options!
|
|
161
162
|
@options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
|
|
162
163
|
@capsules = {}
|
|
163
164
|
@directory = {}
|
|
@@ -177,6 +178,7 @@ module Wurk
|
|
|
177
178
|
|
|
178
179
|
def []=(key, val)
|
|
179
180
|
guard_frozen!
|
|
181
|
+
RedisPool.validate_idle_timeout(val) if key == :redis_idle_timeout
|
|
180
182
|
@options[key] = val
|
|
181
183
|
end
|
|
182
184
|
|
|
@@ -185,6 +187,7 @@ module Wurk
|
|
|
185
187
|
alias has_key? key?
|
|
186
188
|
def merge!(other)
|
|
187
189
|
guard_frozen!
|
|
190
|
+
RedisPool.validate_idle_timeout(other[:redis_idle_timeout]) if other.key?(:redis_idle_timeout)
|
|
188
191
|
@options.merge!(other)
|
|
189
192
|
end
|
|
190
193
|
|
|
@@ -250,6 +253,12 @@ module Wurk
|
|
|
250
253
|
@redis_config = @redis_config.merge(hash.transform_keys(&:to_sym))
|
|
251
254
|
end
|
|
252
255
|
|
|
256
|
+
# Sidekiq's setter for `:redis_idle_timeout`; read when a pool is built, so
|
|
257
|
+
# call it in the initializer.
|
|
258
|
+
def reap_idle_redis_connections(timeout = 60)
|
|
259
|
+
self[:redis_idle_timeout] = timeout
|
|
260
|
+
end
|
|
261
|
+
|
|
253
262
|
def redis_pool
|
|
254
263
|
default_capsule.redis_pool
|
|
255
264
|
end
|
|
@@ -360,6 +369,40 @@ module Wurk
|
|
|
360
369
|
@options[:fetch_poll_interval]
|
|
361
370
|
end
|
|
362
371
|
|
|
372
|
+
# Display-side argument redaction (Wurk::Redact): a callable taking a deep
|
|
373
|
+
# copy of the job Hash and returning the args to show in logs, the
|
|
374
|
+
# dashboard and the fallback error log. nil (default) shows args as-is.
|
|
375
|
+
def redact_args=(hook)
|
|
376
|
+
raise ArgumentError, 'redact_args must be nil or respond to #call' unless hook.nil? || hook.respond_to?(:call)
|
|
377
|
+
|
|
378
|
+
@options[:redact_args] = hook
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
def redact_args
|
|
382
|
+
@options[:redact_args]
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# Seconds a swarm child may go without a heartbeat before the parent
|
|
386
|
+
# replaces it (Swarm::Liveness). false or 0 turns the check off. The floor
|
|
387
|
+
# sits above the heartbeat's own cadence so a child that is merely busy is
|
|
388
|
+
# never read as hung.
|
|
389
|
+
SWARM_HEARTBEAT_TIMEOUT_FLOOR = 20
|
|
390
|
+
|
|
391
|
+
def swarm_heartbeat_timeout=(seconds)
|
|
392
|
+
unless seconds == false || seconds == 0 || # rubocop:disable Style/NumericPredicate
|
|
393
|
+
((seconds.is_a?(Integer) || seconds.is_a?(Float)) && seconds >= SWARM_HEARTBEAT_TIMEOUT_FLOOR)
|
|
394
|
+
raise ArgumentError,
|
|
395
|
+
"swarm_heartbeat_timeout must be false, 0, or a number >= #{SWARM_HEARTBEAT_TIMEOUT_FLOOR} " \
|
|
396
|
+
"(got #{seconds.inspect})"
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
@options[:swarm_heartbeat_timeout] = seconds
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
def swarm_heartbeat_timeout
|
|
403
|
+
@options[:swarm_heartbeat_timeout]
|
|
404
|
+
end
|
|
405
|
+
|
|
363
406
|
# --- Reliability (Sidekiq Pro drop-in toggles) -----------------------
|
|
364
407
|
|
|
365
408
|
# Sidekiq Pro's opt-in toggle for reliable fetch. Already the default in
|
|
@@ -664,7 +707,11 @@ module Wurk
|
|
|
664
707
|
#
|
|
665
708
|
# Off by default — call this in a `configure_server` block to enable.
|
|
666
709
|
# Spec: docs/target/sidekiq-ent.md §7.1.2.
|
|
667
|
-
|
|
710
|
+
#
|
|
711
|
+
# `metrics:` serves the Prometheus scrape on the same listener (on by
|
|
712
|
+
# default). `min_ready:` — children with a fresh heartbeat required for
|
|
713
|
+
# /ready inside a swarm; nil means half the swarm, rounded up.
|
|
714
|
+
def health_check(port:, bind: '0.0.0.0', ready_window: 30, metrics: true, min_ready: nil)
|
|
668
715
|
guard_frozen!
|
|
669
716
|
p = Integer(port)
|
|
670
717
|
rw = Integer(ready_window)
|
|
@@ -674,7 +721,9 @@ module Wurk
|
|
|
674
721
|
b = bind.to_s
|
|
675
722
|
raise ArgumentError, 'bind must be a non-empty string' if b.empty?
|
|
676
723
|
|
|
677
|
-
@options[:health_check_options] = { port: p, bind: b, ready_window: rw
|
|
724
|
+
@options[:health_check_options] = { port: p, bind: b, ready_window: rw,
|
|
725
|
+
metrics: health_check_metrics(metrics),
|
|
726
|
+
min_ready: health_check_min_ready(min_ready) }
|
|
678
727
|
end
|
|
679
728
|
|
|
680
729
|
# --- Lifecycle hooks --------------------------------------------------
|
|
@@ -698,7 +747,7 @@ module Wurk
|
|
|
698
747
|
|
|
699
748
|
def handle_exception(ex, ctx = {})
|
|
700
749
|
if error_handlers.empty?
|
|
701
|
-
logger.error("#{ctx} #{ex.class}: #{ex.message}")
|
|
750
|
+
logger.error("#{redacted_context(ctx)} #{ex.class}: #{ex.message}")
|
|
702
751
|
else
|
|
703
752
|
error_handlers.each do |handler|
|
|
704
753
|
handler.call(ex, ctx, self)
|
|
@@ -708,14 +757,31 @@ module Wurk
|
|
|
708
757
|
end
|
|
709
758
|
end
|
|
710
759
|
|
|
760
|
+
# The error context with the job's args passed through the `redact_args`
|
|
761
|
+
# hook, for the two reporters that print or tag the whole context: the
|
|
762
|
+
# fallback log line and the default ERROR_HANDLER (whose Wurk::Context a
|
|
763
|
+
# JSON log formatter emits). Host-registered error handlers still receive
|
|
764
|
+
# the real ctx — they are the host's own code.
|
|
765
|
+
def redacted_context(ctx)
|
|
766
|
+
hook = @options[:redact_args]
|
|
767
|
+
job = ctx.is_a?(Hash) ? ctx[:job] : nil
|
|
768
|
+
return ctx unless hook && job.is_a?(Hash) && job.key?('args')
|
|
769
|
+
|
|
770
|
+
require_relative 'redact'
|
|
771
|
+
ctx.merge(job: job.merge('args' => Wurk::Redact.args(job, hook)))
|
|
772
|
+
end
|
|
773
|
+
|
|
711
774
|
# --- Configure blocks (Sidekiq.configure_server / _client) -----------
|
|
712
775
|
|
|
713
776
|
def configure_server(&block)
|
|
714
777
|
yield self if block && server?
|
|
715
778
|
end
|
|
716
779
|
|
|
780
|
+
# A web process that also boots the swarm (Wurk::RailsBoot) is flagged
|
|
781
|
+
# server, yet every enqueue it makes from a request is client work, so
|
|
782
|
+
# `client_in_server` lets its configure_client blocks run too.
|
|
717
783
|
def configure_client(&block)
|
|
718
|
-
yield self if block && !server?
|
|
784
|
+
yield self if block && (!server? || @options[:client_in_server])
|
|
719
785
|
end
|
|
720
786
|
|
|
721
787
|
def server?
|
|
@@ -788,6 +854,7 @@ module Wurk
|
|
|
788
854
|
prepare_for_fork!
|
|
789
855
|
@capsules.each_value(&:freeze)
|
|
790
856
|
@capsules.freeze
|
|
857
|
+
warn_on_small_db_pool
|
|
791
858
|
self
|
|
792
859
|
end
|
|
793
860
|
|
|
@@ -801,6 +868,11 @@ module Wurk
|
|
|
801
868
|
|
|
802
869
|
private
|
|
803
870
|
|
|
871
|
+
def validate_constructor_options!
|
|
872
|
+
@options[:global_concurrency] = normalize_global_concurrency(@options[:global_concurrency])
|
|
873
|
+
RedisPool.validate_idle_timeout(@options[:redis_idle_timeout])
|
|
874
|
+
end
|
|
875
|
+
|
|
804
876
|
# One flat fork running the default capsule's queues + concurrency. The
|
|
805
877
|
# railtie boots this when a Rails host mounts the engine without declaring
|
|
806
878
|
# a topology. queue_specs (not queues) so weights survive the round-trip.
|
|
@@ -810,20 +882,78 @@ module Wurk
|
|
|
810
882
|
end
|
|
811
883
|
|
|
812
884
|
# Number of swarm children when the host hasn't declared a topology
|
|
813
|
-
# (Sidekiq Ent §7.2). Defaults to the CPU count
|
|
814
|
-
#
|
|
815
|
-
#
|
|
816
|
-
#
|
|
817
|
-
#
|
|
885
|
+
# (Sidekiq Ent §7.2). Defaults to the usable CPU count — the cgroup quota
|
|
886
|
+
# when one is tighter than the host's cores (Wurk::CpuCount). A whole number
|
|
887
|
+
# is an absolute count; a fractional value is a CPU multiplier (e.g. `0.5`
|
|
888
|
+
# → half the usable CPUs, rounded), supported since Sidekiq 8.0.2.
|
|
889
|
+
# WURK_COUNT is the native name, SIDEKIQ_COUNT the drop-in alias.
|
|
890
|
+
# Unparseable input falls back to the CPU count; the result is floored at 1
|
|
891
|
+
# so the swarm always forks. Logged once, with where the number came from:
|
|
892
|
+
# a pod that forks the wrong number of children is otherwise only visible
|
|
893
|
+
# as throttling or memory pressure.
|
|
818
894
|
def default_child_count
|
|
819
|
-
|
|
820
|
-
|
|
895
|
+
count, source = child_count_and_source
|
|
896
|
+
logger.info { "Wurk swarm: forking #{count} #{count == 1 ? 'child' : 'children'} (#{source})" }
|
|
897
|
+
count
|
|
898
|
+
end
|
|
899
|
+
|
|
900
|
+
def child_count_and_source
|
|
901
|
+
name = ENV['WURK_COUNT'] ? 'WURK_COUNT' : 'SIDEKIQ_COUNT'
|
|
902
|
+
raw = ENV.fetch(name, nil)
|
|
903
|
+
cpus = Wurk::CpuCount.detect
|
|
904
|
+
return [cpus.count, cpus.source] if raw.nil? || raw.strip.empty?
|
|
905
|
+
|
|
906
|
+
env_child_count(name, raw, cpus)
|
|
907
|
+
end
|
|
821
908
|
|
|
909
|
+
def env_child_count(name, raw, cpus)
|
|
822
910
|
value = Float(raw)
|
|
823
|
-
|
|
824
|
-
|
|
911
|
+
return [[value.to_i, 1].max, "#{name}=#{raw}"] if (value % 1).zero?
|
|
912
|
+
|
|
913
|
+
[[(value * cpus.count).round, 1].max, "#{name}=#{raw} x #{cpus.count} CPUs from #{cpus.source}"]
|
|
825
914
|
rescue ArgumentError, TypeError
|
|
826
|
-
|
|
915
|
+
[cpus.count, "#{cpus.source}; #{name}=#{raw.inspect} is not a number"]
|
|
916
|
+
end
|
|
917
|
+
|
|
918
|
+
def health_check_metrics(value)
|
|
919
|
+
raise ArgumentError, "metrics must be true or false (got #{value.inspect})" unless [true, false].include?(value)
|
|
920
|
+
|
|
921
|
+
value
|
|
922
|
+
end
|
|
923
|
+
|
|
924
|
+
def health_check_min_ready(value)
|
|
925
|
+
return nil if value.nil?
|
|
926
|
+
unless value.is_a?(Integer) && value.positive?
|
|
927
|
+
raise ArgumentError, "min_ready must be nil or an Integer >= 1 (got #{value.inspect})"
|
|
928
|
+
end
|
|
929
|
+
|
|
930
|
+
value
|
|
931
|
+
end
|
|
932
|
+
|
|
933
|
+
# Runs where every worker process settles its capsules — a swarm child
|
|
934
|
+
# after its slot is applied, `wurk`, embedded — so the concurrency compared
|
|
935
|
+
# is the one that process will actually run. Every job thread that touches
|
|
936
|
+
# the database checks out its own connection; a pool smaller than the
|
|
937
|
+
# thread count turns into ActiveRecord::ConnectionTimeoutError under load,
|
|
938
|
+
# long after boot. A warning rather than a raise: a job set that never
|
|
939
|
+
# touches the database is a legitimate reason to run a small pool.
|
|
940
|
+
def warn_on_small_db_pool(record_class = (::ActiveRecord::Base if defined?(::ActiveRecord::Base)))
|
|
941
|
+
return unless record_class
|
|
942
|
+
|
|
943
|
+
size = record_class.connection_pool.size
|
|
944
|
+
threads = total_concurrency
|
|
945
|
+
return if size >= threads
|
|
946
|
+
|
|
947
|
+
logger.warn do
|
|
948
|
+
"ActiveRecord connection pool size (#{size}) is smaller than this process's job concurrency " \
|
|
949
|
+
"(#{threads}); jobs that use the database will wait for connections and may raise " \
|
|
950
|
+
'ActiveRecord::ConnectionTimeoutError. Raise `pool:` in config/database.yml to at least ' \
|
|
951
|
+
"#{threads} (plus any other threads that use the database)."
|
|
952
|
+
end
|
|
953
|
+
rescue StandardError
|
|
954
|
+
# No database configured, or an adapter that cannot report its pool:
|
|
955
|
+
# nothing to compare.
|
|
956
|
+
nil
|
|
827
957
|
end
|
|
828
958
|
|
|
829
959
|
# SIDEKIQ_MAXMEM_MB is the drop-in name; WURK_MAXMEM_MB the native alias.
|
|
@@ -897,15 +1027,16 @@ module Wurk
|
|
|
897
1027
|
logger
|
|
898
1028
|
end
|
|
899
1029
|
|
|
900
|
-
# `size
|
|
901
|
-
# key the host set via `config.redis = {...}` — url, the split
|
|
902
|
-
# reconnect_attempts, driver, … — flows through
|
|
903
|
-
# `overrides` win over the
|
|
904
|
-
#
|
|
905
|
-
# telemetry dispatcher.
|
|
1030
|
+
# `size` and the pool label are structural (the caller owns them); every
|
|
1031
|
+
# other key the host set via `config.redis = {...}` — url, the split
|
|
1032
|
+
# timeouts, reconnect_attempts, driver, a Sentinel `name:` … — flows through
|
|
1033
|
+
# to RedisPool (see RedisOptions.pool_kwargs). `overrides` win over the
|
|
1034
|
+
# host config (the web pool pins its own pool_timeout this way). Every pool
|
|
1035
|
+
# built here is wired to the redis-error telemetry dispatcher.
|
|
906
1036
|
def build_redis_pool(size:, name:, **overrides)
|
|
907
1037
|
RedisPool.new(size: size, name: name, on_error: method(:dispatch_redis_error),
|
|
908
|
-
|
|
1038
|
+
redis_idle_timeout: @options[:redis_idle_timeout],
|
|
1039
|
+
**RedisOptions.pool_kwargs(@redis_config), **overrides)
|
|
909
1040
|
end
|
|
910
1041
|
|
|
911
1042
|
# Fan a RedisPool retry/give-up event out to the registered handlers. A
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'etc'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
# How many CPUs this process may actually use — the default swarm size.
|
|
7
|
+
#
|
|
8
|
+
# `Etc.nprocessors` reports the host's cores (or the affinity mask), not the
|
|
9
|
+
# container's share of them: a pod limited to 2 CPUs on a 64-core node reads
|
|
10
|
+
# 64, and a swarm that forks one child per core then runs 64 processes on 2
|
|
11
|
+
# CPUs' worth of CFS quota — throttled, with 32x the memory. The quota lives
|
|
12
|
+
# in the cgroup, so it is read from there and the smaller of the two wins.
|
|
13
|
+
#
|
|
14
|
+
# cgroup v2: `cpu.max` is `<quota> <period>` or `max <period>`, read in this
|
|
15
|
+
# process's own cgroup and every ancestor up to the mount root (a limit set on
|
|
16
|
+
# a parent slice binds the child too), smallest wins. cgroup v1:
|
|
17
|
+
# `cpu.cfs_quota_us` (-1 = unlimited) over `cpu.cfs_period_us`, in the cpu
|
|
18
|
+
# controller's mount. Either way `ceil(quota / period)`, floored at 1: a
|
|
19
|
+
# 1.5-CPU quota can keep two processes busy part of the time, and rounding a
|
|
20
|
+
# 0.5-CPU quota down to zero would fork nothing.
|
|
21
|
+
module CpuCount
|
|
22
|
+
Result = Data.define(:count, :source)
|
|
23
|
+
|
|
24
|
+
ROOT = '/sys/fs/cgroup'
|
|
25
|
+
PROC_SELF_CGROUP = '/proc/self/cgroup'
|
|
26
|
+
V1_CONTROLLER_DIRS = %w[cpu cpu,cpuacct cpuacct,cpu].freeze
|
|
27
|
+
|
|
28
|
+
module_function
|
|
29
|
+
|
|
30
|
+
def detect(root: ROOT, proc_cgroup: PROC_SELF_CGROUP, nprocessors: Etc.nprocessors)
|
|
31
|
+
quota = cgroup_v2_quota(root, proc_cgroup)
|
|
32
|
+
source = 'cgroup v2 cpu.max'
|
|
33
|
+
unless quota
|
|
34
|
+
quota = cgroup_v1_quota(root)
|
|
35
|
+
source = 'cgroup v1 cpu.cfs_quota_us'
|
|
36
|
+
end
|
|
37
|
+
return Result.new(nprocessors, 'Etc.nprocessors') if quota.nil? || quota >= nprocessors
|
|
38
|
+
|
|
39
|
+
Result.new(quota, source)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def cgroup_v2_quota(root, proc_cgroup)
|
|
43
|
+
v2_dirs(root, proc_cgroup).filter_map { |dir| v2_limit(File.join(dir, 'cpu.max')) }.min
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# `0::<path>` is the v2 entry. Inside a container with its own cgroup
|
|
47
|
+
# namespace the path is `/` and the limit sits at the mount root; on a host
|
|
48
|
+
# (or without the namespace) it sits under the full path.
|
|
49
|
+
def v2_dirs(root, proc_cgroup)
|
|
50
|
+
line = read(proc_cgroup)&.lines&.find { |l| l.start_with?('0::') }
|
|
51
|
+
path = line ? line.strip.delete_prefix('0::') : '/'
|
|
52
|
+
dirs = []
|
|
53
|
+
loop do
|
|
54
|
+
dirs << File.join(root, path)
|
|
55
|
+
break if path == '/' || path.empty?
|
|
56
|
+
|
|
57
|
+
path = File.dirname(path)
|
|
58
|
+
end
|
|
59
|
+
dirs.uniq
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def v2_limit(file)
|
|
63
|
+
quota, period = read(file)&.split
|
|
64
|
+
return nil if quota.nil? || quota == 'max'
|
|
65
|
+
|
|
66
|
+
ceil_ratio(quota, period)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def cgroup_v1_quota(root)
|
|
70
|
+
V1_CONTROLLER_DIRS.each do |dir|
|
|
71
|
+
quota = read(File.join(root, dir, 'cpu.cfs_quota_us'))
|
|
72
|
+
next unless quota
|
|
73
|
+
|
|
74
|
+
return nil if quota.strip.start_with?('-')
|
|
75
|
+
|
|
76
|
+
return ceil_ratio(quota, read(File.join(root, dir, 'cpu.cfs_period_us')))
|
|
77
|
+
end
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def ceil_ratio(quota, period)
|
|
82
|
+
quota = Integer(quota.to_s.strip, 10)
|
|
83
|
+
period = Integer(period.to_s.strip, 10)
|
|
84
|
+
return nil unless quota.positive? && period.positive?
|
|
85
|
+
|
|
86
|
+
(quota + period - 1) / period
|
|
87
|
+
rescue ArgumentError
|
|
88
|
+
nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def read(path)
|
|
92
|
+
File.read(path)
|
|
93
|
+
rescue SystemCallError, IOError
|
|
94
|
+
nil
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
data/lib/wurk/cron.rb
CHANGED
|
@@ -271,7 +271,11 @@ module Wurk
|
|
|
271
271
|
class Loop
|
|
272
272
|
attr_reader :lid, :schedule, :klass, :options, :tz
|
|
273
273
|
|
|
274
|
-
|
|
274
|
+
# `paused:` is the loop HASH's `paused` field when read back from Redis —
|
|
275
|
+
# the runtime owns it (Web UI pause/unpause, ent §2.4). A code-declared
|
|
276
|
+
# `paused:` option only seeds that field at first registration, so it
|
|
277
|
+
# must not keep the loop paused once the field says otherwise.
|
|
278
|
+
def initialize(schedule:, klass:, options: {}, tz: nil, lid: nil, paused: nil)
|
|
275
279
|
raise ArgumentError, 'klass must be a String' unless klass.is_a?(String) && !klass.empty?
|
|
276
280
|
|
|
277
281
|
@schedule = schedule
|
|
@@ -280,6 +284,7 @@ module Wurk
|
|
|
280
284
|
@tz = tz
|
|
281
285
|
@parser = Parser.new(schedule)
|
|
282
286
|
@lid = lid || Cron.lid(schedule, klass, @options)
|
|
287
|
+
@paused = paused.nil? ? [true, '1', 1].include?(@options['paused']) : paused
|
|
283
288
|
end
|
|
284
289
|
|
|
285
290
|
def parser
|
|
@@ -287,19 +292,51 @@ module Wurk
|
|
|
287
292
|
end
|
|
288
293
|
|
|
289
294
|
def paused?
|
|
290
|
-
@
|
|
295
|
+
@paused
|
|
291
296
|
end
|
|
292
297
|
|
|
298
|
+
# The queue a tick lands in: the loop's own `queue:`, else the worker's
|
|
299
|
+
# `sidekiq_options` queue, as `perform_async` would pick (ent §2.2).
|
|
293
300
|
def queue
|
|
294
|
-
@options['queue'] || 'default'
|
|
301
|
+
@options['queue'] || worker_option('queue') || 'default'
|
|
295
302
|
end
|
|
296
303
|
|
|
304
|
+
# `args:` is the static argument list splatted into perform. A bare value
|
|
305
|
+
# (a Hash most often) is one argument — `Array({ 'a' => 1 })` would turn
|
|
306
|
+
# it into `[['a', 1]]`.
|
|
297
307
|
def args
|
|
298
|
-
|
|
308
|
+
value = @options['args']
|
|
309
|
+
return [] if value.nil?
|
|
310
|
+
|
|
311
|
+
value.is_a?(::Array) ? value : [value]
|
|
299
312
|
end
|
|
300
313
|
|
|
314
|
+
# The loop's `retry:`, else the worker's, else Sidekiq's default `true`.
|
|
301
315
|
def retry_value
|
|
302
|
-
@options.
|
|
316
|
+
return @options['retry'] if @options.key?('retry')
|
|
317
|
+
|
|
318
|
+
worker_retry = worker_option('retry')
|
|
319
|
+
worker_retry.nil? || worker_retry
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
# The payload one tick pushes. Ent §2.2: a tick is `klass.perform_async(*args)`
|
|
323
|
+
# with the loop's `retry` / `queue` laid over the worker's own options, so
|
|
324
|
+
# the resolved class goes to the client — JobUtil#normalize_item merges its
|
|
325
|
+
# `get_sidekiq_options` (unique_for, encrypt, expires_in, …) only for a
|
|
326
|
+
# Class. Only the overrides the loop actually declares are set.
|
|
327
|
+
def job_item
|
|
328
|
+
worker = worker_class
|
|
329
|
+
item = { 'class' => worker.respond_to?(:get_sidekiq_options) ? worker : @klass, 'args' => args }
|
|
330
|
+
item['queue'] = @options['queue'] if @options.key?('queue')
|
|
331
|
+
item['retry'] = @options['retry'] if @options.key?('retry')
|
|
332
|
+
item
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# The worker constant, or nil when it is not loadable in this process.
|
|
336
|
+
def worker_class
|
|
337
|
+
@klass.split('::').inject(::Object) { |mod, name| mod.const_get(name) }
|
|
338
|
+
rescue ::NameError
|
|
339
|
+
nil
|
|
303
340
|
end
|
|
304
341
|
|
|
305
342
|
def history
|
|
@@ -379,13 +416,18 @@ module Wurk
|
|
|
379
416
|
def self.from_redis(lid, hash)
|
|
380
417
|
h = hash.is_a?(Array) ? hash.each_slice(2).to_h : hash
|
|
381
418
|
opts = h['options'] ? JSON.parse(h['options']) : {}
|
|
382
|
-
opts['paused'] = '1' if h['paused'] == '1'
|
|
383
419
|
tz = h['tz'].to_s.empty? ? nil : h['tz']
|
|
384
|
-
new(lid: lid, schedule: h['schedule'], klass: h['klass'], options: opts, tz: tz
|
|
420
|
+
new(lid: lid, schedule: h['schedule'], klass: h['klass'], options: opts, tz: tz,
|
|
421
|
+
paused: h['paused'] == '1')
|
|
385
422
|
end
|
|
386
423
|
|
|
387
424
|
private
|
|
388
425
|
|
|
426
|
+
def worker_option(key)
|
|
427
|
+
worker = worker_class
|
|
428
|
+
worker.get_sidekiq_options[key] if worker.respond_to?(:get_sidekiq_options)
|
|
429
|
+
end
|
|
430
|
+
|
|
389
431
|
def stringify_options(opts)
|
|
390
432
|
opts.transform_keys(&:to_s)
|
|
391
433
|
end
|
|
@@ -448,18 +490,23 @@ module Wurk
|
|
|
448
490
|
@config = config
|
|
449
491
|
end
|
|
450
492
|
|
|
493
|
+
# Loops are read up front and yielded with no connection held, so the
|
|
494
|
+
# caller's own Redis work (the poller's claim + push) never nests a second
|
|
495
|
+
# checkout. A loop whose stored hash no longer parses is skipped with a
|
|
496
|
+
# warning: one corrupt record must not hide every loop after it.
|
|
451
497
|
def each
|
|
452
498
|
return enum_for(:each) unless block_given?
|
|
453
499
|
|
|
454
|
-
redis do |c|
|
|
455
|
-
|
|
456
|
-
lids.each do |lid|
|
|
500
|
+
rows = redis do |c|
|
|
501
|
+
c.call('SMEMBERS', PERIODIC_KEY).filter_map do |lid|
|
|
457
502
|
h = c.call('HGETALL', "#{LOOP_PREFIX}#{lid}")
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
yield Loop.from_redis(lid, h)
|
|
503
|
+
[lid, h] unless h.nil? || h.empty?
|
|
461
504
|
end
|
|
462
505
|
end
|
|
506
|
+
rows.each do |lid, h|
|
|
507
|
+
loop_obj = parse(lid, h)
|
|
508
|
+
yield loop_obj if loop_obj
|
|
509
|
+
end
|
|
463
510
|
end
|
|
464
511
|
|
|
465
512
|
def size
|
|
@@ -478,6 +525,13 @@ module Wurk
|
|
|
478
525
|
|
|
479
526
|
private
|
|
480
527
|
|
|
528
|
+
def parse(lid, hash)
|
|
529
|
+
Loop.from_redis(lid, hash)
|
|
530
|
+
rescue JSON::ParserError, ArgumentError => e
|
|
531
|
+
Wurk.logger&.warn("[cron] skipping unreadable loop lid=#{lid}: #{e.class}: #{e.message}")
|
|
532
|
+
nil
|
|
533
|
+
end
|
|
534
|
+
|
|
481
535
|
def redis(idempotent: false, &)
|
|
482
536
|
@config ? @config.redis(idempotent:, &) : Wurk.redis(idempotent:, &)
|
|
483
537
|
end
|
|
@@ -513,9 +567,9 @@ module Wurk
|
|
|
513
567
|
end
|
|
514
568
|
|
|
515
569
|
# Once-per-minute tick driver. Only the cluster leader iterates the LoopSet
|
|
516
|
-
# and enqueues; non-leaders return early.
|
|
517
|
-
#
|
|
518
|
-
#
|
|
570
|
+
# and enqueues; non-leaders return early. A slot that wall-clock has passed
|
|
571
|
+
# by more than `MISSED_TICK_THRESHOLD` seconds is skipped with a warning,
|
|
572
|
+
# never backfilled.
|
|
519
573
|
class Poller
|
|
520
574
|
include Component
|
|
521
575
|
|
|
@@ -556,10 +610,18 @@ module Wurk
|
|
|
556
610
|
# `dear-leader`): non-leaders return early and never iterate the LoopSet.
|
|
557
611
|
# The Launcher owns the lock's renewal — the poller no longer runs (or
|
|
558
612
|
# expires) its own.
|
|
613
|
+
#
|
|
614
|
+
# Rescued per loop: the LoopSet order is stable, so one loop that raises
|
|
615
|
+
# on every tick (a push its middleware refuses, a class that no longer
|
|
616
|
+
# loads) would otherwise starve every loop after it, forever.
|
|
559
617
|
def tick
|
|
560
618
|
return unless leader?
|
|
561
619
|
|
|
562
|
-
LoopSet.new(@config).each
|
|
620
|
+
LoopSet.new(@config).each do |lp|
|
|
621
|
+
enqueue_if_due(lp)
|
|
622
|
+
rescue StandardError => e
|
|
623
|
+
handle_exception(e, { context: 'cron-poller', lid: lp.lid, klass: lp.klass })
|
|
624
|
+
end
|
|
563
625
|
rescue StandardError => e
|
|
564
626
|
handle_exception(e, { context: 'cron-poller' })
|
|
565
627
|
end
|
|
@@ -568,6 +630,10 @@ module Wurk
|
|
|
568
630
|
# cached read, so a second poller can reach the same due loop for a few
|
|
569
631
|
# seconds after a handover. Losing the CAS means another tick owns this
|
|
570
632
|
# slot — enqueue nothing.
|
|
633
|
+
#
|
|
634
|
+
# A slot older than the missed-tick threshold is not backfilled (ent §2.6:
|
|
635
|
+
# no backfill on restart, missed ticks across a leader gap are dropped):
|
|
636
|
+
# the marks still advance past it, but nothing is pushed.
|
|
571
637
|
def enqueue_if_due(loop_obj)
|
|
572
638
|
return if loop_obj.paused?
|
|
573
639
|
|
|
@@ -577,8 +643,8 @@ module Wurk
|
|
|
577
643
|
|
|
578
644
|
future = loop_obj.next_fire_after(slot, now)
|
|
579
645
|
return unless claim_fire?(loop_obj, mark, now, future)
|
|
646
|
+
return skip_missed(loop_obj, slot, now) if now - slot > missed_threshold
|
|
580
647
|
|
|
581
|
-
warn_missed_tick(loop_obj, slot, now)
|
|
582
648
|
jid = enqueue_claimed!(loop_obj)
|
|
583
649
|
record_history(loop_obj, jid, now)
|
|
584
650
|
jid
|
|
@@ -609,13 +675,18 @@ module Wurk
|
|
|
609
675
|
[next_fire, mark || next_fire.to_s]
|
|
610
676
|
end
|
|
611
677
|
|
|
612
|
-
|
|
613
|
-
|
|
678
|
+
# Never tighter than a tick and a half: a raised tick interval must not
|
|
679
|
+
# turn every on-time fire into a "missed" one.
|
|
680
|
+
def missed_threshold
|
|
681
|
+
[MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
|
|
682
|
+
end
|
|
614
683
|
|
|
684
|
+
def skip_missed(loop_obj, expected, now)
|
|
615
685
|
logger.warn(
|
|
616
686
|
"[cron] missed tick lid=#{loop_obj.lid} klass=#{loop_obj.klass} " \
|
|
617
|
-
"expected_at=#{expected}
|
|
687
|
+
"expected_at=#{expected} now=#{now} drift=#{now - expected}s — skipped, not backfilled"
|
|
618
688
|
)
|
|
689
|
+
nil
|
|
619
690
|
end
|
|
620
691
|
|
|
621
692
|
# Compare-and-swap the fire marks: only the tick still looking at the
|
|
@@ -647,30 +718,22 @@ module Wurk
|
|
|
647
718
|
end
|
|
648
719
|
|
|
649
720
|
def enqueue!(loop_obj)
|
|
650
|
-
aj = active_job_class(loop_obj.
|
|
721
|
+
aj = active_job_class(loop_obj.worker_class)
|
|
651
722
|
return enqueue_active_job(aj, loop_obj) if aj
|
|
652
723
|
|
|
653
|
-
@client.push(
|
|
654
|
-
'class' => loop_obj.klass,
|
|
655
|
-
'args' => loop_obj.args,
|
|
656
|
-
'queue' => loop_obj.queue,
|
|
657
|
-
'retry' => loop_obj.retry_value
|
|
658
|
-
)
|
|
724
|
+
@client.push(loop_obj.job_item)
|
|
659
725
|
end
|
|
660
726
|
|
|
661
|
-
#
|
|
727
|
+
# The loop's class when it is an ActiveJob::Base subclass, or nil when
|
|
662
728
|
# ActiveJob isn't loaded (standalone wurk) or the class isn't one.
|
|
663
729
|
# sidekiq-cron parity: a cron loop targeting an ActiveJob must enqueue
|
|
664
730
|
# through the AJ adapter so the job runs via Sidekiq::ActiveJob::Wrapper
|
|
665
731
|
# with full callbacks/serialization — a bare `client.push('class' => …)`
|
|
666
732
|
# would make the processor call `Klass.new.perform`, skipping all of AJ.
|
|
667
|
-
def active_job_class(
|
|
733
|
+
def active_job_class(const)
|
|
668
734
|
return nil unless defined?(::ActiveJob::Base)
|
|
669
735
|
|
|
670
|
-
const = name.split('::').inject(::Object) { |mod, c| mod.const_get(c) }
|
|
671
736
|
const if const.is_a?(::Class) && const < ::ActiveJob::Base
|
|
672
|
-
rescue ::NameError
|
|
673
|
-
nil
|
|
674
737
|
end
|
|
675
738
|
|
|
676
739
|
# Enqueue via the AJ adapter (→ wurk). Only override the queue when the
|
|
@@ -679,7 +742,7 @@ module Wurk
|
|
|
679
742
|
# doesn't apply here. Returns the wurk jid (provider_job_id) so the fire
|
|
680
743
|
# history records it; nil if a before_enqueue callback halted the push.
|
|
681
744
|
def enqueue_active_job(klass, loop_obj)
|
|
682
|
-
target = loop_obj.queue
|
|
745
|
+
target = loop_obj.options.key?('queue') ? klass.set(queue: loop_obj.options['queue']) : klass
|
|
683
746
|
job = target.perform_later(*loop_obj.args)
|
|
684
747
|
job.provider_job_id if job.respond_to?(:provider_job_id)
|
|
685
748
|
end
|