wurk 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -9
- data/app/controllers/wurk/api_controller.rb +4 -1
- data/app/controllers/wurk/dashboard_controller.rb +11 -6
- data/exe/wurk +7 -2
- data/exe/wurkswarm +1 -2
- data/lib/generators/wurk/install/install_generator.rb +12 -0
- data/lib/sidekiq/web.rb +5 -0
- data/lib/wurk/boot_checks.rb +93 -0
- data/lib/wurk/capsule.rb +33 -8
- data/lib/wurk/cli.rb +204 -30
- data/lib/wurk/client/buffered.rb +240 -26
- data/lib/wurk/client.rb +16 -5
- data/lib/wurk/compat.rb +15 -0
- data/lib/wurk/configuration.rb +114 -16
- data/lib/wurk/cron.rb +141 -16
- data/lib/wurk/debounce.rb +2 -2
- data/lib/wurk/embedded.rb +8 -2
- data/lib/wurk/encryption.rb +20 -5
- data/lib/wurk/error_coalescer.rb +35 -0
- data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
- data/lib/wurk/fetcher/capped.rb +4 -0
- data/lib/wurk/fetcher/many_queues.rb +17 -1
- data/lib/wurk/fetcher/oom_drain.rb +102 -0
- data/lib/wurk/fetcher/orphan_grace.rb +60 -44
- data/lib/wurk/fetcher/private_list_key.rb +1 -16
- data/lib/wurk/fetcher/reaper.rb +169 -127
- data/lib/wurk/fetcher/reliable.rb +26 -4
- data/lib/wurk/fetcher/self_reconcile.rb +175 -0
- data/lib/wurk/fetcher/unit_of_work.rb +32 -8
- data/lib/wurk/health.rb +23 -4
- data/lib/wurk/health_probe.rb +64 -0
- data/lib/wurk/heartbeat.rb +55 -3
- data/lib/wurk/job/options.rb +9 -1
- data/lib/wurk/job.rb +2 -24
- data/lib/wurk/job_record.rb +17 -9
- data/lib/wurk/job_retry.rb +38 -10
- data/lib/wurk/job_set.rb +13 -5
- data/lib/wurk/launcher.rb +134 -52
- data/lib/wurk/leader.rb +71 -11
- data/lib/wurk/limiter/base.rb +17 -1
- data/lib/wurk/lua/debounce.lua +11 -6
- data/lib/wurk/lua/loader.rb +13 -1
- data/lib/wurk/lua.rb +71 -8
- data/lib/wurk/manager.rb +1 -1
- data/lib/wurk/metrics/process_counters.rb +27 -0
- data/lib/wurk/metrics/prometheus.rb +35 -14
- data/lib/wurk/middleware/current_attributes.rb +26 -10
- data/lib/wurk/middleware/poison_pill.rb +1 -2
- data/lib/wurk/preflight/checks.rb +203 -0
- data/lib/wurk/preflight/command.rb +135 -0
- data/lib/wurk/preflight/keyspace.rb +109 -0
- data/lib/wurk/preflight/owners.rb +74 -0
- data/lib/wurk/preflight/payload_sample.rb +108 -0
- data/lib/wurk/preflight/reader.rb +60 -0
- data/lib/wurk/preflight/report.rb +82 -0
- data/lib/wurk/preflight/verify.rb +161 -0
- data/lib/wurk/preflight.rb +40 -0
- data/lib/wurk/process_set.rb +3 -1
- data/lib/wurk/processor.rb +82 -18
- data/lib/wurk/queue_consumers.rb +134 -0
- data/lib/wurk/rails_boot.rb +40 -2
- data/lib/wurk/rails_reloader.rb +51 -0
- data/lib/wurk/railtie.rb +20 -1
- data/lib/wurk/redact.rb +12 -0
- data/lib/wurk/redis_pool.rb +34 -9
- data/lib/wurk/scheduled.rb +142 -67
- data/lib/wurk/sentry/death_handler.rb +29 -0
- data/lib/wurk/sentry/error_handler.rb +52 -10
- data/lib/wurk/sentry/middleware.rb +8 -1
- data/lib/wurk/sentry/retry_policy.rb +12 -3
- data/lib/wurk/sentry.rb +42 -5
- data/lib/wurk/sorted_entry.rb +38 -13
- data/lib/wurk/swarm/child_boot.rb +44 -31
- data/lib/wurk/swarm/crash_watch.rb +65 -0
- data/lib/wurk/swarm/test_mode_guard.rb +32 -0
- data/lib/wurk/swarm.rb +79 -11
- data/lib/wurk/testing.rb +74 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/config.rb +8 -4
- data/lib/wurk/web/extension.rb +41 -62
- data/lib/wurk/web/helpers.rb +133 -0
- data/lib/wurk/web/paginator.rb +49 -0
- data/lib/wurk/web/rack_app.rb +30 -6
- data/lib/wurk/work_set.rb +6 -0
- data/lib/wurk/worker/setter.rb +11 -6
- data/lib/wurk/worker.rb +17 -1
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
- data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
- data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +53 -30
- data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
- data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
- data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
data/lib/wurk/configuration.rb
CHANGED
|
@@ -9,6 +9,8 @@ require_relative 'topology'
|
|
|
9
9
|
require_relative 'cpu_count'
|
|
10
10
|
require_relative 'redis_options'
|
|
11
11
|
require_relative 'keys'
|
|
12
|
+
require_relative 'error_coalescer'
|
|
13
|
+
require_relative 'metrics/process_counters'
|
|
12
14
|
|
|
13
15
|
module Wurk
|
|
14
16
|
# Owns runtime knobs (concurrency, queues, timeouts, lifecycle events,
|
|
@@ -98,7 +100,11 @@ module Wurk
|
|
|
98
100
|
# the fetch path reads this once at boot, so an app that caps nothing
|
|
99
101
|
# resolves to "no capped queues" before the first fetch and never asks
|
|
100
102
|
# again.
|
|
101
|
-
global_concurrency: {}
|
|
103
|
+
global_concurrency: {},
|
|
104
|
+
# See #swarm_crash_exit_after=. Long enough to outlast a slow rollout or a
|
|
105
|
+
# dependency coming back, short enough that a bad release does not sit
|
|
106
|
+
# looking Running for an afternoon.
|
|
107
|
+
swarm_crash_exit_after: 600
|
|
102
108
|
}.freeze
|
|
103
109
|
|
|
104
110
|
# :fork fires only inside swarm children, after fork + internal AR/Redis
|
|
@@ -151,7 +157,7 @@ module Wurk
|
|
|
151
157
|
Wurk::Metrics::Statsd.reset!
|
|
152
158
|
end
|
|
153
159
|
|
|
154
|
-
def initialize(options = {})
|
|
160
|
+
def initialize(options = {}) # rubocop:disable Metrics/AbcSize
|
|
155
161
|
@options = deep_dup_defaults.merge(options)
|
|
156
162
|
# Through the same door `global_concurrency=` uses, so a cap passed to
|
|
157
163
|
# `Configuration.new` is validated rather than trusted, and the default
|
|
@@ -162,14 +168,16 @@ module Wurk
|
|
|
162
168
|
@options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
|
|
163
169
|
@capsules = {}
|
|
164
170
|
@directory = {}
|
|
165
|
-
@client_chain = Middleware::Chain.new
|
|
166
|
-
@server_chain = Middleware::Chain.new(on_registration: method(:warn_on_late_server_registration))
|
|
171
|
+
@client_chain = Middleware::Chain.new(self)
|
|
172
|
+
@server_chain = Middleware::Chain.new(self, on_registration: method(:warn_on_late_server_registration))
|
|
167
173
|
@redis_config = { url: ENV.fetch('REDIS_URL', 'redis://localhost:6379/0') }
|
|
168
174
|
@web_redis_pool = nil
|
|
169
175
|
@logger = nil
|
|
170
176
|
@thread_priority = DEFAULT_THREAD_PRIORITY
|
|
171
177
|
@telemetry_installed = false
|
|
172
178
|
@frozen = false
|
|
179
|
+
@deferred_server_blocks = []
|
|
180
|
+
@fetch_error_coalescer = ErrorCoalescer.new
|
|
173
181
|
end
|
|
174
182
|
|
|
175
183
|
# --- Hash-like options access -----------------------------------------
|
|
@@ -192,6 +200,7 @@ module Wurk
|
|
|
192
200
|
end
|
|
193
201
|
|
|
194
202
|
def dig(*keys) = @options.dig(*keys)
|
|
203
|
+
def to_json(*) = Wurk.dump_json(@options)
|
|
195
204
|
|
|
196
205
|
# --- Default capsule shortcuts ---------------------------------------
|
|
197
206
|
|
|
@@ -250,6 +259,7 @@ module Wurk
|
|
|
250
259
|
def redis=(hash)
|
|
251
260
|
guard_frozen!
|
|
252
261
|
RedisOptions.validate!(hash)
|
|
262
|
+
warn_on_late_redis_assignment
|
|
253
263
|
@redis_config = @redis_config.merge(hash.transform_keys(&:to_sym))
|
|
254
264
|
end
|
|
255
265
|
|
|
@@ -263,6 +273,11 @@ module Wurk
|
|
|
263
273
|
default_capsule.redis_pool
|
|
264
274
|
end
|
|
265
275
|
|
|
276
|
+
# Parsed INFO, without the pool's own counters RedisPool#info merges in.
|
|
277
|
+
def redis_info
|
|
278
|
+
redis_pool.info.except('size', 'available')
|
|
279
|
+
end
|
|
280
|
+
|
|
266
281
|
# Disconnect and drop every capsule's cached pools (main + fetch) plus the
|
|
267
282
|
# web pool. Used by Wurk::Swarm so the parent never leaks sockets into forks
|
|
268
283
|
# and each child can build fresh ones.
|
|
@@ -272,6 +287,12 @@ module Wurk
|
|
|
272
287
|
@web_redis_pool = nil
|
|
273
288
|
end
|
|
274
289
|
|
|
290
|
+
# Non-terminal counterpart, for a fork in a parent whose threads are live.
|
|
291
|
+
def reload_redis_pools!
|
|
292
|
+
@capsules.each_value(&:reload_redis_pools!)
|
|
293
|
+
@web_redis_pool&.reload!
|
|
294
|
+
end
|
|
295
|
+
|
|
275
296
|
def new_redis_pool(size, name = 'custom')
|
|
276
297
|
build_redis_pool(size: size, name: name)
|
|
277
298
|
end
|
|
@@ -403,6 +424,20 @@ module Wurk
|
|
|
403
424
|
@options[:swarm_heartbeat_timeout]
|
|
404
425
|
end
|
|
405
426
|
|
|
427
|
+
# Seconds every swarm slot may spend crash-looping before the standalone
|
|
428
|
+
# runner reports it and exits non-zero. nil or false keeps respawning forever.
|
|
429
|
+
def swarm_crash_exit_after=(seconds)
|
|
430
|
+
unless seconds.nil? || seconds == false || (seconds.is_a?(Numeric) && seconds.positive?)
|
|
431
|
+
raise ArgumentError, "swarm_crash_exit_after must be nil, false, or a positive number (got #{seconds.inspect})"
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
@options[:swarm_crash_exit_after] = seconds || nil
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
def swarm_crash_exit_after
|
|
438
|
+
@options[:swarm_crash_exit_after]
|
|
439
|
+
end
|
|
440
|
+
|
|
406
441
|
# --- Reliability (Sidekiq Pro drop-in toggles) -----------------------
|
|
407
442
|
|
|
408
443
|
# Sidekiq Pro's opt-in toggle for reliable fetch. Already the default in
|
|
@@ -411,9 +446,8 @@ module Wurk
|
|
|
411
446
|
# callback. It exists so a Pro initializer drops in unchanged instead of
|
|
412
447
|
# raising NoMethodError.
|
|
413
448
|
#
|
|
414
|
-
# NOTE: `reliable_scheduler!` below is NOT a no-op
|
|
415
|
-
#
|
|
416
|
-
# (Wurk::Scheduled::Enq); only the toggle swaps in the atomic promoter.
|
|
449
|
+
# NOTE: `reliable_scheduler!` below is NOT a no-op: it swaps in the atomic
|
|
450
|
+
# promoter, much faster but skipping client middleware on promotion.
|
|
417
451
|
#
|
|
418
452
|
# The optional block is Pro's recovery callback: `|jobstr, pill|`, fired
|
|
419
453
|
# once per orphan recovery (`pill` nil) and once on a poison kill (`pill`
|
|
@@ -426,8 +460,9 @@ module Wurk
|
|
|
426
460
|
|
|
427
461
|
# Pro reliable scheduler (§4): promote due jobs from retry/schedule onto
|
|
428
462
|
# their target queue in a single atomic Lua (ZRANGEBYSCORE+ZREM+LPUSH),
|
|
429
|
-
#
|
|
430
|
-
# pluggable `scheduled_enq` for the atomic promoter;
|
|
463
|
+
# much faster than the default poller but skipping client middleware on
|
|
464
|
+
# promotion. Swaps the pluggable `scheduled_enq` for the atomic promoter;
|
|
465
|
+
# idempotent.
|
|
431
466
|
def reliable_scheduler!(*)
|
|
432
467
|
self[:scheduled_enq] = Wurk::Scheduled::ReliableEnq
|
|
433
468
|
nil
|
|
@@ -765,6 +800,7 @@ module Wurk
|
|
|
765
800
|
attr_writer :logger
|
|
766
801
|
|
|
767
802
|
def handle_exception(ex, ctx = {})
|
|
803
|
+
ctx = redact_jobstr(ctx)
|
|
768
804
|
if error_handlers.empty?
|
|
769
805
|
logger.error("#{redacted_context(ctx)} #{ex.class}: #{ex.message}")
|
|
770
806
|
else
|
|
@@ -776,11 +812,26 @@ module Wurk
|
|
|
776
812
|
end
|
|
777
813
|
end
|
|
778
814
|
|
|
815
|
+
# What the fetch loop reports through. It sleeps 1s and retries on every
|
|
816
|
+
# thread of every worker, so an outage would otherwise call each handler once
|
|
817
|
+
# per thread per second. The count lands in the metric, the handlers see the
|
|
818
|
+
# first of each exception class and one per window after that.
|
|
819
|
+
def report_fetch_error(exception)
|
|
820
|
+
Metrics::ProcessCounters.increment(:fetch_errors)
|
|
821
|
+
suppressed = @fetch_error_coalescer.admit(exception.class)
|
|
822
|
+
return unless suppressed
|
|
823
|
+
|
|
824
|
+
context = { context: 'Error fetching job' }
|
|
825
|
+
context[:suppressed] = suppressed if suppressed.positive?
|
|
826
|
+
handle_exception(exception, context)
|
|
827
|
+
end
|
|
828
|
+
|
|
779
829
|
# The error context with the job's args passed through the `redact_args`
|
|
780
830
|
# hook, for the two reporters that print or tag the whole context: the
|
|
781
831
|
# fallback log line and the default ERROR_HANDLER (whose Wurk::Context a
|
|
782
832
|
# JSON log formatter emits). Host-registered error handlers still receive
|
|
783
|
-
# the real
|
|
833
|
+
# the real `job` — they are the host's own code (`jobstr` is another matter:
|
|
834
|
+
# see #redact_jobstr).
|
|
784
835
|
def redacted_context(ctx)
|
|
785
836
|
hook = @options[:redact_args]
|
|
786
837
|
job = ctx.is_a?(Hash) ? ctx[:job] : nil
|
|
@@ -790,16 +841,36 @@ module Wurk
|
|
|
790
841
|
ctx.merge(job: job.merge('args' => Wurk::Redact.args(job, hook)))
|
|
791
842
|
end
|
|
792
843
|
|
|
844
|
+
# The raw payload a job that failed to parse or dispatch is reported with. Unlike
|
|
845
|
+
# `job`, this reaches host handlers scrubbed too: it is a string nobody expects
|
|
846
|
+
# to carry the args, so handlers forward it to log aggregators and trackers as
|
|
847
|
+
# is. What it cannot be parsed from cannot be scrubbed, so that is withheld.
|
|
848
|
+
def redact_jobstr(ctx)
|
|
849
|
+
hook = @options[:redact_args]
|
|
850
|
+
raw = ctx.is_a?(Hash) ? ctx[:jobstr] : nil
|
|
851
|
+
return ctx unless hook && raw
|
|
852
|
+
|
|
853
|
+
require_relative 'redact'
|
|
854
|
+
ctx.merge(jobstr: Wurk::Redact.jobstr(raw, hook))
|
|
855
|
+
end
|
|
856
|
+
private :redact_jobstr
|
|
857
|
+
|
|
793
858
|
# --- Configure blocks (Sidekiq.configure_server / _client) -----------
|
|
794
859
|
|
|
795
|
-
# The gate is evaluated per call:
|
|
796
|
-
#
|
|
797
|
-
# the block's value. A skip reports itself once per gate per instance —
|
|
860
|
+
# The gate is evaluated per call: an allowed block yields the configuration
|
|
861
|
+
# and returns the block's value. A skip reports itself once per gate per instance —
|
|
798
862
|
# the flags below decide only whether the skip is reported again, and
|
|
799
863
|
# they live outside @options so a frozen configuration can still warn.
|
|
800
|
-
|
|
801
|
-
|
|
864
|
+
#
|
|
865
|
+
# A skipped block is kept for #enter_server_mode. Gems call this at load
|
|
866
|
+
# time, and under the railtie that is Bundler.require, before the
|
|
867
|
+
# `wurk.server_mode` initializer: dropping it left sidekiq-cron, sentry-sidekiq
|
|
868
|
+
# and the like silently absent from a process that does run workers.
|
|
869
|
+
def configure_server(&block)
|
|
870
|
+
return unless block
|
|
802
871
|
return yield self if server?
|
|
872
|
+
|
|
873
|
+
@deferred_server_blocks << block
|
|
803
874
|
return if @configure_server_skip_warned
|
|
804
875
|
|
|
805
876
|
@configure_server_skip_warned = true
|
|
@@ -807,6 +878,11 @@ module Wurk
|
|
|
807
878
|
nil
|
|
808
879
|
end
|
|
809
880
|
|
|
881
|
+
def enter_server_mode
|
|
882
|
+
self[:server] = true
|
|
883
|
+
@deferred_server_blocks.shift.call(self) until @deferred_server_blocks.empty?
|
|
884
|
+
end
|
|
885
|
+
|
|
810
886
|
# A web process that also boots the swarm (Wurk::RailsBoot) is flagged
|
|
811
887
|
# server, yet every enqueue it makes from a request is client work, so
|
|
812
888
|
# `client_in_server` lets its configure_client blocks run too. Same
|
|
@@ -896,6 +972,16 @@ module Wurk
|
|
|
896
972
|
self
|
|
897
973
|
end
|
|
898
974
|
|
|
975
|
+
# A swarm child inherits the parent's prepare_for_fork! freeze, but its copy
|
|
976
|
+
# is its own, and Launcher#run freezes it again. In between it is what a
|
|
977
|
+
# non-swarm process is: open to `:fork` / `:startup` hooks and to gems that
|
|
978
|
+
# prepend Launcher#initialize and write an option, as sidekiq-cron does.
|
|
979
|
+
def reopen_in_forked_child
|
|
980
|
+
@options = @options.dup
|
|
981
|
+
@frozen = false
|
|
982
|
+
self
|
|
983
|
+
end
|
|
984
|
+
|
|
899
985
|
def frozen?
|
|
900
986
|
@frozen
|
|
901
987
|
end
|
|
@@ -906,6 +992,18 @@ module Wurk
|
|
|
906
992
|
|
|
907
993
|
private
|
|
908
994
|
|
|
995
|
+
# Pools read the hash once, when built, so the later assignment silently does
|
|
996
|
+
# nothing to this process's connections until something rebuilds them.
|
|
997
|
+
def warn_on_late_redis_assignment
|
|
998
|
+
return unless @capsules.each_value.any?(&:redis_pools_built?)
|
|
999
|
+
|
|
1000
|
+
logger.warn do
|
|
1001
|
+
'config.redis= called after a Redis pool was already built; existing connections keep the old ' \
|
|
1002
|
+
'settings until the pools are rebuilt (a swarm child rebuilds them, a single process does not). ' \
|
|
1003
|
+
'Set config.redis before anything uses Redis.'
|
|
1004
|
+
end
|
|
1005
|
+
end
|
|
1006
|
+
|
|
909
1007
|
def warn_on_late_server_registration(klass)
|
|
910
1008
|
affected = @capsules.each_value.filter_map do |capsule|
|
|
911
1009
|
capsule.name if capsule.prepared? && capsule.server_middleware_missing?(klass)
|
|
@@ -1023,7 +1121,7 @@ module Wurk
|
|
|
1023
1121
|
threads = total_concurrency
|
|
1024
1122
|
return if size >= threads
|
|
1025
1123
|
|
|
1026
|
-
logger.
|
|
1124
|
+
logger.error do
|
|
1027
1125
|
"ActiveRecord connection pool size (#{size}) is smaller than this process's job concurrency " \
|
|
1028
1126
|
"(#{threads}); jobs that use the database will wait for connections and may raise " \
|
|
1029
1127
|
'ActiveRecord::ConnectionTimeoutError. Raise `pool:` in config/database.yml to at least ' \
|
data/lib/wurk/cron.rb
CHANGED
|
@@ -450,6 +450,21 @@ module Wurk
|
|
|
450
450
|
@pruned_klasses = {}
|
|
451
451
|
end
|
|
452
452
|
|
|
453
|
+
# How late a slot may fire, in seconds, instead of being skipped, for every
|
|
454
|
+
# loop of this cluster's pollers. Default MISSED_TICK_THRESHOLD. A process-wide
|
|
455
|
+
# setting and not a loop option, which would enter the loop's identity.
|
|
456
|
+
#
|
|
457
|
+
# Coerced here, because the poller rescues per loop: a value it could not
|
|
458
|
+
# compare would stop every loop on every tick with nothing failing at boot.
|
|
459
|
+
def grace=(seconds)
|
|
460
|
+
grace = Float(seconds, exception: false)
|
|
461
|
+
unless grace&.finite? && grace.positive?
|
|
462
|
+
raise ArgumentError, "periodic grace must be a positive, finite number of seconds: #{seconds.inspect}"
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
(@config || Wurk.configuration)[:cron_grace] = grace
|
|
466
|
+
end
|
|
467
|
+
|
|
453
468
|
def register(cron, klass, **opts)
|
|
454
469
|
klass_name = klass.is_a?(String) ? klass : klass.to_s
|
|
455
470
|
tz = opts.delete(:tz) || @tz
|
|
@@ -461,8 +476,53 @@ module Wurk
|
|
|
461
476
|
loop_obj
|
|
462
477
|
end
|
|
463
478
|
|
|
479
|
+
# Registers a sidekiq-cron shaped schedule: `{ name => { cron:, class:,
|
|
480
|
+
# queue:, args:, retry:, tz:, status: } }`, or a list of those carrying
|
|
481
|
+
# `name:`. Other keys (`description`, ...) are ignored. Raises ArgumentError
|
|
482
|
+
# on an entry with no cron or class, and on anything but a 5-field crontab.
|
|
483
|
+
def load_hash(schedule)
|
|
484
|
+
entries = schedule
|
|
485
|
+
entries = schedule.map { |name, entry| { 'name' => name }.merge(entry.to_h) } if schedule.is_a?(::Hash)
|
|
486
|
+
entries.map { |entry| register_entry(entry.to_h.transform_keys(&:to_s)) }
|
|
487
|
+
end
|
|
488
|
+
|
|
489
|
+
# The same shape from a YAML file (ERB allowed), like a `config/schedule.yml`.
|
|
490
|
+
def load_file(path)
|
|
491
|
+
require 'erb'
|
|
492
|
+
require 'yaml'
|
|
493
|
+
load_hash(::YAML.safe_load(::ERB.new(::File.read(path)).result, aliases: true) || {})
|
|
494
|
+
end
|
|
495
|
+
|
|
496
|
+
# Call after the last registration to make this block the whole truth:
|
|
497
|
+
# every other loop in the cluster is deleted. Off unless asked because a
|
|
498
|
+
# process registering only part of the schedule would delete the rest, and
|
|
499
|
+
# because during a rolling deploy old and new code then disagree.
|
|
500
|
+
def authoritative!
|
|
501
|
+
return self unless @prune
|
|
502
|
+
|
|
503
|
+
keep = @loops.map(&:lid)
|
|
504
|
+
LoopSet.new(@config).each { |existing| Cron.unregister(existing.lid) unless keep.include?(existing.lid) }
|
|
505
|
+
self
|
|
506
|
+
end
|
|
507
|
+
|
|
464
508
|
private
|
|
465
509
|
|
|
510
|
+
def register_entry(entry)
|
|
511
|
+
cron = entry['cron'].to_s.strip
|
|
512
|
+
klass = (entry['class'] || entry['klass']).to_s
|
|
513
|
+
if cron.empty? || klass.empty?
|
|
514
|
+
raise ArgumentError, "schedule entry #{entry['name'].inspect} needs `cron` and `class`"
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
register(cron, klass, **entry_options(entry))
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
def entry_options(entry)
|
|
521
|
+
opts = { label: entry['name']&.to_s, queue: entry['queue'], args: entry['args'], retry: entry['retry'],
|
|
522
|
+
tz: entry['tz'], paused: entry['status'] == 'disabled' ? true : nil }
|
|
523
|
+
opts.compact
|
|
524
|
+
end
|
|
525
|
+
|
|
466
526
|
# Once per class per boot: everything else registered for that class is
|
|
467
527
|
# a leftover from a previous deploy (the lid hashes schedule+options, so
|
|
468
528
|
# an edited `register` line yields a new loop and orphans the old one).
|
|
@@ -566,10 +626,12 @@ module Wurk
|
|
|
566
626
|
end
|
|
567
627
|
end
|
|
568
628
|
|
|
569
|
-
# Once-per-minute tick driver
|
|
570
|
-
#
|
|
571
|
-
#
|
|
572
|
-
#
|
|
629
|
+
# Once-per-minute tick driver, plus one tick the moment this process wins the
|
|
630
|
+
# lock: at a tick's phase, a leader that lives under a minute would never tick.
|
|
631
|
+
# Only the cluster leader iterates the LoopSet and enqueues; non-leaders return
|
|
632
|
+
# early. A slot that wall-clock has passed by more than `MISSED_TICK_THRESHOLD`
|
|
633
|
+
# seconds is skipped with a warning, never backfilled, and the occurrence due
|
|
634
|
+
# now, if any, fires in its place.
|
|
573
635
|
class Poller
|
|
574
636
|
include Component
|
|
575
637
|
|
|
@@ -580,17 +642,22 @@ module Wurk
|
|
|
580
642
|
# Operators never need to touch this; integration tests shrink it so a
|
|
581
643
|
# due loop fires within the test window instead of waiting a full minute.
|
|
582
644
|
@tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
|
|
583
|
-
@
|
|
645
|
+
@done = false
|
|
646
|
+
@woken = false
|
|
647
|
+
@mutex = ::Mutex.new
|
|
648
|
+
@sleeper = ::ConditionVariable.new
|
|
649
|
+
@on_leader = method(:wake_for_leadership)
|
|
584
650
|
end
|
|
585
651
|
|
|
586
|
-
#
|
|
587
|
-
#
|
|
588
|
-
#
|
|
652
|
+
# The loop waits one interval before its first tick unless this process
|
|
653
|
+
# wins the lock first: let a process that never leads, or exits at once,
|
|
654
|
+
# exit without ticking at all.
|
|
589
655
|
def start
|
|
590
656
|
return @thread if @thread
|
|
591
657
|
|
|
592
|
-
@
|
|
593
|
-
|
|
658
|
+
@mutex.synchronize { @done = false }
|
|
659
|
+
leader_hooks << @on_leader
|
|
660
|
+
@thread = safe_thread('cron-poller') { run_loop }
|
|
594
661
|
end
|
|
595
662
|
|
|
596
663
|
# Blocks until the thread is really gone: the launcher releases the
|
|
@@ -599,10 +666,14 @@ module Wurk
|
|
|
599
666
|
#
|
|
600
667
|
# Cleared only on a confirmed join (Thread#join returns nil on timeout):
|
|
601
668
|
# a wedged thread must stay tracked so #start's guard returns it instead
|
|
602
|
-
# of
|
|
603
|
-
#
|
|
669
|
+
# of clearing @done, which would un-terminate the loop it is still inside
|
|
670
|
+
# and leave two tick threads double-enqueuing the same loops.
|
|
604
671
|
def terminate
|
|
605
|
-
@
|
|
672
|
+
leader_hooks.delete(@on_leader)
|
|
673
|
+
@mutex.synchronize do
|
|
674
|
+
@done = true
|
|
675
|
+
@sleeper.signal
|
|
676
|
+
end
|
|
606
677
|
@thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
|
|
607
678
|
end
|
|
608
679
|
|
|
@@ -634,6 +705,10 @@ module Wurk
|
|
|
634
705
|
# A slot older than the missed-tick threshold is not backfilled (ent §2.6:
|
|
635
706
|
# no backfill on restart, missed ticks across a leader gap are dropped):
|
|
636
707
|
# the marks still advance past it, but nothing is pushed.
|
|
708
|
+
#
|
|
709
|
+
# One claim, one push at most, whichever slot it pushes for: the occurrence
|
|
710
|
+
# due now stands in for a stale one under the stale one's mark, so a poller
|
|
711
|
+
# that read the same mark loses the CAS for both.
|
|
637
712
|
def enqueue_if_due(loop_obj)
|
|
638
713
|
return if loop_obj.paused?
|
|
639
714
|
|
|
@@ -641,9 +716,12 @@ module Wurk
|
|
|
641
716
|
slot, mark = due_slot(loop_obj, now)
|
|
642
717
|
return if slot.nil?
|
|
643
718
|
|
|
644
|
-
|
|
719
|
+
firing = slot_to_fire(loop_obj, slot, now)
|
|
720
|
+
future = loop_obj.next_fire_after(firing || slot, now)
|
|
645
721
|
return unless claim_fire?(loop_obj, mark, now, future)
|
|
646
|
-
|
|
722
|
+
|
|
723
|
+
skip_missed(loop_obj, slot, now) unless firing == slot
|
|
724
|
+
return if firing.nil?
|
|
647
725
|
|
|
648
726
|
jid = enqueue_claimed!(loop_obj)
|
|
649
727
|
record_history(loop_obj, jid, now)
|
|
@@ -663,6 +741,53 @@ module Wurk
|
|
|
663
741
|
|
|
664
742
|
private
|
|
665
743
|
|
|
744
|
+
def run_loop
|
|
745
|
+
wait_for_tick
|
|
746
|
+
until done?
|
|
747
|
+
tick
|
|
748
|
+
wait_for_tick
|
|
749
|
+
end
|
|
750
|
+
end
|
|
751
|
+
|
|
752
|
+
def wait_for_tick
|
|
753
|
+
@mutex.synchronize do
|
|
754
|
+
@sleeper.wait(@mutex, @tick_interval) unless @done || @woken
|
|
755
|
+
@woken = false
|
|
756
|
+
end
|
|
757
|
+
end
|
|
758
|
+
|
|
759
|
+
def done?
|
|
760
|
+
@mutex.synchronize { @done }
|
|
761
|
+
end
|
|
762
|
+
|
|
763
|
+
# Runs on the campaign thread. The gate's cached answer predates the win.
|
|
764
|
+
def wake_for_leadership
|
|
765
|
+
@leader_checked_at = nil
|
|
766
|
+
@mutex.synchronize do
|
|
767
|
+
@woken = true
|
|
768
|
+
@sleeper.signal
|
|
769
|
+
end
|
|
770
|
+
end
|
|
771
|
+
|
|
772
|
+
def leader_hooks
|
|
773
|
+
@config[:lifecycle_events].fetch(:leader)
|
|
774
|
+
end
|
|
775
|
+
|
|
776
|
+
def slot_to_fire(loop_obj, slot, now)
|
|
777
|
+
now - slot > missed_threshold ? latest_due(loop_obj, now) : slot
|
|
778
|
+
end
|
|
779
|
+
|
|
780
|
+
# The newest occurrence still inside the window a late slot may fire in.
|
|
781
|
+
def latest_due(loop_obj, now)
|
|
782
|
+
latest = nil
|
|
783
|
+
occurrence = loop_obj.next_fire_at(now - missed_threshold - 1)
|
|
784
|
+
while occurrence && occurrence <= now
|
|
785
|
+
latest = occurrence
|
|
786
|
+
occurrence = loop_obj.next_fire_at(occurrence)
|
|
787
|
+
end
|
|
788
|
+
latest
|
|
789
|
+
end
|
|
790
|
+
|
|
666
791
|
# The slot this tick would fire and the token that claims it, or nil when
|
|
667
792
|
# the loop has no resolvable occurrence or none is due yet. The token is
|
|
668
793
|
# the stored `nf` when there is one, otherwise the slot we derived — see
|
|
@@ -678,7 +803,7 @@ module Wurk
|
|
|
678
803
|
# Never tighter than a tick and a half: a raised tick interval must not
|
|
679
804
|
# turn every on-time fire into a "missed" one.
|
|
680
805
|
def missed_threshold
|
|
681
|
-
[MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
|
|
806
|
+
[@config[:cron_grace] || MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
|
|
682
807
|
end
|
|
683
808
|
|
|
684
809
|
def skip_missed(loop_obj, expected, now)
|
data/lib/wurk/debounce.rb
CHANGED
|
@@ -81,8 +81,8 @@ module Wurk
|
|
|
81
81
|
reject_shorter_cap!(seconds, cap)
|
|
82
82
|
|
|
83
83
|
argv = [seconds, cap, JobUtil.scheduled_member(job), GRACE]
|
|
84
|
-
# Replay-safe: a re-run finds the member it just wrote
|
|
85
|
-
#
|
|
84
|
+
# Replay-safe: a re-run finds the member it just wrote and ZADDs it
|
|
85
|
+
# over itself, so a pool retry after a lost reply converges on one
|
|
86
86
|
# entry rather than a duplicate — the whole point of doing this in a
|
|
87
87
|
# script rather than a pipeline.
|
|
88
88
|
#
|
data/lib/wurk/embedded.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative 'component'
|
|
4
4
|
require_relative 'launcher'
|
|
5
|
+
require_relative 'swarm/test_mode_guard'
|
|
5
6
|
|
|
6
7
|
module Wurk
|
|
7
8
|
# Embeds Wurk inside an arbitrary Ruby process (Puma, rake task, custom
|
|
@@ -34,13 +35,13 @@ module Wurk
|
|
|
34
35
|
# the worker threads have time to spin up before the host goes back to
|
|
35
36
|
# serving traffic. Mirrors Sidekiq::Embedded#run.
|
|
36
37
|
def run
|
|
38
|
+
Swarm::TestModeGuard.enforce!(logger)
|
|
37
39
|
housekeeping
|
|
38
40
|
fire_event(:startup, reverse: false, reraise: true)
|
|
39
41
|
@launcher = build_launcher
|
|
40
42
|
@launcher.run
|
|
41
43
|
sleep 0.2
|
|
42
|
-
|
|
43
|
-
logger.debug { Thread.list.map(&:name).to_s }
|
|
44
|
+
log_running
|
|
44
45
|
rescue StandardError
|
|
45
46
|
# The host is left believing Wurk never started, so anything the launcher
|
|
46
47
|
# did get up would keep fetching, beating and campaigning for the leader
|
|
@@ -62,6 +63,11 @@ module Wurk
|
|
|
62
63
|
|
|
63
64
|
private
|
|
64
65
|
|
|
66
|
+
def log_running
|
|
67
|
+
logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
|
|
68
|
+
logger.debug { Thread.list.map(&:name).to_s }
|
|
69
|
+
end
|
|
70
|
+
|
|
65
71
|
# Guarded so the caller sees why the boot failed, not why the rollback did.
|
|
66
72
|
def roll_back_partial_boot
|
|
67
73
|
stop
|
data/lib/wurk/encryption.rb
CHANGED
|
@@ -83,6 +83,12 @@ module Wurk
|
|
|
83
83
|
# is what callers see as the dead record's `error_class`.
|
|
84
84
|
class DecryptionError < Error; end
|
|
85
85
|
|
|
86
|
+
# The job says `encrypt: true` but its last argument is not a Wurk envelope:
|
|
87
|
+
# another implementation's ciphertext, or a producer that never enabled
|
|
88
|
+
# crypto. Neither can be told from a plain value, and either way `perform`
|
|
89
|
+
# must not take it for the decrypted argument.
|
|
90
|
+
class NotSealedError < Error; end
|
|
91
|
+
|
|
86
92
|
class << self
|
|
87
93
|
attr_reader :active_version
|
|
88
94
|
|
|
@@ -228,7 +234,10 @@ module Wurk
|
|
|
228
234
|
'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
229
235
|
)
|
|
230
236
|
Wurk::Metrics::Statsd.increment('jobs.encryption_error', tags: ["worker:#{job['class']}"])
|
|
231
|
-
Wurk
|
|
237
|
+
Wurk.logger.error { "#{DEAD_REASON}: #{job['class']} jid=#{job['jid']} sent to the dead set: #{cause.message}" }
|
|
238
|
+
# No trim: a misconfigured producer or a lost key can fail a whole backlog at
|
|
239
|
+
# once, and the cap would destroy jobs that are recoverable once it is fixed.
|
|
240
|
+
Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause, trim: false)
|
|
232
241
|
nil
|
|
233
242
|
end
|
|
234
243
|
|
|
@@ -318,14 +327,19 @@ module Wurk
|
|
|
318
327
|
# with every outer middleware and the processor; leaving plaintext in it
|
|
319
328
|
# let any of them write the secret back to Redis (InterruptHandler's
|
|
320
329
|
# re-push, limiter dead-routing) or hand it to an error reporter. A decrypt
|
|
321
|
-
# failure (missing/rotated key, bad tag
|
|
322
|
-
# so rather than let it bubble into the 25× retry pipeline we route the
|
|
330
|
+
# failure (missing/rotated key, bad tag, a last argument that is no envelope)
|
|
331
|
+
# is terminal and non-retryable, so rather than let it bubble into the 25× retry pipeline we route the
|
|
323
332
|
# job straight to the dead set tagged `encryption_error` and ACK it via
|
|
324
333
|
# JobRetry::Skip — see Wurk::Encryption.route_to_dead. Plaintext args
|
|
325
334
|
# remain visible for triage per §4.6.
|
|
326
335
|
class ServerMiddleware
|
|
327
336
|
include Wurk::Middleware::ServerMiddleware
|
|
328
337
|
|
|
338
|
+
# Never quotes the argument: it may be a secret in the clear.
|
|
339
|
+
NOT_SEALED_MESSAGE = 'job has encrypt: true but its last argument is not a Wurk envelope, so it was not run. ' \
|
|
340
|
+
'A producer without crypto enabled pushed it, or another implementation encrypted it. ' \
|
|
341
|
+
'A retry re-seals the argument as it stands: retry only a job known to hold plaintext'
|
|
342
|
+
|
|
329
343
|
def call(_worker, job, _queue)
|
|
330
344
|
return yield unless Wurk::Encryption.enabled? && job['encrypt']
|
|
331
345
|
|
|
@@ -342,7 +356,8 @@ module Wurk
|
|
|
342
356
|
|
|
343
357
|
def decrypt_last_arg!(job)
|
|
344
358
|
args = job['args']
|
|
345
|
-
return unless args.is_a?(::Array) && !args.empty?
|
|
359
|
+
return unless args.is_a?(::Array) && !args.empty?
|
|
360
|
+
raise NotSealedError, NOT_SEALED_MESSAGE unless Wurk::Encryption.envelope?(args.last)
|
|
346
361
|
|
|
347
362
|
job['args'] = args[0..-2] + [Wurk::Encryption.decrypt(args.last)]
|
|
348
363
|
rescue Wurk::Encryption::Error,
|
|
@@ -350,7 +365,7 @@ module Wurk
|
|
|
350
365
|
::ArgumentError,
|
|
351
366
|
::TypeError,
|
|
352
367
|
::JSON::ParserError => e
|
|
353
|
-
# Wurk::Encryption::Error covers KeyMissingError / DecryptionError.
|
|
368
|
+
# Wurk::Encryption::Error covers KeyMissingError / DecryptionError / NotSealedError.
|
|
354
369
|
# OpenSSL::Cipher::CipherError → tampered ciphertext / bad key (tag mismatch).
|
|
355
370
|
# ArgumentError → Base64.strict_decode64 / Integer() on a malformed envelope.
|
|
356
371
|
# TypeError → Integer(nil) on a missing 'v' field.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
# Lets the first report of a key through, then one per window, and says how
|
|
5
|
+
# many it swallowed in between. Per process: every forked child speaks for
|
|
6
|
+
# itself, so a fleet still reports at worst children x keys per window.
|
|
7
|
+
class ErrorCoalescer
|
|
8
|
+
DEFAULT_WINDOW = 60
|
|
9
|
+
|
|
10
|
+
def initialize(window: DEFAULT_WINDOW, clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
|
|
11
|
+
@window = window
|
|
12
|
+
@clock = clock
|
|
13
|
+
@lock = ::Mutex.new
|
|
14
|
+
@entries = {}
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# The number of reports suppressed since the last one when this one should
|
|
18
|
+
# go out (0 for a first report); nil when this one should be swallowed.
|
|
19
|
+
def admit(key)
|
|
20
|
+
@lock.synchronize do
|
|
21
|
+
now = @clock.call
|
|
22
|
+
entry = (@entries[key] ||= [nil, 0])
|
|
23
|
+
if entry[0] && now - entry[0] < @window
|
|
24
|
+
entry[1] += 1
|
|
25
|
+
next nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
suppressed = entry[1]
|
|
29
|
+
entry[0] = now
|
|
30
|
+
entry[1] = 0
|
|
31
|
+
suppressed
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|