wurk 1.11.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/exe/wurk +4 -2
- data/lib/generators/wurk/install/templates/wurk.rb +5 -3
- data/lib/wurk/boot_checks.rb +130 -5
- data/lib/wurk/cli.rb +72 -10
- data/lib/wurk/configuration.rb +12 -3
- data/lib/wurk/cron.rb +66 -10
- data/lib/wurk/fetcher/capped.rb +8 -3
- data/lib/wurk/fetcher/oom_drain.rb +0 -5
- data/lib/wurk/fetcher/orphan_reclaim.rb +71 -0
- data/lib/wurk/fetcher/reaper.rb +18 -56
- data/lib/wurk/fetcher/self_reconcile.rb +8 -14
- data/lib/wurk/job_util.rb +18 -2
- data/lib/wurk/launcher.rb +7 -4
- data/lib/wurk/lua/reclaim_orphan.lua +31 -0
- data/lib/wurk/lua.rb +5 -20
- data/lib/wurk/middleware/poison_pill.rb +31 -47
- data/lib/wurk/rails_boot.rb +40 -11
- data/lib/wurk/unique.rb +12 -2
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/views/_job_info.erb +16 -0
- data/lib/wurk/web/views/_paging.erb +11 -0
- data/lib/wurk.rb +4 -0
- data/vendor/assets/dashboard/assets/{ArgsValue-CmaXArv8.js → ArgsValue-BJol9S0s.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-nc5yePZW.js → BatchDetail-D_zGQ9Sd.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-DgEcyOZ_.js → Batches-Dt_xC660.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-CUZncvCy.js → Busy-CWRthsmc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CIZbaaZs.js → Cron-qeGr2YAm.js} +1 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +1 -0
- data/vendor/assets/dashboard/assets/{Extension-Doy4bJoA.js → Extension-DONXVLdA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-C4bO26_d.js → FilterBox-BbB6gXHp.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-Dh3SJdFh.js → FlowDetail-CrmnVRp0.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-C5SMRuhq.js → Flows-g4GC8KmE.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cg6sAsJB.js → JobDetailModal-C86k_1wj.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-BIxwDOJM.js → Limiters-zyO-iMlK.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-Qp9q0nF8.js → Metrics-Ckq5fNBv.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-DwjPxFjU.js → NotFound-C8NnyjlY.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-7dabYVe3.js → Profiles-DRl-FXPA.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +1 -0
- data/vendor/assets/dashboard/assets/{Search-BAsVE2qw.js → Search-LaEpj2fM.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-DZs138BV.js → charts-D_PQLoyc.js} +1 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +141 -0
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-DxkLZCIm.js → useResetPageOnEmpty-BiwhHqgh.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-B0kQFNE8.js → utils-CnWZzW2L.js} +1 -1
- data/vendor/assets/dashboard/index.html +1 -1
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +29 -25
- data/vendor/assets/dashboard/assets/Dashboard-BWmfAook.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-CGKsTZHq.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CNX8ZtTH.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-ClVBj0nc.js +0 -1
- data/vendor/assets/dashboard/assets/index-CMzPFxNE.js +0 -141
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: efda965ecab6b73c86eddfd4f0b8acc5f0942bcba46d933aa60e922aae453c1f
|
|
4
|
+
data.tar.gz: 404c2f3b9055edeafedd0c1a98f1933799c89830d0c2f70d08fb4018c183fa4f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1d466f7764855b501fe06a0e3a2586bf4908b6bd65274e74fa5739b77255613801bc6625fdee52f4a4771cc4c1ea49e6f9b171fa364e8fd5fda95997b788da0c
|
|
7
|
+
data.tar.gz: 1333a3f95fc34da5a126caae9ff041a404bb80c221f425177e9034bf16bb4a16abc11fe9ad8aa1a8e18470f8066cc8aba0d9c45de2dea6ee46bbb38795b83a66
|
data/README.md
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
|
|
23
23
|
Wurk is wire-compatible with Sidekiq — same Redis keys, same job JSON, same Ruby DSL. Swap one line in your `Gemfile` and your existing jobs, batches, limiters, cron entries, and live Redis data keep working untouched. The Pro and Enterprise feature sets ship in the same free gem, with no license check and no tiers.
|
|
24
24
|
|
|
25
|
-
**In production:** Wurk runs background work at partner deployments, across many servers, on the fork-based swarm described below. It is not a preview.
|
|
25
|
+
**In production:** Wurk runs background work at partner deployments, across many servers, on the fork-based swarm described below. It is not a preview. Teams running billions of jobs a month on Wurk in production recommend it. Those volumes are user-reported; no production throughput numbers are published yet, and what has been measured is in [docs/benchmarks.md](docs/benchmarks.md).
|
|
26
26
|
|
|
27
27
|
**At scale:** Wurk is built for fleets, not just for one box. Kubernetes `/live` + `/ready` probes are a config line, not a sidecar; a bearer-scoped [HTTP API](docs/api-http.md) lets non-Ruby services enqueue and inspect; [OpenTelemetry](docs/telemetry.md) traces propagate client → server; per-queue [global concurrency caps](docs/rate-limiting.md) hold cluster-wide limits; and monitoring is the dashboard you already mount — live SSE, charts, no separate stack to run. See [Wurk extras](#wurk-extras).
|
|
28
28
|
|
data/exe/wurk
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
# Usage: `exe/wurk -C config/wurk.yml` — worker: one process, thread pool
|
|
6
6
|
# `exe/wurk api --port 7433` — machine HTTP API, runs no jobs
|
|
7
7
|
# `exe/wurk preflight` / `verify` — read-only cutover checks, run no jobs
|
|
8
|
+
# `exe/wurk boot-check -r .` — boot the app as wurkswarm would, then exit (CI)
|
|
8
9
|
|
|
9
10
|
$stdout.sync = true
|
|
10
11
|
$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
|
|
@@ -20,8 +21,9 @@ begin
|
|
|
20
21
|
cli = Wurk::CLI.instance
|
|
21
22
|
cli.parse
|
|
22
23
|
case cli.command
|
|
23
|
-
when 'api'
|
|
24
|
-
|
|
24
|
+
when 'api' then cli.run_api
|
|
25
|
+
when 'boot-check' then exit(cli.run_boot_check)
|
|
26
|
+
else cli.run
|
|
25
27
|
end
|
|
26
28
|
rescue StandardError => e
|
|
27
29
|
raise e if $DEBUG
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
# Wurk configuration. Generated by `bin/rails g wurk:install`.
|
|
4
|
-
#
|
|
5
|
-
#
|
|
4
|
+
# Every option: https://github.com/developerz-ai/wurk/blob/main/docs/configuration.md
|
|
5
|
+
# Coming from Sidekiq? https://github.com/developerz-ai/wurk/blob/main/docs/migrate-from-sidekiq.md
|
|
6
6
|
|
|
7
7
|
Wurk.configure_server do |config|
|
|
8
8
|
# config.redis = { url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0") }
|
|
@@ -19,7 +19,9 @@ Wurk.configure_server do |config|
|
|
|
19
19
|
# By default Wurk forks one worker process per usable CPU (cgroup quota aware) (set WURK_COUNT, or
|
|
20
20
|
# SIDEKIQ_COUNT, to override). For full control declare a topology — `flat`
|
|
21
21
|
# spawns N identical forks; use `slot`s for dedicated queues. Total in-flight
|
|
22
|
-
# jobs = forks × concurrency.
|
|
22
|
+
# jobs = forks × concurrency. Topology and sizing:
|
|
23
|
+
# https://github.com/developerz-ai/wurk/blob/main/docs/configuration.md#topology
|
|
24
|
+
# https://github.com/developerz-ai/wurk/blob/main/docs/deployment.md#capacity-and-sizing
|
|
23
25
|
# config.topology = Wurk::Topology.flat(count: 2, queues: %w[critical default low], concurrency: 10)
|
|
24
26
|
end
|
|
25
27
|
|
data/lib/wurk/boot_checks.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative 'keys'
|
|
4
4
|
require_relative 'queue_consumers'
|
|
5
|
+
require_relative 'capsule'
|
|
5
6
|
|
|
6
7
|
module Wurk
|
|
7
8
|
# What a standalone runner says once at boot, after the app loaded. Log only,
|
|
@@ -13,13 +14,20 @@ module Wurk
|
|
|
13
14
|
MAX_LISTED = 20
|
|
14
15
|
SUBCLASS = ::Module.instance_method(:<)
|
|
15
16
|
NAME = ::Module.instance_method(:name)
|
|
17
|
+
UNIQUE_OPTIONS = %w[unique_for unique_until].freeze
|
|
18
|
+
STRICT_ENV = 'WURK_STRICT'
|
|
16
19
|
|
|
17
20
|
module_function
|
|
18
21
|
|
|
22
|
+
# `info` is nil when there is no Redis to ask (`wurk boot-check` without
|
|
23
|
+
# `--redis`): only the checks that read the loaded app run then.
|
|
19
24
|
def run(config, info)
|
|
20
|
-
|
|
21
|
-
|
|
25
|
+
if info
|
|
26
|
+
log_backend(config.logger, info)
|
|
27
|
+
warn_on_foreign_processes(config)
|
|
28
|
+
end
|
|
22
29
|
warn_on_ignored_locks(config.logger)
|
|
30
|
+
warn_on_ignored_unique(config.logger)
|
|
23
31
|
rescue StandardError => e
|
|
24
32
|
config.logger.warn { "wurk boot checks skipped: #{e.class}: #{e.message}" }
|
|
25
33
|
end
|
|
@@ -73,18 +81,135 @@ module Wurk
|
|
|
73
81
|
end
|
|
74
82
|
end
|
|
75
83
|
|
|
84
|
+
# Only the middleware `unique!` installs reads these options, so without it
|
|
85
|
+
# every push enqueues. Turning it on for the app is a human call, not a
|
|
86
|
+
# boot check's: this only says so.
|
|
87
|
+
def warn_on_ignored_unique(logger)
|
|
88
|
+
return if Wurk::Unique.enabled?
|
|
89
|
+
|
|
90
|
+
classes = job_classes_setting(UNIQUE_OPTIONS)
|
|
91
|
+
return if classes.empty?
|
|
92
|
+
|
|
93
|
+
logger.warn do
|
|
94
|
+
'These jobs set `unique_for:` or `unique_until:` but unique jobs are not enabled, so the options do ' \
|
|
95
|
+
"nothing and duplicates enqueue: #{listing(classes)}. Enable them once at boot, in an initializer: " \
|
|
96
|
+
'`Sidekiq::Enterprise.unique!`.'
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def locked_job_classes = job_classes_setting(%w[lock])
|
|
101
|
+
|
|
102
|
+
# Workers, and ActiveJob classes that take `sidekiq_options` (Wurk::Job::Options).
|
|
76
103
|
# A walk over every class meets classes that redefine these with other arguments
|
|
77
104
|
# (Faker::Travel::Airport.name takes required keywords), and one such class must
|
|
78
105
|
# not cost the app this check.
|
|
79
|
-
def
|
|
106
|
+
def job_classes_setting(options)
|
|
80
107
|
::ObjectSpace.each_object(::Class).filter_map do |klass|
|
|
81
|
-
next unless SUBCLASS.bind_call(klass, Wurk::Worker)
|
|
108
|
+
next unless SUBCLASS.bind_call(klass, Wurk::Worker) || SUBCLASS.bind_call(klass, Wurk::Job::Options)
|
|
82
109
|
|
|
83
110
|
name = NAME.bind_call(klass)
|
|
84
|
-
name if name && klass.get_sidekiq_options.
|
|
111
|
+
name if name && klass.get_sidekiq_options.values_at(*options).any?
|
|
85
112
|
end.sort
|
|
86
113
|
end
|
|
87
114
|
|
|
115
|
+
# The real gem next to Wurk loads both: `Sidekiq::*` is then partly theirs,
|
|
116
|
+
# partly ours. Gems that depend on `sidekiq` pull it in unless the shim gem,
|
|
117
|
+
# which depends on Wurk instead, satisfies the dependency. Runs as Wurk loads.
|
|
118
|
+
def flag_real_sidekiq_gem(logger, specs: ::Gem.loaded_specs, strict: ENV[STRICT_ENV] == '1')
|
|
119
|
+
message = real_sidekiq_message(specs, logger)
|
|
120
|
+
return unless message
|
|
121
|
+
raise Wurk::Error, message if strict
|
|
122
|
+
|
|
123
|
+
logger.error { "#{message} Set #{STRICT_ENV}=1 to refuse to boot instead." }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def real_sidekiq_message(specs, logger)
|
|
127
|
+
real = specs['sidekiq']
|
|
128
|
+
return if real.nil? || real.dependencies.any? { |dep| dep.name == 'wurk' }
|
|
129
|
+
|
|
130
|
+
"The bundle holds the real sidekiq #{real.version} gem, so it loads next to Wurk and the process runs " \
|
|
131
|
+
"half on each. Pulled in by: #{sidekiq_dependents(specs)}. Replace it with the shim in the Gemfile: " \
|
|
132
|
+
"gem 'sidekiq', github: 'developerz-ai/wurk', tag: 'v#{Wurk::VERSION}', " \
|
|
133
|
+
"glob: 'ecosystem/sidekiq-shim/*.gemspec'."
|
|
134
|
+
rescue StandardError => e
|
|
135
|
+
logger.warn { "wurk: could not check the bundle for the real sidekiq gem: #{e.class}: #{e.message}" }
|
|
136
|
+
nil
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def sidekiq_dependents(specs)
|
|
140
|
+
names = specs.each_value.select { |spec| spec.dependencies.any? { |dep| dep.name == 'sidekiq' } }.map(&:name)
|
|
141
|
+
names.empty? ? 'the Gemfile itself' : listing(names.sort)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# The role line of a process that forks the topology's children and runs no
|
|
145
|
+
# jobs itself. Its children each run the threads of one slot.
|
|
146
|
+
def log_swarm_role(config, role, ar_pool: active_record_pool)
|
|
147
|
+
slots = config.topology.slots.map do |slot|
|
|
148
|
+
["#{slot.count} #{slot.count == 1 ? 'process' : 'processes'}", slot.concurrency]
|
|
149
|
+
end
|
|
150
|
+
log_role(config, role, slots, per_process: slots.map(&:last).max, ar_pool: ar_pool)
|
|
151
|
+
rescue StandardError => e
|
|
152
|
+
role_skipped(config.logger, e)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# The role line of a process that runs every capsule's threads itself.
|
|
156
|
+
def log_process_role(config, role, ar_pool: active_record_pool)
|
|
157
|
+
capsules = config.capsules.values
|
|
158
|
+
slots = capsules.map { |cap| [capsules.size > 1 ? "capsule #{cap.name}" : '1 process', cap.concurrency] }
|
|
159
|
+
log_role(config, role, slots, per_process: slots.sum(&:last), ar_pool: ar_pool)
|
|
160
|
+
rescue StandardError => e
|
|
161
|
+
role_skipped(config.logger, e)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def role_skipped(logger, error)
|
|
165
|
+
logger.warn { "wurk role line skipped: #{error.class}: #{error.message}" }
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# One line per process: what it is, and how big. `per_process` is the most
|
|
169
|
+
# job threads one process runs, which is what its Active Record pool covers.
|
|
170
|
+
def log_role(config, role, slots, per_process:, ar_pool:)
|
|
171
|
+
sizes = slots.map do |label, threads|
|
|
172
|
+
"#{label} × #{threads} threads (redis pool #{redis_pool_size(config, threads)}, fetch #{threads})"
|
|
173
|
+
end
|
|
174
|
+
config.logger.info { "wurk: #{role}; #{sizes.join(' + ')}#{"; AR pool #{ar_pool}" if ar_pool}" }
|
|
175
|
+
warn_on_small_ar_pool(config.logger, ar_pool, per_process)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# DEBUG, not INFO: Wurk logs to stdout, and every `rails runner`, rake task
|
|
179
|
+
# and `wurk preflight --app` boots the app; their stdout is someone's output.
|
|
180
|
+
def log_client_only(logger, reason)
|
|
181
|
+
logger.debug { "wurk: client only (#{reason}); this process enqueues and runs no jobs" }
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Capsule#main_pool_size, without building a pool: a swarm parent describes
|
|
185
|
+
# slots whose pools only its children build.
|
|
186
|
+
def redis_pool_size(config, threads)
|
|
187
|
+
config.redis_config[:size] || [threads + Capsule::POOL_HEADROOM, Capsule::MIN_POOL_SIZE].max
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Every job thread that touches the database holds a connection; with fewer
|
|
191
|
+
# in the pool the rest wait, then fail with ActiveRecord::ConnectionTimeoutError.
|
|
192
|
+
def warn_on_small_ar_pool(logger, ar_pool, threads)
|
|
193
|
+
return unless ar_pool && ar_pool < threads
|
|
194
|
+
|
|
195
|
+
logger.warn do
|
|
196
|
+
"The Active Record pool holds #{ar_pool} connections but this process runs up to #{threads} job threads, " \
|
|
197
|
+
'so jobs that use the database wait for a connection and then fail on a timeout. Set `pool:` in ' \
|
|
198
|
+
"config/database.yml to at least #{threads} (plus the web server's threads when jobs run in the web process)."
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# The pool size the app configured, read without opening a connection; nil
|
|
203
|
+
# without Active Record or a configured database. Rails 8.1 renamed `pool`.
|
|
204
|
+
def active_record_pool
|
|
205
|
+
return unless defined?(::ActiveRecord::Base)
|
|
206
|
+
|
|
207
|
+
db = ::ActiveRecord::Base.connection_db_config
|
|
208
|
+
db.respond_to?(:max_connections) ? db.max_connections : db.pool
|
|
209
|
+
rescue ::ActiveRecord::ActiveRecordError
|
|
210
|
+
nil
|
|
211
|
+
end
|
|
212
|
+
|
|
88
213
|
def listing(names)
|
|
89
214
|
extra = names.size - MAX_LISTED
|
|
90
215
|
names.first(MAX_LISTED).join(', ') + (extra.positive? ? " and #{extra} more" : '')
|
data/lib/wurk/cli.rb
CHANGED
|
@@ -32,12 +32,13 @@ module Wurk
|
|
|
32
32
|
# Subcommands. Bare `wurk` runs the worker — the historical shape, and the
|
|
33
33
|
# only one the swarm binaries accept — so a subcommand is never anything
|
|
34
34
|
# but the first argument, and every argument after it is that command's.
|
|
35
|
-
COMMANDS = %w[api health].freeze
|
|
35
|
+
COMMANDS = %w[api boot-check health].freeze
|
|
36
36
|
# Set only across the one re-exec of #load_rails_framework, and removed on arrival.
|
|
37
37
|
PRELOAD_FIRST_ENV = 'WURK_BOOT_PRELOAD_FIRST'
|
|
38
38
|
# `bundle exec` loads the binstub in-process, so argv0 is Bundler's executable, and
|
|
39
39
|
# `ruby -e "load Gem.bin_path(...)"` has `-e` there: the running program can't name it.
|
|
40
40
|
SWARM_EXECUTABLE = ::File.expand_path('../../exe/wurkswarm', __dir__)
|
|
41
|
+
WURK_EXECUTABLE = ::File.expand_path('../../exe/wurk', __dir__)
|
|
41
42
|
|
|
42
43
|
# A refusal this file raises itself, with nothing a trace would add.
|
|
43
44
|
class StartupError < ArgumentError; end
|
|
@@ -110,6 +111,10 @@ module Wurk
|
|
|
110
111
|
['-s', '--server NAME', :api_server, 'Rack handler to serve with (default: the first installed)']
|
|
111
112
|
].freeze
|
|
112
113
|
|
|
114
|
+
BOOT_CHECK_OPTION_FLAGS = [
|
|
115
|
+
['-R', '--redis', :boot_check_redis, 'Also read Redis: version, eviction policy, live peers (read-only)']
|
|
116
|
+
].freeze
|
|
117
|
+
|
|
113
118
|
HEALTH_OPTION_FLAGS = [
|
|
114
119
|
['-w', '--window SECONDS', :health_window, 'A heartbeat older than this does not count (default 60)', :to_i],
|
|
115
120
|
['-m', '--min COUNT', :health_min, 'Processes on this host that must be beating (default 1)', :to_i],
|
|
@@ -118,6 +123,7 @@ module Wurk
|
|
|
118
123
|
].freeze
|
|
119
124
|
|
|
120
125
|
HELP_COMMANDS = "\nCommands:\n " \
|
|
126
|
+
"boot-check Boot the app the way wurkswarm does, then exit; no fork, no fetch (CI)\n " \
|
|
121
127
|
"health Exit 0 if this host's workers are heartbeating\n " \
|
|
122
128
|
"preflight Read a live Redis: what will carry over to Wurk (read-only)\n " \
|
|
123
129
|
"verify Post-cutover check of a live fleet (read-only)\n\nOptions:"
|
|
@@ -209,6 +215,7 @@ module Wurk
|
|
|
209
215
|
validate_redis!
|
|
210
216
|
validate_pool_sizes!
|
|
211
217
|
BootChecks.run(@config, redis_info)
|
|
218
|
+
BootChecks.log_process_role(@config, 'wurk')
|
|
212
219
|
@config[:identity] = identity
|
|
213
220
|
# Force lazy server-middleware chain so worker threads don't race
|
|
214
221
|
# against each other constructing it. Spec: Sidekiq::CLI line 104.
|
|
@@ -245,15 +252,11 @@ module Wurk
|
|
|
245
252
|
# Only the children read it. Rejected here, a typo fails this boot instead of
|
|
246
253
|
# crash-looping the fleet until the crash-loop exit.
|
|
247
254
|
Health.options_from_env
|
|
248
|
-
if boot_app
|
|
249
|
-
load_rails_framework
|
|
250
|
-
preload_bundler_groups
|
|
251
|
-
boot_application
|
|
252
|
-
eager_load_application
|
|
253
|
-
end
|
|
255
|
+
boot_swarm_application if boot_app
|
|
254
256
|
validate_redis!
|
|
255
257
|
validate_pool_sizes!
|
|
256
258
|
BootChecks.run(@config, redis_info)
|
|
259
|
+
log_swarm_role
|
|
257
260
|
@config[:identity] = identity
|
|
258
261
|
warm_up_process if warmup
|
|
259
262
|
@swarm = Wurk::Swarm.new(topology: @config.topology, config: @config, exit_on_crash_loop: true,
|
|
@@ -270,6 +273,27 @@ module Wurk
|
|
|
270
273
|
exit(1) if @swarm.crash_looped?
|
|
271
274
|
end
|
|
272
275
|
|
|
276
|
+
# `wurk boot-check` — #run_swarm's boot up to the fork: the same require
|
|
277
|
+
# order, config file and checks, plus what a worker reaches only later (every
|
|
278
|
+
# constant, the routes), so a CI step finds what would crash-loop a deploy.
|
|
279
|
+
# Forks nothing, fetches nothing, fires no lifecycle event, and reads Redis
|
|
280
|
+
# only with --redis. Returns the exit code; a boot that raises is the binary's
|
|
281
|
+
# failure report.
|
|
282
|
+
def run_boot_check(boot_app: true)
|
|
283
|
+
enter_server_mode
|
|
284
|
+
Health.options_from_env
|
|
285
|
+
if boot_app
|
|
286
|
+
boot_swarm_application
|
|
287
|
+
eager_load_rails
|
|
288
|
+
draw_routes
|
|
289
|
+
end
|
|
290
|
+
validate_pool_sizes!
|
|
291
|
+
BootChecks.run(@config, boot_check_redis_info)
|
|
292
|
+
log_swarm_role
|
|
293
|
+
logger.info { "wurk boot-check: #{@config[:require]} booted as wurkswarm would" }
|
|
294
|
+
0
|
|
295
|
+
end
|
|
296
|
+
|
|
273
297
|
# `wurk health` — see Wurk::HealthProbe. Boots nothing; returns the exit code.
|
|
274
298
|
def run_health
|
|
275
299
|
require_relative 'health_probe'
|
|
@@ -434,10 +458,26 @@ module Wurk
|
|
|
434
458
|
|
|
435
459
|
def validate_pool_sizes!
|
|
436
460
|
@config.capsules.each_pair do |name, cap|
|
|
437
|
-
|
|
461
|
+
size = cap.redis_pool.size
|
|
462
|
+
next if size >= cap.concurrency
|
|
463
|
+
|
|
464
|
+
raise StartupError, "the Redis pool for capsule #{name} holds #{size} connections, fewer than its " \
|
|
465
|
+
"concurrency of #{cap.concurrency}. Set `config.redis = { size: #{cap.concurrency} }` " \
|
|
466
|
+
'or more, or drop `size:` and Wurk sizes the pool itself (concurrency + 5).'
|
|
438
467
|
end
|
|
439
468
|
end
|
|
440
469
|
|
|
470
|
+
def boot_check_redis_info
|
|
471
|
+
return unless @config[:boot_check_redis]
|
|
472
|
+
|
|
473
|
+
validate_redis!
|
|
474
|
+
redis_info
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
def log_swarm_role
|
|
478
|
+
BootChecks.log_swarm_role(@config, @command ? "#{@command} (as wurkswarm parent)" : 'wurkswarm parent')
|
|
479
|
+
end
|
|
480
|
+
|
|
441
481
|
# --- options + config-file --------------------------------------------
|
|
442
482
|
|
|
443
483
|
# Pulled off before OptionParser sees argv, so the banner and the flag
|
|
@@ -448,6 +488,20 @@ module Wurk
|
|
|
448
488
|
COMMANDS.include?(args.first) ? args.shift : nil
|
|
449
489
|
end
|
|
450
490
|
|
|
491
|
+
def boot_swarm_application
|
|
492
|
+
load_rails_framework
|
|
493
|
+
preload_bundler_groups
|
|
494
|
+
boot_application
|
|
495
|
+
eager_load_application
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
# Rails may draw routes only on the first request; a `mount` of a constant
|
|
499
|
+
# only the web process loads must fail here, not on that request.
|
|
500
|
+
def draw_routes
|
|
501
|
+
app = rails_application
|
|
502
|
+
app.reload_routes! if app.respond_to?(:reload_routes!)
|
|
503
|
+
end
|
|
504
|
+
|
|
451
505
|
def parse_health_options(args)
|
|
452
506
|
@health_options = {}
|
|
453
507
|
parser = option_parser(@health_options)
|
|
@@ -547,6 +601,7 @@ module Wurk
|
|
|
547
601
|
o.separator help_intro if help_intro
|
|
548
602
|
define_value_flags(o, opts, @command == 'health' ? HEALTH_OPTION_FLAGS : OPTION_FLAGS)
|
|
549
603
|
define_value_flags(o, opts, API_OPTION_FLAGS) if @command == 'api'
|
|
604
|
+
define_value_flags(o, opts, BOOT_CHECK_OPTION_FLAGS) if @command == 'boot-check'
|
|
550
605
|
define_meta_flags(o, opts)
|
|
551
606
|
end
|
|
552
607
|
end
|
|
@@ -627,7 +682,7 @@ module Wurk
|
|
|
627
682
|
end
|
|
628
683
|
|
|
629
684
|
def restart_state
|
|
630
|
-
{ env: ::ENV.to_h, argv:
|
|
685
|
+
{ env: ::ENV.to_h, argv: [restart_executable, *::ARGV], dir: ::Dir.pwd }
|
|
631
686
|
end
|
|
632
687
|
|
|
633
688
|
def note_preload_first(first_error)
|
|
@@ -644,11 +699,14 @@ module Wurk
|
|
|
644
699
|
env = state[:env].merge(PRELOAD_FIRST_ENV => error.message.lines.first.to_s.strip[0, 300])
|
|
645
700
|
env['BUNDLE_GEMFILE'] ||= ::Bundler.default_gemfile.to_s if defined?(::Bundler)
|
|
646
701
|
ruby = [::RbConfig.ruby, *('-rbundler/setup' if defined?(::Bundler))]
|
|
647
|
-
::Process.exec(env, *ruby,
|
|
702
|
+
::Process.exec(env, *ruby, *state[:argv], unsetenv_others: true, chdir: state[:dir])
|
|
648
703
|
rescue ::SystemCallError
|
|
649
704
|
raise error
|
|
650
705
|
end
|
|
651
706
|
|
|
707
|
+
# The binary that was run: only `wurk boot-check` reaches this with a command.
|
|
708
|
+
def restart_executable = @command ? WURK_EXECUTABLE : SWARM_EXECUTABLE
|
|
709
|
+
|
|
652
710
|
# The app root for a `-r` that is the app directory or its environment file.
|
|
653
711
|
def rails_app_root(require_path)
|
|
654
712
|
path = ::File.expand_path(require_path.to_s)
|
|
@@ -688,6 +746,10 @@ module Wurk
|
|
|
688
746
|
def eager_load_application
|
|
689
747
|
return unless preload_app?
|
|
690
748
|
|
|
749
|
+
eager_load_rails
|
|
750
|
+
end
|
|
751
|
+
|
|
752
|
+
def eager_load_rails
|
|
691
753
|
app = rails_application
|
|
692
754
|
app.eager_load! if app.respond_to?(:eager_load!)
|
|
693
755
|
end
|
data/lib/wurk/configuration.rb
CHANGED
|
@@ -452,7 +452,7 @@ module Wurk
|
|
|
452
452
|
# The optional block is Pro's recovery callback: `|jobstr, pill|`, fired
|
|
453
453
|
# once per orphan recovery (`pill` nil) and once on a poison kill (`pill`
|
|
454
454
|
# responds to .jid/.klass/.count/.queue). The reaper drives it via
|
|
455
|
-
# Wurk::Middleware::PoisonPill.
|
|
455
|
+
# Wurk::Middleware::PoisonPill.recovered!. Spec: docs/target/sidekiq-pro.md §3.1.
|
|
456
456
|
def super_fetch!(*, &block)
|
|
457
457
|
@super_fetch_callback = block if block
|
|
458
458
|
nil
|
|
@@ -473,7 +473,8 @@ module Wurk
|
|
|
473
473
|
# Yields a Wurk::Cron::Manager so the host app can register periodic
|
|
474
474
|
# jobs at boot. Manager state is shared per-process so multiple
|
|
475
475
|
# `config.periodic` blocks accumulate (matches Sidekiq Ent §2.1). This is
|
|
476
|
-
# the native replacement for the `sidekiq-cron` gem.
|
|
476
|
+
# the native replacement for the `sidekiq-cron` gem. Nothing reaches
|
|
477
|
+
# Redis until the server boots (#apply_periodic!).
|
|
477
478
|
#
|
|
478
479
|
# @example Register cron jobs at boot
|
|
479
480
|
# Wurk.configure_server do |config|
|
|
@@ -488,11 +489,19 @@ module Wurk
|
|
|
488
489
|
# Spec: docs/target/sidekiq-ent.md §2.
|
|
489
490
|
def periodic
|
|
490
491
|
require_relative 'cron'
|
|
491
|
-
@periodic_manager ||= Wurk::Cron::Manager.new(self)
|
|
492
|
+
@periodic_manager ||= Wurk::Cron::Manager.new(self, deferred: true)
|
|
492
493
|
yield @periodic_manager if block_given?
|
|
493
494
|
@periodic_manager
|
|
494
495
|
end
|
|
495
496
|
|
|
497
|
+
# Writes the `config.periodic` registrations to Redis. Launcher#boot calls
|
|
498
|
+
# it, so the loops are registered by a booting server, against the Redis
|
|
499
|
+
# settings the app ended up with.
|
|
500
|
+
def apply_periodic!
|
|
501
|
+
@periodic_manager&.apply!
|
|
502
|
+
nil
|
|
503
|
+
end
|
|
504
|
+
|
|
496
505
|
# --- Historical metrics snapshotter (Ent §5) -------------------------
|
|
497
506
|
|
|
498
507
|
HISTORY_DEFAULT_INTERVAL = 30
|
data/lib/wurk/cron.rb
CHANGED
|
@@ -25,7 +25,8 @@ module Wurk
|
|
|
25
25
|
# so a re-registration of the same loop is idempotent.
|
|
26
26
|
# * `Cron::Manager` — registration DSL. `mgr.register(cron, klass, **opts)`
|
|
27
27
|
# with `tz=` mass-setter. Writes to Redis (`periodic` SET + `loops:{lid}`
|
|
28
|
-
# HASH) and prunes superseded loops for the classes it registers
|
|
28
|
+
# HASH) and prunes superseded loops for the classes it registers; the
|
|
29
|
+
# `config.periodic` one defers both until the server boots.
|
|
29
30
|
# * `Cron::LoopSet` — Enumerable view (`each`/`size`/`fetch(lid)`).
|
|
30
31
|
# * `Cron::ConfigTester` — boot-time validator. Verifies cron syntax and
|
|
31
32
|
# that every worker class constant resolves.
|
|
@@ -46,6 +47,10 @@ module Wurk
|
|
|
46
47
|
DEFAULT_TICK_SECONDS = 60
|
|
47
48
|
MISSED_TICK_THRESHOLD = 90
|
|
48
49
|
|
|
50
|
+
# Handed to the error handlers when a due loop names a class this process
|
|
51
|
+
# cannot load, so the occurrence was skipped.
|
|
52
|
+
class UnknownWorkerClass < Wurk::Error; end
|
|
53
|
+
|
|
49
54
|
# 5-field crontab + `@aliases`. No seconds field, no DOW name aliases
|
|
50
55
|
# (sidekiq-ent §2.2 uses numeric DOW only).
|
|
51
56
|
class Parser
|
|
@@ -441,12 +446,17 @@ module Wurk
|
|
|
441
446
|
attr_reader :loops
|
|
442
447
|
|
|
443
448
|
# `prune: false` registers without superseding — ConfigTester uses it so
|
|
444
|
-
# validating a config never deletes a live loop.
|
|
445
|
-
|
|
449
|
+
# validating a config never deletes a live loop. `deferred: true` holds
|
|
450
|
+
# every write until #apply!: `config.periodic` runs while the initializers
|
|
451
|
+
# load, before the app has finished configuring Redis, and in processes
|
|
452
|
+
# that never boot a server (Ent registers on startup).
|
|
453
|
+
def initialize(config = nil, prune: true, deferred: false)
|
|
446
454
|
@config = config
|
|
447
455
|
@loops = []
|
|
448
456
|
@tz = nil
|
|
449
457
|
@prune = prune
|
|
458
|
+
@deferred = deferred
|
|
459
|
+
@authoritative = false
|
|
450
460
|
@pruned_klasses = {}
|
|
451
461
|
end
|
|
452
462
|
|
|
@@ -471,11 +481,22 @@ module Wurk
|
|
|
471
481
|
normalized = opts.transform_keys(&:to_s)
|
|
472
482
|
loop_obj = Loop.new(schedule: cron, klass: klass_name, options: normalized, tz: tz)
|
|
473
483
|
@loops << loop_obj
|
|
474
|
-
|
|
475
|
-
prune_superseded(klass_name)
|
|
484
|
+
persist(loop_obj) unless @deferred
|
|
476
485
|
loop_obj
|
|
477
486
|
end
|
|
478
487
|
|
|
488
|
+
# Writes the registrations held back since `config.periodic` ran, then
|
|
489
|
+
# prunes as #register and #authoritative! would have. Launcher#boot calls
|
|
490
|
+
# it; once applied, later registrations are written as they come.
|
|
491
|
+
def apply!
|
|
492
|
+
return self unless @deferred
|
|
493
|
+
|
|
494
|
+
@deferred = false
|
|
495
|
+
@loops.each { |loop_obj| persist(loop_obj) }
|
|
496
|
+
prune_unregistered if @authoritative
|
|
497
|
+
self
|
|
498
|
+
end
|
|
499
|
+
|
|
479
500
|
# Registers a sidekiq-cron shaped schedule: `{ name => { cron:, class:,
|
|
480
501
|
# queue:, args:, retry:, tz:, status: } }`, or a list of those carrying
|
|
481
502
|
# `name:`. Other keys (`description`, ...) are ignored. Raises ArgumentError
|
|
@@ -500,13 +521,23 @@ module Wurk
|
|
|
500
521
|
def authoritative!
|
|
501
522
|
return self unless @prune
|
|
502
523
|
|
|
503
|
-
|
|
504
|
-
|
|
524
|
+
@authoritative = true
|
|
525
|
+
prune_unregistered unless @deferred
|
|
505
526
|
self
|
|
506
527
|
end
|
|
507
528
|
|
|
508
529
|
private
|
|
509
530
|
|
|
531
|
+
def persist(loop_obj)
|
|
532
|
+
Cron.persist(loop_obj)
|
|
533
|
+
prune_superseded(loop_obj.klass)
|
|
534
|
+
end
|
|
535
|
+
|
|
536
|
+
def prune_unregistered
|
|
537
|
+
keep = @loops.map(&:lid)
|
|
538
|
+
LoopSet.new(@config).each { |existing| Cron.unregister(existing.lid) unless keep.include?(existing.lid) }
|
|
539
|
+
end
|
|
540
|
+
|
|
510
541
|
def register_entry(entry)
|
|
511
542
|
cron = entry['cron'].to_s.strip
|
|
512
543
|
klass = (entry['class'] || entry['klass']).to_s
|
|
@@ -647,6 +678,7 @@ module Wurk
|
|
|
647
678
|
@mutex = ::Mutex.new
|
|
648
679
|
@sleeper = ::ConditionVariable.new
|
|
649
680
|
@on_leader = method(:wake_for_leadership)
|
|
681
|
+
@unknown_reported = ::Set.new
|
|
650
682
|
end
|
|
651
683
|
|
|
652
684
|
# The loop waits one interval before its first tick unless this process
|
|
@@ -723,9 +755,7 @@ module Wurk
|
|
|
723
755
|
skip_missed(loop_obj, slot, now) unless firing == slot
|
|
724
756
|
return if firing.nil?
|
|
725
757
|
|
|
726
|
-
|
|
727
|
-
record_history(loop_obj, jid, now)
|
|
728
|
-
jid
|
|
758
|
+
fire_claimed(loop_obj, now)
|
|
729
759
|
end
|
|
730
760
|
|
|
731
761
|
# Fire one loop right now, bypassing both the leader gate and the
|
|
@@ -814,6 +844,32 @@ module Wurk
|
|
|
814
844
|
nil
|
|
815
845
|
end
|
|
816
846
|
|
|
847
|
+
def fire_claimed(loop_obj, now)
|
|
848
|
+
return skip_unknown_class(loop_obj) unless loop_obj.worker_class
|
|
849
|
+
|
|
850
|
+
jid = enqueue_claimed!(loop_obj)
|
|
851
|
+
record_history(loop_obj, jid, now)
|
|
852
|
+
jid
|
|
853
|
+
end
|
|
854
|
+
|
|
855
|
+
# Ent's tick is `klass.perform_async`, which needs the class. Pushing the
|
|
856
|
+
# bare name instead drops the worker's sidekiq_options, so a job that
|
|
857
|
+
# declares `retry: 0` failed with NameError under the default retries on
|
|
858
|
+
# every tick. The leader boots the app's code, so a class it cannot load
|
|
859
|
+
# is one that was deleted, renamed or rolled back; its loop stays in Redis
|
|
860
|
+
# until `Cron.unregister(lid)` or `authoritative!`. Logged on every
|
|
861
|
+
# occurrence, reported once per loop per process.
|
|
862
|
+
def skip_unknown_class(loop_obj)
|
|
863
|
+
message = "[cron] class #{loop_obj.klass} is not defined in this process lid=#{loop_obj.lid} — " \
|
|
864
|
+
'occurrence skipped; Wurk::Cron.unregister(lid) retires the loop'
|
|
865
|
+
logger.warn(message)
|
|
866
|
+
return unless @unknown_reported.add?(loop_obj.lid)
|
|
867
|
+
|
|
868
|
+
handle_exception(UnknownWorkerClass.new(message),
|
|
869
|
+
{ context: 'cron-poller', lid: loop_obj.lid, klass: loop_obj.klass })
|
|
870
|
+
nil
|
|
871
|
+
end
|
|
872
|
+
|
|
817
873
|
# Compare-and-swap the fire marks: only the tick still looking at the
|
|
818
874
|
# mark it read wins the slot. Manual `#fire` deliberately skips this —
|
|
819
875
|
# an operator-requested run contends with no schedule slot.
|
data/lib/wurk/fetcher/capped.rb
CHANGED
|
@@ -99,7 +99,7 @@ module Wurk
|
|
|
99
99
|
gate = @gates[keys[1]]
|
|
100
100
|
next if gate&.blocked?(now)
|
|
101
101
|
|
|
102
|
-
uow = gate ? claim(public_q, keys, gate, ack
|
|
102
|
+
uow = gate ? claim(public_q, keys, gate, ack) : lmove(public_q, ack)
|
|
103
103
|
ack = nil
|
|
104
104
|
return uow if uow
|
|
105
105
|
end
|
|
@@ -124,7 +124,7 @@ module Wurk
|
|
|
124
124
|
# is deferred: an ACK handed back by a failed flush must not be able to
|
|
125
125
|
# take the hold of whatever this thread is running by then. See
|
|
126
126
|
# QueueSlot.claim_token.
|
|
127
|
-
def claim(public_q, keys, gate, ack
|
|
127
|
+
def claim(public_q, keys, gate, ack)
|
|
128
128
|
priv, name = keys
|
|
129
129
|
token = QueueSlot.claim_token
|
|
130
130
|
script_keys = [gate.slot_key, public_q, priv]
|
|
@@ -137,7 +137,12 @@ module Wurk
|
|
|
137
137
|
# faster than a job finishes somewhere in the cluster, so re-asking on
|
|
138
138
|
# every pass is how a full queue turns into a spin. An empty queue is
|
|
139
139
|
# remembered by nobody, exactly as on the uncapped path.
|
|
140
|
-
|
|
140
|
+
#
|
|
141
|
+
# Stamped with the time the refusal ARRIVED, not the time the question
|
|
142
|
+
# was sent: a slow Redis would otherwise eat the backoff window and
|
|
143
|
+
# the gate would already be expired when the refusal lands — which is
|
|
144
|
+
# the spin this remembers it to avoid.
|
|
145
|
+
gate.block!(::Process.clock_gettime(::Process::CLOCK_MONOTONIC)) if status == AT_CAPACITY
|
|
141
146
|
nil
|
|
142
147
|
rescue StandardError
|
|
143
148
|
# The window #lmove guards: the ACK left its slot but never reached
|
|
@@ -50,7 +50,6 @@ module Wurk
|
|
|
50
50
|
return nil
|
|
51
51
|
LUA
|
|
52
52
|
|
|
53
|
-
RECLAIM = (HEADER + Lua::RELIABLE_RECLAIM).freeze
|
|
54
53
|
REQUEUE = (HEADER + Lua::RELIABLE_REQUEUE).freeze
|
|
55
54
|
|
|
56
55
|
# KEYS = [lock], ARGV = [ttl_ms]. The reaper's sweep lock: refused, no sweep
|
|
@@ -72,10 +71,6 @@ module Wurk
|
|
|
72
71
|
run(conn, error, CLAIM, [Keys::PROCESSES, Component.identity, *keys], [info, beat, HEARTBEAT_TTL])
|
|
73
72
|
end
|
|
74
73
|
|
|
75
|
-
def reclaim(conn, error, source, destination)
|
|
76
|
-
run(conn, error, RECLAIM, [source, destination], [])
|
|
77
|
-
end
|
|
78
|
-
|
|
79
74
|
# Returns 1 when moved, 0 when the job has left the private list since.
|
|
80
75
|
def requeue(conn, error, private_list, public_q, job)
|
|
81
76
|
run(conn, error, REQUEUE, [private_list, public_q], [job])
|