wurk 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/wurk/configuration.rb +7 -0
- data/lib/wurk/redis_options.rb +142 -0
- data/lib/wurk/redis_pool.rb +32 -14
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk.rb +43 -0
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2b1f8eac5cefbd9f645598b9a37dca543aeebb361967d820f369cc14420ad3f6
|
|
4
|
+
data.tar.gz: 813958b643c78cdd2ffa5063798bb3994f6510b1b0d50a0a54db1622f3bb325a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 65ca6b60bf079d626f3887d9e1e12b7d1d1f91ab264c5fcba7c77f54667961159fb8a63c8be89d44e9583ef7c264669e546eb15ab4bc13e392ef85b35dfea6ad
|
|
7
|
+
data.tar.gz: b865d8cd28483589bad0f6d879d80a85c9f9d5cb9b79a494634c90ebaf0f5371169ed7012303dabdc736b32be3596dbffd2ac60b2bbeb3d24c1b40514e483b54
|
data/lib/wurk/configuration.rb
CHANGED
|
@@ -6,6 +6,7 @@ require_relative 'middleware/chain'
|
|
|
6
6
|
require_relative 'capsule'
|
|
7
7
|
require_relative 'context'
|
|
8
8
|
require_relative 'topology'
|
|
9
|
+
require_relative 'redis_options'
|
|
9
10
|
|
|
10
11
|
module Wurk
|
|
11
12
|
# Owns runtime knobs (concurrency, queues, timeouts, lifecycle events,
|
|
@@ -169,8 +170,14 @@ module Wurk
|
|
|
169
170
|
|
|
170
171
|
# --- Redis ------------------------------------------------------------
|
|
171
172
|
|
|
173
|
+
# Validated here, in the process running the initializer, rather than later
|
|
174
|
+
# in whichever process first builds a pool. The swarm's children are the ones
|
|
175
|
+
# that construct pools, so a bad key used to kill every child on boot while
|
|
176
|
+
# the parent stayed up and healthy — Running pod, passing probe, zero jobs
|
|
177
|
+
# processed (#283).
|
|
172
178
|
def redis=(hash)
|
|
173
179
|
guard_frozen!
|
|
180
|
+
RedisOptions.validate!(hash)
|
|
174
181
|
@redis_config = @redis_config.merge(hash.transform_keys(&:to_sym))
|
|
175
182
|
end
|
|
176
183
|
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'redis-client'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
# Translates a Sidekiq-shaped `config.redis` hash into the exact keyword set
|
|
7
|
+
# redis-client accepts.
|
|
8
|
+
#
|
|
9
|
+
# Sidekiq normalizes the hash itself before handing it over
|
|
10
|
+
# (`sidekiq/redis_client_adapter.rb#client_opts`), so initializers in the wild
|
|
11
|
+
# carry keys redis-client has never known. Wurk used to splat the hash straight
|
|
12
|
+
# into `RedisClient.config`, which surfaced as
|
|
13
|
+
# `ArgumentError: unknown keyword: :network_timeout` — and only inside the
|
|
14
|
+
# forked children, which build their own pools (#283). The parent booted fine,
|
|
15
|
+
# the liveness probe passed, and the swarm respawn loop churned forever
|
|
16
|
+
# processing zero jobs. Hence `validate!`, called from Configuration#redis= so
|
|
17
|
+
# a bad hash raises in the parent where someone can actually see it.
|
|
18
|
+
#
|
|
19
|
+
# Reference: sidekiq 7.3 / 8.1 `client_opts` — namespace rejected,
|
|
20
|
+
# size/pool_timeout dropped, `network_timeout` → `timeout`, `master_name` →
|
|
21
|
+
# `name`, role/driver symbolized, `reconnect_attempts ||= 1`.
|
|
22
|
+
module RedisOptions
|
|
23
|
+
# Consumed by the pool layer (RedisPool / Capsule); never a socket concern.
|
|
24
|
+
POOL_KEYS = %i[size name pool_name pool_timeout on_error].freeze
|
|
25
|
+
|
|
26
|
+
# Accepted for Sidekiq parity, then dropped: Sidekiq used `logger` for its
|
|
27
|
+
# own "connecting to Redis with options ..." line and `cluster_safe` to
|
|
28
|
+
# unlock `:nodes`. redis-client has no keyword for either.
|
|
29
|
+
IGNORED_KEYS = %i[logger cluster_safe].freeze
|
|
30
|
+
|
|
31
|
+
# Sidekiq-only spellings this module rewrites into redis-client keywords.
|
|
32
|
+
TRANSLATED_KEYS = %i[network_timeout master_name].freeze
|
|
33
|
+
|
|
34
|
+
# The umbrella socket timeout, under both its names. `network_timeout` is
|
|
35
|
+
# the redis-rb-era spelling every "widen the timeouts for a slow/remote
|
|
36
|
+
# Redis" snippet still uses; `timeout` is redis-client's own.
|
|
37
|
+
UMBRELLA_TIMEOUT_KEYS = %i[network_timeout timeout].freeze
|
|
38
|
+
|
|
39
|
+
# Wurk splits the socket timeouts (#101) and passes all three explicitly, and
|
|
40
|
+
# redis-client lets an explicit `read_timeout` win over `timeout` — so
|
|
41
|
+
# forwarding the umbrella verbatim would silently drop the host's value.
|
|
42
|
+
# Fan it out instead; a host-supplied split timeout still wins over the fan-out.
|
|
43
|
+
SPLIT_TIMEOUT_KEYS = %i[connect_timeout read_timeout write_timeout].freeze
|
|
44
|
+
|
|
45
|
+
# Symbols in redis-client, strings in plenty of YAML-sourced configs.
|
|
46
|
+
SYMBOLIZED_KEYS = %i[driver role].freeze
|
|
47
|
+
|
|
48
|
+
# Keys that mean something in Sidekiq but have no Wurk equivalent. Raise
|
|
49
|
+
# naming the key and its replacement — the alternative is an opaque
|
|
50
|
+
# `unknown keyword:` from three layers down inside a forked child.
|
|
51
|
+
REJECTED_KEYS = {
|
|
52
|
+
namespace: 'Redis namespacing was dropped in Sidekiq 7 and Wurk never implemented it ' \
|
|
53
|
+
'(docs/migrate-from-sidekiq.md §4). Give Wurk its own Redis database ' \
|
|
54
|
+
'(redis://host:6379/1) or its own instance instead.',
|
|
55
|
+
nodes: 'Wurk does not run on Redis Cluster. Point config.redis at a single server with ' \
|
|
56
|
+
'`url:`, or at a Sentinel set with `sentinels:`.'
|
|
57
|
+
}.freeze
|
|
58
|
+
|
|
59
|
+
# Keyword parameter kinds in Method#parameters.
|
|
60
|
+
KEYWORD_PARAMS = %i[key keyreq].freeze
|
|
61
|
+
|
|
62
|
+
class << self
|
|
63
|
+
# The keyword hash for RedisClient.config / RedisClient.sentinel.
|
|
64
|
+
# `defaults` are Wurk's own socket defaults; everything the host supplied
|
|
65
|
+
# wins over them.
|
|
66
|
+
def normalize(options, defaults: {})
|
|
67
|
+
opts = symbolize(options)
|
|
68
|
+
validate!(opts)
|
|
69
|
+
opts = translate(opts)
|
|
70
|
+
|
|
71
|
+
# A default `url` is meaningless next to a sentinel set and actively
|
|
72
|
+
# harmful: SentinelConfig derives the master name and db from it.
|
|
73
|
+
defaults = defaults.except(:url) if sentinel?(opts)
|
|
74
|
+
|
|
75
|
+
defaults.merge(split_timeouts(opts), opts.except(*UMBRELLA_TIMEOUT_KEYS))
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Raises for anything redis-client would reject. Cheap and pure, so it runs
|
|
79
|
+
# in the parent (Configuration#redis=) as well as at pool-build time.
|
|
80
|
+
def validate!(options)
|
|
81
|
+
opts = symbolize(options)
|
|
82
|
+
|
|
83
|
+
REJECTED_KEYS.each do |key, hint|
|
|
84
|
+
raise ArgumentError, "config.redis[:#{key}] is not supported. #{hint}" if opts.key?(key)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
unknown = opts.keys - known_keys
|
|
88
|
+
return if unknown.empty?
|
|
89
|
+
|
|
90
|
+
raise ArgumentError,
|
|
91
|
+
"config.redis: unknown option#{'s' if unknown.size > 1} " \
|
|
92
|
+
"#{unknown.map(&:inspect).join(', ')}. Supported keys: #{known_keys.sort.join(', ')}."
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Sentinel sets go through RedisClient.sentinel — RedisClient.config
|
|
96
|
+
# rejects `sentinels:` outright. Same routing Sidekiq does.
|
|
97
|
+
def sentinel?(client_config)
|
|
98
|
+
client_config.key?(:sentinels)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Every keyword redis-client itself accepts, read off its own signatures so
|
|
102
|
+
# the list can't drift from the installed version. Config#initialize takes
|
|
103
|
+
# a **kwargs rest and forwards to Config::Common, so both have to be walked;
|
|
104
|
+
# SentinelConfig adds the sentinel-only keys.
|
|
105
|
+
def known_keys
|
|
106
|
+
@known_keys ||= [
|
|
107
|
+
::RedisClient::Config, ::RedisClient::Config::Common, ::RedisClient::SentinelConfig
|
|
108
|
+
].flat_map { |mod| keyword_params(mod) }.union(POOL_KEYS, IGNORED_KEYS, TRANSLATED_KEYS)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
private
|
|
112
|
+
|
|
113
|
+
# Drop what redis-client has no keyword for, then rewrite the Sidekiq
|
|
114
|
+
# spellings it does have an equivalent for.
|
|
115
|
+
def translate(opts)
|
|
116
|
+
opts = opts.except(*POOL_KEYS, *IGNORED_KEYS)
|
|
117
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
|
118
|
+
SYMBOLIZED_KEYS.each { |key| opts[key] = opts[key].to_sym if opts[key] }
|
|
119
|
+
opts
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def keyword_params(mod)
|
|
123
|
+
mod.instance_method(:initialize).parameters.filter_map do |kind, key|
|
|
124
|
+
key if KEYWORD_PARAMS.include?(kind)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# The umbrella timeout fanned out across the three split ones. Returns {}
|
|
129
|
+
# when the host set neither, leaving Wurk's defaults in place.
|
|
130
|
+
def split_timeouts(opts)
|
|
131
|
+
umbrella = opts.values_at(*UMBRELLA_TIMEOUT_KEYS).compact.first
|
|
132
|
+
return {} unless umbrella
|
|
133
|
+
|
|
134
|
+
SPLIT_TIMEOUT_KEYS.to_h { |key| [key, umbrella] }
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def symbolize(options)
|
|
138
|
+
options.transform_keys(&:to_sym)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
data/lib/wurk/redis_pool.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'redis-client'
|
|
4
4
|
require 'connection_pool'
|
|
5
5
|
require_relative 'redis_client_adapter'
|
|
6
|
+
require_relative 'redis_options'
|
|
6
7
|
|
|
7
8
|
module Wurk
|
|
8
9
|
# Per-process pool over redis-client + connection_pool. Never share a socket
|
|
@@ -37,6 +38,15 @@ module Wurk
|
|
|
37
38
|
DEFAULT_WRITE_TIMEOUT = 2.5
|
|
38
39
|
DEFAULT_RECONNECT_ATTEMPTS = 1
|
|
39
40
|
|
|
41
|
+
# The floor every pool starts from; any key the host passed wins over it.
|
|
42
|
+
DEFAULT_CLIENT_CONFIG = {
|
|
43
|
+
url: DEFAULT_URL,
|
|
44
|
+
connect_timeout: DEFAULT_CONNECT_TIMEOUT,
|
|
45
|
+
read_timeout: DEFAULT_READ_TIMEOUT,
|
|
46
|
+
write_timeout: DEFAULT_WRITE_TIMEOUT,
|
|
47
|
+
reconnect_attempts: DEFAULT_RECONNECT_ATTEMPTS
|
|
48
|
+
}.freeze
|
|
49
|
+
|
|
40
50
|
# Server-side messages where the connection is closed and the block retried
|
|
41
51
|
# exactly once. READONLY is itself a RedisClient::ConnectionError subclass,
|
|
42
52
|
# so this message match must be tested BEFORE the generic ConnectionError
|
|
@@ -60,9 +70,12 @@ module Wurk
|
|
|
60
70
|
|
|
61
71
|
# Takes the standard Sidekiq `config.redis` hash: `pool_timeout` tunes the
|
|
62
72
|
# ConnectionPool checkout; `connect_timeout`/`read_timeout`/`write_timeout`/
|
|
63
|
-
# `reconnect_attempts` plus any other key (driver, ssl_params,
|
|
64
|
-
#
|
|
65
|
-
#
|
|
73
|
+
# `reconnect_attempts` plus any other redis-client key (driver, ssl_params,
|
|
74
|
+
# sentinels, …) reach the client. Sidekiq-only spellings (`network_timeout`,
|
|
75
|
+
# `master_name`, `logger`, …) are translated or dropped by {RedisOptions};
|
|
76
|
+
# a key redis-client would reject raises there with the key named.
|
|
77
|
+
# `on_error` is an optional callable fired per retry / final give-up with
|
|
78
|
+
# { error:, attempt:, retried:, pool: }.
|
|
66
79
|
def initialize(size:, name: DEFAULT_NAME, on_error: nil, **options)
|
|
67
80
|
@size = size
|
|
68
81
|
@name = name
|
|
@@ -112,24 +125,29 @@ module Wurk
|
|
|
112
125
|
|
|
113
126
|
private
|
|
114
127
|
|
|
115
|
-
# Socket config forwarded to
|
|
116
|
-
# the
|
|
117
|
-
#
|
|
128
|
+
# Socket config forwarded to redis-client. RedisOptions owns the translation
|
|
129
|
+
# of the Sidekiq-shaped hash (network_timeout, master_name, pool-only keys,
|
|
130
|
+
# …) so this class stays about pooling; host-supplied keys win over the
|
|
131
|
+
# defaults.
|
|
118
132
|
def build_client_config(options)
|
|
119
|
-
|
|
120
|
-
url: DEFAULT_URL,
|
|
121
|
-
connect_timeout: DEFAULT_CONNECT_TIMEOUT,
|
|
122
|
-
read_timeout: DEFAULT_READ_TIMEOUT,
|
|
123
|
-
write_timeout: DEFAULT_WRITE_TIMEOUT,
|
|
124
|
-
reconnect_attempts: DEFAULT_RECONNECT_ATTEMPTS
|
|
125
|
-
}.merge(options.except(:pool_timeout)).freeze
|
|
133
|
+
RedisOptions.normalize(options, defaults: DEFAULT_CLIENT_CONFIG).freeze
|
|
126
134
|
end
|
|
127
135
|
|
|
128
136
|
# Wrapped in the CompatClient decorator so `Sidekiq.redis { |c| c.smembers }`
|
|
129
137
|
# method-style commands work like Sidekiq 7+ (#204). Wurk's own code paths
|
|
130
138
|
# use #call, which the decorator forwards.
|
|
131
139
|
def build_client
|
|
132
|
-
RedisClientAdapter::CompatClient.new(
|
|
140
|
+
RedisClientAdapter::CompatClient.new(redis_client_config.new_client)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# A Sentinel set is a different constructor, not a different keyword:
|
|
144
|
+
# `RedisClient.config(sentinels: [...])` raises. Sidekiq routes the same way.
|
|
145
|
+
def redis_client_config
|
|
146
|
+
if RedisOptions.sentinel?(@client_config)
|
|
147
|
+
RedisClient.sentinel(**@client_config)
|
|
148
|
+
else
|
|
149
|
+
RedisClient.config(**@client_config)
|
|
150
|
+
end
|
|
133
151
|
end
|
|
134
152
|
|
|
135
153
|
def safe_close(conn)
|
data/lib/wurk/version.rb
CHANGED
data/lib/wurk.rb
CHANGED
|
@@ -224,6 +224,22 @@ module Wurk
|
|
|
224
224
|
def ent?
|
|
225
225
|
false
|
|
226
226
|
end
|
|
227
|
+
|
|
228
|
+
# Lazily loads the Rails engine the first time something asks for
|
|
229
|
+
# `Wurk::Engine`. See the note above the `require "wurk/rails"` guard at the
|
|
230
|
+
# bottom of this file for why the guard alone isn't enough (#282), and why
|
|
231
|
+
# this loads the engine and not the railtie.
|
|
232
|
+
def const_missing(name)
|
|
233
|
+
return super unless name == :Engine
|
|
234
|
+
return super unless defined?(::Rails::Engine) && defined?(::ActionDispatch::Routing::RouteSet)
|
|
235
|
+
|
|
236
|
+
require_relative 'wurk/engine'
|
|
237
|
+
# If engine.rb somehow didn't define it, fall through to the real NameError
|
|
238
|
+
# rather than recursing back into here.
|
|
239
|
+
return super unless const_defined?(:Engine, false)
|
|
240
|
+
|
|
241
|
+
const_get(:Engine, false)
|
|
242
|
+
end
|
|
227
243
|
end
|
|
228
244
|
end
|
|
229
245
|
|
|
@@ -318,3 +334,30 @@ require_relative 'wurk/compat'
|
|
|
318
334
|
# real Rails host both are loaded by `rails/all` before Bundler.require, so the
|
|
319
335
|
# stricter gate is invisible there.
|
|
320
336
|
require_relative 'wurk/rails' if defined?(Rails::Engine) && defined?(::ActionDispatch::Routing::RouteSet)
|
|
337
|
+
|
|
338
|
+
# The gate above is evaluated exactly once, when this file is first required —
|
|
339
|
+
# which is too early for the standalone runners (#282). `exe/wurk` and
|
|
340
|
+
# `exe/wurkswarm` require "wurk" before Rails exists (the gate is false, nothing
|
|
341
|
+
# Rails-y loads), then boot the host app themselves via
|
|
342
|
+
# Wurk::CLI#boot_rails_application. By then "wurk" is in $LOADED_FEATURES, so the
|
|
343
|
+
# app's own `Bundler.require` is a no-op and the gate never gets a second look. A
|
|
344
|
+
# host that did exactly what the README says — `mount Wurk::Engine => "/wurk"` in
|
|
345
|
+
# config/routes.rb — then dies during boot with `uninitialized constant
|
|
346
|
+
# Wurk::Engine`, from the worker process only. Web processes are unaffected:
|
|
347
|
+
# there Bundler.require runs after railties, so the gate passes.
|
|
348
|
+
#
|
|
349
|
+
# `Wurk.const_missing` (defined in the class << self block above) closes that
|
|
350
|
+
# window without giving up the lean standalone boot: nothing loads until
|
|
351
|
+
# something actually asks for Wurk::Engine, and even then only when Rails is
|
|
352
|
+
# really present. It reuses the same two-part condition as the guard above, so
|
|
353
|
+
# the ecosystem carve-out (rails/engine/railties without ActionDispatch, per
|
|
354
|
+
# sidekiq-cron's test helper) still falls through to a plain NameError instead of
|
|
355
|
+
# crashing on `isolate_namespace`.
|
|
356
|
+
#
|
|
357
|
+
# It deliberately loads `wurk/engine`, not `wurk/rails`: the railtie's
|
|
358
|
+
# `config.after_initialize` is a *global* ActiveSupport load hook registered at
|
|
359
|
+
# require time, and routes are drawn before those hooks run — so dragging the
|
|
360
|
+
# railtie in there would fork a swarm inside a `wurkswarm` parent that is about
|
|
361
|
+
# to fork its own. The runner owns the swarm; the routes file only needs the
|
|
362
|
+
# constant. `defined?(Wurk::Engine)` stays nil until first use, which is what the
|
|
363
|
+
# "standalone stays Rails-free" tests assert.
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wurk
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.3.
|
|
4
|
+
version: 1.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- developerz.ai
|
|
@@ -270,6 +270,7 @@ files:
|
|
|
270
270
|
- lib/wurk/railtie.rb
|
|
271
271
|
- lib/wurk/redis_client_adapter.rb
|
|
272
272
|
- lib/wurk/redis_connection.rb
|
|
273
|
+
- lib/wurk/redis_options.rb
|
|
273
274
|
- lib/wurk/redis_pool.rb
|
|
274
275
|
- lib/wurk/retry_set.rb
|
|
275
276
|
- lib/wurk/scheduled.rb
|