wurk 0.0.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -2
- data/app/controllers/wurk/api/serializers.rb +48 -2
- data/app/controllers/wurk/api_controller.rb +216 -1
- data/app/controllers/wurk/dashboard_controller.rb +20 -2
- data/app/controllers/wurk/extensions_controller.rb +56 -0
- data/app/controllers/wurk/profiles_controller.rb +68 -0
- data/config/routes.rb +54 -1
- data/exe/sidekiqswarm +8 -0
- data/exe/wurkswarm +23 -0
- data/lib/active_job/queue_adapters/wurk_adapter.rb +35 -0
- data/lib/generators/wurk/install/templates/wurk.rb +14 -3
- data/lib/sidekiq/api.rb +4 -0
- data/lib/sidekiq/cli.rb +9 -0
- data/lib/sidekiq/client.rb +4 -0
- data/lib/sidekiq/job.rb +4 -0
- data/lib/sidekiq/launcher.rb +4 -0
- data/lib/sidekiq/middleware/chain.rb +4 -0
- data/lib/sidekiq/middleware/server/statsd.rb +12 -0
- data/lib/sidekiq/rails.rb +10 -0
- data/lib/sidekiq/redis_connection.rb +4 -0
- data/lib/sidekiq/scheduled.rb +4 -0
- data/lib/sidekiq/testing.rb +4 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/web.rb +4 -0
- data/lib/sidekiq/worker.rb +4 -0
- data/lib/sidekiq.rb +16 -0
- data/lib/wurk/batch/callbacks.rb +103 -13
- data/lib/wurk/batch/death_handler.rb +5 -2
- data/lib/wurk/batch/server_middleware.rb +35 -3
- data/lib/wurk/batch/status.rb +9 -0
- data/lib/wurk/batch.rb +23 -1
- data/lib/wurk/capsule.rb +20 -1
- data/lib/wurk/cli.rb +84 -1
- data/lib/wurk/client.rb +20 -17
- data/lib/wurk/compat.rb +44 -2
- data/lib/wurk/component.rb +5 -4
- data/lib/wurk/configuration.rb +120 -3
- data/lib/wurk/cron.rb +51 -9
- data/lib/wurk/dead_set.rb +8 -3
- data/lib/wurk/deploy.rb +8 -4
- data/lib/wurk/encryption.rb +6 -1
- data/lib/wurk/fetcher/reaper.rb +78 -11
- data/lib/wurk/fetcher/reliable.rb +14 -4
- data/lib/wurk/heartbeat.rb +45 -0
- data/lib/wurk/history.rb +174 -0
- data/lib/wurk/iterable_job/active_record_enumerator.rb +71 -0
- data/lib/wurk/iterable_job/csv_enumerator.rb +51 -0
- data/lib/wurk/iterable_job.rb +41 -0
- data/lib/wurk/iterable_job_query.rb +75 -0
- data/lib/wurk/job.rb +8 -0
- data/lib/wurk/job_record.rb +16 -1
- data/lib/wurk/job_set.rb +4 -4
- data/lib/wurk/job_util.rb +15 -6
- data/lib/wurk/keys.rb +10 -0
- data/lib/wurk/launcher.rb +35 -1
- data/lib/wurk/leader.rb +15 -6
- data/lib/wurk/limiter/bucket.rb +14 -3
- data/lib/wurk/limiter/concurrent.rb +1 -1
- data/lib/wurk/limiter/window.rb +2 -1
- data/lib/wurk/limiter.rb +12 -0
- data/lib/wurk/lua/loader.rb +10 -0
- data/lib/wurk/lua.rb +106 -14
- data/lib/wurk/metrics/history.rb +5 -0
- data/lib/wurk/metrics/query.rb +39 -0
- data/lib/wurk/metrics/queue_rollup.rb +151 -0
- data/lib/wurk/metrics/statsd.rb +11 -0
- data/lib/wurk/middleware/current_attributes.rb +29 -6
- data/lib/wurk/middleware/interrupt_handler.rb +5 -0
- data/lib/wurk/middleware/poison_pill.rb +35 -5
- data/lib/wurk/processor.rb +17 -8
- data/lib/wurk/profile_set.rb +65 -0
- data/lib/wurk/profiler.rb +127 -0
- data/lib/wurk/railtie.rb +19 -5
- data/lib/wurk/redis_client_adapter.rb +72 -0
- data/lib/wurk/redis_connection.rb +30 -0
- data/lib/wurk/redis_pool.rb +5 -1
- data/lib/wurk/scheduled.rb +42 -0
- data/lib/wurk/sorted_entry.rb +13 -11
- data/lib/wurk/stats.rb +11 -4
- data/lib/wurk/swarm/child_boot.rb +26 -4
- data/lib/wurk/swarm.rb +1 -1
- data/lib/wurk/transaction_aware_client.rb +69 -0
- data/lib/wurk/unique.rb +49 -7
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/batch_status.rb +42 -0
- data/lib/wurk/web/config.rb +219 -17
- data/lib/wurk/web/enterprise.rb +14 -0
- data/lib/wurk/web/extension.rb +348 -0
- data/lib/wurk/web/rack_app.rb +77 -0
- data/lib/wurk/web.rb +2 -0
- data/lib/wurk/worker/setter.rb +5 -1
- data/lib/wurk/worker.rb +17 -6
- data/lib/wurk.rb +44 -0
- data/vendor/assets/dashboard/assets/fa-brands-400-BP5tdqmh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-nyy7hhHF.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-DRAAbZTg.woff2 +0 -0
- data/vendor/assets/dashboard/assets/index-9CFRWpfG.js +77 -0
- data/vendor/assets/dashboard/assets/index-CW8AFQIv.css +2 -0
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/favicon.png +0 -0
- data/vendor/assets/dashboard/index.html +10 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +42 -6
- data/CHANGELOG.md +0 -67
- data/CONTRIBUTING.md +0 -73
- data/SECURITY.md +0 -39
- data/vendor/assets/dashboard/assets/index-D2XR0iGw.js +0 -60
- data/vendor/assets/dashboard/assets/index-DlPr4YXw.css +0 -1
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../component'
|
|
4
|
+
require_relative '../keys'
|
|
5
|
+
require_relative '../job_record'
|
|
6
|
+
require_relative 'rollup'
|
|
7
|
+
|
|
8
|
+
module Wurk
|
|
9
|
+
module Metrics
|
|
10
|
+
# Leader-only background thread that snapshots each queue's current depth
|
|
11
|
+
# (LLEN) and head-of-line latency into compact per-queue gauge buckets the
|
|
12
|
+
# dashboard's "queue size / latency over time" charts read directly:
|
|
13
|
+
#
|
|
14
|
+
# qm|1m|<epoch> HASH {<queue>|sz, <queue>|lt} TTL 24h
|
|
15
|
+
# qm|5m|<epoch> HASH {<queue>|sz, <queue>|lt} TTL 7d
|
|
16
|
+
# qm|1h|<epoch> HASH {<queue>|sz, <queue>|lt} TTL 30d
|
|
17
|
+
#
|
|
18
|
+
# Unlike Metrics::Rollup (which SUMS counters rolled up from a source),
|
|
19
|
+
# size and latency are GAUGES — point-in-time values — so each tick samples
|
|
20
|
+
# "now" and writes it to the current bucket at every resolution. Within a
|
|
21
|
+
# coarse (5m/1h) bucket the per-minute ticks overwrite, so the bucket holds
|
|
22
|
+
# the latest sample in its window (a "last value" downsample, which is the
|
|
23
|
+
# right summary for a gauge). Leader-gated so N workers don't each sample
|
|
24
|
+
# the same queues every minute. `<epoch>` is the UTC start-of-bucket.
|
|
25
|
+
#
|
|
26
|
+
# Spec: docs/target/sidekiq-ent.md §5.2 (sidekiq.queue.size /
|
|
27
|
+
# sidekiq.queue.latency gauges), §7 Historical tab.
|
|
28
|
+
class QueueRollup
|
|
29
|
+
include Component
|
|
30
|
+
|
|
31
|
+
PREFIX = 'qm'
|
|
32
|
+
SIZE_KIND = 'sz'
|
|
33
|
+
LAT_KIND = 'lt'
|
|
34
|
+
|
|
35
|
+
# Mirror Metrics::Rollup retention so the dashboard's range selector
|
|
36
|
+
# (24h·1m / 7d·5m / 30d·1h) maps 1:1 to both the throughput and the
|
|
37
|
+
# queue-gauge series.
|
|
38
|
+
BUCKETS = Wurk::Metrics::Rollup::BUCKETS
|
|
39
|
+
|
|
40
|
+
DEFAULT_TICK_SECONDS = 60
|
|
41
|
+
|
|
42
|
+
def self.bucket_key(bucket, epoch)
|
|
43
|
+
"#{PREFIX}|#{bucket}|#{epoch}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def initialize(config)
|
|
47
|
+
@config = config
|
|
48
|
+
@done = false
|
|
49
|
+
@mutex = ::Mutex.new
|
|
50
|
+
@sleeper = ::ConditionVariable.new
|
|
51
|
+
@tick_interval = config[:metrics_rollup_interval] || DEFAULT_TICK_SECONDS
|
|
52
|
+
@thread = nil
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def start
|
|
56
|
+
@thread ||= safe_thread('queue-metrics') do # rubocop:disable Naming/MemoizedInstanceVariableName
|
|
57
|
+
wait
|
|
58
|
+
until @done
|
|
59
|
+
tick
|
|
60
|
+
wait
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def terminate
|
|
66
|
+
@mutex.synchronize do
|
|
67
|
+
@done = true
|
|
68
|
+
@sleeper.signal
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Leader-gated: only the elected leader samples, so N workers don't each
|
|
73
|
+
# HSET the same buckets every minute.
|
|
74
|
+
def tick(now: ::Time.now)
|
|
75
|
+
return unless leader?
|
|
76
|
+
|
|
77
|
+
sample(now)
|
|
78
|
+
rescue StandardError => e
|
|
79
|
+
handle_exception(e, { context: 'queue-metrics' })
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# One sampling pass, bypassing the leader gate and the sleep loop. Public
|
|
83
|
+
# so deterministic specs and a manual "sample now" can drive it directly.
|
|
84
|
+
def sample(now = ::Time.now)
|
|
85
|
+
gauges = queue_gauges
|
|
86
|
+
return if gauges.empty?
|
|
87
|
+
|
|
88
|
+
fields = gauges.flat_map do |name, (size, lat)|
|
|
89
|
+
["#{name}|#{SIZE_KIND}", size, "#{name}|#{LAT_KIND}", lat]
|
|
90
|
+
end
|
|
91
|
+
BUCKETS.each do |bucket, (step, ttl)|
|
|
92
|
+
key = self.class.bucket_key(bucket, (now.to_i / step) * step)
|
|
93
|
+
redis do |c|
|
|
94
|
+
c.call('HSET', key, *fields)
|
|
95
|
+
c.call('EXPIRE', key, ttl)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
nil
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
private
|
|
102
|
+
|
|
103
|
+
# [[queue_name, [size, latency_seconds]], ...] for every live queue. The
|
|
104
|
+
# latency is the head-of-line wait — the tail of the LIST is the oldest
|
|
105
|
+
# job — in seconds, rounded for compact storage.
|
|
106
|
+
def queue_gauges
|
|
107
|
+
now_ms = real_ms
|
|
108
|
+
redis do |c|
|
|
109
|
+
names = c.call('SMEMBERS', Keys::QUEUES_SET)
|
|
110
|
+
next [] if names.empty?
|
|
111
|
+
|
|
112
|
+
raw = pipelined_queue_reads(c, names)
|
|
113
|
+
names.each_with_index.map do |name, i|
|
|
114
|
+
[name, [raw[i * 2].to_i, head_latency(raw[(i * 2) + 1], now_ms)]]
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Per queue: LLEN (depth) then LRANGE tail (head-of-line job), in one
|
|
120
|
+
# round trip. Results interleave [len, [tail], len, [tail], …].
|
|
121
|
+
def pipelined_queue_reads(conn, names)
|
|
122
|
+
conn.pipelined do |p|
|
|
123
|
+
names.each do |q|
|
|
124
|
+
p.call('LLEN', Keys.queue(q))
|
|
125
|
+
p.call('LRANGE', Keys.queue(q), -1, -1)
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# A single malformed tail payload (bad JSON, or valid JSON of the wrong
|
|
131
|
+
# shape) yields 0.0 for that queue rather than bubbling up and skipping
|
|
132
|
+
# the whole sampling pass for every queue.
|
|
133
|
+
def head_latency(lrange_result, now_ms)
|
|
134
|
+
payload = lrange_result.is_a?(::Array) ? lrange_result.first : lrange_result
|
|
135
|
+
return 0.0 if payload.nil?
|
|
136
|
+
|
|
137
|
+
parsed = Wurk.load_json(payload)
|
|
138
|
+
enqueued_at = parsed.is_a?(::Hash) ? parsed['enqueued_at'] : nil
|
|
139
|
+
Wurk::JobRecord.latency_from(enqueued_at, now_ms).round(2)
|
|
140
|
+
rescue ::JSON::ParserError, ::TypeError, ::ArgumentError
|
|
141
|
+
0.0
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def wait
|
|
145
|
+
@mutex.synchronize do
|
|
146
|
+
@sleeper.wait(@mutex, @tick_interval) unless @done
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
data/lib/wurk/metrics/statsd.rb
CHANGED
|
@@ -194,4 +194,15 @@ module Wurk
|
|
|
194
194
|
end
|
|
195
195
|
end
|
|
196
196
|
end
|
|
197
|
+
|
|
198
|
+
module Middleware
|
|
199
|
+
# Sidekiq Pro documents the statsd server middleware as
|
|
200
|
+
# `Sidekiq::Middleware::Server::Statsd` (spec §9.1) — the drop-in snippet is
|
|
201
|
+
# `require "sidekiq/middleware/server/statsd"; chain.add Sidekiq::Middleware::Server::Statsd`.
|
|
202
|
+
# Expose that namespace pointing at our implementation. `Sidekiq::Middleware`
|
|
203
|
+
# aliases `Wurk::Middleware`, so this makes the Sidekiq constant resolve too.
|
|
204
|
+
module Server
|
|
205
|
+
Statsd = Wurk::Metrics::Statsd
|
|
206
|
+
end
|
|
207
|
+
end
|
|
197
208
|
end
|
|
@@ -25,6 +25,7 @@ module Wurk
|
|
|
25
25
|
raise ArgumentError, 'persist requires at least one CurrentAttributes class' if classes.empty?
|
|
26
26
|
|
|
27
27
|
config.client_middleware.add(Save, classes)
|
|
28
|
+
config.client_middleware.add(Load, classes)
|
|
28
29
|
config.server_middleware.add(Load, classes)
|
|
29
30
|
end
|
|
30
31
|
|
|
@@ -35,9 +36,12 @@ module Wurk
|
|
|
35
36
|
end
|
|
36
37
|
|
|
37
38
|
# AS::CurrentAttributes#attributes returns a HashWithIndifferentAccess;
|
|
38
|
-
# we coerce to a plain Hash so JSON encoding is predictable.
|
|
39
|
+
# we coerce to a plain Hash so JSON encoding is predictable. `.dup` so
|
|
40
|
+
# the snapshot is a detached copy — Load holds onto the pre-job state
|
|
41
|
+
# and must not see it mutated when it restores the job's attributes,
|
|
42
|
+
# and Save must not alias live attributes into the persisted job hash.
|
|
39
43
|
def snapshot(klass)
|
|
40
|
-
klass.attributes.to_h
|
|
44
|
+
klass.attributes.to_h.dup
|
|
41
45
|
end
|
|
42
46
|
|
|
43
47
|
def restore(klass, attrs)
|
|
@@ -64,8 +68,17 @@ module Wurk
|
|
|
64
68
|
end
|
|
65
69
|
|
|
66
70
|
# Restores each registered CurrentAttributes class for the duration
|
|
67
|
-
# of the inner block, then
|
|
68
|
-
#
|
|
71
|
+
# of the inner block, then puts back whatever was there before — not a
|
|
72
|
+
# blanket reset. On a server thread the prior state is empty, so this is
|
|
73
|
+
# equivalent to resetting; but Load also runs on the CLIENT chain (persist
|
|
74
|
+
# registers it on both), where an enqueue happens mid-request with
|
|
75
|
+
# request-scoped attributes already set. Resetting there would wipe the
|
|
76
|
+
# caller's state right after `perform_async`. Save/restore is what Sidekiq
|
|
77
|
+
# does and is correct on both chains. Restore runs in `ensure` to survive
|
|
78
|
+
# raises and Skip.
|
|
79
|
+
#
|
|
80
|
+
# Registered on BOTH chains, so `call` takes an optional 4th arg: the
|
|
81
|
+
# client chain passes a `redis_pool`, the server chain stops at `queue`.
|
|
69
82
|
class Load
|
|
70
83
|
include Wurk::Middleware::ServerMiddleware
|
|
71
84
|
|
|
@@ -73,15 +86,25 @@ module Wurk
|
|
|
73
86
|
@classes = classes
|
|
74
87
|
end
|
|
75
88
|
|
|
76
|
-
def call(_job_or_class, job, _queue)
|
|
89
|
+
def call(_job_or_class, job, _queue, _redis_pool = nil)
|
|
90
|
+
previous = @classes.map { |klass| CurrentAttributes.snapshot(klass) }
|
|
77
91
|
@classes.each_with_index do |klass, idx|
|
|
78
92
|
CurrentAttributes.restore(klass, job[CurrentAttributes.key_for(idx)])
|
|
79
93
|
end
|
|
80
94
|
yield
|
|
81
95
|
ensure
|
|
82
|
-
|
|
96
|
+
previous&.each_with_index do |attrs, idx|
|
|
97
|
+
@classes[idx].reset
|
|
98
|
+
CurrentAttributes.restore(@classes[idx], attrs)
|
|
99
|
+
end
|
|
83
100
|
end
|
|
84
101
|
end
|
|
85
102
|
end
|
|
86
103
|
end
|
|
87
104
|
end
|
|
105
|
+
|
|
106
|
+
# Drop-in alias. Sidekiq documents the top-level constant
|
|
107
|
+
# `Sidekiq::CurrentAttributes` (with `.persist` and `Save`/`Load` nested),
|
|
108
|
+
# not the `Middleware::`-namespaced form. Set when this opt-in file is
|
|
109
|
+
# required, so `Sidekiq::CurrentAttributes.persist(MyAttrs)` resolves.
|
|
110
|
+
Sidekiq::CurrentAttributes = Wurk::Middleware::CurrentAttributes if defined?(Sidekiq)
|
|
@@ -43,3 +43,8 @@ module Wurk
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
Wurk.configuration.server_middleware.prepend(Wurk::Middleware::InterruptHandler)
|
|
46
|
+
|
|
47
|
+
# Sidekiq exposes this middleware as `Sidekiq::Job::InterruptHandler`. Mirror
|
|
48
|
+
# that name onto `Wurk::Job` (aliased to `Sidekiq::Job` in compat) so the
|
|
49
|
+
# drop-in constant resolves. Spec: docs/target/sidekiq-free.md §10.3.
|
|
50
|
+
Wurk::Job::InterruptHandler = Wurk::Middleware::InterruptHandler
|
|
@@ -29,6 +29,12 @@ module Wurk
|
|
|
29
29
|
KEY_PREFIX = 'super_fetch:recovered:'
|
|
30
30
|
DEAD_RECORD_LIMIT = 100
|
|
31
31
|
|
|
32
|
+
# The `pill` handed to a Pro `super_fetch! { |jobstr, pill| }` recovery
|
|
33
|
+
# callback on the kill path. Responds to .jid/.klass/.count/.queue so a
|
|
34
|
+
# `pill.jid`-style Pro initializer drops in. `.count` shadows Struct#count
|
|
35
|
+
# by design — Pro's API names it that. Spec: sidekiq-pro.md §3.1.
|
|
36
|
+
Pill = ::Struct.new(:jid, :klass, :count, :queue, keyword_init: true) # rubocop:disable Lint/StructNewOverride
|
|
37
|
+
|
|
32
38
|
module_function
|
|
33
39
|
|
|
34
40
|
# Called per recovered orphan job. Returns `:poison` when the threshold
|
|
@@ -37,23 +43,34 @@ module Wurk
|
|
|
37
43
|
# here). Emits `jobs.recovered.fetch` on every call, `jobs.poison`
|
|
38
44
|
# only on the kill path.
|
|
39
45
|
#
|
|
46
|
+
# Fires the Pro `super_fetch!` recovery callback (config.super_fetch_callback)
|
|
47
|
+
# exactly once per call: `(jobstr, nil)` on plain recovery, `(jobstr, pill)`
|
|
48
|
+
# on the kill path. The poison-only `on_poison` Hash callbacks fire
|
|
49
|
+
# independently inside #mark_poison.
|
|
50
|
+
#
|
|
40
51
|
# @param payload [String, Hash] the job JSON or pre-parsed hash.
|
|
41
52
|
# @param queue [String, nil] the public queue name (without `queue:`).
|
|
53
|
+
# @param config [Configuration] config that owns the super_fetch! callback;
|
|
54
|
+
# defaults to the global so non-reaper callers (tests) need not pass it.
|
|
42
55
|
# @return [Symbol] :recovered | :poison
|
|
43
|
-
def track!(payload, queue: nil)
|
|
56
|
+
def track!(payload, queue: nil, config: Wurk.configuration)
|
|
44
57
|
job = parse(payload)
|
|
45
|
-
|
|
58
|
+
unless job
|
|
59
|
+
fire_super_fetch(config, payload, nil)
|
|
60
|
+
return :recovered
|
|
61
|
+
end
|
|
46
62
|
|
|
47
63
|
jid = job['jid']
|
|
48
64
|
klass = job['class']
|
|
49
65
|
emit_recovered_fetch(klass, queue)
|
|
50
|
-
return :recovered if jid.nil? || jid.empty?
|
|
51
66
|
|
|
52
|
-
count = bump_counter(jid)
|
|
53
|
-
if count >= RECOVERY_THRESHOLD
|
|
67
|
+
count = bump_counter(jid) if jid && !jid.empty?
|
|
68
|
+
if count && count >= RECOVERY_THRESHOLD
|
|
54
69
|
mark_poison(payload, job, queue: queue, count: count)
|
|
70
|
+
fire_super_fetch(config, payload, Pill.new(jid: jid, klass: klass, count: count, queue: queue))
|
|
55
71
|
:poison
|
|
56
72
|
else
|
|
73
|
+
fire_super_fetch(config, payload, nil)
|
|
57
74
|
:recovered
|
|
58
75
|
end
|
|
59
76
|
end
|
|
@@ -144,6 +161,19 @@ module Wurk
|
|
|
144
161
|
Wurk.configuration.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
|
|
145
162
|
end
|
|
146
163
|
end
|
|
164
|
+
|
|
165
|
+
# Invoke the Pro recovery callback registered via config.super_fetch! { }.
|
|
166
|
+
# No-op unless one is registered. `jobstr` is the raw job JSON so a Pro
|
|
167
|
+
# `|jobstr, pill|` block sees exactly what Sidekiq Pro hands it.
|
|
168
|
+
def fire_super_fetch(config, payload, pill)
|
|
169
|
+
cb = config.super_fetch_callback
|
|
170
|
+
return unless cb
|
|
171
|
+
|
|
172
|
+
jobstr = payload.is_a?(::String) ? payload : Wurk.dump_json(payload)
|
|
173
|
+
cb.call(jobstr, pill)
|
|
174
|
+
rescue StandardError => e
|
|
175
|
+
config.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
|
|
176
|
+
end
|
|
147
177
|
end
|
|
148
178
|
end
|
|
149
179
|
end
|
data/lib/wurk/processor.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative 'context'
|
|
|
5
5
|
require_relative 'job_logger'
|
|
6
6
|
require_relative 'job_retry'
|
|
7
7
|
require_relative 'keys'
|
|
8
|
+
require_relative 'profiler'
|
|
8
9
|
|
|
9
10
|
module Wurk
|
|
10
11
|
# Inside each Manager, N Processors run in parallel. Each owns one thread,
|
|
@@ -206,14 +207,12 @@ module Wurk
|
|
|
206
207
|
@retrier.global(jobstr, queue) do
|
|
207
208
|
@job_logger.call(job_hash, queue) do
|
|
208
209
|
stats(jobstr, queue) do
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
@retrier.local(instance, jobstr, queue) do
|
|
216
|
-
yield instance
|
|
210
|
+
Wurk::Profiler.call(job_hash) do
|
|
211
|
+
@reloader.call do
|
|
212
|
+
instance = build_instance(job_hash)
|
|
213
|
+
@retrier.local(instance, jobstr, queue) do
|
|
214
|
+
yield instance
|
|
215
|
+
end
|
|
217
216
|
end
|
|
218
217
|
end
|
|
219
218
|
end
|
|
@@ -222,6 +221,16 @@ module Wurk
|
|
|
222
221
|
end
|
|
223
222
|
end
|
|
224
223
|
|
|
224
|
+
# Instantiate the worker and wire its per-job context. Extracted from
|
|
225
|
+
# dispatch so the dispatch onion stays readable.
|
|
226
|
+
def build_instance(job_hash)
|
|
227
|
+
instance = Object.const_get(job_hash['class']).new
|
|
228
|
+
instance.jid = job_hash['jid']
|
|
229
|
+
instance.bid = job_hash['bid'] if instance.respond_to?(:bid=)
|
|
230
|
+
instance._context = self
|
|
231
|
+
instance
|
|
232
|
+
end
|
|
233
|
+
|
|
225
234
|
def execute_job(instance, job_hash, queue)
|
|
226
235
|
@capsule.server_middleware.invoke(instance, job_hash, queue) do
|
|
227
236
|
instance.perform(*job_hash['args'])
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'keys'
|
|
4
|
+
require_relative 'profiler'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# Read-only view over stored job profiles (Sidekiq 8.0+ data API, spec §19.8).
|
|
8
|
+
# `ProfileSet` enumerates the `profiles` ZSET, purging expired members first;
|
|
9
|
+
# `ProfileRecord` wraps one `<token>-<jid>` HASH.
|
|
10
|
+
class ProfileSet
|
|
11
|
+
include Enumerable
|
|
12
|
+
|
|
13
|
+
# Snapshot the (non-expired) member keys at construction. ZREMRANGEBYSCORE
|
|
14
|
+
# drops members whose expiry score has passed before we read the rest.
|
|
15
|
+
def initialize
|
|
16
|
+
@keys = Wurk.redis do |conn|
|
|
17
|
+
conn.call('ZREMRANGEBYSCORE', Keys::PROFILES, '-inf', "(#{::Time.now.to_i}")
|
|
18
|
+
conn.call('ZRANGE', Keys::PROFILES, 0, -1)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def size = @keys.size
|
|
23
|
+
|
|
24
|
+
def each
|
|
25
|
+
return enum_for(:each) unless block_given?
|
|
26
|
+
|
|
27
|
+
Wurk.redis do |conn|
|
|
28
|
+
@keys.each do |key|
|
|
29
|
+
raw = conn.call('HGETALL', key)
|
|
30
|
+
hash = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
|
|
31
|
+
yield ProfileRecord.new(hash) unless hash.empty?
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# One profile record: the metadata fields of a `<token>-<jid>` HASH plus
|
|
38
|
+
# lazy access to the gzipped gecko blob. Spec §19.8.
|
|
39
|
+
class ProfileRecord
|
|
40
|
+
attr_reader :jid, :type, :token, :size, :elapsed
|
|
41
|
+
|
|
42
|
+
def initialize(hash)
|
|
43
|
+
@hash = hash
|
|
44
|
+
@jid = hash['jid']
|
|
45
|
+
@type = hash['type']
|
|
46
|
+
@token = hash['token']
|
|
47
|
+
@size = hash['size'].to_i
|
|
48
|
+
@elapsed = hash['elapsed'].to_i
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def started_at
|
|
52
|
+
ts = @hash['started_at']
|
|
53
|
+
::Time.at(ts.to_i) unless ts.nil? || ts.empty?
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def key = Wurk::Profiler.profile_key(@token, @jid)
|
|
57
|
+
|
|
58
|
+
# The stored blob is gzipped gecko JSON. `data` returns the raw (gzipped)
|
|
59
|
+
# bytes — the web layer streams them straight to the browser with a gzip
|
|
60
|
+
# Content-Encoding. Returns nil if the HASH expired between list and read.
|
|
61
|
+
def data
|
|
62
|
+
Wurk.redis { |conn| conn.call('HGET', key, 'data') }
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'securerandom'
|
|
4
|
+
require 'zlib'
|
|
5
|
+
require 'stringio'
|
|
6
|
+
require 'tempfile'
|
|
7
|
+
require_relative 'keys'
|
|
8
|
+
|
|
9
|
+
module Wurk
|
|
10
|
+
# Job profiling (Sidekiq 8.0+, OSS). When a job is pushed with a `profile`
|
|
11
|
+
# option, the processor wraps `perform` in a Vernier capture; the resulting
|
|
12
|
+
# Firefox-profiler (gecko) JSON is gzipped and stored so the dashboard can
|
|
13
|
+
# hand it to https://profiler.firefox.com for flame-graph inspection.
|
|
14
|
+
#
|
|
15
|
+
# Redis schema (spec §1.7), wire-compat with Sidekiq's Profiles pane:
|
|
16
|
+
#
|
|
17
|
+
# profiles ZSET member = "<token>-<jid>", score = expiry epoch
|
|
18
|
+
# <token>-<jid> HASH jid, type, token, started_at, elapsed, size,
|
|
19
|
+
# sid, data (gzipped gecko JSON)
|
|
20
|
+
#
|
|
21
|
+
# Capture is a no-op unless the `vernier` gem is loaded — profiling is an
|
|
22
|
+
# opt-in, dev/staging tool, so vernier stays an optional dependency.
|
|
23
|
+
module Profiler
|
|
24
|
+
# Stored profiles live this long (score = now + TTL); ProfileSet purges
|
|
25
|
+
# expired members on read.
|
|
26
|
+
TTL = 7 * 24 * 60 * 60 # 7 days
|
|
27
|
+
|
|
28
|
+
class << self
|
|
29
|
+
# Server-side hook called from Processor#dispatch. Returns the perform
|
|
30
|
+
# result. Only captures when the job opted in AND vernier is present —
|
|
31
|
+
# otherwise it's a plain `yield`. Crucially there is NO blanket rescue
|
|
32
|
+
# here: the job's own exceptions (a normal failure, or JobRetry::Skip
|
|
33
|
+
# from the interrupt/expiry middleware) must propagate untouched so the
|
|
34
|
+
# retry/skip flow works. Only the storage step is made failure-safe
|
|
35
|
+
# (see #safe_store).
|
|
36
|
+
def call(job_hash, &)
|
|
37
|
+
label = job_hash['profile']
|
|
38
|
+
return yield unless label && defined?(::Vernier)
|
|
39
|
+
|
|
40
|
+
capture(job_hash, label, &)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Persists a profile. Extracted from capture so it is unit-testable
|
|
44
|
+
# without vernier: tests pass a ready gecko JSON blob. The wide keyword
|
|
45
|
+
# list mirrors the HASH fields one-to-one — collapsing them into an
|
|
46
|
+
# options hash would just hide the schema.
|
|
47
|
+
# rubocop:disable Metrics/ParameterLists
|
|
48
|
+
def store(jid:, type:, gecko_json:, started_at:, elapsed_ms:, token: SecureRandom.hex(8),
|
|
49
|
+
sid: Wurk.configuration[:identity], pool: nil)
|
|
50
|
+
key = profile_key(token, jid)
|
|
51
|
+
gz = gzip(gecko_json)
|
|
52
|
+
with_pool(pool) do |conn|
|
|
53
|
+
conn.call('HSET', key, 'jid', jid, 'type', type, 'token', token,
|
|
54
|
+
'started_at', started_at.to_i, 'elapsed', elapsed_ms.to_i,
|
|
55
|
+
'size', gz.bytesize, 'sid', sid.to_s, 'data', gz)
|
|
56
|
+
conn.call('EXPIRE', key, TTL)
|
|
57
|
+
conn.call('ZADD', Keys::PROFILES, (now + TTL).to_i, key)
|
|
58
|
+
end
|
|
59
|
+
key
|
|
60
|
+
end
|
|
61
|
+
# rubocop:enable Metrics/ParameterLists
|
|
62
|
+
|
|
63
|
+
def profile_key(token, jid)
|
|
64
|
+
"#{token}-#{jid}"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def gzip(str)
|
|
68
|
+
io = StringIO.new(+'', 'wb')
|
|
69
|
+
gz = Zlib::GzipWriter.new(io)
|
|
70
|
+
gz.write(str)
|
|
71
|
+
gz.close
|
|
72
|
+
io.string
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def gunzip(bytes)
|
|
76
|
+
Zlib::GzipReader.new(StringIO.new(bytes)).read
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# Wrap the block in a Vernier capture, write the gecko JSON to a tempfile
|
|
82
|
+
# (Vernier serializes on block exit), then store it. Only reached when
|
|
83
|
+
# vernier is loaded.
|
|
84
|
+
def capture(job_hash, label)
|
|
85
|
+
retval = nil
|
|
86
|
+
started = now
|
|
87
|
+
elapsed_ms = nil
|
|
88
|
+
json = profile_to_json do
|
|
89
|
+
t0 = monotonic_ms
|
|
90
|
+
retval = yield
|
|
91
|
+
elapsed_ms = monotonic_ms - t0
|
|
92
|
+
end
|
|
93
|
+
safe_store(job_hash, label, json, started, elapsed_ms)
|
|
94
|
+
retval
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The job already ran successfully by the time we get here; a Redis hiccup
|
|
98
|
+
# persisting the profile must not turn a green job red. Job exceptions
|
|
99
|
+
# never reach this method — they propagate out of `capture`'s yield.
|
|
100
|
+
def safe_store(job_hash, label, json, started, elapsed_ms)
|
|
101
|
+
store(jid: job_hash['jid'], type: label.to_s, gecko_json: json,
|
|
102
|
+
started_at: started, elapsed_ms: elapsed_ms)
|
|
103
|
+
rescue StandardError => e
|
|
104
|
+
Wurk.configuration.handle_exception(e, context: 'Wurk::Profiler')
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def profile_to_json(&)
|
|
108
|
+
Tempfile.create(['wurk-profile', '.json']) do |file|
|
|
109
|
+
::Vernier.profile(out: file.path, &)
|
|
110
|
+
File.read(file.path)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def with_pool(pool, &)
|
|
115
|
+
pool ? pool.with(&) : Wurk.redis(&)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def now
|
|
119
|
+
::Time.now.to_f
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def monotonic_ms
|
|
123
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :float_millisecond)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
data/lib/wurk/railtie.rb
CHANGED
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require 'rails/railtie'
|
|
4
4
|
|
|
5
5
|
module Wurk
|
|
6
6
|
# Boot the swarm after the host app has fully initialized.
|
|
7
7
|
# Skip when: WURK_DISABLED=1, Rails console mode, or Rails test env.
|
|
8
8
|
# See docs/idea/03-process-model.md for the exact ordering.
|
|
9
9
|
class Railtie < ::Rails::Railtie
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
# This Rails process forks the workers, so it IS the server. Enter server
|
|
11
|
+
# mode BEFORE config/initializers load — otherwise the app's
|
|
12
|
+
# `Sidekiq.configure_server` blocks gate on `config.server?` (still false)
|
|
13
|
+
# and are silently dropped. Gated identically to the swarm boot: a process
|
|
14
|
+
# that won't run workers (disabled / console / test) is not a server.
|
|
15
|
+
initializer 'wurk.server_mode', before: :load_config_initializers do
|
|
16
|
+
Wurk.enter_server_mode unless Wurk::Railtie.skip_boot?
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
config.after_initialize do |_app|
|
|
20
|
+
next if Wurk::Railtie.skip_boot?
|
|
14
21
|
|
|
15
22
|
swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology)
|
|
16
23
|
swarm.boot
|
|
@@ -24,5 +31,12 @@ module Wurk
|
|
|
24
31
|
Wurk.configuration.logger.error { "wurk supervisor thread died: #{e.class}: #{e.message}" }
|
|
25
32
|
end
|
|
26
33
|
end
|
|
34
|
+
|
|
35
|
+
# A process that won't run workers isn't a server: skip both server mode
|
|
36
|
+
# and the swarm boot. Console mode is detected reliably here — the console
|
|
37
|
+
# command file defines ::Rails::Console before initializers run.
|
|
38
|
+
def self.skip_boot?
|
|
39
|
+
ENV['WURK_DISABLED'] == '1' || defined?(::Rails::Console) || ::Rails.env.test?
|
|
40
|
+
end
|
|
27
41
|
end
|
|
28
42
|
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'redis-client'
|
|
4
|
+
require 'redis_client/decorator'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# Command-method compatibility for connections yielded by `Wurk.redis` /
|
|
8
|
+
# `Sidekiq.redis` (#204). Sidekiq 7+ yields RedisClientAdapter::CompatClient,
|
|
9
|
+
# so third-party gems write `conn.hgetall(...)` / `conn.smembers(...)` —
|
|
10
|
+
# method-style commands raw RedisClient doesn't answer (it only has #call).
|
|
11
|
+
# Every pooled connection is wrapped in this decorator at build time
|
|
12
|
+
# (RedisPool#build_client); wurk's own hot paths keep using #call, which the
|
|
13
|
+
# decorator forwards with a single delegation hop.
|
|
14
|
+
#
|
|
15
|
+
# Mirrors sidekiq-8.1.6 lib/sidekiq/redis_client_adapter.rb byte-for-byte in
|
|
16
|
+
# behavior: same fast-path command list, same deprecation warning, same
|
|
17
|
+
# error constants (gems rescue `Sidekiq::RedisClientAdapter::BaseError`).
|
|
18
|
+
class RedisClientAdapter
|
|
19
|
+
BaseError = RedisClient::Error
|
|
20
|
+
CommandError = RedisClient::CommandError
|
|
21
|
+
|
|
22
|
+
DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
|
|
23
|
+
|
|
24
|
+
module CompatMethods
|
|
25
|
+
def info
|
|
26
|
+
@client.call('INFO') { |i| i.lines(chomp: true).map { |l| l.split(':', 2) }.select { |l| l.size == 2 }.to_h }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def evalsha(sha, keys, argv)
|
|
30
|
+
@client.call('EVALSHA', sha, keys.size, *keys, *argv)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# The Redis commands Sidekiq itself uses — defined eagerly so the
|
|
34
|
+
# common ones skip method_missing. Same list as upstream.
|
|
35
|
+
USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
|
|
36
|
+
get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
|
|
37
|
+
lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
|
|
38
|
+
publish rpop rpush sadd scard script set sismember smembers
|
|
39
|
+
srem ttl type unlink zadd zcard zincrby zrange zrem
|
|
40
|
+
zremrangebyrank zremrangebyscore].freeze
|
|
41
|
+
|
|
42
|
+
USED_COMMANDS.each do |name|
|
|
43
|
+
define_method(name) do |*args, **kwargs|
|
|
44
|
+
@client.call(name, *args, **kwargs)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
# `conn.hmset(...)` instead of redis-client's native `conn.call("hmset", ...)`.
|
|
51
|
+
def method_missing(*args, &)
|
|
52
|
+
if DEPRECATED_COMMANDS.include?(args.first)
|
|
53
|
+
warn("[sidekiq#5788] Redis has deprecated the `#{args.first}` command, called at #{caller(1..1)}")
|
|
54
|
+
end
|
|
55
|
+
@client.call(*args, &)
|
|
56
|
+
end
|
|
57
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
|
58
|
+
|
|
59
|
+
def respond_to_missing?(_name, _include_private = false)
|
|
60
|
+
super # We can't tell what is a valid command.
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
|
65
|
+
|
|
66
|
+
class CompatClient
|
|
67
|
+
def config
|
|
68
|
+
@client.config
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|