wurk 0.0.4 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +16 -2
  3. data/app/controllers/wurk/api/serializers.rb +48 -2
  4. data/app/controllers/wurk/api_controller.rb +216 -1
  5. data/app/controllers/wurk/dashboard_controller.rb +20 -2
  6. data/app/controllers/wurk/extensions_controller.rb +56 -0
  7. data/app/controllers/wurk/profiles_controller.rb +68 -0
  8. data/config/routes.rb +54 -1
  9. data/exe/sidekiqswarm +8 -0
  10. data/exe/wurkswarm +23 -0
  11. data/lib/active_job/queue_adapters/wurk_adapter.rb +35 -0
  12. data/lib/generators/wurk/install/templates/wurk.rb +14 -3
  13. data/lib/sidekiq/api.rb +4 -0
  14. data/lib/sidekiq/cli.rb +9 -0
  15. data/lib/sidekiq/client.rb +4 -0
  16. data/lib/sidekiq/job.rb +4 -0
  17. data/lib/sidekiq/launcher.rb +4 -0
  18. data/lib/sidekiq/middleware/chain.rb +4 -0
  19. data/lib/sidekiq/middleware/server/statsd.rb +12 -0
  20. data/lib/sidekiq/rails.rb +10 -0
  21. data/lib/sidekiq/redis_connection.rb +4 -0
  22. data/lib/sidekiq/scheduled.rb +4 -0
  23. data/lib/sidekiq/testing.rb +4 -0
  24. data/lib/sidekiq/version.rb +4 -0
  25. data/lib/sidekiq/web.rb +4 -0
  26. data/lib/sidekiq/worker.rb +4 -0
  27. data/lib/sidekiq.rb +16 -0
  28. data/lib/wurk/batch/callbacks.rb +103 -13
  29. data/lib/wurk/batch/death_handler.rb +5 -2
  30. data/lib/wurk/batch/server_middleware.rb +35 -3
  31. data/lib/wurk/batch/status.rb +9 -0
  32. data/lib/wurk/batch.rb +23 -1
  33. data/lib/wurk/capsule.rb +20 -1
  34. data/lib/wurk/cli.rb +84 -1
  35. data/lib/wurk/client.rb +20 -17
  36. data/lib/wurk/compat.rb +44 -2
  37. data/lib/wurk/component.rb +5 -4
  38. data/lib/wurk/configuration.rb +120 -3
  39. data/lib/wurk/cron.rb +51 -9
  40. data/lib/wurk/dead_set.rb +8 -3
  41. data/lib/wurk/deploy.rb +8 -4
  42. data/lib/wurk/encryption.rb +6 -1
  43. data/lib/wurk/fetcher/reaper.rb +78 -11
  44. data/lib/wurk/fetcher/reliable.rb +14 -4
  45. data/lib/wurk/heartbeat.rb +45 -0
  46. data/lib/wurk/history.rb +174 -0
  47. data/lib/wurk/iterable_job/active_record_enumerator.rb +71 -0
  48. data/lib/wurk/iterable_job/csv_enumerator.rb +51 -0
  49. data/lib/wurk/iterable_job.rb +41 -0
  50. data/lib/wurk/iterable_job_query.rb +75 -0
  51. data/lib/wurk/job.rb +8 -0
  52. data/lib/wurk/job_record.rb +16 -1
  53. data/lib/wurk/job_set.rb +4 -4
  54. data/lib/wurk/job_util.rb +15 -6
  55. data/lib/wurk/keys.rb +10 -0
  56. data/lib/wurk/launcher.rb +35 -1
  57. data/lib/wurk/leader.rb +15 -6
  58. data/lib/wurk/limiter/bucket.rb +14 -3
  59. data/lib/wurk/limiter/concurrent.rb +1 -1
  60. data/lib/wurk/limiter/window.rb +2 -1
  61. data/lib/wurk/limiter.rb +12 -0
  62. data/lib/wurk/lua/loader.rb +10 -0
  63. data/lib/wurk/lua.rb +106 -14
  64. data/lib/wurk/metrics/history.rb +5 -0
  65. data/lib/wurk/metrics/query.rb +39 -0
  66. data/lib/wurk/metrics/queue_rollup.rb +151 -0
  67. data/lib/wurk/metrics/statsd.rb +11 -0
  68. data/lib/wurk/middleware/current_attributes.rb +29 -6
  69. data/lib/wurk/middleware/interrupt_handler.rb +5 -0
  70. data/lib/wurk/middleware/poison_pill.rb +35 -5
  71. data/lib/wurk/processor.rb +17 -8
  72. data/lib/wurk/profile_set.rb +65 -0
  73. data/lib/wurk/profiler.rb +127 -0
  74. data/lib/wurk/railtie.rb +19 -5
  75. data/lib/wurk/redis_client_adapter.rb +72 -0
  76. data/lib/wurk/redis_connection.rb +30 -0
  77. data/lib/wurk/redis_pool.rb +5 -1
  78. data/lib/wurk/scheduled.rb +42 -0
  79. data/lib/wurk/sorted_entry.rb +13 -11
  80. data/lib/wurk/stats.rb +11 -4
  81. data/lib/wurk/swarm/child_boot.rb +26 -4
  82. data/lib/wurk/swarm.rb +1 -1
  83. data/lib/wurk/transaction_aware_client.rb +69 -0
  84. data/lib/wurk/unique.rb +49 -7
  85. data/lib/wurk/version.rb +1 -1
  86. data/lib/wurk/web/batch_status.rb +42 -0
  87. data/lib/wurk/web/config.rb +219 -17
  88. data/lib/wurk/web/enterprise.rb +14 -0
  89. data/lib/wurk/web/extension.rb +348 -0
  90. data/lib/wurk/web/rack_app.rb +77 -0
  91. data/lib/wurk/web.rb +2 -0
  92. data/lib/wurk/worker/setter.rb +5 -1
  93. data/lib/wurk/worker.rb +17 -6
  94. data/lib/wurk.rb +44 -0
  95. data/vendor/assets/dashboard/assets/fa-brands-400-BP5tdqmh.woff2 +0 -0
  96. data/vendor/assets/dashboard/assets/fa-regular-400-nyy7hhHF.woff2 +0 -0
  97. data/vendor/assets/dashboard/assets/fa-solid-900-DRAAbZTg.woff2 +0 -0
  98. data/vendor/assets/dashboard/assets/index-9CFRWpfG.js +77 -0
  99. data/vendor/assets/dashboard/assets/index-CW8AFQIv.css +2 -0
  100. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  101. data/vendor/assets/dashboard/favicon.png +0 -0
  102. data/vendor/assets/dashboard/index.html +10 -3
  103. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  104. metadata +42 -6
  105. data/CHANGELOG.md +0 -67
  106. data/CONTRIBUTING.md +0 -73
  107. data/SECURITY.md +0 -39
  108. data/vendor/assets/dashboard/assets/index-D2XR0iGw.js +0 -60
  109. data/vendor/assets/dashboard/assets/index-DlPr4YXw.css +0 -1
@@ -0,0 +1,151 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../component'
4
+ require_relative '../keys'
5
+ require_relative '../job_record'
6
+ require_relative 'rollup'
7
+
8
+ module Wurk
9
+ module Metrics
10
+ # Leader-only background thread that snapshots each queue's current depth
11
+ # (LLEN) and head-of-line latency into compact per-queue gauge buckets the
12
+ # dashboard's "queue size / latency over time" charts read directly:
13
+ #
14
+ # qm|1m|<epoch> HASH {<queue>|sz, <queue>|lt} TTL 24h
15
+ # qm|5m|<epoch> HASH {<queue>|sz, <queue>|lt} TTL 7d
16
+ # qm|1h|<epoch> HASH {<queue>|sz, <queue>|lt} TTL 30d
17
+ #
18
+ # Unlike Metrics::Rollup (which SUMS counters rolled up from a source),
19
+ # size and latency are GAUGES — point-in-time values — so each tick samples
20
+ # "now" and writes it to the current bucket at every resolution. Within a
21
+ # coarse (5m/1h) bucket the per-minute ticks overwrite, so the bucket holds
22
+ # the latest sample in its window (a "last value" downsample, which is the
23
+ # right summary for a gauge). Leader-gated so N workers don't each sample
24
+ # the same queues every minute. `<epoch>` is the UTC start-of-bucket.
25
+ #
26
+ # Spec: docs/target/sidekiq-ent.md §5.2 (sidekiq.queue.size /
27
+ # sidekiq.queue.latency gauges), §7 Historical tab.
28
+ class QueueRollup
29
+ include Component
30
+
31
+ PREFIX = 'qm'
32
+ SIZE_KIND = 'sz'
33
+ LAT_KIND = 'lt'
34
+
35
+ # Mirror Metrics::Rollup retention so the dashboard's range selector
36
+ # (24h·1m / 7d·5m / 30d·1h) maps 1:1 to both the throughput and the
37
+ # queue-gauge series.
38
+ BUCKETS = Wurk::Metrics::Rollup::BUCKETS
39
+
40
+ DEFAULT_TICK_SECONDS = 60
41
+
42
+ def self.bucket_key(bucket, epoch)
43
+ "#{PREFIX}|#{bucket}|#{epoch}"
44
+ end
45
+
46
+ def initialize(config)
47
+ @config = config
48
+ @done = false
49
+ @mutex = ::Mutex.new
50
+ @sleeper = ::ConditionVariable.new
51
+ @tick_interval = config[:metrics_rollup_interval] || DEFAULT_TICK_SECONDS
52
+ @thread = nil
53
+ end
54
+
55
+ def start
56
+ @thread ||= safe_thread('queue-metrics') do # rubocop:disable Naming/MemoizedInstanceVariableName
57
+ wait
58
+ until @done
59
+ tick
60
+ wait
61
+ end
62
+ end
63
+ end
64
+
65
+ def terminate
66
+ @mutex.synchronize do
67
+ @done = true
68
+ @sleeper.signal
69
+ end
70
+ end
71
+
72
+ # Leader-gated: only the elected leader samples, so N workers don't each
73
+ # HSET the same buckets every minute.
74
+ def tick(now: ::Time.now)
75
+ return unless leader?
76
+
77
+ sample(now)
78
+ rescue StandardError => e
79
+ handle_exception(e, { context: 'queue-metrics' })
80
+ end
81
+
82
+ # One sampling pass, bypassing the leader gate and the sleep loop. Public
83
+ # so deterministic specs and a manual "sample now" can drive it directly.
84
+ def sample(now = ::Time.now)
85
+ gauges = queue_gauges
86
+ return if gauges.empty?
87
+
88
+ fields = gauges.flat_map do |name, (size, lat)|
89
+ ["#{name}|#{SIZE_KIND}", size, "#{name}|#{LAT_KIND}", lat]
90
+ end
91
+ BUCKETS.each do |bucket, (step, ttl)|
92
+ key = self.class.bucket_key(bucket, (now.to_i / step) * step)
93
+ redis do |c|
94
+ c.call('HSET', key, *fields)
95
+ c.call('EXPIRE', key, ttl)
96
+ end
97
+ end
98
+ nil
99
+ end
100
+
101
+ private
102
+
103
+ # [[queue_name, [size, latency_seconds]], ...] for every live queue. The
104
+ # latency is the head-of-line wait — the tail of the LIST is the oldest
105
+ # job — in seconds, rounded for compact storage.
106
+ def queue_gauges
107
+ now_ms = real_ms
108
+ redis do |c|
109
+ names = c.call('SMEMBERS', Keys::QUEUES_SET)
110
+ next [] if names.empty?
111
+
112
+ raw = pipelined_queue_reads(c, names)
113
+ names.each_with_index.map do |name, i|
114
+ [name, [raw[i * 2].to_i, head_latency(raw[(i * 2) + 1], now_ms)]]
115
+ end
116
+ end
117
+ end
118
+
119
+ # Per queue: LLEN (depth) then LRANGE tail (head-of-line job), in one
120
+ # round trip. Results interleave [len, [tail], len, [tail], …].
121
+ def pipelined_queue_reads(conn, names)
122
+ conn.pipelined do |p|
123
+ names.each do |q|
124
+ p.call('LLEN', Keys.queue(q))
125
+ p.call('LRANGE', Keys.queue(q), -1, -1)
126
+ end
127
+ end
128
+ end
129
+
130
+ # A single malformed tail payload (bad JSON, or valid JSON of the wrong
131
+ # shape) yields 0.0 for that queue rather than bubbling up and skipping
132
+ # the whole sampling pass for every queue.
133
+ def head_latency(lrange_result, now_ms)
134
+ payload = lrange_result.is_a?(::Array) ? lrange_result.first : lrange_result
135
+ return 0.0 if payload.nil?
136
+
137
+ parsed = Wurk.load_json(payload)
138
+ enqueued_at = parsed.is_a?(::Hash) ? parsed['enqueued_at'] : nil
139
+ Wurk::JobRecord.latency_from(enqueued_at, now_ms).round(2)
140
+ rescue ::JSON::ParserError, ::TypeError, ::ArgumentError
141
+ 0.0
142
+ end
143
+
144
+ def wait
145
+ @mutex.synchronize do
146
+ @sleeper.wait(@mutex, @tick_interval) unless @done
147
+ end
148
+ end
149
+ end
150
+ end
151
+ end
@@ -194,4 +194,15 @@ module Wurk
194
194
  end
195
195
  end
196
196
  end
197
+
198
+ module Middleware
199
+ # Sidekiq Pro documents the statsd server middleware as
200
+ # `Sidekiq::Middleware::Server::Statsd` (spec §9.1) — the drop-in snippet is
201
+ # `require "sidekiq/middleware/server/statsd"; chain.add Sidekiq::Middleware::Server::Statsd`.
202
+ # Expose that namespace pointing at our implementation. `Sidekiq::Middleware`
203
+ # aliases `Wurk::Middleware`, so this makes the Sidekiq constant resolve too.
204
+ module Server
205
+ Statsd = Wurk::Metrics::Statsd
206
+ end
207
+ end
197
208
  end
@@ -25,6 +25,7 @@ module Wurk
25
25
  raise ArgumentError, 'persist requires at least one CurrentAttributes class' if classes.empty?
26
26
 
27
27
  config.client_middleware.add(Save, classes)
28
+ config.client_middleware.add(Load, classes)
28
29
  config.server_middleware.add(Load, classes)
29
30
  end
30
31
 
@@ -35,9 +36,12 @@ module Wurk
35
36
  end
36
37
 
37
38
  # AS::CurrentAttributes#attributes returns a HashWithIndifferentAccess;
38
- # we coerce to a plain Hash so JSON encoding is predictable.
39
+ # we coerce to a plain Hash so JSON encoding is predictable. `.dup` so
40
+ # the snapshot is a detached copy — Load holds onto the pre-job state
41
+ # and must not see it mutated when it restores the job's attributes,
42
+ # and Save must not alias live attributes into the persisted job hash.
39
43
  def snapshot(klass)
40
- klass.attributes.to_h
44
+ klass.attributes.to_h.dup
41
45
  end
42
46
 
43
47
  def restore(klass, attrs)
@@ -64,8 +68,17 @@ module Wurk
64
68
  end
65
69
 
66
70
  # Restores each registered CurrentAttributes class for the duration
67
- # of the inner block, then resets so the next job in the thread
68
- # starts clean. Reset runs in `ensure` to survive raises and Skip.
71
+ # of the inner block, then puts back whatever was there before not a
72
+ # blanket reset. On a server thread the prior state is empty, so this is
73
+ # equivalent to resetting; but Load also runs on the CLIENT chain (persist
74
+ # registers it on both), where an enqueue happens mid-request with
75
+ # request-scoped attributes already set. Resetting there would wipe the
76
+ # caller's state right after `perform_async`. Save/restore is what Sidekiq
77
+ # does and is correct on both chains. Restore runs in `ensure` to survive
78
+ # raises and Skip.
79
+ #
80
+ # Registered on BOTH chains, so `call` takes an optional 4th arg: the
81
+ # client chain passes a `redis_pool`, the server chain stops at `queue`.
69
82
  class Load
70
83
  include Wurk::Middleware::ServerMiddleware
71
84
 
@@ -73,15 +86,25 @@ module Wurk
73
86
  @classes = classes
74
87
  end
75
88
 
76
- def call(_job_or_class, job, _queue)
89
+ def call(_job_or_class, job, _queue, _redis_pool = nil)
90
+ previous = @classes.map { |klass| CurrentAttributes.snapshot(klass) }
77
91
  @classes.each_with_index do |klass, idx|
78
92
  CurrentAttributes.restore(klass, job[CurrentAttributes.key_for(idx)])
79
93
  end
80
94
  yield
81
95
  ensure
82
- @classes.each(&:reset)
96
+ previous&.each_with_index do |attrs, idx|
97
+ @classes[idx].reset
98
+ CurrentAttributes.restore(@classes[idx], attrs)
99
+ end
83
100
  end
84
101
  end
85
102
  end
86
103
  end
87
104
  end
105
+
106
+ # Drop-in alias. Sidekiq documents the top-level constant
107
+ # `Sidekiq::CurrentAttributes` (with `.persist` and `Save`/`Load` nested),
108
+ # not the `Middleware::`-namespaced form. Set when this opt-in file is
109
+ # required, so `Sidekiq::CurrentAttributes.persist(MyAttrs)` resolves.
110
+ Sidekiq::CurrentAttributes = Wurk::Middleware::CurrentAttributes if defined?(Sidekiq)
@@ -43,3 +43,8 @@ module Wurk
43
43
  end
44
44
 
45
45
  Wurk.configuration.server_middleware.prepend(Wurk::Middleware::InterruptHandler)
46
+
47
+ # Sidekiq exposes this middleware as `Sidekiq::Job::InterruptHandler`. Mirror
48
+ # that name onto `Wurk::Job` (aliased to `Sidekiq::Job` in compat) so the
49
+ # drop-in constant resolves. Spec: docs/target/sidekiq-free.md §10.3.
50
+ Wurk::Job::InterruptHandler = Wurk::Middleware::InterruptHandler
@@ -29,6 +29,12 @@ module Wurk
29
29
  KEY_PREFIX = 'super_fetch:recovered:'
30
30
  DEAD_RECORD_LIMIT = 100
31
31
 
32
+ # The `pill` handed to a Pro `super_fetch! { |jobstr, pill| }` recovery
33
+ # callback on the kill path. Responds to .jid/.klass/.count/.queue so a
34
+ # `pill.jid`-style Pro initializer drops in. `.count` shadows Struct#count
35
+ # by design — Pro's API names it that. Spec: sidekiq-pro.md §3.1.
36
+ Pill = ::Struct.new(:jid, :klass, :count, :queue, keyword_init: true) # rubocop:disable Lint/StructNewOverride
37
+
32
38
  module_function
33
39
 
34
40
  # Called per recovered orphan job. Returns `:poison` when the threshold
@@ -37,23 +43,34 @@ module Wurk
37
43
  # here). Emits `jobs.recovered.fetch` on every call, `jobs.poison`
38
44
  # only on the kill path.
39
45
  #
46
+ # Fires the Pro `super_fetch!` recovery callback (config.super_fetch_callback)
47
+ # exactly once per call: `(jobstr, nil)` on plain recovery, `(jobstr, pill)`
48
+ # on the kill path. The poison-only `on_poison` Hash callbacks fire
49
+ # independently inside #mark_poison.
50
+ #
40
51
  # @param payload [String, Hash] the job JSON or pre-parsed hash.
41
52
  # @param queue [String, nil] the public queue name (without `queue:`).
53
+ # @param config [Configuration] config that owns the super_fetch! callback;
54
+ # defaults to the global so non-reaper callers (tests) need not pass it.
42
55
  # @return [Symbol] :recovered | :poison
43
- def track!(payload, queue: nil)
56
+ def track!(payload, queue: nil, config: Wurk.configuration)
44
57
  job = parse(payload)
45
- return :recovered unless job
58
+ unless job
59
+ fire_super_fetch(config, payload, nil)
60
+ return :recovered
61
+ end
46
62
 
47
63
  jid = job['jid']
48
64
  klass = job['class']
49
65
  emit_recovered_fetch(klass, queue)
50
- return :recovered if jid.nil? || jid.empty?
51
66
 
52
- count = bump_counter(jid)
53
- if count >= RECOVERY_THRESHOLD
67
+ count = bump_counter(jid) if jid && !jid.empty?
68
+ if count && count >= RECOVERY_THRESHOLD
54
69
  mark_poison(payload, job, queue: queue, count: count)
70
+ fire_super_fetch(config, payload, Pill.new(jid: jid, klass: klass, count: count, queue: queue))
55
71
  :poison
56
72
  else
73
+ fire_super_fetch(config, payload, nil)
57
74
  :recovered
58
75
  end
59
76
  end
@@ -144,6 +161,19 @@ module Wurk
144
161
  Wurk.configuration.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
145
162
  end
146
163
  end
164
+
165
+ # Invoke the Pro recovery callback registered via config.super_fetch! { }.
166
+ # No-op unless one is registered. `jobstr` is the raw job JSON so a Pro
167
+ # `|jobstr, pill|` block sees exactly what Sidekiq Pro hands it.
168
+ def fire_super_fetch(config, payload, pill)
169
+ cb = config.super_fetch_callback
170
+ return unless cb
171
+
172
+ jobstr = payload.is_a?(::String) ? payload : Wurk.dump_json(payload)
173
+ cb.call(jobstr, pill)
174
+ rescue StandardError => e
175
+ config.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
176
+ end
147
177
  end
148
178
  end
149
179
  end
@@ -5,6 +5,7 @@ require_relative 'context'
5
5
  require_relative 'job_logger'
6
6
  require_relative 'job_retry'
7
7
  require_relative 'keys'
8
+ require_relative 'profiler'
8
9
 
9
10
  module Wurk
10
11
  # Inside each Manager, N Processors run in parallel. Each owns one thread,
@@ -206,14 +207,12 @@ module Wurk
206
207
  @retrier.global(jobstr, queue) do
207
208
  @job_logger.call(job_hash, queue) do
208
209
  stats(jobstr, queue) do
209
- @reloader.call do
210
- klass = Object.const_get(job_hash['class'])
211
- instance = klass.new
212
- instance.jid = job_hash['jid']
213
- instance.bid = job_hash['bid'] if instance.respond_to?(:bid=)
214
- instance._context = self
215
- @retrier.local(instance, jobstr, queue) do
216
- yield instance
210
+ Wurk::Profiler.call(job_hash) do
211
+ @reloader.call do
212
+ instance = build_instance(job_hash)
213
+ @retrier.local(instance, jobstr, queue) do
214
+ yield instance
215
+ end
217
216
  end
218
217
  end
219
218
  end
@@ -222,6 +221,16 @@ module Wurk
222
221
  end
223
222
  end
224
223
 
224
+ # Instantiate the worker and wire its per-job context. Extracted from
225
+ # dispatch so the dispatch onion stays readable.
226
+ def build_instance(job_hash)
227
+ instance = Object.const_get(job_hash['class']).new
228
+ instance.jid = job_hash['jid']
229
+ instance.bid = job_hash['bid'] if instance.respond_to?(:bid=)
230
+ instance._context = self
231
+ instance
232
+ end
233
+
225
234
  def execute_job(instance, job_hash, queue)
226
235
  @capsule.server_middleware.invoke(instance, job_hash, queue) do
227
236
  instance.perform(*job_hash['args'])
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'keys'
4
+ require_relative 'profiler'
5
+
6
+ module Wurk
7
+ # Read-only view over stored job profiles (Sidekiq 8.0+ data API, spec §19.8).
8
+ # `ProfileSet` enumerates the `profiles` ZSET, purging expired members first;
9
+ # `ProfileRecord` wraps one `<token>-<jid>` HASH.
10
+ class ProfileSet
11
+ include Enumerable
12
+
13
+ # Snapshot the (non-expired) member keys at construction. ZREMRANGEBYSCORE
14
+ # drops members whose expiry score has passed before we read the rest.
15
+ def initialize
16
+ @keys = Wurk.redis do |conn|
17
+ conn.call('ZREMRANGEBYSCORE', Keys::PROFILES, '-inf', "(#{::Time.now.to_i}")
18
+ conn.call('ZRANGE', Keys::PROFILES, 0, -1)
19
+ end
20
+ end
21
+
22
+ def size = @keys.size
23
+
24
+ def each
25
+ return enum_for(:each) unless block_given?
26
+
27
+ Wurk.redis do |conn|
28
+ @keys.each do |key|
29
+ raw = conn.call('HGETALL', key)
30
+ hash = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
31
+ yield ProfileRecord.new(hash) unless hash.empty?
32
+ end
33
+ end
34
+ end
35
+ end
36
+
37
+ # One profile record: the metadata fields of a `<token>-<jid>` HASH plus
38
+ # lazy access to the gzipped gecko blob. Spec §19.8.
39
+ class ProfileRecord
40
+ attr_reader :jid, :type, :token, :size, :elapsed
41
+
42
+ def initialize(hash)
43
+ @hash = hash
44
+ @jid = hash['jid']
45
+ @type = hash['type']
46
+ @token = hash['token']
47
+ @size = hash['size'].to_i
48
+ @elapsed = hash['elapsed'].to_i
49
+ end
50
+
51
+ def started_at
52
+ ts = @hash['started_at']
53
+ ::Time.at(ts.to_i) unless ts.nil? || ts.empty?
54
+ end
55
+
56
+ def key = Wurk::Profiler.profile_key(@token, @jid)
57
+
58
+ # The stored blob is gzipped gecko JSON. `data` returns the raw (gzipped)
59
+ # bytes — the web layer streams them straight to the browser with a gzip
60
+ # Content-Encoding. Returns nil if the HASH expired between list and read.
61
+ def data
62
+ Wurk.redis { |conn| conn.call('HGET', key, 'data') }
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'securerandom'
4
+ require 'zlib'
5
+ require 'stringio'
6
+ require 'tempfile'
7
+ require_relative 'keys'
8
+
9
+ module Wurk
10
+ # Job profiling (Sidekiq 8.0+, OSS). When a job is pushed with a `profile`
11
+ # option, the processor wraps `perform` in a Vernier capture; the resulting
12
+ # Firefox-profiler (gecko) JSON is gzipped and stored so the dashboard can
13
+ # hand it to https://profiler.firefox.com for flame-graph inspection.
14
+ #
15
+ # Redis schema (spec §1.7), wire-compat with Sidekiq's Profiles pane:
16
+ #
17
+ # profiles ZSET member = "<token>-<jid>", score = expiry epoch
18
+ # <token>-<jid> HASH jid, type, token, started_at, elapsed, size,
19
+ # sid, data (gzipped gecko JSON)
20
+ #
21
+ # Capture is a no-op unless the `vernier` gem is loaded — profiling is an
22
+ # opt-in, dev/staging tool, so vernier stays an optional dependency.
23
+ module Profiler
24
+ # Stored profiles live this long (score = now + TTL); ProfileSet purges
25
+ # expired members on read.
26
+ TTL = 7 * 24 * 60 * 60 # 7 days
27
+
28
+ class << self
29
+ # Server-side hook called from Processor#dispatch. Returns the perform
30
+ # result. Only captures when the job opted in AND vernier is present —
31
+ # otherwise it's a plain `yield`. Crucially there is NO blanket rescue
32
+ # here: the job's own exceptions (a normal failure, or JobRetry::Skip
33
+ # from the interrupt/expiry middleware) must propagate untouched so the
34
+ # retry/skip flow works. Only the storage step is made failure-safe
35
+ # (see #safe_store).
36
+ def call(job_hash, &)
37
+ label = job_hash['profile']
38
+ return yield unless label && defined?(::Vernier)
39
+
40
+ capture(job_hash, label, &)
41
+ end
42
+
43
+ # Persists a profile. Extracted from capture so it is unit-testable
44
+ # without vernier: tests pass a ready gecko JSON blob. The wide keyword
45
+ # list mirrors the HASH fields one-to-one — collapsing them into an
46
+ # options hash would just hide the schema.
47
+ # rubocop:disable Metrics/ParameterLists
48
+ def store(jid:, type:, gecko_json:, started_at:, elapsed_ms:, token: SecureRandom.hex(8),
49
+ sid: Wurk.configuration[:identity], pool: nil)
50
+ key = profile_key(token, jid)
51
+ gz = gzip(gecko_json)
52
+ with_pool(pool) do |conn|
53
+ conn.call('HSET', key, 'jid', jid, 'type', type, 'token', token,
54
+ 'started_at', started_at.to_i, 'elapsed', elapsed_ms.to_i,
55
+ 'size', gz.bytesize, 'sid', sid.to_s, 'data', gz)
56
+ conn.call('EXPIRE', key, TTL)
57
+ conn.call('ZADD', Keys::PROFILES, (now + TTL).to_i, key)
58
+ end
59
+ key
60
+ end
61
+ # rubocop:enable Metrics/ParameterLists
62
+
63
+ def profile_key(token, jid)
64
+ "#{token}-#{jid}"
65
+ end
66
+
67
+ def gzip(str)
68
+ io = StringIO.new(+'', 'wb')
69
+ gz = Zlib::GzipWriter.new(io)
70
+ gz.write(str)
71
+ gz.close
72
+ io.string
73
+ end
74
+
75
+ def gunzip(bytes)
76
+ Zlib::GzipReader.new(StringIO.new(bytes)).read
77
+ end
78
+
79
+ private
80
+
81
+ # Wrap the block in a Vernier capture, write the gecko JSON to a tempfile
82
+ # (Vernier serializes on block exit), then store it. Only reached when
83
+ # vernier is loaded.
84
+ def capture(job_hash, label)
85
+ retval = nil
86
+ started = now
87
+ elapsed_ms = nil
88
+ json = profile_to_json do
89
+ t0 = monotonic_ms
90
+ retval = yield
91
+ elapsed_ms = monotonic_ms - t0
92
+ end
93
+ safe_store(job_hash, label, json, started, elapsed_ms)
94
+ retval
95
+ end
96
+
97
+ # The job already ran successfully by the time we get here; a Redis hiccup
98
+ # persisting the profile must not turn a green job red. Job exceptions
99
+ # never reach this method — they propagate out of `capture`'s yield.
100
+ def safe_store(job_hash, label, json, started, elapsed_ms)
101
+ store(jid: job_hash['jid'], type: label.to_s, gecko_json: json,
102
+ started_at: started, elapsed_ms: elapsed_ms)
103
+ rescue StandardError => e
104
+ Wurk.configuration.handle_exception(e, context: 'Wurk::Profiler')
105
+ end
106
+
107
+ def profile_to_json(&)
108
+ Tempfile.create(['wurk-profile', '.json']) do |file|
109
+ ::Vernier.profile(out: file.path, &)
110
+ File.read(file.path)
111
+ end
112
+ end
113
+
114
+ def with_pool(pool, &)
115
+ pool ? pool.with(&) : Wurk.redis(&)
116
+ end
117
+
118
+ def now
119
+ ::Time.now.to_f
120
+ end
121
+
122
+ def monotonic_ms
123
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :float_millisecond)
124
+ end
125
+ end
126
+ end
127
+ end
data/lib/wurk/railtie.rb CHANGED
@@ -1,16 +1,23 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "rails/railtie"
3
+ require 'rails/railtie'
4
4
 
5
5
  module Wurk
6
6
  # Boot the swarm after the host app has fully initialized.
7
7
  # Skip when: WURK_DISABLED=1, Rails console mode, or Rails test env.
8
8
  # See docs/idea/03-process-model.md for the exact ordering.
9
9
  class Railtie < ::Rails::Railtie
10
- config.after_initialize do |app|
11
- next if ENV["WURK_DISABLED"] == "1"
12
- next if defined?(::Rails::Console)
13
- next if ::Rails.env.test?
10
+ # This Rails process forks the workers, so it IS the server. Enter server
11
+ # mode BEFORE config/initializers load — otherwise the app's
12
+ # `Sidekiq.configure_server` blocks gate on `config.server?` (still false)
13
+ # and are silently dropped. Gated identically to the swarm boot: a process
14
+ # that won't run workers (disabled / console / test) is not a server.
15
+ initializer 'wurk.server_mode', before: :load_config_initializers do
16
+ Wurk.enter_server_mode unless Wurk::Railtie.skip_boot?
17
+ end
18
+
19
+ config.after_initialize do |_app|
20
+ next if Wurk::Railtie.skip_boot?
14
21
 
15
22
  swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology)
16
23
  swarm.boot
@@ -24,5 +31,12 @@ module Wurk
24
31
  Wurk.configuration.logger.error { "wurk supervisor thread died: #{e.class}: #{e.message}" }
25
32
  end
26
33
  end
34
+
35
+ # A process that won't run workers isn't a server: skip both server mode
36
+ # and the swarm boot. Console mode is detected reliably here — the console
37
+ # command file defines ::Rails::Console before initializers run.
38
+ def self.skip_boot?
39
+ ENV['WURK_DISABLED'] == '1' || defined?(::Rails::Console) || ::Rails.env.test?
40
+ end
27
41
  end
28
42
  end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'redis-client'
4
+ require 'redis_client/decorator'
5
+
6
+ module Wurk
7
+ # Command-method compatibility for connections yielded by `Wurk.redis` /
8
+ # `Sidekiq.redis` (#204). Sidekiq 7+ yields RedisClientAdapter::CompatClient,
9
+ # so third-party gems write `conn.hgetall(...)` / `conn.smembers(...)` —
10
+ # method-style commands raw RedisClient doesn't answer (it only has #call).
11
+ # Every pooled connection is wrapped in this decorator at build time
12
+ # (RedisPool#build_client); wurk's own hot paths keep using #call, which the
13
+ # decorator forwards with a single delegation hop.
14
+ #
15
+ # Mirrors sidekiq-8.1.6 lib/sidekiq/redis_client_adapter.rb byte-for-byte in
16
+ # behavior: same fast-path command list, same deprecation warning, same
17
+ # error constants (gems rescue `Sidekiq::RedisClientAdapter::BaseError`).
18
+ class RedisClientAdapter
19
+ BaseError = RedisClient::Error
20
+ CommandError = RedisClient::CommandError
21
+
22
+ DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
23
+
24
+ module CompatMethods
25
+ def info
26
+ @client.call('INFO') { |i| i.lines(chomp: true).map { |l| l.split(':', 2) }.select { |l| l.size == 2 }.to_h }
27
+ end
28
+
29
+ def evalsha(sha, keys, argv)
30
+ @client.call('EVALSHA', sha, keys.size, *keys, *argv)
31
+ end
32
+
33
+ # The Redis commands Sidekiq itself uses — defined eagerly so the
34
+ # common ones skip method_missing. Same list as upstream.
35
+ USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
36
+ get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
37
+ lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
38
+ publish rpop rpush sadd scard script set sismember smembers
39
+ srem ttl type unlink zadd zcard zincrby zrange zrem
40
+ zremrangebyrank zremrangebyscore].freeze
41
+
42
+ USED_COMMANDS.each do |name|
43
+ define_method(name) do |*args, **kwargs|
44
+ @client.call(name, *args, **kwargs)
45
+ end
46
+ end
47
+
48
+ private
49
+
50
+ # `conn.hmset(...)` instead of redis-client's native `conn.call("hmset", ...)`.
51
+ def method_missing(*args, &)
52
+ if DEPRECATED_COMMANDS.include?(args.first)
53
+ warn("[sidekiq#5788] Redis has deprecated the `#{args.first}` command, called at #{caller(1..1)}")
54
+ end
55
+ @client.call(*args, &)
56
+ end
57
+ ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
58
+
59
+ def respond_to_missing?(_name, _include_private = false)
60
+ super # We can't tell what is a valid command.
61
+ end
62
+ end
63
+
64
+ CompatClient = RedisClient::Decorator.create(CompatMethods)
65
+
66
+ class CompatClient
67
+ def config
68
+ @client.config
69
+ end
70
+ end
71
+ end
72
+ end