wurk 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -1
  3. data/app/controllers/concerns/wurk/stream_concurrency_guard.rb +22 -5
  4. data/lib/wurk/capsule.rb +40 -12
  5. data/lib/wurk/client/buffered.rb +7 -0
  6. data/lib/wurk/client.rb +102 -32
  7. data/lib/wurk/component.rb +34 -2
  8. data/lib/wurk/configuration.rb +45 -6
  9. data/lib/wurk/context.rb +1 -1
  10. data/lib/wurk/engine.rb +41 -1
  11. data/lib/wurk/errors.rb +15 -0
  12. data/lib/wurk/fetcher/reaper.rb +12 -2
  13. data/lib/wurk/fetcher/reliable.rb +322 -60
  14. data/lib/wurk/fetcher.rb +6 -0
  15. data/lib/wurk/heartbeat.rb +2 -2
  16. data/lib/wurk/job_logger.rb +16 -7
  17. data/lib/wurk/job_set.rb +3 -2
  18. data/lib/wurk/job_util.rb +44 -24
  19. data/lib/wurk/launcher.rb +116 -72
  20. data/lib/wurk/leader.rb +36 -6
  21. data/lib/wurk/logger.rb +1 -1
  22. data/lib/wurk/lua/loader.rb +9 -3
  23. data/lib/wurk/metrics/accumulator.rb +95 -0
  24. data/lib/wurk/metrics/flusher.rb +70 -0
  25. data/lib/wurk/metrics/history.rb +98 -29
  26. data/lib/wurk/metrics/statsd.rb +32 -18
  27. data/lib/wurk/middleware/chain.rb +31 -14
  28. data/lib/wurk/middleware/poison_pill.rb +23 -2
  29. data/lib/wurk/pool_checkout.rb +10 -0
  30. data/lib/wurk/processor.rb +79 -11
  31. data/lib/wurk/profiler.rb +6 -2
  32. data/lib/wurk/queue.rb +8 -0
  33. data/lib/wurk/redis_client_adapter.rb +1 -1
  34. data/lib/wurk/redis_pool.rb +2 -1
  35. data/lib/wurk/shutdown_gate.rb +79 -0
  36. data/lib/wurk/stats.rb +5 -1
  37. data/lib/wurk/swarm/child_boot.rb +24 -4
  38. data/lib/wurk/swarm.rb +88 -14
  39. data/lib/wurk/version.rb +1 -1
  40. data/lib/wurk/web/config.rb +11 -7
  41. data/lib/wurk.rb +2 -8
  42. data/vendor/assets/dashboard/assets/{ArgsValue-D74zX0MI.js → ArgsValue-CcR2ya6e.js} +1 -1
  43. data/vendor/assets/dashboard/assets/{BatchDetail-OmC5NPgw.js → BatchDetail-CUXJUQ3Q.js} +1 -1
  44. data/vendor/assets/dashboard/assets/{Batches-CIpai7St.js → Batches-Cxan6Ngw.js} +1 -1
  45. data/vendor/assets/dashboard/assets/{Busy-A_kwSR6Q.js → Busy-DC5EGM0g.js} +1 -1
  46. data/vendor/assets/dashboard/assets/{Cron-BG7HTqlp.js → Cron-Dlt8tXJA.js} +1 -1
  47. data/vendor/assets/dashboard/assets/Dashboard-DNLu_WCg.js +1 -0
  48. data/vendor/assets/dashboard/assets/{Dead-8J21jMyK.js → Dead-dZ7VGlKS.js} +1 -1
  49. data/vendor/assets/dashboard/assets/{Extension-B4Q9FIQu.js → Extension-DaFpEIJf.js} +1 -1
  50. data/vendor/assets/dashboard/assets/{FilterBox-Fh_Ae7UW.js → FilterBox-CO3aYWIq.js} +1 -1
  51. data/vendor/assets/dashboard/assets/{JobDetailModal-Ceng0PMB.js → JobDetailModal-DSWbT6G0.js} +1 -1
  52. data/vendor/assets/dashboard/assets/{Limiters-CruDWvNZ.js → Limiters-Cb4PKXNR.js} +1 -1
  53. data/vendor/assets/dashboard/assets/Metrics-CCGzgCsT.js +1 -0
  54. data/vendor/assets/dashboard/assets/{Modal-CN3rdKA_.js → Modal-B86q6ruL.js} +1 -1
  55. data/vendor/assets/dashboard/assets/{PageHeader-C44KNMGm.js → PageHeader-fPrCcp_-.js} +1 -1
  56. data/vendor/assets/dashboard/assets/{Profiles-xEVTyS2N.js → Profiles-BnS82nR_.js} +1 -1
  57. data/vendor/assets/dashboard/assets/{Queues-D86FYohJ.js → Queues-CIyPevOy.js} +1 -1
  58. data/vendor/assets/dashboard/assets/{Retries-Bz1O1D-i.js → Retries-DopwXkXl.js} +1 -1
  59. data/vendor/assets/dashboard/assets/{Scheduled-B6h2akTu.js → Scheduled-1-Z7i1zE.js} +1 -1
  60. data/vendor/assets/dashboard/assets/{Search-OOu22e5s.js → Search-ByA6eTma.js} +1 -1
  61. data/vendor/assets/dashboard/assets/{Skeleton-DzR7XNxz.js → Skeleton-bC7HfQ9r.js} +1 -1
  62. data/vendor/assets/dashboard/assets/{charts-BVHHGof7.js → charts-CLLzJ7vK.js} +1 -1
  63. data/vendor/assets/dashboard/assets/index-B1N8hQUh.js +141 -0
  64. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-dVPGEWzn.js → useResetPageOnEmpty-DpBjkf6_.js} +1 -1
  65. data/vendor/assets/dashboard/assets/{useSort-BeYbztkN.js → useSort-DvpwuNQE.js} +1 -1
  66. data/vendor/assets/dashboard/index.html +2 -2
  67. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  68. metadata +30 -26
  69. data/vendor/assets/dashboard/assets/Dashboard-A_ToqHoo.js +0 -1
  70. data/vendor/assets/dashboard/assets/Metrics-CIT7VCoN.js +0 -1
  71. data/vendor/assets/dashboard/assets/index-D_lSDwKw.js +0 -141
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6ec30c1dfc5983b7182b0f5765ec40acd60d9682a8ef69dd2f5f5abb547c9a59
4
- data.tar.gz: 1d6a8578a6743782463582302d240a3be94d4474a2b1ed8c26209ebcc78c2836
3
+ metadata.gz: '0995fd5827870e14584104a8998db58e9927339197460596567c05f2dacc7da0'
4
+ data.tar.gz: 8159d245826bf646a9c6025e4e06a49717f957aabe18dad5d2a5b58ed99818c0
5
5
  SHA512:
6
- metadata.gz: 529fb49080a16a23f3834b99f37d4f34dd2d080e772529cb9afff46f1f0ca1d9694b35804d1a538fdf7ccd9a59e297c0ab3bd712792874fe37a05792fb991a3b
7
- data.tar.gz: a75598d81e4182d798fbf220febbf50c2960134f3273f238f4182fcf4f03baac65613cfc3efdbeb871e14a38f0d20a775b90962315a8a21473a44f5cb4d0c037
6
+ metadata.gz: d45c20c9a8c29695c7ea2e04066dda58454fea0ce56453ed866cd111e38977fb1739ae47429a60b68380fac2f4d7fe85f232b420f92f95cba330c389ee435b5b
7
+ data.tar.gz: 94c5599bc28408e1e5eb30967d1b0fad4806d415a17e515774c057120d0833660f3c2860d9cf0cb2ea320ae3a27bd9ce683385ce5ace1c347d46530a96cd9a1a
data/README.md CHANGED
@@ -6,7 +6,7 @@
6
6
 
7
7
  <p align="center"><strong>Wurk, wurk.</strong> 🪓 <em>Ready to work. Zug zug.</em></p>
8
8
 
9
- <p align="center"><strong>A 100% drop-in replacement for Sidekiq + Sidekiq Pro + Sidekiq Enterprise. Free forever. Faster.</strong></p>
9
+ <p align="center"><strong>A 100% drop-in replacement for Sidekiq + Sidekiq Pro + Sidekiq Enterprise. Free forever.</strong></p>
10
10
 
11
11
  <div align="center">
12
12
 
@@ -21,6 +21,8 @@
21
21
 
22
22
  Wurk is wire-compatible with Sidekiq — same Redis keys, same job JSON, same Ruby DSL. Swap one line in your `Gemfile` and your existing jobs, batches, limiters, cron entries, and live Redis data keep working untouched. The Pro and Enterprise feature sets ship in the same free gem, with no license check and no tiers.
23
23
 
24
+ **On speed:** Wurk is not currently faster than stock Sidekiq — it runs at roughly 0.87×–1.02× depending on workload shape, with parity on CPU and I/O but still behind on framework overhead (noop) and boot time. Numbers, method, and the reproduction command are in [docs/benchmarks.md](docs/benchmarks.md); run them yourself with `rake bench:vs_sidekiq`.
25
+
24
26
  ## Install
25
27
 
26
28
  ```ruby
@@ -7,28 +7,45 @@ module Wurk
7
7
  # we 503 with Retry-After — the SPA's EventSource reconnects (and its polling
8
8
  # fallback honors Retry-After) once a slot frees. Per-process is the right
9
9
  # scope: it's this process's own thread pool we're protecting.
10
+ #
11
+ # Slots are held as thread references rather than tallied in a counter so the
12
+ # cap can heal itself. A stream whose thread is killed mid-flight never
13
+ # reaches the `ensure` below (Puma hard-reaps worker threads past
14
+ # `force_shutdown_after`, and a thread killed inside an uninterruptible read
15
+ # can skip its ensure), which a counter would record as a slot held by nobody
16
+ # — ten of those and `/api/stream` 503s for the life of the process. A dead
17
+ # holder is instead evicted by the next acquire.
10
18
  module StreamConcurrencyGuard
11
19
  extend ActiveSupport::Concern
12
20
 
13
21
  MAX_CONCURRENT_STREAMS = 10
14
22
  RETRY_AFTER_SECONDS = 3
15
23
 
16
- @open = 0
24
+ @holders = []
17
25
  @lock = Mutex.new
18
26
 
19
27
  class << self
20
- # Reserve a stream slot; false when the cap is already reached.
28
+ # Reserve a stream slot for the calling thread; false when the cap is
29
+ # already reached by threads that are still alive.
21
30
  def acquire
22
31
  @lock.synchronize do
23
- return false if @open >= MAX_CONCURRENT_STREAMS
32
+ @holders.keep_if(&:alive?)
33
+ return false if @holders.size >= MAX_CONCURRENT_STREAMS
24
34
 
25
- @open += 1
35
+ @holders << Thread.current
26
36
  true
27
37
  end
28
38
  end
29
39
 
40
+ # Drops one slot held by the calling thread. Acquire and release always
41
+ # bracket a single block on one thread (`#with_stream_slot`), so a call
42
+ # from a thread holding nothing is a no-op rather than a slot taken away
43
+ # from whoever is actually streaming.
30
44
  def release
31
- @lock.synchronize { @open -= 1 if @open.positive? }
45
+ @lock.synchronize do
46
+ index = @holders.rindex(Thread.current)
47
+ @holders.delete_at(index) if index
48
+ end
32
49
  end
33
50
  end
34
51
 
data/lib/wurk/capsule.rb CHANGED
@@ -17,6 +17,11 @@ module Wurk
17
17
  attr_reader :name, :queues, :mode, :weights, :config
18
18
  attr_accessor :concurrency, :fetcher
19
19
 
20
+ # Capsule-hosted components (Manager, Processor, Fetcher) hand their capsule
21
+ # to Component as `config`, and `safe_thread` reads the priority off it.
22
+ # Sidekiq delegates the same accessor (capsule.rb:30).
23
+ def thread_priority = @config.thread_priority
24
+
20
25
  def initialize(name, config)
21
26
  @name = name.to_s
22
27
  @config = config
@@ -25,8 +30,15 @@ module Wurk
25
30
  @mode = :strict
26
31
  @weights = { 'default' => 0 }
27
32
  @fetcher = nil
28
- @redis_pool = nil
29
- @fetch_redis_pool = nil
33
+ # One mutable Hash rather than two ivars: the pools are the only part of a
34
+ # capsule that legitimately changes after Configuration#freeze! — fork
35
+ # closes them, Launcher#stop releases them, an embedded host that boots
36
+ # again rebuilds them. `Object#freeze` is shallow, so the Hash stays
37
+ # writable and freezing a capsule keeps meaning "no more configuration"
38
+ # instead of "these sockets are yours forever". Before this, a reset on a
39
+ # frozen capsule disconnected the pool and then raised FrozenError on the
40
+ # memo, leaving `redis_pool` answering with a shut-down pool for good.
41
+ @pools = {}
30
42
  @client_chain = nil
31
43
  @server_chain = nil
32
44
  end
@@ -64,9 +76,26 @@ module Wurk
64
76
  # by hand; centralizing it here covers the standalone CLI and embedded
65
77
  # paths too (the bug behind a nil `fetcher` in `exe/wurk`). Idempotent.
66
78
  def prepare!
79
+ prepare_shared!
67
80
  @fetcher ||= build_fetcher
68
81
  redis_pool
69
82
  fetch_redis_pool
83
+ self
84
+ end
85
+
86
+ # The half of `prepare!` a forking parent can run on every child's behalf:
87
+ # the chains are a pure function of this capsule's identity, not of the slot
88
+ # (queues + concurrency) a swarm child is assigned later, and `copy_for`
89
+ # opens nothing. Run before the fork, the entries are allocated once and
90
+ # inherited copy-on-write instead of rebuilt in every child.
91
+ #
92
+ # The rest of `prepare!` deliberately stays post-fork: both pools are sized
93
+ # off the slot's concurrency and one built here would hand every child an
94
+ # inherited socket, and `build_fetcher` fires the host's
95
+ # `config[:fetch_setup]` hook, which is per-child — running it in the parent
96
+ # would let a custom fetcher snapshot the wrong queues, or leak whatever the
97
+ # hook opened across the fork. Idempotent.
98
+ def prepare_shared!
70
99
  client_middleware
71
100
  server_middleware
72
101
  self
@@ -99,7 +128,7 @@ module Wurk
99
128
  MIN_POOL_SIZE = 10
100
129
 
101
130
  def redis_pool
102
- @redis_pool ||= build_pool(size: main_pool_size, name: "#{@name}-main")
131
+ @pools[:main] ||= build_pool(size: main_pool_size, name: "#{@name}-main")
103
132
  end
104
133
 
105
134
  # Dedicated pool for the reliable fetcher's blocking BLMOVE: one slot per
@@ -107,18 +136,17 @@ module Wurk
107
136
  # fetch at once. Keeping fetch off the main pool is what lets an idle worker
108
137
  # hold zero main-pool connections again.
109
138
  def fetch_redis_pool
110
- @fetch_redis_pool ||= build_pool(size: @concurrency, name: "#{@name}-fetch")
139
+ @pools[:fetch] ||= build_pool(size: @concurrency, name: "#{@name}-fetch")
111
140
  end
112
141
 
113
- # Disconnect and drop cached pools. Called by Wurk::Swarm just before
114
- # fork (parent side: close inherited sockets) and just after fork
115
- # (child side: rebuild lazily). Connection_pool#shutdown is terminal,
116
- # so dropping the reference is required — `redis_pool` will rebuild.
142
+ # Disconnect and drop cached pools. Called by Wurk::Swarm just before fork
143
+ # (parent side: close inherited sockets), just after fork (child side:
144
+ # rebuild lazily), and by Launcher#stop (release what this process held).
145
+ # Connection_pool#shutdown is terminal, so dropping the reference is
146
+ # required — `redis_pool` will rebuild.
117
147
  def reset_redis_pools!
118
- @redis_pool&.disconnect!
119
- @redis_pool = nil
120
- @fetch_redis_pool&.disconnect!
121
- @fetch_redis_pool = nil
148
+ @pools.each_value(&:disconnect!)
149
+ @pools.clear
122
150
  end
123
151
 
124
152
  def redis(idempotent: false, &)
@@ -119,6 +119,13 @@ module Wurk
119
119
  @overflow_mode = nil
120
120
  @buffer_client_factory = nil
121
121
  end
122
+ # Stop before dropping: an unstopped drainer thread survives with
123
+ # its factory nil'd out from under it and ticks forever against
124
+ # nothing, leaking the thread and everything its closure retains.
125
+ install_mutex.synchronize do
126
+ @drainer&.stop
127
+ @drainer = nil
128
+ end
122
129
  end
123
130
 
124
131
  # Fork hook, called from the `Process._fork` prepend below and from
data/lib/wurk/client.rb CHANGED
@@ -26,6 +26,20 @@ module Wurk
26
26
  SCHEDULED_BATCH_SIZE = 100
27
27
  SPREAD_INTERVAL_FLOOR = 5
28
28
 
29
+ # Batched (`bid`) payloads per EVALSHA pipeline. Ours, not Sidekiq's — it
30
+ # has no batches. Sized to DEFAULT_BATCH_SIZE so no existing caller's
31
+ # round-trip count moves: `push_bulk` already hands #raw_push at most that
32
+ # many payloads, so its batched pipeline stays exactly one round trip.
33
+ #
34
+ # The cap is for the one path that isn't pre-sliced: `autoflush = true`
35
+ # buffers a whole `Batch#jobs` block, so #flush_batched can be handed an
36
+ # unbounded payload set. Unsliced that is one pipeline holding every
37
+ # command and every reply in memory at once, and — Lua being atomic and
38
+ # single-threaded — one uninterrupted server-side sweep that blocks every
39
+ # other client for its duration. Same reasoning as the LIMIT on
40
+ # RELIABLE_SCHEDULE_PROMOTE.
41
+ BATCH_PIPELINE_SLICE = 1_000
42
+
29
43
  # Thread-local slot holding the payloads of the current push whose Redis
30
44
  # write is confirmed applied. {Client::Buffered} subtracts them from the set
31
45
  # it re-buffers when a *later* phase of the same push loses the connection,
@@ -138,7 +152,21 @@ module Wurk
138
152
 
139
153
  private
140
154
 
155
+ # #push and #push_bulk verify at different points and both match Sidekiq
156
+ # exactly: push walks the payload the chain handed back (sidekiq
157
+ # client.rb:101 — normalize → middleware → verify → raw_push), bulk walks it
158
+ # inside the innermost block (sidekiq client.rb:165). Push used to do both,
159
+ # and since `strict_args_mode` defaults to :raise the second full recursive
160
+ # args walk was never skipped.
161
+ #
162
+ # Bulk keeps its walk inside the block on purpose: a client middleware that
163
+ # halts the job there short-circuits the walk, and hoisting it out would
164
+ # raise on args that middleware was about to drop.
141
165
  def invoke_chain(normed)
166
+ @chain.invoke(normed['class'], normed, normed['queue'], pool) { normed }
167
+ end
168
+
169
+ def invoke_chain_verified(normed)
142
170
  @chain.invoke(normed['class'], normed, normed['queue'], pool) do
143
171
  verify_json(normed)
144
172
  normed
@@ -191,7 +219,7 @@ module Wurk
191
219
  item = base.merge('args' => job_args)
192
220
  item['at'] = ats[idx] if ats
193
221
  normed = normalize_item(item)
194
- invoke_chain(normed)
222
+ invoke_chain_verified(normed)
195
223
  end
196
224
  end
197
225
 
@@ -314,6 +342,27 @@ module Wurk
314
342
  push_batched_pipelined(conn, batched, now) unless batched.empty?
315
343
  end
316
344
 
345
+ # One pipeline per BATCH_PIPELINE_SLICE payloads, each marked delivered the
346
+ # moment its reply is in — same contract as push_plain_group, and for the
347
+ # same reason: a slice that Redis already accepted must stay out of the
348
+ # reliable_push ledger, or a later slice's failure would report it as
349
+ # undelivered.
350
+ def push_batched_pipelined(conn, batched, now)
351
+ batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
352
+ eval_batched_slice(conn) { |pipe, eval_method| push_batched(pipe, slice, now, eval_method: eval_method) }
353
+ mark_delivered(slice)
354
+ end
355
+ end
356
+
357
+ # Same slicing and NOSCRIPT recovery as push_batched_pipelined, for the
358
+ # scheduled batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
359
+ def push_batched_scheduled_pipelined(conn, batched)
360
+ batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
361
+ eval_batched_slice(conn) { |pipe, eval_method| push_batched_scheduled(pipe, slice, eval_method: eval_method) }
362
+ mark_delivered(slice)
363
+ end
364
+ end
365
+
317
366
  # Outside of test boots and `SCRIPT FLUSH` the rescue branch is dead
318
367
  # code; the eager `script_load_all` after fork keeps the script cache
319
368
  # hot for the life of the connection. The retry uses EVAL (source-embedded)
@@ -321,24 +370,18 @@ module Wurk
321
370
  # NOSCRIPT a second time under heavy CI load (WorkerTest 3.4/7.2 flake).
322
371
  # `script_load_all` still primes the cache so the *next* pipeline returns
323
372
  # to the EVALSHA fast path.
324
- def push_batched_pipelined(conn, batched, now)
325
- conn.pipelined { |pipe| push_batched(pipe, batched, now) }
326
- rescue RedisClient::CommandError => e
327
- raise unless e.message.to_s.start_with?('NOSCRIPT')
328
-
329
- Wurk::Lua::Loader.script_load_all(conn)
330
- conn.pipelined { |pipe| push_batched(pipe, batched, now, eval_method: :eval_with_source) }
331
- end
332
-
333
- # Same NOSCRIPT-recovery shape as push_batched_pipelined, for the scheduled
334
- # batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
335
- def push_batched_scheduled_pipelined(conn, batched)
336
- conn.pipelined { |pipe| push_batched_scheduled(pipe, batched) }
373
+ #
374
+ # Replaying the slice is safe precisely because every command in it is the
375
+ # same script: a flushed cache NOSCRIPTs all of them and applies none.
376
+ # Recovery is per slice, so the slices already acknowledged above are never
377
+ # re-sent. Mirrors Fetcher::Reliable#requeue_pipelined.
378
+ def eval_batched_slice(conn)
379
+ conn.pipelined { |pipe| yield(pipe, :eval_cached) }
337
380
  rescue RedisClient::CommandError => e
338
381
  raise unless e.message.to_s.start_with?('NOSCRIPT')
339
382
 
340
383
  Wurk::Lua::Loader.script_load_all(conn)
341
- conn.pipelined { |pipe| push_batched_scheduled(pipe, batched, eval_method: :eval_with_source) }
384
+ conn.pipelined { |pipe| yield(pipe, :eval_with_source) }
342
385
  end
343
386
 
344
387
  # One pipeline per queue, marked delivered the moment its reply is in.
@@ -355,27 +398,48 @@ module Wurk
355
398
  # Cost is a round trip per distinct queue. The single-queue push — every
356
399
  # `perform_async`, every same-class `push_bulk` — still writes exactly the
357
400
  # one SADD + LPUSH pipeline it did before.
401
+ #
402
+ # `uniform_queue` short-circuits the common case (one job, or many jobs
403
+ # all destined for the same queue) without paying for the `group_by`
404
+ # Hash + per-group Array allocations; only a genuinely mixed-queue batch
405
+ # falls through to grouping.
358
406
  def push_plain(conn, payloads, now)
359
- payloads.group_by { |j| j['queue'] }.each do |queue, jobs|
360
- serialized = jobs.map do |j|
361
- j['enqueued_at'] = now
362
- Wurk.dump_json(j)
363
- end
364
- conn.pipelined do |pipe|
365
- pipe.call('SADD', 'queues', queue)
366
- pipe.call('LPUSH', "queue:#{queue}", *serialized)
367
- end
368
- mark_delivered(jobs)
407
+ queue = uniform_queue(payloads)
408
+ return push_plain_group(conn, queue, payloads, now) if queue
409
+
410
+ payloads.group_by { |j| j['queue'] }.each { |q, jobs| push_plain_group(conn, q, jobs, now) }
411
+ end
412
+
413
+ def uniform_queue(payloads)
414
+ first = payloads[0]['queue']
415
+ return first if payloads.size == 1
416
+
417
+ first if payloads.all? { |j| j['queue'] == first }
418
+ end
419
+
420
+ def push_plain_group(conn, queue, jobs, now)
421
+ serialized = jobs.map do |j|
422
+ j['enqueued_at'] = now
423
+ Wurk.dump_json(j)
424
+ end
425
+ conn.pipelined do |pipe|
426
+ pipe.call('SADD', 'queues', queue)
427
+ pipe.call('LPUSH', "queue:#{queue}", *serialized)
369
428
  end
429
+ mark_delivered(jobs)
370
430
  end
371
431
 
372
432
  # Batched jobs route through BATCH_PUSH: increments b-<bid> total+pending,
373
433
  # SADDs jid into the live set, registers the queue, LPUSHes the payload —
374
- # all atomically. One Redis round-trip per job (no pipeline grouping)
375
- # because the lua needs per-job KEYS bound. Acceptable cost: batch
376
- # enqueue is not the hot path; correctness is. `eval_method` is the
434
+ # all atomically. The Lua binds per-job KEYS, so grouping N jobs into one
435
+ # EVALSHA isn't available; they ride one pipeline instead (the `conn` here
436
+ # is always the pipeline #push_batched_pipelined opened), so the cost is
437
+ # N commands and one round trip, not N round trips. `eval_method` is the
377
438
  # Wurk::Lua::Loader entry point (`:eval_cached` for the hot EVALSHA path,
378
- # `:eval_with_source` for the EVAL-source retry).
439
+ # `:eval_with_source` for the EVAL-source retry) — neither reads the reply,
440
+ # which is what makes the pipelined form legal: `eval_cached`'s inline
441
+ # NOSCRIPT rescue can't fire against a buffered call, so recovery is the
442
+ # caller's finalize-time rescue.
379
443
  def push_batched(conn, payloads, now, eval_method: :eval_cached)
380
444
  payloads.each do |j|
381
445
  j['enqueued_at'] = now
@@ -394,9 +458,9 @@ module Wurk
394
458
  # total/pending increment registers the job in its batch at creation, and
395
459
  # the ZADD defers it onto `schedule`. Payload is stripped of `at`/
396
460
  # `enqueued_at` exactly like push_scheduled — `enqueued_at` is stamped fresh
397
- # at promotion, never while the job sits scheduled (spec §7.1). One Redis
398
- # round-trip per job because the Lua binds per-job KEYS; scheduled batch
399
- # enqueue is not the hot path.
461
+ # at promotion, never while the job sits scheduled (spec §7.1). Per-job
462
+ # KEYS again, so one EVALSHA per job pipelined by
463
+ # #push_batched_scheduled_pipelined into one round trip per slice.
400
464
  def push_batched_scheduled(conn, payloads, eval_method: :eval_cached)
401
465
  payloads.each do |j|
402
466
  Wurk::Lua::Loader.public_send(
@@ -436,7 +500,13 @@ module Wurk
436
500
  # evict them, and the drain that does land one counts it then — so booking
437
501
  # them here would inflate the counter on an outage and double-count every
438
502
  # payload that later replays.
503
+ #
504
+ # Resolve the client once for the whole batch and bail before touching a
505
+ # payload: unconfigured is the common case, and the tags below cost two
506
+ # Strings and an Array per job for `increment` to immediately drop.
439
507
  def emit_enqueued(payloads, buffered = nil)
508
+ return if Wurk::Metrics::Statsd.safe_client.nil?
509
+
440
510
  payloads = reject_by_identity(payloads, buffered) if buffered && !buffered.empty?
441
511
  payloads.each do |p|
442
512
  Wurk::Metrics::Statsd.increment(
@@ -37,8 +37,34 @@ module Wurk
37
37
 
38
38
  # --- identity -------------------------------------------------------
39
39
 
40
+ # Base36 `thread.object_id ^ pid` — the id in every log line and the key
41
+ # each Processor publishes its in-flight job under. Constant for the life
42
+ # of a thread inside one process and read several times per job, so it is
43
+ # memoized per thread (frozen: it is used as a Hash key, and an unfrozen
44
+ # String key is duped on every store).
45
+ #
46
+ # The pid is memoized alongside it because the thread that calls fork keeps
47
+ # its thread-locals in the child, where the pid — and therefore the tid —
48
+ # has changed. Without the guard a forked child would report the parent's
49
+ # tid and collide with it in `<identity>:work`.
50
+ #
51
+ # Thread-local, not `Thread#[]`: the latter is fiber-local, so a job that
52
+ # runs inside a Fiber (or any Enumerator) would miss the memo and allocate
53
+ # a fresh String on every read — the identity the memo exists to cache is
54
+ # the thread's, and it does not change when a fiber does.
55
+ def self.tid
56
+ thread = Thread.current
57
+ memo = thread.thread_variable_get(:wurk_tid)
58
+ pid = ::Process.pid
59
+ return memo[1] if memo && memo[0] == pid
60
+
61
+ id = (thread.object_id ^ pid).to_s(36).freeze
62
+ thread.thread_variable_set(:wurk_tid, [pid, id].freeze)
63
+ id
64
+ end
65
+
40
66
  def tid
41
- (Thread.current.object_id ^ ::Process.pid).to_s(36)
67
+ Component.tid
42
68
  end
43
69
 
44
70
  def hostname
@@ -110,10 +136,16 @@ module Wurk
110
136
  # Spawns a named thread that runs `block` under `watchdog(name)`. The
111
137
  # parent must retain the returned Thread; otherwise GC may not, but
112
138
  # report_on_exception is disabled so we don't double-log on death.
139
+ #
140
+ # Priority resolution matches Sidekiq (component.rb:44-48): explicit
141
+ # argument, then `config.thread_priority`, then -1. Ruby's default of 0
142
+ # buys a 100ms timeslice; each negative step halves it, so -1 keeps a
143
+ # CPU-heavy capsule from starving its siblings for a whole tick.
113
144
  def safe_thread(name, priority: nil, &block)
145
+ resolved = priority || config.thread_priority || DEFAULT_THREAD_PRIORITY
114
146
  Thread.new do
115
147
  Thread.current.name = name
116
- Thread.current.priority = priority || DEFAULT_THREAD_PRIORITY
148
+ Thread.current.priority = resolved
117
149
  Thread.current.report_on_exception = false
118
150
  watchdog(name, &block)
119
151
  end
@@ -87,7 +87,16 @@ module Wurk
87
87
  # config.dogstatsd = -> { Datadog::Statsd.new('host', 8125) }
88
88
  #
89
89
  # Spec: docs/target/sidekiq-pro.md §9.1.
90
- attr_accessor :dogstatsd
90
+ attr_reader :dogstatsd
91
+
92
+ # Assignment drops Statsd's resolved-client memo. That memo caches the
93
+ # "nothing configured" answer too (so the no-client emit path allocates
94
+ # nothing), which means a builder wired up after the first emit would
95
+ # otherwise never be picked up.
96
+ def dogstatsd=(builder)
97
+ @dogstatsd = builder
98
+ Wurk::Metrics::Statsd.reset!
99
+ end
91
100
 
92
101
  def initialize(options = {})
93
102
  @options = deep_dup_defaults.merge(options)
@@ -200,7 +209,7 @@ module Wurk
200
209
  end
201
210
 
202
211
  def redis(idempotent: false, &)
203
- PoolCheckout.with(redis_pool, idempotent, &)
212
+ PoolCheckout.trusted(redis_pool, idempotent, &)
204
213
  end
205
214
 
206
215
  # --- Web dashboard Redis pool ----------------------------------------
@@ -243,6 +252,11 @@ module Wurk
243
252
  @directory[name] = instance
244
253
  end
245
254
 
255
+ # Memoizes on a miss, and the first miss can land long after boot (an
256
+ # extension resolved on its first tick), which is why `freeze!` leaves
257
+ # `@directory` writable: frozen, that lookup raised FrozenError instead of
258
+ # building the default. `register` — the host-facing half — still refuses
259
+ # writes past the freeze, so the closed surface is unchanged.
246
260
  def lookup(name, default_class = nil)
247
261
  @directory[name] ||= default_class&.new
248
262
  end
@@ -482,14 +496,39 @@ module Wurk
482
496
  mb&.positive? ? mb * 1024 : nil
483
497
  end
484
498
 
499
+ # The pre-fork half of `freeze!`, and the only half a forking parent can
500
+ # run: capsules stay writable until each child has applied its slot
501
+ # (ChildBoot#apply_slot_to_config) and opened its own Redis pools. What is
502
+ # left is slot-independent — the options Hash and every capsule's middleware
503
+ # chains — so the swarm parent settles it once and every child inherits the
504
+ # result copy-on-write instead of allocating and dirtying its own copy.
505
+ # Freezing the options here also makes a post-fork option write raise in the
506
+ # child that wrote it, rather than silently diverging from its siblings.
507
+ #
508
+ # `@directory` is deliberately left out — see #lookup.
509
+ def prepare_for_fork!
510
+ # The capsule every swarm child configures (ChildBoot reaches for it by
511
+ # name), and the one a client-only config builds on its first enqueue.
512
+ # Materialized here so its chains are shared rather than rebuilt N times
513
+ # — and so `freeze!` can't close `@capsules` around a name that is only
514
+ # ever resolved later, which turned that first resolution into a
515
+ # FrozenError on the frozen Hash.
516
+ default_capsule
517
+ @capsules.each_value(&:prepare_shared!)
518
+ @options.freeze
519
+ @frozen = true
520
+ self
521
+ end
522
+
523
+ # Guarded on the capsule table, not `@frozen`: a swarm child reaches this
524
+ # with `prepare_for_fork!` already run in its parent, so `@frozen` is true
525
+ # while the capsules it has just configured are still open.
485
526
  def freeze!
486
- return self if @frozen
527
+ return self if @capsules.frozen?
487
528
 
529
+ prepare_for_fork!
488
530
  @capsules.each_value(&:freeze)
489
531
  @capsules.freeze
490
- @options.freeze
491
- @directory.freeze
492
- @frozen = true
493
532
  self
494
533
  end
495
534
 
data/lib/wurk/context.rb CHANGED
@@ -14,7 +14,7 @@ module Wurk
14
14
  # context is restored on exit, even if the block raises — safe to nest.
15
15
  def self.with(hash)
16
16
  prior = Thread.current[KEY]
17
- Thread.current[KEY] = (prior || {}).merge(hash)
17
+ Thread.current[KEY] = prior ? prior.merge(hash) : hash
18
18
  yield
19
19
  ensure
20
20
  Thread.current[KEY] = prior
data/lib/wurk/engine.rb CHANGED
@@ -25,6 +25,25 @@ module Wurk
25
25
  class AssetMount
26
26
  PREFIX = '/wurk-assets'
27
27
 
28
+ # Vite fingerprints everything it emits into `assets/`
29
+ # (`Dashboard-cGKycyd0.js`), so the bytes behind one of those URLs can
30
+ # never change — cache them for a year and never revalidate. The rest of
31
+ # the bundle (index.html, the favicons, wurk-manifest.json,
32
+ # .vite/manifest.json) keeps a stable name across builds, so it must be
33
+ # revalidated or a client would pin a stale dashboard shell forever.
34
+ FINGERPRINTED_PREFIX = '/assets/'
35
+ IMMUTABLE_CACHE_CONTROL = 'public, max-age=31536000, immutable'
36
+ REVALIDATE_CACHE_CONTROL = 'public, no-cache'
37
+
38
+ # Only a served body gets a cache directive. Rack::Files answers more than
39
+ # file reads: OPTIONS with 200 + Allow, 405 for any other verb
40
+ # (`ALLOWED_VERBS = %w[GET HEAD OPTIONS]`), and 416 for an unsatisfiable
41
+ # Range. 405 is cacheable by default (RFC 9110 §15.5.6), so stamping it
42
+ # `immutable` would let a shared proxy serve "method not allowed" for a
43
+ # year to everyone behind it.
44
+ CACHEABLE_METHODS = %w[GET HEAD].freeze
45
+ CACHEABLE_STATUSES = [200, 206, 304].freeze
46
+
28
47
  def initialize(app, root:)
29
48
  @app = app
30
49
  @files = ::Rack::Files.new(root)
@@ -38,7 +57,28 @@ module Wurk
38
57
  stripped = path.delete_prefix(PREFIX)
39
58
  inner[::Rack::PATH_INFO] = stripped.empty? ? '/' : stripped
40
59
  response = @files.call(inner)
41
- response[0] == 404 ? @app.call(env) : response
60
+ return @app.call(env) if response[0] == 404
61
+
62
+ stamp_cache_control(response, env, stripped)
63
+ response
64
+ end
65
+
66
+ private
67
+
68
+ # This mount is inserted at index 0 of the HOST app's stack (see the
69
+ # initializer below), so Rack::ETag and Rack::ConditionalGet never see
70
+ # these responses — without this the dashboard's fingerprinted bundle
71
+ # shipped with no cache directive at all and was refetched every load.
72
+ def stamp_cache_control(response, env, stripped)
73
+ return unless CACHEABLE_METHODS.include?(env[::Rack::REQUEST_METHOD])
74
+ return unless CACHEABLE_STATUSES.include?(response[0])
75
+
76
+ response[1]['cache-control'] ||=
77
+ if stripped.start_with?(FINGERPRINTED_PREFIX)
78
+ IMMUTABLE_CACHE_CONTROL
79
+ else
80
+ REVALIDATE_CACHE_CONTROL
81
+ end
42
82
  end
43
83
  end
44
84
 
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ class Error < StandardError; end
5
+
6
+ # Raised inside a worker process to abort the run loop. User code must not
7
+ # rescue this — the swarm uses it to signal teardown across thread boundaries.
8
+ # Spec: docs/target/sidekiq-free.md §3.
9
+ #
10
+ # Lives in its own file, loaded before everything else, because Processor
11
+ # freezes its `Thread.handle_interrupt` masks into constants: those are
12
+ # evaluated the moment `wurk/processor` is required, so this class has to
13
+ # exist by then.
14
+ class Shutdown < Interrupt; end
15
+ end
@@ -3,6 +3,7 @@
3
3
  require_relative '../component'
4
4
  require_relative '../keys'
5
5
  require_relative '../middleware/poison_pill'
6
+ require_relative '../timer_loop'
6
7
 
7
8
  module Wurk
8
9
  class Fetcher
@@ -91,13 +92,22 @@ module Wurk
91
92
  @thread
92
93
  end
93
94
 
95
+ # Bounded at TimerLoop::JOIN_TIMEOUT like every other periodic component.
96
+ # The full-keyspace sweep is exactly the tick that outlasts a stop: it
97
+ # SCANs the whole `queue:*|*` keyspace and drains what it finds, so on a
98
+ # large or slow Redis it can still be running when shutdown lands. Waiting
99
+ # it out held the entire process's teardown open past the swarm parent's
100
+ # SHUTDOWN_GRACE — which SIGKILLs the child mid-drain, the one outcome
101
+ # this component exists to recover from. A straggler is left running
102
+ # instead — and still referenced, like every other periodic component, so
103
+ # a restart after a timed-out stop can't spawn a second sweep loop
104
+ # alongside it. Its own tick_once rescues and reports.
94
105
  def stop
95
106
  @mutex.synchronize do
96
107
  @done = true
97
108
  @sleeper.signal
98
109
  end
99
- @thread&.join
100
- @thread = nil
110
+ @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
101
111
  end
102
112
 
103
113
  def running?