wurk 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/app/controllers/concerns/wurk/stream_concurrency_guard.rb +22 -5
- data/lib/wurk/capsule.rb +40 -12
- data/lib/wurk/client/buffered.rb +7 -0
- data/lib/wurk/client.rb +102 -32
- data/lib/wurk/component.rb +34 -2
- data/lib/wurk/configuration.rb +45 -6
- data/lib/wurk/context.rb +1 -1
- data/lib/wurk/engine.rb +41 -1
- data/lib/wurk/errors.rb +15 -0
- data/lib/wurk/fetcher/reaper.rb +12 -2
- data/lib/wurk/fetcher/reliable.rb +322 -60
- data/lib/wurk/fetcher.rb +6 -0
- data/lib/wurk/heartbeat.rb +2 -2
- data/lib/wurk/job_logger.rb +16 -7
- data/lib/wurk/job_set.rb +3 -2
- data/lib/wurk/job_util.rb +44 -24
- data/lib/wurk/launcher.rb +116 -72
- data/lib/wurk/leader.rb +36 -6
- data/lib/wurk/logger.rb +1 -1
- data/lib/wurk/lua/loader.rb +9 -3
- data/lib/wurk/metrics/accumulator.rb +95 -0
- data/lib/wurk/metrics/flusher.rb +70 -0
- data/lib/wurk/metrics/history.rb +98 -29
- data/lib/wurk/metrics/statsd.rb +32 -18
- data/lib/wurk/middleware/chain.rb +31 -14
- data/lib/wurk/middleware/poison_pill.rb +23 -2
- data/lib/wurk/pool_checkout.rb +10 -0
- data/lib/wurk/processor.rb +79 -11
- data/lib/wurk/profiler.rb +6 -2
- data/lib/wurk/queue.rb +8 -0
- data/lib/wurk/redis_client_adapter.rb +1 -1
- data/lib/wurk/redis_pool.rb +2 -1
- data/lib/wurk/shutdown_gate.rb +79 -0
- data/lib/wurk/stats.rb +5 -1
- data/lib/wurk/swarm/child_boot.rb +24 -4
- data/lib/wurk/swarm.rb +88 -14
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/config.rb +11 -7
- data/lib/wurk.rb +2 -8
- data/vendor/assets/dashboard/assets/{ArgsValue-D74zX0MI.js → ArgsValue-CcR2ya6e.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-OmC5NPgw.js → BatchDetail-CUXJUQ3Q.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-CIpai7St.js → Batches-Cxan6Ngw.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-A_kwSR6Q.js → Busy-DC5EGM0g.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-BG7HTqlp.js → Cron-Dlt8tXJA.js} +1 -1
- data/vendor/assets/dashboard/assets/Dashboard-DNLu_WCg.js +1 -0
- data/vendor/assets/dashboard/assets/{Dead-8J21jMyK.js → Dead-dZ7VGlKS.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-B4Q9FIQu.js → Extension-DaFpEIJf.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-Fh_Ae7UW.js → FilterBox-CO3aYWIq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Ceng0PMB.js → JobDetailModal-DSWbT6G0.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-CruDWvNZ.js → Limiters-Cb4PKXNR.js} +1 -1
- data/vendor/assets/dashboard/assets/Metrics-CCGzgCsT.js +1 -0
- data/vendor/assets/dashboard/assets/{Modal-CN3rdKA_.js → Modal-B86q6ruL.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-C44KNMGm.js → PageHeader-fPrCcp_-.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-xEVTyS2N.js → Profiles-BnS82nR_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Queues-D86FYohJ.js → Queues-CIyPevOy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Retries-Bz1O1D-i.js → Retries-DopwXkXl.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-B6h2akTu.js → Scheduled-1-Z7i1zE.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-OOu22e5s.js → Search-ByA6eTma.js} +1 -1
- data/vendor/assets/dashboard/assets/{Skeleton-DzR7XNxz.js → Skeleton-bC7HfQ9r.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-BVHHGof7.js → charts-CLLzJ7vK.js} +1 -1
- data/vendor/assets/dashboard/assets/index-B1N8hQUh.js +141 -0
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-dVPGEWzn.js → useResetPageOnEmpty-DpBjkf6_.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-BeYbztkN.js → useSort-DvpwuNQE.js} +1 -1
- data/vendor/assets/dashboard/index.html +2 -2
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +30 -26
- data/vendor/assets/dashboard/assets/Dashboard-A_ToqHoo.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-CIT7VCoN.js +0 -1
- data/vendor/assets/dashboard/assets/index-D_lSDwKw.js +0 -141
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '0995fd5827870e14584104a8998db58e9927339197460596567c05f2dacc7da0'
|
|
4
|
+
data.tar.gz: 8159d245826bf646a9c6025e4e06a49717f957aabe18dad5d2a5b58ed99818c0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d45c20c9a8c29695c7ea2e04066dda58454fea0ce56453ed866cd111e38977fb1739ae47429a60b68380fac2f4d7fe85f232b420f92f95cba330c389ee435b5b
|
|
7
|
+
data.tar.gz: 94c5599bc28408e1e5eb30967d1b0fad4806d415a17e515774c057120d0833660f3c2860d9cf0cb2ea320ae3a27bd9ce683385ce5ace1c347d46530a96cd9a1a
|
data/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
<p align="center"><strong>Wurk, wurk.</strong> 🪓 <em>Ready to work. Zug zug.</em></p>
|
|
8
8
|
|
|
9
|
-
<p align="center"><strong>A 100% drop-in replacement for Sidekiq + Sidekiq Pro + Sidekiq Enterprise. Free forever
|
|
9
|
+
<p align="center"><strong>A 100% drop-in replacement for Sidekiq + Sidekiq Pro + Sidekiq Enterprise. Free forever.</strong></p>
|
|
10
10
|
|
|
11
11
|
<div align="center">
|
|
12
12
|
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
|
|
22
22
|
Wurk is wire-compatible with Sidekiq — same Redis keys, same job JSON, same Ruby DSL. Swap one line in your `Gemfile` and your existing jobs, batches, limiters, cron entries, and live Redis data keep working untouched. The Pro and Enterprise feature sets ship in the same free gem, with no license check and no tiers.
|
|
23
23
|
|
|
24
|
+
**On speed:** Wurk is not currently faster than stock Sidekiq — it runs at roughly 0.87×–1.02× depending on workload shape, with parity on CPU and I/O but still behind on framework overhead (noop) and boot time. Numbers, method, and the reproduction command are in [docs/benchmarks.md](docs/benchmarks.md); run them yourself with `rake bench:vs_sidekiq`.
|
|
25
|
+
|
|
24
26
|
## Install
|
|
25
27
|
|
|
26
28
|
```ruby
|
|
@@ -7,28 +7,45 @@ module Wurk
|
|
|
7
7
|
# we 503 with Retry-After — the SPA's EventSource reconnects (and its polling
|
|
8
8
|
# fallback honors Retry-After) once a slot frees. Per-process is the right
|
|
9
9
|
# scope: it's this process's own thread pool we're protecting.
|
|
10
|
+
#
|
|
11
|
+
# Slots are held as thread references rather than tallied in a counter so the
|
|
12
|
+
# cap can heal itself. A stream whose thread is killed mid-flight never
|
|
13
|
+
# reaches the `ensure` below (Puma hard-reaps worker threads past
|
|
14
|
+
# `force_shutdown_after`, and a thread killed inside an uninterruptible read
|
|
15
|
+
# can skip its ensure), which a counter would record as a slot held by nobody
|
|
16
|
+
# — ten of those and `/api/stream` 503s for the life of the process. A dead
|
|
17
|
+
# holder is instead evicted by the next acquire.
|
|
10
18
|
module StreamConcurrencyGuard
|
|
11
19
|
extend ActiveSupport::Concern
|
|
12
20
|
|
|
13
21
|
MAX_CONCURRENT_STREAMS = 10
|
|
14
22
|
RETRY_AFTER_SECONDS = 3
|
|
15
23
|
|
|
16
|
-
@
|
|
24
|
+
@holders = []
|
|
17
25
|
@lock = Mutex.new
|
|
18
26
|
|
|
19
27
|
class << self
|
|
20
|
-
# Reserve a stream slot; false when the cap is
|
|
28
|
+
# Reserve a stream slot for the calling thread; false when the cap is
|
|
29
|
+
# already reached by threads that are still alive.
|
|
21
30
|
def acquire
|
|
22
31
|
@lock.synchronize do
|
|
23
|
-
|
|
32
|
+
@holders.keep_if(&:alive?)
|
|
33
|
+
return false if @holders.size >= MAX_CONCURRENT_STREAMS
|
|
24
34
|
|
|
25
|
-
@
|
|
35
|
+
@holders << Thread.current
|
|
26
36
|
true
|
|
27
37
|
end
|
|
28
38
|
end
|
|
29
39
|
|
|
40
|
+
# Drops one slot held by the calling thread. Acquire and release always
|
|
41
|
+
# bracket a single block on one thread (`#with_stream_slot`), so a call
|
|
42
|
+
# from a thread holding nothing is a no-op rather than a slot taken away
|
|
43
|
+
# from whoever is actually streaming.
|
|
30
44
|
def release
|
|
31
|
-
@lock.synchronize
|
|
45
|
+
@lock.synchronize do
|
|
46
|
+
index = @holders.rindex(Thread.current)
|
|
47
|
+
@holders.delete_at(index) if index
|
|
48
|
+
end
|
|
32
49
|
end
|
|
33
50
|
end
|
|
34
51
|
|
data/lib/wurk/capsule.rb
CHANGED
|
@@ -17,6 +17,11 @@ module Wurk
|
|
|
17
17
|
attr_reader :name, :queues, :mode, :weights, :config
|
|
18
18
|
attr_accessor :concurrency, :fetcher
|
|
19
19
|
|
|
20
|
+
# Capsule-hosted components (Manager, Processor, Fetcher) hand their capsule
|
|
21
|
+
# to Component as `config`, and `safe_thread` reads the priority off it.
|
|
22
|
+
# Sidekiq delegates the same accessor (capsule.rb:30).
|
|
23
|
+
def thread_priority = @config.thread_priority
|
|
24
|
+
|
|
20
25
|
def initialize(name, config)
|
|
21
26
|
@name = name.to_s
|
|
22
27
|
@config = config
|
|
@@ -25,8 +30,15 @@ module Wurk
|
|
|
25
30
|
@mode = :strict
|
|
26
31
|
@weights = { 'default' => 0 }
|
|
27
32
|
@fetcher = nil
|
|
28
|
-
|
|
29
|
-
|
|
33
|
+
# One mutable Hash rather than two ivars: the pools are the only part of a
|
|
34
|
+
# capsule that legitimately changes after Configuration#freeze! — fork
|
|
35
|
+
# closes them, Launcher#stop releases them, an embedded host that boots
|
|
36
|
+
# again rebuilds them. `Object#freeze` is shallow, so the Hash stays
|
|
37
|
+
# writable and freezing a capsule keeps meaning "no more configuration"
|
|
38
|
+
# instead of "these sockets are yours forever". Before this, a reset on a
|
|
39
|
+
# frozen capsule disconnected the pool and then raised FrozenError on the
|
|
40
|
+
# memo, leaving `redis_pool` answering with a shut-down pool for good.
|
|
41
|
+
@pools = {}
|
|
30
42
|
@client_chain = nil
|
|
31
43
|
@server_chain = nil
|
|
32
44
|
end
|
|
@@ -64,9 +76,26 @@ module Wurk
|
|
|
64
76
|
# by hand; centralizing it here covers the standalone CLI and embedded
|
|
65
77
|
# paths too (the bug behind a nil `fetcher` in `exe/wurk`). Idempotent.
|
|
66
78
|
def prepare!
|
|
79
|
+
prepare_shared!
|
|
67
80
|
@fetcher ||= build_fetcher
|
|
68
81
|
redis_pool
|
|
69
82
|
fetch_redis_pool
|
|
83
|
+
self
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# The half of `prepare!` a forking parent can run on every child's behalf:
|
|
87
|
+
# the chains are a pure function of this capsule's identity, not of the slot
|
|
88
|
+
# (queues + concurrency) a swarm child is assigned later, and `copy_for`
|
|
89
|
+
# opens nothing. Run before the fork, the entries are allocated once and
|
|
90
|
+
# inherited copy-on-write instead of rebuilt in every child.
|
|
91
|
+
#
|
|
92
|
+
# The rest of `prepare!` deliberately stays post-fork: both pools are sized
|
|
93
|
+
# off the slot's concurrency and one built here would hand every child an
|
|
94
|
+
# inherited socket, and `build_fetcher` fires the host's
|
|
95
|
+
# `config[:fetch_setup]` hook, which is per-child — running it in the parent
|
|
96
|
+
# would let a custom fetcher snapshot the wrong queues, or leak whatever the
|
|
97
|
+
# hook opened across the fork. Idempotent.
|
|
98
|
+
def prepare_shared!
|
|
70
99
|
client_middleware
|
|
71
100
|
server_middleware
|
|
72
101
|
self
|
|
@@ -99,7 +128,7 @@ module Wurk
|
|
|
99
128
|
MIN_POOL_SIZE = 10
|
|
100
129
|
|
|
101
130
|
def redis_pool
|
|
102
|
-
@
|
|
131
|
+
@pools[:main] ||= build_pool(size: main_pool_size, name: "#{@name}-main")
|
|
103
132
|
end
|
|
104
133
|
|
|
105
134
|
# Dedicated pool for the reliable fetcher's blocking BLMOVE: one slot per
|
|
@@ -107,18 +136,17 @@ module Wurk
|
|
|
107
136
|
# fetch at once. Keeping fetch off the main pool is what lets an idle worker
|
|
108
137
|
# hold zero main-pool connections again.
|
|
109
138
|
def fetch_redis_pool
|
|
110
|
-
@
|
|
139
|
+
@pools[:fetch] ||= build_pool(size: @concurrency, name: "#{@name}-fetch")
|
|
111
140
|
end
|
|
112
141
|
|
|
113
|
-
# Disconnect and drop cached pools. Called by Wurk::Swarm just before
|
|
114
|
-
#
|
|
115
|
-
#
|
|
116
|
-
# so dropping the reference is
|
|
142
|
+
# Disconnect and drop cached pools. Called by Wurk::Swarm just before fork
|
|
143
|
+
# (parent side: close inherited sockets), just after fork (child side:
|
|
144
|
+
# rebuild lazily), and by Launcher#stop (release what this process held).
|
|
145
|
+
# Connection_pool#shutdown is terminal, so dropping the reference is
|
|
146
|
+
# required — `redis_pool` will rebuild.
|
|
117
147
|
def reset_redis_pools!
|
|
118
|
-
@
|
|
119
|
-
@
|
|
120
|
-
@fetch_redis_pool&.disconnect!
|
|
121
|
-
@fetch_redis_pool = nil
|
|
148
|
+
@pools.each_value(&:disconnect!)
|
|
149
|
+
@pools.clear
|
|
122
150
|
end
|
|
123
151
|
|
|
124
152
|
def redis(idempotent: false, &)
|
data/lib/wurk/client/buffered.rb
CHANGED
|
@@ -119,6 +119,13 @@ module Wurk
|
|
|
119
119
|
@overflow_mode = nil
|
|
120
120
|
@buffer_client_factory = nil
|
|
121
121
|
end
|
|
122
|
+
# Stop before dropping: an unstopped drainer thread survives with
|
|
123
|
+
# its factory nil'd out from under it and ticks forever against
|
|
124
|
+
# nothing, leaking the thread and everything its closure retains.
|
|
125
|
+
install_mutex.synchronize do
|
|
126
|
+
@drainer&.stop
|
|
127
|
+
@drainer = nil
|
|
128
|
+
end
|
|
122
129
|
end
|
|
123
130
|
|
|
124
131
|
# Fork hook, called from the `Process._fork` prepend below and from
|
data/lib/wurk/client.rb
CHANGED
|
@@ -26,6 +26,20 @@ module Wurk
|
|
|
26
26
|
SCHEDULED_BATCH_SIZE = 100
|
|
27
27
|
SPREAD_INTERVAL_FLOOR = 5
|
|
28
28
|
|
|
29
|
+
# Batched (`bid`) payloads per EVALSHA pipeline. Ours, not Sidekiq's — it
|
|
30
|
+
# has no batches. Sized to DEFAULT_BATCH_SIZE so no existing caller's
|
|
31
|
+
# round-trip count moves: `push_bulk` already hands #raw_push at most that
|
|
32
|
+
# many payloads, so its batched pipeline stays exactly one round trip.
|
|
33
|
+
#
|
|
34
|
+
# The cap is for the one path that isn't pre-sliced: `autoflush = true`
|
|
35
|
+
# buffers a whole `Batch#jobs` block, so #flush_batched can be handed an
|
|
36
|
+
# unbounded payload set. Unsliced that is one pipeline holding every
|
|
37
|
+
# command and every reply in memory at once, and — Lua being atomic and
|
|
38
|
+
# single-threaded — one uninterrupted server-side sweep that blocks every
|
|
39
|
+
# other client for its duration. Same reasoning as the LIMIT on
|
|
40
|
+
# RELIABLE_SCHEDULE_PROMOTE.
|
|
41
|
+
BATCH_PIPELINE_SLICE = 1_000
|
|
42
|
+
|
|
29
43
|
# Thread-local slot holding the payloads of the current push whose Redis
|
|
30
44
|
# write is confirmed applied. {Client::Buffered} subtracts them from the set
|
|
31
45
|
# it re-buffers when a *later* phase of the same push loses the connection,
|
|
@@ -138,7 +152,21 @@ module Wurk
|
|
|
138
152
|
|
|
139
153
|
private
|
|
140
154
|
|
|
155
|
+
# #push and #push_bulk verify at different points and both match Sidekiq
|
|
156
|
+
# exactly: push walks the payload the chain handed back (sidekiq
|
|
157
|
+
# client.rb:101 — normalize → middleware → verify → raw_push), bulk walks it
|
|
158
|
+
# inside the innermost block (sidekiq client.rb:165). Push used to do both,
|
|
159
|
+
# and since `strict_args_mode` defaults to :raise the second full recursive
|
|
160
|
+
# args walk was never skipped.
|
|
161
|
+
#
|
|
162
|
+
# Bulk keeps its walk inside the block on purpose: a client middleware that
|
|
163
|
+
# halts the job there short-circuits the walk, and hoisting it out would
|
|
164
|
+
# raise on args that middleware was about to drop.
|
|
141
165
|
def invoke_chain(normed)
|
|
166
|
+
@chain.invoke(normed['class'], normed, normed['queue'], pool) { normed }
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def invoke_chain_verified(normed)
|
|
142
170
|
@chain.invoke(normed['class'], normed, normed['queue'], pool) do
|
|
143
171
|
verify_json(normed)
|
|
144
172
|
normed
|
|
@@ -191,7 +219,7 @@ module Wurk
|
|
|
191
219
|
item = base.merge('args' => job_args)
|
|
192
220
|
item['at'] = ats[idx] if ats
|
|
193
221
|
normed = normalize_item(item)
|
|
194
|
-
|
|
222
|
+
invoke_chain_verified(normed)
|
|
195
223
|
end
|
|
196
224
|
end
|
|
197
225
|
|
|
@@ -314,6 +342,27 @@ module Wurk
|
|
|
314
342
|
push_batched_pipelined(conn, batched, now) unless batched.empty?
|
|
315
343
|
end
|
|
316
344
|
|
|
345
|
+
# One pipeline per BATCH_PIPELINE_SLICE payloads, each marked delivered the
|
|
346
|
+
# moment its reply is in — same contract as push_plain_group, and for the
|
|
347
|
+
# same reason: a slice that Redis already accepted must stay out of the
|
|
348
|
+
# reliable_push ledger, or a later slice's failure would report it as
|
|
349
|
+
# undelivered.
|
|
350
|
+
def push_batched_pipelined(conn, batched, now)
|
|
351
|
+
batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
|
|
352
|
+
eval_batched_slice(conn) { |pipe, eval_method| push_batched(pipe, slice, now, eval_method: eval_method) }
|
|
353
|
+
mark_delivered(slice)
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# Same slicing and NOSCRIPT recovery as push_batched_pipelined, for the
|
|
358
|
+
# scheduled batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
|
|
359
|
+
def push_batched_scheduled_pipelined(conn, batched)
|
|
360
|
+
batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
|
|
361
|
+
eval_batched_slice(conn) { |pipe, eval_method| push_batched_scheduled(pipe, slice, eval_method: eval_method) }
|
|
362
|
+
mark_delivered(slice)
|
|
363
|
+
end
|
|
364
|
+
end
|
|
365
|
+
|
|
317
366
|
# Outside of test boots and `SCRIPT FLUSH` the rescue branch is dead
|
|
318
367
|
# code; the eager `script_load_all` after fork keeps the script cache
|
|
319
368
|
# hot for the life of the connection. The retry uses EVAL (source-embedded)
|
|
@@ -321,24 +370,18 @@ module Wurk
|
|
|
321
370
|
# NOSCRIPT a second time under heavy CI load (WorkerTest 3.4/7.2 flake).
|
|
322
371
|
# `script_load_all` still primes the cache so the *next* pipeline returns
|
|
323
372
|
# to the EVALSHA fast path.
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
conn.pipelined { |pipe|
|
|
331
|
-
end
|
|
332
|
-
|
|
333
|
-
# Same NOSCRIPT-recovery shape as push_batched_pipelined, for the scheduled
|
|
334
|
-
# batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
|
|
335
|
-
def push_batched_scheduled_pipelined(conn, batched)
|
|
336
|
-
conn.pipelined { |pipe| push_batched_scheduled(pipe, batched) }
|
|
373
|
+
#
|
|
374
|
+
# Replaying the slice is safe precisely because every command in it is the
|
|
375
|
+
# same script: a flushed cache NOSCRIPTs all of them and applies none.
|
|
376
|
+
# Recovery is per slice, so the slices already acknowledged above are never
|
|
377
|
+
# re-sent. Mirrors Fetcher::Reliable#requeue_pipelined.
|
|
378
|
+
def eval_batched_slice(conn)
|
|
379
|
+
conn.pipelined { |pipe| yield(pipe, :eval_cached) }
|
|
337
380
|
rescue RedisClient::CommandError => e
|
|
338
381
|
raise unless e.message.to_s.start_with?('NOSCRIPT')
|
|
339
382
|
|
|
340
383
|
Wurk::Lua::Loader.script_load_all(conn)
|
|
341
|
-
conn.pipelined { |pipe|
|
|
384
|
+
conn.pipelined { |pipe| yield(pipe, :eval_with_source) }
|
|
342
385
|
end
|
|
343
386
|
|
|
344
387
|
# One pipeline per queue, marked delivered the moment its reply is in.
|
|
@@ -355,27 +398,48 @@ module Wurk
|
|
|
355
398
|
# Cost is a round trip per distinct queue. The single-queue push — every
|
|
356
399
|
# `perform_async`, every same-class `push_bulk` — still writes exactly the
|
|
357
400
|
# one SADD + LPUSH pipeline it did before.
|
|
401
|
+
#
|
|
402
|
+
# `uniform_queue` short-circuits the common case (one job, or many jobs
|
|
403
|
+
# all destined for the same queue) without paying for the `group_by`
|
|
404
|
+
# Hash + per-group Array allocations; only a genuinely mixed-queue batch
|
|
405
|
+
# falls through to grouping.
|
|
358
406
|
def push_plain(conn, payloads, now)
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
407
|
+
queue = uniform_queue(payloads)
|
|
408
|
+
return push_plain_group(conn, queue, payloads, now) if queue
|
|
409
|
+
|
|
410
|
+
payloads.group_by { |j| j['queue'] }.each { |q, jobs| push_plain_group(conn, q, jobs, now) }
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def uniform_queue(payloads)
|
|
414
|
+
first = payloads[0]['queue']
|
|
415
|
+
return first if payloads.size == 1
|
|
416
|
+
|
|
417
|
+
first if payloads.all? { |j| j['queue'] == first }
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
def push_plain_group(conn, queue, jobs, now)
|
|
421
|
+
serialized = jobs.map do |j|
|
|
422
|
+
j['enqueued_at'] = now
|
|
423
|
+
Wurk.dump_json(j)
|
|
424
|
+
end
|
|
425
|
+
conn.pipelined do |pipe|
|
|
426
|
+
pipe.call('SADD', 'queues', queue)
|
|
427
|
+
pipe.call('LPUSH', "queue:#{queue}", *serialized)
|
|
369
428
|
end
|
|
429
|
+
mark_delivered(jobs)
|
|
370
430
|
end
|
|
371
431
|
|
|
372
432
|
# Batched jobs route through BATCH_PUSH: increments b-<bid> total+pending,
|
|
373
433
|
# SADDs jid into the live set, registers the queue, LPUSHes the payload —
|
|
374
|
-
# all atomically.
|
|
375
|
-
#
|
|
376
|
-
#
|
|
434
|
+
# all atomically. The Lua binds per-job KEYS, so grouping N jobs into one
|
|
435
|
+
# EVALSHA isn't available; they ride one pipeline instead (the `conn` here
|
|
436
|
+
# is always the pipeline #push_batched_pipelined opened), so the cost is
|
|
437
|
+
# N commands and one round trip, not N round trips. `eval_method` is the
|
|
377
438
|
# Wurk::Lua::Loader entry point (`:eval_cached` for the hot EVALSHA path,
|
|
378
|
-
# `:eval_with_source` for the EVAL-source retry)
|
|
439
|
+
# `:eval_with_source` for the EVAL-source retry) — neither reads the reply,
|
|
440
|
+
# which is what makes the pipelined form legal: `eval_cached`'s inline
|
|
441
|
+
# NOSCRIPT rescue can't fire against a buffered call, so recovery is the
|
|
442
|
+
# caller's finalize-time rescue.
|
|
379
443
|
def push_batched(conn, payloads, now, eval_method: :eval_cached)
|
|
380
444
|
payloads.each do |j|
|
|
381
445
|
j['enqueued_at'] = now
|
|
@@ -394,9 +458,9 @@ module Wurk
|
|
|
394
458
|
# total/pending increment registers the job in its batch at creation, and
|
|
395
459
|
# the ZADD defers it onto `schedule`. Payload is stripped of `at`/
|
|
396
460
|
# `enqueued_at` exactly like push_scheduled — `enqueued_at` is stamped fresh
|
|
397
|
-
# at promotion, never while the job sits scheduled (spec §7.1).
|
|
398
|
-
#
|
|
399
|
-
#
|
|
461
|
+
# at promotion, never while the job sits scheduled (spec §7.1). Per-job
|
|
462
|
+
# KEYS again, so one EVALSHA per job — pipelined by
|
|
463
|
+
# #push_batched_scheduled_pipelined into one round trip per slice.
|
|
400
464
|
def push_batched_scheduled(conn, payloads, eval_method: :eval_cached)
|
|
401
465
|
payloads.each do |j|
|
|
402
466
|
Wurk::Lua::Loader.public_send(
|
|
@@ -436,7 +500,13 @@ module Wurk
|
|
|
436
500
|
# evict them, and the drain that does land one counts it then — so booking
|
|
437
501
|
# them here would inflate the counter on an outage and double-count every
|
|
438
502
|
# payload that later replays.
|
|
503
|
+
#
|
|
504
|
+
# Resolve the client once for the whole batch and bail before touching a
|
|
505
|
+
# payload: unconfigured is the common case, and the tags below cost two
|
|
506
|
+
# Strings and an Array per job for `increment` to immediately drop.
|
|
439
507
|
def emit_enqueued(payloads, buffered = nil)
|
|
508
|
+
return if Wurk::Metrics::Statsd.safe_client.nil?
|
|
509
|
+
|
|
440
510
|
payloads = reject_by_identity(payloads, buffered) if buffered && !buffered.empty?
|
|
441
511
|
payloads.each do |p|
|
|
442
512
|
Wurk::Metrics::Statsd.increment(
|
data/lib/wurk/component.rb
CHANGED
|
@@ -37,8 +37,34 @@ module Wurk
|
|
|
37
37
|
|
|
38
38
|
# --- identity -------------------------------------------------------
|
|
39
39
|
|
|
40
|
+
# Base36 `thread.object_id ^ pid` — the id in every log line and the key
|
|
41
|
+
# each Processor publishes its in-flight job under. Constant for the life
|
|
42
|
+
# of a thread inside one process and read several times per job, so it is
|
|
43
|
+
# memoized per thread (frozen: it is used as a Hash key, and an unfrozen
|
|
44
|
+
# String key is duped on every store).
|
|
45
|
+
#
|
|
46
|
+
# The pid is memoized alongside it because the thread that calls fork keeps
|
|
47
|
+
# its thread-locals in the child, where the pid — and therefore the tid —
|
|
48
|
+
# has changed. Without the guard a forked child would report the parent's
|
|
49
|
+
# tid and collide with it in `<identity>:work`.
|
|
50
|
+
#
|
|
51
|
+
# Thread-local, not `Thread#[]`: the latter is fiber-local, so a job that
|
|
52
|
+
# runs inside a Fiber (or any Enumerator) would miss the memo and allocate
|
|
53
|
+
# a fresh String on every read — the identity the memo exists to cache is
|
|
54
|
+
# the thread's, and it does not change when a fiber does.
|
|
55
|
+
def self.tid
|
|
56
|
+
thread = Thread.current
|
|
57
|
+
memo = thread.thread_variable_get(:wurk_tid)
|
|
58
|
+
pid = ::Process.pid
|
|
59
|
+
return memo[1] if memo && memo[0] == pid
|
|
60
|
+
|
|
61
|
+
id = (thread.object_id ^ pid).to_s(36).freeze
|
|
62
|
+
thread.thread_variable_set(:wurk_tid, [pid, id].freeze)
|
|
63
|
+
id
|
|
64
|
+
end
|
|
65
|
+
|
|
40
66
|
def tid
|
|
41
|
-
|
|
67
|
+
Component.tid
|
|
42
68
|
end
|
|
43
69
|
|
|
44
70
|
def hostname
|
|
@@ -110,10 +136,16 @@ module Wurk
|
|
|
110
136
|
# Spawns a named thread that runs `block` under `watchdog(name)`. The
|
|
111
137
|
# parent must retain the returned Thread; otherwise GC may not, but
|
|
112
138
|
# report_on_exception is disabled so we don't double-log on death.
|
|
139
|
+
#
|
|
140
|
+
# Priority resolution matches Sidekiq (component.rb:44-48): explicit
|
|
141
|
+
# argument, then `config.thread_priority`, then -1. Ruby's default of 0
|
|
142
|
+
# buys a 100ms timeslice; each negative step halves it, so -1 keeps a
|
|
143
|
+
# CPU-heavy capsule from starving its siblings for a whole tick.
|
|
113
144
|
def safe_thread(name, priority: nil, &block)
|
|
145
|
+
resolved = priority || config.thread_priority || DEFAULT_THREAD_PRIORITY
|
|
114
146
|
Thread.new do
|
|
115
147
|
Thread.current.name = name
|
|
116
|
-
Thread.current.priority =
|
|
148
|
+
Thread.current.priority = resolved
|
|
117
149
|
Thread.current.report_on_exception = false
|
|
118
150
|
watchdog(name, &block)
|
|
119
151
|
end
|
data/lib/wurk/configuration.rb
CHANGED
|
@@ -87,7 +87,16 @@ module Wurk
|
|
|
87
87
|
# config.dogstatsd = -> { Datadog::Statsd.new('host', 8125) }
|
|
88
88
|
#
|
|
89
89
|
# Spec: docs/target/sidekiq-pro.md §9.1.
|
|
90
|
-
|
|
90
|
+
attr_reader :dogstatsd
|
|
91
|
+
|
|
92
|
+
# Assignment drops Statsd's resolved-client memo. That memo caches the
|
|
93
|
+
# "nothing configured" answer too (so the no-client emit path allocates
|
|
94
|
+
# nothing), which means a builder wired up after the first emit would
|
|
95
|
+
# otherwise never be picked up.
|
|
96
|
+
def dogstatsd=(builder)
|
|
97
|
+
@dogstatsd = builder
|
|
98
|
+
Wurk::Metrics::Statsd.reset!
|
|
99
|
+
end
|
|
91
100
|
|
|
92
101
|
def initialize(options = {})
|
|
93
102
|
@options = deep_dup_defaults.merge(options)
|
|
@@ -200,7 +209,7 @@ module Wurk
|
|
|
200
209
|
end
|
|
201
210
|
|
|
202
211
|
def redis(idempotent: false, &)
|
|
203
|
-
PoolCheckout.
|
|
212
|
+
PoolCheckout.trusted(redis_pool, idempotent, &)
|
|
204
213
|
end
|
|
205
214
|
|
|
206
215
|
# --- Web dashboard Redis pool ----------------------------------------
|
|
@@ -243,6 +252,11 @@ module Wurk
|
|
|
243
252
|
@directory[name] = instance
|
|
244
253
|
end
|
|
245
254
|
|
|
255
|
+
# Memoizes on a miss, and the first miss can land long after boot (an
|
|
256
|
+
# extension resolved on its first tick), which is why `freeze!` leaves
|
|
257
|
+
# `@directory` writable: frozen, that lookup raised FrozenError instead of
|
|
258
|
+
# building the default. `register` — the host-facing half — still refuses
|
|
259
|
+
# writes past the freeze, so the closed surface is unchanged.
|
|
246
260
|
def lookup(name, default_class = nil)
|
|
247
261
|
@directory[name] ||= default_class&.new
|
|
248
262
|
end
|
|
@@ -482,14 +496,39 @@ module Wurk
|
|
|
482
496
|
mb&.positive? ? mb * 1024 : nil
|
|
483
497
|
end
|
|
484
498
|
|
|
499
|
+
# The pre-fork half of `freeze!`, and the only half a forking parent can
|
|
500
|
+
# run: capsules stay writable until each child has applied its slot
|
|
501
|
+
# (ChildBoot#apply_slot_to_config) and opened its own Redis pools. What is
|
|
502
|
+
# left is slot-independent — the options Hash and every capsule's middleware
|
|
503
|
+
# chains — so the swarm parent settles it once and every child inherits the
|
|
504
|
+
# result copy-on-write instead of allocating and dirtying its own copy.
|
|
505
|
+
# Freezing the options here also makes a post-fork option write raise in the
|
|
506
|
+
# child that wrote it, rather than silently diverging from its siblings.
|
|
507
|
+
#
|
|
508
|
+
# `@directory` is deliberately left out — see #lookup.
|
|
509
|
+
def prepare_for_fork!
|
|
510
|
+
# The capsule every swarm child configures (ChildBoot reaches for it by
|
|
511
|
+
# name), and the one a client-only config builds on its first enqueue.
|
|
512
|
+
# Materialized here so its chains are shared rather than rebuilt N times
|
|
513
|
+
# — and so `freeze!` can't close `@capsules` around a name that is only
|
|
514
|
+
# ever resolved later, which turned that first resolution into a
|
|
515
|
+
# FrozenError on the frozen Hash.
|
|
516
|
+
default_capsule
|
|
517
|
+
@capsules.each_value(&:prepare_shared!)
|
|
518
|
+
@options.freeze
|
|
519
|
+
@frozen = true
|
|
520
|
+
self
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
# Guarded on the capsule table, not `@frozen`: a swarm child reaches this
|
|
524
|
+
# with `prepare_for_fork!` already run in its parent, so `@frozen` is true
|
|
525
|
+
# while the capsules it has just configured are still open.
|
|
485
526
|
def freeze!
|
|
486
|
-
return self if @frozen
|
|
527
|
+
return self if @capsules.frozen?
|
|
487
528
|
|
|
529
|
+
prepare_for_fork!
|
|
488
530
|
@capsules.each_value(&:freeze)
|
|
489
531
|
@capsules.freeze
|
|
490
|
-
@options.freeze
|
|
491
|
-
@directory.freeze
|
|
492
|
-
@frozen = true
|
|
493
532
|
self
|
|
494
533
|
end
|
|
495
534
|
|
data/lib/wurk/context.rb
CHANGED
|
@@ -14,7 +14,7 @@ module Wurk
|
|
|
14
14
|
# context is restored on exit, even if the block raises — safe to nest.
|
|
15
15
|
def self.with(hash)
|
|
16
16
|
prior = Thread.current[KEY]
|
|
17
|
-
Thread.current[KEY] =
|
|
17
|
+
Thread.current[KEY] = prior ? prior.merge(hash) : hash
|
|
18
18
|
yield
|
|
19
19
|
ensure
|
|
20
20
|
Thread.current[KEY] = prior
|
data/lib/wurk/engine.rb
CHANGED
|
@@ -25,6 +25,25 @@ module Wurk
|
|
|
25
25
|
class AssetMount
|
|
26
26
|
PREFIX = '/wurk-assets'
|
|
27
27
|
|
|
28
|
+
# Vite fingerprints everything it emits into `assets/`
|
|
29
|
+
# (`Dashboard-cGKycyd0.js`), so the bytes behind one of those URLs can
|
|
30
|
+
# never change — cache them for a year and never revalidate. The rest of
|
|
31
|
+
# the bundle (index.html, the favicons, wurk-manifest.json,
|
|
32
|
+
# .vite/manifest.json) keeps a stable name across builds, so it must be
|
|
33
|
+
# revalidated or a client would pin a stale dashboard shell forever.
|
|
34
|
+
FINGERPRINTED_PREFIX = '/assets/'
|
|
35
|
+
IMMUTABLE_CACHE_CONTROL = 'public, max-age=31536000, immutable'
|
|
36
|
+
REVALIDATE_CACHE_CONTROL = 'public, no-cache'
|
|
37
|
+
|
|
38
|
+
# Only a served body gets a cache directive. Rack::Files answers more than
|
|
39
|
+
# file reads: OPTIONS with 200 + Allow, 405 for any other verb
|
|
40
|
+
# (`ALLOWED_VERBS = %w[GET HEAD OPTIONS]`), and 416 for an unsatisfiable
|
|
41
|
+
# Range. 405 is cacheable by default (RFC 9110 §15.5.6), so stamping it
|
|
42
|
+
# `immutable` would let a shared proxy serve "method not allowed" for a
|
|
43
|
+
# year to everyone behind it.
|
|
44
|
+
CACHEABLE_METHODS = %w[GET HEAD].freeze
|
|
45
|
+
CACHEABLE_STATUSES = [200, 206, 304].freeze
|
|
46
|
+
|
|
28
47
|
def initialize(app, root:)
|
|
29
48
|
@app = app
|
|
30
49
|
@files = ::Rack::Files.new(root)
|
|
@@ -38,7 +57,28 @@ module Wurk
|
|
|
38
57
|
stripped = path.delete_prefix(PREFIX)
|
|
39
58
|
inner[::Rack::PATH_INFO] = stripped.empty? ? '/' : stripped
|
|
40
59
|
response = @files.call(inner)
|
|
41
|
-
|
|
60
|
+
return @app.call(env) if response[0] == 404
|
|
61
|
+
|
|
62
|
+
stamp_cache_control(response, env, stripped)
|
|
63
|
+
response
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
# This mount is inserted at index 0 of the HOST app's stack (see the
|
|
69
|
+
# initializer below), so Rack::ETag and Rack::ConditionalGet never see
|
|
70
|
+
# these responses — without this the dashboard's fingerprinted bundle
|
|
71
|
+
# shipped with no cache directive at all and was refetched every load.
|
|
72
|
+
def stamp_cache_control(response, env, stripped)
|
|
73
|
+
return unless CACHEABLE_METHODS.include?(env[::Rack::REQUEST_METHOD])
|
|
74
|
+
return unless CACHEABLE_STATUSES.include?(response[0])
|
|
75
|
+
|
|
76
|
+
response[1]['cache-control'] ||=
|
|
77
|
+
if stripped.start_with?(FINGERPRINTED_PREFIX)
|
|
78
|
+
IMMUTABLE_CACHE_CONTROL
|
|
79
|
+
else
|
|
80
|
+
REVALIDATE_CACHE_CONTROL
|
|
81
|
+
end
|
|
42
82
|
end
|
|
43
83
|
end
|
|
44
84
|
|
data/lib/wurk/errors.rb
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
class Error < StandardError; end
|
|
5
|
+
|
|
6
|
+
# Raised inside a worker process to abort the run loop. User code must not
|
|
7
|
+
# rescue this — the swarm uses it to signal teardown across thread boundaries.
|
|
8
|
+
# Spec: docs/target/sidekiq-free.md §3.
|
|
9
|
+
#
|
|
10
|
+
# Lives in its own file, loaded before everything else, because Processor
|
|
11
|
+
# freezes its `Thread.handle_interrupt` masks into constants: those are
|
|
12
|
+
# evaluated the moment `wurk/processor` is required, so this class has to
|
|
13
|
+
# exist by then.
|
|
14
|
+
class Shutdown < Interrupt; end
|
|
15
|
+
end
|
data/lib/wurk/fetcher/reaper.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative '../component'
|
|
4
4
|
require_relative '../keys'
|
|
5
5
|
require_relative '../middleware/poison_pill'
|
|
6
|
+
require_relative '../timer_loop'
|
|
6
7
|
|
|
7
8
|
module Wurk
|
|
8
9
|
class Fetcher
|
|
@@ -91,13 +92,22 @@ module Wurk
|
|
|
91
92
|
@thread
|
|
92
93
|
end
|
|
93
94
|
|
|
95
|
+
# Bounded at TimerLoop::JOIN_TIMEOUT like every other periodic component.
|
|
96
|
+
# The full-keyspace sweep is exactly the tick that outlasts a stop: it
|
|
97
|
+
# SCANs the whole `queue:*|*` keyspace and drains what it finds, so on a
|
|
98
|
+
# large or slow Redis it can still be running when shutdown lands. Waiting
|
|
99
|
+
# it out held the entire process's teardown open past the swarm parent's
|
|
100
|
+
# SHUTDOWN_GRACE — which SIGKILLs the child mid-drain, the one outcome
|
|
101
|
+
# this component exists to recover from. A straggler is left running
|
|
102
|
+
# instead — and still referenced, like every other periodic component, so
|
|
103
|
+
# a restart after a timed-out stop can't spawn a second sweep loop
|
|
104
|
+
# alongside it. Its own tick_once rescues and reports.
|
|
94
105
|
def stop
|
|
95
106
|
@mutex.synchronize do
|
|
96
107
|
@done = true
|
|
97
108
|
@sleeper.signal
|
|
98
109
|
end
|
|
99
|
-
@thread&.join
|
|
100
|
-
@thread = nil
|
|
110
|
+
@thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
|
|
101
111
|
end
|
|
102
112
|
|
|
103
113
|
def running?
|