wurk 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/lib/wurk/batch/callbacks.rb +82 -12
- data/lib/wurk/batch/death_handler.rb +7 -4
- data/lib/wurk/batch/server_middleware.rb +1 -1
- data/lib/wurk/batch.rb +121 -15
- data/lib/wurk/capsule.rb +5 -4
- data/lib/wurk/cli.rb +48 -14
- data/lib/wurk/client/buffered.rb +193 -43
- data/lib/wurk/client.rb +87 -14
- data/lib/wurk/compat.rb +1 -1
- data/lib/wurk/component.rb +2 -2
- data/lib/wurk/configuration.rb +10 -2
- data/lib/wurk/cron.rb +94 -37
- data/lib/wurk/deploy.rb +5 -3
- data/lib/wurk/embedded.rb +13 -0
- data/lib/wurk/fetcher/reaper.rb +113 -56
- data/lib/wurk/fetcher/reliable.rb +62 -9
- data/lib/wurk/heartbeat.rb +22 -10
- data/lib/wurk/history.rb +13 -1
- data/lib/wurk/launcher.rb +133 -66
- data/lib/wurk/leader.rb +29 -10
- data/lib/wurk/limiter/base.rb +8 -10
- data/lib/wurk/limiter/bucket.rb +1 -1
- data/lib/wurk/limiter/concurrent.rb +27 -22
- data/lib/wurk/limiter/window.rb +13 -11
- data/lib/wurk/limiter.rb +7 -4
- data/lib/wurk/lua.rb +97 -14
- data/lib/wurk/manager.rb +29 -13
- data/lib/wurk/metrics/history.rb +4 -3
- data/lib/wurk/metrics/queue_rollup.rb +13 -1
- data/lib/wurk/metrics/rollup.rb +13 -1
- data/lib/wurk/middleware/interrupt_handler.rb +7 -6
- data/lib/wurk/middleware/poison_pill.rb +70 -29
- data/lib/wurk/middleware.rb +2 -2
- data/lib/wurk/pool_checkout.rb +29 -0
- data/lib/wurk/process_set.rb +10 -5
- data/lib/wurk/processor.rb +6 -0
- data/lib/wurk/profiler.rb +3 -2
- data/lib/wurk/queue.rb +10 -7
- data/lib/wurk/rails_boot.rb +38 -7
- data/lib/wurk/redis_client_adapter.rb +48 -4
- data/lib/wurk/redis_options.rb +142 -0
- data/lib/wurk/redis_pool.rb +102 -39
- data/lib/wurk/scheduled.rb +30 -2
- data/lib/wurk/stats.rb +14 -9
- data/lib/wurk/swarm/child_boot.rb +12 -0
- data/lib/wurk/swarm.rb +174 -33
- data/lib/wurk/timer_loop.rb +14 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +58 -6
- data/lib/wurk/web/extension.rb +1 -1
- data/lib/wurk/web/search.rb +5 -3
- data/lib/wurk.rb +53 -2
- data/vendor/assets/dashboard/assets/{BatchDetail-YRymNsrB.js → BatchDetail-OmC5NPgw.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-HY4hHdQU.js → Batches-CIpai7St.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-FCEN1Bpx.js → Busy-A_kwSR6Q.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-DO3J2zcp.js → Cron-BG7HTqlp.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-B9rOrkzk.js → Dashboard-A_ToqHoo.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Bi4GGk9a.js → Dead-8J21jMyK.js} +1 -1
- data/vendor/assets/dashboard/assets/Extension-B4Q9FIQu.js +1 -0
- data/vendor/assets/dashboard/assets/{FilterBox-IJkHYpdm.js → FilterBox-Fh_Ae7UW.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-DS1ypyoc.js → JobDetailModal-Ceng0PMB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Nz7UbNeJ.js → Limiters-CruDWvNZ.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-BBTDxcaE.js → Metrics-CIT7VCoN.js} +1 -1
- data/vendor/assets/dashboard/assets/Modal-CN3rdKA_.js +1 -0
- data/vendor/assets/dashboard/assets/{Queues-D9PH_THs.js → Queues-D86FYohJ.js} +1 -1
- data/vendor/assets/dashboard/assets/{Retries-CAKzDgYG.js → Retries-Bz1O1D-i.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DTYw1X8S.js → Scheduled-B6h2akTu.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-c4vFxDG_.js → Search-OOu22e5s.js} +1 -1
- data/vendor/assets/dashboard/assets/index-BdiUEDXX.css +1 -0
- data/vendor/assets/dashboard/assets/index-D_lSDwKw.js +141 -0
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-B_FsMah6.js → useResetPageOnEmpty-dVPGEWzn.js} +1 -1
- data/vendor/assets/dashboard/index.html +2 -2
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +22 -20
- data/vendor/assets/dashboard/assets/Extension-BSv8ddW_.js +0 -1
- data/vendor/assets/dashboard/assets/Modal-Crrsu64-.js +0 -1
- data/vendor/assets/dashboard/assets/index-BxjdeuOa.css +0 -1
- data/vendor/assets/dashboard/assets/index-DQu7WY9y.js +0 -141
data/lib/wurk/swarm.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'monitor'
|
|
4
|
+
|
|
3
5
|
require_relative 'component'
|
|
4
6
|
require_relative 'launcher'
|
|
5
7
|
require_relative 'fetcher/reliable'
|
|
@@ -42,6 +44,16 @@ module Wurk
|
|
|
42
44
|
MEMORY_CHECK_INTERVAL = 10
|
|
43
45
|
DEFAULT_SHUTDOWN_TIMEOUT = 25
|
|
44
46
|
|
|
47
|
+
# The supervisor's own Redis pool: one connection, disjoint from every
|
|
48
|
+
# capsule pool. Post-boot the parent's only Redis use is the heartbeat probe
|
|
49
|
+
# (`heartbeat_seen?`) — one SISMEMBER per restart tick — and routing it
|
|
50
|
+
# through the capsule main pool reopened, at capsule size (>= 10), the very
|
|
51
|
+
# sockets step 3 had just closed; the next respawn/restart fork then
|
|
52
|
+
# inherited them. Dedicated and torn down before every fork, so no child
|
|
53
|
+
# ever sees a parent socket.
|
|
54
|
+
SUPERVISOR_POOL_SIZE = 1
|
|
55
|
+
SUPERVISOR_POOL_NAME = 'swarm-supervisor'
|
|
56
|
+
|
|
45
57
|
# Children each hard_shutdown after their own drain deadline (bulk_requeue
|
|
46
58
|
# + a 3s ensure window + heartbeat cleanup); the parent must not SIGKILL
|
|
47
59
|
# them mid-tail, so its own wait always extends past theirs by this much.
|
|
@@ -52,7 +64,7 @@ module Wurk
|
|
|
52
64
|
# whole swarm.
|
|
53
65
|
SWARM_SIGNALS = { 'TERM' => :term, 'INT' => :term, 'TSTP' => :tstp, 'USR1' => :usr1, 'USR2' => :usr2 }.freeze
|
|
54
66
|
|
|
55
|
-
attr_reader :topology
|
|
67
|
+
attr_reader :topology
|
|
56
68
|
|
|
57
69
|
def initialize(topology:, config: Wurk.configuration, memory_limit: config.memory_limit_kb,
|
|
58
70
|
shutdown_timeout: DEFAULT_SHUTDOWN_TIMEOUT)
|
|
@@ -60,15 +72,24 @@ module Wurk
|
|
|
60
72
|
@config = config
|
|
61
73
|
@memory_limit = memory_limit
|
|
62
74
|
@shutdown_timeout = shutdown_timeout
|
|
75
|
+
# One reentrant lock covers the child table AND the restart machine: they
|
|
76
|
+
# are mutually recursive (Restart#advance calls back into `describe` and
|
|
77
|
+
# `spawn`, which read and write `@children`), so separate locks would be
|
|
78
|
+
# taken in opposite orders — enqueue holds children→restart, advance holds
|
|
79
|
+
# restart→children — and deadlock. Monitor's reentrancy is what lets those
|
|
80
|
+
# callbacks re-enter. Never held across a sleep: `wait_for_children` and
|
|
81
|
+
# the supervise tick sleep outside it.
|
|
82
|
+
@lock = ::Monitor.new
|
|
63
83
|
@children = {}
|
|
64
84
|
@assignments = []
|
|
85
|
+
@owner_pid = nil
|
|
65
86
|
@stopping = false
|
|
87
|
+
@shutdown_requested = false
|
|
66
88
|
@quieted = false
|
|
67
89
|
@last_memory_check = 0
|
|
68
|
-
@signal_read = nil
|
|
69
|
-
@signal_write = nil
|
|
70
90
|
@respawn_backoff = Backoff.new(base: RESPAWN_BACKOFF)
|
|
71
91
|
@restart = build_restart
|
|
92
|
+
init_fork_unsafe_handles
|
|
72
93
|
end
|
|
73
94
|
|
|
74
95
|
# `install_signals:` is false in tests so the integration suite can
|
|
@@ -85,29 +106,65 @@ module Wurk
|
|
|
85
106
|
raise ArgumentError, 'Topology has no slots' if @topology.empty?
|
|
86
107
|
|
|
87
108
|
@assignments = @topology.assignments.freeze
|
|
109
|
+
@owner_pid = ::Process.pid
|
|
88
110
|
install_signal_handlers if install_signals
|
|
89
111
|
close_parent_sockets
|
|
90
112
|
fork_children
|
|
91
|
-
|
|
113
|
+
child_pids
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# A snapshot, not the live table. Callers read this off the supervise
|
|
117
|
+
# thread, which inserts (respawn, restart) and deletes (reap) on every tick;
|
|
118
|
+
# handing out the live Hash lets them iterate it mid-mutation.
|
|
119
|
+
def children
|
|
120
|
+
@lock.synchronize { @children.dup }
|
|
92
121
|
end
|
|
93
122
|
|
|
94
123
|
def supervise
|
|
124
|
+
return unless owner?
|
|
125
|
+
|
|
95
126
|
until done?
|
|
96
127
|
drain_signals
|
|
128
|
+
shutdown if @shutdown_requested && !@stopping
|
|
97
129
|
reap_children
|
|
98
130
|
spawn_due_respawns
|
|
99
|
-
|
|
131
|
+
advance_restart unless @stopping
|
|
100
132
|
check_memory_pressure
|
|
101
133
|
sleep SUPERVISE_TICK
|
|
102
134
|
end
|
|
103
135
|
end
|
|
104
136
|
|
|
105
137
|
def shutdown(timeout: @shutdown_timeout)
|
|
138
|
+
return unless owner?
|
|
139
|
+
|
|
106
140
|
@stopping = true
|
|
107
|
-
@restart.abort
|
|
141
|
+
@lock.synchronize { @restart.abort }
|
|
108
142
|
relay_signal('TERM')
|
|
109
143
|
wait_for_children(timeout + SHUTDOWN_GRACE)
|
|
110
144
|
hard_kill_stragglers
|
|
145
|
+
close_supervisor_pool
|
|
146
|
+
close_signal_pipe
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Cross-thread drain request: raise a flag the supervise loop observes on
|
|
150
|
+
# its next tick instead of draining on the caller's thread. `shutdown`
|
|
151
|
+
# walks and clears the child table that the supervise thread is
|
|
152
|
+
# concurrently mutating (reap, respawn, restart), so two threads inside it
|
|
153
|
+
# race. Every caller that isn't the supervise thread — rails_boot's
|
|
154
|
+
# `at_exit`, which fires on the host's main thread — comes through here.
|
|
155
|
+
def request_shutdown
|
|
156
|
+
@shutdown_requested = true
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Only the process that forked the children may supervise or signal them.
|
|
160
|
+
# A forked child inherits this object along with the host's `at_exit` hooks
|
|
161
|
+
# — and rails_boot registers one that drains the swarm — so ChildBoot's
|
|
162
|
+
# `exit` reaches a full drain inside the child. There `@children` holds the
|
|
163
|
+
# child's SIBLINGS, not its own children: unguarded, that TERMs them, stalls
|
|
164
|
+
# the whole shutdown timeout waiting on PIDs it can never reap, then SIGKILLs
|
|
165
|
+
# whatever survived.
|
|
166
|
+
def owner?
|
|
167
|
+
::Process.pid == @owner_pid
|
|
111
168
|
end
|
|
112
169
|
|
|
113
170
|
# TSTP quiet is one-way and GLOBAL (spec §21.3): it must survive respawns
|
|
@@ -124,17 +181,44 @@ module Wurk
|
|
|
124
181
|
# heartbeat → TERM the old child → await its drain) without blocking the
|
|
125
182
|
# supervise thread, so TERM stays responsive throughout the cycle.
|
|
126
183
|
def rolling_restart
|
|
127
|
-
@restart.enqueue(@children.keys)
|
|
184
|
+
@lock.synchronize { @restart.enqueue(@children.keys) }
|
|
128
185
|
end
|
|
129
186
|
|
|
130
187
|
private
|
|
131
188
|
|
|
189
|
+
# The handles the parent must never let cross a fork: the signal self-pipe
|
|
190
|
+
# (an inherited trap would write into the PARENT's supervise loop) and the
|
|
191
|
+
# supervisor's Redis socket. Grouped because they share one rule — opened in
|
|
192
|
+
# the parent, dropped before or at every fork by `close_supervisor_pool` and
|
|
193
|
+
# `close_signal_pipe`.
|
|
194
|
+
def init_fork_unsafe_handles
|
|
195
|
+
@signal_read = nil
|
|
196
|
+
@signal_write = nil
|
|
197
|
+
@supervisor_pool = nil
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def advance_restart
|
|
201
|
+
@lock.synchronize { @restart.advance }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def child_pids
|
|
205
|
+
@lock.synchronize { @children.keys }
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def child_meta(pid)
|
|
209
|
+
@lock.synchronize { @children[pid] }
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def any_children?
|
|
213
|
+
@lock.synchronize { !@children.empty? }
|
|
214
|
+
end
|
|
215
|
+
|
|
132
216
|
def build_restart
|
|
133
217
|
Restart.new(Restart::Config.new(
|
|
134
218
|
spawn: method(:spawn_child),
|
|
135
219
|
kill: method(:safe_kill),
|
|
136
220
|
heartbeat: method(:heartbeat_seen?),
|
|
137
|
-
describe: ->(pid) {
|
|
221
|
+
describe: ->(pid) { child_meta(pid) },
|
|
138
222
|
now: method(:monotonic),
|
|
139
223
|
logger: logger,
|
|
140
224
|
heartbeat_wait: HEARTBEAT_WAIT,
|
|
@@ -168,7 +252,7 @@ module Wurk
|
|
|
168
252
|
# Returns the child PID; never returns in the child (ChildBoot exits).
|
|
169
253
|
def spawn_child(slot, idx)
|
|
170
254
|
pid = fork_child(slot, idx)
|
|
171
|
-
@children[pid] = { slot: slot, index: idx, spawned_at: monotonic }
|
|
255
|
+
@lock.synchronize { @children[pid] = { slot: slot, index: idx, spawned_at: monotonic } }
|
|
172
256
|
pid
|
|
173
257
|
end
|
|
174
258
|
|
|
@@ -176,17 +260,21 @@ module Wurk
|
|
|
176
260
|
# from the parent, it is race-free even if the parent dies the instant
|
|
177
261
|
# after fork (getppid in the child could already return the reaper). The
|
|
178
262
|
# child's OrphanGuard compares live getppid against it.
|
|
263
|
+
#
|
|
264
|
+
# The supervisor pool closes here rather than in `close_parent_sockets`
|
|
265
|
+
# because that runs once, at boot: every later respawn / rolling-restart
|
|
266
|
+
# fork happens after the heartbeat probe has reopened it.
|
|
179
267
|
def fork_child(slot, idx)
|
|
180
268
|
parent_pid = ::Process.pid
|
|
269
|
+
close_supervisor_pool
|
|
181
270
|
pid = ::Process.fork
|
|
182
271
|
return pid if pid
|
|
183
272
|
|
|
184
273
|
# Drop the parent's self-pipe first: the inherited traps still write to
|
|
185
274
|
# it, so a signal landing in the window before ChildBoot resets them
|
|
186
275
|
# would surface in the PARENT's supervise loop (an operator TERMing one
|
|
187
|
-
# child pid would drain the whole swarm).
|
|
188
|
-
|
|
189
|
-
@signal_write&.close
|
|
276
|
+
# child pid would drain the whole swarm). Dropped, the trap write no-ops.
|
|
277
|
+
close_signal_pipe
|
|
190
278
|
ChildBoot.new(@config, slot, idx, parent_pid: parent_pid, start_quiet: @quieted).run
|
|
191
279
|
exit 0 # unreachable; ChildBoot exits explicitly
|
|
192
280
|
end
|
|
@@ -204,12 +292,25 @@ module Wurk
|
|
|
204
292
|
end
|
|
205
293
|
end
|
|
206
294
|
|
|
295
|
+
# Both FDs, once. The traps stay installed and keep firing after a drain, so
|
|
296
|
+
# clear the ivars BEFORE closing: `emit_signal` then writes to nothing
|
|
297
|
+
# instead of a dead FD, and a `drain_signals` loop mid-tick (TERM handled,
|
|
298
|
+
# asking for the next buffered signal) stops instead of reading a closed
|
|
299
|
+
# pipe. Called on shutdown and, in the child, immediately after fork.
|
|
300
|
+
def close_signal_pipe
|
|
301
|
+
read = @signal_read
|
|
302
|
+
write = @signal_write
|
|
303
|
+
@signal_read = @signal_write = nil
|
|
304
|
+
read&.close
|
|
305
|
+
write&.close
|
|
306
|
+
end
|
|
307
|
+
|
|
207
308
|
# Non-blocking self-pipe write from trap context: a blocking `puts` could
|
|
208
309
|
# stall signal delivery if the pipe fills. `exception: false` returns
|
|
209
310
|
# :wait_writable instead of raising when full (drop the coalescible
|
|
210
311
|
# duplicate); a closed pipe during shutdown is ignored too.
|
|
211
312
|
def emit_signal(sig)
|
|
212
|
-
@signal_write
|
|
313
|
+
@signal_write&.write_nonblock("#{sig}\n", exception: false)
|
|
213
314
|
rescue ::IOError, ::Errno::EPIPE, ::Errno::EBADF
|
|
214
315
|
nil
|
|
215
316
|
end
|
|
@@ -222,17 +323,29 @@ module Wurk
|
|
|
222
323
|
when :term then shutdown
|
|
223
324
|
when :tstp then quiet_swarm
|
|
224
325
|
when :usr1 then rolling_restart
|
|
225
|
-
when :usr2
|
|
326
|
+
when :usr2
|
|
327
|
+
reopen_logs
|
|
328
|
+
relay_signal('USR2')
|
|
226
329
|
end
|
|
227
330
|
end
|
|
228
331
|
end
|
|
229
332
|
|
|
230
333
|
# One buffered line per pending signal, non-blocking (wait_readable(0)).
|
|
231
|
-
# nil once the pipe is drained, ending the loop for this tick.
|
|
334
|
+
# nil once the pipe is drained, ending the loop for this tick. Reads through
|
|
335
|
+
# a local because the pipe can vanish mid-drain: a buffered TERM runs
|
|
336
|
+
# `shutdown`, which closes it, and the loop then asks for the next signal.
|
|
232
337
|
def read_pending_signal
|
|
233
|
-
|
|
338
|
+
read = @signal_read
|
|
339
|
+
return nil unless read&.wait_readable(0)
|
|
234
340
|
|
|
235
|
-
|
|
341
|
+
read.gets&.strip
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def reopen_logs
|
|
345
|
+
log = @config.logger
|
|
346
|
+
log.reopen if log.respond_to?(:reopen)
|
|
347
|
+
rescue StandardError
|
|
348
|
+
nil
|
|
236
349
|
end
|
|
237
350
|
|
|
238
351
|
# Reap every exited child this tick (not one), so a fleet-wide death
|
|
@@ -251,14 +364,16 @@ module Wurk
|
|
|
251
364
|
end
|
|
252
365
|
|
|
253
366
|
def on_child_exit(pid, status)
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
367
|
+
@lock.synchronize do
|
|
368
|
+
meta = @children.delete(pid)
|
|
369
|
+
return unless meta
|
|
370
|
+
return if @restart.claim_exit(pid)
|
|
371
|
+
|
|
372
|
+
if @stopping
|
|
373
|
+
logger.info { "swarm: child #{pid} exited (status=#{status.exitstatus})" }
|
|
374
|
+
else
|
|
375
|
+
schedule_respawn(pid, status, meta)
|
|
376
|
+
end
|
|
262
377
|
end
|
|
263
378
|
end
|
|
264
379
|
|
|
@@ -305,7 +420,7 @@ module Wurk
|
|
|
305
420
|
return if now - @last_memory_check < MEMORY_CHECK_INTERVAL
|
|
306
421
|
|
|
307
422
|
@last_memory_check = now
|
|
308
|
-
|
|
423
|
+
child_pids.each { |pid| recycle_if_bloated(pid) }
|
|
309
424
|
end
|
|
310
425
|
|
|
311
426
|
# Route a bloated child through the restart state machine (same path as a
|
|
@@ -317,7 +432,7 @@ module Wurk
|
|
|
317
432
|
return if rss.nil? || rss < @memory_limit
|
|
318
433
|
|
|
319
434
|
logger.warn { "swarm: child #{pid} RSS #{rss}KB >= #{@memory_limit}KB; recycling" }
|
|
320
|
-
@restart.enqueue([pid])
|
|
435
|
+
@lock.synchronize { @restart.enqueue([pid]) }
|
|
321
436
|
end
|
|
322
437
|
|
|
323
438
|
def pid_rss_kb(pid)
|
|
@@ -329,10 +444,15 @@ module Wurk
|
|
|
329
444
|
end
|
|
330
445
|
|
|
331
446
|
def relay_signal(sig)
|
|
332
|
-
|
|
447
|
+
child_pids.each { |pid| safe_kill(pid, sig) }
|
|
333
448
|
end
|
|
334
449
|
|
|
450
|
+
# The one place the supervisor signals a child: relay_signal,
|
|
451
|
+
# hard_kill_stragglers and the restart machine's `kill:` callback all funnel
|
|
452
|
+
# through here, so the owner check covers every path at a single choke point.
|
|
335
453
|
def safe_kill(pid, sig)
|
|
454
|
+
return unless owner?
|
|
455
|
+
|
|
336
456
|
::Process.kill(sig, pid)
|
|
337
457
|
rescue Errno::ESRCH, Errno::EPERM
|
|
338
458
|
nil
|
|
@@ -340,15 +460,20 @@ module Wurk
|
|
|
340
460
|
|
|
341
461
|
def wait_for_children(timeout)
|
|
342
462
|
deadline = monotonic + timeout
|
|
343
|
-
while monotonic < deadline &&
|
|
463
|
+
while monotonic < deadline && any_children?
|
|
344
464
|
reap_children
|
|
345
465
|
sleep 0.1
|
|
346
466
|
end
|
|
347
467
|
end
|
|
348
468
|
|
|
469
|
+
# Kill and forget atomically: a child landing in the table between the walk
|
|
470
|
+
# and the clear would be dropped from it without ever being signalled —
|
|
471
|
+
# untracked and still alive.
|
|
349
472
|
def hard_kill_stragglers
|
|
350
|
-
@
|
|
351
|
-
|
|
473
|
+
@lock.synchronize do
|
|
474
|
+
@children.each_key { |pid| safe_kill(pid, 'KILL') }
|
|
475
|
+
@children.clear
|
|
476
|
+
end
|
|
352
477
|
end
|
|
353
478
|
|
|
354
479
|
# Has the child written its first heartbeat yet? One non-blocking SISMEMBER,
|
|
@@ -360,17 +485,33 @@ module Wurk
|
|
|
360
485
|
# forces progress.
|
|
361
486
|
def heartbeat_seen?(pid)
|
|
362
487
|
identity = "#{hostname}:#{pid}:#{Component::PROCESS_NONCE}"
|
|
363
|
-
|
|
488
|
+
supervisor_pool.with { |c| c.call('SISMEMBER', Keys::PROCESSES, identity) } == 1
|
|
364
489
|
rescue StandardError
|
|
365
490
|
false
|
|
366
491
|
end
|
|
367
492
|
|
|
493
|
+
def supervisor_pool
|
|
494
|
+
@supervisor_pool ||= @config.new_redis_pool(SUPERVISOR_POOL_SIZE, SUPERVISOR_POOL_NAME)
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
# ConnectionPool#shutdown is terminal, so the reference has to go with the
|
|
498
|
+
# socket — the next probe rebuilds. Cleared before disconnecting so a raise
|
|
499
|
+
# mid-teardown can't leave a shut-down pool wired up, which would make every
|
|
500
|
+
# later probe raise PoolShuttingDownError instead of reconnecting.
|
|
501
|
+
def close_supervisor_pool
|
|
502
|
+
pool = @supervisor_pool
|
|
503
|
+
@supervisor_pool = nil
|
|
504
|
+
pool&.disconnect!
|
|
505
|
+
rescue StandardError => e
|
|
506
|
+
logger.warn { "swarm: supervisor pool close failed: #{e.class}: #{e.message}" }
|
|
507
|
+
end
|
|
508
|
+
|
|
368
509
|
def monotonic
|
|
369
510
|
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
370
511
|
end
|
|
371
512
|
|
|
372
513
|
def done?
|
|
373
|
-
@stopping &&
|
|
514
|
+
@stopping && !any_children?
|
|
374
515
|
end
|
|
375
516
|
end
|
|
376
517
|
end
|
data/lib/wurk/timer_loop.rb
CHANGED
|
@@ -15,6 +15,13 @@ module Wurk
|
|
|
15
15
|
# @thread ||= safe_thread('my-loop') { @timer.run { tick } }
|
|
16
16
|
# def terminate = @timer.terminate
|
|
17
17
|
class TimerLoop
|
|
18
|
+
# Deadline every periodic component gives its loop thread when stopping.
|
|
19
|
+
# Bounded rather than an open-ended join: a tick blocked on a wedged Redis
|
|
20
|
+
# must not hold the process's whole shutdown open. Also used by
|
|
21
|
+
# Scheduled::Poller, which hand-rolls its wait (the interval is randomized
|
|
22
|
+
# per iteration) but stops on the same terms.
|
|
23
|
+
JOIN_TIMEOUT = 5
|
|
24
|
+
|
|
18
25
|
def initialize(interval)
|
|
19
26
|
@interval = interval
|
|
20
27
|
@done = false
|
|
@@ -40,6 +47,13 @@ module Wurk
|
|
|
40
47
|
end
|
|
41
48
|
end
|
|
42
49
|
|
|
50
|
+
# Clears a previous #terminate. Hosts call this before respawning, so a
|
|
51
|
+
# start-after-stop cycle spawns a thread that ticks instead of one that
|
|
52
|
+
# sees the stale done flag and exits immediately.
|
|
53
|
+
def reset
|
|
54
|
+
@mutex.synchronize { @done = false }
|
|
55
|
+
end
|
|
56
|
+
|
|
43
57
|
def wait
|
|
44
58
|
@mutex.synchronize do
|
|
45
59
|
@sleeper.wait(@mutex, @interval) unless @done
|
data/lib/wurk/version.rb
CHANGED
data/lib/wurk/web/enterprise.rb
CHANGED
|
@@ -22,21 +22,72 @@ module Wurk
|
|
|
22
22
|
module Limits
|
|
23
23
|
module_function
|
|
24
24
|
|
|
25
|
-
#
|
|
26
|
-
#
|
|
25
|
+
# Names per sweep round trip. Bounds both halves: the probe's pipeline
|
|
26
|
+
# (whose replies the client buffers) and the removal script, which is
|
|
27
|
+
# atomic — an unbatched pass over a set that leaked for months would
|
|
28
|
+
# block every other Redis client for its whole length. A healthy set
|
|
29
|
+
# fits in one slice, so the common case still costs one probe.
|
|
30
|
+
SWEEP_BATCH = 512
|
|
31
|
+
|
|
32
|
+
# @return [Array<String>] live limiter names, optionally filtered to
|
|
33
|
+
# those whose name contains the case-insensitive substring `filter`.
|
|
34
|
+
#
|
|
35
|
+
# Listing is also the sweep. `lmtr-list` membership has no TTL of its
|
|
36
|
+
# own while `lmtr:<name>` expires with the limiter's ttl, so every
|
|
37
|
+
# interpolated name (`stripe-#{user_id}`) the spec blesses would leave a
|
|
38
|
+
# permanent member behind. Names whose metadata is gone are dropped from
|
|
39
|
+
# the SET and from the result, so the dashboard also stops rendering a
|
|
40
|
+
# row with nothing behind it. The liveness probe rides along with the
|
|
41
|
+
# SMEMBERS the listing already pays for — both are O(set) — and once the
|
|
42
|
+
# sweep has run the set is back down to live limiters.
|
|
27
43
|
def list(filter: nil)
|
|
28
|
-
names = Wurk.redis { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }.sort
|
|
44
|
+
names = sweep(Wurk.redis(idempotent: true) { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }).sort
|
|
29
45
|
return names if filter.nil? || filter.to_s.empty?
|
|
30
46
|
|
|
31
47
|
needle = filter.to_s.downcase
|
|
32
48
|
names.select { |n| n.downcase.include?(needle) }
|
|
33
49
|
end
|
|
34
50
|
|
|
51
|
+
# Drop the names whose metadata is gone, return the ones still live.
|
|
52
|
+
def sweep(names)
|
|
53
|
+
names.each_slice(SWEEP_BATCH).flat_map { |slice| sweep_slice(slice) }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def sweep_slice(names)
|
|
57
|
+
live, dead = partition_live(names)
|
|
58
|
+
sweep_batch(dead) unless dead.empty?
|
|
59
|
+
live
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def partition_live(names)
|
|
63
|
+
flags = Wurk.redis(idempotent: true) do |c|
|
|
64
|
+
c.pipelined { |pipe| names.each { |n| pipe.call('EXISTS', meta_key(n)) } }
|
|
65
|
+
end
|
|
66
|
+
names.partition.with_index { |_, i| flags[i].to_i == 1 }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Claims apply-safety, unlike ProcessSet's comparable prune: the script
|
|
70
|
+
# re-decides liveness inside its own atomic step, so a replay can only
|
|
71
|
+
# remove names that are still dead.
|
|
72
|
+
def sweep_batch(names)
|
|
73
|
+
Wurk.redis(idempotent: true) do |c|
|
|
74
|
+
Wurk::Lua::Loader.eval_cached(
|
|
75
|
+
c, :limiter_list_sweep,
|
|
76
|
+
keys: [Wurk::Limiter::LIST_KEY],
|
|
77
|
+
argv: ['lmtr:', *names]
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
35
82
|
def metadata(name)
|
|
36
|
-
raw = Wurk.redis { |c| c.call('HGETALL',
|
|
83
|
+
raw = Wurk.redis(idempotent: true) { |c| c.call('HGETALL', meta_key(name)) }
|
|
37
84
|
raw.is_a?(Array) ? raw.each_slice(2).to_h : raw
|
|
38
85
|
end
|
|
39
86
|
|
|
87
|
+
def meta_key(name)
|
|
88
|
+
"lmtr:#{name}"
|
|
89
|
+
end
|
|
90
|
+
|
|
40
91
|
# Stats-key reset: drops every `lmtr-stats:` / state key for the
|
|
41
92
|
# named limiter while keeping its metadata + LIST membership so the
|
|
42
93
|
# row remains in the UI. Mirrors the §1.5 `#reset` surface — the
|
|
@@ -51,8 +102,9 @@ module Wurk
|
|
|
51
102
|
end
|
|
52
103
|
|
|
53
104
|
# Read-only reconstruction (`register: false`) from the persisted
|
|
54
|
-
# metadata. Returns nil when the meta HASH is gone or malformed —
|
|
55
|
-
#
|
|
105
|
+
# metadata. Returns nil when the meta HASH is gone or malformed — the
|
|
106
|
+
# name a caller holds came from a page `list` rendered earlier, and the
|
|
107
|
+
# metadata can expire between the two.
|
|
56
108
|
def rebuild(name)
|
|
57
109
|
meta = metadata(name)
|
|
58
110
|
return nil if meta.nil? || meta.empty?
|
data/lib/wurk/web/extension.rb
CHANGED
|
@@ -148,7 +148,7 @@ module Wurk
|
|
|
148
148
|
%(<time class="ltr" dir="ltr" title="#{iso}" datetime="#{iso}">#{iso}</time>)
|
|
149
149
|
end
|
|
150
150
|
|
|
151
|
-
def redis(&) = ::Wurk.redis(&)
|
|
151
|
+
def redis(idempotent: false, &) = ::Wurk.redis(idempotent:, &)
|
|
152
152
|
def product_version = ::Wurk::VERSION
|
|
153
153
|
def number_with_delimiter(num) = num.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
|
|
154
154
|
end
|
data/lib/wurk/web/search.rb
CHANGED
|
@@ -103,10 +103,12 @@ module Wurk
|
|
|
103
103
|
# we scan a bounded prefix and never full-walk the LIST from a keystroke.
|
|
104
104
|
def scan_queue_list(queue_name, &)
|
|
105
105
|
key = Keys.queue(queue_name)
|
|
106
|
-
stop_at = queue_scan_stop(Wurk.redis { |c| c.call('LLEN', key) })
|
|
106
|
+
stop_at = queue_scan_stop(Wurk.redis(idempotent: true) { |c| c.call('LLEN', key) })
|
|
107
107
|
start = 0
|
|
108
108
|
while start < stop_at
|
|
109
|
-
slice = Wurk.redis
|
|
109
|
+
slice = Wurk.redis(idempotent: true) do |c|
|
|
110
|
+
c.call('LRANGE', key, start, [start + QUEUE_PAGE, stop_at].min - 1)
|
|
111
|
+
end
|
|
110
112
|
break if slice.empty?
|
|
111
113
|
|
|
112
114
|
slice.each(&)
|
|
@@ -134,7 +136,7 @@ module Wurk
|
|
|
134
136
|
set = sorted_set_for(kind)
|
|
135
137
|
cursor = '0'
|
|
136
138
|
loop do
|
|
137
|
-
cursor, page = Wurk.redis { |c| c.call('ZSCAN', set.name, cursor, 'COUNT', ZSCAN_PAGE) }
|
|
139
|
+
cursor, page = Wurk.redis(idempotent: true) { |c| c.call('ZSCAN', set.name, cursor, 'COUNT', ZSCAN_PAGE) }
|
|
138
140
|
emit_sorted_hits(kind, set, page, &)
|
|
139
141
|
@scanned += page.size / 2
|
|
140
142
|
break if cursor == '0'
|
data/lib/wurk.rb
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
require_relative 'wurk/version'
|
|
9
9
|
require_relative 'wurk/keys'
|
|
10
10
|
require_relative 'wurk/redis_pool'
|
|
11
|
+
require_relative 'wurk/pool_checkout'
|
|
11
12
|
require_relative 'wurk/redis_connection'
|
|
12
13
|
require_relative 'wurk/middleware'
|
|
13
14
|
require_relative 'wurk/middleware/chain'
|
|
@@ -126,8 +127,15 @@ module Wurk
|
|
|
126
127
|
end
|
|
127
128
|
alias default_configuration configuration
|
|
128
129
|
|
|
129
|
-
|
|
130
|
-
|
|
130
|
+
# `idempotent: true` asserts the block is safe to re-run after a command may
|
|
131
|
+
# already have applied server-side, buying back the full connection backoff
|
|
132
|
+
# (see RedisPool#with). Every wrapper down the chain — Capsule,
|
|
133
|
+
# Configuration, Component, middleware, Sidekiq.redis — forwards it, so a
|
|
134
|
+
# caller can claim apply-safety wherever it holds; PoolCheckout drops the
|
|
135
|
+
# claim for a host-supplied pool that wouldn't understand it. The zero-arg
|
|
136
|
+
# shape is the drop-in contract: `Sidekiq.redis { |c| ... }` never changes.
|
|
137
|
+
def redis(idempotent: false, &)
|
|
138
|
+
PoolCheckout.with(redis_pool, idempotent, &)
|
|
131
139
|
end
|
|
132
140
|
|
|
133
141
|
# Capsule-aware pool lookup. Thread-local override wins so per-capsule
|
|
@@ -224,6 +232,22 @@ module Wurk
|
|
|
224
232
|
def ent?
|
|
225
233
|
false
|
|
226
234
|
end
|
|
235
|
+
|
|
236
|
+
# Lazily loads the Rails engine the first time something asks for
|
|
237
|
+
# `Wurk::Engine`. See the note above the `require "wurk/rails"` guard at the
|
|
238
|
+
# bottom of this file for why the guard alone isn't enough (#282), and why
|
|
239
|
+
# this loads the engine and not the railtie.
|
|
240
|
+
def const_missing(name)
|
|
241
|
+
return super unless name == :Engine
|
|
242
|
+
return super unless defined?(::Rails::Engine) && defined?(::ActionDispatch::Routing::RouteSet)
|
|
243
|
+
|
|
244
|
+
require_relative 'wurk/engine'
|
|
245
|
+
# If engine.rb somehow didn't define it, fall through to the real NameError
|
|
246
|
+
# rather than recursing back into here.
|
|
247
|
+
return super unless const_defined?(:Engine, false)
|
|
248
|
+
|
|
249
|
+
const_get(:Engine, false)
|
|
250
|
+
end
|
|
227
251
|
end
|
|
228
252
|
end
|
|
229
253
|
|
|
@@ -318,3 +342,30 @@ require_relative 'wurk/compat'
|
|
|
318
342
|
# real Rails host both are loaded by `rails/all` before Bundler.require, so the
|
|
319
343
|
# stricter gate is invisible there.
|
|
320
344
|
require_relative 'wurk/rails' if defined?(Rails::Engine) && defined?(::ActionDispatch::Routing::RouteSet)
|
|
345
|
+
|
|
346
|
+
# The gate above is evaluated exactly once, when this file is first required —
|
|
347
|
+
# which is too early for the standalone runners (#282). `exe/wurk` and
|
|
348
|
+
# `exe/wurkswarm` require "wurk" before Rails exists (the gate is false, nothing
|
|
349
|
+
# Rails-y loads), then boot the host app themselves via
|
|
350
|
+
# Wurk::CLI#boot_rails_application. By then "wurk" is in $LOADED_FEATURES, so the
|
|
351
|
+
# app's own `Bundler.require` is a no-op and the gate never gets a second look. A
|
|
352
|
+
# host that did exactly what the README says — `mount Wurk::Engine => "/wurk"` in
|
|
353
|
+
# config/routes.rb — then dies during boot with `uninitialized constant
|
|
354
|
+
# Wurk::Engine`, from the worker process only. Web processes are unaffected:
|
|
355
|
+
# there Bundler.require runs after railties, so the gate passes.
|
|
356
|
+
#
|
|
357
|
+
# `Wurk.const_missing` (defined in the class << self block above) closes that
|
|
358
|
+
# window without giving up the lean standalone boot: nothing loads until
|
|
359
|
+
# something actually asks for Wurk::Engine, and even then only when Rails is
|
|
360
|
+
# really present. It reuses the same two-part condition as the guard above, so
|
|
361
|
+
# the ecosystem carve-out (rails/engine/railties without ActionDispatch, per
|
|
362
|
+
# sidekiq-cron's test helper) still falls through to a plain NameError instead of
|
|
363
|
+
# crashing on `isolate_namespace`.
|
|
364
|
+
#
|
|
365
|
+
# It deliberately loads `wurk/engine`, not `wurk/rails`: the railtie's
|
|
366
|
+
# `config.after_initialize` is a *global* ActiveSupport load hook registered at
|
|
367
|
+
# require time, and routes are drawn before those hooks run — so dragging the
|
|
368
|
+
# railtie in there would fork a swarm inside a `wurkswarm` parent that is about
|
|
369
|
+
# to fork its own. The runner owns the swarm; the routes file only needs the
|
|
370
|
+
# constant. `defined?(Wurk::Engine)` stays nil until first use, which is what the
|
|
371
|
+
# "standalone stays Rails-free" tests assert.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{A as e,F as t,J as n,M as r,O as i,_ as a,c as o,f as s,k as c,r as l,s as u,v as d,w as f}from"./Skeleton-DzR7XNxz.js";import{o as p,s as m}from"./index-
|
|
1
|
+
import{A as e,F as t,J as n,M as r,O as i,_ as a,c as o,f as s,k as c,r as l,s as u,v as d,w as f}from"./Skeleton-DzR7XNxz.js";import{o as p,s as m}from"./index-D_lSDwKw.js";import{a as h}from"./utils-DDJC7tJV.js";var g=f(`<div style=display:flex;flex-direction:column;gap:2px><span style=font-size:11px;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em></span><span style=font-variant-numeric:tabular-nums>`),_=f(`<div style=margin-top:1.5rem><h2 style="font-size:14px;margin:0 0 0.5rem"> <span class="badge badge-muted"></span></h2><div class=table-wrapper><table><tbody>`),v=f(`<tr><td style=font-family:monospace;font-size:12px>`),y=f(`<div><div class=section-header style=gap:0.75rem></div><div style=margin-top:1rem>`),b=f(`<div class=empty-state style=color:var(--danger)>`),x=f(`<span class="badge badge-success">`),S=f(`<span class="badge badge-danger">`),C=f(`<p style=color:var(--text-muted);margin-top:0>`),w=f(`<div><div class=section-header style=gap:0.75rem><h1 class=page-title style=margin:0;font-family:monospace;font-size:16px></h1></div><div style="display:grid;grid-template-columns:repeat(auto-fill, minmax(140px, 1fr));gap:1rem;margin-top:1rem">`),T=f(`<span class="badge badge-accent">%`);function E(e){return(()=>{var t=g(),n=t.firstChild,r=n.nextSibling;return a(n,()=>e.label),a(r,()=>e.children),t})()}function D(n){return t(e,{get when(){return n.jids.length>0},get children(){var e=_(),r=e.firstChild,o=r.firstChild,s=o.nextSibling,c=r.nextSibling.firstChild.firstChild;return a(r,()=>n.label,o),a(s,()=>n.jids.length),a(c,t(i,{get each(){return n.jids},children:e=>(()=>{var t=v(),n=t.firstChild;return a(n,e),t})()})),e}})}function O(){let i=m(),f=()=>i.bid??``;n(()=>{document.title=`${u(`nav.batches`)} — Wurk`});let g=s(()=>({queryKey:[`batch`,f()],queryFn:()=>fetch(`${o()}/api/batches/${encodeURIComponent(f())}`).then(e=>{if(!e.ok)throw Error(`HTTP ${e.status}`);return e.json()})}));return t(r,{get children(){return[t(c,{get when(){return g.isPending},get children(){var e=y(),n=e.firstChild,r=n.nextSibling;return a(n,t(p,{href:`/batches`,class:`btn btn-sm`,get children(){return[`← `,d(()=>u(`nav.batches`))]}})),a(r,t(l,{count:8})),e}}),t(c,{get when(){return g.isError||!g.data},get children(){var e=b();return a(e,()=>u(`common.error`)),e}}),t(c,{get when(){return g.data},children:n=>{let r=()=>n().total-n().pending,i=()=>n().total>0?Math.round(r()/n().total*100):0;return(()=>{var r=w(),o=r.firstChild,s=o.firstChild,c=o.nextSibling;return a(o,t(p,{href:`/batches`,class:`btn btn-sm`,get children(){return[`← `,d(()=>u(`nav.batches`))]}}),s),a(s,()=>n().bid),a(o,t(e,{get when(){return n().complete},get fallback(){return(()=>{var e=T(),t=e.firstChild;return a(e,i,t),e})()},get children(){var e=x();return a(e,()=>u(`batches.complete`)),e}}),null),a(o,t(e,{get when(){return n().invalidated},get children(){var e=S();return a(e,()=>u(`batches.invalidated`)),e}}),null),a(r,t(e,{get when(){return n().description},get children(){var e=C();return a(e,()=>n().description),e}}),c),a(c,t(E,{get label(){return u(`table.total`)},get children(){return n().total.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.pending`)},get children(){return n().pending.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.failures`)},get children(){return n().failures.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`batches.children`)},get children(){return n().child_count.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.created`)},get children(){return d(()=>!!n().created_at)()?h(n().created_at):`—`}}),null),a(c,t(E,{get label(){return u(`batches.completed`)},get children(){return d(()=>!!n().complete_at)()?h(n().complete_at):`—`}}),null),a(c,t(e,{get when(){return n().parent_bid},get children(){return t(E,{get label(){return u(`batches.parent`)},get children(){return n().parent_bid}})}}),null),a(c,t(e,{get when(){return n().tags.length>0},get children(){return t(E,{get label(){return u(`batches.tags`)},get children(){return n().tags.join(`, `)}})}}),null),a(r,t(D,{get label(){return u(`batches.failed_jobs`)},get jids(){return n().failed_jids}}),null),a(r,t(D,{get label(){return u(`batches.dead_jobs`)},get jids(){return n().dead_jids}}),null),r})()}})]}})}export{O as default};
|