wurk 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -0
  3. data/lib/wurk/batch/callbacks.rb +82 -12
  4. data/lib/wurk/batch/death_handler.rb +7 -4
  5. data/lib/wurk/batch/server_middleware.rb +1 -1
  6. data/lib/wurk/batch.rb +121 -15
  7. data/lib/wurk/capsule.rb +5 -4
  8. data/lib/wurk/cli.rb +48 -14
  9. data/lib/wurk/client/buffered.rb +193 -43
  10. data/lib/wurk/client.rb +87 -14
  11. data/lib/wurk/compat.rb +1 -1
  12. data/lib/wurk/component.rb +2 -2
  13. data/lib/wurk/configuration.rb +10 -2
  14. data/lib/wurk/cron.rb +94 -37
  15. data/lib/wurk/deploy.rb +5 -3
  16. data/lib/wurk/embedded.rb +13 -0
  17. data/lib/wurk/fetcher/reaper.rb +113 -56
  18. data/lib/wurk/fetcher/reliable.rb +62 -9
  19. data/lib/wurk/heartbeat.rb +22 -10
  20. data/lib/wurk/history.rb +13 -1
  21. data/lib/wurk/launcher.rb +133 -66
  22. data/lib/wurk/leader.rb +29 -10
  23. data/lib/wurk/limiter/base.rb +8 -10
  24. data/lib/wurk/limiter/bucket.rb +1 -1
  25. data/lib/wurk/limiter/concurrent.rb +27 -22
  26. data/lib/wurk/limiter/window.rb +13 -11
  27. data/lib/wurk/limiter.rb +7 -4
  28. data/lib/wurk/lua.rb +97 -14
  29. data/lib/wurk/manager.rb +29 -13
  30. data/lib/wurk/metrics/history.rb +4 -3
  31. data/lib/wurk/metrics/queue_rollup.rb +13 -1
  32. data/lib/wurk/metrics/rollup.rb +13 -1
  33. data/lib/wurk/middleware/interrupt_handler.rb +7 -6
  34. data/lib/wurk/middleware/poison_pill.rb +70 -29
  35. data/lib/wurk/middleware.rb +2 -2
  36. data/lib/wurk/pool_checkout.rb +29 -0
  37. data/lib/wurk/process_set.rb +10 -5
  38. data/lib/wurk/processor.rb +6 -0
  39. data/lib/wurk/profiler.rb +3 -2
  40. data/lib/wurk/queue.rb +10 -7
  41. data/lib/wurk/rails_boot.rb +38 -7
  42. data/lib/wurk/redis_client_adapter.rb +48 -4
  43. data/lib/wurk/redis_options.rb +142 -0
  44. data/lib/wurk/redis_pool.rb +102 -39
  45. data/lib/wurk/scheduled.rb +30 -2
  46. data/lib/wurk/stats.rb +14 -9
  47. data/lib/wurk/swarm/child_boot.rb +12 -0
  48. data/lib/wurk/swarm.rb +174 -33
  49. data/lib/wurk/timer_loop.rb +14 -0
  50. data/lib/wurk/version.rb +1 -1
  51. data/lib/wurk/web/enterprise.rb +58 -6
  52. data/lib/wurk/web/extension.rb +1 -1
  53. data/lib/wurk/web/search.rb +5 -3
  54. data/lib/wurk.rb +53 -2
  55. data/vendor/assets/dashboard/assets/{BatchDetail-YRymNsrB.js → BatchDetail-OmC5NPgw.js} +1 -1
  56. data/vendor/assets/dashboard/assets/{Batches-HY4hHdQU.js → Batches-CIpai7St.js} +1 -1
  57. data/vendor/assets/dashboard/assets/{Busy-FCEN1Bpx.js → Busy-A_kwSR6Q.js} +1 -1
  58. data/vendor/assets/dashboard/assets/{Cron-DO3J2zcp.js → Cron-BG7HTqlp.js} +1 -1
  59. data/vendor/assets/dashboard/assets/{Dashboard-B9rOrkzk.js → Dashboard-A_ToqHoo.js} +1 -1
  60. data/vendor/assets/dashboard/assets/{Dead-Bi4GGk9a.js → Dead-8J21jMyK.js} +1 -1
  61. data/vendor/assets/dashboard/assets/Extension-B4Q9FIQu.js +1 -0
  62. data/vendor/assets/dashboard/assets/{FilterBox-IJkHYpdm.js → FilterBox-Fh_Ae7UW.js} +1 -1
  63. data/vendor/assets/dashboard/assets/{JobDetailModal-DS1ypyoc.js → JobDetailModal-Ceng0PMB.js} +1 -1
  64. data/vendor/assets/dashboard/assets/{Limiters-Nz7UbNeJ.js → Limiters-CruDWvNZ.js} +1 -1
  65. data/vendor/assets/dashboard/assets/{Metrics-BBTDxcaE.js → Metrics-CIT7VCoN.js} +1 -1
  66. data/vendor/assets/dashboard/assets/Modal-CN3rdKA_.js +1 -0
  67. data/vendor/assets/dashboard/assets/{Queues-D9PH_THs.js → Queues-D86FYohJ.js} +1 -1
  68. data/vendor/assets/dashboard/assets/{Retries-CAKzDgYG.js → Retries-Bz1O1D-i.js} +1 -1
  69. data/vendor/assets/dashboard/assets/{Scheduled-DTYw1X8S.js → Scheduled-B6h2akTu.js} +1 -1
  70. data/vendor/assets/dashboard/assets/{Search-c4vFxDG_.js → Search-OOu22e5s.js} +1 -1
  71. data/vendor/assets/dashboard/assets/index-BdiUEDXX.css +1 -0
  72. data/vendor/assets/dashboard/assets/index-D_lSDwKw.js +141 -0
  73. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-B_FsMah6.js → useResetPageOnEmpty-dVPGEWzn.js} +1 -1
  74. data/vendor/assets/dashboard/index.html +2 -2
  75. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  76. metadata +22 -20
  77. data/vendor/assets/dashboard/assets/Extension-BSv8ddW_.js +0 -1
  78. data/vendor/assets/dashboard/assets/Modal-Crrsu64-.js +0 -1
  79. data/vendor/assets/dashboard/assets/index-BxjdeuOa.css +0 -1
  80. data/vendor/assets/dashboard/assets/index-DQu7WY9y.js +0 -141
data/lib/wurk/swarm.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'monitor'
4
+
3
5
  require_relative 'component'
4
6
  require_relative 'launcher'
5
7
  require_relative 'fetcher/reliable'
@@ -42,6 +44,16 @@ module Wurk
42
44
  MEMORY_CHECK_INTERVAL = 10
43
45
  DEFAULT_SHUTDOWN_TIMEOUT = 25
44
46
 
47
+ # The supervisor's own Redis pool: one connection, disjoint from every
48
+ # capsule pool. Post-boot the parent's only Redis use is the heartbeat probe
49
+ # (`heartbeat_seen?`) — one SISMEMBER per restart tick — and routing it
50
+ # through the capsule main pool reopened, at capsule size (>= 10), the very
51
+ # sockets step 3 had just closed; the next respawn/restart fork then
52
+ # inherited them. Dedicated and torn down before every fork, so no child
53
+ # ever sees a parent socket.
54
+ SUPERVISOR_POOL_SIZE = 1
55
+ SUPERVISOR_POOL_NAME = 'swarm-supervisor'
56
+
45
57
  # Children each hard_shutdown after their own drain deadline (bulk_requeue
46
58
  # + a 3s ensure window + heartbeat cleanup); the parent must not SIGKILL
47
59
  # them mid-tail, so its own wait always extends past theirs by this much.
@@ -52,7 +64,7 @@ module Wurk
52
64
  # whole swarm.
53
65
  SWARM_SIGNALS = { 'TERM' => :term, 'INT' => :term, 'TSTP' => :tstp, 'USR1' => :usr1, 'USR2' => :usr2 }.freeze
54
66
 
55
- attr_reader :topology, :children
67
+ attr_reader :topology
56
68
 
57
69
  def initialize(topology:, config: Wurk.configuration, memory_limit: config.memory_limit_kb,
58
70
  shutdown_timeout: DEFAULT_SHUTDOWN_TIMEOUT)
@@ -60,15 +72,24 @@ module Wurk
60
72
  @config = config
61
73
  @memory_limit = memory_limit
62
74
  @shutdown_timeout = shutdown_timeout
75
+ # One reentrant lock covers the child table AND the restart machine: they
76
+ # are mutually recursive (Restart#advance calls back into `describe` and
77
+ # `spawn`, which read and write `@children`), so separate locks would be
78
+ # taken in opposite orders — enqueue holds children→restart, advance holds
79
+ # restart→children — and deadlock. Monitor's reentrancy is what lets those
80
+ # callbacks re-enter. Never held across a sleep: `wait_for_children` and
81
+ # the supervise tick sleep outside it.
82
+ @lock = ::Monitor.new
63
83
  @children = {}
64
84
  @assignments = []
85
+ @owner_pid = nil
65
86
  @stopping = false
87
+ @shutdown_requested = false
66
88
  @quieted = false
67
89
  @last_memory_check = 0
68
- @signal_read = nil
69
- @signal_write = nil
70
90
  @respawn_backoff = Backoff.new(base: RESPAWN_BACKOFF)
71
91
  @restart = build_restart
92
+ init_fork_unsafe_handles
72
93
  end
73
94
 
74
95
  # `install_signals:` is false in tests so the integration suite can
@@ -85,29 +106,65 @@ module Wurk
85
106
  raise ArgumentError, 'Topology has no slots' if @topology.empty?
86
107
 
87
108
  @assignments = @topology.assignments.freeze
109
+ @owner_pid = ::Process.pid
88
110
  install_signal_handlers if install_signals
89
111
  close_parent_sockets
90
112
  fork_children
91
- @children.keys
113
+ child_pids
114
+ end
115
+
116
+ # A snapshot, not the live table. Callers read this off the supervise
117
+ # thread, which inserts (respawn, restart) and deletes (reap) on every tick;
118
+ # handing out the live Hash lets them iterate it mid-mutation.
119
+ def children
120
+ @lock.synchronize { @children.dup }
92
121
  end
93
122
 
94
123
  def supervise
124
+ return unless owner?
125
+
95
126
  until done?
96
127
  drain_signals
128
+ shutdown if @shutdown_requested && !@stopping
97
129
  reap_children
98
130
  spawn_due_respawns
99
- @restart.advance unless @stopping
131
+ advance_restart unless @stopping
100
132
  check_memory_pressure
101
133
  sleep SUPERVISE_TICK
102
134
  end
103
135
  end
104
136
 
105
137
  def shutdown(timeout: @shutdown_timeout)
138
+ return unless owner?
139
+
106
140
  @stopping = true
107
- @restart.abort
141
+ @lock.synchronize { @restart.abort }
108
142
  relay_signal('TERM')
109
143
  wait_for_children(timeout + SHUTDOWN_GRACE)
110
144
  hard_kill_stragglers
145
+ close_supervisor_pool
146
+ close_signal_pipe
147
+ end
148
+
149
+ # Cross-thread drain request: raise a flag the supervise loop observes on
150
+ # its next tick instead of draining on the caller's thread. `shutdown`
151
+ # walks and clears the child table that the supervise thread is
152
+ # concurrently mutating (reap, respawn, restart), so two threads inside it
153
+ # race. Every caller that isn't the supervise thread — rails_boot's
154
+ # `at_exit`, which fires on the host's main thread — comes through here.
155
+ def request_shutdown
156
+ @shutdown_requested = true
157
+ end
158
+
159
+ # Only the process that forked the children may supervise or signal them.
160
+ # A forked child inherits this object along with the host's `at_exit` hooks
161
+ # — and rails_boot registers one that drains the swarm — so ChildBoot's
162
+ # `exit` reaches a full drain inside the child. There `@children` holds the
163
+ # child's SIBLINGS, not its own children: unguarded, that TERMs them, stalls
164
+ # the whole shutdown timeout waiting on PIDs it can never reap, then SIGKILLs
165
+ # whatever survived.
166
+ def owner?
167
+ ::Process.pid == @owner_pid
111
168
  end
112
169
 
113
170
  # TSTP quiet is one-way and GLOBAL (spec §21.3): it must survive respawns
@@ -124,17 +181,44 @@ module Wurk
124
181
  # heartbeat → TERM the old child → await its drain) without blocking the
125
182
  # supervise thread, so TERM stays responsive throughout the cycle.
126
183
  def rolling_restart
127
- @restart.enqueue(@children.keys)
184
+ @lock.synchronize { @restart.enqueue(@children.keys) }
128
185
  end
129
186
 
130
187
  private
131
188
 
189
+ # The handles the parent must never let cross a fork: the signal self-pipe
190
+ # (an inherited trap would write into the PARENT's supervise loop) and the
191
+ # supervisor's Redis socket. Grouped because they share one rule — opened in
192
+ # the parent, dropped before or at every fork by `close_supervisor_pool` and
193
+ # `close_signal_pipe`.
194
+ def init_fork_unsafe_handles
195
+ @signal_read = nil
196
+ @signal_write = nil
197
+ @supervisor_pool = nil
198
+ end
199
+
200
+ def advance_restart
201
+ @lock.synchronize { @restart.advance }
202
+ end
203
+
204
+ def child_pids
205
+ @lock.synchronize { @children.keys }
206
+ end
207
+
208
+ def child_meta(pid)
209
+ @lock.synchronize { @children[pid] }
210
+ end
211
+
212
+ def any_children?
213
+ @lock.synchronize { !@children.empty? }
214
+ end
215
+
132
216
  def build_restart
133
217
  Restart.new(Restart::Config.new(
134
218
  spawn: method(:spawn_child),
135
219
  kill: method(:safe_kill),
136
220
  heartbeat: method(:heartbeat_seen?),
137
- describe: ->(pid) { @children[pid] },
221
+ describe: ->(pid) { child_meta(pid) },
138
222
  now: method(:monotonic),
139
223
  logger: logger,
140
224
  heartbeat_wait: HEARTBEAT_WAIT,
@@ -168,7 +252,7 @@ module Wurk
168
252
  # Returns the child PID; never returns in the child (ChildBoot exits).
169
253
  def spawn_child(slot, idx)
170
254
  pid = fork_child(slot, idx)
171
- @children[pid] = { slot: slot, index: idx, spawned_at: monotonic }
255
+ @lock.synchronize { @children[pid] = { slot: slot, index: idx, spawned_at: monotonic } }
172
256
  pid
173
257
  end
174
258
 
@@ -176,17 +260,21 @@ module Wurk
176
260
  # from the parent, it is race-free even if the parent dies the instant
177
261
  # after fork (getppid in the child could already return the reaper). The
178
262
  # child's OrphanGuard compares live getppid against it.
263
+ #
264
+ # The supervisor pool closes here rather than in `close_parent_sockets`
265
+ # because that runs once, at boot: every later respawn / rolling-restart
266
+ # fork happens after the heartbeat probe has reopened it.
179
267
  def fork_child(slot, idx)
180
268
  parent_pid = ::Process.pid
269
+ close_supervisor_pool
181
270
  pid = ::Process.fork
182
271
  return pid if pid
183
272
 
184
273
  # Drop the parent's self-pipe first: the inherited traps still write to
185
274
  # it, so a signal landing in the window before ChildBoot resets them
186
275
  # would surface in the PARENT's supervise loop (an operator TERMing one
187
- # child pid would drain the whole swarm). Closed, the trap write no-ops.
188
- @signal_read&.close
189
- @signal_write&.close
276
+ # child pid would drain the whole swarm). Dropped, the trap write no-ops.
277
+ close_signal_pipe
190
278
  ChildBoot.new(@config, slot, idx, parent_pid: parent_pid, start_quiet: @quieted).run
191
279
  exit 0 # unreachable; ChildBoot exits explicitly
192
280
  end
@@ -204,12 +292,25 @@ module Wurk
204
292
  end
205
293
  end
206
294
 
295
+ # Both FDs, once. The traps stay installed and keep firing after a drain, so
296
+ # clear the ivars BEFORE closing: `emit_signal` then writes to nothing
297
+ # instead of a dead FD, and a `drain_signals` loop mid-tick (TERM handled,
298
+ # asking for the next buffered signal) stops instead of reading a closed
299
+ # pipe. Called on shutdown and, in the child, immediately after fork.
300
+ def close_signal_pipe
301
+ read = @signal_read
302
+ write = @signal_write
303
+ @signal_read = @signal_write = nil
304
+ read&.close
305
+ write&.close
306
+ end
307
+
207
308
  # Non-blocking self-pipe write from trap context: a blocking `puts` could
208
309
  # stall signal delivery if the pipe fills. `exception: false` returns
209
310
  # :wait_writable instead of raising when full (drop the coalescible
210
311
  # duplicate); a closed pipe during shutdown is ignored too.
211
312
  def emit_signal(sig)
212
- @signal_write.write_nonblock("#{sig}\n", exception: false)
313
+ @signal_write&.write_nonblock("#{sig}\n", exception: false)
213
314
  rescue ::IOError, ::Errno::EPIPE, ::Errno::EBADF
214
315
  nil
215
316
  end
@@ -222,17 +323,29 @@ module Wurk
222
323
  when :term then shutdown
223
324
  when :tstp then quiet_swarm
224
325
  when :usr1 then rolling_restart
225
- when :usr2 then relay_signal('USR2')
326
+ when :usr2
327
+ reopen_logs
328
+ relay_signal('USR2')
226
329
  end
227
330
  end
228
331
  end
229
332
 
230
333
  # One buffered line per pending signal, non-blocking (wait_readable(0)).
231
- # nil once the pipe is drained, ending the loop for this tick.
334
+ # nil once the pipe is drained, ending the loop for this tick. Reads through
335
+ # a local because the pipe can vanish mid-drain: a buffered TERM runs
336
+ # `shutdown`, which closes it, and the loop then asks for the next signal.
232
337
  def read_pending_signal
233
- return nil unless @signal_read.wait_readable(0)
338
+ read = @signal_read
339
+ return nil unless read&.wait_readable(0)
234
340
 
235
- @signal_read.gets&.strip
341
+ read.gets&.strip
342
+ end
343
+
344
+ def reopen_logs
345
+ log = @config.logger
346
+ log.reopen if log.respond_to?(:reopen)
347
+ rescue StandardError
348
+ nil
236
349
  end
237
350
 
238
351
  # Reap every exited child this tick (not one), so a fleet-wide death
@@ -251,14 +364,16 @@ module Wurk
251
364
  end
252
365
 
253
366
  def on_child_exit(pid, status)
254
- meta = @children.delete(pid)
255
- return unless meta
256
- return if @restart.claim_exit(pid)
257
-
258
- if @stopping
259
- logger.info { "swarm: child #{pid} exited (status=#{status.exitstatus})" }
260
- else
261
- schedule_respawn(pid, status, meta)
367
+ @lock.synchronize do
368
+ meta = @children.delete(pid)
369
+ return unless meta
370
+ return if @restart.claim_exit(pid)
371
+
372
+ if @stopping
373
+ logger.info { "swarm: child #{pid} exited (status=#{status.exitstatus})" }
374
+ else
375
+ schedule_respawn(pid, status, meta)
376
+ end
262
377
  end
263
378
  end
264
379
 
@@ -305,7 +420,7 @@ module Wurk
305
420
  return if now - @last_memory_check < MEMORY_CHECK_INTERVAL
306
421
 
307
422
  @last_memory_check = now
308
- @children.dup.each_key { |pid| recycle_if_bloated(pid) }
423
+ child_pids.each { |pid| recycle_if_bloated(pid) }
309
424
  end
310
425
 
311
426
  # Route a bloated child through the restart state machine (same path as a
@@ -317,7 +432,7 @@ module Wurk
317
432
  return if rss.nil? || rss < @memory_limit
318
433
 
319
434
  logger.warn { "swarm: child #{pid} RSS #{rss}KB >= #{@memory_limit}KB; recycling" }
320
- @restart.enqueue([pid])
435
+ @lock.synchronize { @restart.enqueue([pid]) }
321
436
  end
322
437
 
323
438
  def pid_rss_kb(pid)
@@ -329,10 +444,15 @@ module Wurk
329
444
  end
330
445
 
331
446
  def relay_signal(sig)
332
- @children.each_key { |pid| safe_kill(pid, sig) }
447
+ child_pids.each { |pid| safe_kill(pid, sig) }
333
448
  end
334
449
 
450
+ # The one place the supervisor signals a child: relay_signal,
451
+ # hard_kill_stragglers and the restart machine's `kill:` callback all funnel
452
+ # through here, so the owner check covers every path at a single choke point.
335
453
  def safe_kill(pid, sig)
454
+ return unless owner?
455
+
336
456
  ::Process.kill(sig, pid)
337
457
  rescue Errno::ESRCH, Errno::EPERM
338
458
  nil
@@ -340,15 +460,20 @@ module Wurk
340
460
 
341
461
  def wait_for_children(timeout)
342
462
  deadline = monotonic + timeout
343
- while monotonic < deadline && @children.any?
463
+ while monotonic < deadline && any_children?
344
464
  reap_children
345
465
  sleep 0.1
346
466
  end
347
467
  end
348
468
 
469
+ # Kill and forget atomically: a child landing in the table between the walk
470
+ # and the clear would be dropped from it without ever being signalled —
471
+ # untracked and still alive.
349
472
  def hard_kill_stragglers
350
- @children.each_key { |pid| safe_kill(pid, 'KILL') }
351
- @children.clear
473
+ @lock.synchronize do
474
+ @children.each_key { |pid| safe_kill(pid, 'KILL') }
475
+ @children.clear
476
+ end
352
477
  end
353
478
 
354
479
  # Has the child written its first heartbeat yet? One non-blocking SISMEMBER,
@@ -360,17 +485,33 @@ module Wurk
360
485
  # forces progress.
361
486
  def heartbeat_seen?(pid)
362
487
  identity = "#{hostname}:#{pid}:#{Component::PROCESS_NONCE}"
363
- @config.redis { |c| c.call('SISMEMBER', Keys::PROCESSES, identity) } == 1
488
+ supervisor_pool.with { |c| c.call('SISMEMBER', Keys::PROCESSES, identity) } == 1
364
489
  rescue StandardError
365
490
  false
366
491
  end
367
492
 
493
+ def supervisor_pool
494
+ @supervisor_pool ||= @config.new_redis_pool(SUPERVISOR_POOL_SIZE, SUPERVISOR_POOL_NAME)
495
+ end
496
+
497
+ # ConnectionPool#shutdown is terminal, so the reference has to go with the
498
+ # socket — the next probe rebuilds. Cleared before disconnecting so a raise
499
+ # mid-teardown can't leave a shut-down pool wired up, which would make every
500
+ # later probe raise PoolShuttingDownError instead of reconnecting.
501
+ def close_supervisor_pool
502
+ pool = @supervisor_pool
503
+ @supervisor_pool = nil
504
+ pool&.disconnect!
505
+ rescue StandardError => e
506
+ logger.warn { "swarm: supervisor pool close failed: #{e.class}: #{e.message}" }
507
+ end
508
+
368
509
  def monotonic
369
510
  ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
370
511
  end
371
512
 
372
513
  def done?
373
- @stopping && @children.empty?
514
+ @stopping && !any_children?
374
515
  end
375
516
  end
376
517
  end
@@ -15,6 +15,13 @@ module Wurk
15
15
  # @thread ||= safe_thread('my-loop') { @timer.run { tick } }
16
16
  # def terminate = @timer.terminate
17
17
  class TimerLoop
18
+ # Deadline every periodic component gives its loop thread when stopping.
19
+ # Bounded rather than an open-ended join: a tick blocked on a wedged Redis
20
+ # must not hold the process's whole shutdown open. Also used by
21
+ # Scheduled::Poller, which hand-rolls its wait (the interval is randomized
22
+ # per iteration) but stops on the same terms.
23
+ JOIN_TIMEOUT = 5
24
+
18
25
  def initialize(interval)
19
26
  @interval = interval
20
27
  @done = false
@@ -40,6 +47,13 @@ module Wurk
40
47
  end
41
48
  end
42
49
 
50
+ # Clears a previous #terminate. Hosts call this before respawning, so a
51
+ # start-after-stop cycle spawns a thread that ticks instead of one that
52
+ # sees the stale done flag and exits immediately.
53
+ def reset
54
+ @mutex.synchronize { @done = false }
55
+ end
56
+
43
57
  def wait
44
58
  @mutex.synchronize do
45
59
  @sleeper.wait(@mutex, @interval) unless @done
data/lib/wurk/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Wurk
4
- VERSION = "1.3.0"
4
+ VERSION = "1.4.0"
5
5
  end
@@ -22,21 +22,72 @@ module Wurk
22
22
  module Limits
23
23
  module_function
24
24
 
25
- # @return [Array<String>] limiter names, optionally filtered to those
26
- # whose name contains the case-insensitive substring `filter`.
25
+ # Names per sweep round trip. Bounds both halves: the probe's pipeline
26
+ # (whose replies the client buffers) and the removal script, which is
27
+ # atomic — an unbatched pass over a set that leaked for months would
28
+ # block every other Redis client for its whole length. A healthy set
29
+ # fits in one slice, so the common case still costs one probe.
30
+ SWEEP_BATCH = 512
31
+
32
+ # @return [Array<String>] live limiter names, optionally filtered to
33
+ # those whose name contains the case-insensitive substring `filter`.
34
+ #
35
+ # Listing is also the sweep. `lmtr-list` membership has no TTL of its
36
+ # own while `lmtr:<name>` expires with the limiter's ttl, so every
37
+ # interpolated name (`stripe-#{user_id}`) the spec blesses would leave a
38
+ # permanent member behind. Names whose metadata is gone are dropped from
39
+ # the SET and from the result, so the dashboard also stops rendering a
40
+ # row with nothing behind it. The liveness probe rides along with the
41
+ # SMEMBERS the listing already pays for — both are O(set) — and once the
42
+ # sweep has run the set is back down to live limiters.
27
43
  def list(filter: nil)
28
- names = Wurk.redis { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }.sort
44
+ names = sweep(Wurk.redis(idempotent: true) { |c| c.call('SMEMBERS', Wurk::Limiter::LIST_KEY) }).sort
29
45
  return names if filter.nil? || filter.to_s.empty?
30
46
 
31
47
  needle = filter.to_s.downcase
32
48
  names.select { |n| n.downcase.include?(needle) }
33
49
  end
34
50
 
51
+ # Drop the names whose metadata is gone, return the ones still live.
52
+ def sweep(names)
53
+ names.each_slice(SWEEP_BATCH).flat_map { |slice| sweep_slice(slice) }
54
+ end
55
+
56
+ def sweep_slice(names)
57
+ live, dead = partition_live(names)
58
+ sweep_batch(dead) unless dead.empty?
59
+ live
60
+ end
61
+
62
+ def partition_live(names)
63
+ flags = Wurk.redis(idempotent: true) do |c|
64
+ c.pipelined { |pipe| names.each { |n| pipe.call('EXISTS', meta_key(n)) } }
65
+ end
66
+ names.partition.with_index { |_, i| flags[i].to_i == 1 }
67
+ end
68
+
69
+ # Claims apply-safety, unlike ProcessSet's comparable prune: the script
70
+ # re-decides liveness inside its own atomic step, so a replay can only
71
+ # remove names that are still dead.
72
+ def sweep_batch(names)
73
+ Wurk.redis(idempotent: true) do |c|
74
+ Wurk::Lua::Loader.eval_cached(
75
+ c, :limiter_list_sweep,
76
+ keys: [Wurk::Limiter::LIST_KEY],
77
+ argv: ['lmtr:', *names]
78
+ )
79
+ end
80
+ end
81
+
35
82
  def metadata(name)
36
- raw = Wurk.redis { |c| c.call('HGETALL', "lmtr:#{name}") }
83
+ raw = Wurk.redis(idempotent: true) { |c| c.call('HGETALL', meta_key(name)) }
37
84
  raw.is_a?(Array) ? raw.each_slice(2).to_h : raw
38
85
  end
39
86
 
87
+ def meta_key(name)
88
+ "lmtr:#{name}"
89
+ end
90
+
40
91
  # Stats-key reset: drops every `lmtr-stats:` / state key for the
41
92
  # named limiter while keeping its metadata + LIST membership so the
42
93
  # row remains in the UI. Mirrors the §1.5 `#reset` surface — the
@@ -51,8 +102,9 @@ module Wurk
51
102
  end
52
103
 
53
104
  # Read-only reconstruction (`register: false`) from the persisted
54
- # metadata. Returns nil when the meta HASH is gone or malformed —
55
- # LIST membership has no TTL, so a name can outlive its metadata.
105
+ # metadata. Returns nil when the meta HASH is gone or malformed — the
106
+ # name a caller holds came from a page `list` rendered earlier, and the
107
+ # metadata can expire between the two.
56
108
  def rebuild(name)
57
109
  meta = metadata(name)
58
110
  return nil if meta.nil? || meta.empty?
@@ -148,7 +148,7 @@ module Wurk
148
148
  %(<time class="ltr" dir="ltr" title="#{iso}" datetime="#{iso}">#{iso}</time>)
149
149
  end
150
150
 
151
- def redis(&) = ::Wurk.redis(&)
151
+ def redis(idempotent: false, &) = ::Wurk.redis(idempotent:, &)
152
152
  def product_version = ::Wurk::VERSION
153
153
  def number_with_delimiter(num) = num.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
154
154
  end
@@ -103,10 +103,12 @@ module Wurk
103
103
  # we scan a bounded prefix and never full-walk the LIST from a keystroke.
104
104
  def scan_queue_list(queue_name, &)
105
105
  key = Keys.queue(queue_name)
106
- stop_at = queue_scan_stop(Wurk.redis { |c| c.call('LLEN', key) })
106
+ stop_at = queue_scan_stop(Wurk.redis(idempotent: true) { |c| c.call('LLEN', key) })
107
107
  start = 0
108
108
  while start < stop_at
109
- slice = Wurk.redis { |c| c.call('LRANGE', key, start, [start + QUEUE_PAGE, stop_at].min - 1) }
109
+ slice = Wurk.redis(idempotent: true) do |c|
110
+ c.call('LRANGE', key, start, [start + QUEUE_PAGE, stop_at].min - 1)
111
+ end
110
112
  break if slice.empty?
111
113
 
112
114
  slice.each(&)
@@ -134,7 +136,7 @@ module Wurk
134
136
  set = sorted_set_for(kind)
135
137
  cursor = '0'
136
138
  loop do
137
- cursor, page = Wurk.redis { |c| c.call('ZSCAN', set.name, cursor, 'COUNT', ZSCAN_PAGE) }
139
+ cursor, page = Wurk.redis(idempotent: true) { |c| c.call('ZSCAN', set.name, cursor, 'COUNT', ZSCAN_PAGE) }
138
140
  emit_sorted_hits(kind, set, page, &)
139
141
  @scanned += page.size / 2
140
142
  break if cursor == '0'
data/lib/wurk.rb CHANGED
@@ -8,6 +8,7 @@
8
8
  require_relative 'wurk/version'
9
9
  require_relative 'wurk/keys'
10
10
  require_relative 'wurk/redis_pool'
11
+ require_relative 'wurk/pool_checkout'
11
12
  require_relative 'wurk/redis_connection'
12
13
  require_relative 'wurk/middleware'
13
14
  require_relative 'wurk/middleware/chain'
@@ -126,8 +127,15 @@ module Wurk
126
127
  end
127
128
  alias default_configuration configuration
128
129
 
129
- def redis(&)
130
- redis_pool.with(&)
130
+ # `idempotent: true` asserts the block is safe to re-run after a command may
131
+ # already have applied server-side, buying back the full connection backoff
132
+ # (see RedisPool#with). Every wrapper down the chain — Capsule,
133
+ # Configuration, Component, middleware, Sidekiq.redis — forwards it, so a
134
+ # caller can claim apply-safety wherever it holds; PoolCheckout drops the
135
+ # claim for a host-supplied pool that wouldn't understand it. The zero-arg
136
+ # shape is the drop-in contract: `Sidekiq.redis { |c| ... }` never changes.
137
+ def redis(idempotent: false, &)
138
+ PoolCheckout.with(redis_pool, idempotent, &)
131
139
  end
132
140
 
133
141
  # Capsule-aware pool lookup. Thread-local override wins so per-capsule
@@ -224,6 +232,22 @@ module Wurk
224
232
  def ent?
225
233
  false
226
234
  end
235
+
236
+ # Lazily loads the Rails engine the first time something asks for
237
+ # `Wurk::Engine`. See the note above the `require "wurk/rails"` guard at the
238
+ # bottom of this file for why the guard alone isn't enough (#282), and why
239
+ # this loads the engine and not the railtie.
240
+ def const_missing(name)
241
+ return super unless name == :Engine
242
+ return super unless defined?(::Rails::Engine) && defined?(::ActionDispatch::Routing::RouteSet)
243
+
244
+ require_relative 'wurk/engine'
245
+ # If engine.rb somehow didn't define it, fall through to the real NameError
246
+ # rather than recursing back into here.
247
+ return super unless const_defined?(:Engine, false)
248
+
249
+ const_get(:Engine, false)
250
+ end
227
251
  end
228
252
  end
229
253
 
@@ -318,3 +342,30 @@ require_relative 'wurk/compat'
318
342
  # real Rails host both are loaded by `rails/all` before Bundler.require, so the
319
343
  # stricter gate is invisible there.
320
344
  require_relative 'wurk/rails' if defined?(Rails::Engine) && defined?(::ActionDispatch::Routing::RouteSet)
345
+
346
+ # The gate above is evaluated exactly once, when this file is first required —
347
+ # which is too early for the standalone runners (#282). `exe/wurk` and
348
+ # `exe/wurkswarm` require "wurk" before Rails exists (the gate is false, nothing
349
+ # Rails-y loads), then boot the host app themselves via
350
+ # Wurk::CLI#boot_rails_application. By then "wurk" is in $LOADED_FEATURES, so the
351
+ # app's own `Bundler.require` is a no-op and the gate never gets a second look. A
352
+ # host that did exactly what the README says — `mount Wurk::Engine => "/wurk"` in
353
+ # config/routes.rb — then dies during boot with `uninitialized constant
354
+ # Wurk::Engine`, from the worker process only. Web processes are unaffected:
355
+ # there Bundler.require runs after railties, so the gate passes.
356
+ #
357
+ # `Wurk.const_missing` (defined in the class << self block above) closes that
358
+ # window without giving up the lean standalone boot: nothing loads until
359
+ # something actually asks for Wurk::Engine, and even then only when Rails is
360
+ # really present. It reuses the same two-part condition as the guard above, so
361
+ # the ecosystem carve-out (rails/engine/railties without ActionDispatch, per
362
+ # sidekiq-cron's test helper) still falls through to a plain NameError instead of
363
+ # crashing on `isolate_namespace`.
364
+ #
365
+ # It deliberately loads `wurk/engine`, not `wurk/rails`: the railtie's
366
+ # `config.after_initialize` is a *global* ActiveSupport load hook registered at
367
+ # require time, and routes are drawn before those hooks run — so dragging the
368
+ # railtie in there would fork a swarm inside a `wurkswarm` parent that is about
369
+ # to fork its own. The runner owns the swarm; the routes file only needs the
370
+ # constant. `defined?(Wurk::Engine)` stays nil until first use, which is what the
371
+ # "standalone stays Rails-free" tests assert.
@@ -1 +1 @@
1
- import{A as e,F as t,J as n,M as r,O as i,_ as a,c as o,f as s,k as c,r as l,s as u,v as d,w as f}from"./Skeleton-DzR7XNxz.js";import{o as p,s as m}from"./index-DQu7WY9y.js";import{a as h}from"./utils-DDJC7tJV.js";var g=f(`<div style=display:flex;flex-direction:column;gap:2px><span style=font-size:11px;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em></span><span style=font-variant-numeric:tabular-nums>`),_=f(`<div style=margin-top:1.5rem><h2 style="font-size:14px;margin:0 0 0.5rem"> <span class="badge badge-muted"></span></h2><div class=table-wrapper><table><tbody>`),v=f(`<tr><td style=font-family:monospace;font-size:12px>`),y=f(`<div><div class=section-header style=gap:0.75rem></div><div style=margin-top:1rem>`),b=f(`<div class=empty-state style=color:var(--danger)>`),x=f(`<span class="badge badge-success">`),S=f(`<span class="badge badge-danger">`),C=f(`<p style=color:var(--text-muted);margin-top:0>`),w=f(`<div><div class=section-header style=gap:0.75rem><h1 class=page-title style=margin:0;font-family:monospace;font-size:16px></h1></div><div style="display:grid;grid-template-columns:repeat(auto-fill, minmax(140px, 1fr));gap:1rem;margin-top:1rem">`),T=f(`<span class="badge badge-accent">%`);function E(e){return(()=>{var t=g(),n=t.firstChild,r=n.nextSibling;return a(n,()=>e.label),a(r,()=>e.children),t})()}function D(n){return t(e,{get when(){return n.jids.length>0},get children(){var e=_(),r=e.firstChild,o=r.firstChild,s=o.nextSibling,c=r.nextSibling.firstChild.firstChild;return a(r,()=>n.label,o),a(s,()=>n.jids.length),a(c,t(i,{get each(){return n.jids},children:e=>(()=>{var t=v(),n=t.firstChild;return a(n,e),t})()})),e}})}function O(){let i=m(),f=()=>i.bid??``;n(()=>{document.title=`${u(`nav.batches`)} — Wurk`});let g=s(()=>({queryKey:[`batch`,f()],queryFn:()=>fetch(`${o()}/api/batches/${encodeURIComponent(f())}`).then(e=>{if(!e.ok)throw Error(`HTTP ${e.status}`);return e.json()})}));return t(r,{get children(){return[t(c,{get when(){return g.isPending},get children(){var e=y(),n=e.firstChild,r=n.nextSibling;return a(n,t(p,{href:`/batches`,class:`btn btn-sm`,get children(){return[`← `,d(()=>u(`nav.batches`))]}})),a(r,t(l,{count:8})),e}}),t(c,{get when(){return g.isError||!g.data},get children(){var e=b();return a(e,()=>u(`common.error`)),e}}),t(c,{get when(){return g.data},children:n=>{let r=()=>n().total-n().pending,i=()=>n().total>0?Math.round(r()/n().total*100):0;return(()=>{var r=w(),o=r.firstChild,s=o.firstChild,c=o.nextSibling;return a(o,t(p,{href:`/batches`,class:`btn btn-sm`,get children(){return[`← `,d(()=>u(`nav.batches`))]}}),s),a(s,()=>n().bid),a(o,t(e,{get when(){return n().complete},get fallback(){return(()=>{var e=T(),t=e.firstChild;return a(e,i,t),e})()},get children(){var e=x();return a(e,()=>u(`batches.complete`)),e}}),null),a(o,t(e,{get when(){return n().invalidated},get children(){var e=S();return a(e,()=>u(`batches.invalidated`)),e}}),null),a(r,t(e,{get when(){return n().description},get children(){var e=C();return a(e,()=>n().description),e}}),c),a(c,t(E,{get label(){return u(`table.total`)},get children(){return n().total.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.pending`)},get children(){return n().pending.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.failures`)},get children(){return n().failures.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`batches.children`)},get children(){return n().child_count.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.created`)},get children(){return d(()=>!!n().created_at)()?h(n().created_at):`—`}}),null),a(c,t(E,{get label(){return u(`batches.completed`)},get children(){return d(()=>!!n().complete_at)()?h(n().complete_at):`—`}}),null),a(c,t(e,{get when(){return n().parent_bid},get children(){return t(E,{get label(){return u(`batches.parent`)},get children(){return n().parent_bid}})}}),null),a(c,t(e,{get when(){return n().tags.length>0},get children(){return t(E,{get label(){return u(`batches.tags`)},get children(){return n().tags.join(`, `)}})}}),null),a(r,t(D,{get label(){return u(`batches.failed_jobs`)},get jids(){return n().failed_jids}}),null),a(r,t(D,{get label(){return u(`batches.dead_jobs`)},get jids(){return n().dead_jids}}),null),r})()}})]}})}export{O as default};
1
+ import{A as e,F as t,J as n,M as r,O as i,_ as a,c as o,f as s,k as c,r as l,s as u,v as d,w as f}from"./Skeleton-DzR7XNxz.js";import{o as p,s as m}from"./index-D_lSDwKw.js";import{a as h}from"./utils-DDJC7tJV.js";var g=f(`<div style=display:flex;flex-direction:column;gap:2px><span style=font-size:11px;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em></span><span style=font-variant-numeric:tabular-nums>`),_=f(`<div style=margin-top:1.5rem><h2 style="font-size:14px;margin:0 0 0.5rem"> <span class="badge badge-muted"></span></h2><div class=table-wrapper><table><tbody>`),v=f(`<tr><td style=font-family:monospace;font-size:12px>`),y=f(`<div><div class=section-header style=gap:0.75rem></div><div style=margin-top:1rem>`),b=f(`<div class=empty-state style=color:var(--danger)>`),x=f(`<span class="badge badge-success">`),S=f(`<span class="badge badge-danger">`),C=f(`<p style=color:var(--text-muted);margin-top:0>`),w=f(`<div><div class=section-header style=gap:0.75rem><h1 class=page-title style=margin:0;font-family:monospace;font-size:16px></h1></div><div style="display:grid;grid-template-columns:repeat(auto-fill, minmax(140px, 1fr));gap:1rem;margin-top:1rem">`),T=f(`<span class="badge badge-accent">%`);function E(e){return(()=>{var t=g(),n=t.firstChild,r=n.nextSibling;return a(n,()=>e.label),a(r,()=>e.children),t})()}function D(n){return t(e,{get when(){return n.jids.length>0},get children(){var e=_(),r=e.firstChild,o=r.firstChild,s=o.nextSibling,c=r.nextSibling.firstChild.firstChild;return a(r,()=>n.label,o),a(s,()=>n.jids.length),a(c,t(i,{get each(){return n.jids},children:e=>(()=>{var t=v(),n=t.firstChild;return a(n,e),t})()})),e}})}function O(){let i=m(),f=()=>i.bid??``;n(()=>{document.title=`${u(`nav.batches`)} — Wurk`});let g=s(()=>({queryKey:[`batch`,f()],queryFn:()=>fetch(`${o()}/api/batches/${encodeURIComponent(f())}`).then(e=>{if(!e.ok)throw Error(`HTTP ${e.status}`);return e.json()})}));return t(r,{get children(){return[t(c,{get when(){return g.isPending},get children(){var e=y(),n=e.firstChild,r=n.nextSibling;return a(n,t(p,{href:`/batches`,class:`btn btn-sm`,get children(){return[`← `,d(()=>u(`nav.batches`))]}})),a(r,t(l,{count:8})),e}}),t(c,{get when(){return g.isError||!g.data},get children(){var e=b();return a(e,()=>u(`common.error`)),e}}),t(c,{get when(){return g.data},children:n=>{let r=()=>n().total-n().pending,i=()=>n().total>0?Math.round(r()/n().total*100):0;return(()=>{var r=w(),o=r.firstChild,s=o.firstChild,c=o.nextSibling;return a(o,t(p,{href:`/batches`,class:`btn btn-sm`,get children(){return[`← `,d(()=>u(`nav.batches`))]}}),s),a(s,()=>n().bid),a(o,t(e,{get when(){return n().complete},get fallback(){return(()=>{var e=T(),t=e.firstChild;return a(e,i,t),e})()},get children(){var e=x();return a(e,()=>u(`batches.complete`)),e}}),null),a(o,t(e,{get when(){return n().invalidated},get children(){var e=S();return a(e,()=>u(`batches.invalidated`)),e}}),null),a(r,t(e,{get when(){return n().description},get children(){var e=C();return a(e,()=>n().description),e}}),c),a(c,t(E,{get label(){return u(`table.total`)},get children(){return n().total.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.pending`)},get children(){return n().pending.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.failures`)},get children(){return n().failures.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`batches.children`)},get children(){return n().child_count.toLocaleString()}}),null),a(c,t(E,{get label(){return u(`table.created`)},get children(){return d(()=>!!n().created_at)()?h(n().created_at):`—`}}),null),a(c,t(E,{get label(){return u(`batches.completed`)},get children(){return d(()=>!!n().complete_at)()?h(n().complete_at):`—`}}),null),a(c,t(e,{get when(){return n().parent_bid},get children(){return t(E,{get label(){return u(`batches.parent`)},get children(){return n().parent_bid}})}}),null),a(c,t(e,{get when(){return n().tags.length>0},get children(){return t(E,{get label(){return u(`batches.tags`)},get children(){return n().tags.join(`, `)}})}}),null),a(r,t(D,{get label(){return u(`batches.failed_jobs`)},get jids(){return n().failed_jids}}),null),a(r,t(D,{get label(){return u(`batches.dead_jobs`)},get jids(){return n().dead_jids}}),null),r})()}})]}})}export{O as default};