wurk 1.10.0 → 1.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/wurk/cron.rb CHANGED
@@ -25,7 +25,8 @@ module Wurk
25
25
  # so a re-registration of the same loop is idempotent.
26
26
  # * `Cron::Manager` — registration DSL. `mgr.register(cron, klass, **opts)`
27
27
  # with `tz=` mass-setter. Writes to Redis (`periodic` SET + `loops:{lid}`
28
- # HASH) and prunes superseded loops for the classes it registers.
28
+ # HASH) and prunes superseded loops for the classes it registers; the
29
+ # `config.periodic` one defers both until the server boots.
29
30
  # * `Cron::LoopSet` — Enumerable view (`each`/`size`/`fetch(lid)`).
30
31
  # * `Cron::ConfigTester` — boot-time validator. Verifies cron syntax and
31
32
  # that every worker class constant resolves.
@@ -46,6 +47,10 @@ module Wurk
46
47
  DEFAULT_TICK_SECONDS = 60
47
48
  MISSED_TICK_THRESHOLD = 90
48
49
 
50
+ # Handed to the error handlers when a due loop names a class this process
51
+ # cannot load, so the occurrence was skipped.
52
+ class UnknownWorkerClass < Wurk::Error; end
53
+
49
54
  # 5-field crontab + `@aliases`. No seconds field, no DOW name aliases
50
55
  # (sidekiq-ent §2.2 uses numeric DOW only).
51
56
  class Parser
@@ -441,26 +446,57 @@ module Wurk
441
446
  attr_reader :loops
442
447
 
443
448
  # `prune: false` registers without superseding — ConfigTester uses it so
444
- # validating a config never deletes a live loop.
445
- def initialize(config = nil, prune: true)
449
+ # validating a config never deletes a live loop. `deferred: true` holds
450
+ # every write until #apply!: `config.periodic` runs while the initializers
451
+ # load, before the app has finished configuring Redis, and in processes
452
+ # that never boot a server (Ent registers on startup).
453
+ def initialize(config = nil, prune: true, deferred: false)
446
454
  @config = config
447
455
  @loops = []
448
456
  @tz = nil
449
457
  @prune = prune
458
+ @deferred = deferred
459
+ @authoritative = false
450
460
  @pruned_klasses = {}
451
461
  end
452
462
 
463
+ # How late a slot may fire, in seconds, instead of being skipped, for every
464
+ # loop of this cluster's pollers. Default MISSED_TICK_THRESHOLD. A process-wide
465
+ # setting and not a loop option, which would enter the loop's identity.
466
+ #
467
+ # Coerced here, because the poller rescues per loop: a value it could not
468
+ # compare would stop every loop on every tick with nothing failing at boot.
469
+ def grace=(seconds)
470
+ grace = Float(seconds, exception: false)
471
+ unless grace&.finite? && grace.positive?
472
+ raise ArgumentError, "periodic grace must be a positive, finite number of seconds: #{seconds.inspect}"
473
+ end
474
+
475
+ (@config || Wurk.configuration)[:cron_grace] = grace
476
+ end
477
+
453
478
  def register(cron, klass, **opts)
454
479
  klass_name = klass.is_a?(String) ? klass : klass.to_s
455
480
  tz = opts.delete(:tz) || @tz
456
481
  normalized = opts.transform_keys(&:to_s)
457
482
  loop_obj = Loop.new(schedule: cron, klass: klass_name, options: normalized, tz: tz)
458
483
  @loops << loop_obj
459
- Cron.persist(loop_obj)
460
- prune_superseded(klass_name)
484
+ persist(loop_obj) unless @deferred
461
485
  loop_obj
462
486
  end
463
487
 
488
+ # Writes the registrations held back since `config.periodic` ran, then
489
+ # prunes as #register and #authoritative! would have. Launcher#boot calls
490
+ # it; once applied, later registrations are written as they come.
491
+ def apply!
492
+ return self unless @deferred
493
+
494
+ @deferred = false
495
+ @loops.each { |loop_obj| persist(loop_obj) }
496
+ prune_unregistered if @authoritative
497
+ self
498
+ end
499
+
464
500
  # Registers a sidekiq-cron shaped schedule: `{ name => { cron:, class:,
465
501
  # queue:, args:, retry:, tz:, status: } }`, or a list of those carrying
466
502
  # `name:`. Other keys (`description`, ...) are ignored. Raises ArgumentError
@@ -485,13 +521,23 @@ module Wurk
485
521
  def authoritative!
486
522
  return self unless @prune
487
523
 
488
- keep = @loops.map(&:lid)
489
- LoopSet.new(@config).each { |existing| Cron.unregister(existing.lid) unless keep.include?(existing.lid) }
524
+ @authoritative = true
525
+ prune_unregistered unless @deferred
490
526
  self
491
527
  end
492
528
 
493
529
  private
494
530
 
531
+ def persist(loop_obj)
532
+ Cron.persist(loop_obj)
533
+ prune_superseded(loop_obj.klass)
534
+ end
535
+
536
+ def prune_unregistered
537
+ keep = @loops.map(&:lid)
538
+ LoopSet.new(@config).each { |existing| Cron.unregister(existing.lid) unless keep.include?(existing.lid) }
539
+ end
540
+
495
541
  def register_entry(entry)
496
542
  cron = entry['cron'].to_s.strip
497
543
  klass = (entry['class'] || entry['klass']).to_s
@@ -611,10 +657,12 @@ module Wurk
611
657
  end
612
658
  end
613
659
 
614
- # Once-per-minute tick driver. Only the cluster leader iterates the LoopSet
615
- # and enqueues; non-leaders return early. A slot that wall-clock has passed
616
- # by more than `MISSED_TICK_THRESHOLD` seconds is skipped with a warning,
617
- # never backfilled.
660
+ # Once-per-minute tick driver, plus one tick the moment this process wins the
661
+ # lock: at a tick's phase, a leader that lives under a minute would never tick.
662
+ # Only the cluster leader iterates the LoopSet and enqueues; non-leaders return
663
+ # early. A slot that wall-clock has passed by more than `MISSED_TICK_THRESHOLD`
664
+ # seconds is skipped with a warning, never backfilled, and the occurrence due
665
+ # now, if any, fires in its place.
618
666
  class Poller
619
667
  include Component
620
668
 
@@ -625,17 +673,23 @@ module Wurk
625
673
  # Operators never need to touch this; integration tests shrink it so a
626
674
  # due loop fires within the test window instead of waiting a full minute.
627
675
  @tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
628
- @timer = TimerLoop.new(@tick_interval)
676
+ @done = false
677
+ @woken = false
678
+ @mutex = ::Mutex.new
679
+ @sleeper = ::ConditionVariable.new
680
+ @on_leader = method(:wake_for_leadership)
681
+ @unknown_reported = ::Set.new
629
682
  end
630
683
 
631
- # TimerLoop waits one interval before the first tick: don't fire a
632
- # catch-up burst the instant we boot (the leader is barely settled), and
633
- # let a short-lived process exit without ticking at all.
684
+ # The loop waits one interval before its first tick unless this process
685
+ # wins the lock first: let a process that never leads, or exits at once,
686
+ # exit without ticking at all.
634
687
  def start
635
688
  return @thread if @thread
636
689
 
637
- @timer.reset
638
- @thread = safe_thread('cron-poller') { @timer.run { tick } }
690
+ @mutex.synchronize { @done = false }
691
+ leader_hooks << @on_leader
692
+ @thread = safe_thread('cron-poller') { run_loop }
639
693
  end
640
694
 
641
695
  # Blocks until the thread is really gone: the launcher releases the
@@ -644,10 +698,14 @@ module Wurk
644
698
  #
645
699
  # Cleared only on a confirmed join (Thread#join returns nil on timeout):
646
700
  # a wedged thread must stay tracked so #start's guard returns it instead
647
- # of calling @timer.reset, which would un-terminate the loop it is still
648
- # inside and leave two tick threads double-enqueuing the same loops.
701
+ # of clearing @done, which would un-terminate the loop it is still inside
702
+ # and leave two tick threads double-enqueuing the same loops.
649
703
  def terminate
650
- @timer.terminate
704
+ leader_hooks.delete(@on_leader)
705
+ @mutex.synchronize do
706
+ @done = true
707
+ @sleeper.signal
708
+ end
651
709
  @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
652
710
  end
653
711
 
@@ -679,6 +737,10 @@ module Wurk
679
737
  # A slot older than the missed-tick threshold is not backfilled (ent §2.6:
680
738
  # no backfill on restart, missed ticks across a leader gap are dropped):
681
739
  # the marks still advance past it, but nothing is pushed.
740
+ #
741
+ # One claim, one push at most, whichever slot it pushes for: the occurrence
742
+ # due now stands in for a stale one under the stale one's mark, so a poller
743
+ # that read the same mark loses the CAS for both.
682
744
  def enqueue_if_due(loop_obj)
683
745
  return if loop_obj.paused?
684
746
 
@@ -686,13 +748,14 @@ module Wurk
686
748
  slot, mark = due_slot(loop_obj, now)
687
749
  return if slot.nil?
688
750
 
689
- future = loop_obj.next_fire_after(slot, now)
751
+ firing = slot_to_fire(loop_obj, slot, now)
752
+ future = loop_obj.next_fire_after(firing || slot, now)
690
753
  return unless claim_fire?(loop_obj, mark, now, future)
691
- return skip_missed(loop_obj, slot, now) if now - slot > missed_threshold
692
754
 
693
- jid = enqueue_claimed!(loop_obj)
694
- record_history(loop_obj, jid, now)
695
- jid
755
+ skip_missed(loop_obj, slot, now) unless firing == slot
756
+ return if firing.nil?
757
+
758
+ fire_claimed(loop_obj, now)
696
759
  end
697
760
 
698
761
  # Fire one loop right now, bypassing both the leader gate and the
@@ -708,6 +771,53 @@ module Wurk
708
771
 
709
772
  private
710
773
 
774
+ def run_loop
775
+ wait_for_tick
776
+ until done?
777
+ tick
778
+ wait_for_tick
779
+ end
780
+ end
781
+
782
+ def wait_for_tick
783
+ @mutex.synchronize do
784
+ @sleeper.wait(@mutex, @tick_interval) unless @done || @woken
785
+ @woken = false
786
+ end
787
+ end
788
+
789
+ def done?
790
+ @mutex.synchronize { @done }
791
+ end
792
+
793
+ # Runs on the campaign thread. The gate's cached answer predates the win.
794
+ def wake_for_leadership
795
+ @leader_checked_at = nil
796
+ @mutex.synchronize do
797
+ @woken = true
798
+ @sleeper.signal
799
+ end
800
+ end
801
+
802
+ def leader_hooks
803
+ @config[:lifecycle_events].fetch(:leader)
804
+ end
805
+
806
+ def slot_to_fire(loop_obj, slot, now)
807
+ now - slot > missed_threshold ? latest_due(loop_obj, now) : slot
808
+ end
809
+
810
+ # The newest occurrence still inside the window a late slot may fire in.
811
+ def latest_due(loop_obj, now)
812
+ latest = nil
813
+ occurrence = loop_obj.next_fire_at(now - missed_threshold - 1)
814
+ while occurrence && occurrence <= now
815
+ latest = occurrence
816
+ occurrence = loop_obj.next_fire_at(occurrence)
817
+ end
818
+ latest
819
+ end
820
+
711
821
  # The slot this tick would fire and the token that claims it, or nil when
712
822
  # the loop has no resolvable occurrence or none is due yet. The token is
713
823
  # the stored `nf` when there is one, otherwise the slot we derived — see
@@ -723,7 +833,7 @@ module Wurk
723
833
  # Never tighter than a tick and a half: a raised tick interval must not
724
834
  # turn every on-time fire into a "missed" one.
725
835
  def missed_threshold
726
- [MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
836
+ [@config[:cron_grace] || MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
727
837
  end
728
838
 
729
839
  def skip_missed(loop_obj, expected, now)
@@ -734,6 +844,32 @@ module Wurk
734
844
  nil
735
845
  end
736
846
 
847
+ def fire_claimed(loop_obj, now)
848
+ return skip_unknown_class(loop_obj) unless loop_obj.worker_class
849
+
850
+ jid = enqueue_claimed!(loop_obj)
851
+ record_history(loop_obj, jid, now)
852
+ jid
853
+ end
854
+
855
+ # Ent's tick is `klass.perform_async`, which needs the class. Pushing the
856
+ # bare name instead drops the worker's sidekiq_options, so a job that
857
+ # declares `retry: 0` failed with NameError under the default retries on
858
+ # every tick. The leader boots the app's code, so a class it cannot load
859
+ # is one that was deleted, renamed or rolled back; its loop stays in Redis
860
+ # until `Cron.unregister(lid)` or `authoritative!`. Logged on every
861
+ # occurrence, reported once per loop per process.
862
+ def skip_unknown_class(loop_obj)
863
+ message = "[cron] class #{loop_obj.klass} is not defined in this process lid=#{loop_obj.lid} — " \
864
+ 'occurrence skipped; Wurk::Cron.unregister(lid) retires the loop'
865
+ logger.warn(message)
866
+ return unless @unknown_reported.add?(loop_obj.lid)
867
+
868
+ handle_exception(UnknownWorkerClass.new(message),
869
+ { context: 'cron-poller', lid: loop_obj.lid, klass: loop_obj.klass })
870
+ nil
871
+ end
872
+
737
873
  # Compare-and-swap the fire marks: only the tick still looking at the
738
874
  # mark it read wins the slot. Manual `#fire` deliberately skips this —
739
875
  # an operator-requested run contends with no schedule slot.
data/lib/wurk/embedded.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require_relative 'component'
4
4
  require_relative 'launcher'
5
+ require_relative 'swarm/test_mode_guard'
5
6
 
6
7
  module Wurk
7
8
  # Embeds Wurk inside an arbitrary Ruby process (Puma, rake task, custom
@@ -34,13 +35,13 @@ module Wurk
34
35
  # the worker threads have time to spin up before the host goes back to
35
36
  # serving traffic. Mirrors Sidekiq::Embedded#run.
36
37
  def run
38
+ Swarm::TestModeGuard.enforce!(logger)
37
39
  housekeeping
38
40
  fire_event(:startup, reverse: false, reraise: true)
39
41
  @launcher = build_launcher
40
42
  @launcher.run
41
43
  sleep 0.2
42
- logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
43
- logger.debug { Thread.list.map(&:name).to_s }
44
+ log_running
44
45
  rescue StandardError
45
46
  # The host is left believing Wurk never started, so anything the launcher
46
47
  # did get up would keep fetching, beating and campaigning for the leader
@@ -62,6 +63,11 @@ module Wurk
62
63
 
63
64
  private
64
65
 
66
+ def log_running
67
+ logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
68
+ logger.debug { Thread.list.map(&:name).to_s }
69
+ end
70
+
65
71
  # Guarded so the caller sees why the boot failed, not why the rollback did.
66
72
  def roll_back_partial_boot
67
73
  stop
@@ -83,6 +83,12 @@ module Wurk
83
83
  # is what callers see as the dead record's `error_class`.
84
84
  class DecryptionError < Error; end
85
85
 
86
+ # The job says `encrypt: true` but its last argument is not a Wurk envelope:
87
+ # another implementation's ciphertext, or a producer that never enabled
88
+ # crypto. Neither can be told from a plain value, and either way `perform`
89
+ # must not take it for the decrypted argument.
90
+ class NotSealedError < Error; end
91
+
86
92
  class << self
87
93
  attr_reader :active_version
88
94
 
@@ -228,7 +234,10 @@ module Wurk
228
234
  'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
229
235
  )
230
236
  Wurk::Metrics::Statsd.increment('jobs.encryption_error', tags: ["worker:#{job['class']}"])
231
- Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause)
237
+ Wurk.logger.error { "#{DEAD_REASON}: #{job['class']} jid=#{job['jid']} sent to the dead set: #{cause.message}" }
238
+ # No trim: a misconfigured producer or a lost key can fail a whole backlog at
239
+ # once, and the cap would destroy jobs that are recoverable once it is fixed.
240
+ Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause, trim: false)
232
241
  nil
233
242
  end
234
243
 
@@ -318,14 +327,19 @@ module Wurk
318
327
  # with every outer middleware and the processor; leaving plaintext in it
319
328
  # let any of them write the secret back to Redis (InterruptHandler's
320
329
  # re-push, limiter dead-routing) or hand it to an error reporter. A decrypt
321
- # failure (missing/rotated key, bad tag) is terminal and non-retryable,
322
- # so rather than let it bubble into the 25× retry pipeline we route the
330
+ # failure (missing/rotated key, bad tag, a last argument that is no envelope)
331
+ # is terminal and non-retryable, so rather than let it bubble into the 25× retry pipeline we route the
323
332
  # job straight to the dead set tagged `encryption_error` and ACK it via
324
333
  # JobRetry::Skip — see Wurk::Encryption.route_to_dead. Plaintext args
325
334
  # remain visible for triage per §4.6.
326
335
  class ServerMiddleware
327
336
  include Wurk::Middleware::ServerMiddleware
328
337
 
338
+ # Never quotes the argument: it may be a secret in the clear.
339
+ NOT_SEALED_MESSAGE = 'job has encrypt: true but its last argument is not a Wurk envelope, so it was not run. ' \
340
+ 'A producer without crypto enabled pushed it, or another implementation encrypted it. ' \
341
+ 'A retry re-seals the argument as it stands: retry only a job known to hold plaintext'
342
+
329
343
  def call(_worker, job, _queue)
330
344
  return yield unless Wurk::Encryption.enabled? && job['encrypt']
331
345
 
@@ -342,7 +356,8 @@ module Wurk
342
356
 
343
357
  def decrypt_last_arg!(job)
344
358
  args = job['args']
345
- return unless args.is_a?(::Array) && !args.empty? && Wurk::Encryption.envelope?(args.last)
359
+ return unless args.is_a?(::Array) && !args.empty?
360
+ raise NotSealedError, NOT_SEALED_MESSAGE unless Wurk::Encryption.envelope?(args.last)
346
361
 
347
362
  job['args'] = args[0..-2] + [Wurk::Encryption.decrypt(args.last)]
348
363
  rescue Wurk::Encryption::Error,
@@ -350,7 +365,7 @@ module Wurk
350
365
  ::ArgumentError,
351
366
  ::TypeError,
352
367
  ::JSON::ParserError => e
353
- # Wurk::Encryption::Error covers KeyMissingError / DecryptionError.
368
+ # Wurk::Encryption::Error covers KeyMissingError / DecryptionError / NotSealedError.
354
369
  # OpenSSL::Cipher::CipherError → tampered ciphertext / bad key (tag mismatch).
355
370
  # ArgumentError → Base64.strict_decode64 / Integer() on a malformed envelope.
356
371
  # TypeError → Integer(nil) on a missing 'v' field.
@@ -99,7 +99,7 @@ module Wurk
99
99
  gate = @gates[keys[1]]
100
100
  next if gate&.blocked?(now)
101
101
 
102
- uow = gate ? claim(public_q, keys, gate, ack, now) : lmove(public_q, ack)
102
+ uow = gate ? claim(public_q, keys, gate, ack) : lmove(public_q, ack)
103
103
  ack = nil
104
104
  return uow if uow
105
105
  end
@@ -124,7 +124,7 @@ module Wurk
124
124
  # is deferred: an ACK handed back by a failed flush must not be able to
125
125
  # take the hold of whatever this thread is running by then. See
126
126
  # QueueSlot.claim_token.
127
- def claim(public_q, keys, gate, ack, now)
127
+ def claim(public_q, keys, gate, ack)
128
128
  priv, name = keys
129
129
  token = QueueSlot.claim_token
130
130
  script_keys = [gate.slot_key, public_q, priv]
@@ -137,7 +137,12 @@ module Wurk
137
137
  # faster than a job finishes somewhere in the cluster, so re-asking on
138
138
  # every pass is how a full queue turns into a spin. An empty queue is
139
139
  # remembered by nobody, exactly as on the uncapped path.
140
- gate.block!(now) if status == AT_CAPACITY
140
+ #
141
+ # Stamped with the time the refusal ARRIVED, not the time the question
142
+ # was sent: a slow Redis would otherwise eat the backoff window and
143
+ # the gate would already be expired when the refusal lands — which is
144
+ # the spin this remembers it to avoid.
145
+ gate.block!(::Process.clock_gettime(::Process::CLOCK_MONOTONIC)) if status == AT_CAPACITY
141
146
  nil
142
147
  rescue StandardError
143
148
  # The window #lmove guards: the ACK left its slot but never reached
data/lib/wurk/launcher.rb CHANGED
@@ -99,10 +99,12 @@ module Wurk
99
99
  # 4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
100
100
  # not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
101
101
  # so it stays ahead of the periodic loops.
102
- # 5. start the periodic loops. None of them ticks at zero — TimerLoop#run
103
- # waits an interval before its first yield, Scheduled::Poller waits
104
- # INITIAL_WAIT, and Leader waits DEFAULT_INITIAL_WAIT — so starting
105
- # them last costs a few thread spawns and nothing else. Both leader-
102
+ # 5. write the `config.periodic` registrations (Ent registers loops on
103
+ # startup), then start the periodic loops. None of them ticks at
104
+ # zero — TimerLoop#run waits an interval before its first yield,
105
+ # Scheduled::Poller waits INITIAL_WAIT, and Leader waits
106
+ # DEFAULT_INITIAL_WAIT — so starting them last costs a few thread
107
+ # spawns and nothing else. Both leader-
106
108
  # gated pollers are also safe to start before leadership settles: a
107
109
  # non-leader tick just returns early.
108
110
  # 6. start the health probe server LAST so the listener doesn't
@@ -240,6 +242,7 @@ module Wurk
240
242
  # Run on a background thread so /ready probe isn't delayed by a large
241
243
  # orphan sweep (the reaper's reclaim is atomic, but can scan many entries).
242
244
  @boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
245
+ @config.apply_periodic!
243
246
  start_periodic_components
244
247
  end
245
248
  @health_server&.start
data/lib/wurk/leader.rb CHANGED
@@ -3,6 +3,7 @@
3
3
  require 'securerandom'
4
4
  require 'socket'
5
5
  require_relative 'component'
6
+ require_relative 'fetcher/oom_drain'
6
7
  require_relative 'lua'
7
8
  require_relative 'pool_checkout'
8
9
  require_relative 'timer_loop'
@@ -21,8 +22,12 @@ module Wurk
21
22
  # *can* benefit from a guard has one available; the token is best-effort
22
23
  # too — it is never re-read on subsequent acquires, only on transitions.
23
24
  #
24
- # Cadence per spec: renew every 15s while leader, recheck every 60s as
25
- # follower, lock TTL 30s. The campaign itself starts `initial_wait` seconds
25
+ # Cadence per spec: renew every 15s while leader, lock TTL 30s. A follower
26
+ # rechecks when the lease it last read runs out, at most 60s later, and that
27
+ # is what bounds the leaderless time after a SIGKILL: the lease's remaining
28
+ # 15-30s and about a second. Waiting out a flat 60s added up to 60s more. The
29
+ # lock itself is untouched, so a follower still cannot win before the lease
30
+ # has lapsed. The campaign itself starts `initial_wait` seconds
26
31
  # after `#start` — the spec fixes the renew/follower/TTL cadence but says
27
32
  # nothing about when the first campaign runs, and holding it back keeps a
28
33
  # booting process's opening Redis round trips on the job path. Opt out a
@@ -43,6 +48,17 @@ module Wurk
43
48
  # nothing observes the gap. Tune with `config[:leader_initial_wait]`;
44
49
  # 0 campaigns immediately.
45
50
  DEFAULT_INITIAL_WAIT = 1
51
+ # Followers spread out over this many seconds past the lease's end, so a
52
+ # large fleet doesn't campaign in one instant.
53
+ LEASE_EXPIRY_JITTER = 1.0
54
+ # Keeps a lease read as about to lapse from turning into a busy loop.
55
+ MIN_FOLLOWER_WAIT = 0.05
56
+ # The renewal has to survive a full Redis: a lapsed lease stops every
57
+ # leader-gated consumer, and nothing would renew it until memory came back.
58
+ # Shebang scripts, so by EVAL for the reason Fetcher::OomDrain gives. A server
59
+ # refusing one raises the original OOM, the failure without this.
60
+ OOM_CAMPAIGN = "#{Fetcher::OomDrain::HEADER}#{Lua::SCRIPTS.fetch(:leader_campaign)}".freeze
61
+ OOM_TOKEN = "#{Fetcher::OomDrain::HEADER}return redis.call('INCR', KEYS[1])\n".freeze
46
62
  OPT_OUT_ENV = 'WURK_LEADER' # native opt-out env
47
63
  SIDEKIQ_OPT_OUT_ENV = 'SIDEKIQ_LEADER' # Sidekiq Ent drop-in alias (§6.2/§7.2)
48
64
  THREAD_NAME = 'wurk-leader'
@@ -71,6 +87,8 @@ module Wurk
71
87
  @owner = owner || cluster_identity
72
88
  @held = false
73
89
  @token = nil
90
+ @lease_remaining = nil
91
+ @campaign_failed = false
74
92
  @thread = nil
75
93
  @done = false
76
94
  @mutex = ::Mutex.new
@@ -98,7 +116,7 @@ module Wurk
98
116
  return false unless transition
99
117
 
100
118
  if transition == :gained
101
- @token = redis_call { |c| c.call('INCR', TOKEN_KEY) }
119
+ @token = redis_call { |c| next_token(c) }
102
120
  dispatch_leader_event
103
121
  end
104
122
  true
@@ -191,9 +209,7 @@ module Wurk
191
209
  # if another process currently holds the key. Sets `@held` accordingly
192
210
  # so `leader?` reflects the latest state regardless of branch.
193
211
  def run_set_or_refresh
194
- won = redis_call do |c|
195
- Wurk::Lua::Loader.eval_cached(c, :leader_campaign, keys: [@key], argv: [@owner, @ttl])
196
- end
212
+ won = redis_call { |c| campaign(c) }
197
213
  if won.to_i == 1
198
214
  outcome = @held ? :held : :gained
199
215
  @held = true
@@ -201,9 +217,37 @@ module Wurk
201
217
  end
202
218
 
203
219
  step_down
220
+ read_lease_remaining
204
221
  nil
205
222
  end
206
223
 
224
+ def campaign(conn)
225
+ Wurk::Lua::Loader.eval_cached(conn, :leader_campaign, keys: [@key], argv: [@owner, @ttl])
226
+ rescue RedisClient::OutOfMemoryError => e
227
+ past_maxmemory(e) { conn.call('EVAL', OOM_CAMPAIGN, 1, @key, @owner, @ttl) }
228
+ end
229
+
230
+ def next_token(conn)
231
+ conn.call('INCR', TOKEN_KEY)
232
+ rescue RedisClient::OutOfMemoryError => e
233
+ past_maxmemory(e) { conn.call('EVAL', OOM_TOKEN, 1, TOKEN_KEY) }
234
+ end
235
+
236
+ def past_maxmemory(error)
237
+ yield
238
+ rescue RedisClient::ConnectionError
239
+ raise
240
+ rescue RedisClient::CommandError
241
+ raise error
242
+ end
243
+
244
+ # Seconds until the holder's lease runs out; nil for a lock with no TTL, which
245
+ # only expiry cannot free. A lock gone again already is a zero wait.
246
+ def read_lease_remaining
247
+ pttl = redis_call(idempotent: true) { |c| c.call('PTTL', @key) }
248
+ @lease_remaining = pttl == -1 ? nil : [pttl, 0].max / 1000.0
249
+ end
250
+
207
251
  def step_down
208
252
  @held = false
209
253
  @token = nil
@@ -283,10 +327,14 @@ module Wurk
283
327
  # would leave every leader-gated consumer acting as leader through an
284
328
  # outage long enough for a follower to take over, and would keep this loop
285
329
  # on the short renew cadence. Stepping down makes the next success a fresh
286
- # gain — a new fencing token and a re-fired `:leader` event.
330
+ # gain — a new fencing token and a re-fired `:leader` event. It says nothing
331
+ # about the lease, so the retry is on the renew cadence, not the follower's:
332
+ # a leader that failed one renewal still has time to keep its lease.
287
333
  def tick_once
334
+ @campaign_failed = false
288
335
  acquire
289
336
  rescue StandardError => e
337
+ @campaign_failed = true
290
338
  step_down
291
339
  report(e)
292
340
  end
@@ -296,7 +344,13 @@ module Wurk
296
344
  end
297
345
 
298
346
  def wait_next
299
- wait_for(@held ? @renew_interval : @follower_interval)
347
+ wait_for(@held || @campaign_failed ? @renew_interval : follower_wait)
348
+ end
349
+
350
+ def follower_wait
351
+ return @follower_interval unless @lease_remaining
352
+
353
+ [@lease_remaining + (rand * LEASE_EXPIRY_JITTER), @follower_interval].min.clamp(MIN_FOLLOWER_WAIT, nil)
300
354
  end
301
355
 
302
356
  def wait_for(interval)