wurk 1.10.0 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -9
- data/app/controllers/wurk/dashboard_controller.rb +11 -6
- data/exe/wurk +7 -2
- data/exe/wurkswarm +1 -2
- data/lib/generators/wurk/install/install_generator.rb +12 -0
- data/lib/sidekiq/web.rb +5 -0
- data/lib/wurk/boot_checks.rb +8 -2
- data/lib/wurk/cli.rb +140 -31
- data/lib/wurk/client.rb +16 -5
- data/lib/wurk/compat.rb +4 -0
- data/lib/wurk/configuration.rb +11 -2
- data/lib/wurk/cron.rb +162 -26
- data/lib/wurk/embedded.rb +8 -2
- data/lib/wurk/encryption.rb +20 -5
- data/lib/wurk/fetcher/capped.rb +8 -3
- data/lib/wurk/launcher.rb +7 -4
- data/lib/wurk/leader.rb +62 -8
- data/lib/wurk/preflight/checks.rb +203 -0
- data/lib/wurk/preflight/command.rb +135 -0
- data/lib/wurk/preflight/keyspace.rb +109 -0
- data/lib/wurk/preflight/owners.rb +74 -0
- data/lib/wurk/preflight/payload_sample.rb +108 -0
- data/lib/wurk/preflight/reader.rb +60 -0
- data/lib/wurk/preflight/report.rb +82 -0
- data/lib/wurk/preflight/verify.rb +161 -0
- data/lib/wurk/preflight.rb +40 -0
- data/lib/wurk/swarm/child_boot.rb +2 -0
- data/lib/wurk/swarm/test_mode_guard.rb +32 -0
- data/lib/wurk/testing.rb +12 -4
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/config.rb +8 -4
- data/lib/wurk/web/extension.rb +35 -61
- data/lib/wurk/web/helpers.rb +133 -0
- data/lib/wurk/web/paginator.rb +49 -0
- data/lib/wurk/web/rack_app.rb +30 -6
- data/lib/wurk/web/views/_job_info.erb +16 -0
- data/lib/wurk/web/views/_paging.erb +11 -0
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +15 -1
data/lib/wurk/cron.rb
CHANGED
|
@@ -25,7 +25,8 @@ module Wurk
|
|
|
25
25
|
# so a re-registration of the same loop is idempotent.
|
|
26
26
|
# * `Cron::Manager` — registration DSL. `mgr.register(cron, klass, **opts)`
|
|
27
27
|
# with `tz=` mass-setter. Writes to Redis (`periodic` SET + `loops:{lid}`
|
|
28
|
-
# HASH) and prunes superseded loops for the classes it registers
|
|
28
|
+
# HASH) and prunes superseded loops for the classes it registers; the
|
|
29
|
+
# `config.periodic` one defers both until the server boots.
|
|
29
30
|
# * `Cron::LoopSet` — Enumerable view (`each`/`size`/`fetch(lid)`).
|
|
30
31
|
# * `Cron::ConfigTester` — boot-time validator. Verifies cron syntax and
|
|
31
32
|
# that every worker class constant resolves.
|
|
@@ -46,6 +47,10 @@ module Wurk
|
|
|
46
47
|
DEFAULT_TICK_SECONDS = 60
|
|
47
48
|
MISSED_TICK_THRESHOLD = 90
|
|
48
49
|
|
|
50
|
+
# Handed to the error handlers when a due loop names a class this process
|
|
51
|
+
# cannot load, so the occurrence was skipped.
|
|
52
|
+
class UnknownWorkerClass < Wurk::Error; end
|
|
53
|
+
|
|
49
54
|
# 5-field crontab + `@aliases`. No seconds field, no DOW name aliases
|
|
50
55
|
# (sidekiq-ent §2.2 uses numeric DOW only).
|
|
51
56
|
class Parser
|
|
@@ -441,26 +446,57 @@ module Wurk
|
|
|
441
446
|
attr_reader :loops
|
|
442
447
|
|
|
443
448
|
# `prune: false` registers without superseding — ConfigTester uses it so
|
|
444
|
-
# validating a config never deletes a live loop.
|
|
445
|
-
|
|
449
|
+
# validating a config never deletes a live loop. `deferred: true` holds
|
|
450
|
+
# every write until #apply!: `config.periodic` runs while the initializers
|
|
451
|
+
# load, before the app has finished configuring Redis, and in processes
|
|
452
|
+
# that never boot a server (Ent registers on startup).
|
|
453
|
+
def initialize(config = nil, prune: true, deferred: false)
|
|
446
454
|
@config = config
|
|
447
455
|
@loops = []
|
|
448
456
|
@tz = nil
|
|
449
457
|
@prune = prune
|
|
458
|
+
@deferred = deferred
|
|
459
|
+
@authoritative = false
|
|
450
460
|
@pruned_klasses = {}
|
|
451
461
|
end
|
|
452
462
|
|
|
463
|
+
# How late a slot may fire, in seconds, instead of being skipped, for every
|
|
464
|
+
# loop of this cluster's pollers. Default MISSED_TICK_THRESHOLD. A process-wide
|
|
465
|
+
# setting and not a loop option, which would enter the loop's identity.
|
|
466
|
+
#
|
|
467
|
+
# Coerced here, because the poller rescues per loop: a value it could not
|
|
468
|
+
# compare would stop every loop on every tick with nothing failing at boot.
|
|
469
|
+
def grace=(seconds)
|
|
470
|
+
grace = Float(seconds, exception: false)
|
|
471
|
+
unless grace&.finite? && grace.positive?
|
|
472
|
+
raise ArgumentError, "periodic grace must be a positive, finite number of seconds: #{seconds.inspect}"
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
(@config || Wurk.configuration)[:cron_grace] = grace
|
|
476
|
+
end
|
|
477
|
+
|
|
453
478
|
def register(cron, klass, **opts)
|
|
454
479
|
klass_name = klass.is_a?(String) ? klass : klass.to_s
|
|
455
480
|
tz = opts.delete(:tz) || @tz
|
|
456
481
|
normalized = opts.transform_keys(&:to_s)
|
|
457
482
|
loop_obj = Loop.new(schedule: cron, klass: klass_name, options: normalized, tz: tz)
|
|
458
483
|
@loops << loop_obj
|
|
459
|
-
|
|
460
|
-
prune_superseded(klass_name)
|
|
484
|
+
persist(loop_obj) unless @deferred
|
|
461
485
|
loop_obj
|
|
462
486
|
end
|
|
463
487
|
|
|
488
|
+
# Writes the registrations held back since `config.periodic` ran, then
|
|
489
|
+
# prunes as #register and #authoritative! would have. Launcher#boot calls
|
|
490
|
+
# it; once applied, later registrations are written as they come.
|
|
491
|
+
def apply!
|
|
492
|
+
return self unless @deferred
|
|
493
|
+
|
|
494
|
+
@deferred = false
|
|
495
|
+
@loops.each { |loop_obj| persist(loop_obj) }
|
|
496
|
+
prune_unregistered if @authoritative
|
|
497
|
+
self
|
|
498
|
+
end
|
|
499
|
+
|
|
464
500
|
# Registers a sidekiq-cron shaped schedule: `{ name => { cron:, class:,
|
|
465
501
|
# queue:, args:, retry:, tz:, status: } }`, or a list of those carrying
|
|
466
502
|
# `name:`. Other keys (`description`, ...) are ignored. Raises ArgumentError
|
|
@@ -485,13 +521,23 @@ module Wurk
|
|
|
485
521
|
def authoritative!
|
|
486
522
|
return self unless @prune
|
|
487
523
|
|
|
488
|
-
|
|
489
|
-
|
|
524
|
+
@authoritative = true
|
|
525
|
+
prune_unregistered unless @deferred
|
|
490
526
|
self
|
|
491
527
|
end
|
|
492
528
|
|
|
493
529
|
private
|
|
494
530
|
|
|
531
|
+
def persist(loop_obj)
|
|
532
|
+
Cron.persist(loop_obj)
|
|
533
|
+
prune_superseded(loop_obj.klass)
|
|
534
|
+
end
|
|
535
|
+
|
|
536
|
+
def prune_unregistered
|
|
537
|
+
keep = @loops.map(&:lid)
|
|
538
|
+
LoopSet.new(@config).each { |existing| Cron.unregister(existing.lid) unless keep.include?(existing.lid) }
|
|
539
|
+
end
|
|
540
|
+
|
|
495
541
|
def register_entry(entry)
|
|
496
542
|
cron = entry['cron'].to_s.strip
|
|
497
543
|
klass = (entry['class'] || entry['klass']).to_s
|
|
@@ -611,10 +657,12 @@ module Wurk
|
|
|
611
657
|
end
|
|
612
658
|
end
|
|
613
659
|
|
|
614
|
-
# Once-per-minute tick driver
|
|
615
|
-
#
|
|
616
|
-
#
|
|
617
|
-
#
|
|
660
|
+
# Once-per-minute tick driver, plus one tick the moment this process wins the
|
|
661
|
+
# lock: at a tick's phase, a leader that lives under a minute would never tick.
|
|
662
|
+
# Only the cluster leader iterates the LoopSet and enqueues; non-leaders return
|
|
663
|
+
# early. A slot that wall-clock has passed by more than `MISSED_TICK_THRESHOLD`
|
|
664
|
+
# seconds is skipped with a warning, never backfilled, and the occurrence due
|
|
665
|
+
# now, if any, fires in its place.
|
|
618
666
|
class Poller
|
|
619
667
|
include Component
|
|
620
668
|
|
|
@@ -625,17 +673,23 @@ module Wurk
|
|
|
625
673
|
# Operators never need to touch this; integration tests shrink it so a
|
|
626
674
|
# due loop fires within the test window instead of waiting a full minute.
|
|
627
675
|
@tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
|
|
628
|
-
@
|
|
676
|
+
@done = false
|
|
677
|
+
@woken = false
|
|
678
|
+
@mutex = ::Mutex.new
|
|
679
|
+
@sleeper = ::ConditionVariable.new
|
|
680
|
+
@on_leader = method(:wake_for_leadership)
|
|
681
|
+
@unknown_reported = ::Set.new
|
|
629
682
|
end
|
|
630
683
|
|
|
631
|
-
#
|
|
632
|
-
#
|
|
633
|
-
#
|
|
684
|
+
# The loop waits one interval before its first tick unless this process
|
|
685
|
+
# wins the lock first: let a process that never leads, or exits at once,
|
|
686
|
+
# exit without ticking at all.
|
|
634
687
|
def start
|
|
635
688
|
return @thread if @thread
|
|
636
689
|
|
|
637
|
-
@
|
|
638
|
-
|
|
690
|
+
@mutex.synchronize { @done = false }
|
|
691
|
+
leader_hooks << @on_leader
|
|
692
|
+
@thread = safe_thread('cron-poller') { run_loop }
|
|
639
693
|
end
|
|
640
694
|
|
|
641
695
|
# Blocks until the thread is really gone: the launcher releases the
|
|
@@ -644,10 +698,14 @@ module Wurk
|
|
|
644
698
|
#
|
|
645
699
|
# Cleared only on a confirmed join (Thread#join returns nil on timeout):
|
|
646
700
|
# a wedged thread must stay tracked so #start's guard returns it instead
|
|
647
|
-
# of
|
|
648
|
-
#
|
|
701
|
+
# of clearing @done, which would un-terminate the loop it is still inside
|
|
702
|
+
# and leave two tick threads double-enqueuing the same loops.
|
|
649
703
|
def terminate
|
|
650
|
-
@
|
|
704
|
+
leader_hooks.delete(@on_leader)
|
|
705
|
+
@mutex.synchronize do
|
|
706
|
+
@done = true
|
|
707
|
+
@sleeper.signal
|
|
708
|
+
end
|
|
651
709
|
@thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
|
|
652
710
|
end
|
|
653
711
|
|
|
@@ -679,6 +737,10 @@ module Wurk
|
|
|
679
737
|
# A slot older than the missed-tick threshold is not backfilled (ent §2.6:
|
|
680
738
|
# no backfill on restart, missed ticks across a leader gap are dropped):
|
|
681
739
|
# the marks still advance past it, but nothing is pushed.
|
|
740
|
+
#
|
|
741
|
+
# One claim, one push at most, whichever slot it pushes for: the occurrence
|
|
742
|
+
# due now stands in for a stale one under the stale one's mark, so a poller
|
|
743
|
+
# that read the same mark loses the CAS for both.
|
|
682
744
|
def enqueue_if_due(loop_obj)
|
|
683
745
|
return if loop_obj.paused?
|
|
684
746
|
|
|
@@ -686,13 +748,14 @@ module Wurk
|
|
|
686
748
|
slot, mark = due_slot(loop_obj, now)
|
|
687
749
|
return if slot.nil?
|
|
688
750
|
|
|
689
|
-
|
|
751
|
+
firing = slot_to_fire(loop_obj, slot, now)
|
|
752
|
+
future = loop_obj.next_fire_after(firing || slot, now)
|
|
690
753
|
return unless claim_fire?(loop_obj, mark, now, future)
|
|
691
|
-
return skip_missed(loop_obj, slot, now) if now - slot > missed_threshold
|
|
692
754
|
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
755
|
+
skip_missed(loop_obj, slot, now) unless firing == slot
|
|
756
|
+
return if firing.nil?
|
|
757
|
+
|
|
758
|
+
fire_claimed(loop_obj, now)
|
|
696
759
|
end
|
|
697
760
|
|
|
698
761
|
# Fire one loop right now, bypassing both the leader gate and the
|
|
@@ -708,6 +771,53 @@ module Wurk
|
|
|
708
771
|
|
|
709
772
|
private
|
|
710
773
|
|
|
774
|
+
def run_loop
|
|
775
|
+
wait_for_tick
|
|
776
|
+
until done?
|
|
777
|
+
tick
|
|
778
|
+
wait_for_tick
|
|
779
|
+
end
|
|
780
|
+
end
|
|
781
|
+
|
|
782
|
+
def wait_for_tick
|
|
783
|
+
@mutex.synchronize do
|
|
784
|
+
@sleeper.wait(@mutex, @tick_interval) unless @done || @woken
|
|
785
|
+
@woken = false
|
|
786
|
+
end
|
|
787
|
+
end
|
|
788
|
+
|
|
789
|
+
def done?
|
|
790
|
+
@mutex.synchronize { @done }
|
|
791
|
+
end
|
|
792
|
+
|
|
793
|
+
# Runs on the campaign thread. The gate's cached answer predates the win.
|
|
794
|
+
def wake_for_leadership
|
|
795
|
+
@leader_checked_at = nil
|
|
796
|
+
@mutex.synchronize do
|
|
797
|
+
@woken = true
|
|
798
|
+
@sleeper.signal
|
|
799
|
+
end
|
|
800
|
+
end
|
|
801
|
+
|
|
802
|
+
def leader_hooks
|
|
803
|
+
@config[:lifecycle_events].fetch(:leader)
|
|
804
|
+
end
|
|
805
|
+
|
|
806
|
+
def slot_to_fire(loop_obj, slot, now)
|
|
807
|
+
now - slot > missed_threshold ? latest_due(loop_obj, now) : slot
|
|
808
|
+
end
|
|
809
|
+
|
|
810
|
+
# The newest occurrence still inside the window a late slot may fire in.
|
|
811
|
+
def latest_due(loop_obj, now)
|
|
812
|
+
latest = nil
|
|
813
|
+
occurrence = loop_obj.next_fire_at(now - missed_threshold - 1)
|
|
814
|
+
while occurrence && occurrence <= now
|
|
815
|
+
latest = occurrence
|
|
816
|
+
occurrence = loop_obj.next_fire_at(occurrence)
|
|
817
|
+
end
|
|
818
|
+
latest
|
|
819
|
+
end
|
|
820
|
+
|
|
711
821
|
# The slot this tick would fire and the token that claims it, or nil when
|
|
712
822
|
# the loop has no resolvable occurrence or none is due yet. The token is
|
|
713
823
|
# the stored `nf` when there is one, otherwise the slot we derived — see
|
|
@@ -723,7 +833,7 @@ module Wurk
|
|
|
723
833
|
# Never tighter than a tick and a half: a raised tick interval must not
|
|
724
834
|
# turn every on-time fire into a "missed" one.
|
|
725
835
|
def missed_threshold
|
|
726
|
-
[MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
|
|
836
|
+
[@config[:cron_grace] || MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
|
|
727
837
|
end
|
|
728
838
|
|
|
729
839
|
def skip_missed(loop_obj, expected, now)
|
|
@@ -734,6 +844,32 @@ module Wurk
|
|
|
734
844
|
nil
|
|
735
845
|
end
|
|
736
846
|
|
|
847
|
+
def fire_claimed(loop_obj, now)
|
|
848
|
+
return skip_unknown_class(loop_obj) unless loop_obj.worker_class
|
|
849
|
+
|
|
850
|
+
jid = enqueue_claimed!(loop_obj)
|
|
851
|
+
record_history(loop_obj, jid, now)
|
|
852
|
+
jid
|
|
853
|
+
end
|
|
854
|
+
|
|
855
|
+
# Ent's tick is `klass.perform_async`, which needs the class. Pushing the
|
|
856
|
+
# bare name instead drops the worker's sidekiq_options, so a job that
|
|
857
|
+
# declares `retry: 0` failed with NameError under the default retries on
|
|
858
|
+
# every tick. The leader boots the app's code, so a class it cannot load
|
|
859
|
+
# is one that was deleted, renamed or rolled back; its loop stays in Redis
|
|
860
|
+
# until `Cron.unregister(lid)` or `authoritative!`. Logged on every
|
|
861
|
+
# occurrence, reported once per loop per process.
|
|
862
|
+
def skip_unknown_class(loop_obj)
|
|
863
|
+
message = "[cron] class #{loop_obj.klass} is not defined in this process lid=#{loop_obj.lid} — " \
|
|
864
|
+
'occurrence skipped; Wurk::Cron.unregister(lid) retires the loop'
|
|
865
|
+
logger.warn(message)
|
|
866
|
+
return unless @unknown_reported.add?(loop_obj.lid)
|
|
867
|
+
|
|
868
|
+
handle_exception(UnknownWorkerClass.new(message),
|
|
869
|
+
{ context: 'cron-poller', lid: loop_obj.lid, klass: loop_obj.klass })
|
|
870
|
+
nil
|
|
871
|
+
end
|
|
872
|
+
|
|
737
873
|
# Compare-and-swap the fire marks: only the tick still looking at the
|
|
738
874
|
# mark it read wins the slot. Manual `#fire` deliberately skips this —
|
|
739
875
|
# an operator-requested run contends with no schedule slot.
|
data/lib/wurk/embedded.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative 'component'
|
|
4
4
|
require_relative 'launcher'
|
|
5
|
+
require_relative 'swarm/test_mode_guard'
|
|
5
6
|
|
|
6
7
|
module Wurk
|
|
7
8
|
# Embeds Wurk inside an arbitrary Ruby process (Puma, rake task, custom
|
|
@@ -34,13 +35,13 @@ module Wurk
|
|
|
34
35
|
# the worker threads have time to spin up before the host goes back to
|
|
35
36
|
# serving traffic. Mirrors Sidekiq::Embedded#run.
|
|
36
37
|
def run
|
|
38
|
+
Swarm::TestModeGuard.enforce!(logger)
|
|
37
39
|
housekeeping
|
|
38
40
|
fire_event(:startup, reverse: false, reraise: true)
|
|
39
41
|
@launcher = build_launcher
|
|
40
42
|
@launcher.run
|
|
41
43
|
sleep 0.2
|
|
42
|
-
|
|
43
|
-
logger.debug { Thread.list.map(&:name).to_s }
|
|
44
|
+
log_running
|
|
44
45
|
rescue StandardError
|
|
45
46
|
# The host is left believing Wurk never started, so anything the launcher
|
|
46
47
|
# did get up would keep fetching, beating and campaigning for the leader
|
|
@@ -62,6 +63,11 @@ module Wurk
|
|
|
62
63
|
|
|
63
64
|
private
|
|
64
65
|
|
|
66
|
+
def log_running
|
|
67
|
+
logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
|
|
68
|
+
logger.debug { Thread.list.map(&:name).to_s }
|
|
69
|
+
end
|
|
70
|
+
|
|
65
71
|
# Guarded so the caller sees why the boot failed, not why the rollback did.
|
|
66
72
|
def roll_back_partial_boot
|
|
67
73
|
stop
|
data/lib/wurk/encryption.rb
CHANGED
|
@@ -83,6 +83,12 @@ module Wurk
|
|
|
83
83
|
# is what callers see as the dead record's `error_class`.
|
|
84
84
|
class DecryptionError < Error; end
|
|
85
85
|
|
|
86
|
+
# The job says `encrypt: true` but its last argument is not a Wurk envelope:
|
|
87
|
+
# another implementation's ciphertext, or a producer that never enabled
|
|
88
|
+
# crypto. Neither can be told from a plain value, and either way `perform`
|
|
89
|
+
# must not take it for the decrypted argument.
|
|
90
|
+
class NotSealedError < Error; end
|
|
91
|
+
|
|
86
92
|
class << self
|
|
87
93
|
attr_reader :active_version
|
|
88
94
|
|
|
@@ -228,7 +234,10 @@ module Wurk
|
|
|
228
234
|
'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
229
235
|
)
|
|
230
236
|
Wurk::Metrics::Statsd.increment('jobs.encryption_error', tags: ["worker:#{job['class']}"])
|
|
231
|
-
Wurk
|
|
237
|
+
Wurk.logger.error { "#{DEAD_REASON}: #{job['class']} jid=#{job['jid']} sent to the dead set: #{cause.message}" }
|
|
238
|
+
# No trim: a misconfigured producer or a lost key can fail a whole backlog at
|
|
239
|
+
# once, and the cap would destroy jobs that are recoverable once it is fixed.
|
|
240
|
+
Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause, trim: false)
|
|
232
241
|
nil
|
|
233
242
|
end
|
|
234
243
|
|
|
@@ -318,14 +327,19 @@ module Wurk
|
|
|
318
327
|
# with every outer middleware and the processor; leaving plaintext in it
|
|
319
328
|
# let any of them write the secret back to Redis (InterruptHandler's
|
|
320
329
|
# re-push, limiter dead-routing) or hand it to an error reporter. A decrypt
|
|
321
|
-
# failure (missing/rotated key, bad tag
|
|
322
|
-
# so rather than let it bubble into the 25× retry pipeline we route the
|
|
330
|
+
# failure (missing/rotated key, bad tag, a last argument that is no envelope)
|
|
331
|
+
# is terminal and non-retryable, so rather than let it bubble into the 25× retry pipeline we route the
|
|
323
332
|
# job straight to the dead set tagged `encryption_error` and ACK it via
|
|
324
333
|
# JobRetry::Skip — see Wurk::Encryption.route_to_dead. Plaintext args
|
|
325
334
|
# remain visible for triage per §4.6.
|
|
326
335
|
class ServerMiddleware
|
|
327
336
|
include Wurk::Middleware::ServerMiddleware
|
|
328
337
|
|
|
338
|
+
# Never quotes the argument: it may be a secret in the clear.
|
|
339
|
+
NOT_SEALED_MESSAGE = 'job has encrypt: true but its last argument is not a Wurk envelope, so it was not run. ' \
|
|
340
|
+
'A producer without crypto enabled pushed it, or another implementation encrypted it. ' \
|
|
341
|
+
'A retry re-seals the argument as it stands: retry only a job known to hold plaintext'
|
|
342
|
+
|
|
329
343
|
def call(_worker, job, _queue)
|
|
330
344
|
return yield unless Wurk::Encryption.enabled? && job['encrypt']
|
|
331
345
|
|
|
@@ -342,7 +356,8 @@ module Wurk
|
|
|
342
356
|
|
|
343
357
|
def decrypt_last_arg!(job)
|
|
344
358
|
args = job['args']
|
|
345
|
-
return unless args.is_a?(::Array) && !args.empty?
|
|
359
|
+
return unless args.is_a?(::Array) && !args.empty?
|
|
360
|
+
raise NotSealedError, NOT_SEALED_MESSAGE unless Wurk::Encryption.envelope?(args.last)
|
|
346
361
|
|
|
347
362
|
job['args'] = args[0..-2] + [Wurk::Encryption.decrypt(args.last)]
|
|
348
363
|
rescue Wurk::Encryption::Error,
|
|
@@ -350,7 +365,7 @@ module Wurk
|
|
|
350
365
|
::ArgumentError,
|
|
351
366
|
::TypeError,
|
|
352
367
|
::JSON::ParserError => e
|
|
353
|
-
# Wurk::Encryption::Error covers KeyMissingError / DecryptionError.
|
|
368
|
+
# Wurk::Encryption::Error covers KeyMissingError / DecryptionError / NotSealedError.
|
|
354
369
|
# OpenSSL::Cipher::CipherError → tampered ciphertext / bad key (tag mismatch).
|
|
355
370
|
# ArgumentError → Base64.strict_decode64 / Integer() on a malformed envelope.
|
|
356
371
|
# TypeError → Integer(nil) on a missing 'v' field.
|
data/lib/wurk/fetcher/capped.rb
CHANGED
|
@@ -99,7 +99,7 @@ module Wurk
|
|
|
99
99
|
gate = @gates[keys[1]]
|
|
100
100
|
next if gate&.blocked?(now)
|
|
101
101
|
|
|
102
|
-
uow = gate ? claim(public_q, keys, gate, ack
|
|
102
|
+
uow = gate ? claim(public_q, keys, gate, ack) : lmove(public_q, ack)
|
|
103
103
|
ack = nil
|
|
104
104
|
return uow if uow
|
|
105
105
|
end
|
|
@@ -124,7 +124,7 @@ module Wurk
|
|
|
124
124
|
# is deferred: an ACK handed back by a failed flush must not be able to
|
|
125
125
|
# take the hold of whatever this thread is running by then. See
|
|
126
126
|
# QueueSlot.claim_token.
|
|
127
|
-
def claim(public_q, keys, gate, ack
|
|
127
|
+
def claim(public_q, keys, gate, ack)
|
|
128
128
|
priv, name = keys
|
|
129
129
|
token = QueueSlot.claim_token
|
|
130
130
|
script_keys = [gate.slot_key, public_q, priv]
|
|
@@ -137,7 +137,12 @@ module Wurk
|
|
|
137
137
|
# faster than a job finishes somewhere in the cluster, so re-asking on
|
|
138
138
|
# every pass is how a full queue turns into a spin. An empty queue is
|
|
139
139
|
# remembered by nobody, exactly as on the uncapped path.
|
|
140
|
-
|
|
140
|
+
#
|
|
141
|
+
# Stamped with the time the refusal ARRIVED, not the time the question
|
|
142
|
+
# was sent: a slow Redis would otherwise eat the backoff window and
|
|
143
|
+
# the gate would already be expired when the refusal lands — which is
|
|
144
|
+
# the spin this remembers it to avoid.
|
|
145
|
+
gate.block!(::Process.clock_gettime(::Process::CLOCK_MONOTONIC)) if status == AT_CAPACITY
|
|
141
146
|
nil
|
|
142
147
|
rescue StandardError
|
|
143
148
|
# The window #lmove guards: the ACK left its slot but never reached
|
data/lib/wurk/launcher.rb
CHANGED
|
@@ -99,10 +99,12 @@ module Wurk
|
|
|
99
99
|
# 4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
|
|
100
100
|
# not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
|
|
101
101
|
# so it stays ahead of the periodic loops.
|
|
102
|
-
# 5.
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
102
|
+
# 5. write the `config.periodic` registrations (Ent registers loops on
|
|
103
|
+
# startup), then start the periodic loops. None of them ticks at
|
|
104
|
+
# zero — TimerLoop#run waits an interval before its first yield,
|
|
105
|
+
# Scheduled::Poller waits INITIAL_WAIT, and Leader waits
|
|
106
|
+
# DEFAULT_INITIAL_WAIT — so starting them last costs a few thread
|
|
107
|
+
# spawns and nothing else. Both leader-
|
|
106
108
|
# gated pollers are also safe to start before leadership settles: a
|
|
107
109
|
# non-leader tick just returns early.
|
|
108
110
|
# 6. start the health probe server LAST so the listener doesn't
|
|
@@ -240,6 +242,7 @@ module Wurk
|
|
|
240
242
|
# Run on a background thread so /ready probe isn't delayed by a large
|
|
241
243
|
# orphan sweep (the reaper's reclaim is atomic, but can scan many entries).
|
|
242
244
|
@boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
|
|
245
|
+
@config.apply_periodic!
|
|
243
246
|
start_periodic_components
|
|
244
247
|
end
|
|
245
248
|
@health_server&.start
|
data/lib/wurk/leader.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'securerandom'
|
|
4
4
|
require 'socket'
|
|
5
5
|
require_relative 'component'
|
|
6
|
+
require_relative 'fetcher/oom_drain'
|
|
6
7
|
require_relative 'lua'
|
|
7
8
|
require_relative 'pool_checkout'
|
|
8
9
|
require_relative 'timer_loop'
|
|
@@ -21,8 +22,12 @@ module Wurk
|
|
|
21
22
|
# *can* benefit from a guard has one available; the token is best-effort
|
|
22
23
|
# too — it is never re-read on subsequent acquires, only on transitions.
|
|
23
24
|
#
|
|
24
|
-
# Cadence per spec: renew every 15s while leader,
|
|
25
|
-
#
|
|
25
|
+
# Cadence per spec: renew every 15s while leader, lock TTL 30s. A follower
|
|
26
|
+
# rechecks when the lease it last read runs out, at most 60s later, and that
|
|
27
|
+
# is what bounds the leaderless time after a SIGKILL: the lease's remaining
|
|
28
|
+
# 15-30s and about a second. Waiting out a flat 60s added up to 60s more. The
|
|
29
|
+
# lock itself is untouched, so a follower still cannot win before the lease
|
|
30
|
+
# has lapsed. The campaign itself starts `initial_wait` seconds
|
|
26
31
|
# after `#start` — the spec fixes the renew/follower/TTL cadence but says
|
|
27
32
|
# nothing about when the first campaign runs, and holding it back keeps a
|
|
28
33
|
# booting process's opening Redis round trips on the job path. Opt out a
|
|
@@ -43,6 +48,17 @@ module Wurk
|
|
|
43
48
|
# nothing observes the gap. Tune with `config[:leader_initial_wait]`;
|
|
44
49
|
# 0 campaigns immediately.
|
|
45
50
|
DEFAULT_INITIAL_WAIT = 1
|
|
51
|
+
# Followers spread out over this many seconds past the lease's end, so a
|
|
52
|
+
# large fleet doesn't campaign in one instant.
|
|
53
|
+
LEASE_EXPIRY_JITTER = 1.0
|
|
54
|
+
# Keeps a lease read as about to lapse from turning into a busy loop.
|
|
55
|
+
MIN_FOLLOWER_WAIT = 0.05
|
|
56
|
+
# The renewal has to survive a full Redis: a lapsed lease stops every
|
|
57
|
+
# leader-gated consumer, and nothing would renew it until memory came back.
|
|
58
|
+
# Shebang scripts, so by EVAL for the reason Fetcher::OomDrain gives. A server
|
|
59
|
+
# refusing one raises the original OOM, the failure without this.
|
|
60
|
+
OOM_CAMPAIGN = "#{Fetcher::OomDrain::HEADER}#{Lua::SCRIPTS.fetch(:leader_campaign)}".freeze
|
|
61
|
+
OOM_TOKEN = "#{Fetcher::OomDrain::HEADER}return redis.call('INCR', KEYS[1])\n".freeze
|
|
46
62
|
OPT_OUT_ENV = 'WURK_LEADER' # native opt-out env
|
|
47
63
|
SIDEKIQ_OPT_OUT_ENV = 'SIDEKIQ_LEADER' # Sidekiq Ent drop-in alias (§6.2/§7.2)
|
|
48
64
|
THREAD_NAME = 'wurk-leader'
|
|
@@ -71,6 +87,8 @@ module Wurk
|
|
|
71
87
|
@owner = owner || cluster_identity
|
|
72
88
|
@held = false
|
|
73
89
|
@token = nil
|
|
90
|
+
@lease_remaining = nil
|
|
91
|
+
@campaign_failed = false
|
|
74
92
|
@thread = nil
|
|
75
93
|
@done = false
|
|
76
94
|
@mutex = ::Mutex.new
|
|
@@ -98,7 +116,7 @@ module Wurk
|
|
|
98
116
|
return false unless transition
|
|
99
117
|
|
|
100
118
|
if transition == :gained
|
|
101
|
-
@token = redis_call { |c| c
|
|
119
|
+
@token = redis_call { |c| next_token(c) }
|
|
102
120
|
dispatch_leader_event
|
|
103
121
|
end
|
|
104
122
|
true
|
|
@@ -191,9 +209,7 @@ module Wurk
|
|
|
191
209
|
# if another process currently holds the key. Sets `@held` accordingly
|
|
192
210
|
# so `leader?` reflects the latest state regardless of branch.
|
|
193
211
|
def run_set_or_refresh
|
|
194
|
-
won = redis_call
|
|
195
|
-
Wurk::Lua::Loader.eval_cached(c, :leader_campaign, keys: [@key], argv: [@owner, @ttl])
|
|
196
|
-
end
|
|
212
|
+
won = redis_call { |c| campaign(c) }
|
|
197
213
|
if won.to_i == 1
|
|
198
214
|
outcome = @held ? :held : :gained
|
|
199
215
|
@held = true
|
|
@@ -201,9 +217,37 @@ module Wurk
|
|
|
201
217
|
end
|
|
202
218
|
|
|
203
219
|
step_down
|
|
220
|
+
read_lease_remaining
|
|
204
221
|
nil
|
|
205
222
|
end
|
|
206
223
|
|
|
224
|
+
def campaign(conn)
|
|
225
|
+
Wurk::Lua::Loader.eval_cached(conn, :leader_campaign, keys: [@key], argv: [@owner, @ttl])
|
|
226
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
227
|
+
past_maxmemory(e) { conn.call('EVAL', OOM_CAMPAIGN, 1, @key, @owner, @ttl) }
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def next_token(conn)
|
|
231
|
+
conn.call('INCR', TOKEN_KEY)
|
|
232
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
233
|
+
past_maxmemory(e) { conn.call('EVAL', OOM_TOKEN, 1, TOKEN_KEY) }
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def past_maxmemory(error)
|
|
237
|
+
yield
|
|
238
|
+
rescue RedisClient::ConnectionError
|
|
239
|
+
raise
|
|
240
|
+
rescue RedisClient::CommandError
|
|
241
|
+
raise error
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Seconds until the holder's lease runs out; nil for a lock with no TTL, which
|
|
245
|
+
# only expiry cannot free. A lock gone again already is a zero wait.
|
|
246
|
+
def read_lease_remaining
|
|
247
|
+
pttl = redis_call(idempotent: true) { |c| c.call('PTTL', @key) }
|
|
248
|
+
@lease_remaining = pttl == -1 ? nil : [pttl, 0].max / 1000.0
|
|
249
|
+
end
|
|
250
|
+
|
|
207
251
|
def step_down
|
|
208
252
|
@held = false
|
|
209
253
|
@token = nil
|
|
@@ -283,10 +327,14 @@ module Wurk
|
|
|
283
327
|
# would leave every leader-gated consumer acting as leader through an
|
|
284
328
|
# outage long enough for a follower to take over, and would keep this loop
|
|
285
329
|
# on the short renew cadence. Stepping down makes the next success a fresh
|
|
286
|
-
# gain — a new fencing token and a re-fired `:leader` event.
|
|
330
|
+
# gain — a new fencing token and a re-fired `:leader` event. It says nothing
|
|
331
|
+
# about the lease, so the retry is on the renew cadence, not the follower's:
|
|
332
|
+
# a leader that failed one renewal still has time to keep its lease.
|
|
287
333
|
def tick_once
|
|
334
|
+
@campaign_failed = false
|
|
288
335
|
acquire
|
|
289
336
|
rescue StandardError => e
|
|
337
|
+
@campaign_failed = true
|
|
290
338
|
step_down
|
|
291
339
|
report(e)
|
|
292
340
|
end
|
|
@@ -296,7 +344,13 @@ module Wurk
|
|
|
296
344
|
end
|
|
297
345
|
|
|
298
346
|
def wait_next
|
|
299
|
-
wait_for(@held ? @renew_interval :
|
|
347
|
+
wait_for(@held || @campaign_failed ? @renew_interval : follower_wait)
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
def follower_wait
|
|
351
|
+
return @follower_interval unless @lease_remaining
|
|
352
|
+
|
|
353
|
+
[@lease_remaining + (rand * LEASE_EXPIRY_JITTER), @follower_interval].min.clamp(MIN_FOLLOWER_WAIT, nil)
|
|
300
354
|
end
|
|
301
355
|
|
|
302
356
|
def wait_for(interval)
|