wurk 1.11.1 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/config/routes.rb +8 -0
- data/exe/sidekiq +8 -0
- data/exe/wurk +4 -2
- data/lib/generators/wurk/install/install_generator.rb +56 -0
- data/lib/generators/wurk/install/templates/wurk.rb +5 -3
- data/lib/wurk/boot_checks.rb +210 -5
- data/lib/wurk/cli.rb +110 -10
- data/lib/wurk/configuration.rb +28 -7
- data/lib/wurk/fetcher/oom_drain.rb +0 -5
- data/lib/wurk/fetcher/orphan_reclaim.rb +108 -0
- data/lib/wurk/{middleware → fetcher}/poison_pill.rb +33 -47
- data/lib/wurk/fetcher/reaper.rb +46 -65
- data/lib/wurk/fetcher/reliable.rb +4 -5
- data/lib/wurk/fetcher/self_reconcile.rb +8 -14
- data/lib/wurk/fetcher/unit_of_work.rb +1 -1
- data/lib/wurk/job_logger.rb +4 -2
- data/lib/wurk/job_util.rb +32 -2
- data/lib/wurk/launcher.rb +30 -3
- data/lib/wurk/leader.rb +1 -8
- data/lib/wurk/lua/reclaim_orphan.lua +31 -0
- data/lib/wurk/lua.rb +5 -20
- data/lib/wurk/manager.rb +27 -1
- data/lib/wurk/middleware/timeout.rb +6 -6
- data/lib/wurk/preflight/app_scan.rb +182 -0
- data/lib/wurk/preflight/command.rb +21 -5
- data/lib/wurk/preflight/redis_guard.rb +98 -0
- data/lib/wurk/preflight.rb +9 -5
- data/lib/wurk/rails_boot.rb +45 -11
- data/lib/wurk/redis_options.rb +5 -0
- data/lib/wurk/redis_pool.rb +24 -1
- data/lib/wurk/sentry.rb +11 -0
- data/lib/wurk/swarm.rb +63 -14
- data/lib/wurk/unique.rb +12 -2
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/rack_app.rb +8 -6
- data/lib/wurk/web/stats_routes.rb +93 -0
- data/lib/wurk.rb +5 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-CmaXArv8.js → ArgsValue-BJol9S0s.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-nc5yePZW.js → BatchDetail-D_zGQ9Sd.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-DgEcyOZ_.js → Batches-Dt_xC660.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-CUZncvCy.js → Busy-CWRthsmc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CIZbaaZs.js → Cron-qeGr2YAm.js} +1 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +1 -0
- data/vendor/assets/dashboard/assets/{Extension-Doy4bJoA.js → Extension-DONXVLdA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-C4bO26_d.js → FilterBox-BbB6gXHp.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-Dh3SJdFh.js → FlowDetail-CrmnVRp0.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-C5SMRuhq.js → Flows-g4GC8KmE.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cg6sAsJB.js → JobDetailModal-C86k_1wj.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-BIxwDOJM.js → Limiters-zyO-iMlK.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-Qp9q0nF8.js → Metrics-Ckq5fNBv.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-DwjPxFjU.js → NotFound-C8NnyjlY.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-7dabYVe3.js → Profiles-DRl-FXPA.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +1 -0
- data/vendor/assets/dashboard/assets/{Search-BAsVE2qw.js → Search-LaEpj2fM.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-DZs138BV.js → charts-D_PQLoyc.js} +1 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +141 -0
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-DxkLZCIm.js → useResetPageOnEmpty-BiwhHqgh.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-B0kQFNE8.js → utils-CnWZzW2L.js} +1 -1
- data/vendor/assets/dashboard/index.html +1 -1
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +37 -26
- data/vendor/assets/dashboard/assets/Dashboard-BWmfAook.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-CGKsTZHq.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CNX8ZtTH.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-ClVBj0nc.js +0 -1
- data/vendor/assets/dashboard/assets/index-CMzPFxNE.js +0 -141
data/lib/wurk/configuration.rb
CHANGED
|
@@ -452,7 +452,7 @@ module Wurk
|
|
|
452
452
|
# The optional block is Pro's recovery callback: `|jobstr, pill|`, fired
|
|
453
453
|
# once per orphan recovery (`pill` nil) and once on a poison kill (`pill`
|
|
454
454
|
# responds to .jid/.klass/.count/.queue). The reaper drives it via
|
|
455
|
-
# Wurk::Middleware::PoisonPill.
|
|
455
|
+
# Wurk::Middleware::PoisonPill.recovered!. Spec: docs/target/sidekiq-pro.md §3.1.
|
|
456
456
|
def super_fetch!(*, &block)
|
|
457
457
|
@super_fetch_callback = block if block
|
|
458
458
|
nil
|
|
@@ -877,7 +877,7 @@ module Wurk
|
|
|
877
877
|
# and the like silently absent from a process that does run workers.
|
|
878
878
|
def configure_server(&block)
|
|
879
879
|
return unless block
|
|
880
|
-
return
|
|
880
|
+
return run_server_block(block) if server?
|
|
881
881
|
|
|
882
882
|
@deferred_server_blocks << block
|
|
883
883
|
return if @configure_server_skip_warned
|
|
@@ -889,23 +889,34 @@ module Wurk
|
|
|
889
889
|
|
|
890
890
|
def enter_server_mode
|
|
891
891
|
self[:server] = true
|
|
892
|
-
@deferred_server_blocks.shift
|
|
892
|
+
run_server_block(@deferred_server_blocks.shift) until @deferred_server_blocks.empty?
|
|
893
893
|
end
|
|
894
894
|
|
|
895
895
|
# A web process that also boots the swarm (Wurk::RailsBoot) is flagged
|
|
896
896
|
# server, yet every enqueue it makes from a request is client work, so
|
|
897
|
-
# `client_in_server` lets its configure_client blocks run too.
|
|
898
|
-
#
|
|
897
|
+
# `client_in_server` lets its configure_client blocks run too.
|
|
898
|
+
#
|
|
899
|
+
# In a server, skipping the client block is the Sidekiq contract and the
|
|
900
|
+
# usual case: an app configures both roles. So the skip itself is DEBUG,
|
|
901
|
+
# and BootChecks warns at boot only for the case worth one, a server
|
|
902
|
+
# process in which no configure_server block ran at all (#816).
|
|
899
903
|
def configure_client
|
|
900
904
|
return unless block_given?
|
|
901
905
|
return yield self if !server? || @options[:client_in_server]
|
|
906
|
+
|
|
902
907
|
return if @configure_client_skip_warned
|
|
903
908
|
|
|
904
909
|
@configure_client_skip_warned = true
|
|
905
|
-
logger.
|
|
910
|
+
logger.debug { 'configure_client block skipped: server? is true and client_in_server is false or nil' }
|
|
906
911
|
nil
|
|
907
912
|
end
|
|
908
913
|
|
|
914
|
+
# True once a configure_client block was skipped and no configure_server
|
|
915
|
+
# block ran: the server got none of the settings the app made for a role.
|
|
916
|
+
def client_config_only?
|
|
917
|
+
@configure_client_skip_warned == true && !@configure_server_ran
|
|
918
|
+
end
|
|
919
|
+
|
|
909
920
|
def server?
|
|
910
921
|
@options[:server] == true
|
|
911
922
|
end
|
|
@@ -920,9 +931,14 @@ module Wurk
|
|
|
920
931
|
|
|
921
932
|
def topology=(value)
|
|
922
933
|
guard_frozen!
|
|
934
|
+
@topology_assigned = true
|
|
923
935
|
@topology = value
|
|
924
936
|
end
|
|
925
937
|
|
|
938
|
+
# Whether the app declared a topology, as opposed to the default one
|
|
939
|
+
# #topology builds. Only a swarm reads it.
|
|
940
|
+
def topology_assigned? = @topology_assigned == true
|
|
941
|
+
|
|
926
942
|
# Memory-based child recycling (Sidekiq Ent §7.5): the swarm parent TERMs
|
|
927
943
|
# (and respawns) any child whose RSS exceeds this many MB. Set in code or
|
|
928
944
|
# via SIDEKIQ_MAXMEM_MB (WURK_MAXMEM_MB is the native alias); an explicit
|
|
@@ -1001,6 +1017,11 @@ module Wurk
|
|
|
1001
1017
|
|
|
1002
1018
|
private
|
|
1003
1019
|
|
|
1020
|
+
def run_server_block(block)
|
|
1021
|
+
@configure_server_ran = true
|
|
1022
|
+
block.call(self)
|
|
1023
|
+
end
|
|
1024
|
+
|
|
1004
1025
|
# Pools read the hash once, when built, so the later assignment silently does
|
|
1005
1026
|
# nothing to this process's connections until something rebuilds them.
|
|
1006
1027
|
def warn_on_late_redis_assignment
|
|
@@ -1079,7 +1100,7 @@ module Wurk
|
|
|
1079
1100
|
# as throttling or memory pressure.
|
|
1080
1101
|
def default_child_count
|
|
1081
1102
|
count, source = child_count_and_source
|
|
1082
|
-
logger.info { "Wurk swarm:
|
|
1103
|
+
logger.info { "Wurk swarm size: #{count} #{count == 1 ? 'child' : 'children'} (#{source})" }
|
|
1083
1104
|
count
|
|
1084
1105
|
end
|
|
1085
1106
|
|
|
@@ -50,7 +50,6 @@ module Wurk
|
|
|
50
50
|
return nil
|
|
51
51
|
LUA
|
|
52
52
|
|
|
53
|
-
RECLAIM = (HEADER + Lua::RELIABLE_RECLAIM).freeze
|
|
54
53
|
REQUEUE = (HEADER + Lua::RELIABLE_REQUEUE).freeze
|
|
55
54
|
|
|
56
55
|
# KEYS = [lock], ARGV = [ttl_ms]. The reaper's sweep lock: refused, no sweep
|
|
@@ -72,10 +71,6 @@ module Wurk
|
|
|
72
71
|
run(conn, error, CLAIM, [Keys::PROCESSES, Component.identity, *keys], [info, beat, HEARTBEAT_TTL])
|
|
73
72
|
end
|
|
74
73
|
|
|
75
|
-
def reclaim(conn, error, source, destination)
|
|
76
|
-
run(conn, error, RECLAIM, [source, destination], [])
|
|
77
|
-
end
|
|
78
|
-
|
|
79
74
|
# Returns 1 when moved, 0 when the job has left the private list since.
|
|
80
75
|
def requeue(conn, error, private_list, public_q, job)
|
|
81
76
|
run(conn, error, REQUEUE, [private_list, public_q], [job])
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../keys'
|
|
4
|
+
require_relative '../lua'
|
|
5
|
+
require_relative 'poison_pill'
|
|
6
|
+
require_relative '../metrics/process_counters'
|
|
7
|
+
require_relative 'oom_drain'
|
|
8
|
+
|
|
9
|
+
module Wurk
|
|
10
|
+
class Fetcher
|
|
11
|
+
# Puts one payload from a private list back where it belongs, for both ways
|
|
12
|
+
# an orphan is found: the Reaper draining a dead owner's list, and
|
|
13
|
+
# SelfReconcile returning a payload its own live process lost track of.
|
|
14
|
+
#
|
|
15
|
+
# The poison-pill verdict is taken in the same script that moves the job
|
|
16
|
+
# (`lua/reclaim_orphan.lua`), so a job past the cap goes straight from the
|
|
17
|
+
# private list to the dead set and is never visible on its public queue. The
|
|
18
|
+
# notifications (death handlers, `super_fetch!`, `on_poison`) follow once
|
|
19
|
+
# the script has returned, for a job no worker can be running.
|
|
20
|
+
class OrphanReclaim
|
|
21
|
+
THRESHOLD = Middleware::PoisonPill::RECOVERY_THRESHOLD
|
|
22
|
+
TTL = Middleware::PoisonPill::RECOVERY_TTL
|
|
23
|
+
|
|
24
|
+
def initialize(config)
|
|
25
|
+
@config = config
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Returns true when this call took the payload off the private list, false
|
|
29
|
+
# when it had already left (another reaper, an ACK).
|
|
30
|
+
#
|
|
31
|
+
# Sent once, never re-sent on a fresh socket: a replay of a script that
|
|
32
|
+
# already ran finds the payload gone and answers "someone else moved it",
|
|
33
|
+
# which skipped the death handlers of a kill (#908), and a list holding two
|
|
34
|
+
# identical copies would move the second. A reply lost after the script ran
|
|
35
|
+
# is settled by reading back what it wrote instead (#settle_lost_reply).
|
|
36
|
+
def call(private_list, public_q, payload)
|
|
37
|
+
job = Middleware::PoisonPill.parse(payload)
|
|
38
|
+
keys = move_keys(private_list, public_q, job)
|
|
39
|
+
count = moved_count(keys, payload, now)
|
|
40
|
+
return false unless count
|
|
41
|
+
|
|
42
|
+
Metrics::ProcessCounters.increment(:reaper_reclaimed)
|
|
43
|
+
unless count == :uncounted
|
|
44
|
+
Middleware::PoisonPill.recovered!(payload, job, count: count, queue: queue_name(public_q),
|
|
45
|
+
config: @config)
|
|
46
|
+
end
|
|
47
|
+
true
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def moved_count(keys, payload, score)
|
|
53
|
+
@config.redis { |conn| move(conn, keys, payload, score) }
|
|
54
|
+
rescue RedisClient::ConnectionError => e
|
|
55
|
+
settle_lost_reply(keys, payload, score, e)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def move_keys(private_list, public_q, job)
|
|
59
|
+
keys = [private_list, public_q, Keys::DEAD]
|
|
60
|
+
jid = job && job['jid']
|
|
61
|
+
keys << Middleware::PoisonPill.counter_key(jid) if jid.is_a?(String) && !jid.empty?
|
|
62
|
+
keys
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Past maxmemory the counter cannot be written, nor the job killed: it goes
|
|
66
|
+
# back uncounted rather than waiting on the memory it would help free.
|
|
67
|
+
def move(conn, keys, payload, score)
|
|
68
|
+
Wurk::Lua::Loader.eval_once(conn, :reclaim_orphan, keys: keys, argv: [payload, THRESHOLD, TTL, score])
|
|
69
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
70
|
+
:uncounted if OomDrain.requeue(conn, e, keys[0], keys[1], payload) == 1
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# The connection dropped around the script, so it ran or it did not. Only a
|
|
74
|
+
# kill needs settling: the job is gone from the private list, so no later
|
|
75
|
+
# sweep would find it, and the death handlers (a batch's `:death`, a unique
|
|
76
|
+
# lock's release) would never fire. The script wrote the dead-set member
|
|
77
|
+
# at the score this call chose, so that exact score proves the kill was
|
|
78
|
+
# this call's. Returns the recovery count for a kill this call made; nil
|
|
79
|
+
# for a requeue or a move someone else made, which need nothing more;
|
|
80
|
+
# raises the original error when the payload is still in the private list,
|
|
81
|
+
# which is the next pass's to move.
|
|
82
|
+
def settle_lost_reply(keys, payload, score, error)
|
|
83
|
+
private_list, _public_q, dead, counter = keys
|
|
84
|
+
still_there, dead_score, count = @config.redis(idempotent: true) do |conn|
|
|
85
|
+
conn.pipelined do |pipe|
|
|
86
|
+
pipe.call('LPOS', private_list, payload, 'RANK', -1)
|
|
87
|
+
pipe.call('ZSCORE', dead, payload)
|
|
88
|
+
pipe.call('GET', counter) if counter
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
# Exact on purpose: the same double, written by the script and read back.
|
|
92
|
+
return count.to_i if dead_score && Float(dead_score) == Float(score) # rubocop:disable Lint/FloatComparison
|
|
93
|
+
raise error if still_there
|
|
94
|
+
|
|
95
|
+
nil
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# DeadSet#kill's score format, so the entry sorts with every other kill.
|
|
99
|
+
def now
|
|
100
|
+
::Process.clock_gettime(::Process::CLOCK_REALTIME).to_s
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def queue_name(public_q)
|
|
104
|
+
public_q.delete_prefix(Keys::QUEUE_PREFIX)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -9,19 +9,22 @@ require_relative '../dead_set'
|
|
|
9
9
|
module Wurk
|
|
10
10
|
module Middleware
|
|
11
11
|
# Pro parity (§3.2): poison-pill detection for reliable-fetch orphans.
|
|
12
|
-
# When a job is recovered out of a
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
# `jobs.poison` is emitted to statsd, and recovery callbacks fire so
|
|
12
|
+
# When a job is recovered out of a private list, a per-jid counter at
|
|
13
|
+
# `super_fetch:recovered:<jid>` goes up by one with a 72h TTL. Recoveries up
|
|
14
|
+
# to RECOVERY_THRESHOLD (3) requeue; the one that takes the counter past it
|
|
15
|
+
# (the 4th orphaning) is the poison pill: the payload is moved to the dead
|
|
16
|
+
# set, `jobs.poison` is emitted to statsd, and recovery callbacks fire so
|
|
17
17
|
# operators can be paged.
|
|
18
18
|
#
|
|
19
19
|
# Counter key TTL is wire-compat with Sidekiq Pro — third-party tooling
|
|
20
20
|
# that watches `super_fetch:recovered:*` expects 72h.
|
|
21
21
|
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
22
|
+
# Not a chain middleware, so the file lives with the fetcher; the constant
|
|
23
|
+
# keeps its Middleware path because Pro initializers name it. No
|
|
24
|
+
# server-middleware registration: the producer is Fetcher::OrphanReclaim,
|
|
25
|
+
# whose script bumps the counter and takes the requeue-or-kill decision in
|
|
26
|
+
# the same step as the move, then hands the outcome to {recovered!} for the
|
|
27
|
+
# notifications. The consumer is the ACK —
|
|
25
28
|
# `Fetcher::Reliable::UnitOfWork#acknowledge` pipelines {clear_in} next to
|
|
26
29
|
# its LREM, so an attempt that finished starts the next one at zero
|
|
27
30
|
# without costing a round trip of its own.
|
|
@@ -44,37 +47,34 @@ module Wurk
|
|
|
44
47
|
|
|
45
48
|
module_function
|
|
46
49
|
|
|
47
|
-
# Called per recovered orphan job
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
50
|
+
# Called per recovered orphan job, once Fetcher::OrphanReclaim has moved
|
|
51
|
+
# it: `count` is the recovery counter its script wrote (0 for a payload with
|
|
52
|
+
# no jid). Returns `:poison` when the count is past the threshold — the
|
|
53
|
+
# script already put the job in the dead set, never on its queue — and
|
|
54
|
+
# `:recovered` otherwise. Emits `jobs.recovered.fetch` for every parseable
|
|
55
|
+
# job, `jobs.poison` only on the kill path.
|
|
52
56
|
#
|
|
53
57
|
# Fires the Pro `super_fetch!` recovery callback (config.super_fetch_callback)
|
|
54
58
|
# exactly once per call: `(jobstr, nil)` on plain recovery, `(jobstr, pill)`
|
|
55
59
|
# on the kill path. The poison-only `on_poison` Hash callbacks fire
|
|
56
60
|
# independently inside #mark_poison.
|
|
57
61
|
#
|
|
58
|
-
# @param payload [String
|
|
62
|
+
# @param payload [String] the job JSON as it sat in the private list.
|
|
63
|
+
# @param job [Hash, nil] the payload parsed by {parse}.
|
|
64
|
+
# @param count [Integer] the recovery count after this recovery.
|
|
59
65
|
# @param queue [String, nil] the public queue name (without `queue:`).
|
|
60
|
-
# @param config [Configuration] config that owns the super_fetch! callback
|
|
61
|
-
# defaults to the global so non-reaper callers (tests) need not pass it.
|
|
66
|
+
# @param config [Configuration] config that owns the super_fetch! callback.
|
|
62
67
|
# @return [Symbol] :recovered | :poison
|
|
63
|
-
def
|
|
64
|
-
job = parse(payload)
|
|
68
|
+
def recovered!(payload, job, count:, queue: nil, config: Wurk.configuration)
|
|
65
69
|
unless job
|
|
66
70
|
fire_super_fetch(config, payload, nil)
|
|
67
71
|
return :recovered
|
|
68
72
|
end
|
|
69
73
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
emit('jobs.recovered.fetch', klass, queue)
|
|
73
|
-
|
|
74
|
-
count = bump_counter(jid) if jid && !jid.empty?
|
|
75
|
-
if count && count > RECOVERY_THRESHOLD
|
|
74
|
+
emit('jobs.recovered.fetch', job['class'], queue)
|
|
75
|
+
if count > RECOVERY_THRESHOLD
|
|
76
76
|
mark_poison(payload, job, queue: queue, count: count)
|
|
77
|
-
fire_super_fetch(config, payload, Pill.new(jid: jid, klass:
|
|
77
|
+
fire_super_fetch(config, payload, Pill.new(jid: job['jid'], klass: job['class'], count: count, queue: queue))
|
|
78
78
|
:poison
|
|
79
79
|
else
|
|
80
80
|
fire_super_fetch(config, payload, nil)
|
|
@@ -168,29 +168,14 @@ module Wurk
|
|
|
168
168
|
|
|
169
169
|
# ---- internals --------------------------------------------------
|
|
170
170
|
|
|
171
|
+
# The job Hash, or nil for anything that is not a JSON object.
|
|
171
172
|
def parse(payload)
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
Wurk.load_json(payload)
|
|
173
|
+
job = Wurk.load_json(payload)
|
|
174
|
+
job if job.is_a?(::Hash)
|
|
176
175
|
rescue ::JSON::ParserError
|
|
177
176
|
nil
|
|
178
177
|
end
|
|
179
178
|
|
|
180
|
-
# No apply-safety claim: INCR is additive, so a block replayed after a
|
|
181
|
-
# lost reply bumps twice and can carry a healthy job past
|
|
182
|
-
# RECOVERY_THRESHOLD into the dead set. Raising instead only leaves the
|
|
183
|
-
# count short — Reaper#drain rescues, and the job is already back on its
|
|
184
|
-
# public queue, so it just misses one poison check.
|
|
185
|
-
def bump_counter(jid)
|
|
186
|
-
key = counter_key(jid)
|
|
187
|
-
Wurk.redis do |conn|
|
|
188
|
-
count = conn.call('INCR', key).to_i
|
|
189
|
-
conn.call('EXPIRE', key, RECOVERY_TTL)
|
|
190
|
-
count
|
|
191
|
-
end
|
|
192
|
-
end
|
|
193
|
-
|
|
194
179
|
def emit(metric, klass, queue)
|
|
195
180
|
tags = []
|
|
196
181
|
tags << "class:#{klass}" if klass
|
|
@@ -198,6 +183,9 @@ module Wurk
|
|
|
198
183
|
Wurk::Metrics::Statsd.increment(metric, tags: tags.empty? ? nil : tags)
|
|
199
184
|
end
|
|
200
185
|
|
|
186
|
+
# The script already put `payload` in the dead set; DeadSet#kill writes the
|
|
187
|
+
# same member again, so the trim and the death handlers stay DeadSet's.
|
|
188
|
+
#
|
|
201
189
|
# Death handlers fire (`notify_failure` defaults to true). A poison kill
|
|
202
190
|
# is a death like any other exhaustion: suppressing it strands every
|
|
203
191
|
# batch that owns the job — Batch::DeathHandler is a death handler, so a
|
|
@@ -206,8 +194,7 @@ module Wurk
|
|
|
206
194
|
# (docs/target/sidekiq-pro.md §12); recorded in docs/idea/parity-divergences.md.
|
|
207
195
|
def mark_poison(payload, job, queue:, count:)
|
|
208
196
|
emit('jobs.poison', job['class'], queue)
|
|
209
|
-
|
|
210
|
-
Wurk::DeadSet.new.kill(json, ex: poisoned_error(job['class'], count))
|
|
197
|
+
Wurk::DeadSet.new.kill(payload, ex: poisoned_error(job['class'], count))
|
|
211
198
|
fire_callbacks(jid: job['jid'], klass: job['class'], count: count, queue: queue)
|
|
212
199
|
end
|
|
213
200
|
|
|
@@ -231,8 +218,7 @@ module Wurk
|
|
|
231
218
|
cb = config.super_fetch_callback
|
|
232
219
|
return unless cb
|
|
233
220
|
|
|
234
|
-
|
|
235
|
-
cb.call(jobstr, pill)
|
|
221
|
+
cb.call(payload, pill)
|
|
236
222
|
rescue StandardError => e
|
|
237
223
|
config.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
|
|
238
224
|
end
|
data/lib/wurk/fetcher/reaper.rb
CHANGED
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative '../component'
|
|
4
4
|
require_relative '../keys'
|
|
5
|
-
require_relative '../middleware/poison_pill'
|
|
6
|
-
require_relative '../metrics/process_counters'
|
|
7
5
|
require_relative '../timer_loop'
|
|
8
6
|
require_relative 'private_list_key'
|
|
9
7
|
require_relative 'unparseable_keys'
|
|
10
8
|
require_relative 'orphan_grace'
|
|
11
9
|
require_relative 'oom_drain'
|
|
10
|
+
require_relative 'orphan_reclaim'
|
|
12
11
|
require_relative 'self_reconcile'
|
|
12
|
+
require_relative 'reliable'
|
|
13
13
|
|
|
14
14
|
module Wurk
|
|
15
15
|
class Fetcher
|
|
@@ -29,9 +29,10 @@ module Wurk
|
|
|
29
29
|
# ignores a stale `processes` SET member — the SET carries no TTL, so
|
|
30
30
|
# the member lingers after its heartbeat hash expires (60s) until
|
|
31
31
|
# `ProcessSet#cleanup` prunes it. A `kill -9`ed sibling is reclaimed
|
|
32
|
-
# the moment the supervisor reaps it, not 60s later.
|
|
33
|
-
# unrelated process
|
|
34
|
-
#
|
|
32
|
+
# the moment the supervisor reaps it, not 60s later. Pid reuse by an
|
|
33
|
+
# unrelated process reads a dead owner as live; a swarm child does not
|
|
34
|
+
# depend on this check, since its parent drains its lists by exact key
|
|
35
|
+
# the moment it reaps it (#reclaim_exited), before the pid can be reused.
|
|
35
36
|
# * any other incarnation — its pid means nothing in our namespace, so we
|
|
36
37
|
# trust the heartbeat: the owner is alive iff its identity's `info`
|
|
37
38
|
# hash still exists. Such reclaim waits out the 60s heartbeat TTL, the
|
|
@@ -45,10 +46,10 @@ module Wurk
|
|
|
45
46
|
# process's own clock (OrphanGrace), and its identity must still have no
|
|
46
47
|
# `info` hash. Our own incarnation skips both: kill(0) is not a snapshot.
|
|
47
48
|
#
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
# its worker every time can't loop forever.
|
|
49
|
+
# Every move goes through OrphanReclaim, which caps a job at
|
|
50
|
+
# Middleware::PoisonPill::RECOVERY_THRESHOLD recoveries within 72h: past the
|
|
51
|
+
# cap the job goes from the private list straight to the dead set instead of
|
|
52
|
+
# its queue, so a job that crashes its worker every time can't loop forever.
|
|
52
53
|
#
|
|
53
54
|
# One sweep an interval: one SCAN of `queue:*|*` acting on every list, gated
|
|
54
55
|
# by a cluster lock so about one process sweeps per interval. Every list, and
|
|
@@ -62,7 +63,7 @@ module Wurk
|
|
|
62
63
|
# of a job.
|
|
63
64
|
#
|
|
64
65
|
# Spec: docs/target/sidekiq-pro.md §3.2.
|
|
65
|
-
class Reaper
|
|
66
|
+
class Reaper
|
|
66
67
|
include Component
|
|
67
68
|
|
|
68
69
|
# Sweep cadence in seconds. 60s matches the heartbeat TTL — the floor below
|
|
@@ -103,8 +104,8 @@ module Wurk
|
|
|
103
104
|
@sleeper = ::ConditionVariable.new
|
|
104
105
|
@unparseable = UnparseableKeys.new(config)
|
|
105
106
|
@next_sweep_at = 0.0
|
|
106
|
-
@
|
|
107
|
-
|
|
107
|
+
@orphan_reclaim = OrphanReclaim.new(config)
|
|
108
|
+
@self_reconcile = SelfReconcile.new(config, confirm: reconcile_confirm, stop: -> { done? })
|
|
108
109
|
end
|
|
109
110
|
|
|
110
111
|
# Spawns the sweep loop. Idempotent. The loop waits one interval before
|
|
@@ -173,13 +174,30 @@ module Wurk
|
|
|
173
174
|
sweep.tap { wake_for_settling }
|
|
174
175
|
end
|
|
175
176
|
|
|
176
|
-
#
|
|
177
|
-
#
|
|
178
|
-
#
|
|
179
|
-
#
|
|
180
|
-
#
|
|
177
|
+
# The sweep a booting process runs, under the same lock as the periodic one:
|
|
178
|
+
# a rolling restart or a deploy of N processes is one keyspace pass, not N.
|
|
179
|
+
# A replacement's sweep is no longer what recovers the swarm child it
|
|
180
|
+
# replaces; the parent did that by exact key when it reaped the child
|
|
181
|
+
# (#reclaim_exited). What this pass still finds is other incarnations'
|
|
182
|
+
# lists, which wait on the heartbeat TTL and the grace either way.
|
|
181
183
|
def boot_reclaim!
|
|
182
|
-
|
|
184
|
+
reap
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Drains the private lists of a process known to be dead: a swarm child its
|
|
188
|
+
# parent has just reaped. The parent knows the pid, the queues the child
|
|
189
|
+
# served and, since the nonce is inherited across fork, the rest of each
|
|
190
|
+
# list's name, so neither a SCAN nor a liveness check is needed. One
|
|
191
|
+
# pipelined LLEN finds the lists holding anything (none, after a drained
|
|
192
|
+
# exit) and raises rather than paying the pool's backoff once per list
|
|
193
|
+
# when Redis is away. Returns the jobs moved.
|
|
194
|
+
def reclaim_exited(pid, queue_names)
|
|
195
|
+
lists = queue_names.to_h do |name|
|
|
196
|
+
public_q = Keys.queue(name)
|
|
197
|
+
[Reliable.private_queue_name(public_q, pid: pid), public_q]
|
|
198
|
+
end
|
|
199
|
+
lengths = redis(idempotent: true) { |c| c.pipelined { |p| lists.each_key { |l| p.call('LLEN', l) } } }
|
|
200
|
+
lists.zip(lengths).sum { |(list, public_q), len| len.positive? ? drain(list, public_q) : 0 }
|
|
183
201
|
end
|
|
184
202
|
|
|
185
203
|
# Lists whose grace has run out, whichever sweep started it. Runs on every
|
|
@@ -313,27 +331,19 @@ module Wurk
|
|
|
313
331
|
identity.split(':')[0..1].join(':')
|
|
314
332
|
end
|
|
315
333
|
|
|
316
|
-
# Drain one orphaned private list back to its public queue
|
|
317
|
-
#
|
|
318
|
-
#
|
|
319
|
-
#
|
|
320
|
-
#
|
|
334
|
+
# Drain one orphaned private list back to its public queue, a job at a
|
|
335
|
+
# time from the tail: peek it, then OrphanReclaim moves it, to the public
|
|
336
|
+
# tail or, past the poison cap, to the dead set. The decision is taken
|
|
337
|
+
# before the job is visible, so a worker parked in BLMOVE can never take a
|
|
338
|
+
# job that is being killed. A peeked job that has left by the time it is
|
|
339
|
+
# moved (another reaper drained it) costs one more peek.
|
|
321
340
|
#
|
|
322
|
-
#
|
|
323
|
-
#
|
|
324
|
-
# one already on the public tail, so a job that kills its worker every time
|
|
325
|
-
# would get a free recovery past the cap. A raise instead lands in the
|
|
326
|
-
# rescue below, and the next sweep re-drains what's left.
|
|
341
|
+
# A raise lands in the rescue below; the job stays in the private list
|
|
342
|
+
# and the next sweep re-drains what's left.
|
|
327
343
|
def drain(private_list, public_q)
|
|
328
|
-
queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
|
|
329
344
|
count = 0
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
break unless job
|
|
333
|
-
|
|
334
|
-
count += 1
|
|
335
|
-
Metrics::ProcessCounters.increment(:reaper_reclaimed)
|
|
336
|
-
poison_off(public_q, job, queue_name)
|
|
345
|
+
while (job = redis(idempotent: true) { |c| c.call('LINDEX', private_list, -1) })
|
|
346
|
+
count += 1 if @orphan_reclaim.call(private_list, public_q, job)
|
|
337
347
|
end
|
|
338
348
|
count
|
|
339
349
|
rescue StandardError => e
|
|
@@ -341,35 +351,6 @@ module Wurk
|
|
|
341
351
|
count
|
|
342
352
|
end
|
|
343
353
|
|
|
344
|
-
# A full Redis must not hold a dead worker's jobs back from the workers
|
|
345
|
-
# still able to drain the queue.
|
|
346
|
-
def reclaim_one(conn, private_list, public_q)
|
|
347
|
-
Wurk::Lua::Loader.eval_cached(conn, :reliable_reclaim, keys: [private_list, public_q], argv: [])
|
|
348
|
-
rescue RedisClient::OutOfMemoryError => e
|
|
349
|
-
OomDrain.reclaim(conn, e, private_list, public_q)
|
|
350
|
-
end
|
|
351
|
-
|
|
352
|
-
# SelfReconcile put a payload back: the same accounting as a reclaim, so a
|
|
353
|
-
# job that kills its processor thread every time still hits the poison cap.
|
|
354
|
-
def recovered(public_q, job)
|
|
355
|
-
Metrics::ProcessCounters.increment(:reaper_reclaimed)
|
|
356
|
-
poison_off(public_q, job, public_q.delete_prefix(Keys::QUEUE_PREFIX))
|
|
357
|
-
end
|
|
358
|
-
|
|
359
|
-
# The job is back on its queue whatever happens here. Past maxmemory the
|
|
360
|
-
# count can't be taken (nor the job killed, leaving it where it is): raising
|
|
361
|
-
# would end the drain at one job per pass, with the rest of the list waiting
|
|
362
|
-
# on memory they would help free.
|
|
363
|
-
def poison_off(public_q, job, queue_name)
|
|
364
|
-
return unless Middleware::PoisonPill.track!(job, queue: queue_name, config: @config) == :poison
|
|
365
|
-
|
|
366
|
-
# track! already ZADDed the payload to the dead set; pull the copy we
|
|
367
|
-
# just moved onto the public tail so it isn't also re-run.
|
|
368
|
-
redis { |c| c.call('LREM', public_q, -1, job) }
|
|
369
|
-
rescue RedisClient::OutOfMemoryError => e
|
|
370
|
-
report(e)
|
|
371
|
-
end
|
|
372
|
-
|
|
373
354
|
def acquire_lock?
|
|
374
355
|
ttl = (@interval * LOCK_TTL_RATIO * 1000).ceil
|
|
375
356
|
redis do |c|
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'socket'
|
|
4
3
|
require_relative '../component'
|
|
5
4
|
require_relative '../pid_cache'
|
|
6
5
|
require_relative '../keys'
|
|
@@ -80,10 +79,10 @@ module Wurk
|
|
|
80
79
|
# reaper's `kill(0)` liveness check would read a dead owner's list as
|
|
81
80
|
# live (jobs stranded) or a live owner's as dead (job run twice). Keys
|
|
82
81
|
# written before the nonce existed stay reclaimable — PrivateListKey
|
|
83
|
-
# accepts both shapes.
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
"#{public_queue}|#{
|
|
82
|
+
# accepts both shapes. `pid:` lets a swarm parent name a reaped child's
|
|
83
|
+
# lists (Reaper#reclaim_exited).
|
|
84
|
+
def self.private_queue_name(public_queue, index = 0, pid: ::Process.pid)
|
|
85
|
+
"#{public_queue}|#{Component.hostname}|#{pid}|#{Component::PROCESS_NONCE}|#{index}"
|
|
87
86
|
end
|
|
88
87
|
|
|
89
88
|
# Guards the generation bump only. Fetchers read the counter without it —
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative '../keys'
|
|
4
|
-
require_relative '
|
|
4
|
+
require_relative 'orphan_reclaim'
|
|
5
5
|
|
|
6
6
|
module Wurk
|
|
7
7
|
class Fetcher
|
|
@@ -34,13 +34,14 @@ module Wurk
|
|
|
34
34
|
|
|
35
35
|
Suspect = Struct.new(:copies, :since)
|
|
36
36
|
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
|
|
37
|
+
# A payload put back goes through the same poison accounting as a reclaim
|
|
38
|
+
# (OrphanReclaim), so a job that kills its processor thread every time
|
|
39
|
+
# still hits the poison cap.
|
|
40
|
+
def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false })
|
|
40
41
|
@config = config
|
|
41
42
|
@confirm = confirm
|
|
42
43
|
@stop = stop
|
|
43
|
-
@
|
|
44
|
+
@orphan_reclaim = OrphanReclaim.new(config)
|
|
44
45
|
@suspects = {}
|
|
45
46
|
end
|
|
46
47
|
|
|
@@ -154,16 +155,9 @@ module Wurk
|
|
|
154
155
|
moved
|
|
155
156
|
end
|
|
156
157
|
|
|
157
|
-
# LPOS-guarded, so a
|
|
158
|
-
# time, and a payload settled since the last read stays settled.
|
|
158
|
+
# LPOS-guarded, so a payload settled since the last read stays settled.
|
|
159
159
|
def requeue(list, public_q, payload)
|
|
160
|
-
|
|
161
|
-
Wurk::Lua::Loader.eval_cached(conn, :reliable_requeue, keys: [list, public_q], argv: [payload])
|
|
162
|
-
rescue RedisClient::OutOfMemoryError => e
|
|
163
|
-
OomDrain.requeue(conn, e, list, public_q, payload)
|
|
164
|
-
end.to_i
|
|
165
|
-
@on_moved&.call(public_q, payload) if applied == 1
|
|
166
|
-
applied
|
|
160
|
+
@orphan_reclaim.call(list, public_q, payload) ? 1 : 0
|
|
167
161
|
end
|
|
168
162
|
|
|
169
163
|
def forget_all
|
data/lib/wurk/job_logger.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Wurk
|
|
|
12
12
|
# 1. prepare(job_hash) { ... } → sets thread-local context, applies
|
|
13
13
|
# per-job log_level, yields to the rest of dispatch.
|
|
14
14
|
# 2. call(item, queue) { ... } → wraps the actual perform with the
|
|
15
|
-
# start/done/fail log line trio and the elapsed-
|
|
15
|
+
# start/done/fail log line trio and the elapsed-seconds measurement.
|
|
16
16
|
#
|
|
17
17
|
# Skipping default logging is controlled by config[:skip_default_job_logging];
|
|
18
18
|
# the prepare step still runs so context/log_level still apply.
|
|
@@ -68,8 +68,10 @@ module Wurk
|
|
|
68
68
|
|
|
69
69
|
private
|
|
70
70
|
|
|
71
|
+
# Seconds, as upstream logs it (`elapsed=0.007`): log scrapers and
|
|
72
|
+
# dashboards built on Sidekiq's line read the field that way.
|
|
71
73
|
def elapsed(start)
|
|
72
|
-
(
|
|
74
|
+
(::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
|
|
73
75
|
end
|
|
74
76
|
|
|
75
77
|
# The attribute list is a host setting (defaults to %w[bid tags]), so the
|