wurk 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -9
- data/app/controllers/wurk/api_controller.rb +4 -1
- data/app/controllers/wurk/dashboard_controller.rb +11 -6
- data/exe/wurk +7 -2
- data/exe/wurkswarm +1 -2
- data/lib/generators/wurk/install/install_generator.rb +12 -0
- data/lib/sidekiq/web.rb +5 -0
- data/lib/wurk/boot_checks.rb +93 -0
- data/lib/wurk/capsule.rb +33 -8
- data/lib/wurk/cli.rb +204 -30
- data/lib/wurk/client/buffered.rb +240 -26
- data/lib/wurk/client.rb +16 -5
- data/lib/wurk/compat.rb +15 -0
- data/lib/wurk/configuration.rb +114 -16
- data/lib/wurk/cron.rb +141 -16
- data/lib/wurk/debounce.rb +2 -2
- data/lib/wurk/embedded.rb +8 -2
- data/lib/wurk/encryption.rb +20 -5
- data/lib/wurk/error_coalescer.rb +35 -0
- data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
- data/lib/wurk/fetcher/capped.rb +4 -0
- data/lib/wurk/fetcher/many_queues.rb +17 -1
- data/lib/wurk/fetcher/oom_drain.rb +102 -0
- data/lib/wurk/fetcher/orphan_grace.rb +60 -44
- data/lib/wurk/fetcher/private_list_key.rb +1 -16
- data/lib/wurk/fetcher/reaper.rb +169 -127
- data/lib/wurk/fetcher/reliable.rb +26 -4
- data/lib/wurk/fetcher/self_reconcile.rb +175 -0
- data/lib/wurk/fetcher/unit_of_work.rb +32 -8
- data/lib/wurk/health.rb +23 -4
- data/lib/wurk/health_probe.rb +64 -0
- data/lib/wurk/heartbeat.rb +55 -3
- data/lib/wurk/job/options.rb +9 -1
- data/lib/wurk/job.rb +2 -24
- data/lib/wurk/job_record.rb +17 -9
- data/lib/wurk/job_retry.rb +38 -10
- data/lib/wurk/job_set.rb +13 -5
- data/lib/wurk/launcher.rb +134 -52
- data/lib/wurk/leader.rb +71 -11
- data/lib/wurk/limiter/base.rb +17 -1
- data/lib/wurk/lua/debounce.lua +11 -6
- data/lib/wurk/lua/loader.rb +13 -1
- data/lib/wurk/lua.rb +71 -8
- data/lib/wurk/manager.rb +1 -1
- data/lib/wurk/metrics/process_counters.rb +27 -0
- data/lib/wurk/metrics/prometheus.rb +35 -14
- data/lib/wurk/middleware/current_attributes.rb +26 -10
- data/lib/wurk/middleware/poison_pill.rb +1 -2
- data/lib/wurk/preflight/checks.rb +203 -0
- data/lib/wurk/preflight/command.rb +135 -0
- data/lib/wurk/preflight/keyspace.rb +109 -0
- data/lib/wurk/preflight/owners.rb +74 -0
- data/lib/wurk/preflight/payload_sample.rb +108 -0
- data/lib/wurk/preflight/reader.rb +60 -0
- data/lib/wurk/preflight/report.rb +82 -0
- data/lib/wurk/preflight/verify.rb +161 -0
- data/lib/wurk/preflight.rb +40 -0
- data/lib/wurk/process_set.rb +3 -1
- data/lib/wurk/processor.rb +82 -18
- data/lib/wurk/queue_consumers.rb +134 -0
- data/lib/wurk/rails_boot.rb +40 -2
- data/lib/wurk/rails_reloader.rb +51 -0
- data/lib/wurk/railtie.rb +20 -1
- data/lib/wurk/redact.rb +12 -0
- data/lib/wurk/redis_pool.rb +34 -9
- data/lib/wurk/scheduled.rb +142 -67
- data/lib/wurk/sentry/death_handler.rb +29 -0
- data/lib/wurk/sentry/error_handler.rb +52 -10
- data/lib/wurk/sentry/middleware.rb +8 -1
- data/lib/wurk/sentry/retry_policy.rb +12 -3
- data/lib/wurk/sentry.rb +42 -5
- data/lib/wurk/sorted_entry.rb +38 -13
- data/lib/wurk/swarm/child_boot.rb +44 -31
- data/lib/wurk/swarm/crash_watch.rb +65 -0
- data/lib/wurk/swarm/test_mode_guard.rb +32 -0
- data/lib/wurk/swarm.rb +79 -11
- data/lib/wurk/testing.rb +74 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/config.rb +8 -4
- data/lib/wurk/web/extension.rb +41 -62
- data/lib/wurk/web/helpers.rb +133 -0
- data/lib/wurk/web/paginator.rb +49 -0
- data/lib/wurk/web/rack_app.rb +30 -6
- data/lib/wurk/work_set.rb +6 -0
- data/lib/wurk/worker/setter.rb +11 -6
- data/lib/wurk/worker.rb +17 -1
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
- data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
- data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +53 -30
- data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
- data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
- data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'oom_drain'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
class Fetcher
|
|
7
|
+
# {Fetcher::Reliable}'s fetch once Redis has refused one with OOM: a full
|
|
8
|
+
# Redis refuses LMOVE, which would stop every worker draining the queues that
|
|
9
|
+
# filled it. See OomDrain for what makes this possible. Only reached after an
|
|
10
|
+
# OutOfMemoryError, so it costs a healthy fetch nothing.
|
|
11
|
+
module AtMaxmemory
|
|
12
|
+
# Seconds between error reports while draining a full Redis. Every thread
|
|
13
|
+
# lands here on every fetch for as long as it lasts.
|
|
14
|
+
REPORT_INTERVAL = 60
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
# Costs the wall clock a BLMOVE would have on an empty pass, for the reason
|
|
19
|
+
# #retrieve_work's short-circuits do. An ACK is an LREM, which a full Redis
|
|
20
|
+
# still takes.
|
|
21
|
+
def claim_at_maxmemory(queues, error)
|
|
22
|
+
flush_pending_acks
|
|
23
|
+
report_maxmemory(error)
|
|
24
|
+
fetchable = fetchable_at_maxmemory(queues)
|
|
25
|
+
raise error if fetchable.empty?
|
|
26
|
+
|
|
27
|
+
index, job = claim_past_maxmemory(fetchable, error)
|
|
28
|
+
return unit_at_maxmemory(fetchable[index - 1], job) if job
|
|
29
|
+
|
|
30
|
+
sleep poll_interval
|
|
31
|
+
nil
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# A pipeline applies everything it can before raising; the guard makes what
|
|
35
|
+
# already went back a no-op here.
|
|
36
|
+
def requeue_past_maxmemory(units, error)
|
|
37
|
+
config.redis(idempotent: true) do |conn|
|
|
38
|
+
units.each { |uow| OomDrain.requeue(conn, error, queue_keys(uow.queue).first, uow.queue, uow.job) }
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def claim_past_maxmemory(fetchable, error)
|
|
43
|
+
keys = fetchable.flat_map { |public_q| [public_q, queue_keys(public_q).first] }
|
|
44
|
+
config.redis(idempotent: true) do |conn|
|
|
45
|
+
OomDrain.claim(conn, error, keys: keys, info: maxmemory_info, beat: ::Time.now.to_f.to_s)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def unit_at_maxmemory(public_q, job)
|
|
50
|
+
priv, name = queue_keys(public_q)
|
|
51
|
+
unit_of_work(public_q, priv, name, job)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Capped overrides this: a cap cannot be honoured by a script that skips
|
|
55
|
+
# the ledger's write.
|
|
56
|
+
def fetchable_at_maxmemory(queues)
|
|
57
|
+
queues
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# What the claim script writes if this process has no heartbeat hash (it
|
|
61
|
+
# booted into the full Redis). The first ordinary beat replaces it.
|
|
62
|
+
def maxmemory_info
|
|
63
|
+
Wurk.dump_json(
|
|
64
|
+
'hostname' => hostname, 'pid' => ::Process.pid, 'identity' => Component.identity,
|
|
65
|
+
'started_at' => ::Time.now.to_f, 'tag' => '', 'labels' => [], 'concurrency' => config.concurrency,
|
|
66
|
+
'queues' => config.queues.uniq, 'version' => Wurk::VERSION, 'embedded' => false
|
|
67
|
+
)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def report_maxmemory(error)
|
|
71
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
72
|
+
return if @oom_reported_at && now - @oom_reported_at < REPORT_INTERVAL
|
|
73
|
+
|
|
74
|
+
@oom_reported_at = now
|
|
75
|
+
handle_exception(error, { context: 'Redis is at maxmemory: draining with allow-oom scripts' })
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
data/lib/wurk/fetcher/capped.rb
CHANGED
|
@@ -210,6 +210,10 @@ module Wurk
|
|
|
210
210
|
blocked ? [poll_interval, CAPPED_BACKOFF].min : poll_interval
|
|
211
211
|
end
|
|
212
212
|
|
|
213
|
+
def fetchable_at_maxmemory(queues)
|
|
214
|
+
queues.select { |public_q| gate_for(public_q).nil? }
|
|
215
|
+
end
|
|
216
|
+
|
|
213
217
|
def gate_for(public_q)
|
|
214
218
|
@gates[queue_keys(public_q)[1]]
|
|
215
219
|
end
|
|
@@ -23,6 +23,16 @@ module Wurk
|
|
|
23
23
|
module ManyQueues
|
|
24
24
|
NO_ARGV = [].freeze
|
|
25
25
|
|
|
26
|
+
# How long a script pass that found every queue empty answers for the whole
|
|
27
|
+
# fetcher. The script sends two keys per queue, per thread, per poll: 120 KB
|
|
28
|
+
# a pass at 1000 queues, which at concurrency 500 cost a process 70% of a
|
|
29
|
+
# core and Redis 26% of one while idle. A thread skipping it parks in BLMOVE
|
|
30
|
+
# on the first queue for the whole poll, unaware of anything that lands on a
|
|
31
|
+
# later queue meanwhile, so a job can wait out a poll interval (about 2s)
|
|
32
|
+
# with a thread idle. That is the same bound an empty script pass gives it.
|
|
33
|
+
# Not a config knob.
|
|
34
|
+
REST_MISS_BACKOFF = 0.25
|
|
35
|
+
|
|
26
36
|
private
|
|
27
37
|
|
|
28
38
|
# Queues 2..n. Two queues is one more LMOVE — a one-key script costs more
|
|
@@ -34,11 +44,17 @@ module Wurk
|
|
|
34
44
|
def claim_rest(queues)
|
|
35
45
|
return lmove(queues[1]) if queues.size == 2
|
|
36
46
|
|
|
47
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
48
|
+
return nil if now < @rest_empty_until
|
|
49
|
+
|
|
37
50
|
keys = claim_keys(queues)
|
|
38
51
|
index, job = config.redis(idempotent: true) do |conn|
|
|
39
52
|
Wurk::Lua::Loader.eval_cached(conn, :fetch_first, keys: keys, argv: NO_ARGV)
|
|
40
53
|
end
|
|
41
|
-
|
|
54
|
+
unless job
|
|
55
|
+
@rest_empty_until = now + REST_MISS_BACKOFF
|
|
56
|
+
return nil
|
|
57
|
+
end
|
|
42
58
|
|
|
43
59
|
public_q = queues[index]
|
|
44
60
|
priv, name = queue_keys(public_q)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../keys'
|
|
4
|
+
require_relative '../lua'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
class Fetcher
|
|
8
|
+
# The way out of a full Redis.
|
|
9
|
+
#
|
|
10
|
+
# LMOVE and ZADD are `denyoom` commands, so at `maxmemory` under the
|
|
11
|
+
# `noeviction` policy Wurk requires, every worker would be refused its next
|
|
12
|
+
# job and could therefore never free the memory that is refusing it. A
|
|
13
|
+
# Redis 7 script flagged `allow-oom` runs those commands anyway, and moving a
|
|
14
|
+
# job between two lists grows nothing but a small key.
|
|
15
|
+
#
|
|
16
|
+
# Deliberately not under lib/wurk/lua/: everything there is uploaded at boot,
|
|
17
|
+
# and Dragonfly refuses to compile a `#!lua` shebang, so one script in that
|
|
18
|
+
# directory would crash every child at boot. The same refusal is why these
|
|
19
|
+
# go by EVAL and not EVALSHA: a path taken only while Redis is full has no
|
|
20
|
+
# use for a NOSCRIPT recovery arm. A server that refuses the script raises
|
|
21
|
+
# the original OOM, which is exactly the behaviour without this class.
|
|
22
|
+
module OomDrain
|
|
23
|
+
HEADER = "#!lua flags=allow-oom\n"
|
|
24
|
+
|
|
25
|
+
# Heartbeat::TTL_SECONDS. Spelled out because the fetcher loads without the
|
|
26
|
+
# heartbeat, which pulls in the Processor.
|
|
27
|
+
HEARTBEAT_TTL = 60
|
|
28
|
+
|
|
29
|
+
# Claiming proves this process alive, and it has to say so itself: the
|
|
30
|
+
# heartbeat's own HSET is refused too. The hash then lapses (a booting
|
|
31
|
+
# process never gets one), other hosts' reapers drain the jobs it is
|
|
32
|
+
# running, and the swarm's liveness check sees its beat stand still.
|
|
33
|
+
#
|
|
34
|
+
# KEYS = [processes, identity, public, private, public, private, ...]
|
|
35
|
+
# ARGV = [info_json, beat, ttl_seconds]
|
|
36
|
+
# Returns {n, payload} for the n-th queue, nil when all were empty.
|
|
37
|
+
CLAIM = <<~LUA.freeze
|
|
38
|
+
#{HEADER}if redis.call('EXISTS', KEYS[2]) == 0 then
|
|
39
|
+
redis.call('HSET', KEYS[2], 'info', ARGV[1])
|
|
40
|
+
redis.call('SADD', KEYS[1], KEYS[2])
|
|
41
|
+
end
|
|
42
|
+
redis.call('HSET', KEYS[2], 'beat', ARGV[2])
|
|
43
|
+
redis.call('EXPIRE', KEYS[2], ARGV[3])
|
|
44
|
+
for i = 3, #KEYS, 2 do
|
|
45
|
+
local job = redis.call('LMOVE', KEYS[i], KEYS[i + 1], 'RIGHT', 'LEFT')
|
|
46
|
+
if job then
|
|
47
|
+
return { (i - 1) / 2, job }
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
return nil
|
|
51
|
+
LUA
|
|
52
|
+
|
|
53
|
+
RECLAIM = (HEADER + Lua::RELIABLE_RECLAIM).freeze
|
|
54
|
+
REQUEUE = (HEADER + Lua::RELIABLE_REQUEUE).freeze
|
|
55
|
+
|
|
56
|
+
# KEYS = [lock], ARGV = [ttl_ms]. The reaper's sweep lock: refused, no sweep
|
|
57
|
+
# would ever run past maxmemory. Same reply as the SET it stands in for.
|
|
58
|
+
LOCK = <<~LUA.freeze
|
|
59
|
+
#{HEADER}return redis.call('SET', KEYS[1], '1', 'NX', 'PX', ARGV[1])
|
|
60
|
+
LUA
|
|
61
|
+
|
|
62
|
+
# KEYS = [sorted_set], ARGV = [score, member]
|
|
63
|
+
ZADD = <<~LUA.freeze
|
|
64
|
+
#{HEADER}return redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2])
|
|
65
|
+
LUA
|
|
66
|
+
|
|
67
|
+
module_function
|
|
68
|
+
|
|
69
|
+
# `error` is the OOM that sent the caller here, raised again if the server
|
|
70
|
+
# cannot run the script. Returns [queue_index, payload] or nil.
|
|
71
|
+
def claim(conn, error, keys:, info:, beat:)
|
|
72
|
+
run(conn, error, CLAIM, [Keys::PROCESSES, Component.identity, *keys], [info, beat, HEARTBEAT_TTL])
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def reclaim(conn, error, source, destination)
|
|
76
|
+
run(conn, error, RECLAIM, [source, destination], [])
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Returns 1 when moved, 0 when the job has left the private list since.
|
|
80
|
+
def requeue(conn, error, private_list, public_q, job)
|
|
81
|
+
run(conn, error, REQUEUE, [private_list, public_q], [job])
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def lock(conn, error, key, ttl_ms)
|
|
85
|
+
run(conn, error, LOCK, [key], [ttl_ms])
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def zadd(conn, error, key, score, member)
|
|
89
|
+
run(conn, error, ZADD, [key], [score, member])
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def run(conn, error, script, keys, argv)
|
|
93
|
+
conn.call('EVAL', script, keys.size, *keys, *argv)
|
|
94
|
+
rescue RedisClient::ConnectionError
|
|
95
|
+
raise
|
|
96
|
+
rescue RedisClient::CommandError
|
|
97
|
+
raise error
|
|
98
|
+
end
|
|
99
|
+
private_class_method :run
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -2,65 +2,81 @@
|
|
|
2
2
|
|
|
3
3
|
module Wurk
|
|
4
4
|
class Fetcher
|
|
5
|
-
# The Reaper's
|
|
6
|
-
#
|
|
7
|
-
# a moment before its first heartbeat lands, and draining its list in that
|
|
8
|
-
# window runs the job twice, concurrently.
|
|
5
|
+
# The Reaper's wait between judging a list orphaned by its heartbeat and
|
|
6
|
+
# draining it.
|
|
9
7
|
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
# is
|
|
15
|
-
#
|
|
16
|
-
#
|
|
8
|
+
# A live owner can read as having no heartbeat hash: its first beat failed on
|
|
9
|
+
# a Redis blip, CPU-bound jobs held the beat back, or Redis restarted from an
|
|
10
|
+
# older snapshot. Its next beat lands within the wait, and that spares it.
|
|
11
|
+
#
|
|
12
|
+
# The clock is this process's own and it is shared by nothing else, so it
|
|
13
|
+
# costs the same on every server and at every fleet size. That is the
|
|
14
|
+
# difference from what it replaces, OBJECT IDLETIME: any read of the list
|
|
15
|
+
# (a monitor's LLEN) reset that clock, some servers do not have the command,
|
|
16
|
+
# and a clock kept per sweep only advanced in a process that won the sweep
|
|
17
|
+
# lock twice. The loop thread therefore finishes what any sweep here started,
|
|
18
|
+
# with or without the lock (Reaper#drain_settled).
|
|
17
19
|
class OrphanGrace
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
'orphan is drained one sweep later than it would be. Expected under an LFU ' \
|
|
21
|
-
'maxmemory policy or on servers without OBJECT (e.g. Dragonfly).'
|
|
22
|
-
|
|
23
|
-
# Lists remembered for the fallback clock. A list leaves on the pass
|
|
24
|
-
# that clears it; one judged orphaned and then found alive can linger,
|
|
25
|
-
# so the memory is bounded and simply restarts when full — every clock
|
|
26
|
-
# restarting is the conservative direction.
|
|
20
|
+
# Lists remembered at once. Past it the memory restarts, and every clock
|
|
21
|
+
# restarting is the conservative direction: the next sweep judges them anew.
|
|
27
22
|
MEMORY = 10_000
|
|
28
23
|
|
|
29
|
-
|
|
30
|
-
|
|
24
|
+
Entry = Struct.new(:since, :owner)
|
|
25
|
+
|
|
26
|
+
def initialize(grace)
|
|
31
27
|
@grace = grace
|
|
32
|
-
@
|
|
33
|
-
@
|
|
28
|
+
@lock = ::Mutex.new
|
|
29
|
+
@entries = {}
|
|
34
30
|
end
|
|
35
31
|
|
|
36
|
-
# true while the list must be left alone.
|
|
37
|
-
|
|
32
|
+
# true while the list must be left alone. `owner` is what #take_settled
|
|
33
|
+
# hands back when the wait is over.
|
|
34
|
+
def recent?(key, owner)
|
|
38
35
|
return false unless @grace.positive?
|
|
39
|
-
return local_recent?(key) unless @idletime
|
|
40
36
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
37
|
+
now = monotonic
|
|
38
|
+
@lock.synchronize do
|
|
39
|
+
@entries.clear if @entries.size >= MEMORY && !@entries.key?(key)
|
|
40
|
+
entry = (@entries[key] ||= Entry.new(now, owner))
|
|
41
|
+
next true if now - entry.since < @grace
|
|
42
|
+
|
|
43
|
+
@entries.delete(key)
|
|
44
|
+
false
|
|
45
|
+
end
|
|
47
46
|
end
|
|
48
47
|
|
|
49
|
-
|
|
48
|
+
# The owner came back: its next judgement starts a new wait.
|
|
49
|
+
def forget(key)
|
|
50
|
+
@lock.synchronize { @entries.delete(key) }
|
|
51
|
+
end
|
|
50
52
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
53
|
+
# [[key, owner], ...] whose wait is over, removed from memory.
|
|
54
|
+
def take_settled
|
|
55
|
+
now = monotonic
|
|
56
|
+
@lock.synchronize do
|
|
57
|
+
settled = @entries.select { |_, entry| now - entry.since >= @grace }
|
|
58
|
+
settled.each_key { |key| @entries.delete(key) }
|
|
59
|
+
settled.map { |key, entry| [key, entry.owner] }
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Seconds until the next wait ends, nil with nothing waiting.
|
|
64
|
+
def next_due_in
|
|
65
|
+
now = monotonic
|
|
66
|
+
@lock.synchronize do
|
|
67
|
+
oldest = @entries.each_value.map(&:since).min
|
|
68
|
+
oldest && [(oldest + @grace) - now, 0].max
|
|
69
|
+
end
|
|
70
|
+
end
|
|
56
71
|
|
|
57
|
-
|
|
58
|
-
|
|
72
|
+
def size
|
|
73
|
+
@lock.synchronize { @entries.size }
|
|
59
74
|
end
|
|
60
75
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def monotonic
|
|
79
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
64
80
|
end
|
|
65
81
|
end
|
|
66
82
|
end
|
|
@@ -38,19 +38,6 @@ module Wurk
|
|
|
38
38
|
nil
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
# `<public_q>|<host>|<pid>|<nonce>|<idx>` → [host, pid, nonce] (pid as
|
|
42
|
-
# Integer), or all-nil when the suffix isn't a well-formed owner tail.
|
|
43
|
-
# Splitting the suffix off the known public-queue prefix tolerates a `|`
|
|
44
|
-
# inside the queue name itself, and leaves the tail unambiguous: exactly
|
|
45
|
-
# 4 segments for the current shape, exactly 3 for the pre-nonce one.
|
|
46
|
-
def parse_owner(public_q, key)
|
|
47
|
-
suffix = key.delete_prefix("#{public_q}|")
|
|
48
|
-
return [nil, nil, nil] if suffix == key
|
|
49
|
-
|
|
50
|
-
host, pid, nonce = owner_tails(suffix.split('|')).first
|
|
51
|
-
[host, pid, nonce]
|
|
52
|
-
end
|
|
53
|
-
|
|
54
41
|
# Owner segments of a private-list key, taken from the right, as
|
|
55
42
|
# [host, pid, nonce, segment_count] readings in preference order (empty
|
|
56
43
|
# when nothing parses). The wide shape is preferred: an all-digit nonce is
|
|
@@ -67,9 +54,7 @@ module Wurk
|
|
|
67
54
|
# The nonce slot must carry PROCESS_NONCE's exact shape (NONCE_PATTERN):
|
|
68
55
|
# on a pre-nonce key read wide it holds the real pid, which is never 12
|
|
69
56
|
# hex, so the guard is what falls such a key through to narrow_tail
|
|
70
|
-
# instead of misreading it.
|
|
71
|
-
# because parse_owner reads through owner_tails too — one guard, and the
|
|
72
|
-
# two parsers agree on every input.
|
|
57
|
+
# instead of misreading it.
|
|
73
58
|
def wide_tail(parts)
|
|
74
59
|
[parts[-4], parts[-3].to_i, parts[-2], 4] if parts.size >= 4 && integer?(parts[-3]) && nonce?(parts[-2])
|
|
75
60
|
end
|