wurk 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +9 -9
  3. data/app/controllers/wurk/api_controller.rb +4 -1
  4. data/app/controllers/wurk/dashboard_controller.rb +11 -6
  5. data/exe/wurk +7 -2
  6. data/exe/wurkswarm +1 -2
  7. data/lib/generators/wurk/install/install_generator.rb +12 -0
  8. data/lib/sidekiq/web.rb +5 -0
  9. data/lib/wurk/boot_checks.rb +93 -0
  10. data/lib/wurk/capsule.rb +33 -8
  11. data/lib/wurk/cli.rb +204 -30
  12. data/lib/wurk/client/buffered.rb +240 -26
  13. data/lib/wurk/client.rb +16 -5
  14. data/lib/wurk/compat.rb +15 -0
  15. data/lib/wurk/configuration.rb +114 -16
  16. data/lib/wurk/cron.rb +141 -16
  17. data/lib/wurk/debounce.rb +2 -2
  18. data/lib/wurk/embedded.rb +8 -2
  19. data/lib/wurk/encryption.rb +20 -5
  20. data/lib/wurk/error_coalescer.rb +35 -0
  21. data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
  22. data/lib/wurk/fetcher/capped.rb +4 -0
  23. data/lib/wurk/fetcher/many_queues.rb +17 -1
  24. data/lib/wurk/fetcher/oom_drain.rb +102 -0
  25. data/lib/wurk/fetcher/orphan_grace.rb +60 -44
  26. data/lib/wurk/fetcher/private_list_key.rb +1 -16
  27. data/lib/wurk/fetcher/reaper.rb +169 -127
  28. data/lib/wurk/fetcher/reliable.rb +26 -4
  29. data/lib/wurk/fetcher/self_reconcile.rb +175 -0
  30. data/lib/wurk/fetcher/unit_of_work.rb +32 -8
  31. data/lib/wurk/health.rb +23 -4
  32. data/lib/wurk/health_probe.rb +64 -0
  33. data/lib/wurk/heartbeat.rb +55 -3
  34. data/lib/wurk/job/options.rb +9 -1
  35. data/lib/wurk/job.rb +2 -24
  36. data/lib/wurk/job_record.rb +17 -9
  37. data/lib/wurk/job_retry.rb +38 -10
  38. data/lib/wurk/job_set.rb +13 -5
  39. data/lib/wurk/launcher.rb +134 -52
  40. data/lib/wurk/leader.rb +71 -11
  41. data/lib/wurk/limiter/base.rb +17 -1
  42. data/lib/wurk/lua/debounce.lua +11 -6
  43. data/lib/wurk/lua/loader.rb +13 -1
  44. data/lib/wurk/lua.rb +71 -8
  45. data/lib/wurk/manager.rb +1 -1
  46. data/lib/wurk/metrics/process_counters.rb +27 -0
  47. data/lib/wurk/metrics/prometheus.rb +35 -14
  48. data/lib/wurk/middleware/current_attributes.rb +26 -10
  49. data/lib/wurk/middleware/poison_pill.rb +1 -2
  50. data/lib/wurk/preflight/checks.rb +203 -0
  51. data/lib/wurk/preflight/command.rb +135 -0
  52. data/lib/wurk/preflight/keyspace.rb +109 -0
  53. data/lib/wurk/preflight/owners.rb +74 -0
  54. data/lib/wurk/preflight/payload_sample.rb +108 -0
  55. data/lib/wurk/preflight/reader.rb +60 -0
  56. data/lib/wurk/preflight/report.rb +82 -0
  57. data/lib/wurk/preflight/verify.rb +161 -0
  58. data/lib/wurk/preflight.rb +40 -0
  59. data/lib/wurk/process_set.rb +3 -1
  60. data/lib/wurk/processor.rb +82 -18
  61. data/lib/wurk/queue_consumers.rb +134 -0
  62. data/lib/wurk/rails_boot.rb +40 -2
  63. data/lib/wurk/rails_reloader.rb +51 -0
  64. data/lib/wurk/railtie.rb +20 -1
  65. data/lib/wurk/redact.rb +12 -0
  66. data/lib/wurk/redis_pool.rb +34 -9
  67. data/lib/wurk/scheduled.rb +142 -67
  68. data/lib/wurk/sentry/death_handler.rb +29 -0
  69. data/lib/wurk/sentry/error_handler.rb +52 -10
  70. data/lib/wurk/sentry/middleware.rb +8 -1
  71. data/lib/wurk/sentry/retry_policy.rb +12 -3
  72. data/lib/wurk/sentry.rb +42 -5
  73. data/lib/wurk/sorted_entry.rb +38 -13
  74. data/lib/wurk/swarm/child_boot.rb +44 -31
  75. data/lib/wurk/swarm/crash_watch.rb +65 -0
  76. data/lib/wurk/swarm/test_mode_guard.rb +32 -0
  77. data/lib/wurk/swarm.rb +79 -11
  78. data/lib/wurk/testing.rb +74 -0
  79. data/lib/wurk/version.rb +1 -1
  80. data/lib/wurk/web/config.rb +8 -4
  81. data/lib/wurk/web/extension.rb +41 -62
  82. data/lib/wurk/web/helpers.rb +133 -0
  83. data/lib/wurk/web/paginator.rb +49 -0
  84. data/lib/wurk/web/rack_app.rb +30 -6
  85. data/lib/wurk/work_set.rb +6 -0
  86. data/lib/wurk/worker/setter.rb +11 -6
  87. data/lib/wurk/worker.rb +17 -1
  88. data/lib/wurk.rb +1 -1
  89. data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
  90. data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
  91. data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
  92. data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
  93. data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
  94. data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
  95. data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
  96. data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
  97. data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
  98. data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
  99. data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
  100. data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
  101. data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
  102. data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
  103. data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
  104. data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
  105. data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
  106. data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
  107. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
  108. data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
  109. data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
  110. data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
  111. data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
  112. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
  113. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
  114. data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
  115. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
  116. data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
  117. data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
  118. data/vendor/assets/dashboard/index.html +3 -3
  119. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  120. metadata +53 -30
  121. data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
  122. data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
  123. data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'oom_drain'
4
+
5
+ module Wurk
6
+ class Fetcher
7
+ # {Fetcher::Reliable}'s fetch once Redis has refused one with OOM: a full
8
+ # Redis refuses LMOVE, which would stop every worker draining the queues that
9
+ # filled it. See OomDrain for what makes this possible. Only reached after an
10
+ # OutOfMemoryError, so it costs a healthy fetch nothing.
11
+ module AtMaxmemory
12
+ # Seconds between error reports while draining a full Redis. Every thread
13
+ # lands here on every fetch for as long as it lasts.
14
+ REPORT_INTERVAL = 60
15
+
16
+ private
17
+
18
+ # Costs the wall clock a BLMOVE would have on an empty pass, for the reason
19
+ # #retrieve_work's short-circuits do. An ACK is an LREM, which a full Redis
20
+ # still takes.
21
+ def claim_at_maxmemory(queues, error)
22
+ flush_pending_acks
23
+ report_maxmemory(error)
24
+ fetchable = fetchable_at_maxmemory(queues)
25
+ raise error if fetchable.empty?
26
+
27
+ index, job = claim_past_maxmemory(fetchable, error)
28
+ return unit_at_maxmemory(fetchable[index - 1], job) if job
29
+
30
+ sleep poll_interval
31
+ nil
32
+ end
33
+
34
+ # A pipeline applies everything it can before raising; the guard makes what
35
+ # already went back a no-op here.
36
+ def requeue_past_maxmemory(units, error)
37
+ config.redis(idempotent: true) do |conn|
38
+ units.each { |uow| OomDrain.requeue(conn, error, queue_keys(uow.queue).first, uow.queue, uow.job) }
39
+ end
40
+ end
41
+
42
+ def claim_past_maxmemory(fetchable, error)
43
+ keys = fetchable.flat_map { |public_q| [public_q, queue_keys(public_q).first] }
44
+ config.redis(idempotent: true) do |conn|
45
+ OomDrain.claim(conn, error, keys: keys, info: maxmemory_info, beat: ::Time.now.to_f.to_s)
46
+ end
47
+ end
48
+
49
+ def unit_at_maxmemory(public_q, job)
50
+ priv, name = queue_keys(public_q)
51
+ unit_of_work(public_q, priv, name, job)
52
+ end
53
+
54
+ # Capped overrides this: a cap cannot be honoured by a script that skips
55
+ # the ledger's write.
56
+ def fetchable_at_maxmemory(queues)
57
+ queues
58
+ end
59
+
60
+ # What the claim script writes if this process has no heartbeat hash (it
61
+ # booted into the full Redis). The first ordinary beat replaces it.
62
+ def maxmemory_info
63
+ Wurk.dump_json(
64
+ 'hostname' => hostname, 'pid' => ::Process.pid, 'identity' => Component.identity,
65
+ 'started_at' => ::Time.now.to_f, 'tag' => '', 'labels' => [], 'concurrency' => config.concurrency,
66
+ 'queues' => config.queues.uniq, 'version' => Wurk::VERSION, 'embedded' => false
67
+ )
68
+ end
69
+
70
+ def report_maxmemory(error)
71
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
72
+ return if @oom_reported_at && now - @oom_reported_at < REPORT_INTERVAL
73
+
74
+ @oom_reported_at = now
75
+ handle_exception(error, { context: 'Redis is at maxmemory: draining with allow-oom scripts' })
76
+ end
77
+ end
78
+ end
79
+ end
@@ -210,6 +210,10 @@ module Wurk
210
210
  blocked ? [poll_interval, CAPPED_BACKOFF].min : poll_interval
211
211
  end
212
212
 
213
+ def fetchable_at_maxmemory(queues)
214
+ queues.select { |public_q| gate_for(public_q).nil? }
215
+ end
216
+
213
217
  def gate_for(public_q)
214
218
  @gates[queue_keys(public_q)[1]]
215
219
  end
@@ -23,6 +23,16 @@ module Wurk
23
23
  module ManyQueues
24
24
  NO_ARGV = [].freeze
25
25
 
26
+ # How long a script pass that found every queue empty answers for the whole
27
+ # fetcher. The script sends two keys per queue, per thread, per poll: 120 KB
28
+ # a pass at 1000 queues, which at concurrency 500 cost a process 70% of a
29
+ # core and Redis 26% of one while idle. A thread skipping it parks in BLMOVE
30
+ # on the first queue for the whole poll, unaware of anything that lands on a
31
+ # later queue meanwhile, so a job can wait out a poll interval (about 2s)
32
+ # with a thread idle. That is the same bound an empty script pass gives it.
33
+ # Not a config knob.
34
+ REST_MISS_BACKOFF = 0.25
35
+
26
36
  private
27
37
 
28
38
  # Queues 2..n. Two queues is one more LMOVE — a one-key script costs more
@@ -34,11 +44,17 @@ module Wurk
34
44
  def claim_rest(queues)
35
45
  return lmove(queues[1]) if queues.size == 2
36
46
 
47
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
48
+ return nil if now < @rest_empty_until
49
+
37
50
  keys = claim_keys(queues)
38
51
  index, job = config.redis(idempotent: true) do |conn|
39
52
  Wurk::Lua::Loader.eval_cached(conn, :fetch_first, keys: keys, argv: NO_ARGV)
40
53
  end
41
- return nil unless job
54
+ unless job
55
+ @rest_empty_until = now + REST_MISS_BACKOFF
56
+ return nil
57
+ end
42
58
 
43
59
  public_q = queues[index]
44
60
  priv, name = queue_keys(public_q)
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../keys'
4
+ require_relative '../lua'
5
+
6
+ module Wurk
7
+ class Fetcher
8
+ # The way out of a full Redis.
9
+ #
10
+ # LMOVE and ZADD are `denyoom` commands, so at `maxmemory` under the
11
+ # `noeviction` policy Wurk requires, every worker would be refused its next
12
+ # job and could therefore never free the memory that is refusing it. A
13
+ # Redis 7 script flagged `allow-oom` runs those commands anyway, and moving a
14
+ # job between two lists grows nothing but a small key.
15
+ #
16
+ # Deliberately not under lib/wurk/lua/: everything there is uploaded at boot,
17
+ # and Dragonfly refuses to compile a `#!lua` shebang, so one script in that
18
+ # directory would crash every child at boot. The same refusal is why these
19
+ # go by EVAL and not EVALSHA: a path taken only while Redis is full has no
20
+ # use for a NOSCRIPT recovery arm. A server that refuses the script raises
21
+ # the original OOM, which is exactly the behaviour without this class.
22
+ module OomDrain
23
+ HEADER = "#!lua flags=allow-oom\n"
24
+
25
+ # Heartbeat::TTL_SECONDS. Spelled out because the fetcher loads without the
26
+ # heartbeat, which pulls in the Processor.
27
+ HEARTBEAT_TTL = 60
28
+
29
+ # Claiming proves this process alive, and it has to say so itself: the
30
+ # heartbeat's own HSET is refused too. The hash then lapses (a booting
31
+ # process never gets one), other hosts' reapers drain the jobs it is
32
+ # running, and the swarm's liveness check sees its beat stand still.
33
+ #
34
+ # KEYS = [processes, identity, public, private, public, private, ...]
35
+ # ARGV = [info_json, beat, ttl_seconds]
36
+ # Returns {n, payload} for the n-th queue, nil when all were empty.
37
+ CLAIM = <<~LUA.freeze
38
+ #{HEADER}if redis.call('EXISTS', KEYS[2]) == 0 then
39
+ redis.call('HSET', KEYS[2], 'info', ARGV[1])
40
+ redis.call('SADD', KEYS[1], KEYS[2])
41
+ end
42
+ redis.call('HSET', KEYS[2], 'beat', ARGV[2])
43
+ redis.call('EXPIRE', KEYS[2], ARGV[3])
44
+ for i = 3, #KEYS, 2 do
45
+ local job = redis.call('LMOVE', KEYS[i], KEYS[i + 1], 'RIGHT', 'LEFT')
46
+ if job then
47
+ return { (i - 1) / 2, job }
48
+ end
49
+ end
50
+ return nil
51
+ LUA
52
+
53
+ RECLAIM = (HEADER + Lua::RELIABLE_RECLAIM).freeze
54
+ REQUEUE = (HEADER + Lua::RELIABLE_REQUEUE).freeze
55
+
56
+ # KEYS = [lock], ARGV = [ttl_ms]. The reaper's sweep lock: refused, no sweep
57
+ # would ever run past maxmemory. Same reply as the SET it stands in for.
58
+ LOCK = <<~LUA.freeze
59
+ #{HEADER}return redis.call('SET', KEYS[1], '1', 'NX', 'PX', ARGV[1])
60
+ LUA
61
+
62
+ # KEYS = [sorted_set], ARGV = [score, member]
63
+ ZADD = <<~LUA.freeze
64
+ #{HEADER}return redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2])
65
+ LUA
66
+
67
+ module_function
68
+
69
+ # `error` is the OOM that sent the caller here, raised again if the server
70
+ # cannot run the script. Returns [queue_index, payload] or nil.
71
+ def claim(conn, error, keys:, info:, beat:)
72
+ run(conn, error, CLAIM, [Keys::PROCESSES, Component.identity, *keys], [info, beat, HEARTBEAT_TTL])
73
+ end
74
+
75
+ def reclaim(conn, error, source, destination)
76
+ run(conn, error, RECLAIM, [source, destination], [])
77
+ end
78
+
79
+ # Returns 1 when moved, 0 when the job has left the private list since.
80
+ def requeue(conn, error, private_list, public_q, job)
81
+ run(conn, error, REQUEUE, [private_list, public_q], [job])
82
+ end
83
+
84
+ def lock(conn, error, key, ttl_ms)
85
+ run(conn, error, LOCK, [key], [ttl_ms])
86
+ end
87
+
88
+ def zadd(conn, error, key, score, member)
89
+ run(conn, error, ZADD, [key], [score, member])
90
+ end
91
+
92
+ def run(conn, error, script, keys, argv)
93
+ conn.call('EVAL', script, keys.size, *keys, *argv)
94
+ rescue RedisClient::ConnectionError
95
+ raise
96
+ rescue RedisClient::CommandError
97
+ raise error
98
+ end
99
+ private_class_method :run
100
+ end
101
+ end
102
+ end
@@ -2,65 +2,81 @@
2
2
 
3
3
  module Wurk
4
4
  class Fetcher
5
- # The Reaper's boot-grace check: has this heartbeat-judged orphan been
6
- # left alone for at least `grace` seconds? A booting worker can claim a job
7
- # a moment before its first heartbeat lands, and draining its list in that
8
- # window runs the job twice, concurrently.
5
+ # The Reaper's wait between judging a list orphaned by its heartbeat and
6
+ # draining it.
9
7
  #
10
- # The answer comes from OBJECT IDLETIME, which a live owner resets on every
11
- # claim and ACK. Some servers refuse it — Redis under an LFU maxmemory
12
- # policy, Dragonfly with no OBJECT at all — and the grace must not quietly
13
- # vanish with it. Without IDLETIME the clock is this reaper's own: a list
14
- # is drained only once it has been judged orphaned for `grace` seconds
15
- # across sweeps, by which time a booting owner has beaten and the
16
- # heartbeat re-check spares it. The switch is logged once.
8
+ # A live owner can read as having no heartbeat hash: its first beat failed on
9
+ # a Redis blip, CPU-bound jobs held the beat back, or Redis restarted from an
10
+ # older snapshot. Its next beat lands within the wait, and that spares it.
11
+ #
12
+ # The clock is this process's own and it is shared by nothing else, so it
13
+ # costs the same on every server and at every fleet size. That is the
14
+ # difference from what it replaces, OBJECT IDLETIME: any read of the list
15
+ # (a monitor's LLEN) reset that clock, some servers do not have the command,
16
+ # and a clock kept per sweep only advanced in a process that won the sweep
17
+ # lock twice. The loop thread therefore finishes what any sweep here started,
18
+ # with or without the lock (Reaper#drain_settled).
17
19
  class OrphanGrace
18
- NO_IDLETIME_WARNING = 'reaper: OBJECT IDLETIME is unavailable on this Redis (%s); the boot grace ' \
19
- 'before reclaiming a private list now runs on this process\'s own clock, so an ' \
20
- 'orphan is drained one sweep later than it would be. Expected under an LFU ' \
21
- 'maxmemory policy or on servers without OBJECT (e.g. Dragonfly).'
22
-
23
- # Lists remembered for the fallback clock. A list leaves on the pass
24
- # that clears it; one judged orphaned and then found alive can linger,
25
- # so the memory is bounded and simply restarts when full — every clock
26
- # restarting is the conservative direction.
20
+ # Lists remembered at once. Past it the memory restarts, and every clock
21
+ # restarting is the conservative direction: the next sweep judges them anew.
27
22
  MEMORY = 10_000
28
23
 
29
- def initialize(config, grace)
30
- @config = config
24
+ Entry = Struct.new(:since, :owner)
25
+
26
+ def initialize(grace)
31
27
  @grace = grace
32
- @idletime = true
33
- @first_seen = {}
28
+ @lock = ::Mutex.new
29
+ @entries = {}
34
30
  end
35
31
 
36
- # true while the list must be left alone.
37
- def recent?(key)
32
+ # true while the list must be left alone. `owner` is what #take_settled
33
+ # hands back when the wait is over.
34
+ def recent?(key, owner)
38
35
  return false unless @grace.positive?
39
- return local_recent?(key) unless @idletime
40
36
 
41
- idle = @config.redis(idempotent: true) { |c| c.call('OBJECT', 'IDLETIME', key) }
42
- # nil: the list is already gone, nothing to drain.
43
- idle.nil? || idle < @grace
44
- rescue RedisClient::CommandError => e
45
- idletime_unavailable!(e)
46
- local_recent?(key)
37
+ now = monotonic
38
+ @lock.synchronize do
39
+ @entries.clear if @entries.size >= MEMORY && !@entries.key?(key)
40
+ entry = (@entries[key] ||= Entry.new(now, owner))
41
+ next true if now - entry.since < @grace
42
+
43
+ @entries.delete(key)
44
+ false
45
+ end
47
46
  end
48
47
 
49
- private
48
+ # The owner came back: its next judgement starts a new wait.
49
+ def forget(key)
50
+ @lock.synchronize { @entries.delete(key) }
51
+ end
50
52
 
51
- def local_recent?(key)
52
- now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
53
- @first_seen.clear if @first_seen.size >= MEMORY && !@first_seen.key?(key)
54
- seen = (@first_seen[key] ||= now)
55
- return true if now - seen < @grace
53
+ # [[key, owner], ...] whose wait is over, removed from memory.
54
+ def take_settled
55
+ now = monotonic
56
+ @lock.synchronize do
57
+ settled = @entries.select { |_, entry| now - entry.since >= @grace }
58
+ settled.each_key { |key| @entries.delete(key) }
59
+ settled.map { |key, entry| [key, entry.owner] }
60
+ end
61
+ end
62
+
63
+ # Seconds until the next wait ends, nil with nothing waiting.
64
+ def next_due_in
65
+ now = monotonic
66
+ @lock.synchronize do
67
+ oldest = @entries.each_value.map(&:since).min
68
+ oldest && [(oldest + @grace) - now, 0].max
69
+ end
70
+ end
56
71
 
57
- @first_seen.delete(key)
58
- false
72
+ def size
73
+ @lock.synchronize { @entries.size }
59
74
  end
60
75
 
61
- def idletime_unavailable!(error)
62
- @idletime = false
63
- @config.logger.warn { format(NO_IDLETIME_WARNING, error.message.lines.first&.strip) }
76
+ private
77
+
78
+ def monotonic
79
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
64
80
  end
65
81
  end
66
82
  end
@@ -38,19 +38,6 @@ module Wurk
38
38
  nil
39
39
  end
40
40
 
41
- # `<public_q>|<host>|<pid>|<nonce>|<idx>` → [host, pid, nonce] (pid as
42
- # Integer), or all-nil when the suffix isn't a well-formed owner tail.
43
- # Splitting the suffix off the known public-queue prefix tolerates a `|`
44
- # inside the queue name itself, and leaves the tail unambiguous: exactly
45
- # 4 segments for the current shape, exactly 3 for the pre-nonce one.
46
- def parse_owner(public_q, key)
47
- suffix = key.delete_prefix("#{public_q}|")
48
- return [nil, nil, nil] if suffix == key
49
-
50
- host, pid, nonce = owner_tails(suffix.split('|')).first
51
- [host, pid, nonce]
52
- end
53
-
54
41
  # Owner segments of a private-list key, taken from the right, as
55
42
  # [host, pid, nonce, segment_count] readings in preference order (empty
56
43
  # when nothing parses). The wide shape is preferred: an all-digit nonce is
@@ -67,9 +54,7 @@ module Wurk
67
54
  # The nonce slot must carry PROCESS_NONCE's exact shape (NONCE_PATTERN):
68
55
  # on a pre-nonce key read wide it holds the real pid, which is never 12
69
56
  # hex, so the guard is what falls such a key through to narrow_tail
70
- # instead of misreading it. Guarded here rather than in parse_full
71
- # because parse_owner reads through owner_tails too — one guard, and the
72
- # two parsers agree on every input.
57
+ # instead of misreading it.
73
58
  def wide_tail(parts)
74
59
  [parts[-4], parts[-3].to_i, parts[-2], 4] if parts.size >= 4 && integer?(parts[-3]) && nonce?(parts[-2])
75
60
  end