wurk 1.10.0 → 1.11.0

data/lib/wurk/cron.rb

@@ -450,6 +450,21 @@ module Wurk
         @pruned_klasses = {}
       end
 
+      # How late a slot may fire, in seconds, instead of being skipped, for every
+      # loop of this cluster's pollers. Default MISSED_TICK_THRESHOLD. A process-wide
+      # setting and not a loop option, which would enter the loop's identity.
+      #
+      # Coerced here, because the poller rescues per loop: a value it could not
+      # compare would stop every loop on every tick with nothing failing at boot.
+      def grace=(seconds)
+        grace = Float(seconds, exception: false)
+        unless grace&.finite? && grace.positive?
+          raise ArgumentError, "periodic grace must be a positive, finite number of seconds: #{seconds.inspect}"
+        end
+
+        (@config || Wurk.configuration)[:cron_grace] = grace
+      end
+
       def register(cron, klass, **opts)
         klass_name = klass.is_a?(String) ? klass : klass.to_s
         tz = opts.delete(:tz) || @tz
@@ -611,10 +626,12 @@ module Wurk
       end
     end
 
-    # Once-per-minute tick driver. Only the cluster leader iterates the LoopSet
-    # and enqueues; non-leaders return early. A slot that wall-clock has passed
-    # by more than `MISSED_TICK_THRESHOLD` seconds is skipped with a warning,
-    # never backfilled.
+    # Once-per-minute tick driver, plus one tick the moment this process wins the
+    # lock: at a tick's phase, a leader that lives under a minute would never tick.
+    # Only the cluster leader iterates the LoopSet and enqueues; non-leaders return
+    # early. A slot that wall-clock has passed by more than `MISSED_TICK_THRESHOLD`
+    # seconds is skipped with a warning, never backfilled, and the occurrence due
+    # now, if any, fires in its place.
     class Poller
       include Component
 
@@ -625,17 +642,22 @@ module Wurk
         # Operators never need to touch this; integration tests shrink it so a
         # due loop fires within the test window instead of waiting a full minute.
         @tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
-        @timer = TimerLoop.new(@tick_interval)
+        @done = false
+        @woken = false
+        @mutex = ::Mutex.new
+        @sleeper = ::ConditionVariable.new
+        @on_leader = method(:wake_for_leadership)
       end
 
-      # TimerLoop waits one interval before the first tick: don't fire a
-      # catch-up burst the instant we boot (the leader is barely settled), and
-      # let a short-lived process exit without ticking at all.
+      # The loop waits one interval before its first tick unless this process
+      # wins the lock first: let a process that never leads, or exits at once,
+      # exit without ticking at all.
       def start
         return @thread if @thread
 
-        @timer.reset
-        @thread = safe_thread('cron-poller') { @timer.run { tick } }
+        @mutex.synchronize { @done = false }
+        leader_hooks << @on_leader
+        @thread = safe_thread('cron-poller') { run_loop }
       end
 
       # Blocks until the thread is really gone: the launcher releases the
@@ -644,10 +666,14 @@ module Wurk
       #
       # Cleared only on a confirmed join (Thread#join returns nil on timeout):
       # a wedged thread must stay tracked so #start's guard returns it instead
-      # of calling @timer.reset, which would un-terminate the loop it is still
-      # inside and leave two tick threads double-enqueuing the same loops.
+      # of clearing @done, which would un-terminate the loop it is still inside
+      # and leave two tick threads double-enqueuing the same loops.
       def terminate
-        @timer.terminate
+        leader_hooks.delete(@on_leader)
+        @mutex.synchronize do
+          @done = true
+          @sleeper.signal
+        end
         @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
@@ -679,6 +705,10 @@ module Wurk
       # A slot older than the missed-tick threshold is not backfilled (ent §2.6:
       # no backfill on restart, missed ticks across a leader gap are dropped):
       # the marks still advance past it, but nothing is pushed.
+      #
+      # One claim, one push at most, whichever slot it pushes for: the occurrence
+      # due now stands in for a stale one under the stale one's mark, so a poller
+      # that read the same mark loses the CAS for both.
       def enqueue_if_due(loop_obj)
         return if loop_obj.paused?
 
@@ -686,9 +716,12 @@ module Wurk
         slot, mark = due_slot(loop_obj, now)
         return if slot.nil?
 
-        future = loop_obj.next_fire_after(slot, now)
+        firing = slot_to_fire(loop_obj, slot, now)
+        future = loop_obj.next_fire_after(firing || slot, now)
         return unless claim_fire?(loop_obj, mark, now, future)
-        return skip_missed(loop_obj, slot, now) if now - slot > missed_threshold
+
+        skip_missed(loop_obj, slot, now) unless firing == slot
+        return if firing.nil?
 
         jid = enqueue_claimed!(loop_obj)
         record_history(loop_obj, jid, now)
@@ -708,6 +741,53 @@ module Wurk
 
       private
 
+      def run_loop
+        wait_for_tick
+        until done?
+          tick
+          wait_for_tick
+        end
+      end
+
+      def wait_for_tick
+        @mutex.synchronize do
+          @sleeper.wait(@mutex, @tick_interval) unless @done || @woken
+          @woken = false
+        end
+      end
+
+      def done?
+        @mutex.synchronize { @done }
+      end
+
+      # Runs on the campaign thread. The gate's cached answer predates the win.
+      def wake_for_leadership
+        @leader_checked_at = nil
+        @mutex.synchronize do
+          @woken = true
+          @sleeper.signal
+        end
+      end
+
+      def leader_hooks
+        @config[:lifecycle_events].fetch(:leader)
+      end
+
+      def slot_to_fire(loop_obj, slot, now)
+        now - slot > missed_threshold ? latest_due(loop_obj, now) : slot
+      end
+
+      # The newest occurrence still inside the window a late slot may fire in.
+      def latest_due(loop_obj, now)
+        latest = nil
+        occurrence = loop_obj.next_fire_at(now - missed_threshold - 1)
+        while occurrence && occurrence <= now
+          latest = occurrence
+          occurrence = loop_obj.next_fire_at(occurrence)
+        end
+        latest
+      end
+
       # The slot this tick would fire and the token that claims it, or nil when
       # the loop has no resolvable occurrence or none is due yet. The token is
       # the stored `nf` when there is one, otherwise the slot we derived — see
@@ -723,7 +803,7 @@ module Wurk
       # Never tighter than a tick and a half: a raised tick interval must not
       # turn every on-time fire into a "missed" one.
       def missed_threshold
-        [MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
+        [@config[:cron_grace] || MISSED_TICK_THRESHOLD, @tick_interval * 1.5].max
       end
 
       def skip_missed(loop_obj, expected, now)

data/lib/wurk/embedded.rb

@@ -2,6 +2,7 @@
 
 require_relative 'component'
 require_relative 'launcher'
+require_relative 'swarm/test_mode_guard'
 
 module Wurk
   # Embeds Wurk inside an arbitrary Ruby process (Puma, rake task, custom
@@ -34,13 +35,13 @@ module Wurk
     # the worker threads have time to spin up before the host goes back to
     # serving traffic. Mirrors Sidekiq::Embedded#run.
     def run
+      Swarm::TestModeGuard.enforce!(logger)
       housekeeping
       fire_event(:startup, reverse: false, reraise: true)
       @launcher = build_launcher
       @launcher.run
       sleep 0.2
-      logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
-      logger.debug { Thread.list.map(&:name).to_s }
+      log_running
     rescue StandardError
       # The host is left believing Wurk never started, so anything the launcher
       # did get up would keep fetching, beating and campaigning for the leader
@@ -62,6 +63,11 @@ module Wurk
 
     private
 
+    def log_running
+      logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
+      logger.debug { Thread.list.map(&:name).to_s }
+    end
+
     # Guarded so the caller sees why the boot failed, not why the rollback did.
     def roll_back_partial_boot
       stop

data/lib/wurk/encryption.rb

@@ -83,6 +83,12 @@ module Wurk
     # is what callers see as the dead record's `error_class`.
     class DecryptionError < Error; end
 
+    # The job says `encrypt: true` but its last argument is not a Wurk envelope:
+    # another implementation's ciphertext, or a producer that never enabled
+    # crypto. Neither can be told from a plain value, and either way `perform`
+    # must not take it for the decrypted argument.
+    class NotSealedError < Error; end
+
     class << self
       attr_reader :active_version
 
@@ -228,7 +234,10 @@ module Wurk
           'failed_at' => ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
         )
         Wurk::Metrics::Statsd.increment('jobs.encryption_error', tags: ["worker:#{job['class']}"])
-        Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause)
+        Wurk.logger.error { "#{DEAD_REASON}: #{job['class']} jid=#{job['jid']} sent to the dead set: #{cause.message}" }
+        # No trim: a misconfigured producer or a lost key can fail a whole backlog at
+        # once, and the cap would destroy jobs that are recoverable once it is fixed.
+        Wurk::DeadSet.new.kill(Wurk.dump_json(record), ex: cause, trim: false)
         nil
       end
 
@@ -318,14 +327,19 @@ module Wurk
     # with every outer middleware and the processor; leaving plaintext in it
     # let any of them write the secret back to Redis (InterruptHandler's
     # re-push, limiter dead-routing) or hand it to an error reporter. A decrypt
-    # failure (missing/rotated key, bad tag) is terminal and non-retryable,
-    # so rather than let it bubble into the 25× retry pipeline we route the
+    # failure (missing/rotated key, bad tag, a last argument that is no envelope)
+    # is terminal and non-retryable, so rather than let it bubble into the 25× retry pipeline we route the
     # job straight to the dead set tagged `encryption_error` and ACK it via
     # JobRetry::Skip — see Wurk::Encryption.route_to_dead. Plaintext args
     # remain visible for triage per §4.6.
     class ServerMiddleware
       include Wurk::Middleware::ServerMiddleware
 
+      # Never quotes the argument: it may be a secret in the clear.
+      NOT_SEALED_MESSAGE = 'job has encrypt: true but its last argument is not a Wurk envelope, so it was not run. ' \
+                           'A producer without crypto enabled pushed it, or another implementation encrypted it. ' \
+                           'A retry re-seals the argument as it stands: retry only a job known to hold plaintext'
+
       def call(_worker, job, _queue)
         return yield unless Wurk::Encryption.enabled? && job['encrypt']
 
@@ -342,7 +356,8 @@ module Wurk
 
       def decrypt_last_arg!(job)
         args = job['args']
-        return unless args.is_a?(::Array) && !args.empty? && Wurk::Encryption.envelope?(args.last)
+        return unless args.is_a?(::Array) && !args.empty?
+        raise NotSealedError, NOT_SEALED_MESSAGE unless Wurk::Encryption.envelope?(args.last)
 
         job['args'] = args[0..-2] + [Wurk::Encryption.decrypt(args.last)]
       rescue Wurk::Encryption::Error,
@@ -350,7 +365,7 @@ module Wurk
              ::ArgumentError,
              ::TypeError,
              ::JSON::ParserError => e
-        # Wurk::Encryption::Error covers KeyMissingError / DecryptionError.
+        # Wurk::Encryption::Error covers KeyMissingError / DecryptionError / NotSealedError.
         # OpenSSL::Cipher::CipherError → tampered ciphertext / bad key (tag mismatch).
         # ArgumentError → Base64.strict_decode64 / Integer() on a malformed envelope.
         # TypeError    → Integer(nil) on a missing 'v' field.

data/lib/wurk/leader.rb

@@ -3,6 +3,7 @@
 require 'securerandom'
 require 'socket'
 require_relative 'component'
+require_relative 'fetcher/oom_drain'
 require_relative 'lua'
 require_relative 'pool_checkout'
 require_relative 'timer_loop'
@@ -21,8 +22,12 @@ module Wurk
   # *can* benefit from a guard has one available; the token is best-effort
   # too — it is never re-read on subsequent acquires, only on transitions.
   #
-  # Cadence per spec: renew every 15s while leader, recheck every 60s as
-  # follower, lock TTL 30s. The campaign itself starts `initial_wait` seconds
+  # Cadence per spec: renew every 15s while leader, lock TTL 30s. A follower
+  # rechecks when the lease it last read runs out, at most 60s later, and that
+  # is what bounds the leaderless time after a SIGKILL: the lease's remaining
+  # 15-30s and about a second. Waiting out a flat 60s added up to 60s more. The
+  # lock itself is untouched, so a follower still cannot win before the lease
+  # has lapsed. The campaign itself starts `initial_wait` seconds
   # after `#start` — the spec fixes the renew/follower/TTL cadence but says
   # nothing about when the first campaign runs, and holding it back keeps a
   # booting process's opening Redis round trips on the job path. Opt out a
@@ -43,6 +48,17 @@ module Wurk
     # nothing observes the gap. Tune with `config[:leader_initial_wait]`;
     # 0 campaigns immediately.
     DEFAULT_INITIAL_WAIT = 1
+    # Followers spread out over this many seconds past the lease's end, so a
+    # large fleet doesn't campaign in one instant.
+    LEASE_EXPIRY_JITTER = 1.0
+    # Keeps a lease read as about to lapse from turning into a busy loop.
+    MIN_FOLLOWER_WAIT = 0.05
+    # The renewal has to survive a full Redis: a lapsed lease stops every
+    # leader-gated consumer, and nothing would renew it until memory came back.
+    # Shebang scripts, so by EVAL for the reason Fetcher::OomDrain gives. A server
+    # refusing one raises the original OOM, the failure without this.
+    OOM_CAMPAIGN = "#{Fetcher::OomDrain::HEADER}#{Lua::SCRIPTS.fetch(:leader_campaign)}".freeze
+    OOM_TOKEN = "#{Fetcher::OomDrain::HEADER}return redis.call('INCR', KEYS[1])\n".freeze
     OPT_OUT_ENV = 'WURK_LEADER'            # native opt-out env
     SIDEKIQ_OPT_OUT_ENV = 'SIDEKIQ_LEADER' # Sidekiq Ent drop-in alias (§6.2/§7.2)
     THREAD_NAME = 'wurk-leader'
@@ -71,6 +87,8 @@ module Wurk
       @owner = owner || cluster_identity
       @held = false
       @token = nil
+      @lease_remaining = nil
+      @campaign_failed = false
       @thread = nil
       @done = false
       @mutex = ::Mutex.new
@@ -98,7 +116,7 @@ module Wurk
       return false unless transition
 
       if transition == :gained
-        @token = redis_call { |c| c.call('INCR', TOKEN_KEY) }
+        @token = redis_call { |c| next_token(c) }
         dispatch_leader_event
       end
       true
@@ -191,9 +209,7 @@ module Wurk
     # if another process currently holds the key. Sets `@held` accordingly
     # so `leader?` reflects the latest state regardless of branch.
     def run_set_or_refresh
-      won = redis_call do |c|
-        Wurk::Lua::Loader.eval_cached(c, :leader_campaign, keys: [@key], argv: [@owner, @ttl])
-      end
+      won = redis_call { |c| campaign(c) }
       if won.to_i == 1
         outcome = @held ? :held : :gained
         @held = true
@@ -201,9 +217,37 @@ module Wurk
       end
 
       step_down
+      read_lease_remaining
       nil
     end
 
+    def campaign(conn)
+      Wurk::Lua::Loader.eval_cached(conn, :leader_campaign, keys: [@key], argv: [@owner, @ttl])
+    rescue RedisClient::OutOfMemoryError => e
+      past_maxmemory(e) { conn.call('EVAL', OOM_CAMPAIGN, 1, @key, @owner, @ttl) }
+    end
+
+    def next_token(conn)
+      conn.call('INCR', TOKEN_KEY)
+    rescue RedisClient::OutOfMemoryError => e
+      past_maxmemory(e) { conn.call('EVAL', OOM_TOKEN, 1, TOKEN_KEY) }
+    end
+
+    def past_maxmemory(error)
+      yield
+    rescue RedisClient::ConnectionError
+      raise
+    rescue RedisClient::CommandError
+      raise error
+    end
+
+    # Seconds until the holder's lease runs out; nil for a lock with no TTL, which
+    # only expiry cannot free. A lock gone again already is a zero wait.
+    def read_lease_remaining
+      pttl = redis_call(idempotent: true) { |c| c.call('PTTL', @key) }
+      @lease_remaining = pttl == -1 ? nil : [pttl, 0].max / 1000.0
+    end
+
     def step_down
       @held = false
       @token = nil
@@ -283,10 +327,14 @@ module Wurk
     # would leave every leader-gated consumer acting as leader through an
     # outage long enough for a follower to take over, and would keep this loop
     # on the short renew cadence. Stepping down makes the next success a fresh
-    # gain — a new fencing token and a re-fired `:leader` event.
+    # gain — a new fencing token and a re-fired `:leader` event. It says nothing
+    # about the lease, so the retry is on the renew cadence, not the follower's:
+    # a leader that failed one renewal still has time to keep its lease.
     def tick_once
+      @campaign_failed = false
       acquire
     rescue StandardError => e
+      @campaign_failed = true
       step_down
       report(e)
     end
@@ -296,7 +344,13 @@ module Wurk
     end
 
     def wait_next
-      wait_for(@held ? @renew_interval : @follower_interval)
+      wait_for(@held || @campaign_failed ? @renew_interval : follower_wait)
+    end
+
+    def follower_wait
+      return @follower_interval unless @lease_remaining
+
+      [@lease_remaining + (rand * LEASE_EXPIRY_JITTER), @follower_interval].min.clamp(MIN_FOLLOWER_WAIT, nil)
     end
 
     def wait_for(interval)

data/lib/wurk/preflight/checks.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require_relative '../boot_checks'
+require_relative '../keys'
+require_relative 'payload_sample'
+
+module Wurk
+  module Preflight
+    # The questions a preflight answers, each reading Redis through the Reader
+    # and writing to the Report. Run before the cutover, so a live stock fleet is
+    # normal here (Verify is the one that expects Wurk).
+    class Checks
+      MIN_REDIS = ::Gem::Version.new('7.0.0')
+      DRAGONFLY_FULL_RATIO = 0.8
+
+      # Add-ons that are known not to work on Wurk, or that must be gone by cutover.
+      KNOWN_BROKEN = {
+        'sidekiq-pro' => 'licensed gem: remove it, Wurk replaces it',
+        'sidekiq-ent' => 'licensed gem: remove it, Wurk replaces it',
+        'sidekiq-status' => 'untested on Wurk, and its web tab expects stock ERB rendering; prefer `track:`',
+        'sidekiq-throttled' => 'patches stock fetch classes, which Wurk does not use; use Sidekiq::Limiter'
+      }.freeze
+
+      def initialize(reader:, report:, keyspace:, owners:, sample:)
+        @reader = reader
+        @report = report
+        @keyspace = keyspace
+        @owners = owners
+        @sample = sample
+      end
+
+      def run
+        server
+        processes
+        scan_coverage
+        private_lists
+        payloads
+        batches
+        locks
+      end
+
+      def app(specs: ::Gem.loaded_specs)
+        add_ons(specs)
+        ignored_locks
+      end
+
+      private
+
+      def server
+        info = parse_info(@reader.call('INFO'))
+        name, version = BootChecks.backend(info)
+        @report.section('redis', server_section(info, name, version))
+        version_check(name, version)
+        topology(info)
+        eviction(info['maxmemory_policy'], name)
+        dragonfly_memory(info) if name == 'dragonfly'
+      end
+
+      def server_section(info, name, version)
+        { 'backend' => name, 'version' => version, 'mode' => info['redis_mode'], 'role' => info['role'],
+          'eviction' => info['maxmemory_policy'], 'used_memory' => info['used_memory'].to_i,
+          'maxmemory' => info['maxmemory'].to_i, 'keys' => @reader.call('DBSIZE') }
+      end
+
+      def parse_info(text)
+        text.each_line.with_object({}) do |line, out|
+          key, value = line.strip.split(':', 2)
+          out[key] = value if value
+        end
+      end
+
+      def version_check(name, version)
+        return unless name == 'redis' && ::Gem::Version.correct?(version) && ::Gem::Version.new(version) < MIN_REDIS
+
+        @report.warning('redis_version', "Redis #{version} is below the required 7.0.")
+      end
+
+      def topology(info)
+        return unless info['cluster_enabled'] == '1' || info['redis_mode'] == 'cluster'
+
+        @report.warning('redis_cluster', 'Redis Cluster is not supported: `nodes:` raises. Use standalone or Sentinel.')
+      end
+
+      def eviction(policy, name)
+        return if policy == 'noeviction'
+
+        if policy.nil?
+          @report.info('eviction_unknown', "#{name} does not report maxmemory_policy: confirm it cannot evict keys.")
+        else
+          @report.warning('eviction', "maxmemory_policy is #{policy}: an evicting policy can silently drop queues.")
+        end
+      end
+
+      def dragonfly_memory(info)
+        used = info['used_memory'].to_f
+        max = info['maxmemory'].to_f
+        if max.positive? && used / max >= DRAGONFLY_FULL_RATIO
+          @report.warning('dragonfly_memory', "Dragonfly is at #{(100 * used / max).round}% of maxmemory. Near the " \
+                                              'limit it can drop a job mid-move: see docs/deployment.md.')
+        else
+          @report.info('dragonfly', 'Dragonfly: keep it well below maxmemory and alert on it. See docs/deployment.md.')
+        end
+      end
+
+      def processes # rubocop:disable Metrics/AbcSize
+        wurk, other = @owners.processes.partition(&:wurk)
+        by_version = @owners.processes.group_by { |p| "#{p.wurk ? 'wurk' : 'other'} #{p.version}" }
+        @report.section('processes', { 'live' => @owners.processes.size, 'stale_members' => @owners.stale.size,
+                                       'by_version' => by_version.transform_values(&:size) })
+        return unless wurk.any? && other.any?
+
+        @report.warning('mixed_fleet', "#{other.size} non-Wurk and #{wurk.size} Wurk process(es) share this Redis.")
+      end
+
+      def scan_coverage
+        @report.section('keyspace', @keyspace.to_h)
+        return unless @keyspace.truncated?
+
+        @report.warning('scan_truncated', "Stopped at #{@keyspace.scanned} keys: every count below is a lower bound " \
+                                          'and this run cannot call the keyspace clean. Raise --limit.')
+      end
+
+      def private_lists
+        lists = @owners.classify(@keyspace.private_lists)
+        examples = lists.transform_values { |keys| keys.first(10) }
+        @report.section('private_lists', lists.transform_values(&:size).merge('samples' => examples))
+        name_unparseable(lists['unparseable'])
+        return if lists['reclaimable'].empty?
+
+        @report.warning('reclaimable_lists', "#{lists['reclaimable'].size} private list(s) have no live owner. Wurk " \
+                                             'reclaims these shapes after its grace; stock Sidekiq never does.')
+      end
+
+      def name_unparseable(keys)
+        return if keys.empty?
+
+        @report.warning('unparseable_lists', "#{keys.size} key(s) look like private lists in a shape Wurk cannot " \
+                                             "read, and it will not reclaim them: #{keys.first(10).join(', ')}. " \
+                                             'Inspect them; if their owners are gone, move the jobs back by hand.')
+      end
+
+      def payloads
+        sampled = PayloadSample.new(@reader, per_source: @sample).take(@owners.queue_names)
+        @report.section('payloads', sampled.to_h)
+        sampled.report_to(@report, foreign: @owners.foreign.any?)
+      end
+
+      def batches # rubocop:disable Metrics/AbcSize
+        hashes = batch_hashes
+        fields = @reader.pipeline(hashes.map { |key| ['HMGET', key, 'pending', 'callbacks'] })
+        in_flight = hashes.zip(fields).select { |_, (pending, _)| pending.to_i.positive? }
+        bare = in_flight.reject { |_, (_, callbacks)| callbacks }.map(&:first)
+        @report.section('batches', { 'seen' => @keyspace.counts['batches'], 'inspected' => hashes.size,
+                                     'in_flight' => in_flight.size, 'without_callbacks' => bare.size })
+        return if bare.empty?
+
+        @report.warning('batch_without_callbacks',
+                        "#{bare.size} in-flight batch(es) have no `callbacks` field, e.g. #{bare.first}. Wurk finds " \
+                        'no callbacks to fire on these: let them finish before the cutover.')
+      end
+
+      def batch_hashes
+        rows = @keyspace.batches.first(@sample)
+        rows.zip(@reader.pipeline(rows.map { |key| ['TYPE', key] })).filter_map { |key, type| key if type == 'hash' }
+      end
+
+      def locks
+        counts = @keyspace.counts
+        @report.section('locks', counts.slice('unique_sha1', 'unique_sha256', 'unique_other', 'gem_locks', 'gem_keys'))
+        if counts['unique_sha1'].positive?
+          @report.warning('unique_sha1', "#{counts['unique_sha1']} `unique:` lock(s) carry a SHA1-length digest. " \
+                                         "Wurk's digest is SHA256, so it does not see them: one duplicate per " \
+                                         'identity until their TTL.')
+        end
+        return unless counts['gem_locks'].positive?
+
+        @report.info('gem_locks', "#{counts['gem_locks']} sidekiq-unique-jobs lock(s). Keep the gem and they carry " \
+                                  'over. Drop it and they stay held: UNLINK `uniquejobs:*` before any rollback.')
+      end
+
+      def add_ons(specs)
+        found = KNOWN_BROKEN.select { |name, _| specs.key?(name) }
+        real = specs['sidekiq']
+        @report.section('app', { 'gems_seen' => specs.size, 'known_broken' => found.keys })
+        found.each { |name, why| @report.warning('known_broken_gem', "#{name}: #{why}.") }
+        return unless real && !real.full_gem_path.include?('sidekiq-shim')
+
+        @report.warning('real_sidekiq_gem', "The bundle holds a released sidekiq #{real.version}: it would load next " \
+                                            'to Wurk. Use the shim gem. See docs/migrate-from-sidekiq.md.')
+      end
+
+      def ignored_locks
+        return if defined?(::SidekiqUniqueJobs)
+
+        classes = BootChecks.locked_job_classes
+        return if classes.empty?
+
+        @report.warning('ignored_lock', "#{classes.size} class(es) set `lock:` with sidekiq-unique-jobs not loaded, " \
+                                        "so duplicates run: #{BootChecks.listing(classes)}.")
+      end
+    end
+  end
+end