wurk 1.7.5 → 1.8.0

data/lib/wurk/fetcher/reliable.rb

@@ -7,6 +7,7 @@ require_relative '../keys'
 require_relative '../lua'
 require_relative '../fetcher'
 require_relative 'capped'
+require_relative 'many_queues'
 
 module Wurk
   class Fetcher
@@ -23,10 +24,13 @@ module Wurk
     # the paths that must send a held ACK before they stop fetching, and
     # docs/idea/parity-divergences.md for the window that widens.
     #
-    # Priority handling: iterate queues_cmd in order with non-blocking
-    # LMOVE, then fall back to a blocking BLMOVE on the first queue so an
-    # empty poll doesn't spin Redis. BLMOVE has no multi-key form, so
-    # blocking on a single queue is the best Redis gives us. The block
+    # Priority handling: walk queues_cmd in order with a non-blocking claim —
+    # an LMOVE on the first queue, then one fetch_first script call over all
+    # the others (ManyQueues) — then fall back to a blocking BLMOVE
+    # on the first queue so an empty poll doesn't spin Redis. BLMOVE has no
+    # multi-key form, so blocking on a single queue is the best Redis gives us:
+    # a job landing on a later queue while every thread is parked waits for
+    # the next pass, at most one block timeout. The block
     # timeout defaults to TIMEOUT (2s) and is overridable per the Pro
     # super_fetch §3.3 `config.fetch_poll_interval` knob.
     #
@@ -40,6 +44,7 @@ module Wurk
       # that caps nothing, which is a stronger guarantee that the unconfigured
       # path is unchanged than a second loop kept in step by hand.
       prepend Capped
+      include ManyQueues
 
       # Default BLMOVE block timeout; overridable via config.fetch_poll_interval.
       TIMEOUT = 2
@@ -72,7 +77,7 @@ module Wurk
       # namespaces are in play: a restarted container reuses both, so the
       # reaper's `kill(0)` liveness check would read a dead owner's list as
       # live (jobs stranded) or a live owner's as dead (job run twice). Keys
-      # written before the nonce existed stay reclaimable — Reaper#parse_owner
+      # written before the nonce existed stay reclaimable — PrivateListKey
       # accepts both shapes.
       def self.private_queue_name(public_queue, index = 0)
         host = ENV['DYNO'] || Socket.gethostname
@@ -112,6 +117,7 @@ module Wurk
         @queue_keys_pid = ::Process.pid
         @prefixed_queues = nil
         @prefixed_source = nil
+        @claim_keys = nil
       end
 
       # Take custody of a finished job's LREM instead of sending it now. One
@@ -246,20 +252,20 @@ module Wurk
         config.config.handle_exception(ex, ctx)
       end
 
-      # Non-blocking pass over the fetchable queues. The pending ACK rides the
-      # first LMOVE and only the first: the walk is one fetch, so a second copy
-      # of the same LREM would be a wasted command on every queue we find
-      # empty. By the time we fall through to BLMOVE this thread is holding
-      # nothing — which is the point, since a blocking call can't join a
-      # pipeline and would strand the ACK for a whole poll interval.
+      # Non-blocking pass over the fetchable queues, then the blocking
+      # fall-through. The pending ACK rides the first LMOVE and only the first.
+      # By the time we fall through to BLMOVE this thread is holding nothing —
+      # which is the point, since a blocking call can't join a pipeline and
+      # would strand the ACK for a whole poll interval.
+      #
+      # The first queue is always a bare LMOVE, so the busy-first-queue steady
+      # state (and the one-queue capsule `rake bench` gates) costs exactly what
+      # it did. Only a miss there pays for the rest — see ManyQueues.
       def walk(queues)
         ack = take_pending_ack
-        queues.each do |public_q|
-          uow = lmove(public_q, ack)
-          ack = nil
-          return uow if uow
-        end
-        blmove(queues.first)
+        uow = lmove(queues.first, ack)
+        uow ||= claim_rest(queues) if queues.size > 1
+        uow || blmove(queues.first)
       end
 
       # Exactly one, even when a failed flush left this thread holding two: the

data/lib/wurk/fetcher/unit_of_work.rb

@@ -98,8 +98,26 @@ module Wurk
           config.redis(idempotent: true) { |conn| conn.call('ZREM', slot_key, slot_token) }
         end
 
+        # Puts a claimed job back without running it — the public half of the
+        # Sidekiq UnitOfWork contract that throttling/rate-limiting gems call
+        # (sidekiq-throttled's requeue path, anything holding a BasicFetch UoW,
+        # which is this one under the `Sidekiq::BasicFetch` alias). Sidekiq's
+        # BasicFetch has no private list, so its bare RPUSH is the whole move;
+        # here the private copy has to go in the same step, or the job sits in
+        # both lists and runs twice — once from the public queue, once when the
+        # reaper reclaims this process's private list.
+        #
+        # LREM then RPUSH in one MULTI, unconditionally: unlike bulk_requeue
+        # (which races a Processor ACKing the same UoW and so guards the push on
+        # the LREM), the caller here owns the unit, and a job whose ACK already
+        # went out must still be re-queued rather than dropped.
         def requeue
-          config.redis { |conn| conn.call('RPUSH', queue, job) }
+          config.redis do |conn|
+            conn.multi do |tx|
+              tx.call('LREM', private_queue, LREM_COUNT, job)
+              tx.call('RPUSH', queue, job)
+            end
+          end
         end
       end
     end

data/lib/wurk/fetcher/unparseable_keys.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative '../keys'
+
+module Wurk
+  class Fetcher
+    # The Reaper's report on a `queue:*|*` key that is not a private list it
+    # can read the owner of — a Sidekiq Pro super_fetch list in a shape Wurk
+    # does not write, say, left behind by a live migration. Nothing will ever
+    # reclaim its jobs, so it is said once per key per process (WARN, with the
+    # list's length) rather than skipped in silence on every sweep.
+    #
+    # A public queue whose own name contains `|` matches the same SCAN pattern,
+    # so a key whose name is in the `queues` SET is not reported.
+    class UnparseableKeys
+      # Distinct keys remembered as already reported. Bounded so a keyspace
+      # full of odd keys cannot grow it without limit; past the bound the
+      # memory is cleared and keys are reported again.
+      MEMORY = 1000
+
+      def initialize(config)
+        @config = config
+        @reported = ::Set.new
+      end
+
+      def report(key)
+        return if @reported.include?(key)
+
+        public_queue, size = inspect_key(key)
+        return if public_queue
+
+        @reported.clear if @reported.size >= MEMORY
+        @reported << key
+        @config.logger.warn do
+          "reaper: #{key.inspect} looks like a reliable-fetch private list but does not match " \
+            "queue:<queue>|<host>|<pid>|<nonce>|<index>; its #{size} job(s) will not be recovered " \
+            'automatically. Inspect it and LMOVE them back onto their queue by hand if they belong to a dead process.'
+        end
+      rescue StandardError => e
+        @config.handle_exception(e, context: 'wurk-reaper')
+      end
+
+      private
+
+      def inspect_key(key)
+        member, size = @config.redis(idempotent: true) do |conn|
+          conn.pipelined do |pipe|
+            pipe.call('SISMEMBER', Keys::QUEUES_SET, key.delete_prefix(Keys::QUEUE_PREFIX))
+            pipe.call('LLEN', key)
+          end
+        end
+        [member == 1, size]
+      end
+    end
+  end
+end

data/lib/wurk/flow/completion.rb

@@ -59,19 +59,44 @@ module Wurk
       # spent: a replay after a lost reply finds the node already out of the
       # state it claims from and writes nothing, so the pool may retry a
       # connection error rather than give up on a callback that fires once.
+      #
+      # The graph around the node is read first so the script is handed every
+      # key it may write as a declared key (Redis Cluster and Dragonfly refuse
+      # one built inside Lua). Safe to read ahead: a node's jid, its
+      # dependents, and their bids and queues are all fixed at creation; the
+      # counters and states the decision turns on are read inside the script.
       def advance(fid, index)
         Wurk.redis(idempotent: true) do |conn|
+          node_key = Keys.flow_node(fid, index)
+          jid, dependents = conn.call('HMGET', node_key, 'jid', 'dependents')
+          deps = dependents ? Wurk.load_json(dependents) : []
           Wurk::Lua::Loader.eval_cached(
             conn, :flow_advance,
-            keys: [Keys.flow(fid), 'queues'],
-            argv: [index, now_seconds, now_millis, Keys::STATUS_PREFIX]
+            keys: [Keys.flow(fid), 'queues', Keys::SCHEDULE, Keys.flow_dead(fid), node_key,
+                   Keys.status(jid.to_s), *dependent_keys(conn, fid, deps)],
+            argv: [index, now_seconds, now_millis, *deps.map(&:to_s)]
           )
         end
       end
 
+      # Four keys per dependent, in `deps` order: its record, its batch, the
+      # batch's live-jid set, and the queue it is released onto.
+      def dependent_keys(conn, fid, deps)
+        return [] if deps.empty?
+
+        rows = conn.pipelined { |pipe| deps.each { |d| pipe.call('HMGET', Keys.flow_node(fid, d), 'bid', 'queue') } }
+        deps.zip(rows).flat_map do |d, (bid, queue)|
+          [Keys.flow_node(fid, d), "b-#{bid}", "b-#{bid}-jids", Keys.queue(queue)]
+        end
+      end
+
       def mark_dead(fid, index)
         Wurk.redis(idempotent: true) do |conn|
-          Wurk::Lua::Loader.eval_cached(conn, :flow_fail, keys: [Keys.flow(fid)], argv: [index, now_seconds])
+          Wurk::Lua::Loader.eval_cached(
+            conn, :flow_fail,
+            keys: [Keys.flow(fid), Keys.flow_node(fid, index), Keys.flow_dead(fid)],
+            argv: [index, now_seconds]
+          )
         end
       end
 

data/lib/wurk/flow/creation.rb

@@ -160,13 +160,24 @@ module Wurk
       # pool may retry a connection error that a non-idempotent write would have
       # to give up on.
       def write(payloads)
-        keys = [Keys.flow(@flow.fid), Keys::FLOWS_SET, 'batches', 'queues']
+        keys = [Keys.flow(@flow.fid), Keys::FLOWS_SET, 'batches', 'queues', Keys::SCHEDULE, *node_keys(payloads)]
         argv = script_argv(payloads)
         Wurk.redis(idempotent: true) do |conn|
           Wurk::Lua::Loader.eval_cached(conn, :flow_create, keys: keys, argv: argv)
         end
       end
 
+      # Four declared keys per node, in the same topological order as the
+      # envelopes in {#script_argv}: its record, its batch, the batch's live-jid
+      # set, and its queue.
+      def node_keys(payloads)
+        @flow.nodes.flat_map do |node|
+          payload = payloads[node.index]
+          bid = payload['bid']
+          [Keys.flow_node(@flow.fid, node.index), "b-#{bid}", "b-#{bid}-jids", Keys.queue(payload['queue'])]
+        end
+      end
+
       def script_argv(payloads)
         now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
         cutoff, rank = Wurk::Batch.trim_bounds
@@ -185,19 +196,28 @@ module Wurk
       # `enqueued_at` marks arrival on an immediate queue, so only the nodes
       # this write actually queues carry one. A waiting node's stored payload
       # gets its stamp from the push that releases it.
+      #
+      # A node declared with `at:` is deferred, root or not, so its payload is
+      # stored as the `schedule` member Client#push would write — `at` is the
+      # score, and the promoter stamps `enqueued_at` — and the epoch rides
+      # beside it for whichever script releases the node.
       def envelope(node, payload)
-        payload['enqueued_at'] = now_in_millis if node.root?
+        payload['enqueued_at'] = now_in_millis if node.root? && !payload.key?('at')
         Wurk.dump_json(node_fields(node, payload).merge(edge_fields(node)))
       end
 
       def node_fields(node, payload)
-        json = Wurk.dump_json(payload)
+        json = stored_payload(payload)
         { 'i' => node.index.to_s, 'name' => node.name.to_s, 'class' => payload['class'],
           'queue' => payload['queue'], 'jid' => payload['jid'], 'bid' => payload['bid'],
-          'state' => node.root? ? ENQUEUED : WAITING, 'payload' => json,
+          'state' => node.root? ? ENQUEUED : WAITING, 'payload' => json, 'at' => payload['at'].to_s,
           'desc' => node.label, 'cb' => callbacks_json(node), 'pipe' => pipe_field(node, json) }
       end
 
+      def stored_payload(payload)
+        payload.key?('at') ? JobUtil.scheduled_member(payload) : Wurk.dump_json(payload)
+      end
+
       # Empty for an ordinary node, and the sentinel — exactly as it appears in
       # the stored bytes — for a piped one, which is both the flag the release
       # tests and the needle it splices at.

data/lib/wurk/flow.rb

@@ -179,14 +179,36 @@ module Wurk
       def abandon(fid) # rubocop:disable Naming/PredicateMethod
         now = ::Process.clock_gettime(::Process::CLOCK_REALTIME).to_s
         released = Wurk.redis(idempotent: true) do |conn|
+          bids = node_bids(conn, fid)
           Wurk::Lua::Loader.eval_cached(
             conn, :flow_abandon,
-            keys: [Keys.flow(fid), 'batches', 'dead-batches'],
-            argv: [now, *Wurk::Batch::KEY_SUFFIXES]
+            keys: [Keys.flow(fid), 'batches', 'dead-batches', *abandoned_keys(fid, bids)],
+            argv: [now, *bids.compact]
           )
         end
         released.to_i >= 0
       end
+
+      private
+
+      # Every node's bid, in index order, nil where a record is gone. Read ahead
+      # of the script so it can be handed declared keys: a node's bid is fixed at
+      # creation, and the script's own state claim still decides whether
+      # anything is released.
+      def node_bids(conn, fid)
+        total = conn.call('HGET', Keys.flow(fid), 'total').to_i
+        return [] if total.zero?
+
+        conn.pipelined { |pipe| total.times { |i| pipe.call('HGET', Keys.flow_node(fid, i), 'bid') } }
+      end
+
+      def abandoned_keys(fid, bids)
+        keys = bids.each_with_index.flat_map do |bid, i|
+          batch = bid ? ["b-#{bid}", *Wurk::Batch::KEY_SUFFIXES.map { |suffix| "b-#{bid}-#{suffix}" }] : []
+          [Keys.flow_node(fid, i), *batch]
+        end
+        keys << Keys.flow_dead(fid)
+      end
     end
 
     def initialize(&block)

data/lib/wurk/health.rb

@@ -2,6 +2,7 @@
 
 require 'socket'
 require 'json'
+require_relative 'metrics/prometheus'
 
 module Wurk
   # Thin HTTP listener for k8s liveness/readiness probes. Optional, off by
@@ -10,10 +11,16 @@ module Wurk
   # Endpoints:
   #   * GET /live  → 200 while the Launcher is running (not in quiet/stop).
   #   * GET /ready → 200 only when Redis is reachable AND the heartbeat has
-  #                  fired within `ready_window` seconds. 503 otherwise.
+  #                  fired within `ready_window` seconds AND, inside a swarm
+  #                  child, enough of the swarm's children have (see
+  #                  Health::Fleet). 503 otherwise.
+  #   * GET /metrics → Prometheus text exposition (Wurk::Metrics::Prometheus).
+  #                  On whenever the listener is; `metrics: false` in
+  #                  `:health_check_options` turns it into a 404.
   # Anything else returns 404 JSON.
   #
-  # The server uses a raw TCPServer and one accept thread. No Rack, no
+  # The server uses a raw TCPServer, one accept thread and a short-lived
+  # thread per connection (bounded, deadline-capped). No Rack, no
   # dependencies — it lives inside every worker process where Rails may or
   # may not exist (standalone CLI, Embedded, swarm child). Bound to a
   # dedicated port so it does not collide with the host application's HTTP.
@@ -23,6 +30,19 @@ module Wurk
     DEFAULT_PORT          = 7433
     DEFAULT_BIND          = '0.0.0.0'
     DEFAULT_READY_WINDOW  = 30
+    JSON_TYPE             = 'application/json'
+
+    class << self
+      # Children the swarm was configured with, set by Swarm::ChildBoot inside
+      # each child (inherited from nowhere: the parent never runs a listener).
+      # nil outside a swarm child — standalone and embedded processes are a
+      # fleet of one and /ready judges them on their own heartbeat alone.
+      attr_accessor :fleet_size
+
+      def json(status, **extra)
+        ::JSON.generate({ status: status }.merge(extra))
+      end
+    end
 
     # The HTTP listener. Owns one TCPServer + one accept thread. Idempotent
     # start/stop; safe to call from Launcher#run / Launcher#stop.
@@ -32,6 +52,13 @@ module Wurk
       # that probes come back quickly after the owner dies, long enough not to
       # spin. See #start_retry_loop.
       RETRY_INTERVAL = 5
+      # Overall budget for one request head. A kubelet probe sends its head in
+      # one segment; anything slower is broken or hostile.
+      REQUEST_TIMEOUT   = 1.0
+      MAX_REQUEST_BYTES = 8192
+      MAX_CONNECTIONS   = 16
+      REASONS = { 200 => 'OK', 404 => 'Not Found', 405 => 'Method Not Allowed',
+                  503 => 'Service Unavailable' }.freeze
 
       attr_reader :port, :bind
 
@@ -47,6 +74,9 @@ module Wurk
         @thread         = nil
         @retry_thread   = nil
         @done           = false
+        @slots          = ::Mutex.new
+        @connections    = 0
+        @fleet          = nil
       end
 
       def start
@@ -133,7 +163,7 @@ module Wurk
 
           begin
             client, _addr = @server.accept_nonblock(exception: false)
-            handle(client) if client
+            dispatch(client) if client
           rescue ::IO::WaitReadable
             next
           rescue ::StandardError => e
@@ -145,41 +175,105 @@ module Wurk
         # Server was closed during shutdown — expected.
       end
 
-      def handle(client) # rubocop:disable Metrics/AbcSize
-        return unless client.wait_readable(1.0)
+      # Each connection gets its own short-lived thread so a client that
+      # dribbles bytes (slowloris) cannot hold the accept loop — and with it
+      # every kubelet probe — hostage. Past MAX_CONNECTIONS the socket is
+      # closed unanswered rather than queued: a probe that loses that race
+      # retries, an attacker gains nothing.
+      def dispatch(client)
+        return client.close unless claim_slot
+
+        spawn_handler(client)
+      rescue ::ThreadError
+        release_slot
+        client.close
+      end
 
-        request_line = client.gets("\r\n")
-        return if request_line.nil?
+      def spawn_handler(client)
+        ::Thread.new(client) do |sock|
+          ::Thread.current.name = 'wurk-health-conn'
+          ::Thread.current.report_on_exception = false
+          handle(sock)
+        ensure
+          release_slot
+        end
+      end
 
-        method, path, = request_line.strip.split(' ', 3)
-        # Drain remaining headers; ignore the body (probes don't send one).
-        # Wait for readability before each gets so a stalled client can't
-        # block the single accept thread mid-headers.
-        loop do
-          break unless client.wait_readable(1.0)
-
-          line = client.gets("\r\n")
-          break if line.nil? || line == "\r\n"
+      def claim_slot
+        @slots.synchronize do
+          return false if @connections >= MAX_CONNECTIONS
+
+          @connections += 1
+          true
         end
+      end
 
-        body, status = response_for(method, path)
-        write_response(client, status, body)
+      def release_slot
+        @slots.synchronize { @connections -= 1 }
+      end
+
+      def handle(client)
+        request_line = read_request_line(client)
+        return if request_line.nil?
+
+        method, path, = request_line.strip.split(' ', 3)
+        body, status, type = response_for(method, path)
+        write_response(client, status, body, type || JSON_TYPE)
       rescue ::StandardError => e
         logger&.error { "Wurk::Health request: #{e.class}: #{e.message}" }
       ensure
         client&.close
       end
 
+      # Reads the head under one overall deadline and a MAX_REQUEST_BYTES cap.
+      # Headers are read only to drain them (closing a socket with unread bytes
+      # makes the kernel answer RST, which can eat the response); the answer
+      # depends on the request line alone, so a client that sends that line
+      # and then stalls is still answered when the deadline expires.
+      def read_request_line(client)
+        buffer = +''
+        deadline = monotonic + REQUEST_TIMEOUT
+        until buffer.include?("\r\n\r\n") || buffer.bytesize >= MAX_REQUEST_BYTES
+          chunk = read_chunk(client, MAX_REQUEST_BYTES - buffer.bytesize, deadline)
+          break unless chunk
+
+          buffer << chunk
+        end
+        buffer[/\A[^\r\n]*(?=\r\n)/]
+      end
+
+      # nil once the deadline passes or the peer closes.
+      def read_chunk(client, max_bytes, deadline)
+        remaining = deadline - monotonic
+        return unless remaining.positive? && client.wait_readable(remaining)
+
+        chunk = client.read_nonblock(max_bytes, exception: false)
+        chunk == :wait_readable ? '' : chunk
+      end
+
+      def monotonic
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
       def response_for(method, path)
         return [json('error', message: 'method not allowed'), 405] unless method == 'GET'
 
         case path
         when '/live'  then live_response
         when '/ready' then ready_response
-        else [json('error', message: 'not found', path: path), 404]
+        when '/metrics' then fleet.metrics_response || not_found(path)
+        else not_found(path)
         end
       end
 
+      def not_found(path)
+        [json('error', message: 'not found', path: path), 404]
+      end
+
+      def fleet
+        @fleet ||= Fleet.new(@config, @ready_window)
+      end
+
       def live_response
         if @launcher.stopping?
           [json('down', check: 'live', reason: 'stopping'), 503]
@@ -189,15 +283,10 @@ module Wurk
       end
 
       def ready_response
-        redis_ok = ping_redis
-        beat_fresh = heartbeat_fresh?
+        return [json('down', check: 'ready', reason: 'redis unreachable'), 503] unless ping_redis
+        return [json('down', check: 'ready', reason: 'heartbeat stale'), 503] unless heartbeat_fresh?
 
-        if redis_ok && beat_fresh
-          [json('ok', check: 'ready'), 200]
-        else
-          reason = redis_ok ? 'heartbeat stale' : 'redis unreachable'
-          [json('down', check: 'ready', reason: reason), 503]
-        end
+        fleet.ready_response
       end
 
       def ping_redis
@@ -216,22 +305,13 @@ module Wurk
         (::Time.now.to_f - last) < @ready_window
       end
 
-      def json(status, **extra)
-        ::JSON.generate({ status: status }.merge(extra))
-      end
-
-      def write_response(client, status, body)
-        reason = case status
-                 when 200 then 'OK'
-                 when 404 then 'Not Found'
-                 when 405 then 'Method Not Allowed'
-                 when 503 then 'Service Unavailable'
-                 else 'Status'
-                 end
+      def json(...) = Health.json(...)
 
+      def write_response(client, status, body, type = JSON_TYPE)
+        reason = REASONS.fetch(status, 'Status')
         client.write(
           "HTTP/1.1 #{status} #{reason}\r\n" \
-          "Content-Type: application/json\r\n" \
+          "Content-Type: #{type}\r\n" \
           "Content-Length: #{body.bytesize}\r\n" \
           "Connection: close\r\n\r\n" \
           "#{body}"
@@ -242,5 +322,67 @@ module Wurk
         @config&.logger
       end
     end
+
+    # The swarm-wide half of the listener: the /metrics body and the fleet
+    # check behind /ready. One Prometheus collector serves both, so a scrape and
+    # a probe landing in the same second cost one Redis refresh between them.
+    #
+    # A swarm pod is one probe target however many children it runs, and the
+    # listener lives in whichever child won the port. Judging readiness on that
+    # child alone said "ready" while its siblings crash-looped. Ready now also
+    # needs `min_ready` children (default: half the fleet, rounded up) with a
+    # heartbeat inside `ready_window` — a majority-ish bar rather than all, so
+    # one slot in respawn backoff does not pull a pod that is still doing most
+    # of its work out of a rollout. /live stays local on purpose: failing it
+    # restarts the whole pod, and replacing one wedged child is the swarm
+    # parent's job (Swarm::Liveness).
+    class Fleet
+      def initialize(config, ready_window)
+        @config = config
+        @ready_window = ready_window
+        @collector = nil
+      end
+
+      # [body, status, content type]; nil when `metrics: false`.
+      def metrics_response
+        return nil if option(:metrics) == false
+
+        body = collector.render(expected_children: Health.fleet_size, fresh_window: @ready_window)
+        [body, 200, Metrics::Prometheus::CONTENT_TYPE]
+      end
+
+      # The /ready answer once the local checks (Redis, own heartbeat) passed.
+      def ready_response
+        expected = Health.fleet_size
+        return [Health.json('ok', check: 'ready'), 200] unless expected
+
+        needed = min_ready(expected)
+        fresh = collector.fresh_local(collector.snapshot, @ready_window).size
+        if fresh >= needed
+          [Health.json('ok', check: 'ready', children: fresh, expected: expected), 200]
+        else
+          [Health.json('down', check: 'ready', reason: 'too few live children', children: fresh,
+                               needed: needed, expected: expected), 503]
+        end
+      end
+
+      private
+
+      def collector
+        @collector ||= Metrics::Prometheus.new(@config)
+      end
+
+      def min_ready(expected)
+        configured = option(:min_ready)
+        return [Integer(configured), expected].min if configured
+
+        (expected / 2.0).ceil
+      end
+
+      def option(key)
+        opts = @config.respond_to?(:[]) ? @config[:health_check_options] : nil
+        opts.is_a?(Hash) ? opts[key] : nil
+      end
+    end
   end
 end