kicks_liveness 0.1.2 → 0.1.4

data/lib/kicks_liveness/monitor.rb

@@ -17,15 +17,18 @@ module KicksLiveness
       @consumers = consumers
       @config = config
       @heartbeat = heartbeat || Heartbeat.new
-      @attempts = Attempts.new(@heartbeat.dir)
+      @attempts = Attempts.new(@heartbeat.dir, generation: @heartbeat.topology.generation)
     end
 
     # Declares the expected fork count and starts the tick thread.
-    # @return [Thread]
+    # @return [Thread, nil] nil when the PID topology is proven unsupported
     def start!
+      return reject_topology! if @heartbeat.topology.unsafe?
+
       attempt, elapsed = @attempts.record!(@slot)
       @heartbeat.declare!(@processes)
       attempt > 1 ? report_respawn(attempt, elapsed) : report_first_start
+      report_topology
 
       Thread.new do
         Thread.current.name = 'kicks-liveness'
@@ -38,18 +41,11 @@ module KicksLiveness
     #
     # @return [Boolean] whether the process is healthy at this moment
     def tick!
-      # Re-declared on every tick, not only at startup. This file is the probe's
-      # only source for how many forks to expect, and nothing else restores it:
-      # if the directory is wiped, `touch!` brings back the slot marks while
-      # `expected` stays missing, and the probe reports "worker has not started"
-      # for the rest of the pod's life. It also lets a respawned set of forks
-      # correct the count after the supervisor was told to run fewer of them.
+      # Restore expected after directory removal; refreshing a slot alone
+      # cannot recover a probe whose expected count is missing.
       @heartbeat.declare!(@processes)
 
-      if Registry.stopping?
-        shutdown_tick!
-        return true
-      end
+      return shutdown_tick! if Registry.stopping?
 
       healthy = Registry.healthy?(@consumers)
       if healthy
@@ -63,26 +59,30 @@ module KicksLiveness
 
     private
 
+    # Publish unsafe evidence without recording a start or launching the loop.
+    def reject_topology!
+      # Reported before the state is published: a marks directory that cannot be
+      # written to is exactly the case where the log line is all the operator
+      # will get.
+      report_topology
+      @heartbeat.reject_topology!
+      nil
+    end
+
+    # Claim before logging so simultaneous forks cannot report the same state.
+    def report_topology
+      level, message = TopologyReport.new(@heartbeat.dir, @heartbeat.topology).claim!
+
+      log(level, message) if level
+    end
+
     def report_first_start
       log(:info, "started: dir=#{@heartbeat.dir} max_age=#{@heartbeat.max_age}s tick=#{@config.tick}s " \
                  "processes=#{@processes} consumers=#{@consumers}")
     end
 
-    # A slot that starts again without ever having become healthy is a respawn
-    # loop: the supervisor brings the fork back after `start_worker_delay`,
-    # which is a fraction of a second, so the process dies long before any
-    # single monitor's grace period expires. Two things follow.
-    #
-    # The start line and the first transition are suppressed. Seeding
-    # `@previous` is what does the second part: without it the first tick reads
-    # a nil previous state and logs `waiting for N consumers`, which in a storm
-    # means hundreds of identical INFO lines a minute and no diagnosis in any of
-    # them.
-    #
-    # The escalation is driven off the marks directory instead of off
-    # `unhealthy_ticks`, because that counter dies with the fork and can never
-    # reach the threshold. Reporting once per grace window rather than once per
-    # respawn is the difference between one line a minute and five a second.
+    # Seed @previous to suppress repeated waiting messages. Escalation uses the
+    # persistent attempt window because unhealthy_ticks dies with each fork.
     def report_respawn(attempt, elapsed)
       @previous = false
 
@@ -96,8 +96,7 @@ module KicksLiveness
       @config.startup_grace_ticks * interval
     end
 
-    # Once the slot is healthy the respawn counter has done its job, and the
-    # next start of this slot deserves to be treated as a fresh one.
+    # Clear once after recovery so a later failure starts a new unhealthy run.
     def clear_attempt_once!
       return if @attempt_cleared
 
@@ -109,27 +108,32 @@ module KicksLiveness
       @unhealthy_ticks ||= 0
     end
 
-    # From the moment shutdown begins the consumer count says nothing about
-    # health: the workers unsubscribe one by one while the pool drains, and the
-    # process is doing exactly what it was told. So the predicate is dropped and
-    # the mark is kept fresh, which leaves a slow drain the whole of
-    # terminationGracePeriodSeconds instead of having the probe cut it short at
-    # max_age.
-    def shutdown_tick!
+    # The first stop callback, not a delayed tick, starts the monotonic deadline.
+    # Expiry stops refreshes; ordinary mark staleness then fails the probe.
+    # @see file:docs/LIMITATIONS.md#graceful-shutdown-has-a-finite-heartbeat-exemption
+    def shutdown_tick! # rubocop:disable Naming/PredicateMethod -- Publishes the heartbeat and records transitions.
+      @shutdown_deadline ||= Registry.stopping_at + @config.shutdown_grace
+      report_shutdown_once
+
+      if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= @shutdown_deadline
+        log(:error, 'shutdown grace expired: heartbeat is no longer refreshed') unless @shutdown_expired
+        @shutdown_expired = true
+        return false
+      end
+
       @heartbeat.touch!(@slot)
+      true
+    end
 
+    def report_shutdown_once
       return if @shutdown_reported
 
       log(:info, 'shutting down: consumers are no longer checked')
       @shutdown_reported = true
     end
 
-    # Nothing inside this loop may be allowed to kill the thread. It is the only
-    # thing writing the mark, and its death is close to invisible: the exception
-    # surfaces as a bare backtrace on stderr, never through the gem's logger,
-    # while the pod restarts on every probe budget with nothing in the
-    # application log to explain it. `sleep` used to sit outside the guarded
-    # block, which is exactly how a non-positive tick killed the monitor.
+    # Guard tick and sleep together so an exception cannot kill the only
+    # heartbeat writer. Recovery uses the default interval.
     def run_loop
       loop do
         tick!
@@ -147,16 +151,13 @@ module KicksLiveness
       tick.is_a?(Numeric) && tick.positive? ? tick : Configuration::DEFAULT_TICK
     end
 
-    # Events are logged, not the pulse — the pulse lives in the mark's mtime.
-    # ERROR is reserved for genuine failure, because a logger usually comes up
-    # with LOG_LEVEL defaulting to error.
+    # Log transitions and startup escalation; the periodic pulse is the mtime.
     def report(healthy)
       if healthy != @previous
         report_transition(healthy)
         @previous = healthy
       elsif unhealthy_ticks == @config.startup_grace_ticks
-        # Startup took longer than normal, which is a failure by now. Staying
-        # silent would leave a pod that never comes up saying nothing about it.
+        # Report once when a surviving monitor exhausts startup grace.
         log(:error, "not healthy #{unhealthy_ticks * @config.tick}s, expected #{@consumers} consumers")
       end
     end
@@ -174,10 +175,8 @@ module KicksLiveness
     def log(level, message)
       @config.resolved_logger&.public_send(level, "[liveness] slot #{@slot}: #{message}")
     rescue StandardError
-      # Logging is diagnostic, while the heartbeat is the liveness contract. A
-      # broken custom logger must not prevent a mark from being written or kill
-      # the only thread that can refresh it. There is deliberately no fallback
-      # log here: calling the same logger again would only repeat the failure.
+      # Logger failures must not stop the heartbeat writer. Do not retry the
+      # same logger from its failure path.
       nil
     end
   end

data/lib/kicks_liveness/registry.rb

@@ -9,16 +9,13 @@ module KicksLiveness
     # Initialised at file-load time: single-threaded, and before any fork.
     @workers = []
     @mutex = Mutex.new
-    @stopping = false
+    @stopping_at = nil
 
     class << self
       # Records a worker as subscribed.
       #
-      # Registering the same object twice would break the size comparison in
-      # {healthy?} and leave the process permanently unhealthy, so identity is
-      # checked first. +equal?+ rather than +==+ on purpose: this is a registry
-      # of objects, and a worker class is free to define equality however it
-      # likes.
+      # Duplicate objects must not inflate the expected-count check. Use identity
+      # rather than == because worker classes can override equality.
       #
       # @param worker [Sneakers::Worker]
       # @return [Array] the registry
@@ -36,25 +33,26 @@ module KicksLiveness
         @mutex.synchronize { @workers.delete(worker) }
       end
 
-      # @return [Integer] how many workers are currently registered
-      def size
-        @mutex.synchronize { @workers.size }
-      end
-
-      # Records that this process is shutting down on purpose.
-      #
-      # Without it a graceful shutdown is indistinguishable from a failure: the
-      # registry empties as each worker unsubscribes, so the monitor would see
-      # an incomplete set and report a genuine fault on every deploy.
+      # Mark deliberate shutdown before unsubscription changes the predicate.
       #
       # @return [true]
-      def stopping!
-        @stopping = true
+      def stopping! # rubocop:disable Naming/PredicateMethod -- Records the stop notification.
+        @mutex.synchronize { @stopping_at ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        true
       end
 
       # @return [Boolean] whether shutdown has begun
       def stopping?
-        @stopping
+        !stopping_at.nil?
+      end
+
+      # Captured by the first stop callback, even if the monitor's next tick is
+      # delayed. Repeated callbacks cannot extend the shutdown exemption. This
+      # is process-local: a sibling replacement fork has not been asked to stop.
+      #
+      # @return [Float, nil] monotonic time of the first stop notification
+      def stopping_at
+        @mutex.synchronize { @stopping_at }
       end
 
       # Whether this process is consuming everything it is supposed to consume.

data/lib/kicks_liveness/topology_report.rb

@@ -0,0 +1,110 @@
+require 'fileutils'
+
+module KicksLiveness
+  # Claims worker topology reports across fork respawns. Kept outside
+  # {Heartbeat} because the probe does not report these diagnostics.
+  #
+  # @see file:docs/LIMITATIONS.md#container-generations-need-linux-procfs-and-a-private-pid-namespace
+  # @api private
+  class TopologyReport
+    # @param dir [String] marks directory
+    # @param topology [ContainerGeneration::Result] what this worker proved
+    def initialize(dir, topology)
+      @dir = dir
+      @topology = topology
+    end
+
+    # Claims the right to report this state, once.
+    #
+    # Unverifiable results use a generation key; unavailable and unsafe results
+    # have no trustworthy generation and use only their state. Verification
+    # clears the flag so a later degradation can be reported again.
+    #
+    # @return [Array(Symbol, String), nil] the level and the line to log, or nil
+    #   when there is nothing new to say
+    def claim!
+      key = key_for(@topology)
+
+      unless key
+        forget! if @topology.verified?
+        return nil
+      end
+
+      return nil unless claim(key)
+
+      report_for(@topology)
+    end
+
+    private
+
+    def path
+      File.join(@dir, 'topology-report')
+    end
+
+    def key_for(topology)
+      case topology.state
+      when :unsafe then 'unsafe'
+      when :unverifiable then "unverifiable #{topology.generation}"
+      when :unavailable then 'unavailable'
+      end
+    end
+
+    # Only unsafe changes the probe verdict and is reported at ERROR.
+    def report_for(topology)
+      case topology.state
+      when :unsafe
+        [:error, "#{Heartbeat::UNSUPPORTED_TOPOLOGY} (#{topology.detail}); " \
+                 'liveness probes fail until the pod is recreated without those settings']
+      when :unverifiable
+        [:warn, "PID topology unverified (#{topology.detail}): the container generation still scopes the heartbeat, " \
+                'but shareProcessNamespace and hostPID cannot be ruled out']
+      else
+        [:warn, "no container generation (#{topology.detail}): falling back to freshness only, " \
+                'which cannot reject a heartbeat inherited from a previous container']
+      end
+    end
+
+    # The runner starts all of its forks together, so the read and write have
+    # to be one inter-process operation: an atomic rename protects readers from
+    # a partial file, but does not stop every fork reading the old key and each
+    # claiming the same report. The state file is also the lock file, so no
+    # second path is added to the marks contract. Once it has existed it is
+    # cleared by truncation rather than unlink: every process must keep locking
+    # the same inode.
+    def claim(key)
+      synchronize(create: true) do |state|
+        state.rewind
+        next false if state.read.chomp == key
+
+        state.rewind
+        state.truncate(0)
+        state.write("#{key}\n")
+        state.flush
+        true
+      end
+    rescue StandardError
+      # A report that cannot be remembered must still be made. The cost is
+      # repetition on the next start, not silence about a lost guarantee.
+      true
+    end
+
+    def forget!
+      return unless File.exist?(path)
+
+      synchronize(create: false) { |state| state.truncate(0) }
+    rescue StandardError
+      nil
+    end
+
+    def synchronize(create:)
+      FileUtils.mkdir_p(@dir)
+      flags = File::RDWR
+      flags |= File::CREAT if create
+      File.open(path, flags, 0o644) do |state|
+        raise IOError, "cannot lock #{path}" unless state.flock(File::LOCK_EX)
+
+        yield state
+      end
+    end
+  end
+end

data/lib/kicks_liveness/version.rb

@@ -1,4 +1,4 @@
 module KicksLiveness
   # @return [String] gem version
-  VERSION = '0.1.2'.freeze
+  VERSION = '0.1.4'.freeze
 end

data/lib/kicks_liveness.rb

@@ -1,6 +1,7 @@
 require_relative 'kicks_liveness/version'
 require_relative 'kicks_liveness/heartbeat'
 require_relative 'kicks_liveness/attempts'
+require_relative 'kicks_liveness/topology_report'
 require_relative 'kicks_liveness/configuration'
 require_relative 'kicks_liveness/registry'
 require_relative 'kicks_liveness/monitor'
@@ -76,7 +77,8 @@ module KicksLiveness
     # @param slot [Integer] supervisor slot of this fork
     # @param processes [Integer] how many forks the probe must wait for
     # @param consumers [Integer] how many workers must subscribe here
-    # @return [Thread, nil] nil when disabled by configuration
+    # @return [Thread, nil] nil when disabled by configuration, or when the PID
+    #   topology is proven unsupported
     # @api private
     def start!(slot:, processes:, consumers:)
       return unless config.enabled?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kicks_liveness
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - PoroshkinaVV
@@ -39,13 +39,14 @@ files:
 - lib/kicks_liveness/probe.rb
 - lib/kicks_liveness/railtie.rb
 - lib/kicks_liveness/registry.rb
+- lib/kicks_liveness/topology_report.rb
 - lib/kicks_liveness/version.rb
 homepage: https://github.com/PoroshkinaVV/kicks_liveness
 licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/PoroshkinaVV/kicks_liveness
-  documentation_uri: https://rubydoc.info/gems/kicks_liveness/0.1.2
+  documentation_uri: https://rubydoc.info/gems/kicks_liveness/0.1.4
   changelog_uri: https://github.com/PoroshkinaVV/kicks_liveness/blob/main/CHANGELOG.md
   bug_tracker_uri: https://github.com/PoroshkinaVV/kicks_liveness/issues
   rubygems_mfa_required: 'true'