kicks_liveness 0.1.0 → 0.1.2

data/lib/kicks_liveness/heartbeat.rb

@@ -1,5 +1,57 @@
 module KicksLiveness
-  # The liveness mark on the filesystem: written by the worker, read by the
+  # Identifies an ordinary Linux container incarnation shared by the worker and
+  # exec probes without writing anything to the marks directory. This assumes
+  # the container owns PID 1; shared and host PID namespaces are documented as
+  # unsupported in +docs/LIMITATIONS.md+.
+  # @api private
+  module ContainerGeneration
+    module_function
+
+    # @return [String, nil] current private-PID Linux container incarnation, or
+    #   nil when procfs does not expose one
+    # @api private
+    def current
+      mount_namespace = File.readlink('/proc/self/ns/mnt')
+      stat = File.read('/proc/1/stat')
+      closing_parenthesis = stat.rindex(') ')
+      return unless closing_parenthesis
+
+      # After the command in parentheses, field 3 (`state`) is index 0. Process
+      # start time is field 22, therefore index 19 in this tail.
+      started_at = stat[(closing_parenthesis + 2)..].split[19]
+      return unless started_at && Integer(started_at).positive?
+
+      "#{mount_namespace}:#{started_at}"
+    rescue StandardError
+      nil
+    end
+  end
+
+  # Generation-specific parts of the filesystem contract.
+  # @api private
+  module GenerationGuard
+    private
+
+    def generation_path
+      File.join(@dir, 'generation')
+    end
+
+    def declared_generation
+      File.read(generation_path)
+    rescue StandardError
+      nil
+    end
+
+    def current_generation?
+      !@generation || declared_generation == @generation
+    end
+
+    def previous_generation?(path)
+      @generation && File.read(path)[/\bgeneration=(\S+)/, 1] != @generation
+    end
+  end
+
+  # The liveness marks on the filesystem: written by the worker, read by the
   # probe.
   #
   # The directory must live on tmpfs — in Kubernetes, an emptyDir with
@@ -17,6 +69,8 @@ module KicksLiveness
   #
   # @see file:docs/DESIGN.md#why-the-heartbeat-file-has-no-require-of-its-own
   class Heartbeat
+    include GenerationGuard
+
     # @return [String] marks directory used when the environment says nothing
     DEFAULT_DIR = '/opt/app/tmp/health'.freeze
     # @return [Integer] seconds after which a mark is stale, by default
@@ -29,6 +83,16 @@ module KicksLiveness
       tick: 'KICKS_LIVENESS_TICK'
     }.freeze
 
+    # Linux exposes a stable identifier shared by a private-PID container and
+    # its exec probes. A restarted container gets a new identifier even though
+    # its Kubernetes emptyDir survives.
+    # @return [String, nil] current private-PID container incarnation, or nil
+    #   off Linux
+    # @api private
+    def self.container_generation
+      ContainerGeneration.current
+    end
+
     # An empty string counts as unset: in a ConfigMap that is what you get by
     # declaring a key and leaving it blank.
     #
@@ -72,9 +136,15 @@ module KicksLiveness
 
     # @param dir [String] marks directory
     # @param max_age [Integer] seconds after which a mark is considered stale
-    def initialize(dir: Heartbeat.env_dir, max_age: Heartbeat.env_max_age)
+    # @param generation [String, nil] container incarnation; injected in specs
+    def initialize(
+      dir: Heartbeat.env_dir,
+      max_age: Heartbeat.env_max_age,
+      generation: Heartbeat.container_generation
+    )
       @dir = dir
       @max_age = max_age
+      @generation = generation
     end
 
     attr_reader :dir, :max_age
@@ -93,10 +163,8 @@ module KicksLiveness
     # @return [void]
     def declare!(processes)
       make_dir
-      # The pid keeps concurrent forks from sharing the temporary file.
-      tmp = "#{expected_path}.#{Process.pid}"
-      File.write(tmp, processes)
-      File.rename(tmp, expected_path)
+      atomic_write(generation_path, @generation) if @generation
+      atomic_write(expected_path, processes)
     end
 
     # Refreshes this fork's mark.
@@ -106,14 +174,18 @@ module KicksLiveness
     # PID in the name that file would stay stale forever and the probe would fail
     # permanently.
     #
-    # The contents exist only for a human running <tt>kubectl exec ... cat</tt>;
-    # the probe decides on mtime alone.
+    # The timestamp, pid and slot exist for a human running
+    # <tt>kubectl exec ... cat</tt>. The generation is also checked by the probe:
+    # a Kubernetes emptyDir survives a container restart, so freshness alone
+    # cannot distinguish this process from the one that just exited.
     #
     # @param slot [Integer] supervisor slot of this fork
     # @return [Integer] bytes written
     def touch!(slot)
       make_dir
-      File.write(slot_path(slot), "#{Time.now.utc.strftime('%FT%TZ')} pid=#{Process.pid} slot=#{slot}\n")
+      contents = "#{Time.now.utc.strftime('%FT%TZ')} pid=#{Process.pid} slot=#{slot}"
+      contents = "#{contents} generation=#{@generation}" if @generation
+      atomic_write(slot_path(slot), "#{contents}\n")
     end
 
     # The probe side: is every declared fork's mark present and fresh?
@@ -126,13 +198,9 @@ module KicksLiveness
     def check(now: Time.now.utc)
       processes = expected
       return [false, "no #{expected_path}: worker has not started yet"] unless processes&.positive?
+      return [false, 'heartbeat belongs to a previous container: worker has not started yet'] unless current_generation?
 
-      problems = (0...processes).filter_map do |slot|
-        age = age_of(slot_path(slot), now)
-        next "worker-#{slot} missing" if age.nil?
-
-        "worker-#{slot} stale #{age.round}s > #{@max_age}s" if age > @max_age
-      end
+      problems = (0...processes).filter_map { |slot| problem_for(slot, now) }
 
       problems.empty? ? [true, "#{processes} process(es) healthy"] : [false, problems.join('; ')]
     end
@@ -169,6 +237,25 @@ module KicksLiveness
       nil
     end
 
+    def problem_for(slot, now)
+      path = slot_path(slot)
+      age = age_of(path, now)
+      return "worker-#{slot} missing" if age.nil?
+      return "worker-#{slot} belongs to a previous container" if previous_generation?(path)
+
+      "worker-#{slot} stale #{age.round}s > #{@max_age}s" if age > @max_age
+    rescue StandardError
+      "worker-#{slot} unreadable"
+    end
+
+    def atomic_write(path, contents)
+      # The pid keeps concurrent forks from sharing the temporary file.
+      tmp = "#{path}.#{Process.pid}"
+      bytes = File.write(tmp, contents)
+      File.rename(tmp, path)
+      bytes
+    end
+
     def age_of(path, now)
       now - File.mtime(path)
     rescue StandardError

data/lib/kicks_liveness/hooks.rb

@@ -60,7 +60,7 @@ module KicksLiveness
             consumers: kicks_liveness_expected_consumers
           )
         rescue StandardError => e
-          KicksLiveness.config.resolved_logger&.error("[liveness] failed to start: #{e.class}: #{e.message}")
+          report_start_failure(e)
         end
       end
 
@@ -80,6 +80,17 @@ module KicksLiveness
 
       private
 
+      def report_start_failure(error)
+        KicksLiveness.config.resolved_logger&.error(
+          "[liveness] failed to start: #{error.class}: #{error.message}"
+        )
+      rescue StandardError
+        # This is already the failure path. Neither resolving the configuration
+        # again nor a broken logger may let liveness instrumentation stop the
+        # worker process it is meant to observe.
+        nil
+      end
+
       # The same set the worker gem itself builds its workers from, so the
       # queue list is never duplicated and cannot drift. An array of classes
       # under sneakers:run, a callable registry under sneakers:active_job.

data/lib/kicks_liveness/monitor.rb

@@ -173,6 +173,12 @@ module KicksLiveness
 
     def log(level, message)
       @config.resolved_logger&.public_send(level, "[liveness] slot #{@slot}: #{message}")
+    rescue StandardError
+      # Logging is diagnostic, while the heartbeat is the liveness contract. A
+      # broken custom logger must not prevent a mark from being written or kill
+      # the only thread that can refresh it. There is deliberately no fallback
+      # log here: calling the same logger again would only repeat the failure.
+      nil
     end
   end
 end

data/lib/kicks_liveness/version.rb

@@ -1,4 +1,4 @@
 module KicksLiveness
   # @return [String] gem version
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.1.2'.freeze
 end

data/lib/kicks_liveness.rb

@@ -9,8 +9,8 @@ require_relative 'kicks_liveness/hooks'
 # Liveness probe for Kicks and Sneakers workers, backed by a tmpfs heartbeat.
 #
 # The worker publishes a mark from inside its own process, checking its Bunny
-# consumers in memory; the probe reads only the mark's mtime. No Rails, no call
-# to the broker.
+# consumers in memory; the probe reads only the container generation and mark
+# mtimes. No Rails, no call to the broker.
 #
 # @see file:docs/SETUP.md
 # @see file:docs/DESIGN.md
@@ -46,10 +46,13 @@ module KicksLiveness
     # <tt>gem 'kicks', require: false</tt> the to_prepare hook runs earlier and
     # would fail on NameError.
     #
-    # @raise [LoadError] if neither +kicks+ nor +sneakers+ is available
+    # @raise [LoadError] if neither worker gem is available, or if both +kicks+
+    #   and +sneakers+ are activated
     # @return [Module]
     # @see file:docs/SETUP.md#installing-the-hooks
     def install!
+      reject_ambiguous_worker_gems!
+
       begin
         require 'sneakers'
         require 'sneakers/workergroup'
@@ -80,6 +83,15 @@ module KicksLiveness
 
       Monitor.new(slot: slot, processes: processes, consumers: consumers, config: config).start!
     end
+
+    private
+
+    def reject_ambiguous_worker_gems!
+      return unless defined?(Gem.loaded_specs)
+      return unless Gem.loaded_specs.key?('kicks') && Gem.loaded_specs.key?('sneakers')
+
+      raise LoadError, 'kicks_liveness cannot run with both kicks and sneakers activated; install exactly one'
+    end
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kicks_liveness
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - PoroshkinaVV
@@ -11,12 +11,8 @@ date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: |
   A liveness probe for RabbitMQ worker pods that loads no Rails and never talks
-  to the broker. The worker publishes a heartbeat to tmpfs from inside its own
-  process, checking its Bunny consumers in memory; the probe only reads the
-  file's mtime and runs as `bundle exec kicks-liveness`. It loads no application
-  code; aside from Bundler and the interpreter it loads only the gem's small
-  probe files, while the state itself comes from tmpfs. A broker hiccup cannot
-  restart every replica at once.
+  to the broker. Workers check their Bunny consumers in memory and publish a
+  heartbeat to tmpfs; the probe only reads that heartbeat.
 email:
 - lera.poroshkina@mail.ru
 executables:
@@ -24,6 +20,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".yardopts"
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
@@ -48,7 +45,7 @@ licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/PoroshkinaVV/kicks_liveness
-  documentation_uri: https://rubydoc.info/gems/kicks_liveness/0.1.0
+  documentation_uri: https://rubydoc.info/gems/kicks_liveness/0.1.2
   changelog_uri: https://github.com/PoroshkinaVV/kicks_liveness/blob/main/CHANGELOG.md
   bug_tracker_uri: https://github.com/PoroshkinaVV/kicks_liveness/issues
   rubygems_mfa_required: 'true'