kicks_liveness 0.1.2 → 0.1.4

data/lib/kicks_liveness/heartbeat.rb

@@ -1,95 +1,182 @@
 module KicksLiveness
-  # Identifies an ordinary Linux container incarnation shared by the worker and
-  # exec probes without writing anything to the marks directory. This assumes
-  # the container owns PID 1; shared and host PID namespaces are documented as
-  # unsupported in +docs/LIMITATIONS.md+.
+  # Identifies the container generation and PID topology from Linux procfs,
+  # independently in the worker and each exec probe.
+  #
+  # @see file:docs/LIMITATIONS.md#container-generations-need-linux-procfs-and-a-private-pid-namespace
   # @api private
   module ContainerGeneration
-    module_function
-
-    # @return [String, nil] current private-PID Linux container incarnation, or
-    #   nil when procfs does not expose one
+    # @return [String] this process's mount namespace link
+    SELF_NAMESPACE = '/proc/self/ns/mnt'.freeze
+    # @return [String] PID 1's mount namespace link
+    PID1_NAMESPACE = '/proc/1/ns/mnt'.freeze
+    # @return [String] PID 1's status line, which carries its start time
+    PID1_STAT = '/proc/1/stat'.freeze
+
+    # One procfs inspection, as a value.
+    #
+    # +:verified+:: matching mount namespaces, with a generation.
+    # +:unsafe+:: different mount namespaces; no trustworthy generation.
+    # +:unverifiable+:: a generation, but PID 1's namespace could not be read.
+    # +:unavailable+:: Linux procfs yielded no generation.
+    # +:non_linux+:: another operating system.
+    #
     # @api private
-    def current
-      mount_namespace = File.readlink('/proc/self/ns/mnt')
-      stat = File.read('/proc/1/stat')
-      closing_parenthesis = stat.rindex(') ')
-      return unless closing_parenthesis
-
-      # After the command in parentheses, field 3 (`state`) is index 0. Process
-      # start time is field 22, therefore index 19 in this tail.
-      started_at = stat[(closing_parenthesis + 2)..].split[19]
-      return unless started_at && Integer(started_at).positive?
-
-      "#{mount_namespace}:#{started_at}"
-    rescue StandardError
-      nil
-    end
-  end
-
-  # Generation-specific parts of the filesystem contract.
-  # @api private
-  module GenerationGuard
-    private
-
-    def generation_path
-      File.join(@dir, 'generation')
+    class Result
+      class << self
+        # @param generation [String]
+        # @return [Result]
+        def verified(generation) = new(:verified, generation: generation)
+
+        # @param detail [String] which namespaces differ
+        # @return [Result]
+        def unsafe(detail) = new(:unsafe, detail: detail)
+
+        # @param generation [String]
+        # @param detail [String] what could not be read
+        # @return [Result]
+        def unverifiable(generation, detail) = new(:unverifiable, generation: generation, detail: detail)
+
+        # @param detail [String] what could not be read
+        # @return [Result]
+        def unavailable(detail) = new(:unavailable, detail: detail)
+
+        # @return [Result]
+        def non_linux = new(:non_linux)
+      end
+
+      # @return [Symbol] +:verified+, +:unsafe+, +:unverifiable+, +:unavailable+ or +:non_linux+
+      attr_reader :state
+      # @return [String, nil] the container incarnation, in the two states that have one
+      attr_reader :generation
+      # @return [String, nil] what could not be read, or which namespaces differ
+      attr_reader :detail
+
+      # @param state [Symbol]
+      # @param generation [String, nil]
+      # @param detail [String, nil]
+      def initialize(state, generation: nil, detail: nil)
+        @state = state
+        @generation = generation
+        @detail = detail
+        freeze
+      end
+
+      # @return [Boolean]
+      def verified? = @state == :verified
+
+      # @return [Boolean]
+      def unsafe? = @state == :unsafe
     end
 
-    def declared_generation
-      File.read(generation_path)
-    rescue StandardError
-      nil
-    end
-
-    def current_generation?
-      !@generation || declared_generation == @generation
-    end
-
-    def previous_generation?(path)
-      @generation && File.read(path)[/\bgeneration=(\S+)/, 1] != @generation
+    class << self
+      # Reads the three procfs entries the protocol needs, in the one order that
+      # cannot be talked out of a rejection: an unsafe topology is decided before
+      # anything is derived from PID 1, because in that topology PID 1 belongs to
+      # the pod sandbox or the node and its start time means nothing here.
+      #
+      # @param linux [Boolean] injected in specs, where procfs is stubbed
+      # @return [Result]
+      def current(linux: linux?)
+        return Result.non_linux unless linux
+
+        mine, mine_error = read_link(SELF_NAMESPACE)
+        pid1, pid1_error = read_link(PID1_NAMESPACE)
+        return Result.unsafe("PID 1 is in #{pid1}, this process in #{mine}") if another_namespace?(mine, pid1)
+
+        started_at, stat_error = pid1_started_at
+        return Result.unavailable(mine_error || stat_error) unless mine && started_at
+
+        generation = "#{mine}:#{started_at}"
+        pid1 ? Result.verified(generation) : Result.unverifiable(generation, pid1_error)
+      end
+
+      private
+
+      # Missing procfs does not identify the OS. JRuby reports "java", so use
+      # its predefined RbConfig to identify the host. CRuby on Linux answers
+      # first, including under --disable-gems where RbConfig may be undefined.
+      # https://bugs.ruby-lang.org/issues/8090
+      def linux?
+        return true if RUBY_PLATFORM.include?('linux')
+        return false unless defined?(RbConfig)
+
+        RbConfig::CONFIG['host_os'].to_s.include?('linux')
+      end
+
+      # Only a comparison both sides of which were read can prove anything.
+      # @param mine [String, nil] this process's mount namespace
+      # @param pid1 [String, nil] PID 1's mount namespace
+      # @return [Boolean]
+      def another_namespace?(mine, pid1)
+        !mine.nil? && !pid1.nil? && mine != pid1
+      end
+
+      # @param path [String] a procfs namespace link
+      # @return [Array(String, nil), Array(nil, String)] the link target, or why it could not be read
+      def read_link(path)
+        [File.readlink(path), nil]
+      rescue StandardError => e
+        [nil, "cannot read #{path} (#{e.class})"]
+      end
+
+      # @return [Array(String, nil), Array(nil, String)] PID 1's start time, or why it is missing
+      def pid1_started_at
+        started_at = start_time_field(File.read(PID1_STAT))
+        return [started_at.to_s, nil] if started_at&.positive?
+        return [nil, 'PID 1 start time is zero; possible hostPID or virtualized procfs'] if started_at&.zero?
+
+        [nil, "cannot parse #{PID1_STAT}"]
+      rescue StandardError => e
+        [nil, "cannot read #{PID1_STAT} (#{e.class})"]
+      end
+
+      # @param stat [String] the contents of /proc/1/stat
+      # @return [Integer, nil]
+      def start_time_field(stat)
+        closing_parenthesis = stat.rindex(') ')
+        return unless closing_parenthesis
+
+        # After the command in parentheses, field 3 (`state`) is index 0. Process
+        # start time is field 22, therefore index 19 in this tail.
+        started_at = stat[(closing_parenthesis + 2)..].split[19]
+        Integer(started_at) if started_at
+      rescue ArgumentError, TypeError
+        nil
+      end
     end
   end
 
-  # The liveness marks on the filesystem: written by the worker, read by the
-  # probe.
-  #
-  # The directory must live on tmpfs — in Kubernetes, an emptyDir with
-  # <tt>medium: Memory</tt>. Put it on a real disk and the probe starts
-  # depending on the disk again, and a stalling disk is one of the most common
-  # causes of false restarts.
+  # Filesystem contract shared by the worker and probe. The marks directory
+  # must be on tmpfs (an emptyDir with medium: Memory in Kubernetes).
   #
-  # This file deliberately runs no +require+ on the probe's path and refers to
-  # nothing else in the gem: the probe can load it alone, with no application
-  # code and no Rails behind it. That keeps the optional direct form at ~43 ms
-  # and lets it run on a bare interpreter with +--disable-gems+ when the image
-  # allows it -- whether Bundler is in the picture depends on where the image
-  # put its gems, not on this file. The one +require+ it does contain is lazy,
-  # in a branch only the worker reaches.
+  # The probe loads this file alone, including under --disable-gems. Keep its
+  # read path free of requires and references to other gem code; fileutils is
+  # loaded lazily only when the worker needs to create parent directories.
   #
   # @see file:docs/DESIGN.md#why-the-heartbeat-file-has-no-require-of-its-own
   class Heartbeat
-    include GenerationGuard
-
     # @return [String] marks directory used when the environment says nothing
     DEFAULT_DIR = '/opt/app/tmp/health'.freeze
     # @return [Integer] seconds after which a mark is stale, by default
     DEFAULT_MAX_AGE = 45
 
+    # Shared verdict for unsafe topology evidence from either participant.
+    # @return [String]
+    UNSUPPORTED_TOPOLOGY = 'unsupported PID topology: pod must not set shareProcessNamespace or hostPID'.freeze
+
     # @return [Hash{Symbol => String}] setting name to environment variable
     ENV_NAMES = {
       dir: 'KICKS_LIVENESS_DIR',
       max_age: 'KICKS_LIVENESS_MAX_AGE',
-      tick: 'KICKS_LIVENESS_TICK'
+      tick: 'KICKS_LIVENESS_TICK',
+      shutdown_grace: 'KICKS_LIVENESS_SHUTDOWN_GRACE'
     }.freeze
 
-    # Linux exposes a stable identifier shared by a private-PID container and
-    # its exec probes. A restarted container gets a new identifier even though
-    # its Kubernetes emptyDir survives.
-    # @return [String, nil] current private-PID container incarnation, or nil
-    #   off Linux
+    # Inspects this process's container generation and PID topology.
+    # @return [ContainerGeneration::Result] what this process can prove about
+    #   its own incarnation
     # @api private
-    def self.container_generation
+    def self.topology
       ContainerGeneration.current
     end
 
@@ -103,15 +190,9 @@ module KicksLiveness
       value unless value.nil? || value.empty?
     end
 
-    # Values arrive from a ConfigMap, and a typo there is no reason to bring a
-    # worker down — garbage falls back to the default.
-    #
-    # Both settings are durations, so a value has to be parseable *and*
-    # positive. Zero and negative numbers parse perfectly well and are the more
-    # dangerous half: a tick of zero turns the monitor into a hot loop, a
-    # negative one used to kill the monitor thread on its first sleep, and a
-    # negative max_age makes every mark stale on arrival, so the probe can never
-    # pass again.
+    # Environment durations require positive integers. Invalid input falls
+    # back to the default; accepting zero or negatives can stop the monitor
+    # or make healthy marks stale immediately.
     #
     # @param key [Symbol] one of the keys of {ENV_NAMES}
     # @param default [Integer]
@@ -136,68 +217,90 @@ module KicksLiveness
 
     # @param dir [String] marks directory
     # @param max_age [Integer] seconds after which a mark is considered stale
-    # @param generation [String, nil] container incarnation; injected in specs
+    # @param topology [ContainerGeneration::Result] what this process proved
+    #   about its incarnation; injected in specs
     def initialize(
       dir: Heartbeat.env_dir,
       max_age: Heartbeat.env_max_age,
-      generation: Heartbeat.container_generation
+      topology: Heartbeat.topology
     )
       @dir = dir
       @max_age = max_age
-      @generation = generation
+      @topology = topology
+      @generation = topology.generation
     end
 
-    attr_reader :dir, :max_age
+    # @return [String] marks directory
+    attr_reader :dir
+    # @return [Integer] seconds after which a mark is considered stale
+    attr_reader :max_age
+    # @return [ContainerGeneration::Result] what this process proved about its
+    #   incarnation, and the incarnation itself where there is one
+    attr_reader :topology
 
-    # Records how many forks the probe must wait for. Every fork writes the same
-    # value. Without it the probe would pass as soon as any single mark was
-    # fresh, while half the workers had not subscribed yet.
-    #
-    # Written through a rename, because a plain write truncates first: a probe
-    # reading in that window finds the file empty and reports that the worker
-    # has not started. The window is real and recurring — every fork rewrites
-    # this file on every tick, while the liveness probe reads it on a schedule
-    # of its own. Rename is atomic on tmpfs.
+    # Declares how many fork marks the probe requires. Atomic publication
+    # prevents a concurrent probe from reading a truncated count or generation.
     #
     # @param processes [Integer]
-    # @return [void]
+    # @return [Integer] bytes written
     def declare!(processes)
+      return reject_topology! if @topology.unsafe?
+
       make_dir
       atomic_write(generation_path, @generation) if @generation
       atomic_write(expected_path, processes)
     end
 
-    # Refreshes this fork's mark.
-    #
-    # The file is named by supervisor slot, not by PID: a fork killed with
-    # SIGKILL is respawned into the same slot and overwrites its own file. With a
-    # PID in the name that file would stay stale forever and the probe would fail
-    # permanently.
-    #
-    # The timestamp, pid and slot exist for a human running
-    # <tt>kubectl exec ... cat</tt>. The generation is also checked by the probe:
-    # a Kubernetes emptyDir survives a container restart, so freshness alone
-    # cannot distinguish this process from the one that just exited.
+    # Refreshes the supervisor slot, so a respawn replaces the old mark instead
+    # of leaving a stale PID-named file. The probe checks generation and mtime;
+    # the remaining contents are diagnostic.
     #
     # @param slot [Integer] supervisor slot of this fork
     # @return [Integer] bytes written
     def touch!(slot)
+      return reject_topology! if @topology.unsafe?
+
       make_dir
       contents = "#{Time.now.utc.strftime('%FT%TZ')} pid=#{Process.pid} slot=#{slot}"
       contents = "#{contents} generation=#{@generation}" if @generation
       atomic_write(slot_path(slot), "#{contents}\n")
     end
 
+    # Publishes the terminal fail-closed state for a proven unsafe topology,
+    # instead of any mark.
+    #
+    # Explicit evidence is needed because inherited generation and slot marks
+    # may still be fresh. It precedes normal writes and is never removed during
+    # the marks volume's lifetime.
+    #
+    # @return [Integer, nil] bytes written, or nil when this process has not
+    #   proven the topology unsafe and so has nothing to publish
+    # @api private
+    def reject_topology!
+      return unless @topology.unsafe?
+
+      make_dir
+      atomic_write(topology_path, "#{[UNSUPPORTED_TOPOLOGY, @topology.detail].compact.join("\n")}\n")
+    end
+
     # The probe side: is every declared fork's mark present and fresh?
     #
-    # The returned message names slots the way the files are named, so that a
-    # human reading the Unhealthy event knows which file to look at.
+    # Published or local unsafe evidence takes precedence over every generation
+    # and freshness check, including when this probe cannot verify the topology.
     #
     # @param now [Time] injected in specs
     # @return [Array(Boolean, String)] health, and the reason to print on stdout
     def check(now: Time.now.utc)
+      return [false, UNSUPPORTED_TOPOLOGY] if unsupported_topology?
+
       processes = expected
-      return [false, "no #{expected_path}: worker has not started yet"] unless processes&.positive?
+      unless processes&.positive?
+        return [
+          false,
+          "no #{expected_path}: worker has not started yet " \
+          '(is KicksLiveness.install! called before the runner starts? outside Rails it is not automatic)'
+        ]
+      end
       return [false, 'heartbeat belongs to a previous container: worker has not started yet'] unless current_generation?
 
       problems = (0...processes).filter_map { |slot| problem_for(slot, now) }
@@ -207,13 +310,8 @@ module KicksLiveness
 
     private
 
-    # One syscall on the happy path, and whichever fork gets there first wins.
-    #
-    # +Dir.mkdir+ creates a single level and raises ENOENT when the parent is
-    # missing, which any nested KICKS_LIVENESS_DIR hits. FileUtils is required
-    # here rather than at the top of the file: only the worker ever creates the
-    # directory, and the probe's path through {#check} must stay free of
-    # requires.
+    # Try one mkdir first; competing forks may already have created it.
+    # Only missing parent directories need fileutils, loaded on the worker path.
     def make_dir
       Dir.mkdir(@dir)
     rescue Errno::EEXIST
@@ -231,12 +329,44 @@ module KicksLiveness
       File.join(@dir, "worker-#{slot}")
     end
 
+    def generation_path
+      File.join(@dir, 'generation')
+    end
+
+    def topology_path
+      File.join(@dir, 'topology-unsafe')
+    end
+
+    # The worker's published proof first, then this process's own: either is
+    # enough, and no weaker result from the other side can take it back.
+    #
+    # Existence is the whole signal, so this guard cannot be defeated by a
+    # partial read and holds even where the file itself is not readable. Its
+    # contents are there for a human running `cat`.
+    def unsupported_topology?
+      File.exist?(topology_path) || @topology.unsafe?
+    end
+
     def expected
       Integer(File.read(expected_path))
     rescue StandardError
       nil
     end
 
+    def declared_generation
+      File.read(generation_path)
+    rescue StandardError
+      nil
+    end
+
+    def current_generation?
+      !@generation || declared_generation == @generation
+    end
+
+    def previous_generation?(path)
+      @generation && File.read(path)[/\bgeneration=(\S+)/, 1] != @generation
+    end
+
     def problem_for(slot, now)
       path = slot_path(slot)
       age = age_of(path, now)

data/lib/kicks_liveness/hooks.rb

@@ -25,29 +25,17 @@ module KicksLiveness
 
     # Prepended to +Sneakers::WorkerGroup+.
     #
-    # +ServerEngine::Server#create_worker+ calls +w.extend(worker_module)+, so
-    # +WorkerGroup+ lands in the singleton class of the instance rather than
-    # being included in a class. A prepend to the module still sits ahead of it
-    # in that lookup chain; +worker_id+ and +config+ come from
-    # +ServerEngine::Worker+.
+    # ServerEngine extends each worker instance with WorkerGroup. A prepend to
+    # the module still intercepts that singleton-class chain; worker_id and
+    # config come from ServerEngine::Worker.
     #
     # @see file:docs/DESIGN.md#where-the-hooks-attach
     # @api private
     module WorkerGroup
-      # Starts the monitor from +after_fork+ rather than from +run+.
-      #
-      # +Sneakers::WorkerGroup#run+ calls +after_fork+ as its very first
-      # statement and only afterwards resolves +worker_classes+, so this is the
-      # earliest point in the fork at which the application's own +after_fork+
-      # hook has already run. That ordering matters because resolving the
-      # expected consumer count may run application code: under
-      # sneakers:active_job the set is a callable registry, and the standard use
-      # of +after_fork+ is to re-establish the connections a fork inherited.
-      # Resolving it first would run that code against a fork that is not ready.
-      #
-      # Wrapped in a rescue: instrumentation has no right to prevent a worker
-      # from starting, so the worst outcome is a missing mark and a pod restart
-      # rather than a pod that stays up with silent queues.
+      # Run after the application's after_fork restores inherited connections.
+      # Resolving callable worker_classes earlier could run application code before
+      # those connections are ready. Rescue startup errors so instrumentation
+      # cannot prevent the worker itself from starting.
       #
       # @return [void]
       def after_fork
@@ -64,13 +52,8 @@ module KicksLiveness
         end
       end
 
-      # Marks the shutdown as deliberate before the workers unsubscribe.
-      #
-      # +Sneakers::WorkerGroup#stop+ calls +stop+ on each worker, and
-      # +Sneakers::Worker#stop+ waits for the thread pool to drain, which takes
-      # as long as the longest in-flight job. The registry empties at the start
-      # of that, so without this flag the monitor would report a fault on every
-      # ordinary deploy.
+      # Set the flag before any worker unsubscribes or starts draining, so the
+      # monitor does not treat deliberate removal as a consumer failure.
       #
       # @return [void]
       def stop
@@ -91,9 +74,8 @@ module KicksLiveness
         nil
       end
 
-      # The same set the worker gem itself builds its workers from, so the
-      # queue list is never duplicated and cannot drift. An array of classes
-      # under sneakers:run, a callable registry under sneakers:active_job.
+      # Use the worker gem's own set: an array for sneakers:run, a callable registry
+      # for sneakers:active_job. This must run after the application's after_fork.
       def kicks_liveness_expected_consumers
         classes = config[:worker_classes]
         classes = classes.call if classes.respond_to?(:call)