usps-support 0.2.49 → 0.2.50

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5764f3cd03bbd09cf0cc9be4873c9ecba705c6f4a5676058a6577de01999a66f
-  data.tar.gz: 50784833a152814d04f12ad8785f2d4c6199b0b88af40ead54c7d0e8c6a0cdf9
+  metadata.gz: 634d21f58442e20b6f15886c8b0b91e92a0308d2d455422e9970fca70cf6e8ed
+  data.tar.gz: fdd939321295a783fea487db554a5009cde3b42e612fa67e3137b29eff9ee5e8
 SHA512:
-  metadata.gz: 1ad1f146fd493edbeb08d110abb5de5b825cb5ab93ef494c6f8e12b554615a056940acf8d9ef9a3f39f64fb6ea2adba5ac7f8870282cecd4dd838f2aa01547fd
-  data.tar.gz: 6d0a9a31c61db44949f1852853f53e2eb29b6b47cb2aa828398cdc1492f81d29c511d250eae688463a5f018b418a64949bd071bec798db7bd8ff23cb790b98db
+  metadata.gz: 20322eb372cbca9ede2c3e02d8be07e98d0e39f0a4e3bb5775c9d598c01c17a2fc7bc706c6dffb8969012bd2b09a423202886a27ba030649017cc39a13f81795
+  data.tar.gz: d6622c12395f58e9427c058351de395188bdd08d6206d411b0f5b22ab2b1391149738f6a7ba061fa0e1f8dd04d5df9d29f6a03963a049a9c8284f6fe8724b9ce

data/app/controllers/usps/support/health_controller.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 require 'net/http'
+require 'timeout'
 
 module Usps::Support
   # Base controller for external synthetic health probes (see the infrastructure repo's
   # doc/application_health_probes.md). Unlike Rails' /up (boot-only) and nginx's static
   # /instance-health-check, this exercises an app's *critical dependencies* and returns the sentinel
-  # "HEALTHCHECK_OK <app>" only when every check passes — a Route 53 HTTPS_STR_MATCH probe asserts
-  # the "HEALTHCHECK_OK" prefix, so a quiet app-level failure flips the Statuspage component even
-  # with no organic traffic.
+  # "HEALTHCHECK_OK <app>" only when every check passes, so a Route 53 HTTPS_STR_MATCH probe
+  # asserting that prefix flips the Statuspage component even with no organic traffic.
   #
   # Host apps subclass this and declare which dependencies to probe with a CHECKS constant — a list
   # of symbols, each naming a #check_<symbol> method:
@@ -17,15 +17,18 @@ module Usps::Support
   #     CHECKS = %i[database redis hq_database imis].freeze
   #   end
   #
-  # and route to it (`get 'health' => 'health#show'`). Each check_* primitive returns a boolean or
-  # raises; the base times every check and treats a raise as a failed check, so the action always
-  # responds. Apps run only the checks they list — admin-console has no HQ connection, so it omits
-  # :hq_database; an iMIS-backed app adds :imis. For a bespoke dependency, define a #check_<name> in
-  # the subclass (e.g. wrapping #check_http(url)) and add <name> to CHECKS.
+  # and route to it (`get 'health' => 'health#show'`). For a bespoke dependency, define a
+  # #check_<name> in the subclass (e.g. wrapping #check_http(url)) and add <name> to CHECKS.
   #
-  # Every probe's per-component results (ok, latency, error) are also handed to
-  # Usps::Support::HealthDiagnostics, which — when enabled — publishes them to CloudWatch so a past
-  # failure can be attributed to a specific component after it has recovered.
+  # Checks run on a background thread, not the request thread: a per-process refresher updates a
+  # cached snapshot every health_refresh_interval seconds and #show serves that snapshot, so a slow
+  # or wedged dependency can't tie up the Passenger worker pool. Each check also runs under
+  # health_check_timeout, and a raise or timeout reads as a failed check (its error captured) so the
+  # action always responds. See doc/health_check_refresh_design.md.
+  #
+  # Per-component results (ok, latency, error) are handed to Usps::Support::HealthDiagnostics once
+  # per refresh which — when enabled — publishes them to CloudWatch so a past failure can be
+  # attributed to a specific component after it has recovered.
   #
   # Inherits ActionController::Base directly — NOT the host's ApplicationController — to bypass
   # Devise auth, Pundit authorization, PaperTrail, and especially `allow_browser`, whose
@@ -34,47 +37,103 @@ module Usps::Support
   class HealthController < ActionController::Base # rubocop:disable Rails/ApplicationController
     SENTINEL_PREFIX = 'HEALTHCHECK_OK'
 
+    # Raised when a single check overruns the per-check timeout; recorded as a failed check.
+    class CheckTimeout < StandardError; end
+
     # Symbols naming the dependency checks to run; each invokes the matching #check_<symbol>.
     # Override per app by defining a CHECKS constant. Default probes the primary database only,
     # which presumably every app needs.
     CHECKS = %i[database].freeze
 
+    # Tunables (overridable per subclass, e.g. `self.health_check_timeout = 1` or in a test):
+    #   health_check_timeout      — per-check budget in seconds; a check that overruns is a failure
+    #   health_refresh_interval   — seconds between background refreshes of the cached snapshot
+    #   health_snapshot_max_age   — a cached snapshot older than this is refreshed inline before serving
+    #   health_background_refresh — run the background refresher (false in tests for synchronous runs)
+    class_attribute :health_check_timeout, default: 2
+    class_attribute :health_refresh_interval, default: 15
+    class_attribute :health_snapshot_max_age, default: 45
+    class_attribute :health_background_refresh, default: true
+
+    class << self
+      # One monitor per concrete subclass, per process. Built lazily so the background thread starts
+      # after Passenger forks its workers (threads do not survive fork).
+      def health_monitor
+        klass = self
+        @health_monitor ||= HealthMonitor.new(
+          interval: health_refresh_interval,
+          max_age: health_snapshot_max_age,
+          background: health_background_refresh,
+          runner: -> { klass.new.send(:run_and_record) }
+        )
+      end
+
+      # Drops the memoized monitor (stopping its thread). For tests.
+      def reset_health_monitor!
+        @health_monitor&.stop
+        @health_monitor = nil
+      end
+    end
+
     def show
-      results = checks
-      healthy = results.values.all?
+      data = self.class.health_monitor.current
+      return render_unavailable unless data
 
-      body = "#{healthy ? "#{SENTINEL_PREFIX} #{app_name}" : "HEALTHCHECK_FAIL #{app_name}"}\n"
-      results.each { |name, ok| body << "#{name}=#{ok ? 'ok' : 'fail'}\n" }
+      render_health(data)
+    end
+
+  private
+
+    # Renders the sentinel line, per-check status lines, and cache/retry headers from a snapshot.
+    def render_health(data)
+      healthy = data[:results].values.all?
+
+      body = "#{healthy ? "#{SENTINEL_PREFIX} #{data[:app]}" : "HEALTHCHECK_FAIL #{data[:app]}"}\n"
+      data[:results].each { |name, ok| body << "#{name}=#{ok ? 'ok' : 'fail'}\n" }
 
       response.set_header('Cache-Control', 'no-store')
       response.set_header('Retry-After', '30') unless healthy
-      HealthDiagnostics.record(app: app_name, components: @measurements)
       render(plain: body, content_type: 'text/plain', status: healthy ? :ok : :service_unavailable)
     end
 
-  private
+    # Fallback for the (rare) case where not even the first refresh could produce a snapshot.
+    def render_unavailable
+      response.set_header('Cache-Control', 'no-store')
+      response.set_header('Retry-After', '30')
+      render(plain: "HEALTHCHECK_FAIL #{app_name}\nsnapshot=unavailable\n",
+             content_type: 'text/plain', status: :service_unavailable)
+    end
+
+    # One round of checks + diagnostics, invoked by the monitor's runner (background thread, or the
+    # inline cold/stale path). Wrapped in the Rails executor so ActiveRecord connections checked out
+    # by the DB checks are returned even though this runs outside a request. Returns the render data.
+    def run_and_record
+      data = nil
+      ::Rails.application.executor.wrap { data = perform_checks }
+      HealthDiagnostics.record(app: data[:app], components: data[:measurements])
+      data
+    end
 
-    # Runs each configured check, returning { label => boolean }. Reads the subclass's CHECKS
-    # constant, so apps pick checks by listing symbols rather than overriding this method. Each run
-    # is timed and its outcome captured in @measurements for HealthDiagnostics.
-    def checks
-      @measurements = []
-      self.class::CHECKS.index_with { |name| measure(name) }
+    # Runs each configured check, returning { app:, results: { label => boolean }, measurements: }.
+    def perform_checks
+      measurements = []
+      results = self.class::CHECKS.index_with { |name| measure(name, measurements) }
+      { app: app_name, results:, measurements: }
     end
 
-    # Runs a single named check, timing it and recording its outcome. A raised error reads as a
-    # failed check and its class/message is captured, so a dependency that blows up is attributable
-    # rather than a 500.
-    def measure(name)
+    # Runs a single named check under the per-check timeout, timing it and recording its outcome. A
+    # raised error or a timeout reads as a failed check (its class/message captured), so a dependency
+    # that blows up or hangs is attributable rather than a 500 or a stalled worker.
+    def measure(name, measurements)
       started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       begin
-        ok = send(:"check_#{name}") ? true : false
+        ok = Timeout.timeout(health_check_timeout, CheckTimeout) { send(:"check_#{name}") } ? true : false
         error = nil
       rescue StandardError => e
         ok = false
         error = "#{e.class}: #{e.message}"
       end
-      @measurements << { label: name.to_s, ok:, latency_ms: elapsed_ms(started), error: }
+      measurements << { label: name.to_s, ok:, latency_ms: elapsed_ms(started), error: }
       ok
     end
 
@@ -85,8 +144,9 @@ module Usps::Support
     def app_name = Rails.application.class.module_parent_name.underscore.dasherize
 
     # --- Check primitives. Each returns true on success and returns false or raises on failure;
-    # #measure catches the raise, so a dead dependency reads as a failed check (with its error
-    # captured) rather than a 500. Named check_<name>, not <name>?, to drive CHECKS dispatch. ---
+    # #measure catches the raise/timeout, so a dead or slow dependency reads as a failed check (with
+    # its error captured) rather than a 500. Named check_<name>, not <name>?, to drive CHECKS
+    # dispatch. ---
 
     # rubocop:disable Naming/PredicateMethod
 

data/lib/usps/support/health_monitor.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Usps
+  module Support
+    # Keeps a process-local snapshot of the latest health-check results so /health can answer from
+    # memory instead of running the checks on the request thread. See
+    # doc/health_check_refresh_design.md for the rationale.
+    #
+    # runner is a callable that runs one full round of checks and returns whatever the caller needs
+    # to render (HealthController returns { app:, results:, measurements: }). It is expected to bound
+    # its own checks; the monitor adds no timeout of its own.
+    #
+    # One monitor exists per concrete HealthController subclass, per process. It is created lazily on
+    # the first request, so the background thread starts after Passenger forks its workers (threads
+    # do not survive fork).
+    class HealthMonitor
+      Snapshot = Struct.new(:data, :at, keyword_init: true)
+
+      # interval:   seconds between background refreshes
+      # max_age:    a snapshot older than this is refreshed inline before use, so a wedged or
+      #             never-started background thread can't serve dangerously old data
+      # runner:     callable that runs one round of checks and returns the render data
+      # background: start the background refresher (false in tests, so refreshes are synchronous)
+      # clock:      monotonic clock (seconds) to read; injectable for deterministic staleness tests
+      def initialize(interval:, max_age:, runner:, background: true, clock: nil)
+        @interval = interval
+        @max_age = max_age
+        @runner = runner
+        @background = background
+        @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        @mutex = Mutex.new
+      end
+
+      # The latest results. Serves a fresh cached snapshot without touching any dependency; on a cold
+      # (first call) or stale snapshot, refreshes inline so the answer is never dangerously old. Also
+      # starts the background refresher on first use. Returns the runner's data, or nil only if the
+      # very first refresh failed and there is no prior snapshot to fall back on.
+      def current
+        ensure_running if @background
+
+        snapshot = @snapshot
+        return snapshot.data if snapshot && fresh?(snapshot)
+
+        refresh
+      end
+
+      # Runs one round of checks and stores the snapshot; returns the fresh data. On a runner error,
+      # keeps the last good snapshot and returns its data (nil if there was none), so a transient
+      # failure to run the checks never blanks out /health.
+      def refresh
+        @mutex.synchronize do
+          data = @runner.call
+          @snapshot = Snapshot.new(data:, at: @clock.call)
+          data
+        end
+      rescue StandardError
+        @snapshot&.data
+      end
+
+      def fresh?(snapshot) = (@clock.call - snapshot.at) <= @max_age
+
+      # Stops the background refresher. Used by tests; also safe to call on shutdown.
+      def stop
+        @mutex.synchronize do
+          @thread&.kill
+          @thread = nil
+        end
+      end
+
+    private
+
+      # Starts the background refresher thread; exercised in prod (and the background spec) rather
+      # than counted as a unit, matching HealthDiagnostics' delivery worker.
+      # :nocov:
+      def ensure_running
+        return if @thread&.alive?
+
+        @mutex.synchronize do
+          return if @thread&.alive?
+
+          @thread = Thread.new do
+            loop do
+              refresh
+              sleep(@interval)
+            end
+          end
+        end
+      end
+      # :nocov:
+    end
+  end
+end

data/lib/usps/support/version.rb

@@ -2,6 +2,6 @@
 
 module Usps
   module Support
-    VERSION = '0.2.49'
+    VERSION = '0.2.50'
   end
 end

data/lib/usps/support.rb

@@ -18,6 +18,7 @@ require_relative 'support/helpers'
 require_relative 'support/models'
 require_relative 'support/lib'
 require_relative 'support/health_diagnostics'
+require_relative 'support/health_monitor'
 
 # The Engine require lives in `usps/all`, not here. This file can
 # legitimately be required before Rails is loaded (e.g. from .simplecov to pull

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: usps-support
 version: !ruby/object:Gem::Version
-  version: 0.2.49
+  version: 0.2.50
 platform: ruby
 authors:
 - Julian Fiander
@@ -46,7 +46,7 @@ dependencies:
         version: '0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.1
+        version: 0.15.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -56,7 +56,7 @@ dependencies:
         version: '0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.1
+        version: 0.15.0
 - !ruby/object:Gem::Dependency
   name: usps-jwt_auth
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +98,7 @@ files:
 - lib/usps/support/db/websites_schema.rb
 - lib/usps/support/engine.rb
 - lib/usps/support/health_diagnostics.rb
+- lib/usps/support/health_monitor.rb
 - lib/usps/support/helpers.rb
 - lib/usps/support/helpers/badges_helper.rb
 - lib/usps/support/helpers/flags_helper.rb