usps-support 0.2.50 → 0.2.51

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 634d21f58442e20b6f15886c8b0b91e92a0308d2d455422e9970fca70cf6e8ed
-  data.tar.gz: fdd939321295a783fea487db554a5009cde3b42e612fa67e3137b29eff9ee5e8
+  metadata.gz: 3b6c4b733379c3005f044d0d8e780dcb4b9f1c37124f97ff61eaf3ab5844ba81
+  data.tar.gz: 00c8be1ab0bc6fa06aac0a195a431ed055300d1be7e44441a13d106045bfc01d
 SHA512:
-  metadata.gz: 20322eb372cbca9ede2c3e02d8be07e98d0e39f0a4e3bb5775c9d598c01c17a2fc7bc706c6dffb8969012bd2b09a423202886a27ba030649017cc39a13f81795
-  data.tar.gz: d6622c12395f58e9427c058351de395188bdd08d6206d411b0f5b22ab2b1391149738f6a7ba061fa0e1f8dd04d5df9d29f6a03963a049a9c8284f6fe8724b9ce
+  metadata.gz: 1bfca2dd2afc4080aa03b98fb2c1fa88d3049e1c9db9415ae6bd096b14196135718f1846af24900e6db82f82a8c30c3055bd866aa5e187666def7999834b03b7
+  data.tar.gz: 758b292a1469fd630c0b2281f5ee9db88d0b97a96d2339c19983c4aabd9e1426f12a54762c7547e8b8217abc80d3a6b4629984aa7e52457b7d0475dae4843455

data/app/controllers/concerns/usps/support/dependency_gate.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Usps::Support
+  # Fails requests fast, with the branded 503 page, while a dependency the app cannot work without
+  # is known to be down — instead of letting every request discover that for itself by blocking on
+  # the dead dependency.
+  #
+  # ## Why
+  #
+  # The HQ member database is in the request path of *every authenticated request*:
+  # `authenticate_user_from_jwt!` resolves the JWT's certificate through `Members::Member.find`.
+  # While HQ is unreachable each of those requests blocks until the database client's connect
+  # timeout expires, then raises a connection error that ApplicationController rescues into the 503
+  # page. The page is correct; paying a connect timeout to produce it is what hurts. The web tier
+  # serves a small, fixed number of requests concurrently, so a per-request stall of seconds is
+  # enough to occupy every worker: requests queue, nginx gives up on the upstream, and an app that
+  # is *up and answering correctly* reads to users as hard down.
+  #
+  # This gate closes that gap: the app already *knows* HQ is down, because HealthController's
+  # monitor probes it on a background thread and leaves the verdict in memory (see
+  # health_check_refresh_design.md). Consulting that costs no I/O and no worker time, so the request
+  # returns the same 503 immediately instead of after a timeout.
+  #
+  # ## Contract
+  #
+  # - **/health is untouched.** The gate is a *reader* of the snapshot; it never runs a check, never
+  #   changes what /health probes, renders, or reports to CloudWatch. The synthetic probes and
+  #   Statuspage components keep behaving exactly as they do today — the app's own gating must never
+  #   become the thing that decides whether we call ourselves up.
+  # - **Never blocks.** It reads the last completed snapshot only (HealthMonitor#cached), so a
+  #   process that has not yet run a round serves the request normally rather than paying for a
+  #   round on the request thread.
+  # - **Fails open.** No snapshot, a stale one (a wedged refresher), or any error reading it means
+  #   "no opinion" and the request proceeds — the pre-existing `rescue_from` for the connection
+  #   errors is still the backstop, so failing open costs latency, never correctness.
+  # - **Requires a sustained failure.** Gating opens only after `dependency_gate_threshold`
+  #   consecutive failed rounds, so a single slow round — one that merely overran its check budget —
+  #   can't blackhole a healthy app. It closes again on the first passing round, with no deploy or
+  #   restart needed.
+  #
+  # ## Wiring
+  #
+  #   class ApplicationController < ActionController::Base
+  #     include Usps::Support::DependencyGate
+  #     self.dependency_gate_title = 'Exams System'
+  #   end
+  #
+  # `gated_dependencies` defaults to `%i[hq_database]` — the only check that is in the user request
+  # path. iMIS deliberately is not: it is an external service reached outside the VPN and is touched
+  # only by rescued admin reports and Sidekiq jobs, so an iMIS outage must not gate the app.
+  #
+  # Actions that genuinely do not need the gated dependency opt out the same way they opt out of
+  # authentication, and for the same reason — keeping work that still functions during an HQ outage
+  # (e.g. the unauthenticated online-examination flow, which lives entirely in RDS) working:
+  #
+  #   skip_before_action :require_gated_dependencies!, only: %i[exam submit_response]
+  #
+  module DependencyGate
+    extend ActiveSupport::Concern
+
+    included do
+      # Checks (by their HealthController CHECKS name) that must be up for a request to be served.
+      class_attribute :gated_dependencies, default: %i[hq_database]
+
+      # Consecutive failed check rounds before the gate opens; rounds are health_refresh_interval
+      # apart, so this trades detection speed against tolerance for a one-off slow round.
+      class_attribute :dependency_gate_threshold, default: 2
+
+      # Title for the rendered error page, matching what the rescue path uses ('Exams System').
+      class_attribute :dependency_gate_title, default: nil
+
+      # The app's HealthController subclass, whose monitor owns the snapshot. Resolved lazily (and
+      # by name) so this loads cleanly in an app whose controllers autoload later.
+      class_attribute :dependency_gate_health_controller, default: -> { ::HealthController }
+
+      # Runs before every other filter — notably before authentication, which is itself what reaches
+      # HQ and would block.
+      prepend_before_action :require_gated_dependencies!
+    end
+
+    # Logs gate open/close transitions once per process rather than once per request: during an
+    # outage every request is gated, and a per-request line would bury everything else in the log.
+    # A race between threads can duplicate a line; that is cheaper than a lock on the request path.
+    def self.log_transition(down)
+      return if @down == down
+
+      was_open = @down.present?
+      @down = down
+
+      if down.any?
+        ::Rails.logger.warn("[dependency-gate] open: #{down.join(', ')} down, serving 503 until recovered")
+      elsif was_open
+        ::Rails.logger.warn('[dependency-gate] closed: gated dependencies recovered, serving normally')
+      end
+      # A process whose first request is already healthy has nothing to announce.
+    end
+
+    # Forgets the last logged state. For tests.
+    def self.reset_transition_log! = @down = nil
+
+  private
+
+    def require_gated_dependencies!
+      down = unavailable_dependencies
+      DependencyGate.log_transition(down)
+      return if down.empty?
+
+      render_dependencies_unavailable
+    end
+
+    # Gated dependencies whose failure has persisted for at least the threshold. Anything the
+    # snapshot has no entry for — unknown, cold, or stale — is absent from the status hash and so
+    # reads as available (fail open).
+    def unavailable_dependencies
+      status = dependency_gate_status
+
+      gated_dependencies.select { |name| status.dig(name, :consecutive_failures).to_i >= dependency_gate_threshold }
+    end
+
+    # Deliberately swallows everything: a gate that raises would turn a degraded app into a broken
+    # one, which is the opposite of the point.
+    def dependency_gate_status
+      dependency_gate_health_controller.call.dependency_status
+    rescue StandardError => e
+      ::Rails.logger.warn("[dependency-gate] status unavailable (#{e.class}: #{e.message}); serving normally")
+      {}
+    end
+
+    # The same branded 503 the ActiveRecord rescue path renders, so a gated request is
+    # indistinguishable to a user (and to a browser, which honours Retry-After) from one that
+    # discovered the outage the slow way. No Bugsnag notify: unlike the rescue path this fires on
+    # *every* request for the duration of an outage, and the CloudWatch alarm and Statuspage
+    # component already own that signal.
+    #
+    # The `error` layout is required — the application layout reads `current_user`, which would
+    # reconnect to the unavailable HQ database from inside the handler.
+    def render_dependencies_unavailable
+      @title = dependency_gate_title
+      response.set_header('Retry-After', '30')
+
+      respond_to do |format|
+        format.html { render('errors/service_unavailable', layout: 'error', status: :service_unavailable) }
+        format.json { render(json: { error: 'Service unavailable.' }, status: :service_unavailable) }
+        format.any  { head(:service_unavailable) }
+      end
+    end
+  end
+end

data/app/controllers/usps/support/health_controller.rb

@@ -49,11 +49,15 @@ module Usps::Support
     #   health_check_timeout      — per-check budget in seconds; a check that overruns is a failure
     #   health_refresh_interval   — seconds between background refreshes of the cached snapshot
     #   health_snapshot_max_age   — a cached snapshot older than this is refreshed inline before serving
-    #   health_background_refresh — run the background refresher (false in tests for synchronous runs)
+    #   health_background_refresh — run the background refresher; off under test, where refreshes run
+    #                               synchronously and an unannounced thread probing real dependencies
+    #                               (DependencyGate starts one from any request) would be a surprise
+    #   health_status_max_age     — .dependency_status ignores a snapshot older than this (fails open)
     class_attribute :health_check_timeout, default: 2
     class_attribute :health_refresh_interval, default: 15
     class_attribute :health_snapshot_max_age, default: 45
-    class_attribute :health_background_refresh, default: true
+    class_attribute :health_background_refresh, default: !::Rails.env.test?
+    class_attribute :health_status_max_age, default: 90
 
     class << self
       # One monitor per concrete subclass, per process. Built lazily so the background thread starts
@@ -68,10 +72,37 @@ module Usps::Support
         )
       end
 
-      # Drops the memoized monitor (stopping its thread). For tests.
+      # Read-only, non-blocking view of the last completed round, for callers on a *user* request
+      # thread — specifically Usps::Support::DependencyGate, which uses it to fail a request fast
+      # rather than discovering the outage by blocking on the dead dependency itself:
+      #
+      #   { hq_database: { ok: false, consecutive_failures: 4 }, database: { ok: true, ... } }
+      #
+      # Runs no check and takes no lock; it reads whatever the background refresher last stored.
+      # Returns {} — "no opinion", so callers fail *open* — when no round has completed yet (cold
+      # worker) or the snapshot has aged past health_status_max_age, which means the refresher itself
+      # is wedged and its last verdict is no longer evidence about the dependency.
+      def dependency_status
+        snapshot = health_monitor.cached
+        return {} unless snapshot && health_monitor.age_of(snapshot) <= health_status_max_age
+
+        snapshot.data&.[](:status) || {}
+      end
+
+      # Consecutive failed rounds per check, updated once per refresh so a gate can require a
+      # *sustained* failure and ignore a single slow round. Called only from the monitor's runner,
+      # and the monitor serialises refreshes under its own mutex, so this needs no lock of its own.
+      def record_streaks(results)
+        @streaks ||= Hash.new(0)
+        results.each { |name, ok| ok ? @streaks[name] = 0 : @streaks[name] += 1 }
+        results.to_h { |name, ok| [name, { ok:, consecutive_failures: @streaks[name] }] }
+      end
+
+      # Drops the memoized monitor (stopping its thread) and the failure streaks. For tests.
       def reset_health_monitor!
         @health_monitor&.stop
         @health_monitor = nil
+        @streaks = nil
       end
     end
 
@@ -114,11 +145,13 @@ module Usps::Support
       data
     end
 
-    # Runs each configured check, returning { app:, results: { label => boolean }, measurements: }.
+    # Runs each configured check, returning { app:, results: { label => boolean }, measurements:,
+    # status: }. Only :results feeds the rendered body; :status carries the per-check failure streaks
+    # that .dependency_status hands to the request-path gate.
     def perform_checks
       measurements = []
       results = self.class::CHECKS.index_with { |name| measure(name, measurements) }
-      { app: app_name, results:, measurements: }
+      { app: app_name, results:, measurements:, status: self.class.record_streaks(results) }
     end
 
     # Runs a single named check under the per-check timeout, timing it and recording its outcome. A

data/lib/usps/support/health_monitor.rb

@@ -59,6 +59,22 @@ module Usps
 
       def fresh?(snapshot) = (@clock.call - snapshot.at) <= @max_age
 
+      # The last completed snapshot, or nil if none has been taken yet -- *without* refreshing, even
+      # when stale. #current is for /health, which must answer accurately and may pay for an inline
+      # round; this is for callers on a user request thread (Usps::Support::DependencyGate), where
+      # running a check would reintroduce exactly the stall they exist to avoid. Starts the
+      # background refresher on first use so a process that never serves /health still gets one.
+      # Callers judge staleness themselves via #age_of.
+      def cached
+        ensure_running if @background
+
+        @snapshot
+      end
+
+      # Seconds since the snapshot was taken, on the monitor's own (monotonic) clock -- callers can't
+      # compare `snapshot.at` against a wall clock of their own.
+      def age_of(snapshot) = @clock.call - snapshot.at
+
       # Stops the background refresher. Used by tests; also safe to call on shutdown.
       def stop
         @mutex.synchronize do

data/lib/usps/support/version.rb

@@ -2,6 +2,6 @@
 
 module Usps
   module Support
-    VERSION = '0.2.50'
+    VERSION = '0.2.51'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: usps-support
 version: !ruby/object:Gem::Version
-  version: 0.2.50
+  version: 0.2.51
 platform: ruby
 authors:
 - Julian Fiander
@@ -86,6 +86,7 @@ extra_rdoc_files: []
 files:
 - Readme.md
 - app/controllers/concerns/usps/support/admin_menu.rb
+- app/controllers/concerns/usps/support/dependency_gate.rb
 - app/controllers/usps/support/admins_controller.rb
 - app/controllers/usps/support/health_controller.rb
 - app/models/usps/support/policy/admin_context.rb