usps-support 0.2.47 → 0.2.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d7e4b37785b4d1f70b3721aa387bb0101b42fc54965e540ba3ad3f830490d44
-  data.tar.gz: 3b804cf40fd405311f71ee157e6eb364019bf464cd49a4ef052e98646b8fba75
+  metadata.gz: 5764f3cd03bbd09cf0cc9be4873c9ecba705c6f4a5676058a6577de01999a66f
+  data.tar.gz: 50784833a152814d04f12ad8785f2d4c6199b0b88af40ead54c7d0e8c6a0cdf9
 SHA512:
-  metadata.gz: 1960575ba4e2b6408a7370867f26257de717f4203051f7f5ebb539d324b309c51e555106351e3e21ac1bf145432d7209d742ed8758a30bd068ca1207ac2fa4cb
-  data.tar.gz: 3c40989a2707bedf404c19bf0a6f2d3b425f611db081f9f590dbef8e359987927d33a5cf58cde9175b2dffe7a0678b9bc7b080075c4894588ccc401d76801f2a
+  metadata.gz: 1ad1f146fd493edbeb08d110abb5de5b825cb5ab93ef494c6f8e12b554615a056940acf8d9ef9a3f39f64fb6ea2adba5ac7f8870282cecd4dd838f2aa01547fd
+  data.tar.gz: 6d0a9a31c61db44949f1852853f53e2eb29b6b47cb2aa828398cdc1492f81d29c511d250eae688463a5f018b418a64949bd071bec798db7bd8ff23cb790b98db

data/app/controllers/usps/support/health_controller.rb

@@ -10,17 +10,22 @@ module Usps::Support
   # the "HEALTHCHECK_OK" prefix, so a quiet app-level failure flips the Statuspage component even
   # with no organic traffic.
   #
-  # Host apps subclass this and override #checks to pick which dependencies to probe, e.g.
+  # Host apps subclass this and declare which dependencies to probe with a CHECKS constant — a list
+  # of symbols, each naming a #check_<symbol> method:
   #
   #   class HealthController < Usps::Support::HealthController
-  #     private
-  #     def checks = { database: check_database, redis: check_redis }
+  #     CHECKS = %i[database redis hq_database imis].freeze
   #   end
   #
-  # and route to it (`get 'health' => 'health#show'`). The check_* primitives below each return a
-  # boolean and never raise, so the action always responds. Apps run only the checks for the
-  # dependencies they actually configure — admin-console has no HQ connection, so it omits
-  # check_hq_database; an iMIS-backed app adds check_imis; a custom downstream is check_http(url).
+  # and route to it (`get 'health' => 'health#show'`). Each check_* primitive returns a boolean or
+  # raises; the base times every check and treats a raise as a failed check, so the action always
+  # responds. Apps run only the checks they list — admin-console has no HQ connection, so it omits
+  # :hq_database; an iMIS-backed app adds :imis. For a bespoke dependency, define a #check_<name> in
+  # the subclass (e.g. wrapping #check_http(url)) and add <name> to CHECKS.
+  #
+  # Every probe's per-component results (ok, latency, error) are also handed to
+  # Usps::Support::HealthDiagnostics, which — when enabled — publishes them to CloudWatch so a past
+  # failure can be attributed to a specific component after it has recovered.
   #
   # Inherits ActionController::Base directly — NOT the host's ApplicationController — to bypass
   # Devise auth, Pundit authorization, PaperTrail, and especially `allow_browser`, whose
@@ -29,6 +34,11 @@ module Usps::Support
   class HealthController < ActionController::Base # rubocop:disable Rails/ApplicationController
     SENTINEL_PREFIX = 'HEALTHCHECK_OK'
 
+    # Symbols naming the dependency checks to run; each invokes the matching #check_<symbol>.
+    # Override per app by defining a CHECKS constant. Default probes the primary database only,
+    # which presumably every app needs.
+    CHECKS = %i[database].freeze
+
     def show
       results = checks
       healthy = results.values.all?
@@ -38,24 +48,47 @@ module Usps::Support
 
       response.set_header('Cache-Control', 'no-store')
       response.set_header('Retry-After', '30') unless healthy
+      HealthDiagnostics.record(app: app_name, components: @measurements)
       render(plain: body, content_type: 'text/plain', status: healthy ? :ok : :service_unavailable)
     end
 
   private
 
-    # Dependencies to probe, as { label => boolean }.
-    #
-    # Override per app.
-    #
-    # Default is the primary database only, which presumably every app needs.
-    def checks = { database: check_database }
+    # Runs each configured check, returning { label => boolean }. Reads the subclass's CHECKS
+    # constant, so apps pick checks by listing symbols rather than overriding this method. Each run
+    # is timed and its outcome captured in @measurements for HealthDiagnostics.
+    def checks
+      @measurements = []
+      self.class::CHECKS.index_with { |name| measure(name) }
+    end
+
+    # Runs a single named check, timing it and recording its outcome. A raised error reads as a
+    # failed check and its class/message is captured, so a dependency that blows up is attributable
+    # rather than a 500.
+    def measure(name)
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      begin
+        ok = send(:"check_#{name}") ? true : false
+        error = nil
+      rescue StandardError => e
+        ok = false
+        error = "#{e.class}: #{e.message}"
+      end
+      @measurements << { label: name.to_s, ok:, latency_ms: elapsed_ms(started), error: }
+      ok
+    end
+
+    def elapsed_ms(started) = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round(1)
 
     # Derives "admin-console" from the host application's module (e.g. AdminConsole), so the sentinel
     # identifies which app answered without per-app configuration.
     def app_name = Rails.application.class.module_parent_name.underscore.dasherize
 
-    # --- Check primitives. Each returns true/false and swallows its own errors, so a dead
-    # dependency reads as a failed check rather than a 500. ---
+    # --- Check primitives. Each returns true on success and returns false or raises on failure;
+    # #measure catches the raise, so a dead dependency reads as a failed check (with its error
+    # captured) rather than a 500. Named check_<name>, not <name>?, to drive CHECKS dispatch. ---
+
+    # rubocop:disable Naming/PredicateMethod
 
     # Primary application database (RDS).
     def check_database = check_connection(::ActiveRecord::Base)
@@ -67,8 +100,6 @@ module Usps::Support
     def check_redis
       ::Sidekiq.redis { it.call('PING') }
       true
-    rescue StandardError
-      false
     end
 
     # iMIS API liveness — authenticating proves network + credentials + the token endpoint, against
@@ -76,24 +107,18 @@ module Usps::Support
     def check_imis
       ::Usps::Imis::Api.new.auth_token
       true
-    rescue StandardError
-      false
     end
 
     # Generic downstream HTTP liveness for a custom dependency.
-    def check_http(url)
-      ::Net::HTTP.get_response(URI(url)).is_a?(::Net::HTTPSuccess)
-    rescue StandardError
-      false
-    end
+    def check_http(url) = ::Net::HTTP.get_response(URI(url)).is_a?(::Net::HTTPSuccess)
 
     # Checks out a connection from the given model's pool and runs a trivial query — proves the
     # connection is live, which a boot-time check skips.
     def check_connection(model)
       model.connection.execute('SELECT 1')
       true
-    rescue StandardError
-      false
     end
+
+    # rubocop:enable Naming/PredicateMethod
   end
 end

data/lib/usps/support/health_diagnostics.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'socket'
+
+module Usps
+  module Support
+    # Fire-and-forget publisher of per-component health-check results to CloudWatch Logs in
+    # Embedded Metric Format (EMF). CloudWatch extracts a per-(App, Component) metric from each line
+    # while the raw JSON — status, latency, error — stays queryable in Logs Insights. See the
+    # infrastructure repo's doc/health_component_metrics.md.
+    #
+    # Why: /health returns a single 503 when ANY dependency fails, and Route 53's last-failure-reason
+    # keeps only the most recent probe, so once the dependency recovers we can no longer tell WHICH
+    # component failed (e.g. the Exams/Merit Marks flaps, where hq_database and imis are indistinct
+    # from AWS telemetry). This records each component every probe, so a past incident is attributable.
+    #
+    # Disabled unless HEALTH_DIAGNOSTICS_LOG_GROUP is set, so dev/test and unprovisioned apps are
+    # no-ops. Delivery runs on a single background thread off the request path; anything that goes
+    # wrong (queue full, SDK missing, API error) drops the record. Diagnostics must never slow or
+    # fail /health — that endpoint is exactly what stalls during an incident.
+    module HealthDiagnostics
+      NAMESPACE = 'USPS/HealthCheck'
+      MAX_QUEUE = 1_000
+
+      # Exponential backoff bounds for the delivery thread, so a persistent failure (e.g. missing
+      # logs:CreateLogStream) throttles to one retry per MAX_BACKOFF instead of one per probe.
+      BASE_BACKOFF_SECONDS = 1
+      MAX_BACKOFF_SECONDS  = 60
+
+      class << self
+        # Enqueue one probe's component results for delivery. Non-blocking; returns nil.
+        # components: [{ label:, ok:, latency_ms:, error: }, ...]
+        def record(app:, components:)
+          return unless enabled?
+
+          enqueue(app: app, components: components, timestamp_ms: now_ms)
+          nil
+        end
+
+        def enabled? = !log_group.nil?
+
+        def log_group = ENV.fetch('HEALTH_DIAGNOSTICS_LOG_GROUP', nil)
+
+        # Builds the EMF log events for one probe. Pure — the background worker just ships these.
+        def emf_events(app:, components:, timestamp_ms:)
+          components.map do |component|
+            message = {
+              '_aws' => {
+                'Timestamp' => timestamp_ms,
+                'CloudWatchMetrics' => [{
+                  'Namespace' => NAMESPACE,
+                  'Dimensions' => [%w[App Component]],
+                  'Metrics' => [
+                    { 'Name' => 'Unhealthy', 'Unit' => 'Count' },
+                    { 'Name' => 'LatencyMs', 'Unit' => 'Milliseconds' }
+                  ]
+                }]
+              },
+              'App' => app,
+              'Component' => component[:label],
+              'Unhealthy' => component[:ok] ? 0 : 1,
+              'LatencyMs' => component[:latency_ms],
+              'status' => component[:ok] ? 'ok' : 'fail'
+            }
+            message['error'] = component[:error] if component[:error]
+            { timestamp: timestamp_ms, message: message.to_json }
+          end
+        end
+
+      private
+
+        def now_ms = (Time.now.to_f * 1000).to_i
+
+        # :nocov: -- background delivery: a live thread + the AWS SDK, exercised in prod, not units.
+        def enqueue(**item)
+          queue.push(item) if queue.size < MAX_QUEUE
+        end
+
+        def queue
+          return @queue if @queue
+
+          mutex.synchronize { @queue ||= start_worker }
+        end
+
+        def mutex = @mutex ||= Mutex.new
+
+        def start_worker
+          queue = Thread::Queue.new
+          Thread.new do
+            failures = 0
+            loop do
+              if ship(queue.pop)
+                failures = 0
+              else
+                failures += 1
+                sleep([BASE_BACKOFF_SECONDS * (2**(failures - 1)), MAX_BACKOFF_SECONDS].min)
+              end
+            end
+          end
+          queue
+        end
+
+        # Ships one probe's events; returns true on success, false (dropping the batch) on any error.
+        def ship(item)
+          ensure_stream
+          client.put_log_events(
+            log_group_name: log_group,
+            log_stream_name: stream_name,
+            log_events: emf_events(**item)
+          )
+          true
+        rescue StandardError
+          @stream_ensured = false # force a re-create next time
+          false
+        end
+
+        def client
+          require 'aws-sdk-cloudwatchlogs'
+          @client ||= Aws::CloudWatchLogs::Client.new(region: region)
+        end
+
+        def region = ENV['HEALTH_DIAGNOSTICS_REGION'] || ENV['AWS_REGION'] || 'us-east-1'
+
+        # One stream per process, so concurrent writers never contend for the same stream.
+        def stream_name = @stream_name ||= "#{Socket.gethostname}/#{Process.pid}"
+
+        def ensure_stream
+          return if @stream_ensured
+
+          client.create_log_stream(log_group_name: log_group, log_stream_name: stream_name)
+          @stream_ensured = true
+        rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+          @stream_ensured = true
+        end
+        # :nocov:
+      end
+    end
+  end
+end

data/lib/usps/support/version.rb

@@ -2,6 +2,6 @@
 
 module Usps
   module Support
-    VERSION = '0.2.47'
+    VERSION = '0.2.49'
   end
 end

data/lib/usps/support.rb

@@ -17,6 +17,7 @@ end
 require_relative 'support/helpers'
 require_relative 'support/models'
 require_relative 'support/lib'
+require_relative 'support/health_diagnostics'
 
 # The Engine require lives in `usps/all`, not here. This file can
 # legitimately be required before Rails is loaded (e.g. from .simplecov to pull

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: usps-support
 version: !ruby/object:Gem::Version
-  version: 0.2.47
+  version: 0.2.49
 platform: ruby
 authors:
 - Julian Fiander
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudwatchlogs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: usps-imis-api
   requirement: !ruby/object:Gem::Requirement
@@ -83,6 +97,7 @@ files:
 - lib/usps/support/db/members_schema.rb
 - lib/usps/support/db/websites_schema.rb
 - lib/usps/support/engine.rb
+- lib/usps/support/health_diagnostics.rb
 - lib/usps/support/helpers.rb
 - lib/usps/support/helpers/badges_helper.rb
 - lib/usps/support/helpers/flags_helper.rb