usps-support 0.2.49 → 0.2.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3b6c4b733379c3005f044d0d8e780dcb4b9f1c37124f97ff61eaf3ab5844ba81
|
|
4
|
+
data.tar.gz: 00c8be1ab0bc6fa06aac0a195a431ed055300d1be7e44441a13d106045bfc01d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1bfca2dd2afc4080aa03b98fb2c1fa88d3049e1c9db9415ae6bd096b14196135718f1846af24900e6db82f82a8c30c3055bd866aa5e187666def7999834b03b7
|
|
7
|
+
data.tar.gz: 758b292a1469fd630c0b2281f5ee9db88d0b97a96d2339c19983c4aabd9e1426f12a54762c7547e8b8217abc80d3a6b4629984aa7e52457b7d0475dae4843455
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Usps::Support
|
|
4
|
+
# Fails requests fast, with the branded 503 page, while a dependency the app cannot work without
|
|
5
|
+
# is known to be down — instead of letting every request discover that for itself by blocking on
|
|
6
|
+
# the dead dependency.
|
|
7
|
+
#
|
|
8
|
+
# ## Why
|
|
9
|
+
#
|
|
10
|
+
# The HQ member database is in the request path of *every authenticated request*:
|
|
11
|
+
# `authenticate_user_from_jwt!` resolves the JWT's certificate through `Members::Member.find`.
|
|
12
|
+
# While HQ is unreachable each of those requests blocks until the database client's connect
|
|
13
|
+
# timeout expires, then raises a connection error that ApplicationController rescues into the 503
|
|
14
|
+
# page. The page is correct; paying a connect timeout to produce it is what hurts. The web tier
|
|
15
|
+
# serves a small, fixed number of requests concurrently, so a per-request stall of seconds is
|
|
16
|
+
# enough to occupy every worker: requests queue, nginx gives up on the upstream, and an app that
|
|
17
|
+
# is *up and answering correctly* reads to users as hard down.
|
|
18
|
+
#
|
|
19
|
+
# This gate closes that gap: the app already *knows* HQ is down, because HealthController's
|
|
20
|
+
# monitor probes it on a background thread and leaves the verdict in memory (see
|
|
21
|
+
# health_check_refresh_design.md). Consulting that costs no I/O and no worker time, so the request
|
|
22
|
+
# returns the same 503 immediately instead of after a timeout.
|
|
23
|
+
#
|
|
24
|
+
# ## Contract
|
|
25
|
+
#
|
|
26
|
+
# - **/health is untouched.** The gate is a *reader* of the snapshot; it never runs a check, never
|
|
27
|
+
# changes what /health probes, renders, or reports to CloudWatch. The synthetic probes and
|
|
28
|
+
# Statuspage components keep behaving exactly as they do today — the app's own gating must never
|
|
29
|
+
# become the thing that decides whether we call ourselves up.
|
|
30
|
+
# - **Never blocks.** It reads the last completed snapshot only (HealthMonitor#cached), so a
|
|
31
|
+
# process that has not yet run a round serves the request normally rather than paying for a
|
|
32
|
+
# round on the request thread.
|
|
33
|
+
# - **Fails open.** No snapshot, a stale one (a wedged refresher), or any error reading it means
|
|
34
|
+
# "no opinion" and the request proceeds — the pre-existing `rescue_from` for the connection
|
|
35
|
+
# errors is still the backstop, so failing open costs latency, never correctness.
|
|
36
|
+
# - **Requires a sustained failure.** Gating opens only after `dependency_gate_threshold`
|
|
37
|
+
# consecutive failed rounds, so a single slow round — one that merely overran its check budget —
|
|
38
|
+
# can't blackhole a healthy app. It closes again on the first passing round, with no deploy or
|
|
39
|
+
# restart needed.
|
|
40
|
+
#
|
|
41
|
+
# ## Wiring
|
|
42
|
+
#
|
|
43
|
+
# class ApplicationController < ActionController::Base
|
|
44
|
+
# include Usps::Support::DependencyGate
|
|
45
|
+
# self.dependency_gate_title = 'Exams System'
|
|
46
|
+
# end
|
|
47
|
+
#
|
|
48
|
+
# `gated_dependencies` defaults to `%i[hq_database]` — the only check that is in the user request
|
|
49
|
+
# path. iMIS deliberately is not: it is an external service reached outside the VPN and is touched
|
|
50
|
+
# only by rescued admin reports and Sidekiq jobs, so an iMIS outage must not gate the app.
|
|
51
|
+
#
|
|
52
|
+
# Actions that genuinely do not need the gated dependency opt out the same way they opt out of
|
|
53
|
+
# authentication, and for the same reason — keeping work that still functions during an HQ outage
|
|
54
|
+
# (e.g. the unauthenticated online-examination flow, which lives entirely in RDS) working:
|
|
55
|
+
#
|
|
56
|
+
# skip_before_action :require_gated_dependencies!, only: %i[exam submit_response]
|
|
57
|
+
#
|
|
58
|
+
module DependencyGate
|
|
59
|
+
extend ActiveSupport::Concern
|
|
60
|
+
|
|
61
|
+
included do
|
|
62
|
+
# Checks (by their HealthController CHECKS name) that must be up for a request to be served.
|
|
63
|
+
class_attribute :gated_dependencies, default: %i[hq_database]
|
|
64
|
+
|
|
65
|
+
# Consecutive failed check rounds before the gate opens; rounds are health_refresh_interval
|
|
66
|
+
# apart, so this trades detection speed against tolerance for a one-off slow round.
|
|
67
|
+
class_attribute :dependency_gate_threshold, default: 2
|
|
68
|
+
|
|
69
|
+
# Title for the rendered error page, matching what the rescue path uses ('Exams System').
|
|
70
|
+
class_attribute :dependency_gate_title, default: nil
|
|
71
|
+
|
|
72
|
+
# The app's HealthController subclass, whose monitor owns the snapshot. Resolved lazily (and
|
|
73
|
+
# by name) so this loads cleanly in an app whose controllers autoload later.
|
|
74
|
+
class_attribute :dependency_gate_health_controller, default: -> { ::HealthController }
|
|
75
|
+
|
|
76
|
+
# Runs before every other filter — notably before authentication, which is itself what reaches
|
|
77
|
+
# HQ and would block.
|
|
78
|
+
prepend_before_action :require_gated_dependencies!
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Logs gate open/close transitions once per process rather than once per request: during an
|
|
82
|
+
# outage every request is gated, and a per-request line would bury everything else in the log.
|
|
83
|
+
# A race between threads can duplicate a line; that is cheaper than a lock on the request path.
|
|
84
|
+
def self.log_transition(down)
|
|
85
|
+
return if @down == down
|
|
86
|
+
|
|
87
|
+
was_open = @down.present?
|
|
88
|
+
@down = down
|
|
89
|
+
|
|
90
|
+
if down.any?
|
|
91
|
+
::Rails.logger.warn("[dependency-gate] open: #{down.join(', ')} down, serving 503 until recovered")
|
|
92
|
+
elsif was_open
|
|
93
|
+
::Rails.logger.warn('[dependency-gate] closed: gated dependencies recovered, serving normally')
|
|
94
|
+
end
|
|
95
|
+
# A process whose first request is already healthy has nothing to announce.
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Forgets the last logged state. For tests.
|
|
99
|
+
def self.reset_transition_log! = @down = nil
|
|
100
|
+
|
|
101
|
+
private
|
|
102
|
+
|
|
103
|
+
def require_gated_dependencies!
|
|
104
|
+
down = unavailable_dependencies
|
|
105
|
+
DependencyGate.log_transition(down)
|
|
106
|
+
return if down.empty?
|
|
107
|
+
|
|
108
|
+
render_dependencies_unavailable
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Gated dependencies whose failure has persisted for at least the threshold. Anything the
|
|
112
|
+
# snapshot has no entry for — unknown, cold, or stale — is absent from the status hash and so
|
|
113
|
+
# reads as available (fail open).
|
|
114
|
+
def unavailable_dependencies
|
|
115
|
+
status = dependency_gate_status
|
|
116
|
+
|
|
117
|
+
gated_dependencies.select { |name| status.dig(name, :consecutive_failures).to_i >= dependency_gate_threshold }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Deliberately swallows everything: a gate that raises would turn a degraded app into a broken
|
|
121
|
+
# one, which is the opposite of the point.
|
|
122
|
+
def dependency_gate_status
|
|
123
|
+
dependency_gate_health_controller.call.dependency_status
|
|
124
|
+
rescue StandardError => e
|
|
125
|
+
::Rails.logger.warn("[dependency-gate] status unavailable (#{e.class}: #{e.message}); serving normally")
|
|
126
|
+
{}
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# The same branded 503 the ActiveRecord rescue path renders, so a gated request is
|
|
130
|
+
# indistinguishable to a user (and to a browser, which honours Retry-After) from one that
|
|
131
|
+
# discovered the outage the slow way. No Bugsnag notify: unlike the rescue path this fires on
|
|
132
|
+
# *every* request for the duration of an outage, and the CloudWatch alarm and Statuspage
|
|
133
|
+
# component already own that signal.
|
|
134
|
+
#
|
|
135
|
+
# The `error` layout is required — the application layout reads `current_user`, which would
|
|
136
|
+
# reconnect to the unavailable HQ database from inside the handler.
|
|
137
|
+
def render_dependencies_unavailable
|
|
138
|
+
@title = dependency_gate_title
|
|
139
|
+
response.set_header('Retry-After', '30')
|
|
140
|
+
|
|
141
|
+
respond_to do |format|
|
|
142
|
+
format.html { render('errors/service_unavailable', layout: 'error', status: :service_unavailable) }
|
|
143
|
+
format.json { render(json: { error: 'Service unavailable.' }, status: :service_unavailable) }
|
|
144
|
+
format.any { head(:service_unavailable) }
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'net/http'
|
|
4
|
+
require 'timeout'
|
|
4
5
|
|
|
5
6
|
module Usps::Support
|
|
6
7
|
# Base controller for external synthetic health probes (see the infrastructure repo's
|
|
7
8
|
# doc/application_health_probes.md). Unlike Rails' /up (boot-only) and nginx's static
|
|
8
9
|
# /instance-health-check, this exercises an app's *critical dependencies* and returns the sentinel
|
|
9
|
-
# "HEALTHCHECK_OK <app>" only when every check passes
|
|
10
|
-
#
|
|
11
|
-
# with no organic traffic.
|
|
10
|
+
# "HEALTHCHECK_OK <app>" only when every check passes, so a Route 53 HTTPS_STR_MATCH probe
|
|
11
|
+
# asserting that prefix flips the Statuspage component even with no organic traffic.
|
|
12
12
|
#
|
|
13
13
|
# Host apps subclass this and declare which dependencies to probe with a CHECKS constant — a list
|
|
14
14
|
# of symbols, each naming a #check_<symbol> method:
|
|
@@ -17,15 +17,18 @@ module Usps::Support
|
|
|
17
17
|
# CHECKS = %i[database redis hq_database imis].freeze
|
|
18
18
|
# end
|
|
19
19
|
#
|
|
20
|
-
# and route to it (`get 'health' => 'health#show'`).
|
|
21
|
-
#
|
|
22
|
-
# responds. Apps run only the checks they list — admin-console has no HQ connection, so it omits
|
|
23
|
-
# :hq_database; an iMIS-backed app adds :imis. For a bespoke dependency, define a #check_<name> in
|
|
24
|
-
# the subclass (e.g. wrapping #check_http(url)) and add <name> to CHECKS.
|
|
20
|
+
# and route to it (`get 'health' => 'health#show'`). For a bespoke dependency, define a
|
|
21
|
+
# #check_<name> in the subclass (e.g. wrapping #check_http(url)) and add <name> to CHECKS.
|
|
25
22
|
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
23
|
+
# Checks run on a background thread, not the request thread: a per-process refresher updates a
|
|
24
|
+
# cached snapshot every health_refresh_interval seconds and #show serves that snapshot, so a slow
|
|
25
|
+
# or wedged dependency can't tie up the Passenger worker pool. Each check also runs under
|
|
26
|
+
# health_check_timeout, and a raise or timeout reads as a failed check (its error captured) so the
|
|
27
|
+
# action always responds. See doc/health_check_refresh_design.md.
|
|
28
|
+
#
|
|
29
|
+
# Per-component results (ok, latency, error) are handed to Usps::Support::HealthDiagnostics once
|
|
30
|
+
# per refresh which — when enabled — publishes them to CloudWatch so a past failure can be
|
|
31
|
+
# attributed to a specific component after it has recovered.
|
|
29
32
|
#
|
|
30
33
|
# Inherits ActionController::Base directly — NOT the host's ApplicationController — to bypass
|
|
31
34
|
# Devise auth, Pundit authorization, PaperTrail, and especially `allow_browser`, whose
|
|
@@ -34,47 +37,136 @@ module Usps::Support
|
|
|
34
37
|
class HealthController < ActionController::Base # rubocop:disable Rails/ApplicationController
|
|
35
38
|
SENTINEL_PREFIX = 'HEALTHCHECK_OK'
|
|
36
39
|
|
|
40
|
+
# Raised when a single check overruns the per-check timeout; recorded as a failed check.
|
|
41
|
+
class CheckTimeout < StandardError; end
|
|
42
|
+
|
|
37
43
|
# Symbols naming the dependency checks to run; each invokes the matching #check_<symbol>.
|
|
38
44
|
# Override per app by defining a CHECKS constant. Default probes the primary database only,
|
|
39
45
|
# which presumably every app needs.
|
|
40
46
|
CHECKS = %i[database].freeze
|
|
41
47
|
|
|
48
|
+
# Tunables (overridable per subclass, e.g. `self.health_check_timeout = 1` or in a test):
|
|
49
|
+
# health_check_timeout — per-check budget in seconds; a check that overruns is a failure
|
|
50
|
+
# health_refresh_interval — seconds between background refreshes of the cached snapshot
|
|
51
|
+
# health_snapshot_max_age — a cached snapshot older than this is refreshed inline before serving
|
|
52
|
+
# health_background_refresh — run the background refresher; off under test, where refreshes run
|
|
53
|
+
# synchronously and an unannounced thread probing real dependencies
|
|
54
|
+
# (DependencyGate starts one from any request) would be a surprise
|
|
55
|
+
# health_status_max_age — .dependency_status ignores a snapshot older than this (fails open)
|
|
56
|
+
class_attribute :health_check_timeout, default: 2
|
|
57
|
+
class_attribute :health_refresh_interval, default: 15
|
|
58
|
+
class_attribute :health_snapshot_max_age, default: 45
|
|
59
|
+
class_attribute :health_background_refresh, default: !::Rails.env.test?
|
|
60
|
+
class_attribute :health_status_max_age, default: 90
|
|
61
|
+
|
|
62
|
+
class << self
|
|
63
|
+
# One monitor per concrete subclass, per process. Built lazily so the background thread starts
|
|
64
|
+
# after Passenger forks its workers (threads do not survive fork).
|
|
65
|
+
def health_monitor
|
|
66
|
+
klass = self
|
|
67
|
+
@health_monitor ||= HealthMonitor.new(
|
|
68
|
+
interval: health_refresh_interval,
|
|
69
|
+
max_age: health_snapshot_max_age,
|
|
70
|
+
background: health_background_refresh,
|
|
71
|
+
runner: -> { klass.new.send(:run_and_record) }
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Read-only, non-blocking view of the last completed round, for callers on a *user* request
|
|
76
|
+
# thread — specifically Usps::Support::DependencyGate, which uses it to fail a request fast
|
|
77
|
+
# rather than discovering the outage by blocking on the dead dependency itself:
|
|
78
|
+
#
|
|
79
|
+
# { hq_database: { ok: false, consecutive_failures: 4 }, database: { ok: true, ... } }
|
|
80
|
+
#
|
|
81
|
+
# Runs no check and takes no lock; it reads whatever the background refresher last stored.
|
|
82
|
+
# Returns {} — "no opinion", so callers fail *open* — when no round has completed yet (cold
|
|
83
|
+
# worker) or the snapshot has aged past health_status_max_age, which means the refresher itself
|
|
84
|
+
# is wedged and its last verdict is no longer evidence about the dependency.
|
|
85
|
+
def dependency_status
|
|
86
|
+
snapshot = health_monitor.cached
|
|
87
|
+
return {} unless snapshot && health_monitor.age_of(snapshot) <= health_status_max_age
|
|
88
|
+
|
|
89
|
+
snapshot.data&.[](:status) || {}
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Consecutive failed rounds per check, updated once per refresh so a gate can require a
|
|
93
|
+
# *sustained* failure and ignore a single slow round. Called only from the monitor's runner,
|
|
94
|
+
# and the monitor serialises refreshes under its own mutex, so this needs no lock of its own.
|
|
95
|
+
def record_streaks(results)
|
|
96
|
+
@streaks ||= Hash.new(0)
|
|
97
|
+
results.each { |name, ok| ok ? @streaks[name] = 0 : @streaks[name] += 1 }
|
|
98
|
+
results.to_h { |name, ok| [name, { ok:, consecutive_failures: @streaks[name] }] }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Drops the memoized monitor (stopping its thread) and the failure streaks. For tests.
|
|
102
|
+
def reset_health_monitor!
|
|
103
|
+
@health_monitor&.stop
|
|
104
|
+
@health_monitor = nil
|
|
105
|
+
@streaks = nil
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
42
109
|
def show
|
|
43
|
-
|
|
44
|
-
|
|
110
|
+
data = self.class.health_monitor.current
|
|
111
|
+
return render_unavailable unless data
|
|
45
112
|
|
|
46
|
-
|
|
47
|
-
|
|
113
|
+
render_health(data)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
# Renders the sentinel line, per-check status lines, and cache/retry headers from a snapshot.
|
|
119
|
+
def render_health(data)
|
|
120
|
+
healthy = data[:results].values.all?
|
|
121
|
+
|
|
122
|
+
body = "#{healthy ? "#{SENTINEL_PREFIX} #{data[:app]}" : "HEALTHCHECK_FAIL #{data[:app]}"}\n"
|
|
123
|
+
data[:results].each { |name, ok| body << "#{name}=#{ok ? 'ok' : 'fail'}\n" }
|
|
48
124
|
|
|
49
125
|
response.set_header('Cache-Control', 'no-store')
|
|
50
126
|
response.set_header('Retry-After', '30') unless healthy
|
|
51
|
-
HealthDiagnostics.record(app: app_name, components: @measurements)
|
|
52
127
|
render(plain: body, content_type: 'text/plain', status: healthy ? :ok : :service_unavailable)
|
|
53
128
|
end
|
|
54
129
|
|
|
55
|
-
|
|
130
|
+
# Fallback for the (rare) case where not even the first refresh could produce a snapshot.
|
|
131
|
+
def render_unavailable
|
|
132
|
+
response.set_header('Cache-Control', 'no-store')
|
|
133
|
+
response.set_header('Retry-After', '30')
|
|
134
|
+
render(plain: "HEALTHCHECK_FAIL #{app_name}\nsnapshot=unavailable\n",
|
|
135
|
+
content_type: 'text/plain', status: :service_unavailable)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# One round of checks + diagnostics, invoked by the monitor's runner (background thread, or the
|
|
139
|
+
# inline cold/stale path). Wrapped in the Rails executor so ActiveRecord connections checked out
|
|
140
|
+
# by the DB checks are returned even though this runs outside a request. Returns the render data.
|
|
141
|
+
def run_and_record
|
|
142
|
+
data = nil
|
|
143
|
+
::Rails.application.executor.wrap { data = perform_checks }
|
|
144
|
+
HealthDiagnostics.record(app: data[:app], components: data[:measurements])
|
|
145
|
+
data
|
|
146
|
+
end
|
|
56
147
|
|
|
57
|
-
# Runs each configured check, returning { label => boolean }
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
def
|
|
61
|
-
|
|
62
|
-
self.class::CHECKS.index_with { |name| measure(name) }
|
|
148
|
+
# Runs each configured check, returning { app:, results: { label => boolean }, measurements:,
|
|
149
|
+
# status: }. Only :results feeds the rendered body; :status carries the per-check failure streaks
|
|
150
|
+
# that .dependency_status hands to the request-path gate.
|
|
151
|
+
def perform_checks
|
|
152
|
+
measurements = []
|
|
153
|
+
results = self.class::CHECKS.index_with { |name| measure(name, measurements) }
|
|
154
|
+
{ app: app_name, results:, measurements:, status: self.class.record_streaks(results) }
|
|
63
155
|
end
|
|
64
156
|
|
|
65
|
-
# Runs a single named check, timing it and recording its outcome. A
|
|
66
|
-
# failed check
|
|
67
|
-
# rather than a 500.
|
|
68
|
-
def measure(name)
|
|
157
|
+
# Runs a single named check under the per-check timeout, timing it and recording its outcome. A
|
|
158
|
+
# raised error or a timeout reads as a failed check (its class/message captured), so a dependency
|
|
159
|
+
# that blows up or hangs is attributable rather than a 500 or a stalled worker.
|
|
160
|
+
def measure(name, measurements)
|
|
69
161
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
70
162
|
begin
|
|
71
|
-
ok = send(:"check_#{name}") ? true : false
|
|
163
|
+
ok = Timeout.timeout(health_check_timeout, CheckTimeout) { send(:"check_#{name}") } ? true : false
|
|
72
164
|
error = nil
|
|
73
165
|
rescue StandardError => e
|
|
74
166
|
ok = false
|
|
75
167
|
error = "#{e.class}: #{e.message}"
|
|
76
168
|
end
|
|
77
|
-
|
|
169
|
+
measurements << { label: name.to_s, ok:, latency_ms: elapsed_ms(started), error: }
|
|
78
170
|
ok
|
|
79
171
|
end
|
|
80
172
|
|
|
@@ -85,8 +177,9 @@ module Usps::Support
|
|
|
85
177
|
def app_name = Rails.application.class.module_parent_name.underscore.dasherize
|
|
86
178
|
|
|
87
179
|
# --- Check primitives. Each returns true on success and returns false or raises on failure;
|
|
88
|
-
# #measure catches the raise, so a dead dependency reads as a failed check (with
|
|
89
|
-
# captured) rather than a 500. Named check_<name>, not <name>?, to drive CHECKS
|
|
180
|
+
# #measure catches the raise/timeout, so a dead or slow dependency reads as a failed check (with
|
|
181
|
+
# its error captured) rather than a 500. Named check_<name>, not <name>?, to drive CHECKS
|
|
182
|
+
# dispatch. ---
|
|
90
183
|
|
|
91
184
|
# rubocop:disable Naming/PredicateMethod
|
|
92
185
|
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Usps
|
|
4
|
+
module Support
|
|
5
|
+
# Keeps a process-local snapshot of the latest health-check results so /health can answer from
|
|
6
|
+
# memory instead of running the checks on the request thread. See
|
|
7
|
+
# doc/health_check_refresh_design.md for the rationale.
|
|
8
|
+
#
|
|
9
|
+
# runner is a callable that runs one full round of checks and returns whatever the caller needs
|
|
10
|
+
# to render (HealthController returns { app:, results:, measurements: }). It is expected to bound
|
|
11
|
+
# its own checks; the monitor adds no timeout of its own.
|
|
12
|
+
#
|
|
13
|
+
# One monitor exists per concrete HealthController subclass, per process. It is created lazily on
|
|
14
|
+
# the first request, so the background thread starts after Passenger forks its workers (threads
|
|
15
|
+
# do not survive fork).
|
|
16
|
+
class HealthMonitor
|
|
17
|
+
Snapshot = Struct.new(:data, :at, keyword_init: true)
|
|
18
|
+
|
|
19
|
+
# interval: seconds between background refreshes
|
|
20
|
+
# max_age: a snapshot older than this is refreshed inline before use, so a wedged or
|
|
21
|
+
# never-started background thread can't serve dangerously old data
|
|
22
|
+
# runner: callable that runs one round of checks and returns the render data
|
|
23
|
+
# background: start the background refresher (false in tests, so refreshes are synchronous)
|
|
24
|
+
# clock: monotonic clock (seconds) to read; injectable for deterministic staleness tests
|
|
25
|
+
def initialize(interval:, max_age:, runner:, background: true, clock: nil)
|
|
26
|
+
@interval = interval
|
|
27
|
+
@max_age = max_age
|
|
28
|
+
@runner = runner
|
|
29
|
+
@background = background
|
|
30
|
+
@clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
31
|
+
@mutex = Mutex.new
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# The latest results. Serves a fresh cached snapshot without touching any dependency; on a cold
|
|
35
|
+
# (first call) or stale snapshot, refreshes inline so the answer is never dangerously old. Also
|
|
36
|
+
# starts the background refresher on first use. Returns the runner's data, or nil only if the
|
|
37
|
+
# very first refresh failed and there is no prior snapshot to fall back on.
|
|
38
|
+
def current
|
|
39
|
+
ensure_running if @background
|
|
40
|
+
|
|
41
|
+
snapshot = @snapshot
|
|
42
|
+
return snapshot.data if snapshot && fresh?(snapshot)
|
|
43
|
+
|
|
44
|
+
refresh
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Runs one round of checks and stores the snapshot; returns the fresh data. On a runner error,
|
|
48
|
+
# keeps the last good snapshot and returns its data (nil if there was none), so a transient
|
|
49
|
+
# failure to run the checks never blanks out /health.
|
|
50
|
+
def refresh
|
|
51
|
+
@mutex.synchronize do
|
|
52
|
+
data = @runner.call
|
|
53
|
+
@snapshot = Snapshot.new(data:, at: @clock.call)
|
|
54
|
+
data
|
|
55
|
+
end
|
|
56
|
+
rescue StandardError
|
|
57
|
+
@snapshot&.data
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def fresh?(snapshot) = (@clock.call - snapshot.at) <= @max_age
|
|
61
|
+
|
|
62
|
+
# The last completed snapshot, or nil if none has been taken yet -- *without* refreshing, even
|
|
63
|
+
# when stale. #current is for /health, which must answer accurately and may pay for an inline
|
|
64
|
+
# round; this is for callers on a user request thread (Usps::Support::DependencyGate), where
|
|
65
|
+
# running a check would reintroduce exactly the stall they exist to avoid. Starts the
|
|
66
|
+
# background refresher on first use so a process that never serves /health still gets one.
|
|
67
|
+
# Callers judge staleness themselves via #age_of.
|
|
68
|
+
def cached
|
|
69
|
+
ensure_running if @background
|
|
70
|
+
|
|
71
|
+
@snapshot
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Seconds since the snapshot was taken, on the monitor's own (monotonic) clock -- callers can't
|
|
75
|
+
# compare `snapshot.at` against a wall clock of their own.
|
|
76
|
+
def age_of(snapshot) = @clock.call - snapshot.at
|
|
77
|
+
|
|
78
|
+
# Stops the background refresher. Used by tests; also safe to call on shutdown.
|
|
79
|
+
def stop
|
|
80
|
+
@mutex.synchronize do
|
|
81
|
+
@thread&.kill
|
|
82
|
+
@thread = nil
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
# Starts the background refresher thread; exercised in prod (and the background spec) rather
|
|
89
|
+
# than counted as a unit, matching HealthDiagnostics' delivery worker.
|
|
90
|
+
# :nocov:
|
|
91
|
+
def ensure_running
|
|
92
|
+
return if @thread&.alive?
|
|
93
|
+
|
|
94
|
+
@mutex.synchronize do
|
|
95
|
+
return if @thread&.alive?
|
|
96
|
+
|
|
97
|
+
@thread = Thread.new do
|
|
98
|
+
loop do
|
|
99
|
+
refresh
|
|
100
|
+
sleep(@interval)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
# :nocov:
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
data/lib/usps/support/version.rb
CHANGED
data/lib/usps/support.rb
CHANGED
|
@@ -18,6 +18,7 @@ require_relative 'support/helpers'
|
|
|
18
18
|
require_relative 'support/models'
|
|
19
19
|
require_relative 'support/lib'
|
|
20
20
|
require_relative 'support/health_diagnostics'
|
|
21
|
+
require_relative 'support/health_monitor'
|
|
21
22
|
|
|
22
23
|
# The Engine require lives in `usps/all`, not here. This file can
|
|
23
24
|
# legitimately be required before Rails is loaded (e.g. from .simplecov to pull
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: usps-support
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.51
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Julian Fiander
|
|
@@ -46,7 +46,7 @@ dependencies:
|
|
|
46
46
|
version: '0'
|
|
47
47
|
- - ">="
|
|
48
48
|
- !ruby/object:Gem::Version
|
|
49
|
-
version: 0.
|
|
49
|
+
version: 0.15.0
|
|
50
50
|
type: :runtime
|
|
51
51
|
prerelease: false
|
|
52
52
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -56,7 +56,7 @@ dependencies:
|
|
|
56
56
|
version: '0'
|
|
57
57
|
- - ">="
|
|
58
58
|
- !ruby/object:Gem::Version
|
|
59
|
-
version: 0.
|
|
59
|
+
version: 0.15.0
|
|
60
60
|
- !ruby/object:Gem::Dependency
|
|
61
61
|
name: usps-jwt_auth
|
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -86,6 +86,7 @@ extra_rdoc_files: []
|
|
|
86
86
|
files:
|
|
87
87
|
- Readme.md
|
|
88
88
|
- app/controllers/concerns/usps/support/admin_menu.rb
|
|
89
|
+
- app/controllers/concerns/usps/support/dependency_gate.rb
|
|
89
90
|
- app/controllers/usps/support/admins_controller.rb
|
|
90
91
|
- app/controllers/usps/support/health_controller.rb
|
|
91
92
|
- app/models/usps/support/policy/admin_context.rb
|
|
@@ -98,6 +99,7 @@ files:
|
|
|
98
99
|
- lib/usps/support/db/websites_schema.rb
|
|
99
100
|
- lib/usps/support/engine.rb
|
|
100
101
|
- lib/usps/support/health_diagnostics.rb
|
|
102
|
+
- lib/usps/support/health_monitor.rb
|
|
101
103
|
- lib/usps/support/helpers.rb
|
|
102
104
|
- lib/usps/support/helpers/badges_helper.rb
|
|
103
105
|
- lib/usps/support/helpers/flags_helper.rb
|