elasticgraph-health_check 0.18.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3810ecb7062490ded525ea2bfee19dac4a408227baebae6b3ca0e6cf43fc6b92
+  data.tar.gz: 3d793cfdca8a02dcbb328d4e0a6f8f5608c8e3ad45ffe86e9269f8955a9b205c
+SHA512:
+  metadata.gz: 60ff8f6a90f1d260b8fac16a8677ea7c9db922cdee2f3c2cc614dd5be3ea7da1ca914f77aadc825a9b51c42fba69b7bd3d602eed027e5560288f0c68a3479d2f
+  data.tar.gz: b30042926be384bddee2172bfc5d7f6a4fa696878b2f58b82328a5988ea784b6a908ea70671f55eb5e47daadb24e86b57f254cee3bb92227e5829d5245951d01

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,82 @@
+# ElasticGraph::HealthCheck
+
+Provides a component that can act as a health check for high availability deployments. The HealthCheck component
+returns a summary status of either `healthy`, `degraded`, or `unhealthy` for the endpoint.
+
+The intended semantics of these statuses
+map to the corresponding Envoy statuses, see
+[the Envoy documentation for more details](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking),
+but in short `degraded` maps to "endpoint is impaired, do not use unless you have no other choice" and `unhealthy` maps to "endpoint is hard
+down/should not be used under any circumstances".
+
+The returned status is the worst of the status values from the individual sub-checks:
+1. The datastore clusters' own health statuses. The datastore clusters reflect their status as green/yellow/red. See
+   [the Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#cluster-health-api-response-body)
+   for details on the meaning of these statuses.
+   - `green` maps to `healthy`, `yellow` to `degraded`, and `red` to `unhealthy`.
+
+2. The recency of data present in ElasticGraph indices. The HealthCheck configuration specifies the expected "max recency" for items within an
+   index.
+   - If no records have been indexed within the specified period, the HealthCheck component will consider the index to be in a `degraded` status.
+
+As mentioned above, the returned status is the worst status of these two checks. E.g. if the datastore cluster(s) are all `green`, but a recency check fails, the
+overall status will be `degraded`. If the recency checks pass, but at least one datastore cluster is `red`, an `unhealthy` status will be returned.
+
+## Integration
+
+To use, simply register the `EnvoyExtension` when defining your schema:
+
+```ruby
+require(envoy_extension_path = "elastic_graph/health_check/envoy_extension")
+schema.register_graphql_extension ElasticGraph::HealthCheck::EnvoyExtension,
+  defined_at: envoy_extension_path,
+  http_path_segment: "/_status"
+```
+
+## Configuration
+
+These checks are configurable. The following configuration will be used as an example:
+
+```
+health_check:
+  clusters_to_consider: ["widgets-cluster"]
+  data_recency_checks:
+    Widget:
+      timestamp_field: createdAt
+      expected_max_recency_seconds: 30
+```
+
+- `clusters_to_consider` configures the first check (datastore cluster health), and specifies which clusters' health status is monitored.
+- `data_recency_checks` configures the second check (data recency), and configures the recency check described above. In this example, if no new "Widgets"
+  are indexed for thirty seconds (perhaps because of an infrastructure issue), a `degraded` status will be returned.
+  - Note that this setting is most appropriate for types where you expect a steady stream of indexing (and where the absence of new records is indicative
+    of some kind of failure).
+
+## Behavior when datastore clusters are inaccessible
+
+A given ElasticGraph GraphQL endpoint does not necessarily have access to all datastore clusters - more specifically, the endpoint will only have access
+to clusters present in the `datastore.clusters` configuration map.
+
+If a health check is configured for either a cluster or type that the GraphQL endpoint does not have access to, the respective check will be skipped. This is appropriate,
+as since the GraphQL endpoint does not have access to the cluster/type, the cluster's/type's health is immaterial.
+
+For example, with the following configuration:
+
+```
+datastore:
+  clusters:
+    widgets-cluster: { ... }
+    # components-cluster: { ... } ### Not available, commented out.
+health_check:
+  clusters_to_consider: ["widgets-cluster", "components-cluster"]
+  data_recency_checks:
+    Component:
+      timestamp_field: createdAt
+      expected_max_recency_seconds: 10
+    Widget:
+      timestamp_field: createdAt
+      expected_max_recency_seconds: 30
+```
+
+... the `components-cluster` datastore status health check will be skipped, as will the Component recency check. However the `widgets-cluster`/`Widget` health
+checks will proceed as normal.

data/elasticgraph-health_check.gemspec

@@ -0,0 +1,23 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../gemspec_helper"
+
+ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :extension) do |spec, eg_version|
+  spec.summary = "An ElasticGraph extension that provides a health check for high availability deployments."
+
+  spec.add_dependency "elasticgraph-datastore_core", eg_version
+  spec.add_dependency "elasticgraph-graphql", eg_version
+  spec.add_dependency "elasticgraph-support", eg_version
+
+  spec.add_development_dependency "elasticgraph-admin", eg_version
+  spec.add_development_dependency "elasticgraph-elasticsearch", eg_version
+  spec.add_development_dependency "elasticgraph-indexer", eg_version
+  spec.add_development_dependency "elasticgraph-opensearch", eg_version
+  spec.add_development_dependency "elasticgraph-schema_definition", eg_version
+end

data/lib/elastic_graph/health_check/config.rb

@@ -0,0 +1,48 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module HealthCheck
+    class Config < ::Data.define(
+      # The list of clusters to perform datastore status health checks on. A `green` status maps to `healthy`, a
+      # `yellow` status maps to `degraded`, and a `red` status maps to `unhealthy`. The returned status is the minimum
+      # status from all clusters in the list (a `yellow` cluster and a `green` cluster will result in a `degraded` status).
+      #
+      # Example: ["cluster-one", "cluster-two"]
+      :clusters_to_consider,
+      # A map of types to perform recency checks on. If no new records for that type have been indexed within the specified
+      # period, a `degraded` status will be returned.
+      #
+      # Example: { Widget: { timestamp_field: createdAt, expected_max_recency_seconds: 30 }}
+      :data_recency_checks
+    )
+      EMPTY = new([], {})
+
+      def self.from_parsed_yaml(config_hash)
+        config_hash = config_hash.fetch("health_check") { return EMPTY }
+
+        new(
+          clusters_to_consider: config_hash.fetch("clusters_to_consider"),
+          data_recency_checks: config_hash.fetch("data_recency_checks").transform_values do |value_hash|
+            DataRecencyCheck.from(value_hash)
+          end
+        )
+      end
+
+      DataRecencyCheck = ::Data.define(:expected_max_recency_seconds, :timestamp_field) do
+        # @implements DataRecencyCheck
+        def self.from(config_hash)
+          new(
+            expected_max_recency_seconds: config_hash.fetch("expected_max_recency_seconds"),
+            timestamp_field: config_hash.fetch("timestamp_field")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/health_check/constants.rb

@@ -0,0 +1,46 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+# Enumerates constants that are used from multiple places in ElasticGraph::HealthCheck.
+module ElasticGraph
+  module HealthCheck
+    # List of datastore cluster health fields from:
+    # https://www.elastic.co/guide/en/elasticsearch/reference/7.10/cluster-health.html#cluster-health-api-response-body
+    #
+    # This is expressed as a constant so that we can use it in dynamic ways in a few places
+    # (such as in a test; we want an acceptance test to fetch all these fields to make
+    # sure they work, and having them defined this way makes that easier).
+    #
+    # To get this list, this javascript was used in the chrome console:
+    #
+    # Array.from(document.querySelectorAll('div.variablelist')[2].querySelectorAll(':scope > dl.variablelist > dt')).map(x => x.innerText)
+    #
+    # (Feel free to use/change that as needed if/when you update this list in the future based on a newer datastore version.)
+    #
+    # Note: `discovered_master` is a new boolean field that AWS OpenSearch seems to add to the cluster health response.
+    # It was observed on the response on 2022-04-18.
+    DATASTORE_CLUSTER_HEALTH_FIELDS = %i[
+      cluster_name
+      status
+      timed_out
+      number_of_nodes
+      number_of_data_nodes
+      active_primary_shards
+      active_shards
+      relocating_shards
+      initializing_shards
+      unassigned_shards
+      delayed_unassigned_shards
+      number_of_pending_tasks
+      number_of_in_flight_fetch
+      task_max_waiting_in_queue_millis
+      active_shards_percent_as_number
+      discovered_master
+    ].to_set
+  end
+end

data/lib/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rb

@@ -0,0 +1,61 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "delegate"
+require "elastic_graph/graphql/http_endpoint"
+require "uri"
+
+module ElasticGraph
+  module HealthCheck
+    module EnvoyExtension
+      # Intercepts HTTP requests so that a health check can be performed if it's a GET request to the configured health check path.
+      # The HTTP response follows Envoy HTTP health check guidelines:
+      #
+      # https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking
+      class GraphQLHTTPEndpointDecorator < DelegateClass(GraphQL::HTTPEndpoint)
+        def initialize(http_endpoint, health_check_http_path_segment:, health_checker:, logger:)
+          super(http_endpoint)
+          @health_check_http_path_segment = health_check_http_path_segment.delete_prefix("/").delete_suffix("/")
+          @health_checker = health_checker
+          @logger = logger
+        end
+
+        __skip__ =
+          def process(request, **)
+            if request.http_method == :get && URI(request.url).path.split("/").include?(@health_check_http_path_segment)
+              perform_health_check
+            else
+              super
+            end
+          end
+
+        private
+
+        RESPONSES_BY_HEALTH_STATUS_CATEGORY = {
+          healthy: [200, "Healthy!", {}],
+          unhealthy: [500, "Unhealthy!", {}],
+          # https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/health_checking#degraded-health
+          degraded: [200, "Degraded.", {"x-envoy-degraded" => "true"}]
+        }
+
+        def perform_health_check
+          status = @health_checker.check_health
+          @logger.info status.to_loggable_description
+
+          status, message, headers = RESPONSES_BY_HEALTH_STATUS_CATEGORY.fetch(status.category)
+
+          GraphQL::HTTPResponse.new(
+            status_code: status,
+            headers: headers.merge("Content-Type" => "text/plain"),
+            body: message
+          )
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/health_check/envoy_extension.rb

@@ -0,0 +1,43 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+require "elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator"
+require "elastic_graph/health_check/health_checker"
+
+module ElasticGraph
+  module HealthCheck
+    # An extension module that hooks into the HTTP endpoint to provide Envoy health checks.
+    module EnvoyExtension
+      def graphql_http_endpoint
+        @graphql_http_endpoint ||=
+          begin
+            http_path_segment = config.extension_settings.dig("health_check", "http_path_segment")
+            http_path_segment ||= runtime_metadata
+              .graphql_extension_modules
+              .find { |ext_mod| ext_mod.extension_class == EnvoyExtension }
+              &.extension_config
+              &.dig(:http_path_segment)
+
+            if http_path_segment.nil?
+              raise ElasticGraph::ConfigSettingNotSetError, "Health check `http_path_segment` is not configured. " \
+                "Either set under `health_check` in YAML config or pass it along if you register the `EnvoyExtension` " \
+                "via `register_graphql_extension`."
+            end
+
+            GraphQLHTTPEndpointDecorator.new(
+              super,
+              health_check_http_path_segment: http_path_segment,
+              health_checker: HealthChecker.build_from(self),
+              logger: logger
+            )
+          end
+      end
+    end
+  end
+end

data/lib/elastic_graph/health_check/health_checker.rb

@@ -0,0 +1,221 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+require "elastic_graph/health_check/config"
+require "elastic_graph/health_check/health_status"
+require "elastic_graph/support/threading"
+require "time"
+
+module ElasticGraph
+  module HealthCheck
+    class HealthChecker
+      # Static factory method that builds a HealthChecker from an ElasticGraph::GraphQL instance.
+      def self.build_from(graphql)
+        new(
+          schema: graphql.schema,
+          config: HealthCheck::Config.from_parsed_yaml(graphql.config.extension_settings),
+          datastore_search_router: graphql.datastore_search_router,
+          datastore_query_builder: graphql.datastore_query_builder,
+          datastore_clients_by_name: graphql.datastore_core.clients_by_name,
+          clock: graphql.clock,
+          logger: graphql.logger
+        )
+      end
+
+      def initialize(
+        schema:,
+        config:,
+        datastore_search_router:,
+        datastore_query_builder:,
+        datastore_clients_by_name:,
+        clock:,
+        logger:
+      )
+        @schema = schema
+        @datastore_search_router = datastore_search_router
+        @datastore_query_builder = datastore_query_builder
+        @datastore_clients_by_name = datastore_clients_by_name
+        @clock = clock
+        @logger = logger
+        @indexed_document_types_by_name = @schema.indexed_document_types.to_h { |t| [t.name.to_s, t] }
+
+        @config = validate_and_normalize_config(config)
+      end
+
+      def check_health
+        recency_queries_by_type_name = @config.data_recency_checks.to_h do |type_name, recency_config|
+          [type_name, build_recency_query_for(type_name, recency_config)]
+        end
+
+        recency_results_by_query, *cluster_healths = execute_in_parallel(
+          lambda { @datastore_search_router.msearch(recency_queries_by_type_name.values) },
+          *@config.clusters_to_consider.map do |cluster|
+            lambda { [cluster, @datastore_clients_by_name.fetch(cluster).get_cluster_health] }
+          end
+        )
+
+        HealthStatus.new(
+          cluster_health_by_name: build_cluster_health_by_name(cluster_healths.to_h),
+          latest_record_by_type: build_latest_record_by_type(recency_results_by_query, recency_queries_by_type_name)
+        )
+      end
+
+      private
+
+      def build_recency_query_for(type_name, recency_config)
+        type = @indexed_document_types_by_name.fetch(type_name)
+
+        @datastore_query_builder.new_query(
+          search_index_definitions: type.search_index_definitions,
+          filter: build_index_optimization_filter_for(recency_config),
+          requested_fields: ["id", recency_config.timestamp_field],
+          document_pagination: {first: 1},
+          sort: [{recency_config.timestamp_field => {"order" => "desc"}}]
+        )
+      end
+
+      # To make the recency query more optimal, we filter on the timestamp field. This can provide
+      # a couple optimizations:
+      #
+      # - If its a rollover index and the timestamp is the field we use for rollover, this allows
+      #   the ElasticGraph query engine to hit only a subset of indices for better perf.
+      # - We've been told (by AWS support) that sorting a larger result set if more expensive than
+      #   a small result set (presumably larger than filtering cost) so even if we can't limit what
+      #   indices we hit with this, it should still be helpful.
+      #
+      # However, there's a bit of a risk of not actually finding the latest record if we include
+      # this filter. What we have here is a compromise: we "lookback" up to 100 times the
+      # `expected_max_recency_seconds`. For example, if that's set at 30, we'd search the last 3000
+      # seconds of data, which should be plenty of lookback for most cases, while still allowing
+      # a filter optimization. Once the latest record is more than 100 times older than our threshold
+      # the exact age of it is less interesting, anyway.
+      def build_index_optimization_filter_for(recency_config)
+        lookback_timestamp = @clock.now - (recency_config.expected_max_recency_seconds * 100)
+        {recency_config.timestamp_field => {"gte" => lookback_timestamp.iso8601}}
+      end
+
+      def execute_in_parallel(*lambdas)
+        Support::Threading.parallel_map(lambdas) { |l| l.call }
+      end
+
+      def build_cluster_health_by_name(cluster_healths)
+        cluster_healths.transform_values do |health|
+          health_status_fields = DATASTORE_CLUSTER_HEALTH_FIELDS.to_h do |field_name|
+            [field_name, health[field_name.to_s]]
+          end
+
+          HealthStatus::ClusterHealth.new(**health_status_fields)
+        end
+      end
+
+      def build_latest_record_by_type(recency_results_by_query, recency_queries_by_type_name)
+        recency_queries_by_type_name.to_h do |type_name, query|
+          config = @config.data_recency_checks.fetch(type_name)
+
+          latest_record = if (latest_doc = recency_results_by_query.fetch(query).first)
+            timestamp = ::Time.iso8601(latest_doc.fetch(config.timestamp_field))
+
+            HealthStatus::LatestRecord.new(
+              id: latest_doc.id,
+              timestamp: timestamp,
+              seconds_newer_than_required: timestamp - (@clock.now - config.expected_max_recency_seconds)
+            )
+          end
+
+          [type_name, latest_record]
+        end
+      end
+
+      def validate_and_normalize_config(config)
+        unrecognized_cluster_names = config.clusters_to_consider - all_known_clusters
+
+        # @type var errors: ::Array[::String]
+        errors = []
+
+        if unrecognized_cluster_names.any?
+          errors << "`health_check.clusters_to_consider` contains " \
+            "unrecognized cluster names: #{unrecognized_cluster_names.join(", ")}"
+        end
+
+        # Here, we determine which of the specified `clusters_to_consider` are actually available for datastore health checks (green/yellow/red).
+        # Before partitioning, we remove `unrecognized_cluster_names` as those will be reported through a separate error mechanism (above).
+        #
+        # Below, `available_clusters_to_consider` will replace `clusters_to_consider` in the returned `Config` instance.
+        available_clusters_to_consider, unavailable_clusters_to_consider =
+          (config.clusters_to_consider - unrecognized_cluster_names).partition { |it| @datastore_clients_by_name.key?(it) }
+
+        if unavailable_clusters_to_consider.any?
+          @logger.warn("#{unavailable_clusters_to_consider.length} cluster(s) were unavailable for health-checking: #{unavailable_clusters_to_consider.join(", ")}")
+        end
+
+        valid_type_names, invalid_type_names = config
+          .data_recency_checks.keys
+          .partition { |type| @indexed_document_types_by_name.key?(type) }
+
+        if invalid_type_names.any?
+          errors << "Some `health_check.data_recency_checks` types are not recognized indexed types: " \
+            "#{invalid_type_names.join(", ")}"
+        end
+
+        # It is possible to configure a GraphQL endpoint that has a healthcheck set up on type A, but doesn't actually
+        # have access to the datastore cluster that backs type A. In that case, we want to skip the health check - if the endpoint
+        # can't access type A, its health (or unhealth) is immaterial.
+        #
+        # So below, filter to types that have all of their datastore clusters available for querying.
+        available_type_names, unavailable_type_names = valid_type_names.partition do |type_name|
+          @indexed_document_types_by_name[type_name].search_index_definitions.all? do |search_index_definition|
+            @datastore_clients_by_name.key?(search_index_definition.cluster_to_query.to_s)
+          end
+        end
+
+        if unavailable_type_names.any?
+          @logger.warn("#{unavailable_type_names.length} type(s) were unavailable for health-checking: #{unavailable_type_names.join(", ")}")
+        end
+
+        # @type var invalid_timestamp_fields_by_type: ::Hash[::String, ::String]
+        invalid_timestamp_fields_by_type = {}
+        # @type var normalized_data_recency_checks: ::Hash[::String, Config::DataRecencyCheck]
+        normalized_data_recency_checks = {}
+
+        available_type_names.each do |type|
+          check = config.data_recency_checks.fetch(type)
+          field = @indexed_document_types_by_name
+            .fetch(type)
+            .fields_by_name[check.timestamp_field]
+
+          if field&.type&.unwrap_fully&.name.to_s == "DateTime"
+            # Convert the config so that we have a reference to the index field name.
+            normalized_data_recency_checks[type] = check.with(timestamp_field: field.name_in_index.to_s)
+          else
+            invalid_timestamp_fields_by_type[type] = check.timestamp_field
+          end
+        end
+
+        if invalid_timestamp_fields_by_type.any?
+          errors << "Some `health_check.data_recency_checks` entries have invalid timestamp fields: " \
+            "#{invalid_timestamp_fields_by_type.map { |k, v| "#{k} (#{v})" }.join(", ")}"
+        end
+
+        raise ConfigError, errors.join("\n\n") unless errors.empty?
+        config.with(
+          data_recency_checks: normalized_data_recency_checks,
+          clusters_to_consider: available_clusters_to_consider
+        )
+      end
+
+      def all_known_clusters
+        @all_known_clusters ||= @indexed_document_types_by_name.flat_map do |_, index_type|
+          index_type.search_index_definitions.flat_map do |it|
+            [it.cluster_to_query] + it.clusters_to_index_into
+          end
+        end + @datastore_clients_by_name.keys
+      end
+    end
+  end
+end

data/lib/elastic_graph/health_check/health_status.rb

@@ -0,0 +1,90 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/health_check/constants"
+
+module ElasticGraph
+  module HealthCheck
+    # Encapsulates all of the status information for an ElasticGraph GraphQL endpoint.
+    # Computes a `category` for the status of the ElasticGraph endpoint.
+    #
+    #   - unhealthy: the endpoint should not be used
+    #   - degraded: the endpoint can be used, but prefer a healthy endpoint over it
+    #   - healthy: the endpoint should be used
+    class HealthStatus < ::Data.define(:cluster_health_by_name, :latest_record_by_type, :category)
+      def initialize(cluster_health_by_name:, latest_record_by_type:)
+        super(
+          cluster_health_by_name: cluster_health_by_name,
+          latest_record_by_type: latest_record_by_type,
+          category: compute_category(cluster_health_by_name, latest_record_by_type)
+        )
+      end
+
+      def to_loggable_description
+        latest_record_descriptions = latest_record_by_type
+          .sort_by(&:first) # sort by type name
+          .map { |type, record| record&.to_loggable_description(type) || "Latest #{type} (missing)" }
+          .map { |description| "- #{description}" }
+
+        cluster_health_descriptions = cluster_health_by_name
+          .sort_by(&:first) # sort by cluster name
+          .map { |name, health| "\n- #{health.to_loggable_description(name)}" }
+
+        <<~EOS.strip.gsub("\n\n\n", "\n")
+          HealthStatus: #{category} (checked #{cluster_health_by_name.size} clusters, #{latest_record_by_type.size} latest records)
+          #{latest_record_descriptions.join("\n")}
+          #{cluster_health_descriptions.join("\n")}
+        EOS
+      end
+
+      private
+
+      def compute_category(cluster_health_by_name, latest_record_by_type)
+        cluster_statuses = cluster_health_by_name.values.map(&:status)
+        return :unhealthy if cluster_statuses.include?("red")
+
+        return :degraded if cluster_statuses.include?("yellow")
+        return :degraded if latest_record_by_type.values.any? { |v| v.nil? || v.too_old? }
+
+        :healthy
+      end
+
+      # Encapsulates the status information for a single datastore cluster.
+      ClusterHealth = ::Data.define(*DATASTORE_CLUSTER_HEALTH_FIELDS.to_a) do
+        # @implements ClusterHealth
+
+        def to_loggable_description(name)
+          field_values = to_h.map { |field, value| "  #{field}: #{value.inspect}" }
+          "#{name} cluster health (#{status}):\n#{field_values.join("\n")}"
+        end
+      end
+
+      # Encapsulates information about the latest record of a type.
+      LatestRecord = ::Data.define(
+        :id, # the id of the record
+        :timestamp, # the record's timestamp
+        :seconds_newer_than_required # the recency of the record relative to expectation; positive == more recent
+      ) do
+        # @implements LatestRecord
+        def to_loggable_description(type)
+          rounded_age = seconds_newer_than_required.round(2).abs
+
+          if too_old?
+            "Latest #{type} (too old): #{id} / #{timestamp.iso8601} (#{rounded_age}s too old)"
+          else
+            "Latest #{type} (recent enough): #{id} / #{timestamp.iso8601} (#{rounded_age}s newer than required)"
+          end
+        end
+
+        def too_old?
+          seconds_newer_than_required < 0
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,428 @@
+--- !ruby/object:Gem::Specification
+name: elasticgraph-health_check
+version: !ruby/object:Gem::Version
+  version: 0.18.0.0
+platform: ruby
+authors:
+- Myron Marston
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2024-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubocop-factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+- !ruby/object:Gem::Dependency
+  name: steep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: coderay
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: flatware-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: super_diff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: httpx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: method_source
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rspec-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.3.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.3.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-datastore_core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-graphql
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-admin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-indexer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-opensearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-schema_definition
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+description: 
+email:
+- myron@squareup.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- elasticgraph-health_check.gemspec
+- lib/elastic_graph/health_check/config.rb
+- lib/elastic_graph/health_check/constants.rb
+- lib/elastic_graph/health_check/envoy_extension.rb
+- lib/elastic_graph/health_check/envoy_extension/graphql_http_endpoint_decorator.rb
+- lib/elastic_graph/health_check/health_checker.rb
+- lib/elastic_graph/health_check/health_status.rb
+homepage: 
+licenses:
+- MIT
+metadata:
+  gem_category: extension
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key: 
+specification_version: 4
+summary: An ElasticGraph extension that provides a health check for high availability
+  deployments.
+test_files: []