elasticgraph-datastore_core 0.18.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7bd50dac79cefe5466066b048ebb298df524a33a904082b0871af9429a7521ed
+  data.tar.gz: 666da201ff6ef33c81c2d7b4926068913188f735ec1f9777ed68255c3433bc2a
+SHA512:
+  metadata.gz: 9ec51fffb49a31152bfa0718bfb16d0063c964ca24128710639d287dac4d10ccc4ea0fbe969caae6f7c39df28ddb80b3a2bc225c2c7af1c4bb7d6f5baf5c1459
+  data.tar.gz: 671573cb43b2880450bac584ba4edd21154188a2de245bd195569cc166a0e323038caba947cc7d74bbf0b403b78952b37a800bea81fe602f125188dffa529275

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# ElasticGraph::DatastoreCore
+
+Contains the core datastore logic used by the rest of ElasticGraph.

data/elasticgraph-datastore_core.gemspec

@@ -0,0 +1,21 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../gemspec_helper"
+
+ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version|
+  spec.summary = "ElasticGraph gem containing the core datastore support types and logic."
+
+  spec.add_dependency "elasticgraph-schema_artifacts", eg_version
+  spec.add_dependency "elasticgraph-support", eg_version
+
+  spec.add_development_dependency "elasticgraph-admin", eg_version
+  spec.add_development_dependency "elasticgraph-elasticsearch", eg_version
+  spec.add_development_dependency "elasticgraph-opensearch", eg_version
+  spec.add_development_dependency "elasticgraph-schema_definition", eg_version
+end

data/lib/elastic_graph/datastore_core/config.rb

@@ -0,0 +1,58 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/datastore_core/configuration/client_faraday_adapter"
+require "elastic_graph/datastore_core/configuration/cluster_definition"
+require "elastic_graph/datastore_core/configuration/index_definition"
+require "elastic_graph/error"
+
+module ElasticGraph
+  class DatastoreCore
+    # Defines the configuration related to datastores.
+    class Config < ::Data.define(
+      # Configuration of the faraday adapter to use with the datastore client.
+      :client_faraday_adapter,
+      # Map of datastore cluster definitions, keyed by cluster name. The names will be referenced within
+      # `index_definitions` by `query_cluster` and `index_into_clusters` to identify
+      # datastore clusters. Each definition has a `url` and `settings`. `settings` contains datastore
+      # settings in the flattened name form, e.g. `"cluster.max_shards_per_node": 2000`.
+      :clusters,
+      # Map of index definition names to `IndexDefinition` objects containing customizations
+      # for the named index definitions for this environment.
+      :index_definitions,
+      # Determines if we log requests/responses to/from the datastore.
+      # Defaults to `false`.
+      :log_traffic,
+      # Passed down to the datastore client, controls the number of times ElasticGraph attempts a call against
+      # the datastore before failing. Retrying a handful of times is generally advantageous, since some sporadic
+      # failures are expected during the course of operation, and better to retry than fail the entire call.
+      # Defaults to 3.
+      :max_client_retries
+    )
+      # Helper method to build an instance from parsed YAML config.
+      def self.from_parsed_yaml(parsed_yaml)
+        parsed_yaml = parsed_yaml.fetch("datastore")
+        extra_keys = parsed_yaml.keys - EXPECTED_KEYS
+
+        unless extra_keys.empty?
+          raise ConfigError, "Unknown `datastore` config settings: #{extra_keys.join(", ")}"
+        end
+
+        new(
+          client_faraday_adapter: Configuration::ClientFaradayAdapter.from_parsed_yaml(parsed_yaml),
+          clusters: Configuration::ClusterDefinition.definitions_by_name_hash_from(parsed_yaml.fetch("clusters")),
+          index_definitions: Configuration::IndexDefinition.definitions_by_name_hash_from(parsed_yaml.fetch("index_definitions")),
+          log_traffic: parsed_yaml.fetch("log_traffic", false),
+          max_client_retries: parsed_yaml.fetch("max_client_retries", 3)
+        )
+      end
+
+      EXPECTED_KEYS = members.map(&:to_s)
+    end
+  end
+end

data/lib/elastic_graph/datastore_core/configuration/client_faraday_adapter.rb

@@ -0,0 +1,38 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  class DatastoreCore
+    module Configuration
+      class ClientFaradayAdapter < ::Data.define(
+        # The faraday adapter to use with the datastore client, such as `httpx` or `typhoeus`.
+        # For more info, see:
+        # https://github.com/elastic/elasticsearch-ruby/commit/a7bbdbf2a96168c1b33dca46ee160d2d4d75ada0
+        :name,
+        # A Ruby library to require which provides the named adapter (optional).
+        :require
+      )
+        def self.from_parsed_yaml(parsed_yaml)
+          parsed_yaml = parsed_yaml.fetch("client_faraday_adapter") || {}
+          extra_keys = parsed_yaml.keys - EXPECTED_KEYS
+
+          unless extra_keys.empty?
+            raise ConfigError, "Unknown `datastore.client_faraday_adapter` config settings: #{extra_keys.join(", ")}"
+          end
+
+          new(
+            name: parsed_yaml["name"]&.to_sym,
+            require: parsed_yaml["require"]
+          )
+        end
+
+        EXPECTED_KEYS = members.map(&:to_s)
+      end
+    end
+  end
+end

data/lib/elastic_graph/datastore_core/configuration/cluster_definition.rb

@@ -0,0 +1,52 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+
+module ElasticGraph
+  class DatastoreCore
+    module Configuration
+      class ClusterDefinition < ::Data.define(:url, :backend_client_class, :settings)
+        def self.from_hash(hash)
+          extra_keys = hash.keys - EXPECTED_KEYS
+
+          unless extra_keys.empty?
+            raise ConfigError, "Unknown `datastore.clusters` config settings: #{extra_keys.join(", ")}"
+          end
+
+          backend_name = hash["backend"]
+          backend_client_class =
+            case backend_name
+            when "elasticsearch"
+              require "elastic_graph/elasticsearch/client"
+              Elasticsearch::Client
+            when "opensearch"
+              require "elastic_graph/opensearch/client"
+              OpenSearch::Client
+            else
+              raise ConfigError, "Unknown `datastore.clusters` backend: `#{backend_name}`. Valid backends are `elasticsearch` and `opensearch`."
+            end
+
+          new(
+            url: hash.fetch("url"),
+            backend_client_class: backend_client_class,
+            settings: hash.fetch("settings")
+          )
+        end
+
+        def self.definitions_by_name_hash_from(cluster_def_hash_by_name)
+          cluster_def_hash_by_name.transform_values do |cluster_def_hash|
+            from_hash(cluster_def_hash)
+          end
+        end
+
+        EXPECTED_KEYS = members.map(&:to_s) - ["backend_client_class"] + ["backend"]
+      end
+    end
+  end
+end

data/lib/elastic_graph/datastore_core/configuration/index_definition.rb

@@ -0,0 +1,110 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/support/time_set"
+require "elastic_graph/error"
+require "time"
+
+module ElasticGraph
+  class DatastoreCore
+    module Configuration
+      # Defines environment-specific customizations for an index definition.
+      #
+      # - ignore_routing_values: routing values for which we will ignore routing as configured on the index.
+      #   This is intended to be used when a single routing value contains such a large portion of the dataset that it creates lopsided shards.
+      #   By including that routing value in this config setting, it'll spread that value's data across all shards instead of concentrating it on a single shard.
+      # - query_cluster: named search cluster to be used for queries on this index.
+      # - index_into_cluster: named search clusters to index data into.
+      # - setting_overrides: overrides for index (or index template) settings.
+      # - setting_overrides_by_timestamp: overrides for index template settings for specific dates,
+      #   allowing us to have different settings than the template for some timestamp.
+      # - custom_timestamp_ranges: defines indices for a custom timestamp range (rather than relying
+      #   on the configured rollover frequency).
+      # - use_updates_for_indexing: when `true`, opts the index into using the `update` API instead of the `index` API for indexing.
+      #   (Defaults to `true`).
+      class IndexDefinition < ::Data.define(
+        :ignore_routing_values,
+        :query_cluster,
+        :index_into_clusters,
+        :setting_overrides,
+        :setting_overrides_by_timestamp,
+        :custom_timestamp_ranges,
+        :use_updates_for_indexing
+      )
+        def initialize(ignore_routing_values:, **rest)
+          __skip__ = super(ignore_routing_values: ignore_routing_values.to_set, **rest)
+
+          # Verify the custom ranges are disjoint.
+          # Yeah, this is O(N^2), which isn't great, but we expect a _very_ small number of custom
+          # ranges (0-2) so this should be ok.
+          return if custom_timestamp_ranges
+            .map(&:time_set)
+            .combination(2)
+            .none? do |s1_s2|
+              s1, s2 = s1_s2
+              s1.intersect?(s2)
+            end
+
+          raise ConfigError, "Your configured `custom_timestamp_ranges` are not disjoint, as required."
+        end
+
+        def without_env_overrides
+          with(setting_overrides: {}, setting_overrides_by_timestamp: {}, custom_timestamp_ranges: [])
+        end
+
+        def custom_timestamp_range_for(timestamp)
+          custom_timestamp_ranges.find do |range|
+            range.time_set.member?(timestamp)
+          end
+        end
+
+        def self.definitions_by_name_hash_from(index_def_hash_by_name)
+          index_def_hash_by_name.transform_values do |index_def_hash|
+            __skip__ = from(**index_def_hash.transform_keys(&:to_sym))
+          end
+        end
+
+        def self.from(custom_timestamp_ranges:, use_updates_for_indexing: true, **rest)
+          __skip__ = new(
+            custom_timestamp_ranges: CustomTimestampRange.ranges_from(custom_timestamp_ranges),
+            use_updates_for_indexing: use_updates_for_indexing,
+            **rest
+          )
+        end
+
+        # Represents an index definition that is based on a custom timestamp range.
+        class CustomTimestampRange < ::Data.define(:index_name_suffix, :setting_overrides, :time_set)
+          def initialize(index_name_suffix:, setting_overrides:, time_set:)
+            super
+
+            if time_set.empty?
+              raise ConfigError, "Custom timestamp range with suffix `#{index_name_suffix}` is invalid: no timestamps exist in it."
+            end
+          end
+
+          def self.ranges_from(range_hashes)
+            range_hashes.map do |range_hash|
+              __skip__ = from(**range_hash.transform_keys(&:to_sym))
+            end
+          end
+
+          private_class_method def self.from(index_name_suffix:, setting_overrides:, **predicates_hash)
+            if predicates_hash.empty?
+              raise ConfigSettingNotSetError, "Custom timestamp range with suffix `#{index_name_suffix}` lacks boundary definitions."
+            end
+
+            range_options = predicates_hash.transform_values { |iso8601_string| ::Time.iso8601(iso8601_string) }
+            time_set = Support::TimeSet.of_range(**range_options)
+
+            new(index_name_suffix: index_name_suffix, setting_overrides: setting_overrides, time_set: time_set)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/datastore_core/index_config_normalizer.rb

@@ -0,0 +1,79 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  class DatastoreCore
+    module IndexConfigNormalizer
+      # These are settings that the datastore exposes when you fetch an index, but that you can
+      # never set. We need to ignore them when figuring out what settings to update.
+      #
+      # Note: `index.routing.allocation.include._tier_preference` is not a read-only setting, but
+      # we want to treat it as one, because (1) Elasticsearch 7.10+ sets it and (2) we do not want
+      # to ever write it at this time.
+      #
+      # Note: `index.history.uuid` is a weird setting that sometimes shows up in managed AWS OpenSearch
+      # clusters, but only on _some_ indices. It's not documented and we don't want to mess with it here,
+      # so we want to treat it as a read only setting.
+      READ_ONLY_SETTINGS = %w[
+        index.creation_date
+        index.history.uuid
+        index.provided_name
+        index.replication.type
+        index.routing.allocation.include._tier_preference
+        index.uuid
+        index.version.created
+        index.version.upgraded
+      ]
+
+      # Normalizes the provided index configuration so that it is in a stable form that we can compare to what
+      # the datastore returns when we query it for the configuration of an index. This includes:
+      #
+      # - Dropping read-only settings that we never interact with but that the datastore automatically sets on an index.
+      #   Omitting them makes it easier for us to compare our desired configuration to what is in the datastore.
+      # - Converting setting values to a normalized string form. The datastore oddly returns setting values as strings
+      #   (e.g. `"false"` or `"7"` instead of `false` or `7`), so this matches that behavior.
+      # - Drops `type: object` from a mapping when there are `properties` because the datastore omits it in that
+      #   situation, treating it as the default type.
+      def self.normalize(index_config)
+        if (settings = index_config["settings"])
+          index_config = index_config.merge("settings" => normalize_settings(settings))
+        end
+
+        if (mappings = index_config["mappings"])
+          index_config = index_config.merge("mappings" => normalize_mappings(mappings))
+        end
+
+        index_config
+      end
+
+      def self.normalize_mappings(mappings)
+        return mappings unless (properties = mappings["properties"])
+
+        mappings = mappings.except("type") if mappings["type"] == "object"
+        mappings.merge("properties" => properties.transform_values { |prop| normalize_mappings(prop) })
+      end
+
+      def self.normalize_settings(settings)
+        settings
+          .except(*READ_ONLY_SETTINGS)
+          .to_h { |name, value| [name, normalize_setting_value(value)] }
+      end
+
+      private_class_method def self.normalize_setting_value(value)
+        case value
+        when nil
+          nil
+        when ::Array
+          value.map { |v| normalize_setting_value(v) }
+        else
+          value.to_s
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/datastore_core/index_definition/base.rb

@@ -0,0 +1,162 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/datastore_core/index_config_normalizer"
+require "elastic_graph/error"
+require "elastic_graph/support/hash_util"
+
+module ElasticGraph
+  class DatastoreCore
+    module IndexDefinition
+      # This module contains common implementation logic for both the rollover and non-rollover
+      # implementations of the common IndexDefinition type.
+      module Base
+        # Returns any setting overrides for this index from the environment-specific config file,
+        # after flattening it so that it can be directly used in a create index request.
+        def flattened_env_setting_overrides
+          @flattened_env_setting_overrides ||= Support::HashUtil.flatten_and_stringify_keys(
+            env_index_config.setting_overrides,
+            prefix: "index"
+          )
+        end
+
+        # Gets the routing value for the given `prepared_record`. Notably, `prepared_record` must be previously
+        # prepared with an `Indexer::RecordPreparer` in order to ensure that it uses internal index
+        # field names (to align with `route_with_path`/`route_with` which also use the internal name) rather
+        # than the public field name (which can differ).
+        def routing_value_for_prepared_record(prepared_record, route_with_path: route_with, id_path: "id")
+          return nil unless has_custom_routing?
+
+          unless route_with_path
+            raise ConfigError, "`#{self}` uses custom routing, but `route_with_path` is misconfigured (was `nil`)"
+          end
+
+          config_routing_value = Support::HashUtil.fetch_value_at_path(prepared_record, route_with_path).to_s
+          return config_routing_value unless ignored_values_for_routing.include?(config_routing_value)
+
+          Support::HashUtil.fetch_value_at_path(prepared_record, id_path).to_s
+        end
+
+        def has_custom_routing?
+          route_with != "id"
+        end
+
+        # Indicates if a search on this index definition may hit incomplete documents. An incomplete document
+        # can occur when multiple event types flow into the same index. An index that has only one source type
+        # can never have incomplete documents, but an index that has 2 or more sources can have incomplete
+        # documents when the "primary" event type hasn't yet been received for a document.
+        #
+        # This case is notable because we need to apply automatic filtering in order to hide documents that are
+        # not yet complete.
+        #
+        # Note: determining this value sometimes requires that we query the datastore for the record of all
+        # sources that an index has ever had. This value changes very, very rarely, and we don't want to slow
+        # down every GraphQL query by adding the extra query against the datastore, so we cache the value here.
+        def searches_could_hit_incomplete_docs?
+          return @searches_could_hit_incomplete_docs if defined?(@searches_could_hit_incomplete_docs)
+
+          if current_sources.size > 1
+            # We know that incomplete docs are possible, without needing to check sources recorded in `_meta`.
+            @searches_could_hit_incomplete_docs = true
+          else
+            # While our current configuration can't produce incomplete documents, some may already exist in the index
+            # if we previously had some `sourced_from` fields (but no longer have them). Here we check for the sources
+            # we've recorded in `_meta` to account for that.
+            client = datastore_clients_by_name.fetch(cluster_to_query)
+            recorded_sources = mappings_in_datastore(client).dig("_meta", "ElasticGraph", "sources") || []
+            sources = recorded_sources.union(current_sources.to_a)
+
+            @searches_could_hit_incomplete_docs = sources.size > 1
+          end
+        end
+
+        def cluster_to_query
+          env_index_config.query_cluster
+        end
+
+        def clusters_to_index_into
+          env_index_config.index_into_clusters.tap do |clusters_to_index_into|
+            raise ConfigError, "No `index_into_clusters` defined for #{self} in env_index_config" unless clusters_to_index_into
+          end
+        end
+
+        def use_updates_for_indexing?
+          env_index_config.use_updates_for_indexing
+        end
+
+        def ignored_values_for_routing
+          env_index_config.ignore_routing_values
+        end
+
+        # Returns a list of all defined datastore clusters this index resides within.
+        def all_accessible_cluster_names
+          @all_accessible_cluster_names ||=
+            # Using `_` because steep doesn't understand that `compact` removes nils.
+            (clusters_to_index_into + [_ = cluster_to_query]).compact.uniq.select do |name|
+              defined_clusters.include?(name)
+            end
+        end
+
+        def accessible_cluster_names_to_index_into
+          @accessible_cluster_names_to_index_into ||= clusters_to_index_into.select do |name|
+            defined_clusters.include?(name)
+          end
+        end
+
+        # Indicates whether not the index is be accessible from GraphQL queries, by virtue of
+        # the `cluster_to_query` being a defined cluster or not. This will be used to
+        # hide GraphQL schema elements that can't be queried when our config omits the means
+        # to query an index (e.g. due to lacking a configured URL).
+        def accessible_from_queries?
+          return false unless (cluster = cluster_to_query)
+          defined_clusters.include?(cluster)
+        end
+
+        # Returns a list of indices related to this template in the datastore cluster this
+        # index definition is configured to query. Note that for performance reasons, this method
+        # memoizes the result of querying the datastore for its current list of indices, and as
+        # a result the return value may be out of date. If it is absolutely essential that you get
+        # an up-to-date list of related indices, use `related_rollover_indices(datastore_client`) instead of
+        # this method.
+        #
+        # Note, however, that indices generally change *very* rarely (say, monthly or yearly) and as such
+        # this will very rarely be out of date, even with the memoization.
+        def known_related_query_rollover_indices
+          @known_related_query_rollover_indices ||= cluster_to_query&.then do |name|
+            # For query purposes, we only want indices that exist. If we return a query that is defined in our configuration
+            # but does not exist, and that gets used in a search index expression (even for the purposes of excluding it!),
+            # the datastore will return an error.
+            related_rollover_indices(datastore_clients_by_name.fetch(name), only_if_exists: true)
+          end || []
+        end
+
+        # Returns a set of all of the field paths to subfields of the special `LIST_COUNTS_FIELD`
+        # that contains the element counts of all list fields. The returned set is filtered based
+        # on the provided `source` to only contain the paths of fields that are populated by the
+        # given source.
+        def list_counts_field_paths_for_source(source)
+          @list_counts_field_paths_for_source ||= {} # : ::Hash[::String, ::Set[::String]]
+          @list_counts_field_paths_for_source[source] ||= identify_list_counts_field_paths_for_source(source)
+        end
+
+        def to_s
+          "#<#{self.class.name} #{name}>"
+        end
+        alias_method :inspect, :to_s
+
+        private
+
+        def identify_list_counts_field_paths_for_source(source)
+          fields_by_path.filter_map do |path, field|
+            path if field.source == source && path.split(".").include?(LIST_COUNTS_FIELD)
+          end.to_set
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/datastore_core/index_definition/index.rb

@@ -0,0 +1,64 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/datastore_core/index_config_normalizer"
+require "elastic_graph/datastore_core/index_definition/base"
+require "elastic_graph/support/memoizable_data"
+
+module ElasticGraph
+  class DatastoreCore
+    module IndexDefinition
+      class Index < Support::MemoizableData.define(
+        :name, :route_with, :default_sort_clauses, :current_sources, :fields_by_path,
+        :env_index_config, :defined_clusters, :datastore_clients_by_name
+      )
+        # `Data.define` provides all these methods:
+        # @dynamic name, route_with, default_sort_clauses, current_sources, fields_by_path, env_index_config, defined_clusters, datastore_clients_by_name, initialize
+
+        # `include IndexDefinition::Base` provides all these methods. Steep should be able to detect it
+        # but can't for some reason so we have to declare them with `@dynamic`.
+        # @dynamic flattened_env_setting_overrides, routing_value_for_prepared_record, has_custom_routing?, cluster_to_query, use_updates_for_indexing?
+        # @dynamic clusters_to_index_into, all_accessible_cluster_names, ignored_values_for_routing, searches_could_hit_incomplete_docs?
+        # @dynamic accessible_cluster_names_to_index_into, accessible_from_queries?, known_related_query_rollover_indices, list_counts_field_paths_for_source
+        include IndexDefinition::Base
+
+        def mappings_in_datastore(datastore_client)
+          IndexConfigNormalizer.normalize_mappings(datastore_client.get_index(name)["mappings"] || {})
+        end
+
+        # `ignore_unavailable: true` is needed to prevent errors when we delete non-existing non-rollover indices
+        def delete_from_datastore(datastore_client)
+          datastore_client.delete_indices(name)
+        end
+
+        # Indicates if this is a rollover index definition.
+        #
+        # Use of this is considered a mild code smell.  When feasible, it's generally better to
+        # implement a new polymorphic API on the IndexDefinition interface, rather
+        # then branching on the value of this predicate.
+        def rollover_index_template?
+          false
+        end
+
+        def index_expression_for_search
+          name
+        end
+
+        # Returns an index name to use for write operations.
+        def index_name_for_writes(record, timestamp_field_path: nil)
+          name
+        end
+
+        # A concrete index has no related indices (really only rollover indices do).
+        def related_rollover_indices(datastore_client, only_if_exists: false)
+          []
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/datastore_core/index_definition/rollover_index.rb

@@ -0,0 +1,48 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "delegate"
+require "elastic_graph/datastore_core/index_definition/index"
+
+module ElasticGraph
+  class DatastoreCore
+    module IndexDefinition
+      # Represents a concrete index for specific time range, derived from a RolloverIndexTemplate.
+      class RolloverIndex < DelegateClass(Index)
+        # @dynamic time_set
+        attr_reader :time_set
+
+        def initialize(index, time_set)
+          super(index)
+          @time_set = time_set
+        end
+
+        # We need to override `==` so that two `RolloverIndex` objects that wrap the same `Index` object are
+        # considered equal. Oddly enough, the `DelegateClass` implementation of `==` returns `true` if `other`
+        # is the wrapped object, but not if it's another instance of the same `DelegateClass` wrapping the same
+        # instance.
+        #
+        # https://github.com/ruby/ruby/blob/v3_0_3/lib/delegate.rb#L156-L159
+        #
+        # We need this because we want two `RolloverIndex` instances that wrap the same
+        # underlying `Index` instance to be considered equal (something a test relies upon,
+        # but also generally useful and expected).
+        def ==(other)
+          if RolloverIndex === other
+            __getobj__ == other.__getobj__ && time_set == other.time_set
+          else
+            # :nocov: -- this method isn't explicitly covered by tests (not worth writing a test just to cover this line).
+            super
+            # :nocov:
+          end
+        end
+        alias_method :eql?, :==
+      end
+    end
+  end
+end