elasticgraph-admin 0.18.0.0

data/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb

@@ -0,0 +1,247 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/admin/cluster_configurator/action_reporter"
+require "elastic_graph/admin/index_definition_configurator/for_index"
+require "elastic_graph/datastore_core/index_config_normalizer"
+require "elastic_graph/indexer/hash_differ"
+require "elastic_graph/support/hash_util"
+
+module ElasticGraph
+  class Admin
+    module IndexDefinitionConfigurator
+      # Responsible for managing an index template's configuration, including both mappings and settings.
+      class ForIndexTemplate
+        # @dynamic index_template
+
+        attr_reader :index_template
+
+        def initialize(datastore_client, index_template, env_agnostic_index_config_parent, output, clock)
+          @datastore_client = datastore_client
+          @index_template = index_template
+          @env_agnostic_index_config_parent = env_agnostic_index_config_parent
+          @env_agnostic_index_config = env_agnostic_index_config_parent.fetch("template")
+          @reporter = ClusterConfigurator::ActionReporter.new(output)
+          @output = output
+          @clock = clock
+        end
+
+        # Attempts to idempotently update the index configuration to the desired configuration
+        # exposed by the `IndexDefinition` object. Based on the configuration of the passed index
+        # and the state of the index in the datastore, does one of the following:
+        #
+        #   - If the index did not already exist: creates the index with the desired mappings and settings.
+        #   - If the desired mapping has fewer fields than what is in the index: raises an exception,
+        #     because the datastore provides no way to remove fields from a mapping and it would be confusing
+        #     for this method to silently ignore the issue.
+        #   - If the settings have desired changes: updates the settings, restoring any setting that
+        #     no longer has a desired value to its default.
+        #   - If the mapping has desired changes: updates the mappings.
+        #
+        # Note that any of the writes to the index may fail. There are many things that cannot
+        # be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt
+        # to validate those things ahead of time and instead rely on the datastore to fail if an invalid operation
+        # is attempted.
+        def configure!
+          related_index_configurators.each(&:configure!)
+
+          # there is no partial update for index template config and the same API both creates and updates it
+          put_index_template if has_mapping_updates? || settings_updates.any?
+        end
+
+        def validate
+          errors = related_index_configurators.flat_map(&:validate)
+
+          return errors unless index_template_exists?
+
+          errors << cannot_modify_mapping_field_type_error if mapping_type_changes.any?
+
+          errors
+        end
+
+        private
+
+        def put_index_template
+          desired_template_config_payload = Support::HashUtil.deep_merge(
+            desired_config_parent,
+            {"template" => {"mappings" => merge_properties(desired_mapping, current_mapping)}}
+          )
+
+          action_description = "Updated index template: `#{@index_template.name}`:\n#{config_diff}"
+
+          if mapping_removals.any?
+            action_description += "\n\nNote: the extra fields listed here will not actually get removed. " \
+              "Mapping removals are unsupported (but ElasticGraph will leave them alone and they'll cause no problems)."
+          end
+
+          @datastore_client.put_index_template(name: @index_template.name, body: desired_template_config_payload)
+          report_action action_description
+        end
+
+        def cannot_modify_mapping_field_type_error
+          "The datastore does not support modifying the type of a field from an existing index definition. " \
+          "You are attempting to update type of fields (#{mapping_type_changes.inspect}) from the #{@index_template.name} index definition."
+        end
+
+        def index_template_exists?
+          !current_config_parent.empty?
+        end
+
+        def mapping_removals
+          @mapping_removals ||= mapping_fields_from(current_mapping) - mapping_fields_from(desired_mapping)
+        end
+
+        def mapping_type_changes
+          @mapping_type_changes ||= begin
+            flattened_current = Support::HashUtil.flatten_and_stringify_keys(current_mapping)
+            flattened_desired = Support::HashUtil.flatten_and_stringify_keys(desired_mapping)
+
+            flattened_current.keys.select do |key|
+              key.end_with?(".type") && flattened_desired.key?(key) && flattened_desired[key] != flattened_current[key]
+            end
+          end
+        end
+
+        def has_mapping_updates?
+          current_mapping != desired_mapping
+        end
+
+        def settings_updates
+          @settings_updates ||= begin
+            # Updating a setting to null will cause the datastore to restore the default value of the setting.
+            restore_to_defaults = (current_settings.keys - desired_settings.keys).to_h { |key| [key, nil] }
+            desired_settings.select { |key, value| current_settings[key] != value }.merge(restore_to_defaults)
+          end
+        end
+
+        def mapping_fields_from(mapping_hash, prefix = "")
+          (mapping_hash["properties"] || []).flat_map do |key, params|
+            field = prefix + key
+            if params.key?("properties")
+              [field] + mapping_fields_from(params, "#{field}.")
+            else
+              [field]
+            end
+          end
+        end
+
+        def desired_mapping
+          desired_config_parent.fetch("template").fetch("mappings")
+        end
+
+        def desired_settings
+          @desired_settings ||= desired_config_parent.fetch("template").fetch("settings")
+        end
+
+        def desired_config_parent
+          @desired_config_parent ||= begin
+            # _meta is place where we can record state on the index mapping in the datastore.
+            # We want to maintain `_meta.ElasticGraph.sources` as an append-only set of all sources that have ever
+            # been configured to flow into an index, so that we can remember whether or not an index which currently
+            # has no `sourced_from` from fields ever did. This is necessary for our automatic filtering of multi-source
+            # indexes.
+            previously_recorded_sources = current_mapping.dig("_meta", "ElasticGraph", "sources") || []
+            sources = previously_recorded_sources.union(@index_template.current_sources.to_a).sort
+
+            env_agnostic_index_config_with_meta =
+              DatastoreCore::IndexConfigNormalizer.normalize(Support::HashUtil.deep_merge(@env_agnostic_index_config, {
+                "mappings" => {"_meta" => {"ElasticGraph" => {"sources" => sources}}},
+                "settings" => @index_template.flattened_env_setting_overrides
+              }))
+
+            @env_agnostic_index_config_parent.merge({"template" => env_agnostic_index_config_with_meta})
+          end
+        end
+
+        def current_mapping
+          current_config_parent.dig("template", "mappings") || {}
+        end
+
+        def current_settings
+          @current_settings ||= current_config_parent.dig("template", "settings")
+        end
+
+        def current_config_parent
+          @current_config_parent ||= begin
+            config = @datastore_client.get_index_template(@index_template.name)
+            if (template = config.dig("template"))
+              config.merge({"template" => DatastoreCore::IndexConfigNormalizer.normalize(template)})
+            else
+              config
+            end
+          end
+        end
+
+        def config_diff
+          @config_diff ||= Indexer::HashDiffer.diff(current_config_parent, desired_config_parent) || "(no diff)"
+        end
+
+        def report_action(message)
+          @reporter.report_action(message)
+        end
+
+        # Helper method used to merge properties between a _desired_ configuration and a _current_ configuration.
+        # This is used when we are figuring out how to update an index template. We do not want to delete existing
+        # fields from a template--while the datastore would allow it, our schema evolution strategy depends upon
+        # us not dropping old unused fields. The datastore doesn't allow it on indices, anyway (though it does allow
+        # it on index templates). We've ran into trouble (a near SEV) when allowing the logic here to delete an unused
+        # field from an index template. The indexer "mapping completeness" check started failing because an old version
+        # of the code (from back when the field in question was still used) noticed the expected field was missing and
+        # started failing on every event.
+        #
+        # This helps us avoid that problem by retaining any currently existing fields.
+        #
+        # Long term, if we want to support fully "garbage collecting" these old fields on templates, we will need
+        # to have them get dropped in a follow up step. We could have our `update_datastore_config` script notice that
+        # the deployed prod indexers are at a version that will tolerate the fields being dropped, or support it
+        # via an opt-in flag or something.
+        def merge_properties(desired_object, current_object)
+          desired_properties = desired_object.fetch("properties") { _ = {} }
+          current_properties = current_object.fetch("properties") { _ = {} }
+
+          merged_properties = desired_properties.merge(current_properties) do |key, desired, current|
+            if current.is_a?(::Hash) && current.key?("properties") && desired.key?("properties")
+              merge_properties(desired, current)
+            else
+              desired
+            end
+          end
+
+          desired_object.merge("properties" => merged_properties)
+        end
+
+        def related_index_configurators
+          # Here we fan out and get a configurator for each related index. These are generally concrete
+          # index that are based on a template, either via being specified in our config YAML, or via
+          # auto creation at indexing time.
+          #
+          # Note that it should not matter whether the related indices are configured before of after
+          # its rollover template; our use of index maintenance mode below prevents new indidces from
+          # being auto-created while this configuration process runs.
+          @related_index_configurators ||= begin
+            rollover_indices = @index_template.related_rollover_indices(@datastore_client)
+
+            # When we have a rollover index, it's important that we make at least one concrete index. Otherwise, if any
+            # queries come in before the first event is indexed, we won't have any concrete indices to search, and
+            # the datastore returns a response that differs from normal in that case. It particularly creates trouble
+            # for aggregation queries since the response format it expects is quite complex.
+            #
+            # Here we create a concrete index for the current timestamp if there are no concrete indices yet.
+            if rollover_indices.empty?
+              rollover_indices = [@index_template.related_rollover_index_for_timestamp(@clock.now.getutc.iso8601)].compact
+            end
+
+            rollover_indices.map do |index|
+              IndexDefinitionConfigurator::ForIndex.new(@datastore_client, index, @env_agnostic_index_config, @output)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/admin/index_definition_configurator.rb

@@ -0,0 +1,24 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/admin/index_definition_configurator/for_index"
+require "elastic_graph/admin/index_definition_configurator/for_index_template"
+
+module ElasticGraph
+  class Admin
+    module IndexDefinitionConfigurator
+      def self.new(datastore_client, index_def, env_agnostic_index_config, output, clock)
+        if index_def.rollover_index_template?
+          ForIndexTemplate.new(datastore_client, _ = index_def, env_agnostic_index_config, output, clock)
+        else
+          ForIndex.new(datastore_client, _ = index_def, env_agnostic_index_config, output)
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/admin/rake_tasks.rb

@@ -0,0 +1,129 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/admin"
+require "elastic_graph/support/from_yaml_file"
+require "rake/tasklib"
+
+module ElasticGraph
+  class Admin
+    class RakeTasks < ::Rake::TaskLib
+      extend Support::FromYamlFile::ForRakeTasks.new(ElasticGraph::Admin)
+
+      attr_reader :output, :prototype_index_names
+
+      def initialize(prototype_index_names: [], output: $stdout, &load_admin)
+        @output = output
+        @prototype_index_names = prototype_index_names.to_set
+        @load_admin = load_admin
+
+        define_tasks
+      end
+
+      private
+
+      def define_tasks
+        namespace :clusters do
+          namespace :configure do
+            desc "Performs the configuration of datastore clusters, including indices, settings, and scripts"
+            task :perform do
+              print_in_color "#{"=" * 80}\nNOTE: Performing datastore cluster updates for real!\n#{"=" * 80}", RED_COLOR_CODE
+
+              index_defs = update_clusters_for(admin)
+              output.puts "Finished updating datastore clusters. Validating index consistency..."
+              admin.datastore_indexing_router.validate_mapping_completeness_of!(:all_accessible_cluster_names, *index_defs)
+              output.puts "Done."
+            end
+
+            desc "Dry-runs the configuration of datastore clusters, including indices, settings, and scripts"
+            task :dry_run do
+              print_in_color "#{"=" * 80}\nNOTE: In dry-run mode. The updates reported below will actually be no-ops.\n#{"=" * 80}", GREEN_COLOR_CODE
+              update_clusters_for(admin.with_dry_run_datastore_clients)
+              print_in_color "#{"=" * 80}\nNOTE: This was dry-run mode. The updates reported above were actually no-ops.\n#{"=" * 80}", GREEN_COLOR_CODE
+            end
+          end
+        end
+
+        namespace :indices do
+          desc "Drops all prototype index definitions on all datastore clusters"
+          task :drop_prototypes do
+            require "elastic_graph/support/threading"
+
+            prototype_indices = admin
+              .datastore_core
+              .index_definitions_by_name.values
+              .select { |index| prototype_index_names.include?(index.name) }
+              .reject { |index| index.all_accessible_cluster_names.empty? }
+
+            output.puts "Disabling rollover index auto creation for all clusters"
+            admin.cluster_settings_manager.start_index_maintenance_mode!(:all_clusters)
+            output.puts "Disabled rollover index auto creation for all clusters"
+
+            output.puts "Dropping the following prototype index definitions: #{prototype_indices.map(&:name).join(",")}"
+            Support::Threading.parallel_map(prototype_indices) do |prototype_index_def|
+              delete_index_def_in_all_accessible_clusters(prototype_index_def)
+            end
+
+            output.puts "Finished dropping all prototype index definitions"
+          end
+
+          desc "Drops the specified index definition on the specified datastore cluster"
+          task :drop, :index_def_name, :cluster_name do |_, args|
+            index_def_name = args.fetch(:index_def_name)
+            cluster_name = args.fetch(:cluster_name)
+            datastore_client = admin.datastore_core.clients_by_name.fetch(cluster_name) do |key|
+              raise IndexOperationError, "Cluster named `#{key}` does not exist. Valid clusters: #{admin.datastore_core.clients_by_name.keys}."
+            end
+
+            index_def = admin.datastore_core.index_definitions_by_name.fetch(index_def_name)
+            unless prototype_index_names.include?(index_def.name)
+              raise IndexOperationError, "Unable to drop live index #{index_def_name}. Deleting a live index is extremely dangerous. " \
+                "Please ensure this is indeed intended, add the index name to the `prototype_index_names` list and retry."
+            end
+
+            output.puts "Disabling rollover index auto creation for this cluster"
+            admin.cluster_settings_manager.in_index_maintenance_mode(cluster_name) do
+              output.puts "Disabled rollover index auto creation for this cluster"
+              output.puts "Dropping index #{index_def}"
+              index_def.delete_from_datastore(datastore_client)
+              output.puts "Dropped index #{index_def}"
+            end
+            output.puts "Re-enabled rollover index auto creation for this cluster"
+          end
+        end
+      end
+
+      # See https://en.wikipedia.org/wiki/ANSI_escape_code#Colors for full list.
+      RED_COLOR_CODE = 31
+      GREEN_COLOR_CODE = 32
+
+      def update_clusters_for(admin)
+        configurator = admin.cluster_configurator
+
+        configurator.accessible_index_definitions.tap do |index_defs|
+          output.puts "The following index definitions will be configured:\n#{index_defs.map(&:name).join("\n")}"
+          configurator.configure_cluster(@output)
+        end
+      end
+
+      def print_in_color(message, color_code)
+        @output.puts "\033[#{color_code}m#{message}\033[0m"
+      end
+
+      def delete_index_def_in_all_accessible_clusters(index_def)
+        index_def.all_accessible_cluster_names.each do |cluster_name|
+          index_def.delete_from_datastore(admin.datastore_core.clients_by_name.fetch(cluster_name))
+        end
+      end
+
+      def admin
+        @admin ||= @load_admin.call
+      end
+    end
+  end
+end

data/lib/elastic_graph/admin.rb

@@ -0,0 +1,97 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/datastore_core"
+require "elastic_graph/support/from_yaml_file"
+require "time"
+
+module ElasticGraph
+  # The entry point into this library. Create an instance of this class to get access to
+  # the public interfaces provided by this library.
+  class Admin
+    extend Support::FromYamlFile
+
+    # @dynamic datastore_core, schema_artifacts
+    attr_reader :datastore_core, :schema_artifacts
+
+    # A factory method that builds an Admin instance from the given parsed YAML config.
+    # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`).
+    def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
+      new(datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :admin, &datastore_client_customization_block))
+    end
+
+    def initialize(datastore_core:, monotonic_clock: nil, clock: ::Time)
+      @datastore_core = datastore_core
+      @monotonic_clock = monotonic_clock
+      @clock = clock
+      @schema_artifacts = @datastore_core.schema_artifacts
+    end
+
+    def cluster_configurator
+      @cluster_configurator ||= begin
+        require "elastic_graph/admin/cluster_configurator"
+        ClusterConfigurator.new(
+          datastore_clients_by_name: @datastore_core.clients_by_name,
+          index_defs: @datastore_core.index_definitions_by_name.values,
+          index_configurations_by_name: schema_artifacts.indices,
+          index_template_configurations_by_name: schema_artifacts.index_templates,
+          scripts: schema_artifacts.datastore_scripts,
+          cluster_settings_manager: cluster_settings_manager,
+          clock: @clock
+        )
+      end
+    end
+
+    def cluster_settings_manager
+      @cluster_settings_manager ||= begin
+        require "elastic_graph/admin/cluster_configurator/cluster_settings_manager"
+        ClusterConfigurator::ClusterSettingsManager.new(
+          datastore_clients_by_name: @datastore_core.clients_by_name,
+          datastore_config: @datastore_core.config,
+          logger: @datastore_core.logger
+        )
+      end
+    end
+
+    def datastore_indexing_router
+      @datastore_indexing_router ||= begin
+        require "elastic_graph/indexer/datastore_indexing_router"
+        Indexer::DatastoreIndexingRouter.new(
+          datastore_clients_by_name: datastore_core.clients_by_name,
+          mappings_by_index_def_name: schema_artifacts.index_mappings_by_index_def_name,
+          monotonic_clock: monotonic_clock,
+          logger: datastore_core.logger
+        )
+      end
+    end
+
+    def monotonic_clock
+      @monotonic_clock ||= begin
+        require "elastic_graph/support/monotonic_clock"
+        Support::MonotonicClock.new
+      end
+    end
+
+    # Returns an alternate `Admin` instance with the datastore clients replaced with
+    # alternate implementations that turn all write operations into no-ops.
+    def with_dry_run_datastore_clients
+      require "elastic_graph/admin/datastore_client_dry_run_decorator"
+      dry_run_clients_by_name = @datastore_core.clients_by_name.transform_values do |client|
+        DatastoreClientDryRunDecorator.new(client)
+      end
+
+      Admin.new(datastore_core: DatastoreCore.new(
+        config: datastore_core.config,
+        logger: datastore_core.logger,
+        schema_artifacts: datastore_core.schema_artifacts,
+        clients_by_name: dry_run_clients_by_name,
+        client_customization_block: datastore_core.client_customization_block
+      ))
+    end
+  end
+end