elasticgraph-admin 0.18.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6202cf82573c7c499ad9615f590f14272dda1b605617f5680bcec1f97a648be6
+  data.tar.gz: eb0f47afd2cafb1a91b868097ac6e65f51ba4bef76ff307762c6f10bc2a1fbaa
+SHA512:
+  metadata.gz: 641c7cf2668a7ae77b4aa7e53d333193e09e052c834cff99792dd75a2213a9b6c441c021665ebf6cc9a5b425368ff8ecec8e1af48bc8cc7d8b7dfdfa8d8a13ff
+  data.tar.gz: ed4aede0dfa748e7a6f97bf71d51112e6b5e1d97c44a8fd2a08f928063c23b878c4bcae1d8ae4872094229e304328422250885ae43e92fdb054264ee6925ac69

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# ElasticGraph::Admin
+
+Provides datastore administrative tasks for ElasticGraph.

data/elasticgraph-admin.gemspec

@@ -0,0 +1,23 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../gemspec_helper"
+
+ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :core) do |spec, eg_version|
+  spec.summary = "ElasticGraph gem that provides datastore administrative tasks, to keep a datastore up-to-date with an ElasticGraph schema."
+
+  spec.add_dependency "elasticgraph-datastore_core", eg_version
+  spec.add_dependency "elasticgraph-indexer", eg_version
+  spec.add_dependency "elasticgraph-schema_artifacts", eg_version
+  spec.add_dependency "elasticgraph-support", eg_version
+  spec.add_dependency "rake", "~> 13.2"
+
+  spec.add_development_dependency "elasticgraph-elasticsearch", eg_version
+  spec.add_development_dependency "elasticgraph-opensearch", eg_version
+  spec.add_development_dependency "elasticgraph-schema_definition", eg_version
+end

data/lib/elastic_graph/admin/cluster_configurator/action_reporter.rb

@@ -0,0 +1,23 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  class Admin
+    class ClusterConfigurator
+      class ActionReporter
+        def initialize(output)
+          @output = output
+        end
+
+        def report_action(message)
+          @output.puts "#{message.chomp}\n#{"=" * 80}\n"
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/admin/cluster_configurator/cluster_settings_manager.rb

@@ -0,0 +1,99 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+
+module ElasticGraph
+  class Admin
+    class ClusterConfigurator
+      # Responsible for updating datastore cluster settings based on the mode EG is in, maintenance mode or indexing mode
+      class ClusterSettingsManager
+        def initialize(datastore_clients_by_name:, datastore_config:, logger:)
+          @datastore_clients_by_name = datastore_clients_by_name
+          @datastore_config = datastore_config
+          @logger = logger
+        end
+
+        # Starts index maintenance mode, if it has not already been started. This method is idempotent.
+        #
+        # In index maintenance mode, you can safely delete or update the index configuration without
+        # worrying about indices being auto-created with dynamic mappings (e.g. due to an indexing
+        # race condition). While in this mode, indexing operations on documents that fall into new rollover
+        # indices may fail since the auto-creation of those indices is disabled.
+        #
+        # `cluster_spec` can be the name of a specific cluster (as a string) or `:all_clusters`.
+        def start_index_maintenance_mode!(cluster_spec)
+          cluster_names_for(cluster_spec).each do |cluster_name|
+            datastore_client_named(cluster_name).put_persistent_cluster_settings(desired_cluster_settings(cluster_name))
+          end
+        end
+
+        # Ends index maintenance mode, if it has not already ended. This method is idempotent.
+        #
+        # Outside of this mode, you cannot safely delete or update the index configuration. However,
+        # new rollover indices will correctly be auto-created as documents that fall in new months or
+        # years are indexed.
+        #
+        # `cluster_spec` can be the name of a specific cluster (as a string) or `:all_clusters`.
+        def end_index_maintenance_mode!(cluster_spec)
+          cluster_names_for(cluster_spec).each do |cluster_name|
+            datastore_client_named(cluster_name).put_persistent_cluster_settings(
+              desired_cluster_settings(cluster_name, auto_create_index_patterns: ["*#{ROLLOVER_INDEX_INFIX_MARKER}*"])
+            )
+          end
+        end
+
+        # Runs a block in index maintenance mode. Should be used to wrap any code that updates your index configuration.
+        #
+        # `cluster_spec` can be the name of a specific cluster (as a string) or `:all_clusters`.
+        def in_index_maintenance_mode(cluster_spec)
+          start_index_maintenance_mode!(cluster_spec)
+
+          begin
+            yield
+          rescue => e
+            @logger.warn "WARNING: ClusterSettingsManager#in_index_maintenance_mode is not able to exit index maintenance mode due to exception #{e}.\n A bit of manual cleanup may be required (although a re-try should be idempotent)."
+            raise # re-raise the same error
+          else
+            # Note: we intentionally do not end maintenance mode in an `ensure` block, because if an exception
+            # happens while we `yield`, we do _not_ want to exit maintenance mode. Exiting maintenance mode
+            # could put us in a state where indices are dynamically created when we do not want them to be.
+            end_index_maintenance_mode!(cluster_spec)
+          end
+        end
+
+        private
+
+        def desired_cluster_settings(cluster_name, auto_create_index_patterns: [])
+          {
+            # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
+            #
+            # We generally want to disable automatic index creation in order to require all indices to be properly
+            # defined and configured. However, we must allow kibana to create some indices for it to be usable
+            # (https://discuss.elastic.co/t/elasticsearchs-action-auto-create-index-setting-impact-on-kibana/117701).
+            "action.auto_create_index" => ([".kibana*"] + auto_create_index_patterns).map { |p| "+#{p}" }.join(",")
+          }.merge(@datastore_config.clusters.fetch(cluster_name).settings)
+        end
+
+        def datastore_client_named(cluster_name)
+          @datastore_clients_by_name.fetch(cluster_name) do
+            raise ClusterOperationError,
+              "Unknown datastore cluster name: `#{cluster_name}`. Valid cluster names: #{@datastore_clients_by_name.keys}"
+          end
+        end
+
+        def cluster_names_for(cluster_spec)
+          case cluster_spec
+          when :all_clusters then @datastore_clients_by_name.keys
+          else [cluster_spec]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/admin/cluster_configurator/script_configurator.rb

@@ -0,0 +1,54 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/admin/cluster_configurator/action_reporter"
+require "elastic_graph/error"
+
+module ElasticGraph
+  class Admin
+    class ClusterConfigurator
+      class ScriptConfigurator
+        def initialize(datastore_client:, script_context:, script_id:, script:, output:)
+          @datastore_client = datastore_client
+          @script_context = script_context
+          @script_id = script_id
+          @script = script
+          @action_reporter = ActionReporter.new(output)
+        end
+
+        def validate
+          case existing_datastore_script
+          when :not_found, @script
+            []
+          else
+            [
+              "#{@script_context} script #{@script_id} already exists in the datastore but has different contents. " \
+                "\n\nScript in the datastore:\n#{::YAML.dump(existing_datastore_script)}" \
+                "\n\nDesired script:\n#{::YAML.dump(@script)}"
+            ]
+          end
+        end
+
+        def configure!
+          if existing_datastore_script == :not_found
+            @datastore_client.put_script(id: @script_id, body: {script: @script}, context: @script_context)
+            @action_reporter.report_action "Stored #{@script_context} script: #{@script_id}"
+          end
+        end
+
+        private
+
+        def existing_datastore_script
+          @existing_datastore_script ||= @datastore_client
+            .get_script(id: @script_id)
+            &.fetch("script") || :not_found
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/admin/cluster_configurator.rb

@@ -0,0 +1,104 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/admin/cluster_configurator/script_configurator"
+require "elastic_graph/admin/index_definition_configurator"
+require "elastic_graph/error"
+require "stringio"
+
+module ElasticGraph
+  class Admin
+    # Facade responsible for overall cluster configuration. Delegates to other classes as
+    # necessary to configure different aspects of the cluster (such as index configuration,
+    # cluster settings, etc).
+    class ClusterConfigurator
+      def initialize(
+        datastore_clients_by_name:,
+        index_defs:,
+        index_configurations_by_name:,
+        index_template_configurations_by_name:,
+        scripts:,
+        cluster_settings_manager:,
+        clock:
+      )
+        @datastore_clients_by_name = datastore_clients_by_name
+        @index_defs = index_defs
+        @index_configurations_by_name = index_configurations_by_name.merge(index_template_configurations_by_name)
+        @scripts_by_id = scripts
+        @cluster_settings_manager = cluster_settings_manager
+        @clock = clock
+      end
+
+      # Attempts to configure all aspects of the datastore cluster. Known/expected failure
+      # cases are pre-validated so that an error can be raised before applying any changes to
+      # any indices, so that we hopefully don't wind up in a "partially configured" state.
+      def configure_cluster(output)
+        # Note: we do not want to cache `index_configurators_for` here in a variable, because it's important
+        # for our tests that different instances are used for `validate` vs `configure!`. That's the case because
+        # each `index_configurator` memoizes some datastore responses (e.g. when it fetches the settings or
+        # mappings for an index...). In our tests, we use different datastore clients that connect to the same
+        # datastore server, and that means that when we reuse the same `index_configurator`, the datastore
+        # index winds up being mutated (via another client) in between `validate` and `configure!` breaking assumptions
+        # of the datastore response memoization. By using different index configurators for the two steps it
+        # avoids some odd bugs.
+        script_configurators = script_configurators_for(output)
+
+        errors = script_configurators.flat_map(&:validate) + index_definition_configurators_for(output).flat_map(&:validate)
+
+        if errors.any?
+          error_descriptions = errors.map.with_index do |error, index|
+            "#{index + 1}): #{error}"
+          end.join("\n#{"=" * 80}\n\n")
+
+          raise ClusterOperationError, "Got #{errors.size} validation error(s):\n\n#{error_descriptions}"
+        end
+
+        script_configurators.each(&:configure!)
+
+        @cluster_settings_manager.in_index_maintenance_mode(:all_clusters) do
+          index_definition_configurators_for(output).each(&:configure!)
+        end
+      end
+
+      def accessible_index_definitions
+        @accessible_index_definitions ||= @index_defs.reject { |i| i.all_accessible_cluster_names.empty? }
+      end
+
+      private
+
+      def script_configurators_for(output)
+        # It's a bit tricky to know which datastore cluster a script is needed in (the script metadata
+        # doesn't store that), but storing a script in a cluster that doesn't need it causes no harm. The
+        # id of each script contains the hash of its contents so there's no possibility of different clusters
+        # needing a script with the same `id` to have different contents. So here we create a script configurator
+        # for each datastore client.
+        @datastore_clients_by_name.values.flat_map do |datastore_client|
+          @scripts_by_id.map do |id, payload|
+            ScriptConfigurator.new(
+              datastore_client: datastore_client,
+              script_context: payload.fetch("context"),
+              script_id: id,
+              script: payload.fetch("script"),
+              output: output
+            )
+          end
+        end
+      end
+
+      def index_definition_configurators_for(output)
+        @index_defs.flat_map do |index_def|
+          env_agnostic_config = @index_configurations_by_name.fetch(index_def.name)
+
+          index_def.all_accessible_cluster_names.map do |cluster_name|
+            IndexDefinitionConfigurator.new(@datastore_clients_by_name.fetch(cluster_name), index_def, env_agnostic_config, output, @clock)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb

@@ -0,0 +1,76 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "forwardable"
+
+module ElasticGraph
+  class Admin
+    # Decorator that wraps a datastore client in order to implement dry run behavior.
+    # All write operations are implemented as no-ops, while read operations are passed through
+    # to the wrapped datastore client.
+    #
+    # We prefer this over having to check a `dry_run` flag in many places because that's
+    # easy to forget. One mistake and a dry run isn't truly a dry run!
+    #
+    # In contrast, this gives us a strong guarantee that dry run mode truly avoids mutating
+    # any datastore state. This decorator specifically picks and chooses which operations it
+    # allows.
+    #
+    # - Read operations are forwarded to the wrapped datastore client.
+    # - Write operations are implemented as no-ops.
+    #
+    # If/when the calling code evolves to call a new method on this, it'll trigger
+    # `NoMethodError`, giving us a good chance to evaluate how this decorator should
+    # support a particular API. This is also why this doesn't use Ruby's `delegate` library,
+    # because we don't want methods automatically delegated; we want to opt-in to only the read-only methods.
+    class DatastoreClientDryRunDecorator
+      extend Forwardable
+
+      def initialize(wrapped_client)
+        @wrapped_client = wrapped_client
+      end
+
+      # Cluster APIs
+      def_delegators :@wrapped_client, :get_flat_cluster_settings, :get_cluster_health
+
+      def put_persistent_cluster_settings(*) = nil
+
+      # Script APIs
+      def_delegators :@wrapped_client, :get_script
+
+      def put_script(*) = nil
+
+      def delete_script(*) = nil
+
+      # Index Template APIs
+      def_delegators :@wrapped_client, :get_index_template
+
+      def delete_index_template(*) = nil
+
+      def put_index_template(*) = nil
+
+      # Index APIs
+      def_delegators :@wrapped_client, :get_index, :list_indices_matching
+
+      def delete_indices(*) = nil
+
+      def create_index(*) = nil
+
+      def put_index_mapping(*) = nil
+
+      def put_index_settings(*) = nil
+
+      # Document APIs
+      def_delegators :@wrapped_client, :get, :search, :msearch
+
+      def delete_all_documents(*) = nil
+
+      def bulk(*) = nil
+    end
+  end
+end

data/lib/elastic_graph/admin/index_definition_configurator/for_index.rb

@@ -0,0 +1,194 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/admin/cluster_configurator/action_reporter"
+require "elastic_graph/datastore_core/index_config_normalizer"
+require "elastic_graph/indexer/hash_differ"
+require "elastic_graph/support/hash_util"
+
+module ElasticGraph
+  class Admin
+    module IndexDefinitionConfigurator
+      # Responsible for managing an index's configuration, including both mappings and settings.
+      class ForIndex
+        # @dynamic index
+
+        attr_reader :index
+
+        def initialize(datastore_client, index, env_agnostic_index_config, output)
+          @datastore_client = datastore_client
+          @index = index
+          @env_agnostic_index_config = env_agnostic_index_config
+          @reporter = ClusterConfigurator::ActionReporter.new(output)
+        end
+
+        # Attempts to idempotently update the index configuration to the desired configuration
+        # exposed by the `IndexDefinition` object. Based on the configuration of the passed index
+        # and the state of the index in the datastore, does one of the following:
+        #
+        #   - If the index did not already exist: creates the index with the desired mappings and settings.
+        #   - If the desired mapping has fewer fields than what is in the index: raises an exception,
+        #     because the datastore provides no way to remove fields from a mapping and it would be confusing
+        #     for this method to silently ignore the issue.
+        #   - If the settings have desired changes: updates the settings, restoring any setting that
+        #     no longer has a desired value to its default.
+        #   - If the mapping has desired changes: updates the mappings.
+        #
+        # Note that any of the writes to the index may fail. There are many things that cannot
+        # be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt
+        # to validate those things ahead of time and instead rely on the datastore to fail if an invalid operation
+        # is attempted.
+        def configure!
+          return create_new_index unless index_exists?
+
+          # Update settings before mappings, to front-load the API call that is more likely to fail.
+          # Our `validate` method guards against mapping changes that are known to be disallowed by
+          # the datastore, but it is much harder to validate that for settings, because there are so
+          # many settings, and there is not clear documentation that outlines all settings, which can
+          # be updated on existing indices, etc.
+          #
+          # If we get a failure, we'd rather it happen before any changes are applied to the index, instead
+          # of applying the mappings and then failing on the settings.
+          update_settings if settings_updates.any?
+
+          update_mapping if has_mapping_updates?
+        end
+
+        def validate
+          if index_exists? && mapping_type_changes.any?
+            [cannot_modify_mapping_field_type_error]
+          else
+            []
+          end
+        end
+
+        private
+
+        def create_new_index
+          @datastore_client.create_index(index: @index.name, body: desired_config)
+          report_action "Created index: `#{@index.name}`"
+        end
+
+        def update_mapping
+          @datastore_client.put_index_mapping(index: @index.name, body: desired_mapping)
+          action_description = "Updated mappings for index `#{@index.name}`:\n#{mapping_diff}"
+
+          if mapping_removals.any?
+            action_description += "\n\nNote: the extra fields listed here will not actually get removed. " \
+              "Mapping removals are unsupported (but ElasticGraph will leave them alone and they'll cause no problems)."
+          end
+
+          report_action action_description
+        end
+
+        def update_settings
+          @datastore_client.put_index_settings(index: @index.name, body: settings_updates)
+          report_action "Updated settings for index `#{@index.name}`:\n#{settings_diff}"
+        end
+
+        def cannot_modify_mapping_field_type_error
+          "The datastore does not support modifying the type of a field from an existing index definition. " \
+          "You are attempting to update type of fields (#{mapping_type_changes.inspect}) from the #{@index.name} index definition."
+        end
+
+        def index_exists?
+          !current_config.empty?
+        end
+
+        def mapping_removals
+          @mapping_removals ||= mapping_fields_from(current_mapping) - mapping_fields_from(desired_mapping)
+        end
+
+        def mapping_type_changes
+          @mapping_type_changes ||= begin
+            flattened_current = Support::HashUtil.flatten_and_stringify_keys(current_mapping)
+            flattened_desired = Support::HashUtil.flatten_and_stringify_keys(desired_mapping)
+
+            flattened_current.keys.select do |key|
+              key.end_with?(".type") && flattened_desired.key?(key) && flattened_desired[key] != flattened_current[key]
+            end
+          end
+        end
+
+        def has_mapping_updates?
+          current_mapping != desired_mapping
+        end
+
+        def settings_updates
+          @settings_updates ||= begin
+            # Updating a setting to null will cause the datastore to restore the default value of the setting.
+            restore_to_defaults = (current_settings.keys - desired_settings.keys).to_h { |key| [key, nil] }
+            desired_settings.select { |key, value| current_settings[key] != value }.merge(restore_to_defaults)
+          end
+        end
+
+        def mapping_fields_from(mapping_hash, prefix = "")
+          (mapping_hash["properties"] || []).flat_map do |key, params|
+            field = prefix + key
+            if params.key?("properties")
+              [field] + mapping_fields_from(params, "#{field}.")
+            else
+              [field]
+            end
+          end
+        end
+
+        def desired_mapping
+          desired_config.fetch("mappings")
+        end
+
+        def desired_settings
+          @desired_settings ||= desired_config.fetch("settings")
+        end
+
+        def desired_config
+          @desired_config ||= begin
+            # _meta is place where we can record state on the index mapping in the datastore.
+            # We want to maintain `_meta.ElasticGraph.sources` as an append-only set of all sources that have ever
+            # been configured to flow into an index, so that we can remember whether or not an index which currently
+            # has no `sourced_from` from fields ever did. This is necessary for our automatic filtering of multi-source
+            # indexes.
+            previously_recorded_sources = current_mapping.dig("_meta", "ElasticGraph", "sources") || []
+            sources = previously_recorded_sources.union(@index.current_sources.to_a).sort
+
+            DatastoreCore::IndexConfigNormalizer.normalize(Support::HashUtil.deep_merge(@env_agnostic_index_config, {
+              "mappings" => {"_meta" => {"ElasticGraph" => {"sources" => sources}}},
+              "settings" => @index.flattened_env_setting_overrides
+            }))
+          end
+        end
+
+        def current_mapping
+          current_config["mappings"] || {}
+        end
+
+        def current_settings
+          @current_settings ||= current_config["settings"]
+        end
+
+        def current_config
+          @current_config ||= DatastoreCore::IndexConfigNormalizer.normalize(
+            @datastore_client.get_index(@index.name)
+          )
+        end
+
+        def mapping_diff
+          @mapping_diff ||= Indexer::HashDiffer.diff(current_mapping, desired_mapping) || "(no diff)"
+        end
+
+        def settings_diff
+          @settings_diff ||= Indexer::HashDiffer.diff(current_settings, desired_settings) || "(no diff)"
+        end
+
+        def report_action(message)
+          @reporter.report_action(message)
+        end
+      end
+    end
+  end
+end