esse 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0d63306d9028972cd58998169513b15c761f1ecd6fe67d2d6d04a6eb995cdae
-  data.tar.gz: f4877f14109d22c40dbec51eef17edc86d60231d488929cc773749deeb8cb6b4
+  metadata.gz: b6c9d974dbe345e36fc5b8f286185735377309176016d9662900cec7510dd788
+  data.tar.gz: 01ad7d8d5ee1d849e0dc67cdf1c812c7c44aa686e8b71af5f85137c75e911842
 SHA512:
-  metadata.gz: ee0b45292911de3db131af146d212e36f94ad8782021c89e20da74813135ebff6c586996bd33e355bb6af0e190db6a3be34eba1aebb4fa51f75abb4adf4cf1a1
-  data.tar.gz: 70c69e8941c07ec18f3d14164ef211308ea72752bc22e1fa6e20dc0578d6c43aa549e681cca1177b148c6e37b9d2420f21f5a9ba9da4414a70a6814196a3d2e4
+  metadata.gz: b67855b65130b69443a88fe5f033c5437066f84b3ee377169c3ecb8f10c336b1f58e02751aa7fda5393eab308fbd022cc440465cfaa759a29fc943cdc85dc2b3
+  data.tar.gz: c509f2960292c11144a0092befd0fd8c074faa341b0d4ffb028eb9819561fbf28d5af07c388b7712dcc7e576214fa679c009cd54757d364ca967674de643dedd

data/lib/esse/cli/event_listener.rb

@@ -95,6 +95,13 @@ module Esse
         end
         print_message(stats.join(', ') + '.')
       end
+
+      def elasticsearch_reindex(event)
+        print_message '[%<runtime>s] Reindex from %<from>s to %<to>s successfuly completed',
+          from: colorize(event[:request].dig(:body, :source, :index), :bold),
+          to: colorize(event[:request].dig(:body, :dest, :index), :bold),
+          runtime: formatted_runtime(event[:runtime])
+      end
     end
   end
 end

data/lib/esse/cli/index.rb

@@ -16,14 +16,18 @@ module Esse
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :import, type: :boolean, default: true, desc: 'Import documents before point alias to the new index'
+      option :reindex, type: :boolean, default: false, desc: 'Use _reindex API to import documents from the old index to the new index'
       option :optimize, type: :boolean, default: true, desc: 'Optimize index before import documents by disabling refresh_interval and setting number_of_replicas to 0'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=refresh_interval:1s,number_of_replicas:0'
       def reset(*index_classes)
         require_relative 'index/reset'
-        Reset.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        if opts[:import] && opts[:reindex]
+          raise ArgumentError, 'You cannot use --import and --reindex together'
+        end
+        Reset.new(indices: index_classes, **opts).run
       end
 
-      # @TODO Add reindex task to create a new index and import documents from the old index using _reindex API
-
       desc 'create *INDEX_CLASSES', 'Creates indices for the given classes'
       long_desc <<-DESC
         Creates index and applies mapping and settings for the given classes.
@@ -33,9 +37,11 @@ module Esse
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :alias, type: :boolean, default: false, aliases: '-a', desc: 'Update alias after create index'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1,index.number_of_replicas:0'
       def create(*index_classes)
         require_relative 'index/create'
-        Create.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        Create.new(indices: index_classes, **opts).run
       end
 
       desc 'delete *INDEX_CLASSES', 'Deletes indices for the given classes'
@@ -58,9 +64,11 @@ module Esse
       desc 'update_settings *INDEX_CLASS', 'Closes the index for read/write operations, updates the index settings, and open it again'
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :type, type: :string, default: nil, aliases: '-t', desc: 'Document Type to update mapping for'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1,index.number_of_replicas:0'
       def update_settings(*index_classes)
         require_relative 'index/update_settings'
-        UpdateSettings.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        UpdateSettings.new(indices: index_classes, **opts).run
       end
 
       desc 'update_mapping *INDEX_CLASS', 'Create or update a mapping'
@@ -89,8 +97,8 @@ module Esse
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :context, type: :hash, default: {}, required: true, desc: 'List of options to pass to the index class'
       option :repo, type: :string, default: nil, alias: '-r', desc: 'Repository to use for import'
-      option :eager_include_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to include to the bulk index request'
-      option :lazy_update_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to bulk update after the bulk index request'
+      option :eager_include_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes'
+      option :lazy_update_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes'
       def import(*index_classes)
         require_relative 'index/import'
         opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)

data/lib/esse/cli/templates/config.rb.erb

@@ -13,8 +13,10 @@ Esse.configure do |config|
 
     # Global index settings
     #   cluster.settings = {
-    #     number_of_shards: 5,
-    #     number_of_replicas: 0,
+    #     index: {
+    #       number_of_shards: 5,
+    #       number_of_replicas: 0,
+    #     }
     #   }
 
     # Global index mappings

data/lib/esse/config.rb

@@ -10,8 +10,11 @@ module Esse
   #       cluster.index_prefix = 'backend'
   #       cluster.client = Elasticsearch::Client.new
   #       cluster.settings = {
-  #         number_of_shards: 2,
-  #         number_of_replicas: 0
+  #         index: {
+  #           number_of_shards: 2,
+  #           number_of_replicas: 1
+  #         },
+  #         analysis: { ... }
   #       }
   #       cluster.mappings = {
   #         dynamic_templates: [...]

data/lib/esse/document.rb

@@ -100,7 +100,10 @@ module Esse
     def inspect
       attributes = %i[id routing source].map do |attr|
         value = send(attr)
-        "#{attr}: #{value.inspect}" if value
+        next unless value
+        "#{attr}: #{value.inspect}"
+      rescue
+        nil
       end.compact.join(', ')
       attributes << " mutations: #{@__mutations__.inspect}" if @__mutations__
       "#<#{self.class.name || 'Esse::Document'} #{attributes}>"

data/lib/esse/events.rb

@@ -56,5 +56,7 @@ module Esse
     register_event 'elasticsearch.exist'
     register_event 'elasticsearch.count'
     register_event 'elasticsearch.get'
+    register_event 'elasticsearch.reindex'
+    register_event 'elasticsearch.update_by_query'
   end
 end

data/lib/esse/index/documents.rb

@@ -236,6 +236,20 @@ module Esse
         count
       end
 
+      # Update documents by query
+      #
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      #
+      # @return [Hash] The elasticsearch response hash
+      def update_by_query(suffix: nil, **options)
+        definition = {
+          index: index_name(suffix: suffix),
+        }.merge(options)
+        cluster.may_update_type!(definition)
+        cluster.api.update_by_query(**definition)
+      end
+
       protected
 
       def document?(doc)

data/lib/esse/index/indices.rb

@@ -26,10 +26,10 @@ module Esse
       #
       # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
       # @see Esse::Transport#create_index
-      def create_index(suffix: nil, body: nil, **options)
+      def create_index(suffix: nil, body: nil, settings: nil, **options)
         options = CREATE_INDEX_RESERVED_KEYWORDS.merge(options)
         name = build_real_index_name(suffix)
-        definition = body || [settings_hash, mappings_hash].reduce(&:merge)
+        definition = body || [settings_hash(settings: settings), mappings_hash].reduce(&:merge)
 
         if options.delete(:alias) && name != index_name
           definition[:aliases] = { index_name => {} }
@@ -48,34 +48,38 @@ module Esse
       # @return [Hash] the elasticsearch response
       #
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html
-      def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: false, **options)
+      def reset_index(suffix: index_suffix, settings: nil, optimize: true, import: true, reindex: false, refresh: nil, **options)
         cluster.throw_error_when_readonly!
 
         suffix ||= Esse.timestamp
         suffix = Esse.timestamp while index_exist?(suffix: suffix)
 
-        if optimize
-          definition = [settings_hash, mappings_hash].reduce(&:merge)
+        if optimize && import
+          definition = [settings_hash(settings: settings), mappings_hash].reduce(&:merge)
           number_of_replicas = definition.dig(Esse::SETTING_ROOT_KEY, :index, :number_of_replicas)
           refresh_interval = definition.dig(Esse::SETTING_ROOT_KEY, :index, :refresh_interval)
           new_number_of_replicas = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:number_of_replicas] = 0
           new_refresh_interval = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:refresh_interval] = '-1'
           create_index(**options, suffix: suffix, alias: false, body: definition)
         else
-          create_index(**options, suffix: suffix, alias: false)
+          create_index(**options, suffix: suffix, alias: false, settings: settings)
         end
 
         if index_exist? && aliases.none?
           cluster.api.delete_index(index: index_name)
         end
         if import
-          import(**options, suffix: suffix)
-        elsif reindex && (_from = indices_pointing_to_alias).any?
-          # @TODO: Reindex using the reindex API
+          import(**options, suffix: suffix, refresh: refresh)
+        elsif reindex && (source_indexes = indices_pointing_to_alias).any?
+          reindex_kwargs = reindex.is_a?(Hash) ? reindex : {}
+          reindex_kwargs[:wait_for_completion] = true unless reindex_kwargs.key?(:wait_for_completion)
+          source_indexes.each do |from|
+            cluster.api.reindex(**options, body: { source: { index: from }, dest: { index: index_name(suffix: suffix) } }, refresh: refresh)
+          end
         end
 
-        if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval
-          update_settings(suffix: suffix)
+        if optimize && import && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval
+          update_settings(suffix: suffix, settings: settings)
           refresh(suffix: suffix)
         end
 
@@ -152,16 +156,17 @@ module Esse
       #
       # @param :suffix [String, nil] :suffix The index suffix
       # @see Esse::Transport#update_settings
-      def update_settings(suffix: nil, **options)
+      def update_settings(suffix: nil, settings: nil, **options)
         response = nil
 
-        settings = HashUtils.deep_transform_keys(settings_hash.fetch(Esse::SETTING_ROOT_KEY), &:to_s)
+        settings = HashUtils.deep_transform_keys(settings_hash(settings: settings).fetch(Esse::SETTING_ROOT_KEY), &:to_sym)
         if options[:body]
-          settings = settings.merge(HashUtils.deep_transform_keys(options.delete(:body), &:to_s))
+          body = HashUtils.deep_transform_keys(options.delete(:body), &:to_sym)
+          settings = HashUtils.deep_merge(settings, body)
         end
-        settings.delete('number_of_shards') # Can't change number of shards for an index
-        settings['index']&.delete('number_of_shards')
-        analysis = settings.delete('analysis')
+        settings.delete(:number_of_shards) # Can't change number of shards for an index
+        settings[:index]&.delete(:number_of_shards)
+        analysis = settings.delete(:analysis)
 
         if settings.any?
           response = cluster.api.update_settings(index: index_name(suffix: suffix), body: settings, **options)

data/lib/esse/index/settings.rb

@@ -4,9 +4,28 @@ module Esse
   # https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/indices/put_settings.rb
   class Index
     module ClassMethods
-      def settings_hash
+      # Elasticsearch supports passing index.* related settings directly in the body of the request.
+      # We are moving it to the index key to make it more explicit and to be the source-of-truth when merging settings.
+      # So the settings `{ number_of_shards: 1 }` will be transformed to `{ index: { number_of_shards: 1 } }`
+      INDEX_SIMPLIFIED_SETTINGS = %i[
+        number_of_shards
+        number_of_replicas
+        refresh_interval
+      ].freeze
+
+      def settings_hash(settings: nil)
         hash = setting.body
-        { Esse::SETTING_ROOT_KEY => (hash.key?(Esse::SETTING_ROOT_KEY) ? hash[Esse::SETTING_ROOT_KEY] : hash) }
+        values = (hash.key?(Esse::SETTING_ROOT_KEY) ? hash[Esse::SETTING_ROOT_KEY] : hash)
+        values = HashUtils.explode_keys(values)
+        if settings.is_a?(Hash)
+          values = HashUtils.deep_merge(values, HashUtils.explode_keys(settings))
+        end
+        INDEX_SIMPLIFIED_SETTINGS.each do |key|
+          next unless values.key?(key)
+
+          (values[:index] ||= {}).merge!(key => values.delete(key))
+        end
+        { Esse::SETTING_ROOT_KEY => values }
       end
 
       # Define /_settings definition by each index.
@@ -18,7 +37,7 @@ module Esse
       #
       #   class UserIndex < Esse::Index
       #     settings {
-      #       number_of_replicas: 4,
+      #       index: { number_of_replicas: 4 }
       #     }
       #   end
       #

data/lib/esse/primitives/hash_utils.rb

@@ -47,5 +47,15 @@ module Esse
         end
       end
     end
+
+    def explode_keys(hash, separator = '.')
+      hash.each_with_object({}) do |(key, value), result|
+        is_symbol = key.is_a?(Symbol)
+        keys = key.to_s.split(separator)
+        last_key = keys.pop
+        current = keys.reduce(result) { |memo, k| memo[is_symbol ? k.to_sym : k] ||= {} }
+        current[is_symbol ? last_key.to_sym : last_key] = value
+      end
+    end
   end
 end

data/lib/esse/transport/indices.rb

@@ -185,6 +185,83 @@ module Esse
           payload[:response] = coerce_exception { client.indices.put_settings(**opts) }
         end
       end
+
+      # Allows to copy documents from one index to another, optionally filtering the source
+      # documents by a query, changing the destination index settings, or fetching the
+      # documents from a remote cluster.
+      #
+      # @option arguments [Boolean] :refresh Should the affected indexes be refreshed?
+      # @option arguments [Time] :timeout Time each individual bulk request should wait for shards that are unavailable.
+      # @option arguments [String] :wait_for_active_shards Sets the number of shard copies that must be active before proceeding with the reindex operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)
+      # @option arguments [Boolean] :wait_for_completion Should the request should block until the reindex is complete.
+      # @option arguments [Number] :requests_per_second The throttle to set on this request in sub-requests per second. -1 means no throttle.
+      # @option arguments [Time] :scroll Control how long to keep the search context alive
+      # @option arguments [Number|string] :slices The number of slices this task should be divided into. Defaults to 1, meaning the task isn't sliced into subtasks. Can be set to `auto`.
+      # @option arguments [Number] :max_docs Maximum number of documents to process (default: all documents)
+      # @option arguments [Hash] :headers Custom HTTP headers
+      # @option arguments [Hash] :body The search definition using the Query DSL and the prototype for the index request. (*Required*)
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html
+      def reindex(body:, **options)
+        throw_error_when_readonly!
+
+        Esse::Events.instrument('elasticsearch.reindex') do |payload|
+          payload[:request] = opts = options.merge(body: body)
+          payload[:response] = coerce_exception { client.reindex(**opts) }
+        end
+      end
+
+      # Performs an update on every document in the index without changing the source,
+      # for example to pick up a mapping change.
+      #
+      # @option arguments [List] :index A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices (*Required*)
+      # @option arguments [String] :analyzer The analyzer to use for the query string
+      # @option arguments [Boolean] :analyze_wildcard Specify whether wildcard and prefix queries should be analyzed (default: false)
+      # @option arguments [String] :default_operator The default operator for query string query (AND or OR) (options: AND, OR)
+      # @option arguments [String] :df The field to use as default where no field prefix is given in the query string
+      # @option arguments [Number] :from Starting offset (default: 0)
+      # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when unavailable (missing or closed)
+      # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)
+      # @option arguments [String] :conflicts What to do when the update by query hits version conflicts? (options: abort, proceed)
+      # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that are open, closed or both. (options: open, closed, hidden, none, all)
+      # @option arguments [Boolean] :lenient Specify whether format-based query failures (such as providing text to a numeric field) should be ignored
+      # @option arguments [String] :pipeline Ingest pipeline to set on index requests made by this action. (default: none)
+      # @option arguments [String] :preference Specify the node or shard the operation should be performed on (default: random)
+      # @option arguments [String] :q Query in the Lucene query string syntax
+      # @option arguments [List] :routing A comma-separated list of specific routing values
+      # @option arguments [Time] :scroll Specify how long a consistent view of the index should be maintained for scrolled search
+      # @option arguments [String] :search_type Search operation type (options: query_then_fetch, dfs_query_then_fetch)
+      # @option arguments [Time] :search_timeout Explicit timeout for each search request. Defaults to no timeout.
+      # @option arguments [Number] :size Deprecated, please use `max_docs` instead
+      # @option arguments [Number] :max_docs Maximum number of documents to process (default: all documents)
+      # @option arguments [List] :sort A comma-separated list of <field>:<direction> pairs
+      # @option arguments [List] :_source True or false to return the _source field or not, or a list of fields to return
+      # @option arguments [List] :_source_excludes A list of fields to exclude from the returned _source field
+      # @option arguments [List] :_source_includes A list of fields to extract and return from the _source field
+      # @option arguments [Number] :terminate_after The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early.
+      # @option arguments [List] :stats Specific 'tag' of the request for logging and statistical purposes
+      # @option arguments [Boolean] :version Specify whether to return document version as part of a hit
+      # @option arguments [Boolean] :version_type Should the document increment the version number (internal) on hit or not (reindex)
+      # @option arguments [Boolean] :request_cache Specify if request cache should be used for this request or not, defaults to index level setting
+      # @option arguments [Boolean] :refresh Should the affected indexes be refreshed?
+      # @option arguments [Time] :timeout Time each individual bulk request should wait for shards that are unavailable.
+      # @option arguments [String] :wait_for_active_shards Sets the number of shard copies that must be active before proceeding with the update by query operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)
+      # @option arguments [Number] :scroll_size Size on the scroll request powering the update by query
+      # @option arguments [Boolean] :wait_for_completion Should the request should block until the update by query operation is complete.
+      # @option arguments [Number] :requests_per_second The throttle to set on this request in sub-requests per second. -1 means no throttle.
+      # @option arguments [Number|string] :slices The number of slices this task should be divided into. Defaults to 1, meaning the task isn't sliced into subtasks. Can be set to `auto`.
+      # @option arguments [Hash] :headers Custom HTTP headers
+      # @option arguments [Hash] :body The search definition using the Query DSL
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html
+      def update_by_query(index:, **options)
+        throw_error_when_readonly!
+
+        Esse::Events.instrument('elasticsearch.update_by_query') do |payload|
+          payload[:request] = opts = options.merge(index: index)
+          payload[:response] = coerce_exception { client.update_by_query(**opts) }
+        end
+      end
     end
 
     include InstanceMethods

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.3.4'
+  VERSION = '0.3.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2024-07-18 00:00:00.000000000 Z
+date: 2024-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json