esse 0.3.4 → 0.4.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0d63306d9028972cd58998169513b15c761f1ecd6fe67d2d6d04a6eb995cdae
-  data.tar.gz: f4877f14109d22c40dbec51eef17edc86d60231d488929cc773749deeb8cb6b4
+  metadata.gz: f6023e2f4fe6b5d8e45dd09e7c98bb250d8862ea3adc769f119011efc07191c5
+  data.tar.gz: 852a46c60435d7c0848254d6b79a4a29ee89efe9e0700975d7d651660ad874d8
 SHA512:
-  metadata.gz: ee0b45292911de3db131af146d212e36f94ad8782021c89e20da74813135ebff6c586996bd33e355bb6af0e190db6a3be34eba1aebb4fa51f75abb4adf4cf1a1
-  data.tar.gz: 70c69e8941c07ec18f3d14164ef211308ea72752bc22e1fa6e20dc0578d6c43aa549e681cca1177b148c6e37b9d2420f21f5a9ba9da4414a70a6814196a3d2e4
+  metadata.gz: 40b5c6d0ed7ff8ce793cd8be088c01f9377c62cf58366b201c8ee988f04e9f2f3e53118a48c165857394dc100c26fd06b300d653f2afdead076430d19210c554
+  data.tar.gz: f3b4f4acadea569ef1108c1d3d3a2745c965177c29f307e5076357d89aec1f050f778933a350ccb75c0128a295d0f1349899452ea041551fae8ae574fb75793d

data/lib/esse/cli/event_listener.rb

@@ -95,6 +95,13 @@ module Esse
         end
         print_message(stats.join(', ') + '.')
       end
+
+      def elasticsearch_reindex(event)
+        print_message '[%<runtime>s] Reindex from %<from>s to %<to>s successfuly completed',
+          from: colorize(event[:request].dig(:body, :source, :index), :bold),
+          to: colorize(event[:request].dig(:body, :dest, :index), :bold),
+          runtime: formatted_runtime(event[:runtime])
+      end
     end
   end
 end

data/lib/esse/cli/index.rb

@@ -16,14 +16,18 @@ module Esse
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :import, type: :boolean, default: true, desc: 'Import documents before point alias to the new index'
+      option :reindex, type: :boolean, default: false, desc: 'Use _reindex API to import documents from the old index to the new index'
       option :optimize, type: :boolean, default: true, desc: 'Optimize index before import documents by disabling refresh_interval and setting number_of_replicas to 0'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=refresh_interval:1s,number_of_replicas:0'
       def reset(*index_classes)
         require_relative 'index/reset'
-        Reset.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        if opts[:import] && opts[:reindex]
+          raise ArgumentError, 'You cannot use --import and --reindex together'
+        end
+        Reset.new(indices: index_classes, **opts).run
       end
 
-      # @TODO Add reindex task to create a new index and import documents from the old index using _reindex API
-
       desc 'create *INDEX_CLASSES', 'Creates indices for the given classes'
       long_desc <<-DESC
         Creates index and applies mapping and settings for the given classes.
@@ -33,9 +37,11 @@ module Esse
       DESC
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :alias, type: :boolean, default: false, aliases: '-a', desc: 'Update alias after create index'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1,index.number_of_replicas:0'
       def create(*index_classes)
         require_relative 'index/create'
-        Create.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        Create.new(indices: index_classes, **opts).run
       end
 
       desc 'delete *INDEX_CLASSES', 'Deletes indices for the given classes'
@@ -58,9 +64,11 @@ module Esse
       desc 'update_settings *INDEX_CLASS', 'Closes the index for read/write operations, updates the index settings, and open it again'
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :type, type: :string, default: nil, aliases: '-t', desc: 'Document Type to update mapping for'
+      option :settings, type: :hash, default: nil, desc: 'List of settings to pass to the index class. Example: --settings=index.refresh_interval:-1,index.number_of_replicas:0'
       def update_settings(*index_classes)
         require_relative 'index/update_settings'
-        UpdateSettings.new(indices: index_classes, **options.to_h.transform_keys(&:to_sym)).run
+        opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+        UpdateSettings.new(indices: index_classes, **opts).run
       end
 
       desc 'update_mapping *INDEX_CLASS', 'Create or update a mapping'
@@ -89,18 +97,17 @@ module Esse
       option :suffix, type: :string, default: nil, aliases: '-s', desc: 'Suffix to append to index name'
       option :context, type: :hash, default: {}, required: true, desc: 'List of options to pass to the index class'
       option :repo, type: :string, default: nil, alias: '-r', desc: 'Repository to use for import'
-      option :eager_include_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to include to the bulk index request'
-      option :lazy_update_document_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to bulk update after the bulk index request'
+      option :preload_lazy_attributes, type: :string, default: nil, desc: 'Command separated list of lazy document attributes to preload using search API before the bulk import. Or pass `true` to preload all lazy attributes'
+      option :eager_load_lazy_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes'
+      option :update_lazy_attributes, type: :string, default: nil, desc: 'Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes'
+
       def import(*index_classes)
         require_relative 'index/import'
         opts = HashUtils.deep_transform_keys(options.to_h, &:to_sym)
-        opts.delete(:lazy_update_document_attributes) if opts[:lazy_update_document_attributes] == 'false'
-        opts.delete(:eager_include_document_attributes) if opts[:eager_include_document_attributes] == 'false'
-        if (val = opts[:eager_include_document_attributes])
-          opts[:eager_include_document_attributes] = (val == 'true') ? true : val.split(',')
-        end
-        if (val = opts[:lazy_update_document_attributes])
-          opts[:lazy_update_document_attributes] = (val == 'true') ? true : val.split(',')
+        %i[preload_lazy_attributes eager_load_lazy_attributes update_lazy_attributes].each do |key|
+          if (val = opts.delete(key)) && val != 'false'
+            opts[key] = (val == 'true') ? true : val.split(',')
+          end
         end
         Import.new(indices: index_classes, **opts).run
       end

data/lib/esse/cli/templates/config.rb.erb

@@ -13,8 +13,10 @@ Esse.configure do |config|
 
     # Global index settings
     #   cluster.settings = {
-    #     number_of_shards: 5,
-    #     number_of_replicas: 0,
+    #     index: {
+    #       number_of_shards: 5,
+    #       number_of_replicas: 0,
+    #     }
     #   }
 
     # Global index mappings

data/lib/esse/config.rb

@@ -10,8 +10,11 @@ module Esse
   #       cluster.index_prefix = 'backend'
   #       cluster.client = Elasticsearch::Client.new
   #       cluster.settings = {
-  #         number_of_shards: 2,
-  #         number_of_replicas: 0
+  #         index: {
+  #           number_of_shards: 2,
+  #           number_of_replicas: 1
+  #         },
+  #         analysis: { ... }
   #       }
   #       cluster.mappings = {
   #         dynamic_templates: [...]

data/lib/esse/core.rb

@@ -6,6 +6,7 @@ module Esse
   require_relative 'primitives'
   require_relative 'collection'
   require_relative 'document'
+  require_relative 'document_for_partial_update'
   require_relative 'document_lazy_attribute'
   require_relative 'lazy_document_header'
   require_relative 'hash_document'
@@ -91,4 +92,10 @@ module Esse
 
     !!(object.is_a?(Esse::Document) && object.id)
   end
+
+  def self.document_match_with_header?(document, id, routing, type)
+    id && id.to_s == document.id.to_s &&
+      routing == document.routing &&
+      (LazyDocumentHeader::ACCEPTABLE_DOC_TYPES.include?(document.type) && LazyDocumentHeader::ACCEPTABLE_DOC_TYPES.include?(type) || document.type == type)
+  end
 end

data/lib/esse/document.rb

@@ -2,11 +2,13 @@
 
 module Esse
   class Document
+    MUTATIONS_FALLBACK = {}.freeze
+
     attr_reader :object, :options
 
     def initialize(object, **options)
       @object = object
-      @options = options
+      @options = options.freeze
     end
 
     # @return [String, Number] the document ID
@@ -84,11 +86,16 @@ module Esse
       id.nil?
     end
 
-    def ==(other)
-      other.is_a?(self.class) && (
-        id == other.id && type == other.type && routing == other.routing && meta == other.meta && source == other.source
-      )
+    def eql?(other, match_lazy_doc_header: false)
+      if match_lazy_doc_header
+        other.eql?(self)
+      else
+        other.is_a?(Esse::Document) && (
+          id.to_s == other.id.to_s && type == other.type && routing == other.routing && meta == other.meta
+        )
+      end
     end
+    alias_method :==, :eql?
 
     def doc_header
       { _id: id }.tap do |h|
@@ -97,10 +104,17 @@ module Esse
       end
     end
 
+    def document_for_partial_update(source)
+      DocumentForPartialUpdate.new(self, source: source)
+    end
+
     def inspect
       attributes = %i[id routing source].map do |attr|
         value = send(attr)
-        "#{attr}: #{value.inspect}" if value
+        next unless value
+        "#{attr}: #{value.inspect}"
+      rescue
+        nil
       end.compact.join(', ')
       attributes << " mutations: #{@__mutations__.inspect}" if @__mutations__
       "#<#{self.class.name || 'Esse::Document'} #{attributes}>"
@@ -112,7 +126,9 @@ module Esse
       instance_variable_set(:@__mutated_source__, nil)
     end
 
-    protected
+    def mutations
+      @__mutations__ || MUTATIONS_FALLBACK
+    end
 
     def mutated_source
       return source unless @__mutations__

data/lib/esse/document_for_partial_update.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Esse
+  class DocumentForPartialUpdate < Esse::Document
+    extend Forwardable
+
+    def_delegators :object, :id, :type, :routing, :options
+
+    attr_reader :source
+
+    def initialize(lazy_header, source:)
+      @source = source
+      super(lazy_header)
+    end
+  end
+end

data/lib/esse/events.rb

@@ -56,5 +56,7 @@ module Esse
     register_event 'elasticsearch.exist'
     register_event 'elasticsearch.count'
     register_event 'elasticsearch.get'
+    register_event 'elasticsearch.reindex'
+    register_event 'elasticsearch.update_by_query'
   end
 end

data/lib/esse/import/bulk.rb

@@ -1,27 +1,35 @@
 module Esse
   module Import
     class Bulk
-      def initialize(type: nil, index: nil, delete: nil, create: nil, update: nil)
-        @index = Array(index).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+      def self.build_from_documents(type: nil, index: nil, delete: nil, create: nil, update: nil)
+        index = Array(index).select(&Esse.method(:document?)).reject(&:ignore_on_index?).map do |doc|
           value = doc.to_bulk
           value[:_type] ||= type if type
-          { index: value }
+          value
         end
-        @create = Array(create).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+        create = Array(create).select(&Esse.method(:document?)).reject(&:ignore_on_index?).map do |doc|
           value = doc.to_bulk
           value[:_type] ||= type if type
-          { create: value }
+          value
         end
-        @update = Array(update).select(&method(:valid_doc?)).reject(&:ignore_on_index?).map do |doc|
+        update = Array(update).select(&Esse.method(:document?)).reject(&:ignore_on_index?).map do |doc|
           value = doc.to_bulk(operation: :update)
           value[:_type] ||= type if type
-          { update: value }
+          value
         end
-        @delete = Array(delete).select(&method(:valid_doc?)).reject(&:ignore_on_delete?).map do |doc|
+        delete = Array(delete).select(&Esse.method(:document?)).reject(&:ignore_on_delete?).map do |doc|
           value = doc.to_bulk(data: false)
           value[:_type] ||= type if type
-          { delete: value }
+          value
         end
+        new(index: index, delete: delete, create: create, update: update)
+      end
+
+      def initialize(index: nil, delete: nil, create: nil, update: nil)
+        @index = Esse::ArrayUtils.wrap(index).map { |payload| { index: payload } }
+        @create = Esse::ArrayUtils.wrap(create).map { |payload| { create: payload } }
+        @update = Esse::ArrayUtils.wrap(update).map { |payload| { update: payload } }
+        @delete = Esse::ArrayUtils.wrap(delete).map { |payload| { delete: payload } }
       end
 
       # Return an array of RequestBody instances
@@ -68,10 +76,6 @@ module Esse
 
       private
 
-      def valid_doc?(doc)
-        Esse.document?(doc)
-      end
-
       def optimistic_request
         request = Import::RequestBodyAsJson.new
         request.create = @create

data/lib/esse/index/documents.rb

@@ -114,7 +114,7 @@ module Esse
       def update(doc = nil, suffix: nil, **options)
         if document?(doc)
           options[:id] = doc.id
-          options[:body] = { doc: doc.source }
+          options[:body] = { doc: doc.mutated_source }
           options[:type] = doc.type if doc.type?
           options[:routing] = doc.routing if doc.routing?
         end
@@ -140,7 +140,7 @@ module Esse
       def index(doc = nil, suffix: nil, **options)
         if document?(doc)
           options[:id] = doc.id
-          options[:body] = doc.source
+          options[:body] = doc.mutated_source
           options[:type] = doc.type if doc.type?
           options[:routing] = doc.routing if doc.routing?
         end
@@ -171,13 +171,53 @@ module Esse
         }.merge(options)
         cluster.may_update_type!(definition)
 
+        to_index = []
+        to_create = []
+        to_update = []
+        to_delete = []
+        Esse::ArrayUtils.wrap(index).each do |doc|
+          if doc.is_a?(Hash)
+            to_index << doc
+          elsif Esse.document?(doc) && !doc.ignore_on_index?
+            hash = doc.to_bulk
+            hash[:_type] ||= type if type
+            to_index << hash
+          end
+        end
+        Esse::ArrayUtils.wrap(create).each do |doc|
+          if doc.is_a?(Hash)
+            to_create << doc
+          elsif Esse.document?(doc) && !doc.ignore_on_index?
+            hash = doc.to_bulk
+            hash[:_type] ||= type if type
+            to_create << hash
+          end
+        end
+        Esse::ArrayUtils.wrap(update).each do |doc|
+          if doc.is_a?(Hash)
+            to_update << doc
+          elsif Esse.document?(doc) && !doc.ignore_on_index?
+            hash = doc.to_bulk(operation: :update)
+            hash[:_type] ||= type if type
+            to_update << hash
+          end
+        end
+        Esse::ArrayUtils.wrap(delete).each do |doc|
+          if doc.is_a?(Hash)
+            to_delete << doc
+          elsif Esse.document?(doc) && !doc.ignore_on_delete?
+            hash = doc.to_bulk(data: false)
+            hash[:_type] ||= type if type
+            to_delete << hash
+          end
+        end
+
         # @TODO Wrap the return in a some other Stats object with more information
         Esse::Import::Bulk.new(
-          **definition.slice(:type),
-          create: create,
-          delete: delete,
-          index: index,
-          update: update,
+          create: to_create,
+          delete: to_delete,
+          index: to_index,
+          update: to_update,
         ).each_request do |request_body|
           cluster.api.bulk(**definition, body: request_body.body) do |event_payload|
             event_payload[:body_stats] = request_body.stats
@@ -198,36 +238,61 @@ module Esse
       # @option [String, nil] :suffix The index suffix. Defaults to the nil.
       # @option [Hash] :context The collection context. This value will be passed as argument to the collection
       #   May be SQL condition or any other filter you have defined on the collection.
+      # @option [Boolean, Array<String>] :eager_load_lazy_attributes A list of lazy document attributes to include to the bulk index request.
+      #   Or pass `true` to include all lazy attributes.
+      # @option [Boolean, Array<String>] :update_lazy_attributes A list of lazy document attributes to bulk update each after the bulk import.
+      #   Or pass `true` to update all lazy attributes.
+      # @option [Boolean, Array<String>] :preload_lazy_attributes A list of lazy document attributes to preload using search API before the bulk import.
+      #   Or pass `true` to preload all lazy attributes.
       # @return [Numeric] The number of documents imported
-      def import(*repo_types, context: {}, eager_include_document_attributes: false, lazy_update_document_attributes: false, suffix: nil, **options)
+      def import(*repo_types, context: {}, eager_load_lazy_attributes: false, update_lazy_attributes: false, preload_lazy_attributes: false, suffix: nil, **options)
         repo_types = repo_hash.keys if repo_types.empty?
         count = 0
 
+        if options.key?(:eager_include_document_attributes)
+          warn 'The `eager_include_document_attributes` option is deprecated. Use `eager_load_lazy_attributes` instead.'
+          eager_load_lazy_attributes = options.delete(:eager_include_document_attributes)
+        end
+        if options.key?(:lazy_update_document_attributes)
+          warn 'The `lazy_update_document_attributes` option is deprecated. Use `update_lazy_attributes` instead.'
+          update_lazy_attributes = options.delete(:lazy_update_document_attributes)
+        end
+
         repo_hash.slice(*repo_types).each do |repo_name, repo|
-          doc_attrs = {eager: [], lazy: []}
-          doc_attrs[:eager] = repo.lazy_document_attribute_names(eager_include_document_attributes)
-          doc_attrs[:lazy] = repo.lazy_document_attribute_names(lazy_update_document_attributes)
-          doc_attrs[:lazy] -= doc_attrs[:eager]
+          # Elasticsearch 6.x and older have multiple types per index.
+          # This gem supports multiple types per index for backward compatibility, but we recommend to update
+          # your elasticsearch to a at least 7.x version and use a single type per index.
+          #
+          # Note that the repository name will be used as the document type.
+          # mapping_default_type
+          bulk_kwargs = { suffix: suffix, type: repo_name, **options }
+          cluster.may_update_type!(bulk_kwargs)
 
           context ||= {}
-          context[:lazy_attributes] = doc_attrs[:eager] if doc_attrs[:eager].any?
+          context[:eager_load_lazy_attributes] = eager_load_lazy_attributes
+          context[:preload_lazy_attributes] = preload_lazy_attributes
           repo.each_serialized_batch(**context) do |batch|
-            # Elasticsearch 6.x and older have multiple types per index.
-            # This gem supports multiple types per index for backward compatibility, but we recommend to update
-            # your elasticsearch to a at least 7.x version and use a single type per index.
-            #
-            # Note that the repository name will be used as the document type.
-            # mapping_default_type
-            kwargs = { suffix: suffix, type: repo_name, **options }
-            cluster.may_update_type!(kwargs)
-
-            bulk(**kwargs, index: batch)
-
-            doc_attrs[:lazy].each do |attr_name|
-              partial_docs = repo.documents_for_lazy_attribute(attr_name, batch.reject(&:ignore_on_index?))
-              next if partial_docs.empty?
-
-              bulk(**kwargs, update: partial_docs)
+            bulk(**bulk_kwargs, index: batch)
+
+            if update_lazy_attributes != false
+              attrs = repo.lazy_document_attribute_names(update_lazy_attributes)
+              attrs -= repo.lazy_document_attribute_names(eager_load_lazy_attributes)
+              update_attrs = attrs.each_with_object(Hash.new { |h, k| h[k] = {} }) do |attr_name, memo|
+                filtered_docs = batch.reject do |doc|
+                  doc.ignore_on_index? || doc.mutations.key?(attr_name)
+                end
+                next if filtered_docs.empty?
+
+                repo.retrieve_lazy_attribute_values(attr_name, filtered_docs).each do |doc, value|
+                  memo[doc.doc_header][attr_name] = value
+                end
+              end
+              if update_attrs.any?
+                bulk_update = update_attrs.map do |header, values|
+                  header.merge(data: {doc: values})
+                end
+                bulk(**bulk_kwargs, update: bulk_update)
+              end
             end
 
             count += batch.size
@@ -236,6 +301,20 @@ module Esse
         count
       end
 
+      # Update documents by query
+      #
+      # @param options [Hash] Hash of paramenters that will be passed along to elasticsearch request
+      # @option [String, nil] :suffix The index suffix. Defaults to the nil.
+      #
+      # @return [Hash] The elasticsearch response hash
+      def update_by_query(suffix: nil, **options)
+        definition = {
+          index: index_name(suffix: suffix),
+        }.merge(options)
+        cluster.may_update_type!(definition)
+        cluster.api.update_by_query(**definition)
+      end
+
       protected
 
       def document?(doc)

data/lib/esse/index/indices.rb

@@ -26,10 +26,10 @@ module Esse
       #
       # @see http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
       # @see Esse::Transport#create_index
-      def create_index(suffix: nil, body: nil, **options)
+      def create_index(suffix: nil, body: nil, settings: nil, **options)
         options = CREATE_INDEX_RESERVED_KEYWORDS.merge(options)
         name = build_real_index_name(suffix)
-        definition = body || [settings_hash, mappings_hash].reduce(&:merge)
+        definition = body || [settings_hash(settings: settings), mappings_hash].reduce(&:merge)
 
         if options.delete(:alias) && name != index_name
           definition[:aliases] = { index_name => {} }
@@ -48,34 +48,40 @@ module Esse
       # @return [Hash] the elasticsearch response
       #
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html
-      def reset_index(suffix: index_suffix, optimize: true, import: true, reindex: false, **options)
+      def reset_index(suffix: index_suffix, settings: nil, optimize: true, import: true, reindex: false, refresh: nil, **options)
         cluster.throw_error_when_readonly!
 
         suffix ||= Esse.timestamp
         suffix = Esse.timestamp while index_exist?(suffix: suffix)
 
-        if optimize
-          definition = [settings_hash, mappings_hash].reduce(&:merge)
+        if optimize && import
+          definition = [settings_hash(settings: settings), mappings_hash].reduce(&:merge)
           number_of_replicas = definition.dig(Esse::SETTING_ROOT_KEY, :index, :number_of_replicas)
           refresh_interval = definition.dig(Esse::SETTING_ROOT_KEY, :index, :refresh_interval)
           new_number_of_replicas = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:number_of_replicas] = 0
           new_refresh_interval = ((definition[Esse::SETTING_ROOT_KEY] ||= {})[:index] ||= {})[:refresh_interval] = '-1'
           create_index(**options, suffix: suffix, alias: false, body: definition)
         else
-          create_index(**options, suffix: suffix, alias: false)
+          create_index(**options, suffix: suffix, alias: false, settings: settings)
         end
 
         if index_exist? && aliases.none?
           cluster.api.delete_index(index: index_name)
         end
         if import
-          import(**options, suffix: suffix)
-        elsif reindex && (_from = indices_pointing_to_alias).any?
-          # @TODO: Reindex using the reindex API
+          import_kwargs = import.is_a?(Hash) ? import : {}
+          import_kwargs[:refresh] ||= refresh if refresh
+          import(**options, **import_kwargs, suffix: suffix)
+        elsif reindex && (source_indexes = indices_pointing_to_alias).any?
+          reindex_kwargs = reindex.is_a?(Hash) ? reindex : {}
+          reindex_kwargs[:wait_for_completion] = true unless reindex_kwargs.key?(:wait_for_completion)
+          source_indexes.each do |from|
+            cluster.api.reindex(**options, body: { source: { index: from }, dest: { index: index_name(suffix: suffix) } }, refresh: refresh)
+          end
         end
 
-        if optimize && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval
-          update_settings(suffix: suffix)
+        if optimize && import && number_of_replicas != new_number_of_replicas || refresh_interval != new_refresh_interval
+          update_settings(suffix: suffix, settings: settings)
           refresh(suffix: suffix)
         end
 
@@ -152,16 +158,17 @@ module Esse
       #
       # @param :suffix [String, nil] :suffix The index suffix
       # @see Esse::Transport#update_settings
-      def update_settings(suffix: nil, **options)
+      def update_settings(suffix: nil, settings: nil, **options)
         response = nil
 
-        settings = HashUtils.deep_transform_keys(settings_hash.fetch(Esse::SETTING_ROOT_KEY), &:to_s)
+        settings = HashUtils.deep_transform_keys(settings_hash(settings: settings).fetch(Esse::SETTING_ROOT_KEY), &:to_sym)
         if options[:body]
-          settings = settings.merge(HashUtils.deep_transform_keys(options.delete(:body), &:to_s))
+          body = HashUtils.deep_transform_keys(options.delete(:body), &:to_sym)
+          settings = HashUtils.deep_merge(settings, body)
         end
-        settings.delete('number_of_shards') # Can't change number of shards for an index
-        settings['index']&.delete('number_of_shards')
-        analysis = settings.delete('analysis')
+        settings.delete(:number_of_shards) # Can't change number of shards for an index
+        settings[:index]&.delete(:number_of_shards)
+        analysis = settings.delete(:analysis)
 
         if settings.any?
           response = cluster.api.update_settings(index: index_name(suffix: suffix), body: settings, **options)

data/lib/esse/index/object_document_mapper.rb

@@ -24,7 +24,7 @@ module Esse
       # @return [Enumerator] All serialized entries
       def documents(repo_name = nil, **kwargs)
         Enumerator.new do |yielder|
-          each_serialized_batch(repo_name, **kwargs) do |documents, **_collection_kargs|
+          each_serialized_batch(repo_name, **kwargs) do |documents|
             documents.each { |document| yielder.yield(document) }
           end
         end

data/lib/esse/index/settings.rb

@@ -4,9 +4,28 @@ module Esse
   # https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/indices/put_settings.rb
   class Index
     module ClassMethods
-      def settings_hash
+      # Elasticsearch supports passing index.* related settings directly in the body of the request.
+      # We are moving it to the index key to make it more explicit and to be the source-of-truth when merging settings.
+      # So the settings `{ number_of_shards: 1 }` will be transformed to `{ index: { number_of_shards: 1 } }`
+      INDEX_SIMPLIFIED_SETTINGS = %i[
+        number_of_shards
+        number_of_replicas
+        refresh_interval
+      ].freeze
+
+      def settings_hash(settings: nil)
         hash = setting.body
-        { Esse::SETTING_ROOT_KEY => (hash.key?(Esse::SETTING_ROOT_KEY) ? hash[Esse::SETTING_ROOT_KEY] : hash) }
+        values = (hash.key?(Esse::SETTING_ROOT_KEY) ? hash[Esse::SETTING_ROOT_KEY] : hash)
+        values = HashUtils.explode_keys(values)
+        if settings.is_a?(Hash)
+          values = HashUtils.deep_merge(values, HashUtils.explode_keys(settings))
+        end
+        INDEX_SIMPLIFIED_SETTINGS.each do |key|
+          next unless values.key?(key)
+
+          (values[:index] ||= {}).merge!(key => values.delete(key))
+        end
+        { Esse::SETTING_ROOT_KEY => values }
       end
 
       # Define /_settings definition by each index.
@@ -18,7 +37,7 @@ module Esse
       #
       #   class UserIndex < Esse::Index
       #     settings {
-      #       number_of_replicas: 4,
+      #       index: { number_of_replicas: 4 }
       #     }
       #   end
       #

data/lib/esse/lazy_document_header.rb

@@ -2,11 +2,19 @@
 
 module Esse
   class LazyDocumentHeader
+    ACCEPTABLE_CLASSES = [Esse::LazyDocumentHeader, Esse::Document].freeze
+    ACCEPTABLE_DOC_TYPES = [nil, '_doc', 'doc'].freeze
+
     def self.coerce_each(values)
+      values = Esse::ArrayUtils.wrap(values)
+      return values if values.all? do |value|
+        ACCEPTABLE_CLASSES.any? { |klass| value.is_a?(klass) }
+      end
+
       arr = []
-      Esse::ArrayUtils.wrap(values).map do |value|
+      values.flatten.map do |value|
         instance = coerce(value)
-        arr << instance if instance&.valid?
+        arr << instance if instance && !instance.id.nil?
       end
       arr
     end
@@ -17,57 +25,61 @@ module Esse
       if value.is_a?(Esse::LazyDocumentHeader)
         value
       elsif value.is_a?(Esse::Document)
-        new(value.doc_header)
+        value
       elsif value.is_a?(Hash)
         resp = value.transform_keys do |key|
           case key
           when :_id, :id, '_id', 'id'
-            :_id
+            :id
           when :_routing, :routing, '_routing', 'routing'
             :routing
           when :_type, :type, '_type', 'type'
-            :_type
+            :type
           else
             key.to_sym
           end
         end
-        new(resp)
+        resp[:id] ||= nil
+        new(**resp)
       elsif String === value || Integer === value
-        new(_id: value)
+        new(id: value)
       end
     end
 
-    def initialize(attributes)
-      @attributes = attributes
-    end
+    attr_reader :id, :type, :routing, :options
 
-    def valid?
-      !@attributes[:_id].nil?
+    def initialize(id:, type: nil, routing: nil, **extra_attributes)
+      @id = id
+      @type = type
+      @routing = routing
+      @options = extra_attributes.freeze
     end
 
     def to_h
-      @attributes
-    end
-
-    def id
-      @attributes.fetch(:_id)
-    end
-
-    def type
-      @attributes[:_type]
+      options.merge(_id: id).tap do |hash|
+        hash[:_type] = type if type
+        hash[:routing] = routing if routing
+      end
     end
 
-    def routing
-      @attributes[:routing]
+    def document_for_partial_update(source)
+      Esse::DocumentForPartialUpdate.new(self, source: source)
     end
 
-    def to_doc(source = {})
-      HashDocument.new(source.merge(@attributes))
+    def doc_header
+      { _id: id }.tap do |hash|
+        hash[:_type] = type if type
+        hash[:routing] = routing if routing
+      end
     end
 
-    def eql?(other)
-      self.class == other.class && @attributes == other.instance_variable_get(:@attributes)
+    def eql?(other, **)
+      ACCEPTABLE_CLASSES.any? { |klass| other.is_a?(klass) } &&
+        id.to_s == other.id.to_s &&
+        routing == other.routing &&
+        ((ACCEPTABLE_DOC_TYPES.include?(type) && ACCEPTABLE_DOC_TYPES.include?(other.type)) || type == other.type)
     end
     alias_method :==, :eql?
   end
 end
+

data/lib/esse/primitives/hash_utils.rb

@@ -47,5 +47,15 @@ module Esse
         end
       end
     end
+
+    def explode_keys(hash, separator = '.')
+      hash.each_with_object({}) do |(key, value), result|
+        is_symbol = key.is_a?(Symbol)
+        keys = key.to_s.split(separator)
+        last_key = keys.pop
+        current = keys.reduce(result) { |memo, k| memo[is_symbol ? k.to_sym : k] ||= {} }
+        current[is_symbol ? last_key.to_sym : last_key] = value
+      end
+    end
   end
 end

data/lib/esse/repository/documents.rb

@@ -16,7 +16,7 @@ module Esse
 
       def documents_for_lazy_attribute(name, ids_or_doc_headers)
         retrieve_lazy_attribute_values(name, ids_or_doc_headers).map do |doc_header, datum|
-          doc_header.to_doc(name => datum)
+          doc_header.document_for_partial_update(name => datum)
         end
       end
 
@@ -36,11 +36,10 @@ module Esse
         return [] unless result.is_a?(Hash)
 
         result.each_with_object({}) do |(key, value), memo|
-          if key.is_a?(LazyDocumentHeader) && (doc = docs.find { |d| d == key || d.id == key.id })
-            memo[doc] = value
-          elsif (doc = docs.find { |d| d.id == key })
-            memo[doc] = value
-          end
+          val = docs.find { |doc| doc.eql?(key, match_lazy_doc_header: true) || doc.id == key }
+          next unless val
+
+          memo[val] = value
         end
       end
     end

data/lib/esse/repository/object_document_mapper.rb

@@ -69,27 +69,65 @@ module Esse
         @collection_proc = collection_klass || block
       end
 
+      # Expose the collection class to let external plugins and extensions to access it.
+      # @return [Class, nil] The collection class
+      # IDEA: When collection is defined as a block, it should setup a class with the block content.
+      def collection_class
+        return unless @collection_proc.is_a?(Class)
+
+        @collection_proc
+      end
+
       # Wrap collection data into serialized batches
       #
       # @param [Hash] kwargs The context
       # @return [Enumerator] The enumerator
       # @yield [Array, **context] serialized collection and the optional context from the collection
-      def each_serialized_batch(lazy_attributes: false, **kwargs)
+      def each_serialized_batch(eager_load_lazy_attributes: false, preload_lazy_attributes: false, **kwargs)
+        if kwargs.key?(:lazy_attributes)
+          warn 'The `lazy_attributes` option is deprecated. Use `eager_load_lazy_attributes` instead.'
+          eager_load_lazy_attributes = kwargs.delete(:lazy_attributes)
+        end
+
+        lazy_attrs_to_eager_load = lazy_document_attribute_names(eager_load_lazy_attributes)
+        lazy_attrs_to_search_preload = lazy_document_attribute_names(preload_lazy_attributes)
+        lazy_attrs_to_search_preload -= lazy_attrs_to_eager_load
+
         each_batch(**kwargs) do |*args|
           batch, collection_context = args
           collection_context ||= {}
           entries = [*batch].map { |entry| serialize(entry, **collection_context) }.compact
-          if lazy_attributes
-            attrs = lazy_attributes.is_a?(Array) ? lazy_attributes : lazy_document_attribute_names(lazy_attributes)
-            attrs.each do |attr_name|
-              retrieve_lazy_attribute_values(attr_name, entries).each do |doc_header, value|
-                doc = entries.find { |d| doc_header.id.to_s == d.id.to_s && doc_header.type == d.type && doc_header.routing == d.routing }
-                doc&.mutate(attr_name) { value }
+          lazy_attrs_to_eager_load.each do |attr_name|
+            retrieve_lazy_attribute_values(attr_name, entries).each do |doc_header, value|
+              doc = entries.find { |d| d.eql?(doc_header, match_lazy_doc_header: true) }
+              doc&.mutate(attr_name) { value }
+            end
+          end
+
+          if lazy_attrs_to_search_preload.any?
+            entries.group_by(&:routing).each do |routing, docs|
+              search_request = {
+                query: { ids: { values: docs.map(&:id) } },
+                size: docs.size,
+                _source: lazy_attrs_to_search_preload
+              }
+              search_request[:routing] = routing if routing
+              index.search(**search_request).response.hits.each do |hit|
+                header = [hit['_id'], hit['_routing'], hit['_type']]
+                next if header[0].nil?
+
+                hit.dig('_source')&.each do |attr_name, attr_value|
+                  real_attr_name = lazy_document_attribute_names(attr_name).first
+                  next if real_attr_name.nil?
+
+                  doc = entries.find { |d| Esse.document_match_with_header?(d, *header) }
+                  doc&.mutate(real_attr_name) { attr_value }
+                end
               end
             end
           end
 
-          yield entries, **kwargs
+          yield entries
         end
       end
 
@@ -101,7 +139,7 @@ module Esse
       # @return [Enumerator] All serialized entries
       def documents(**kwargs)
         Enumerator.new do |yielder|
-          each_serialized_batch(**kwargs) do |docs, **_collection_kargs|
+          each_serialized_batch(**kwargs) do |docs|
             docs.each { |document| yielder.yield(document) }
           end
         end

data/lib/esse/transport/indices.rb

@@ -185,6 +185,83 @@ module Esse
           payload[:response] = coerce_exception { client.indices.put_settings(**opts) }
         end
       end
+
+      # Allows to copy documents from one index to another, optionally filtering the source
+      # documents by a query, changing the destination index settings, or fetching the
+      # documents from a remote cluster.
+      #
+      # @option arguments [Boolean] :refresh Should the affected indexes be refreshed?
+      # @option arguments [Time] :timeout Time each individual bulk request should wait for shards that are unavailable.
+      # @option arguments [String] :wait_for_active_shards Sets the number of shard copies that must be active before proceeding with the reindex operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)
+      # @option arguments [Boolean] :wait_for_completion Should the request should block until the reindex is complete.
+      # @option arguments [Number] :requests_per_second The throttle to set on this request in sub-requests per second. -1 means no throttle.
+      # @option arguments [Time] :scroll Control how long to keep the search context alive
+      # @option arguments [Number|string] :slices The number of slices this task should be divided into. Defaults to 1, meaning the task isn't sliced into subtasks. Can be set to `auto`.
+      # @option arguments [Number] :max_docs Maximum number of documents to process (default: all documents)
+      # @option arguments [Hash] :headers Custom HTTP headers
+      # @option arguments [Hash] :body The search definition using the Query DSL and the prototype for the index request. (*Required*)
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html
+      def reindex(body:, **options)
+        throw_error_when_readonly!
+
+        Esse::Events.instrument('elasticsearch.reindex') do |payload|
+          payload[:request] = opts = options.merge(body: body)
+          payload[:response] = coerce_exception { client.reindex(**opts) }
+        end
+      end
+
+      # Performs an update on every document in the index without changing the source,
+      # for example to pick up a mapping change.
+      #
+      # @option arguments [List] :index A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices (*Required*)
+      # @option arguments [String] :analyzer The analyzer to use for the query string
+      # @option arguments [Boolean] :analyze_wildcard Specify whether wildcard and prefix queries should be analyzed (default: false)
+      # @option arguments [String] :default_operator The default operator for query string query (AND or OR) (options: AND, OR)
+      # @option arguments [String] :df The field to use as default where no field prefix is given in the query string
+      # @option arguments [Number] :from Starting offset (default: 0)
+      # @option arguments [Boolean] :ignore_unavailable Whether specified concrete indices should be ignored when unavailable (missing or closed)
+      # @option arguments [Boolean] :allow_no_indices Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)
+      # @option arguments [String] :conflicts What to do when the update by query hits version conflicts? (options: abort, proceed)
+      # @option arguments [String] :expand_wildcards Whether to expand wildcard expression to concrete indices that are open, closed or both. (options: open, closed, hidden, none, all)
+      # @option arguments [Boolean] :lenient Specify whether format-based query failures (such as providing text to a numeric field) should be ignored
+      # @option arguments [String] :pipeline Ingest pipeline to set on index requests made by this action. (default: none)
+      # @option arguments [String] :preference Specify the node or shard the operation should be performed on (default: random)
+      # @option arguments [String] :q Query in the Lucene query string syntax
+      # @option arguments [List] :routing A comma-separated list of specific routing values
+      # @option arguments [Time] :scroll Specify how long a consistent view of the index should be maintained for scrolled search
+      # @option arguments [String] :search_type Search operation type (options: query_then_fetch, dfs_query_then_fetch)
+      # @option arguments [Time] :search_timeout Explicit timeout for each search request. Defaults to no timeout.
+      # @option arguments [Number] :size Deprecated, please use `max_docs` instead
+      # @option arguments [Number] :max_docs Maximum number of documents to process (default: all documents)
+      # @option arguments [List] :sort A comma-separated list of <field>:<direction> pairs
+      # @option arguments [List] :_source True or false to return the _source field or not, or a list of fields to return
+      # @option arguments [List] :_source_excludes A list of fields to exclude from the returned _source field
+      # @option arguments [List] :_source_includes A list of fields to extract and return from the _source field
+      # @option arguments [Number] :terminate_after The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early.
+      # @option arguments [List] :stats Specific 'tag' of the request for logging and statistical purposes
+      # @option arguments [Boolean] :version Specify whether to return document version as part of a hit
+      # @option arguments [Boolean] :version_type Should the document increment the version number (internal) on hit or not (reindex)
+      # @option arguments [Boolean] :request_cache Specify if request cache should be used for this request or not, defaults to index level setting
+      # @option arguments [Boolean] :refresh Should the affected indexes be refreshed?
+      # @option arguments [Time] :timeout Time each individual bulk request should wait for shards that are unavailable.
+      # @option arguments [String] :wait_for_active_shards Sets the number of shard copies that must be active before proceeding with the update by query operation. Defaults to 1, meaning the primary shard only. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)
+      # @option arguments [Number] :scroll_size Size on the scroll request powering the update by query
+      # @option arguments [Boolean] :wait_for_completion Should the request should block until the update by query operation is complete.
+      # @option arguments [Number] :requests_per_second The throttle to set on this request in sub-requests per second. -1 means no throttle.
+      # @option arguments [Number|string] :slices The number of slices this task should be divided into. Defaults to 1, meaning the task isn't sliced into subtasks. Can be set to `auto`.
+      # @option arguments [Hash] :headers Custom HTTP headers
+      # @option arguments [Hash] :body The search definition using the Query DSL
+      #
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html
+      def update_by_query(index:, **options)
+        throw_error_when_readonly!
+
+        Esse::Events.instrument('elasticsearch.update_by_query') do |payload|
+          payload[:request] = opts = options.merge(index: index)
+          payload[:response] = coerce_exception { client.update_by_query(**opts) }
+        end
+      end
     end
 
     include InstanceMethods

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.3.4'
+  VERSION = '0.4.0.rc1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0.rc1
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2024-07-18 00:00:00.000000000 Z
+date: 2024-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -248,6 +248,7 @@ files:
 - lib/esse/deprecations/repository_backend_delegator.rb
 - lib/esse/deprecations/serializer.rb
 - lib/esse/document.rb
+- lib/esse/document_for_partial_update.rb
 - lib/esse/document_lazy_attribute.rb
 - lib/esse/dynamic_template.rb
 - lib/esse/errors.rb
@@ -318,9 +319,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3.1
 signing_key: