esse 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc6a554b787bfb51e5568bd590790171cc88ad8c5ce432e1de2c3ef68a5d1def
-  data.tar.gz: cac82a25d6e8dd6dad3667a734d58be098f4b7020db76472359f7bdb4fe2067c
+  metadata.gz: 15b531b7c6876d665c12fd773d25fa68acda30c4c92cf7999790315f4fe35b2d
+  data.tar.gz: 9520b76a128e752bed3db6d2a1a96e38e2107ca790c6d0c0327c2690d91b21d7
 SHA512:
-  metadata.gz: f26afad34715ed37cec3f1d1cad11ca6e05d2921a6f31884d9ea9f1136ef239ba7661e29796c37abf69ea19aa5f5407884cf88b108bbfa211e5639e371d492ed
-  data.tar.gz: 13b1921d85723d3c33bacc54c7bcf95885a196c3a62c095a5ea3be5a5d5061b8cd4eb8fbb1bdbf1bb91c141ed4457f9a1baab5ec121dd7e4f4dc1c1c423277d2
+  metadata.gz: 865aa67f404a82e573009e73186dd60ee9c3f978ba3c93b39337184c2b8c9c9415bc116573c5eb4f21cb7e4ffc7e19aae4957e54fe7b8aa99a8804e86469b13f
+  data.tar.gz: bcacb23ae7b1b1a073298a59090455e30d6aa9531b93a815871762ed9e3364de873795c47ca0b2d495d8d9394110fd623011d3e4e3054a34ddf89b5e35947ded

data/lib/esse/import/bulk.rb

@@ -13,13 +13,16 @@ module Esse
       # In case of timeout error, will retry with an exponential backoff using the following formula:
       #  wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1)) seconds. It will retry up to max_retries times that is default 4.
       #
-      # Too large bulk requests will be split into multiple requests with only one attempt.
+      # Too large bulk requests will first be split into multiple size-balanced requests; if that still
+      # returns 413, the bulk is retried one document per request as a last resort. Only after a single
+      # document still returns 413 does the error bubble up.
       #
       # @yield [RequestBody] A request body instance
-      def each_request(max_retries: 4, last_retry_in_small_chunks: true)
+      def each_request(max_retries: 4, last_retry_in_small_chunks: true, last_retry_per_document: true)
         # @TODO create indexes when by checking all the index suffixes (if mapping is not empty)
         requests = [optimistic_request]
         retry_count = 0
+        too_large_retry_count = 0
 
         begin
           requests.each do |request|
@@ -37,12 +40,28 @@ module Esse
           sleep(wait_interval)
           retry
         rescue Esse::Transport::RequestEntityTooLargeError => e
-          retry_count += 1
-          raise e if retry_count > 1 # only retry once on this error
-          requests = balance_requests_size(e)
+          too_large_retry_count += 1
+          raise e if too_large_retry_count > 2
+
+          if too_large_retry_count == 1
+            balanced = balance_requests_size(e)
+            if balanced && !balanced.empty?
+              requests = balanced
+              Esse.logger.warn <<~MSG
+                Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(' + ')}.
+                Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+              MSG
+              retry
+            end
+            raise e unless last_retry_per_document
+            too_large_retry_count = 2
+          end
+
+          raise e unless last_retry_per_document
+          requests = requests_per_document
           Esse.logger.warn <<~MSG
-            Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(' + ')}.
-            Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+            Request entity too large after balancing, retrying one document per request as a last resort.
+            If a single document still exceeds the bulk size, the error will be raised.
           MSG
           retry
         end
@@ -60,45 +79,57 @@ module Esse
       end
 
       def requests_in_small_chunks(chunk_size: 1)
+        arr = build_per_document_requests(chunk_size: chunk_size)
+        Esse.logger.warn <<~MSG
+          Retrying the last request in small chunks of #{chunk_size} documents.
+          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
+        MSG
+        arr
+      end
+
+      def requests_per_document
+        build_per_document_requests(chunk_size: 1)
+      end
+
+      def build_per_document_requests(chunk_size: 1)
         arr = []
         @create.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.create = slice } }
         @index.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.index = slice } }
         @update.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.update = slice } }
         @delete.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.delete = slice } }
-        Esse.logger.warn <<~MSG
-          Retrying the last request in small chunks of #{chunk_size} documents.
-          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
-        MSG
         arr
       end
 
-      # @return [Array<RequestBody>]
+      # @return [Array<RequestBody>, nil] balanced requests, or nil when the error message has no parseable byte limit
       def balance_requests_size(err)
-        if (bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i) > 0
-          requests = (@create + @index + @update + @delete).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
-            operation, meta = as_json.to_a.first
-            meta = meta.dup
-            data = meta.delete(:data)
-            piece = MultiJson.dump(operation => meta)
-            piece << "\n" << MultiJson.dump(data) if data
-            if piece.bytesize > bulk_size
-              Esse.logger.warn <<~MSG
-                The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
-                Consider increasing the bulk size or reducing the document size. The document will be ignored during this import.
-              MSG
-              next
-            end
+        bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i
+        return nil unless bulk_size > 0
 
-            if result.last.body.bytesize + piece.bytesize > bulk_size
-              result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
-            else
-              result[-1].add(operation, piece)
-            end
+        requests = (@create + @index + @update + @delete).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
+          operation, meta = as_json.to_a.first
+          meta = meta.dup
+          data = meta.delete(:data)
+          piece = MultiJson.dump(operation => meta)
+          piece << "\n" << MultiJson.dump(data) if data
+
+          if piece.bytesize > bulk_size
+            Esse.logger.warn <<~MSG
+              The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
+              It will be sent in its own request; if the cluster rejects it, the error will be raised.
+            MSG
+            result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+            result.push(Import::RequestBodyRaw.new)
+            next
+          end
+
+          if result.last.body.bytesize + piece.bytesize > bulk_size
+            result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+          else
+            result[-1].add(operation, piece)
           end
-          requests.each(&:finalize)
-        else
-          raise err
         end
+        requests.reject! { |r| r.body.empty? }
+        requests.each(&:finalize)
       end
     end
   end

data/lib/esse/index/settings.rb

@@ -4,29 +4,21 @@ module Esse
   # https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/indices/put_settings.rb
   class Index
     module ClassMethods
-      # Elasticsearch supports passing index.* related settings directly in the body of the request.
-      # We are moving it to the index key to make it more explicit and to be the source-of-truth when merging settings.
-      # So the settings `{ number_of_shards: 1 }` will be transformed to `{ index: { number_of_shards: 1 } }`
-      INDEX_SIMPLIFIED_SETTINGS = %i[
-        number_of_shards
-        number_of_replicas
-        refresh_interval
-        mapping
-      ].freeze
+      # Backwards-compatible alias. The canonical list now lives on
+      # +Esse::IndexSetting::INDEX_SIMPLIFIED_SETTINGS+ so that the merge
+      # logic and the simplified-key promotion stay in sync.
+      INDEX_SIMPLIFIED_SETTINGS = Esse::IndexSetting::INDEX_SIMPLIFIED_SETTINGS
 
       def settings_hash(settings: nil)
-        hash = setting.body
-        values = (hash.key?(Esse::SETTING_ROOT_KEY) ? hash[Esse::SETTING_ROOT_KEY] : hash)
-        values = HashUtils.explode_keys(values)
-        if settings.is_a?(Hash)
-          values = HashUtils.deep_merge(values, HashUtils.explode_keys(settings))
-        end
-        INDEX_SIMPLIFIED_SETTINGS.each do |key|
-          next unless values.key?(key)
-          value = values.delete(key)
-          next if value.nil?
+        # Normalize each side (global vs local) separately before merging so
+        # a flat global key (e.g. top-level :number_of_shards) cannot clobber
+        # an explicit nested local value (e.g. :index => { :number_of_shards => 8 }).
+        global = Esse::IndexSetting.normalize(setting.globals)
+        local = Esse::IndexSetting.normalize(setting.to_h)
+        values = HashUtils.deep_merge(global, local)
 
-          (values[:index] ||= {}).merge!(key => value)
+        if settings.is_a?(Hash)
+          values = HashUtils.deep_merge(values, Esse::IndexSetting.normalize(settings))
         end
 
         if values[:index].is_a?(Hash)

data/lib/esse/index_setting.rb

@@ -3,6 +3,17 @@
 module Esse
   # https://www.elastic.co/guide/en/elasticsearch/reference/1.7/indices.html
   class IndexSetting
+    # Top-level keys that Elasticsearch/OpenSearch accept either flat or nested
+    # under `index:`. We always promote them to the nested form so that values
+    # from different sources (cluster globals vs per-index template) merge
+    # predictably regardless of which form each side was authored in.
+    INDEX_SIMPLIFIED_SETTINGS = %i[
+      number_of_shards
+      number_of_replicas
+      refresh_interval
+      mapping
+    ].freeze
+
     # @param [Hash] options
     # @option options [Proc] :globals  A proc that will be called to load global settings
     # @option options [Array] :paths   A list of paths to load settings from
@@ -35,6 +46,38 @@ module Esse
       HashUtils.deep_merge(global, local)
     end
 
+    # Returns the raw (unsymbolized) global settings as supplied by the
+    # +globals+ proc. Public so that callers like
+    # +Esse::Index.settings_hash+ can normalize it independently before
+    # merging it with the local template — preventing a flat global value
+    # from clobbering a nested local value once both are merged.
+    def globals
+      @globals.call || {}
+    end
+
+    # Normalize a settings hash by:
+    #   * symbolizing keys
+    #   * stripping the `:settings` root if present
+    #   * exploding dotted keys ('index.number_of_replicas' -> { index: { number_of_replicas: ... } })
+    #   * promoting simplified flat keys (number_of_shards, etc.) into the
+    #     nested `:index` form, preserving any value already present under
+    #     `:index` (we never overwrite an explicit nested setting with a
+    #     flat value from the same source).
+    def self.normalize(hash)
+      values = HashUtils.deep_transform_keys(hash || {}, &:to_sym)
+      values = values[Esse::SETTING_ROOT_KEY] if values.key?(Esse::SETTING_ROOT_KEY)
+      values = HashUtils.explode_keys(values)
+      INDEX_SIMPLIFIED_SETTINGS.each do |key|
+        next unless values.key?(key)
+        value = values.delete(key)
+        next if value.nil?
+
+        values[:index] ||= {}
+        values[:index][key] = value unless values[:index].key?(key)
+      end
+      values
+    end
+
     protected
 
     def from_template

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2026-03-18 00:00:00.000000000 Z
+date: 2026-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json