esse 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fc6f63594da661b35bc22ab0c83655ded02b8ebe2e20ec5612a0927934156e9
-  data.tar.gz: c46f9a70e262d40a9c60c7ce72be3e3b816d18c536f6e3fe770bd06c06c5f57b
+  metadata.gz: 15b531b7c6876d665c12fd773d25fa68acda30c4c92cf7999790315f4fe35b2d
+  data.tar.gz: 9520b76a128e752bed3db6d2a1a96e38e2107ca790c6d0c0327c2690d91b21d7
 SHA512:
-  metadata.gz: 8525a1ba74d9452163e87e1acf1bf621654f2bb8b9a7811e6eca94fc452c47f9255808b2c45be701112be085f8d3176243a685fcfb6647a8e9845699a727911d
-  data.tar.gz: 3d0be64e631a47c07758341b34fc86eedffd76bbd12c4734016d9303c256d9c8470929fc564bdace28bb21f072e44d0193cbee06e8e808675a695b46aac92c38
+  metadata.gz: 865aa67f404a82e573009e73186dd60ee9c3f978ba3c93b39337184c2b8c9c9415bc116573c5eb4f21cb7e4ffc7e19aae4957e54fe7b8aa99a8804e86469b13f
+  data.tar.gz: bcacb23ae7b1b1a073298a59090455e30d6aa9531b93a815871762ed9e3364de873795c47ca0b2d495d8d9394110fd623011d3e4e3054a34ddf89b5e35947ded

data/lib/esse/import/bulk.rb

@@ -13,13 +13,16 @@ module Esse
       # In case of timeout error, will retry with an exponential backoff using the following formula:
       #  wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1)) seconds. It will retry up to max_retries times that is default 4.
       #
-      # Too large bulk requests will be split into multiple requests with only one attempt.
+      # Too large bulk requests will first be split into multiple size-balanced requests; if that still
+      # returns 413, the bulk is retried one document per request as a last resort. Only after a single
+      # document still returns 413 does the error bubble up.
       #
       # @yield [RequestBody] A request body instance
-      def each_request(max_retries: 4, last_retry_in_small_chunks: true)
+      def each_request(max_retries: 4, last_retry_in_small_chunks: true, last_retry_per_document: true)
         # @TODO create indexes when by checking all the index suffixes (if mapping is not empty)
         requests = [optimistic_request]
         retry_count = 0
+        too_large_retry_count = 0
 
         begin
           requests.each do |request|
@@ -37,12 +40,28 @@ module Esse
           sleep(wait_interval)
           retry
         rescue Esse::Transport::RequestEntityTooLargeError => e
-          retry_count += 1
-          raise e if retry_count > 1 # only retry once on this error
-          requests = balance_requests_size(e)
+          too_large_retry_count += 1
+          raise e if too_large_retry_count > 2
+
+          if too_large_retry_count == 1
+            balanced = balance_requests_size(e)
+            if balanced && !balanced.empty?
+              requests = balanced
+              Esse.logger.warn <<~MSG
+                Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(' + ')}.
+                Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+              MSG
+              retry
+            end
+            raise e unless last_retry_per_document
+            too_large_retry_count = 2
+          end
+
+          raise e unless last_retry_per_document
+          requests = requests_per_document
           Esse.logger.warn <<~MSG
-            Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(' + ')}.
-            Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+            Request entity too large after balancing, retrying one document per request as a last resort.
+            If a single document still exceeds the bulk size, the error will be raised.
           MSG
           retry
         end
@@ -60,45 +79,57 @@ module Esse
       end
 
       def requests_in_small_chunks(chunk_size: 1)
+        arr = build_per_document_requests(chunk_size: chunk_size)
+        Esse.logger.warn <<~MSG
+          Retrying the last request in small chunks of #{chunk_size} documents.
+          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
+        MSG
+        arr
+      end
+
+      def requests_per_document
+        build_per_document_requests(chunk_size: 1)
+      end
+
+      def build_per_document_requests(chunk_size: 1)
         arr = []
         @create.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.create = slice } }
         @index.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.index = slice } }
         @update.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.update = slice } }
         @delete.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.delete = slice } }
-        Esse.logger.warn <<~MSG
-          Retrying the last request in small chunks of #{chunk_size} documents.
-          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
-        MSG
         arr
       end
 
-      # @return [Array<RequestBody>]
+      # @return [Array<RequestBody>, nil] balanced requests, or nil when the error message has no parseable byte limit
       def balance_requests_size(err)
-        if (bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i) > 0
-          requests = (@create + @index + @update + @delete).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
-            operation, meta = as_json.to_a.first
-            meta = meta.dup
-            data = meta.delete(:data)
-            piece = MultiJson.dump(operation => meta)
-            piece << "\n" << MultiJson.dump(data) if data
-            if piece.bytesize > bulk_size
-              Esse.logger.warn <<~MSG
-                The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
-                Consider increasing the bulk size or reducing the document size. The document will be ignored during this import.
-              MSG
-              next
-            end
+        bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i
+        return nil unless bulk_size > 0
 
-            if result.last.body.bytesize + piece.bytesize > bulk_size
-              result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
-            else
-              result[-1].add(operation, piece)
-            end
+        requests = (@create + @index + @update + @delete).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
+          operation, meta = as_json.to_a.first
+          meta = meta.dup
+          data = meta.delete(:data)
+          piece = MultiJson.dump(operation => meta)
+          piece << "\n" << MultiJson.dump(data) if data
+
+          if piece.bytesize > bulk_size
+            Esse.logger.warn <<~MSG
+              The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
+              It will be sent in its own request; if the cluster rejects it, the error will be raised.
+            MSG
+            result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+            result.push(Import::RequestBodyRaw.new)
+            next
+          end
+
+          if result.last.body.bytesize + piece.bytesize > bulk_size
+            result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+          else
+            result[-1].add(operation, piece)
           end
-          requests.each(&:finalize)
-        else
-          raise err
         end
+        requests.reject! { |r| r.body.empty? }
+        requests.each(&:finalize)
       end
     end
   end

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.4.1'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2026-04-29 00:00:00.000000000 Z
+date: 2026-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json