esse 0.4.1 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fc6f63594da661b35bc22ab0c83655ded02b8ebe2e20ec5612a0927934156e9
-  data.tar.gz: c46f9a70e262d40a9c60c7ce72be3e3b816d18c536f6e3fe770bd06c06c5f57b
+  metadata.gz: d5aa2b7107d8bc92810414bacbca1c821024e8771b50be5f5c0afb4b66349874
+  data.tar.gz: 7ea6f01e96dc3e7a260f4b2cc8198f88231920dbd3e96a1d2493df34eb1bfeaa
 SHA512:
-  metadata.gz: 8525a1ba74d9452163e87e1acf1bf621654f2bb8b9a7811e6eca94fc452c47f9255808b2c45be701112be085f8d3176243a685fcfb6647a8e9845699a727911d
-  data.tar.gz: 3d0be64e631a47c07758341b34fc86eedffd76bbd12c4734016d9303c256d9c8470929fc564bdace28bb21f072e44d0193cbee06e8e808675a695b46aac92c38
+  metadata.gz: bce16d5dd68a36e0f75215994a96746982d84529fddd0f667660ca644cf8ca3c89c79211db91499161323c025528a48e2b371bddf743a4e19de4a95988121196
+  data.tar.gz: e87a03b8fc0c10fcbbfb7945483c740af2958a2991a1ea5330503f41ca8390988110dbddce59a62cb61987cf020b9f24bb4a02c3770d5dc2bed56bf124163226

data/lib/esse/config.rb

@@ -43,7 +43,7 @@ module Esse
   #   end
   class Config
     DEFAULT_CLUSTER_ID = :default
-    ATTRIBUTES = %i[indices_directory bulk_wait_interval].freeze
+    ATTRIBUTES = %i[indices_directory bulk_wait_interval bulk_retry_on_failure_max_retries bulk_retry_on_failure_wait].freeze
 
     # The location of the indices. Defaults to the `app/indices`
     attr_reader :indices_directory
@@ -51,9 +51,17 @@ module Esse
     # wait a given period between posting pages to give Elasticsearch time to catch up.
     attr_reader :bulk_wait_interval
 
+    # number of retries on transient server errors (502, 503, 504, 429) before raising
+    attr_reader :bulk_retry_on_failure_max_retries
+
+    # base wait in seconds between transient-error retries (doubles each attempt)
+    attr_reader :bulk_retry_on_failure_wait
+
     def initialize
       self.indices_directory = 'app/indices'
       self.bulk_wait_interval = 0.1
+      self.bulk_retry_on_failure_max_retries = 3
+      self.bulk_retry_on_failure_wait = 2.0
       @clusters = {}
       cluster(DEFAULT_CLUSTER_ID) # initialize the :default client
     end
@@ -81,6 +89,14 @@ module Esse
       @bulk_wait_interval = value.to_f
     end
 
+    def bulk_retry_on_failure_max_retries=(value)
+      @bulk_retry_on_failure_max_retries = value.to_i
+    end
+
+    def bulk_retry_on_failure_wait=(value)
+      @bulk_retry_on_failure_wait = value.to_f
+    end
+
     def load(arg)
       case arg
       when Hash

data/lib/esse/errors.rb

@@ -57,6 +57,7 @@ module Esse
       'ExpectationFailed' => 'ExpectationFailedError', # 417
       'ImATeapot' => 'ImATeapotError', # 418
       'TooManyConnectionsFromThisIP' => 'TooManyConnectionsFromThisIPError', # 421
+      'TooManyRequests' => 'TooManyRequestsError', # 429
       'UpgradeRequired' => 'UpgradeRequiredError', # 426
       'BlockedByWindowsParentalControls' => 'BlockedByWindowsParentalControlsError', # 450
       'RequestHeaderTooLarge' => 'RequestHeaderTooLargeError', # 494

data/lib/esse/import/bulk.rb

@@ -13,13 +13,18 @@ module Esse
       # In case of timeout error, will retry with an exponential backoff using the following formula:
       #  wait_interval = (retry_count**4) + 15 + (rand(10) * (retry_count + 1)) seconds. It will retry up to max_retries times that is default 4.
       #
-      # Too large bulk requests will be split into multiple requests with only one attempt.
+      # Too large bulk requests will first be split into multiple size-balanced requests; if that still
+      # returns 413, the bulk is retried one document per request as a last resort. Only after a single
+      # document still returns 413 does the error bubble up.
       #
       # @yield [RequestBody] A request body instance
-      def each_request(max_retries: 4, last_retry_in_small_chunks: true)
+      def each_request(max_retries: 4, last_retry_in_small_chunks: true, last_retry_per_document: true,
+                       retry_on_failure_max_retries: 3, retry_on_failure_wait: 2.0)
         # @TODO create indexes when by checking all the index suffixes (if mapping is not empty)
         requests = [optimistic_request]
         retry_count = 0
+        too_large_retry_count = 0
+        transient_failure_count = 0
 
         begin
           requests.each do |request|
@@ -37,14 +42,40 @@ module Esse
           sleep(wait_interval)
           retry
         rescue Esse::Transport::RequestEntityTooLargeError => e
-          retry_count += 1
-          raise e if retry_count > 1 # only retry once on this error
-          requests = balance_requests_size(e)
+          too_large_retry_count += 1
+          raise e if too_large_retry_count > 2
+
+          if too_large_retry_count == 1
+            balanced = balance_requests_size(e)
+            if balanced && !balanced.empty?
+              requests = balanced
+              Esse.logger.warn <<~MSG
+                Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(' + ')}.
+                Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+              MSG
+              retry
+            end
+            raise e unless last_retry_per_document
+            too_large_retry_count = 2
+          end
+
+          raise e unless last_retry_per_document
+          requests = requests_per_document
           Esse.logger.warn <<~MSG
-            Request entity too large, retrying with a bulk with: #{requests.map(&:bytesize).join(' + ')}.
-            Note that this cause performance degradation, consider adjusting the batch_size of the index or increasing the bulk size.
+            Request entity too large after balancing, retrying one document per request as a last resort.
+            If a single document still exceeds the bulk size, the error will be raised.
           MSG
           retry
+        rescue Esse::Transport::BadGatewayError,
+               Esse::Transport::ServiceUnavailableError,
+               Esse::Transport::GatewayTimeoutError,
+               Esse::Transport::TooManyRequestsError => e
+          transient_failure_count += 1
+          raise e if transient_failure_count >= retry_on_failure_max_retries
+          wait = retry_on_failure_wait * (2**(transient_failure_count - 1))
+          Esse.logger.warn "#{e.class} error, retrying in #{wait}s (attempt #{transient_failure_count}/#{retry_on_failure_max_retries})"
+          sleep(wait)
+          retry
         end
       end
 
@@ -60,45 +91,57 @@ module Esse
       end
 
       def requests_in_small_chunks(chunk_size: 1)
+        arr = build_per_document_requests(chunk_size: chunk_size)
+        Esse.logger.warn <<~MSG
+          Retrying the last request in small chunks of #{chunk_size} documents.
+          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
+        MSG
+        arr
+      end
+
+      def requests_per_document
+        build_per_document_requests(chunk_size: 1)
+      end
+
+      def build_per_document_requests(chunk_size: 1)
         arr = []
         @create.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.create = slice } }
         @index.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.index = slice } }
         @update.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.update = slice } }
         @delete.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.delete = slice } }
-        Esse.logger.warn <<~MSG
-          Retrying the last request in small chunks of #{chunk_size} documents.
-          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
-        MSG
         arr
       end
 
-      # @return [Array<RequestBody>]
+      # @return [Array<RequestBody>, nil] balanced requests, or nil when the error message has no parseable byte limit
       def balance_requests_size(err)
-        if (bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i) > 0
-          requests = (@create + @index + @update + @delete).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
-            operation, meta = as_json.to_a.first
-            meta = meta.dup
-            data = meta.delete(:data)
-            piece = MultiJson.dump(operation => meta)
-            piece << "\n" << MultiJson.dump(data) if data
-            if piece.bytesize > bulk_size
-              Esse.logger.warn <<~MSG
-                The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
-                Consider increasing the bulk size or reducing the document size. The document will be ignored during this import.
-              MSG
-              next
-            end
+        bulk_size = err.message.scan(/exceeded.(\d+).bytes/).dig(0, 0).to_i
+        return nil unless bulk_size > 0
 
-            if result.last.body.bytesize + piece.bytesize > bulk_size
-              result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
-            else
-              result[-1].add(operation, piece)
-            end
+        requests = (@create + @index + @update + @delete).each_with_object([Import::RequestBodyRaw.new]) do |as_json, result|
+          operation, meta = as_json.to_a.first
+          meta = meta.dup
+          data = meta.delete(:data)
+          piece = MultiJson.dump(operation => meta)
+          piece << "\n" << MultiJson.dump(data) if data
+
+          if piece.bytesize > bulk_size
+            Esse.logger.warn <<~MSG
+              The document #{meta.inspect} size is #{piece.bytesize} bytes, which exceeds the maximum bulk size of #{bulk_size} bytes.
+              It will be sent in its own request; if the cluster rejects it, the error will be raised.
+            MSG
+            result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+            result.push(Import::RequestBodyRaw.new)
+            next
+          end
+
+          if result.last.body.bytesize + piece.bytesize > bulk_size
+            result.push(Import::RequestBodyRaw.new.tap { |r| r.add(operation, piece) })
+          else
+            result[-1].add(operation, piece)
           end
-          requests.each(&:finalize)
-        else
-          raise err
         end
+        requests.reject! { |r| r.body.empty? }
+        requests.each(&:finalize)
       end
     end
   end

data/lib/esse/index/attributes.rb

@@ -72,6 +72,22 @@ module Esse
         @bulk_wait_interval = value.to_f
       end
 
+      def bulk_retry_on_failure_max_retries
+        @bulk_retry_on_failure_max_retries || Esse.config.bulk_retry_on_failure_max_retries
+      end
+
+      def bulk_retry_on_failure_max_retries=(value)
+        @bulk_retry_on_failure_max_retries = value.to_i
+      end
+
+      def bulk_retry_on_failure_wait
+        @bulk_retry_on_failure_wait || Esse.config.bulk_retry_on_failure_wait
+      end
+
+      def bulk_retry_on_failure_wait=(value)
+        @bulk_retry_on_failure_wait = value.to_f
+      end
+
       def mapping_single_type=(value)
         @mapping_single_type = !!value
       end

data/lib/esse/index/documents.rb

@@ -256,7 +256,10 @@ module Esse
           delete: to_delete,
           index: to_index,
           update: to_update,
-        ).each_request do |request_body|
+        ).each_request(
+          retry_on_failure_max_retries: bulk_retry_on_failure_max_retries,
+          retry_on_failure_wait: bulk_retry_on_failure_wait,
+        ) do |request_body|
           cluster.api.bulk(**definition, body: request_body.body) do |event_payload|
             event_payload[:body_stats] = request_body.stats
             if bulk_wait_interval > 0

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.4.1'
+  VERSION = '0.5.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.1
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2026-04-29 00:00:00.000000000 Z
+date: 2026-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json