esse 0.5.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15b531b7c6876d665c12fd773d25fa68acda30c4c92cf7999790315f4fe35b2d
-  data.tar.gz: 9520b76a128e752bed3db6d2a1a96e38e2107ca790c6d0c0327c2690d91b21d7
+  metadata.gz: fd1807aa9b2b2c92564d5e35d5a62282c67b9d25883fdc6166ca8981ee9008ac
+  data.tar.gz: 1cf17acb25d5b6bcf88e0999fc29c90a564b55d31ac51dce148f397c17ced4c4
 SHA512:
-  metadata.gz: 865aa67f404a82e573009e73186dd60ee9c3f978ba3c93b39337184c2b8c9c9415bc116573c5eb4f21cb7e4ffc7e19aae4957e54fe7b8aa99a8804e86469b13f
-  data.tar.gz: bcacb23ae7b1b1a073298a59090455e30d6aa9531b93a815871762ed9e3364de873795c47ca0b2d495d8d9394110fd623011d3e4e3054a34ddf89b5e35947ded
+  metadata.gz: bd72ab5edf509428c62ea598305a8351105c0073fa63a2246eb9eecb84b99fa2115355b1c7f881afdd500196a0d31482a7be7ebb8f4b92743eca3a8bd775840c
+  data.tar.gz: 94f0e1fed7ccaac67b8795a349ef51ab0cbfa9251fc219ffa49510a25a7e3ea52e44ae3ec386a7437014a5314ae65a0a20bfca542babddbb124a3381e49fd1c0

data/lib/esse/config.rb

@@ -43,7 +43,7 @@ module Esse
   #   end
   class Config
     DEFAULT_CLUSTER_ID = :default
-    ATTRIBUTES = %i[indices_directory bulk_wait_interval].freeze
+    ATTRIBUTES = %i[indices_directory bulk_wait_interval bulk_retry_on_failure_max_retries bulk_retry_on_failure_wait].freeze
 
     # The location of the indices. Defaults to the `app/indices`
     attr_reader :indices_directory
@@ -51,9 +51,17 @@ module Esse
     # wait a given period between posting pages to give Elasticsearch time to catch up.
     attr_reader :bulk_wait_interval
 
+    # number of retries on transient server errors (502, 503, 504, 429) before raising
+    attr_reader :bulk_retry_on_failure_max_retries
+
+    # base wait in seconds between transient-error retries (doubles each attempt)
+    attr_reader :bulk_retry_on_failure_wait
+
     def initialize
       self.indices_directory = 'app/indices'
       self.bulk_wait_interval = 0.1
+      self.bulk_retry_on_failure_max_retries = 3
+      self.bulk_retry_on_failure_wait = 2.0
       @clusters = {}
       cluster(DEFAULT_CLUSTER_ID) # initialize the :default client
     end
@@ -81,6 +89,14 @@ module Esse
       @bulk_wait_interval = value.to_f
     end
 
+    def bulk_retry_on_failure_max_retries=(value)
+      @bulk_retry_on_failure_max_retries = value.to_i
+    end
+
+    def bulk_retry_on_failure_wait=(value)
+      @bulk_retry_on_failure_wait = value.to_f
+    end
+
     def load(arg)
       case arg
       when Hash

data/lib/esse/errors.rb

@@ -57,6 +57,7 @@ module Esse
       'ExpectationFailed' => 'ExpectationFailedError', # 417
       'ImATeapot' => 'ImATeapotError', # 418
       'TooManyConnectionsFromThisIP' => 'TooManyConnectionsFromThisIPError', # 421
+      'TooManyRequests' => 'TooManyRequestsError', # 429
       'UpgradeRequired' => 'UpgradeRequiredError', # 426
       'BlockedByWindowsParentalControls' => 'BlockedByWindowsParentalControlsError', # 450
       'RequestHeaderTooLarge' => 'RequestHeaderTooLargeError', # 494

data/lib/esse/import/bulk.rb

@@ -18,11 +18,13 @@ module Esse
       # document still returns 413 does the error bubble up.
       #
       # @yield [RequestBody] A request body instance
-      def each_request(max_retries: 4, last_retry_in_small_chunks: true, last_retry_per_document: true)
+      def each_request(max_retries: 4, last_retry_in_small_chunks: true, last_retry_per_document: true,
+                       retry_on_failure_max_retries: 3, retry_on_failure_wait: 2.0)
         # @TODO create indexes when by checking all the index suffixes (if mapping is not empty)
         requests = [optimistic_request]
         retry_count = 0
         too_large_retry_count = 0
+        transient_failure_count = 0
 
         begin
           requests.each do |request|
@@ -59,11 +61,23 @@ module Esse
 
           raise e unless last_retry_per_document
           requests = requests_per_document
+          ids = document_ids
+          id_info = ids.any? ? " (document IDs: #{ids.join(', ')})" : ''
           Esse.logger.warn <<~MSG
-            Request entity too large after balancing, retrying one document per request as a last resort.
+            Request entity too large after balancing, retrying one document per request as a last resort#{id_info}.
             If a single document still exceeds the bulk size, the error will be raised.
           MSG
           retry
+        rescue Esse::Transport::BadGatewayError,
+               Esse::Transport::ServiceUnavailableError,
+               Esse::Transport::GatewayTimeoutError,
+               Esse::Transport::TooManyRequestsError => e
+          transient_failure_count += 1
+          raise e if transient_failure_count >= retry_on_failure_max_retries
+          wait = retry_on_failure_wait * (2**(transient_failure_count - 1))
+          Esse.logger.warn "#{e.class} error, retrying in #{wait}s (attempt #{transient_failure_count}/#{retry_on_failure_max_retries})"
+          sleep(wait)
+          retry
         end
       end
 
@@ -80,9 +94,11 @@ module Esse
 
       def requests_in_small_chunks(chunk_size: 1)
         arr = build_per_document_requests(chunk_size: chunk_size)
+        ids = document_ids
+        id_info = ids.any? ? " (document IDs: #{ids.join(', ')})" : ''
         Esse.logger.warn <<~MSG
-          Retrying the last request in small chunks of #{chunk_size} documents.
-          This is a last resort to avoid timeout errors, consider increasing the bulk size or reducing the batch size.
+          Timeout error, retrying one document per request as a last resort#{id_info}.
+          Consider increasing the bulk size or reducing the batch size.
         MSG
         arr
       end
@@ -91,6 +107,10 @@ module Esse
         build_per_document_requests(chunk_size: 1)
       end
 
+      def document_ids
+        (@create + @index + @update + @delete).filter_map { |op| op.values.first[:_id] }
+      end
+
       def build_per_document_requests(chunk_size: 1)
         arr = []
         @create.each_slice(chunk_size) { |slice| arr << Import::RequestBodyAsJson.new.tap { |r| r.create = slice } }

data/lib/esse/index/attributes.rb

@@ -72,6 +72,22 @@ module Esse
         @bulk_wait_interval = value.to_f
       end
 
+      def bulk_retry_on_failure_max_retries
+        @bulk_retry_on_failure_max_retries || Esse.config.bulk_retry_on_failure_max_retries
+      end
+
+      def bulk_retry_on_failure_max_retries=(value)
+        @bulk_retry_on_failure_max_retries = value.to_i
+      end
+
+      def bulk_retry_on_failure_wait
+        @bulk_retry_on_failure_wait || Esse.config.bulk_retry_on_failure_wait
+      end
+
+      def bulk_retry_on_failure_wait=(value)
+        @bulk_retry_on_failure_wait = value.to_f
+      end
+
       def mapping_single_type=(value)
         @mapping_single_type = !!value
       end

data/lib/esse/index/documents.rb

@@ -256,7 +256,10 @@ module Esse
           delete: to_delete,
           index: to_index,
           update: to_update,
-        ).each_request do |request_body|
+        ).each_request(
+          retry_on_failure_max_retries: bulk_retry_on_failure_max_retries,
+          retry_on_failure_wait: bulk_retry_on_failure_wait,
+        ) do |request_body|
           cluster.api.bulk(**definition, body: request_body.body) do |event_payload|
             event_payload[:body_stats] = request_body.stats
             if bulk_wait_interval > 0

data/lib/esse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Esse
-  VERSION = '0.5.0'
+  VERSION = '0.5.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: esse
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - Marcos G. Zimmermann
 autorequire:
 bindir: exec
 cert_chain: []
-date: 2026-05-15 00:00:00.000000000 Z
+date: 2026-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json