test-opensearch 0.0.0.18 → 0.0.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91c18a291793fda6a46f006bef41d93aa3f94ae6818d2b69ad27df0b5b4f6614
-  data.tar.gz: f6854e958070eaf6596b603f5aee2deacf41b68c70308a524adab089b658e609
+  metadata.gz: 833739abc81a6562071c9047e266225855f059853b5071cd11452cff393b4a1a
+  data.tar.gz: 75c81ae847a0d5a3e12d8e3e211a7614778ee419ee5ef10f8aa2c194b3063d6f
 SHA512:
-  metadata.gz: b98d89745b6419d64a2fb6e745791a715604e0c6aa28d2f52022648d71c322ca5c0b5c778f090e4a3ec88b3726a0a932357bd9aa41f900a4c8c807b9d24d01f7
-  data.tar.gz: 1c303c1587a238ab2c9042ae3d2e0efe7ad838fc7b53c0253db281f29ec6f43a461451b2475eb818e6c8fb610f3f71c4c7e82fd5412e42ea9bef309176629f22
+  metadata.gz: e6d93a2323842bf0258558d0da05da7bd76a3b95bfbbd191943cfc9c6203e131db9ee49e0455a7426c7626ca8668d4800c3cb0d3dc0b9ec283d171698b44b3d5
+  data.tar.gz: ce93e37096e8ebf4c502990fc172b1c6523b8ba6f285a8e3bc7e9bdb9ef219a45d4385dee1ab4b377e686e51e949990b8f6b95bf22c16e0cf49e20a312cb0e8d

data/README.OpenSearchInput.md

@@ -24,7 +24,16 @@
   + [docinfo_fields](#docinfo_fields)
   + [docinfo_target](#docinfo_target)
   + [docinfo](#docinfo)
-  + [infinite_check_connection](#infinite_check_connection)
+  + [check_connection](#check_connection)
+  + [retry_forever](#retry_forever)
+  + [retry_timeout](#retry_timeout)
+  + [retry_max_times](#retry_max_times)
+  + [retry_type](#retry_type)
+  + [retry_wait](#retry_wait)
+  + [retry_exponential_backoff_base](#retry_exponential_backoff_base)
+  + [retry_max_interval](#retry_max_interval)
+  + [retry_randomize](#retry_randomize)
+
 * [Advanced Usage](#advanced-usage)
 
 ## Usage
@@ -275,14 +284,87 @@ This parameter specifies whether docinfo information including or not. The defau
 docinfo false
 ```
 
-### infinite_check_connection
+### check_connection
+
+The parameter for checking on connection availability with Elasticsearch or Opensearch hosts. The default value is `true`.
+
+```
+check_connection true
+```
+### retry_forever
+
+The parameter If true, plugin will ignore retry_timeout and retry_max_times options and retry forever. The default value is `true`.
+
+```
+retry_forever true
+```
+
+### retry_timeout
+
+The parameter maximum time (seconds) to retry again the failed try, until the plugin discards the retry.
+If the next retry is going to exceed this time limit, the last retry will be made at exactly this time limit..
+The default value is `72h`.
+72hours == 17 times with exponential backoff (not to change default behavior)
+
+```
+retry_timeout 72 * 60 * 60
+```
+
+### retry_max_times
+
+The parameter maximum number of times to retry the failed try. The default value is `5`
+
+```
+retry_max_times 5
+```
+
+### retry_type
 
-The parameter infinite checking on connection availability with Elasticsearch or opensearch hosts, every  request_timeout (default 5) seconds. The default value is `true`.
+The parameter needs for how long need to wait (time in seconds) to retry again:
+`exponential_backoff`: wait in seconds will become large exponentially per failure,
+`periodic`: plugin will retry periodically with fixed intervals (configured via retry_wait). The default value is `:exponential_backoff`
+Periodic -> fixed :retry_wait
+Exponential backoff: k is number of retry times
+c: constant factor, @retry_wait
+b: base factor, @retry_exponential_backoff_base
+k: times
+total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
 
 ```
-infinite_check_connection true
+retry_type :exponential_backoff
 ```
 
+### retry_wait
+
+The parameter needs for wait in seconds before the next retry to again or constant factor of exponential backoff. The default value is  `5`
+
+```
+retry_wait 5
+```
+
+### retry_exponential_backoff_base
+
+The parameter The base number of exponential backoff for retries. The default value is  `2`
+
+```
+retry_exponential_backoff_base 2
+```
+
+### retry_max_interval
+
+The parameter maximum interval (seconds) for exponential backoff between retries while failing. The default value is  `nil`
+
+```
+retry_max_interval nil
+```
+
+### retry_randomize
+
+The parameter If true, the output plugin will retry after randomized interval not to do burst retries. The default value is  `false`
+
+```
+retry_randomize false
+```
 
 ## Advanced Usage
 

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'test-opensearch'
-  s.version       = '0.0.0.18'
+  s.version       = '0.0.0.21'
   s.authors       = ['Oleh']
   s.email         = ['imcotop@icloud.com']
   s.description   = %q{test}

data/lib/fluent/plugin/in_opensearch.rb

@@ -204,28 +204,24 @@ module Fluent::Plugin
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path
       end
-      live_hosts = hosts.select { |host| reachable_host?(host) }
+      live_hosts = @check_connection ? hosts.select { |host| reachable_host?(host) } : hosts
       {
         hosts: live_hosts
       }
     end
 
     def reachable_host?(host)
-      if @check_connection
-        client = OpenSearch::Client.new(
-          host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
-          user: host[:user],
-          password: host[:password],
-          reload_connections: @reload_connections,
-          request_timeout: @request_timeout,
-          resurrect_after: @resurrect_after,
-          reload_on_failure: @reload_on_failure,
-          transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
-        )
-        client.ping
-      else
-        true
-      end
+      client = OpenSearch::Client.new(
+        host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
+        user: host[:user],
+        password: host[:password],
+        reload_connections: @reload_connections,
+        request_timeout: @request_timeout,
+        resurrect_after: @resurrect_after,
+        reload_on_failure: @reload_on_failure,
+        transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
+      )
+      client.ping
     rescue => e
       log.warn "Failed to connect to #{host[:scheme]}://#{host[:host]}:#{host[:port]}: #{e.message}"
       false
@@ -353,15 +349,13 @@ module Fluent::Plugin
       end
     rescue Faraday::ConnectionFailed, OpenSearch::Transport::Transport::Error => e
       @retry.step
-      # Check if the retry limit has been reached
-      if @retry.limit?
-        raise "Hit limit for retries. retry_times: #{@retry.steps} error: #{e.message}"
-      else
-        msg = "failed to connect or search."
-        log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: e.message)
-        sleep(@retry.next_time - Time.now)
-        retry
-      end
+      #Raise error if the retry limit has been reached
+      raise "Hit limit for retries. retry_times: #{@retry.steps}, error: #{e.message}" if @retry.limit?
+
+      #Retry if the retry limit hasn't been reached
+      log.warn("failed to connect or search.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: e.message)
+      sleep(@retry.next_time - Time.now)
+      retry
     end
 
     def run_slice(slice_id=nil)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: test-opensearch
 version: !ruby/object:Gem::Version
-  version: 0.0.0.18
+  version: 0.0.0.21
 platform: ruby
 authors:
 - Oleh