test-opensearch 0.0.0.17 → 0.0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af727e2a72e5f71b170ca5e37f65b84f8ae40fcb69eff636d2a70e44ce8426be
-  data.tar.gz: 2467954baa65a7a74ee254f0f67dbe16aef370b34c434d005d8af13215c268a2
+  metadata.gz: e64802826ae07863828975be01dbb8fee1bc0e4d8fef81ad195fe658ede210e0
+  data.tar.gz: cd18ef4da46e20bd059a3fcf5d9b2791e630e4c37704cc4af17e2480e717113b
 SHA512:
-  metadata.gz: 3db5fe46be07f5a966cf0db77dfa09d3c300e33960cfa7f951f836c71a7074a6eecd49855569d0dbeac52b75ee5c0f324ccb5db3111c07c95f6689ea0e8dfe5f
-  data.tar.gz: 4fb822bf6adf28269fbf13c434c2b06ea51e715ea9801fc76ca78524ffeda5d8a876e07607d453137da51e7b3d70edc17fa7308e3589a4a776a20b9709d2288d
+  metadata.gz: d79fbfa7dd1f59e79db7c53e422cf3f6f4ad8d62b09f7b20566cfec33077230971083b2691dbb57b081344005b0ad09200a2916eba1a35628b18d3cbcf039858
+  data.tar.gz: ade76a78db35219afeb6cba5cd4db1bcfb14470c9a0f7d1daf82d9a7ae065602fcb9577d36f6e81448f456f2e773837ac48f35be70627c761f3dd6f76ba0108c

data/README.OpenSearchInput.md

@@ -24,7 +24,16 @@
   + [docinfo_fields](#docinfo_fields)
   + [docinfo_target](#docinfo_target)
   + [docinfo](#docinfo)
-  + [infinite_check_connection](#infinite_check_connection)
+  + [check_connection](#check_connection)
+  + [retry_forever](#retry_forever)
+  + [retry_timeout](#retry_timeout)
+  + [retry_max_times](#retry_max_times)
+  + [retry_type](#retry_type)
+  + [retry_wait](#retry_wait)
+  + [retry_exponential_backoff_base](#retry_exponential_backoff_base)
+  + [retry_max_interval](#retry_max_interval)
+  + [retry_randomize](#retry_randomize)
+
 * [Advanced Usage](#advanced-usage)
 
 ## Usage
@@ -275,14 +284,87 @@ This parameter specifies whether docinfo information including or not. The defau
 docinfo false
 ```
 
-### infinite_check_connection
+### check_connection
+
+The parameter for checking on connection availability with Elasticsearch or Opensearch hosts. The default value is `true`.
+
+```
+check_connection true
+```
+### retry_forever
+
+The parameter If true, plugin will ignore retry_timeout and retry_max_times options and retry forever. The default value is `true`.
+
+```
+retry_forever true
+```
+
+### retry_timeout
+
+The parameter maximum time (seconds) to retry again the failed try, until the plugin discards the retry.
+If the next retry is going to exceed this time limit, the last retry will be made at exactly this time limit..
+The default value is `72h`.
+72hours == 17 times with exponential backoff (not to change default behavior)
+
+```
+retry_timeout 72 * 60 * 60
+```
+
+### retry_max_times
+
+The parameter maximum number of times to retry the failed try. The default value is `5`
+
+```
+retry_max_times 5
+```
+
+### retry_type
 
-The parameter infinite checking on connection availability with Elasticsearch or opensearch hosts, every  request_timeout (default 5) seconds. The default value is `true`.
+The parameter needs for how long need to wait (time in seconds) to retry again:
+`exponential_backoff`: wait in seconds will become large exponentially per failure,
+`periodic:` plugin will retry periodically with fixed intervals (configured via retry_wait). The default value is `:periodic`
+Periodic -> fixed :retry_wait
+Exponential backoff: k is number of retry times
+c: constant factor, @retry_wait
+b: base factor, @retry_exponential_backoff_base
+k: times
+total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
 
 ```
-infinite_check_connection true
+retry_type :periodic
 ```
 
+### retry_wait
+
+The parameter needs for wait in seconds before the next retry to again or constant factor of exponential backoff. The default value is  `5`
+
+```
+retry_wait 5
+```
+
+### retry_exponential_backoff_base
+
+The parameter The base number of exponential backoff for retries. The default value is  `2`
+
+```
+retry_exponential_backoff_base 2
+```
+
+### retry_max_interval
+
+The parameter maximum interval (seconds) for exponential backoff between retries while failing. The default value is  `nil`
+
+```
+retry_max_interval nil
+```
+
+### retry_randomize
+
+The parameter If true, the output plugin will retry after randomized interval not to do burst retries. The default value is  `false`
+
+```
+retry_randomize false
+```
 
 ## Advanced Usage
 

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'test-opensearch'
-  s.version       = '0.0.0.17'
+  s.version       = '0.0.0.20'
   s.authors       = ['Oleh']
   s.email         = ['imcotop@icloud.com']
   s.description   = %q{test}

data/lib/fluent/plugin/in_opensearch.rb

@@ -83,9 +83,9 @@ module Fluent::Plugin
     config_param :docinfo, :bool, :default => false
     config_param :check_connection, :bool, :default => true
     config_param :retry_forever, :bool, default: true, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry forever.'
-    config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry to flush while failing, until plugin discards buffer chunks.'
+    config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry'
     # 72hours == 17 times with exponential backoff (not to change default behavior)
-    config_param :retry_max_times, :integer, default: 5, desc: 'The maximum number of times to retry to flush while failing.'
+    config_param :retry_max_times, :integer, default: 5, desc: 'The maximum number of times to retry'
     # exponential backoff sequence will be initialized at the time of this threshold
     config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff
     ### Periodic -> fixed :retry_wait
@@ -94,7 +94,7 @@ module Fluent::Plugin
     # b: base factor, @retry_exponential_backoff_base
     # k: times
     # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
-    config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
+    config_param :retry_wait, :time, default: 5, desc: 'Seconds to wait before next retry , or constant factor of exponential backoff.'
     config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
     config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
     config_param :retry_randomize, :bool, default: false, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
@@ -204,28 +204,24 @@ module Fluent::Plugin
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path
       end
-      live_hosts = hosts.select { |host| reachable_host?(host) }
+      live_hosts = @check_connection ? hosts.select { |host| reachable_host?(host) } : hosts
       {
         hosts: live_hosts
       }
     end
 
     def reachable_host?(host)
-      if @check_connection
-        client = OpenSearch::Client.new(
-          host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
-          user: host[:user],
-          password: host[:password],
-          reload_connections: @reload_connections,
-          request_timeout: @request_timeout,
-          resurrect_after: @resurrect_after,
-          reload_on_failure: @reload_on_failure,
-          transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
-        )
-        client.ping
-      else
-        true
-      end
+      client = OpenSearch::Client.new(
+        host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
+        user: host[:user],
+        password: host[:password],
+        reload_connections: @reload_connections,
+        request_timeout: @request_timeout,
+        resurrect_after: @resurrect_after,
+        reload_on_failure: @reload_on_failure,
+        transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
+      )
+      client.ping
     rescue => e
       log.warn "Failed to connect to #{host[:scheme]}://#{host[:host]}:#{host[:port]}: #{e.message}"
       false
@@ -353,15 +349,13 @@ module Fluent::Plugin
       end
     rescue Faraday::ConnectionFailed, OpenSearch::Transport::Transport::Error => e
       @retry.step
-      # Check if the retry limit has been reached
-      if @retry.limit?
-        raise Fluent::ConfigError, "Hit limit for retries. retry_times: #{@retry.steps} error: #{e.message}"
-      else
-        msg = "failed to connect or search."
-        log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: e.message)
-        sleep(@retry.next_time - Time.now)
-        retry
-      end
+      #Raise error if the retry limit has been reached
+      raise "Hit limit for retries. retry_times: #{@retry.limit?}, error: #{e.message}" if @retry.limit?
+
+      #Retry if the retry limit hasn't been reached
+      log.warn("failed to connect or search.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: e.message)
+      sleep(@retry.next_time - Time.now)
+      retry
     end
 
     def run_slice(slice_id=nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: test-opensearch
 version: !ruby/object:Gem::Version
-  version: 0.0.0.17
+  version: 0.0.0.20
 platform: ruby
 authors:
 - Oleh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-04-19 00:00:00.000000000 Z
+date: 2024-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd