test-opensearch 0.0.0.2 → 0.0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7580053f641548484430925f843889791431d992ea33d0f2208d0050df99f11
-  data.tar.gz: 0aa937c2c3f0d2b46f51f13a089c131967d87452738ce1c2450349437c144733
+  metadata.gz: 4e56816c3f750e8e813fbcf64e6589ca799a2b94f2b00a07ef0ae8395a4da14b
+  data.tar.gz: 0340bdd6b8a12e2ed032437484656f61b37c5cca2355bc01cadd408225619b90
 SHA512:
-  metadata.gz: 4f1c1c7c5e3b0545a16b00a97d794ddd19810db75353c5e1ec81f95720028f447c5eed44987e34a965acdf84a05916fd0f277d6135fc63f14c6cda3f2e1af1e8
-  data.tar.gz: 6cebc7d1d09d249b1bb441a8dd15632a643769b6e8a645884b9c1bd52dc604016cce477de2a61e1d188e52d909ade417f8ef9d7f30e64ae8cde49f333f60a6a2
+  metadata.gz: b5cabd951822174fa8cf36902279bcf41db4a3a839b783ccdc072f4dfb6816c40d502261053343d71df9e5a1d26e3d330adfcbae839bbaa9b3974f576f226d11
+  data.tar.gz: 58305a974872fd798841bd9df1dccf0099c50282e6bab96d8dcd3155e0ad05c0796c7e856bec379d643c6dad03caf2a0057577a3981734f4bc0b2c3100df0bac

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'test-opensearch'
-  s.version       = '0.0.0.2'
+  s.version       = '0.0.0.3'
   s.authors       = ['Oleh']
   s.email         = ['imcotop@icloud.com']
   s.description   = %q{test}

data/lib/fluent/plugin/in_opensearch.rb

@@ -26,9 +26,10 @@
 
 require 'opensearch'
 
-# require 'faraday/excon'
+require 'faraday/excon'
 require 'fluent/log-ext'
 require 'fluent/plugin/input'
+require 'fluent/plugin_helper'
 require_relative 'opensearch_constants'
 
 module Fluent::Plugin
@@ -39,7 +40,7 @@ module Fluent::Plugin
     DEFAULT_STORAGE_TYPE = 'local'
     METADATA = "@metadata".freeze
 
-    helpers :timer, :thread
+    helpers :timer, :thread, :retry_state
 
     Fluent::Plugin.register_input('opensearch', self)
 
@@ -80,7 +81,24 @@ module Fluent::Plugin
     config_param :docinfo_fields, :array, :default => ['_index', '_type', '_id']
     config_param :docinfo_target, :string, :default => METADATA
     config_param :docinfo, :bool, :default => false
-    config_param :infinite_check_connection, :bool, :default => true
+    config_param :check_connection, :bool, :default => true
+    config_param :retry_forever, :bool, default: true, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry forever.'
+    config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry to flush while failing, until plugin discards buffer chunks.'
+    # 72hours == 17 times with exponential backoff (not to change default behavior)
+    config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.'
+    config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.'
+    # exponential backoff sequence will be initialized at the time of this threshold
+    config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :periodic
+    ### Periodic -> fixed :retry_wait
+    ### Exponential backoff: k is number of retry times
+    # c: constant factor, @retry_wait
+    # b: base factor, @retry_exponential_backoff_base
+    # k: times
+    # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
+    config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
+    config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
+    config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
+    config_param :retry_randomize, :bool, default: false, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
 
     include Fluent::Plugin::OpenSearchConstants
 
@@ -93,6 +111,7 @@ module Fluent::Plugin
 
       @timestamp_parser = create_time_parser
       @backend_options = backend_options
+      @retry = retry_state(@retry_randomize)
 
       raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
 
@@ -139,6 +158,15 @@ module Fluent::Plugin
       raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
     end
 
+    def retry_state(randomize)
+      retry_state_create(
+        :input_retries,@retry_type,@retry_type,@retry_timeout,
+        forever: @retry_forever,max_steps: @retry_max_times,
+        max_interval: @retry_max_interval,backoff_base: @retry_exponential_backoff_base,
+        randomize: randomize
+      )
+    end
+
     def get_escaped_userinfo(host_str)
       if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
         m["scheme"] +
@@ -177,12 +205,33 @@ module Fluent::Plugin
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path
       end
-
+      live_hosts = hosts.select { |host| reachable_host?(host) }
       {
-        hosts: hosts
+        hosts: live_hosts
       }
     end
 
+    def reachable_host?(host)
+      if @check_connection
+        client = OpenSearch::Client.new(
+          host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
+          user: host[:user],
+          password: host[:password],
+          reload_connections: @reload_connections,
+          request_timeout: @request_timeout,
+          resurrect_after: @resurrect_after,
+          reload_on_failure: @reload_on_failure,
+          transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
+        )
+        client.ping
+      else
+        true
+      end
+    rescue => e
+      log.warn "Failed to connect to #{host[:scheme]}://#{host[:host]}:#{host[:port]}: #{e.message}"
+      false
+    end
+
     def emit_error_label_event(&block)
       # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
       if emit_error_label_event
@@ -303,30 +352,39 @@ module Fluent::Plugin
           run_slice(slice_id)
         end
       end
-    rescue Faraday::ConnectionFailed => e
-      log.warn "Connection to OpenSearch failed during search in the 'run' method: #{e.message}. Retrying..."
-      retry
+    rescue => e
+      @retry.step
+      # Check if the retry limit has been reached
+      if @retry.limit?
+        msg = "Hit limit for retries."
+        log.warn msg, retry_times: @retry.steps, error: e.message
+        break
+      else
+        msg = "failed to connect or search."
+        log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: error)
+        sleep(@retry.next_time - Time.now)
+        retry
+      end
     end
 
     def run_slice(slice_id=nil)
-      client.ping
-      # slice_query = @base_query
-      # slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @num_slices}) unless slice_id.nil?
-      # result = client.search(@options.merge(:body => Yajl.dump(slice_query) ))
-      # es = Fluent::MultiEventStream.new
-
-      # result["hits"]["hits"].each {|hit| process_events(hit, es)}
-      # has_hits = result['hits']['hits'].any?
-      # scroll_id = result['_scroll_id']
-
-      # while has_hits && scroll_id
-      #   result = process_next_scroll_request(es, scroll_id)
-      #   has_hits = result['has_hits']
-      #   scroll_id = result['_scroll_id']
-      # end
-
-      # router.emit_stream(@tag, es)
-      # clear_scroll(scroll_id)
+      slice_query = @base_query
+      slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @num_slices}) unless slice_id.nil?
+      result = client.search(@options.merge(:body => Yajl.dump(slice_query) ))
+      es = Fluent::MultiEventStream.new
+
+      result["hits"]["hits"].each {|hit| process_events(hit, es)}
+      has_hits = result['hits']['hits'].any?
+      scroll_id = result['_scroll_id']
+
+      while has_hits && scroll_id
+        result = process_next_scroll_request(es, scroll_id)
+        has_hits = result['has_hits']
+        scroll_id = result['_scroll_id']
+      end
+
+      router.emit_stream(@tag, es)
+      clear_scroll(scroll_id)
     end
 
     def clear_scroll(scroll_id)

data/test/plugin/test_in_opensearch.rb

@@ -39,7 +39,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
   CONFIG = %[
     tag raw.opensearch
     interval 2
-    infinite_check_connection false
+    check_connection false
   ]
 
   def setup
@@ -191,7 +191,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     john
       password doe
       tag      raw.opensearch
-      infinite_check_connection false
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -230,7 +230,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     john
       password doe
       tag      raw.opensearch
-      infinite_check_connection false
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -252,7 +252,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     %{j+hn}
       password %{d@e}
       tag      raw.opensearch
-      infinite_check_connection false
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -275,7 +275,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       path     /es/
       port     123
       tag      raw.opensearch
-      infinite_check_connection false
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -300,7 +300,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     default_user
       password default_password
       tag      raw.opensearch
-      infinite_check_connection false
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -329,7 +329,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     default_user
       password default_password
       tag      raw.opensearch
-      infinite_check_connection false
+      check_connection false
     }
     instance = driver(config).instance
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: test-opensearch
 version: !ruby/object:Gem::Version
-  version: 0.0.0.2
+  version: 0.0.0.3
 platform: ruby
 authors:
 - Oleh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-04-15 00:00:00.000000000 Z
+date: 2024-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd