fluent-plugin-opensearch 1.1.4 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 578e5c63061d5f1cb3f3729dee4ea630564dc60d82f6b047c8e4b23830fdf16a
-  data.tar.gz: a502d46eb5a3a59c5c4f65f9009623ff32fde1de4cfda7e181437a8720bdbda1
+  metadata.gz: d0a251b65daac387b7520c49475a881440181bbe44da46849d8bdee61b9a4374
+  data.tar.gz: ec313c1dba6a723a4c5fd4cf672abaf7e1be9785e29176c31f419b253949f539
 SHA512:
-  metadata.gz: 77b3373041396440cf8debaea8fd2ab95b94df950cbfe6105f65d45cce7092a6f89bd019b38d4022051c651541d3d21918625ea1b9939f58108101eae90a027f
-  data.tar.gz: 2e746cd797f5d24f08dbb386ea42b9bff506831c56f9f4b92dcf99dde321b9b613c7473cbad39bd1f6d338ae6dd094a87f838141306111ab49b5a76886346918
+  metadata.gz: 1e40ac1b0816b42fa6d84a3cfd72eb3219ce688a80e15e110670e32a6ee7dd97a20542dd6d056a443f92d173e16f3a29b614a249e64ba2babb18243d3ad4ba7f
+  data.tar.gz: 342f988468e7fb9a47ef2111e88a474b9d7a97d310b3112173e1ca7dc510dea3c1a0ddf6f0c7515c5769e389a01a68f761540576e0856f718e06e2e83ab0f712

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -5,7 +5,7 @@ about: Suggest an idea for this project
 ---
 
 (check apply)
-- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+- [ ] read [the contribution guideline](https://github.com/fluent/fluent-plugin-opensearch/blob/master/CONTRIBUTING.md)
 
 **Is your feature request related to a problem? Please describe.**
 

data/.github/workflows/linux.yml

@@ -22,5 +22,6 @@ jobs:
         CI: true
       run: |
         gem install bundler rake
+        bundle config set path 'vendor/bundle'
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/History.md

@@ -2,6 +2,14 @@
 
 ### [Unreleased]
 
+### 1.1.5
+ - Fix bundle installation error on GitHub Action (#143)
+ - Prevent AWS credentials refresh from stopping on exception (#142)
+ - Added a retry logic and a service availability check function for high availability. (#136)
+ - out\_opensearch\_data\_stream: Early return on empty body (#131)
+ - Docs/cleanup issue templates (#119)
+ - pin dependency opensearch-ruby to 3.0.1 (#116)
+
 ### 1.1.4
  - test: remove minitest to correct misjudge of the framework by flexmock (#114)
  - Add logic to write method of out_opensearch_data_stream (#109)

data/README.OpenSearchInput.md

@@ -24,6 +24,16 @@
   + [docinfo_fields](#docinfo_fields)
   + [docinfo_target](#docinfo_target)
   + [docinfo](#docinfo)
+  + [check_connection](#check_connection)
+  + [retry_forever](#retry_forever)
+  + [retry_timeout](#retry_timeout)
+  + [retry_max_times](#retry_max_times)
+  + [retry_type](#retry_type)
+  + [retry_wait](#retry_wait)
+  + [retry_exponential_backoff_base](#retry_exponential_backoff_base)
+  + [retry_max_interval](#retry_max_interval)
+  + [retry_randomize](#retry_randomize)
+
 * [Advanced Usage](#advanced-usage)
 
 ## Usage
@@ -274,6 +284,88 @@ This parameter specifies whether docinfo information including or not. The defau
 docinfo false
 ```
 
+### check_connection
+
+The parameter for checking on connection availability with Elasticsearch or Opensearch hosts. The default value is `true`.
+
+```
+check_connection true
+```
+### retry_forever
+
+The parameter If true, plugin will ignore retry_timeout and retry_max_times options and retry forever. The default value is `true`.
+
+```
+retry_forever true
+```
+
+### retry_timeout
+
+The parameter maximum time (seconds) to retry again the failed try, until the plugin discards the retry.
+If the next retry is going to exceed this time limit, the last retry will be made at exactly this time limit..
+The default value is `72h`.
+72hours == 17 times with exponential backoff (not to change default behavior)
+
+```
+retry_timeout 72 * 60 * 60
+```
+
+### retry_max_times
+
+The parameter maximum number of times to retry the failed try. The default value is `5`
+
+```
+retry_max_times 5
+```
+
+### retry_type
+
+The parameter needs for how long need to wait (time in seconds) to retry again:
+`exponential_backoff`: wait in seconds will become large exponentially per failure,
+`periodic`: plugin will retry periodically with fixed intervals (configured via retry_wait). The default value is `:exponential_backoff`
+Periodic -> fixed :retry_wait
+Exponential backoff: k is number of retry times
+c: constant factor, @retry_wait
+b: base factor, @retry_exponential_backoff_base
+k: times
+total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
+
+```
+retry_type exponential_backoff
+```
+
+### retry_wait
+
+The parameter needs for wait in seconds before the next retry to again or constant factor of exponential backoff. The default value is  `5`
+
+```
+retry_wait 5
+```
+
+### retry_exponential_backoff_base
+
+The parameter The base number of exponential backoff for retries. The default value is  `2`
+
+```
+retry_exponential_backoff_base 2
+```
+
+### retry_max_interval
+
+The parameter maximum interval (seconds) for exponential backoff between retries while failing. The default value is  `nil`
+
+```
+retry_max_interval nil
+```
+
+### retry_randomize
+
+The parameter If true, the plugin will retry after randomized interval not to do burst retries. The default value is  `false`
+
+```
+retry_randomize false
+```
+
 ## Advanced Usage
 
 OpenSearch Input plugin and OpenSearch output plugin can combine to transfer records into another cluster.

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.1.4'
+  s.version       = '1.1.5'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
 
   s.add_runtime_dependency 'fluentd', '>= 0.14.22'
-  s.add_runtime_dependency 'opensearch-ruby'
+  s.add_runtime_dependency 'opensearch-ruby', '>= 3.0.1'
   s.add_runtime_dependency "aws-sdk-core", "~> 3"
   s.add_runtime_dependency 'excon', '>= 0'
   s.add_runtime_dependency 'faraday', '>= 2.0.0'
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rake', '>= 0'
   s.add_development_dependency 'webrick', '~> 1.7.0'
-  s.add_development_dependency 'webmock', '~> 3'
+  s.add_development_dependency 'webmock', '~> 3.18.1'
   s.add_development_dependency 'test-unit', '~> 3.3.0'
   s.add_development_dependency 'minitest', '~> 5.8'
   s.add_development_dependency 'flexmock', '~> 2.0'

data/lib/fluent/plugin/in_opensearch.rb

@@ -29,6 +29,7 @@ require 'opensearch'
 require 'faraday/excon'
 require 'fluent/log-ext'
 require 'fluent/plugin/input'
+require 'fluent/plugin_helper'
 require_relative 'opensearch_constants'
 
 module Fluent::Plugin
@@ -39,7 +40,7 @@ module Fluent::Plugin
     DEFAULT_STORAGE_TYPE = 'local'
     METADATA = "@metadata".freeze
 
-    helpers :timer, :thread
+    helpers :timer, :thread, :retry_state
 
     Fluent::Plugin.register_input('opensearch', self)
 
@@ -80,6 +81,23 @@ module Fluent::Plugin
     config_param :docinfo_fields, :array, :default => ['_index', '_type', '_id']
     config_param :docinfo_target, :string, :default => METADATA
     config_param :docinfo, :bool, :default => false
+    config_param :check_connection, :bool, :default => true
+    config_param :retry_forever, :bool, default: true, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry forever.'
+    config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry'
+    # 72hours == 17 times with exponential backoff (not to change default behavior)
+    config_param :retry_max_times, :integer, default: 5, desc: 'The maximum number of times to retry'
+    # exponential backoff sequence will be initialized at the time of this threshold
+    config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff
+    ### Periodic -> fixed :retry_wait
+    ### Exponential backoff: k is number of retry times
+    # c: constant factor, @retry_wait
+    # b: base factor, @retry_exponential_backoff_base
+    # k: times
+    # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
+    config_param :retry_wait, :time, default: 5, desc: 'Seconds to wait before next retry , or constant factor of exponential backoff.'
+    config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
+    config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
+    config_param :retry_randomize, :bool, default: false, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
 
     include Fluent::Plugin::OpenSearchConstants
 
@@ -92,6 +110,7 @@ module Fluent::Plugin
 
       @timestamp_parser = create_time_parser
       @backend_options = backend_options
+      @retry = nil
 
       raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
 
@@ -138,6 +157,15 @@ module Fluent::Plugin
       raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
     end
 
+    def retry_state(randomize)
+      retry_state_create(
+        :input_retries, @retry_type, @retry_wait, @retry_timeout,
+        forever: @retry_forever, max_steps: @retry_max_times,
+        max_interval: @retry_max_interval, backoff_base: @retry_exponential_backoff_base,
+        randomize: randomize
+      )
+    end
+
     def get_escaped_userinfo(host_str)
       if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
         m["scheme"] +
@@ -176,12 +204,29 @@ module Fluent::Plugin
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path
       end
-
+      live_hosts = @check_connection ? hosts.select { |host| reachable_host?(host) } : hosts
       {
-        hosts: hosts
+        hosts: live_hosts
       }
     end
 
+    def reachable_host?(host)
+      client = OpenSearch::Client.new(
+        host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
+        user: host[:user],
+        password: host[:password],
+        reload_connections: @reload_connections,
+        request_timeout: @request_timeout,
+        resurrect_after: @resurrect_after,
+        reload_on_failure: @reload_on_failure,
+        transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
+      )
+      client.ping
+    rescue => e
+      log.warn "Failed to connect to #{host[:scheme]}://#{host[:host]}:#{host[:port]}: #{e.message}"
+      false
+    end
+
     def emit_error_label_event(&block)
       # If `emit_error_label_event` is specified as false, error event emittions are not occurred.
       if emit_error_label_event
@@ -292,6 +337,25 @@ module Fluent::Plugin
       return true
     end
 
+    def update_retry_state(error=nil)
+      if error
+        unless @retry
+          @retry = retry_state(@retry_randomize)
+        end
+        @retry.step
+        #Raise error if the retry limit has been reached
+        raise "Hit limit for retries. retry_times: #{@retry.steps}, error: #{error.message}" if @retry.limit?
+        #Retry if the limit hasn't been reached
+        log.warn("failed to connect or search.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: error.message)
+        sleep(@retry.next_time - Time.now)
+      else
+        unless @retry.nil?
+          log.debug("retry succeeded.")
+          @retry = nil
+        end
+      end
+    end
+
     def run
       return run_slice if @num_slices <= 1
 
@@ -302,6 +366,9 @@ module Fluent::Plugin
           run_slice(slice_id)
         end
       end
+    rescue Faraday::ConnectionFailed, OpenSearch::Transport::Transport::Error => error
+      update_retry_state(error)
+      retry
     end
 
     def run_slice(slice_id=nil)
@@ -321,7 +388,15 @@ module Fluent::Plugin
       end
 
       router.emit_stream(@tag, es)
+      clear_scroll(scroll_id)
+      update_retry_state
+    end
+
+    def clear_scroll(scroll_id)
       client.clear_scroll(scroll_id: scroll_id) if scroll_id
+    rescue => e
+      # ignore & log any clear_scroll errors
+      log.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
     end
 
     def process_scroll_request(scroll_id)

data/lib/fluent/plugin/out_opensearch.rb

@@ -350,7 +350,11 @@ module Fluent::Plugin
 
             @credential_mutex.synchronize do
               @_os = nil
-              @_aws_credentials = aws_credentials(@endpoint)
+              begin
+                @_aws_credentials = aws_credentials(@endpoint)
+              rescue => e
+                log.error("Failed to get new AWS credentials: #{e}")
+              end
             end
           end
         end

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -201,6 +201,8 @@ module Fluent::Plugin
         end
       end
 
+      return if bulk_message.to_s.empty?
+
       params = {
         index: data_stream_name,
         body: bulk_message

data/test/plugin/test_in_opensearch.rb

@@ -39,6 +39,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
   CONFIG = %[
     tag raw.opensearch
     interval 2
+    check_connection false
   ]
 
   def setup
@@ -190,6 +191,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     john
       password doe
       tag      raw.opensearch
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -228,6 +230,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     john
       password doe
       tag      raw.opensearch
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -249,6 +252,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     %{j+hn}
       password %{d@e}
       tag      raw.opensearch
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -271,6 +275,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       path     /es/
       port     123
       tag      raw.opensearch
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -295,6 +300,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     default_user
       password default_password
       tag      raw.opensearch
+      check_connection false
     }
     instance = driver(config).instance
 
@@ -323,6 +329,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
       user     default_user
       password default_password
       tag      raw.opensearch
+      check_connection false
     }
     instance = driver(config).instance
 

data/test/plugin/test_out_opensearch.rb

@@ -3977,4 +3977,26 @@ class OpenSearchOutputTest < Test::Unit::TestCase
       end
     }
   end
+
+  def test_no_aws_credentials_refresh_exception
+    # See https://github.com/fluent/fluent-plugin-opensearch/issues/129
+    endpoint_config =
+      Fluent::Config::Element.new('endpoint', '', {
+                                    'url' => "https://search-opensearch.aws.example.com/",
+                                    'region' => "local",
+                                    'access_key_id' => 'YOUR_AWESOME_KEY',
+                                    'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                    'refresh_credentials_interval' => '0',
+                                  }, [])
+    config = Fluent::Config::Element.new('ROOT', '**', { '@type' => 'opensearch' },
+                                         [endpoint_config])
+    # aws_credentials will be called twice in
+    # OpenSearchOutput#configure call, and in the 2nd call was changed not
+    # to emit exception. (instead, logging error) so check the error logs
+    flexmock(Fluent::Plugin::OpenSearchOutput).new_instances.should_receive(:aws_credentials)
+      .and_return(true).and_raise(::RuntimeError.new("No valid AWS credentials found."))
+    d = driver(config)
+    d.run
+    assert { d.logs.any?(/\[error\]: Failed to get new AWS credentials: No valid AWS credentials found.\n/) }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.5
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-21 00:00:00.000000000 Z
+date: 2024-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.1
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: 3.18.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: 3.18.1
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -203,6 +203,7 @@ files:
 - ".editorconfig"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/PULL_REQUEST_TEMPLATE/pull_request_template.md"
 - ".github/workflows/coverage.yaml"
 - ".github/workflows/issue-auto-closer.yml"
 - ".github/workflows/linux.yml"
@@ -212,9 +213,7 @@ files:
 - CONTRIBUTING.md
 - Gemfile
 - History.md
-- ISSUE_TEMPLATE.md
 - LICENSE.txt
-- PULL_REQUEST_TEMPLATE.md
 - README.OpenSearchGenID.md
 - README.OpenSearchInput.md
 - README.Troubleshooting.md

data/ISSUE_TEMPLATE.md

@@ -1,26 +0,0 @@
-(check apply)
-- [ ] read [the contribution guideline](https://github.com/fluent/fluent-plugin-opensearch/blob/master/CONTRIBUTING.md)
-
-#### Problem
-
-...
-
-#### Steps to replicate
-
-Provide example config and message
-
-#### Expected Behavior or What you need to ask
-
-...
-
-#### Using Fluentd and OpenSearch plugin versions
-
-* OS version
-* Bare Metal or within Docker or Kubernetes or others?
-* Fluentd v1.0 or later
-  * paste result of ``fluentd --version`` or ``td-agent --version``
-* OpenSearch plugin version
-  * paste boot log of fluentd or td-agent
-  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
-* OpenSearch version (optional)
-* OpenSearch template(s) (optional)