fluent-plugin-elasticsearch 5.2.4 → 5.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f822334b8b2ed8169045953bd05cb383b6dd2068b513140429777f55e0f8d6bc
-  data.tar.gz: a8741a46f4f9fe0dd990c38844ac53d19fea9eb27cf7b3505f26d29d0ed7b83f
+  metadata.gz: a475837f544383a6d7c59f43f31c9d79a7c6154b2b8c851c266a9bbbe393e50a
+  data.tar.gz: 300ad2d07e3009f9d50433ceecee0e410fbd129d75c86d71646e659f58f19df3
 SHA512:
-  metadata.gz: ed5b593be5147c505fae9b563f87e831db8f110887026bab90a2ece444a1462cbd51a079134b62dab3e21c5a57ce78eabc371d9165f3d765ffe8c51493bc1344
-  data.tar.gz: 1fed7a68045103ed5a5bd9b49fe283468a87c45a2a68bf7a4922881deba2e93cedd9923afc046159465b5552d4d0bd4d47b3a86826b6622fcae02974eaa0ac91
+  metadata.gz: d1b12f9ee8582952388aadb13e06ef2538677bb761e9e55f1ea41214442c224b3334252a2b78bc35db6a334c19fa2183fff4a9311bd55bf8edeb44cbab30a1b5
+  data.tar.gz: eff7db364b8c307f01935b18221effc879e65d9ef300caba3cbdfc3a1c0fe7d84f3f4d8741fdb934bed553f58d0475a76b235598145e9f39c56c124766015540

data/.github/workflows/issue-auto-closer.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Autoclose issues that did not follow issue template
-      uses: roots/issue-closer-action@v1.1
+      uses: roots/issue-closer-action@v1.2
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."

data/.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0' ]
+        ruby: [ '3.0', '3.1', '3.2' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0' ]
+        ruby: [ '3.0', '3.1', '3.2' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0' ]
+        ruby: [ '3.0', '3.1', '3.2' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/Gemfile

@@ -4,7 +4,6 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'simplecov', require: false
-gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
 gem "elasticsearch-xpack" if ENV["USE_XPACK"]

data/History.md

@@ -2,6 +2,21 @@
 
 ### [Unreleased]
 
+### 5.3.0
+
+- Unpin `faraday` from v1, upgrade to v2. (#1012)
+  Note that if you can't migrate other plugins from `faraday` v1 yet, need to keep
+  fluent-plugin-elasticsearch v5.2.5.
+
+### 5.2.5
+
+- Update the name of parameter about `thread_pool`.
+  It should be `thread_pool.write.queue_size` instead of `thread_pool.bulk.queue_size` (#1000)
+- Replace obsolete `File.exists?` with `File.exist?` (#998)
+- Support `compression_level` for `elasticsearch_data_stream` (#995)
+- Honor `hosts` parameter for `elasticsearch_data_stream` (#1008)
+- Drop outdated coveralls (#1009)
+
 ### 5.2.4
 - Pin Faraday 1.10 (#987)
 - Increase errors metric on error response in data stream (#986)

data/README.md

@@ -4,7 +4,6 @@
 ![Testing on Windows](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Windows/badge.svg?branch=master)
 ![Testing on macOS](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20macOS/badge.svg?branch=master)
 ![Testing on Ubuntu](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Ubuntu/badge.svg?branch=master)
-[![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
 [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
 
 Send your logs to Elasticsearch (and search them with Kibana maybe?)
@@ -126,6 +125,8 @@ Current maintainers: [Hiroshi Hatake | @cosmo0920](https://github.com/cosmo0920)
 | >= 2.0.0 && < 3.2.3          | >= v0.14.20 | >= 2.1 |
 |  < 2.0.0                     | >= v0.12.0  | >= 1.9 |
 
+NOTE: Since fluent-plugin-elasticsearch 5.3.0, it requires faraday 2.0 or later.
+
 NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0.12 branch if you encountered 1.x version's bug.
 
 NOTE: This documentation is for fluent-plugin-elasticsearch 2.x or later. For 1.x documentation, please see [v0.12 branch](https://github.com/uken/fluent-plugin-elasticsearch/tree/v0.12).
@@ -201,7 +202,7 @@ You can specify Elasticsearch port by this parameter.
 ### cloud_id
 
 ```
-cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng== 
+cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng==
 ```
 
 You can specify Elasticsearch cloud_id by this parameter.
@@ -1179,11 +1180,11 @@ Advanced users can increase its capacity, but normal users should follow default
 
 If you want to increase it and forcibly retrying bulk request, please consider to change `unrecoverable_error_types` parameter from default value.
 
-Change default value of `thread_pool.bulk.queue_size` in elasticsearch.yml:
+Change default value of `thread_pool.write.queue_size` in elasticsearch.yml:
 e.g.)
 
 ```yaml
-thread_pool.bulk.queue_size: 1000
+thread_pool.write.queue_size: 1000
 ```
 
 Then, remove `es_rejected_execution_exception` from `unrecoverable_error_types` parameter:
@@ -1531,7 +1532,7 @@ This parameter is mandatory for `elasticsearch_data_stream`.
 
 ### data_stream_template_name
 
-You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template. 
+You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template.
 
 Default value is `data_stream_name`.
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.2.4'
+  s.version       = '5.3.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -23,7 +23,8 @@ Gem::Specification.new do |s|
   s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
 
   s.add_runtime_dependency 'fluentd', '>= 0.14.22'
-  s.add_runtime_dependency "faraday", "~> 1.10"
+  s.add_runtime_dependency 'faraday', '>= 2.0.0'
+  s.add_runtime_dependency 'faraday-excon', '>= 2.0.0'
   s.add_runtime_dependency 'excon', '>= 0'
   s.add_runtime_dependency 'elasticsearch'
 

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -3,7 +3,7 @@ require_relative './elasticsearch_error'
 
 module Fluent::ElasticsearchIndexTemplate
   def get_template(template_file)
-    if !File.exists?(template_file)
+    if !File.exist?(template_file)
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
@@ -11,7 +11,7 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def get_custom_template(template_file, customize_template)
-    if !File.exists?(template_file)
+    if !File.exist?(template_file)
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')

data/lib/fluent/plugin/in_elasticsearch.rb

@@ -1,5 +1,6 @@
 require 'elasticsearch'
 
+require 'faraday/excon'
 require 'fluent/log-ext'
 require 'fluent/plugin/input'
 require_relative 'elasticsearch_constants'

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -32,6 +32,8 @@ begin
 rescue LoadError
 end
 
+require 'faraday/excon'
+
 module Fluent::Plugin
   class ElasticsearchOutput < Output
     class RecoverableRequestFailure < StandardError; end
@@ -471,7 +473,7 @@ EOC
       when :excon
         { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
       when :typhoeus
-        require 'typhoeus'
+        require 'faraday/typhoeus'
         { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
       end
     rescue LoadError => ex

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -219,7 +219,11 @@ module Fluent::Plugin
       data_stream_ilm_name = @data_stream_ilm_name
       host = nil
       if @use_placeholder
-        host = extract_placeholders(@host, chunk)
+        host = if @hosts
+                 extract_placeholders(@hosts, chunk)
+               else
+                 extract_placeholders(@host, chunk)
+               end
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
         data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
@@ -257,12 +261,18 @@ module Fluent::Plugin
         end
       end
 
+      prepared_data = if compression
+        gzip(bulk_message)
+      else
+        bulk_message
+      end
+
       params = {
         index: data_stream_name,
-        body: bulk_message
+        body: prepared_data
       }
       begin
-        response = client(host).bulk(params)
+        response = client(host, compression).bulk(params)
         if response['errors']
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
           @num_errors_metrics.inc

data/test/helper.rb

@@ -5,9 +5,6 @@ SimpleCov.start do
   end
 end
 
-require 'coveralls'
-Coveralls.wear!
-
 # needs to be after simplecov but before test/unit, because fluentd sets default
 # encoding to ASCII-8BIT, but coverall might load git data which could contain a
 # UTF-8 character

data/test/plugin/test_out_elasticsearch.rb

@@ -871,55 +871,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
-  class GetElasticsearchVersionTest < self
-    def create_driver(conf='', client_version="\"5.0\"")
-      # For request stub to detect compatibility.
-      @client_version ||= client_version
-      # Ensure original implementation existence.
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def detect_es_major_version
-          @_es_info ||= client.info
-          unless version = @_es_info.dig("version", "number")
-            version = @default_elasticsearch_version
-          end
-          version.to_i
-        end
-      CODE
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def client_library_version
-          #{@client_version}
-        end
-      CODE
-      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
-    end
-
-    def test_retry_get_es_version
-      config = %{
-        host            logs.google.com
-        port            778
-        scheme          https
-        path            /es/
-        user            john
-        password        doe
-        verify_es_version_at_startup true
-        max_retry_get_es_version 3
-      }
-
-      connection_resets = 0
-      stub_request(:get, "https://logs.google.com:778/es//").
-        with(basic_auth: ['john', 'doe']) do |req|
-        connection_resets += 1
-        raise Faraday::ConnectionFailed, "Test message"
-      end
-
-      assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
-        create_driver(config)
-      end
-
-      assert_equal(4, connection_resets)
-    end
-  end
-
   class GetElasticsearchIncompatibleVersionTest < self
     def create_driver(conf='', client_version="7.14")
       # For request stub to detect compatibility.
@@ -981,63 +932,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
-  class GetElasticsearchVersionWithFallbackTest < self
-    def create_driver(conf='', client_version="\"5.0\"")
-      # For request stub to detect compatibility.
-      @client_version ||= client_version
-      # Ensure original implementation existence.
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def detect_es_major_version
-          @_es_info ||= client.info
-          unless version = @_es_info.dig("version", "number")
-            version = @default_elasticsearch_version
-          end
-          version.to_i
-        end
-      CODE
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def client_library_version
-          #{@client_version}
-        end
-      CODE
-      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
-    end
-
-    data("Elasticsearch 5" => ["5.0", 5],
-         "Elasticsearch 6" => ["6.0", 6],
-         "Elasticsearch 7" => ["7.0", 7],
-         "Elasticsearch 8" => ["8.0", 8])
-    def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
-      client_version, es_major_version = data
-      config = %{
-        host            logs.google.com
-        port            778
-        scheme          https
-        path            /es/
-        user            john
-        password        doe
-        verify_es_version_at_startup true
-        max_retry_get_es_version 2
-        fail_on_detecting_es_version_retry_exceed false
-        default_elasticsearch_version #{es_major_version}
-        @log_level info
-      }
-
-      connection_resets = 0
-      stub_request(:get, "https://logs.google.com:778/es//").
-        with(basic_auth: ['john', 'doe']) do |req|
-        connection_resets += 1
-        raise Faraday::ConnectionFailed, "Test message"
-      end
-
-      d = create_driver(config, client_version)
-
-      assert_equal es_major_version, d.instance.default_elasticsearch_version
-      assert_equal 3, connection_resets
-      assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
-    end
-  end
-
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])
   def test_template_already_present(data)
@@ -2143,7 +2037,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
       assert_equal ["logstash-tag1", "logstash-tag2"], driver.instance.alias_indexes
 
-      assert_requested(elastic_request)
+      assert_requested(elastic_request, at_least_times: 1, at_most_times: 2)
     end
 
 
@@ -3056,7 +2950,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       assert_equal('mylogs-test-tag2', index_cmds.first['index']['_index'])
       assert_equal ["mylogs-test-tag1", "mylogs-test-tag2"], driver.instance.alias_indexes
 
-      assert_requested(elastic_request)
+      assert_requested(elastic_request, at_least_times: 1, at_most_times: 2)
     end
 
     data("legacy_template" => [true, "_template"],
@@ -3375,49 +3269,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
-  data("legacy_template" => [true, "_template"],
-       "new_template"    => [false, "_index_template"])
-  def test_template_retry_install_fails(data)
-    use_legacy_template_flag, endpoint = data
-    if !use_legacy_template_flag && Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.8.0")
-      omit "elastisearch-ruby v7.8.0 or later is needed."
-    end
-    cwd = File.dirname(__FILE__)
-    template_file = if use_legacy_template_flag
-                      File.join(cwd, 'test_template.json')
-                    else
-                      File.join(cwd, 'test_index_template.json')
-                    end
-
-    config = %{
-      host            logs.google.com
-      port            778
-      scheme          https
-      path            /es/
-      user            john
-      password        doe
-      template_name   logstash
-      template_file   #{template_file}
-      max_retry_putting_template 3
-      use_legacy_template #{use_legacy_template_flag}
-    }
-
-    connection_resets = 0
-    # check if template exists
-    stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
-      .with(basic_auth: ['john', 'doe']) do |req|
-      connection_resets += 1
-      raise Faraday::ConnectionFailed, "Test message"
-    end
-    stub_elastic_info("https://logs.google.com:778/es//")
-
-    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
-      driver(config)
-    end
-
-    assert_equal(4, connection_resets)
-  end
-
   transport_errors_handled_separately = [TRANSPORT_CLASS::Transport::Errors::NotFound]
   transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map { |err| [err, TRANSPORT_CLASS::Transport::Errors.const_get(err)]  }
   transport_errors_hash = Hash[transport_errors.select { |err| !transport_errors_handled_separately.include?(err[1]) } ]
@@ -3460,47 +3311,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(1, retries)
   end
 
-  data("legacy_template" => [true, "_template"],
-       "new_template"    => [false, "_index_template"])
-  def test_template_retry_install_does_not_fail(data)
-    use_legacy_template_flag, endpoint = data
-    if !use_legacy_template_flag && Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.8.0")
-      omit "elastisearch-ruby v7.8.0 or later is needed."
-    end
-    cwd = File.dirname(__FILE__)
-    template_file = if use_legacy_template_flag
-                      File.join(cwd, 'test_template.json')
-                    else
-                      File.join(cwd, 'test_index_template.json')
-                    end
-
-    config = %{
-      host            logs.google.com
-      port            778
-      scheme          https
-      path            /es/
-      user            john
-      password        doe
-      template_name   logstash
-      template_file   #{template_file}
-      max_retry_putting_template 3
-      fail_on_putting_template_retry_exceed false
-      use_legacy_template #{use_legacy_template_flag}
-    }
-
-    connection_resets = 0
-    # check if template exists
-    stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
-      .with(basic_auth: ['john', 'doe']) do |req|
-      connection_resets += 1
-      raise Faraday::ConnectionFailed, "Test message"
-    end
-    stub_elastic_info("https://logs.google.com:778/es//")
-
-    driver(config)
-
-    assert_equal(4, connection_resets)
-  end
 
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -57,6 +57,14 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     }.configure(conf)
   end
 
+  def elasticsearch_transport_layer_decoupling?
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+  end
+
+  def elastic_transport_layer?
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+  end
+
   def sample_data_stream
     {
       'data_streams': [
@@ -172,6 +180,18 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_data_stream(datastream_name)
   end
 
+  def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
+    if @index_command_counts == nil
+      @index_command_counts = {}
+      @index_command_counts.default = 0
+    end
+
+    stub_request(:post, url).with do |req|
+      index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_command_counts[url] += index_cmds.size
+    end
+  end
+
   def data_stream_supported?
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
   end
@@ -525,6 +545,114 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  def test_configure_compression
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_compression
+    }
+
+    stub_default
+
+    assert_equal true, driver(config).instance.compression
+  end
+
+  def test_check_compression_strategy
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_speed
+    }
+
+    stub_default
+    stub_bulk_feed
+
+    assert_equal Zlib::BEST_SPEED, driver(config).instance.compression_strategy
+  end
+
+  def test_check_content_encoding_header_with_compression
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_compression
+    }
+
+    stub_default
+    stub_request(:post, "http://localhost:9200/foo/_bulk").
+      to_return(status: 200, body: "", headers: {})
+
+    instance = driver(config).instance
+
+    if elastic_transport_layer?
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
+
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    if elastic_transport_layer?
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
+  end
+
+  def test_check_compression_option_is_passed_to_transport
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_compression
+    }
+
+    stub_default
+    stub_request(:post, "http://localhost:9200/foo/_bulk").
+      to_return(status: 200, body: "", headers: {})
+
+    instance = driver(config).instance
+
+    if elastic_transport_layer?
+      assert_equal false, instance.client.transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal false, instance.client.transport.transport.options[:compression]
+    else
+      assert_equal false, instance.client.transport.options[:compression]
+    end
+
+
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    if elastic_transport_layer?
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
+    else
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    end
+  end
+
   def test_existent_data_stream
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -712,40 +840,93 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records.length
   end
 
-  def test_template_retry_install_fails
-    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+  def test_placeholder_writes_to_multi_hosts
+    stub_default("foo_bar", "foo_tpl_bar")
+    hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
+    hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
+    hosts.each do |host_info|
+      host, port = host_info
+      stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/foo_bar/_bulk")
+      stub_elastic_info("http://#{host}:#{port}/")
+      stub_request(:get, "http://#{host}:#{port}/_data_stream/foo_bar").
+        to_return(status: 200, body: "", headers: {})
+    end
+
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo_${key1}',
+      'data_stream_template_name' => 'foo_tpl_${key1}',
+      'hosts' => "#{hosts_string}"
+    }, [config_element('buffer', 'tag,key1', {
+      'timekey' => '1d'
+    }, [])])
+    driver(conf).run(default_tag: 'test') do
+      hashes = {
+        'age' => rand(100),
+        'key1' => 'bar'
+      }
+      1000.times do
+        driver.feed(sample_record.merge(hashes))
+      end
+    end
+
+    # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
+    # it's Fluentd test driver's constraint
+    # so @index_command_counts.size is always 1
+    assert(@index_command_counts.size > 0, "not working with hosts options")
+
+    total = 0
+    @index_command_counts.each do |_, count|
+      total += count
+    end
+    assert_equal(2000, total)
+  end
 
-    cwd = File.dirname(__FILE__)
-    template_file = File.join(cwd, 'test_index_template.json')
+  # gzip compress data
+  def gzip(string, strategy)
+    wio = StringIO.new("w")
+    w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
+    w_gz.write(string)
+    w_gz.close
+    wio.string
+  end
+
+  def test_writes_to_data_stream_with_compression
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
     config = %{
-      host                       logs.google.com
-      port                       778
-      scheme                     https
       data_stream_name           foo
-      data_stream_ilm_name       foo_ilm_policy
       data_stream_template_name  foo_tpl
-      user                       john
-      password                   doe
-      template_name              logstash
-      template_file              #{template_file}
-      max_retry_putting_template 3
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          default_compression
     }
 
-    connection_resets = 0
-    # check if template exists
-    stub_request(:get, "https://logs.google.com:778/_index_template/logstash")
-      .with(basic_auth: ['john', 'doe']) do |req|
-      connection_resets += 1
-      raise Faraday::ConnectionFailed, "Test message"
-    end
-    stub_elastic_info("https://logs.google.com:778/")
+    bodystr = %({
+          "took" : 500,
+          "errors" : false,
+          "items" : [
+            {
+              "create": {
+                "_index" : "fluentd",
+                "_type"  : "fluentd"
+              }
+            }
+           ]
+        })
 
-    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
-      driver(config)
+    compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
+
+    stub_default
+    elastic_request = stub_request(:post, "http://localhost:9200/foo/_bulk").
+        to_return(:status => 200, :headers => {'Content-Type' => 'application/json'}, :body => compressed_body)
+
+    driver(config)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
     end
 
-    assert_equal(4, connection_resets)
+    assert_requested(elastic_request)
   end
 
   def test_doesnt_update_ilm_policy_if_overwrite_unset

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.2.4
+  version: 5.3.0
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-18 00:00:00.000000000 Z
+date: 2023-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,16 +30,30 @@ dependencies:
   name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: faraday-excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
@@ -161,7 +175,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".coveralls.yml"
 - ".editorconfig"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
@@ -238,7 +251,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector

data/.coveralls.yml

@@ -1,2 +0,0 @@
-service_name: travis-ci
-