fluent-plugin-elasticsearch 5.2.3 → 5.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e2737dc1ced4c50a3db85d71c83351e77a606d581fbd768261ca6613b1700f8
-  data.tar.gz: 49a84ff1ea184c4afd43e69fb6cbb89559926bbd6ac063196bd777281be17d2b
+  metadata.gz: 9e6354575b148be2666c68a6a3c72fcda76139517570203751217cb130fcc070
+  data.tar.gz: 32a0c118516bb5dca54e402e655309565d08f2e1d01caf467a39f2c15d58f26f
 SHA512:
-  metadata.gz: 157dbe3ab067ec279f2051a8b6c6ac25538821903b48e96a09687b547270796d7f6972d0707225c99cd36694e8a8668d32dc9ebba3599d24f08c814712c0849a
-  data.tar.gz: 73c611531aa95d5d03d8bd5146ab5f6812c30b3be634f5d1ca139b2ffc6206ccebb8915c4fa1e78d2d53706f6c14b7203342d235bd217a89ac9bf0135e3fec21
+  metadata.gz: 444ef6890245dd454adde27b32478369c913f8a4a4b55bd51025390fcf8850ffceb08203fd85265a3fbbdb9f8f4e7bf77526da2119860de1a6c35fe525275ef9
+  data.tar.gz: a78b3e755f9a9f4eef63fb694cc39ceb6db5bc3d565f46f1934b234d00ddb48ea76235db8623f84e70a64afd844f29dc90b75d4d237194054e0777a92f5d199c

data/.github/workflows/issue-auto-closer.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Autoclose issues that did not follow issue template
-      uses: roots/issue-closer-action@v1.1
+      uses: roots/issue-closer-action@v1.2
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."

data/.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0' ]
+        ruby: [ '2.7', '3.0', '3.1', '3.2' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0' ]
+        ruby: [ '2.7', '3.0', '3.1', '3.2' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0' ]
+        ruby: [ '2.7', '3.0', '3.1', '3.2' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/Gemfile

@@ -4,7 +4,6 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'simplecov', require: false
-gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
 gem "elasticsearch-xpack" if ENV["USE_XPACK"]

data/History.md

@@ -2,6 +2,19 @@
 
 ### [Unreleased]
 
+### 5.2.5
+
+- Update the name of parameter about `thread_pool`.
+  It should be `thread_pool.write.queue_size` instead of `thread_pool.bulk.queue_size` (#1000)
+- Replace obsolete `File.exists?` with `File.exist?` (#998)
+- Support `compression_level` for `elasticsearch_data_stream` (#995)
+- Honor `hosts` parameter for `elasticsearch_data_stream` (#1008)
+- Drop outdated coveralls (#1009)
+
+### 5.2.4
+- Pin Faraday 1.10 (#987)
+- Increase errors metric on error response in data stream (#986)
+
 ### 5.2.3
 - Bump actions/checkout from 2 to 3 (#978)
 - chore: Included githubactions in the dependabot config (#977)

data/README.md

@@ -4,7 +4,6 @@
 ![Testing on Windows](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Windows/badge.svg?branch=master)
 ![Testing on macOS](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20macOS/badge.svg?branch=master)
 ![Testing on Ubuntu](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Ubuntu/badge.svg?branch=master)
-[![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
 [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
 
 Send your logs to Elasticsearch (and search them with Kibana maybe?)
@@ -201,7 +200,7 @@ You can specify Elasticsearch port by this parameter.
 ### cloud_id
 
 ```
-cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng== 
+cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng==
 ```
 
 You can specify Elasticsearch cloud_id by this parameter.
@@ -1179,11 +1178,11 @@ Advanced users can increase its capacity, but normal users should follow default
 
 If you want to increase it and forcibly retrying bulk request, please consider to change `unrecoverable_error_types` parameter from default value.
 
-Change default value of `thread_pool.bulk.queue_size` in elasticsearch.yml:
+Change default value of `thread_pool.write.queue_size` in elasticsearch.yml:
 e.g.)
 
 ```yaml
-thread_pool.bulk.queue_size: 1000
+thread_pool.write.queue_size: 1000
 ```
 
 Then, remove `es_rejected_execution_exception` from `unrecoverable_error_types` parameter:
@@ -1531,7 +1530,7 @@ This parameter is mandatory for `elasticsearch_data_stream`.
 
 ### data_stream_template_name
 
-You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template. 
+You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template.
 
 Default value is `data_stream_name`.
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.2.3'
+  s.version       = '5.2.5'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
 
   s.add_runtime_dependency 'fluentd', '>= 0.14.22'
+  s.add_runtime_dependency "faraday", "~> 1.10"
   s.add_runtime_dependency 'excon', '>= 0'
   s.add_runtime_dependency 'elasticsearch'
 

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -3,7 +3,7 @@ require_relative './elasticsearch_error'
 
 module Fluent::ElasticsearchIndexTemplate
   def get_template(template_file)
-    if !File.exists?(template_file)
+    if !File.exist?(template_file)
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
@@ -11,7 +11,7 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def get_custom_template(template_file, customize_template)
-    if !File.exists?(template_file)
+    if !File.exist?(template_file)
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -219,7 +219,11 @@ module Fluent::Plugin
       data_stream_ilm_name = @data_stream_ilm_name
       host = nil
       if @use_placeholder
-        host = extract_placeholders(@host, chunk)
+        host = if @hosts
+                 extract_placeholders(@hosts, chunk)
+               else
+                 extract_placeholders(@host, chunk)
+               end
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
         data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
@@ -257,14 +261,21 @@ module Fluent::Plugin
         end
       end
 
+      prepared_data = if compression
+        gzip(bulk_message)
+      else
+        bulk_message
+      end
+
       params = {
         index: data_stream_name,
-        body: bulk_message
+        body: prepared_data
       }
       begin
-        response = client(host).bulk(params)
+        response = client(host, compression).bulk(params)
         if response['errors']
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
+          @num_errors_metrics.inc
         end
       rescue => e
         raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"

data/test/helper.rb

@@ -5,9 +5,6 @@ SimpleCov.start do
   end
 end
 
-require 'coveralls'
-Coveralls.wear!
-
 # needs to be after simplecov but before test/unit, because fluentd sets default
 # encoding to ASCII-8BIT, but coverall might load git data which could contain a
 # UTF-8 character

data/test/plugin/test_out_elasticsearch.rb

@@ -871,55 +871,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
-  class GetElasticsearchVersionTest < self
-    def create_driver(conf='', client_version="\"5.0\"")
-      # For request stub to detect compatibility.
-      @client_version ||= client_version
-      # Ensure original implementation existence.
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def detect_es_major_version
-          @_es_info ||= client.info
-          unless version = @_es_info.dig("version", "number")
-            version = @default_elasticsearch_version
-          end
-          version.to_i
-        end
-      CODE
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def client_library_version
-          #{@client_version}
-        end
-      CODE
-      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
-    end
-
-    def test_retry_get_es_version
-      config = %{
-        host            logs.google.com
-        port            778
-        scheme          https
-        path            /es/
-        user            john
-        password        doe
-        verify_es_version_at_startup true
-        max_retry_get_es_version 3
-      }
-
-      connection_resets = 0
-      stub_request(:get, "https://logs.google.com:778/es//").
-        with(basic_auth: ['john', 'doe']) do |req|
-        connection_resets += 1
-        raise Faraday::ConnectionFailed, "Test message"
-      end
-
-      assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
-        create_driver(config)
-      end
-
-      assert_equal(4, connection_resets)
-    end
-  end
-
   class GetElasticsearchIncompatibleVersionTest < self
     def create_driver(conf='', client_version="7.14")
       # For request stub to detect compatibility.
@@ -981,63 +932,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
-  class GetElasticsearchVersionWithFallbackTest < self
-    def create_driver(conf='', client_version="\"5.0\"")
-      # For request stub to detect compatibility.
-      @client_version ||= client_version
-      # Ensure original implementation existence.
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def detect_es_major_version
-          @_es_info ||= client.info
-          unless version = @_es_info.dig("version", "number")
-            version = @default_elasticsearch_version
-          end
-          version.to_i
-        end
-      CODE
-      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
-        def client_library_version
-          #{@client_version}
-        end
-      CODE
-      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
-    end
-
-    data("Elasticsearch 5" => ["5.0", 5],
-         "Elasticsearch 6" => ["6.0", 6],
-         "Elasticsearch 7" => ["7.0", 7],
-         "Elasticsearch 8" => ["8.0", 8])
-    def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
-      client_version, es_major_version = data
-      config = %{
-        host            logs.google.com
-        port            778
-        scheme          https
-        path            /es/
-        user            john
-        password        doe
-        verify_es_version_at_startup true
-        max_retry_get_es_version 2
-        fail_on_detecting_es_version_retry_exceed false
-        default_elasticsearch_version #{es_major_version}
-        @log_level info
-      }
-
-      connection_resets = 0
-      stub_request(:get, "https://logs.google.com:778/es//").
-        with(basic_auth: ['john', 'doe']) do |req|
-        connection_resets += 1
-        raise Faraday::ConnectionFailed, "Test message"
-      end
-
-      d = create_driver(config, client_version)
-
-      assert_equal es_major_version, d.instance.default_elasticsearch_version
-      assert_equal 3, connection_resets
-      assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
-    end
-  end
-
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])
   def test_template_already_present(data)
@@ -2143,7 +2037,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
       assert_equal ["logstash-tag1", "logstash-tag2"], driver.instance.alias_indexes
 
-      assert_requested(elastic_request)
+      assert_requested(elastic_request, at_least_times: 1, at_most_times: 2)
     end
 
 
@@ -3056,7 +2950,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       assert_equal('mylogs-test-tag2', index_cmds.first['index']['_index'])
       assert_equal ["mylogs-test-tag1", "mylogs-test-tag2"], driver.instance.alias_indexes
 
-      assert_requested(elastic_request)
+      assert_requested(elastic_request, at_least_times: 1, at_most_times: 2)
     end
 
     data("legacy_template" => [true, "_template"],
@@ -3375,49 +3269,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
-  data("legacy_template" => [true, "_template"],
-       "new_template"    => [false, "_index_template"])
-  def test_template_retry_install_fails(data)
-    use_legacy_template_flag, endpoint = data
-    if !use_legacy_template_flag && Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.8.0")
-      omit "elastisearch-ruby v7.8.0 or later is needed."
-    end
-    cwd = File.dirname(__FILE__)
-    template_file = if use_legacy_template_flag
-                      File.join(cwd, 'test_template.json')
-                    else
-                      File.join(cwd, 'test_index_template.json')
-                    end
-
-    config = %{
-      host            logs.google.com
-      port            778
-      scheme          https
-      path            /es/
-      user            john
-      password        doe
-      template_name   logstash
-      template_file   #{template_file}
-      max_retry_putting_template 3
-      use_legacy_template #{use_legacy_template_flag}
-    }
-
-    connection_resets = 0
-    # check if template exists
-    stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
-      .with(basic_auth: ['john', 'doe']) do |req|
-      connection_resets += 1
-      raise Faraday::ConnectionFailed, "Test message"
-    end
-    stub_elastic_info("https://logs.google.com:778/es//")
-
-    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
-      driver(config)
-    end
-
-    assert_equal(4, connection_resets)
-  end
-
   transport_errors_handled_separately = [TRANSPORT_CLASS::Transport::Errors::NotFound]
   transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map { |err| [err, TRANSPORT_CLASS::Transport::Errors.const_get(err)]  }
   transport_errors_hash = Hash[transport_errors.select { |err| !transport_errors_handled_separately.include?(err[1]) } ]
@@ -3460,47 +3311,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(1, retries)
   end
 
-  data("legacy_template" => [true, "_template"],
-       "new_template"    => [false, "_index_template"])
-  def test_template_retry_install_does_not_fail(data)
-    use_legacy_template_flag, endpoint = data
-    if !use_legacy_template_flag && Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.8.0")
-      omit "elastisearch-ruby v7.8.0 or later is needed."
-    end
-    cwd = File.dirname(__FILE__)
-    template_file = if use_legacy_template_flag
-                      File.join(cwd, 'test_template.json')
-                    else
-                      File.join(cwd, 'test_index_template.json')
-                    end
-
-    config = %{
-      host            logs.google.com
-      port            778
-      scheme          https
-      path            /es/
-      user            john
-      password        doe
-      template_name   logstash
-      template_file   #{template_file}
-      max_retry_putting_template 3
-      fail_on_putting_template_retry_exceed false
-      use_legacy_template #{use_legacy_template_flag}
-    }
-
-    connection_resets = 0
-    # check if template exists
-    stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
-      .with(basic_auth: ['john', 'doe']) do |req|
-      connection_resets += 1
-      raise Faraday::ConnectionFailed, "Test message"
-    end
-    stub_elastic_info("https://logs.google.com:778/es//")
-
-    driver(config)
-
-    assert_equal(4, connection_resets)
-  end
 
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -57,6 +57,14 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     }.configure(conf)
   end
 
+  def elasticsearch_transport_layer_decoupling?
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+  end
+
+  def elastic_transport_layer?
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+  end
+
   def sample_data_stream
     {
       'data_streams': [
@@ -172,6 +180,18 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_data_stream(datastream_name)
   end
 
+  def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
+    if @index_command_counts == nil
+      @index_command_counts = {}
+      @index_command_counts.default = 0
+    end
+
+    stub_request(:post, url).with do |req|
+      index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_command_counts[url] += index_cmds.size
+    end
+  end
+
   def data_stream_supported?
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
   end
@@ -525,6 +545,114 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  def test_configure_compression
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_compression
+    }
+
+    stub_default
+
+    assert_equal true, driver(config).instance.compression
+  end
+
+  def test_check_compression_strategy
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_speed
+    }
+
+    stub_default
+    stub_bulk_feed
+
+    assert_equal Zlib::BEST_SPEED, driver(config).instance.compression_strategy
+  end
+
+  def test_check_content_encoding_header_with_compression
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_compression
+    }
+
+    stub_default
+    stub_request(:post, "http://localhost:9200/foo/_bulk").
+      to_return(status: 200, body: "", headers: {})
+
+    instance = driver(config).instance
+
+    if elastic_transport_layer?
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
+
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    if elastic_transport_layer?
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
+  end
+
+  def test_check_compression_option_is_passed_to_transport
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          best_compression
+    }
+
+    stub_default
+    stub_request(:post, "http://localhost:9200/foo/_bulk").
+      to_return(status: 200, body: "", headers: {})
+
+    instance = driver(config).instance
+
+    if elastic_transport_layer?
+      assert_equal false, instance.client.transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal false, instance.client.transport.transport.options[:compression]
+    else
+      assert_equal false, instance.client.transport.options[:compression]
+    end
+
+
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    if elastic_transport_layer?
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
+    else
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    end
+  end
+
   def test_existent_data_stream
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -712,40 +840,93 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records.length
   end
 
-  def test_template_retry_install_fails
-    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+  def test_placeholder_writes_to_multi_hosts
+    stub_default("foo_bar", "foo_tpl_bar")
+    hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
+    hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
+    hosts.each do |host_info|
+      host, port = host_info
+      stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/foo_bar/_bulk")
+      stub_elastic_info("http://#{host}:#{port}/")
+      stub_request(:get, "http://#{host}:#{port}/_data_stream/foo_bar").
+        to_return(status: 200, body: "", headers: {})
+    end
+
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo_${key1}',
+      'data_stream_template_name' => 'foo_tpl_${key1}',
+      'hosts' => "#{hosts_string}"
+    }, [config_element('buffer', 'tag,key1', {
+      'timekey' => '1d'
+    }, [])])
+    driver(conf).run(default_tag: 'test') do
+      hashes = {
+        'age' => rand(100),
+        'key1' => 'bar'
+      }
+      1000.times do
+        driver.feed(sample_record.merge(hashes))
+      end
+    end
+
+    # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
+    # it's Fluentd test driver's constraint
+    # so @index_command_counts.size is always 1
+    assert(@index_command_counts.size > 0, "not working with hosts options")
+
+    total = 0
+    @index_command_counts.each do |_, count|
+      total += count
+    end
+    assert_equal(2000, total)
+  end
 
-    cwd = File.dirname(__FILE__)
-    template_file = File.join(cwd, 'test_index_template.json')
+  # gzip compress data
+  def gzip(string, strategy)
+    wio = StringIO.new("w")
+    w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
+    w_gz.write(string)
+    w_gz.close
+    wio.string
+  end
+
+  def test_writes_to_data_stream_with_compression
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
     config = %{
-      host                       logs.google.com
-      port                       778
-      scheme                     https
       data_stream_name           foo
-      data_stream_ilm_name       foo_ilm_policy
       data_stream_template_name  foo_tpl
-      user                       john
-      password                   doe
-      template_name              logstash
-      template_file              #{template_file}
-      max_retry_putting_template 3
+      data_stream_ilm_name       foo_ilm_policy
+      compression_level          default_compression
     }
 
-    connection_resets = 0
-    # check if template exists
-    stub_request(:get, "https://logs.google.com:778/_index_template/logstash")
-      .with(basic_auth: ['john', 'doe']) do |req|
-      connection_resets += 1
-      raise Faraday::ConnectionFailed, "Test message"
-    end
-    stub_elastic_info("https://logs.google.com:778/")
+    bodystr = %({
+          "took" : 500,
+          "errors" : false,
+          "items" : [
+            {
+              "create": {
+                "_index" : "fluentd",
+                "_type"  : "fluentd"
+              }
+            }
+           ]
+        })
 
-    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
-      driver(config)
+    compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
+
+    stub_default
+    elastic_request = stub_request(:post, "http://localhost:9200/foo/_bulk").
+        to_return(:status => 200, :headers => {'Content-Type' => 'application/json'}, :body => compressed_body)
+
+    driver(config)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
     end
 
-    assert_equal(4, connection_resets)
+    assert_requested(elastic_request)
   end
 
   def test_doesnt_update_ilm_policy_if_overwrite_unset

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.2.3
+  version: 5.2.5
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-16 00:00:00.000000000 Z
+date: 2023-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -26,6 +26,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.22
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
@@ -147,7 +161,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".coveralls.yml"
 - ".editorconfig"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
@@ -224,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector

data/.coveralls.yml

@@ -1,2 +0,0 @@
-service_name: travis-ci
-