fluent-plugin-elasticsearch 4.0.7 → 4.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6a25db7c9a8c6d5d31365939cb60092440a299f4bacb52f1c3896927aa501a7
-  data.tar.gz: ed08dc5d2d526a9211a76dd36c0db7a90f1e50a17d6cc634790d3e6cba286175
+  metadata.gz: 1979708479afe32ade449552ade837bc9cfc8acff343cd248008abc579b794d3
+  data.tar.gz: 6ed1a94f6e8553ca72fb75e319ee954f5a8a676ef4ccc0a6a65ca4d041c101c2
 SHA512:
-  metadata.gz: c00a1d839a4adc7d47b3032099a81f6e73fe126ef0711d61518fe9034b915a39d9b6f50d19cc7a5dd39f8e903e243b745fc615a097794d4d456ba2a2a71a54c5
-  data.tar.gz: 84501757defe9cd9e78f95843cc39f3cfa68a32c09ef0235764c2e2bc2bc9e37a59241bd40ba74a87de8bbf968ea172db5e6aed55b233833029683213b0040ec
+  metadata.gz: e9957264a1ade9aa7e3b7cac60157c0adc9d9b2cd9e6d3d72ce73ca480e6ee32415bb94ad83c95ee7419e6b02b6695c28c39e2696945906e35e25c43421f70ca
+  data.tar.gz: 0efced0623b3312f1d767c82bb3ac9647e0082389bc6457052a4c74f5063a57ba21ff4effe881a18996c0a3f256ade017ec68b9df7423cb5c63869b3e49c1b07

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/macos.yml

@@ -0,0 +1,26 @@
+name: Testing on macOS
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6' ]
+        os:
+          - macOS-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/windows.yml

@@ -0,0 +1,26 @@
+name: Testing on Windows
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6' ]
+        os:
+          - windows-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 4.0.8
+- Handle compressable connection usable state (#743)
+- Use newer tls protocol versions (#739)
+- Add GitHub Actions file (#740)
+
 ### 4.0.7
 - Added http_backend_excon_nonblock config in out_elasticsearch (#733)
 

data/README.md

@@ -881,7 +881,11 @@ ssl_min_version TLSv1_2
 
 Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
 
-If they are *not* specified in the Elasticsearch plugin configuration, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
+If they are *not* specified in the Elasticsearch plugin configuration, `ssl_max_version` and `ssl_min_version` is set up with:
+
+In Elasticsearch plugin v4.0.8 or later with Ruby 2.5 or later environment, `ssl_max_version` should be `TLSv1_3` and `ssl_min_version` should be `TLSv1_2`.
+
+From Elasticsearch plugin v4.0.4 to v4.0.7 with Ruby 2.5 or later environment, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
 
 
 ### Proxy Support

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.7'
+  s.version       = '4.0.8'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_tls.rb

@@ -10,7 +10,7 @@ module Fluent::Plugin
                                [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
                              end
 
-    DEFAULT_VERSION = :TLSv1
+    DEFAULT_VERSION = :TLSv1_2
     METHODS_MAP = begin
                     # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
                     map = {
@@ -48,8 +48,8 @@ module Fluent::Plugin
       if USE_TLS_MINMAX_VERSION
         case
         when ssl_min_version.nil? && ssl_max_version.nil?
-          ssl_min_version = METHODS_MAP[ssl_version]
-          ssl_max_version = METHODS_MAP[ssl_version]
+          ssl_min_version = METHODS_MAP[:TLSv1_2]
+          ssl_max_version = METHODS_MAP[:TLSv1_3]
         when ssl_min_version && ssl_max_version.nil?
           raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
         when ssl_min_version.nil? && ssl_max_version

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -55,6 +55,7 @@ module Fluent::Plugin
     attr_reader :alias_indexes
     attr_reader :template_names
     attr_reader :ssl_version_options
+    attr_reader :compressable_connection
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -343,6 +344,7 @@ EOC
       @routing_key_name = configure_routing_key_name
       @meta_config_map = create_meta_config_map
       @current_config = nil
+      @compressable_connection = false
 
       @ignore_exception_classes = @ignore_exceptions.map do |exception|
         unless Object.const_defined?(exception)
@@ -511,13 +513,15 @@ EOC
       return Time.at(event_time).to_datetime
     end
 
-    def client(host = nil)
+    def client(host = nil, compress_connection = false)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
       @_es = nil unless is_existing_connection(connection_options[:hosts])
+      @_es = nil unless @compressable_connection == compress_connection
 
       @_es ||= begin
+        @compressable_connection = compress_connection
         @current_config = connection_options[:hosts].clone
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
         local_reload_connections = @reload_connections
@@ -525,7 +529,7 @@ EOC
           local_reload_connections = @reload_after
         end
 
-        gzip_headers = if compression
+        gzip_headers = if compress_connection
                          {'Content-Encoding' => 'gzip'}
                        else
                          {}
@@ -551,7 +555,7 @@ EOC
                                                                               },
                                                                               sniffer_class: @sniffer_class,
                                                                               serializer_class: @serializer_class,
-                                                                              compression: compression,
+                                                                              compression: compress_connection,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -947,7 +951,7 @@ EOC
                           data
                         end
 
-        response = client(info.host).bulk body: prepared_data, index: info.index
+        response = client(info.host, compression).bulk body: prepared_data, index: info.index
         log.on_trace { log.trace "bulk response: #{response}" }
 
         if response['errors']

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -35,16 +35,18 @@ module Fluent::Plugin
     end
 
 
-    def client(host = nil)
+    def client(host = nil, compress_connection = false)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
       @_es = nil unless is_existing_connection(connection_options[:hosts])
+      @_es = nil unless @compressable_connection == compress_connection
 
       @_es ||= begin
+        @compressable_connection = compress_connection
         @current_config = connection_options[:hosts].clone
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
-        gzip_headers = if compression
+        gzip_headers = if compress_connection
                          {'Content-Encoding' => 'gzip'}
                        else
                          {}
@@ -67,7 +69,7 @@ module Fluent::Plugin
                                                                                 password: @password,
                                                                                 scheme: @scheme
                                                                               },
-                                                                              compression: compression,
+                                                                              compression: compress_connection,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -228,7 +230,7 @@ module Fluent::Plugin
                         else
                           data
                         end
-        response = client(host).bulk body: prepared_data, index: index
+        response = client(host, compression).bulk body: prepared_data, index: index
         if response['errors']
           log.error "Could not push log to Elasticsearch: #{response}"
         end

data/test/plugin/test_elasticsearch_tls.rb

@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
       d = driver('')
       ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
       if @use_tls_minmax_version
-        assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION,
-                      min_version: OpenSSL::SSL::TLS1_VERSION}, ssl_version_options)
+        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                      min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
       else
         assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
       end

data/test/plugin/test_out_elasticsearch.rb

@@ -230,7 +230,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_nil instance.ssl_max_version
     assert_nil instance.ssl_min_version
     if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
-      assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
+      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
                    instance.ssl_version_options)
     else
       assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
@@ -284,7 +284,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+
+    stub_request(:post, "http://localhost:9200/_bulk").
+      to_return(status: 200, body: "", headers: {})
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
   end
 
   test 'check compression option is passed to transport' do
@@ -295,7 +304,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal true, instance.client.transport.options[:compression]
+    assert_equal false, instance.client.transport.options[:compression]
+
+    stub_request(:post, "http://localhost:9200/_bulk").
+      to_return(status: 200, body: "", headers: {})
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    assert_equal true, instance.client(nil, compressable).transport.options[:compression]
   end
 
   test 'configure Content-Type' do

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -101,7 +101,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_nil instance.ssl_max_version
     assert_nil instance.ssl_min_version
     if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
-      assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
+      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
                    instance.ssl_version_options)
     else
       assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
@@ -147,7 +147,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+
+    stub_request(:post, "http://localhost:9200/_bulk").
+      to_return(status: 200, body: "", headers: {})
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
   end
 
   test 'check compression option is passed to transport' do
@@ -158,7 +167,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal true, instance.client.transport.options[:compression]
+    assert_equal false, instance.client.transport.options[:compression]
+
+    stub_request(:post, "http://localhost:9200/_bulk").
+      to_return(status: 200, body: "", headers: {})
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    assert_equal true, instance.client(nil, compressable).transport.options[:compression]
   end
 
   test 'configure Content-Type' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.7
+  version: 4.0.8
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2020-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -135,6 +135,9 @@ extra_rdoc_files: []
 files:
 - ".coveralls.yml"
 - ".editorconfig"
+- ".github/workflows/linux.yml"
+- ".github/workflows/macos.yml"
+- ".github/workflows/windows.yml"
 - ".gitignore"
 - ".travis.yml"
 - CONTRIBUTING.md