fluent-plugin-elasticsearch 3.6.0 → 3.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22474d12620f4cc8c2a2224b19e5e98534bd38d86f9489ace710b50139ea1422
-  data.tar.gz: 85c5043ca7ab0905efb9e2093c3b04d83dadae645e5aa2e580dbd601db77fe88
+  metadata.gz: 1073f1770d5c380f496b57f111e1792812b870392c5996111465ce9cd2cfa0c6
+  data.tar.gz: 1f8e70ec6516ba252d7f2b722ec7c85d3830e3eb0b616b67c909663ac5db9461
 SHA512:
-  metadata.gz: f22d3fe7adbb44c8b38f9c61757857937c3293c762dfba6842081e08df65ceddd3c75a0bfbf791c325fa1d82380e6d5b09ed7a3a774f03706a095cb8553323ac
-  data.tar.gz: 5f418e39091ac5bdceb2869604480e9b822c7654ba35da44c0ac636b88148168bee352c8bd7c3197fc5642019ab2a145430b4a88a722902c609457f1930eed2d
+  metadata.gz: 291e4a936373fc096718c4fa9db816874a7e53d2f950b9a8e5bfa3f7a6830d98a4d79ab4b36d16b5615cfd831e364b5e545ff91a49866f26805a5aae73670338
+  data.tar.gz: ea66679ef5406d950923e9f255532116f5378619b712f063c64b01997313aecf5ec53c3e04a7c1313125cda4370e058c200d539bd741c2afd55a08015a42c26b

data/History.md

@@ -1,6 +1,11 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+### 3.6.1
+- retry upsert on recoverable error. (#667)
+- Allow `_index` in chunk_keys (#665)
+- Support compression feature (#664)
+
 ### 3.6.0
 - Set order in newly created templates (#660)
 - Merge Support index lifecycle management into master (#659)

data/README.md

@@ -66,6 +66,7 @@ Current maintainers: @cosmo0920
   + [include_index_in_url](#include_index_in_url)
   + [http_backend](#http_backend)
   + [prefer_oj_serializer](#prefer_oj_serializer)
+  + [compression_level](#compression_level)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
   + [Buffer options](#buffer-options)
@@ -792,6 +793,13 @@ Default value is `excon` which is default http_backend of elasticsearch plugin.
 http_backend typhoeus
 ```
 
+### compression_level
+You can add gzip compression of output data. In this case `default_compression`, `best_compression` or `best speed` option should be chosen. 
+By default there is no compression, default value for this option is `no_compression`
+```
+compression_level best_compression
+``` 
+
 ### prefer_oj_serializer
 
 With default behavior, Elasticsearch client uses `Yajl` as JSON encoder/decoder.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.6.0'
+  s.version       = '3.6.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -16,6 +16,10 @@ Gem::Specification.new do |s|
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ['lib']
 
+  if s.respond_to?(:metadata)
+    s.metadata["changelog_uri"] = "https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md"
+  end
+
   s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
 
   s.add_runtime_dependency 'fluentd', '>= 0.14.22'

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -60,6 +60,8 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         write_operation = @plugin.write_operation
       elsif INDEX_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(CREATE_OP)
         write_operation = CREATE_OP
+      elsif UPSERT_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(UPDATE_OP)
+        write_operation = UPDATE_OP
       elsif item.nil?
         stats[:errors_nil_resp] += 1
         next

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -18,6 +18,7 @@ require 'fluent/event'
 require 'fluent/error'
 require 'fluent/time'
 require 'fluent/log-ext'
+require 'zlib'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error'
 require_relative 'elasticsearch_error_handler'
@@ -148,6 +149,7 @@ EOC
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
     config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
+    config_param :compression_level, :enum, {list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression}
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
     config_param :ilm_policy, :hash, :default => {}
@@ -170,7 +172,7 @@ EOC
       compat_parameters_convert(conf, :buffer)
 
       super
-      raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
+      raise Fluent::ConfigError, "'tag' or '_index' in chunk_keys is required." if not @buffer_config.chunk_keys.include? "tag" and not @buffer_config.chunk_keys.include? "_index"
 
       @time_parser = create_time_parser
       @backend_options = backend_options
@@ -327,6 +329,18 @@ EOC
           alias_method :split_request?, :split_request_size_check?
         end
       end
+
+      version_arr = Elasticsearch::Transport::VERSION.split('.')
+
+      if (version_arr[0].to_i < 7) || (version_arr[0].to_i == 7 && version_arr[1].to_i < 2)
+        if compression
+          raise Fluent::ConfigError, <<-EOC
+            Cannot use compression with elasticsearch-transport plugin version < 7.2.0
+            Your elasticsearch-transport plugin version version is #{Elasticsearch::Transport::VERSION}.
+            Please consider to upgrade ES client.
+          EOC
+        end
+      end
     end
 
     def placeholder?(name, param)
@@ -338,6 +352,23 @@ EOC
       end
     end
 
+    def compression
+      !(@compression_level == :no_compression)
+    end
+
+    def compression_strategy
+      case @compression_level
+      when :default_compression
+        Zlib::DEFAULT_COMPRESSION
+      when :best_compression
+        Zlib::BEST_COMPRESSION
+      when :best_speed
+        Zlib::BEST_SPEED
+      else
+        Zlib::NO_COMPRESSION
+      end
+    end
+
     def backend_options
       case @http_backend
       when :excon
@@ -438,7 +469,14 @@ EOC
         if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
           local_reload_connections = @reload_after
         end
-        headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers)
+
+        gzip_headers = if compression
+                         {'Content-Encoding' => 'gzip'}
+                       else
+                         {}
+                       end
+        headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
+
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: local_reload_connections,
@@ -456,6 +494,7 @@ EOC
                                                                               },
                                                                               sniffer_class: @sniffer_class,
                                                                               serializer_class: @serializer_class,
+                                                                              compression: compression,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -768,6 +807,15 @@ EOC
       [parent_object, path[-1]]
     end
 
+    # gzip compress data
+    def gzip(string)
+      wio = StringIO.new("w")
+      w_gz = Zlib::GzipWriter.new(wio, strategy = compression_strategy)
+      w_gz.write(string)
+      w_gz.close
+      wio.string
+    end
+
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
@@ -791,7 +839,14 @@ EOC
       begin
 
         log.on_trace { log.trace "bulk request: #{data}" }
-        response = client(info.host).bulk body: data, index: info.index
+
+        prepared_data = if compression
+                          gzip(data)
+                        else
+                          data
+                        end
+
+        response = client(info.host).bulk body: prepared_data, index: info.index
         log.on_trace { log.trace "bulk response: #{response}" }
 
         if response['errors']

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -44,6 +44,12 @@ module Fluent::Plugin
       @_es ||= begin
         @current_config = connection_options[:hosts].clone
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
+        gzip_headers = if compression
+                         {'Content-Encoding' => 'gzip'}
+                       else
+                         {}
+                       end
+        headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: @reload_connections,
@@ -51,14 +57,15 @@ module Fluent::Plugin
                                                                               resurrect_after: @resurrect_after,
                                                                               logger: @transport_logger,
                                                                               transport_options: {
-                                                                                headers: { 'Content-Type' => @content_type.to_s },
+                                                                                headers: headers,
                                                                                 request: { timeout: @request_timeout },
                                                                                 ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
                                                                               },
                                                                               http: {
                                                                                 user: @user,
                                                                                 password: @password
-                                                                              }
+                                                                              },
+                                                                              compression: compression,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -210,7 +217,12 @@ module Fluent::Plugin
 
     def send_bulk(data, host, index)
       begin
-        response = client(host).bulk body: data, index: index
+        prepared_data = if compression
+                          gzip(data)
+                        else
+                          data
+                        end
+        response = client(host).bulk body: prepared_data, index: index
         if response['errors']
           log.error "Could not push log to Elasticsearch: #{response}"
         end

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -7,9 +7,10 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   class TestPlugin
     attr_reader :log
-    attr_reader :write_operation, :error_events
+    attr_reader :error_events
     attr_accessor :unrecoverable_error_types
     attr_accessor :log_es_400_reason
+    attr_accessor :write_operation
     def initialize(log, log_es_400_reason = false)
       @log = log
       @write_operation = 'index'
@@ -522,4 +523,125 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   end
 
+  def test_retry_error_upsert
+    @plugin.write_operation = 'upsert'
+    records = []
+    error_records = Hash.new(false)
+    error_records.merge!({0=>true, 4=>true, 9=>true})
+    10.times do |i|
+      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
+    end
+    chunk = MockChunk.new(records)
+
+    response = parse_response(%({
+      "took" : 1,
+      "errors" : true,
+      "items" : [
+        {
+          "update" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "1",
+            "status" : 201
+          }
+        },
+        {
+          "update" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "2",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "update" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "3",
+            "status" : 409,
+            "error" : {
+              "type":"version_conflict_engine_exception",
+              "reason":"document already exists"
+            }
+          }
+        },
+        {
+          "update" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "5",
+            "status" : 500,
+            "error" : {
+              "reason":"unrecognized error - no type field"
+            }
+          }
+        },
+        {
+          "update" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "6",
+            "status" : 400,
+            "error" : {
+              "type" : "mapper_parsing_exception",
+              "reason":"failed to parse"
+            }
+          }
+        },
+        {
+          "update" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "7",
+            "status" : 400,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "update" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "8",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        }
+      ]
+    }))
+
+    begin
+      failed = false
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+    rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
+      failed = true
+      records = [].tap do |records|
+        next unless e.respond_to?(:retry_stream)
+        e.retry_stream.each {|time, record| records << record}
+      end
+      puts records
+      assert_equal 3, records.length
+      assert_equal 2, records[0]['_id']
+      # upsert is retried in case of conflict error.
+      assert_equal 3, records[1]['_id']
+      assert_equal 8, records[2]['_id']
+      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
+      assert_equal 3, error_ids.length
+      assert_equal [5, 6, 7], error_ids
+      @plugin.error_events.collect {|h| h[:error]}.each do |e|
+        assert_true e.respond_to?(:backtrace)
+      end
+    end
+    assert_true failed
+  end
 end

data/test/plugin/test_out_elasticsearch.rb

@@ -232,6 +232,44 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
     assert_false instance.log_es_400_reason
     assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
+    assert_false instance.compression
+    assert_equal :no_compression, instance.compression_level
+  end
+
+  test 'configure compression' do
+    config = %{
+      compression_level best_compression
+    }
+    instance = driver(config).instance
+
+    assert_equal true, instance.compression
+  end
+
+  test 'check compression strategy' do
+    config = %{
+      compression_level best_speed
+    }
+    instance = driver(config).instance
+
+    assert_equal Zlib::BEST_SPEED, instance.compression_strategy
+  end
+
+  test 'check content-encoding header with compression' do
+    config = %{
+      compression_level best_compression
+    }
+    instance = driver(config).instance
+
+    assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+  end
+
+  test 'check compression option is passed to transport' do
+    config = %{
+      compression_level best_compression
+    }
+    instance = driver(config).instance
+
+    assert_equal true, instance.client.transport.options[:compression]
   end
 
   test 'configure Content-Type' do
@@ -341,23 +379,65 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
-  test 'lack of tag in chunk_keys' do
-    assert_raise_message(/'tag' in chunk_keys is required./) do
-      driver(Fluent::Config::Element.new(
-               'ROOT', '', {
-                 '@type' => 'elasticsearch',
-                 'host' => 'log.google.com',
-                 'port' => 777,
-                 'scheme' => 'https',
-                 'path' => '/es/',
-                 'user' => 'john',
-                 'password' => 'doe',
-               }, [
-                 Fluent::Config::Element.new('buffer', 'mykey', {
-                                               'chunk_keys' => 'mykey'
-                                             }, [])
-               ]
-             ))
+  sub_test_case 'chunk_keys requirement' do
+    test 'tag in chunk_keys' do
+      assert_nothing_raised do
+        driver(Fluent::Config::Element.new(
+                 'ROOT', '', {
+                   '@type' => 'elasticsearch',
+                   'host' => 'log.google.com',
+                   'port' => 777,
+                   'scheme' => 'https',
+                   'path' => '/es/',
+                   'user' => 'john',
+                   'password' => 'doe',
+                 }, [
+                   Fluent::Config::Element.new('buffer', 'tag', {
+                                                 'chunk_keys' => 'tag'
+                                               }, [])
+                 ]
+               ))
+      end
+    end
+
+    test '_index in chunk_keys' do
+      assert_nothing_raised do
+        driver(Fluent::Config::Element.new(
+                 'ROOT', '', {
+                   '@type' => 'elasticsearch',
+                   'host' => 'log.google.com',
+                   'port' => 777,
+                   'scheme' => 'https',
+                   'path' => '/es/',
+                   'user' => 'john',
+                   'password' => 'doe',
+                 }, [
+                   Fluent::Config::Element.new('buffer', '_index', {
+                                                 'chunk_keys' => '_index'
+                                               }, [])
+                 ]
+               ))
+      end
+    end
+
+    test 'lack of tag and _index in chunk_keys' do
+      assert_raise_message(/'tag' or '_index' in chunk_keys is required./) do
+        driver(Fluent::Config::Element.new(
+                 'ROOT', '', {
+                   '@type' => 'elasticsearch',
+                   'host' => 'log.google.com',
+                   'port' => 777,
+                   'scheme' => 'https',
+                   'path' => '/es/',
+                   'user' => 'john',
+                   'password' => 'doe',
+                 }, [
+                   Fluent::Config::Element.new('buffer', 'mykey', {
+                                                 'chunk_keys' => 'mykey'
+                                               }, [])
+                 ]
+               ))
+      end
     end
   end
 
@@ -1762,6 +1842,47 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_name, index_cmds.first['index']['_index'])
   end
 
+  # gzip compress data
+  def gzip(string, strategy)
+    wio = StringIO.new("w")
+    w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
+    w_gz.write(string)
+    w_gz.close
+    wio.string
+  end
+
+
+  def test_writes_to_default_index_with_compression
+    config = %[
+      compression_level default_compression
+    ]
+
+    bodystr = %({
+          "took" : 500,
+          "errors" : false,
+          "items" : [
+            {
+              "create": {
+                "_index" : "fluentd",
+                "_type"  : "fluentd"
+              }
+            }
+           ]
+        })
+
+    compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
+
+    elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+        to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
+
+    driver(config)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+
+    assert_requested(elastic_request)
+  end
+
   data('Elasticsearch 6' => [6, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME],
        'Elasticsearch 7' => [7, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME_ES_7x],
        'Elasticsearch 8' => [8, nil],

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -104,6 +104,44 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_false instance.with_transporter_log
     assert_equal :"application/json", instance.content_type
     assert_equal :excon, instance.http_backend
+    assert_false instance.compression
+    assert_equal :no_compression, instance.compression_level
+  end
+
+  test 'configure compression' do
+    config = %{
+      compression_level best_compression
+    }
+    instance = driver(config).instance
+
+    assert_equal true, instance.compression
+  end
+
+  test 'check compression strategy' do
+    config = %{
+      compression_level best_speed
+    }
+    instance = driver(config).instance
+
+    assert_equal Zlib::BEST_SPEED, instance.compression_strategy
+  end
+
+  test 'check content-encoding header with compression' do
+    config = %{
+      compression_level best_compression
+    }
+    instance = driver(config).instance
+
+    assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+  end
+
+  test 'check compression option is passed to transport' do
+    config = %{
+      compression_level best_compression
+    }
+    instance = driver(config).instance
+
+    assert_equal true, instance.client.transport.options[:compression]
   end
 
   test 'configure Content-Type' do
@@ -305,6 +343,46 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal('fluentd', index_cmds.first['index']['_index'])
   end
 
+  # gzip compress data
+  def gzip(string, strategy)
+    wio = StringIO.new("w")
+    w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
+    w_gz.write(string)
+    w_gz.close
+    wio.string
+  end
+
+  def test_writes_to_default_index_with_compression
+    config = %[
+      compression_level default_compression
+    ]
+
+    bodystr = %({
+          "took" : 500,
+          "errors" : false,
+          "items" : [
+            {
+              "create": {
+                "_index" : "fluentd",
+                "_type"  : "fluentd"
+              }
+            }
+           ]
+        })
+
+    compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
+
+    elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+        to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
+
+    driver(config)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+
+    assert_requested(elastic_request)
+  end
+
   def test_writes_to_default_type
     stub_elastic
     driver.run(default_tag: 'test') do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.6.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-06 00:00:00.000000000 Z
+date: 2019-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -170,7 +170,8 @@ files:
 homepage: https://github.com/uken/fluent-plugin-elasticsearch
 licenses:
 - Apache-2.0
-metadata: {}
+metadata:
+  changelog_uri: https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md
 post_install_message: 
 rdoc_options: []
 require_paths: