fluent-plugin-elasticsearch 2.6.1 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfdcf38e16814e5b5e71a312978677c190277fcfe6be1a830d65ed4ea9d764c2
-  data.tar.gz: 3a241a5a1e020b74384c7b72c4caac7e84a7b53465256ea28c36fc509187fa0e
+  metadata.gz: 6c45d1357073cd62a59770c94e106099a367b504e4fcc306868a51cad18b76a1
+  data.tar.gz: ffeb4647996e9da73b432d7b9baae5766e4973801b7ee59e097e26bf927aff38
 SHA512:
-  metadata.gz: d4e4ca1522f9b39efe8e1197d289f967a426b24db2c01bfc80552cc5e7582561a8b0275da1beb371b061f05c86dc98a71ff0142c790ca0907e26affc93df0f39
-  data.tar.gz: ce63c24535a2174731f6e4a1d385f5b1cc79996987c003daaaa5e5ced8a46bf65e4721e1fa3de1815eaf0fcdf06b0ccf15b8963a0ccdb85647e308f9b192157c
+  metadata.gz: 82bc86a0116891984e93587d77f39fcd8e86112a6688bcc10e52b99114a0d98bcade90643083815f0929533802cb956ac1f3aa8fa150cf7bba4e2d55094fb9f1
+  data.tar.gz: f74034825181ddfa92968de7eaa041ff35e17797a5aa496e396f645bae25f57ee5b9eff5cc86bd1c82ea0fc306f5991232fef5ff927d06130b5f61229ec40d29

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.7.0
+- Configureable content type (#367)
+
 ### 2.6.1
 - Prevent error when using template in elasticsearch_dynamic for elementally use case (#363)
 

data/README.md

@@ -51,6 +51,7 @@ Current maintainers: @cosmo0920
   + [time_parse_error_tag](#time_parse_error_tag)
   + [reconnect_on_error](#reconnect_on_error)
   + [with_transporter_log](#with_transporter_log)
+  + [content_type](#content_type)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
   + [Buffer options](#buffer-options)
@@ -570,6 +571,16 @@ We recommend to set this true if you start to debug this plugin.
 with_transporter_log true
 ```
 
+### content_type
+
+With `content_type application/json`, elasticsearch plugin adds `application/json` as `Content-Type` in payload.
+
+Default value is `application/x-ndjson` which is encourage to use Elasticsearch bulk request.
+
+```
+content_type application/json
+```
+
 ### Client/host certificate options
 
 Need to verify Elasticsearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.6.1'
+  s.version       = '2.7.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -75,6 +75,7 @@ module Fluent::Plugin
     config_param :reconnect_on_error, :bool, :default => false
     config_param :pipeline, :string, :default => nil
     config_param :with_transporter_log, :bool, :default => false
+    config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/x-ndjson"
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -207,7 +208,7 @@ module Fluent::Plugin
                                                                               retry_on_failure: 5,
                                                                               logger: @transport_logger,
                                                                               transport_options: {
-                                                                                headers: { 'Content-Type' => 'application/json' },
+                                                                                headers: { 'Content-Type' => @content_type.to_s },
                                                                                 request: { timeout: @request_timeout },
                                                                                 ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
                                                                               }

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -54,7 +54,7 @@ module Fluent::Plugin
                                                                               retry_on_failure: 5,
                                                                               logger: @transport_logger,
                                                                               transport_options: {
-                                                                                headers: { 'Content-Type' => 'application/json' },
+                                                                                headers: { 'Content-Type' => @content_type.to_s },
                                                                                 request: { timeout: @request_timeout },
                                                                                 ssl: { verify: @ssl_verify, ca_file: @ca_file }
                                                                               }

data/test/plugin/test_out_elasticsearch.rb

@@ -212,6 +212,24 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_nil instance.client_cert
     assert_nil instance.client_key_pass
     assert_false instance.with_transporter_log
+    assert_equal :"application/x-ndjson", instance.content_type
+  end
+
+  test 'configure Content-Type' do
+    config = %{
+      content_type application/json
+    }
+    instance = driver(config).instance
+    assert_equal :"application/json", instance.content_type
+  end
+
+  test 'invalid Content-Type' do
+    config = %{
+      content_type nonexistent/invalid
+    }
+    assert_raise(Fluent::ConfigError) {
+      instance = driver(config).instance
+    }
   end
 
   test 'lack of tag in chunk_keys' do
@@ -625,7 +643,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})
     elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
-      with(headers: { "Content-Type" => "application/json" })
+      with(headers: { "Content-Type" => "application/x-ndjson" })
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -85,6 +85,24 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_nil instance.client_cert
     assert_nil instance.client_key_pass
     assert_false instance.with_transporter_log
+    assert_equal :"application/x-ndjson", instance.content_type
+  end
+
+  test 'configure Content-Type' do
+    config = %{
+      content_type application/json
+    }
+    instance = driver(config).instance
+    assert_equal :"application/json", instance.content_type
+  end
+
+  test 'invalid Content-Type' do
+    config = %{
+      content_type nonexistent/invalid
+    }
+    assert_raise(Fluent::ConfigError) {
+      instance = driver(config).instance
+    }
   end
 
   def test_defaults
@@ -220,7 +238,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})
     elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
-      with(headers: { "Content-Type" => "application/json" })
+      with(headers: { "Content-Type" => "application/x-ndjson" })
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.7.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-20 00:00:00.000000000 Z
+date: 2018-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd