fluent-plugin-elasticsearch 2.7.0 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c45d1357073cd62a59770c94e106099a367b504e4fcc306868a51cad18b76a1
-  data.tar.gz: ffeb4647996e9da73b432d7b9baae5766e4973801b7ee59e097e26bf927aff38
+  metadata.gz: bdf35ea1b2e3e85372e0f5af8febdc9920e38090053b7e818d714d32d9df6428
+  data.tar.gz: f43d98b89ee686f14bfe3b544f929cefc2b50d28d064cd9c40a3502aadf0774b
 SHA512:
-  metadata.gz: 82bc86a0116891984e93587d77f39fcd8e86112a6688bcc10e52b99114a0d98bcade90643083815f0929533802cb956ac1f3aa8fa150cf7bba4e2d55094fb9f1
-  data.tar.gz: f74034825181ddfa92968de7eaa041ff35e17797a5aa496e396f645bae25f57ee5b9eff5cc86bd1c82ea0fc306f5991232fef5ff927d06130b5f61229ec40d29
+  metadata.gz: 8d4b5829f6bfcc13d8e25a60fca57422068a1449376690776dbe6623346ae058226563b215ad4926b97680b632515516309e6d50414d2d030e4d559359ae077a
+  data.tar.gz: 763d3a4838093c99d92a2b4f3382335ff95ddd4e20d0c4a2378a42b3cb830a4c6654501662920035f9299a28e492372d5671792f0098f9f9e4566035d24533af

data/History.md

@@ -4,6 +4,10 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.8.0
+- Remove outdated generating hash id support module (#373)
+- Check Elasticsearch major version (#371)
+
 ### 2.7.0
 - Configureable content type (#367)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.7.0'
+  s.version       = '2.8.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -13,7 +13,6 @@ require 'fluent/plugin/output'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
-require_relative 'generate_hash_id_support'
 
 module Fluent::Plugin
   class ElasticsearchOutput < Output
@@ -24,6 +23,8 @@ module Fluent::Plugin
     Fluent::Plugin.register_output('elasticsearch', self)
 
     DEFAULT_BUFFER_TYPE = "memory"
+    DEFAULT_ELASTICSEARCH_VERSION = 5 # For compatibility.
+    DEFAULT_TYPE_NAME = "_doc".freeze
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -33,7 +34,10 @@ module Fluent::Plugin
     config_param :scheme, :string, :default => 'http'
     config_param :hosts, :string, :default => nil
     config_param :target_index_key, :string, :default => nil
-    config_param :target_type_key, :string, :default => nil
+    config_param :target_type_key, :string, :default => nil,
+                 :deprecated => <<EOC
+Elasticsearch 7.x or above will ignore this config. Please use fixed type_name instead.
+EOC
     config_param :time_key_format, :string, :default => nil
     config_param :time_precision, :integer, :default => 9
     config_param :include_timestamp, :bool, :default => false
@@ -42,7 +46,7 @@ module Fluent::Plugin
     config_param :logstash_prefix_separator, :string, :default => '-'
     config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
     config_param :utc_index, :bool, :default => true
-    config_param :type_name, :string, :default => "fluentd"
+    config_param :type_name, :string, :default => DEFAULT_TYPE_NAME
     config_param :index_name, :string, :default => "fluentd"
     config_param :id_key, :string, :default => nil
     config_param :write_operation, :string, :default => "index"
@@ -84,7 +88,6 @@ module Fluent::Plugin
     end
 
     include Fluent::ElasticsearchIndexTemplate
-    include Fluent::Plugin::GenerateHashIdSupport
     include Fluent::Plugin::ElasticsearchConstants
 
     def initialize
@@ -151,6 +154,17 @@ module Fluent::Plugin
         log_level = conf['@log_level'] || conf['log_level']
         log.warn "Consider to specify log_level with @log_level." unless log_level
       end
+
+      @last_seen_major_version = detect_es_major_version rescue DEFAULT_ELASTICSEARCH_VERSION
+      if @last_seen_major_version >= 7 && @type_name != DEFAULT_TYPE_NAME
+        log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+        @type_name = '_doc'.freeze
+      end
+    end
+
+    def detect_es_major_version
+      @_es_info ||= client.info
+      @_es_info["version"]["number"].to_i
     end
 
     def convert_compat_id_key(key)
@@ -356,6 +370,7 @@ module Fluent::Plugin
       tag = chunk.metadata.tag
       logstash_prefix, index_name, type_name = expand_placeholders(chunk.metadata)
       @error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
+      @last_seen_major_version = detect_es_major_version rescue DEFAULT_ELASTICSEARCH_VERSION
 
       chunk.msgpack_each do |time, record|
         @error.records += 1
@@ -404,8 +419,20 @@ module Fluent::Plugin
         target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
         if target_type_parent && target_type_parent[target_type_child_key]
           target_type = target_type_parent.delete(target_type_child_key)
+          if @last_seen_major_version == 6
+            log.warn "Detected ES 6.x: `@type_name` will be used as the document `_type`."
+            target_type = type_name
+          elsif @last_seen_major_version >= 7
+            log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+            target_type = '_doc'.freeze
+          end
         else
-          target_type = type_name
+          if @last_seen_major_version >= 7 && target_type != DEFAULT_TYPE_NAME
+            log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+            target_type = '_doc'.freeze
+          else
+            target_type = type_name
+          end
         end
 
         meta.clear
@@ -453,6 +480,7 @@ module Fluent::Plugin
         if retries < 2
           retries += 1
           @_es = nil
+          @_es_info = nil
           log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
           sleep 2**retries
           retry
@@ -460,6 +488,7 @@ module Fluent::Plugin
         raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
       rescue Exception
         @_es = nil if @reconnect_on_error
+        @_es_info = nil if @reconnect_on_error
         raise
       end
     end

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -13,8 +13,6 @@ module Fluent::Plugin
     DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
     DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
 
-    include Fluent::Plugin::GenerateHashIdSupport
-
     attr_reader :dynamic_config
 
     def configure(conf)

data/test/plugin/test_out_elasticsearch.rb

@@ -19,7 +19,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
     log.out.logs.slice!(0, log.out.logs.length)
   end
 
-  def driver(conf='')
+  def driver(conf='', es_version=5)
+    # For request stub to detect compatibility.
+    @es_version ||= es_version
+    Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+      def detect_es_major_version
+        #{@es_version}
+      end
+    CODE
     @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
       # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
       if !defined?(Fluent::Plugin::Output)
@@ -232,6 +239,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
+  test 'Detected Elasticsearch 7' do
+    config = %{
+      type_name changed
+    }
+    instance = driver(config, 7).instance
+    assert_equal '_doc', instance.type_name
+  end
+
   test 'lack of tag in chunk_keys' do
     assert_raise_message(/'tag' in chunk_keys is required./) do
       driver(Fluent::Config::Element.new(
@@ -252,21 +267,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
-  test 'with invaild generate id config' do
-    assert_raise_message(/Use bundled filter-elasticsearch-genid instead./) do
-      driver.configure(Fluent::Config::Element.new(
-                         'ROOT', '', {
-                           '@type' => 'elasticsearch',
-                           'id_key' =>'id_mismatch',
-                         }, [
-                           Fluent::Config::Element.new('hash', '', {
-                                                         'hash_id_key' => '_hash',
-                                                       }, [])
-                         ]
-                       ))
-    end
-  end
-
   def test_template_already_present
     config = %{
       host            logs.google.com
@@ -665,7 +665,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('fluentd', index_cmds.first['index']['_type'])
+    assert_equal('_doc', index_cmds.first['index']['_type'])
   end
 
   def test_writes_to_speficied_index
@@ -678,27 +678,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
-  class AdditionalHashIdMechanismTest < self
-    data("default"            => {"hash_id_key" => '_id'},
-         "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
-        )
-    def test_writes_with_genrate_hash(data)
-      assert_raise_message(/Use bundled filter-elasticsearch-genid instead./) do
-        driver.configure(Fluent::Config::Element.new(
-                           'ROOT', '', {
-                             '@type' => 'elasticsearch',
-                             'id_key' => data["hash_id_key"],
-                           }, [
-                             Fluent::Config::Element.new('hash', '', {
-                                                           'keys' => ['request_id'],
-                                                           'hash_id_key' => data["hash_id_key"],
-                                                         }, [])
-                           ]
-                         ))
-      end
-    end
-  end
-
   class IndexNamePlaceholdersTest < self
     def test_writes_to_speficied_index_with_tag_placeholder
       driver.configure("index_name myindex.${tag}\n")
@@ -836,35 +815,46 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_speficied_type
-    driver.configure("type_name mytype\n")
+  data("border"        => {"es_version" => 6, "_type" => "mytype"},
+       "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+      )
+  def test_writes_to_speficied_type(data)
+    driver('', data["es_version"]).configure("type_name mytype\n")
     stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('mytype', index_cmds.first['index']['_type'])
+    assert_equal(data['_type'], index_cmds.first['index']['_type'])
   end
 
-  def test_writes_to_speficied_type_with_placeholders
-    driver.configure("type_name mytype.${tag}\n")
+  data("border"        => {"es_version" => 6, "_type" => "mytype.test"},
+       "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+      )
+  def test_writes_to_speficied_type_with_placeholders(data)
+    driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
     stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('mytype.test', index_cmds.first['index']['_type'])
+    assert_equal(data['_type'], index_cmds.first['index']['_type'])
   end
 
-  def test_writes_to_target_type_key
-    driver.configure("target_type_key @target_type\n")
+  data("old"           => {"es_version" => 2, "_type" => "local-override"},
+       "old_behavior"  => {"es_version" => 5, "_type" => "local-override"},
+       "border"        => {"es_version" => 6, "_type" => "_doc"},
+       "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+      )
+  def test_writes_to_target_type_key(data)
+    driver('', data["es_version"]).configure("target_type_key @target_type\n")
     stub_elastic_ping
     stub_elastic
     record = sample_record.clone
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('@target_type' => 'local-override'))
     end
-    assert_equal('local-override', index_cmds.first['index']['_type'])
+    assert_equal(data["_type"], index_cmds.first['index']['_type'])
     assert_nil(index_cmds[1]['@target_type'])
   end
 
@@ -875,7 +865,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('fluentd', index_cmds.first['index']['_type'])
+    assert_equal('_doc', index_cmds.first['index']['_type'])
   end
 
   def test_writes_to_target_type_key_fallack_to_type_name
@@ -889,8 +879,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
-  def test_writes_to_target_type_key_nested
-    driver.configure("target_type_key kubernetes.labels.log_type\n")
+  data("old"           => {"es_version" => 2, "_type" => "local-override"},
+       "old_behavior"  => {"es_version" => 5, "_type" => "local-override"},
+       "border"        => {"es_version" => 6, "_type" => "_doc"},
+       "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+      )
+  def test_writes_to_target_type_key_nested(data)
+    driver('', data["es_version"]).configure("target_type_key kubernetes.labels.log_type\n")
     stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
@@ -900,7 +895,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
         }
       }))
     end
-    assert_equal('local-override', index_cmds.first['index']['_type'])
+    assert_equal(data["_type"], index_cmds.first['index']['_type'])
     assert_nil(index_cmds[1]['kubernetes']['labels']['log_type'])
   end
 
@@ -915,7 +910,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
         }
       }))
     end
-    assert_equal('fluentd', index_cmds.first['index']['_type'])
+    assert_equal('_doc', index_cmds.first['index']['_type'])
   end
 
   def test_writes_to_speficied_host

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -16,7 +16,14 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     @driver = nil
   end
 
-  def driver(conf='')
+  def driver(conf='', es_version=5)
+    # For request stub to detect compatibility.
+    @es_version ||= es_version
+    Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
+      def detect_es_major_version
+        #{@es_version}
+      end
+    CODE
     @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutputDynamic) {
       # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
       if !defined?(Fluent::Plugin::Output)
@@ -105,6 +112,14 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
   end
 
+  test 'Detected Elasticsearch 7' do
+    config = %{
+      type_name changed
+    }
+    instance = driver(config, 7).instance
+    assert_equal '_doc', instance.type_name
+  end
+
   def test_defaults
     config = %{
       host     logs.google.com
@@ -260,7 +275,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('fluentd', index_cmds.first['index']['_type'])
+    assert_equal('_doc', index_cmds.first['index']['_type'])
   end
 
   def test_writes_to_specified_index
@@ -343,27 +358,6 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(2000, total)
   end
 
-  class AdditionalHashIdMechanismTest < self
-    data("default"            => {"hash_id_key" => '_id'},
-         "custom hash_id_key" => {"hash_id_key" => '_hash_id'},
-        )
-    def test_writes_with_genrate_hash(data)
-      assert_raise_message(/Use bundled filter-elasticsearch-genid instead./) do
-        driver.configure(Fluent::Config::Element.new(
-                           'ROOT', '', {
-                             '@type' => 'elasticsearch',
-                             'id_key' => data["hash_id_key"],
-                           }, [
-                             Fluent::Config::Element.new('hash', '', {
-                                                           'keys' => ['request_id'],
-                                                           'hash_id_key' => data["hash_id_key"],
-                                                         }, [])
-                           ]
-                         ))
-      end
-    end
-  end
-
   def test_makes_bulk_request
     stub_elastic_ping
     stub_elastic

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.7.0
+  version: 2.8.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-26 00:00:00.000000000 Z
+date: 2018-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -148,7 +148,6 @@ files:
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
 - lib/fluent/plugin/filter_elasticsearch_genid.rb
-- lib/fluent/plugin/generate_hash_id_support.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb

data/lib/fluent/plugin/generate_hash_id_support.rb

@@ -1,24 +0,0 @@
-require 'securerandom'
-require 'base64'
-
-module Fluent
-  module Plugin
-    module GenerateHashIdSupport
-      def self.included(klass)
-        klass.instance_eval {
-          config_section :hash, param_name: :hash_config, required: false, multi: false do
-            config_param :hash_id_key, :string, default: '_hash',
-                         obsoleted: "Use bundled filter-elasticsearch-genid instead."
-          end
-        }
-      end
-
-      def generate_hash_id_key(record)
-        s = ""
-        s += Base64.strict_encode64(SecureRandom.uuid)
-        record[@hash_config.hash_id_key] = s
-        record
-      end
-    end
-  end
-end