fluent-plugin-elasticsearch 1.9.4 → 5.0.3

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -0,0 +1,74 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_elasticsearch'
+
+class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
+  attr_accessor :index_cmds
+
+  def setup
+    Fluent::Test.setup
+    @driver = nil
+    log = Fluent::Engine.log
+    log.out.logs.slice!(0, log.out.logs.length)
+  end
+
+  def stub_elastic(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).with do |req|
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
+    body ="{\"version\":{\"number\":\"#{version}\"}}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
+  def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
+    stub_request(:get, url).to_return(:status => [404, "Not Found"])
+  end
+
+  def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
+    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
+  end
+
+  def sample_record(content={})
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
+  end
+
+  def driver(conf='')
+    @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
+      # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+      if !defined?(Fluent::Plugin::Output)
+        def format(tag, time, record)
+          [time, record].to_msgpack
+        end
+      end
+    }.configure(conf)
+  end
+
+  def test_fallback_on_info
+    stub_elastic_info_not_found("http://localhost:9202/")
+    stub_elastic_info_unavailable("http://localhost:9201/")
+    stub_elastic_info
+    stub_elastic
+    config = %[
+      hosts localhost:9202,localhost:9201,localhost:9200
+      selector_class_name Fluent::Plugin::ElasticseatchFallbackSelector
+      @log_level debug
+      with_transporter_log true
+      reload_connections true
+      reload_after 10
+      catch_transport_exception_on_retry false # For fallback testing
+    ]
+    assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
+      driver(config)
+    end
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal(2, index_cmds.length)
+    assert_equal("fluentd", index_cmds.first['index']['_index'])
+  end
+
+  # TODO: on feed phase test case
+end

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -0,0 +1,66 @@
+require_relative '../helper'
+require 'elasticsearch'
+require 'fluent/plugin/elasticsearch_index_lifecycle_management'
+
+class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
+  include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
+
+  def setup
+    begin
+      require "elasticsearch/xpack"
+    rescue LoadError
+      omit "ILM testcase needs elasticsearch-xpack gem."
+    end
+    if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+      omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+    end
+    Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
+      def client
+        Elasticsearch::Client.new url: 'localhost:9200'
+      end
+      def log
+        log_device = Fluent::Test::DummyLogDevice.new
+        dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
+        logger = ServerEngine::DaemonLogger.new(log_device, dl_opts)
+        Fluent::Log.new(logger)
+      end
+    CODE
+  end
+
+  def test_xpack_info
+    stub_request(:get, "http://localhost:9200/_xpack").
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
+    assert_equal(expected, xpack_info)
+  end
+
+  def test_verify_ilm_working
+    stub_request(:get, "http://localhost:9200/_xpack").
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    assert_nothing_raised { verify_ilm_working }
+  end
+
+  def test_ilm_policy_doesnt_exists
+    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
+      to_return(:status => 404, :body => "", :headers => {})
+    assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
+  end
+
+  def test_ilm_policy_exists
+    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
+      to_return(:status => 200, :body => "", :headers => {})
+    assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
+  end
+
+  def test_create_ilm_policy
+    stub_request(:get, "http://localhost:9200/_ilm/policy/fluent-policy").
+      to_return(:status => 404, :body => "", :headers => {})
+    stub_request(:put, "http://localhost:9200/_ilm/policy/fluent-policy").
+      with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
+         :headers => {'Content-Type'=>'application/json'}).
+      to_return(:status => 200, :body => "", :headers => {})
+    create_ilm_policy("fluent-policy")
+
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
+  end
+end

data/test/plugin/test_elasticsearch_tls.rb

@@ -0,0 +1,145 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/output'
+require 'fluent/plugin/elasticsearch_tls'
+
+class TestElasticsearchTLS < Test::Unit::TestCase
+
+  class TestTLSModuleOutput < Fluent::Plugin::Output
+    include Fluent::Plugin::ElasticsearchTLS
+
+    def initialize
+      super
+      @emit_streams = []
+    end
+
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
+    end
+  end
+
+  setup do
+    Fluent::Test.setup
+    @use_tls_minmax_version = begin
+                                map = {
+                                  TLSv1: OpenSSL::SSL::TLS1_VERSION,
+                                  TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
+                                  TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
+                                }
+                                map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                                true
+                              rescue NameError
+                                false
+                              end
+    @enabled_tlsv1_3 = begin
+                         map = {TLSv1_3: OpenSSL::SSL::TLS1_3_VERSION}
+                         true
+                       rescue NameError
+                         false
+                       end
+  end
+
+  def driver(conf='')
+    Fluent::Test::Driver::Output.new(TestTLSModuleOutput).configure(conf)
+  end
+
+  test 'configure' do
+    assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, driver.instance.ssl_version
+    assert_nil driver.instance.ssl_max_version
+    assert_nil driver.instance.ssl_min_version
+  end
+
+  test 'check USE_TLS_MINMAX_VERSION value' do
+    assert_equal @use_tls_minmax_version, Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
+  end
+
+  sub_test_case 'set_tls_minmax_version_config' do
+    test 'default' do
+      d = driver('')
+      ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
+      if @use_tls_minmax_version
+        if @enabled_tlsv1_3
+          assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+        else
+          assert_equal({max_version: nil,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+
+        end
+      else
+        assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
+      end
+    end
+
+    test 'errorous cases' do
+      if @use_tls_minmax_version
+        assert_raise(Fluent::ConfigError) do
+          d = driver(%{ssl_max_version TLSv1_2})
+          d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                   d.instance.ssl_max_version,
+                                                   d.instance.ssl_min_version)
+        end
+        assert_raise(Fluent::ConfigError) do
+          d = driver(%{ssl_min_version TLSv1_2})
+          d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                   d.instance.ssl_max_version,
+                                                   d.instance.ssl_min_version)
+        end
+      else
+        d1 = driver(%{
+          ssl_max_version TLSv1_2
+          @log_level info
+        })
+        d1.instance.set_tls_minmax_version_config(d1.instance.ssl_version,
+                                                  d1.instance.ssl_max_version,
+                                                  d1.instance.ssl_min_version)
+
+        d1.logs.any? {|a| a.include?("'ssl_max_version' does not have any effect in this environment.") }
+        d2 = driver(%{
+          ssl_min_version TLSv1_2
+          @log_level info
+        })
+        d2.instance.set_tls_minmax_version_config(d2.instance.ssl_version,
+                                                  d2.instance.ssl_max_version,
+                                                  d2.instance.ssl_min_version)
+        d2.logs.any? {|a| a.include?("'ssl_min_version' does not have any effect in this environment.") }
+      end
+    end
+
+    test 'min_version & max_version' do
+      config = %{
+        ssl_max_version TLSv1_2
+        ssl_min_version TLSv1_1
+      }
+      d = driver(config)
+      ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                                     d.instance.ssl_max_version,
+                                                                     d.instance.ssl_min_version)
+      if @use_tls_minmax_version
+        assert_equal({max_version: OpenSSL::SSL::TLS1_2_VERSION,
+                      min_version: OpenSSL::SSL::TLS1_1_VERSION}, ssl_version_options)
+      else
+        assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
+      end
+    end
+
+    test 'TLSv1.3' do
+      omit "openssl gem does not support TLSv1.3" unless @enabled_tlsv1_3
+      config = %{
+        ssl_max_version TLSv1_3
+        ssl_min_version TLSv1_2
+      }
+      d = driver(config)
+      ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                                     d.instance.ssl_max_version,
+                                                                     d.instance.ssl_min_version)
+      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                    min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+
+    end
+  end
+end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -0,0 +1,215 @@
+require_relative '../helper'
+require 'date'
+require 'fluent/test/helpers'
+require 'json'
+require 'fluent/test/driver/filter'
+require 'flexmock/test_unit'
+require 'fluent/plugin/filter_elasticsearch_genid'
+
+class ElasticsearchGenidFilterTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='')
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
+  end
+
+  test "invalid configuration" do
+    assert_raise(Fluent::ConfigError) do
+      create_driver("use_record_as_seed true")
+    end
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal '_hash', d.instance.hash_id_key
+  end
+
+  data("default" => {"hash_id_key" => "_hash"},
+       "custom_key" => {"hash_id_key" => "_edited"},
+      )
+  def test_filter(data)
+    d = create_driver("hash_id_key #{data["hash_id_key"]}")
+    flexmock(SecureRandom).should_receive(:uuid)
+      .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
+    time = event_time("2017-10-15 15:00:23.34567890 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, sample_record)
+    end
+    assert_equal(Base64.strict_encode64(SecureRandom.uuid),
+                 d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+  end
+
+  class UseRecordAsSeedTest < self
+    data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
+         "sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
+         "sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
+         "sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
+        )
+    def test_simple(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
+         "sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
+         "sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
+         "sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
+    def test_record_with_tag(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
+         "sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
+         "sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
+         "sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
+    def test_record_with_time(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
+         "sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
+         "sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
+         "sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
+
+  class UseEntireRecordAsSeedTest < self
+    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
+         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
+         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
+         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    def test_record
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
+         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
+         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
+         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    def test_record_with_tag
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
+         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
+         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
+         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    def test_record_with_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
+         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
+         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
+         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
+end