fluent-plugin-elasticsearch 2.10.3 → 2.10.4

data/test/helper.rb

@@ -1,24 +1,24 @@
-require 'simplecov'
-SimpleCov.start do
-  add_filter do |src|
-    !(src.filename =~ /^#{SimpleCov.root}\/lib/)
-  end
-end
-
-require 'coveralls'
-Coveralls.wear!
-
-# needs to be after simplecov but before test/unit, because fluentd sets default
-# encoding to ASCII-8BIT, but coverall might load git data which could contain a
-# UTF-8 character
-at_exit do
-  Encoding.default_internal = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_internal)
-  Encoding.default_external = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_external)
-end
-
-require 'test/unit'
-require 'fluent/test'
-require 'minitest/pride'
-
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
+require 'simplecov'
+SimpleCov.start do
+  add_filter do |src|
+    !(src.filename =~ /^#{SimpleCov.root}\/lib/)
+  end
+end
+
+require 'coveralls'
+Coveralls.wear!
+
+# needs to be after simplecov but before test/unit, because fluentd sets default
+# encoding to ASCII-8BIT, but coverall might load git data which could contain a
+# UTF-8 character
+at_exit do
+  Encoding.default_internal = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_internal)
+  Encoding.default_external = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_external)
+end
+
+require 'test/unit'
+require 'fluent/test'
+require 'minitest/pride'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -1,195 +1,195 @@
-require 'helper'
-require 'fluent/plugin/out_elasticsearch'
-require 'fluent/plugin/elasticsearch_error_handler'
-require 'json'
-
-class TestElasticsearchErrorHandler < Test::Unit::TestCase
-
-  class TestPlugin
-    attr_reader :log
-    attr_reader :write_operation, :error_events
-    def initialize(log)
-      @log = log
-      @write_operation = 'index'
-      @error_events = []
-    end
-
-    def router
-      self
-    end
-
-    def emit_error_event(tag, time, record, e)
-       @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
-    end
-
-    def process_message(tag, meta, header, time, record, bulk_message, extracted_values)
-      if record.has_key?('raise') && record['raise']
-        raise Exception('process_message')
-      end
-    end
-  end
-
-  class MockChunk
-    def initialize(records)
-      @records = records
-      @index = 0
-    end
-    def msgpack_each
-      @records.each { |item| yield(item[:time],item[:record]) }
-    end
-  end
-
-  def setup
-    Fluent::Test.setup
-    @log_device = Fluent::Test::DummyLogDevice.new
-    dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
-    logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
-    @log = Fluent::Log.new(logger)
-    @plugin = TestPlugin.new(@log)
-    @handler = Fluent::Plugin::ElasticsearchErrorHandler.new(@plugin)
-  end
-
-  def parse_response(value)
-    JSON.parse(value)
-  end
-
-  def test_dlq_400_responses
-    records = [{time: 123, record: {"foo" => "bar"}}]
-    response = parse_response(%({
-      "took" : 0,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "status" : 400,
-            "_type"  : "bar",
-              "reason":"unrecognized error"
-            }
-        }
-      ]
-     }))
-    chunk = MockChunk.new(records)
-    dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
-    assert_equal(1, @plugin.error_events.size)
-    assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
-  end
-
-  def test_retry_error
-    records = []
-    error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
-    10.times do |i|
-      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
-    end
-    chunk = MockChunk.new(records)
-
-    response = parse_response(%({
-      "took" : 1,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "1",
-            "status" : 201
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "2",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "3",
-            "status" : 409
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "5",
-            "status" : 500,
-            "error" : {
-              "reason":"unrecognized error - no type field"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "6",
-            "status" : 429,
-            "error" : {
-              "type" : "es_rejected_execution_exception",
-              "reason":"unable to fulfill request at this time, try again later"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "7",
-            "status" : 400,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "8",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        }
-      ]
-    }))
-
-    begin
-      failed = false
-      dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
-    rescue Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
-      failed = true
-      records = [].tap do |records|
-        e.retry_stream.each {|time, record| records << record}
-      end
-      assert_equal 3, records.length
-      assert_equal 2, records[0]['_id']
-      assert_equal 6, records[1]['_id']
-      assert_equal 8, records[2]['_id']
-      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 2, error_ids.length
-      assert_equal 5, error_ids[0]
-      assert_equal 7, error_ids[1]
-      @plugin.error_events.collect {|h| h[:error]}.each do |e|
-        assert_true e.respond_to?(:backtrace)
-      end
-    end
-    assert_true failed
-
-  end
-
-end
+require 'helper'
+require 'fluent/plugin/out_elasticsearch'
+require 'fluent/plugin/elasticsearch_error_handler'
+require 'json'
+
+class TestElasticsearchErrorHandler < Test::Unit::TestCase
+
+  class TestPlugin
+    attr_reader :log
+    attr_reader :write_operation, :error_events
+    def initialize(log)
+      @log = log
+      @write_operation = 'index'
+      @error_events = []
+    end
+
+    def router
+      self
+    end
+
+    def emit_error_event(tag, time, record, e)
+       @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
+    end
+
+    def process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+      if record.has_key?('raise') && record['raise']
+        raise Exception('process_message')
+      end
+    end
+  end
+
+  class MockChunk
+    def initialize(records)
+      @records = records
+      @index = 0
+    end
+    def msgpack_each
+      @records.each { |item| yield(item[:time],item[:record]) }
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+    @log_device = Fluent::Test::DummyLogDevice.new
+    dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
+    logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+    @log = Fluent::Log.new(logger)
+    @plugin = TestPlugin.new(@log)
+    @handler = Fluent::Plugin::ElasticsearchErrorHandler.new(@plugin)
+  end
+
+  def parse_response(value)
+    JSON.parse(value)
+  end
+
+  def test_dlq_400_responses
+    records = [{time: 123, record: {"foo" => "bar"}}]
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 400,
+            "_type"  : "bar",
+              "reason":"unrecognized error"
+            }
+        }
+      ]
+     }))
+    chunk = MockChunk.new(records)
+    dummy_extracted_values = []
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    assert_equal(1, @plugin.error_events.size)
+    assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
+  end
+
+  def test_retry_error
+    records = []
+    error_records = Hash.new(false)
+    error_records.merge!({0=>true, 4=>true, 9=>true})
+    10.times do |i|
+      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
+    end
+    chunk = MockChunk.new(records)
+
+    response = parse_response(%({
+      "took" : 1,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "1",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "2",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "3",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "5",
+            "status" : 500,
+            "error" : {
+              "reason":"unrecognized error - no type field"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "6",
+            "status" : 429,
+            "error" : {
+              "type" : "es_rejected_execution_exception",
+              "reason":"unable to fulfill request at this time, try again later"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "7",
+            "status" : 400,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "8",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        }
+      ]
+    }))
+
+    begin
+      failed = false
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+    rescue Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
+      failed = true
+      records = [].tap do |records|
+        e.retry_stream.each {|time, record| records << record}
+      end
+      assert_equal 3, records.length
+      assert_equal 2, records[0]['_id']
+      assert_equal 6, records[1]['_id']
+      assert_equal 8, records[2]['_id']
+      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
+      assert_equal 2, error_ids.length
+      assert_equal 5, error_ids[0]
+      assert_equal 7, error_ids[1]
+      @plugin.error_events.collect {|h| h[:error]}.each do |e|
+        assert_true e.respond_to?(:backtrace)
+      end
+    end
+    assert_true failed
+
+  end
+
+end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -1,44 +1,44 @@
-require 'helper'
-require 'date'
-require 'fluent/test/helpers'
-require 'json'
-require 'fluent/test/driver/filter'
-require 'flexmock/test_unit'
-require 'fluent/plugin/filter_elasticsearch_genid'
-
-class ElasticsearchGenidFilterTest < Test::Unit::TestCase
-  include FlexMock::TestCase
-  include Fluent::Test::Helpers
-
-  def setup
-    Fluent::Test.setup
-  end
-
-  def create_driver(conf='')
-    Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
-  end
-
-  def sample_record
-    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal '_hash', d.instance.hash_id_key
-  end
-
-  data("default" => {"hash_id_key" => "_hash"},
-       "custom_key" => {"hash_id_key" => "_edited"},
-      )
-  def test_filter(data)
-    d = create_driver("hash_id_key #{data["hash_id_key"]}")
-    flexmock(SecureRandom).should_receive(:uuid)
-      .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
-    time = event_time("2017-10-15 15:00:23.34567890 UTC")
-    d.run(default_tag: 'test') do
-      d.feed(time, sample_record)
-    end
-    assert_equal(Base64.strict_encode64(SecureRandom.uuid),
-                 d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
-  end
-end
+require 'helper'
+require 'date'
+require 'fluent/test/helpers'
+require 'json'
+require 'fluent/test/driver/filter'
+require 'flexmock/test_unit'
+require 'fluent/plugin/filter_elasticsearch_genid'
+
+class ElasticsearchGenidFilterTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='')
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal '_hash', d.instance.hash_id_key
+  end
+
+  data("default" => {"hash_id_key" => "_hash"},
+       "custom_key" => {"hash_id_key" => "_edited"},
+      )
+  def test_filter(data)
+    d = create_driver("hash_id_key #{data["hash_id_key"]}")
+    flexmock(SecureRandom).should_receive(:uuid)
+      .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
+    time = event_time("2017-10-15 15:00:23.34567890 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, sample_record)
+    end
+    assert_equal(Base64.strict_encode64(SecureRandom.uuid),
+                 d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+  end
+end