fluent-plugin-elasticsearch 4.0.6 → 4.0.11

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -55,6 +55,7 @@ module Fluent::Plugin
     attr_reader :alias_indexes
     attr_reader :template_names
     attr_reader :ssl_version_options
+    attr_reader :compressable_connection
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -89,6 +90,7 @@ EOC
     config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
     config_param :utc_index, :bool, :default => true
     config_param :type_name, :string, :default => DEFAULT_TYPE_NAME
+    config_param :suppress_type_name, :bool, :default => false
     config_param :index_name, :string, :default => "fluentd"
     config_param :id_key, :string, :default => nil
     config_param :write_operation, :string, :default => "index"
@@ -143,6 +145,7 @@ see: https://github.com/elastic/elasticsearch-ruby/pull/514
 EOC
     config_param :include_index_in_url, :bool, :default => false
     config_param :http_backend, :enum, list: [:excon, :typhoeus], :default => :excon
+    config_param :http_backend_excon_nonblock, :bool, :default => true
     config_param :validate_client_version, :bool, :default => false
     config_param :prefer_oj_serializer, :bool, :default => false
     config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "es_rejected_execution_exception"]
@@ -158,6 +161,7 @@ EOC
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
     config_param :ilm_policy, :hash, :default => {}
+    config_param :ilm_policies, :hash, :default => {}
     config_param :ilm_policy_overwrite, :bool, :default => false
     config_param :truncate_caches_interval, :time, :default => nil
 
@@ -218,6 +222,11 @@ EOC
         log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
       end
 
+      raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
+
+      unless @ilm_policy.empty?
+        @ilm_policies = { @ilm_policy_id => @ilm_policy }
+      end
       @alias_indexes = []
       @template_names = []
       if !dry_run?
@@ -226,14 +235,14 @@ EOC
             raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
           end
           if @ilm_policy.empty? && @ilm_policy_overwrite
-            raise Fluent::ConfigError, "ilm_policy_overwrite can work with non empty ilm_policy. Specify non-empty ilm policy into ilm_policy. "
+            raise Fluent::ConfigError, "ilm_policy_overwrite requires a non empty ilm_policy."
           end
           if @logstash_format || placeholder_substitution_needed_for_template?
             class << self
               alias_method :template_installation, :template_installation_actual
             end
           else
-            template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
+            template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
           end
           verify_ilm_working if @enable_ilm
         elsif @templates
@@ -296,16 +305,20 @@ EOC
                                  else
                                    @default_elasticsearch_version
                                  end
-      if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
-      end
-      if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
-        @type_name = '_doc'.freeze
-      end
-      if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.info "Detected ES 8.x or above: This parameter has no effect."
+      if @suppress_type_name && @last_seen_major_version >= 7
         @type_name = nil
+      else
+        if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
+        end
+        if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
+          @type_name = '_doc'.freeze
+        end
+        if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.info "Detected ES 8.x or above: This parameter has no effect."
+          @type_name = nil
+        end
       end
 
       if @validate_client_version && !dry_run?
@@ -327,6 +340,13 @@ EOC
         end
       end
 
+      if @ssl_version && @scheme == :https
+        if !@http_backend_excon_nonblock
+          log.warn "TLS handshake will be stucked with block connection.
+                    Consider to set `http_backend_excon_nonblock` as true"
+        end
+      end
+
       # Consider missing the prefix of "$." in nested key specifiers.
       @id_key = convert_compat_id_key(@id_key) if @id_key
       @parent_key = convert_compat_id_key(@parent_key) if @parent_key
@@ -335,6 +355,7 @@ EOC
       @routing_key_name = configure_routing_key_name
       @meta_config_map = create_meta_config_map
       @current_config = nil
+      @compressable_connection = false
 
       @ignore_exception_classes = @ignore_exceptions.map do |exception|
         unless Object.const_defined?(exception)
@@ -409,7 +430,7 @@ EOC
     def backend_options
       case @http_backend
       when :excon
-        { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
+        { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
       when :typhoeus
         require 'typhoeus'
         { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
@@ -503,13 +524,15 @@ EOC
       return Time.at(event_time).to_datetime
     end
 
-    def client(host = nil)
+    def client(host = nil, compress_connection = false)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
       @_es = nil unless is_existing_connection(connection_options[:hosts])
+      @_es = nil unless @compressable_connection == compress_connection
 
       @_es ||= begin
+        @compressable_connection = compress_connection
         @current_config = connection_options[:hosts].clone
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
         local_reload_connections = @reload_connections
@@ -517,7 +540,7 @@ EOC
           local_reload_connections = @reload_after
         end
 
-        gzip_headers = if compression
+        gzip_headers = if compress_connection
                          {'Content-Encoding' => 'gzip'}
                        else
                          {}
@@ -543,7 +566,7 @@ EOC
                                                                               },
                                                                               sniffer_class: @sniffer_class,
                                                                               serializer_class: @serializer_class,
-                                                                              compression: compression,
+                                                                              compression: compress_connection,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -707,7 +730,12 @@ EOC
       else
         pipeline = nil
       end
-      return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
+      if @ilm_policy_id
+        ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
+      else
+        ilm_policy_id = nil
+      end
+      return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
     end
 
     def multi_workers_ready?
@@ -733,9 +761,9 @@ EOC
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url
-                   RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze])
+                   RequestInfo.new(host, meta.delete("_index".freeze), meta.delete("_alias".freeze))
                  else
-                   RequestInfo.new(host, nil, meta["_index".freeze])
+                   RequestInfo.new(host, nil, meta.delete("_alias".freeze))
                  end
 
           if split_request?(bulk_message, info)
@@ -781,7 +809,7 @@ EOC
     end
 
     def process_message(tag, meta, header, time, record, extracted_values)
-      logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
+      logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
         record = flatten_record(record)
@@ -804,17 +832,19 @@ EOC
 
       target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
       if target_index_parent && target_index_parent[target_index_child_key]
-        target_index = target_index_parent.delete(target_index_child_key)
+        target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
       elsif @logstash_format
         dt = dt.new_offset(0) if @utc_index
         target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
+        target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
       else
-        target_index = index_name
+        target_index_alias = target_index = index_name
       end
 
       # Change target_index to lower-case since Elasticsearch doesn't
       # allow upper-case characters in index names.
       target_index = target_index.downcase
+      target_index_alias = target_index_alias.downcase
       if @include_tag_key
         record[@tag_key] = tag
       end
@@ -833,7 +863,9 @@ EOC
           target_type = nil
         end
       else
-        if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+        if @suppress_type_name && @last_seen_major_version >= 7
+          target_type = nil
+        elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
         elsif @last_seen_major_version >= 8
@@ -847,6 +879,7 @@ EOC
       meta.clear
       meta["_index".freeze] = target_index
       meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
+      meta["_alias".freeze] = target_index_alias
 
       if @pipeline
         meta["pipeline".freeze] = pipeline
@@ -889,16 +922,17 @@ EOC
         placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
         placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
         placeholder?(:deflector_alias, @deflector_alias.to_s) ||
-        placeholder?(:application_name, @application_name.to_s)
+        placeholder?(:application_name, @application_name.to_s) ||
+        placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
       log.debug("Need substitution: #{need_substitution}")
       need_substitution
     end
 
-    def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
+    def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
       # for safety.
     end
 
-    def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
+    def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
       if template_name && @template_file
         if @alias_indexes.include? deflector_alias
           log.debug("Index alias #{deflector_alias} already exists (cached)")
@@ -907,11 +941,12 @@ EOC
         else
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             if customize_template
-              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
+              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
             else
-              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
+              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
             end
-            create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, @ilm_policy_overwrite, host)
+            ilm_policy = @ilm_policies[ilm_policy_id] || {}
+            create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
           end
           @alias_indexes << deflector_alias unless deflector_alias.nil?
           @template_names << template_name unless template_name.nil?
@@ -922,11 +957,11 @@ EOC
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
-      logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
+      logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
       if deflector_alias
-        template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
+        template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
       else
-        template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
+        template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
       end
 
       begin
@@ -939,7 +974,7 @@ EOC
                           data
                         end
 
-        response = client(info.host).bulk body: prepared_data, index: info.index
+        response = client(info.host, compression).bulk body: prepared_data, index: info.index
         log.on_trace { log.trace "bulk response: #{response}" }
 
         if response['errors']

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -35,16 +35,18 @@ module Fluent::Plugin
     end
 
 
-    def client(host = nil)
+    def client(host = nil, compress_connection = false)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
       @_es = nil unless is_existing_connection(connection_options[:hosts])
+      @_es = nil unless @compressable_connection == compress_connection
 
       @_es ||= begin
+        @compressable_connection = compress_connection
         @current_config = connection_options[:hosts].clone
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
-        gzip_headers = if compression
+        gzip_headers = if compress_connection
                          {'Content-Encoding' => 'gzip'}
                        else
                          {}
@@ -67,7 +69,7 @@ module Fluent::Plugin
                                                                                 password: @password,
                                                                                 scheme: @scheme
                                                                               },
-                                                                              compression: compression,
+                                                                              compression: compress_connection,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -228,7 +230,7 @@ module Fluent::Plugin
                         else
                           data
                         end
-        response = client(host).bulk body: prepared_data, index: index
+        response = client(host, compression).bulk body: prepared_data, index: index
         if response['errors']
           log.error "Could not push log to Elasticsearch: #{response}"
         end

data/test/plugin/test_elasticsearch_tls.rb

@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
       d = driver('')
       ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
       if @use_tls_minmax_version
-        assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION,
-                      min_version: OpenSSL::SSL::TLS1_VERSION}, ssl_version_options)
+        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                      min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
       else
         assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
       end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
   end
 
+  test "invalid configuration" do
+    assert_raise(Fluent::ConfigError) do
+      create_driver("use_record_as_seed true")
+    end
+  end
+
   def sample_record
     {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
   end
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
     assert_equal(Base64.strict_encode64(SecureRandom.uuid),
                  d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
   end
+
+  class UseRecordAsSeedTest < self
+    data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
+         "sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
+         "sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
+         "sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
+        )
+    def test_simple(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
+         "sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
+         "sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
+         "sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
+    def test_record_with_tag(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
+         "sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
+         "sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
+         "sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
+    def test_record_with_time(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
+         "sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
+         "sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
+         "sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
+
+  class UseEntireRecordAsSeedTest < self
+    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
+         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
+         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
+         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    def test_record
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
+         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
+         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
+         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    def test_record_with_tag
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
+         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
+         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
+         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    def test_record_with_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
+         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
+         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
+         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
 end