fluent-plugin-elasticsearch 4.0.9 → 4.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f76e6d69d85cace43250e57e272d26e492e4ae5d48856306687efc487279dd4d
-  data.tar.gz: c6f583de5857aba3c9a01bdde72e16c851d52cdc07b1151052155356954155ec
+  metadata.gz: a12dc6596a56b7486d45129b64187021215433d2e7aa5428e96712fb63a345b5
+  data.tar.gz: 6fe5296f4a0afa23e863399f6d1cc9eeaa8f70c55e5cb979ca8db1bbf7a8988f
 SHA512:
-  metadata.gz: 58003e22128dc44d615351ca0b36bd6bc2495d21b146f08713db6f8cfc77323192c38c841df446f30ec6d0edde89cced9c51a676f523e44d7488444ec9f2e737
-  data.tar.gz: 9b34567bea973a765fb2dbf699dadd26d80fd50b45b7b1179469d60425c67602e3d288f8da25305c089c20eb6daf236b900931ca0abd503d2665a5d6ab153c12
+  metadata.gz: 4f5d200eb1dee56f9f4fc05efc46cdbf440e0e2df7152e527985b124ac91ff74b0d263bee36f69fddec9f5ccc3730032145d51a95cc22fc00817f909fd4c2a7a
+  data.tar.gz: 11b0453ea6cbcfaf46e71866a96daf4ceba8f3877dd23e7aa124ea214f056b1b7f56bd8082ae273883a7403a2f1752c64beb377a2a3564c1d30757a8a5f101fa

data/.github/workflows/issue-auto-closer.yml

@@ -0,0 +1,12 @@
+name: Autocloser
+on: [issues]
+jobs:
+  autoclose:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Autoclose issues that did not follow issue template
+      uses: roots/issue-closer-action@v1.1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
+        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"

data/History.md

@@ -2,6 +2,15 @@
 
 ### [Unreleased]
 
+### 4.0.10
+- filter_elasticsearch_genid: Use entire record as hash seed (#777)
+- Suppress type in meta with suppress_type_name parameter (#774)
+- filter\_elasticsearch\_genid: Add hash generation mechanism from events (#773)
+- Clean up error text (#772)
+- Use GitHub Actions badges instead of Travis' (#760)
+- Add issue auto closer workflow (#759)
+- Document required permissions (#757)
+
 ### 4.0.9
 - Add possibility to configure multiple ILM policies (#753)
 - Document required permissions (#757)

data/README.ElasticsearchGenID.md

@@ -0,0 +1,116 @@
+## Index
+
+* [Usage](#usage)
+* [Configuration](#configuration)
+  + [hash_id_key](#hash_id_key)
+  + [include_tag_in_seed](#include_tag_in_seed)
+  + [include_time_in_seed](#include_time_in_seed)
+  + [use_record_as_seed](#use_record_as_seed)
+  + [use_entire_record](#use_entire_record)
+  + [record_keys](#record_keys)
+  + [separator](#separator)
+  + [hash_type](#hash_type)
+* [Advanced Usage](#advanced-usage)
+
+## Usage
+
+In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
+
+```
+<source>
+  @type elasticsearch_genid
+  hash_id_key _hash
+  include_tag_in_seed false
+  include_time_in_seed false
+  use_record_as_seed false
+  use_entire_record false
+  record_keys []
+  separator _
+  hash_type md5
+</match>
+```
+
+## Configuration
+
+### hash_id_key
+
+```
+hash_id_key _id
+```
+
+You can specify generated hash storing key.
+
+### include_tag_in_seed
+
+```
+include_tag_in_seed true
+```
+
+You can specify to use tag for hash generation seed.
+
+### include_time_in_seed
+
+```
+include_time_in_seed true
+```
+
+You can specify to use time for hash generation seed.
+
+### use_record_as_seed
+
+```
+use_record_as_seed true
+```
+
+You can specify to use record in events for hash generation seed. This parameter should be used with [record_keys](#record_keys) parameter in practice.
+
+### record_keys
+
+```
+record_keys request_id,pipeline_id
+```
+
+You can specify keys which are record in events for hash generation seed. This parameter should be used with [use_record_as_seed](#use_record_as_seed) parameter in practice.
+
+### use_entire_record
+
+```
+use_entire_record true
+```
+
+You can specify to use entire record in events for hash generation seed.
+
+
+### separator
+
+```
+separator |
+```
+
+You can specify separator charactor to creating seed for hash generation.
+
+### hash_type
+
+```
+hash_type sha1
+```
+
+You can specify hash algorithm.
+
+## Advanced Usage
+
+Elasticsearch GenID plugin can handle record contents differing with the following parameters:
+
+```aconf
+<filter the.awesome.your.routing.tag>
+  @type elasticsearch_genid
+  use_entire_record true
+  hash_type sha1
+  hash_id_key _hash
+  separator _
+  inc_time_as_key true
+  inc_tag_as_key true
+</filter>
+```
+
+The above configuration can handle tag, time, and record differing and generate different base64 encoded hash per record.

data/README.md

@@ -1,7 +1,9 @@
 # Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
-[![Build Status](https://travis-ci.org/uken/fluent-plugin-elasticsearch.png?branch=master)](https://travis-ci.org/uken/fluent-plugin-elasticsearch)
+![Testing on Windows](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Windows/badge.svg?branch=master)
+![Testing on macOS](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20macOS/badge.svg?branch=master)
+![Testing on Ubuntu](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Ubuntu/badge.svg?branch=master)
 [![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
 [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
 
@@ -31,6 +33,7 @@ Current maintainers: @cosmo0920
   + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
   + [include_timestamp](#include_timestamp)
   + [utc_index](#utc_index)
+  + [suppress_type_name](#suppress_type_name)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
   + [template_name](#template_name)
@@ -97,6 +100,7 @@ Current maintainers: @cosmo0920
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
+* [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
 * [Elasticsearch permissions](#elasticsearch-permissions)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
@@ -351,6 +355,20 @@ utc_index true
 
 By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
 
+### suppress_type_name
+
+In Elasticsearch 7.x, Elasticsearch cluster complains the following types removal warnings:
+
+```json
+{"type": "deprecation", "timestamp": "2020-07-03T08:02:20,830Z", "level": "WARN", "component": "o.e.d.a.b.BulkRequestParser", "cluster.name": "docker-cluster", "node.name": "70dd5c6b94c3", "message": "[types removal] Specifying types in bulk requests is deprecated.", "cluster.uuid": "NoJJmtzfTtSzSMv0peG8Wg", "node.id": "VQ-PteHmTVam2Pnbg7xWHw"  }
+```
+
+This can be suppressed with:
+
+```
+suppress_type_name true
+```
+
 ### target_index_key
 
 Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
@@ -985,7 +1003,7 @@ If you use Fluentd directly, you must pass the following lines as Fluentd comman
 
 ```
 sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
-$ fluentd -r $sniffer" [AND YOUR OTHER OPTIONS]
+$ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
 ```
 
 ### Reload After
@@ -1243,6 +1261,10 @@ Default value is `nil`.
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
 
+## Configuration - Elasticsearch Filter GenID
+
+See [Elasticsearch Filter GenID document](README.ElasticsearchGenID.md)
+
 ## Elasticsearch permissions
 
 If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.9'
+  s.version       = '4.0.10'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -7,6 +7,13 @@ module Fluent::Plugin
     Fluent::Plugin.register_filter('elasticsearch_genid', self)
 
     config_param :hash_id_key, :string, :default => '_hash'
+    config_param :include_tag_in_seed, :bool, :default => false
+    config_param :include_time_in_seed, :bool, :default => false
+    config_param :use_record_as_seed, :bool, :default => false
+    config_param :use_entire_record, :bool, :default => false
+    config_param :record_keys, :array, :default => []
+    config_param :separator, :string, :default => '_'
+    config_param :hash_type, :enum, list: [:md5, :sha1, :sha256, :sha512], :default => :sha1
 
     def initialize
       super
@@ -14,12 +21,57 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+
+      if !@use_entire_record
+        if @record_keys.empty? && @use_record_as_seed
+          raise Fluent::ConfigError, "When using record as hash seed, users must specify `record_keys`."
+        end
+      end
+
+      if @use_record_as_seed
+        class << self
+          alias_method :filter, :filter_seed_as_record
+        end
+      else
+        class << self
+          alias_method :filter, :filter_simple
+        end
+      end
     end
 
     def filter(tag, time, record)
+      # for safety.
+    end
+
+    def filter_simple(tag, time, record)
       record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
       record
     end
 
+    def filter_seed_as_record(tag, time, record)
+      seed = ""
+      seed += tag + separator if @include_tag_in_seed
+      seed += time.to_s + separator if @include_time_in_seed
+      if @use_entire_record
+        record.each {|k,v| seed += "|#{k}|#{v}"}
+      else
+        seed += record_keys.map {|k| record[k]}.join(separator)
+      end
+      record[@hash_id_key] = Base64.strict_encode64(encode_hash(@hash_type, seed))
+      record
+    end
+
+    def encode_hash(type, seed)
+      case type
+      when :md5
+        Digest::MD5.digest(seed)
+      when :sha1
+        Digest::SHA1.digest(seed)
+      when :sha256
+        Digest::SHA256.digest(seed)
+      when :sha512
+        Digest::SHA512.digest(seed)
+      end
+    end
   end
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -90,6 +90,7 @@ EOC
     config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
     config_param :utc_index, :bool, :default => true
     config_param :type_name, :string, :default => DEFAULT_TYPE_NAME
+    config_param :suppress_type_name, :bool, :default => false
     config_param :index_name, :string, :default => "fluentd"
     config_param :id_key, :string, :default => nil
     config_param :write_operation, :string, :default => "index"
@@ -234,7 +235,7 @@ EOC
             raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
           end
           if @ilm_policy.empty? && @ilm_policy_overwrite
-            raise Fluent::ConfigError, "ilm_policy_overwrite can work with non empty ilm_policy. Specify non-empty ilm policy into ilm_policy. "
+            raise Fluent::ConfigError, "ilm_policy_overwrite requires a non empty ilm_policy."
           end
           if @logstash_format || placeholder_substitution_needed_for_template?
             class << self
@@ -304,16 +305,20 @@ EOC
                                  else
                                    @default_elasticsearch_version
                                  end
-      if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
-      end
-      if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
-        @type_name = '_doc'.freeze
-      end
-      if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.info "Detected ES 8.x or above: This parameter has no effect."
+      if @suppress_type_name && @last_seen_major_version >= 7
         @type_name = nil
+      else
+        if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
+        end
+        if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
+          @type_name = '_doc'.freeze
+        end
+        if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.info "Detected ES 8.x or above: This parameter has no effect."
+          @type_name = nil
+        end
       end
 
       if @validate_client_version && !dry_run?
@@ -856,7 +861,9 @@ EOC
           target_type = nil
         end
       else
-        if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+        if @suppress_type_name && @last_seen_major_version >= 7
+          target_type = nil
+        elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
         elsif @last_seen_major_version >= 8

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
   end
 
+  test "invalid configuration" do
+    assert_raise(Fluent::ConfigError) do
+      create_driver("use_record_as_seed true")
+    end
+  end
+
   def sample_record
     {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
   end
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
     assert_equal(Base64.strict_encode64(SecureRandom.uuid),
                  d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
   end
+
+  class UseRecordAsSeedTest < self
+    data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
+         "sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
+         "sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
+         "sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
+        )
+    def test_simple(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
+         "sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
+         "sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
+         "sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
+    def test_record_with_tag(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
+         "sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
+         "sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
+         "sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
+    def test_record_with_time(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
+         "sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
+         "sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
+         "sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
+
+  class UseEntireRecordAsSeedTest < self
+    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
+         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
+         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
+         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    def test_record
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
+         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
+         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
+         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    def test_record_with_tag
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
+         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
+         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
+         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    def test_record_with_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
+         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
+         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
+         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
 end

data/test/plugin/test_out_elasticsearch.rb

@@ -3147,6 +3147,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   data("border"        => {"es_version" => 6, "_type" => "mytype"},
        "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+       "to be nil"     => {"es_version" => 8, "_type" => nil},
       )
   def test_writes_to_speficied_type(data)
     driver('', data["es_version"]).configure("type_name mytype\n")
@@ -3159,6 +3160,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   data("border"        => {"es_version" => 6, "_type" => "mytype.test"},
        "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+       "to be nil"     => {"es_version" => 8, "_type" => nil},
       )
   def test_writes_to_speficied_type_with_placeholders(data)
     driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
@@ -3169,6 +3171,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(data['_type'], index_cmds.first['index']['_type'])
   end
 
+  data("border"        => {"es_version" => 6, "_type" => "mytype.test"},
+       "fixed_behavior"=> {"es_version" => 7, "_type" => nil},
+       "to be nil"     => {"es_version" => 8, "_type" => nil},
+      )
+  def test_writes_to_speficied_type_with_suppress_type_name(data)
+    driver('', data["es_version"])
+      .configure("type_name mytype.${tag}\nsuppress_type_name true")
+    stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal(data['_type'], index_cmds.first['index']['_type'])
+  end
+
   data("old"           => {"es_version" => 2, "_type" => "local-override"},
        "old_behavior"  => {"es_version" => 5, "_type" => "local-override"},
        "border"        => {"es_version" => 6, "_type" => "fluentd"},

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.9
+  version: 4.0.10
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-27 00:00:00.000000000 Z
+date: 2020-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -135,6 +135,7 @@ extra_rdoc_files: []
 files:
 - ".coveralls.yml"
 - ".editorconfig"
+- ".github/workflows/issue-auto-closer.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/macos.yml"
 - ".github/workflows/windows.yml"
@@ -146,6 +147,7 @@ files:
 - ISSUE_TEMPLATE.md
 - LICENSE.txt
 - PULL_REQUEST_TEMPLATE.md
+- README.ElasticsearchGenID.md
 - README.ElasticsearchInput.md
 - README.md
 - Rakefile