fluent-plugin-elasticsearch 4.2.0 → 4.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46901e5f885a46c37ea29dca2968d53fd4026dec637f007936cabdd3ee070c5c
-  data.tar.gz: 21f3d3e9dd22752bcab060ea46ba12e8309147c2ddbb3be4aea656b527aff103
+  metadata.gz: 9eb6b06e90f827162d669e94fb9a622c09f12b35e0a05f82c961ad870e3f9726
+  data.tar.gz: c7964330b33c36176e4b1238e4ba4f2d55e60e6ee6da4fec6799f25d943ab85f
 SHA512:
-  metadata.gz: ed30b7f203f9c102fdedb0a28c88210f172e9bccab3f153de81ba88b821a065e1cfdb8caccc097fbb83745069c7c5cd052e70879501ebd83b472f55c835b6116
-  data.tar.gz: 3ded3c8f6041224546a71df34878148a14b0f2e47970b86efb11d8b4fe4008764680534149b1c56c1651bc7dbfffc0df26ad7a459c87e739c91f391d8da83f42
+  metadata.gz: fd5d961731f5866a7c841f9cbeb11f8434868988862cb9929af4cfbe174a7ce65c38d125b91dc8952120be2da4b5ad08e89c127f9c65b9e38585185acccd1cdc
+  data.tar.gz: 0c15c5670b0a2a9eda90b6962768c8db7e510e2da4b2434e3cb841d2fa5b15cf52a8f65f35b4236ab01cb8721d640c690f3bf753a4f4f269b2c802d9f2d7a9b6

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 4.2.1
+- Update a broken link (#821)
+- Include chunk_id in records (#820)
+- Fix a failing testcase (#818)
+
 ### 4.2.0
 - ci: Add Ruby 2.7 jobs (#812)
 - Support Elasticsearch new style template (#810)

data/README.md

@@ -102,6 +102,9 @@ Current maintainers: @cosmo0920
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
   + [use_legacy_template](#use_legacy_template)
+  + [metadata section](#metadata-section)
+    + [include_chunk_id](#include_chunk_id)
+    + [chunk_id_key](#chunk_id_key)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
 * [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
 * [Elasticsearch permissions](#elasticsearch-permissions)
@@ -921,7 +924,7 @@ Starting with version 0.8.0, this gem uses excon, which supports proxy with envi
 
 ### Buffer options
 
-`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/v0.14/articles/output-plugin-overview) and use `compat_parameters` plugin helper. It adds the following options:
+`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/output#overview) and use `compat_parameters` plugin helper. It adds the following options:
 
 ```
 buffer_type memory
@@ -1316,6 +1319,51 @@ Composable template documentation is [Put Index Template API | Elasticsearch Ref
 
 Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
 
+## <metadata\> section
+
+Users can specify whether including `chunk_id` information into records or not:
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+    # chunk_id_key chunk_id # Default value is "chunk_id".
+  </metadata>
+</match>
+```
+
+### include_chunk_id
+
+Whether including `chunk_id` for not. Default value is `false`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+  </metadata>
+</match>
+```
+
+
+### chunk_id_key
+
+Specify `chunk_id_key` to store `chunk_id` information into records. Default value is `chunk_id`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id
+    chunk_id_key chunk_hex
+  </metadata>
+</match>
+```
+
 ## Configuration - Elasticsearch Input
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.2.0'
+  s.version       = '4.2.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -18,6 +18,7 @@ require 'fluent/plugin/output'
 require 'fluent/event'
 require 'fluent/error'
 require 'fluent/time'
+require 'fluent/unique_id'
 require 'fluent/log-ext'
 require 'zlib'
 require_relative 'elasticsearch_constants'
@@ -171,6 +172,11 @@ EOC
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
 
+    config_section :metadata, param_name: :metainfo, multi: false do
+      config_param :include_chunk_id, :bool, :default => false
+      config_param :chunk_id_key, :string, :default => "chunk_id".freeze
+    end
+
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
       config_set_default :chunk_keys, ['tag']
@@ -771,6 +777,15 @@ EOC
       true
     end
 
+    def inject_chunk_id_to_record_if_needed(record, chunk_id)
+      if @metainfo&.include_chunk_id
+        record[@metainfo.chunk_id_key] = chunk_id
+        record
+      else
+        record
+      end
+    end
+
     def write(chunk)
       bulk_message_count = Hash.new { |h,k| h[k] = 0 }
       bulk_message = Hash.new { |h,k| h[k] = '' }
@@ -778,6 +793,7 @@ EOC
       meta = {}
 
       tag = chunk.metadata.tag
+      chunk_id = dump_unique_id_hex(chunk.unique_id)
       extracted_values = expand_placeholders(chunk)
       host = if @hosts
                extract_placeholders(@hosts, chunk)
@@ -787,6 +803,9 @@ EOC
 
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
+
+        record = inject_chunk_id_to_record_if_needed(record, chunk_id)
+
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url

data/test/plugin/test_elasticsearch_tls.rb

@@ -62,8 +62,14 @@ class TestElasticsearchTLS < Test::Unit::TestCase
       d = driver('')
       ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
       if @use_tls_minmax_version
-        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
-                      min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+        if @enabled_tlsv1_3
+          assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+        else
+          assert_equal({max_version: nil,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+
+        end
       else
         assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
       end

data/test/plugin/test_out_elasticsearch.rb

@@ -230,8 +230,13 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_nil instance.ssl_max_version
     assert_nil instance.ssl_min_version
     if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
-      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
-                   instance.ssl_version_options)
+      if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
+                     instance.ssl_version_options)
+      else
+        assert_equal({max_version: nil, min_version: OpenSSL::SSL::TLS1_2_VERSION},
+                     instance.ssl_version_options)
+      end
     else
       assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
                    instance.ssl_version_options)
@@ -1873,7 +1878,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       else
         stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"index_patterns\":\"task_definition-2020.09-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"task_definition-2020.09\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
+               body: "{\"index_patterns\":\"task_definition-#{date_str}-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"task_definition-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
           to_return(:status => 200, :body => "", :headers => {})
       end
       # check if alias exists
@@ -3404,6 +3409,39 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_requested(request, times: 2)
   end
 
+  def test_writes_with_record_metadata
+    chunk_id_key = "metadata_key".freeze
+    driver.configure(Fluent::Config::Element.new(
+                       'ROOT', '', {
+                         '@type' => 'elasticsearch',
+                       }, [
+                         Fluent::Config::Element.new('metadata', '', {
+                                                       'include_chunk_id' => true,
+                                                       'chunk_id_key' => chunk_id_key,
+                                                     }, [])
+                       ]
+                     ))
+    stub_request(:post, "http://localhost:9200/_bulk").
+      with(
+        body: /{"index":{"_index":"fluentd","_type":"fluentd"}}\n{"age":26,"request_id":"42","parent_id":"parent","routing_id":"routing","#{chunk_id_key}":".*"}\n/) do |req|
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+    end
+    driver.run(default_tag: 'test', shutdown: false) do
+      driver.feed(sample_record)
+    end
+    assert_true index_cmds[1].has_key?(chunk_id_key)
+    first_chunk_id = index_cmds[1].fetch(chunk_id_key)
+
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_true index_cmds[1].has_key?(chunk_id_key)
+    second_chunk_id = index_cmds[1].fetch(chunk_id_key)
+    assert do
+      first_chunk_id != second_chunk_id
+    end
+  end
+
   def test_writes_with_huge_records_but_uncheck
     driver.configure(Fluent::Config::Element.new(
                        'ROOT', '', {

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -101,8 +101,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_nil instance.ssl_max_version
     assert_nil instance.ssl_min_version
     if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
-      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
-                   instance.ssl_version_options)
+      if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
+                     instance.ssl_version_options)
+      else
+        assert_equal({max_version: nil, min_version: OpenSSL::SSL::TLS1_2_VERSION},
+                     instance.ssl_version_options)
+      end
     else
       assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
                    instance.ssl_version_options)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.2.1
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-23 00:00:00.000000000 Z
+date: 2020-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd