fluent-plugin-elasticsearch 3.7.0 → 3.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3edaea3928f2cc09c7a908d870afdc2cac2a6d15f2e3f36cc7aa2f27962da79f
-  data.tar.gz: '09c000b05842db4f08a546f09adb3fe725d50fcf213435d3c7fc0dfa94753551'
+  metadata.gz: 66d34ebe18eabf21cd55dbe1c0f3d4f469298e3ada1072849068c1d490e62650
+  data.tar.gz: f91f6d38cc821642ddd6bc2fda232534b14d070fa9694db328bdef68d982d93a
 SHA512:
-  metadata.gz: 148893479963ccf22397b41be92ea10dbe49e852d54412daa10659e8eb52c8bd1f0ac9546549ec28dde9df53a40f41e6fc45746eb95c10ab8e885e8e38d60537
-  data.tar.gz: f7e31c37963276c11162d78aa45b52c71feaaf409dd548b72cfd32c652b784008ccc0d0777da0f050817f3d1803023e523c3d693d193a21725aeeb3849518fd6
+  metadata.gz: d83715d884fd22a0cb487e0eea16d37679800ce477bfcc888546d1856e48dd8bfa45106826731008b5b3d67cb2408a4d7eca9740a92076ec877a0f999252a512
+  data.tar.gz: d5e476b7ec45883dceeecffeb529d43c1b5a16a9a6c203139ed42891a8e702bc74861edcced7bfd87340e4f3150de0f5a7f7d9ecff8840d1dab9e1348db11601

data/History.md

@@ -1,6 +1,11 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+### 3.7.1
+- Make conpatible for Fluentd v1.8 (#677)
+- Handle flatten_hashes in elasticsearch_dynamic (#675)
+- Handle empty index_date_pattern parameter (#674)
+
 ### 3.7.0
 - Tweak for cosmetic change (#671)
 - Fix access to Elasticsearch::Transport::VERSION with explicit top level class path (#670)

data/README.ElasticsearchInput.md

@@ -284,7 +284,7 @@ Elasticsearch Input plugin and Elasticsearch output plugin can combine to transf
   host transferred-cluster.local
   port 9200
   index_name ${$.@metadata._index}
-  # type_name ${$.@metadata._type} # This parameter is optional due to Removal of mapping types since ES7.
+  type_name ${$.@metadata._type} # This parameter will be deprecated due to Removal of mapping types since ES7.
   id_key ${$.@metadata._id} # This parameter is needed for prevent duplicated records.
   <buffer tag, $.@metadata._index, $.@metadata._type, $.@metadata._id>
     @type memory # should use file buffer for preventing chunk lost

data/README.md

@@ -434,10 +434,15 @@ for example: <logstash-default-{now/d}-000001>. Overriding this changes the roll
 "now/w{xxxx.ww}" would create weekly rollover indexes instead of daily.
 
 This setting only takes effect when combined with the [rollover_index](#rollover_index) setting.
+
+And rollover\_index is also used in Lifecycle Index Management(ILM) feature.
 ```
 index_date_pattern "now/w{xxxx.ww}" # defaults to "now/d"
 ```
 
+If empty string(`""`) is specified in `index_date_patter`, index date pattern is not used.
+Elasticsearch plugin just creates <`index_prefix`-`application_name`-000001> rollover index instead of <`index_prefix`-`application_name`-`{index_date_pattern}`-000001>.
+
 If [customize_template](#customize_template) is set, then this parameter will be in effect otherwise ignored.
 
 ### deflector_alias

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.7.0'
+  s.version       = '3.7.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -130,7 +130,11 @@ module Fluent::ElasticsearchIndexTemplate
   def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, host)
     if rollover_index
       if !client.indices.exists_alias(:name => deflector_alias_name)
-        index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+        if index_date_pattern.empty?
+          index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-000001>'
+        else
+          index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+        end
         indexcreation(index_name_temp, host)
         body = {}
         body = rollover_alias_payload(deflector_alias_name) if enable_ilm

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -206,7 +206,7 @@ EOC
       end
 
       @alias_indexes = []
-      if !Fluent::Engine.dry_run_mode
+      if !dry_run?
         if @template_name && @template_file
           if @rollover_index
             raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
@@ -259,7 +259,7 @@ EOC
       end
 
       @last_seen_major_version =
-        if @verify_es_version_at_startup && !Fluent::Engine.dry_run_mode
+        if @verify_es_version_at_startup && !dry_run?
           retry_operate(@max_retry_get_es_version) do
             detect_es_major_version
           end
@@ -278,7 +278,7 @@ EOC
         @type_name = nil
       end
 
-      if @validate_client_version && !Fluent::Engine.dry_run_mode
+      if @validate_client_version && !dry_run?
         if @last_seen_major_version != client_library_version.to_i
           raise Fluent::ConfigError, <<-EOC
             Detected ES #{@last_seen_major_version} but you use ES client #{client_library_version}.
@@ -343,6 +343,14 @@ EOC
       end
     end
 
+    def dry_run?
+      if Fluent::Engine.respond_to?(:dry_run_mode)
+        Fluent::Engine.dry_run_mode
+      elsif Fluent::Engine.respond_to?(:supervisor_mode)
+        Fluent::Engine.supervisor_mode
+      end
+    end
+
     def placeholder?(name, param)
       begin
         placeholder_validate!(name, param)

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -132,6 +132,10 @@ module Fluent::Plugin
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
+        if @flatten_hashes
+          record = flatten_record(record)
+        end
+
         begin
           # evaluate all configurations here
           DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|

data/test/plugin/test_out_elasticsearch.rb

@@ -672,6 +672,77 @@ class ElasticsearchOutput < Test::Unit::TestCase
       assert_requested(elastic_request)
     end
 
+    def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   logstash
+        template_file   #{template_file}
+        rollover_index  true
+        index_date_pattern ""
+        deflector_alias myapp_deflector
+        enable_ilm      true
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
+        to_return(status: 200, body: "", headers: {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/_alias/myapp_deflector").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed(sample_record)
+      end
+
+      assert_requested(elastic_request)
+    end
+
     def test_template_create_with_rollover_index_and_custom_ilm
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -465,6 +465,40 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(2000, total)
   end
 
+  def test_nested_record_with_flattening_on
+    driver.configure("flatten_hashes true
+                      flatten_hashes_separator |")
+
+    original_hash =  {"foo" => {"bar" => "baz"}, "people" => [
+      {"age" => "25", "height" => "1ft"},
+      {"age" => "30", "height" => "2ft"}
+    ]}
+
+    expected_output = {"foo|bar"=>"baz", "people" => [
+      {"age" => "25", "height" => "1ft"},
+      {"age" => "30", "height" => "2ft"}
+    ]}
+
+    stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(original_hash)
+    end
+    assert_equal expected_output, index_cmds[1]
+  end
+
+  def test_nested_record_with_flattening_off
+    # flattening off by default
+
+    original_hash =  {"foo" => {"bar" => "baz"}}
+    expected_output = {"foo" => {"bar" => "baz"}}
+
+    stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(original_hash)
+    end
+    assert_equal expected_output, index_cmds[1]
+  end
+
   def test_makes_bulk_request
     stub_elastic
     driver.run(default_tag: 'test') do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 3.7.1
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-18 00:00:00.000000000 Z
+date: 2019-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd