fluent-plugin-elasticsearch 4.2.1 → 4.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9eb6b06e90f827162d669e94fb9a622c09f12b35e0a05f82c961ad870e3f9726
-  data.tar.gz: c7964330b33c36176e4b1238e4ba4f2d55e60e6ee6da4fec6799f25d943ab85f
+  metadata.gz: d54b3f207af54f6fb5d3119f2d6d649478e25471fda03c56e00f9206ae72914e
+  data.tar.gz: 82eea8ed3938919f6860582698ce1253214531c806c132723e8845adfb45e2e0
 SHA512:
-  metadata.gz: fd5d961731f5866a7c841f9cbeb11f8434868988862cb9929af4cfbe174a7ce65c38d125b91dc8952120be2da4b5ad08e89c127f9c65b9e38585185acccd1cdc
-  data.tar.gz: 0c15c5670b0a2a9eda90b6962768c8db7e510e2da4b2434e3cb841d2fa5b15cf52a8f65f35b4236ab01cb8721d640c690f3bf753a4f4f269b2c802d9f2d7a9b6
+  metadata.gz: c4fd2785f716d97cb8c44845c51f933f7d0ee69fe3e2911f85bf621d043015e815639350e363ce42866b224905fc20ae12b3666d302e5f1f07937f047123b20d
+  data.tar.gz: 15d34dc457c5a98b77f2142b271d3f4ef5f611e38caeb4f225f12dcfc2c3771dea0153bae3ebe99a83cc71c6e414d67e39fea1d2fa1fa430049f90839c677578

data/History.md

@@ -2,6 +2,12 @@
 
 ### [Unreleased]
 
+### 4.2.2
+- Remove unnecessary nil check (#826)
+- Fix ILM rollover index template pattern to apply index_separator (#825)
+- Fix ILM rollover alias creation when a placeholder is used in index_name (#823)
+- Add a note about the pitfalls of per-date indexes used with ILM (#822)
+
 ### 4.2.1
 - Update a broken link (#821)
 - Include chunk_id in records (#820)

data/README.md

@@ -1881,6 +1881,16 @@ template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
 ```
 
+Note that if you create a new set of indexes every day, the elasticsearch ILM policy system will treat each day separately and will always
+maintain a separate active write index for each day.
+
+If you have a rollover based on max_age, it will continue to roll the indexes for prior dates even if no new documents are indexed.  If you want
+to delete indexes after a period of time, the ILM policy will never delete the current write index regardless of its age, so you would need a separate
+system, such as curator, to actually delete the old indexes.
+
+For this reason, if you put the date into the index names with ILM you should only rollover based on size or number of documents and may need to use
+curator to actually delete old indexes.
+
 #### Fixed ILM indices
 
 Also, users can use fixed ILM indices configuration.
@@ -1996,4 +2006,7 @@ Install dev dependencies:
 $ gem install bundler
 $ bundle install
 $ bundle exec rake test
+# To just run the test you are working on:
+$ bundle exec rake test TEST=test/plugin/test_out_elasticsearch.rb TESTOPTS='--verbose --name=test_custom_template_with_rollover_index_create_and_custom_ilm'
+
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.2.1'
+  s.version       = '4.2.2'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -73,14 +73,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
     inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
 
       log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
@@ -91,7 +92,8 @@ module Fluent::ElasticsearchIndexTemplate
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
       log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
     else
@@ -99,14 +101,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
     template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
     custom_template = if enable_ilm
                         inject_ilm_settings_to_template(deflector_alias_name,
                                                         target_index,
                                                         ilm_policy_id,
                                                         get_custom_template(template_file,
-                                                                            customize_template))
+                                                                            customize_template),
+                                                        index_separator)
                       else
                         get_custom_template(template_file, customize_template)
                       end
@@ -127,9 +130,9 @@ module Fluent::ElasticsearchIndexTemplate
     enable_ilm ? deflector_alias_name : template_name
   end
 
-  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
-    template['index_patterns'] = "#{target_index}-*"
+    template['index_patterns'] = "#{target_index}#{index_separator}*"
     if @use_legacy_template
       template.delete('template') if template.include?('template')
       # Prepare settings Hash
@@ -140,7 +143,7 @@ module Fluent::ElasticsearchIndexTemplate
         log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
       end
       template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
-      template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
+      template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
     else
       # Prepare template.settings Hash
       if !template['template'].key?('settings')
@@ -150,7 +153,7 @@ module Fluent::ElasticsearchIndexTemplate
         log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
       end
       template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
-      template['priority'] = template['priority'] ? template['priority'] + target_index.split('-').length : 100 + target_index.split('-').length
+      template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
     end
     template
   end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -259,7 +259,7 @@ EOC
             template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
           end
           verify_ilm_working if @enable_ilm
-        end 
+        end
         if @templates
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             templates_hash_install(@templates, @template_overwrite)
@@ -982,22 +982,24 @@ EOC
 
     def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
       if template_name && @template_file
-        if !@logstash_format && @alias_indexes.include?(deflector_alias)
-          log.debug("Index alias #{deflector_alias} already exists (cached)")
-        elsif !@logstash_format && @template_names.include?(template_name)
-          log.debug("Template name #{template_name} already exists (cached)")
+        if !@logstash_format && (deflector_alias.nil? || (@alias_indexes.include? deflector_alias)) && (@template_names.include? template_name)
+          if deflector_alias
+            log.debug("Index alias #{deflector_alias} and template #{template_name} already exist (cached)")
+          else
+            log.debug("Template #{template_name} already exists (cached)")
+          end
         else
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             if customize_template
-              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
+              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
             else
-              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
+              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
             end
             ilm_policy = @ilm_policies[ilm_policy_id] || {}
             create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
           end
           @alias_indexes << deflector_alias unless deflector_alias.nil?
-          @template_names << template_name unless template_name.nil?
+          @template_names << template_name
         end
       end
     end

data/test/plugin/test_out_elasticsearch.rb

@@ -1775,30 +1775,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-test").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-tag1").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
       if use_legacy_template_flag
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-test-*\",\"order\":52}").
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-tag1-*\",\"order\":52}").
           to_return(status: 200, body: "", headers: {})
       else
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"index_patterns\":\"logstash-test-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
+               body: "{\"index_patterns\":\"logstash-tag1-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
           to_return(status: 200, body: "", headers: {})
       end
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag1-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-test").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag1-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-tag1").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"logstash-test\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"logstash-tag1\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-tag2").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      if use_legacy_template_flag
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag2\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-tag2-*\",\"order\":52}").
+          to_return(status: 200, body: "", headers: {})
+      else
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"index_patterns\":\"logstash-tag2-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag2\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
+          to_return(status: 200, body: "", headers: {})
+      end      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag2-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag2-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-tag2").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"logstash-tag2\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -1815,11 +1850,104 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
       elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
       driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
+        driver.feed('tag1', event_time, sample_record)
+        driver.feed('tag2', event_time, sample_record)
+        driver.feed('tag1', event_time, sample_record)
+        driver.feed('tag2', event_time, sample_record)
+      end
+      assert_equal('logstash-tag2', index_cmds.first['index']['_index'])
+
+      assert_equal ["logstash-tag1", "logstash-tag2"], driver.instance.alias_indexes
+
+      assert_requested(elastic_request)
+    end
+
+
+    data("legacy_template" => [true, "_template"],
+         "new_template"    => [false, "_index_template"])
+    def test_template_create_with_rollover_index_and_default_ilm_and_placeholders_and_index_separator(data)
+      use_legacy_template_flag, endpoint = data
+      cwd = File.dirname(__FILE__)
+      template_file = if use_legacy_template_flag
+                        File.join(cwd, 'test_template.json')
+                      else
+                        File.join(cwd, 'test_index_template.json')
+                      end
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   logstash
+        template_file   #{template_file}
+        index_date_pattern now/w{xxxx.ww}
+        index_name logstash.${tag}
+        enable_ilm      true
+        index_separator "."
+        use_legacy_template #{use_legacy_template_flag}
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash.tag1").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      if use_legacy_template_flag
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash.tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash.tag1.*\",\"order\":52}").
+          to_return(status: 200, body: "", headers: {})
+      else
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"index_patterns\":\"logstash.tag1.*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash.tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
+          to_return(status: 200, body: "", headers: {})
       end
-      assert_equal('logstash-test', index_cmds.first['index']['_index'])
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash.tag1.default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash.tag1.default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash.tag1").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"logstash.tag1\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
 
-      assert_equal ["logstash-test"], driver.instance.alias_indexes
+      driver(config)
+
+      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed('tag1', event_time, sample_record)
+      end
+      assert_equal('logstash.tag1', index_cmds.first['index']['_index'])
+
+      assert_equal ["logstash.tag1"], driver.instance.alias_indexes
 
       assert_requested(elastic_request)
     end
@@ -2202,7 +2330,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       use_legacy_template #{use_legacy_template_flag}
     }
 
-    timestr = Time.now.strftime("%Y.%m.%d")
+    timestr = Time.now.getutc.strftime("%Y.%m.%d")
     # connection start
     stub_request(:head, "https://logs.google.com:777/es//").
       with(basic_auth: ['john', 'doe']).
@@ -2478,6 +2606,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       stub_request(:head, "https://logs.google.com:777/es//").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
+
+      # test-tag1
       # check if template exists
       stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
         with(basic_auth: ['john', 'doe']).
@@ -2487,31 +2617,78 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # creation of index which can rollover
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag1-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-test-tag1").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      if use_legacy_template_flag
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag1\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-test-tag1-*\"}").
+          to_return(status: 200, body: "", headers: {})
+      else
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"priority\":108,\"index_patterns\":\"mylogs-test-tag1-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag1\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
+          to_return(status: 200, body: "", headers: {})
+      end
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag1-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-test-tag1").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"mylogs-test-tag1\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      # test-tag2
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # creation of index which can rollover
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag2-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-custom-test").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-test-tag2").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
       if use_legacy_template_flag
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
+               body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag2\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-test-tag2-*\"}").
           to_return(status: 200, body: "", headers: {})
       else
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"priority\":108,\"index_patterns\":\"mylogs-custom-test-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
+               body: "{\"priority\":108,\"index_patterns\":\"mylogs-test-tag2-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag2\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
           to_return(status: 200, body: "", headers: {})
       end
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-custom-test").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag2-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-test-tag2").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"mylogs-custom-test\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"mylogs-test-tag2\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -2527,12 +2704,14 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       driver(config)
 
       elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
-      driver.run(default_tag: 'custom-test') do
-        driver.feed(sample_record)
+      driver.run(default_tag: 'test') do
+        driver.feed("test-tag1", event_time, sample_record)
+        driver.feed("test-tag2", event_time, sample_record)
+        driver.feed("test-tag1", event_time, sample_record)
+        driver.feed("test-tag2", event_time, sample_record)
       end
-      assert_equal('mylogs-custom-test', index_cmds.first['index']['_index'])
-
-      assert_equal ["mylogs-custom-test"], driver.instance.alias_indexes
+      assert_equal('mylogs-test-tag2', index_cmds.first['index']['_index'])
+      assert_equal ["mylogs-test-tag1", "mylogs-test-tag2"], driver.instance.alias_indexes
 
       assert_requested(elastic_request)
     end
@@ -4344,7 +4523,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_elastic
 
     ts = "2001/02/03 13:14:01,673+02:00"
-    index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
+    index = "logstash-#{Time.now.getutc.strftime("%Y.%m.%d")}"
 
     flexmock(driver.instance.router).should_receive(:emit_error_event)
       .with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.2.1
+  version: 4.2.2
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-12 00:00:00.000000000 Z
+date: 2020-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd