fluent-plugin-elasticsearch 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e963cd715f4b07ec74b88df45a4b27bd28ea3d9a81a986720eb103baf1f4ab8
-  data.tar.gz: 99adcede30be2bff2bce8eebdfb39e67a14028e46d073516411bfb81bcd623df
+  metadata.gz: 2d857451dc47373e32d9b2833fcb63a6afca5e45c1d131ee229ff8e29e408955
+  data.tar.gz: c0b7f154abe0b1e8288f72a7447c81a1c96ef3c7bb939af9f5964257e413a926
 SHA512:
-  metadata.gz: a0fd61160246d393d0059737a2557ca595d943a5db15a554d12a7c20664b42972c2f466325a1c027cbddb1d3dcc0e6cb69fe3499f506aa95f17a47d809e6a7c9
-  data.tar.gz: 7031ebb1c3fbfcd8bd23662c77ea34b44512297f780f7be632765a30753df7802e677a66bf6d468c6d3c33314938add162a26a16a484e587d1fae416e56ca8b1
+  metadata.gz: 1366c860429fe8da518a618f3651c839d5e151fbdf8025557433e323c84b718b7132fb27ad2c5c40d7d72980dcdb430a3398cb74aa1aafd99e4c4f701564c56d
+  data.tar.gz: caba8636b01815d08a00a81bf8159eaeafa0fce7824f2408d11f813f23504ab63b5ab1076c0a47ebc6f786f0228765a50f26b2076b5d1004710b1b448dace176

data/History.md

@@ -1,6 +1,10 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+### 4.0.1
+- Placeholders for template name and customize template (#708)
+- Add overwriting ilm policy config parameter (#707)
+- Fix a failing ILM config testcase (#706)
 
 ### 4.0.0
 - Restructuring ILM related features (#701)

data/README.md

@@ -91,6 +91,7 @@ Current maintainers: @cosmo0920
   + [enable_ilm](#enable_ilm)
   + [ilm_policy_id](#ilm_policy_id)
   + [ilm_policy](#ilm_policy)
+  + [ilm_policy_overwrite](#ilm_policy_overwrite)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
@@ -1162,6 +1163,14 @@ Default value is `{}`.
 
 **NOTE:** This parameter requests to install elasticsearch-xpack gem.
 
+## ilm_policy_overwrite
+
+Specify whether overwriting ilm policy or not.
+
+Default value is `false`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
 ## Configuration - Elasticsearch Input
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.0'
+  s.version       = '4.0.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
     s.metadata["changelog_uri"] = "https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md"
   end
 
-  s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
+  s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
 
   s.add_runtime_dependency 'fluentd', '>= 0.14.22'
   s.add_runtime_dependency 'excon', '>= 0'

data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb

@@ -1,10 +1,10 @@
 module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
   ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
 
-  def setup_ilm(enable_ilm, policy_id, ilm_policy = default_policy_payload)
+  def setup_ilm(enable_ilm, policy_id, ilm_policy = default_policy_payload, overwrite = false)
     return unless enable_ilm
 
-    create_ilm_policy(policy_id, ilm_policy)
+    create_ilm_policy(policy_id, ilm_policy, overwrite)
   end
 
   def verify_ilm_working
@@ -26,8 +26,8 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
     end
   end
 
-  def create_ilm_policy(policy_id, ilm_policy = default_policy_payload)
-    if !ilm_policy_exists?(policy_id)
+  def create_ilm_policy(policy_id, ilm_policy = default_policy_payload, overwrite = false)
+    if overwrite || !ilm_policy_exists?(policy_id)
       ilm_policy_put(policy_id, ilm_policy)
     end
   end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -127,7 +127,7 @@ module Fluent::ElasticsearchIndexTemplate
     template
   end
 
-  def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, host)
+  def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite, host)
      # ILM request to create alias.
     if rollover_index || enable_ilm
       if !client.indices.exists_alias(:name => deflector_alias_name)
@@ -137,8 +137,7 @@ module Fluent::ElasticsearchIndexTemplate
           index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
         end
         indexcreation(index_name_temp, host)
-        body = {}
-        body = rollover_alias_payload(deflector_alias_name) if enable_ilm
+        body = rollover_alias_payload(deflector_alias_name)
         client.indices.put_alias(:index => index_name_temp, :name => deflector_alias_name,
                                  :body => body)
         log.info("The alias '#{deflector_alias_name}' is created for the index '#{index_name_temp}'")
@@ -146,7 +145,7 @@ module Fluent::ElasticsearchIndexTemplate
           if ilm_policy.empty?
             setup_ilm(enable_ilm, ilm_policy_id)
           else
-            setup_ilm(enable_ilm, ilm_policy_id, ilm_policy)
+            setup_ilm(enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite)
           end
         end
       else

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -52,6 +52,7 @@ module Fluent::Plugin
     RequestInfo = Struct.new(:host, :index, :ilm_index)
 
     attr_reader :alias_indexes
+    attr_reader :template_names
 
     helpers :event_emitter, :compat_parameters, :record_accessor
 
@@ -155,6 +156,7 @@ EOC
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
     config_param :ilm_policy, :hash, :default => {}
+    config_param :ilm_policy_overwrite, :bool, :default => false
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -212,17 +214,21 @@ EOC
       end
 
       @alias_indexes = []
+      @template_names = []
       if !dry_run?
         if @template_name && @template_file
           if @enable_ilm
             raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
           end
+          if @ilm_policy.empty? && @ilm_policy_overwrite
+            raise Fluent::ConfigError, "ilm_policy_overwrite can work with non empty ilm_policy. Specify non-empty ilm policy into ilm_policy. "
+          end
           if @logstash_format || placeholder_substitution_needed_for_template?
             class << self
               alias_method :template_installation, :template_installation_actual
             end
           else
-            template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @application_name, @index_name)
+            template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
           end
           verify_ilm_working if @enable_ilm
         elsif @templates
@@ -358,13 +364,17 @@ EOC
     end
 
     def placeholder?(name, param)
-      begin
-        placeholder_validate!(name, param)
-        true
-      rescue Fluent::ConfigError => e
-        log.debug("'#{name} #{param}' is tested built-in placeholder(s) but there is no valid placeholder(s). error: #{e}")
-        false
+      placeholder_validities = []
+      placeholder_validators(name, param).each do |v|
+        begin
+          v.validate!
+          placeholder_validities << true
+        rescue Fluent::ConfigError => e
+          log.debug("'#{name} #{param}' is tested built-in placeholder(s) but there is no valid placeholder(s). error: #{e}")
+          placeholder_validities << false
+        end
       end
+      placeholder_validities.include?(true)
     end
 
     def compression
@@ -647,6 +657,16 @@ EOC
       else
         type_name = nil
       end
+      if @template_name
+        template_name = extract_placeholders(@template_name, chunk)
+      else
+        template_name = nil
+      end
+      if @customize_template
+        customize_template = @customize_template.each_with_object({}) { |(key, value), hash| hash[key] = extract_placeholders(value, chunk) }
+      else
+        customize_template = nil
+      end
       if @deflector_alias
         deflector_alias = extract_placeholders(@deflector_alias, chunk)
       else
@@ -662,7 +682,7 @@ EOC
       else
         pipeline = nil
       end
-      return logstash_prefix, index_name, type_name, deflector_alias, application_name, pipeline
+      return logstash_prefix, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
     end
 
     def multi_workers_ready?
@@ -736,7 +756,7 @@ EOC
     end
 
     def process_message(tag, meta, header, time, record, extracted_values)
-      logstash_prefix, index_name, type_name, _deflector_alias, _application_name, pipeline = extracted_values
+      logstash_prefix, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
 
       if @flatten_hashes
         record = flatten_record(record)
@@ -837,31 +857,38 @@ EOC
     end
 
     def placeholder_substitution_needed_for_template?
-      placeholder?(:host, @host.to_s) ||
+      need_substitution = placeholder?(:host, @host.to_s) ||
         placeholder?(:index_name, @index_name.to_s) ||
+        placeholder?(:template_name, @template_name.to_s) ||
+        @customize_template&.values&.any? { |value| placeholder?(:customize_template, value.to_s) } ||
         placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
         placeholder?(:deflector_alias, @deflector_alias.to_s) ||
         placeholder?(:application_name, @application_name.to_s)
+      log.debug("Need substitution: #{need_substitution}")
+      need_substitution
     end
 
-    def template_installation(deflector_alias, application_name, target_index, host)
+    def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
       # for safety.
     end
 
-    def template_installation_actual(deflector_alias, application_name, target_index, host=nil)
-      if @template_name && @template_file
+    def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
+      if template_name && @template_file
         if @alias_indexes.include? deflector_alias
           log.debug("Index alias #{deflector_alias} already exists (cached)")
+        elsif @template_names.include? template_name
+          log.debug("Template name #{template_name} already exists (cached)")
         else
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
-            if @customize_template
-              template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
+            if customize_template
+              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
             else
-              template_install(@template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
+              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
             end
-            create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, host)
+            create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, @ilm_policy_overwrite, host)
           end
           @alias_indexes << deflector_alias unless deflector_alias.nil?
+          @template_names << template_name unless template_name.nil?
         end
       end
     end
@@ -869,11 +896,11 @@ EOC
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
-      logstash_prefix, index_name, _type_name, deflector_alias, application_name, _pipeline = extracted_values
+      logstash_prefix, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
       if deflector_alias
-        template_installation(deflector_alias, application_name, index_name, info.host)
+        template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
       else
-        template_installation(info.ilm_index, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
+        template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
       end
 
       begin

data/test/plugin/test_out_elasticsearch.rb

@@ -307,36 +307,96 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
-  test 'valid configuration of index lifecycle management' do
-    cwd = File.dirname(__FILE__)
-    template_file = File.join(cwd, 'test_template.json')
+  sub_test_case 'ILM default config' do
+    setup do
+      begin
+        require "elasticsearch/xpack"
+      rescue LoadError
+        omit "ILM testcase needs elasticsearch-xpack gem."
+      end
+    end
 
-    config = %{
-      enable_ilm    true
-      template_name logstash
-      template_file #{template_file}
-    }
-    stub_request(:get, "http://localhost:9200/_template/fluentd").
-      to_return(status: 200, body: "", headers: {})
-    stub_request(:head, "http://localhost:9200/_alias/fluentd").
-      to_return(status: 404, body: "", headers: {})
-    stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/_alias/fluentd").
-      with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
-      to_return(status: 200, body: "", headers: {})
-    stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
-      to_return(status: 200, body: "", headers: {})
-    stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}',
-                :headers => {"Content-Type"=> "application/json"})
-    stub_request(:get, "http://localhost:9200/_ilm/policy/logstash-policy").
-      to_return(status: 404, body: "", headers: {})
-    stub_request(:put, "http://localhost:9200/_ilm/policy/logstash-policy").
-      with(body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
-      to_return(status: 200, body: "", headers: {})
+    test 'valid configuration of index lifecycle management' do
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
 
-    assert_nothing_raised {
-      driver(config)
-    }
+      config = %{
+        enable_ilm    true
+        template_name logstash
+        template_file #{template_file}
+      }
+      stub_request(:get, "http://localhost:9200/_template/fluentd").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:head, "http://localhost:9200/_alias/fluentd").
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/_alias/fluentd").
+        with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:get, "http://localhost:9200/_xpack").
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}',
+                  :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "http://localhost:9200/_ilm/policy/logstash-policy").
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "http://localhost:9200/_ilm/policy/logstash-policy").
+        with(body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(status: 200, body: "", headers: {})
+
+      assert_nothing_raised {
+        driver(config)
+      }
+    end
+
+    test 'valid configuration of overwriting ilm_policy' do
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = %{
+        enable_ilm    true
+        template_name logstash
+        template_file #{template_file}
+        ilm_policy_overwrite true
+        ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"75gb","max_age": "50d"}}}}}}
+      }
+      stub_request(:get, "http://localhost:9200/_template/fluentd").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:head, "http://localhost:9200/_alias/fluentd").
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/_alias/fluentd").
+        with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:get, "http://localhost:9200/_xpack").
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}',
+                  :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "http://localhost:9200/_ilm/policy/logstash-policy").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:put, "http://localhost:9200/_ilm/policy/logstash-policy").
+        with(body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"75gb\",\"max_age\":\"50d\"}}}}}}").
+        to_return(status: 200, body: "", headers: {})
+
+      assert_nothing_raised {
+        driver(config)
+      }
+    end
+
+    test 'invalid configuration of overwriting ilm_policy' do
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = %{
+        enable_ilm    true
+        template_name logstash
+        template_file #{template_file}
+        ilm_policy_overwrite true
+      }
+
+      assert_raise(Fluent::ConfigError) {
+        driver(config)
+      }
+    end
   end
 
   test 'Detected Elasticsearch 7' do
@@ -399,6 +459,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   sub_test_case "placeholder substitution needed?" do
     data("host placeholder" => ["host", "host-${tag}.google.com"],
+         "index_name_placeholder" => ["index_name", "logstash-${tag}"],
+         "template_name_placeholder" => ["template_name", "logstash-${tag}"],
+         "customize_template" => ["customize_template", '{"<<TAG>>":"${tag}"}'],
          "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${tag}"],
          "deflector_alias_placeholder" => ["deflector_alias", "fluentd-${tag}"],
          "application_name_placeholder" => ["application_name", "fluentd-${tag}"],
@@ -419,6 +482,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
 
     data("host placeholder" => ["host", "host-%Y%m%d.google.com"],
+         "index_name_placeholder" => ["index_name", "logstash-%Y%m%d"],
+         "template_name_placeholder" => ["template_name", "logstash-%Y%m%d"],
+         "customize_template" => ["customize_template", '{"<<TAG>>":"fluentd-%Y%m%d"}'],
          "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-%Y%m%d"],
          "deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
          "application_name_placeholder" => ["application_name", "fluentd-%Y%m%d"],
@@ -440,6 +506,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
 
     data("host placeholder" => ["host", "host-${mykey}.google.com"],
+         "index_name_placeholder" => ["index_name", "logstash-${mykey}"],
+         "template_name_placeholder" => ["template_name", "logstash-${mykey}"],
+         "customize_template" => ["customize_template", '{"<<TAG>>":"${mykey}"}'],
          "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${mykey}"],
          "deflector_alias_placeholder" => ["deflector_alias", "fluentd-${mykey}"],
          "application_name_placeholder" => ["application_name", "fluentd-${mykey}"],
@@ -460,6 +529,31 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
       assert_true driver.instance.placeholder_substitution_needed_for_template?
     end
+
+    data("host placeholder" => ["host", "host-${tag}.google.com"],
+         "index_name_placeholder" => ["index_name", "logstash-${es_index}-%Y%m%d"],
+         "template_name_placeholder" => ["template_name", "logstash-${tag}-%Y%m%d"],
+         "customize_template" => ["customize_template", '{"<<TAG>>":"${es_index}"}'],
+         "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${es_index}-%Y%m%d"],
+         "deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
+         "application_name_placeholder" => ["application_name", "fluentd-${tag}-${es_index}-%Y%m%d"],
+        )
+    test 'mixed placeholder' do |data|
+      param, value = data
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'elasticsearch',
+          param => value
+        }, [
+          Fluent::Config::Element.new('buffer', 'tag,time,es_index', {
+                                        'chunk_keys' => 'es_index',
+                                        'timekey' => '1d',
+                                      }, [])
+        ])
+      driver(config)
+
+      assert_true driver.instance.placeholder_substitution_needed_for_template?
+    end
   end
 
   sub_test_case 'chunk_keys requirement' do
@@ -650,6 +744,63 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
   end
 
+  def test_template_create_with_rollover_index_and_template_related_placeholders
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+   config = %{
+      host               logs.google.com
+      port               777
+      scheme             https
+      path               /es/
+      user               john
+      password           doe
+      template_name      logstash-${tag}
+      template_file      #{template_file}
+      rollover_index     true
+      index_date_pattern ""
+      index_name         fluentd-${tag}
+      deflector_alias    myapp_deflector-${tag}
+    }
+
+    # connection start
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
+    to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # create template
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if alias exists
+    stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # put the alias for the index
+    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E").
+      with(basic_auth: ['john', 'doe']).
+      to_return(status: 200, body: "", headers: {})
+    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/_alias/myapp_deflector-test.template").
+      with(basic_auth: ['john', 'doe'],
+           body: "{\"aliases\":{\"myapp_deflector-test.template\":{\"is_write_index\":true}}}").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver(config)
+
+    elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+    driver.run(default_tag: 'test.template') do
+      driver.feed(sample_record)
+    end
+    assert_equal('fluentd-test.template', index_cmds.first['index']['_index'])
+
+    assert_equal ["myapp_deflector-test.template"], driver.instance.alias_indexes
+    assert_equal ["logstash-test.template"], driver.instance.template_names
+
+    assert_requested(elastic_request)
+  end
+
   class TemplateIndexLifecycleManagementTest < self
     def setup
       begin
@@ -724,6 +875,73 @@ class ElasticsearchOutput < Test::Unit::TestCase
       assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
     end
 
+    def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   logstash
+        template_file   #{template_file}
+        index_date_pattern now/w{xxxx.ww}
+        index_name      logstash
+        enable_ilm      true
+        ilm_policy_overwrite true
+        ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
+        to_return(status: 200, body: "", headers: {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"60gb\",\"max_age\":\"45d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
+    end
+
     def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
@@ -1029,6 +1247,48 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
+  def test_custom_template_create_with_customize_template_related_placeholders
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_alias_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   myapp_alias_template-${tag}
+      template_file   #{template_file}
+      customize_template {"--appid--": "${tag}-logs","--index_prefix--":"${tag}"}
+    }
+
+    # connection start
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template-test.template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+
+    stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test-default-000001%3E").
+      to_return(status: 200, body: "", headers: {})
+
+    driver(config)
+
+    stub_elastic("https://logs.google.com:777/es//_bulk")
+    driver.run(default_tag: 'test.template') do
+      driver.feed(sample_record)
+    end
+
+    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template-test.template", times: 1)
+  end
+
   def test_custom_template_installation_for_host_placeholder
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')
@@ -1302,6 +1562,77 @@ class ElasticsearchOutput < Test::Unit::TestCase
       assert_requested(:put, "https://logs.google.com:777/es//_template/mylogs", times: 1)
     end
 
+    def test_custom_template_with_rollover_index_create_and_default_ilm_and_ilm_policy_overwrite
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_alias_template.json')
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   myapp_alias_template
+        template_file   #{template_file}
+        customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+        index_date_pattern now/w{xxxx.ww}
+        index_name    mylogs
+        application_name myapp
+        ilm_policy_id   fluentd-policy
+        enable_ilm      true
+        ilm_policy_overwrite true
+        ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # creation of index which can rollover
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_template/mylogs").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "https://logs.google.com:777/es//_template/mylogs").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
+        to_return(status: 200, body: "", headers: {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"60gb\",\"max_age\":\"45d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      assert_requested(:put, "https://logs.google.com:777/es//_template/mylogs", times: 1)
+    end
+
     def test_custom_template_with_rollover_index_create_and_default_ilm_with_deflector_alias
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_alias_template.json')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-27 00:00:00.000000000 Z
+date: 2020-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -185,7 +185,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="