fluent-plugin-elasticsearch 2.11.11 → 2.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac2845d66bafd01ff3ff4bcc297e962d005dabc4878b1d24fd85a9cc135c7a98
-  data.tar.gz: b6649d2372541c3f6049baf5be3853a3e2a283422e268f03e9321ce50c2138a2
+  metadata.gz: daf714ffb7c731c2d2792efd8930165c2ca8baa3c387eae815f41d2ccfd89ead
+  data.tar.gz: df13ef1152b10bb2d63e7a870ab9afcbed6f040e6aee2a671f376690d0e616bb
 SHA512:
-  metadata.gz: ab31cd71bc444c47246ea32878f5b65796175feeaa69dc301a87afd2ffe4a51dda014fbeb32e630bbc45763fdf5ff69b38974b12bdfec7afd36ab890942505a2
-  data.tar.gz: 5bcec142ca65c7546139eae23bafddefb027d631c67b1e0062ff36c2f11fca746d5aa11310cb3646c6285017fd8bc26509e59fb1edd3fecfa95aa6bf3bff18fa
+  metadata.gz: b33818935a059b0e150bb5dcc0cbc1179a3b7179d4fd0b29c964d8238efcbaa68cc59b3232f3f326ba855914b2d54d1fd9bdf0eb107a75108a1c38dae8c305e0
+  data.tar.gz: 4d8d1cd2a3700e657c763caf5544344e6bfad1c185879539f48b52a65b84e5ba4dbc8bd524a1934e7d29348a20d7763a4c156519f71bb49c296b3192b2185069

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 2.12.0
+- Decoupling the custom template and rollover index creation #485 (#486)
+
 ### 2.11.11
 - Handle error not to acquire version information (#479)
 

data/README.md

@@ -37,6 +37,9 @@ Current maintainers: @cosmo0920
   + [template_file](#template_file)
   + [template_overwrite](#template_overwrite)
   + [customize_template](#customize_template)
+  + [rollover_index](#rollover_index)
+  + [deflector_alias](#deflector_alias)
+  + [application_name](#application_name)
   + [index_prefix](#index_prefix)
   + [templates](#templates)
   + [max_retry_putting_template](#max_retry_putting_template)
@@ -338,22 +341,52 @@ If `template_file` and `template_name` are set, then this parameter will be igno
 
 ### customize_template
 
-Specify the string and its value to be replaced in form of hash. Can contain multiple templates.
+Specify the string and its value to be replaced in form of hash. Can contain multiple key value pair that would be replaced in the specified template_file.
+This setting only creates template and to add rollover index please check the [rollover_index](#rollover_index) configuration.
 
 ```
 customize_template {"string_1": "subs_value_1", "string_2": "subs_value_2"}
 ```
 
-If `template_file` and `template_name` are set, then this parameter will be in effect otherwise ignored.
+If [template_file](#template_file) and [template_name](#template_name) are set, then this parameter will be in effect otherwise ignored.
+
+### rollover_index
+
+Specify this as true when an index with rollover capability needs to be created. It creates an index with the format <logstash-default-{now/d}-000001> where logstash denotes the index_prefix and default denotes the application_name which can be set.
+'deflector_alias' is a required field for rollover_index set to true.
+'index_prefix' and 'application_name' are optional and defaults to logstash and default respectively.
+```
+rollover_index true # defaults to false
+```
+
+If [customize_template](#customize_template) is set, then this parameter will be in effect otherwise ignored.
+
+### deflector_alias
+
+Specify the deflector alias which would be assigned to the rollover index created. This is useful in case of using the Elasticsearch rollover API 
+```
+deflector_alias test-current
+```
+
+If [rollover_index](#rollover_index) is set, then this parameter will be in effect otherwise ignored.
 
 ### index_prefix
 
+Specify the index prefix for the rollover index to be created.
 ```
 index_prefix mylogs # defaults to "logstash"
 ```
 
-If `customize_template` is set, then this parameter will be in effect otherwise ignored.
+If [rollover_index](#rollover_index) is set, then this parameter will be in effect otherwise ignored.
+
+### application_name
+
+Specify the application name for the rollover index to be created.
+```
+application_name default # defaults to "default"
+```
 
+If [rollover_index](#rollover_index) is set, then this parameter will be in effect otherwise ignored.
 
 ### template_overwrite
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.11.11'
+  s.version       = '2.12.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -68,9 +68,8 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(name, template_file, overwrite, customize_template, index_prefix)
-    template_custom_name=name.downcase+'_alias_template'
-    alias_name=name.downcase+'-current'
+  def template_custom_install(template_name, template_file, overwrite, customize_template, index_prefix, rollover_index, deflector_alias_name, app_name)
+    template_custom_name=template_name.downcase
     if overwrite
       template_put(template_custom_name, get_custom_template(template_file, customize_template))
       log.info("Template '#{template_custom_name}' overwritten with #{template_file}.")
@@ -82,14 +81,17 @@ module Fluent::ElasticsearchIndexTemplate
     else
       log.info("Template configured and already installed.")
     end
-    
-    if !client.indices.exists_alias(:name => alias_name)
-      index_name='<'+index_prefix.downcase+'-'+name.downcase+'-{now/d}-000001>'
-      indexcreation(index_name)
-      client.indices.put_alias(:index => index_name, :name => alias_name)
-      log.info("The alias '#{alias_name}' is created for the index '#{index_name}'")
+    if rollover_index
+      if !client.indices.exists_alias(:name => deflector_alias_name)
+        index_name_temp='<'+index_prefix.downcase+'-'+app_name.downcase+'-{now/d}-000001>'
+        indexcreation(index_name_temp)
+        client.indices.put_alias(:index => index_name_temp, :name => deflector_alias_name)
+        log.info("The alias '#{deflector_alias_name}' is created for the index '#{index_name_temp}'")
+      else
+        log.info("The alias '#{deflector_alias_name}' is already present")
+      end
     else
-      log.info("The alias '#{alias_name}' is already present")
+      log.info("No index and alias creation action performed because rollover_index is set to '#{rollover_index}'")
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -99,7 +99,10 @@ EOC
     config_param :template_file, :string, :default => nil
     config_param :template_overwrite, :bool, :default => false
     config_param :customize_template, :hash, :default => nil
+    config_param :rollover_index, :string, :default => false
+    config_param :deflector_alias, :string, :default => nil
     config_param :index_prefix, :string, :default => "logstash"
+    config_param :application_name, :string, :default => "default"
     config_param :templates, :hash, :default => nil
     config_param :max_retry_putting_template, :integer, :default => 10
     config_param :include_tag_key, :bool, :default => false
@@ -160,10 +163,14 @@ EOC
       end
 
       raise Fluent::ConfigError, "'max_retry_putting_template' must be positive number." if @max_retry_putting_template < 0
+      
       if @template_name && @template_file
         retry_install(@max_retry_putting_template) do
           if @customize_template
-            template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @index_prefix)
+            if @rollover_index
+              raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
+            end
+            template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @index_prefix, @rollover_index, @deflector_alias, @application_name)
           else
             template_install(@template_name, @template_file, @template_overwrite)
           end

data/test/plugin/test_out_elasticsearch.rb

@@ -380,10 +380,44 @@ class ElasticsearchOutput < Test::Unit::TestCase
       path            /es/
       user            john
       password        doe
-      template_name   myapp
+      template_name   myapp_alias_template
       template_file   #{template_file}
       customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 404, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 200, :body => "", :headers => {})
+    
+    driver(config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+  end
+
+  def test_custom_template_with_rollover_index_create
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_alias_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   myapp_alias_template
+      template_file   #{template_file}
+      customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+      rollover_index  true
+      deflector_alias myapp_deflector
       index_prefix    mylogs
+      application_name myapp
     }
 
     # connection start
@@ -399,10 +433,10 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
       to_return(:status => 200, :body => "", :headers => {})
     # check if alias exists
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp-current").
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp_deflector").
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp-current").
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp_deflector").
       to_return(:status => 200, :body => "", :headers => {})
     
     driver(config)
@@ -410,6 +444,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
+
   def test_template_overwrite
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')
@@ -452,11 +487,45 @@ class ElasticsearchOutput < Test::Unit::TestCase
       path            /es/
       user            john
       password        doe
-      template_name   myapp
+      template_name   myapp_alias_template
+      template_file   #{template_file}
+      template_overwrite true
+      customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 200, :body => "", :headers => {})
+    
+    driver(config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+  end
+
+  def test_custom_template_with_rollover_index_overwrite
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   myapp_alias_template
       template_file   #{template_file}
       template_overwrite true
       customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+      deflector_alias myapp_deflector
       index_prefix    mylogs
+      application_name myapp
     }
 
     # connection start
@@ -472,10 +541,10 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
       to_return(:status => 200, :body => "", :headers => {})
     # check if alias exists
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp-current").
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp_deflector").
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp-current").
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp_deflector").
       to_return(:status => 200, :body => "", :headers => {})
   
     driver(config)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.11.11
+  version: 2.12.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-09 00:00:00.000000000 Z
+date: 2018-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd