fluent-plugin-elasticsearch 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df3fc320b12a8944666a94ef4191c2c4f54596fe
-  data.tar.gz: 8e9542c23f48771a1b8c4f9b58d48c3a88368808
+  metadata.gz: af3f58b827423bdfc292675d045d0e88f3bcc051
+  data.tar.gz: 63d6cdcdf4f23cac1c7e932e633df607cc3ac7fd
 SHA512:
-  metadata.gz: 442135842951fd0b3a747a979f52e3093cdc3f3d5eaaceb168b4a422d0dd616998e974b197d69750f9527a5a1372ef2416d5145ba709489dd301bd63af85de54
-  data.tar.gz: 49c2a393a950e02c53f8734ded656deb683c955f70018e565b6fffd6fb4facd107e2d56d1f7231e3120140d8701e76622c8041cb0664abcb3137e19c53af6c47
+  metadata.gz: 38fdabbdd290f4d03f5cc26eae8643c8a7e6a922ebf6d294fc46b3b4a085b77155776c6c48c44d9fa4ca03ffbad4e91c0f3587d34999610fa0214d5635d61c1b
+  data.tar.gz: de3e9167b99936945d290da18400074b9def208d59b6ae01b2a506e644d972d4e5551717f32a77fb969074c6a09c153490ceb940ae88fece749266192d08a45c

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.3.0
+- Allow overwriting existing index template (#239)
+
 ### 2.2.0
 - GA release 2.2.0.
 

data/README.md

@@ -34,6 +34,7 @@ Current maintainers: @cosmo0920
   + [target_type_key](#target_type_key)
   + [template_name](#template_name)
   + [template_file](#template_file)
+  + [template_overwrite](#template_overwrite)
   + [templates](#templates)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
@@ -294,7 +295,7 @@ Similar to `target_index_key` config, find the type name to write to in the reco
 
 ### template_name
 
-The name of the template to define. If a template by the name given is already present, it will be left unchanged.
+The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
 
 This parameter along with template_file allow the plugin to behave similarly to Logstash (it installs a template at creation time) so that raw records are available. See [https://github.com/uken/fluent-plugin-elasticsearch/issues/33](https://github.com/uken/fluent-plugin-elasticsearch/issues/33).
 
@@ -316,6 +317,16 @@ templates { "templane_name_1": "path_to_template_1_file", "templane_name_2": "pa
 
 If `template_file` and `template_name` are set, then this parameter will be ignored.
 
+### template_overwrite
+
+Always update the template, even if it already exists.
+
+```
+template_overwrite true # defaults to false
+```
+
+One of [template_file](#template_file) or [templates](#templates) must also be specified if this is set.
+
 ### request_timeout
 
 You can specify HTTP request timeout.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.2.0'
+  s.version       = '2.3.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -19,7 +19,12 @@ module Fluent::ElasticsearchIndexTemplate
     client.indices.put_template(:name => name, :body => template)
   end
 
-  def template_install(name, template_file)
+  def template_install(name, template_file, overwrite)
+    if overwrite
+      template_put(name, get_template(template_file))
+      log.info("Template '#{name}' overwritten with #{template_file}.")
+      return
+    end
     if !template_exists?(name)
       template_put(name, get_template(template_file))
       log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
@@ -28,9 +33,9 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def templates_hash_install (templates)
+  def templates_hash_install(templates, overwrite)
     templates.each do |key, value|
-      template_install(key, value)
+      template_install(key, value, overwrite)
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -67,6 +67,7 @@ module Fluent::Plugin
     config_param :flatten_hashes_separator, :string, :default => "_"
     config_param :template_name, :string, :default => nil
     config_param :template_file, :string, :default => nil
+    config_param :template_overwrite, :bool, :default => false
     config_param :templates, :hash, :default => nil
     config_param :include_tag_key, :bool, :default => false
     config_param :tag_key, :string, :default => 'tag'
@@ -114,9 +115,9 @@ module Fluent::Plugin
       end
 
       if @template_name && @template_file
-        template_install(@template_name, @template_file)
+        template_install(@template_name, @template_file, @template_overwrite)
       elsif @templates
-        templates_hash_install (@templates)
+        templates_hash_install(@templates, @template_overwrite)
       end
 
       @meta_config_map = create_meta_config_map

data/test/plugin/test_out_elasticsearch.rb

@@ -262,6 +262,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
+
+    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
   end
 
   def test_template_create
@@ -290,6 +292,39 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
+  end
+
+  def test_template_overwrite
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+      template_overwrite true
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver(config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
   end
 
 
@@ -355,6 +390,44 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
   end
 
+  def test_templates_overwrite
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
+      template_overwrite true
+    }
+
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+     # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+      to_return(:status => 200, :body => "", :headers => {}) #exists
+
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver(config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
+  end
+
   def test_templates_not_used
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - diogo