fluent-plugin-elasticsearch 1.12.0 → 1.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd8184a5b89190d9004101efa043bb86fa28b917
-  data.tar.gz: 357d028cdc8fadaecdbb70a766f0925258c3045c
+  metadata.gz: d0d19458ba42dc0503bcbc2d1d5d83f58f125ba5
+  data.tar.gz: c3ca62e3b1801ed4b507ce61efca73a662836445
 SHA512:
-  metadata.gz: 16697587184032cb71ba05add339aceff404c53ba66e9403fdd9d533d8148f38fbccc5304a97ce71ec98ebff41fd989b0f02fec618eaf273a36fc34ddd509229
-  data.tar.gz: f09260ba5ae763a0a18a53c8e6191f477e8dea3bf42cdafa489e1db5275c573d4e5539807c5e14bd66a02baa4b63455a12487aefcf10d642e65e5b800b72d6b4
+  metadata.gz: e919bbe4e67ed88809331b3b1e2c5db6c4f57e03b48bfc0df97edc2980bbf6f0a076e61749acc8f19e9f7ba415836d4d506789a41dc475d5b91d85ee6878b68f
+  data.tar.gz: f4ee41cb99098a2de2a3a98a90b1ebd3977d28dfa9b7c7c52127e1446c73c6dc7b2839489d972d067bfdefde9f1658b11abead46fcaab4efa16c48f10ff6e81b

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 1.13.0
+- Backport allowing to overwrite existing index template (#336)
+
 ### 1.12.0
 - GA release 1.12.0.
 

data/README.md

@@ -32,6 +32,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [target_type_key](#target_type_key)
   + [template_name](#template_name)
   + [template_file](#template_file)
+  + [template_overwrite](#template_overwrite)
   + [templates](#templates)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
@@ -279,7 +280,7 @@ Similar to `target_index_key` config, find the type name to write to in the reco
 
 ### template_name
 
-The name of the template to define. If a template by the name given is already present, it will be left unchanged.
+The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
 
 This parameter along with template_file allow the plugin to behave similarly to Logstash (it installs a template at creation time) so that raw records are available. See [https://github.com/uken/fluent-plugin-elasticsearch/issues/33](https://github.com/uken/fluent-plugin-elasticsearch/issues/33).
 
@@ -301,6 +302,16 @@ templates { "templane_name_1": "path_to_template_1_file", "templane_name_2": "pa
 
 If `template_file` and `template_name` are set, then this parameter will be ignored.
 
+### template_overwrite
+
+Always update the template, even if it already exists.
+
+```
+template_overwrite true # defaults to false
+```
+
+One of [template_file](#template_file) or [templates](#templates) must also be specified if this is set.
+
 ### request_timeout
 
 You can specify HTTP request timeout.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.12.0'
+  s.version       = '1.13.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -19,7 +19,12 @@ module Fluent::ElasticsearchIndexTemplate
     client.indices.put_template(:name => name, :body => template)
   end
 
-  def template_install(name, template_file)
+  def template_install(name, template_file, overwrite)
+    if overwrite
+      template_put(name, get_template(template_file))
+      log.info("Template '#{name}' overwritten with #{template_file}.")
+      return
+    end
     if !template_exists?(name)
       template_put(name, get_template(template_file))
       log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
@@ -28,9 +33,9 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def templates_hash_install (templates)
+  def templates_hash_install(templates, overwrite)
     templates.each do |key, value|
-      template_install(key, value)
+      template_install(key, value, overwrite)
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -62,6 +62,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :flatten_hashes_separator, :string, :default => "_"
   config_param :template_name, :string, :default => nil
   config_param :template_file, :string, :default => nil
+  config_param :template_overwrite, :bool, :default => false
   config_param :templates, :hash, :default => nil
   config_param :include_tag_key, :bool, :default => false
   config_param :tag_key, :string, :default => 'tag'
@@ -98,9 +99,9 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     end
 
     if @template_name && @template_file
-      template_install(@template_name, @template_file)
+      template_install(@template_name, @template_file, @template_overwrite)
     elsif @templates
-      templates_hash_install (@templates)
+      templates_hash_install(@templates, @template_overwrite)
     end
 
     @meta_config_map = create_meta_config_map

data/test/plugin/test_out_elasticsearch.rb

@@ -239,6 +239,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     driver('test', config)
+
+    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
   end
 
   def test_template_create
@@ -267,6 +269,39 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     driver('test', config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
+  end
+
+  def test_template_overwrite
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+      template_overwrite true
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver('test', config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
   end
 
 
@@ -332,6 +367,44 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
   end
 
+  def test_templates_overwrite
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
+      template_overwrite true
+    }
+
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+     # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+      to_return(:status => 200, :body => "", :headers => {}) #exists
+
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver('test', config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
+  end
+
   def test_templates_not_used
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.12.0
+  version: 1.13.0
 platform: ruby
 authors:
 - diogo