fluent-plugin-elasticsearch 2.11.5 → 2.11.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36408493fd7023fcd54d9c3e1ad98f3fa897d7f43dcb0a2d8ebc2e789bbd9518
-  data.tar.gz: 1cf82341e33dc2b713a94181b9450bbdaaf82f02c025b69b7008eb4e45a7314f
+  metadata.gz: 1a97d5261421d51521687b644f6f6b02555ad0f1fd592b24a40f876a8702ba4e
+  data.tar.gz: b31b34473d410112dd2d1d6e05ac51d277953bb2d33e411eaa6621731cbc5052
 SHA512:
-  metadata.gz: 9cfe34fbe403a4ef5233a1e34a42e02e184228a22bf20a79698912176a1bfb4ab03f21fd2be0015a666ced2be46106218013e6946bd5b5351b30d08993c08aca
-  data.tar.gz: 9712390a2f6be181cf642583d01af895469183a5198513c40493fa61982c9e05198a670bbe74cb4e79e545f145c0330e45f463c1f928572b5421703d19344ff8
+  metadata.gz: d9708a487c86d1cdb367789fccf3ae0ff9a7900710e803e4ca2bcc22f5216334775ae806a86e9b08c09f473bda7c15fa3ba3a5df71ccd95a2de6f3a19fc6d3c5
+  data.tar.gz: 541256741abbb757a29ba855a19f1d880240f1b6a05d3583017b3f23b600b1bd987f8f8734f627e34868b4ba8d7083c0534d8c10878c5f0ea6216a47722416d1

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 2.11.6
+- 355 customize template (#431)
+
 ### 2.11.5
 - Uplift Merge pull request #459 from richm/v0.12-simple-sniffer (#461)
 

data/README.md

@@ -36,6 +36,8 @@ Current maintainers: @cosmo0920
   + [template_name](#template_name)
   + [template_file](#template_file)
   + [template_overwrite](#template_overwrite)
+  + [customize_template](#customize_template)
+  + [index_prefix](#index_prefix)
   + [templates](#templates)
   + [max_retry_putting_template](#max_retry_putting_template)
   + [request_timeout](#request_timeout)
@@ -127,12 +129,13 @@ hosts host1:port1,host2:port2,host3:port3
 You can specify multiple Elasticsearch hosts with separator ",".
 
 If you specify multiple hosts, this plugin will load balance updates to Elasticsearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
+**Note:** If you will use scheme https, do not include "https://" in your hosts ie. host "https://domain", this will cause ES cluster to be unreachable and you will recieve an error "Can not reach Elasticsearch cluster"
 
 **Note:** Up until v2.8.5, it was allowed to embed the username/password in the URL. However, this syntax is deprecated as of v2.8.6 because it was found to cause serious connection problems (See #394). Please migrate your settings to use the `user` and `password` field (described below) instead.
 
 ### user, password, path, scheme, ssl_verify
 
-If you specify this option, host and port options are ignored.
+If you specify this option, port options are ignored.
 
 ```
 user demo
@@ -331,6 +334,25 @@ templates { "template_name_1": "path_to_template_1_file", "template_name_2": "pa
 
 If `template_file` and `template_name` are set, then this parameter will be ignored.
 
+### customize_template
+
+Specify the string and its value to be replaced in form of hash. Can contain multiple templates.
+
+```
+customize_template {"string_1": "subs_value_1", "string_2": "subs_value_2"}
+```
+
+If `template_file` and `template_name` are set, then this parameter will be in effect otherwise ignored.
+
+### index_prefix
+
+```
+index_prefix mylogs # defaults to "logstash"
+```
+
+If `customize_template` is set, then this parameter will be in effect otherwise ignored.
+
+
 ### template_overwrite
 
 Always update the template, even if it already exists.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.11.5'
+  s.version       = '2.11.6'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -8,6 +8,17 @@ module Fluent::ElasticsearchIndexTemplate
     JSON.parse(file_contents)
   end
 
+  def get_custom_template(template_file, customize_template)
+    if !File.exists?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    customize_template.each do |key, value|
+      file_contents = file_contents.gsub(key,value.downcase)
+    end
+    JSON.parse(file_contents)
+  end
+
   def template_exists?(name)
     client.indices.get_template(:name => name)
     return true
@@ -37,6 +48,12 @@ module Fluent::ElasticsearchIndexTemplate
     client.indices.put_template(:name => name, :body => template)
   end
 
+  def indexcreation(index_name)
+    client.indices.create(:index => index_name)
+    rescue Elasticsearch::Transport::Transport::Error => e
+      log.error("Error while index creation - #{index_name}: #{e.inspect}")
+  end
+
   def template_install(name, template_file, overwrite)
     if overwrite
       template_put(name, get_template(template_file))
@@ -51,6 +68,31 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
+  def template_custom_install(name, template_file, overwrite, customize_template, index_prefix)
+    template_custom_name=name.downcase+'_alias_template'
+    alias_name=name.downcase+'-current'
+    if overwrite
+      template_put(template_custom_name, get_custom_template(template_file, customize_template))
+      log.info("Template '#{template_custom_name}' overwritten with #{template_file}.")
+      return
+    end
+    if !template_exists?(template_custom_name)
+      template_put(template_custom_name, get_custom_template(template_file, customize_template))
+      log.info("Template configured, but no template installed. Installed '#{template_custom_name}' from #{template_file}.")
+    else
+      log.info("Template configured and already installed.")
+    end
+    
+    if !client.indices.exists_alias(:name => alias_name)
+      index_name='<'+index_prefix.downcase+'-'+name.downcase+'-{now/d}-000001>'
+      indexcreation(index_name)
+      client.indices.put_alias(:index => index_name, :name => alias_name)
+      log.info("The alias '#{alias_name}' is created for the index '#{index_name}'")
+    else
+      log.info("The alias '#{alias_name}' is already present")
+    end
+  end
+
   def templates_hash_install(templates, overwrite)
     templates.each do |key, value|
       template_install(key, value, overwrite)

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -93,6 +93,8 @@ EOC
     config_param :template_name, :string, :default => nil
     config_param :template_file, :string, :default => nil
     config_param :template_overwrite, :bool, :default => false
+    config_param :customize_template, :hash, :default => nil
+    config_param :index_prefix, :string, :default => "logstash"
     config_param :templates, :hash, :default => nil
     config_param :max_retry_putting_template, :integer, :default => 10
     config_param :include_tag_key, :bool, :default => false
@@ -153,7 +155,11 @@ EOC
       raise Fluent::ConfigError, "'max_retry_putting_template' must be positive number." if @max_retry_putting_template < 0
       if @template_name && @template_file
         retry_install(@max_retry_putting_template) do
-          template_install(@template_name, @template_file, @template_overwrite)
+          if @customize_template
+            template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @index_prefix)
+          else
+            template_install(@template_name, @template_file, @template_overwrite)
+          end
         end
       elsif @templates
         retry_install(@max_retry_putting_template) do

data/test/plugin/test_alias_template.json

@@ -0,0 +1,9 @@
+{
+  "order": 5,
+  "template": "--index_prefix-----appid---*",
+  "settings": {},
+  "mappings": {},
+  "aliases": {
+    "--appid---alias": {}
+  }
+}

data/test/plugin/test_out_elasticsearch.rb

@@ -340,6 +340,47 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
   end
 
+  def test_custom_template_create
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_alias_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   myapp
+      template_file   #{template_file}
+      customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+      index_prefix    mylogs
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 404, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation of index which can rollover
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if alias exists
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp-current").
+      to_return(:status => 404, :body => "", :headers => {})
+    # put the alias for the index
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp-current").
+      to_return(:status => 200, :body => "", :headers => {})
+    
+    driver(config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+  end
+
   def test_template_overwrite
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')
@@ -371,6 +412,47 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
   end
 
+  def test_custom_template_overwrite
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   myapp
+      template_file   #{template_file}
+      template_overwrite true
+      customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+      index_prefix    mylogs
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation of index which can rollover
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if alias exists
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp-current").
+      to_return(:status => 404, :body => "", :headers => {})
+    # put the alias for the index
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp-current").
+      to_return(:status => 200, :body => "", :headers => {})
+  
+    driver(config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+  end
 
   def test_template_create_invalid_filename
     config = %{

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.11.5
+  version: 2.11.6
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-14 00:00:00.000000000 Z
+date: 2018-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -152,6 +152,7 @@ files:
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
+- test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
@@ -183,6 +184,7 @@ specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
+- test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb