fluent-plugin-elasticsearch 1.7.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1ce21e5b836181cdc5f32e97e4d3a91bfc31da7
-  data.tar.gz: 77c81ab19d4749da338f481404983f1bcf8b069e
+  metadata.gz: f279ccf2181eb8582234fb5b417439dfd8f22ac9
+  data.tar.gz: e3b8e02a2aa972ac0e07971369dd9381b58401e3
 SHA512:
-  metadata.gz: dd36dacfb1b9c7fe7c860ffe4b2ed0e7bbdd3c5803c2faef0879c7898d8d9f732860e4b54e63125b1dabf8bb7fdc66868babaec7c199b896fbb913b644ae778a
-  data.tar.gz: 90d3c5c2b9c4af19d4c5c19d48288c92aebcb4d80cc83343a2502377f85dfef0e9ca1bf39c3de5a6065456e0a2bdc985a4f312372af2e3ad40fcd05f8df46225
+  metadata.gz: a576bf1cd9ff05130d1f71cabfc6b264ffc53f86cb9780af42ee0aa4f647a24905e5d41c22dd446474896bf2ee17b7ad942011bc2f6dd10cf4231fde348e35aa
+  data.tar.gz: b938736ba3ed0fbf8846e864c5f7273c1b059a930a4b91165d2513e6038de6bdf48968570479047dce33a7ad3fa727a02f48215d3be724a6d60414c024010ae4

data/History.md

@@ -1,6 +1,9 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
-### Future
+### [Unreleased]
+- fix typo in defaults for ssl_verify on elasticsearch_dynamic (#202)
+- add support for `templates` (#196)
+- rename `send` method to `send_bulk` (#206)
 
 ### 1.7.0
 - add support for `template_name` and `template_file` (#194)

data/README.md

@@ -28,6 +28,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [target_type_key](#target_type_key)
   + [template_name](#template_name)
   + [template_file](#template_file)
+  + [templates](#templates)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
   + [reload_on_failure](#reload_on_failure)
@@ -240,6 +241,16 @@ The path to the file containing the template to install.
 
 [template_name](#template_name) must also be specified.
 
+### templates
+
+Specify index templates in form of hash. Can contain multiple templates.
+
+```
+templates { "templane_name_1": "path_to_template_1_file", "templane_name_2": "path_to_template_2_file"}
+```
+
+If `template_file` and `template_name` are set, then this parameter will be ignored.
+
 ### request_timeout
 
 You can specify HTTP request timeout.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.7.0'
+  s.version       = '1.8.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -28,4 +28,10 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
+  def templates_hash_install (templates)
+    templates.each do |key, value|
+      template_install(key, value)
+    end
+  end
+
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -54,6 +54,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :flatten_hashes_separator, :string, :default => "_"
   config_param :template_name, :string, :default => nil
   config_param :template_file, :string, :default => nil
+  config_param :templates, :hash, :default => nil
 
   include Fluent::SetTagKeyMixin
   include Fluent::ElasticsearchIndexTemplate
@@ -86,7 +87,10 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
 
     if @template_name && @template_file
       template_install(@template_name, @template_file)
+    elsif @templates
+      templates_hash_install (@templates)
     end
+
   end
 
   def start
@@ -315,7 +319,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       append_record_to_messages(@write_operation, meta, record, bulk_message)
     end
 
-    send(bulk_message) unless bulk_message.empty?
+    send_bulk(bulk_message) unless bulk_message.empty?
     bulk_message.clear
   end
 
@@ -328,7 +332,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     [parent_object, path[-1]]
   end
 
-  def send(data)
+  def send_bulk(data)
     retries = 0
     begin
       client.bulk body: data

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -15,7 +15,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
   config_param :reload_connections, :string, :default => "true"
   config_param :reload_on_failure, :string, :default => "false"
   config_param :resurrect_after, :string, :default => "60"
-  config_param :ssl_verify, :string, :dfeault => "true"
+  config_param :ssl_verify, :string, :default => "true"
 
   def configure(conf)
     super
@@ -178,12 +178,12 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
     end
 
     bulk_message.each do | hKey, array |
-      send(array, hKey) unless array.empty?
+      send_bulk(array, hKey) unless array.empty?
       array.clear
     end
   end
 
-  def send(data, host)
+  def send_bulk(data, host)
     retries = 0
     begin
       client(host).bulk body: data

data/test/plugin/test_out_elasticsearch.rb

@@ -113,7 +113,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     driver('test', config)
-  end
+  end 
+  
 
   def test_template_create_invalid_filename
     config = %{
@@ -139,6 +140,118 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
+  def test_templates_create
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
+    }
+
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+     # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 404, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 404, :body => "", :headers => {})
+
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+      to_return(:status => 200, :body => "", :headers => {}) #exists
+
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+      to_return(:status => 200, :body => "", :headers => {})
+    
+    driver('test', config)
+    
+    assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
+    assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
+    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
+  end
+  
+  def test_templates_not_used
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+      templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
+    }    
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 404, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 404, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 404, :body => "", :headers => {})
+    #creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver('test', config)
+
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
+
+    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1")
+    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
+  end
+
+  def test_templates_can_be_partially_created_if_error_occurs
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      templates       {"logstash1":"#{template_file}", "logstash2":"/abc" }
+    }
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+     # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 404, :body => "", :headers => {})
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 404, :body => "", :headers => {})
+
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+      to_return(:status => 200, :body => "", :headers => {})
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    assert_raise(RuntimeError) {
+      driver('test', config)
+    }
+    
+    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
+    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
+  end
+
   def test_legacy_hosts_list
     config = %{
       hosts    host1:50,host2:100,host3

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -63,6 +63,26 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal 'doe', instance.password
   end
 
+  def test_defaults
+    config = %{
+      host     logs.google.com
+      scheme   https
+      path     /es/
+      user     john
+      password doe
+    }
+    instance = driver('test', config).instance
+
+    assert_equal "9200", instance.port
+    assert_equal "false", instance.logstash_format
+    assert_equal "true", instance.utc_index
+    assert_equal false, instance.time_key_exclude_timestamp
+    assert_equal "true", instance.reload_connections
+    assert_equal "false", instance.reload_on_failure
+    assert_equal "60", instance.resurrect_after
+    assert_equal "true", instance.ssl_verify
+  end
+
   def test_legacy_hosts_list
     config = %{
       hosts    host1:50,host2:100,host3

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.8.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-12 00:00:00.000000000 Z
+date: 2016-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd