fluent-plugin-elasticsearch 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 421e2c4bb196e72e87a6c6e3c8444844b755fe07
-  data.tar.gz: 8568c9999361d7d0ab68f5916721c91d0312ef16
+  metadata.gz: c95d97728c587e2328005541d5a56a93c935456d
+  data.tar.gz: 27731b0037a18c63b94ed8264bc9845f52cf7d0b
 SHA512:
-  metadata.gz: 9878a3743fb7ff425713cf822e7d16ec877eb9697a3e702b428f2aa62474bc8c3c1ee8e8438770e8b5ec5e5d2e37e15b6a81cce1eafb013ffce9970ee5d98098
-  data.tar.gz: c9cbebe13d2c81e4aa36dd99c666f02d4b0f1d0301ed39ad14dd93cef088a03e949113c6003d635b0ebe603eb6d58521bed47234c7b650603cd87abd4899eb9e
+  metadata.gz: 2e9a5268a43192926f48f8429a766bf16af0102d604cb2b2d436357b72d421a4eecbb01a175a6f0b93627f89d77ebea3832aaf39a2464e7888364b81db6942d4
+  data.tar.gz: 7a11dc8ef1a2af5730f03a556c1b1112ddb8ef2ff708be775d3943674fef680069b707ccf5407f486176c7442b459c86a610b543af95e78f2f42b8312498da1a

data/History.md

@@ -2,6 +2,10 @@
 
 ### Future
 
+### 1.2.0
+- out_elasticsearch_dynamic get memory improvement and fix for race condition (#133)
+- Add `resurrect_after` option (#136)
+
 ### 1.1.0
 - Support SSL client verification and custom CA file (#123)
 - Release experimental `type elasticsearch_dynamic` (#127)

data/README.md

@@ -6,7 +6,9 @@
 [![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
 [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
 
-I wrote this so you can search logs routed through Fluentd.
+Send your logs to ElasticSearch (and search them with Kibana maybe?)
+
+Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-elasticsearch-service](https://github.com/atomita/fluent-plugin-aws-elasticsearch-service)
 
 * [Installation](#installation)
 * [Usage](#usage)
@@ -22,6 +24,7 @@ I wrote this so you can search logs routed through Fluentd.
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
   + [reload_on_failure](#reload_on_failure)
+  + [resurrect_after](#resurrect_after)
   + [include_tag_key, tag_key](#include_tag_key-tag_key)
   + [id_key](#id_key)
   + [Client/host certificate options](#clienthost-certificate-options)
@@ -176,6 +179,14 @@ reload_on_failure true # defaults to false
 Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the
 request, this can be useful to quickly remove a dead node from the list of addresses.
 
+### resurrect_after
+
+You can set in the elasticsearch-transport how often dead connections from the elasticsearch-transport's pool will be resurrected.
+
+```
+resurrect_after 5 # defaults to 60s
+```
+
 ### include_tag_key, tag_key
 
 ```
@@ -240,6 +251,8 @@ retry_wait 1.0
 num_threads 1
 ```
 
+The value for option `buffer_chunk_limit` should not exceed value `http.max_content_length` in your Elasticsearch setup (by default it is 104857600 bytes).
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.1.0'
+  s.version       = '1.2.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -27,6 +27,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :request_timeout, :time, :default => 5
   config_param :reload_connections, :bool, :default => true
   config_param :reload_on_failure, :bool, :default => false
+  config_param :resurrect_after, :time, :default => 60
   config_param :time_key, :string, :default => nil
   config_param :ssl_verify , :bool, :default => true
   config_param :client_key, :string, :default => nil
@@ -57,6 +58,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
                                                                           options: {
                                                                             reload_connections: @reload_connections,
                                                                             reload_on_failure: @reload_on_failure,
+                                                                            resurrect_after: @resurrect_after,
                                                                             retry_on_failure: 5,
                                                                             transport_options: {
                                                                               request: { timeout: @request_timeout },

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -13,20 +13,19 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
   config_param :utc_index, :string, :default => "true"
   config_param :reload_connections, :string, :default => "true"
   config_param :reload_on_failure, :string, :default => "false"
+  config_param :resurrect_after, :string, :default => "60"
   config_param :ssl_verify, :string, :dfeault => "true"
 
   def configure(conf)
     super
 
     # evaluate all configurations here
+    @dynamic_params = self.instance_variables.select { |var| is_valid_expand_param_type(var) }
     @dynamic_config = Hash.new
-    self.instance_variables.each { |var|
-      if is_valid_expand_param_type(var)
-        value = expand_param(self.instance_variable_get(var), nil, nil, nil)
-
-        var = var.to_s.gsub(/@(.+)/){ $1 }
-        @dynamic_config[var] = value
-      end
+    @dynamic_params.each { |var|
+      value = expand_param(self.instance_variable_get(var), nil, nil, nil)
+      var = var[1..-1]
+      @dynamic_config[var] = value
     }
     # end eval all configs
     @current_config = nil
@@ -47,6 +46,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
                                                                           options: {
                                                                             reload_connections: @dynamic_config['reload_connections'],
                                                                             reload_on_failure: @dynamic_config['reload_on_failure'],
+                                                                            resurrect_after: @dynamic_config['resurrect_after'],
                                                                             retry_on_failure: 5,
                                                                             transport_options: {
                                                                               request: { timeout: @dynamic_config['request_timeout'] },
@@ -105,64 +105,63 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
       attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
       attributes.inspect
     end.join(', ')
-  end  
+  end
 
   def write(chunk)
-    
     bulk_message = Hash.new { |h,k| h[k] = [] }
+    dynamic_conf = @dynamic_config.clone
 
     chunk.msgpack_each do |tag, time, record|
       next unless record.is_a? Hash
 
       # evaluate all configurations here
-      self.instance_variables.each { |var|
-        if is_valid_expand_param_type(var)
-          # check here to determine if we should evaluate
-          if @dynamic_config[var[1,var.length-1]] != self.instance_variable_get(var)
-            value = expand_param(self.instance_variable_get(var), tag, time, record)
-            var = var.to_s.gsub(/@(.+)/){ $1 }
-            @dynamic_config[var] = value
-          end
+      @dynamic_params.each { |var|
+        k = var[1..-1]
+        v = self.instance_variable_get(var)
+        # check here to determine if we should evaluate
+        if dynamic_conf[k] != v
+          value = expand_param(v, tag, time, record)
+          dynamic_conf[k] = value
         end
       }
       # end eval all configs
 
-      if eval(@dynamic_config['logstash_format'])
+      if eval(dynamic_conf['logstash_format'])
         if record.has_key?("@timestamp")
           time = Time.parse record["@timestamp"]
-        elsif record.has_key?(@dynamic_config['time_key'])
-          time = Time.parse record[@dynamic_config['time_key']]
-          record['@timestamp'] = record[@dynamic_config['time_key']]
+        elsif record.has_key?(dynamic_conf['time_key'])
+          time = Time.parse record[dynamic_conf['time_key']]
+          record['@timestamp'] = record[dynamic_conf['time_key']]
         else
           record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
         end
 
-        if eval(@dynamic_config['utc_index'])
-          target_index = "#{@dynamic_config['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{@dynamic_config['logstash_dateformat']}")}"
+        if eval(dynamic_conf['utc_index'])
+          target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
         else
-          target_index = "#{@dynamic_config['logstash_prefix']}-#{Time.at(time).strftime("#{@dynamic_config['logstash_dateformat']}")}"
+          target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).strftime("#{dynamic_conf['logstash_dateformat']}")}"
         end
       else
-        target_index = @dynamic_config['index_name']
+        target_index = dynamic_conf['index_name']
       end
 
       if @include_tag_key
-        record.merge!(@dynamic_config['tag_key'] => tag)
+        record.merge!(dynamic_conf['tag_key'] => tag)
       end
 
-      meta = { "index" => {"_index" => target_index, "_type" => @dynamic_config['type_name']} }
-      if @dynamic_config['id_key'] && record[@dynamic_config['id_key']]
-        meta['index']['_id'] = record[@dynamic_config['id_key']]
+      meta = { "index" => {"_index" => target_index, "_type" => dynamic_conf['type_name']} }
+      if dynamic_conf['id_key'] && record[dynamic_conf['id_key']]
+        meta['index']['_id'] = record[dynamic_conf['id_key']]
       end
 
-      if @dynamic_config['parent_key'] && record[@dynamic_config['parent_key']]
-        meta['index']['_parent'] = record[@dynamic_config['parent_key']]
+      if dynamic_conf['parent_key'] && record[dynamic_conf['parent_key']]
+        meta['index']['_parent'] = record[dynamic_conf['parent_key']]
       end
 
-      if @dynamic_config['hosts']
-        host = @dynamic_config['hosts']  
+      if dynamic_conf['hosts']
+        host = dynamic_conf['hosts']
       else
-        host = "#{@dynamic_config['host']}:#{@dynamic_config['port']}"
+        host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
       end
 
       bulk_message[host] << meta
@@ -174,7 +173,6 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
       send(array, hKey) unless array.empty?
       array.clear
     end
-
   end
 
   def send(data, host)
@@ -222,6 +220,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
   end
 
   def is_valid_expand_param_type(param)
+    return false if [:@buffer_type].include?(param)
     return self.instance_variable_get(param).is_a?(String)
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-17 00:00:00.000000000 Z
+date: 2015-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd