fluent-plugin-elasticsearch 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb97386663d594fba5f8370521ec652d8421be24
-  data.tar.gz: ee4c8b7df8eedab7a49928a8501d125acee07d81
+  metadata.gz: 52e6ea7f1fcf3e37be32d06f5d81d5364702cbe5
+  data.tar.gz: 66ce94029829ccf119262929ca50e9e21c4d73ea
 SHA512:
-  metadata.gz: 1e18d1b311ac77e067fead2acd4e4dcbfafad8e1930f60d041195024da6aa8f1993e068af25ad2ce58f95a6c87f746baf254aadbc48a8612c82c43fdf747f7c3
-  data.tar.gz: 226358388cf4cdb2916bbba8c7272e019f257bcb8b964d1e1b839f7a6be4aa6aee9688a917ae20b2f02982863ac43b0a1b9125ce1879baa8bda30a0db4e09e40
+  metadata.gz: b9922d58ba0999c3c03671144b1f4446da3154731e9e5b9dc4bc7dc921d154f382259b2b8b972a21bbcafc35f12265921dadbcba4f592d4fc0f1ebba16566ff5
+  data.tar.gz: d0ec86f473deb9e1a70cca98f3d8194a72f82d08f389246840ae1e25c0dcad9a2c7832b3d54acb6e774652e6c26f50858a9945bca069760f8e972ee01b666515

data/History.md

@@ -3,9 +3,16 @@
 ### Future
 
 
+### 0.5.0
+
+- add full connection URI support (#65)
+- use `@timestamp` for index (#41)
+- add support for elasticsearch gem version 1 (#71)
+- fix connection reset & retry when connection is lost (#67)
+
 ### 0.4.0
 
-- add `request_timeout` config
+- add `request_timeout` config (#59)
 - fix lockup when non-hash values are sent (#52)
 
 ### 0.3.1
@@ -40,7 +47,6 @@
 
 - fix timezone in logstash key
 
-
 ### 0.1.0
 
  - Initial gem release.

data/README.md

@@ -25,18 +25,38 @@ index_name fluentd
 type_name fluentd
 ```
 
+**Index templates**
+
+This plugin creates ElasticSearch indices by merely writing to them. Consider using [Index Templates](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-templates.html) to gain control of what get indexed and how. See [this example](https://github.com/uken/fluent-plugin-elasticsearch/issues/33#issuecomment-38693282) for a good starting point.
+
 **More options:**
 
 ```
 hosts host1:port1,host2:port2,host3:port3
 ```
 
+or
+
+```
+hosts https://customhost.com:443/path,https://username:password@host-failover.com:443
+```
+
 You can specify multiple elasticsearch hosts with separator ",".
 
-If you specify multiple hosts, plugin writes to elasticsearch with load balanced. (it's elasticsearch-ruby's feature, default is round-robin.)
+If you specify multiple hosts, this plugin will load balance updates to elasticsearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
 
 If you specify this option, host and port options are ignored.
 
+```
+user demo
+password secret
+path /elastic_search/
+scheme https
+```
+
+You can specify user and password for HTTP basic auth. If used in conjunction with a hosts list, then these options will be used by default i.e. if you do not provide any of these options within the hosts listed.
+
+
 ```
 logstash_format true # defaults to false
 ```
@@ -53,7 +73,7 @@ By default, the records inserted into index `logstash-YYMMDD`. This option allow
 logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
 ```
 
-By default, when inserting records in logstash format, @timestamp is dynamically created with the time at log ingestion. If you'd like to use a custom time. Include an @timestamp with your record. 
+By default, when inserting records in logstash format, @timestamp is dynamically created with the time at log ingestion. If you'd like to use a custom time. Include an @timestamp with your record.
 
 ```
 {"@timestamp":"2014-04-07T000:00:00-00:00"}

data/fluent-plugin-elasticsearch.gemspec

@@ -3,9 +3,9 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '0.4.0'
+  s.version       = '0.5.0'
   s.authors       = ['diogo', 'pitr']
-  s.email         = ['team@uken.com']
+  s.email         = ['pitr@uken.com', 'diogo@uken.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
   s.summary       = s.description
   s.homepage      = 'https://github.com/uken/fluent-plugin-elasticsearch'
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'fluentd', '~> 0'
   s.add_runtime_dependency 'patron', '~> 0'
-  s.add_runtime_dependency 'elasticsearch', '~> 0'
+  s.add_runtime_dependency 'elasticsearch', '>= 0'
 
   s.add_development_dependency 'rake', '~> 0'
   s.add_development_dependency 'webmock', '~> 1'

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,12 +2,20 @@
 require 'date'
 require 'patron'
 require 'elasticsearch'
+require 'uri'
 
 class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
+  class ConnectionFailure < StandardError; end
+
   Fluent::Plugin.register_output('elasticsearch', self)
 
   config_param :host, :string,  :default => 'localhost'
   config_param :port, :integer, :default => 9200
+  config_param :user, :string, :default => nil
+  config_param :password, :string, :default => nil
+  config_param :path, :string, :default => nil
+  config_param :scheme, :string, :default => 'http'
+  config_param :hosts, :string, :default => nil
   config_param :logstash_format, :bool, :default => false
   config_param :logstash_prefix, :string, :default => "logstash"
   config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
@@ -16,7 +24,6 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :index_name, :string, :default => "fluentd"
   config_param :id_key, :string, :default => nil
   config_param :parent_key, :string, :default => nil
-  config_param :hosts, :string, :default => nil
   config_param :request_timeout, :time, :default => 5
 
   include Fluent::SetTagKeyMixin
@@ -37,26 +44,66 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   def client
     @_es ||= begin
       adapter_conf = lambda {|f| f.adapter :patron }
-      transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new({ hosts: get_hosts,
-                                                                           options: {
-                                                                             reload_connections: true,
-                                                                             retry_on_failure: 5,
-                                                                             transport_options: {
-                                                                               request: { timeout: @request_timeout }
-                                                                             }
-                                                                          }}, &adapter_conf)
-      Elasticsearch::Client.new transport: transport
+      transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
+                                                                          options: {
+                                                                            reload_connections: true,
+                                                                            retry_on_failure: 5,
+                                                                            transport_options: {
+                                                                              request: { timeout: @request_timeout }
+                                                                            }
+                                                                          }), &adapter_conf)
+      es = Elasticsearch::Client.new transport: transport
+
+      begin
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping
+      rescue Faraday::ConnectionFailed => e
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}"
+      end
+
+      log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}"
+      es
     end
-    raise "Can not reach Elasticsearch cluster (#{@host}:#{@port})!" unless @_es.ping
-    @_es
   end
 
-  def get_hosts
-    if @hosts
-        @hosts.split(',').map {|x| hp = x.split(':'); { host: hp[0], port: hp[1] || @port } }.compact
-     else
-       [{host: @host, port: @port }]
-     end
+  def get_connection_options
+    raise "`password` must be present if `user` is present" if @user && !@password
+
+    hosts = if @hosts
+      @hosts.split(',').map do |host_str|
+        # Support legacy hosts format host:port,host:port,host:port...
+        if host_str.match(%r{^[^:]+\:\d+$})
+          {
+            host:   host_str.split(':')[0],
+            port:   (host_str.split(':')[1] || @port).to_i,
+            scheme: @scheme
+          }
+        else
+          # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
+          uri = URI(host_str)
+          %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
+            hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
+            hash
+          end
+        end
+      end.compact
+    else
+      [{host: @host, port: @port, scheme: @scheme}]
+    end.each do |host|
+      host.merge!(user: @user, password: @password) if !host[:user] && @user
+      host.merge!(path: @path) if !host[:path] && @path
+    end
+
+    {
+      hosts: hosts
+    }
+  end
+
+  def connection_options_description
+    get_connection_options[:hosts].map do |host_info|
+      attributes = host_info.dup
+      attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
+      attributes.inspect
+    end.join(', ')
   end
 
   def format(tag, time, record)
@@ -73,7 +120,11 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     chunk.msgpack_each do |tag, time, record|
       next unless record.is_a? Hash
       if @logstash_format
-        record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s}) unless record.has_key?("@timestamp")
+        if record.has_key?("@timestamp")
+          time = Time.parse record["@timestamp"]
+        else
+          record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
+        end
         if @utc_index
           target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
         else
@@ -105,6 +156,18 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   end
 
   def send(data)
-    client.bulk body: data
+    retries = 0
+    begin
+      client.bulk body: data
+    rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e
+      if retries < 2
+        retries += 1
+        @_es = nil
+        log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
+        sleep 2**retries
+        retry
+      end
+      raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+    end
   end
 end

data/test/plugin/test_out_elasticsearch.rb

@@ -6,11 +6,11 @@ require 'fluent/plugin/out_elasticsearch'
 require 'webmock/test_unit'
 require 'date'
 
-require 'helper'
-
-$:.push File.expand_path("../lib", __FILE__)
+$:.push File.expand_path("../..", __FILE__)
 $:.push File.dirname(__FILE__)
 
+require 'helper'
+
 WebMock.disable_net_connect!
 
 class ElasticsearchOutput < Test::Unit::TestCase
@@ -55,6 +55,87 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  def test_configure
+    config = %{
+      host     logs.google.com
+      port     777
+      scheme   https
+      path     /es/
+      user     john
+      password doe
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 'logs.google.com', instance.host
+    assert_equal 777, instance.port
+    assert_equal 'https', instance.scheme
+    assert_equal '/es/', instance.path
+    assert_equal 'john', instance.user
+    assert_equal 'doe', instance.password
+  end
+
+  def test_legacy_hosts_list
+    config = %{
+      hosts    host1:50,host2:100
+      scheme   https
+      path     /es/
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 50, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal '/es/', host2[:path]
+  end
+
+  def test_hosts_list
+    config = %{
+      hosts    https://john:password@host1:443/elastic/,http://host2
+      path     /default_path
+      user     default_user
+      password default_password
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'john', host1[:user]
+    assert_equal 'password', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
+  def test_single_host_params_and_defaults
+    config = %{
+      host     logs.google.com
+      user     john
+      password doe
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 1, instance.get_connection_options[:hosts].length
+    host1 = instance.get_connection_options[:hosts][0]
+
+    assert_equal 'logs.google.com', host1[:host]
+    assert_equal 9200, host1[:port]
+    assert_equal 'http', host1[:scheme]
+    assert_equal 'john', host1[:user]
+    assert_equal 'doe', host1[:password]
+    assert_equal nil, host1[:path]
+  end
+
   def test_writes_to_default_index
     stub_elastic_ping
     stub_elastic
@@ -332,4 +413,23 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.emit("some garbage string")
     driver.run
   end
+
+  def test_connection_failed_retry
+    connection_resets = 0
+
+    stub_elastic_ping(url="http://localhost:9200").with do |req|
+      connection_resets += 1
+    end
+
+    stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      raise Faraday::ConnectionFailed, "Test message"
+    end
+
+    driver.emit(sample_record)
+
+    assert_raise(Fluent::ElasticsearchOutput::ConnectionFailure) {
+      driver.run
+    }
+    assert_equal(connection_resets, 3)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-01 00:00:00.000000000 Z
+date: 2014-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -43,14 +43,14 @@ dependencies:
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -83,7 +83,8 @@ dependencies:
         version: '1'
 description: ElasticSearch output plugin for Fluent event collector
 email:
-- team@uken.com
+- pitr@uken.com
+- diogo@uken.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -119,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: ElasticSearch output plugin for Fluent event collector