fluent-plugin-elasticsearch 0.2.0 → 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c80790f854f11ab486602bfec4e4f96ab56f2f56
+  data.tar.gz: 06163c0e34e87393e84f9a8a04d906dff2fc985b
+SHA512:
+  metadata.gz: 9344c56bfe39072a37402e7e2343a7ec28a9d3417cb14f899bc57744bf91e224d98d5fbf3df048b10d37b36c7bf3edb77812ecd10407c76dad514d0174c98828
+  data.tar.gz: 102828a3a5df25a3c5dcf8462c9a43c5afd802bbe974f2497b5d2c1e9c53f6cf8462165b1a3bad895ed34915e4fdce286cc6976f980085cfe41411fdf25dd2f1

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.1
+
+script: bundle exec ruby -S -Itest test/plugin/test_out_elasticsearch.rb

data/History.md

@@ -1,34 +1,33 @@
-Changelog
-=========
+## Changelog
 
-0.2.0
-=====
+### 0.3.0
+
+- add `parent_key` option (#28)
+- have travis-ci build on multiple rubies (#30)
+- add `utc_index` and `hosts` options, switch to using `elasticsearch` gem (#26, #29)
+
+### 0.2.0
 
 - fix encoding issues with JSON conversion and again when sending to elasticsearch (#19, #21)
 - add logstash_dateformat option (#20)
 
-0.1.4
-=====
+### 0.1.4
 
 - add logstash_prefix option
 
-0.1.3
-=====
+### 0.1.3
 
 - raising an exception on non-success response from elasticsearch
 
-0.1.2
-=====
+### 0.1.2
 
 - add id_key option
 
-0.1.1
-=====
+### 0.1.1
 
 - fix timezone in logstash key
 
 
-0.1.0
-=====
+### 0.1.0
 
  - Initial gem release.

data/README.md

@@ -1,4 +1,4 @@
-# Fluent::Plugin::Elasticsearch
+# Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
 [![Dependency Status](https://gemnasium.com/uken/guard-sidekiq.png)](https://gemnasium.com/uken/fluent-plugin-elasticsearch)
@@ -12,6 +12,8 @@ I wrote this so you can search logs routed through Fluentd.
 
     $ gem install fluent-plugin-elasticsearch
 
+* prerequisite : You need to install [libcurl](http://curl.haxx.se/libcurl/) to work with.
+
 ## Usage
 
 In your fluentd configration, use `type elasticsearch`. Additional configuration is optional, default values would look like this:
@@ -23,9 +25,18 @@ index_name fluentd
 type_name fluentd
 ```
 
-
 **More options:**
 
+```
+hosts host1:port1,host2:port2,host3:port3
+```
+
+You can specify multiple elasticsearch hosts with separator ",".
+
+If you specify multiple hosts, plugin writes to elasticsearch with load balanced. (it's elasticsearch-ruby's feature, default is round-robin.)
+
+If you specify this option, host and port options are ignored.
+
 ```
 logstash_format true # defaults to false
 ```
@@ -44,6 +55,12 @@ logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
 
 By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `logstash-YYYYMM` for a monthly index.
 
+```
+utc_index true
+```
+
+By default, the records inserted into index `logstash-YYMMDD` with utc (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
+
 ---
 
 ```
@@ -117,3 +134,5 @@ Please consider using [fluent-plugin-forest](https://github.com/tagomoris/fluent
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/fluent-plugin-elasticsearch.gemspec

@@ -1,23 +1,25 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
+$:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
-  s.name          = "fluent-plugin-elasticsearch"
-  s.version       = '0.2.0'
-  s.authors       = ["diogo", 'pitr']
-  s.email         = ["team@uken.com"]
+  s.name          = 'fluent-plugin-elasticsearch'
+  s.version       = '0.3.0'
+  s.authors       = ['diogo', 'pitr']
+  s.email         = ['team@uken.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
   s.summary       = s.description
-  s.homepage      = "https://github.com/uken/fluent-plugin-elasticsearch"
+  s.homepage      = 'https://github.com/uken/fluent-plugin-elasticsearch'
   s.license       = 'MIT'
 
   s.files         = `git ls-files`.split($/)
   s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
-  s.require_paths = ["lib"]
+  s.require_paths = ['lib']
 
-  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency 'fluentd', '~> 0'
+  s.add_runtime_dependency 'patron', '~> 0'
+  s.add_runtime_dependency 'elasticsearch', '~> 0'
 
-  s.add_development_dependency "rake"
-  s.add_development_dependency "webmock"
+  s.add_development_dependency 'rake', '~> 0'
+  s.add_development_dependency 'webmock', '~> 0'
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -1,6 +1,7 @@
 # encoding: UTF-8
-require 'net/http'
 require 'date'
+require 'patron'
+require 'elasticsearch'
 
 class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('elasticsearch', self)
@@ -10,9 +11,12 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :logstash_format, :bool, :default => false
   config_param :logstash_prefix, :string, :default => "logstash"
   config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
+  config_param :utc_index, :bool, :default => true
   config_param :type_name, :string, :default => "fluentd"
   config_param :index_name, :string, :default => "fluentd"
   config_param :id_key, :string, :default => nil
+  config_param :parent_key, :string, :default => nil
+  config_param :hosts, :string, :default => nil
 
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
@@ -29,6 +33,20 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     super
   end
 
+  def client
+    @_es ||= Elasticsearch::Client.new :hosts => get_hosts, :reload_connections => true, :adapter => :patron, :retry_on_failure => 5
+    raise "Can not reach Elasticsearch cluster (#{@host}:#{@port})!" unless @_es.ping
+    @_es
+  end
+
+  def get_hosts
+    if @hosts
+        @hosts.split(',').map {|x| x.strip}.compact
+     else
+       ["#{@host}:#{@port}"]
+     end
+  end
+
   def format(tag, time, record)
     [tag, time, record].to_msgpack
   end
@@ -43,7 +61,11 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     chunk.msgpack_each do |tag, time, record|
       if @logstash_format
         record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
-        target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
+        if @utc_index
+          target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
+        else
+          target_index = "#{@logstash_prefix}-#{Time.at(time).strftime("#{@logstash_dateformat}")}"
+        end
       else
         target_index = @index_name
       end
@@ -56,14 +78,20 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       if @id_key && record[@id_key]
         meta['index']['_id'] = record[@id_key]
       end
-      bulk_message << Yajl::Encoder.encode(meta)
-      bulk_message << Yajl::Encoder.encode(record)
+
+      if @parent_key && record[@parent_key]
+        meta['index']['_parent'] = record[@parent_key]
+      end
+
+      bulk_message << meta
+      bulk_message << record
     end
-    bulk_message << ""
 
-    http = Net::HTTP.new(@host, @port.to_i)
-    request = Net::HTTP::Post.new('/_bulk', {'content-type' => 'application/json; charset=utf-8'})
-    request.body = bulk_message.join("\n")
-    http.request(request).value
+    send(bulk_message) unless bulk_message.empty?
+    bulk_message.clear
+  end
+
+  def send(data)
+    client.bulk body: data
   end
 end

data/test/plugin/test_out_elasticsearch.rb

@@ -14,7 +14,7 @@ $:.push File.dirname(__FILE__)
 WebMock.disable_net_connect!
 
 class ElasticsearchOutput < Test::Unit::TestCase
-  attr_accessor :index_cmds, :content_type
+  attr_accessor :index_cmds, :index_command_counts
 
   def setup
     Fluent::Test.setup
@@ -26,12 +26,15 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def sample_record
-    {'age' => 26, 'request_id' => '42'}
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent'}
+  end
+
+  def stub_elastic_ping(url="http://localhost:9200")
+    stub_request(:head, url).with.to_return(:status => 200, :body => "", :headers => {})
   end
 
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
-      @content_type = req.headers["Content-Type"]
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
   end
@@ -40,21 +43,28 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
 
-  def test_writes_to_default_index
-    stub_elastic
-    driver.emit(sample_record)
-    driver.run
-    assert_equal('fluentd', index_cmds.first['index']['_index'])
+  def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
+    if @index_command_counts == nil
+       @index_command_counts = {}
+       @index_command_counts.default = 0
+    end
+
+    stub_request(:post, url).with do |req|
+      index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_command_counts[url] += index_cmds.size
+    end
   end
 
-  def test_wrties_with_proper_content_type
+  def test_writes_to_default_index
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
-    assert_equal("application/json; charset=utf-8", @content_type)
+    assert_equal('fluentd', index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_default_type
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -63,6 +73,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_index
     driver.configure("index_name myindex\n")
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -71,6 +82,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_type
     driver.configure("type_name mytype\n")
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -79,6 +91,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
+    stub_elastic_ping("http://192.168.33.50:9200")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
     driver.emit(sample_record)
     driver.run
@@ -87,13 +100,46 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_port
     driver.configure("port 9201\n")
+    stub_elastic_ping("http://localhost:9201")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
     driver.emit(sample_record)
     driver.run
     assert_requested(elastic_request)
   end
 
+  def test_writes_to_multi_hosts
+    hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
+    hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
+
+    driver.configure("hosts #{hosts_string}")
+
+    hosts.each do |host_info|
+      host, port = host_info
+      stub_elastic_ping("http://#{host}:#{port}")
+      stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
+    end
+
+    1000.times do 
+      driver.emit(sample_record.merge('age'=>rand(100)))
+    end
+
+    driver.run
+
+    # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
+    # it's Fluentd test driver's constraint
+    # so @index_command_counts.size is always 1
+    
+    assert(@index_command_counts.size > 0, "not working with hosts options")
+
+    total = 0
+    @index_command_counts.each do |url, count|
+      total += count
+    end
+    assert_equal(2000, total)
+  end
+
   def test_makes_bulk_request
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.emit(sample_record.merge('age' => 27))
@@ -102,6 +148,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_all_records_are_preserved_in_bulk
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.emit(sample_record.merge('age' => 27))
@@ -114,17 +161,31 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_format true\n")
     time = Time.parse Date.today.to_s
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record, time)
     driver.run
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
+  def test_writes_to_logstash_utc_index
+    driver.configure("logstash_format true\n")
+    driver.configure("utc_index false\n")
+    time = Time.parse Date.today.to_s
+    utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(utc_index, index_cmds.first['index']['_index'])
+  end
+
   def test_writes_to_logstash_index_with_specified_prefix
     driver.configure("logstash_format true\n")
     driver.configure("logstash_prefix myprefix\n")
     time = Time.parse Date.today.to_s
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record, time)
     driver.run
@@ -136,6 +197,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_dateformat %Y.%m\n")
     time = Time.parse Date.today.to_s
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record, time)
     driver.run
@@ -148,6 +210,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_dateformat %Y.%m\n")
     time = Time.parse Date.today.to_s
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record, time)
     driver.run
@@ -155,6 +218,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_doesnt_add_logstash_timestamp_by_default
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -163,6 +227,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_logstash_timestamp_when_configured
     driver.configure("logstash_format true\n")
+    stub_elastic_ping
     stub_elastic
     ts = DateTime.now.to_s
     driver.emit(sample_record)
@@ -172,6 +237,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_doesnt_add_tag_key_by_default
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -180,6 +246,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_tag_key_when_configured
     driver('mytag').configure("include_tag_key true\n")
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -189,6 +256,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_id_key_when_configured
     driver.configure("id_key request_id\n")
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -197,6 +265,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_doesnt_add_id_key_if_missing_when_configured
     driver.configure("id_key another_request_id\n")
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
@@ -204,16 +273,44 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_adds_id_key_when_not_configured
+    stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
     driver.run
     assert(!index_cmds[0]['index'].has_key?('_id'))
   end
 
+  def test_adds_parent_key_when_configured
+    driver.configure("parent_key parent_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal(index_cmds[0]['index']['_parent'], 'parent')
+  end
+
+  def test_doesnt_add_parent_key_if_missing_when_configured
+    driver.configure("parent_key another_parent_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_parent'))
+  end
+
+  def test_adds_parent_key_when_not_configured
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_parent'))
+  end
+
   def test_request_error
+    stub_elastic_ping
     stub_elastic_unavailable
     driver.emit(sample_record)
-    assert_raise(Net::HTTPFatalError) {
+    assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
       driver.run
     }
   end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.2.0
-  prerelease: 
+  version: 0.3.0
 platform: ruby
 authors:
 - diogo
@@ -10,54 +9,76 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-06 00:00:00.000000000 Z
+date: 2014-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: patron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 description: ElasticSearch output plugin for Fluent event collector
@@ -67,7 +88,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - History.md
 - LICENSE.txt
@@ -80,33 +102,26 @@ files:
 homepage: https://github.com/uken/fluent-plugin-elasticsearch
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1091394761449257025
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1091394761449257025
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: ElasticSearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb