fluent-plugin-elasticsearch 0.3.0 → 0.3.1

data/History.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### Future
+
+- do not generate @timestamp if already part of message (#35)
+
 ### 0.3.0
 
 - add `parent_key` option (#28)

data/README.md

@@ -12,7 +12,7 @@ I wrote this so you can search logs routed through Fluentd.
 
     $ gem install fluent-plugin-elasticsearch
 
-* prerequisite : You need to install [libcurl](http://curl.haxx.se/libcurl/) to work with.
+* prerequisite : You need to install [libcurl (libcurl-devel)](http://curl.haxx.se/libcurl/) to work with.
 
 ## Usage
 
@@ -53,6 +53,12 @@ By default, the records inserted into index `logstash-YYMMDD`. This option allow
 logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
 ```
 
+By default, when inserting records in logstash format, @timestamp is dynamically created with the time at log ingestion. If you'd like to use a custom time. Include an @timestamp with your record. 
+
+```
+{"@timestamp":"2014-04-07T000:00:00-00:00"}
+```
+
 By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `logstash-YYYYMM` for a monthly index.
 
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '0.3.0'
+  s.version       = '0.3.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['team@uken.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
@@ -21,5 +21,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'elasticsearch', '~> 0'
 
   s.add_development_dependency 'rake', '~> 0'
-  s.add_development_dependency 'webmock', '~> 0'
+  s.add_development_dependency 'webmock', '~> 1'
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -34,16 +34,24 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   end
 
   def client
-    @_es ||= Elasticsearch::Client.new :hosts => get_hosts, :reload_connections => true, :adapter => :patron, :retry_on_failure => 5
+    @_es ||= begin
+      adapter_conf = lambda {|f| f.adapter :patron }
+      transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new({ hosts: get_hosts,
+                                                                           options: {
+                                                                             reload_connections: true,
+                                                                             retry_on_failure: 5
+                                                                          }}, &adapter_conf)
+      Elasticsearch::Client.new transport: transport
+    end
     raise "Can not reach Elasticsearch cluster (#{@host}:#{@port})!" unless @_es.ping
     @_es
   end
 
   def get_hosts
     if @hosts
-        @hosts.split(',').map {|x| x.strip}.compact
+        @hosts.split(',').map {|x| hp = x.split(':'); { host: hp[0], port: hp[1] || @port } }.compact
      else
-       ["#{@host}:#{@port}"]
+       [{host: @host, port: @port }]
      end
   end
 
@@ -60,7 +68,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
 
     chunk.msgpack_each do |tag, time, record|
       if @logstash_format
-        record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
+        record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s}) unless record.has_key?("@timestamp")
         if @utc_index
           target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
         else

data/test/plugin/test_out_elasticsearch.rb

@@ -119,7 +119,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
     end
 
-    1000.times do 
+    1000.times do
       driver.emit(sample_record.merge('age'=>rand(100)))
     end
 
@@ -128,7 +128,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
     # it's Fluentd test driver's constraint
     # so @index_command_counts.size is always 1
-    
+
     assert(@index_command_counts.size > 0, "not working with hosts options")
 
     total = 0
@@ -169,8 +169,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_logstash_utc_index
-    driver.configure("logstash_format true\n")
-    driver.configure("utc_index false\n")
+    driver.configure("logstash_format true
+                      utc_index false")
     time = Time.parse Date.today.to_s
     utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic_ping
@@ -181,8 +181,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_logstash_index_with_specified_prefix
-    driver.configure("logstash_format true\n")
-    driver.configure("logstash_prefix myprefix\n")
+    driver.configure("logstash_format true
+                      logstash_prefix myprefix")
     time = Time.parse Date.today.to_s
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
@@ -193,8 +193,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
     def test_writes_to_logstash_index_with_specified_dateformat
-    driver.configure("logstash_format true\n")
-    driver.configure("logstash_dateformat %Y.%m\n")
+    driver.configure("logstash_format true
+                      logstash_dateformat %Y.%m")
     time = Time.parse Date.today.to_s
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
@@ -205,9 +205,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
-    driver.configure("logstash_format true\n")
-    driver.configure("logstash_prefix myprefix\n")
-    driver.configure("logstash_dateformat %Y.%m\n")
+    driver.configure("logstash_format true
+                      logstash_prefix myprefix
+                      logstash_dateformat %Y.%m")
     time = Time.parse Date.today.to_s
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
@@ -236,6 +236,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_timestamp_when_included_in_record
+    driver.configure("logstash_format true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
   def test_doesnt_add_tag_key_by_default
     stub_elastic_ping
     stub_elastic

metadata

@@ -1,7 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
+  prerelease: 
 platform: ruby
 authors:
 - diogo
@@ -9,78 +10,88 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-17 00:00:00.000000000 Z
+date: 2014-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: patron
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 description: ElasticSearch output plugin for Fluent event collector
 email:
 - team@uken.com
@@ -88,8 +99,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
+- .gitignore
+- .travis.yml
 - Gemfile
 - History.md
 - LICENSE.txt
@@ -102,26 +113,27 @@ files:
 homepage: https://github.com/uken/fluent-plugin-elasticsearch
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 1.8.23.2
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: ElasticSearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: c80790f854f11ab486602bfec4e4f96ab56f2f56
-  data.tar.gz: 06163c0e34e87393e84f9a8a04d906dff2fc985b
-SHA512:
-  metadata.gz: 9344c56bfe39072a37402e7e2343a7ec28a9d3417cb14f899bc57744bf91e224d98d5fbf3df048b10d37b36c7bf3edb77812ecd10407c76dad514d0174c98828
-  data.tar.gz: 102828a3a5df25a3c5dcf8462c9a43c5afd802bbe974f2497b5d2c1e9c53f6cf8462165b1a3bad895ed34915e4fdce286cc6976f980085cfe41411fdf25dd2f1