fluent-plugin-elasticsearch 0.1.1 → 0.1.2

data/History.md

@@ -0,0 +1,18 @@
+Changelog
+=========
+
+0.1.2
+=====
+
+- add id_key option
+
+0.1.1
+=====
+
+- fix timezone in logstash key
+
+
+0.1.0
+=====
+
+ - Initial gem release.

data/README.md

@@ -51,6 +51,21 @@ The record inserted into elasticsearch would be
 
 ---
 
+```
+id_key request_id # use "request_id" field as a record id in ES
+```
+
+By default, all records inserted into elasticsearch get a random _id. This option allows to use a field in the record as an identifier.
+
+This following record `{"name":"Johnny","request_id":"87d89af7daffad6"}` will trigger the following ElasticSearch command
+
+```
+{ "index" : { "_index" : "logstash-2013.01.01, "_type" : "fluentd", "_id" : "87d89af7daffad6" } }
+{ "name": "Johnny", "request_id": "87d89af7daffad6" }
+```
+
+---
+
 fluentd-plugin-elasticsearch is a buffered output that uses elasticseach's bulk API. So additional buffer configuration would be (with default values):
 
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,8 +3,8 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = "fluent-plugin-elasticsearch"
-  s.version       = '0.1.1'
-  s.authors       = ["diogo"]
+  s.version       = '0.1.2'
+  s.authors       = ["diogo", 'pitr']
   s.email         = ["team@uken.com"]
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
   s.summary       = s.description

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -10,6 +10,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :logstash_format, :bool, :default => false
   config_param :type_name, :string, :default => "fluentd"
   config_param :index_name, :string, :default => "fluentd"
+  config_param :id_key, :string, :default => nil
 
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
@@ -49,7 +50,11 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
         record.merge!(@tag_key => tag)
       end
 
-      bulk_message << { "index" => {"_index" => target_index, "_type" => type_name} }.to_json
+      meta = { "index" => {"_index" => target_index, "_type" => type_name} }
+      if @id_key && record[@id_key]
+        meta['index']['_id'] = record[@id_key]
+      end
+      bulk_message << meta.to_json
       bulk_message << record.to_json
     end
     bulk_message << ""

data/test/plugin/test_out_elasticsearch.rb

@@ -26,7 +26,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def sample_record
-    {'age' => 26}
+    {'age' => 26, 'request_id' => '42'}
   end
 
   def stub_elastic(url="http://localhost:9200/_bulk")
@@ -140,4 +140,27 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert(index_cmds[1].has_key?('tag'))
     assert_equal(index_cmds[1]['tag'], 'mytag')
   end
+
+  def test_adds_id_key_when_configured
+    driver.configure("id_key request_id\n")
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal(index_cmds[0]['index']['_id'], '42')
+  end
+
+  def test_doesnt_add_id_key_if_missing_when_configured
+    driver.configure("id_key another_request_id\n")
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_id'))
+  end
+
+  def test_adds_id_key_when_not_configured
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_id'))
+  end
 end

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
 - diogo
+- pitr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-27 00:00:00.000000000 Z
+date: 2013-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -68,6 +69,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
+- History.md
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -89,7 +91,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4483254378511463978
+      hash: 3762640390142768539
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -98,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4483254378511463978
+      hash: 3762640390142768539
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23